MetaGer.php 62.1 KB
Newer Older
1
<?php
Karl Hasselbring's avatar
Karl Hasselbring committed
2

3
4
5
namespace App;

use App;
6
use Cache;
7
use Carbon;
8
use Illuminate\Http\Request;
9
use Illuminate\Support\Facades\Redis;
10
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix    
Dominik Hebeler committed
11
use LaravelLocalization;
12
use Log;
13
use Predis\Connection\ConnectionException;
14
15
16

class MetaGer
{
Dominik Hebeler's avatar
Dominik Hebeler committed
17
18
    const FETCHQUEUE_KEY = "fetcher.queue";

19
    # Einstellungen für die Suche
20
21
    public $alteredQuery = "";
    public $alterationOverrideQuery = "";
22
23
24
25
26
27
28
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
29
    protected $time = 2000;
30
    protected $hostBlacklist = [];
31
    protected $domainBlacklist = [];
32
33
34
35
    private $urlBlacklist = [];
    protected $stopWords = [];
    protected $phrases = [];
    protected $engines = [];
36
    protected $totalResults = 0;
37
    protected $results = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
38
39
    protected $queryFilter = [];
    protected $parameterFilter = [];
40
    protected $ads = [];
41
    protected $infos = [];
42
43
44
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
45
    protected $availableFoki = [];
46
47
    protected $startCount = 0;
    protected $canCache = false;
48
    # Daten über die Abfrage$
49
    protected $ip;
Dominik Hebeler's avatar
Dominik Hebeler committed
50
    protected $useragent;
51
52
    protected $language;
    protected $agent;
53
    protected $apiKey = "";
Phil Höfer's avatar
Phil Höfer committed
54
    protected $apiAuthorized = false;
55
    protected $next = [];
56
57
58
59
60
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
61
    protected $newtab;
62
    protected $domainsBlacklisted = [];
63
    protected $adDomainsBlacklisted = [];
64
    protected $urlsBlacklisted = [];
65
    protected $adUrlsBlacklisted = [];
66
    protected $url;
67
    protected $fullUrl;
Aria's avatar
Aria committed
68
    protected $enabledSearchengines = [];
69
    protected $languageDetect;
Dominik Hebeler's avatar
Dominik Hebeler committed
70
71
    protected $verificationId;
    protected $verificationCount;
72
    protected $searchUid;
73
    protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult, $redisCurrentResultList;
Dominik Hebeler's avatar
Dominik Hebeler committed
74
    public $starttime;
75

76
    public function __construct($hash = "")
77
    {
78
        # Timer starten
79
        $this->starttime = microtime(true);
80
        # Versuchen Blacklists einzulesen
81
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
82
            $tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
83
            $this->domainsBlacklisted = explode("\n", $tmp);
84
85
            $tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted = explode("\n", $tmp);
86
        } else {
87
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
88
        }
89
90
91
92
93
94
95
96
97
        # Versuchen Blacklists einzulesen
        if (file_exists(config_path() . "/adBlacklistDomains.txt") && file_exists(config_path() . "/adBlacklistUrl.txt")) {
            $tmp = file_get_contents(config_path() . "/adBlacklistDomains.txt");
            $this->adDomainsBlacklisted = explode("\n", $tmp);
            $tmp = file_get_contents(config_path() . "/adBlacklistUrl.txt");
            $this->adUrlsBlacklisted = explode("\n", $tmp);
        } else {
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
        }
98

99
        # Parser Skripte einhängen
100
101
102
103
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
104
                require_once $path;
105
106
107
            }
        }

108
        # Cachebarkeit testen
109
110
111
112
113
114
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
115
116
117
118
119
        if ($hash === "") {
            $this->searchUid = md5(uniqid());
        } else {
            $this->searchUid = $hash;
        }
120
121
122
123
124
125
126
        $redisPrefix = "search";
        # This is a list on which the MetaGer process can do a blocking pop to wait for new results
        $this->redisResultWaitingKey = $redisPrefix . "." . $this->searchUid . ".ready";
        # This is a list of searchengines which have delivered results for this search
        $this->redisResultEngineList = $redisPrefix . "." . $this->searchUid . ".engines";
        # This is the key where the results of the engine are stored as well as some statistical data
        $this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results.";
127
128
        # A list of all search results already delivered to the user (sorted of course)
        $this->redisCurrentResultList = $redisPrefix . "." . $this->searchUid . ".currentResults";
129
    }
130

131
    # Erstellt aus den gesammelten Ergebnissen den View
132
    public function createView($quicktipResults = [])
133
    {
134
135
        # Hiermit werden die evtl. ausgewählten SuMas extrahiert, damit die Input-Boxen richtig gesetzt werden können
        $focusPages = [];
Aria Givi's avatar
Aria Givi committed
136

137
        foreach ($this->request->all() as $key => $value) {
138
139
            if (starts_with($key, 'engine_') && $value === 'on') {
                $focusPages[] = $key;
140
141
142
            }
        }

143
        $viewResults = [];
144
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
145
        foreach ($this->results as $result) {
146
147
148
149
            $viewResults[] = get_object_vars($result);
        }
        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();
150
151
        if ($this->fokus === "bilder") {
            switch ($this->out) {
152
                case 'results':
153
                    return view('resultpages.results_images')
154
155
156
157
158
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
159
                        ->with('apiAuthorized', $this->apiAuthorized)
160
                        ->with('metager', $this)
161
                        ->with('browser', (new Agent())->browser());
162
                default:
163
                    return view('resultpages.resultpage_images')
164
165
166
167
168
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
169
                        ->with('apiAuthorized', $this->apiAuthorized)
170
                        ->with('metager', $this)
171
172
173
174
                        ->with('browser', (new Agent())->browser())
                        ->with('quicktips', $quicktipResults)
                        ->with('focus', $this->fokus)
                        ->with('resultcount', count($this->results));
175
            }
176
177
178
        } else {
            switch ($this->out) {
                case 'results':
179
                    return view('resultpages.results')
180
181
182
183
184
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
185
                        ->with('apiAuthorized', $this->apiAuthorized)
186
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
187
188
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
189
190
                    break;
                case 'results-with-style':
191
                    return view('resultpages.resultpage')
192
193
194
195
196
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
197
                        ->with('apiAuthorized', $this->apiAuthorized)
198
199
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
Dominik Hebeler's avatar
Dominik Hebeler committed
200
201
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
202
                    break;
Phil Höfer's avatar
Phil Höfer committed
203
                case 'rich':
204
                    return view('resultpages.metager3rich')
205
                        ->with('results', $viewResults)
Phil Höfer's avatar
Phil Höfer committed
206
207
208
209
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
210
                        ->with('apiAuthorized', $this->apiAuthorized)
Phil Höfer's avatar
Phil Höfer committed
211
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
212
213
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
Phil Höfer's avatar
Phil Höfer committed
214
                    break;
215
                case 'rss20':
216
                    return view('resultpages.metager3resultsrss20')
217
218
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
Phil Höfer's avatar
Phil Höfer committed
219
                        ->with('apiAuthorized', $this->apiAuthorized)
220
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
221
222
                        ->with('resultcount', sizeof($viewResults))
                        ->with('fokus', $this->fokus);
223
                    break;
224
                case 'api':
Phil Höfer's avatar
Phil Höfer committed
225
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'key' => $this->apiKey, 'apiAuthorized' => $this->apiAuthorized])->header('Content-Type', 'application/xml');
226
                    break;
Aria Givi's avatar
Aria Givi committed
227
                case 'atom10':
Phil Höfer's avatar
Phil Höfer committed
228
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'key' => $this->apiKey, 'apiAuthorized' => true])
229
                        ->header('Content-Type', 'application/xml');
Aria Givi's avatar
Aria Givi committed
230
                    break;
231
                case 'result-count':
232
233
                    # Wir geben die Ergebniszahl und die benötigte Zeit zurück:
                    return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
234
                    break;
235
                default:
236
                    return view('resultpages.resultpage')
237
                        ->with('eingabe', $this->eingabe)
238
                        ->with('focusPages', $focusPages)
239
240
241
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
242
                        ->with('apiAuthorized', $this->apiAuthorized)
243
                        ->with('metager', $this)
244
                        ->with('browser', (new Agent())->browser())
245
                        ->with('quicktips', $quicktipResults)
246
247
                        ->with('resultcount', count($this->results))
                        ->with('focus', $this->fokus);
248
249
                    break;
            }
250
251
252
        }
    }

Phil Höfer's avatar
Phil Höfer committed
253
    public function prepareResults()
254
    {
Phil Höfer's avatar
Phil Höfer committed
255
256
        $engines = $this->engines;
        // combine
257
        $this->combineResults($engines);
Phil Höfer's avatar
Phil Höfer committed
258
        // misc (WiP)
259
        if ($this->fokus == "nachrichten") {
260
261
262
            $this->results = array_filter($this->results, function ($v, $k) {
                return !is_null($v->getRank());
            }, ARRAY_FILTER_USE_BOTH);
263
264
265
266
267
268
269
270
271
272
            uasort($this->results, function ($a, $b) {
                $datea = $a->getDate();
                $dateb = $b->getDate();
                return $dateb - $datea;
            });
        } else {
            uasort($this->results, function ($a, $b) {
                if ($a->getRank() == $b->getRank()) {
                    return 0;
                }
273

274
275
276
                return ($a->getRank() < $b->getRank()) ? 1 : -1;
            });
        }
277

278
279
        # Validate Results
        $newResults = [];
280
281
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
282
                $newResults[] = $result;
283
            }
284
285
286
        }
        $this->results = $newResults;

287
288
289
290
        # Validate Advertisements
        $newResults = [];
        foreach ($this->ads as $ad) {
            if (($ad->strippedHost !== "" && (in_array($ad->strippedHost, $this->adDomainsBlacklisted) ||
291
292
                in_array($ad->strippedLink, $this->adUrlsBlacklisted))) || ($ad->strippedHostAnzeige !== "" && (in_array($ad->strippedHostAnzeige, $this->adDomainsBlacklisted) ||
                in_array($ad->strippedLinkAnzeige, $this->adUrlsBlacklisted)))
Karl Hasselbring's avatar
Karl Hasselbring committed
293
            ) {
294
295
296
297
298
299
                continue;
            }
            $newResults[] = $ad;
        }
        $this->ads = $newResults;

300
        #Adgoal Implementation
301
302
303
        if (!$this->apiAuthorized) {
            $this->results = $this->parseAdgoal($this->results);
        }
304

Dominik Hebeler's avatar
Dominik Hebeler committed
305
306
        # Human Verification
        $this->results = $this->humanVerification($this->results);
307
        $this->ads = $this->humanVerification($this->ads);
Dominik Hebeler's avatar
Dominik Hebeler committed
308

309
        $counter = 0;
310
        $firstRank = 0;
311

312
        if (count($this->results) <= 0) {
313
314
315
316
317
318
            if (strlen($this->site) > 0) {
                $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
                $this->errors[] = trans('metaGer.results.failedSitesearch', ['altSearch' => $no_sitesearch_query]);
            } else {
                $this->errors[] = trans('metaGer.results.failed');
            }
319
        }
320

321
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
322
            $page = $this->page + 1;
323
            $this->next = [
324
325
                'page' => $page,
                'engines' => $this->next,
326
            ];
327
            \App\CacheHelper::put($this->getSearchUid(), serialize($this->next), 60 * 60);
328
329
        } else {
            $this->next = [];
330
        }
331
    }
332

Phil Höfer's avatar
Phil Höfer committed
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
    public function combineResults($engines)
    {
        foreach ($engines as $engine) {
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
            foreach ($engine->results as $result) {
                if ($result->valid) {
                    $this->results[] = $result;
                }
            }
            foreach ($engine->ads as $ad) {
                $this->ads[] = $ad;
            }
        }
    }

353
354
    public function parseAdgoal($results)
    {
355
        $publicKey = getenv('adgoal_public');
356
        $privateKey = getenv('adgoal_private');
357
        if ($publicKey === false) {
358
359
360
            return $results;
        }
        $tldList = "";
361
362
        try {
            foreach ($results as $result) {
363
364
365
                if (!$result->new) {
                    continue;
                }
366
                $link = $result->link;
367
                if (strpos($link, "http") !== 0) {
368
369
370
371
372
373
374
375
376
377
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

378
            # Query
379
            $query = $this->q;
380

381
            $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . urlencode($publicKey) . "&k=" . urlencode($hash) . "&tld=" . urlencode($tldList) . "&q=" . urlencode($query);
382
383
384
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
385
            foreach ($answer as $el) {
386
                $hoster = $el[0];
387
                $hash = $el[1];
388

389
                foreach ($results as $result) {
390
                    if ($result->new && $hoster === $result->tld && !$result->partnershop) {
391
392
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
393
                        if ($result->image !== "") {
394
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . urlencode($hash) . ".gif";
395
                        } else {
396
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . urlencode($hash) . ".gif";
397
398
                        }

399
400
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
401
                        $targetUrl = $result->link;
402

403
                        $gateHash = md5($targetUrl . $privateKey);
404
                        $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . urlencode($publicKey) . "&k=" . urlencode($gateHash) . "&url=" . urlencode($targetUrl) . "&q=" . urlencode($query);
405
                        $result->link = $newLink;
406
407
408
409
                        $result->partnershop = true;
                    }
                }
            }
410
        } catch (\ErrorException $e) {
411
412
413
414
415
            return $results;
        }

        return $results;
    }
416

417
418
    public function humanVerification($results)
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
419
        # Let's check if we need to implement a redirect for human verification
420
421
        if ($this->verificationCount > 10) {
            foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
422
423
424
                $link = $result->link;
                $day = Carbon::now()->day;
                $pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
425
426
427
                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
Dominik Hebeler's avatar
Dominik Hebeler committed
428
                $result->link = $url;
429
                $result->proxyLink = $proxyUrl;
Dominik Hebeler's avatar
Dominik Hebeler committed
430
431
            }
            return $results;
432
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
433
434
435
436
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
437
438
439
440
441
    public function authorize($key)
    {
        $postdata = http_build_query(array(
            'dummy' => rand(),
        ));
Karl Hasselbring's avatar
Karl Hasselbring committed
442
443
444
445
446
447
        $opts = array(
            'http' => array(
                'method' => 'POST',
                'header' => 'Content-type: application/x-www-form-urlencoded',
                'content' => $postdata,
            ),
Phil Höfer's avatar
Phil Höfer committed
448
449
450
451
452
        );

        $context = stream_context_create($opts);

        try {
453
            $link = "https://key.metager3.de/" . urlencode($key) . "/request-permission/api-access";
Phil Höfer's avatar
Phil Höfer committed
454
455
456
457
458
459
460
461
462
463
464
            $result = json_decode(file_get_contents($link, false, $context));
            if ($result->{'api-access'} == true) {
                return true;
            } else {
                return false;
            }
        } catch (\ErrorException $e) {
            return false;
        }
    }

465
466
    public function createQuicktips()
    {
467
        # Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
468
        $quicktips = new \App\Models\Quicktips\Quicktips($this->q, LaravelLocalization::getCurrentLocale(), $this->getTime());
469
470
471
        return $quicktips;
    }

Karl's avatar
Karl committed
472
473
474
475
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

476
477
    public function createSearchEngines(Request $request)
    {
478
        # Wenn es kein Suchwort gibt
Dominik Hebeler's avatar
Dominik Hebeler committed
479
        if (!$request->filled("eingabe") || $this->q === "") {
480
            return;
481
        }
482

Dominik Hebeler's avatar
Dominik Hebeler committed
483
        $this->enabledSearchengines = [];
484
        $overtureEnabled = false;
485

486
487
488
        # Check if selected focus is valid
        if (empty($this->sumaFile->foki->{$this->fokus})) {
            $this->fokus = "web";
489
        }
490

Karl Hasselbring's avatar
Karl Hasselbring committed
491
        $sumaNames = $this->sumaFile->foki->{$this->fokus}->sumas;
492

Karl Hasselbring's avatar
Karl Hasselbring committed
493
494
495
        $sumas = [];
        foreach ($sumaNames as $sumaName) {
            $sumas[$sumaName] = $this->sumaFile->sumas->{$sumaName};
496
497
        }

Karl Hasselbring's avatar
Karl Hasselbring committed
498
499
500
        $this->removeAdsFromListIfAdfree($sumas);

        foreach ($sumas as $sumaName => $suma) {
501
            # Check if this engine is disabled and can't be used
Karl Hasselbring's avatar
Karl Hasselbring committed
502
503
504
505
506
507
508
            $disabled = empty($suma->disabled) ? false : $suma->disabled;
            $autoDisabled = empty($suma->{"auto-disabled"}) ? false : $suma->{"auto-disabled"};
            if (
                $disabled || $autoDisabled
                || \Cookie::get($this->getFokus() . "_engine_" . $sumaName) === "off"
            ) {
                continue;
509
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
510

511
            $valid = true;
Karl Hasselbring's avatar
Karl Hasselbring committed
512
513
514
515

            # Check if this engine can use potentially defined query-filter
            foreach ($this->queryFilter as $filterName => $filter) {
                if (empty($this->sumaFile->filter->{"query-filter"}->$filterName->sumas->$sumaName)) {
516
517
                    $valid = false;
                    break;
Dominik Hebeler's avatar
Dominik Hebeler committed
518
519
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
520
521

            # Check if this engine can use potentially defined parameter-filter
Dominik Hebeler's avatar
Dominik Hebeler committed
522
523
            if ($valid) {
                foreach ($this->parameterFilter as $filterName => $filter) {
524
                    # We need to check if the searchengine supports the parameter value, too
Karl Hasselbring's avatar
Karl Hasselbring committed
525
                    if (empty($filter->sumas->$sumaName) || empty($filter->sumas->{$sumaName}->values->{$filter->value})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
526
527
528
529
530
                        $valid = false;
                        break;
                    }
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
531

532
            # Check if this engine should only be active when filter is used
Karl Hasselbring's avatar
Karl Hasselbring committed
533
            if ($suma->{"filter-opt-in"}) {
534
535
536
                # This search engine should only be used when a parameter filter of it is used
                $validTmp = false;
                foreach ($this->parameterFilter as $filterName => $filter) {
Karl Hasselbring's avatar
Karl Hasselbring committed
537
                    if (!empty($filter->sumas->{$sumaName})) {
538
539
540
541
542
543
544
545
                        $validTmp = true;
                        break;
                    }
                }
                if (!$validTmp) {
                    $valid = false;
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
546
547

            # If the suma is still valid, we can add it
548
            if ($valid) {
Karl Hasselbring's avatar
Karl Hasselbring committed
549
                $this->enabledSearchengines[$sumaName] = $suma;
550
            }
551
        }
552

Karl Hasselbring's avatar
Karl Hasselbring committed
553
554
        # Include Yahoo Ads if Yahoo is not enabled as a searchengine
        if (!$this->apiAuthorized && $this->fokus != "bilder" && empty($this->enabledSearchengines["yahoo"]) && isset($this->sumaFile->sumas->{"yahoo-ads"})) {
555
556
557
            $this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
        }

558
559
560
561
562
563
        # Special case if search engines are disabled
        # Since bing is normally only active if a filter is set but it should be active, too if yahoo is disabled
        if ($this->getFokus() === "web" && empty($this->enabledSearchengines["yahoo"]) && \Cookie::get("web_engine_bing") !== "off") {
            $this->enabledSearchengines["bing"] = $this->sumaFile->sumas->{"bing"};
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
564
        if (sizeof($this->enabledSearchengines) === 0) {
565
566
            $filter = "";
            foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
Dominik Hebeler's avatar
Dominik Hebeler committed
567
                $filter .= trans($this->sumaFile->filter->{"query-filter"}->{$queryFilter}->name) . ",";
568
569
            }
            $filter = rtrim($filter, ",");
Karl Hasselbring's avatar
Karl Hasselbring committed
570
571
            $error = trans('metaGer.engines.noSpecialSearch', [
                'fokus' => trans($this->sumaFile->foki->{$this->fokus}->{"display-name"}),
572
                'filter' => $filter,
Karl Hasselbring's avatar
Karl Hasselbring committed
573
            ]);
574
            $this->errors[] = $error;
575
        }
576
577
        $this->setEngines($request);
    }
578

Karl Hasselbring's avatar
Karl Hasselbring committed
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
    private function removeAdsFromListIfAdfree(&$sumas)
    {
        if ($this->apiAuthorized) {
            foreach ($sumas as $sumaName => $suma) {
                $ads = $suma->ads ?? false;
                if ($ads) {
                    unset($sumas[$sumaName]);

                    $adBackups = $suma->{"ad-backups"} ?? [];
                    $adBackupName = $adBackups->{$this->fokus} ?? null;
                    if (isset($adBackupName)) {
                        $this->sumaFile->sumas->{$adBackupName}->{"filter-opt-in"} = false;
                    }
                }
            }
        }
    }

597
598
    public function setEngines(Request $request, $enabledSearchengines = [])
    {
599
        if ($this->requestIsCached($request)) {
600
            # If this is a page other than 1 the request is "cached"
601
            $engines = $this->getCachedEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
602
            # We need to edit some Options of the Cached Search Engines
Phil Höfer's avatar
Phil Höfer committed
603
            foreach ($engines as $engine) {
604
                $engine->setResultHash($this->getSearchUid());
Dominik Hebeler's avatar
Dominik Hebeler committed
605
            }
606
            $this->engines = $engines;
607
        } else {
608
609
610
            if (sizeof($enabledSearchengines) > 0) {
                $this->enabledSearchengines = $enabledSearchengines;
            }
611
            $this->actuallyCreateSearchEngines($this->enabledSearchengines);
612
        }
613
    }
Dominik Hebeler's avatar
Dominik Hebeler committed
614

615
616
    public function startSearch()
    {
617
        # Wir starten alle Suchen
618
        foreach ($this->engines as $engine) {
619
            $engine->startSearch($this);
620
        }
621
622
    }

623
624
    # Spezielle Suchen und Sumas

625
    public function sumaIsSelected($suma, $request, $custom)
626
    {
627
        if ($custom) {
628
            if ($request->filled("engine_" . strtolower($suma["name"]))) {
629
630
631
632
633
634
635
636
637
638
639
                return true;
            }
        } else {
            $types = explode(",", $suma["type"]);
            if (in_array($this->fokus, $types)) {
                return true;
            }
        }
        return false;
    }

640
    public function actuallyCreateSearchEngines($enabledSearchengines)
641
642
    {
        $engines = [];
643
        foreach ($enabledSearchengines as $engineName => $engine) {
644

645
            if (!isset($engine->{"parser-class"})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
646
647
                die(var_dump($engine));
            }
648
            # Setze Pfad zu Parser
649
            $path = "App\\Models\\parserSkripte\\" . $engine->{"parser-class"};
650
651

            # Prüfe ob Parser vorhanden
652
653
654
            if (!file_exists(app_path() . "/Models/parserSkripte/" . $engine->{"parser-class"} . ".php")) {
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen, da kein Parser existiert");
                $this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine->{"display-name"}]);
655
656
657
658
659
660
                continue;
            }

            # Es wird versucht die Suchengine zu erstellen
            $time = microtime();
            try {
661
                $tmp = new $path($engineName, $engine, $this);
662
            } catch (\ErrorException $e) {
663
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen. " . var_dump($e));
664
665
666
                continue;
            }

667
            $engines[] = $tmp;
668
        }
669
        $this->engines = $engines;
670
671
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
672
673
674
675
676
677
678
    public function getAvailableParameterFilter()
    {
        $parameterFilter = $this->sumaFile->filter->{"parameter-filter"};

        $availableFilter = [];

        foreach ($parameterFilter as $filterName => $filter) {
679
            $values = $filter->values;
Dominik Hebeler's avatar
Dominik Hebeler committed
680
681
682
            # Check if any of the enabled search engines provide this filter
            foreach ($this->enabledSearchengines as $engineName => $engine) {
                if (!empty($filter->sumas->$engineName)) {
683
684
685
686
687
688
689
690
691
692
                    if (empty($availableFilter[$filterName])) {
                        $availableFilter[$filterName] = $filter;
                        unset($availableFilter[$filterName]->values);
                    }
                    if (empty($availableFilter[$filterName]->values)) {
                        $availableFilter[$filterName]->values = (object) ["" => $values->{""}];
                    }
                    foreach ($filter->sumas->{$engineName}->values as $key => $value) {
                        $availableFilter[$filterName]->values->$key = $values->$key;
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
693
694
                }
            }
695
696
            # We will also add the filter from the opt-in search engines (the searchengines that are only used when a filter of it is too)
            foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
697
                if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"} && \Cookie::get($this->getFokus() . "_engine_" . $suma) !== "off") {
698
                    if (!empty($filter->sumas->{$suma})) {
699
700
                        # If the searchengine is disabled this filter shouldn't be available
                        if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
Karl Hasselbring's avatar
Karl Hasselbring committed
701
702
                            || (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)
                        ) {
703
704
                            continue;
                        }
705
706
707
708
709
710
711
712
713
714
                        if (empty($availableFilter[$filterName])) {
                            $availableFilter[$filterName] = $filter;
                            unset($availableFilter[$filterName]->values);
                        }
                        if (empty($availableFilter[$filterName]->values)) {
                            $availableFilter[$filterName]->values = (object) ["" => $values->{""}];
                        }
                        foreach ($filter->sumas->{$suma}->values as $key => $value) {
                            $availableFilter[$filterName]->values->$key = $values->$key;
                        }
715
716
717
                    }
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
718
719
        }

720
721
722
723
724
725
726
727
728
        # Set the current values for the filters
        foreach ($availableFilter as $filterName => $filter) {
            if (\Request::filled($filter->{"get-parameter"})) {
                $filter->value = \Request::input($filter->{"get-parameter"});
            } else if (\Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"}) !== null) {
                $filter->value = \Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"});
            }
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
729
730
731
        return $availableFilter;
    }

732
733
    public function isBildersuche()
    {
734
        return $this->fokus === "bilder";
735
736
737
738
    }

    public function sumaIsAdsuche($suma, $overtureEnabled)
    {
739
        $sumaName = $suma["name"]->__toString();
740
        return
741
742
            $sumaName === "qualigo"
            || $sumaName === "similar_product_ads"
Karl Hasselbring's avatar
Karl Hasselbring committed
743
            || (!$overtureEnabled && $sumaName === "overtureAds");
744
745
746
747
748
    }

    public function sumaIsDisabled($suma)
    {
        return
749
750
        isset($suma['disabled'])
        && $suma['disabled']->__toString() === "1";
751
752
753
754
755
    }

    public function sumaIsOverture($suma)
    {
        return
756
757
        $suma["name"]->__toString() === "overture"
        || $suma["name"]->__toString() === "overtureAds";
758
759
760
761
762
    }

    public function sumaIsNotAdsuche($suma)
    {
        return
763
764
765
        $suma["name"]->__toString() !== "qualigo"
        && $suma["name"]->__toString() !== "similar_product_ads"
        && $suma["name"]->__toString() !== "overtureAds";
766
767
768
769
770
    }

    public function requestIsCached($request)
    {
        return
771
772
773
        $request->filled('next')
        && Cache::has($request->input('next'))
        && unserialize(Cache::get($request->input('next')))['page'] > 1;
774
775
776
777
    }

    public function getCachedEngines($request)
    {
778
        $next = unserialize(Cache::get($request->input('next')));
779
        $this->page = $next['page'];
780
        $engines = $next['engines'];
781
782
783
        return $engines;
    }

784
    public function waitForMainResults()
785
    {
786
        $engines = $this->engines;
787
        $enginesToWaitFor = [];
788
789
790
        $mainEngines = $this->sumaFile->foki->{$this->fokus}->main;
        foreach ($mainEngines as $mainEngine) {
            foreach ($engines as $engine) {
Dominik Hebeler's avatar
Dominik Hebeler committed
791
                if ($engine->name === $mainEngine) {
792
793
                    $enginesToWaitFor[] = $engine;
                }
794
795
            }
        }
796

Phil Höfer's avatar
Phil Höfer committed
797
        $timeStart = microtime(true);
798
        $answered = [];
799
        $results = null;
800
801
802
803
804
805
806
807

        # If there is no main searchengine to wait for or if the only main engine is yahoo-ads we will define a timeout of 1s
        $forceTimeout = null;
        if (sizeof($enginesToWaitFor) === 0 || (sizeof($enginesToWaitFor) === 1 && $enginesToWaitFor[0]->name === "yahoo-ads")) {
            $forceTimeout = 1;
        }

        while (sizeof($enginesToWaitFor) > 0 || ($forceTimeout !== null && (microtime(true) - $timeStart) < $forceTimeout)) {
Dominik Hebeler's avatar
Dominik Hebeler committed
808
            foreach ($enginesToWaitFor as $index => $engine) {
809
                if (Redis::get($engine->hash) !== null) {
Dominik Hebeler's avatar
Dominik Hebeler committed
810
811
812
                    $answered[] = $engine;
                    unset($enginesToWaitFor[$index]);
                    break;
813
                }
814
            }
815
            if ((microtime(true) - $timeStart) >= 2) {
816
                break;
Dominik Hebeler's avatar
Dominik Hebeler committed
817
818
            } else {
                usleep(50 * 1000);
819
            }
820
        }
821
822

        # Now we can add an entry to Redis which defines the starting time and how many engines should answer this request
Dominik Hebeler's avatar
Dominik Hebeler committed
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
        /*
    $pipeline = $redis->pipeline();
    $pipeline->hset($this->getRedisEngineResult() . "status", "startTime", $timeStart);
    $pipeline->hset($this->getRedisEngineResult() . "status", "engineCount", sizeof($engines));
    $pipeline->hset($this->getRedisEngineResult() . "status", "engineDelivered", sizeof($answered));
    # Add the cached engines as answered
    foreach ($engines as $engine) {
    if ($engine->cached) {
    $pipeline->hincrby($this->getRedisEngineResult() . "status", "engineDelivered", 1);
    $pipeline->hincrby($this->getRedisEngineResult() . "status", "engineAnswered", 1);
    }
    }
    foreach ($answered as $engine) {
    $pipeline->hset($this->getRedisEngineResult() . $engine, "delivered", "1");
    }
    $pipeline->execute();*/
839
    }
840

841
    public function retrieveResults()
842
    {
843
        $engines = $this->engines;
844
        # Von geladenen Engines die Ergebnisse holen
845
846
847
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
                try {
848
                    $engine->retrieveResults($this);
849
                } catch (\ErrorException $e) {
850
851
852
                    Log::error($e);
                }
            }
853
854
855
            if (!empty($engine->totalResults) && $engine->totalResults > $this->totalResults) {
                $this->totalResults = $engine->totalResults;
            }
856
857
858
859
            if (!empty($engine->alteredQuery) && !empty($engine->alterationOverrideQuery)) {
                $this->alteredQuery = $engine->alteredQuery;
                $this->alterationOverrideQuery = $engine->alterationOverrideQuery;
            }
860
        }
861
862
    }

Karl Hasselbring's avatar
Karl Hasselbring committed
863
    /*
864
865
     * Ende Suchmaschinenerstellung und Ergebniserhalt
     */
Karl's avatar
Karl committed
866

867
868
    public function parseFormData(Request $request)
    {
869

870
        # Sichert, dass der request in UTF-8 formatiert ist
871
        if ($request->input('encoding', 'utf8') !== "utf8") {
872
873
874
875
876
877
878
            # In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
            $input = $request->all();
            foreach ($input as $key => $value) {
                $input[$key] = mb_convert_encoding("$value", "UTF-8", "ISO-8859-1");
            }
            $request->replace($input);
        }
879
        $this->url = $request->url();
880
        $this->fullUrl = $request->fullUrl();
881
        # Zunächst überprüfen wir die eingegebenen Einstellungen:
882
        # Fokus
883
        $this->fokus = $request->input('focus', 'web');
884
        # Suma-File
885
        if (App::isLocale("en")) {
886
            $this->sumaFile = config_path() . "/sumasEn.json";
887
        } else {
888
            $this->sumaFile = config_path() . "/sumas.json";
889
        }
890
        if (!file_exists($this->sumaFile)) {
891
            die(trans('metaGer.formdata.cantLoad'));
892
893
        } else {
            $this->sumaFile = json_decode(file_get_contents($this->sumaFile));
894
        }
895
        # Sucheingabe
896
        $th