MetaGer.php 70.1 KB
Newer Older
1
<?php
Karl Hasselbring's avatar
Karl Hasselbring committed
2

3
4
5
namespace App;

use App;
6
use Cache;
7
use Carbon;
Davide's avatar
Davide committed
8
use Cookie;
9
use Illuminate\Http\Request;
10
use Illuminate\Support\Facades\Redis;
11
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix    
Dominik Hebeler committed
12
use LaravelLocalization;
13
use Log;
14
use Monospice\LaravelRedisSentinel\RedisSentinel;
15
use Predis\Connection\ConnectionException;
16
17
18

class MetaGer
{
Dominik Hebeler's avatar
Dominik Hebeler committed
19
20
    const FETCHQUEUE_KEY = "fetcher.queue";

21
    # Einstellungen für die Suche
22
23
    public $alteredQuery = "";
    public $alterationOverrideQuery = "";
24
    protected $fokus;
Dominik Hebeler's avatar
update    
Dominik Hebeler committed
25
    protected $test;
26
27
28
29
30
31
    protected $eingabe;
    protected $q;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
32
    protected $time = 2000;
33
    protected $hostBlacklist = [];
34
    protected $domainBlacklist = [];
35
36
37
38
    private $urlBlacklist = [];
    protected $stopWords = [];
    protected $phrases = [];
    protected $engines = [];
39
    protected $totalResults = 0;
40
    protected $results = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
41
42
    protected $queryFilter = [];
    protected $parameterFilter = [];
43
    protected $ads = [];
44
    protected $infos = [];
45
46
47
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
48
    protected $availableFoki = [];
49
50
    protected $startCount = 0;
    protected $canCache = false;
51
    protected $javascript = false;
52
    # Daten über die Abfrage$
53
    protected $ip;
Dominik Hebeler's avatar
Dominik Hebeler committed
54
    protected $useragent;
55
56
    protected $language;
    protected $agent;
57
    protected $apiKey = "";
Phil Höfer's avatar
Phil Höfer committed
58
    protected $apiAuthorized = false;
59
    protected $next = [];
60
61
62
63
64
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
65
    protected $newtab;
66
    protected $domainsBlacklisted = [];
67
    protected $adDomainsBlacklisted = [];
68
    protected $urlsBlacklisted = [];
69
    protected $adUrlsBlacklisted = [];
70
    protected $blacklistDescriptionUrl = [];
71
    protected $url;
72
    protected $fullUrl;
Aria's avatar
Aria committed
73
    protected $enabledSearchengines = [];
74
    protected $languageDetect;
Dominik Hebeler's avatar
Dominik Hebeler committed
75
76
    protected $verificationId;
    protected $verificationCount;
77
    protected $searchUid;
78
79
80
81
    protected $redisResultWaitingKey;
    protected $redisResultEngineList;
    protected $redisEngineResult;
    protected $redisCurrentResultList;
Dominik Hebeler's avatar
Dominik Hebeler committed
82
    public $starttime;
Davide Aprea's avatar
Davide Aprea committed
83
    protected $dummy = false;
84

85
    public function __construct($hash = "")
86
    {
87
        # Timer starten
88
        $this->starttime = microtime(true);
89
        # Versuchen Blacklists einzulesen
90
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
91
            $tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
92
            $this->domainsBlacklisted = explode("\n", $tmp);
93
94
            $tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted = explode("\n", $tmp);
95
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
96

97
        # Versuchen Blacklists einzulesen
Dominik Hebeler's avatar
Dominik Hebeler committed
98
        if (file_exists(config_path() . "/adBlacklistDomains.txt")) {
99
100
            $tmp = file_get_contents(config_path() . "/adBlacklistDomains.txt");
            $this->adDomainsBlacklisted = explode("\n", $tmp);
Dominik Hebeler's avatar
Dominik Hebeler committed
101
102
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
103
        if (file_exists(config_path() . "/adBlacklistUrl.txt")) {
104
105
106
            $tmp = file_get_contents(config_path() . "/adBlacklistUrl.txt");
            $this->adUrlsBlacklisted = explode("\n", $tmp);
        }
107

Dominik Hebeler's avatar
Dominik Hebeler committed
108
        if (file_exists(config_path() . "/blacklistDescriptionUrl.txt")) {
109
110
111
112
            $tmp = file_get_contents(config_path() . "/blacklistDescriptionUrl.txt");
            $this->blacklistDescriptionUrl = explode("\n", $tmp);
        }

113
        # Parser Skripte einhängen
114
115
116
117
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
118
                require_once $path;
119
120
121
            }
        }

122
        # Cachebarkeit testen
123
124
125
126
127
128
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
129
130
131
132
133
        if ($hash === "") {
            $this->searchUid = md5(uniqid());
        } else {
            $this->searchUid = $hash;
        }
134
135
136
137
138
139
140
        $redisPrefix = "search";
        # This is a list on which the MetaGer process can do a blocking pop to wait for new results
        $this->redisResultWaitingKey = $redisPrefix . "." . $this->searchUid . ".ready";
        # This is a list of searchengines which have delivered results for this search
        $this->redisResultEngineList = $redisPrefix . "." . $this->searchUid . ".engines";
        # This is the key where the results of the engine are stored as well as some statistical data
        $this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results.";
141
142
        # A list of all search results already delivered to the user (sorted of course)
        $this->redisCurrentResultList = $redisPrefix . "." . $this->searchUid . ".currentResults";
143
    }
144

145
    # Erstellt aus den gesammelten Ergebnissen den View
146
    public function createView($quicktipResults = [])
147
    {
148
149
        # Hiermit werden die evtl. ausgewählten SuMas extrahiert, damit die Input-Boxen richtig gesetzt werden können
        $focusPages = [];
Aria Givi's avatar
Aria Givi committed
150

151
        foreach ($this->request->all() as $key => $value) {
152
            if (stripos($key, 'engine_') === 0 && $value === 'on') {
153
                $focusPages[] = $key;
154
155
156
            }
        }

157
        $viewResults = [];
158
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
159
        foreach ($this->results as $result) {
160
161
162
163
            $viewResults[] = get_object_vars($result);
        }
        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();
164
165
        if ($this->fokus === "bilder") {
            switch ($this->out) {
166
                case 'results':
167
                    return view('resultpages.results_images')
168
169
170
171
172
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
173
                        ->with('apiAuthorized', $this->apiAuthorized)
174
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
175
                        ->with('imagesearch', true)
176
                        ->with('browser', (new Agent())->browser());
177
                default:
178
                    return view('resultpages.resultpage_images')
179
180
181
182
183
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
184
                        ->with('apiAuthorized', $this->apiAuthorized)
185
                        ->with('metager', $this)
186
                        ->with('browser', (new Agent())->browser())
187
                        ->with('quicktips', $quicktipResults)
188
                        ->with('focus', $this->fokus)
Dominik Hebeler's avatar
Dominik Hebeler committed
189
                        ->with('imagesearch', true)
190
                        ->with('resultcount', count($this->results));
191
            }
192
193
194
        } else {
            switch ($this->out) {
                case 'results':
195
                    return view('resultpages.results')
196
197
198
199
200
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
201
                        ->with('apiAuthorized', $this->apiAuthorized)
202
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
203
204
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
205
206
                    break;
                case 'results-with-style':
207
                    return view('resultpages.resultpage')
208
209
210
211
212
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
213
                        ->with('apiAuthorized', $this->apiAuthorized)
214
215
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
Dominik Hebeler's avatar
Dominik Hebeler committed
216
217
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
218
                    break;
Dominik Hebeler's avatar
Dominik Hebeler committed
219
220
221
222
223
224
225
226
227
                case 'rss20':
                    return view('resultpages.metager3resultsrss20')
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('apiAuthorized', $this->apiAuthorized)
                        ->with('metager', $this)
                        ->with('resultcount', sizeof($viewResults))
                        ->with('fokus', $this->fokus);
                    break;
228
                case 'api':
Dominik Hebeler's avatar
Dominik Hebeler committed
229
                    return view('resultpages.metager3resultsatom10', ['eingabe' => $this->eingabe, 'resultcount' => sizeof($viewResults), 'key' => $this->apiKey, 'metager' => $this]);
230
                    break;
231
                case 'atom10':
Dominik Hebeler's avatar
Dominik Hebeler committed
232
                    return view('resultpages.metager3resultsatom10', ['eingabe' => $this->eingabe, 'resultcount' => sizeof($viewResults), 'key' => $this->apiKey, 'metager' => $this]);
Davide Aprea's avatar
Davide Aprea committed
233
                    break;
234
                case 'result-count':
235
236
                    # Wir geben die Ergebniszahl und die benötigte Zeit zurück:
                    return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
237
                    break;
238
                default:
239
                    return view('resultpages.resultpage')
240
                        ->with('eingabe', $this->eingabe)
241
                        ->with('focusPages', $focusPages)
242
243
244
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
245
                        ->with('apiAuthorized', $this->apiAuthorized)
246
                        ->with('metager', $this)
247
                        ->with('browser', (new Agent())->browser())
248
                        ->with('quicktips', $quicktipResults)
249
250
                        ->with('resultcount', count($this->results))
                        ->with('focus', $this->fokus);
251
252
                    break;
            }
253
254
255
        }
    }

256
    public function prepareResults(&$timings = null)
257
    {
Phil Höfer's avatar
Phil Höfer committed
258
259
        $engines = $this->engines;
        // combine
260
        $this->combineResults($engines);
261
        if (!empty($timings)) {
262
263
            $timings["prepareResults"]["combined results"] = microtime(true) - $timings["starttime"];
        }
Phil Höfer's avatar
Phil Höfer committed
264
        // misc (WiP)
265
        if ($this->fokus == "nachrichten") {
266
267
268
            $this->results = array_filter($this->results, function ($v, $k) {
                return !is_null($v->getRank());
            }, ARRAY_FILTER_USE_BOTH);
269
270
271
272
273
274
275
276
277
278
            uasort($this->results, function ($a, $b) {
                $datea = $a->getDate();
                $dateb = $b->getDate();
                return $dateb - $datea;
            });
        } else {
            uasort($this->results, function ($a, $b) {
                if ($a->getRank() == $b->getRank()) {
                    return 0;
                }
279

280
281
282
                return ($a->getRank() < $b->getRank()) ? 1 : -1;
            });
        }
283
        if (!empty($timings)) {
284
285
            $timings["prepareResults"]["sorted results"] = microtime(true) - $timings["starttime"];
        }
286
287
        # Validate Results
        $newResults = [];
288
289
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
290
                $newResults[] = $result;
291
            }
292
293
        }
        $this->results = $newResults;
294
        if (!empty($timings)) {
295
296
            $timings["prepareResults"]["validated results"] = microtime(true) - $timings["starttime"];
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
297

Davide Aprea's avatar
Davide Aprea committed
298
        $this->duplicationCheck();
Dominik Hebeler's avatar
Dominik Hebeler committed
299
        if (!empty($timings)) {
300
301
            $timings["prepareResults"]["duplications checked"] = microtime(true) - $timings["starttime"];
        }
302
303
304
305
        # Validate Advertisements
        $newResults = [];
        foreach ($this->ads as $ad) {
            if (($ad->strippedHost !== "" && (in_array($ad->strippedHost, $this->adDomainsBlacklisted) ||
Dominik Hebeler's avatar
Dominik Hebeler committed
306
307
                    in_array($ad->strippedLink, $this->adUrlsBlacklisted))) || ($ad->strippedHostAnzeige !== "" && (in_array($ad->strippedHostAnzeige, $this->adDomainsBlacklisted) ||
                    in_array($ad->strippedLinkAnzeige, $this->adUrlsBlacklisted)))
Karl Hasselbring's avatar
Karl Hasselbring committed
308
            ) {
309
310
311
312
                continue;
            }
            $newResults[] = $ad;
        }
313

314
        $this->ads = $newResults;
315
        if (!empty($timings)) {
316
317
            $timings["prepareResults"]["validated ads"] = microtime(true) - $timings["starttime"];
        }
318
        #Adgoal Implementation
319
320
321
        if (empty($this->adgoalLoaded)) {
            $this->adgoalLoaded = false;
        }
322
323

        if (!empty($this->jskey)) {
324
            $js = Redis::connection(config('cache.stores.redis.connection'))->lpop("js" . $this->jskey);
325
326
            if ($js !== null && boolval($js)) {
                $this->javascript = true;
327
            }
328
        }
329

Dominik Hebeler's avatar
Dominik Hebeler committed
330
        # Human Verification
331
332
        $this->humanVerification($this->results);
        $this->humanVerification($this->ads);
333
        if (!empty($timings)) {
334
335
            $timings["prepareResults"]["human verification"] = microtime(true) - $timings["starttime"];
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
336

337
        $counter = 0;
338
        $firstRank = 0;
339

340
        if (count($this->results) <= 0) {
341
342
343
344
345
346
            if (strlen($this->site) > 0) {
                $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
                $this->errors[] = trans('metaGer.results.failedSitesearch', ['altSearch' => $no_sitesearch_query]);
            } else {
                $this->errors[] = trans('metaGer.results.failed');
            }
347
        }
348

349
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
350
            $page = $this->page + 1;
351
            $this->next = [
352
353
                'page' => $page,
                'engines' => $this->next,
354
            ];
355
            Cache::put($this->getSearchUid(), serialize($this->next), 60 * 60);
356
            if (!empty($timings)) {
357
358
                $timings["prepareResults"]["filled cache"] = microtime(true) - $timings["starttime"];
            }
359
360
        } else {
            $this->next = [];
361
        }
362
    }
363

Phil Höfer's avatar
Phil Höfer committed
364
365
366
367
368
369
370
371
372
373
374
    public function combineResults($engines)
    {
        foreach ($engines as $engine) {
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
            foreach ($engine->results as $result) {
                if ($result->valid) {
375
                    $this->results[] = clone $result;
Phil Höfer's avatar
Phil Höfer committed
376
377
378
                }
            }
            foreach ($engine->ads as $ad) {
379
                $this->ads[] = clone $ad;
Phil Höfer's avatar
Phil Höfer committed
380
381
382
383
            }
        }
    }

Davide Aprea's avatar
Davide Aprea committed
384
385
386
    public function duplicationCheck()
    {
        $arr = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
387
        for ($i = 0; $i < count($this->results); $i++) {
Davide Aprea's avatar
Davide Aprea committed
388
389
390
391
392
            $link = $this->results[$i]->link;

            if (strpos($link, "http://") === 0) {
                $link = substr($link, 7);
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
393

Davide Aprea's avatar
Davide Aprea committed
394
395
396
            if (strpos($link, "https://") === 0) {
                $link = substr($link, 8);
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
397

Davide Aprea's avatar
Davide Aprea committed
398
399
400
            if (strpos($link, "www.") === 0) {
                $link = substr($link, 4);
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
401

Davide Aprea's avatar
Davide Aprea committed
402
403
            $link = trim($link, "/");

Dominik Hebeler's avatar
Dominik Hebeler committed
404
            if (isset($arr[$link])) {
Davide Aprea's avatar
Davide Aprea committed
405
406
                $arr[$link]->gefVon[] = $this->results[$i]->gefVon[0];
                $arr[$link]->gefVonLink[] = $this->results[$i]->gefVonLink[0];
407
408

                // The duplicate might already be an adgoal partnershop
Dominik Hebeler's avatar
Dominik Hebeler committed
409
                if ($this->results[$i]->partnershop) {
410
411
412
413
414
415
416
                    # Den Link hinzufügen:
                    $arr[$link]->logo = $this->results[$i]->logo;
                    $arr[$link]->image = $this->results[$i]->image;
                    $arr[$link]->link = $this->results[$i]->link;
                    $arr[$link]->partnershop = $this->results[$i]->partnershop;
                }

Davide Aprea's avatar
Davide Aprea committed
417
418
                array_splice($this->results, $i, 1);
                $i--;
Dominik Hebeler's avatar
Dominik Hebeler committed
419
                if ($arr[$link]->new === true || $this->results[$i]->new === true) {
Dominik Hebeler's avatar
Dominik Hebeler committed
420
                    $arr[$link]->changed = true;
Davide Aprea's avatar
Davide Aprea committed
421
                }
Dominik Hebeler's avatar
Dominik Hebeler committed
422
            } else {
Davide Aprea's avatar
Davide Aprea committed
423
424
425
426
427
                $arr[$link] = &$this->results[$i];
            }
        }
    }

428
429
430
431
432
    /**
     * @param \App\Models\Admitad[] $admitads
     * @param Boolean $wait Wait for Results?
     * @return Boolean whether or not all Admitad Objects are finished
     */
Dominik Hebeler's avatar
Dominik Hebeler committed
433
434
    public function parseAffiliates(&$affiliates)
    {
435
436
        $wait = false;
        $finished = true;
Dominik Hebeler's avatar
Dominik Hebeler committed
437
        if (!$this->javascript) {
438
439
            $wait = true;
        }
440
441
442
        foreach ($affiliates as $affiliate) {
            $affiliate->fetchAffiliates($wait);
            $affiliate->parseAffiliates($this->results);
Dominik Hebeler's avatar
Dominik Hebeler committed
443
            if (!$affiliate->finished) {
444
445
446
447
                $finished = false;
            }
        }
        return $finished;
448
    }
449

450

451
452
453
454
455
456
457
458
459
460
461
    /**
     * Modifies the already filled array of advertisements and
     * includes an advertisement for our donation page.
     * 
     * It will do so everytime when there are other advertisments to mix it in
     * and only in a percentage of cases when there are no other advertisements.
     * 
     * The Position at which our advertisement is placed is random within the other
     * advertisements. In some cases it will be the first ad and in other cases in some
     * other place.
     */
462
463
    public function addDonationAdvertisement()
    {
464
465
466
467
468
469
470
471
        /**
         * If there are no other advertisements we will only display our advertisements 
         * every so often. ~33% in this case
         */
        if (sizeof($this->ads) === 0 && rand(1, 100) > 33) {
            return;
        }

472
473
474
475
476
477
478
479
480
481
482
483
        $donationAd = new \App\Models\Result(
            "MetaGer",
            __("metaGer.ads.own.title"),
            LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), route("spende")),
            LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), route("spende")),
            __("metaGer.ads.own.description"),
            "MetaGer",
            "https://metager.de",
            1
        );
        $adCount = sizeof($this->ads);
        // Put Donation Advertisement to random position
484
        $position = random_int(0, $adCount);
485

486
        array_splice($this->ads, $position, 0, [$donationAd]);
487
488
    }

489
    public function humanVerification(&$results)
490
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
491
        # Let's check if we need to implement a redirect for human verification
492
493
        if ($this->verificationCount > 10) {
            foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
494
495
                $link = $result->link;
                $day = Carbon::now()->day;
496
                $pw = md5($this->verificationId . $day . $link . config("metager.metager.proxy.password"));
497
                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
498
                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . config("metager.metager.proxy.password"));
499
                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
Dominik Hebeler's avatar
Dominik Hebeler committed
500
                $result->link = $url;
501
                $result->proxyLink = $proxyUrl;
Dominik Hebeler's avatar
Dominik Hebeler committed
502
503
504
505
            }
        }
    }

Phil Höfer's avatar
Phil Höfer committed
506
507
    public function authorize($key)
    {
508
        return app('App\Models\Key')->requestPermission();
Phil Höfer's avatar
Phil Höfer committed
509
510
    }

Karl's avatar
Karl committed
511
512
513
514
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

Dominik Hebeler's avatar
Dominik Hebeler committed
515
    public function createSearchEngines(Request $request, &$timings)
516
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
517
518
519
520
        if (!empty($timings)) {
            $timings["createSearchEngines"]["start"] = microtime(true) - $timings["starttime"];
        }

521
        # Wenn es kein Suchwort gibt
Dominik Hebeler's avatar
Dominik Hebeler committed
522
        if (!$request->filled("eingabe") || $this->q === "") {
523
            return;
524
        }
525

Dominik Hebeler's avatar
Dominik Hebeler committed
526
        $this->enabledSearchengines = [];
527
        $overtureEnabled = false;
528

529
530
531
        # Check if selected focus is valid
        if (empty($this->sumaFile->foki->{$this->fokus})) {
            $this->fokus = "web";
532
        }
533

Karl Hasselbring's avatar
Karl Hasselbring committed
534
        $sumaNames = $this->sumaFile->foki->{$this->fokus}->sumas;
535

Karl Hasselbring's avatar
Karl Hasselbring committed
536
537
538
        $sumas = [];
        foreach ($sumaNames as $sumaName) {
            $sumas[$sumaName] = $this->sumaFile->sumas->{$sumaName};
539
540
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
541
542
543
544
        if (!empty($timings)) {
            $timings["createSearchEngines"]["created engine array"] = microtime(true) - $timings["starttime"];
        }

Karl Hasselbring's avatar
Karl Hasselbring committed
545
546
        $this->removeAdsFromListIfAdfree($sumas);

Dominik Hebeler's avatar
Dominik Hebeler committed
547
548
549
550
        if (!empty($timings)) {
            $timings["createSearchEngines"]["removed ads"] = microtime(true) - $timings["starttime"];
        }

Karl Hasselbring's avatar
Karl Hasselbring committed
551
        foreach ($sumas as $sumaName => $suma) {
552
            # Check if this engine is disabled and can't be used
Karl Hasselbring's avatar
Karl Hasselbring committed
553
554
555
556
557
558
559
            $disabled = empty($suma->disabled) ? false : $suma->disabled;
            $autoDisabled = empty($suma->{"auto-disabled"}) ? false : $suma->{"auto-disabled"};
            if (
                $disabled || $autoDisabled
                || \Cookie::get($this->getFokus() . "_engine_" . $sumaName) === "off"
            ) {
                continue;
560
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
561

562
            $valid = true;
Karl Hasselbring's avatar
Karl Hasselbring committed
563
564
565
566

            # Check if this engine can use potentially defined query-filter
            foreach ($this->queryFilter as $filterName => $filter) {
                if (empty($this->sumaFile->filter->{"query-filter"}->$filterName->sumas->$sumaName)) {
567
568
                    $valid = false;
                    break;
Dominik Hebeler's avatar
Dominik Hebeler committed
569
570
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
571
572

            # Check if this engine can use potentially defined parameter-filter
Dominik Hebeler's avatar
Dominik Hebeler committed
573
574
            if ($valid) {
                foreach ($this->parameterFilter as $filterName => $filter) {
575
                    # We need to check if the searchengine supports the parameter value, too
Karl Hasselbring's avatar
Karl Hasselbring committed
576
                    if (empty($filter->sumas->$sumaName) || empty($filter->sumas->{$sumaName}->values->{$filter->value})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
577
578
579
580
581
                        $valid = false;
                        break;
                    }
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
582

583
            # Check if this engine should only be active when filter is used
Karl Hasselbring's avatar
Karl Hasselbring committed
584
            if ($suma->{"filter-opt-in"}) {
585
586
587
                # This search engine should only be used when a parameter filter of it is used
                $validTmp = false;
                foreach ($this->parameterFilter as $filterName => $filter) {
Karl Hasselbring's avatar
Karl Hasselbring committed
588
                    if (!empty($filter->sumas->{$sumaName})) {
589
590
591
592
593
594
595
596
                        $validTmp = true;
                        break;
                    }
                }
                if (!$validTmp) {
                    $valid = false;
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
597
598

            # If the suma is still valid, we can add it
599
            if ($valid) {
Karl Hasselbring's avatar
Karl Hasselbring committed
600
                $this->enabledSearchengines[$sumaName] = $suma;
601
            }
602
        }
603

Dominik Hebeler's avatar
Dominik Hebeler committed
604
605
606
607
        if (!empty($timings)) {
            $timings["createSearchEngines"]["filtered invalid engines"] = microtime(true) - $timings["starttime"];
        }

Karl Hasselbring's avatar
Karl Hasselbring committed
608
609
        # Include Yahoo Ads if Yahoo is not enabled as a searchengine
        if (!$this->apiAuthorized && $this->fokus != "bilder" && empty($this->enabledSearchengines["yahoo"]) && isset($this->sumaFile->sumas->{"yahoo-ads"})) {
610
611
612
            $this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
        }

613
614
        # Special case if search engines are disabled
        # Since bing is normally only active if a filter is set but it should be active, too if yahoo is disabled
Davide Aprea's avatar
Davide Aprea committed
615
        if ($this->getFokus() === "web" && empty($this->enabledSearchengines["yahoo"]) && \Cookie::get("web_engine_bing") !== "off"  && isset($this->sumaFile->sumas->{"bing"})) {
616
617
618
            $this->enabledSearchengines["bing"] = $this->sumaFile->sumas->{"bing"};
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
619
        if (sizeof($this->enabledSearchengines) === 0) {
620
621
            $filter = "";
            foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
Dominik Hebeler's avatar
Dominik Hebeler committed
622
                $filter .= trans($this->sumaFile->filter->{"query-filter"}->{$queryFilter}->name) . ",";
623
624
            }
            $filter = rtrim($filter, ",");
Karl Hasselbring's avatar
Karl Hasselbring committed
625
626
            $error = trans('metaGer.engines.noSpecialSearch', [
                'fokus' => trans($this->sumaFile->foki->{$this->fokus}->{"display-name"}),
627
                'filter' => $filter,
Karl Hasselbring's avatar
Karl Hasselbring committed
628
            ]);
629
            $this->errors[] = $error;
630
        }
631
        $this->setEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
632
633
634
        if (!empty($timings)) {
            $timings["createSearchEngines"]["saved engines"] = microtime(true) - $timings["starttime"];
        }
635
    }
636

Karl Hasselbring's avatar
Karl Hasselbring committed
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
    private function removeAdsFromListIfAdfree(&$sumas)
    {
        if ($this->apiAuthorized) {
            foreach ($sumas as $sumaName => $suma) {
                $ads = $suma->ads ?? false;
                if ($ads) {
                    unset($sumas[$sumaName]);

                    $adBackups = $suma->{"ad-backups"} ?? [];
                    $adBackupName = $adBackups->{$this->fokus} ?? null;
                    if (isset($adBackupName)) {
                        $this->sumaFile->sumas->{$adBackupName}->{"filter-opt-in"} = false;
                    }
                }
            }
        }
    }

655
656
    public function setEngines(Request $request, $enabledSearchengines = [])
    {
657
        if ($this->requestIsCached($request)) {
658
            # If this is a page other than 1 the request is "cached"
659
            $engines = $this->getCachedEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
660
            # We need to edit some Options of the Cached Search Engines
Phil Höfer's avatar
Phil Höfer committed
661
            foreach ($engines as $engine) {
662
                $engine->setResultHash($this->getSearchUid());
Dominik Hebeler's avatar
Dominik Hebeler committed
663
            }
664
            $this->engines = $engines;
665
        } else {
666
667
668
            if (sizeof($enabledSearchengines) > 0) {
                $this->enabledSearchengines = $enabledSearchengines;
            }
669
            $this->actuallyCreateSearchEngines($this->enabledSearchengines);
670
        }
671
    }
Dominik Hebeler's avatar
Dominik Hebeler committed
672

Dominik Hebeler's avatar
Dominik Hebeler committed
673
    public function startSearch(&$timings)
674
    {
675
676
677
678
679
        if (!empty($timings)) {
            $timings["startSearch"]["start"] = microtime(true) - $timings["starttime"];
        }

        # Check all engines for Cached responses
Dominik Hebeler's avatar
Dominik Hebeler committed
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
        $this->checkCache();

        if (!empty($timings)) {
            $timings["startSearch"]["cache checked"] = microtime(true) - $timings["starttime"];
        }

        # Wir starten alle Suchen
        foreach ($this->engines as $engine) {
            $engine->startSearch($this, $timings);
        }
        if (!empty($timings)) {
            $timings["startSearch"]["searches started"] = microtime(true) - $timings["starttime"];
        }
    }

    public function checkCache()
    {
697
698
        if ($this->canCache()) {
            foreach ($this->engines as $engine) {
Dominik Hebeler's avatar
Dominik Hebeler committed
699
                if (Cache::has($engine->hash)) {
700
                    $engine->cached = true;
701
                    $engine->retrieveResults($this, Cache::get($engine->hash));
702
703
704
                }
            }
        }
705
706
    }

707
708
    # Spezielle Suchen und Sumas

709
    public function sumaIsSelected($suma, $request, $custom)
710
    {
711
        if ($custom) {
712
            if ($request->filled("engine_" . strtolower($suma["name"]))) {
713
714
715
716
717
718
719
720
721
722
723
                return true;
            }
        } else {
            $types = explode(",", $suma["type"]);
            if (in_array($this->fokus, $types)) {
                return true;
            }
        }
        return false;
    }

724
    public function actuallyCreateSearchEngines($enabledSearchengines)
725
726
    {
        $engines = [];
727
728
        foreach ($enabledSearchengines as $engineName => $engine) {
            if (!isset($engine->{"parser-class"})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
729
730
                die(var_dump($engine));
            }
731
            # Setze Pfad zu Parser
732
            $path = "App\\Models\\parserSkripte\\" . $engine->{"parser-class"};
733
734

            # Prüfe ob Parser vorhanden
735
736
737
            if (!file_exists(app_path() . "/Models/parserSkripte/" . $engine->{"parser-class"} . ".php")) {
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen, da kein Parser existiert");
                $this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine->{"display-name"}]);
738
739
740
741
742
743
                continue;
            }

            # Es wird versucht die Suchengine zu erstellen
            $time = microtime();
            try {
744
                $tmp = new $path($engineName, $engine, $this);
745
            } catch (\ErrorException $e) {
746
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen. " . $e);
747
748
749
                continue;
            }

750
            $engines[] = $tmp;
751
        }
752
        $this->engines = $engines;
753
754
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
755
756
757
758
759
760
761
    public function getAvailableParameterFilter()
    {
        $parameterFilter = $this->sumaFile->filter->{"parameter-filter"};

        $availableFilter = [];

        foreach ($parameterFilter as $filterName => $filter) {
762
            $values = clone $filter->values;
Dominik Hebeler's avatar
Dominik Hebeler committed
763
764
765
            # Check if any of the enabled search engines provide this filter
            foreach ($this->enabledSearchengines as $engineName => $engine) {
                if (!empty($filter->sumas->$engineName)) {
766
767
                    if (empty($availableFilter[$filterName])) {
                        $availableFilter[$filterName] = $filter;
768
769
                        foreach ($availableFilter[$filterName]->values as $key => $value) {
                            if ($key !== "nofilter") {
770
771
772
                                unset($availableFilter[$filterName]->values->{$key});
                            }
                        }
773
774
                    }
                    if (empty($availableFilter[$filterName]->values)) {
775
                        $availableFilter[$filterName]->values = new \stdClass();
776
777
                    }
                    foreach ($filter->sumas->{$engineName}->values as $key => $value) {
778
                        $availableFilter[$filterName]->values->{$key} = $values->$key;
779
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
780
781
                }
            }
782
783
            # We will also add the filter from the opt-in search engines (the searchengines that are only used when a filter of it is too)
            foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
784
                if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"} && \Cookie::get($this->getFokus() . "_engine_" . $suma) !== "off") {
785
                    if (!empty($filter->sumas->{$suma})) {
786
787
                        # If the searchengine is disabled this filter shouldn't be available
                        if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
Karl Hasselbring's avatar
Karl Hasselbring committed
788
789
                            || (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)
                        ) {
790
791
                            continue;
                        }
792
793
                        if (empty($availableFilter[$filterName])) {
                            $availableFilter[$filterName] = $filter;
794
795
                            foreach ($availableFilter[$filterName]->values as $key => $value) {
                                if ($key !== "nofilter") {
796
797
798
                                    unset($availableFilter[$filterName]->values->{$key});
                                }
                            }
799
800
                        }
                        if (empty($availableFilter[$filterName]->values)) {
801
                            $availableFilter[$filterName]->values = new \stdClass();
802
803
                        }
                        foreach ($filter->sumas->{$suma}->values as $key => $value) {
804
                            $availableFilter[$filterName]->values->{$key} = $values->$key;
805
                        }
806
807
808
                    }
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
809
810
        }

811
812
813
814
        # Set the current values for the filters
        foreach ($availableFilter as $filterName => $filter) {
            if (\Request::filled($filter->{"get-parameter"})) {
                $filter->value = \Request::input($filter->{"get-parameter"});
815
            } elseif (\Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"}) !== null) {
816
817
818
819
                $filter->value = \Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"});
            }
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
820
821
822
        return $availableFilter;
    }

823
824
    public function isBildersuche()
    {
825
        return $this->fokus === "bilder";
826
827
828
829
    }

    public function sumaIsAdsuche($suma, $overtureEnabled)
    {
830
        $sumaName = $suma["name"]->__toString();
831
        return
832
833
            $sumaName === "qualigo"
            || $sumaName === "similar_product_ads"
Karl Hasselbring's avatar
Karl Hasselbring committed
834
            || (!$overtureEnabled && $sumaName === "overtureAds");
835
836
837
838
839
    }

    public function sumaIsDisabled($suma)
    {
        return
Dominik Hebeler's avatar
Dominik Hebeler committed
840
841
            isset($suma['disabled'])
            && $suma['disabled']->__toString() === "1";
842
843
844
845
846
    }

    public function sumaIsOverture($suma)
    {
        return
Dominik Hebeler's avatar
Dominik Hebeler committed
847
848
            $suma["name"]->__toString() === "overture"
            || $suma["name"]->__toString() === "overtureAds";
849
850
851
852
853
    }

    public function sumaIsNotAdsuche($suma)
    {
        return
Dominik Hebeler's avatar
Dominik Hebeler committed
854
855
856
            $suma["name"]->__toString() !== "qualigo"
            && $suma["name"]->__toString() !== "similar_product_ads"
            && $suma["name"]->__toString() !== "overtureAds";
857
858
859
860
861
    }

    public function requestIsCached($request)
    {
        return
Dominik Hebeler's avatar
Dominik Hebeler committed
862
863
864
            $request->filled('next')
            && Cache::has($request->input('next'))
            && unserialize(Cache::get($request->input('next')))['page'] > 1;
865
866
867
868
    }

    public function getCachedEngines($request)
    {
869
        $next = unserialize(Cache::get($request->input('next')));