MetaGer.php 71.5 KB
Newer Older
1
<?php
Karl Hasselbring's avatar
Karl Hasselbring committed
2

3
4
5
namespace App;

use App;
6
use Cache;
7
use Carbon;
Davide's avatar
Davide committed
8
use Cookie;
9
use Illuminate\Http\Request;
10
use Illuminate\Support\Facades\Redis;
11
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix    
Dominik Hebeler committed
12
use LaravelLocalization;
13
use Log;
14
use Monospice\LaravelRedisSentinel\RedisSentinel;
15
use Predis\Connection\ConnectionException;
16
17
18

class MetaGer
{
Dominik Hebeler's avatar
Dominik Hebeler committed
19
20
    const FETCHQUEUE_KEY = "fetcher.queue";

21
    # Einstellungen für die Suche
22
23
    public $alteredQuery = "";
    public $alterationOverrideQuery = "";
24
    protected $fokus;
Dominik Hebeler's avatar
update    
Dominik Hebeler committed
25
    protected $test;
26
27
28
29
30
31
    protected $eingabe;
    protected $q;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
32
    protected $time = 2000;
33
    protected $hostBlacklist = [];
34
    protected $domainBlacklist = [];
35
36
37
38
    private $urlBlacklist = [];
    protected $stopWords = [];
    protected $phrases = [];
    protected $engines = [];
39
    protected $totalResults = 0;
40
    protected $results = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
41
42
    protected $queryFilter = [];
    protected $parameterFilter = [];
43
    protected $ads = [];
44
    protected $infos = [];
45
46
47
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
48
    protected $availableFoki = [];
49
50
    protected $startCount = 0;
    protected $canCache = false;
51
    # Daten über die Abfrage$
52
    protected $ip;
Dominik Hebeler's avatar
Dominik Hebeler committed
53
    protected $useragent;
54
55
    protected $language;
    protected $agent;
56
    protected $apiKey = "";
Phil Höfer's avatar
Phil Höfer committed
57
    protected $apiAuthorized = false;
58
    protected $next = [];
59
60
61
62
63
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
64
    protected $newtab;
65
    protected $domainsBlacklisted = [];
66
    protected $adDomainsBlacklisted = [];
67
    protected $urlsBlacklisted = [];
68
    protected $adUrlsBlacklisted = [];
69
    protected $url;
70
    protected $fullUrl;
Aria's avatar
Aria committed
71
    protected $enabledSearchengines = [];
72
    protected $languageDetect;
Dominik Hebeler's avatar
Dominik Hebeler committed
73
74
    protected $verificationId;
    protected $verificationCount;
75
    protected $searchUid;
76
77
78
79
    protected $redisResultWaitingKey;
    protected $redisResultEngineList;
    protected $redisEngineResult;
    protected $redisCurrentResultList;
Dominik Hebeler's avatar
Dominik Hebeler committed
80
    public $starttime;
81

82
    public function __construct($hash = "")
83
    {
84
        # Timer starten
85
        $this->starttime = microtime(true);
86
        # Versuchen Blacklists einzulesen
87
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
88
            $tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
89
            $this->domainsBlacklisted = explode("\n", $tmp);
90
91
            $tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted = explode("\n", $tmp);
92
        } else {
93
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
94
        }
95
96
97
98
99
100
101
102
103
        # Versuchen Blacklists einzulesen
        if (file_exists(config_path() . "/adBlacklistDomains.txt") && file_exists(config_path() . "/adBlacklistUrl.txt")) {
            $tmp = file_get_contents(config_path() . "/adBlacklistDomains.txt");
            $this->adDomainsBlacklisted = explode("\n", $tmp);
            $tmp = file_get_contents(config_path() . "/adBlacklistUrl.txt");
            $this->adUrlsBlacklisted = explode("\n", $tmp);
        } else {
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
        }
104

105
        # Parser Skripte einhängen
106
107
108
109
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
110
                require_once $path;
111
112
113
            }
        }

114
        # Cachebarkeit testen
115
116
117
118
119
120
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
121
122
123
124
125
        if ($hash === "") {
            $this->searchUid = md5(uniqid());
        } else {
            $this->searchUid = $hash;
        }
126
127
128
129
130
131
132
        $redisPrefix = "search";
        # This is a list on which the MetaGer process can do a blocking pop to wait for new results
        $this->redisResultWaitingKey = $redisPrefix . "." . $this->searchUid . ".ready";
        # This is a list of searchengines which have delivered results for this search
        $this->redisResultEngineList = $redisPrefix . "." . $this->searchUid . ".engines";
        # This is the key where the results of the engine are stored as well as some statistical data
        $this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results.";
133
134
        # A list of all search results already delivered to the user (sorted of course)
        $this->redisCurrentResultList = $redisPrefix . "." . $this->searchUid . ".currentResults";
135
    }
136

137
    # Erstellt aus den gesammelten Ergebnissen den View
138
    public function createView($quicktipResults = [])
139
    {
140
141
        # Hiermit werden die evtl. ausgewählten SuMas extrahiert, damit die Input-Boxen richtig gesetzt werden können
        $focusPages = [];
Aria Givi's avatar
Aria Givi committed
142

143
        foreach ($this->request->all() as $key => $value) {
144
145
            if (starts_with($key, 'engine_') && $value === 'on') {
                $focusPages[] = $key;
146
147
148
            }
        }

149
        $viewResults = [];
150
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
151
        foreach ($this->results as $result) {
152
153
154
155
            $viewResults[] = get_object_vars($result);
        }
        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();
156
157
        if ($this->fokus === "bilder") {
            switch ($this->out) {
158
                case 'results':
159
                    return view('resultpages.results_images')
160
161
162
163
164
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
165
                        ->with('apiAuthorized', $this->apiAuthorized)
166
                        ->with('metager', $this)
167
                        ->with('browser', (new Agent())->browser());
168
                default:
169
                    return view('resultpages.resultpage_images')
170
171
172
173
174
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
175
                        ->with('apiAuthorized', $this->apiAuthorized)
176
                        ->with('metager', $this)
177
                        ->with('browser', (new Agent())->browser())
178
                        ->with('quicktips', $quicktipResults)
179
180
                        ->with('focus', $this->fokus)
                        ->with('resultcount', count($this->results));
181
            }
182
183
184
        } else {
            switch ($this->out) {
                case 'results':
185
                    return view('resultpages.results')
186
187
188
189
190
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
191
                        ->with('apiAuthorized', $this->apiAuthorized)
192
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
193
194
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
195
196
                    break;
                case 'results-with-style':
197
                    return view('resultpages.resultpage')
198
199
200
201
202
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
203
                        ->with('apiAuthorized', $this->apiAuthorized)
204
205
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
Dominik Hebeler's avatar
Dominik Hebeler committed
206
207
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
208
                    break;
209
                /* WIP
210
                case 'api':
Phil Höfer's avatar
Phil Höfer committed
211
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'key' => $this->apiKey, 'apiAuthorized' => $this->apiAuthorized])->header('Content-Type', 'application/xml');
212
                    break;
213
214
215
216
                case 'atom10':
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'key' => $this->apiKey, 'apiAuthorized' => true])
                        ->header('Content-Type', 'application/xml');
                    break;*/
217
                case 'result-count':
218
219
                    # Wir geben die Ergebniszahl und die benötigte Zeit zurück:
                    return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
220
                    break;
221
                default:
222
                    return view('resultpages.resultpage')
223
                        ->with('eingabe', $this->eingabe)
224
                        ->with('focusPages', $focusPages)
225
226
227
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
228
                        ->with('apiAuthorized', $this->apiAuthorized)
229
                        ->with('metager', $this)
230
                        ->with('browser', (new Agent())->browser())
231
                        ->with('quicktips', $quicktipResults)
232
233
                        ->with('resultcount', count($this->results))
                        ->with('focus', $this->fokus);
234
235
                    break;
            }
236
237
238
        }
    }

239
    public function prepareResults(&$timings = null)
240
    {
Phil Höfer's avatar
Phil Höfer committed
241
242
        $engines = $this->engines;
        // combine
243
        $this->combineResults($engines);
244
        if (!empty($timings)) {
245
246
            $timings["prepareResults"]["combined results"] = microtime(true) - $timings["starttime"];
        }
Phil Höfer's avatar
Phil Höfer committed
247
        // misc (WiP)
248
        if ($this->fokus == "nachrichten") {
249
250
251
            $this->results = array_filter($this->results, function ($v, $k) {
                return !is_null($v->getRank());
            }, ARRAY_FILTER_USE_BOTH);
252
253
254
255
256
257
258
259
260
261
            uasort($this->results, function ($a, $b) {
                $datea = $a->getDate();
                $dateb = $b->getDate();
                return $dateb - $datea;
            });
        } else {
            uasort($this->results, function ($a, $b) {
                if ($a->getRank() == $b->getRank()) {
                    return 0;
                }
262

263
264
265
                return ($a->getRank() < $b->getRank()) ? 1 : -1;
            });
        }
266
        if (!empty($timings)) {
267
268
            $timings["prepareResults"]["sorted results"] = microtime(true) - $timings["starttime"];
        }
269
270
        # Validate Results
        $newResults = [];
271
272
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
273
                $newResults[] = $result;
274
            }
275
276
        }
        $this->results = $newResults;
277
        if (!empty($timings)) {
278
279
            $timings["prepareResults"]["validated results"] = microtime(true) - $timings["starttime"];
        }
280
        
Davide Aprea's avatar
Davide Aprea committed
281
        $this->duplicationCheck();
282
283
284
        if(!empty($timings)){
            $timings["prepareResults"]["duplications checked"] = microtime(true) - $timings["starttime"];
        }
285
286
287
288
        # Validate Advertisements
        $newResults = [];
        foreach ($this->ads as $ad) {
            if (($ad->strippedHost !== "" && (in_array($ad->strippedHost, $this->adDomainsBlacklisted) ||
289
290
                in_array($ad->strippedLink, $this->adUrlsBlacklisted))) || ($ad->strippedHostAnzeige !== "" && (in_array($ad->strippedHostAnzeige, $this->adDomainsBlacklisted) ||
                in_array($ad->strippedLinkAnzeige, $this->adUrlsBlacklisted)))
Karl Hasselbring's avatar
Karl Hasselbring committed
291
            ) {
292
293
294
295
296
                continue;
            }
            $newResults[] = $ad;
        }
        $this->ads = $newResults;
297
        if (!empty($timings)) {
298
299
            $timings["prepareResults"]["validated ads"] = microtime(true) - $timings["starttime"];
        }
300
        #Adgoal Implementation
301
302
303
304
305
        if (empty($this->adgoalLoaded)) {
            $this->adgoalLoaded = false;
        }
        if (!$this->apiAuthorized && !$this->adgoalLoaded) {
            if (empty($this->adgoalHash)) {
Dominik Hebeler's avatar
Dominik Hebeler committed
306
307
308
309
310
                if (!empty($this->jskey)) {
                    $js = Redis::connection('cache')->lpop("js" . $this->jskey);
                    if ($js !== null && boolval($js)) {
                        $this->javascript = true;
                    }
311
                }
312
                $this->adgoalHash = $this->startAdgoal($this->results);
313
                if (!empty($timings)) {
314
315
                    $timings["prepareResults"]["started adgoal"] = microtime(true) - $timings["starttime"];
                }
316
            }
317
        
Dominik Hebeler's avatar
Dominik Hebeler committed
318
            if (!$this->javascript) {
319
                $this->adgoalLoaded = $this->parseAdgoal($this->results, $this->adgoalHash, true);
320
                if (!empty($timings)) {
321
322
                    $timings["prepareResults"]["parsed adgoal"] = microtime(true) - $timings["starttime"];
                }
323
324
            } else {
                $this->adgoalLoaded = $this->parseAdgoal($this->results, $this->adgoalHash, false);
325
                if (!empty($timings)) {
326
327
                    $timings["prepareResults"]["parsed adgoal"] = microtime(true) - $timings["starttime"];
                }
328
329
330
            }
        } else {
            $this->adgoalLoaded = true;
331
        }
332

Dominik Hebeler's avatar
Dominik Hebeler committed
333
        # Human Verification
334
335
        $this->humanVerification($this->results);
        $this->humanVerification($this->ads);
336
        if (!empty($timings)) {
337
338
            $timings["prepareResults"]["human verification"] = microtime(true) - $timings["starttime"];
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
339

340
        $counter = 0;
341
        $firstRank = 0;
342

343
        if (count($this->results) <= 0) {
344
345
346
347
348
349
            if (strlen($this->site) > 0) {
                $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
                $this->errors[] = trans('metaGer.results.failedSitesearch', ['altSearch' => $no_sitesearch_query]);
            } else {
                $this->errors[] = trans('metaGer.results.failed');
            }
350
        }
351

352
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
353
            $page = $this->page + 1;
354
            $this->next = [
355
356
                'page' => $page,
                'engines' => $this->next,
357
            ];
358
            Cache::put($this->getSearchUid(), serialize($this->next), 60 * 60);
359
            if (!empty($timings)) {
360
361
                $timings["prepareResults"]["filled cache"] = microtime(true) - $timings["starttime"];
            }
362
363
        } else {
            $this->next = [];
364
        }
365
    }
366

Phil Höfer's avatar
Phil Höfer committed
367
368
369
370
371
372
373
374
375
376
377
    public function combineResults($engines)
    {
        foreach ($engines as $engine) {
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
            foreach ($engine->results as $result) {
                if ($result->valid) {
378
                    $this->results[] = clone $result;
Phil Höfer's avatar
Phil Höfer committed
379
380
381
                }
            }
            foreach ($engine->ads as $ad) {
382
                $this->ads[] = clone $ad;
Phil Höfer's avatar
Phil Höfer committed
383
384
385
386
            }
        }
    }

Davide Aprea's avatar
Davide Aprea committed
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
    public function duplicationCheck()
    {
        $arr = [];
        for($i = 0; $i < count($this->results); $i++) {

            $link = $this->results[$i]->link;

            if (strpos($link, "http://") === 0) {
                $link = substr($link, 7);
            }
    
            if (strpos($link, "https://") === 0) {
                $link = substr($link, 8);
            }
    
            if (strpos($link, "www.") === 0) {
                $link = substr($link, 4);
            }
    
            $link = trim($link, "/");
            $hash = md5($link);

            if(isset($arr[$link])){
                $arr[$link]->gefVon[] = $this->results[$i]->gefVon[0];
                $arr[$link]->gefVonLink[] = $this->results[$i]->gefVonLink[0];
Davide Aprea's avatar
Davide Aprea committed
412
413
                array_splice($this->results, $i, 1);
                $i--;
Davide Aprea's avatar
Davide Aprea committed
414
415
416
                if($arr[$link]->new === true || $this->results[$i]->new === true){
                    $arr[$link]->changed = true;
                }
Davide Aprea's avatar
Davide Aprea committed
417
418
419
420
421
422
            }else{
                $arr[$link] = &$this->results[$i];
            }
        }
    }

423
    public function startAdgoal(&$results)
424
    {
425
        $publicKey = getenv('adgoal_public');
426
        $privateKey = getenv('adgoal_private');
427
        if ($publicKey === false) {
428
            return true;
429
430
        }
        $tldList = "";
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
        foreach ($results as $result) {
            if (!$result->new) {
                continue;
            }
            $link = $result->link;
            if (strpos($link, "http") !== 0) {
                $link = "http://" . $link;
            }
            $tldList .= parse_url($link, PHP_URL_HOST) . ",";
            $result->tld = parse_url($link, PHP_URL_HOST);
        }
        $tldList = rtrim($tldList, ",");

        # Hashwert
        $hash = md5("meta" . $publicKey . $tldList . "GER");
446

447
448
449
450
451
452
453
454
455
456
457
458
459
460
        # Query
        $query = $this->q;

        $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . urlencode($publicKey) . "&k=" . urlencode($hash) . "&tld=" . urlencode($tldList) . "&q=" . urlencode($query);

        // Submit fetch job to worker
        $mission = [
            "resulthash" => $hash,
            "url" => $link,
            "useragent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0",
            "username" => null,
            "password" => null,
            "headers" => null,
            "cacheDuration" => 60,
461
            "name" => "Adgoal",
462
463
464
465
466
467
468
469
470
471
472
473
474
        ];
        $mission = json_encode($mission);
        Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);

        return $hash;
    }

    public function parseAdgoal(&$results, $hash, $waitForResult)
    {
        # Wait for result
        $startTime = microtime(true);
        $answer = null;

475
        # Hash is true if Adgoal request wasn't started in the first place
476
        if ($hash === true) {
477
478
479
            return true;
        }

480
481
        if ($waitForResult) {
            while (microtime(true) - $startTime < 5) {
482
                $answer = Cache::get($hash);
483
484
485
486
487
488
489
                if ($answer === null) {
                    usleep(50 * 1000);
                } else {
                    break;
                }
            }
        } else {
490
            $answer = Cache::get($hash);
491
492
493
494
        }
        if ($answer === null) {
            return false;
        }
495
        try {
496
497
498
499
500
            $answer = json_decode($answer);
            $publicKey = getenv('adgoal_public');
            $privateKey = getenv('adgoal_private');

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
501
            foreach ($results as $result) {
502
                $link = $result->link;
503
504
505
                $result->tld = parse_url($link, PHP_URL_HOST);
            }

506
            foreach ($answer as $el) {
507
                $hoster = $el[0];
508
                $hash = $el[1];
509

510
                foreach ($results as $result) {
511
                    if ($hoster === $result->tld && !$result->partnershop) {
512
513
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
514
                        if ($result->image !== "") {
515
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . urlencode($hash) . ".gif";
516
                        } else {
517
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . urlencode($hash) . ".gif";
518
519
                        }

520
                        # Den Link hinzufügen:
521
                        $targetUrl = $result->link;
522
523
                        # Query
                        $query = $this->q;
524

525
                        $gateHash = md5($targetUrl . $privateKey);
526
                        $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . urlencode($publicKey) . "&k=" . urlencode($gateHash) . "&url=" . urlencode($targetUrl) . "&q=" . urlencode($query);
527
                        $result->link = $newLink;
528
                        $result->partnershop = true;
Davide Aprea's avatar
Davide Aprea committed
529
                        $result->changed = true;
530
531
532
                    }
                }
            }
533
        } catch (\ErrorException $e) {
534
            Log::error($e->getMessage());
Dominik Hebeler's avatar
Dominik Hebeler committed
535
        } finally {
536
            $requestTime = microtime(true) - $startTime;
Dominik Hebeler's avatar
Dominik Hebeler committed
537
            \App\PrometheusExporter::Duration($requestTime, "adgoal");
538
        }
539
        return true;
540
    }
541

542
    public function humanVerification(&$results)
543
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
544
        # Let's check if we need to implement a redirect for human verification
545
546
        if ($this->verificationCount > 10) {
            foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
547
548
549
                $link = $result->link;
                $day = Carbon::now()->day;
                $pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
550
551
552
                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
Dominik Hebeler's avatar
Dominik Hebeler committed
553
                $result->link = $url;
554
                $result->proxyLink = $proxyUrl;
Dominik Hebeler's avatar
Dominik Hebeler committed
555
556
557
558
            }
        }
    }

Phil Höfer's avatar
Phil Höfer committed
559
560
    public function authorize($key)
    {
561
        return app('App\Models\Key')->requestPermission();
Phil Höfer's avatar
Phil Höfer committed
562
563
    }

Karl's avatar
Karl committed
564
565
566
567
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

Dominik Hebeler's avatar
Dominik Hebeler committed
568
    public function createSearchEngines(Request $request, &$timings)
569
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
570
571
572
573
        if (!empty($timings)) {
            $timings["createSearchEngines"]["start"] = microtime(true) - $timings["starttime"];
        }

574
        # Wenn es kein Suchwort gibt
Dominik Hebeler's avatar
Dominik Hebeler committed
575
        if (!$request->filled("eingabe") || $this->q === "") {
576
            return;
577
        }
578

Dominik Hebeler's avatar
Dominik Hebeler committed
579
        $this->enabledSearchengines = [];
580
        $overtureEnabled = false;
581

582
583
584
        # Check if selected focus is valid
        if (empty($this->sumaFile->foki->{$this->fokus})) {
            $this->fokus = "web";
585
        }
586

Karl Hasselbring's avatar
Karl Hasselbring committed
587
        $sumaNames = $this->sumaFile->foki->{$this->fokus}->sumas;
588

Karl Hasselbring's avatar
Karl Hasselbring committed
589
590
591
        $sumas = [];
        foreach ($sumaNames as $sumaName) {
            $sumas[$sumaName] = $this->sumaFile->sumas->{$sumaName};
592
593
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
594
595
596
597
        if (!empty($timings)) {
            $timings["createSearchEngines"]["created engine array"] = microtime(true) - $timings["starttime"];
        }

Karl Hasselbring's avatar
Karl Hasselbring committed
598
599
        $this->removeAdsFromListIfAdfree($sumas);

Dominik Hebeler's avatar
Dominik Hebeler committed
600
601
602
603
        if (!empty($timings)) {
            $timings["createSearchEngines"]["removed ads"] = microtime(true) - $timings["starttime"];
        }

Karl Hasselbring's avatar
Karl Hasselbring committed
604
        foreach ($sumas as $sumaName => $suma) {
605
            # Check if this engine is disabled and can't be used
Karl Hasselbring's avatar
Karl Hasselbring committed
606
607
608
609
610
611
612
            $disabled = empty($suma->disabled) ? false : $suma->disabled;
            $autoDisabled = empty($suma->{"auto-disabled"}) ? false : $suma->{"auto-disabled"};
            if (
                $disabled || $autoDisabled
                || \Cookie::get($this->getFokus() . "_engine_" . $sumaName) === "off"
            ) {
                continue;
613
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
614

615
            $valid = true;
Karl Hasselbring's avatar
Karl Hasselbring committed
616
617
618
619

            # Check if this engine can use potentially defined query-filter
            foreach ($this->queryFilter as $filterName => $filter) {
                if (empty($this->sumaFile->filter->{"query-filter"}->$filterName->sumas->$sumaName)) {
620
621
                    $valid = false;
                    break;
Dominik Hebeler's avatar
Dominik Hebeler committed
622
623
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
624
625

            # Check if this engine can use potentially defined parameter-filter
Dominik Hebeler's avatar
Dominik Hebeler committed
626
627
            if ($valid) {
                foreach ($this->parameterFilter as $filterName => $filter) {
628
                    # We need to check if the searchengine supports the parameter value, too
Karl Hasselbring's avatar
Karl Hasselbring committed
629
                    if (empty($filter->sumas->$sumaName) || empty($filter->sumas->{$sumaName}->values->{$filter->value})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
630
631
632
633
634
                        $valid = false;
                        break;
                    }
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
635

636
            # Check if this engine should only be active when filter is used
Karl Hasselbring's avatar
Karl Hasselbring committed
637
            if ($suma->{"filter-opt-in"}) {
638
639
640
                # This search engine should only be used when a parameter filter of it is used
                $validTmp = false;
                foreach ($this->parameterFilter as $filterName => $filter) {
Karl Hasselbring's avatar
Karl Hasselbring committed
641
                    if (!empty($filter->sumas->{$sumaName})) {
642
643
644
645
646
647
648
649
                        $validTmp = true;
                        break;
                    }
                }
                if (!$validTmp) {
                    $valid = false;
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
650
651

            # If the suma is still valid, we can add it
652
            if ($valid) {
Karl Hasselbring's avatar
Karl Hasselbring committed
653
                $this->enabledSearchengines[$sumaName] = $suma;
654
            }
655
        }
656

Dominik Hebeler's avatar
Dominik Hebeler committed
657
658
659
660
        if (!empty($timings)) {
            $timings["createSearchEngines"]["filtered invalid engines"] = microtime(true) - $timings["starttime"];
        }

Karl Hasselbring's avatar
Karl Hasselbring committed
661
662
        # Include Yahoo Ads if Yahoo is not enabled as a searchengine
        if (!$this->apiAuthorized && $this->fokus != "bilder" && empty($this->enabledSearchengines["yahoo"]) && isset($this->sumaFile->sumas->{"yahoo-ads"})) {
663
664
665
            $this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
        }

666
667
668
669
670
671
        # Special case if search engines are disabled
        # Since bing is normally only active if a filter is set but it should be active, too if yahoo is disabled
        if ($this->getFokus() === "web" && empty($this->enabledSearchengines["yahoo"]) && \Cookie::get("web_engine_bing") !== "off") {
            $this->enabledSearchengines["bing"] = $this->sumaFile->sumas->{"bing"};
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
672
        if (sizeof($this->enabledSearchengines) === 0) {
673
674
            $filter = "";
            foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
Dominik Hebeler's avatar
Dominik Hebeler committed
675
                $filter .= trans($this->sumaFile->filter->{"query-filter"}->{$queryFilter}->name) . ",";
676
677
            }
            $filter = rtrim($filter, ",");
Karl Hasselbring's avatar
Karl Hasselbring committed
678
679
            $error = trans('metaGer.engines.noSpecialSearch', [
                'fokus' => trans($this->sumaFile->foki->{$this->fokus}->{"display-name"}),
680
                'filter' => $filter,
Karl Hasselbring's avatar
Karl Hasselbring committed
681
            ]);
682
            $this->errors[] = $error;
683
        }
684
        $this->setEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
685
686
687
        if (!empty($timings)) {
            $timings["createSearchEngines"]["saved engines"] = microtime(true) - $timings["starttime"];
        }
688
    }
689

Karl Hasselbring's avatar
Karl Hasselbring committed
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
    private function removeAdsFromListIfAdfree(&$sumas)
    {
        if ($this->apiAuthorized) {
            foreach ($sumas as $sumaName => $suma) {
                $ads = $suma->ads ?? false;
                if ($ads) {
                    unset($sumas[$sumaName]);

                    $adBackups = $suma->{"ad-backups"} ?? [];
                    $adBackupName = $adBackups->{$this->fokus} ?? null;
                    if (isset($adBackupName)) {
                        $this->sumaFile->sumas->{$adBackupName}->{"filter-opt-in"} = false;
                    }
                }
            }
        }
    }

708
709
    public function setEngines(Request $request, $enabledSearchengines = [])
    {
710
        if ($this->requestIsCached($request)) {
711
            # If this is a page other than 1 the request is "cached"
712
            $engines = $this->getCachedEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
713
            # We need to edit some Options of the Cached Search Engines
Phil Höfer's avatar
Phil Höfer committed
714
            foreach ($engines as $engine) {
715
                $engine->setResultHash($this->getSearchUid());
Dominik Hebeler's avatar
Dominik Hebeler committed
716
            }
717
            $this->engines = $engines;
718
        } else {
719
720
721
            if (sizeof($enabledSearchengines) > 0) {
                $this->enabledSearchengines = $enabledSearchengines;
            }
722
            $this->actuallyCreateSearchEngines($this->enabledSearchengines);
723
        }
724
    }
Dominik Hebeler's avatar
Dominik Hebeler committed
725

Dominik Hebeler's avatar
Dominik Hebeler committed
726
    public function startSearch(&$timings)
727
    {
728
729
730
731
732
        if (!empty($timings)) {
            $timings["startSearch"]["start"] = microtime(true) - $timings["starttime"];
        }

        # Check all engines for Cached responses
Dominik Hebeler's avatar
Dominik Hebeler committed
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
        $this->checkCache();

        if (!empty($timings)) {
            $timings["startSearch"]["cache checked"] = microtime(true) - $timings["starttime"];
        }

        # Wir starten alle Suchen
        foreach ($this->engines as $engine) {
            $engine->startSearch($this, $timings);
        }
        if (!empty($timings)) {
            $timings["startSearch"]["searches started"] = microtime(true) - $timings["starttime"];
        }
    }

    public function checkCache()
    {
750
751
752
753
754
755
756
757
758
759
760
761
762
        if ($this->canCache()) {
            $keys = [];
            foreach ($this->engines as $engine) {
                $keys[] = $engine->hash;
            }
            $cacheValues = Cache::many($keys);
            foreach ($this->engines as $engine) {
                if ($cacheValues[$engine->hash] !== null) {
                    $engine->cached = true;
                    $engine->retrieveResults($this, $cacheValues[$engine->hash]);
                }
            }
        }
763
764
    }

765
766
    # Spezielle Suchen und Sumas

767
    public function sumaIsSelected($suma, $request, $custom)
768
    {
769
        if ($custom) {
770
            if ($request->filled("engine_" . strtolower($suma["name"]))) {
771
772
773
774
775
776
777
778
779
780
781
                return true;
            }
        } else {
            $types = explode(",", $suma["type"]);
            if (in_array($this->fokus, $types)) {
                return true;
            }
        }
        return false;
    }

782
    public function actuallyCreateSearchEngines($enabledSearchengines)
783
784
    {
        $engines = [];
785
786
        foreach ($enabledSearchengines as $engineName => $engine) {
            if (!isset($engine->{"parser-class"})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
787
788
                die(var_dump($engine));
            }
789
            # Setze Pfad zu Parser
790
            $path = "App\\Models\\parserSkripte\\" . $engine->{"parser-class"};
791
792

            # Prüfe ob Parser vorhanden
793
794
795
            if (!file_exists(app_path() . "/Models/parserSkripte/" . $engine->{"parser-class"} . ".php")) {
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen, da kein Parser existiert");
                $this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine->{"display-name"}]);
796
797
798
799
800
801
                continue;
            }

            # Es wird versucht die Suchengine zu erstellen
            $time = microtime();
            try {
802
                $tmp = new $path($engineName, $engine, $this);
803
            } catch (\ErrorException $e) {
804
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen. " . $e);
805
806
807
                continue;
            }

808
            $engines[] = $tmp;
809
        }
810
        $this->engines = $engines;
811
812
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
813
814
815
816
817
818
819
    public function getAvailableParameterFilter()
    {
        $parameterFilter = $this->sumaFile->filter->{"parameter-filter"};

        $availableFilter = [];

        foreach ($parameterFilter as $filterName => $filter) {
820
            $values = clone $filter->values;
Dominik Hebeler's avatar
Dominik Hebeler committed
821
822
823
            # Check if any of the enabled search engines provide this filter
            foreach ($this->enabledSearchengines as $engineName => $engine) {
                if (!empty($filter->sumas->$engineName)) {
824
825
                    if (empty($availableFilter[$filterName])) {
                        $availableFilter[$filterName] = $filter;
826
827
                        foreach ($availableFilter[$filterName]->values as $key => $value) {
                            if ($key !== "nofilter") {
828
829
830
                                unset($availableFilter[$filterName]->values->{$key});
                            }
                        }
831
832
                    }
                    if (empty($availableFilter[$filterName]->values)) {
833
                        $availableFilter[$filterName]->values = new \stdClass();
834
835
                    }
                    foreach ($filter->sumas->{$engineName}->values as $key => $value) {
836
                        $availableFilter[$filterName]->values->{$key} = $values->$key;
837
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
838
839
                }
            }
840
841
            # We will also add the filter from the opt-in search engines (the searchengines that are only used when a filter of it is too)
            foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
842
                if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"} && \Cookie::get($this->getFokus() . "_engine_" . $suma) !== "off") {
843
                    if (!empty($filter->sumas->{$suma})) {
844
845
                        # If the searchengine is disabled this filter shouldn't be available
                        if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
Karl Hasselbring's avatar
Karl Hasselbring committed
846
847
                            || (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)
                        ) {
848
849
                            continue;
                        }
850
851
                        if (empty($availableFilter[$filterName])) {
                            $availableFilter[$filterName] = $filter;
852
853
                            foreach ($availableFilter[$filterName]->values as $key => $value) {
                                if ($key !== "nofilter") {
854
855
856
                                    unset($availableFilter[$filterName]->values->{$key});
                                }
                            }
857
858
                        }
                        if (empty($availableFilter[$filterName]->values)) {
859
                            $availableFilter[$filterName]->values = new \stdClass();
860
861
                        }
                        foreach ($filter->sumas->{$suma}->values as $key => $value) {
862
                            $availableFilter[$filterName]->values->{$key} = $values->$key;
863
                        }
864
865
866
                    }
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
867
868
        }

869
870
871
872
        # Set the current values for the filters
        foreach ($availableFilter as $filterName => $filter) {
            if (\Request::filled($filter->{"get-parameter"})) {
                $filter->value = \Request::input($filter->{"get-parameter"});
873
            } elseif (\Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"}) !== null) {
874
875
876
877
                $filter->value = \Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"});
            }
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
878
879
880
        return $availableFilter;
    }

881
882
    public function isBildersuche()
    {
883
        return $this->fokus === "bilder";
884
885
886
887
    }

    public function sumaIsAdsuche($suma, $overtureEnabled)
    {