MetaGer.php 71.4 KB
Newer Older
1
<?php
Karl Hasselbring's avatar
Karl Hasselbring committed
2

3
4
5
namespace App;

use App;
6
use Cache;
7
use Carbon;
8
use Illuminate\Http\Request;
9
use Illuminate\Support\Facades\Redis;
10
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix    
Dominik Hebeler committed
11
use LaravelLocalization;
12
use Log;
13
use Monospice\LaravelRedisSentinel\RedisSentinel;
14
use Predis\Connection\ConnectionException;
15
16
17

class MetaGer
{
Dominik Hebeler's avatar
Dominik Hebeler committed
18
19
    const FETCHQUEUE_KEY = "fetcher.queue";

20
    # Einstellungen für die Suche
21
22
    public $alteredQuery = "";
    public $alterationOverrideQuery = "";
23
    protected $fokus;
Dominik Hebeler's avatar
update    
Dominik Hebeler committed
24
    protected $test;
25
26
27
28
29
30
    protected $eingabe;
    protected $q;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
31
    protected $time = 2000;
32
    protected $hostBlacklist = [];
33
    protected $domainBlacklist = [];
34
35
36
37
    private $urlBlacklist = [];
    protected $stopWords = [];
    protected $phrases = [];
    protected $engines = [];
38
    protected $totalResults = 0;
39
    protected $results = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
40
41
    protected $queryFilter = [];
    protected $parameterFilter = [];
42
    protected $ads = [];
43
    protected $infos = [];
44
45
46
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
47
    protected $availableFoki = [];
48
49
    protected $startCount = 0;
    protected $canCache = false;
50
    # Daten über die Abfrage$
51
    protected $ip;
Dominik Hebeler's avatar
Dominik Hebeler committed
52
    protected $useragent;
53
54
    protected $language;
    protected $agent;
55
    protected $apiKey = "";
Phil Höfer's avatar
Phil Höfer committed
56
    protected $apiAuthorized = false;
57
    protected $next = [];
58
59
60
61
62
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
63
    protected $newtab;
64
    protected $domainsBlacklisted = [];
65
    protected $adDomainsBlacklisted = [];
66
    protected $urlsBlacklisted = [];
67
    protected $adUrlsBlacklisted = [];
68
    protected $url;
69
    protected $fullUrl;
Aria's avatar
Aria committed
70
    protected $enabledSearchengines = [];
71
    protected $languageDetect;
Dominik Hebeler's avatar
Dominik Hebeler committed
72
73
    protected $verificationId;
    protected $verificationCount;
74
    protected $searchUid;
75
    protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult, $redisCurrentResultList;
Dominik Hebeler's avatar
Dominik Hebeler committed
76
    public $starttime;
77

78
    public function __construct($hash = "")
79
    {
80
        # Timer starten
81
        $this->starttime = microtime(true);
82
        # Versuchen Blacklists einzulesen
83
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
84
            $tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
85
            $this->domainsBlacklisted = explode("\n", $tmp);
86
87
            $tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted = explode("\n", $tmp);
88
        } else {
89
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
90
        }
91
92
93
94
95
96
97
98
99
        # Versuchen Blacklists einzulesen
        if (file_exists(config_path() . "/adBlacklistDomains.txt") && file_exists(config_path() . "/adBlacklistUrl.txt")) {
            $tmp = file_get_contents(config_path() . "/adBlacklistDomains.txt");
            $this->adDomainsBlacklisted = explode("\n", $tmp);
            $tmp = file_get_contents(config_path() . "/adBlacklistUrl.txt");
            $this->adUrlsBlacklisted = explode("\n", $tmp);
        } else {
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
        }
100

101
        # Parser Skripte einhängen
102
103
104
105
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
106
                require_once $path;
107
108
109
            }
        }

110
        # Cachebarkeit testen
111
112
113
114
115
116
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
117
118
119
120
121
        if ($hash === "") {
            $this->searchUid = md5(uniqid());
        } else {
            $this->searchUid = $hash;
        }
122
123
124
125
126
127
128
        $redisPrefix = "search";
        # This is a list on which the MetaGer process can do a blocking pop to wait for new results
        $this->redisResultWaitingKey = $redisPrefix . "." . $this->searchUid . ".ready";
        # This is a list of searchengines which have delivered results for this search
        $this->redisResultEngineList = $redisPrefix . "." . $this->searchUid . ".engines";
        # This is the key where the results of the engine are stored as well as some statistical data
        $this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results.";
129
130
        # A list of all search results already delivered to the user (sorted of course)
        $this->redisCurrentResultList = $redisPrefix . "." . $this->searchUid . ".currentResults";
131
    }
132

133
    # Erstellt aus den gesammelten Ergebnissen den View
134
    public function createView()
135
    {
136
137
        # Hiermit werden die evtl. ausgewählten SuMas extrahiert, damit die Input-Boxen richtig gesetzt werden können
        $focusPages = [];
Aria Givi's avatar
Aria Givi committed
138

139
        foreach ($this->request->all() as $key => $value) {
140
141
            if (starts_with($key, 'engine_') && $value === 'on') {
                $focusPages[] = $key;
142
143
144
            }
        }

145
        $viewResults = [];
146
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
147
        foreach ($this->results as $result) {
148
149
150
151
            $viewResults[] = get_object_vars($result);
        }
        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();
152
153
        if ($this->fokus === "bilder") {
            switch ($this->out) {
154
                case 'results':
155
                    return view('resultpages.results_images')
156
157
158
159
160
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
161
                        ->with('apiAuthorized', $this->apiAuthorized)
162
                        ->with('metager', $this)
163
                        ->with('browser', (new Agent())->browser());
164
                default:
165
                    return view('resultpages.resultpage_images')
166
167
168
169
170
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
171
                        ->with('apiAuthorized', $this->apiAuthorized)
172
                        ->with('metager', $this)
173
                        ->with('browser', (new Agent())->browser())
174
                        ->with('quicktips', action('MetaGerSearch@quicktips', ["search" => $this->eingabe]))
175
176
                        ->with('focus', $this->fokus)
                        ->with('resultcount', count($this->results));
177
            }
178
179
180
        } else {
            switch ($this->out) {
                case 'results':
181
                    return view('resultpages.results')
182
183
184
185
186
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
187
                        ->with('apiAuthorized', $this->apiAuthorized)
188
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
189
190
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
191
192
                    break;
                case 'results-with-style':
193
                    return view('resultpages.resultpage')
194
195
196
197
198
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
199
                        ->with('apiAuthorized', $this->apiAuthorized)
200
201
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
Dominik Hebeler's avatar
Dominik Hebeler committed
202
203
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
204
                    break;
Phil Höfer's avatar
Phil Höfer committed
205
                case 'rich':
206
                    return view('resultpages.metager3rich')
207
                        ->with('results', $viewResults)
Phil Höfer's avatar
Phil Höfer committed
208
209
210
211
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
212
                        ->with('apiAuthorized', $this->apiAuthorized)
Phil Höfer's avatar
Phil Höfer committed
213
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
214
215
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
Phil Höfer's avatar
Phil Höfer committed
216
                    break;
217
                case 'rss20':
218
                    return view('resultpages.metager3resultsrss20')
219
220
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
Phil Höfer's avatar
Phil Höfer committed
221
                        ->with('apiAuthorized', $this->apiAuthorized)
222
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
223
224
                        ->with('resultcount', sizeof($viewResults))
                        ->with('fokus', $this->fokus);
225
                    break;
226
                case 'api':
Phil Höfer's avatar
Phil Höfer committed
227
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'key' => $this->apiKey, 'apiAuthorized' => $this->apiAuthorized])->header('Content-Type', 'application/xml');
228
                    break;
Aria Givi's avatar
Aria Givi committed
229
                case 'atom10':
Phil Höfer's avatar
Phil Höfer committed
230
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'key' => $this->apiKey, 'apiAuthorized' => true])
231
                        ->header('Content-Type', 'application/xml');
Aria Givi's avatar
Aria Givi committed
232
                    break;
233
                case 'result-count':
234
235
                    # Wir geben die Ergebniszahl und die benötigte Zeit zurück:
                    return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
236
                    break;
237
                default:
238
                    return view('resultpages.resultpage')
239
                        ->with('eingabe', $this->eingabe)
240
                        ->with('focusPages', $focusPages)
241
242
243
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
244
                        ->with('apiAuthorized', $this->apiAuthorized)
245
                        ->with('metager', $this)
246
                        ->with('browser', (new Agent())->browser())
247
                        ->with('quicktips', action('MetaGerSearch@quicktips', ["search" => $this->eingabe, "quotes" => $this->sprueche]))
248
249
                        ->with('resultcount', count($this->results))
                        ->with('focus', $this->fokus);
250
251
                    break;
            }
252
253
254
        }
    }

255
    public function prepareResults(&$timings = null)
256
    {
Phil Höfer's avatar
Phil Höfer committed
257
258
        $engines = $this->engines;
        // combine
259
        $this->combineResults($engines);
260
261
262
        if(!empty($timings)){
            $timings["prepareResults"]["combined results"] = microtime(true) - $timings["starttime"];
        }
Phil Höfer's avatar
Phil Höfer committed
263
        // misc (WiP)
264
        if ($this->fokus == "nachrichten") {
265
266
267
            $this->results = array_filter($this->results, function ($v, $k) {
                return !is_null($v->getRank());
            }, ARRAY_FILTER_USE_BOTH);
268
269
270
271
272
273
274
275
276
277
            uasort($this->results, function ($a, $b) {
                $datea = $a->getDate();
                $dateb = $b->getDate();
                return $dateb - $datea;
            });
        } else {
            uasort($this->results, function ($a, $b) {
                if ($a->getRank() == $b->getRank()) {
                    return 0;
                }
278

279
280
281
                return ($a->getRank() < $b->getRank()) ? 1 : -1;
            });
        }
282
283
284
        if(!empty($timings)){
            $timings["prepareResults"]["sorted results"] = microtime(true) - $timings["starttime"];
        }
285
286
        # Validate Results
        $newResults = [];
287
288
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
289
                $newResults[] = $result;
290
            }
291
292
        }
        $this->results = $newResults;
293
294
295
        if(!empty($timings)){
            $timings["prepareResults"]["validated results"] = microtime(true) - $timings["starttime"];
        }
296
        
Davide Aprea's avatar
Davide Aprea committed
297
        $this->duplicationCheck();
298
299
300
        if(!empty($timings)){
            $timings["prepareResults"]["duplications checked"] = microtime(true) - $timings["starttime"];
        }
301
302
303
304
        # Validate Advertisements
        $newResults = [];
        foreach ($this->ads as $ad) {
            if (($ad->strippedHost !== "" && (in_array($ad->strippedHost, $this->adDomainsBlacklisted) ||
305
306
                in_array($ad->strippedLink, $this->adUrlsBlacklisted))) || ($ad->strippedHostAnzeige !== "" && (in_array($ad->strippedHostAnzeige, $this->adDomainsBlacklisted) ||
                in_array($ad->strippedLinkAnzeige, $this->adUrlsBlacklisted)))
Karl Hasselbring's avatar
Karl Hasselbring committed
307
            ) {
308
309
310
311
312
                continue;
            }
            $newResults[] = $ad;
        }
        $this->ads = $newResults;
313
314
315
        if(!empty($timings)){
            $timings["prepareResults"]["validated ads"] = microtime(true) - $timings["starttime"];
        }
316
        #Adgoal Implementation
317
318
319
320
321
        if (empty($this->adgoalLoaded)) {
            $this->adgoalLoaded = false;
        }
        if (!$this->apiAuthorized && !$this->adgoalLoaded) {
            if (empty($this->adgoalHash)) {
Dominik Hebeler's avatar
Dominik Hebeler committed
322
323
324
325
326
                if (!empty($this->jskey)) {
                    $js = Redis::connection('cache')->lpop("js" . $this->jskey);
                    if ($js !== null && boolval($js)) {
                        $this->javascript = true;
                    }
327
                }
328
                $this->adgoalHash = $this->startAdgoal($this->results);
329
330
331
                if(!empty($timings)){
                    $timings["prepareResults"]["started adgoal"] = microtime(true) - $timings["starttime"];
                }
332
            }
333
        
Dominik Hebeler's avatar
Dominik Hebeler committed
334
            if (!$this->javascript) {
335
                $this->adgoalLoaded = $this->parseAdgoal($this->results, $this->adgoalHash, true);
336
337
338
                if(!empty($timings)){
                    $timings["prepareResults"]["parsed adgoal"] = microtime(true) - $timings["starttime"];
                }
339
340
            } else {
                $this->adgoalLoaded = $this->parseAdgoal($this->results, $this->adgoalHash, false);
341
342
343
                if(!empty($timings)){
                    $timings["prepareResults"]["parsed adgoal"] = microtime(true) - $timings["starttime"];
                }
344
345
346
            }
        } else {
            $this->adgoalLoaded = true;
347
        }
348

Dominik Hebeler's avatar
Dominik Hebeler committed
349
        # Human Verification
350
351
        $this->humanVerification($this->results);
        $this->humanVerification($this->ads);
352
353
354
        if(!empty($timings)){
            $timings["prepareResults"]["human verification"] = microtime(true) - $timings["starttime"];
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
355

356
        $counter = 0;
357
        $firstRank = 0;
358

359
        if (count($this->results) <= 0) {
360
361
362
363
364
365
            if (strlen($this->site) > 0) {
                $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
                $this->errors[] = trans('metaGer.results.failedSitesearch', ['altSearch' => $no_sitesearch_query]);
            } else {
                $this->errors[] = trans('metaGer.results.failed');
            }
366
        }
367

368
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
369
            $page = $this->page + 1;
370
            $this->next = [
371
372
                'page' => $page,
                'engines' => $this->next,
373
            ];
374
            Cache::put($this->getSearchUid(), serialize($this->next), 60 * 60);
375
376
377
            if(!empty($timings)){
                $timings["prepareResults"]["filled cache"] = microtime(true) - $timings["starttime"];
            }
378
379
        } else {
            $this->next = [];
380
        }
381
    }
382

Phil Höfer's avatar
Phil Höfer committed
383
384
385
386
387
388
389
390
391
392
393
    public function combineResults($engines)
    {
        foreach ($engines as $engine) {
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
            foreach ($engine->results as $result) {
                if ($result->valid) {
394
                    $this->results[] = clone $result;
Phil Höfer's avatar
Phil Höfer committed
395
396
397
                }
            }
            foreach ($engine->ads as $ad) {
398
                $this->ads[] = clone $ad;
Phil Höfer's avatar
Phil Höfer committed
399
400
401
402
            }
        }
    }

Davide Aprea's avatar
Davide Aprea committed
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
    public function duplicationCheck()
    {
        $arr = [];
        for($i = 0; $i < count($this->results); $i++) {

            $link = $this->results[$i]->link;

            if (strpos($link, "http://") === 0) {
                $link = substr($link, 7);
            }
    
            if (strpos($link, "https://") === 0) {
                $link = substr($link, 8);
            }
    
            if (strpos($link, "www.") === 0) {
                $link = substr($link, 4);
            }
    
            $link = trim($link, "/");
            $hash = md5($link);

            if(isset($arr[$link])){
                $arr[$link]->gefVon[] = $this->results[$i]->gefVon[0];
                $arr[$link]->gefVonLink[] = $this->results[$i]->gefVonLink[0];
Davide Aprea's avatar
Davide Aprea committed
428
429
                array_splice($this->results, $i, 1);
                $i--;
Davide Aprea's avatar
Davide Aprea committed
430
431
432
433
434
435
            }else{
                $arr[$link] = &$this->results[$i];
            }
        }
    }

436
    public function startAdgoal(&$results)
437
    {
438
        $publicKey = getenv('adgoal_public');
439
        $privateKey = getenv('adgoal_private');
440
        if ($publicKey === false) {
441
            return true;
442
443
        }
        $tldList = "";
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
        foreach ($results as $result) {
            if (!$result->new) {
                continue;
            }
            $link = $result->link;
            if (strpos($link, "http") !== 0) {
                $link = "http://" . $link;
            }
            $tldList .= parse_url($link, PHP_URL_HOST) . ",";
            $result->tld = parse_url($link, PHP_URL_HOST);
        }
        $tldList = rtrim($tldList, ",");

        # Hashwert
        $hash = md5("meta" . $publicKey . $tldList . "GER");
459

460
461
462
463
464
465
466
467
468
469
470
471
472
473
        # Query
        $query = $this->q;

        $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . urlencode($publicKey) . "&k=" . urlencode($hash) . "&tld=" . urlencode($tldList) . "&q=" . urlencode($query);

        // Submit fetch job to worker
        $mission = [
            "resulthash" => $hash,
            "url" => $link,
            "useragent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0",
            "username" => null,
            "password" => null,
            "headers" => null,
            "cacheDuration" => 60,
474
            "name" => "Adgoal",
475
476
477
478
479
480
481
482
483
484
485
486
487
        ];
        $mission = json_encode($mission);
        Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);

        return $hash;
    }

    public function parseAdgoal(&$results, $hash, $waitForResult)
    {
        # Wait for result
        $startTime = microtime(true);
        $answer = null;

488
489
490
491
492
        # Hash is true if Adgoal request wasn't started in the first place
        if($hash === true){
            return true;
        }

493
494
        if ($waitForResult) {
            while (microtime(true) - $startTime < 5) {
495
                $answer = Redis::get($hash);
496
497
498
499
500
501
502
                if ($answer === null) {
                    usleep(50 * 1000);
                } else {
                    break;
                }
            }
        } else {
503
            $answer = Redis::get($hash);
504
505
506
507
        }
        if ($answer === null) {
            return false;
        }
508
        try {
509
510
511
512
513
            $answer = json_decode($answer);
            $publicKey = getenv('adgoal_public');
            $privateKey = getenv('adgoal_private');

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
514
            foreach ($results as $result) {
515
                $link = $result->link;
516
517
518
                $result->tld = parse_url($link, PHP_URL_HOST);
            }

519
            foreach ($answer as $el) {
520
                $hoster = $el[0];
521
                $hash = $el[1];
522

523
                foreach ($results as $result) {
524
                    if ($hoster === $result->tld && !$result->partnershop) {
525
526
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
527
                        if ($result->image !== "") {
528
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . urlencode($hash) . ".gif";
529
                        } else {
530
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . urlencode($hash) . ".gif";
531
532
                        }

533
                        # Den Link hinzufügen:
534
                        $targetUrl = $result->link;
535
536
                        # Query
                        $query = $this->q;
537

538
                        $gateHash = md5($targetUrl . $privateKey);
539
                        $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . urlencode($publicKey) . "&k=" . urlencode($gateHash) . "&url=" . urlencode($targetUrl) . "&q=" . urlencode($query);
540
                        $result->link = $newLink;
541
                        $result->partnershop = true;
542
                        $result->adgoalChanged = true;
543
544
545
                    }
                }
            }
546
        } catch (\ErrorException $e) {
547
            Log::error($e->getMessage());
Dominik Hebeler's avatar
Dominik Hebeler committed
548
        } finally {
549
            $requestTime = microtime(true) - $startTime;
Dominik Hebeler's avatar
Dominik Hebeler committed
550
            \App\PrometheusExporter::Duration($requestTime, "adgoal");
551
        }
552
        return true;
553
554

    }
555

556
    public function humanVerification(&$results)
557
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
558
        # Let's check if we need to implement a redirect for human verification
559
560
        if ($this->verificationCount > 10) {
            foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
561
562
563
                $link = $result->link;
                $day = Carbon::now()->day;
                $pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
564
565
566
                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
Dominik Hebeler's avatar
Dominik Hebeler committed
567
                $result->link = $url;
568
                $result->proxyLink = $proxyUrl;
Dominik Hebeler's avatar
Dominik Hebeler committed
569
570
571
572
            }
        }
    }

Phil Höfer's avatar
Phil Höfer committed
573
574
575
576
577
    public function authorize($key)
    {
        $postdata = http_build_query(array(
            'dummy' => rand(),
        ));
Karl Hasselbring's avatar
Karl Hasselbring committed
578
579
580
581
582
583
        $opts = array(
            'http' => array(
                'method' => 'POST',
                'header' => 'Content-type: application/x-www-form-urlencoded',
                'content' => $postdata,
            ),
Phil Höfer's avatar
Phil Höfer committed
584
585
586
587
588
        );

        $context = stream_context_create($opts);

        try {
589
            $link = "https://key.metager3.de/" . urlencode($key) . "/request-permission/api-access";
Phil Höfer's avatar
Phil Höfer committed
590
591
592
593
594
595
596
597
598
599
600
            $result = json_decode(file_get_contents($link, false, $context));
            if ($result->{'api-access'} == true) {
                return true;
            } else {
                return false;
            }
        } catch (\ErrorException $e) {
            return false;
        }
    }

Karl's avatar
Karl committed
601
602
603
604
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

Dominik Hebeler's avatar
Dominik Hebeler committed
605
    public function createSearchEngines(Request $request, &$timings)
606
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
607
608
609
610
        if (!empty($timings)) {
            $timings["createSearchEngines"]["start"] = microtime(true) - $timings["starttime"];
        }

611
        # Wenn es kein Suchwort gibt
Dominik Hebeler's avatar
Dominik Hebeler committed
612
        if (!$request->filled("eingabe") || $this->q === "") {
613
            return;
614
        }
615

Dominik Hebeler's avatar
Dominik Hebeler committed
616
        $this->enabledSearchengines = [];
617
        $overtureEnabled = false;
618

619
620
621
        # Check if selected focus is valid
        if (empty($this->sumaFile->foki->{$this->fokus})) {
            $this->fokus = "web";
622
        }
623

Karl Hasselbring's avatar
Karl Hasselbring committed
624
        $sumaNames = $this->sumaFile->foki->{$this->fokus}->sumas;
625

Karl Hasselbring's avatar
Karl Hasselbring committed
626
627
628
        $sumas = [];
        foreach ($sumaNames as $sumaName) {
            $sumas[$sumaName] = $this->sumaFile->sumas->{$sumaName};
629
630
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
631
632
633
634
        if (!empty($timings)) {
            $timings["createSearchEngines"]["created engine array"] = microtime(true) - $timings["starttime"];
        }

Karl Hasselbring's avatar
Karl Hasselbring committed
635
636
        $this->removeAdsFromListIfAdfree($sumas);

Dominik Hebeler's avatar
Dominik Hebeler committed
637
638
639
640
        if (!empty($timings)) {
            $timings["createSearchEngines"]["removed ads"] = microtime(true) - $timings["starttime"];
        }

Karl Hasselbring's avatar
Karl Hasselbring committed
641
        foreach ($sumas as $sumaName => $suma) {
642
            # Check if this engine is disabled and can't be used
Karl Hasselbring's avatar
Karl Hasselbring committed
643
644
645
646
647
648
649
            $disabled = empty($suma->disabled) ? false : $suma->disabled;
            $autoDisabled = empty($suma->{"auto-disabled"}) ? false : $suma->{"auto-disabled"};
            if (
                $disabled || $autoDisabled
                || \Cookie::get($this->getFokus() . "_engine_" . $sumaName) === "off"
            ) {
                continue;
650
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
651

652
            $valid = true;
Karl Hasselbring's avatar
Karl Hasselbring committed
653
654
655
656

            # Check if this engine can use potentially defined query-filter
            foreach ($this->queryFilter as $filterName => $filter) {
                if (empty($this->sumaFile->filter->{"query-filter"}->$filterName->sumas->$sumaName)) {
657
658
                    $valid = false;
                    break;
Dominik Hebeler's avatar
Dominik Hebeler committed
659
660
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
661
662

            # Check if this engine can use potentially defined parameter-filter
Dominik Hebeler's avatar
Dominik Hebeler committed
663
664
            if ($valid) {
                foreach ($this->parameterFilter as $filterName => $filter) {
665
                    # We need to check if the searchengine supports the parameter value, too
Karl Hasselbring's avatar
Karl Hasselbring committed
666
                    if (empty($filter->sumas->$sumaName) || empty($filter->sumas->{$sumaName}->values->{$filter->value})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
667
668
669
670
671
                        $valid = false;
                        break;
                    }
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
672

673
            # Check if this engine should only be active when filter is used
Karl Hasselbring's avatar
Karl Hasselbring committed
674
            if ($suma->{"filter-opt-in"}) {
675
676
677
                # This search engine should only be used when a parameter filter of it is used
                $validTmp = false;
                foreach ($this->parameterFilter as $filterName => $filter) {
Karl Hasselbring's avatar
Karl Hasselbring committed
678
                    if (!empty($filter->sumas->{$sumaName})) {
679
680
681
682
683
684
685
686
                        $validTmp = true;
                        break;
                    }
                }
                if (!$validTmp) {
                    $valid = false;
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
687
688

            # If the suma is still valid, we can add it
689
            if ($valid) {
Karl Hasselbring's avatar
Karl Hasselbring committed
690
                $this->enabledSearchengines[$sumaName] = $suma;
691
            }
692
        }
693

Dominik Hebeler's avatar
Dominik Hebeler committed
694
695
696
697
        if (!empty($timings)) {
            $timings["createSearchEngines"]["filtered invalid engines"] = microtime(true) - $timings["starttime"];
        }

Karl Hasselbring's avatar
Karl Hasselbring committed
698
699
        # Include Yahoo Ads if Yahoo is not enabled as a searchengine
        if (!$this->apiAuthorized && $this->fokus != "bilder" && empty($this->enabledSearchengines["yahoo"]) && isset($this->sumaFile->sumas->{"yahoo-ads"})) {
700
701
702
            $this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
        }

703
704
705
706
707
708
        # Special case if search engines are disabled
        # Since bing is normally only active if a filter is set but it should be active, too if yahoo is disabled
        if ($this->getFokus() === "web" && empty($this->enabledSearchengines["yahoo"]) && \Cookie::get("web_engine_bing") !== "off") {
            $this->enabledSearchengines["bing"] = $this->sumaFile->sumas->{"bing"};
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
709
        if (sizeof($this->enabledSearchengines) === 0) {
710
711
            $filter = "";
            foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
Dominik Hebeler's avatar
Dominik Hebeler committed
712
                $filter .= trans($this->sumaFile->filter->{"query-filter"}->{$queryFilter}->name) . ",";
713
714
            }
            $filter = rtrim($filter, ",");
Karl Hasselbring's avatar
Karl Hasselbring committed
715
716
            $error = trans('metaGer.engines.noSpecialSearch', [
                'fokus' => trans($this->sumaFile->foki->{$this->fokus}->{"display-name"}),
717
                'filter' => $filter,
Karl Hasselbring's avatar
Karl Hasselbring committed
718
            ]);
719
            $this->errors[] = $error;
720
        }
721
        $this->setEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
722
723
724
725
        if (!empty($timings)) {
            $timings["createSearchEngines"]["saved engines"] = microtime(true) - $timings["starttime"];
        }

726
    }
727

Karl Hasselbring's avatar
Karl Hasselbring committed
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
    private function removeAdsFromListIfAdfree(&$sumas)
    {
        if ($this->apiAuthorized) {
            foreach ($sumas as $sumaName => $suma) {
                $ads = $suma->ads ?? false;
                if ($ads) {
                    unset($sumas[$sumaName]);

                    $adBackups = $suma->{"ad-backups"} ?? [];
                    $adBackupName = $adBackups->{$this->fokus} ?? null;
                    if (isset($adBackupName)) {
                        $this->sumaFile->sumas->{$adBackupName}->{"filter-opt-in"} = false;
                    }
                }
            }
        }
    }

746
747
    public function setEngines(Request $request, $enabledSearchengines = [])
    {
748
        if ($this->requestIsCached($request)) {
749
            # If this is a page other than 1 the request is "cached"
750
            $engines = $this->getCachedEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
751
            # We need to edit some Options of the Cached Search Engines
Phil Höfer's avatar
Phil Höfer committed
752
            foreach ($engines as $engine) {
753
                $engine->setResultHash($this->getSearchUid());
Dominik Hebeler's avatar
Dominik Hebeler committed
754
            }
755
            $this->engines = $engines;
756
        } else {
757
758
759
            if (sizeof($enabledSearchengines) > 0) {
                $this->enabledSearchengines = $enabledSearchengines;
            }
760
            $this->actuallyCreateSearchEngines($this->enabledSearchengines);
761
        }
762
    }
Dominik Hebeler's avatar
Dominik Hebeler committed
763

Dominik Hebeler's avatar
Dominik Hebeler committed
764
    public function startSearch(&$timings)
765
    {
766
767
768
769
770
        if (!empty($timings)) {
            $timings["startSearch"]["start"] = microtime(true) - $timings["starttime"];
        }

        # Check all engines for Cached responses
Dominik Hebeler's avatar
Dominik Hebeler committed
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
        $this->checkCache();

        if (!empty($timings)) {
            $timings["startSearch"]["cache checked"] = microtime(true) - $timings["starttime"];
        }

        # Wir starten alle Suchen
        foreach ($this->engines as $engine) {
            $engine->startSearch($this, $timings);
        }
        if (!empty($timings)) {
            $timings["startSearch"]["searches started"] = microtime(true) - $timings["starttime"];
        }

    }

    public function checkCache()
    {
789
790
791
792
793
794
795
796
797
798
799
800
801
802
        if ($this->canCache()) {
            $keys = [];
            foreach ($this->engines as $engine) {
                $keys[] = $engine->hash;
            }
            $cacheValues = Cache::many($keys);
            foreach ($this->engines as $engine) {
                if ($cacheValues[$engine->hash] !== null) {
                    $engine->cached = true;
                    $engine->retrieveResults($this, $cacheValues[$engine->hash]);
                }
            }
        }

803
804
    }

805
806
    # Spezielle Suchen und Sumas

807
    public function sumaIsSelected($suma, $request, $custom)
808
    {
809
        if ($custom) {
810
            if ($request->filled("engine_" . strtolower($suma["name"]))) {
811
812
813
814
815
816
817
818
819
820
821
                return true;
            }
        } else {
            $types = explode(",", $suma["type"]);
            if (in_array($this->fokus, $types)) {
                return true;
            }
        }
        return false;
    }

822
    public function actuallyCreateSearchEngines($enabledSearchengines)
823
824
    {
        $engines = [];
825
        foreach ($enabledSearchengines as $engineName => $engine) {
826

827
            if (!isset($engine->{"parser-class"})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
828
829
                die(var_dump($engine));
            }
830
            # Setze Pfad zu Parser
831
            $path = "App\\Models\\parserSkripte\\" . $engine->{"parser-class"};
832
833

            # Prüfe ob Parser vorhanden
834
835
836
            if (!file_exists(app_path() . "/Models/parserSkripte/" . $engine->{"parser-class"} . ".php")) {
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen, da kein Parser existiert");
                $this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine->{"display-name"}]);
837
838
839
840
841
842
                continue;
            }

            # Es wird versucht die Suchengine zu erstellen
            $time = microtime();
            try {
843
                $tmp = new $path($engineName, $engine, $this);
844
            } catch (\ErrorException $e) {
845
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen. " . $e);
846
847
848
                continue;
            }

849
            $engines[] = $tmp;
850
        }
851
        $this->engines = $engines;
852
853
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
854
855
856
857
858
859
860
    public function getAvailableParameterFilter()
    {
        $parameterFilter = $this->sumaFile->filter->{"parameter-filter"};

        $availableFilter = [];

        foreach ($parameterFilter as $filterName => $filter) {
861
            $values = clone $filter->values;
Dominik Hebeler's avatar
Dominik Hebeler committed
862
863
864
            # Check if any of the enabled search engines provide this filter
            foreach ($this->enabledSearchengines as $engineName => $engine) {
                if (!empty($filter->sumas->$engineName)) {
865
866
                    if (empty($availableFilter[$filterName])) {
                        $availableFilter[$filterName] = $filter;
867
868
                        foreach ($availableFilter[$filterName]->values as $key => $value) {
                            if ($key !== "nofilter") {
869
870
871
                                unset($availableFilter[$filterName]->values->{$key});
                            }
                        }
872
873
                    }
                    if (empty($availableFilter[$filterName]->values)) {
874
                        $availableFilter[$filterName]->values = new \stdClass();
875
876
                    }
                    foreach ($filter->sumas->{$engineName}->values as $key => $value) {
877
                        $availableFilter[$filterName]->values->{$key} = $values->$key;
878
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
879
880
                }
            }
881
882
            # We will also add the filter from the opt-in search engines (the searchengines that are only used when a filter of it is too)
            foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
883
                if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"} && \Cookie::get($this->getFokus() . "_engine_" . $suma) !== "off") {
884
                    if (!empty($filter->sumas->{$suma})) {
885
886
                        # If the searchengine is disabled this filter shouldn't be available
                        if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
Karl Hasselbring's avatar
Karl Hasselbring committed
887
888
                            || (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)
                        ) {
889
890
                            continue;
                        }
891
892
                        if (empty($availableFilter[$filterName])) {
                            $availableFilter[$filterName] = $filter;
893
894
                            foreach ($availableFilter[$filterName]->values as $key => $value) {
                                if ($key !== "nofilter") {
895
896
897
                                    unset($availableFilter[$filterName]->values->{$key});
                                }
                            }
898
899
                        }
                        if (empty($availableFilter[$filterName]->values)) {
900
                            $availableFilter[$filterName<