MetaGer.php 73.4 KB
Newer Older
1
<?php
Karl Hasselbring's avatar
Karl Hasselbring committed
2

3
4
5
namespace App;

use App;
6
use Cache;
7
use Carbon;
Davide's avatar
Davide committed
8
use Cookie;
9
use Illuminate\Http\Request;
10
use Illuminate\Support\Facades\Redis;
11
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix    
Dominik Hebeler committed
12
use LaravelLocalization;
13
use Log;
14
use Monospice\LaravelRedisSentinel\RedisSentinel;
15
use Predis\Connection\ConnectionException;
16
17
18

class MetaGer
{
Dominik Hebeler's avatar
Dominik Hebeler committed
19
20
    const FETCHQUEUE_KEY = "fetcher.queue";

21
    # Einstellungen für die Suche
22
23
    public $alteredQuery = "";
    public $alterationOverrideQuery = "";
24
    protected $fokus;
Dominik Hebeler's avatar
update    
Dominik Hebeler committed
25
    protected $test;
26
27
28
29
30
31
    protected $eingabe;
    protected $q;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
32
    protected $time = 2000;
33
    protected $hostBlacklist = [];
34
    protected $domainBlacklist = [];
35
36
37
38
    private $urlBlacklist = [];
    protected $stopWords = [];
    protected $phrases = [];
    protected $engines = [];
39
    protected $totalResults = 0;
40
    protected $results = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
41
42
    protected $queryFilter = [];
    protected $parameterFilter = [];
43
    protected $ads = [];
44
    protected $infos = [];
45
46
47
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
48
    protected $availableFoki = [];
49
50
    protected $startCount = 0;
    protected $canCache = false;
51
    # Daten über die Abfrage$
52
    protected $ip;
Dominik Hebeler's avatar
Dominik Hebeler committed
53
    protected $useragent;
54
55
    protected $language;
    protected $agent;
56
    protected $apiKey = "";
Phil Höfer's avatar
Phil Höfer committed
57
    protected $apiAuthorized = false;
58
    protected $next = [];
59
60
61
62
63
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
64
    protected $newtab;
65
    protected $domainsBlacklisted = [];
66
    protected $adDomainsBlacklisted = [];
67
    protected $urlsBlacklisted = [];
68
    protected $adUrlsBlacklisted = [];
69
    protected $url;
70
    protected $fullUrl;
Aria's avatar
Aria committed
71
    protected $enabledSearchengines = [];
72
    protected $languageDetect;
Dominik Hebeler's avatar
Dominik Hebeler committed
73
74
    protected $verificationId;
    protected $verificationCount;
75
    protected $searchUid;
76
77
78
79
    protected $redisResultWaitingKey;
    protected $redisResultEngineList;
    protected $redisEngineResult;
    protected $redisCurrentResultList;
Dominik Hebeler's avatar
Dominik Hebeler committed
80
    public $starttime;
81

82
    public function __construct($hash = "")
83
    {
84
        # Timer starten
85
        $this->starttime = microtime(true);
86
        # Versuchen Blacklists einzulesen
87
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
88
            $tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
89
            $this->domainsBlacklisted = explode("\n", $tmp);
90
91
            $tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted = explode("\n", $tmp);
92
        } else {
93
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
94
        }
95
96
97
98
99
100
101
102
103
        # Versuchen Blacklists einzulesen
        if (file_exists(config_path() . "/adBlacklistDomains.txt") && file_exists(config_path() . "/adBlacklistUrl.txt")) {
            $tmp = file_get_contents(config_path() . "/adBlacklistDomains.txt");
            $this->adDomainsBlacklisted = explode("\n", $tmp);
            $tmp = file_get_contents(config_path() . "/adBlacklistUrl.txt");
            $this->adUrlsBlacklisted = explode("\n", $tmp);
        } else {
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
        }
104

105
        # Parser Skripte einhängen
106
107
108
109
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
110
                require_once $path;
111
112
113
            }
        }

114
        # Cachebarkeit testen
115
116
117
118
119
120
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
121
122
123
124
125
        if ($hash === "") {
            $this->searchUid = md5(uniqid());
        } else {
            $this->searchUid = $hash;
        }
126
127
128
129
130
131
132
        $redisPrefix = "search";
        # This is a list on which the MetaGer process can do a blocking pop to wait for new results
        $this->redisResultWaitingKey = $redisPrefix . "." . $this->searchUid . ".ready";
        # This is a list of searchengines which have delivered results for this search
        $this->redisResultEngineList = $redisPrefix . "." . $this->searchUid . ".engines";
        # This is the key where the results of the engine are stored as well as some statistical data
        $this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results.";
133
134
        # A list of all search results already delivered to the user (sorted of course)
        $this->redisCurrentResultList = $redisPrefix . "." . $this->searchUid . ".currentResults";
135
    }
136

137
    # Erstellt aus den gesammelten Ergebnissen den View
138
    public function createView($quicktipResults = [])
139
    {
140
141
        # Hiermit werden die evtl. ausgewählten SuMas extrahiert, damit die Input-Boxen richtig gesetzt werden können
        $focusPages = [];
Aria Givi's avatar
Aria Givi committed
142

143
        foreach ($this->request->all() as $key => $value) {
144
145
            if (starts_with($key, 'engine_') && $value === 'on') {
                $focusPages[] = $key;
146
147
148
            }
        }

149
        $viewResults = [];
150
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
151
        foreach ($this->results as $result) {
152
153
154
155
            $viewResults[] = get_object_vars($result);
        }
        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();
156
157
        if ($this->fokus === "bilder") {
            switch ($this->out) {
158
                case 'results':
159
                    return view('resultpages.results_images')
160
161
162
163
164
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
165
                        ->with('apiAuthorized', $this->apiAuthorized)
166
                        ->with('metager', $this)
167
                        ->with('browser', (new Agent())->browser());
168
                default:
169
                    return view('resultpages.resultpage_images')
170
171
172
173
174
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
175
                        ->with('apiAuthorized', $this->apiAuthorized)
176
                        ->with('metager', $this)
177
                        ->with('browser', (new Agent())->browser())
178
                        ->with('quicktips', $quicktipResults)
179
180
                        ->with('focus', $this->fokus)
                        ->with('resultcount', count($this->results));
181
            }
182
183
184
        } else {
            switch ($this->out) {
                case 'results':
185
                    return view('resultpages.results')
186
187
188
189
190
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
191
                        ->with('apiAuthorized', $this->apiAuthorized)
192
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
193
194
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
195
196
                    break;
                case 'results-with-style':
197
                    return view('resultpages.resultpage')
198
199
200
201
202
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
203
                        ->with('apiAuthorized', $this->apiAuthorized)
204
205
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
Dominik Hebeler's avatar
Dominik Hebeler committed
206
207
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
208
                    break;
Phil Höfer's avatar
Phil Höfer committed
209
                case 'rich':
210
                    return view('resultpages.metager3rich')
211
                        ->with('results', $viewResults)
Phil Höfer's avatar
Phil Höfer committed
212
213
214
215
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
216
                        ->with('apiAuthorized', $this->apiAuthorized)
Phil Höfer's avatar
Phil Höfer committed
217
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
218
219
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
Phil Höfer's avatar
Phil Höfer committed
220
                    break;
221
                case 'rss20':
222
                    return view('resultpages.metager3resultsrss20')
223
224
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
Phil Höfer's avatar
Phil Höfer committed
225
                        ->with('apiAuthorized', $this->apiAuthorized)
226
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
227
228
                        ->with('resultcount', sizeof($viewResults))
                        ->with('fokus', $this->fokus);
229
                    break;
230
                case 'api':
Phil Höfer's avatar
Phil Höfer committed
231
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'key' => $this->apiKey, 'apiAuthorized' => $this->apiAuthorized])->header('Content-Type', 'application/xml');
232
                    break;
Aria Givi's avatar
Aria Givi committed
233
                case 'atom10':
Phil Höfer's avatar
Phil Höfer committed
234
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'key' => $this->apiKey, 'apiAuthorized' => true])
235
                        ->header('Content-Type', 'application/xml');
Aria Givi's avatar
Aria Givi committed
236
                    break;
237
                case 'result-count':
238
239
                    # Wir geben die Ergebniszahl und die benötigte Zeit zurück:
                    return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
240
                    break;
241
                default:
242
                    return view('resultpages.resultpage')
243
                        ->with('eingabe', $this->eingabe)
244
                        ->with('focusPages', $focusPages)
245
246
247
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
248
                        ->with('apiAuthorized', $this->apiAuthorized)
249
                        ->with('metager', $this)
250
                        ->with('browser', (new Agent())->browser())
251
                        ->with('quicktips', $quicktipResults)
252
253
                        ->with('resultcount', count($this->results))
                        ->with('focus', $this->fokus);
254
255
                    break;
            }
256
257
258
        }
    }

259
    public function prepareResults(&$timings = null)
260
    {
Phil Höfer's avatar
Phil Höfer committed
261
262
        $engines = $this->engines;
        // combine
263
        $this->combineResults($engines);
264
        if (!empty($timings)) {
265
266
            $timings["prepareResults"]["combined results"] = microtime(true) - $timings["starttime"];
        }
Phil Höfer's avatar
Phil Höfer committed
267
        // misc (WiP)
268
        if ($this->fokus == "nachrichten") {
269
270
271
            $this->results = array_filter($this->results, function ($v, $k) {
                return !is_null($v->getRank());
            }, ARRAY_FILTER_USE_BOTH);
272
273
274
275
276
277
278
279
280
281
            uasort($this->results, function ($a, $b) {
                $datea = $a->getDate();
                $dateb = $b->getDate();
                return $dateb - $datea;
            });
        } else {
            uasort($this->results, function ($a, $b) {
                if ($a->getRank() == $b->getRank()) {
                    return 0;
                }
282

283
284
285
                return ($a->getRank() < $b->getRank()) ? 1 : -1;
            });
        }
286
        if (!empty($timings)) {
287
288
            $timings["prepareResults"]["sorted results"] = microtime(true) - $timings["starttime"];
        }
289
290
        # Validate Results
        $newResults = [];
291
292
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
293
                $newResults[] = $result;
294
            }
295
296
        }
        $this->results = $newResults;
297
        if (!empty($timings)) {
298
299
            $timings["prepareResults"]["validated results"] = microtime(true) - $timings["starttime"];
        }
300
        
Davide Aprea's avatar
Davide Aprea committed
301
        $this->duplicationCheck();
Dominik Hebeler's avatar
Dominik Hebeler committed
302
        if (!empty($timings)) {
303
304
            $timings["prepareResults"]["duplications checked"] = microtime(true) - $timings["starttime"];
        }
305
306
307
308
        # Validate Advertisements
        $newResults = [];
        foreach ($this->ads as $ad) {
            if (($ad->strippedHost !== "" && (in_array($ad->strippedHost, $this->adDomainsBlacklisted) ||
309
310
                in_array($ad->strippedLink, $this->adUrlsBlacklisted))) || ($ad->strippedHostAnzeige !== "" && (in_array($ad->strippedHostAnzeige, $this->adDomainsBlacklisted) ||
                in_array($ad->strippedLinkAnzeige, $this->adUrlsBlacklisted)))
Karl Hasselbring's avatar
Karl Hasselbring committed
311
            ) {
312
313
314
315
316
                continue;
            }
            $newResults[] = $ad;
        }
        $this->ads = $newResults;
317
        if (!empty($timings)) {
318
319
            $timings["prepareResults"]["validated ads"] = microtime(true) - $timings["starttime"];
        }
320
        #Adgoal Implementation
321
322
323
324
325
        if (empty($this->adgoalLoaded)) {
            $this->adgoalLoaded = false;
        }
        if (!$this->apiAuthorized && !$this->adgoalLoaded) {
            if (empty($this->adgoalHash)) {
Dominik Hebeler's avatar
Dominik Hebeler committed
326
327
328
329
330
                if (!empty($this->jskey)) {
                    $js = Redis::connection('cache')->lpop("js" . $this->jskey);
                    if ($js !== null && boolval($js)) {
                        $this->javascript = true;
                    }
331
                }
332
                $this->adgoalHash = $this->startAdgoal($this->results);
333
                if (!empty($timings)) {
334
335
                    $timings["prepareResults"]["started adgoal"] = microtime(true) - $timings["starttime"];
                }
336
            }
337
        
Dominik Hebeler's avatar
Dominik Hebeler committed
338
            if (!$this->javascript) {
339
                $this->adgoalLoaded = $this->parseAdgoal($this->results, $this->adgoalHash, true);
340
                if (!empty($timings)) {
341
342
                    $timings["prepareResults"]["parsed adgoal"] = microtime(true) - $timings["starttime"];
                }
343
344
            } else {
                $this->adgoalLoaded = $this->parseAdgoal($this->results, $this->adgoalHash, false);
345
                if (!empty($timings)) {
346
347
                    $timings["prepareResults"]["parsed adgoal"] = microtime(true) - $timings["starttime"];
                }
348
349
350
            }
        } else {
            $this->adgoalLoaded = true;
351
        }
352

Dominik Hebeler's avatar
Dominik Hebeler committed
353
        # Human Verification
354
355
        $this->humanVerification($this->results);
        $this->humanVerification($this->ads);
356
        if (!empty($timings)) {
357
358
            $timings["prepareResults"]["human verification"] = microtime(true) - $timings["starttime"];
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
359

360
        $counter = 0;
361
        $firstRank = 0;
362

363
        if (count($this->results) <= 0) {
364
365
366
367
368
369
            if (strlen($this->site) > 0) {
                $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
                $this->errors[] = trans('metaGer.results.failedSitesearch', ['altSearch' => $no_sitesearch_query]);
            } else {
                $this->errors[] = trans('metaGer.results.failed');
            }
370
        }
371

372
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
373
            $page = $this->page + 1;
374
            $this->next = [
375
376
                'page' => $page,
                'engines' => $this->next,
377
            ];
378
            Cache::put($this->getSearchUid(), serialize($this->next), 60 * 60);
379
            if (!empty($timings)) {
380
381
                $timings["prepareResults"]["filled cache"] = microtime(true) - $timings["starttime"];
            }
382
383
        } else {
            $this->next = [];
384
        }
385
    }
386

Phil Höfer's avatar
Phil Höfer committed
387
388
389
390
391
392
393
394
395
396
397
    public function combineResults($engines)
    {
        foreach ($engines as $engine) {
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
            foreach ($engine->results as $result) {
                if ($result->valid) {
398
                    $this->results[] = clone $result;
Phil Höfer's avatar
Phil Höfer committed
399
400
401
                }
            }
            foreach ($engine->ads as $ad) {
402
                $this->ads[] = clone $ad;
Phil Höfer's avatar
Phil Höfer committed
403
404
405
406
            }
        }
    }

Davide Aprea's avatar
Davide Aprea committed
407
408
409
    public function duplicationCheck()
    {
        $arr = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
410
        for ($i = 0; $i < count($this->results); $i++) {
Davide Aprea's avatar
Davide Aprea committed
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
            $link = $this->results[$i]->link;

            if (strpos($link, "http://") === 0) {
                $link = substr($link, 7);
            }
    
            if (strpos($link, "https://") === 0) {
                $link = substr($link, 8);
            }
    
            if (strpos($link, "www.") === 0) {
                $link = substr($link, 4);
            }
    
            $link = trim($link, "/");
            $hash = md5($link);

Dominik Hebeler's avatar
Dominik Hebeler committed
428
            if (isset($arr[$link])) {
Davide Aprea's avatar
Davide Aprea committed
429
430
                $arr[$link]->gefVon[] = $this->results[$i]->gefVon[0];
                $arr[$link]->gefVonLink[] = $this->results[$i]->gefVonLink[0];
Davide Aprea's avatar
Davide Aprea committed
431
432
                array_splice($this->results, $i, 1);
                $i--;
Dominik Hebeler's avatar
Dominik Hebeler committed
433
                if ($arr[$link]->new === true || $this->results[$i]->new === true) {
Davide Aprea's avatar
Davide Aprea committed
434
435
                    $arr[$link]->changed = true;
                }
Dominik Hebeler's avatar
Dominik Hebeler committed
436
            } else {
Davide Aprea's avatar
Davide Aprea committed
437
438
439
440
441
                $arr[$link] = &$this->results[$i];
            }
        }
    }

442
    public function startAdgoal(&$results)
443
    {
444
        $publicKey = getenv('adgoal_public');
445
        $privateKey = getenv('adgoal_private');
446
        if ($publicKey === false) {
447
            return true;
448
449
        }
        $tldList = "";
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
        foreach ($results as $result) {
            if (!$result->new) {
                continue;
            }
            $link = $result->link;
            if (strpos($link, "http") !== 0) {
                $link = "http://" . $link;
            }
            $tldList .= parse_url($link, PHP_URL_HOST) . ",";
            $result->tld = parse_url($link, PHP_URL_HOST);
        }
        $tldList = rtrim($tldList, ",");

        # Hashwert
        $hash = md5("meta" . $publicKey . $tldList . "GER");
465

466
467
468
469
470
471
472
473
474
475
476
477
478
479
        # Query
        $query = $this->q;

        $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . urlencode($publicKey) . "&k=" . urlencode($hash) . "&tld=" . urlencode($tldList) . "&q=" . urlencode($query);

        // Submit fetch job to worker
        $mission = [
            "resulthash" => $hash,
            "url" => $link,
            "useragent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0",
            "username" => null,
            "password" => null,
            "headers" => null,
            "cacheDuration" => 60,
480
            "name" => "Adgoal",
481
482
483
484
485
486
487
488
489
490
491
492
493
        ];
        $mission = json_encode($mission);
        Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);

        return $hash;
    }

    public function parseAdgoal(&$results, $hash, $waitForResult)
    {
        # Wait for result
        $startTime = microtime(true);
        $answer = null;

494
        # Hash is true if Adgoal request wasn't started in the first place
495
        if ($hash === true) {
496
497
498
            return true;
        }

499
500
        if ($waitForResult) {
            while (microtime(true) - $startTime < 5) {
501
                $answer = Cache::get($hash);
502
503
504
505
506
507
508
                if ($answer === null) {
                    usleep(50 * 1000);
                } else {
                    break;
                }
            }
        } else {
509
            $answer = Cache::get($hash);
510
511
512
513
        }
        if ($answer === null) {
            return false;
        }
514
        try {
515
516
517
518
519
            $answer = json_decode($answer);
            $publicKey = getenv('adgoal_public');
            $privateKey = getenv('adgoal_private');

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
520
            foreach ($results as $result) {
521
                $link = $result->link;
522
523
524
                $result->tld = parse_url($link, PHP_URL_HOST);
            }

525
            foreach ($answer as $el) {
526
                $hoster = $el[0];
527
                $hash = $el[1];
528

529
                foreach ($results as $result) {
530
                    if ($hoster === $result->tld && !$result->partnershop) {
531
532
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
533
                        if ($result->image !== "") {
534
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . urlencode($hash) . ".gif";
535
                        } else {
536
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . urlencode($hash) . ".gif";
537
538
                        }

539
                        # Den Link hinzufügen:
540
                        $targetUrl = $result->link;
541
542
                        # Query
                        $query = $this->q;
543

544
                        $gateHash = md5($targetUrl . $privateKey);
545
                        $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . urlencode($publicKey) . "&k=" . urlencode($gateHash) . "&url=" . urlencode($targetUrl) . "&q=" . urlencode($query);
546
                        $result->link = $newLink;
547
                        $result->partnershop = true;
Davide Aprea's avatar
Davide Aprea committed
548
                        $result->changed = true;
549
550
551
                    }
                }
            }
552
        } catch (\ErrorException $e) {
553
            Log::error($e->getMessage());
Dominik Hebeler's avatar
Dominik Hebeler committed
554
        } finally {
555
            $requestTime = microtime(true) - $startTime;
Dominik Hebeler's avatar
Dominik Hebeler committed
556
            \App\PrometheusExporter::Duration($requestTime, "adgoal");
557
        }
558
        return true;
559
    }
560

561
    public function humanVerification(&$results)
562
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
563
        # Let's check if we need to implement a redirect for human verification
564
565
        if ($this->verificationCount > 10) {
            foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
566
567
568
                $link = $result->link;
                $day = Carbon::now()->day;
                $pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
569
570
571
                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
Dominik Hebeler's avatar
Dominik Hebeler committed
572
                $result->link = $url;
573
                $result->proxyLink = $proxyUrl;
Dominik Hebeler's avatar
Dominik Hebeler committed
574
575
576
577
            }
        }
    }

Phil Höfer's avatar
Phil Höfer committed
578
579
580
581
582
    public function authorize($key)
    {
        $postdata = http_build_query(array(
            'dummy' => rand(),
        ));
Karl Hasselbring's avatar
Karl Hasselbring committed
583
584
585
586
587
588
        $opts = array(
            'http' => array(
                'method' => 'POST',
                'header' => 'Content-type: application/x-www-form-urlencoded',
                'content' => $postdata,
            ),
Phil Höfer's avatar
Phil Höfer committed
589
590
591
592
593
        );

        $context = stream_context_create($opts);

        try {
594
            $link = "https://key.metager3.de/" . urlencode($key) . "/request-permission/api-access";
Phil Höfer's avatar
Phil Höfer committed
595
596
597
598
599
600
601
602
603
604
605
            $result = json_decode(file_get_contents($link, false, $context));
            if ($result->{'api-access'} == true) {
                return true;
            } else {
                return false;
            }
        } catch (\ErrorException $e) {
            return false;
        }
    }

Karl's avatar
Karl committed
606
607
608
609
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

Dominik Hebeler's avatar
Dominik Hebeler committed
610
    public function createSearchEngines(Request $request, &$timings)
611
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
612
613
614
615
        if (!empty($timings)) {
            $timings["createSearchEngines"]["start"] = microtime(true) - $timings["starttime"];
        }

616
        # Wenn es kein Suchwort gibt
Dominik Hebeler's avatar
Dominik Hebeler committed
617
        if (!$request->filled("eingabe") || $this->q === "") {
618
            return;
619
        }
620

Dominik Hebeler's avatar
Dominik Hebeler committed
621
        $this->enabledSearchengines = [];
622
        $overtureEnabled = false;
623

624
625
626
        # Check if selected focus is valid
        if (empty($this->sumaFile->foki->{$this->fokus})) {
            $this->fokus = "web";
627
        }
628

Karl Hasselbring's avatar
Karl Hasselbring committed
629
        $sumaNames = $this->sumaFile->foki->{$this->fokus}->sumas;
630

Karl Hasselbring's avatar
Karl Hasselbring committed
631
632
633
        $sumas = [];
        foreach ($sumaNames as $sumaName) {
            $sumas[$sumaName] = $this->sumaFile->sumas->{$sumaName};
634
635
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
636
637
638
639
        if (!empty($timings)) {
            $timings["createSearchEngines"]["created engine array"] = microtime(true) - $timings["starttime"];
        }

Karl Hasselbring's avatar
Karl Hasselbring committed
640
641
        $this->removeAdsFromListIfAdfree($sumas);

Dominik Hebeler's avatar
Dominik Hebeler committed
642
643
644
645
        if (!empty($timings)) {
            $timings["createSearchEngines"]["removed ads"] = microtime(true) - $timings["starttime"];
        }

Karl Hasselbring's avatar
Karl Hasselbring committed
646
        foreach ($sumas as $sumaName => $suma) {
647
            # Check if this engine is disabled and can't be used
Karl Hasselbring's avatar
Karl Hasselbring committed
648
649
650
651
652
653
654
            $disabled = empty($suma->disabled) ? false : $suma->disabled;
            $autoDisabled = empty($suma->{"auto-disabled"}) ? false : $suma->{"auto-disabled"};
            if (
                $disabled || $autoDisabled
                || \Cookie::get($this->getFokus() . "_engine_" . $sumaName) === "off"
            ) {
                continue;
655
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
656

657
            $valid = true;
Karl Hasselbring's avatar
Karl Hasselbring committed
658
659
660
661

            # Check if this engine can use potentially defined query-filter
            foreach ($this->queryFilter as $filterName => $filter) {
                if (empty($this->sumaFile->filter->{"query-filter"}->$filterName->sumas->$sumaName)) {
662
663
                    $valid = false;
                    break;
Dominik Hebeler's avatar
Dominik Hebeler committed
664
665
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
666
667

            # Check if this engine can use potentially defined parameter-filter
Dominik Hebeler's avatar
Dominik Hebeler committed
668
669
            if ($valid) {
                foreach ($this->parameterFilter as $filterName => $filter) {
670
                    # We need to check if the searchengine supports the parameter value, too
Karl Hasselbring's avatar
Karl Hasselbring committed
671
                    if (empty($filter->sumas->$sumaName) || empty($filter->sumas->{$sumaName}->values->{$filter->value})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
672
673
674
675
676
                        $valid = false;
                        break;
                    }
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
677

678
            # Check if this engine should only be active when filter is used
Karl Hasselbring's avatar
Karl Hasselbring committed
679
            if ($suma->{"filter-opt-in"}) {
680
681
682
                # This search engine should only be used when a parameter filter of it is used
                $validTmp = false;
                foreach ($this->parameterFilter as $filterName => $filter) {
Karl Hasselbring's avatar
Karl Hasselbring committed
683
                    if (!empty($filter->sumas->{$sumaName})) {
684
685
686
687
688
689
690
691
                        $validTmp = true;
                        break;
                    }
                }
                if (!$validTmp) {
                    $valid = false;
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
692
693

            # If the suma is still valid, we can add it
694
            if ($valid) {
Karl Hasselbring's avatar
Karl Hasselbring committed
695
                $this->enabledSearchengines[$sumaName] = $suma;
696
            }
697
        }
698

Dominik Hebeler's avatar
Dominik Hebeler committed
699
700
701
702
        if (!empty($timings)) {
            $timings["createSearchEngines"]["filtered invalid engines"] = microtime(true) - $timings["starttime"];
        }

Karl Hasselbring's avatar
Karl Hasselbring committed
703
704
        # Include Yahoo Ads if Yahoo is not enabled as a searchengine
        if (!$this->apiAuthorized && $this->fokus != "bilder" && empty($this->enabledSearchengines["yahoo"]) && isset($this->sumaFile->sumas->{"yahoo-ads"})) {
705
706
707
            $this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
        }

708
709
710
711
712
713
        # Special case if search engines are disabled
        # Since bing is normally only active if a filter is set but it should be active, too if yahoo is disabled
        if ($this->getFokus() === "web" && empty($this->enabledSearchengines["yahoo"]) && \Cookie::get("web_engine_bing") !== "off") {
            $this->enabledSearchengines["bing"] = $this->sumaFile->sumas->{"bing"};
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
714
        if (sizeof($this->enabledSearchengines) === 0) {
715
716
            $filter = "";
            foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
Dominik Hebeler's avatar
Dominik Hebeler committed
717
                $filter .= trans($this->sumaFile->filter->{"query-filter"}->{$queryFilter}->name) . ",";
718
719
            }
            $filter = rtrim($filter, ",");
Karl Hasselbring's avatar
Karl Hasselbring committed
720
721
            $error = trans('metaGer.engines.noSpecialSearch', [
                'fokus' => trans($this->sumaFile->foki->{$this->fokus}->{"display-name"}),
722
                'filter' => $filter,
Karl Hasselbring's avatar
Karl Hasselbring committed
723
            ]);
724
            $this->errors[] = $error;
725
        }
726
        $this->setEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
727
728
729
        if (!empty($timings)) {
            $timings["createSearchEngines"]["saved engines"] = microtime(true) - $timings["starttime"];
        }
730
    }
731

Karl Hasselbring's avatar
Karl Hasselbring committed
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
    private function removeAdsFromListIfAdfree(&$sumas)
    {
        if ($this->apiAuthorized) {
            foreach ($sumas as $sumaName => $suma) {
                $ads = $suma->ads ?? false;
                if ($ads) {
                    unset($sumas[$sumaName]);

                    $adBackups = $suma->{"ad-backups"} ?? [];
                    $adBackupName = $adBackups->{$this->fokus} ?? null;
                    if (isset($adBackupName)) {
                        $this->sumaFile->sumas->{$adBackupName}->{"filter-opt-in"} = false;
                    }
                }
            }
        }
    }

750
751
    public function setEngines(Request $request, $enabledSearchengines = [])
    {
752
        if ($this->requestIsCached($request)) {
753
            # If this is a page other than 1 the request is "cached"
754
            $engines = $this->getCachedEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
755
            # We need to edit some Options of the Cached Search Engines
Phil Höfer's avatar
Phil Höfer committed
756
            foreach ($engines as $engine) {
757
                $engine->setResultHash($this->getSearchUid());
Dominik Hebeler's avatar
Dominik Hebeler committed
758
            }
759
            $this->engines = $engines;
760
        } else {
761
762
763
            if (sizeof($enabledSearchengines) > 0) {
                $this->enabledSearchengines = $enabledSearchengines;
            }
764
            $this->actuallyCreateSearchEngines($this->enabledSearchengines);
765
        }
766
    }
Dominik Hebeler's avatar
Dominik Hebeler committed
767

Dominik Hebeler's avatar
Dominik Hebeler committed
768
    public function startSearch(&$timings)
769
    {
770
771
772
773
774
        if (!empty($timings)) {
            $timings["startSearch"]["start"] = microtime(true) - $timings["starttime"];
        }

        # Check all engines for Cached responses
Dominik Hebeler's avatar
Dominik Hebeler committed
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
        $this->checkCache();

        if (!empty($timings)) {
            $timings["startSearch"]["cache checked"] = microtime(true) - $timings["starttime"];
        }

        # Wir starten alle Suchen
        foreach ($this->engines as $engine) {
            $engine->startSearch($this, $timings);
        }
        if (!empty($timings)) {
            $timings["startSearch"]["searches started"] = microtime(true) - $timings["starttime"];
        }
    }

    public function checkCache()
    {
792
793
794
795
796
797
798
799
800
801
802
803
804
        if ($this->canCache()) {
            $keys = [];
            foreach ($this->engines as $engine) {
                $keys[] = $engine->hash;
            }
            $cacheValues = Cache::many($keys);
            foreach ($this->engines as $engine) {
                if ($cacheValues[$engine->hash] !== null) {
                    $engine->cached = true;
                    $engine->retrieveResults($this, $cacheValues[$engine->hash]);
                }
            }
        }
805
806
    }

807
808
    # Spezielle Suchen und Sumas

809
    public function sumaIsSelected($suma, $request, $custom)
810
    {
811
        if ($custom) {
812
            if ($request->filled("engine_" . strtolower($suma["name"]))) {
813
814
815
816
817
818
819
820
821
822
823
                return true;
            }
        } else {
            $types = explode(",", $suma["type"]);
            if (in_array($this->fokus, $types)) {
                return true;
            }
        }
        return false;
    }

824
    public function actuallyCreateSearchEngines($enabledSearchengines)
825
826
    {
        $engines = [];
827
828
        foreach ($enabledSearchengines as $engineName => $engine) {
            if (!isset($engine->{"parser-class"})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
829
830
                die(var_dump($engine));
            }
831
            # Setze Pfad zu Parser
832
            $path = "App\\Models\\parserSkripte\\" . $engine->{"parser-class"};
833
834

            # Prüfe ob Parser vorhanden
835
836
837
            if (!file_exists(app_path() . "/Models/parserSkripte/" . $engine->{"parser-class"} . ".php")) {
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen, da kein Parser existiert");
                $this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine->{"display-name"}]);
838
839
840
841
842
843
                continue;
            }

            # Es wird versucht die Suchengine zu erstellen
            $time = microtime();
            try {
844
                $tmp = new $path($engineName, $engine, $this);
845
            } catch (\ErrorException $e) {
846
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen. " . $e);
847
848
849
                continue;
            }

850
            $engines[] = $tmp;
851
        }
852
        $this->engines = $engines;
853
854
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
855
856
857
858
859
860
861
    public function getAvailableParameterFilter()
    {
        $parameterFilter = $this->sumaFile->filter->{"parameter-filter"};

        $availableFilter = [];

        foreach ($parameterFilter as $filterName => $filter) {
862
            $values = clone $filter->values;
Dominik Hebeler's avatar
Dominik Hebeler committed
863
864
865
            # Check if any of the enabled search engines provide this filter
            foreach ($this->enabledSearchengines as $engineName => $engine) {
                if (!empty($filter->sumas->$engineName)) {
866
867
                    if (empty($availableFilter[$filterName])) {
                        $availableFilter[$filterName] = $filter;
868
869
                        foreach ($availableFilter[$filterName]->values as $key => $value) {
                            if ($key !== "nofilter") {
870
871
872
                                unset($availableFilter[$filterName]->values->{$key});
                            }
                        }
873
874
                    }
                    if (empty($availableFilter[$filterName]->values)) {
875
                        $availableFilter[$filterName]->values = new \stdClass();
876
877
                    }
                    foreach ($filter->sumas->{$engineName}->values as $key => $value) {
878
                        $availableFilter[$filterName]->values->{$key} = $values->$key;
879
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
880