MetaGer.php 57.4 KB
Newer Older
1
2
3
4
<?php
namespace App;

use App;
5
use Cache;
6
use Carbon;
7
use Illuminate\Http\Request;
Phil Höfer's avatar
Phil Höfer committed
8
use Illuminate\Support\Facades\Redis;
9
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix    
Dominik Hebeler committed
10
use LaravelLocalization;
11
use Log;
12
use Predis\Connection\ConnectionException;
13
14
15

class MetaGer
{
16
17
18
19
20
21
22
23
    # Einstellungen für die Suche
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
24
    protected $time = 2000;
25
    protected $hostBlacklist = [];
26
    protected $domainBlacklist = [];
27
28
29
30
    private $urlBlacklist = [];
    protected $stopWords = [];
    protected $phrases = [];
    protected $engines = [];
31
    protected $totalResults = 0;
32
    protected $results = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
33
34
    protected $queryFilter = [];
    protected $parameterFilter = [];
35
36
37
38
    protected $ads = [];
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
39
    protected $availableFoki = [];
40
41
    protected $startCount = 0;
    protected $canCache = false;
42
    # Daten über die Abfrage$
43
    protected $ip;
Dominik Hebeler's avatar
Dominik Hebeler committed
44
    protected $useragent;
45
46
    protected $language;
    protected $agent;
47
    protected $apiKey = "";
Phil Höfer's avatar
Phil Höfer committed
48
    protected $apiAuthorized = false;
49
50
51
52
53
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
54
    protected $newtab;
55
    protected $domainsBlacklisted = [];
56
    protected $urlsBlacklisted = [];
57
    protected $url;
58
    protected $fullUrl;
59
    protected $languageDetect;
Dominik Hebeler's avatar
Dominik Hebeler committed
60
61
    protected $verificationId;
    protected $verificationCount;
62

63
64
    public function __construct()
    {
65
        # Timer starten
66
        $this->starttime = microtime(true);
67
        # Versuchen Blacklists einzulesen
68
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
69
            $tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
70
            $this->domainsBlacklisted = explode("\n", $tmp);
71
72
            $tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted = explode("\n", $tmp);
73
        } else {
74
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
75
76
        }

77
        # Parser Skripte einhängen
78
79
80
81
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
82
                require_once $path;
83
84
85
            }
        }

86
        # Cachebarkeit testen
87
88
89
90
91
92
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
93
    }
94

95
    # Erstellt aus den gesammelten Ergebnissen den View
96
    public function createView($quicktipResults = [])
97
    {
98
99
        # Hiermit werden die evtl. ausgewählten SuMas extrahiert, damit die Input-Boxen richtig gesetzt werden können
        $focusPages = [];
Aria Givi's avatar
Aria Givi committed
100

101
        foreach ($this->request->all() as $key => $value) {
102
103
            if (starts_with($key, 'engine_') && $value === 'on') {
                $focusPages[] = $key;
104
105
106
            }
        }

107
        $viewResults = [];
108
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
109
        foreach ($this->results as $result) {
110
111
112
113
            $viewResults[] = get_object_vars($result);
        }
        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();
114
115
        if ($this->fokus === "bilder") {
            switch ($this->out) {
116
                case 'results':
117
                    return view('resultpages.results_images')
118
119
120
121
122
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
123
                        ->with('apiAuthorized', $this->apiAuthorized)
124
                        ->with('metager', $this)
125
                        ->with('browser', (new Agent())->browser());
126
                default:
127
                    return view('resultpages.resultpage_images')
128
129
130
131
132
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
133
                        ->with('apiAuthorized', $this->apiAuthorized)
134
                        ->with('metager', $this)
135
136
137
138
                        ->with('browser', (new Agent())->browser())
                        ->with('quicktips', $quicktipResults)
                        ->with('focus', $this->fokus)
                        ->with('resultcount', count($this->results));
139
            }
140
141
142
        } else {
            switch ($this->out) {
                case 'results':
143
                    return view('resultpages.results')
144
145
146
147
148
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
149
                        ->with('apiAuthorized', $this->apiAuthorized)
150
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
151
152
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
153
154
                    break;
                case 'results-with-style':
155
                    return view('resultpages.resultpage')
156
157
158
159
160
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
161
                        ->with('apiAuthorized', $this->apiAuthorized)
162
163
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
Dominik Hebeler's avatar
Dominik Hebeler committed
164
165
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
166
                    break;
Phil Höfer's avatar
Phil Höfer committed
167
                case 'rich':
168
                    return view('resultpages.metager3rich')
169
                        ->with('results', $viewResults)
Phil Höfer's avatar
Phil Höfer committed
170
171
172
173
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
174
                        ->with('apiAuthorized', $this->apiAuthorized)
Phil Höfer's avatar
Phil Höfer committed
175
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
176
177
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
Phil Höfer's avatar
Phil Höfer committed
178
                    break;
179
                case 'rss20':
180
                    return view('resultpages.metager3resultsrss20')
181
182
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
Phil Höfer's avatar
Phil Höfer committed
183
                        ->with('apiAuthorized', $this->apiAuthorized)
184
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
185
186
                        ->with('resultcount', sizeof($viewResults))
                        ->with('fokus', $this->fokus);
187
                    break;
188
189
190
                case 'api':
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'apiAuthorized' => $this->apiAuthorized])->header('Content-Type', 'application/xml');
                    break;
Aria Givi's avatar
Aria Givi committed
191
                case 'atom10':
192
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'apiAuthorized' => true])
193
                        ->header('Content-Type', 'application/xml');
Aria Givi's avatar
Aria Givi committed
194
                    break;
195
                case 'result-count':
196
197
                    # Wir geben die Ergebniszahl und die benötigte Zeit zurück:
                    return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
198
                    break;
199
                default:
200
                    return view('resultpages.resultpage')
201
                        ->with('eingabe', $this->eingabe)
202
                        ->with('focusPages', $focusPages)
203
204
205
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
206
                        ->with('apiAuthorized', $this->apiAuthorized)
207
                        ->with('metager', $this)
208
                        ->with('browser', (new Agent())->browser())
209
                        ->with('quicktips', $quicktipResults)
210
211
                        ->with('resultcount', count($this->results))
                        ->with('focus', $this->fokus);
212
213
                    break;
            }
214
215
216
        }
    }

Phil Höfer's avatar
Phil Höfer committed
217
    public function prepareResults()
218
    {
Phil Höfer's avatar
Phil Höfer committed
219
        $engines = $this->engines;
220

Phil Höfer's avatar
Phil Höfer committed
221
222
        // combine
        $combinedResults = $this->combineResults($engines);
223
224
225
226

        # Wir bestimmen die Sprache eines jeden Suchergebnisses
        $this->results = $this->addLangCodes($this->results);

Phil Höfer's avatar
Phil Höfer committed
227
228
229
230
231
        // sort
        //$sortedResults = $this->sortResults($engines);
        // filter
        // augment (boost&adgoal)
        // authorize
Phil Höfer's avatar
Phil Höfer committed
232
233
234
        if ($this->apiKey) {
            $this->apiAuthorized = $this->authorize($this->apiKey);
        }
Phil Höfer's avatar
Phil Höfer committed
235
        // misc (WiP)
236
        if ($this->fokus == "nachrichten") {
237
238
239
            $this->results = array_filter($this->results, function ($v, $k) {
                return !is_null($v->getRank());
            }, ARRAY_FILTER_USE_BOTH);
240
241
242
243
244
245
246
247
248
249
            uasort($this->results, function ($a, $b) {
                $datea = $a->getDate();
                $dateb = $b->getDate();
                return $dateb - $datea;
            });
        } else {
            uasort($this->results, function ($a, $b) {
                if ($a->getRank() == $b->getRank()) {
                    return 0;
                }
250

251
252
253
                return ($a->getRank() < $b->getRank()) ? 1 : -1;
            });
        }
254

255
256
        # Validate Results
        $newResults = [];
257
258
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
259
                $newResults[] = $result;
260
261
            }

262
263
264
        }
        $this->results = $newResults;

265
        #Adgoal Implementation
266
267
268
        if (!$this->apiAuthorized) {
            $this->results = $this->parseAdgoal($this->results);
        }
269

Dominik Hebeler's avatar
Dominik Hebeler committed
270
271
        # Human Verification
        $this->results = $this->humanVerification($this->results);
272
        $this->ads = $this->humanVerification($this->ads);
Dominik Hebeler's avatar
Dominik Hebeler committed
273

274
        $counter = 0;
275
        $firstRank = 0;
276

277
        if (isset($this->startForwards)) {
278
            $this->startCount = $this->startForwards;
279
        } elseif (isset($this->startBackwards)) {
280
            $this->startCount = $this->startBackwards - count($this->results) - 1;
281
        } else {
282
283
284
            $this->startCount = 0;
        }

285
286
        foreach ($this->results as $result) {
            if ($counter === 0) {
287
                $firstRank = $result->rank;
288
289
            }

290
            $counter++;
291
            $result->number = $counter + $this->startCount;
292
            $confidence = 0;
293
294
295
            if ($firstRank > 0) {
                $confidence = $result->rank / $firstRank;
            } else {
296
                $confidence = 0;
297
298
299
            }

            if ($confidence > 0.65) {
300
                $result->color = "#FF4000";
301
            } elseif ($confidence > 0.4) {
302
                $result->color = "#FF0080";
303
            } elseif ($confidence > 0.2) {
304
                $result->color = "#C000C0";
305
            } else {
306
                $result->color = "#000000";
307
308
            }

309
310
        }

311
        if ($this->validated) {
312
            $this->ads = [];
313
        }
314
315

        if (count($this->results) <= 0) {
316
317
318
319
320
321
            if (strlen($this->site) > 0) {
                $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
                $this->errors[] = trans('metaGer.results.failedSitesearch', ['altSearch' => $no_sitesearch_query]);
            } else {
                $this->errors[] = trans('metaGer.results.failed');
            }
322
        }
323

324
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
325
            $page = $this->page + 1;
326
            $this->next = [
327
                'page' => $page,
328
                'startForwards' => $this->results[count($this->results) - 1]->number,
329
                'engines' => $this->next,
330
331
            ];
            Cache::put(md5(serialize($this->next)), serialize($this->next), 60);
332
333
        } else {
            $this->next = [];
334
335
        }

336
    }
337

338
339
    private function addLangCodes($results)
    {
340
341
342
343
344
        # Wenn es keine Ergebnisse gibt, brauchen wir uns gar nicht erst zu bemühen
        if (sizeof($results) === 0) {
            return $results;
        }

345
346
347
348
349
        # Bei der Spracheinstellung "all" wird nicht gefiltert
        if ($this->getLang() === "all") {
            return $results;
        } else {
            # Ansonsten müssen wir jedem Result einen Sprachcode hinzufügen
350
            $id = 0;
351
352
353
354
355
            $langStrings = [];
            foreach ($results as $result) {
                # Wir geben jedem Ergebnis eine ID um später die Sprachcodes zuordnen zu können
                $result->id = $id;

Dominik Hebeler's avatar
Dominik Hebeler committed
356
                $langStrings["result_" . $id] = utf8_encode($result->getLangString());
357
358
359
360
361
362
363
364

                $id++;
            }
            # Wir schreiben die Strings in eine temporäre JSON-Datei,
            # Da das Array unter umständen zu groß ist für eine direkte Übergabe an das Skript
            $filename = "/tmp/" . getmypid();
            file_put_contents($filename, json_encode($langStrings));
            $langDetectorPath = app_path() . "/Models/lang.pl";
365
366
            $lang = exec("echo '$filename' | $langDetectorPath");
            $lang = json_decode($lang, true);
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383

            # Wir haben nun die Sprachcodes der einzelnen Ergebnisse.
            # Diese müssen wir nur noch korrekt zuordnen, dann sind wir fertig.
            foreach ($lang as $key => $langCode) {
                # Prefix vom Key entfernen:
                $id = intval(str_replace("result_", "", $key));
                foreach ($this->results as $result) {
                    if ($result->id === $id) {
                        $result->langCode = $langCode;
                        break;
                    }
                }
            }
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
    public function combineResults($engines)
    {
        foreach ($engines as $engine) {
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
            foreach ($engine->results as $result) {
                if ($result->valid) {
                    $this->results[] = $result;
                }
            }
            foreach ($engine->ads as $ad) {
                $this->ads[] = $ad;
            }
        }
    }

404
405
    public function parseAdgoal($results)
    {
406
        $publicKey = getenv('adgoal_public');
407
        $privateKey = getenv('adgoal_private');
408
        if ($publicKey === false) {
409
410
411
            return $results;
        }
        $tldList = "";
412
413
        try {
            foreach ($results as $result) {
414
                $link = $result->anzeigeLink;
415
                if (strpos($link, "http") !== 0) {
416
417
418
419
420
421
422
423
424
425
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

426
            # Query
427
428
            $query = urlencode($this->q);

429
            $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . $publicKey . "&k=" . $hash . "&tld=" . $tldList . "&q=" . $query;
430
431
432
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
433
            foreach ($answer as $el) {
434
                $hoster = $el[0];
435
                $hash = $el[1];
436

437
                foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
438
                    if ($hoster === $result->tld && !$result->partnershop) {
439
440
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
441
                        if ($result->image !== "") {
442
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . $hash . ".gif";
443
                        } else {
444
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . $hash . ".gif";
445
446
                        }

447
448
449
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
                        $targetUrl = $result->anzeigeLink;
450
                        if (strpos($targetUrl, "http") !== 0) {
451
                            $targetUrl = "http://" . $targetUrl;
452
453
                        }

454
455
456
                        $gateHash = md5($targetUrl . $privateKey);
                        $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . $publicKey . "&k=" . $gateHash . "&url=" . urlencode($targetUrl) . "&q=" . $query;
                        $result->link = $newLink;
457
458
459
460
                        $result->partnershop = true;
                    }
                }
            }
461
        } catch (\ErrorException $e) {
462
463
464
465
466
            return $results;
        }

        return $results;
    }
467

468
469
    public function humanVerification($results)
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
470
        # Let's check if we need to implement a redirect for human verification
471
472
        if ($this->verificationCount > 10) {
            foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
473
474
475
                $link = $result->link;
                $day = Carbon::now()->day;
                $pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
476
477
478
                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
Dominik Hebeler's avatar
Dominik Hebeler committed
479
                $result->link = $url;
480
                $result->proxyLink = $proxyUrl;
Dominik Hebeler's avatar
Dominik Hebeler committed
481
482
            }
            return $results;
483
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
484
485
486
487
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
488
489
490
491
492
493
    public function authorize($key)
    {
        $postdata = http_build_query(array(
            'dummy' => rand(),
        ));
        $opts = array('http' => array(
494
495
            'method' => 'POST',
            'header' => 'Content-type: application/x-www-form-urlencoded',
Phil Höfer's avatar
Phil Höfer committed
496
497
498
499
500
501
502
            'content' => $postdata,
        ),
        );

        $context = stream_context_create($opts);

        try {
503
            $link = "https://key.metager3.de/" . urlencode($key) . "/request-permission/api-access";
Phil Höfer's avatar
Phil Höfer committed
504
505
506
507
508
509
510
511
512
513
514
515
            $result = json_decode(file_get_contents($link, false, $context));
            if ($result->{'api-access'} == true) {
                return true;
            } else {
                return false;
            }

        } catch (\ErrorException $e) {
            return false;
        }
    }

516
517
    public function createQuicktips()
    {
518
519
520
521
522
        # Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
        $quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime(), $this->getHashCode());
        return $quicktips;
    }

Karl's avatar
Karl committed
523
524
525
526
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

527
528
    public function createSearchEngines(Request $request)
    {
529
        # Wenn es kein Suchwort gibt
Dominik Hebeler's avatar
Dominik Hebeler committed
530
        if (!$request->filled("eingabe") || $this->q === "") {
531
            return;
532
        }
533

Dominik Hebeler's avatar
Dominik Hebeler committed
534
        $this->enabledSearchengines = [];
535
        $overtureEnabled = false;
536

537
538
539
        # Check if selected focus is valid
        if (empty($this->sumaFile->foki->{$this->fokus})) {
            $this->fokus = "web";
540
541
        }

542
543
        foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
            # Check if this engine is disabled and can't be used
544
            $disabled = empty($this->sumaFile->sumas->{$suma}->disabled) ? false : $this->sumaFile->sumas->{$suma}->disabled;
545
546
            if ($disabled) {
                continue;
547
548
            }

549
550
551
            # Check if this engine can use eventually defined query-filter
            $valid = true;
            foreach ($this->queryFilter as $queryFilter => $filter) {
Dominik Hebeler's avatar
Dominik Hebeler committed
552
                if (empty($this->sumaFile->filter->{"query-filter"}->$queryFilter->sumas->$suma)) {
553
554
                    $valid = false;
                    break;
Dominik Hebeler's avatar
Dominik Hebeler committed
555
556
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
557
558
559
560
561
562
563
564
565
            # Check if this engine can use eventually defined parameter-filter
            if ($valid) {
                foreach ($this->parameterFilter as $filterName => $filter) {
                    if (empty($filter->sumas->$suma)) {
                        $valid = false;
                        break;
                    }
                }
            }
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
            # Check if this engine should only be active when filter is used
            if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"}) {
                # This search engine should only be used when a parameter filter of it is used
                $validTmp = false;
                foreach ($this->parameterFilter as $filterName => $filter) {
                    if (!empty($filter->sumas->{$suma})) {
                        $validTmp = true;
                        break;
                    }
                }
                if (!$validTmp) {
                    $valid = false;
                }

            }
581
582
            # If it can we add it
            if ($valid) {
Dominik Hebeler's avatar
Dominik Hebeler committed
583
                $this->enabledSearchengines[$suma] = $this->sumaFile->sumas->{$suma};
584
            }
585

586
        }
587

Dominik Hebeler's avatar
Dominik Hebeler committed
588
        if (sizeof($this->enabledSearchengines) === 0) {
589
590
            $filter = "";
            foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
Dominik Hebeler's avatar
Dominik Hebeler committed
591
                $filter .= trans($this->sumaFile->filter->{"query-filter"}->{$queryFilter}->name) . ",";
592
593
594
595
596
            }
            $filter = rtrim($filter, ",");
            $error = trans('metaGer.engines.noSpecialSearch', ['fokus' => trans($this->sumaFile->foki->{$this->fokus}->{"display-name"}),
                'filter' => $filter]);
            $this->errors[] = $error;
597
        }
598

599
        $engines = [];
600
        $typeslist = [];
601
        $counter = 0;
602

603
604
        if ($this->requestIsCached($request)) {
            $engines = $this->getCachedEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
605
            # We need to edit some Options of the Cached Search Engines
Phil Höfer's avatar
Phil Höfer committed
606
            foreach ($engines as $engine) {
Dominik Hebeler's avatar
Dominik Hebeler committed
607
608
                $engine->setResultHash($this->getHashCode());
            }
609
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
610
            $engines = $this->actuallyCreateSearchEngines($this->enabledSearchengines);
611
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
612

613
        # Wir starten alle Suchen
614
615
        foreach ($engines as $engine) {
            $engine->startSearch($this);
616
        }
617

618
619
620
621
622
        /* Wir warten auf die Antwort der Suchmaschinen
         * Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
         * Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
         * $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
         * welche Suchmaschinen nicht rechtzeitig geantwortet haben.
623
624
         */

625
        $enginesToLoad = [];
626
        $canBreak = false;
627
628
629
630
631
        foreach ($engines as $engine) {
            if ($engine->cached) {
                if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
                    $canBreak = true;
                }
632
633
            } else {
                $enginesToLoad[$engine->name] = false;
634
635
636
637
638
639
            }
        }

        $this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);

        $this->retrieveResults($engines);
640
641
642
643
644
        foreach ($engines as $engine) {
            if (!empty($engine->totalResults) && $engine->totalResults > $this->totalResults) {
                $this->totalResults = $engine->totalResults;
            }
        }
645
646
    }

647
648
    # Spezielle Suchen und Sumas

649
    public function sumaIsSelected($suma, $request, $custom)
650
    {
651
        if ($custom) {
652
            if ($request->filled("engine_" . strtolower($suma["name"]))) {
653
654
655
656
657
658
659
660
661
662
663
                return true;
            }
        } else {
            $types = explode(",", $suma["type"]);
            if (in_array($this->fokus, $types)) {
                return true;
            }
        }
        return false;
    }

664
    public function actuallyCreateSearchEngines($enabledSearchengines)
665
666
    {
        $engines = [];
667
        foreach ($enabledSearchengines as $engineName => $engine) {
668

669
            if (!isset($engine->{"parser-class"})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
670
671
                die(var_dump($engine));
            }
672
            # Setze Pfad zu Parser
673
            $path = "App\\Models\\parserSkripte\\" . $engine->{"parser-class"};
674
675

            # Prüfe ob Parser vorhanden
676
677
678
            if (!file_exists(app_path() . "/Models/parserSkripte/" . $engine->{"parser-class"} . ".php")) {
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen, da kein Parser existiert");
                $this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine->{"display-name"}]);
679
680
681
682
683
684
                continue;
            }

            # Es wird versucht die Suchengine zu erstellen
            $time = microtime();
            try {
685
                $tmp = new $path($engineName, $engine, $this);
686
            } catch (\ErrorException $e) {
687
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen. " . var_dump($e));
688
689
690
                continue;
            }

691
            $engines[] = $tmp;
692
693
694
695
        }
        return $engines;
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
696
697
698
699
700
701
702
703
704
705
706
707
708
    public function getAvailableParameterFilter()
    {
        $parameterFilter = $this->sumaFile->filter->{"parameter-filter"};

        $availableFilter = [];

        foreach ($parameterFilter as $filterName => $filter) {
            # Check if any of the enabled search engines provide this filter
            foreach ($this->enabledSearchengines as $engineName => $engine) {
                if (!empty($filter->sumas->$engineName)) {
                    $availableFilter[$filterName] = $filter;
                }
            }
709
710
711
712
713
714
715
716
            # We will also add the filter from the opt-in search engines (the searchengines that are only used when a filter of it is too)
            foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
                if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"}) {
                    if (!empty($filter->sumas->{$suma})) {
                        $availableFilter[$filterName] = $filter;
                    }
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
717
718
719
720
721
        }

        return $availableFilter;
    }

722
723
    public function isBildersuche()
    {
724
        return $this->fokus === "bilder";
725
726
727
728
    }

    public function sumaIsAdsuche($suma, $overtureEnabled)
    {
729
        $sumaName = $suma["name"]->__toString();
730
        return
731
732
            $sumaName === "qualigo"
            || $sumaName === "similar_product_ads"
Karl Hasselbring's avatar
Karl Hasselbring committed
733
            || (!$overtureEnabled && $sumaName === "overtureAds");
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
    }

    public function sumaIsDisabled($suma)
    {
        return
        isset($suma['disabled'])
        && $suma['disabled']->__toString() === "1";
    }

    public function sumaIsOverture($suma)
    {
        return
        $suma["name"]->__toString() === "overture"
        || $suma["name"]->__toString() === "overtureAds";
    }

    public function sumaIsNotAdsuche($suma)
    {
        return
        $suma["name"]->__toString() !== "qualigo"
        && $suma["name"]->__toString() !== "similar_product_ads"
        && $suma["name"]->__toString() !== "overtureAds";
    }

    public function requestIsCached($request)
    {
        return
Dominik Hebeler's avatar
Dominik Hebeler committed
761
        $request->filled('next')
762
763
764
765
766
767
        && Cache::has($request->input('next'))
        && unserialize(Cache::get($request->input('next')))['page'] > 1;
    }

    public function getCachedEngines($request)
    {
768
        $next = unserialize(Cache::get($request->input('next')));
769
        $this->page = $next['page'];
770
        $engines = $next['engines'];
771
772
773
774
775
776
777
778
779
        if (isset($next['startForwards'])) {
            $this->startForwards = $next['startForwards'];
        }
        if (isset($next['startBackwards'])) {
            $this->startBackwards = $next['startBackwards'];
        }
        return $engines;
    }

780
    # Passt den Suchfokus an, falls für einen Fokus genau alle vorhandenen Sumas eingeschaltet sind
781
782
    public function adjustFocus($sumas, $enabledSearchengines)
    {
783
784
        # Findet für alle Foki die enthaltenen Sumas
        $foki = []; # [fokus][suma] => [suma]
785
        foreach ($sumas as $suma) {
786
            if ((!$this->sumaIsDisabled($suma)) && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
787
                if (isset($suma['type'])) {
788
789
790
                    # Wenn foki für diese Suchmaschine angegeben sind
                    $focuses = explode(",", $suma['type']->__toString());
                    foreach ($focuses as $foc) {
791
792
793
794
795
                        if (isset($suma['minismCollection'])) {
                            $foki[$foc][] = "minism";
                        } else {
                            $foki[$foc][] = $suma['name']->__toString();
                        }
796
                    }
797
                } else {
798
                    # Wenn keine foki für diese Suchmaschine angegeben sind
799
800
801
802
803
                    if (isset($suma['minismCollection'])) {
                        $foki["andere"][] = "minism";
                    } else {
                        $foki["andere"][] = $suma['name']->__toString();
                    }
804
805
806
807
                }
            }
        }

808
        # Findet die Namen der aktuell eingeschalteten Sumas
809
        $realEngNames = [];
810
        foreach ($enabledSearchengines as $realEng) {
811
            $nam = $realEng["name"]->__toString();
Karl Hasselbring's avatar
Karl Hasselbring committed
812
            if ($nam !== "qualigo" && $nam !== "overtureAds") {
813
814
815
                $realEngNames[] = $nam;
            }
        }
816

817
        # Anschließend werden diese beiden Listen verglichen (jeweils eine der Fokuslisten für jeden Fokus), um herauszufinden ob sie vielleicht identisch sind. Ist dies der Fall, so hat der Nutzer anscheinend Suchmaschinen eines kompletten Fokus eingestellt. Der Fokus wird dementsprechend angepasst.
818
        foreach ($foki as $fok => $engines) {
819
            $isFokus = true;
820
            $fokiEngNames = [];
821
            foreach ($engines as $eng) {
822
823
                $fokiEngNames[] = $eng;
            }
824
            # Jede eingeschaltete Engine ist für diesen Fokus geeignet
825
            foreach ($fokiEngNames as $fen) {
826
827
                # Bei Bildersuchen ist uns egal, ob alle Suchmaschinen aus dem Suchfokus eingeschaltet sind, da wir sie eh als Bildersuche anzeigen müssen
                if (!in_array($fen, $realEngNames) && $fok !== "bilder") {
828
829
830
                    $isFokus = false;
                }
            }
831
            # Jede im Fokus erwartete Engine ist auch eingeschaltet
832
833
            foreach ($realEngNames as $ren) {
                if (!in_array($ren, $fokiEngNames)) {
834
835
836
                    $isFokus = false;
                }
            }
837
            # Wenn die Listen identisch sind, setze den Fokus um
838
            if ($isFokus) {
839
840
841
                $this->fokus = $fok;
            }
        }
842
    }
843

844
845
    public function waitForResults($enginesToLoad, $overtureEnabled, $canBreak)
    {
846

Phil Höfer's avatar
Phil Höfer committed
847
        $timeStart = microtime(true);
848
        $results = null;
849
        while (true) {
850
851
852
            $results = Redis::hgetall('search.' . $this->getHashCode());

            $ready = true;
Phil Höfer's avatar
Phil Höfer committed
853
            // When every
854
            $connected = true;
Phil Höfer's avatar
Phil Höfer committed
855
856
            foreach ($results as $key => $value) {
                if ($value === "waiting" || $value === "connected") {
857
                    $ready = false;
858
                }
Phil Höfer's avatar
Phil Höfer committed
859
                if ($value === "waiting") {
860
                    $connected = false;
861
                }
862
            }
863

864
865
            // If $ready is false at this point, we're waiting for more searchengines
            // But we have to check for the timeout, too
Phil Höfer's avatar
Phil Höfer committed
866
867
868
869
            if (!$connected) {
                $timeStart = microtime(true);
            }

870
871
872
873
            $time = (microtime(true) - $timeStart) * 1000;
            // We will apply the timeout only if it's not Yahoo we're waiting for since they are one the most
            // important search engines.
            $canTimeout = !((isset($results["overture"]) && $results["overture"] === "waiting") || (isset($results["overtureAds"]) && $results["overtureAds"] === "waiting"));
Phil Höfer's avatar
Phil Höfer committed
874
875
876
            if ($time > $this->time && $canTimeout) {
                $ready = true;
            }
877

Phil Höfer's avatar
Phil Höfer committed
878
            if ($ready) {
879
880
881
882
                break;
            }
            usleep(50000);
        }
883
884

        # Wir haben nun so lange wie möglich gewartet. Wir registrieren nun noch die Suchmaschinen, die geanwortet haben.
885
        foreach ($results as $key => $value) {
886
887
888
            $enginesToLoad[$key] = true;
        }
        $this->enginesToLoad = $enginesToLoad;
889
    }
890

891
892
893
    public function retrieveResults($engines)
    {
        # Von geladenen Engines die Ergebnisse holen
894
895
896
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
                try {
897
                    $engine->retrieveResults($this);
898
                } catch (\ErrorException $e) {
899
900
901
902
                    Log::error($e);
                }
            }
        }
903

904
        $this->engines = $engines;
905
906
    }

907