MetaGer.php 55.9 KB
Newer Older
1
2
3
4
<?php
namespace App;

use App;
5
use Cache;
6
use Carbon;
7
use Illuminate\Http\Request;
Phil Höfer's avatar
Phil Höfer committed
8
use Illuminate\Support\Facades\Redis;
9
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix    
Dominik Hebeler committed
10
use LaravelLocalization;
11
use Log;
12
use Predis\Connection\ConnectionException;
13
14
15

class MetaGer
{
16
17
18
19
20
21
22
23
    # Einstellungen für die Suche
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
24
    protected $time = 2000;
25
    protected $hostBlacklist = [];
26
    protected $domainBlacklist = [];
27
28
29
30
31
    private $urlBlacklist = [];
    protected $stopWords = [];
    protected $phrases = [];
    protected $engines = [];
    protected $results = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
32
33
    protected $queryFilter = [];
    protected $parameterFilter = [];
34
35
36
37
    protected $ads = [];
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
38
    protected $availableFoki = [];
39
40
    protected $startCount = 0;
    protected $canCache = false;
41
    # Daten über die Abfrage$
42
    protected $ip;
Dominik Hebeler's avatar
Dominik Hebeler committed
43
    protected $useragent;
44
45
    protected $language;
    protected $agent;
46
    protected $apiKey = "";
Phil Höfer's avatar
Phil Höfer committed
47
    protected $apiAuthorized = false;
48
49
50
51
52
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
53
    protected $newtab;
54
    protected $domainsBlacklisted = [];
55
    protected $urlsBlacklisted = [];
56
    protected $url;
57
    protected $fullUrl;
58
    protected $languageDetect;
Dominik Hebeler's avatar
Dominik Hebeler committed
59
60
    protected $verificationId;
    protected $verificationCount;
61

62
63
    public function __construct()
    {
64
        # Timer starten
65
        $this->starttime = microtime(true);
66
        # Versuchen Blacklists einzulesen
67
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
68
            $tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
69
            $this->domainsBlacklisted = explode("\n", $tmp);
70
71
            $tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted = explode("\n", $tmp);
72
        } else {
73
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
74
75
        }

76
        # Parser Skripte einhängen
77
78
79
80
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
81
                require_once $path;
82
83
84
            }
        }

85
        # Cachebarkeit testen
86
87
88
89
90
91
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
92
    }
93

94
    # Erstellt aus den gesammelten Ergebnissen den View
95
    public function createView($quicktipResults = [])
96
    {
97
98
        # Hiermit werden die evtl. ausgewählten SuMas extrahiert, damit die Input-Boxen richtig gesetzt werden können
        $focusPages = [];
Aria Givi's avatar
Aria Givi committed
99

100
        foreach ($this->request->all() as $key => $value) {
101
102
            if (starts_with($key, 'engine_') && $value === 'on') {
                $focusPages[] = $key;
103
104
105
            }
        }

106
        $viewResults = [];
107
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
108
        foreach ($this->results as $result) {
109
110
111
112
            $viewResults[] = get_object_vars($result);
        }
        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();
113
114
        if ($this->fokus === "bilder") {
            switch ($this->out) {
115
                case 'results':
116
                    return view('resultpages.results_images')
117
118
119
120
121
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
122
                        ->with('apiAuthorized', $this->apiAuthorized)
123
                        ->with('metager', $this)
124
                        ->with('browser', (new Agent())->browser());
125
                default:
126
                    return view('resultpages.resultpage_images')
127
128
129
130
131
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
132
                        ->with('apiAuthorized', $this->apiAuthorized)
133
                        ->with('metager', $this)
134
135
136
137
                        ->with('browser', (new Agent())->browser())
                        ->with('quicktips', $quicktipResults)
                        ->with('focus', $this->fokus)
                        ->with('resultcount', count($this->results));
138
            }
139
140
141
        } else {
            switch ($this->out) {
                case 'results':
142
                    return view('resultpages.results')
143
144
145
146
147
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
148
                        ->with('apiAuthorized', $this->apiAuthorized)
149
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
150
151
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
152
153
                    break;
                case 'results-with-style':
154
                    return view('resultpages.resultpage')
155
156
157
158
159
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
160
                        ->with('apiAuthorized', $this->apiAuthorized)
161
162
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
Dominik Hebeler's avatar
Dominik Hebeler committed
163
164
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
165
                    break;
Phil Höfer's avatar
Phil Höfer committed
166
                case 'rich':
167
                    return view('resultpages.metager3rich')
168
                        ->with('results', $viewResults)
Phil Höfer's avatar
Phil Höfer committed
169
170
171
172
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
173
                        ->with('apiAuthorized', $this->apiAuthorized)
Phil Höfer's avatar
Phil Höfer committed
174
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
175
176
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
Phil Höfer's avatar
Phil Höfer committed
177
                    break;
178
                case 'rss20':
179
                    return view('resultpages.metager3resultsrss20')
180
181
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
Phil Höfer's avatar
Phil Höfer committed
182
                        ->with('apiAuthorized', $this->apiAuthorized)
183
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
184
185
                        ->with('resultcount', sizeof($viewResults))
                        ->with('fokus', $this->fokus);
186
                    break;
187
188
189
                case 'api':
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'apiAuthorized' => $this->apiAuthorized])->header('Content-Type', 'application/xml');
                    break;
Aria Givi's avatar
Aria Givi committed
190
                case 'atom10':
191
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'apiAuthorized' => true])
192
                        ->header('Content-Type', 'application/xml');
Aria Givi's avatar
Aria Givi committed
193
                    break;
194
                case 'result-count':
195
196
                    # Wir geben die Ergebniszahl und die benötigte Zeit zurück:
                    return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
197
                    break;
198
                default:
199
                    return view('resultpages.resultpage')
200
                        ->with('eingabe', $this->eingabe)
201
                        ->with('focusPages', $focusPages)
202
203
204
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
205
                        ->with('apiAuthorized', $this->apiAuthorized)
206
                        ->with('metager', $this)
207
                        ->with('browser', (new Agent())->browser())
208
                        ->with('quicktips', $quicktipResults)
209
210
                        ->with('resultcount', count($this->results))
                        ->with('focus', $this->fokus);
211
212
                    break;
            }
213
214
215
        }
    }

Phil Höfer's avatar
Phil Höfer committed
216
    public function prepareResults()
217
    {
Phil Höfer's avatar
Phil Höfer committed
218
        $engines = $this->engines;
219

Phil Höfer's avatar
Phil Höfer committed
220
221
        // combine
        $combinedResults = $this->combineResults($engines);
222
223
224
225

        # Wir bestimmen die Sprache eines jeden Suchergebnisses
        $this->results = $this->addLangCodes($this->results);

Phil Höfer's avatar
Phil Höfer committed
226
227
228
229
230
        // sort
        //$sortedResults = $this->sortResults($engines);
        // filter
        // augment (boost&adgoal)
        // authorize
Phil Höfer's avatar
Phil Höfer committed
231
232
233
        if ($this->apiKey) {
            $this->apiAuthorized = $this->authorize($this->apiKey);
        }
Phil Höfer's avatar
Phil Höfer committed
234
        // misc (WiP)
235
        if ($this->fokus == "nachrichten") {
236
237
238
            $this->results = array_filter($this->results, function ($v, $k) {
                return !is_null($v->getRank());
            }, ARRAY_FILTER_USE_BOTH);
239
240
241
242
243
244
245
246
247
248
            uasort($this->results, function ($a, $b) {
                $datea = $a->getDate();
                $dateb = $b->getDate();
                return $dateb - $datea;
            });
        } else {
            uasort($this->results, function ($a, $b) {
                if ($a->getRank() == $b->getRank()) {
                    return 0;
                }
249

250
251
252
                return ($a->getRank() < $b->getRank()) ? 1 : -1;
            });
        }
253

254
255
        # Validate Results
        $newResults = [];
256
257
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
258
                $newResults[] = $result;
259
260
            }

261
262
263
        }
        $this->results = $newResults;

264
        #Adgoal Implementation
265
266
267
        if (!$this->apiAuthorized) {
            $this->results = $this->parseAdgoal($this->results);
        }
268

Dominik Hebeler's avatar
Dominik Hebeler committed
269
270
        # Human Verification
        $this->results = $this->humanVerification($this->results);
271
        $this->ads = $this->humanVerification($this->ads);
Dominik Hebeler's avatar
Dominik Hebeler committed
272

273
        $counter = 0;
274
        $firstRank = 0;
275

276
        if (isset($this->startForwards)) {
277
            $this->startCount = $this->startForwards;
278
        } elseif (isset($this->startBackwards)) {
279
            $this->startCount = $this->startBackwards - count($this->results) - 1;
280
        } else {
281
282
283
            $this->startCount = 0;
        }

284
285
        foreach ($this->results as $result) {
            if ($counter === 0) {
286
                $firstRank = $result->rank;
287
288
            }

289
            $counter++;
290
            $result->number = $counter + $this->startCount;
291
            $confidence = 0;
292
293
294
            if ($firstRank > 0) {
                $confidence = $result->rank / $firstRank;
            } else {
295
                $confidence = 0;
296
297
298
            }

            if ($confidence > 0.65) {
299
                $result->color = "#FF4000";
300
            } elseif ($confidence > 0.4) {
301
                $result->color = "#FF0080";
302
            } elseif ($confidence > 0.2) {
303
                $result->color = "#C000C0";
304
            } else {
305
                $result->color = "#000000";
306
307
            }

308
309
        }

310
        if ($this->validated) {
311
            $this->ads = [];
312
        }
313
314

        if (count($this->results) <= 0) {
315
316
317
318
319
320
            if (strlen($this->site) > 0) {
                $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
                $this->errors[] = trans('metaGer.results.failedSitesearch', ['altSearch' => $no_sitesearch_query]);
            } else {
                $this->errors[] = trans('metaGer.results.failed');
            }
321
        }
322

323
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
324
            $page = $this->page + 1;
325
            $this->next = [
326
                'page' => $page,
327
                'startForwards' => $this->results[count($this->results) - 1]->number,
328
                'engines' => $this->next,
329
330
            ];
            Cache::put(md5(serialize($this->next)), serialize($this->next), 60);
331
332
        } else {
            $this->next = [];
333
334
        }

335
    }
336

337
338
    private function addLangCodes($results)
    {
339
340
341
342
343
        # Wenn es keine Ergebnisse gibt, brauchen wir uns gar nicht erst zu bemühen
        if (sizeof($results) === 0) {
            return $results;
        }

344
345
346
347
348
        # Bei der Spracheinstellung "all" wird nicht gefiltert
        if ($this->getLang() === "all") {
            return $results;
        } else {
            # Ansonsten müssen wir jedem Result einen Sprachcode hinzufügen
349
            $id = 0;
350
351
352
353
354
            $langStrings = [];
            foreach ($results as $result) {
                # Wir geben jedem Ergebnis eine ID um später die Sprachcodes zuordnen zu können
                $result->id = $id;

Dominik Hebeler's avatar
Dominik Hebeler committed
355
                $langStrings["result_" . $id] = utf8_encode($result->getLangString());
356
357
358
359
360
361
362
363

                $id++;
            }
            # Wir schreiben die Strings in eine temporäre JSON-Datei,
            # Da das Array unter umständen zu groß ist für eine direkte Übergabe an das Skript
            $filename = "/tmp/" . getmypid();
            file_put_contents($filename, json_encode($langStrings));
            $langDetectorPath = app_path() . "/Models/lang.pl";
364
365
            $lang = exec("echo '$filename' | $langDetectorPath");
            $lang = json_decode($lang, true);
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382

            # Wir haben nun die Sprachcodes der einzelnen Ergebnisse.
            # Diese müssen wir nur noch korrekt zuordnen, dann sind wir fertig.
            foreach ($lang as $key => $langCode) {
                # Prefix vom Key entfernen:
                $id = intval(str_replace("result_", "", $key));
                foreach ($this->results as $result) {
                    if ($result->id === $id) {
                        $result->langCode = $langCode;
                        break;
                    }
                }
            }
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
    public function combineResults($engines)
    {
        foreach ($engines as $engine) {
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
            foreach ($engine->results as $result) {
                if ($result->valid) {
                    $this->results[] = $result;
                }
            }
            foreach ($engine->ads as $ad) {
                $this->ads[] = $ad;
            }
        }
    }

403
404
    public function parseAdgoal($results)
    {
405
        $publicKey = getenv('adgoal_public');
406
        $privateKey = getenv('adgoal_private');
407
        if ($publicKey === false) {
408
409
410
            return $results;
        }
        $tldList = "";
411
412
        try {
            foreach ($results as $result) {
413
                $link = $result->anzeigeLink;
414
                if (strpos($link, "http") !== 0) {
415
416
417
418
419
420
421
422
423
424
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

425
            # Query
426
427
            $query = urlencode($this->q);

428
            $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . $publicKey . "&k=" . $hash . "&tld=" . $tldList . "&q=" . $query;
429
430
431
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
432
            foreach ($answer as $el) {
433
                $hoster = $el[0];
434
                $hash = $el[1];
435

436
                foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
437
                    if ($hoster === $result->tld && !$result->partnershop) {
438
439
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
440
                        if ($result->image !== "") {
441
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . $hash . ".gif";
442
                        } else {
443
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . $hash . ".gif";
444
445
                        }

446
447
448
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
                        $targetUrl = $result->anzeigeLink;
449
                        if (strpos($targetUrl, "http") !== 0) {
450
                            $targetUrl = "http://" . $targetUrl;
451
452
                        }

453
454
455
                        $gateHash = md5($targetUrl . $privateKey);
                        $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . $publicKey . "&k=" . $gateHash . "&url=" . urlencode($targetUrl) . "&q=" . $query;
                        $result->link = $newLink;
456
457
458
459
                        $result->partnershop = true;
                    }
                }
            }
460
        } catch (\ErrorException $e) {
461
462
463
464
465
            return $results;
        }

        return $results;
    }
466

467
468
    public function humanVerification($results)
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
469
        # Let's check if we need to implement a redirect for human verification
470
471
        if ($this->verificationCount > 10) {
            foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
472
473
474
                $link = $result->link;
                $day = Carbon::now()->day;
                $pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
475
476
477
                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
Dominik Hebeler's avatar
Dominik Hebeler committed
478
                $result->link = $url;
479
                $result->proxyLink = $proxyUrl;
Dominik Hebeler's avatar
Dominik Hebeler committed
480
481
            }
            return $results;
482
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
483
484
485
486
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
487
488
489
490
491
492
    public function authorize($key)
    {
        $postdata = http_build_query(array(
            'dummy' => rand(),
        ));
        $opts = array('http' => array(
493
494
            'method' => 'POST',
            'header' => 'Content-type: application/x-www-form-urlencoded',
Phil Höfer's avatar
Phil Höfer committed
495
496
497
498
499
500
501
            'content' => $postdata,
        ),
        );

        $context = stream_context_create($opts);

        try {
502
            $link = "https://key.metager3.de/" . urlencode($key) . "/request-permission/api-access";
Phil Höfer's avatar
Phil Höfer committed
503
504
505
506
507
508
509
510
511
512
513
514
            $result = json_decode(file_get_contents($link, false, $context));
            if ($result->{'api-access'} == true) {
                return true;
            } else {
                return false;
            }

        } catch (\ErrorException $e) {
            return false;
        }
    }

515
516
    public function createQuicktips()
    {
517
518
519
520
521
        # Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
        $quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime(), $this->getHashCode());
        return $quicktips;
    }

Karl's avatar
Karl committed
522
523
524
525
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

526
527
    public function createSearchEngines(Request $request)
    {
528
        # Wenn es kein Suchwort gibt
Dominik Hebeler's avatar
Dominik Hebeler committed
529
        if (!$request->filled("eingabe") || $this->q === "") {
530
            return;
531
        }
532

Dominik Hebeler's avatar
Dominik Hebeler committed
533
        $this->enabledSearchengines = [];
534
        $overtureEnabled = false;
535

536
537
538
        # Check if selected focus is valid
        if (empty($this->sumaFile->foki->{$this->fokus})) {
            $this->fokus = "web";
539
540
        }

541
542
543
544
545
        foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
            # Check if this engine is disabled and can't be used
            $disabled = empty($suma->disabled) ? false : $suma->disabled;
            if ($disabled) {
                continue;
546
547
            }

548
549
550
            # Check if this engine can use eventually defined query-filter
            $valid = true;
            foreach ($this->queryFilter as $queryFilter => $filter) {
Dominik Hebeler's avatar
Dominik Hebeler committed
551
                if (empty($this->sumaFile->filter->{"query-filter"}->$queryFilter->sumas->$suma)) {
552
553
                    $valid = false;
                    break;
Dominik Hebeler's avatar
Dominik Hebeler committed
554
555
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
556
557
558
559
560
561
562
563
564
            # Check if this engine can use eventually defined parameter-filter
            if ($valid) {
                foreach ($this->parameterFilter as $filterName => $filter) {
                    if (empty($filter->sumas->$suma)) {
                        $valid = false;
                        break;
                    }
                }
            }
565
566
            # If it can we add it
            if ($valid) {
Dominik Hebeler's avatar
Dominik Hebeler committed
567
                $this->enabledSearchengines[$suma] = $this->sumaFile->sumas->{$suma};
568
            }
569

570
        }
571

Dominik Hebeler's avatar
Dominik Hebeler committed
572
        if (sizeof($this->enabledSearchengines) === 0) {
573
574
            $filter = "";
            foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
Dominik Hebeler's avatar
Dominik Hebeler committed
575
                $filter .= trans($this->sumaFile->filter->{"query-filter"}->{$queryFilter}->name) . ",";
576
577
578
579
580
            }
            $filter = rtrim($filter, ",");
            $error = trans('metaGer.engines.noSpecialSearch', ['fokus' => trans($this->sumaFile->foki->{$this->fokus}->{"display-name"}),
                'filter' => $filter]);
            $this->errors[] = $error;
581
        }
582

583
        $engines = [];
584
        $typeslist = [];
585
        $counter = 0;
586

587
588
        if ($this->requestIsCached($request)) {
            $engines = $this->getCachedEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
589
            # We need to edit some Options of the Cached Search Engines
Phil Höfer's avatar
Phil Höfer committed
590
            foreach ($engines as $engine) {
Dominik Hebeler's avatar
Dominik Hebeler committed
591
592
                $engine->setResultHash($this->getHashCode());
            }
593
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
594
            $engines = $this->actuallyCreateSearchEngines($this->enabledSearchengines);
595
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
596

597
        # Wir starten alle Suchen
598
599
        foreach ($engines as $engine) {
            $engine->startSearch($this);
600
        }
601

602
603
604
605
606
        /* Wir warten auf die Antwort der Suchmaschinen
         * Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
         * Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
         * $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
         * welche Suchmaschinen nicht rechtzeitig geantwortet haben.
607
608
         */

609
        $enginesToLoad = [];
610
        $canBreak = false;
611
612
613
614
615
        foreach ($engines as $engine) {
            if ($engine->cached) {
                if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
                    $canBreak = true;
                }
616
617
            } else {
                $enginesToLoad[$engine->name] = false;
618
619
620
621
622
623
624
625
            }
        }

        $this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);

        $this->retrieveResults($engines);
    }

626
627
    # Spezielle Suchen und Sumas

628
    public function sumaIsSelected($suma, $request, $custom)
629
    {
630
        if ($custom) {
631
            if ($request->filled("engine_" . strtolower($suma["name"]))) {
632
633
634
635
636
637
638
639
640
641
642
                return true;
            }
        } else {
            $types = explode(",", $suma["type"]);
            if (in_array($this->fokus, $types)) {
                return true;
            }
        }
        return false;
    }

643
    public function actuallyCreateSearchEngines($enabledSearchengines)
644
645
    {
        $engines = [];
646
        foreach ($enabledSearchengines as $engineName => $engine) {
647

648
            if (!isset($engine->{"parser-class"})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
649
650
                die(var_dump($engine));
            }
651
            # Setze Pfad zu Parser
652
            $path = "App\\Models\\parserSkripte\\" . $engine->{"parser-class"};
653
654

            # Prüfe ob Parser vorhanden
655
656
657
            if (!file_exists(app_path() . "/Models/parserSkripte/" . $engine->{"parser-class"} . ".php")) {
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen, da kein Parser existiert");
                $this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine->{"display-name"}]);
658
659
660
661
662
663
                continue;
            }

            # Es wird versucht die Suchengine zu erstellen
            $time = microtime();
            try {
664
                $tmp = new $path($engineName, $engine, $this);
665
            } catch (\ErrorException $e) {
666
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen. " . var_dump($e));
667
668
669
                continue;
            }

670
            $engines[] = $tmp;
671
672
673
674
        }
        return $engines;
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
    public function getAvailableParameterFilter()
    {
        $parameterFilter = $this->sumaFile->filter->{"parameter-filter"};

        $availableFilter = [];

        foreach ($parameterFilter as $filterName => $filter) {
            # Check if any of the enabled search engines provide this filter
            foreach ($this->enabledSearchengines as $engineName => $engine) {
                if (!empty($filter->sumas->$engineName)) {
                    $availableFilter[$filterName] = $filter;
                }
            }
        }

        return $availableFilter;
    }

693
694
    public function isBildersuche()
    {
695
        return $this->fokus === "bilder";
696
697
698
699
    }

    public function sumaIsAdsuche($suma, $overtureEnabled)
    {
700
        $sumaName = $suma["name"]->__toString();
701
        return
702
703
            $sumaName === "qualigo"
            || $sumaName === "similar_product_ads"
Karl Hasselbring's avatar
Karl Hasselbring committed
704
            || (!$overtureEnabled && $sumaName === "overtureAds");
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
    }

    public function sumaIsDisabled($suma)
    {
        return
        isset($suma['disabled'])
        && $suma['disabled']->__toString() === "1";
    }

    public function sumaIsOverture($suma)
    {
        return
        $suma["name"]->__toString() === "overture"
        || $suma["name"]->__toString() === "overtureAds";
    }

    public function sumaIsNotAdsuche($suma)
    {
        return
        $suma["name"]->__toString() !== "qualigo"
        && $suma["name"]->__toString() !== "similar_product_ads"
        && $suma["name"]->__toString() !== "overtureAds";
    }

    public function requestIsCached($request)
    {
        return
Dominik Hebeler's avatar
Dominik Hebeler committed
732
        $request->filled('next')
733
734
735
736
737
738
        && Cache::has($request->input('next'))
        && unserialize(Cache::get($request->input('next')))['page'] > 1;
    }

    public function getCachedEngines($request)
    {
739
        $next = unserialize(Cache::get($request->input('next')));
740
        $this->page = $next['page'];
741
        $engines = $next['engines'];
742
743
744
745
746
747
748
749
750
        if (isset($next['startForwards'])) {
            $this->startForwards = $next['startForwards'];
        }
        if (isset($next['startBackwards'])) {
            $this->startBackwards = $next['startBackwards'];
        }
        return $engines;
    }

751
    # Passt den Suchfokus an, falls für einen Fokus genau alle vorhandenen Sumas eingeschaltet sind
752
753
    public function adjustFocus($sumas, $enabledSearchengines)
    {
754
755
        # Findet für alle Foki die enthaltenen Sumas
        $foki = []; # [fokus][suma] => [suma]
756
        foreach ($sumas as $suma) {
757
            if ((!$this->sumaIsDisabled($suma)) && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
758
                if (isset($suma['type'])) {
759
760
761
                    # Wenn foki für diese Suchmaschine angegeben sind
                    $focuses = explode(",", $suma['type']->__toString());
                    foreach ($focuses as $foc) {
762
763
764
765
766
                        if (isset($suma['minismCollection'])) {
                            $foki[$foc][] = "minism";
                        } else {
                            $foki[$foc][] = $suma['name']->__toString();
                        }
767
                    }
768
                } else {
769
                    # Wenn keine foki für diese Suchmaschine angegeben sind
770
771
772
773
774
                    if (isset($suma['minismCollection'])) {
                        $foki["andere"][] = "minism";
                    } else {
                        $foki["andere"][] = $suma['name']->__toString();
                    }
775
776
777
778
                }
            }
        }

779
        # Findet die Namen der aktuell eingeschalteten Sumas
780
        $realEngNames = [];
781
        foreach ($enabledSearchengines as $realEng) {
782
            $nam = $realEng["name"]->__toString();
Karl Hasselbring's avatar
Karl Hasselbring committed
783
            if ($nam !== "qualigo" && $nam !== "overtureAds") {
784
785
786
                $realEngNames[] = $nam;
            }
        }
787

788
        # Anschließend werden diese beiden Listen verglichen (jeweils eine der Fokuslisten für jeden Fokus), um herauszufinden ob sie vielleicht identisch sind. Ist dies der Fall, so hat der Nutzer anscheinend Suchmaschinen eines kompletten Fokus eingestellt. Der Fokus wird dementsprechend angepasst.
789
        foreach ($foki as $fok => $engines) {
790
            $isFokus = true;
791
            $fokiEngNames = [];
792
            foreach ($engines as $eng) {
793
794
                $fokiEngNames[] = $eng;
            }
795
            # Jede eingeschaltete Engine ist für diesen Fokus geeignet
796
            foreach ($fokiEngNames as $fen) {
797
798
                # Bei Bildersuchen ist uns egal, ob alle Suchmaschinen aus dem Suchfokus eingeschaltet sind, da wir sie eh als Bildersuche anzeigen müssen
                if (!in_array($fen, $realEngNames) && $fok !== "bilder") {
799
800
801
                    $isFokus = false;
                }
            }
802
            # Jede im Fokus erwartete Engine ist auch eingeschaltet
803
804
            foreach ($realEngNames as $ren) {
                if (!in_array($ren, $fokiEngNames)) {
805
806
807
                    $isFokus = false;
                }
            }
808
            # Wenn die Listen identisch sind, setze den Fokus um
809
            if ($isFokus) {
810
811
812
                $this->fokus = $fok;
            }
        }
813
    }
814

815
816
    public function waitForResults($enginesToLoad, $overtureEnabled, $canBreak)
    {
817

Phil Höfer's avatar
Phil Höfer committed
818
        $timeStart = microtime(true);
819
        $results = null;
820
        while (true) {
821
822
823
            $results = Redis::hgetall('search.' . $this->getHashCode());

            $ready = true;
Phil Höfer's avatar
Phil Höfer committed
824
            // When every
825
            $connected = true;
Phil Höfer's avatar
Phil Höfer committed
826
827
            foreach ($results as $key => $value) {
                if ($value === "waiting" || $value === "connected") {
828
                    $ready = false;
829
                }
Phil Höfer's avatar
Phil Höfer committed
830
                if ($value === "waiting") {
831
                    $connected = false;
832
                }
833
            }
834

835
836
            // If $ready is false at this point, we're waiting for more searchengines
            // But we have to check for the timeout, too
Phil Höfer's avatar
Phil Höfer committed
837
838
839
840
            if (!$connected) {
                $timeStart = microtime(true);
            }

841
842
843
844
            $time = (microtime(true) - $timeStart) * 1000;
            // We will apply the timeout only if it's not Yahoo we're waiting for since they are one the most
            // important search engines.
            $canTimeout = !((isset($results["overture"]) && $results["overture"] === "waiting") || (isset($results["overtureAds"]) && $results["overtureAds"] === "waiting"));
Phil Höfer's avatar
Phil Höfer committed
845
846
847
            if ($time > $this->time && $canTimeout) {
                $ready = true;
            }
848

Phil Höfer's avatar
Phil Höfer committed
849
            if ($ready) {
850
851
852
853
                break;
            }
            usleep(50000);
        }
854
855

        # Wir haben nun so lange wie möglich gewartet. Wir registrieren nun noch die Suchmaschinen, die geanwortet haben.
856
        foreach ($results as $key => $value) {
857
858
859
            $enginesToLoad[$key] = true;
        }
        $this->enginesToLoad = $enginesToLoad;
860
    }
861

862
863
864
    public function retrieveResults($engines)
    {
        # Von geladenen Engines die Ergebnisse holen
865
866
867
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
                try {
868
                    $engine->retrieveResults($this);
869
                } catch (\ErrorException $e) {
870
871
872
873
                    Log::error($e);
                }
            }
        }
874

875
        $this->engines = $engines;
876
877
    }

878
879
880
/*
 * Ende Suchmaschinenerstellung und Ergebniserhalt
 */
Karl's avatar
Karl committed
881

882
883
    public function parseFormData(Request $request)
    {
884

885
        # Sichert, dass der request in UTF-8 formatiert ist
886
        if ($request->input('encoding', 'utf8') !== "utf8") {
887
888
889
890
891
892
893
            # In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
            $input = $request->all();
            foreach ($input as $key => $value) {
                $input[$key] = mb_convert_encoding("$value", "UTF-8", "ISO-8859-1");
            }
            $request->replace($input);
        }
894
        $this->url = $request->url();
895
        $this->fullUrl = $request->fullUrl();
896
        # Zunächst überprüfen wir die eingegebenen Einstellungen:
897
        # Fokus
898
        $this->fokus = $request->input('focus', 'web');