MetaGer.php 57.5 KB
Newer Older
1
2
3
4
<?php
namespace App;

use App;
5
use Cache;
6
use Carbon;
7
use Illuminate\Http\Request;
Phil Höfer's avatar
Phil Höfer committed
8
use Illuminate\Support\Facades\Redis;
9
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix    
Dominik Hebeler committed
10
use LaravelLocalization;
11
use Log;
12
use Predis\Connection\ConnectionException;
13
14
15

class MetaGer
{
16
17
18
19
20
21
22
23
    # Einstellungen für die Suche
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
24
    protected $time = 2000;
25
    protected $hostBlacklist = [];
26
    protected $domainBlacklist = [];
27
28
29
30
    private $urlBlacklist = [];
    protected $stopWords = [];
    protected $phrases = [];
    protected $engines = [];
31
    protected $totalResults = 0;
32
    protected $results = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
33
34
    protected $queryFilter = [];
    protected $parameterFilter = [];
35
36
37
38
    protected $ads = [];
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
39
    protected $availableFoki = [];
40
41
    protected $startCount = 0;
    protected $canCache = false;
42
    # Daten über die Abfrage$
43
    protected $ip;
Dominik Hebeler's avatar
Dominik Hebeler committed
44
    protected $useragent;
45
46
    protected $language;
    protected $agent;
47
    protected $apiKey = "";
Phil Höfer's avatar
Phil Höfer committed
48
    protected $apiAuthorized = false;
49
50
51
52
53
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
54
    protected $newtab;
55
    protected $domainsBlacklisted = [];
56
    protected $urlsBlacklisted = [];
57
    protected $url;
58
    protected $fullUrl;
59
    protected $languageDetect;
Dominik Hebeler's avatar
Dominik Hebeler committed
60
61
    protected $verificationId;
    protected $verificationCount;
62

63
64
    public function __construct()
    {
65
        # Timer starten
66
        $this->starttime = microtime(true);
67
        # Versuchen Blacklists einzulesen
68
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
69
            $tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
70
            $this->domainsBlacklisted = explode("\n", $tmp);
71
72
            $tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted = explode("\n", $tmp);
73
        } else {
74
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
75
76
        }

77
        # Parser Skripte einhängen
78
79
80
81
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
82
                require_once $path;
83
84
85
            }
        }

86
        # Cachebarkeit testen
87
88
89
90
91
92
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
93
    }
94

95
    # Erstellt aus den gesammelten Ergebnissen den View
96
    public function createView($quicktipResults = [])
97
    {
98
99
        # Hiermit werden die evtl. ausgewählten SuMas extrahiert, damit die Input-Boxen richtig gesetzt werden können
        $focusPages = [];
Aria Givi's avatar
Aria Givi committed
100

101
        foreach ($this->request->all() as $key => $value) {
102
103
            if (starts_with($key, 'engine_') && $value === 'on') {
                $focusPages[] = $key;
104
105
106
            }
        }

107
        $viewResults = [];
108
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
109
        foreach ($this->results as $result) {
110
111
112
113
            $viewResults[] = get_object_vars($result);
        }
        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();
114
115
        if ($this->fokus === "bilder") {
            switch ($this->out) {
116
                case 'results':
117
                    return view('resultpages.results_images')
118
119
120
121
122
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
123
                        ->with('apiAuthorized', $this->apiAuthorized)
124
                        ->with('metager', $this)
125
                        ->with('browser', (new Agent())->browser());
126
                default:
127
                    return view('resultpages.resultpage_images')
128
129
130
131
132
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
133
                        ->with('apiAuthorized', $this->apiAuthorized)
134
                        ->with('metager', $this)
135
136
137
138
                        ->with('browser', (new Agent())->browser())
                        ->with('quicktips', $quicktipResults)
                        ->with('focus', $this->fokus)
                        ->with('resultcount', count($this->results));
139
            }
140
141
142
        } else {
            switch ($this->out) {
                case 'results':
143
                    return view('resultpages.results')
144
145
146
147
148
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
149
                        ->with('apiAuthorized', $this->apiAuthorized)
150
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
151
152
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
153
154
                    break;
                case 'results-with-style':
155
                    return view('resultpages.resultpage')
156
157
158
159
160
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
161
                        ->with('apiAuthorized', $this->apiAuthorized)
162
163
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
Dominik Hebeler's avatar
Dominik Hebeler committed
164
165
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
166
                    break;
Phil Höfer's avatar
Phil Höfer committed
167
                case 'rich':
168
                    return view('resultpages.metager3rich')
169
                        ->with('results', $viewResults)
Phil Höfer's avatar
Phil Höfer committed
170
171
172
173
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
174
                        ->with('apiAuthorized', $this->apiAuthorized)
Phil Höfer's avatar
Phil Höfer committed
175
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
176
177
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
Phil Höfer's avatar
Phil Höfer committed
178
                    break;
179
                case 'rss20':
180
                    return view('resultpages.metager3resultsrss20')
181
182
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
Phil Höfer's avatar
Phil Höfer committed
183
                        ->with('apiAuthorized', $this->apiAuthorized)
184
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
185
186
                        ->with('resultcount', sizeof($viewResults))
                        ->with('fokus', $this->fokus);
187
                    break;
188
189
190
                case 'api':
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'apiAuthorized' => $this->apiAuthorized])->header('Content-Type', 'application/xml');
                    break;
Aria Givi's avatar
Aria Givi committed
191
                case 'atom10':
192
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'apiAuthorized' => true])
193
                        ->header('Content-Type', 'application/xml');
Aria Givi's avatar
Aria Givi committed
194
                    break;
195
                case 'result-count':
196
197
                    # Wir geben die Ergebniszahl und die benötigte Zeit zurück:
                    return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
198
                    break;
199
                default:
200
                    return view('resultpages.resultpage')
201
                        ->with('eingabe', $this->eingabe)
202
                        ->with('focusPages', $focusPages)
203
204
205
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
206
                        ->with('apiAuthorized', $this->apiAuthorized)
207
                        ->with('metager', $this)
208
                        ->with('browser', (new Agent())->browser())
209
                        ->with('quicktips', $quicktipResults)
210
211
                        ->with('resultcount', count($this->results))
                        ->with('focus', $this->fokus);
212
213
                    break;
            }
214
215
216
        }
    }

Phil Höfer's avatar
Phil Höfer committed
217
    public function prepareResults()
218
    {
Phil Höfer's avatar
Phil Höfer committed
219
        $engines = $this->engines;
220

Phil Höfer's avatar
Phil Höfer committed
221
222
        // combine
        $combinedResults = $this->combineResults($engines);
223
224
225
226

        # Wir bestimmen die Sprache eines jeden Suchergebnisses
        $this->results = $this->addLangCodes($this->results);

Phil Höfer's avatar
Phil Höfer committed
227
228
229
230
231
        // sort
        //$sortedResults = $this->sortResults($engines);
        // filter
        // augment (boost&adgoal)
        // authorize
Phil Höfer's avatar
Phil Höfer committed
232
233
234
        if ($this->apiKey) {
            $this->apiAuthorized = $this->authorize($this->apiKey);
        }
Phil Höfer's avatar
Phil Höfer committed
235
        // misc (WiP)
236
        if ($this->fokus == "nachrichten") {
237
238
239
            $this->results = array_filter($this->results, function ($v, $k) {
                return !is_null($v->getRank());
            }, ARRAY_FILTER_USE_BOTH);
240
241
242
243
244
245
246
247
248
249
            uasort($this->results, function ($a, $b) {
                $datea = $a->getDate();
                $dateb = $b->getDate();
                return $dateb - $datea;
            });
        } else {
            uasort($this->results, function ($a, $b) {
                if ($a->getRank() == $b->getRank()) {
                    return 0;
                }
250

251
252
253
                return ($a->getRank() < $b->getRank()) ? 1 : -1;
            });
        }
254

255
256
        # Validate Results
        $newResults = [];
257
258
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
259
                $newResults[] = $result;
260
261
            }

262
263
264
        }
        $this->results = $newResults;

265
        #Adgoal Implementation
266
267
268
        if (!$this->apiAuthorized) {
            $this->results = $this->parseAdgoal($this->results);
        }
269

Dominik Hebeler's avatar
Dominik Hebeler committed
270
271
        # Human Verification
        $this->results = $this->humanVerification($this->results);
272
        $this->ads = $this->humanVerification($this->ads);
Dominik Hebeler's avatar
Dominik Hebeler committed
273

274
        $counter = 0;
275
        $firstRank = 0;
276

277
        if (isset($this->startForwards)) {
278
            $this->startCount = $this->startForwards;
279
        } elseif (isset($this->startBackwards)) {
280
            $this->startCount = $this->startBackwards - count($this->results) - 1;
281
        } else {
282
283
284
            $this->startCount = 0;
        }

285
286
        foreach ($this->results as $result) {
            if ($counter === 0) {
287
                $firstRank = $result->rank;
288
289
            }

290
            $counter++;
291
            $result->number = $counter + $this->startCount;
292
            $confidence = 0;
293
294
295
            if ($firstRank > 0) {
                $confidence = $result->rank / $firstRank;
            } else {
296
                $confidence = 0;
297
298
299
            }

            if ($confidence > 0.65) {
300
                $result->color = "#FF4000";
301
            } elseif ($confidence > 0.4) {
302
                $result->color = "#FF0080";
303
            } elseif ($confidence > 0.2) {
304
                $result->color = "#C000C0";
305
            } else {
306
                $result->color = "#000000";
307
308
            }

309
310
        }

311
        if (count($this->results) <= 0) {
312
313
314
315
316
317
            if (strlen($this->site) > 0) {
                $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
                $this->errors[] = trans('metaGer.results.failedSitesearch', ['altSearch' => $no_sitesearch_query]);
            } else {
                $this->errors[] = trans('metaGer.results.failed');
            }
318
        }
319

320
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
321
            $page = $this->page + 1;
322
            $this->next = [
323
                'page' => $page,
324
                'startForwards' => $this->results[count($this->results) - 1]->number,
325
                'engines' => $this->next,
326
327
            ];
            Cache::put(md5(serialize($this->next)), serialize($this->next), 60);
328
329
        } else {
            $this->next = [];
330
331
        }

332
    }
333

334
335
    private function addLangCodes($results)
    {
336
337
338
339
340
        # Wenn es keine Ergebnisse gibt, brauchen wir uns gar nicht erst zu bemühen
        if (sizeof($results) === 0) {
            return $results;
        }

341
342
343
344
345
        # Bei der Spracheinstellung "all" wird nicht gefiltert
        if ($this->getLang() === "all") {
            return $results;
        } else {
            # Ansonsten müssen wir jedem Result einen Sprachcode hinzufügen
346
            $id = 0;
347
348
349
350
351
            $langStrings = [];
            foreach ($results as $result) {
                # Wir geben jedem Ergebnis eine ID um später die Sprachcodes zuordnen zu können
                $result->id = $id;

Dominik Hebeler's avatar
Dominik Hebeler committed
352
                $langStrings["result_" . $id] = utf8_encode($result->getLangString());
353
354
355
356
357
358
359
360

                $id++;
            }
            # Wir schreiben die Strings in eine temporäre JSON-Datei,
            # Da das Array unter umständen zu groß ist für eine direkte Übergabe an das Skript
            $filename = "/tmp/" . getmypid();
            file_put_contents($filename, json_encode($langStrings));
            $langDetectorPath = app_path() . "/Models/lang.pl";
361
362
            $lang = exec("echo '$filename' | $langDetectorPath");
            $lang = json_decode($lang, true);
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379

            # Wir haben nun die Sprachcodes der einzelnen Ergebnisse.
            # Diese müssen wir nur noch korrekt zuordnen, dann sind wir fertig.
            foreach ($lang as $key => $langCode) {
                # Prefix vom Key entfernen:
                $id = intval(str_replace("result_", "", $key));
                foreach ($this->results as $result) {
                    if ($result->id === $id) {
                        $result->langCode = $langCode;
                        break;
                    }
                }
            }
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
    public function combineResults($engines)
    {
        foreach ($engines as $engine) {
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
            foreach ($engine->results as $result) {
                if ($result->valid) {
                    $this->results[] = $result;
                }
            }
            foreach ($engine->ads as $ad) {
                $this->ads[] = $ad;
            }
        }
    }

400
401
    public function parseAdgoal($results)
    {
402
        $publicKey = getenv('adgoal_public');
403
        $privateKey = getenv('adgoal_private');
404
        if ($publicKey === false) {
405
406
407
            return $results;
        }
        $tldList = "";
408
409
        try {
            foreach ($results as $result) {
410
                $link = $result->anzeigeLink;
411
                if (strpos($link, "http") !== 0) {
412
413
414
415
416
417
418
419
420
421
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

422
            # Query
423
424
            $query = urlencode($this->q);

425
            $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . $publicKey . "&k=" . $hash . "&tld=" . $tldList . "&q=" . $query;
426
427
428
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
429
            foreach ($answer as $el) {
430
                $hoster = $el[0];
431
                $hash = $el[1];
432

433
                foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
434
                    if ($hoster === $result->tld && !$result->partnershop) {
435
436
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
437
                        if ($result->image !== "") {
438
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . $hash . ".gif";
439
                        } else {
440
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . $hash . ".gif";
441
442
                        }

443
444
445
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
                        $targetUrl = $result->anzeigeLink;
446
                        if (strpos($targetUrl, "http") !== 0) {
447
                            $targetUrl = "http://" . $targetUrl;
448
449
                        }

450
451
452
                        $gateHash = md5($targetUrl . $privateKey);
                        $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . $publicKey . "&k=" . $gateHash . "&url=" . urlencode($targetUrl) . "&q=" . $query;
                        $result->link = $newLink;
453
454
455
456
                        $result->partnershop = true;
                    }
                }
            }
457
        } catch (\ErrorException $e) {
458
459
460
461
462
            return $results;
        }

        return $results;
    }
463

464
465
    public function humanVerification($results)
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
466
        # Let's check if we need to implement a redirect for human verification
467
468
        if ($this->verificationCount > 10) {
            foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
469
470
471
                $link = $result->link;
                $day = Carbon::now()->day;
                $pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
472
473
474
                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
Dominik Hebeler's avatar
Dominik Hebeler committed
475
                $result->link = $url;
476
                $result->proxyLink = $proxyUrl;
Dominik Hebeler's avatar
Dominik Hebeler committed
477
478
            }
            return $results;
479
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
480
481
482
483
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
484
485
486
487
488
489
    public function authorize($key)
    {
        $postdata = http_build_query(array(
            'dummy' => rand(),
        ));
        $opts = array('http' => array(
490
491
            'method' => 'POST',
            'header' => 'Content-type: application/x-www-form-urlencoded',
Phil Höfer's avatar
Phil Höfer committed
492
493
494
495
496
497
498
            'content' => $postdata,
        ),
        );

        $context = stream_context_create($opts);

        try {
499
            $link = "https://key.metager3.de/" . urlencode($key) . "/request-permission/api-access";
Phil Höfer's avatar
Phil Höfer committed
500
501
502
503
504
505
506
507
508
509
510
511
            $result = json_decode(file_get_contents($link, false, $context));
            if ($result->{'api-access'} == true) {
                return true;
            } else {
                return false;
            }

        } catch (\ErrorException $e) {
            return false;
        }
    }

512
513
    public function createQuicktips()
    {
514
515
516
517
518
        # Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
        $quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime(), $this->getHashCode());
        return $quicktips;
    }

Karl's avatar
Karl committed
519
520
521
522
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

523
524
    public function createSearchEngines(Request $request)
    {
525
        # Wenn es kein Suchwort gibt
Dominik Hebeler's avatar
Dominik Hebeler committed
526
        if (!$request->filled("eingabe") || $this->q === "") {
527
            return;
528
        }
529

Dominik Hebeler's avatar
Dominik Hebeler committed
530
        $this->enabledSearchengines = [];
531
        $overtureEnabled = false;
532

533
534
535
        # Check if selected focus is valid
        if (empty($this->sumaFile->foki->{$this->fokus})) {
            $this->fokus = "web";
536
537
        }

538
539
        foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
            # Check if this engine is disabled and can't be used
540
            $disabled = empty($this->sumaFile->sumas->{$suma}->disabled) ? false : $this->sumaFile->sumas->{$suma}->disabled;
541
542
            $autoDisabled = empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) ? false : $this->sumaFile->sumas->{$suma}->{"auto-disabled"};
            if ($disabled || $autoDisabled) {
543
                continue;
544
545
            }

546
547
548
            # Check if this engine can use eventually defined query-filter
            $valid = true;
            foreach ($this->queryFilter as $queryFilter => $filter) {
Dominik Hebeler's avatar
Dominik Hebeler committed
549
                if (empty($this->sumaFile->filter->{"query-filter"}->$queryFilter->sumas->$suma)) {
550
551
                    $valid = false;
                    break;
Dominik Hebeler's avatar
Dominik Hebeler committed
552
553
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
554
555
556
            # Check if this engine can use eventually defined parameter-filter
            if ($valid) {
                foreach ($this->parameterFilter as $filterName => $filter) {
557
558
559
                    # We need to check if the searchengine supports the parameter value, too
                    $value = $request->input($filter->{"get-parameter"}, "");
                    if (empty($filter->sumas->$suma) || empty($filter->sumas->{$suma}->values->{$value})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
560
561
562
563
564
                        $valid = false;
                        break;
                    }
                }
            }
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
            # Check if this engine should only be active when filter is used
            if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"}) {
                # This search engine should only be used when a parameter filter of it is used
                $validTmp = false;
                foreach ($this->parameterFilter as $filterName => $filter) {
                    if (!empty($filter->sumas->{$suma})) {
                        $validTmp = true;
                        break;
                    }
                }
                if (!$validTmp) {
                    $valid = false;
                }

            }
580
581
            # If it can we add it
            if ($valid) {
Dominik Hebeler's avatar
Dominik Hebeler committed
582
                $this->enabledSearchengines[$suma] = $this->sumaFile->sumas->{$suma};
583
            }
584

585
        }
586

587
588
589
590
591
592
593
        # Implements Yahoo Ads if Yahoo is not enabled as a searchengine
        if (empty($this->enabledSearchengines["yahoo"]) && $this->fokus != "bilder" && !empty($this->sumaFile->sumas->{"yahoo-ads"})) {
            $this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
        }

        #die(var_dump($this->enabledSearchengines));

Dominik Hebeler's avatar
Dominik Hebeler committed
594
        if (sizeof($this->enabledSearchengines) === 0) {
595
596
            $filter = "";
            foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
Dominik Hebeler's avatar
Dominik Hebeler committed
597
                $filter .= trans($this->sumaFile->filter->{"query-filter"}->{$queryFilter}->name) . ",";
598
599
600
601
602
            }
            $filter = rtrim($filter, ",");
            $error = trans('metaGer.engines.noSpecialSearch', ['fokus' => trans($this->sumaFile->foki->{$this->fokus}->{"display-name"}),
                'filter' => $filter]);
            $this->errors[] = $error;
603
        }
604

605
        $engines = [];
606
        $typeslist = [];
607
        $counter = 0;
608

609
610
        if ($this->requestIsCached($request)) {
            $engines = $this->getCachedEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
611
            # We need to edit some Options of the Cached Search Engines
Phil Höfer's avatar
Phil Höfer committed
612
            foreach ($engines as $engine) {
Dominik Hebeler's avatar
Dominik Hebeler committed
613
614
                $engine->setResultHash($this->getHashCode());
            }
615
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
616
            $engines = $this->actuallyCreateSearchEngines($this->enabledSearchengines);
617
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
618

619
        # Wir starten alle Suchen
620
621
        foreach ($engines as $engine) {
            $engine->startSearch($this);
622
        }
623

624
625
626
627
628
        /* Wir warten auf die Antwort der Suchmaschinen
         * Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
         * Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
         * $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
         * welche Suchmaschinen nicht rechtzeitig geantwortet haben.
629
630
         */

631
        $enginesToLoad = [];
632
        $canBreak = false;
633
634
635
636
637
        foreach ($engines as $engine) {
            if ($engine->cached) {
                if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
                    $canBreak = true;
                }
638
639
            } else {
                $enginesToLoad[$engine->name] = false;
640
641
642
643
644
645
            }
        }

        $this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);

        $this->retrieveResults($engines);
646
647
648
649
650
        foreach ($engines as $engine) {
            if (!empty($engine->totalResults) && $engine->totalResults > $this->totalResults) {
                $this->totalResults = $engine->totalResults;
            }
        }
651
652
    }

653
654
    # Spezielle Suchen und Sumas

655
    public function sumaIsSelected($suma, $request, $custom)
656
    {
657
        if ($custom) {
658
            if ($request->filled("engine_" . strtolower($suma["name"]))) {
659
660
661
662
663
664
665
666
667
668
669
                return true;
            }
        } else {
            $types = explode(",", $suma["type"]);
            if (in_array($this->fokus, $types)) {
                return true;
            }
        }
        return false;
    }

670
    public function actuallyCreateSearchEngines($enabledSearchengines)
671
672
    {
        $engines = [];
673
        foreach ($enabledSearchengines as $engineName => $engine) {
674

675
            if (!isset($engine->{"parser-class"})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
676
677
                die(var_dump($engine));
            }
678
            # Setze Pfad zu Parser
679
            $path = "App\\Models\\parserSkripte\\" . $engine->{"parser-class"};
680
681

            # Prüfe ob Parser vorhanden
682
683
684
            if (!file_exists(app_path() . "/Models/parserSkripte/" . $engine->{"parser-class"} . ".php")) {
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen, da kein Parser existiert");
                $this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine->{"display-name"}]);
685
686
687
688
689
690
                continue;
            }

            # Es wird versucht die Suchengine zu erstellen
            $time = microtime();
            try {
691
                $tmp = new $path($engineName, $engine, $this);
692
            } catch (\ErrorException $e) {
693
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen. " . var_dump($e));
694
695
696
                continue;
            }

697
            $engines[] = $tmp;
698
699
700
701
        }
        return $engines;
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
702
703
704
705
706
707
708
709
710
711
712
713
714
    public function getAvailableParameterFilter()
    {
        $parameterFilter = $this->sumaFile->filter->{"parameter-filter"};

        $availableFilter = [];

        foreach ($parameterFilter as $filterName => $filter) {
            # Check if any of the enabled search engines provide this filter
            foreach ($this->enabledSearchengines as $engineName => $engine) {
                if (!empty($filter->sumas->$engineName)) {
                    $availableFilter[$filterName] = $filter;
                }
            }
715
716
717
718
            # We will also add the filter from the opt-in search engines (the searchengines that are only used when a filter of it is too)
            foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
                if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"}) {
                    if (!empty($filter->sumas->{$suma})) {
719
720
721
722
723
                        # If the searchengine is disabled this filter shouldn't be available
                        if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
                            || (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)) {
                            continue;
                        }
724
725
726
727
                        $availableFilter[$filterName] = $filter;
                    }
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
728
729
730
731
732
        }

        return $availableFilter;
    }

733
734
    public function isBildersuche()
    {
735
        return $this->fokus === "bilder";
736
737
738
739
    }

    public function sumaIsAdsuche($suma, $overtureEnabled)
    {
740
        $sumaName = $suma["name"]->__toString();
741
        return
742
743
            $sumaName === "qualigo"
            || $sumaName === "similar_product_ads"
Karl Hasselbring's avatar
Karl Hasselbring committed
744
            || (!$overtureEnabled && $sumaName === "overtureAds");
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
    }

    public function sumaIsDisabled($suma)
    {
        return
        isset($suma['disabled'])
        && $suma['disabled']->__toString() === "1";
    }

    public function sumaIsOverture($suma)
    {
        return
        $suma["name"]->__toString() === "overture"
        || $suma["name"]->__toString() === "overtureAds";
    }

    public function sumaIsNotAdsuche($suma)
    {
        return
        $suma["name"]->__toString() !== "qualigo"
        && $suma["name"]->__toString() !== "similar_product_ads"
        && $suma["name"]->__toString() !== "overtureAds";
    }

    public function requestIsCached($request)
    {
        return
Dominik Hebeler's avatar
Dominik Hebeler committed
772
        $request->filled('next')
773
774
775
776
777
778
        && Cache::has($request->input('next'))
        && unserialize(Cache::get($request->input('next')))['page'] > 1;
    }

    public function getCachedEngines($request)
    {
779
        $next = unserialize(Cache::get($request->input('next')));
780
        $this->page = $next['page'];
781
        $engines = $next['engines'];
782
783
784
785
786
787
788
789
790
        if (isset($next['startForwards'])) {
            $this->startForwards = $next['startForwards'];
        }
        if (isset($next['startBackwards'])) {
            $this->startBackwards = $next['startBackwards'];
        }
        return $engines;
    }

791
    # Passt den Suchfokus an, falls für einen Fokus genau alle vorhandenen Sumas eingeschaltet sind
792
793
    public function adjustFocus($sumas, $enabledSearchengines)
    {
794
795
        # Findet für alle Foki die enthaltenen Sumas
        $foki = []; # [fokus][suma] => [suma]
796
        foreach ($sumas as $suma) {
797
            if ((!$this->sumaIsDisabled($suma)) && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
798
                if (isset($suma['type'])) {
799
800
801
                    # Wenn foki für diese Suchmaschine angegeben sind
                    $focuses = explode(",", $suma['type']->__toString());
                    foreach ($focuses as $foc) {
802
803
804
805
806
                        if (isset($suma['minismCollection'])) {
                            $foki[$foc][] = "minism";
                        } else {
                            $foki[$foc][] = $suma['name']->__toString();
                        }
807
                    }
808
                } else {
809
                    # Wenn keine foki für diese Suchmaschine angegeben sind
810
811
812
813
814
                    if (isset($suma['minismCollection'])) {
                        $foki["andere"][] = "minism";
                    } else {
                        $foki["andere"][] = $suma['name']->__toString();
                    }
815
816
817
818
                }
            }
        }

819
        # Findet die Namen der aktuell eingeschalteten Sumas
820
        $realEngNames = [];
821
        foreach ($enabledSearchengines as $realEng) {
822
            $nam = $realEng["name"]->__toString();
Karl Hasselbring's avatar
Karl Hasselbring committed
823
            if ($nam !== "qualigo" && $nam !== "overtureAds") {
824
825
826
                $realEngNames[] = $nam;
            }
        }
827

828
        # Anschließend werden diese beiden Listen verglichen (jeweils eine der Fokuslisten für jeden Fokus), um herauszufinden ob sie vielleicht identisch sind. Ist dies der Fall, so hat der Nutzer anscheinend Suchmaschinen eines kompletten Fokus eingestellt. Der Fokus wird dementsprechend angepasst.
829
        foreach ($foki as $fok => $engines) {
830
            $isFokus = true;
831
            $fokiEngNames = [];
832
            foreach ($engines as $eng) {
833
834
                $fokiEngNames[] = $eng;
            }
835
            # Jede eingeschaltete Engine ist für diesen Fokus geeignet
836
            foreach ($fokiEngNames as $fen) {
837
838
                # Bei Bildersuchen ist uns egal, ob alle Suchmaschinen aus dem Suchfokus eingeschaltet sind, da wir sie eh als Bildersuche anzeigen müssen
                if (!in_array($fen, $realEngNames) && $fok !== "bilder") {
839
840
841
                    $isFokus = false;
                }
            }
842
            # Jede im Fokus erwartete Engine ist auch eingeschaltet
843
844
            foreach ($realEngNames as $ren) {
                if (!in_array($ren, $fokiEngNames)) {
845
846
847
                    $isFokus = false;
                }
            }
848
            # Wenn die Listen identisch sind, setze den Fokus um
849
            if ($isFokus) {
850
851
852
                $this->fokus = $fok;
            }
        }
853
    }
854

855
856
    public function waitForResults($enginesToLoad, $overtureEnabled, $canBreak)
    {
857

Phil Höfer's avatar
Phil Höfer committed
858
        $timeStart = microtime(true);
859
        $results = null;
860
        while (true) {
861
862
863
            $results = Redis::hgetall('search.' . $this->getHashCode());

            $ready = true;
Phil Höfer's avatar
Phil Höfer committed
864
            // When every
865
            $connected = true;
Phil Höfer's avatar
Phil Höfer committed
866
867
            foreach ($results as $key => $value) {
                if ($value === "waiting" || $value === "connected") {
868
                    $ready = false;
869
                }
Phil Höfer's avatar
Phil Höfer committed
870
                if ($value === "waiting") {
871
                    $connected = false;
872
                }
873
            }
874

875
876
            // If $ready is false at this point, we're waiting for more searchengines
            // But we have to check for the timeout, too
Phil Höfer's avatar
Phil Höfer committed
877
878
879
880
            if (!$connected) {
                $timeStart = microtime(true);
            }

881
882
883
884
            $time = (microtime(true) - $timeStart) * 1000;
            // We will apply the timeout only if it's not Yahoo we're waiting for since they are one the most
            // important search engines.
            $canTimeout = !((isset($results["overture"]) && $results["overture"] === "waiting") || (isset($results["overtureAds"]) && $results["overtureAds"] === "waiting"));
Phil Höfer's avatar
Phil Höfer committed
885
886
887
            if ($time > $this->time && $canTimeout) {
                $ready = true;
            }
888

Phil Höfer's avatar
Phil Höfer committed
889
            if ($ready) {
890
891
892
893
                break;
            }
            usleep(50000);
        }
894
895

        # Wir haben nun so lange wie möglich gewartet. Wir registrieren nun noch die Suchmaschinen, die geanwortet haben.
896
        foreach ($results as $key => $value) {
897
898
899
            $enginesToLoad[$key] = true;
        }
        $this->enginesToLoad = $enginesToLoad;
900
    }
901

902
903
904
    public function retrieveResults($engines)
    {
        # Von geladenen Engines die Ergebnisse holen
905
906
907
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
                try {
908
                    $engine->retrieveResults($this);