MetaGer.php 57.5 KB
Newer Older
1
2
3
4
<?php
namespace App;

use App;
5
use Cache;
6
use Carbon;
7
use Illuminate\Http\Request;
Phil Höfer's avatar
Phil Höfer committed
8
use Illuminate\Support\Facades\Redis;
9
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix    
Dominik Hebeler committed
10
use LaravelLocalization;
11
use Log;
12
use Predis\Connection\ConnectionException;
13
14
15

class MetaGer
{
16
17
18
19
20
21
22
23
    # Einstellungen für die Suche
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
24
    protected $time = 2000;
25
    protected $hostBlacklist = [];
26
    protected $domainBlacklist = [];
27
28
29
30
    private $urlBlacklist = [];
    protected $stopWords = [];
    protected $phrases = [];
    protected $engines = [];
31
    protected $totalResults = 0;
32
    protected $results = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
33
34
    protected $queryFilter = [];
    protected $parameterFilter = [];
35
36
37
38
    protected $ads = [];
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
39
    protected $availableFoki = [];
40
41
    protected $startCount = 0;
    protected $canCache = false;
42
    # Daten über die Abfrage$
43
    protected $ip;
Dominik Hebeler's avatar
Dominik Hebeler committed
44
    protected $useragent;
45
46
    protected $language;
    protected $agent;
47
    protected $apiKey = "";
Phil Höfer's avatar
Phil Höfer committed
48
    protected $apiAuthorized = false;
49
50
51
52
53
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
54
    protected $newtab;
55
    protected $domainsBlacklisted = [];
56
    protected $urlsBlacklisted = [];
57
    protected $url;
58
    protected $fullUrl;
59
    protected $languageDetect;
Dominik Hebeler's avatar
Dominik Hebeler committed
60
61
    protected $verificationId;
    protected $verificationCount;
62

63
64
    public function __construct()
    {
65
        # Timer starten
66
        $this->starttime = microtime(true);
67
        # Versuchen Blacklists einzulesen
68
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
69
            $tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
70
            $this->domainsBlacklisted = explode("\n", $tmp);
71
72
            $tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted = explode("\n", $tmp);
73
        } else {
74
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
75
76
        }

77
        # Parser Skripte einhängen
78
79
80
81
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
82
                require_once $path;
83
84
85
            }
        }

86
        # Cachebarkeit testen
87
88
89
90
91
92
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
93
    }
94

95
    # Erstellt aus den gesammelten Ergebnissen den View
96
    public function createView($quicktipResults = [])
97
    {
98
99
        # Hiermit werden die evtl. ausgewählten SuMas extrahiert, damit die Input-Boxen richtig gesetzt werden können
        $focusPages = [];
Aria Givi's avatar
Aria Givi committed
100

101
        foreach ($this->request->all() as $key => $value) {
102
103
            if (starts_with($key, 'engine_') && $value === 'on') {
                $focusPages[] = $key;
104
105
106
            }
        }

107
        $viewResults = [];
108
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
109
        foreach ($this->results as $result) {
110
111
112
113
            $viewResults[] = get_object_vars($result);
        }
        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();
114
115
        if ($this->fokus === "bilder") {
            switch ($this->out) {
116
                case 'results':
117
                    return view('resultpages.results_images')
118
119
120
121
122
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
123
                        ->with('apiAuthorized', $this->apiAuthorized)
124
                        ->with('metager', $this)
125
                        ->with('browser', (new Agent())->browser());
126
                default:
127
                    return view('resultpages.resultpage_images')
128
129
130
131
132
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
133
                        ->with('apiAuthorized', $this->apiAuthorized)
134
                        ->with('metager', $this)
135
136
137
138
                        ->with('browser', (new Agent())->browser())
                        ->with('quicktips', $quicktipResults)
                        ->with('focus', $this->fokus)
                        ->with('resultcount', count($this->results));
139
            }
140
141
142
        } else {
            switch ($this->out) {
                case 'results':
143
                    return view('resultpages.results')
144
145
146
147
148
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
149
                        ->with('apiAuthorized', $this->apiAuthorized)
150
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
151
152
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
153
154
                    break;
                case 'results-with-style':
155
                    return view('resultpages.resultpage')
156
157
158
159
160
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
161
                        ->with('apiAuthorized', $this->apiAuthorized)
162
163
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
Dominik Hebeler's avatar
Dominik Hebeler committed
164
165
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
166
                    break;
Phil Höfer's avatar
Phil Höfer committed
167
                case 'rich':
168
                    return view('resultpages.metager3rich')
169
                        ->with('results', $viewResults)
Phil Höfer's avatar
Phil Höfer committed
170
171
172
173
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
174
                        ->with('apiAuthorized', $this->apiAuthorized)
Phil Höfer's avatar
Phil Höfer committed
175
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
176
177
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
Phil Höfer's avatar
Phil Höfer committed
178
                    break;
179
                case 'rss20':
180
                    return view('resultpages.metager3resultsrss20')
181
182
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
Phil Höfer's avatar
Phil Höfer committed
183
                        ->with('apiAuthorized', $this->apiAuthorized)
184
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
185
186
                        ->with('resultcount', sizeof($viewResults))
                        ->with('fokus', $this->fokus);
187
                    break;
188
189
190
                case 'api':
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'apiAuthorized' => $this->apiAuthorized])->header('Content-Type', 'application/xml');
                    break;
Aria Givi's avatar
Aria Givi committed
191
                case 'atom10':
192
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'apiAuthorized' => true])
193
                        ->header('Content-Type', 'application/xml');
Aria Givi's avatar
Aria Givi committed
194
                    break;
195
                case 'result-count':
196
197
                    # Wir geben die Ergebniszahl und die benötigte Zeit zurück:
                    return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
198
                    break;
199
                default:
200
                    return view('resultpages.resultpage')
201
                        ->with('eingabe', $this->eingabe)
202
                        ->with('focusPages', $focusPages)
203
204
205
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
206
                        ->with('apiAuthorized', $this->apiAuthorized)
207
                        ->with('metager', $this)
208
                        ->with('browser', (new Agent())->browser())
209
                        ->with('quicktips', $quicktipResults)
210
211
                        ->with('resultcount', count($this->results))
                        ->with('focus', $this->fokus);
212
213
                    break;
            }
214
215
216
        }
    }

Phil Höfer's avatar
Phil Höfer committed
217
    public function prepareResults()
218
    {
Phil Höfer's avatar
Phil Höfer committed
219
        $engines = $this->engines;
220

Phil Höfer's avatar
Phil Höfer committed
221
222
        // combine
        $combinedResults = $this->combineResults($engines);
223
224
225
226

        # Wir bestimmen die Sprache eines jeden Suchergebnisses
        $this->results = $this->addLangCodes($this->results);

Phil Höfer's avatar
Phil Höfer committed
227
228
229
230
231
        // sort
        //$sortedResults = $this->sortResults($engines);
        // filter
        // augment (boost&adgoal)
        // authorize
Phil Höfer's avatar
Phil Höfer committed
232
233
234
        if ($this->apiKey) {
            $this->apiAuthorized = $this->authorize($this->apiKey);
        }
Phil Höfer's avatar
Phil Höfer committed
235
        // misc (WiP)
236
        if ($this->fokus == "nachrichten") {
237
238
239
            $this->results = array_filter($this->results, function ($v, $k) {
                return !is_null($v->getRank());
            }, ARRAY_FILTER_USE_BOTH);
240
241
242
243
244
245
246
247
248
249
            uasort($this->results, function ($a, $b) {
                $datea = $a->getDate();
                $dateb = $b->getDate();
                return $dateb - $datea;
            });
        } else {
            uasort($this->results, function ($a, $b) {
                if ($a->getRank() == $b->getRank()) {
                    return 0;
                }
250

251
252
253
                return ($a->getRank() < $b->getRank()) ? 1 : -1;
            });
        }
254

255
256
        # Validate Results
        $newResults = [];
257
258
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
259
                $newResults[] = $result;
260
261
            }

262
263
264
        }
        $this->results = $newResults;

265
        #Adgoal Implementation
266
267
268
        if (!$this->apiAuthorized) {
            $this->results = $this->parseAdgoal($this->results);
        }
269

Dominik Hebeler's avatar
Dominik Hebeler committed
270
271
        # Human Verification
        $this->results = $this->humanVerification($this->results);
272
        $this->ads = $this->humanVerification($this->ads);
Dominik Hebeler's avatar
Dominik Hebeler committed
273

274
        $counter = 0;
275
        $firstRank = 0;
276

277
        if (isset($this->startForwards)) {
278
            $this->startCount = $this->startForwards;
279
        } elseif (isset($this->startBackwards)) {
280
            $this->startCount = $this->startBackwards - count($this->results) - 1;
281
        } else {
282
283
284
            $this->startCount = 0;
        }

285
286
        foreach ($this->results as $result) {
            if ($counter === 0) {
287
                $firstRank = $result->rank;
288
289
            }

290
            $counter++;
291
            $result->number = $counter + $this->startCount;
292
            $confidence = 0;
293
294
295
            if ($firstRank > 0) {
                $confidence = $result->rank / $firstRank;
            } else {
296
                $confidence = 0;
297
298
299
            }

            if ($confidence > 0.65) {
300
                $result->color = "#FF4000";
301
            } elseif ($confidence > 0.4) {
302
                $result->color = "#FF0080";
303
            } elseif ($confidence > 0.2) {
304
                $result->color = "#C000C0";
305
            } else {
306
                $result->color = "#000000";
307
308
            }

309
310
        }

311
        if ($this->validated) {
312
            $this->ads = [];
313
        }
314
315

        if (count($this->results) <= 0) {
316
317
318
319
320
321
            if (strlen($this->site) > 0) {
                $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
                $this->errors[] = trans('metaGer.results.failedSitesearch', ['altSearch' => $no_sitesearch_query]);
            } else {
                $this->errors[] = trans('metaGer.results.failed');
            }
322
        }
323

324
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
325
            $page = $this->page + 1;
326
            $this->next = [
327
                'page' => $page,
328
                'startForwards' => $this->results[count($this->results) - 1]->number,
329
                'engines' => $this->next,
330
331
            ];
            Cache::put(md5(serialize($this->next)), serialize($this->next), 60);
332
333
        } else {
            $this->next = [];
334
335
        }

336
    }
337

338
339
    private function addLangCodes($results)
    {
340
341
342
343
344
        # Wenn es keine Ergebnisse gibt, brauchen wir uns gar nicht erst zu bemühen
        if (sizeof($results) === 0) {
            return $results;
        }

345
346
347
348
349
        # Bei der Spracheinstellung "all" wird nicht gefiltert
        if ($this->getLang() === "all") {
            return $results;
        } else {
            # Ansonsten müssen wir jedem Result einen Sprachcode hinzufügen
350
            $id = 0;
351
352
353
354
355
            $langStrings = [];
            foreach ($results as $result) {
                # Wir geben jedem Ergebnis eine ID um später die Sprachcodes zuordnen zu können
                $result->id = $id;

Dominik Hebeler's avatar
Dominik Hebeler committed
356
                $langStrings["result_" . $id] = utf8_encode($result->getLangString());
357
358
359
360
361
362
363
364

                $id++;
            }
            # Wir schreiben die Strings in eine temporäre JSON-Datei,
            # Da das Array unter umständen zu groß ist für eine direkte Übergabe an das Skript
            $filename = "/tmp/" . getmypid();
            file_put_contents($filename, json_encode($langStrings));
            $langDetectorPath = app_path() . "/Models/lang.pl";
365
366
            $lang = exec("echo '$filename' | $langDetectorPath");
            $lang = json_decode($lang, true);
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383

            # Wir haben nun die Sprachcodes der einzelnen Ergebnisse.
            # Diese müssen wir nur noch korrekt zuordnen, dann sind wir fertig.
            foreach ($lang as $key => $langCode) {
                # Prefix vom Key entfernen:
                $id = intval(str_replace("result_", "", $key));
                foreach ($this->results as $result) {
                    if ($result->id === $id) {
                        $result->langCode = $langCode;
                        break;
                    }
                }
            }
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
    public function combineResults($engines)
    {
        foreach ($engines as $engine) {
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
            foreach ($engine->results as $result) {
                if ($result->valid) {
                    $this->results[] = $result;
                }
            }
            foreach ($engine->ads as $ad) {
                $this->ads[] = $ad;
            }
        }
    }

404
405
    public function parseAdgoal($results)
    {
406
        $publicKey = getenv('adgoal_public');
407
        $privateKey = getenv('adgoal_private');
408
        if ($publicKey === false) {
409
410
411
            return $results;
        }
        $tldList = "";
412
413
        try {
            foreach ($results as $result) {
414
                $link = $result->anzeigeLink;
415
                if (strpos($link, "http") !== 0) {
416
417
418
419
420
421
422
423
424
425
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

426
            # Query
427
428
            $query = urlencode($this->q);

429
            $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . $publicKey . "&k=" . $hash . "&tld=" . $tldList . "&q=" . $query;
430
431
432
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
433
            foreach ($answer as $el) {
434
                $hoster = $el[0];
435
                $hash = $el[1];
436

437
                foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
438
                    if ($hoster === $result->tld && !$result->partnershop) {
439
440
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
441
                        if ($result->image !== "") {
442
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . $hash . ".gif";
443
                        } else {
444
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . $hash . ".gif";
445
446
                        }

447
448
449
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
                        $targetUrl = $result->anzeigeLink;
450
                        if (strpos($targetUrl, "http") !== 0) {
451
                            $targetUrl = "http://" . $targetUrl;
452
453
                        }

454
455
456
                        $gateHash = md5($targetUrl . $privateKey);
                        $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . $publicKey . "&k=" . $gateHash . "&url=" . urlencode($targetUrl) . "&q=" . $query;
                        $result->link = $newLink;
457
458
459
460
                        $result->partnershop = true;
                    }
                }
            }
461
        } catch (\ErrorException $e) {
462
463
464
465
466
            return $results;
        }

        return $results;
    }
467

468
469
    public function humanVerification($results)
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
470
        # Let's check if we need to implement a redirect for human verification
471
472
        if ($this->verificationCount > 10) {
            foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
473
474
475
                $link = $result->link;
                $day = Carbon::now()->day;
                $pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
476
477
478
                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
Dominik Hebeler's avatar
Dominik Hebeler committed
479
                $result->link = $url;
480
                $result->proxyLink = $proxyUrl;
Dominik Hebeler's avatar
Dominik Hebeler committed
481
482
            }
            return $results;
483
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
484
485
486
487
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
488
489
490
491
492
493
    public function authorize($key)
    {
        $postdata = http_build_query(array(
            'dummy' => rand(),
        ));
        $opts = array('http' => array(
494
495
            'method' => 'POST',
            'header' => 'Content-type: application/x-www-form-urlencoded',
Phil Höfer's avatar
Phil Höfer committed
496
497
498
499
500
501
502
            'content' => $postdata,
        ),
        );

        $context = stream_context_create($opts);

        try {
503
            $link = "https://key.metager3.de/" . urlencode($key) . "/request-permission/api-access";
Phil Höfer's avatar
Phil Höfer committed
504
505
506
507
508
509
510
511
512
513
514
515
            $result = json_decode(file_get_contents($link, false, $context));
            if ($result->{'api-access'} == true) {
                return true;
            } else {
                return false;
            }

        } catch (\ErrorException $e) {
            return false;
        }
    }

516
517
    public function createQuicktips()
    {
518
519
520
521
522
        # Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
        $quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime(), $this->getHashCode());
        return $quicktips;
    }

Karl's avatar
Karl committed
523
524
525
526
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

527
528
    public function createSearchEngines(Request $request)
    {
529
        # Wenn es kein Suchwort gibt
Dominik Hebeler's avatar
Dominik Hebeler committed
530
        if (!$request->filled("eingabe") || $this->q === "") {
531
            return;
532
        }
533

Dominik Hebeler's avatar
Dominik Hebeler committed
534
        $this->enabledSearchengines = [];
535
        $overtureEnabled = false;
536

537
538
539
        # Check if selected focus is valid
        if (empty($this->sumaFile->foki->{$this->fokus})) {
            $this->fokus = "web";
540
541
        }

542
543
        foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
            # Check if this engine is disabled and can't be used
544
            $disabled = empty($this->sumaFile->sumas->{$suma}->disabled) ? false : $this->sumaFile->sumas->{$suma}->disabled;
545
546
            $autoDisabled = empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) ? false : $this->sumaFile->sumas->{$suma}->{"auto-disabled"};
            if ($disabled || $autoDisabled) {
547
                continue;
548
549
            }

550
551
552
            # Check if this engine can use eventually defined query-filter
            $valid = true;
            foreach ($this->queryFilter as $queryFilter => $filter) {
Dominik Hebeler's avatar
Dominik Hebeler committed
553
                if (empty($this->sumaFile->filter->{"query-filter"}->$queryFilter->sumas->$suma)) {
554
555
                    $valid = false;
                    break;
Dominik Hebeler's avatar
Dominik Hebeler committed
556
557
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
558
559
560
561
562
563
564
565
566
            # Check if this engine can use eventually defined parameter-filter
            if ($valid) {
                foreach ($this->parameterFilter as $filterName => $filter) {
                    if (empty($filter->sumas->$suma)) {
                        $valid = false;
                        break;
                    }
                }
            }
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
            # Check if this engine should only be active when filter is used
            if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"}) {
                # This search engine should only be used when a parameter filter of it is used
                $validTmp = false;
                foreach ($this->parameterFilter as $filterName => $filter) {
                    if (!empty($filter->sumas->{$suma})) {
                        $validTmp = true;
                        break;
                    }
                }
                if (!$validTmp) {
                    $valid = false;
                }

            }
582
583
            # If it can we add it
            if ($valid) {
Dominik Hebeler's avatar
Dominik Hebeler committed
584
                $this->enabledSearchengines[$suma] = $this->sumaFile->sumas->{$suma};
585
            }
586

587
        }
588

Dominik Hebeler's avatar
Dominik Hebeler committed
589
        if (sizeof($this->enabledSearchengines) === 0) {
590
591
            $filter = "";
            foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
Dominik Hebeler's avatar
Dominik Hebeler committed
592
                $filter .= trans($this->sumaFile->filter->{"query-filter"}->{$queryFilter}->name) . ",";
593
594
595
596
597
            }
            $filter = rtrim($filter, ",");
            $error = trans('metaGer.engines.noSpecialSearch', ['fokus' => trans($this->sumaFile->foki->{$this->fokus}->{"display-name"}),
                'filter' => $filter]);
            $this->errors[] = $error;
598
        }
599

600
        $engines = [];
601
        $typeslist = [];
602
        $counter = 0;
603

604
605
        if ($this->requestIsCached($request)) {
            $engines = $this->getCachedEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
606
            # We need to edit some Options of the Cached Search Engines
Phil Höfer's avatar
Phil Höfer committed
607
            foreach ($engines as $engine) {
Dominik Hebeler's avatar
Dominik Hebeler committed
608
609
                $engine->setResultHash($this->getHashCode());
            }
610
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
611
            $engines = $this->actuallyCreateSearchEngines($this->enabledSearchengines);
612
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
613

614
        # Wir starten alle Suchen
615
616
        foreach ($engines as $engine) {
            $engine->startSearch($this);
617
        }
618

619
620
621
622
623
        /* Wir warten auf die Antwort der Suchmaschinen
         * Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
         * Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
         * $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
         * welche Suchmaschinen nicht rechtzeitig geantwortet haben.
624
625
         */

626
        $enginesToLoad = [];
627
        $canBreak = false;
628
629
630
631
632
        foreach ($engines as $engine) {
            if ($engine->cached) {
                if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
                    $canBreak = true;
                }
633
634
            } else {
                $enginesToLoad[$engine->name] = false;
635
636
637
638
639
640
            }
        }

        $this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);

        $this->retrieveResults($engines);
641
642
643
644
645
        foreach ($engines as $engine) {
            if (!empty($engine->totalResults) && $engine->totalResults > $this->totalResults) {
                $this->totalResults = $engine->totalResults;
            }
        }
646
647
    }

648
649
    # Spezielle Suchen und Sumas

650
    public function sumaIsSelected($suma, $request, $custom)
651
    {
652
        if ($custom) {
653
            if ($request->filled("engine_" . strtolower($suma["name"]))) {
654
655
656
657
658
659
660
661
662
663
664
                return true;
            }
        } else {
            $types = explode(",", $suma["type"]);
            if (in_array($this->fokus, $types)) {
                return true;
            }
        }
        return false;
    }

665
    public function actuallyCreateSearchEngines($enabledSearchengines)
666
667
    {
        $engines = [];
668
        foreach ($enabledSearchengines as $engineName => $engine) {
669

670
            if (!isset($engine->{"parser-class"})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
671
672
                die(var_dump($engine));
            }
673
            # Setze Pfad zu Parser
674
            $path = "App\\Models\\parserSkripte\\" . $engine->{"parser-class"};
675
676

            # Prüfe ob Parser vorhanden
677
678
679
            if (!file_exists(app_path() . "/Models/parserSkripte/" . $engine->{"parser-class"} . ".php")) {
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen, da kein Parser existiert");
                $this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine->{"display-name"}]);
680
681
682
683
684
685
                continue;
            }

            # Es wird versucht die Suchengine zu erstellen
            $time = microtime();
            try {
686
                $tmp = new $path($engineName, $engine, $this);
687
            } catch (\ErrorException $e) {
688
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen. " . var_dump($e));
689
690
691
                continue;
            }

692
            $engines[] = $tmp;
693
694
695
696
        }
        return $engines;
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
697
698
699
700
701
702
703
704
705
706
707
708
709
    public function getAvailableParameterFilter()
    {
        $parameterFilter = $this->sumaFile->filter->{"parameter-filter"};

        $availableFilter = [];

        foreach ($parameterFilter as $filterName => $filter) {
            # Check if any of the enabled search engines provide this filter
            foreach ($this->enabledSearchengines as $engineName => $engine) {
                if (!empty($filter->sumas->$engineName)) {
                    $availableFilter[$filterName] = $filter;
                }
            }
710
711
712
713
            # We will also add the filter from the opt-in search engines (the searchengines that are only used when a filter of it is too)
            foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
                if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"}) {
                    if (!empty($filter->sumas->{$suma})) {
714
715
716
717
718
                        # If the searchengine is disabled this filter shouldn't be available
                        if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
                            || (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)) {
                            continue;
                        }
719
720
721
722
                        $availableFilter[$filterName] = $filter;
                    }
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
723
724
725
726
727
        }

        return $availableFilter;
    }

728
729
    public function isBildersuche()
    {
730
        return $this->fokus === "bilder";
731
732
733
734
    }

    public function sumaIsAdsuche($suma, $overtureEnabled)
    {
735
        $sumaName = $suma["name"]->__toString();
736
        return
737
738
            $sumaName === "qualigo"
            || $sumaName === "similar_product_ads"
Karl Hasselbring's avatar
Karl Hasselbring committed
739
            || (!$overtureEnabled && $sumaName === "overtureAds");
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
    }

    public function sumaIsDisabled($suma)
    {
        return
        isset($suma['disabled'])
        && $suma['disabled']->__toString() === "1";
    }

    public function sumaIsOverture($suma)
    {
        return
        $suma["name"]->__toString() === "overture"
        || $suma["name"]->__toString() === "overtureAds";
    }

    public function sumaIsNotAdsuche($suma)
    {
        return
        $suma["name"]->__toString() !== "qualigo"
        && $suma["name"]->__toString() !== "similar_product_ads"
        && $suma["name"]->__toString() !== "overtureAds";
    }

    public function requestIsCached($request)
    {
        return
Dominik Hebeler's avatar
Dominik Hebeler committed
767
        $request->filled('next')
768
769
770
771
772
773
        && Cache::has($request->input('next'))
        && unserialize(Cache::get($request->input('next')))['page'] > 1;
    }

    public function getCachedEngines($request)
    {
774
        $next = unserialize(Cache::get($request->input('next')));
775
        $this->page = $next['page'];
776
        $engines = $next['engines'];
777
778
779
780
781
782
783
784
785
        if (isset($next['startForwards'])) {
            $this->startForwards = $next['startForwards'];
        }
        if (isset($next['startBackwards'])) {
            $this->startBackwards = $next['startBackwards'];
        }
        return $engines;
    }

786
    # Passt den Suchfokus an, falls für einen Fokus genau alle vorhandenen Sumas eingeschaltet sind
787
788
    public function adjustFocus($sumas, $enabledSearchengines)
    {
789
790
        # Findet für alle Foki die enthaltenen Sumas
        $foki = []; # [fokus][suma] => [suma]
791
        foreach ($sumas as $suma) {
792
            if ((!$this->sumaIsDisabled($suma)) && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
793
                if (isset($suma['type'])) {
794
795
796
                    # Wenn foki für diese Suchmaschine angegeben sind
                    $focuses = explode(",", $suma['type']->__toString());
                    foreach ($focuses as $foc) {
797
798
799
800
801
                        if (isset($suma['minismCollection'])) {
                            $foki[$foc][] = "minism";
                        } else {
                            $foki[$foc][] = $suma['name']->__toString();
                        }
802
                    }
803
                } else {
804
                    # Wenn keine foki für diese Suchmaschine angegeben sind
805
806
807
808
809
                    if (isset($suma['minismCollection'])) {
                        $foki["andere"][] = "minism";
                    } else {
                        $foki["andere"][] = $suma['name']->__toString();
                    }
810
811
812
813
                }
            }
        }

814
        # Findet die Namen der aktuell eingeschalteten Sumas
815
        $realEngNames = [];
816
        foreach ($enabledSearchengines as $realEng) {
817
            $nam = $realEng["name"]->__toString();
Karl Hasselbring's avatar
Karl Hasselbring committed
818
            if ($nam !== "qualigo" && $nam !== "overtureAds") {
819
820
821
                $realEngNames[] = $nam;
            }
        }
822

823
        # Anschließend werden diese beiden Listen verglichen (jeweils eine der Fokuslisten für jeden Fokus), um herauszufinden ob sie vielleicht identisch sind. Ist dies der Fall, so hat der Nutzer anscheinend Suchmaschinen eines kompletten Fokus eingestellt. Der Fokus wird dementsprechend angepasst.
824
        foreach ($foki as $fok => $engines) {
825
            $isFokus = true;
826
            $fokiEngNames = [];
827
            foreach ($engines as $eng) {
828
829
                $fokiEngNames[] = $eng;
            }
830
            # Jede eingeschaltete Engine ist für diesen Fokus geeignet
831
            foreach ($fokiEngNames as $fen) {
832
833
                # Bei Bildersuchen ist uns egal, ob alle Suchmaschinen aus dem Suchfokus eingeschaltet sind, da wir sie eh als Bildersuche anzeigen müssen
                if (!in_array($fen, $realEngNames) && $fok !== "bilder") {
834
835
836
                    $isFokus = false;
                }
            }
837
            # Jede im Fokus erwartete Engine ist auch eingeschaltet
838
839
            foreach ($realEngNames as $ren) {
                if (!in_array($ren, $fokiEngNames)) {
840
841
842
                    $isFokus = false;
                }
            }
843
            # Wenn die Listen identisch sind, setze den Fokus um
844
            if ($isFokus) {
845
846
847
                $this->fokus = $fok;
            }
        }
848
    }
849

850
851
    public function waitForResults($enginesToLoad, $overtureEnabled, $canBreak)
    {
852

Phil Höfer's avatar
Phil Höfer committed
853
        $timeStart = microtime(true);
854
        $results = null;
855
        while (true) {
856
857
858
            $results = Redis::hgetall('search.' . $this->getHashCode());

            $ready = true;
Phil Höfer's avatar
Phil Höfer committed
859
            // When every
860
            $connected = true;
Phil Höfer's avatar
Phil Höfer committed
861
862
            foreach ($results as $key => $value) {
                if ($value === "waiting" || $value === "connected") {
863
                    $ready = false;
864
                }
Phil Höfer's avatar
Phil Höfer committed
865
                if ($value === "waiting") {
866
                    $connected = false;
867
                }
868
            }
869

870
871
            // If $ready is false at this point, we're waiting for more searchengines
            // But we have to check for the timeout, too
Phil Höfer's avatar
Phil Höfer committed
872
873
874
875
            if (!$connected) {
                $timeStart = microtime(true);
            }

876
877
878
879
            $time = (microtime(true) - $timeStart) * 1000;
            // We will apply the timeout only if it's not Yahoo we're waiting for since they are one the most
            // important search engines.
            $canTimeout = !((isset($results["overture"]) && $results["overture"] === "waiting") || (isset($results["overtureAds"]) && $results["overtureAds"] === "waiting"));
Phil Höfer's avatar
Phil Höfer committed
880
881
882
            if ($time > $this->time && $canTimeout) {
                $ready = true;
            }
883

Phil Höfer's avatar
Phil Höfer committed
884
            if ($ready) {
885
886
887
888
                break;
            }
            usleep(50000);
        }
889
890

        # Wir haben nun so lange wie möglich gewartet. Wir registrieren nun noch die Suchmaschinen, die geanwortet haben.
891
        foreach ($results as $key => $value) {
892
893
894
            $enginesToLoad[$key] = true;
        }
        $this->enginesToLoad = $enginesToLoad;
895
    }
896

897
898
899
    public function retrieveResults($engines)
    {
        # Von geladenen Engines die Ergebnisse holen
900
901
902
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
                try {
903
                    $engine->retrieveResults($this);
904
                } catch (\ErrorException $e) {