MetaGer.php 37.8 KB
Newer Older
1
2
3
4
<?php
namespace App;

use App;
5
use App\lib\TextLanguageDetect\TextLanguageDetect;
6
use Cache;
7
8
use Illuminate\Http\Request;
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix    
Dominik Hebeler committed
9
use LaravelLocalization;
10
use Log;
11
use Predis\Connection\ConnectionException;
12
use Redis;
13
14
15

class MetaGer
{
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
    # Einstellungen für die Suche
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $category;
    protected $time;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
    protected $hostBlacklist   = [];
    protected $domainBlacklist = [];
    protected $stopWords       = [];
    protected $phrases         = [];
    protected $engines         = [];
    protected $results         = [];
    protected $ads             = [];
    protected $warnings        = [];
    protected $errors          = [];
    protected $addedHosts      = [];
36
    protected $startCount      = 0;
37
    protected $canCache        = false;
38
39
40
41
42
43
44
45
46
    # Daten über die Abfrage
    protected $ip;
    protected $language;
    protected $agent;
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
47
    protected $domainsBlacklisted = [];
48
    protected $urlsBlacklisted    = [];
49
50
51
    protected $url;
    protected $languageDetect;

52
53
54
55
    public function __construct()
    {
        $this->starttime = microtime(true);
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
56
            # Blacklists einlesen:
57
            $tmp                      = file_get_contents(config_path() . "/blacklistDomains.txt");
58
            $this->domainsBlacklisted = explode("\n", $tmp);
59
60
61
            $tmp                      = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted    = explode("\n", $tmp);
        } else {
62
63
64
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
        }

65
66
67
68
69
70
71
72
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
                require $path;
            }
        }

73
74
        $this->languageDetect = new TextLanguageDetect();
        $this->languageDetect->setNameMode("2");
75
76
77
78
79
80
81

        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
82
    }
83

84
    public function getHashCode()
85
86
87
88
89
    {
        $string = url()->full();
        return md5($string);
    }

90
    public function rankAll()
91
    {
92
        foreach ($this->engines as $engine) {
93
94
95
            $engine->rank($this);
        }
    }
96

97
98
99
    public function createView()
    {
        $viewResults = [];
100

101
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
102
        foreach ($this->results as $result) {
103
104
105
106
107
108
            $viewResults[] = get_object_vars($result);
        }

        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();

109
110
        if ($this->fokus === "bilder") {
            switch ($this->out) {
111
112
113
114
115
116
117
118
                case 'results':
                    return view('metager3bilderresults')
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
119
                        ->with('browser', (new Agent())->browser());
120
121
122
123
124
125
126
127
                default:
                    return view('metager3bilder')
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
128
                        ->with('browser', (new Agent())->browser());
129
130
131
132
133
134
135
136
137
138
139
140
            }
        }

        switch ($this->out) {
            case 'results':
                return view('metager3results')
                    ->with('results', $viewResults)
                    ->with('eingabe', $this->eingabe)
                    ->with('mobile', $this->mobile)
                    ->with('warnings', $this->warnings)
                    ->with('errors', $this->errors)
                    ->with('metager', $this)
141
                    ->with('browser', (new Agent())->browser());
142
143
144
145
146
147
148
149
150
151
                break;
            case 'results-with-style':
                return view('metager3')
                    ->with('results', $viewResults)
                    ->with('eingabe', $this->eingabe)
                    ->with('mobile', $this->mobile)
                    ->with('warnings', $this->warnings)
                    ->with('errors', $this->errors)
                    ->with('metager', $this)
                    ->with('suspendheader', "yes")
152
                    ->with('browser', (new Agent())->browser());
153
154
155
156
157
158
159
160
                break;
            default:
                return view('metager3')
                    ->with('eingabe', $this->eingabe)
                    ->with('mobile', $this->mobile)
                    ->with('warnings', $this->warnings)
                    ->with('errors', $this->errors)
                    ->with('metager', $this)
161
                    ->with('browser', (new Agent())->browser());
162
163
                break;
        }
164
    }
165

166
167
168
169
170
171
    private function createLogs()
    {
        $redis = Redis::connection('redisLogs');
        try
        {
            $logEntry = "";
172
            $logEntry .= "[" . date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y"))) . "]";
173
174
175
176
177
178
            $logEntry .= " pid=" . getmypid();
            $logEntry .= " ref=" . $this->request->header('Referer');
            $useragent = $this->request->header('User-Agent');
            $useragent = str_replace("(", " ", $useragent);
            $useragent = str_replace(")", " ", $useragent);
            $useragent = str_replace(" ", "", $useragent);
179
            $logEntry .= " time=" . round((microtime(true) - $this->starttime), 2) . " serv=" . $this->fokus;
180
181
            $logEntry .= " search=" . $this->eingabe;
            $redis->rpush('logs.search', $logEntry);
182
        } catch (\Exception $e) {
183
184
185
186
            return;
        }
    }

187
    public function removeInvalids()
188
189
    {
        $results = [];
190
191
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
192
                $results[] = $result;
193
194
            }

195
196
197
        }
        #$this->results = $results;
    }
198

199
200
201
    public function combineResults()
    {
        foreach ($this->engines as $engine) {
202
203
204
205
206
207
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
208
209
            foreach ($engine->results as $result) {
                if ($result->valid) {
210
                    $this->results[] = $result;
211
                }
212
            }
213
            foreach ($engine->ads as $ad) {
214
215
                $this->ads[] = $ad;
            }
216
        }
217

218
219
        uasort($this->results, function ($a, $b) {
            if ($a->getRank() == $b->getRank()) {
220
                return 0;
221
222
            }

223
224
225
226
            return ($a->getRank() < $b->getRank()) ? 1 : -1;
        });
        # Validate Results
        $newResults = [];
227
228
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
229
                $newResults[] = $result;
230
231
            }

232
233
234
        }
        $this->results = $newResults;

235
236
237
238
239
240
        # Boost implementation
        $this->results = $this->parseBoost($this->results);

        #Adgoal Implementation
        $this->results = $this->parseAdgoal($this->results);

241
        $counter   = 0;
242
        $firstRank = 0;
243

244
        if (isset($this->startForwards)) {
245
            $this->startCount = $this->startForwards;
246
        } elseif (isset($this->startBackwards)) {
247
            $this->startCount = $this->startBackwards - count($this->results) - 1;
248
        } else {
249
250
251
            $this->startCount = 0;
        }

252
253
        foreach ($this->results as $result) {
            if ($counter === 0) {
254
                $firstRank = $result->rank;
255
256
            }

257
            $counter++;
258
            $result->number = $counter + $this->startCount;
259
260
261
262
            $confidence     = 0;
            if ($firstRank > 0) {
                $confidence = $result->rank / $firstRank;
            } else {
263
                $confidence = 0;
264
265
266
            }

            if ($confidence > 0.65) {
267
                $result->color = "#FF4000";
268
            } elseif ($confidence > 0.4) {
269
                $result->color = "#FF0080";
270
            } elseif ($confidence > 0.2) {
271
                $result->color = "#C000C0";
272
            } else {
273
                $result->color = "#000000";
274
275
            }

276
277
        }

278
        if (LaravelLocalization::getCurrentLocale() === "en") {
279
280
281
            $this->ads = [];
        }

282
        $this->validated = false;
283
        if (isset($this->password)) {
284
285
286
            # Wir bieten einen bezahlten API-Zugriff an, bei dem dementsprechend die Werbung ausgeblendet wurde:
            # Aktuell ist es nur die Uni-Mainz. Deshalb überprüfen wir auch nur diese.
            $password = getenv('mainz');
287
            $eingabe  = $this->eingabe;
288
            $password = md5($eingabe . $password);
289
290
            if ($this->password === $password) {
                $this->ads       = [];
291
292
293
                $this->validated = true;
            }
        }
294
295

        if (count($this->results) <= 0) {
Dominik Hebeler's avatar
Dominik Hebeler committed
296
            $this->errors[] = "Leider konnten wir zu Ihrer Sucheingabe keine passenden Ergebnisse finden.";
297
        }
298

299
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
300
            $page       = $this->page + 1;
301
            $this->next = [
302
303
304
                'page'          => $page,
                'startForwards' => $this->results[count($this->results) - 1]->number,
                'engines'       => $this->next,
305
306
            ];
            Cache::put(md5(serialize($this->next)), serialize($this->next), 60);
307
308
        } else {
            $this->next = [];
309
310
        }

311
    }
312

313
314
    public function parseBoost($results)
    {
315
316
317
318
319
320
321
322
323
324
325
326
        foreach ($results as $result) {
            if (preg_match('/^(http[s]?\:\/\/)?(www.)?amazon\.de/', $result->anzeigeLink)) {
                if (preg_match('/\?/', $result->anzeigeLink)) {
                    $result->link .= '&tag=boostmg01-21';
                } else {
                    $result->link .= '?tag=boostmg01-21';
                }
                $result->partnershop = true;

            }
        }
        return $results;
327
328
329
    }
    public function parseAdgoal($results)
    {
330
        $publicKey  = getenv('adgoal_public');
331
        $privateKey = getenv('adgoal_private');
332
        if ($publicKey === false) {
333
334
335
            return $results;
        }
        $tldList = "";
336
337
        try {
            foreach ($results as $result) {
338
                $link = $result->anzeigeLink;
339
                if (strpos($link, "http") !== 0) {
340
341
342
343
344
345
346
347
348
349
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

350
            # Query
351
352
            $query = urlencode($this->q);

353
            $link   = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . $publicKey . "&k=" . $hash . "&tld=" . $tldList . "&q=" . $query;
354
355
356
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
357
            foreach ($answer as $el) {
358
                $hoster = $el[0];
359
                $hash   = $el[1];
360

361
362
                foreach ($results as $result) {
                    if ($hoster === $result->tld) {
363
364
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
365
                        if ($result->image !== "") {
366
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . $hash . ".gif";
367
                        } else {
368
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . $hash . ".gif";
369
370
                        }

371
372
373
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
                        $targetUrl = $result->anzeigeLink;
374
                        if (strpos($targetUrl, "http") !== 0) {
375
                            $targetUrl = "http://" . $targetUrl;
376
377
378
379
380
                        }

                        $gateHash            = md5($targetUrl . $privateKey);
                        $newLink             = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . $publicKey . "&k=" . $gateHash . "&url=" . urlencode($targetUrl) . "&q=" . $query;
                        $result->link        = $newLink;
381
382
383
384
                        $result->partnershop = true;
                    }
                }
            }
385
        } catch (\ErrorException $e) {
386
387
388
389
390
            return $results;
        }

        return $results;
    }
391

392
393
    public function createSearchEngines(Request $request)
    {
394

395
        if (!$request->has("eingabe")) {
396
            return;
397
        }
398

399
400
        # Überprüfe, welche Sumas eingeschaltet sind
        $xml                  = simplexml_load_file($this->sumaFile);
401
        $enabledSearchengines = [];
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
        $overtureEnabled      = false;
        $countSumas           = 0;
        $sumas                = $xml->xpath("suma");
        if ($this->fokus === "angepasst") {
            foreach ($sumas as $suma) {
                if ($request->has($suma["name"])
                    || ($this->fokus !== "bilder"
                        && ($suma["name"]->__toString() === "qualigo"
                            || $suma["name"]->__toString() === "similar_product_ads"
                            || (!$overtureEnabled && $suma["name"]->__toString() === "overtureAds")
                        )
                    )
                ) {

                    if (!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1")) {
                        if ($suma["name"]->__toString() === "overture" || $suma["name"]->__toString() === "overtureAds") {
                            $overtureEnabled = true;
419
                        }
420
                        if ($suma["name"]->__toString() !== "qualigo" && $suma["name"]->__toString() !== "similar_product_ads" && $suma["name"]->__toString() !== "overtureAds") {
421
                            $countSumas += 1;
422
423
                        }

424
425
426
427
                        $enabledSearchengines[] = $suma;
                    }
                }
            }
428
429
430
431
432
433
434
435
436
437
438
439
440
441
        } else {
            foreach ($sumas as $suma) {
                $types = explode(",", $suma["type"]);
                if (in_array($this->fokus, $types)
                    || ($this->fokus !== "bilder"
                        && ($suma["name"]->__toString() === "qualigo"
                            || $suma["name"]->__toString() === "similar_product_ads"
                            || (!$overtureEnabled && $suma["name"]->__toString() === "overtureAds")
                        )
                    )
                ) {
                    if (!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1")) {
                        if ($suma["name"]->__toString() === "overture" || $suma["name"]->__toString() === "overtureAds") {
                            $overtureEnabled = true;
442
                        }
443
                        if ($suma["name"]->__toString() !== "qualigo" && $suma["name"]->__toString() !== "similar_product_ads" && $suma["name"]->__toString() !== "overtureAds") {
444
                            $countSumas += 1;
445
446
                        }

447
448
449
450
451
452
453
454
                        $enabledSearchengines[] = $suma;
                    }
                }
            }
        }

        # Sonderregelung für alle Suchmaschinen, die zu den Minisuchern gehören. Diese können alle gemeinsam über einen Link abgefragt werden
        $subcollections = [];
455
456
457
        $tmp            = [];
        foreach ($enabledSearchengines as $engine) {
            if (isset($engine['minismCollection'])) {
458
                $subcollections[] = $engine['minismCollection']->__toString();
459
            } else {
460
                $tmp[] = $engine;
461
462
            }

463
464
        }
        $enabledSearchengines = $tmp;
465
466
467
468
        if (sizeof($subcollections) > 0) {
            $count                        = sizeof($subcollections) * 10;
            $minisucherEngine             = $xml->xpath('suma[@name="minism"]')[0];
            $subcollections               = urlencode("(" . implode(" OR ", $subcollections) . ")");
469
470
            $minisucherEngine["formData"] = str_replace("<<SUBCOLLECTIONS>>", $subcollections, $minisucherEngine["formData"]);
            $minisucherEngine["formData"] = str_replace("<<COUNT>>", $count, $minisucherEngine["formData"]);
471
            $enabledSearchengines[]       = $minisucherEngine;
472
473
474
475
        }

        #die(var_dump($enabledSearchengines));

476
        if ($countSumas <= 0) {
477
478
            $this->errors[] = "Achtung: Sie haben in ihren Einstellungen keine Suchmaschine ausgewählt.";
        }
479
        $engines = [];
480

481
        $siteSearchFailed = false;
482
        if (strlen($this->site) > 0) {
483
484
            # Wenn eine Sitesearch durchgeführt werden soll, überprüfen wir ob eine der Suchmaschinen überhaupt eine Sitesearch unterstützt:
            $enginesWithSite = 0;
485
486
            foreach ($enabledSearchengines as $engine) {
                if (isset($engine['hasSiteSearch']) && $engine['hasSiteSearch']->__toString() === "1") {
487
488
489
                    $enginesWithSite++;
                }
            }
490
491
            if ($enginesWithSite === 0) {
                $this->errors[]   = "Sie wollten eine Sitesearch auf " . $this->site . " durchführen. Leider unterstützen die eingestellten Suchmaschinen diese nicht. Sie können <a href=\"" . $this->generateSearchLink("web", false) . "\">hier</a> die Sitesearch im Web-Fokus durchführen. Es werden ihnen Ergebnisse ohne Sitesearch angezeigt.";
492
                $siteSearchFailed = true;
493
            } else {
494
495
496
497
498
499
                $this->warnings[] = "Sie führen eine Sitesearch durch. Es werden nur Ergebnisse von der Seite: <a href=\"http://" . $this->site . "\" target=\"_blank\">\"" . $this->site . "\"</a> angezeigt.";
            }

        }

        $typeslist = [];
500
        $counter   = 0;
501

502
503
        if ($request->has('next') && Cache::has($request->input('next')) && unserialize(Cache::get($request->input('next')))['page'] > 1) {
            $next       = unserialize(Cache::get($request->input('next')));
504
            $this->page = $next['page'];
505
506
            $engines    = $next['engines'];
            if (isset($next['startForwards'])) {
507
                $this->startForwards = $next['startForwards'];
508
509
510
            }

            if (isset($next['startBackwards'])) {
511
                $this->startBackwards = $next['startBackwards'];
512
513
            }

514
515
        } else {
            foreach ($enabledSearchengines as $engine) {
516

517
                if (!$siteSearchFailed && strlen($this->site) > 0 && (!isset($engine['hasSiteSearch']) || $engine['hasSiteSearch']->__toString() === "0")) {
518

519
520
521
522
                    continue;
                }
                # Wenn diese Suchmaschine gar nicht eingeschaltet sein soll
                $path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString());
523

524
525
526
527
                if (!file_exists(app_path() . "/Models/parserSkripte/" . ucfirst($engine["package"]->__toString()) . ".php")) {
                    Log::error("Konnte " . $engine["name"] . " nicht abfragen, da kein Parser existiert");
                    continue;
                }
528

529
                $time = microtime();
530

531
532
533
534
535
536
537
                try
                {
                    $tmp = new $path($engine, $this);
                } catch (\ErrorException $e) {
                    Log::error("Konnte " . $engine["name"] . " nicht abfragen." . var_dump($e));
                    continue;
                }
538

539
540
541
542
543
544
545
                if ($tmp->enabled && isset($this->debug)) {
                    $this->warnings[] = $tmp->service . "   Connection_Time: " . $tmp->connection_time . "    Write_Time: " . $tmp->write_time . " Insgesamt:" . ((microtime() - $time) / 1000);
                }

                if ($tmp->isEnabled()) {
                    $engines[] = $tmp;
                }
546
547

            }
548
        }
549

550
551
552
        # Wir starten die Suche manuell:
        foreach ($engines as $engine) {
            $engine->startSearch($this);
553
        }
554

555
556
        # Jetzt werden noch alle Kategorien der Settings durchgegangen und die jeweils enthaltenen namen der Suchmaschinen gespeichert.
        $foki = [];
557
558
559
        foreach ($sumas as $suma) {
            if ((!isset($suma['disabled']) || $suma['disabled'] === "") && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
                if (isset($suma['type'])) {
560
                    $f = explode(",", $suma['type']->__toString());
561
562
                    foreach ($f as $tmp) {
                        $name                                    = $suma['name']->__toString();
563
564
                        $foki[$tmp][$suma['name']->__toString()] = $name;
                    }
565
566
                } else {
                    $name                                        = $suma['name']->__toString();
567
568
569
570
571
572
573
                    $foki["andere"][$suma['name']->__toString()] = $name;
                }
            }
        }

        # Es werden auch die Namen der aktuell aktiven Suchmaschinen abgespeichert.
        $realEngNames = [];
574
        foreach ($enabledSearchengines as $realEng) {
575
            $nam = $realEng["name"]->__toString();
576
            if ($nam !== "qualigo" && $nam !== "overtureAds") {
577
578
579
580
                $realEngNames[] = $nam;
            }
        }
        # Anschließend werden diese beiden Listen verglichen (jeweils eine der Fokuslisten für jeden Fokus), um herauszufinden ob sie vielleicht identisch sind. Ist dies der Fall, so hat der Nutzer anscheinend Suchmaschinen eines kompletten Fokus eingestellt. Der Fokus wird dementsprechend angepasst.
581
582
        foreach ($foki as $fok => $engs) {
            $isFokus      = true;
583
            $fokiEngNames = [];
584
            foreach ($engs as $eng) {
585
586
                $fokiEngNames[] = $eng;
            }
587
588
            foreach ($fokiEngNames as $fen) {
                if (!in_array($fen, $realEngNames)) {
589
590
591
                    $isFokus = false;
                }
            }
592
593
            foreach ($realEngNames as $ren) {
                if (!in_array($ren, $fokiEngNames)) {
594
595
596
                    $isFokus = false;
                }
            }
597
            if ($isFokus) {
598
599
600
601
602
603
604
605
606
                $this->fokus = $fok;
            }
        }

        # Nun passiert ein elementarer Schritt.
        # Wir warten auf die Antwort der Suchmaschinen, da wir vorher nicht weiter machen können.
        # aber natürlich nicht ewig.
        # Die Verbindung steht zu diesem Zeitpunkt und auch unsere Request wurde schon gesendet.
        # Wir geben der Suchmaschine nun bis zu 500ms Zeit zu antworten.
607
608
609

        # Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
        $enginesToLoad = 0;
610
611
612
        $canBreak      = false;
        foreach ($engines as $engine) {
            if ($engine->cached) {
613
                $enginesToLoad--;
614
                if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
615
                    $canBreak = true;
616
617
                }

618
619
620
            }
        }
        $enginesToLoad += count($engines);
621
        $loadedEngines = 0;
622
623
624
        $timeStart     = microtime(true);
        while (true) {
            $time          = (microtime(true) - $timeStart) * 1000;
625
            $loadedEngines = intval(Redis::hlen('search.' . $this->getHashCode()));
626
            if ($overtureEnabled && (Redis::hexists('search.' . $this->getHashCode(), 'overture') || Redis::hexists('search.' . $this->getHashCode(), 'overtureAds'))) {
627
                $canBreak = true;
628
            }
629
630

            # Abbruchbedingung
631
632
            if ($time < 500) {
                if (($enginesToLoad === 0 || $loadedEngines >= $enginesToLoad) && $canBreak) {
633
                    break;
634
635
636
637
                }

            } elseif ($time >= 500 && $time < $this->time) {
                if (($enginesToLoad === 0 || ($loadedEngines / ($enginesToLoad * 1.0)) >= 0.8) && $canBreak) {
638
                    break;
639
640
641
                }

            } else {
642
643
644
645
646
                break;
            }
            usleep(50000);
        }

647
648
649
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
                try {
650
                    $engine->retrieveResults($this);
651
                } catch (\ErrorException $e) {
652
                    Log::error($e);
653

654
655
656
                }
            }
        }
657

658
        # und verwerfen den Rest:
659
660
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
661
                $engine->shutdown();
662
663
            }

664
665
666
        }

        $this->engines = $engines;
667
668
669
670
671
672
673
674
675
676
677
678
    }

    public function parseFormData(Request $request)
    {
        if ($request->input('encoding', '') !== "utf8") {
            # In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
            $input = $request->all();
            foreach ($input as $key => $value) {
                $input[$key] = mb_convert_encoding("$value", "UTF-8", "ISO-8859-1");
            }
            $request->replace($input);
        }
679
        $this->url = $request->url();
680
        # Zunächst überprüfen wir die eingegebenen Einstellungen:
681
682
        # FOKUS
        $this->fokus = trans('fokiNames.'
683
684
            . $request->input('focus', 'web'));
        if (strpos($this->fokus, ".")) {
685
686
687
688
            $this->fokus = trans('fokiNames.web');
        }

        # SUMA-FILE
689
        if (App::isLocale("en")) {
690
            $this->sumaFile = config_path() . "/sumas.xml";
691
        } else {
692
693
            $this->sumaFile = config_path() . "/sumas.xml";
        }
694
        if (!file_exists($this->sumaFile)) {
695
696
697
698
699
            die("Suma-File konnte nicht gefunden werden");
        }

        # Sucheingabe:
        $this->eingabe = trim($request->input('eingabe', ''));
700
        if (strlen($this->eingabe) === 0) {
701
702
703
704
705
706
707
708
            $this->warnings[] = 'Achtung: Sie haben keinen Suchbegriff eingegeben. Sie können ihre Suchbegriffe oben eingeben und es erneut versuchen.';
        }
        $this->q = $this->eingabe;

        # IP:
        $this->ip = $request->ip();

        # Language:
709
        if (isset($_SERVER['HTTP_LANGUAGE'])) {
710
            $this->language = $_SERVER['HTTP_LANGUAGE'];
711
        } else {
712
713
714
715
716
717
            $this->language = "";
        }
        # Category
        $this->category = $request->input('category', '');
        # Request Times:
        $this->time = $request->input('time', 1000);
718

719
        # Page
720
        $this->page = 1;
721
722
        # Lang
        $this->lang = $request->input('lang', 'all');
723
724
        if ($this->lang !== "de" && $this->lang !== "en" && $this->lang !== "all") {
            $this->lang = "all";
725
        }
726
        $this->agent  = new Agent();
727
728
729
730
        $this->mobile = $this->agent->isMobile();

        #Sprüche
        $this->sprueche = $request->input('sprueche', 'off');
731
        if ($this->sprueche === "off") {
732
            $this->sprueche = true;
733
        } else {
734
            $this->sprueche = false;
735
736
        }

737
738
739
740
        # Ergebnisse pro Seite:
        $this->resultCount = $request->input('resultCount', '20');

        # Manchmal müssen wir Parameter anpassen um den Sucheinstellungen gerecht zu werden:
741
742
743
        if ($request->has('dart')) {
            $this->time       = 10000;
            $this->warnings[] = "Hinweis: Sie haben Dart-Europe aktiviert. Die Suche kann deshalb länger dauern und die maximale Suchzeit wurde auf 10 Sekunden hochgesetzt.";
744
        }
745
746
        if ($this->time <= 500 || $this->time > 20000) {
            $this->time = 1000;
747
        }
748
749
750
751
752
753
754
755
756
        if ($request->has('minism') && ($request->has('fportal') || $request->has('harvest'))) {
            $input    = $request->all();
            $newInput = [];
            foreach ($input as $key => $value) {
                if ($key !== "fportal" && $key !== "harvest") {
                    $newInput[$key] = $value;
                }
            }
            $request->replace($newInput);
757
        }
758
759
        if (App::isLocale("en")) {
            $this->sprueche = "off";
760
        }
761
762
        if ($this->resultCount <= 0 || $this->resultCount > 200) {
            $this->resultCount = 1000;
763
        }
764
765
766
767
768
769
        if ($request->has('onenewspageAll') || $request->has('onenewspageGermanyAll')) {
            $this->time  = 5000;
            $this->cache = "cache";
        }
        if ($request->has('tab')) {
            if ($request->input('tab') === "off") {
770
                $this->tab = "_blank";
771
            } else {
772
773
                $this->tab = "_self";
            }
774
        } else {
775
776
            $this->tab = "_blank";
        }
777
        if ($request->has('password')) {
778
            $this->password = $request->input('password');
779
780
781
        }

        if ($request->has('quicktips')) {
782
            $this->quicktips = false;
783
        } else {
784
            $this->quicktips = true;
785
        }
786
787

        $this->out = $request->input('out', "html");
788
        if ($this->out !== "html" && $this->out !== "json" && $this->out !== "results" && $this->out !== "results-with-style") {
789
            $this->out = "html";
790
791
        }

792
        $this->request = $request;
793
794
795
796
797
798
799
800
801
802
    }

    public function checkSpecialSearches(Request $request)
    {
        # Site Search:
        if (preg_match("/(.*)\bsite:(\S+)(.*)/si", $this->q, $match)) {
            $this->site = $match[2];
            $this->q    = $match[1] . $match[3];
        }
        if ($request->has('site')) {
803
804
            $this->site = $request->input('site');
        }
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
        # Wenn die Suchanfrage um das Schlüsselwort "-host:*" ergänzt ist, sollen bestimmte Hosts nicht eingeblendet werden
        # Wir prüfen, ob das hier der Fall ist:
        while (preg_match("/(.*)(^|\s)-host:(\S+)(.*)/si", $this->q, $match)) {
            $this->hostBlacklist[] = $match[3];
            $this->q               = $match[1] . $match[4];
        }
        if (sizeof($this->hostBlacklist) > 0) {
            $hostString = "";
            foreach ($this->hostBlacklist as $host) {
                $hostString .= $host . ", ";
            }
            $hostString       = rtrim($hostString, ", ");
            $this->warnings[] = "Ergebnisse von folgenden Hosts werden nicht angezeigt: \"" . $hostString . "\"";
        }
        # Wenn die Suchanfrage um das Schlüsselwort "-domain:*" ergänzt ist, sollen bestimmte Domains nicht eingeblendet werden
        # Wir prüfen, ob das hier der Fall ist:
        while (preg_match("/(.*)(^|\s)-domain:(\S+)(.*)/si", $this->q, $match)) {
            $this->domainBlacklist[] = $match[3];
            $this->q                 = $match[1] . $match[4];
        }
        if (sizeof($this->domainBlacklist) > 0) {
            $domainString = "";
            foreach ($this->domainBlacklist as $domain) {
                $domainString .= $domain . ", ";
            }
            $domainString     = rtrim($domainString, ", ");
            $this->warnings[] = "Ergebnisse von folgenden Domains werden nicht angezeigt: \"" . $domainString . "\"";
        }

        # Alle mit "-" gepräfixten Worte sollen aus der Suche ausgeschlossen werden.
        # Wir prüfen, ob das hier der Fall ist:
        while (preg_match("/(.*)(^|\s)-(\S+)(.*)/si", $this->q, $match)) {
            $this->stopWords[] = $match[3];
            $this->q           = $match[1] . $match[4];
        }
        if (sizeof($this->stopWords) > 0) {
            $stopwordsString = "";
            foreach ($this->stopWords as $stopword) {
                $stopwordsString .= $stopword . ", ";
            }
            $stopwordsString  = rtrim($stopwordsString, ", ");
            $this->warnings[] = "Sie machen eine Ausschlusssuche. Ergebnisse mit folgenden Wörtern werden nicht angezeigt: \"" . $stopwordsString . "\"";
        }

        # Meldung über eine Phrasensuche
        $p   = "";
851
        $tmp = $this->q;
852
853
        while (preg_match("/(.*)\"(.+)\"(.*)/si", $tmp, $match)) {
            $tmp             = $match[1] . $match[3];
854
            $this->phrases[] = strtolower($match[2]);
855
856
        }
        foreach ($this->phrases as $phrase) {
857
858
859
            $p .= "\"$phrase\", ";
        }
        $p = rtrim($p, ", ");
860
        if (sizeof($this->phrases) > 0) {
861
            $this->warnings[] = "Sie führen eine Phrasensuche durch: $p";
862
        }
863

864
865
866
    }

    public function getFokus()
867
868
869
870
    {
        return $this->fokus;
    }

871
    public function getIp()
872
873
874
875
    {
        return $this->ip;
    }

876
    public function getEingabe()
877
878
879
880
    {
        return $this->eingabe;
    }

881
    public function getQ()
882
    {
883
        return $this->q;
884
885
    }

886
    public function getUrl()
887
888
889
    {
        return $this->url;
    }
890
    public function getTime()
891
892
893
894
    {
        return $this->time;
    }

895
    public function getLanguage()
896
897
898
899
    {
        return $this->language;
    }

900
    public function getLang()
901
902
903
904
    {
        return $this->lang;
    }

905
    public function getSprueche()
906
907
908
909
    {
        return $this->sprueche;
    }

910
    public function getCategory()
911
912
913
914
    {
        return $this->category;
    }

915
    public function getPhrases()
916
917
918
    {
        return $this->phrases;
    }
919
920
921
922
    public function getPage()
    {
        return $this->page;
    }
923

924
    public function getSumaFile()
925
926
927
928
    {
        return $this->sumaFile;
    }

929
    public function getUserHostBlacklist()
930
931
932
933
    {
        return $this->hostBlacklist;
    }

934
    public function getUserDomainBlacklist()
935
936
937
938
    {
        return $this->domainBlacklist;
    }

939
    public function getDomainBlacklist()
940
941
942
943
    {
        return $this->domainsBlacklisted;
    }

944
    public function getUrlBlacklist()
945
946
947
    {
        return $this->urlsBlacklisted;
    }
948
    public function getLanguageDetect()
949
950
951
    {
        return $this->languageDetect;
    }
952
    public function getStopWords()
953
954
955
956
957
    {
        return $this->stopWords;
    }
    public function getHostCount($host)
    {
958
        if (isset($this->addedHosts[$host])) {
959
            return $this->addedHosts[$host];
960
        } else {
961
962
963
            return 0;
        }
    }
964
965
966
967
    public function getStartCount()
    {
        return $this->startCount;
    }
968
969
970
    public function addHostCount($host)
    {
        $hash = md5($host);
971
        if (isset($this->addedHosts[$hash])) {
972
            $this->addedHosts[$hash] += 1;
973
        } else {
974
975
976
            $this->addedHosts[$hash] = 1;
        }
    }
977
978
979
980
    public function canCache()
    {
        return $this->canCache;
    }
981
982
983
984
985
986
    public function getSite()
    {
        return $this->site;
    }
    public function addLink($link)
    {
987
        if (strpos($link, "http://") === 0) {
988
            $link = substr($link, 7);
989
990
991
        }

        if (strpos($link, "https://") === 0) {
992
            $link = substr($link, 8);
993
994
995
        }

        if (strpos($link, "www.") === 0) {
996
            $link = substr($link, 4);
997
998
        }

999
1000
        $link = trim($link, "/");
        $hash = md5($link);
1001
        if (isset($this->addedLinks[$hash])) {
1002
            return false;