MetaGer.php 37.8 KB
Newer Older
1 2 3 4
<?php
namespace App;

use App;
5
use App\lib\TextLanguageDetect\TextLanguageDetect;
6
use Cache;
7 8
use Illuminate\Http\Request;
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix  
Dominik Hebeler committed
9
use LaravelLocalization;
10 11
use Log;
use Redis;
12 13 14

class MetaGer
{
15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
    # Einstellungen für die Suche
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $category;
    protected $time;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
    protected $hostBlacklist   = [];
    protected $domainBlacklist = [];
    protected $stopWords       = [];
    protected $phrases         = [];
    protected $engines         = [];
    protected $results         = [];
    protected $ads             = [];
    protected $warnings        = [];
    protected $errors          = [];
    protected $addedHosts      = [];
35
    protected $startCount      = 0;
36 37 38 39 40 41 42 43 44
    # Daten über die Abfrage
    protected $ip;
    protected $language;
    protected $agent;
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
45
    protected $domainsBlacklisted = [];
46
    protected $urlsBlacklisted    = [];
47 48 49
    protected $url;
    protected $languageDetect;

50 51 52 53
    public function __construct()
    {
        $this->starttime = microtime(true);
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
54
            # Blacklists einlesen:
55
            $tmp                      = file_get_contents(config_path() . "/blacklistDomains.txt");
56
            $this->domainsBlacklisted = explode("\n", $tmp);
57 58 59
            $tmp                      = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted    = explode("\n", $tmp);
        } else {
60 61 62
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
        }

63 64 65 66 67 68 69 70
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
                require $path;
            }
        }

71 72
        $this->languageDetect = new TextLanguageDetect();
        $this->languageDetect->setNameMode("2");
73
    }
74

75
    public function getHashCode()
76 77 78 79 80
    {
        $string = url()->full();
        return md5($string);
    }

81
    public function rankAll()
82
    {
83
        foreach ($this->engines as $engine) {
84 85 86
            $engine->rank($this);
        }
    }
87

88 89 90
    public function createView()
    {
        $viewResults = [];
91

92
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
93
        foreach ($this->results as $result) {
94 95 96 97 98 99
            $viewResults[] = get_object_vars($result);
        }

        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();

100 101
        if ($this->fokus === "bilder") {
            switch ($this->out) {
102 103 104 105 106 107 108 109
                case 'results':
                    return view('metager3bilderresults')
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
110
                        ->with('browser', (new Agent())->browser());
111 112 113 114 115 116 117 118
                default:
                    return view('metager3bilder')
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
119
                        ->with('browser', (new Agent())->browser());
120 121 122 123 124 125 126 127 128 129 130 131
            }
        }

        switch ($this->out) {
            case 'results':
                return view('metager3results')
                    ->with('results', $viewResults)
                    ->with('eingabe', $this->eingabe)
                    ->with('mobile', $this->mobile)
                    ->with('warnings', $this->warnings)
                    ->with('errors', $this->errors)
                    ->with('metager', $this)
132
                    ->with('browser', (new Agent())->browser());
133 134 135 136 137 138 139 140 141 142
                break;
            case 'results-with-style':
                return view('metager3')
                    ->with('results', $viewResults)
                    ->with('eingabe', $this->eingabe)
                    ->with('mobile', $this->mobile)
                    ->with('warnings', $this->warnings)
                    ->with('errors', $this->errors)
                    ->with('metager', $this)
                    ->with('suspendheader', "yes")
143
                    ->with('browser', (new Agent())->browser());
144 145 146 147 148 149 150 151
                break;
            default:
                return view('metager3')
                    ->with('eingabe', $this->eingabe)
                    ->with('mobile', $this->mobile)
                    ->with('warnings', $this->warnings)
                    ->with('errors', $this->errors)
                    ->with('metager', $this)
152
                    ->with('browser', (new Agent())->browser());
153 154
                break;
        }
155
    }
156

157 158 159 160 161 162
    private function createLogs()
    {
        $redis = Redis::connection('redisLogs');
        try
        {
            $logEntry = "";
163
            $logEntry .= "[" . date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y"))) . "]";
164 165 166 167 168 169
            $logEntry .= " pid=" . getmypid();
            $logEntry .= " ref=" . $this->request->header('Referer');
            $useragent = $this->request->header('User-Agent');
            $useragent = str_replace("(", " ", $useragent);
            $useragent = str_replace(")", " ", $useragent);
            $useragent = str_replace(" ", "", $useragent);
170
            $logEntry .= " time=" . round((microtime(true) - $this->starttime), 2) . " serv=" . $this->fokus;
171 172
            $logEntry .= " search=" . $this->eingabe;
            $redis->rpush('logs.search', $logEntry);
173
        } catch (\Exception $e) {
174 175 176 177
            return;
        }
    }

178
    public function removeInvalids()
179 180
    {
        $results = [];
181 182
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
183
                $results[] = $result;
184 185
            }

186 187 188
        }
        #$this->results = $results;
    }
189

190 191 192
    public function combineResults()
    {
        foreach ($this->engines as $engine) {
193 194 195 196 197 198
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
199 200
            foreach ($engine->results as $result) {
                if ($result->valid) {
201
                    $this->results[] = $result;
202
                }
203
            }
204
            foreach ($engine->ads as $ad) {
205 206
                $this->ads[] = $ad;
            }
207
        }
208

209 210
        uasort($this->results, function ($a, $b) {
            if ($a->getRank() == $b->getRank()) {
211
                return 0;
212 213
            }

214 215 216 217
            return ($a->getRank() < $b->getRank()) ? 1 : -1;
        });
        # Validate Results
        $newResults = [];
218 219
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
220
                $newResults[] = $result;
221 222
            }

223 224 225
        }
        $this->results = $newResults;

226 227 228 229 230 231
        # Boost implementation
        $this->results = $this->parseBoost($this->results);

        #Adgoal Implementation
        $this->results = $this->parseAdgoal($this->results);

232
        $counter   = 0;
233
        $firstRank = 0;
234

235
        if (isset($this->startForwards)) {
236
            $this->startCount = $this->startForwards;
237
        } elseif (isset($this->startBackwards)) {
238
            $this->startCount = $this->startBackwards - count($this->results) - 1;
239
        } else {
240 241 242
            $this->startCount = 0;
        }

243 244
        foreach ($this->results as $result) {
            if ($counter === 0) {
245
                $firstRank = $result->rank;
246 247
            }

248
            $counter++;
249
            $result->number = $counter + $this->startCount;
250 251 252 253
            $confidence     = 0;
            if ($firstRank > 0) {
                $confidence = $result->rank / $firstRank;
            } else {
254
                $confidence = 0;
255 256 257
            }

            if ($confidence > 0.65) {
258
                $result->color = "#FF4000";
259
            } elseif ($confidence > 0.4) {
260
                $result->color = "#FF0080";
261
            } elseif ($confidence > 0.2) {
262
                $result->color = "#C000C0";
263
            } else {
264
                $result->color = "#000000";
265 266
            }

267 268
        }

269
        if (LaravelLocalization::getCurrentLocale() === "en") {
270 271 272
            $this->ads = [];
        }

273
        $this->validated = false;
274
        if (isset($this->password)) {
275 276 277
            # Wir bieten einen bezahlten API-Zugriff an, bei dem dementsprechend die Werbung ausgeblendet wurde:
            # Aktuell ist es nur die Uni-Mainz. Deshalb überprüfen wir auch nur diese.
            $password = getenv('mainz');
278
            $eingabe  = $this->eingabe;
279
            $password = md5($eingabe . $password);
280 281
            if ($this->password === $password) {
                $this->ads       = [];
282 283 284
                $this->validated = true;
            }
        }
285 286

        if (count($this->results) <= 0) {
Dominik Hebeler's avatar
Dominik Hebeler committed
287
            $this->errors[] = "Leider konnten wir zu Ihrer Sucheingabe keine passenden Ergebnisse finden.";
288
        }
289

290 291
        if (isset($this->last) && count($this->last) > 0) {
            $page       = $this->page - 1;
292
            $this->last = [
293 294 295
                'page'           => $page,
                'startBackwards' => $this->results[0]->number,
                'engines'        => $this->last,
296 297 298 299
            ];
            Cache::put(md5(serialize($this->last)), serialize($this->last), 60);
        }

300 301
        if (isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
            $page       = $this->page + 1;
302
            $this->next = [
303 304 305
                'page'          => $page,
                'startForwards' => $this->results[count($this->results) - 1]->number,
                'engines'       => $this->next,
306 307 308 309
            ];
            Cache::put(md5(serialize($this->next)), serialize($this->next), 60);
        }

310
    }
311

312 313
    public function parseBoost($results)
    {
314 315 316 317 318 319 320 321 322 323 324 325
        foreach ($results as $result) {
            if (preg_match('/^(http[s]?\:\/\/)?(www.)?amazon\.de/', $result->anzeigeLink)) {
                if (preg_match('/\?/', $result->anzeigeLink)) {
                    $result->link .= '&tag=boostmg01-21';
                } else {
                    $result->link .= '?tag=boostmg01-21';
                }
                $result->partnershop = true;

            }
        }
        return $results;
326 327 328
    }
    public function parseAdgoal($results)
    {
329
        $publicKey  = getenv('adgoal_public');
330
        $privateKey = getenv('adgoal_private');
331
        if ($publicKey === false) {
332 333 334
            return $results;
        }
        $tldList = "";
335 336
        try {
            foreach ($results as $result) {
337
                $link = $result->anzeigeLink;
338
                if (strpos($link, "http") !== 0) {
339 340 341 342 343 344 345 346 347 348
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

349
            # Query
350 351
            $query = urlencode($this->q);

352
            $link   = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . $publicKey . "&k=" . $hash . "&tld=" . $tldList . "&q=" . $query;
353 354 355
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
356
            foreach ($answer as $el) {
357
                $hoster = $el[0];
358
                $hash   = $el[1];
359

360 361
                foreach ($results as $result) {
                    if ($hoster === $result->tld) {
362 363
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
364
                        if ($result->image !== "") {
365
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . $hash . ".gif";
366
                        } else {
367
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . $hash . ".gif";
368 369
                        }

370 371 372
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
                        $targetUrl = $result->anzeigeLink;
373
                        if (strpos($targetUrl, "http") !== 0) {
374
                            $targetUrl = "http://" . $targetUrl;
375 376 377 378 379
                        }

                        $gateHash            = md5($targetUrl . $privateKey);
                        $newLink             = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . $publicKey . "&k=" . $gateHash . "&url=" . urlencode($targetUrl) . "&q=" . $query;
                        $result->link        = $newLink;
380 381 382 383
                        $result->partnershop = true;
                    }
                }
            }
384
        } catch (\ErrorException $e) {
385 386 387 388 389
            return $results;
        }

        return $results;
    }
390

391 392
    public function createSearchEngines(Request $request)
    {
393

394
        if (!$request->has("eingabe")) {
395
            return;
396
        }
397

398 399
        # Überprüfe, welche Sumas eingeschaltet sind
        $xml                  = simplexml_load_file($this->sumaFile);
400
        $enabledSearchengines = [];
401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417
        $overtureEnabled      = false;
        $countSumas           = 0;
        $sumas                = $xml->xpath("suma");
        if ($this->fokus === "angepasst") {
            foreach ($sumas as $suma) {
                if ($request->has($suma["name"])
                    || ($this->fokus !== "bilder"
                        && ($suma["name"]->__toString() === "qualigo"
                            || $suma["name"]->__toString() === "similar_product_ads"
                            || (!$overtureEnabled && $suma["name"]->__toString() === "overtureAds")
                        )
                    )
                ) {

                    if (!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1")) {
                        if ($suma["name"]->__toString() === "overture" || $suma["name"]->__toString() === "overtureAds") {
                            $overtureEnabled = true;
418
                        }
419
                        if ($suma["name"]->__toString() !== "qualigo" && $suma["name"]->__toString() !== "similar_product_ads" && $suma["name"]->__toString() !== "overtureAds") {
420
                            $countSumas += 1;
421 422
                        }

423 424 425 426
                        $enabledSearchengines[] = $suma;
                    }
                }
            }
427 428 429 430 431 432 433 434 435 436 437 438 439 440
        } else {
            foreach ($sumas as $suma) {
                $types = explode(",", $suma["type"]);
                if (in_array($this->fokus, $types)
                    || ($this->fokus !== "bilder"
                        && ($suma["name"]->__toString() === "qualigo"
                            || $suma["name"]->__toString() === "similar_product_ads"
                            || (!$overtureEnabled && $suma["name"]->__toString() === "overtureAds")
                        )
                    )
                ) {
                    if (!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1")) {
                        if ($suma["name"]->__toString() === "overture" || $suma["name"]->__toString() === "overtureAds") {
                            $overtureEnabled = true;
441
                        }
442
                        if ($suma["name"]->__toString() !== "qualigo" && $suma["name"]->__toString() !== "similar_product_ads" && $suma["name"]->__toString() !== "overtureAds") {
443
                            $countSumas += 1;
444 445
                        }

446 447 448 449 450 451 452 453
                        $enabledSearchengines[] = $suma;
                    }
                }
            }
        }

        # Sonderregelung für alle Suchmaschinen, die zu den Minisuchern gehören. Diese können alle gemeinsam über einen Link abgefragt werden
        $subcollections = [];
454 455 456
        $tmp            = [];
        foreach ($enabledSearchengines as $engine) {
            if (isset($engine['minismCollection'])) {
457
                $subcollections[] = $engine['minismCollection']->__toString();
458
            } else {
459
                $tmp[] = $engine;
460 461
            }

462 463
        }
        $enabledSearchengines = $tmp;
464 465 466 467
        if (sizeof($subcollections) > 0) {
            $count                        = sizeof($subcollections) * 10;
            $minisucherEngine             = $xml->xpath('suma[@name="minism"]')[0];
            $subcollections               = urlencode("(" . implode(" OR ", $subcollections) . ")");
468 469
            $minisucherEngine["formData"] = str_replace("<<SUBCOLLECTIONS>>", $subcollections, $minisucherEngine["formData"]);
            $minisucherEngine["formData"] = str_replace("<<COUNT>>", $count, $minisucherEngine["formData"]);
470
            $enabledSearchengines[]       = $minisucherEngine;
471 472 473 474
        }

        #die(var_dump($enabledSearchengines));

475
        if ($countSumas <= 0) {
476 477
            $this->errors[] = "Achtung: Sie haben in ihren Einstellungen keine Suchmaschine ausgewählt.";
        }
478
        $engines = [];
479

480
        $siteSearchFailed = false;
481
        if (strlen($this->site) > 0) {
482 483
            # Wenn eine Sitesearch durchgeführt werden soll, überprüfen wir ob eine der Suchmaschinen überhaupt eine Sitesearch unterstützt:
            $enginesWithSite = 0;
484 485
            foreach ($enabledSearchengines as $engine) {
                if (isset($engine['hasSiteSearch']) && $engine['hasSiteSearch']->__toString() === "1") {
486 487 488
                    $enginesWithSite++;
                }
            }
489 490
            if ($enginesWithSite === 0) {
                $this->errors[]   = "Sie wollten eine Sitesearch auf " . $this->site . " durchführen. Leider unterstützen die eingestellten Suchmaschinen diese nicht. Sie können <a href=\"" . $this->generateSearchLink("web", false) . "\">hier</a> die Sitesearch im Web-Fokus durchführen. Es werden ihnen Ergebnisse ohne Sitesearch angezeigt.";
491
                $siteSearchFailed = true;
492
            } else {
493 494 495 496 497 498
                $this->warnings[] = "Sie führen eine Sitesearch durch. Es werden nur Ergebnisse von der Seite: <a href=\"http://" . $this->site . "\" target=\"_blank\">\"" . $this->site . "\"</a> angezeigt.";
            }

        }

        $typeslist = [];
499
        $counter   = 0;
500

501 502
        if ($request->has('next') && Cache::has($request->input('next')) && unserialize(Cache::get($request->input('next')))['page'] > 1) {
            $next       = unserialize(Cache::get($request->input('next')));
503
            $this->page = $next['page'];
504 505
            $engines    = $next['engines'];
            if (isset($next['startForwards'])) {
506
                $this->startForwards = $next['startForwards'];
507 508 509
            }

            if (isset($next['startBackwards'])) {
510
                $this->startBackwards = $next['startBackwards'];
511 512
            }

513 514
        } else {
            foreach ($enabledSearchengines as $engine) {
515

516
                if (!$siteSearchFailed && strlen($this->site) > 0 && (!isset($engine['hasSiteSearch']) || $engine['hasSiteSearch']->__toString() === "0")) {
517

518 519 520 521
                    continue;
                }
                # Wenn diese Suchmaschine gar nicht eingeschaltet sein soll
                $path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString());
522

523 524 525 526
                if (!file_exists(app_path() . "/Models/parserSkripte/" . ucfirst($engine["package"]->__toString()) . ".php")) {
                    Log::error("Konnte " . $engine["name"] . " nicht abfragen, da kein Parser existiert");
                    continue;
                }
527

528
                $time = microtime();
529

530 531 532 533 534 535 536
                try
                {
                    $tmp = new $path($engine, $this);
                } catch (\ErrorException $e) {
                    Log::error("Konnte " . $engine["name"] . " nicht abfragen." . var_dump($e));
                    continue;
                }
537

538 539 540 541 542 543 544
                if ($tmp->enabled && isset($this->debug)) {
                    $this->warnings[] = $tmp->service . "   Connection_Time: " . $tmp->connection_time . "    Write_Time: " . $tmp->write_time . " Insgesamt:" . ((microtime() - $time) / 1000);
                }

                if ($tmp->isEnabled()) {
                    $engines[] = $tmp;
                }
545 546

            }
547
        }
548

549 550 551
        # Wir starten die Suche manuell:
        foreach ($engines as $engine) {
            $engine->startSearch($this);
552
        }
553

554 555
        # Jetzt werden noch alle Kategorien der Settings durchgegangen und die jeweils enthaltenen namen der Suchmaschinen gespeichert.
        $foki = [];
556 557 558
        foreach ($sumas as $suma) {
            if ((!isset($suma['disabled']) || $suma['disabled'] === "") && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
                if (isset($suma['type'])) {
559
                    $f = explode(",", $suma['type']->__toString());
560 561
                    foreach ($f as $tmp) {
                        $name                                    = $suma['name']->__toString();
562 563
                        $foki[$tmp][$suma['name']->__toString()] = $name;
                    }
564 565
                } else {
                    $name                                        = $suma['name']->__toString();
566 567 568 569 570 571 572
                    $foki["andere"][$suma['name']->__toString()] = $name;
                }
            }
        }

        # Es werden auch die Namen der aktuell aktiven Suchmaschinen abgespeichert.
        $realEngNames = [];
573
        foreach ($enabledSearchengines as $realEng) {
574
            $nam = $realEng["name"]->__toString();
575
            if ($nam !== "qualigo" && $nam !== "overtureAds") {
576 577 578 579
                $realEngNames[] = $nam;
            }
        }
        # Anschließend werden diese beiden Listen verglichen (jeweils eine der Fokuslisten für jeden Fokus), um herauszufinden ob sie vielleicht identisch sind. Ist dies der Fall, so hat der Nutzer anscheinend Suchmaschinen eines kompletten Fokus eingestellt. Der Fokus wird dementsprechend angepasst.
580 581
        foreach ($foki as $fok => $engs) {
            $isFokus      = true;
582
            $fokiEngNames = [];
583
            foreach ($engs as $eng) {
584 585
                $fokiEngNames[] = $eng;
            }
586 587
            foreach ($fokiEngNames as $fen) {
                if (!in_array($fen, $realEngNames)) {
588 589 590
                    $isFokus = false;
                }
            }
591 592
            foreach ($realEngNames as $ren) {
                if (!in_array($ren, $fokiEngNames)) {
593 594 595
                    $isFokus = false;
                }
            }
596
            if ($isFokus) {
597 598 599 600 601 602 603 604 605
                $this->fokus = $fok;
            }
        }

        # Nun passiert ein elementarer Schritt.
        # Wir warten auf die Antwort der Suchmaschinen, da wir vorher nicht weiter machen können.
        # aber natürlich nicht ewig.
        # Die Verbindung steht zu diesem Zeitpunkt und auch unsere Request wurde schon gesendet.
        # Wir geben der Suchmaschine nun bis zu 500ms Zeit zu antworten.
606 607 608

        # Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
        $enginesToLoad = 0;
609 610 611
        $canBreak      = false;
        foreach ($engines as $engine) {
            if ($engine->cached) {
612
                $enginesToLoad--;
613
                if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
614
                    $canBreak = true;
615 616
                }

617 618 619
            }
        }
        $enginesToLoad += count($engines);
620
        $loadedEngines = 0;
621 622 623
        $timeStart     = microtime(true);
        while (true) {
            $time          = (microtime(true) - $timeStart) * 1000;
624
            $loadedEngines = intval(Redis::hlen('search.' . $this->getHashCode()));
625
            if ($overtureEnabled && (Redis::hexists('search.' . $this->getHashCode(), 'overture') || Redis::hexists('search.' . $this->getHashCode(), 'overtureAds'))) {
626
                $canBreak = true;
627
            }
628 629

            # Abbruchbedingung
630 631
            if ($time < 500) {
                if (($enginesToLoad === 0 || $loadedEngines >= $enginesToLoad) && $canBreak) {
632
                    break;
633 634 635 636
                }

            } elseif ($time >= 500 && $time < $this->time) {
                if (($enginesToLoad === 0 || ($loadedEngines / ($enginesToLoad * 1.0)) >= 0.8) && $canBreak) {
637
                    break;
638 639 640
                }

            } else {
641 642 643 644 645
                break;
            }
            usleep(50000);
        }

646 647 648
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
                try {
649
                    $engine->retrieveResults($this);
650
                } catch (\ErrorException $e) {
651
                    Log::error($e);
652

653 654 655
                }
            }
        }
656

657
        # und verwerfen den Rest:
658 659
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
660
                $engine->shutdown();
661 662
            }

663 664 665
        }

        $this->engines = $engines;
666 667 668 669 670 671 672 673 674 675 676 677
    }

    public function parseFormData(Request $request)
    {
        if ($request->input('encoding', '') !== "utf8") {
            # In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
            $input = $request->all();
            foreach ($input as $key => $value) {
                $input[$key] = mb_convert_encoding("$value", "UTF-8", "ISO-8859-1");
            }
            $request->replace($input);
        }
678
        $this->url = $request->url();
679
        # Zunächst überprüfen wir die eingegebenen Einstellungen:
680 681
        # FOKUS
        $this->fokus = trans('fokiNames.'
682 683
            . $request->input('focus', 'web'));
        if (strpos($this->fokus, ".")) {
684 685 686 687
            $this->fokus = trans('fokiNames.web');
        }

        # SUMA-FILE
688
        if (App::isLocale("en")) {
689
            $this->sumaFile = config_path() . "/sumas.xml";
690
        } else {
691 692
            $this->sumaFile = config_path() . "/sumas.xml";
        }
693
        if (!file_exists($this->sumaFile)) {
694 695 696 697 698
            die("Suma-File konnte nicht gefunden werden");
        }

        # Sucheingabe:
        $this->eingabe = trim($request->input('eingabe', ''));
699
        if (strlen($this->eingabe) === 0) {
700 701 702 703 704 705 706 707
            $this->warnings[] = 'Achtung: Sie haben keinen Suchbegriff eingegeben. Sie können ihre Suchbegriffe oben eingeben und es erneut versuchen.';
        }
        $this->q = $this->eingabe;

        # IP:
        $this->ip = $request->ip();

        # Language:
708
        if (isset($_SERVER['HTTP_LANGUAGE'])) {
709
            $this->language = $_SERVER['HTTP_LANGUAGE'];
710
        } else {
711 712 713 714 715 716
            $this->language = "";
        }
        # Category
        $this->category = $request->input('category', '');
        # Request Times:
        $this->time = $request->input('time', 1000);
717

718
        # Page
719
        $this->page = 1;
720 721
        # Lang
        $this->lang = $request->input('lang', 'all');
722 723
        if ($this->lang !== "de" && $this->lang !== "en" && $this->lang !== "all") {
            $this->lang = "all";
724
        }
725
        $this->agent  = new Agent();
726 727 728 729
        $this->mobile = $this->agent->isMobile();

        #Sprüche
        $this->sprueche = $request->input('sprueche', 'off');
730
        if ($this->sprueche === "off") {
731
            $this->sprueche = true;
732
        } else {
733
            $this->sprueche = false;
734 735
        }

736 737 738 739
        # Ergebnisse pro Seite:
        $this->resultCount = $request->input('resultCount', '20');

        # Manchmal müssen wir Parameter anpassen um den Sucheinstellungen gerecht zu werden:
740 741 742
        if ($request->has('dart')) {
            $this->time       = 10000;
            $this->warnings[] = "Hinweis: Sie haben Dart-Europe aktiviert. Die Suche kann deshalb länger dauern und die maximale Suchzeit wurde auf 10 Sekunden hochgesetzt.";
743
        }
744 745
        if ($this->time <= 500 || $this->time > 20000) {
            $this->time = 1000;
746
        }
747 748 749 750 751 752 753 754 755
        if ($request->has('minism') && ($request->has('fportal') || $request->has('harvest'))) {
            $input    = $request->all();
            $newInput = [];
            foreach ($input as $key => $value) {
                if ($key !== "fportal" && $key !== "harvest") {
                    $newInput[$key] = $value;
                }
            }
            $request->replace($newInput);
756
        }
757 758
        if (App::isLocale("en")) {
            $this->sprueche = "off";
759
        }
760 761
        if ($this->resultCount <= 0 || $this->resultCount > 200) {
            $this->resultCount = 1000;
762
        }
763 764 765 766 767 768
        if ($request->has('onenewspageAll') || $request->has('onenewspageGermanyAll')) {
            $this->time  = 5000;
            $this->cache = "cache";
        }
        if ($request->has('tab')) {
            if ($request->input('tab') === "off") {
769
                $this->tab = "_blank";
770
            } else {
771 772
                $this->tab = "_self";
            }
773
        } else {
774 775
            $this->tab = "_blank";
        }
776
        if ($request->has('password')) {
777
            $this->password = $request->input('password');
778 779 780
        }

        if ($request->has('quicktips')) {
781
            $this->quicktips = false;
782
        } else {
783
            $this->quicktips = true;
784
        }
785 786

        $this->out = $request->input('out', "html");
787
        if ($this->out !== "html" && $this->out !== "json" && $this->out !== "results" && $this->out !== "results-with-style") {
788
            $this->out = "html";
789 790
        }

791
        $this->request = $request;
792 793 794 795 796 797 798 799 800 801
    }

    public function checkSpecialSearches(Request $request)
    {
        # Site Search:
        if (preg_match("/(.*)\bsite:(\S+)(.*)/si", $this->q, $match)) {
            $this->site = $match[2];
            $this->q    = $match[1] . $match[3];
        }
        if ($request->has('site')) {
802 803
            $this->site = $request->input('site');
        }
804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849
        # Wenn die Suchanfrage um das Schlüsselwort "-host:*" ergänzt ist, sollen bestimmte Hosts nicht eingeblendet werden
        # Wir prüfen, ob das hier der Fall ist:
        while (preg_match("/(.*)(^|\s)-host:(\S+)(.*)/si", $this->q, $match)) {
            $this->hostBlacklist[] = $match[3];
            $this->q               = $match[1] . $match[4];
        }
        if (sizeof($this->hostBlacklist) > 0) {
            $hostString = "";
            foreach ($this->hostBlacklist as $host) {
                $hostString .= $host . ", ";
            }
            $hostString       = rtrim($hostString, ", ");
            $this->warnings[] = "Ergebnisse von folgenden Hosts werden nicht angezeigt: \"" . $hostString . "\"";
        }
        # Wenn die Suchanfrage um das Schlüsselwort "-domain:*" ergänzt ist, sollen bestimmte Domains nicht eingeblendet werden
        # Wir prüfen, ob das hier der Fall ist:
        while (preg_match("/(.*)(^|\s)-domain:(\S+)(.*)/si", $this->q, $match)) {
            $this->domainBlacklist[] = $match[3];
            $this->q                 = $match[1] . $match[4];
        }
        if (sizeof($this->domainBlacklist) > 0) {
            $domainString = "";
            foreach ($this->domainBlacklist as $domain) {
                $domainString .= $domain . ", ";
            }
            $domainString     = rtrim($domainString, ", ");
            $this->warnings[] = "Ergebnisse von folgenden Domains werden nicht angezeigt: \"" . $domainString . "\"";
        }

        # Alle mit "-" gepräfixten Worte sollen aus der Suche ausgeschlossen werden.
        # Wir prüfen, ob das hier der Fall ist:
        while (preg_match("/(.*)(^|\s)-(\S+)(.*)/si", $this->q, $match)) {
            $this->stopWords[] = $match[3];
            $this->q           = $match[1] . $match[4];
        }
        if (sizeof($this->stopWords) > 0) {
            $stopwordsString = "";
            foreach ($this->stopWords as $stopword) {
                $stopwordsString .= $stopword . ", ";
            }
            $stopwordsString  = rtrim($stopwordsString, ", ");
            $this->warnings[] = "Sie machen eine Ausschlusssuche. Ergebnisse mit folgenden Wörtern werden nicht angezeigt: \"" . $stopwordsString . "\"";
        }

        # Meldung über eine Phrasensuche
        $p   = "";
850
        $tmp = $this->q;
851 852
        while (preg_match("/(.*)\"(.+)\"(.*)/si", $tmp, $match)) {
            $tmp             = $match[1] . $match[3];
853
            $this->phrases[] = strtolower($match[2]);
854 855
        }
        foreach ($this->phrases as $phrase) {
856 857 858
            $p .= "\"$phrase\", ";
        }
        $p = rtrim($p, ", ");
859
        if (sizeof($this->phrases) > 0) {
860
            $this->warnings[] = "Sie führen eine Phrasensuche durch: $p";
861
        }
862

863 864 865
    }

    public function getFokus()
866 867 868 869
    {
        return $this->fokus;
    }

870
    public function getIp()
871 872 873 874
    {
        return $this->ip;
    }

875
    public function getEingabe()
876 877 878 879
    {
        return $this->eingabe;
    }

880
    public function getQ()
881
    {
882
        return $this->q;
883 884
    }

885
    public function getUrl()
886 887 888
    {
        return $this->url;
    }
889
    public function getTime()
890 891 892 893
    {
        return $this->time;
    }

894
    public function getLanguage()
895 896 897 898
    {
        return $this->language;
    }

899
    public function getLang()
900 901 902 903
    {
        return $this->lang;
    }

904
    public function getSprueche()
905 906 907 908
    {
        return $this->sprueche;
    }

909
    public function getCategory()
910 911 912 913
    {
        return $this->category;
    }

914
    public function getPhrases()
915 916 917
    {
        return $this->phrases;
    }
918 919 920 921
    public function getPage()
    {
        return $this->page;
    }
922

923
    public function getSumaFile()
924 925 926 927
    {
        return $this->sumaFile;
    }

928
    public function getUserHostBlacklist()
929 930 931 932
    {
        return $this->hostBlacklist;
    }

933
    public function getUserDomainBlacklist()
934 935 936 937