MetaGer.php 37.9 KB
Newer Older
1 2 3 4
<?php
namespace App;

use App;
5
use App\lib\TextLanguageDetect\TextLanguageDetect;
6
use Cache;
7 8
use Illuminate\Http\Request;
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix  
Dominik Hebeler committed
9
use LaravelLocalization;
10 11
use Log;
use Redis;
12 13 14

class MetaGer
{
15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
    # Einstellungen für die Suche
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $category;
    protected $time;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
    protected $hostBlacklist   = [];
    protected $domainBlacklist = [];
    protected $stopWords       = [];
    protected $phrases         = [];
    protected $engines         = [];
    protected $results         = [];
    protected $ads             = [];
    protected $warnings        = [];
    protected $errors          = [];
    protected $addedHosts      = [];
35
    protected $startCount = 0;
36 37 38 39 40 41 42 43 44
    # Daten über die Abfrage
    protected $ip;
    protected $language;
    protected $agent;
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
45
    protected $domainsBlacklisted = [];
46
    protected $urlsBlacklisted    = [];
47 48 49
    protected $url;
    protected $languageDetect;

50 51 52 53
    public function __construct()
    {
        $this->starttime = microtime(true);
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
54
            # Blacklists einlesen:
55
            $tmp                      = file_get_contents(config_path() . "/blacklistDomains.txt");
56
            $this->domainsBlacklisted = explode("\n", $tmp);
57 58 59
            $tmp                      = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted    = explode("\n", $tmp);
        } else {
60 61 62
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
        }

63 64 65 66 67 68 69 70
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
                require $path;
            }
        }

71 72
        $this->languageDetect = new TextLanguageDetect();
        $this->languageDetect->setNameMode("2");
73
    }
74

75
    public function getHashCode()
76 77 78 79 80
    {
        $string = url()->full();
        return md5($string);
    }

81
    public function rankAll()
82
    {
83
        foreach ($this->engines as $engine) {
84 85 86
            $engine->rank($this);
        }
    }
87

88 89 90
    public function createView()
    {
        $viewResults = [];
91

92
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
93
        foreach ($this->results as $result) {
94 95 96 97 98 99
            $viewResults[] = get_object_vars($result);
        }

        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();

100 101
        if ($this->fokus === "bilder") {
            switch ($this->out) {
102 103 104 105 106 107 108 109
                case 'results':
                    return view('metager3bilderresults')
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
110
                        ->with('browser', (new Agent())->browser());
111 112 113 114 115 116 117 118
                default:
                    return view('metager3bilder')
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
119
                        ->with('browser', (new Agent())->browser());
120 121 122 123 124 125 126 127 128 129 130 131
            }
        }

        switch ($this->out) {
            case 'results':
                return view('metager3results')
                    ->with('results', $viewResults)
                    ->with('eingabe', $this->eingabe)
                    ->with('mobile', $this->mobile)
                    ->with('warnings', $this->warnings)
                    ->with('errors', $this->errors)
                    ->with('metager', $this)
132
                    ->with('browser', (new Agent())->browser());
133 134 135 136 137 138 139 140 141 142
                break;
            case 'results-with-style':
                return view('metager3')
                    ->with('results', $viewResults)
                    ->with('eingabe', $this->eingabe)
                    ->with('mobile', $this->mobile)
                    ->with('warnings', $this->warnings)
                    ->with('errors', $this->errors)
                    ->with('metager', $this)
                    ->with('suspendheader', "yes")
143
                    ->with('browser', (new Agent())->browser());
144 145 146 147 148 149 150 151
                break;
            default:
                return view('metager3')
                    ->with('eingabe', $this->eingabe)
                    ->with('mobile', $this->mobile)
                    ->with('warnings', $this->warnings)
                    ->with('errors', $this->errors)
                    ->with('metager', $this)
152
                    ->with('browser', (new Agent())->browser());
153 154
                break;
        }
155
    }
156

157 158 159 160 161 162
    private function createLogs()
    {
        $redis = Redis::connection('redisLogs');
        try
        {
            $logEntry = "";
163
            $logEntry .= "[" . date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y"))) . "]";
164 165 166 167 168 169
            $logEntry .= " pid=" . getmypid();
            $logEntry .= " ref=" . $this->request->header('Referer');
            $useragent = $this->request->header('User-Agent');
            $useragent = str_replace("(", " ", $useragent);
            $useragent = str_replace(")", " ", $useragent);
            $useragent = str_replace(" ", "", $useragent);
170
            $logEntry .= " time=" . round((microtime(true) - $this->starttime), 2) . " serv=" . $this->fokus;
171 172
            $logEntry .= " search=" . $this->eingabe;
            $redis->rpush('logs.search', $logEntry);
173
        } catch (\Exception $e) {
174 175 176 177
            return;
        }
    }

178
    public function removeInvalids()
179 180
    {
        $results = [];
181 182
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
183
                $results[] = $result;
184 185
            }

186 187 188
        }
        #$this->results = $results;
    }
189

190 191 192
    public function combineResults()
    {
        foreach ($this->engines as $engine) {
193 194 195 196 197 198
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
199 200
            foreach ($engine->results as $result) {
                if ($result->valid) {
201
                    $this->results[] = $result;
202
                }
203
            }
204
            foreach ($engine->ads as $ad) {
205 206
                $this->ads[] = $ad;
            }
207
        }
208

209 210
        uasort($this->results, function ($a, $b) {
            if ($a->getRank() == $b->getRank()) {
211
                return 0;
212 213
            }

214 215 216 217
            return ($a->getRank() < $b->getRank()) ? 1 : -1;
        });
        # Validate Results
        $newResults = [];
218 219
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
220
                $newResults[] = $result;
221 222
            }

223 224 225
        }
        $this->results = $newResults;

226
        $counter   = 0;
227
        $firstRank = 0;
228 229 230 231 232 233 234 235 236 237

        if(isset($this->startForwards))
        {
            $this->startCount = $this->startForwards;
        }elseif (isset($this->startBackwards)) {
            $this->startCount = $this->startBackwards - count($this->results) - 1;
        }else{
            $this->startCount = 0;
        }

238 239
        foreach ($this->results as $result) {
            if ($counter === 0) {
240
                $firstRank = $result->rank;
241 242
            }

243
            $counter++;
244
            $result->number = $counter + $this->startCount;
245 246 247 248
            $confidence     = 0;
            if ($firstRank > 0) {
                $confidence = $result->rank / $firstRank;
            } else {
249
                $confidence = 0;
250 251 252
            }

            if ($confidence > 0.65) {
253
                $result->color = "#FF4000";
254
            } elseif ($confidence > 0.4) {
255
                $result->color = "#FF0080";
256
            } elseif ($confidence > 0.2) {
257
                $result->color = "#C000C0";
258
            } else {
259
                $result->color = "#000000";
260 261
            }

262 263
        }

264
        if (LaravelLocalization::getCurrentLocale() === "en") {
265 266 267
            $this->ads = [];
        }

268
        $this->validated = false;
269
        if (isset($this->password)) {
270 271 272
            # Wir bieten einen bezahlten API-Zugriff an, bei dem dementsprechend die Werbung ausgeblendet wurde:
            # Aktuell ist es nur die Uni-Mainz. Deshalb überprüfen wir auch nur diese.
            $password = getenv('mainz');
273
            $eingabe  = $this->eingabe;
274
            $password = md5($eingabe . $password);
275 276
            if ($this->password === $password) {
                $this->ads       = [];
277 278 279
                $this->validated = true;
            }
        }
280 281

        if (count($this->results) <= 0) {
Dominik Hebeler's avatar
Dominik Hebeler committed
282
            $this->errors[] = "Leider konnten wir zu Ihrer Sucheingabe keine passenden Ergebnisse finden.";
283
        }
284

285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306
        if( isset($this->last) && count($this->last) > 0 )
        {
            $page = $this->page - 1;
            $this->last = [
                'page'  =>  $page,
                'startBackwards'     =>  $this->results[0]->number,
                'engines' => $this->last,
            ];
            Cache::put(md5(serialize($this->last)), serialize($this->last), 60);
        }

        if( isset($this->next) && count($this->next) > 0 && count($this->results) > 0)
        {
            $page = $this->page + 1;
            $this->next = [
                'page'  =>  $page,
                'startForwards'     =>  $this->results[count($this->results)-1]->number,
                'engines' => $this->next,
            ];
            Cache::put(md5(serialize($this->next)), serialize($this->next), 60);
        }

307
    }
308

309 310
    public function parseBoost($results)
    {
311 312 313 314 315 316 317 318 319 320 321 322
        foreach ($results as $result) {
            if (preg_match('/^(http[s]?\:\/\/)?(www.)?amazon\.de/', $result->anzeigeLink)) {
                if (preg_match('/\?/', $result->anzeigeLink)) {
                    $result->link .= '&tag=boostmg01-21';
                } else {
                    $result->link .= '?tag=boostmg01-21';
                }
                $result->partnershop = true;

            }
        }
        return $results;
323 324 325
    }
    public function parseAdgoal($results)
    {
326
        $publicKey  = getenv('adgoal_public');
327
        $privateKey = getenv('adgoal_private');
328
        if ($publicKey === false) {
329 330 331
            return $results;
        }
        $tldList = "";
332 333
        try {
            foreach ($results as $result) {
334
                $link = $result->anzeigeLink;
335
                if (strpos($link, "http") !== 0) {
336 337 338 339 340 341 342 343 344 345
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

346
            # Query
347 348
            $query = urlencode($this->q);

349
            $link   = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . $publicKey . "&k=" . $hash . "&tld=" . $tldList . "&q=" . $query;
350 351 352
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
353
            foreach ($answer as $el) {
354
                $hoster = $el[0];
355
                $hash   = $el[1];
356

357 358
                foreach ($results as $result) {
                    if ($hoster === $result->tld) {
359 360
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
361
                        if ($result->image !== "") {
362
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . $hash . ".gif";
363
                        } else {
364
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . $hash . ".gif";
365 366
                        }

367 368 369
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
                        $targetUrl = $result->anzeigeLink;
370
                        if (strpos($targetUrl, "http") !== 0) {
371
                            $targetUrl = "http://" . $targetUrl;
372 373 374 375 376
                        }

                        $gateHash            = md5($targetUrl . $privateKey);
                        $newLink             = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . $publicKey . "&k=" . $gateHash . "&url=" . urlencode($targetUrl) . "&q=" . $query;
                        $result->link        = $newLink;
377 378 379 380
                        $result->partnershop = true;
                    }
                }
            }
381
        } catch (\ErrorException $e) {
382 383 384 385 386
            return $results;
        }

        return $results;
    }
387

388 389
    public function createSearchEngines(Request $request)
    {
390

391
        if (!$request->has("eingabe")) {
392
            return;
393
        }
394

395 396
        # Überprüfe, welche Sumas eingeschaltet sind
        $xml                  = simplexml_load_file($this->sumaFile);
397
        $enabledSearchengines = [];
398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414
        $overtureEnabled      = false;
        $countSumas           = 0;
        $sumas                = $xml->xpath("suma");
        if ($this->fokus === "angepasst") {
            foreach ($sumas as $suma) {
                if ($request->has($suma["name"])
                    || ($this->fokus !== "bilder"
                        && ($suma["name"]->__toString() === "qualigo"
                            || $suma["name"]->__toString() === "similar_product_ads"
                            || (!$overtureEnabled && $suma["name"]->__toString() === "overtureAds")
                        )
                    )
                ) {

                    if (!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1")) {
                        if ($suma["name"]->__toString() === "overture" || $suma["name"]->__toString() === "overtureAds") {
                            $overtureEnabled = true;
415
                        }
416
                        if ($suma["name"]->__toString() !== "qualigo" && $suma["name"]->__toString() !== "similar_product_ads" && $suma["name"]->__toString() !== "overtureAds") {
417
                            $countSumas += 1;
418 419
                        }

420 421 422 423
                        $enabledSearchengines[] = $suma;
                    }
                }
            }
424 425 426 427 428 429 430 431 432 433 434 435 436 437
        } else {
            foreach ($sumas as $suma) {
                $types = explode(",", $suma["type"]);
                if (in_array($this->fokus, $types)
                    || ($this->fokus !== "bilder"
                        && ($suma["name"]->__toString() === "qualigo"
                            || $suma["name"]->__toString() === "similar_product_ads"
                            || (!$overtureEnabled && $suma["name"]->__toString() === "overtureAds")
                        )
                    )
                ) {
                    if (!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1")) {
                        if ($suma["name"]->__toString() === "overture" || $suma["name"]->__toString() === "overtureAds") {
                            $overtureEnabled = true;
438
                        }
439
                        if ($suma["name"]->__toString() !== "qualigo" && $suma["name"]->__toString() !== "similar_product_ads" && $suma["name"]->__toString() !== "overtureAds") {
440
                            $countSumas += 1;
441 442
                        }

443 444 445 446 447 448 449 450
                        $enabledSearchengines[] = $suma;
                    }
                }
            }
        }

        # Sonderregelung für alle Suchmaschinen, die zu den Minisuchern gehören. Diese können alle gemeinsam über einen Link abgefragt werden
        $subcollections = [];
451 452 453
        $tmp            = [];
        foreach ($enabledSearchengines as $engine) {
            if (isset($engine['minismCollection'])) {
454
                $subcollections[] = $engine['minismCollection']->__toString();
455
            } else {
456
                $tmp[] = $engine;
457 458
            }

459 460
        }
        $enabledSearchengines = $tmp;
461 462 463 464
        if (sizeof($subcollections) > 0) {
            $count                        = sizeof($subcollections) * 10;
            $minisucherEngine             = $xml->xpath('suma[@name="minism"]')[0];
            $subcollections               = urlencode("(" . implode(" OR ", $subcollections) . ")");
465 466
            $minisucherEngine["formData"] = str_replace("<<SUBCOLLECTIONS>>", $subcollections, $minisucherEngine["formData"]);
            $minisucherEngine["formData"] = str_replace("<<COUNT>>", $count, $minisucherEngine["formData"]);
467
            $enabledSearchengines[]       = $minisucherEngine;
468 469 470 471
        }

        #die(var_dump($enabledSearchengines));

472
        if ($countSumas <= 0) {
473 474
            $this->errors[] = "Achtung: Sie haben in ihren Einstellungen keine Suchmaschine ausgewählt.";
        }
475
        $engines = [];
476

477
        $siteSearchFailed = false;
478
        if (strlen($this->site) > 0) {
479 480
            # Wenn eine Sitesearch durchgeführt werden soll, überprüfen wir ob eine der Suchmaschinen überhaupt eine Sitesearch unterstützt:
            $enginesWithSite = 0;
481 482
            foreach ($enabledSearchengines as $engine) {
                if (isset($engine['hasSiteSearch']) && $engine['hasSiteSearch']->__toString() === "1") {
483 484 485
                    $enginesWithSite++;
                }
            }
486 487
            if ($enginesWithSite === 0) {
                $this->errors[]   = "Sie wollten eine Sitesearch auf " . $this->site . " durchführen. Leider unterstützen die eingestellten Suchmaschinen diese nicht. Sie können <a href=\"" . $this->generateSearchLink("web", false) . "\">hier</a> die Sitesearch im Web-Fokus durchführen. Es werden ihnen Ergebnisse ohne Sitesearch angezeigt.";
488
                $siteSearchFailed = true;
489
            } else {
490 491 492 493 494 495
                $this->warnings[] = "Sie führen eine Sitesearch durch. Es werden nur Ergebnisse von der Seite: <a href=\"http://" . $this->site . "\" target=\"_blank\">\"" . $this->site . "\"</a> angezeigt.";
            }

        }

        $typeslist = [];
496
        $counter   = 0;
497

498 499 500 501 502 503 504 505 506 507
        if ($request->has('next') && Cache::has($request->input('next')) && unserialize(Cache::get($request->input('next')))['page'] > 1 ) {
            $next = unserialize(Cache::get($request->input('next')));
            $this->page = $next['page'];
            $engines = $next['engines'];
            if(isset($next['startForwards']))
                $this->startForwards = $next['startForwards'];
            if(isset($next['startBackwards']))
                $this->startBackwards = $next['startBackwards'];
        } else {
            foreach ($enabledSearchengines as $engine) {
508

509
                if (!$siteSearchFailed && strlen($this->site) > 0 && (!isset($engine['hasSiteSearch']) || $engine['hasSiteSearch']->__toString() === "0")) {
510

511 512 513 514
                    continue;
                }
                # Wenn diese Suchmaschine gar nicht eingeschaltet sein soll
                $path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString());
515

516 517 518 519
                if (!file_exists(app_path() . "/Models/parserSkripte/" . ucfirst($engine["package"]->__toString()) . ".php")) {
                    Log::error("Konnte " . $engine["name"] . " nicht abfragen, da kein Parser existiert");
                    continue;
                }
520

521
                $time = microtime();
522

523 524 525 526 527 528 529
                try
                {
                    $tmp = new $path($engine, $this);
                } catch (\ErrorException $e) {
                    Log::error("Konnte " . $engine["name"] . " nicht abfragen." . var_dump($e));
                    continue;
                }
530

531 532 533 534 535 536 537
                if ($tmp->enabled && isset($this->debug)) {
                    $this->warnings[] = $tmp->service . "   Connection_Time: " . $tmp->connection_time . "    Write_Time: " . $tmp->write_time . " Insgesamt:" . ((microtime() - $time) / 1000);
                }

                if ($tmp->isEnabled()) {
                    $engines[] = $tmp;
                }
538 539

            }
540
        }
541

542 543 544
        # Wir starten die Suche manuell:
        foreach ($engines as $engine) {
            $engine->startSearch($this);
545
        }
546

547 548
        # Jetzt werden noch alle Kategorien der Settings durchgegangen und die jeweils enthaltenen namen der Suchmaschinen gespeichert.
        $foki = [];
549 550 551
        foreach ($sumas as $suma) {
            if ((!isset($suma['disabled']) || $suma['disabled'] === "") && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
                if (isset($suma['type'])) {
552
                    $f = explode(",", $suma['type']->__toString());
553 554
                    foreach ($f as $tmp) {
                        $name                                    = $suma['name']->__toString();
555 556
                        $foki[$tmp][$suma['name']->__toString()] = $name;
                    }
557 558
                } else {
                    $name                                        = $suma['name']->__toString();
559 560 561 562 563 564 565
                    $foki["andere"][$suma['name']->__toString()] = $name;
                }
            }
        }

        # Es werden auch die Namen der aktuell aktiven Suchmaschinen abgespeichert.
        $realEngNames = [];
566
        foreach ($enabledSearchengines as $realEng) {
567
            $nam = $realEng["name"]->__toString();
568
            if ($nam !== "qualigo" && $nam !== "overtureAds") {
569 570 571 572
                $realEngNames[] = $nam;
            }
        }
        # Anschließend werden diese beiden Listen verglichen (jeweils eine der Fokuslisten für jeden Fokus), um herauszufinden ob sie vielleicht identisch sind. Ist dies der Fall, so hat der Nutzer anscheinend Suchmaschinen eines kompletten Fokus eingestellt. Der Fokus wird dementsprechend angepasst.
573 574
        foreach ($foki as $fok => $engs) {
            $isFokus      = true;
575
            $fokiEngNames = [];
576
            foreach ($engs as $eng) {
577 578
                $fokiEngNames[] = $eng;
            }
579 580
            foreach ($fokiEngNames as $fen) {
                if (!in_array($fen, $realEngNames)) {
581 582 583
                    $isFokus = false;
                }
            }
584 585
            foreach ($realEngNames as $ren) {
                if (!in_array($ren, $fokiEngNames)) {
586 587 588
                    $isFokus = false;
                }
            }
589
            if ($isFokus) {
590 591 592 593 594 595 596 597 598
                $this->fokus = $fok;
            }
        }

        # Nun passiert ein elementarer Schritt.
        # Wir warten auf die Antwort der Suchmaschinen, da wir vorher nicht weiter machen können.
        # aber natürlich nicht ewig.
        # Die Verbindung steht zu diesem Zeitpunkt und auch unsere Request wurde schon gesendet.
        # Wir geben der Suchmaschine nun bis zu 500ms Zeit zu antworten.
599 600 601

        # Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
        $enginesToLoad = 0;
602 603 604
        $canBreak      = false;
        foreach ($engines as $engine) {
            if ($engine->cached) {
605
                $enginesToLoad--;
606
                if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
607
                    $canBreak = true;
608 609
                }

610 611 612
            }
        }
        $enginesToLoad += count($engines);
613
        $loadedEngines = 0;
614 615 616
        $timeStart     = microtime(true);
        while (true) {
            $time          = (microtime(true) - $timeStart) * 1000;
617
            $loadedEngines = intval(Redis::hlen('search.' . $this->getHashCode()));
618
            if ($overtureEnabled && (Redis::hexists('search.' . $this->getHashCode(), 'overture') || Redis::hexists('search.' . $this->getHashCode(), 'overtureAds'))) {
619
                $canBreak = true;
620
            }
621 622

            # Abbruchbedingung
623 624
            if ($time < 500) {
                if (($enginesToLoad === 0 || $loadedEngines >= $enginesToLoad) && $canBreak) {
625
                    break;
626 627 628 629
                }

            } elseif ($time >= 500 && $time < $this->time) {
                if (($enginesToLoad === 0 || ($loadedEngines / ($enginesToLoad * 1.0)) >= 0.8) && $canBreak) {
630
                    break;
631 632 633
                }

            } else {
634 635 636 637 638
                break;
            }
            usleep(50000);
        }

639 640 641
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
                try {
642
                    $engine->retrieveResults($this);
643
                } catch (\ErrorException $e) {
644
                    Log::error($e);
645

646 647 648
                }
            }
        }
649

650
        # und verwerfen den Rest:
651 652
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
653
                $engine->shutdown();
654 655
            }

656 657 658
        }

        $this->engines = $engines;
659 660 661 662 663 664 665 666 667 668 669 670
    }

    public function parseFormData(Request $request)
    {
        if ($request->input('encoding', '') !== "utf8") {
            # In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
            $input = $request->all();
            foreach ($input as $key => $value) {
                $input[$key] = mb_convert_encoding("$value", "UTF-8", "ISO-8859-1");
            }
            $request->replace($input);
        }
671
        $this->url = $request->url();
672
        # Zunächst überprüfen wir die eingegebenen Einstellungen:
673 674
        # FOKUS
        $this->fokus = trans('fokiNames.'
675 676
            . $request->input('focus', 'web'));
        if (strpos($this->fokus, ".")) {
677 678 679 680
            $this->fokus = trans('fokiNames.web');
        }

        # SUMA-FILE
681
        if (App::isLocale("en")) {
682
            $this->sumaFile = config_path() . "/sumas.xml";
683
        } else {
684 685
            $this->sumaFile = config_path() . "/sumas.xml";
        }
686
        if (!file_exists($this->sumaFile)) {
687 688 689 690 691
            die("Suma-File konnte nicht gefunden werden");
        }

        # Sucheingabe:
        $this->eingabe = trim($request->input('eingabe', ''));
692
        if (strlen($this->eingabe) === 0) {
693 694 695 696 697 698 699 700
            $this->warnings[] = 'Achtung: Sie haben keinen Suchbegriff eingegeben. Sie können ihre Suchbegriffe oben eingeben und es erneut versuchen.';
        }
        $this->q = $this->eingabe;

        # IP:
        $this->ip = $request->ip();

        # Language:
701
        if (isset($_SERVER['HTTP_LANGUAGE'])) {
702
            $this->language = $_SERVER['HTTP_LANGUAGE'];
703
        } else {
704 705 706 707 708 709
            $this->language = "";
        }
        # Category
        $this->category = $request->input('category', '');
        # Request Times:
        $this->time = $request->input('time', 1000);
710

711
        # Page
712
        $this->page = 1;
713 714
        # Lang
        $this->lang = $request->input('lang', 'all');
715 716
        if ($this->lang !== "de" && $this->lang !== "en" && $this->lang !== "all") {
            $this->lang = "all";
717
        }
718
        $this->agent  = new Agent();
719 720 721 722
        $this->mobile = $this->agent->isMobile();

        #Sprüche
        $this->sprueche = $request->input('sprueche', 'off');
723
        if ($this->sprueche === "off") {
724
            $this->sprueche = true;
725
        } else {
726
            $this->sprueche = false;
727 728
        }

729 730 731 732
        # Ergebnisse pro Seite:
        $this->resultCount = $request->input('resultCount', '20');

        # Manchmal müssen wir Parameter anpassen um den Sucheinstellungen gerecht zu werden:
733 734 735
        if ($request->has('dart')) {
            $this->time       = 10000;
            $this->warnings[] = "Hinweis: Sie haben Dart-Europe aktiviert. Die Suche kann deshalb länger dauern und die maximale Suchzeit wurde auf 10 Sekunden hochgesetzt.";
736
        }
737 738
        if ($this->time <= 500 || $this->time > 20000) {
            $this->time = 1000;
739
        }
740 741 742 743 744 745 746 747 748
        if ($request->has('minism') && ($request->has('fportal') || $request->has('harvest'))) {
            $input    = $request->all();
            $newInput = [];
            foreach ($input as $key => $value) {
                if ($key !== "fportal" && $key !== "harvest") {
                    $newInput[$key] = $value;
                }
            }
            $request->replace($newInput);
749
        }
750 751
        if (App::isLocale("en")) {
            $this->sprueche = "off";
752
        }
753 754
        if ($this->resultCount <= 0 || $this->resultCount > 200) {
            $this->resultCount = 1000;
755
        }
756 757 758 759 760 761
        if ($request->has('onenewspageAll') || $request->has('onenewspageGermanyAll')) {
            $this->time  = 5000;
            $this->cache = "cache";
        }
        if ($request->has('tab')) {
            if ($request->input('tab') === "off") {
762
                $this->tab = "_blank";
763
            } else {
764 765
                $this->tab = "_self";
            }
766
        } else {
767 768
            $this->tab = "_blank";
        }
769
        if ($request->has('password')) {
770
            $this->password = $request->input('password');
771 772 773
        }

        if ($request->has('quicktips')) {
774
            $this->quicktips = false;
775
        } else {
776
            $this->quicktips = true;
777
        }
778 779

        $this->out = $request->input('out', "html");
780
        if ($this->out !== "html" && $this->out !== "json" && $this->out !== "results" && $this->out !== "results-with-style") {
781
            $this->out = "html";
782 783
        }

784
        $this->request = $request;
785 786 787 788 789 790 791 792 793 794
    }

    public function checkSpecialSearches(Request $request)
    {
        # Site Search:
        if (preg_match("/(.*)\bsite:(\S+)(.*)/si", $this->q, $match)) {
            $this->site = $match[2];
            $this->q    = $match[1] . $match[3];
        }
        if ($request->has('site')) {
795 796
            $this->site = $request->input('site');
        }
797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842
        # Wenn die Suchanfrage um das Schlüsselwort "-host:*" ergänzt ist, sollen bestimmte Hosts nicht eingeblendet werden
        # Wir prüfen, ob das hier der Fall ist:
        while (preg_match("/(.*)(^|\s)-host:(\S+)(.*)/si", $this->q, $match)) {
            $this->hostBlacklist[] = $match[3];
            $this->q               = $match[1] . $match[4];
        }
        if (sizeof($this->hostBlacklist) > 0) {
            $hostString = "";
            foreach ($this->hostBlacklist as $host) {
                $hostString .= $host . ", ";
            }
            $hostString       = rtrim($hostString, ", ");
            $this->warnings[] = "Ergebnisse von folgenden Hosts werden nicht angezeigt: \"" . $hostString . "\"";
        }
        # Wenn die Suchanfrage um das Schlüsselwort "-domain:*" ergänzt ist, sollen bestimmte Domains nicht eingeblendet werden
        # Wir prüfen, ob das hier der Fall ist:
        while (preg_match("/(.*)(^|\s)-domain:(\S+)(.*)/si", $this->q, $match)) {
            $this->domainBlacklist[] = $match[3];
            $this->q                 = $match[1] . $match[4];
        }
        if (sizeof($this->domainBlacklist) > 0) {
            $domainString = "";
            foreach ($this->domainBlacklist as $domain) {
                $domainString .= $domain . ", ";
            }
            $domainString     = rtrim($domainString, ", ");
            $this->warnings[] = "Ergebnisse von folgenden Domains werden nicht angezeigt: \"" . $domainString . "\"";
        }

        # Alle mit "-" gepräfixten Worte sollen aus der Suche ausgeschlossen werden.
        # Wir prüfen, ob das hier der Fall ist:
        while (preg_match("/(.*)(^|\s)-(\S+)(.*)/si", $this->q, $match)) {
            $this->stopWords[] = $match[3];
            $this->q           = $match[1] . $match[4];
        }
        if (sizeof($this->stopWords) > 0) {
            $stopwordsString = "";
            foreach ($this->stopWords as $stopword) {
                $stopwordsString .= $stopword . ", ";
            }
            $stopwordsString  = rtrim($stopwordsString, ", ");
            $this->warnings[] = "Sie machen eine Ausschlusssuche. Ergebnisse mit folgenden Wörtern werden nicht angezeigt: \"" . $stopwordsString . "\"";
        }

        # Meldung über eine Phrasensuche
        $p   = "";
843
        $tmp = $this->q;
844 845
        while (preg_match("/(.*)\"(.+)\"(.*)/si", $tmp, $match)) {
            $tmp             = $match[1] . $match[3];
846
            $this->phrases[] = strtolower($match[2]);
847 848
        }
        foreach ($this->phrases as $phrase) {
849 850 851
            $p .= "\"$phrase\", ";
        }
        $p = rtrim($p, ", ");
852
        if (sizeof($this->phrases) > 0) {
853
            $this->warnings[] = "Sie führen eine Phrasensuche durch: $p";
854
        }
855

856 857 858
    }

    public function getFokus()
859 860 861 862
    {
        return $this->fokus;
    }

863
    public function getIp()
864 865 866 867
    {
        return $this->ip;
    }

868
    public function getEingabe()
869 870 871 872
    {
        return $this->eingabe;
    }

873
    public function getQ()
874
    {
875
        return $this->q;
876 877
    }

878
    public function getUrl()
879 880 881
    {
        return $this->url;
    }
882
    public function getTime()
883 884 885 886
    {
        return $this->time;
    }

887
    public function getLanguage()
888 889 890 891
    {
        return $this->language;
    }

892
    public function getLang()
893 894 895 896
    {
        return $this->lang;
    }

897
    public function getSprueche()
898 899 900 901
    {
        return $this->sprueche;
    }

902
    public function getCategory()
903 904 905 906
    {
        return $this->category;
    }

907
    public function getPhrases()
908 909 910
    {
        return $this->phrases;
    }
911 912 913 914
    public function getPage()
    {
        return $this->page;
    }
915

916
    public function getSumaFile()
917 918 919 920
    {
        return $this->sumaFile;
    }

921
    public function getUserHostBlacklist()
922 923 924 925
    {
        return $this->hostBlacklist;
    }

926
    public function getUserDomainBlacklist()
927 928 929 930
    {
        return $this->domainBlacklist;
    }

931
    public function getDomainBlacklist()
932 933 934 935
    {
        return $this->domainsBlacklisted;
    }

936
    public function getUrlBlacklist()
937 938 939
    {
        return $this->urlsBlacklisted;
    }
940
    public function getLanguageDetect()
941 942 943
    {
        return $this->languageDetect;
    }
944
    public function getStopWords()
945 946 947 948 949
    {
        return $this->stopWords;
    }
    public function getHostCount($host)
    {
950
        if (isset($this->addedHosts[$host])) {
951
            return $this->addedHosts[$host];
952
        } else {
953 954 955
            return 0;
        }
    }
956 957 958 959
    public function getStartCount()
    {
        return $this->startCount;
    }
960