MetaGer.php 38.7 KB
Newer Older
1 2 3 4
<?php
namespace App;

use App;
5
use App\lib\TextLanguageDetect\TextLanguageDetect;
6
use Cache;
7 8
use Illuminate\Http\Request;
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix  
Dominik Hebeler committed
9
use LaravelLocalization;
10
use Log;
11
use Predis\Connection\ConnectionException;
12
use Redis;
13 14 15

class MetaGer
{
16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
    # Einstellungen für die Suche
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $category;
    protected $time;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
    protected $hostBlacklist   = [];
    protected $domainBlacklist = [];
    protected $stopWords       = [];
    protected $phrases         = [];
    protected $engines         = [];
    protected $results         = [];
    protected $ads             = [];
    protected $warnings        = [];
    protected $errors          = [];
    protected $addedHosts      = [];
36
    protected $startCount      = 0;
37
    protected $canCache        = false;
38 39 40 41 42 43 44 45 46
    # Daten über die Abfrage
    protected $ip;
    protected $language;
    protected $agent;
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
47
    protected $domainsBlacklisted = [];
48
    protected $urlsBlacklisted    = [];
49 50 51
    protected $url;
    protected $languageDetect;

52 53
    public function __construct()
    {
54
        # Timer starten
55
        $this->starttime = microtime(true);
56 57

        # Versuchen Blacklists einzulesen
58 59
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
            $tmp                      = file_get_contents(config_path() . "/blacklistDomains.txt");
60
            $this->domainsBlacklisted = explode("\n", $tmp);
61 62 63
            $tmp                      = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted    = explode("\n", $tmp);
        } else {
64
            Log::warning(trans('metaGer.blacklist.failed'));
65 66
        }

67
        # Parser Skripte einhängen
68 69 70 71 72 73 74 75
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
                require $path;
            }
        }

76
        # Spracherkennung starten
77 78
        $this->languageDetect = new TextLanguageDetect();
        $this->languageDetect->setNameMode("2");
79

80
        # Cachebarkeit testen
81 82 83 84 85 86
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
87
    }
88

89
    # Erstellt aus den gesammelten Ergebnissen den View
90 91 92
    public function createView()
    {
        $viewResults = [];
93

94
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
95
        foreach ($this->results as $result) {
96 97 98 99 100 101
            $viewResults[] = get_object_vars($result);
        }

        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();

102 103
        if ($this->fokus === "bilder") {
            switch ($this->out) {
104 105 106 107 108 109 110 111
                case 'results':
                    return view('metager3bilderresults')
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
112
                        ->with('browser', (new Agent())->browser());
113 114 115 116 117 118 119 120
                default:
                    return view('metager3bilder')
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
121
                        ->with('browser', (new Agent())->browser());
122
            }
123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
        } else {
            switch ($this->out) {
                case 'results':
                    return view('metager3results')
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
                        ->with('browser', (new Agent())->browser());
                    break;
                case 'results-with-style':
                    return view('metager3')
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
                        ->with('browser', (new Agent())->browser());
                    break;
                default:
                    return view('metager3')
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
                        ->with('browser', (new Agent())->browser());
                    break;
            }
156 157 158
        }
    }

159 160 161
    public function combineResults()
    {
        foreach ($this->engines as $engine) {
162 163 164 165 166 167
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
168 169
            foreach ($engine->results as $result) {
                if ($result->valid) {
170
                    $this->results[] = $result;
171
                }
172
            }
173
            foreach ($engine->ads as $ad) {
174 175
                $this->ads[] = $ad;
            }
176
        }
177

178 179
        uasort($this->results, function ($a, $b) {
            if ($a->getRank() == $b->getRank()) {
180
                return 0;
181 182
            }

183 184
            return ($a->getRank() < $b->getRank()) ? 1 : -1;
        });
185

186 187
        # Validate Results
        $newResults = [];
188 189
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
190
                $newResults[] = $result;
191 192
            }

193 194 195
        }
        $this->results = $newResults;

196 197 198 199 200 201
        # Boost implementation
        $this->results = $this->parseBoost($this->results);

        #Adgoal Implementation
        $this->results = $this->parseAdgoal($this->results);

202
        $counter   = 0;
203
        $firstRank = 0;
204

205
        if (isset($this->startForwards)) {
206
            $this->startCount = $this->startForwards;
207
        } elseif (isset($this->startBackwards)) {
208
            $this->startCount = $this->startBackwards - count($this->results) - 1;
209
        } else {
210 211 212
            $this->startCount = 0;
        }

213 214
        foreach ($this->results as $result) {
            if ($counter === 0) {
215
                $firstRank = $result->rank;
216 217
            }

218
            $counter++;
219
            $result->number = $counter + $this->startCount;
220 221 222 223
            $confidence     = 0;
            if ($firstRank > 0) {
                $confidence = $result->rank / $firstRank;
            } else {
224
                $confidence = 0;
225 226 227
            }

            if ($confidence > 0.65) {
228
                $result->color = "#FF4000";
229
            } elseif ($confidence > 0.4) {
230
                $result->color = "#FF0080";
231
            } elseif ($confidence > 0.2) {
232
                $result->color = "#C000C0";
233
            } else {
234
                $result->color = "#000000";
235 236
            }

237 238
        }

239
        if (LaravelLocalization::getCurrentLocale() === "en") {
240 241 242
            $this->ads = [];
        }

243
        $this->validated = false;
244
        if (isset($this->password)) {
245 246 247
            # Wir bieten einen bezahlten API-Zugriff an, bei dem dementsprechend die Werbung ausgeblendet wurde:
            # Aktuell ist es nur die Uni-Mainz. Deshalb überprüfen wir auch nur diese.
            $password = getenv('mainz');
248
            $eingabe  = $this->eingabe;
249
            $password = md5($eingabe . $password);
250 251
            if ($this->password === $password) {
                $this->ads       = [];
252 253 254
                $this->validated = true;
            }
        }
255 256

        if (count($this->results) <= 0) {
257
            $this->errors[] = trans('metaGer.results.failed');
258
        }
259

260
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
261
            $page       = $this->page + 1;
262
            $this->next = [
263 264 265
                'page'          => $page,
                'startForwards' => $this->results[count($this->results) - 1]->number,
                'engines'       => $this->next,
266 267
            ];
            Cache::put(md5(serialize($this->next)), serialize($this->next), 60);
268 269
        } else {
            $this->next = [];
270 271
        }

272
    }
273

274 275
    public function parseBoost($results)
    {
276 277 278 279 280 281 282 283 284 285 286 287
        foreach ($results as $result) {
            if (preg_match('/^(http[s]?\:\/\/)?(www.)?amazon\.de/', $result->anzeigeLink)) {
                if (preg_match('/\?/', $result->anzeigeLink)) {
                    $result->link .= '&tag=boostmg01-21';
                } else {
                    $result->link .= '?tag=boostmg01-21';
                }
                $result->partnershop = true;

            }
        }
        return $results;
288
    }
289

290 291
    public function parseAdgoal($results)
    {
292
        $publicKey  = getenv('adgoal_public');
293
        $privateKey = getenv('adgoal_private');
294
        if ($publicKey === false) {
295 296 297
            return $results;
        }
        $tldList = "";
298 299
        try {
            foreach ($results as $result) {
300
                $link = $result->anzeigeLink;
301
                if (strpos($link, "http") !== 0) {
302 303 304 305 306 307 308 309 310 311
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

312
            # Query
313 314
            $query = urlencode($this->q);

315
            $link   = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . $publicKey . "&k=" . $hash . "&tld=" . $tldList . "&q=" . $query;
316 317 318
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
319
            foreach ($answer as $el) {
320
                $hoster = $el[0];
321
                $hash   = $el[1];
322

323 324
                foreach ($results as $result) {
                    if ($hoster === $result->tld) {
325 326
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
327
                        if ($result->image !== "") {
328
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . $hash . ".gif";
329
                        } else {
330
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . $hash . ".gif";
331 332
                        }

333 334 335
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
                        $targetUrl = $result->anzeigeLink;
336
                        if (strpos($targetUrl, "http") !== 0) {
337
                            $targetUrl = "http://" . $targetUrl;
338 339 340 341 342
                        }

                        $gateHash            = md5($targetUrl . $privateKey);
                        $newLink             = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . $publicKey . "&k=" . $gateHash . "&url=" . urlencode($targetUrl) . "&q=" . $query;
                        $result->link        = $newLink;
343 344 345 346
                        $result->partnershop = true;
                    }
                }
            }
347
        } catch (\ErrorException $e) {
348 349 350 351 352
            return $results;
        }

        return $results;
    }
353

Karl's avatar
Karl committed
354 355 356 357
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

358 359 360
    public function createSearchEngines(Request $request)
    {
        if (!$request->has("eingabe")) {
361
            return;
362
        }
363

364 365
        # Überprüfe, welche Sumas eingeschaltet sind
        $xml                  = simplexml_load_file($this->sumaFile);
366
        $enabledSearchengines = [];
367
        $overtureEnabled      = false;
368
        $sumaCount            = 0;
369
        $sumas                = $xml->xpath("suma");
370

Karl's avatar
Karl committed
371 372 373 374 375 376 377 378
        /* Erstellt die Liste der eingestellten Sumas
         * Der einzige Unterschied bei angepasstem Suchfokus ist, dass nicht nach den Typen einer Suma, sondern den im Request mitgegebenen Typen entschieden wird
         * Ansonsten wird genau das selbe geprüft und gemacht:
         * Handelt es sich um spezielle Suchmaschinen die immer an sein müssen
         * Wenn es Overture ist vermerken dass Overture an ist
         * Suma Zähler erhöhen
         * Zu Liste hinzufügen
         */
379 380
        foreach ($sumas as $suma) {
            if ($this->fokus === "angepasst") {
381 382 383 384 385 386 387 388 389 390 391 392
                if ($request->has($suma["name"])
                    || ($this->fokus !== "bilder"
                        && ($suma["name"]->__toString() === "qualigo"
                            || $suma["name"]->__toString() === "similar_product_ads"
                            || (!$overtureEnabled && $suma["name"]->__toString() === "overtureAds")
                        )
                    )
                ) {

                    if (!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1")) {
                        if ($suma["name"]->__toString() === "overture" || $suma["name"]->__toString() === "overtureAds") {
                            $overtureEnabled = true;
393
                        }
394
                        if ($suma["name"]->__toString() !== "qualigo" && $suma["name"]->__toString() !== "similar_product_ads" && $suma["name"]->__toString() !== "overtureAds") {
395
                            $sumaCount += 1;
396
                        }
397 398 399
                        $enabledSearchengines[] = $suma;
                    }
                }
400
            } else {
401 402 403 404 405 406 407 408 409 410 411 412
                $types = explode(",", $suma["type"]);
                if (in_array($this->fokus, $types)
                    || ($this->fokus !== "bilder"
                        && ($suma["name"]->__toString() === "qualigo"
                            || $suma["name"]->__toString() === "similar_product_ads"
                            || (!$overtureEnabled && $suma["name"]->__toString() === "overtureAds")
                        )
                    )
                ) {
                    if (!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1")) {
                        if ($suma["name"]->__toString() === "overture" || $suma["name"]->__toString() === "overtureAds") {
                            $overtureEnabled = true;
413
                        }
414
                        if ($suma["name"]->__toString() !== "qualigo" && $suma["name"]->__toString() !== "similar_product_ads" && $suma["name"]->__toString() !== "overtureAds") {
415
                            $sumaCount += 1;
416
                        }
417 418 419 420 421 422 423 424
                        $enabledSearchengines[] = $suma;
                    }
                }
            }
        }

        # Sonderregelung für alle Suchmaschinen, die zu den Minisuchern gehören. Diese können alle gemeinsam über einen Link abgefragt werden
        $subcollections = [];
425 426 427
        $tmp            = [];
        foreach ($enabledSearchengines as $engine) {
            if (isset($engine['minismCollection'])) {
428
                $subcollections[] = $engine['minismCollection']->__toString();
429
            } else {
430
                $tmp[] = $engine;
431 432
            }

433 434
        }
        $enabledSearchengines = $tmp;
435 436 437 438
        if (sizeof($subcollections) > 0) {
            $count                        = sizeof($subcollections) * 10;
            $minisucherEngine             = $xml->xpath('suma[@name="minism"]')[0];
            $subcollections               = urlencode("(" . implode(" OR ", $subcollections) . ")");
439 440
            $minisucherEngine["formData"] = str_replace("<<SUBCOLLECTIONS>>", $subcollections, $minisucherEngine["formData"]);
            $minisucherEngine["formData"] = str_replace("<<COUNT>>", $count, $minisucherEngine["formData"]);
441
            $enabledSearchengines[]       = $minisucherEngine;
442 443
        }

444
        if ($sumaCount <= 0) {
445
            $this->errors[] = trans('metaGer.settings.noneSelected');
446
        }
447
        $engines = [];
448

449 450
        # Wenn eine Sitesearch durchgeführt werden soll, überprüfen wir ob eine der Suchmaschinen überhaupt eine Sitesearch unterstützt
        $siteSearchFailed = $this->checkCanNotSitesearch($enabledSearchengines);
451 452

        $typeslist = [];
453
        $counter   = 0;
454

455 456
        if ($request->has('next') && Cache::has($request->input('next')) && unserialize(Cache::get($request->input('next')))['page'] > 1) {
            $next       = unserialize(Cache::get($request->input('next')));
457
            $this->page = $next['page'];
458 459
            $engines    = $next['engines'];
            if (isset($next['startForwards'])) {
460
                $this->startForwards = $next['startForwards'];
461 462 463
            }

            if (isset($next['startBackwards'])) {
464
                $this->startBackwards = $next['startBackwards'];
465 466
            }

467 468
        } else {
            foreach ($enabledSearchengines as $engine) {
469

Karl's avatar
Karl committed
470
                # Wenn diese Suchmaschine gar nicht eingeschaltet sein soll
471 472 473
                if (!$siteSearchFailed && strlen($this->site) > 0 && (!isset($engine['hasSiteSearch']) || $engine['hasSiteSearch']->__toString() === "0")) {
                    continue;
                }
Karl's avatar
Karl committed
474 475

                # Setze Pfad zu Parser
476
                $path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString());
477

Karl's avatar
Karl committed
478
                # Prüfe ob Parser vorhanden
479
                if (!file_exists(app_path() . "/Models/parserSkripte/" . ucfirst($engine["package"]->__toString()) . ".php")) {
480
                    Log::error(trans('metaGer.engines.noParser', ['engine' => $engine["name"]]));
481 482
                    continue;
                }
483

Karl's avatar
Karl committed
484
                # Es wird versucht die Suchengine zu erstellen
485
                $time = microtime();
Karl's avatar
Karl committed
486
                try {
487 488
                    $tmp = new $path($engine, $this);
                } catch (\ErrorException $e) {
489
                    Log::error(trans('metaGer.engines.cantQuery', ['engine' => $engine["name"], 'error' => var_dump($e)]));
490 491
                    continue;
                }
492

Karl's avatar
Karl committed
493
                # Ausgabe bei Debug-Modus
494 495 496 497
                if ($tmp->enabled && isset($this->debug)) {
                    $this->warnings[] = $tmp->service . "   Connection_Time: " . $tmp->connection_time . "    Write_Time: " . $tmp->write_time . " Insgesamt:" . ((microtime() - $time) / 1000);
                }

Karl's avatar
Karl committed
498
                # Wenn die neu erstellte Engine eingeschaltet ist, wird sie der Liste hinzugefügt
499 500 501
                if ($tmp->isEnabled()) {
                    $engines[] = $tmp;
                }
502 503

            }
504
        }
505

506 507 508
        # Wir starten die Suche manuell:
        foreach ($engines as $engine) {
            $engine->startSearch($this);
509
        }
510

511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
        $this->adjustFocus($sumas, $enabledSearchengines);

        /* Nun passiert ein elementarer Schritt.
         * Wir warten auf die Antwort der Suchmaschinen, da wir vorher nicht weiter machen können.
         * Aber natürlich nicht ewig.
         * Die Verbindung steht zu diesem Zeitpunkt und auch unsere Request wurde schon gesendet.
         * Wir geben der Suchmaschine nun bis zu 500ms Zeit zu antworten.
         */

        # Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
        $enginesToLoad = 0;
        $canBreak      = false;
        foreach ($engines as $engine) {
            if ($engine->cached) {
                $enginesToLoad--;
                if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
                    $canBreak = true;
                }
            }
        }

        $enginesToLoad += count($engines);

        $this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);

        $this->retrieveResults($engines);
    }

    public function adjustFocus($sumas, $enabledSearchengines)
    {
541 542
        # Jetzt werden noch alle Kategorien der Settings durchgegangen und die jeweils enthaltenen namen der Suchmaschinen gespeichert.
        $foki = [];
543 544 545
        foreach ($sumas as $suma) {
            if ((!isset($suma['disabled']) || $suma['disabled'] === "") && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
                if (isset($suma['type'])) {
546
                    $f = explode(",", $suma['type']->__toString());
547 548
                    foreach ($f as $tmp) {
                        $name                                    = $suma['name']->__toString();
549 550
                        $foki[$tmp][$suma['name']->__toString()] = $name;
                    }
551 552
                } else {
                    $name                                        = $suma['name']->__toString();
553 554 555 556 557 558 559
                    $foki["andere"][$suma['name']->__toString()] = $name;
                }
            }
        }

        # Es werden auch die Namen der aktuell aktiven Suchmaschinen abgespeichert.
        $realEngNames = [];
560
        foreach ($enabledSearchengines as $realEng) {
561
            $nam = $realEng["name"]->__toString();
562
            if ($nam !== "qualigo" && $nam !== "overtureAds") {
563 564 565
                $realEngNames[] = $nam;
            }
        }
566

567
        # Anschließend werden diese beiden Listen verglichen (jeweils eine der Fokuslisten für jeden Fokus), um herauszufinden ob sie vielleicht identisch sind. Ist dies der Fall, so hat der Nutzer anscheinend Suchmaschinen eines kompletten Fokus eingestellt. Der Fokus wird dementsprechend angepasst.
568 569
        foreach ($foki as $fok => $engs) {
            $isFokus      = true;
570
            $fokiEngNames = [];
571
            foreach ($engs as $eng) {
572 573
                $fokiEngNames[] = $eng;
            }
574 575
            foreach ($fokiEngNames as $fen) {
                if (!in_array($fen, $realEngNames)) {
576 577 578
                    $isFokus = false;
                }
            }
579 580
            foreach ($realEngNames as $ren) {
                if (!in_array($ren, $fokiEngNames)) {
581 582 583
                    $isFokus = false;
                }
            }
584
            if ($isFokus) {
585 586 587
                $this->fokus = $fok;
            }
        }
588
    }
589

590 591 592 593 594 595 596
    public function checkCanNotSitesearch($enabledSearchengines)
    {
        if (strlen($this->site) > 0) {
            $enginesWithSite = 0;
            foreach ($enabledSearchengines as $engine) {
                if (isset($engine['hasSiteSearch']) && $engine['hasSiteSearch']->__toString() === "1") {
                    $enginesWithSite++;
597
                }
598 599
            }
            if ($enginesWithSite === 0) {
600
                $this->errors[] = trans('metaGer.sitesearch.failed', ['site' => $this->site, 'searchLink' => $this->generateSearchLink("web", false)]);
601 602
                return true;
            } else {
603
                $this->warnings[] = trans('metaGer.sitesearch.success', ['site' => $this->site]);
604
                return false;
605 606
            }
        }
607 608 609 610
    }

    public function waitForResults($enginesToLoad, $overtureEnabled, $canBreak)
    {
611
        $loadedEngines = 0;
612 613 614
        $timeStart     = microtime(true);
        while (true) {
            $time          = (microtime(true) - $timeStart) * 1000;
615
            $loadedEngines = intval(Redis::hlen('search.' . $this->getHashCode()));
616
            if ($overtureEnabled && (Redis::hexists('search.' . $this->getHashCode(), 'overture') || Redis::hexists('search.' . $this->getHashCode(), 'overtureAds'))) {
617
                $canBreak = true;
618
            }
619 620

            # Abbruchbedingung
621 622
            if ($time < 500) {
                if (($enginesToLoad === 0 || $loadedEngines >= $enginesToLoad) && $canBreak) {
623
                    break;
624 625 626 627
                }

            } elseif ($time >= 500 && $time < $this->time) {
                if (($enginesToLoad === 0 || ($loadedEngines / ($enginesToLoad * 1.0)) >= 0.8) && $canBreak) {
628
                    break;
629 630 631
                }

            } else {
632 633 634 635
                break;
            }
            usleep(50000);
        }
636
    }
637

638 639 640
    public function retrieveResults($engines)
    {
        # Von geladenen Engines die Ergebnisse holen
641 642 643
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
                try {
644
                    $engine->retrieveResults($this);
645
                } catch (\ErrorException $e) {
646 647 648 649
                    Log::error($e);
                }
            }
        }
650

651
        # Nicht fertige Engines verwefen
652 653
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
654
                $engine->shutdown();
655
            }
656 657 658
        }

        $this->engines = $engines;
659 660
    }

Karl's avatar
Karl committed
661 662 663 664
    /*
     * Ende
     */

665 666
    public function parseFormData(Request $request)
    {
667
        # Sichert, dass der request in UTF-8 formatiert ist
668 669 670 671 672 673 674 675
        if ($request->input('encoding', '') !== "utf8") {
            # In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
            $input = $request->all();
            foreach ($input as $key => $value) {
                $input[$key] = mb_convert_encoding("$value", "UTF-8", "ISO-8859-1");
            }
            $request->replace($input);
        }
676
        $this->url = $request->url();
677
        # Zunächst überprüfen wir die eingegebenen Einstellungen:
678
        # Fokus
679
        $this->fokus = $request->input('focus', 'web');
680
        # Suma-File
681
        if (App::isLocale("en")) {
682
            $this->sumaFile = config_path() . "/sumas.xml";
683
        } else {
684 685
            $this->sumaFile = config_path() . "/sumas.xml";
        }
686
        if (!file_exists($this->sumaFile)) {
687
            die(trans('metaGer.formdata.cantLoad'));
688
        }
689
        # Sucheingabe
690
        $this->eingabe = trim($request->input('eingabe', ''));
691
        if (strlen($this->eingabe) === 0) {
692
            $this->warnings[] = trans('metaGer.formdata.noSearch');
693 694
        }
        $this->q = $this->eingabe;
695
        # IP
696
        $this->ip = $request->ip();
697
        # Language
698
        if (isset($_SERVER['HTTP_LANGUAGE'])) {
699
            $this->language = $_SERVER['HTTP_LANGUAGE'];
700
        } else {
701 702 703 704
            $this->language = "";
        }
        # Category
        $this->category = $request->input('category', '');
705
        # Request Times
706 707
        $this->time = $request->input('time', 1000);
        # Page
708
        $this->page = 1;
709 710
        # Lang
        $this->lang = $request->input('lang', 'all');
711 712
        if ($this->lang !== "de" && $this->lang !== "en" && $this->lang !== "all") {
            $this->lang = "all";
713
        }
714
        $this->agent  = new Agent();
715
        $this->mobile = $this->agent->isMobile();
716
        # Sprüche
717
        $this->sprueche = $request->input('sprueche', 'off');
718
        if ($this->sprueche === "off") {
719
            $this->sprueche = true;
720
        } else {
721
            $this->sprueche = false;
722
        }
723 724 725
        # Ergebnisse pro Seite:
        $this->resultCount = $request->input('resultCount', '20');
        # Manchmal müssen wir Parameter anpassen um den Sucheinstellungen gerecht zu werden:
726 727
        if ($request->has('dart')) {
            $this->time       = 10000;
728
            $this->warnings[] = trans('metaGer.formdata.dartEurope');
729
        }
730 731
        if ($this->time <= 500 || $this->time > 20000) {
            $this->time = 1000;
732
        }
733 734 735 736 737 738 739 740 741
        if ($request->has('minism') && ($request->has('fportal') || $request->has('harvest'))) {
            $input    = $request->all();
            $newInput = [];
            foreach ($input as $key => $value) {
                if ($key !== "fportal" && $key !== "harvest") {
                    $newInput[$key] = $value;
                }
            }
            $request->replace($newInput);
742
        }
743 744
        if (App::isLocale("en")) {
            $this->sprueche = "off";
745
        }
746 747
        if ($this->resultCount <= 0 || $this->resultCount > 200) {
            $this->resultCount = 1000;
748
        }
749 750 751 752 753 754
        if ($request->has('onenewspageAll') || $request->has('onenewspageGermanyAll')) {
            $this->time  = 5000;
            $this->cache = "cache";
        }
        if ($request->has('tab')) {
            if ($request->input('tab') === "off") {
755
                $this->tab = "_blank";
756
            } else {
757 758
                $this->tab = "_self";
            }
759
        } else {
760 761
            $this->tab = "_blank";
        }
762
        if ($request->has('password')) {
763
            $this->password = $request->input('password');
764 765
        }
        if ($request->has('quicktips')) {
766
            $this->quicktips = false;
767
        } else {
768
            $this->quicktips = true;
769
        }
770
        $this->out = $request->input('out', "html");
771
        # Standard output format html
772
        if ($this->out !== "html" && $this->out !== "json" && $this->out !== "results" && $this->out !== "results-with-style") {
773
            $this->out = "html";
774
        }
775
        $this->request = $request;
776 777 778 779
    }

    public function checkSpecialSearches(Request $request)
    {
780
        # Site Search
781 782 783 784 785
        if (preg_match("/(.*)\bsite:(\S+)(.*)/si", $this->q, $match)) {
            $this->site = $match[2];
            $this->q    = $match[1] . $match[3];
        }
        if ($request->has('site')) {
786 787
            $this->site = $request->input('site');
        }
788 789

        # Host Blacklisting
790 791 792 793 794 795 796 797 798 799 800
        # Wenn die Suchanfrage um das Schlüsselwort "-host:*" ergänzt ist, sollen bestimmte Hosts nicht eingeblendet werden
        while (preg_match("/(.*)(^|\s)-host:(\S+)(.*)/si", $this->q, $match)) {
            $this->hostBlacklist[] = $match[3];
            $this->q               = $match[1] . $match[4];
        }
        if (sizeof($this->hostBlacklist) > 0) {
            $hostString = "";
            foreach ($this->hostBlacklist as $host) {
                $hostString .= $host . ", ";
            }
            $hostString       = rtrim($hostString, ", ");
801
            $this->warnings[] = trans('metaGer.formdata.hostBlacklist', ['host' => $hostString]);
802
        }
803 804

        # Domain Blacklisting
805 806 807 808 809 810 811 812 813 814 815
        # Wenn die Suchanfrage um das Schlüsselwort "-domain:*" ergänzt ist, sollen bestimmte Domains nicht eingeblendet werden
        while (preg_match("/(.*)(^|\s)-domain:(\S+)(.*)/si", $this->q, $match)) {
            $this->domainBlacklist[] = $match[3];
            $this->q                 = $match[1] . $match[4];
        }
        if (sizeof($this->domainBlacklist) > 0) {
            $domainString = "";
            foreach ($this->domainBlacklist as $domain) {
                $domainString .= $domain . ", ";
            }
            $domainString     = rtrim($domainString, ", ");
816
            $this->warnings[] = trans('metaGer.formdata.domainBlacklist', ['domain' => $domainString]);
817 818
        }