MetaGer.php 58 KB
Newer Older
1 2 3 4
<?php
namespace App;

use App;
5
use Cache;
6
use Carbon;
7
use Illuminate\Http\Request;
Phil Höfer's avatar
Phil Höfer committed
8
use Illuminate\Support\Facades\Redis;
9
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix  
Dominik Hebeler committed
10
use LaravelLocalization;
11
use Log;
12
use Predis\Connection\ConnectionException;
13 14 15

class MetaGer
{
16 17 18 19 20 21 22 23 24 25
    # Einstellungen für die Suche
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $category;
    protected $time;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
26
    protected $hostBlacklist = [];
27
    protected $domainBlacklist = [];
28 29 30 31 32 33 34 35 36
    private $urlBlacklist = [];
    protected $stopWords = [];
    protected $phrases = [];
    protected $engines = [];
    protected $results = [];
    protected $ads = [];
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
37
    protected $availableFoki = [];
38 39
    protected $startCount = 0;
    protected $canCache = false;
40
    # Daten über die Abfrage$
41
    protected $ip;
Dominik Hebeler's avatar
Dominik Hebeler committed
42
    protected $useragent;
43 44
    protected $language;
    protected $agent;
45
    protected $apiKey = "";
Phil Höfer's avatar
Phil Höfer committed
46
    protected $apiAuthorized = false;
47 48 49 50 51
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
52
    protected $maps;
53
    protected $newtab;
54
    protected $domainsBlacklisted = [];
55
    protected $urlsBlacklisted = [];
56
    protected $url;
57
    protected $fullUrl;
58
    protected $languageDetect;
Dominik Hebeler's avatar
Dominik Hebeler committed
59 60
    protected $verificationId;
    protected $verificationCount;
61

62 63
    public function __construct()
    {
64
        # Timer starten
65
        $this->starttime = microtime(true);
66
        # Versuchen Blacklists einzulesen
67
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
68
            $tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
69
            $this->domainsBlacklisted = explode("\n", $tmp);
70 71
            $tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted = explode("\n", $tmp);
72
        } else {
73
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
74 75
        }

76
        # Parser Skripte einhängen
77 78 79 80
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
81
                require_once $path;
82 83 84
            }
        }

85
        # Cachebarkeit testen
86 87 88 89 90 91
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
92
    }
93

94
    # Erstellt aus den gesammelten Ergebnissen den View
95
    public function createView($quicktipResults = [])
96
    {
97 98
        # Hiermit werden die evtl. ausgewählten SuMas extrahiert, damit die Input-Boxen richtig gesetzt werden können
        $focusPages = [];
Aria Givi's avatar
Aria Givi committed
99

100 101
        foreach ($this->request->all() as $key => $value) {
            if ($value === 'on') {
Aria Givi's avatar
Aria Givi committed
102
                $focusPages[] = str_replace('param_', '', str_replace('engine_', '', $key));
103 104 105
            }
        }

106
        $viewResults = [];
107
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
108
        foreach ($this->results as $result) {
109 110 111 112
            $viewResults[] = get_object_vars($result);
        }
        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();
113 114
        if ($this->fokus === "bilder") {
            switch ($this->out) {
115
                case 'results':
116
                    return view('resultpages.results_images')
117 118 119 120 121
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
122
                        ->with('apiAuthorized', $this->apiAuthorized)
123
                        ->with('metager', $this)
124
                        ->with('browser', (new Agent())->browser());
125
                default:
126
                    return view('resultpages.resultpage_images')
127 128 129 130 131
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
132
                        ->with('apiAuthorized', $this->apiAuthorized)
133
                        ->with('metager', $this)
134
                        ->with('browser', (new Agent())->browser());
135
            }
136 137 138
        } else {
            switch ($this->out) {
                case 'results':
139
                    return view('resultpages.results')
140 141 142 143 144
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
145
                        ->with('apiAuthorized', $this->apiAuthorized)
146
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
147 148
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
149 150
                    break;
                case 'results-with-style':
151
                    return view('resultpages.resultpage')
152 153 154 155 156
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
157
                        ->with('apiAuthorized', $this->apiAuthorized)
158 159
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
Dominik Hebeler's avatar
Dominik Hebeler committed
160 161
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
162
                    break;
Phil Höfer's avatar
Phil Höfer committed
163
                case 'rich':
164
                    return view('resultpages.metager3rich')
165
                        ->with('results', $viewResults)
Phil Höfer's avatar
Phil Höfer committed
166 167 168 169
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
170
                        ->with('apiAuthorized', $this->apiAuthorized)
Phil Höfer's avatar
Phil Höfer committed
171
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
172 173
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
Phil Höfer's avatar
Phil Höfer committed
174
                    break;
175
                case 'rss20':
176
                    return view('resultpages.metager3resultsrss20')
177 178
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
Phil Höfer's avatar
Phil Höfer committed
179
                        ->with('apiAuthorized', $this->apiAuthorized)
180
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
181 182
                        ->with('resultcount', sizeof($viewResults))
                        ->with('fokus', $this->fokus);
183
                    break;
Aria Givi's avatar
Aria Givi committed
184
                case 'atom10':
185 186
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults)])
                        ->header('Content-Type', 'application/xml');
Aria Givi's avatar
Aria Givi committed
187
                    break;
188
                case 'result-count':
189 190
                    # Wir geben die Ergebniszahl und die benötigte Zeit zurück:
                    return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
191
                    break;
192
                default:
193
                    return view('resultpages.resultpage')
194
                        ->with('eingabe', $this->eingabe)
195
                        ->with('focusPages', $focusPages)
196 197 198
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
199
                        ->with('apiAuthorized', $this->apiAuthorized)
200
                        ->with('metager', $this)
201
                        ->with('browser', (new Agent())->browser())
202
                        ->with('quicktips', $quicktipResults)
203 204
                        ->with('resultcount', count($this->results))
                        ->with('focus', $this->fokus);
205 206
                    break;
            }
207 208 209
        }
    }

Phil Höfer's avatar
Phil Höfer committed
210
    public function prepareResults()
211
    {
Phil Höfer's avatar
Phil Höfer committed
212
        $engines = $this->engines;
213

Phil Höfer's avatar
Phil Höfer committed
214 215
        // combine
        $combinedResults = $this->combineResults($engines);
216 217 218 219

        # Wir bestimmen die Sprache eines jeden Suchergebnisses
        $this->results = $this->addLangCodes($this->results);

Phil Höfer's avatar
Phil Höfer committed
220 221 222 223 224
        // sort
        //$sortedResults = $this->sortResults($engines);
        // filter
        // augment (boost&adgoal)
        // authorize
Phil Höfer's avatar
Phil Höfer committed
225 226 227
        if ($this->apiKey) {
            $this->apiAuthorized = $this->authorize($this->apiKey);
        }
Phil Höfer's avatar
Phil Höfer committed
228
        // misc (WiP)
229
        if ($this->fokus == "nachrichten") {
230 231 232
            $this->results = array_filter($this->results, function ($v, $k) {
                return !is_null($v->getRank());
            }, ARRAY_FILTER_USE_BOTH);
233 234 235 236 237 238 239 240 241 242
            uasort($this->results, function ($a, $b) {
                $datea = $a->getDate();
                $dateb = $b->getDate();
                return $dateb - $datea;
            });
        } else {
            uasort($this->results, function ($a, $b) {
                if ($a->getRank() == $b->getRank()) {
                    return 0;
                }
243

244 245 246
                return ($a->getRank() < $b->getRank()) ? 1 : -1;
            });
        }
247

248 249
        # Validate Results
        $newResults = [];
250 251
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
252
                $newResults[] = $result;
253 254
            }

255 256 257
        }
        $this->results = $newResults;

258
        #Adgoal Implementation
259 260 261
        if (!$this->apiAuthorized) {
            $this->results = $this->parseAdgoal($this->results);
        }
262

Dominik Hebeler's avatar
Dominik Hebeler committed
263 264
        # Human Verification
        $this->results = $this->humanVerification($this->results);
265
        $this->ads = $this->humanVerification($this->ads);
Dominik Hebeler's avatar
Dominik Hebeler committed
266

267
        $counter = 0;
268
        $firstRank = 0;
269

270
        if (isset($this->startForwards)) {
271
            $this->startCount = $this->startForwards;
272
        } elseif (isset($this->startBackwards)) {
273
            $this->startCount = $this->startBackwards - count($this->results) - 1;
274
        } else {
275 276 277
            $this->startCount = 0;
        }

278 279
        foreach ($this->results as $result) {
            if ($counter === 0) {
280
                $firstRank = $result->rank;
281 282
            }

283
            $counter++;
284
            $result->number = $counter + $this->startCount;
285
            $confidence = 0;
286 287 288
            if ($firstRank > 0) {
                $confidence = $result->rank / $firstRank;
            } else {
289
                $confidence = 0;
290 291 292
            }

            if ($confidence > 0.65) {
293
                $result->color = "#FF4000";
294
            } elseif ($confidence > 0.4) {
295
                $result->color = "#FF0080";
296
            } elseif ($confidence > 0.2) {
297
                $result->color = "#C000C0";
298
            } else {
299
                $result->color = "#000000";
300 301
            }

302 303
        }

304
        if ($this->validated) {
305 306
            $this->ads = [];
            $this->maps = false;
307
        }
308 309

        if (count($this->results) <= 0) {
310 311 312 313 314 315
            if (strlen($this->site) > 0) {
                $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
                $this->errors[] = trans('metaGer.results.failedSitesearch', ['altSearch' => $no_sitesearch_query]);
            } else {
                $this->errors[] = trans('metaGer.results.failed');
            }
316
        }
317

318
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
319
            $page = $this->page + 1;
320
            $this->next = [
321
                'page' => $page,
322
                'startForwards' => $this->results[count($this->results) - 1]->number,
323
                'engines' => $this->next,
324 325
            ];
            Cache::put(md5(serialize($this->next)), serialize($this->next), 60);
326 327
        } else {
            $this->next = [];
328 329
        }

330
    }
331

332 333
    private function addLangCodes($results)
    {
334 335 336 337 338
        # Wenn es keine Ergebnisse gibt, brauchen wir uns gar nicht erst zu bemühen
        if (sizeof($results) === 0) {
            return $results;
        }

339 340 341 342 343
        # Bei der Spracheinstellung "all" wird nicht gefiltert
        if ($this->getLang() === "all") {
            return $results;
        } else {
            # Ansonsten müssen wir jedem Result einen Sprachcode hinzufügen
344
            $id = 0;
345 346 347 348 349
            $langStrings = [];
            foreach ($results as $result) {
                # Wir geben jedem Ergebnis eine ID um später die Sprachcodes zuordnen zu können
                $result->id = $id;

Dominik Hebeler's avatar
Dominik Hebeler committed
350
                $langStrings["result_" . $id] = utf8_encode($result->getLangString());
351 352 353 354 355 356 357 358

                $id++;
            }
            # Wir schreiben die Strings in eine temporäre JSON-Datei,
            # Da das Array unter umständen zu groß ist für eine direkte Übergabe an das Skript
            $filename = "/tmp/" . getmypid();
            file_put_contents($filename, json_encode($langStrings));
            $langDetectorPath = app_path() . "/Models/lang.pl";
359 360
            $lang = exec("echo '$filename' | $langDetectorPath");
            $lang = json_decode($lang, true);
361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377

            # Wir haben nun die Sprachcodes der einzelnen Ergebnisse.
            # Diese müssen wir nur noch korrekt zuordnen, dann sind wir fertig.
            foreach ($lang as $key => $langCode) {
                # Prefix vom Key entfernen:
                $id = intval(str_replace("result_", "", $key));
                foreach ($this->results as $result) {
                    if ($result->id === $id) {
                        $result->langCode = $langCode;
                        break;
                    }
                }
            }
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397
    public function combineResults($engines)
    {
        foreach ($engines as $engine) {
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
            foreach ($engine->results as $result) {
                if ($result->valid) {
                    $this->results[] = $result;
                }
            }
            foreach ($engine->ads as $ad) {
                $this->ads[] = $ad;
            }
        }
    }

398 399
    public function parseAdgoal($results)
    {
400
        $publicKey = getenv('adgoal_public');
401
        $privateKey = getenv('adgoal_private');
402
        if ($publicKey === false) {
403 404 405
            return $results;
        }
        $tldList = "";
406 407
        try {
            foreach ($results as $result) {
408
                $link = $result->anzeigeLink;
409
                if (strpos($link, "http") !== 0) {
410 411 412 413 414 415 416 417 418 419
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

420
            # Query
421 422
            $query = urlencode($this->q);

423
            $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . $publicKey . "&k=" . $hash . "&tld=" . $tldList . "&q=" . $query;
424 425 426
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
427
            foreach ($answer as $el) {
428
                $hoster = $el[0];
429
                $hash = $el[1];
430

431
                foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
432
                    if ($hoster === $result->tld && !$result->partnershop) {
433 434
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
435
                        if ($result->image !== "") {
436
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . $hash . ".gif";
437
                        } else {
438
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . $hash . ".gif";
439 440
                        }

441 442 443
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
                        $targetUrl = $result->anzeigeLink;
444
                        if (strpos($targetUrl, "http") !== 0) {
445
                            $targetUrl = "http://" . $targetUrl;
446 447
                        }

448 449 450
                        $gateHash = md5($targetUrl . $privateKey);
                        $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . $publicKey . "&k=" . $gateHash . "&url=" . urlencode($targetUrl) . "&q=" . $query;
                        $result->link = $newLink;
451 452 453 454
                        $result->partnershop = true;
                    }
                }
            }
455
        } catch (\ErrorException $e) {
456 457 458 459 460
            return $results;
        }

        return $results;
    }
461

462 463
    public function humanVerification($results)
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
464
        # Let's check if we need to implement a redirect for human verification
465 466
        if ($this->verificationCount > 10) {
            foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
467 468 469
                $link = $result->link;
                $day = Carbon::now()->day;
                $pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
470 471 472
                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
Dominik Hebeler's avatar
Dominik Hebeler committed
473
                $result->link = $url;
474
                $result->proxyLink = $proxyUrl;
Dominik Hebeler's avatar
Dominik Hebeler committed
475 476
            }
            return $results;
477
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
478 479 480 481
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
482 483 484 485 486 487
    public function authorize($key)
    {
        $postdata = http_build_query(array(
            'dummy' => rand(),
        ));
        $opts = array('http' => array(
488 489
            'method' => 'POST',
            'header' => 'Content-type: application/x-www-form-urlencoded',
Phil Höfer's avatar
Phil Höfer committed
490 491 492 493 494 495 496
            'content' => $postdata,
        ),
        );

        $context = stream_context_create($opts);

        try {
497
            $link = "https://key.metager3.de/" . urlencode($key) . "/request-permission/api-access";
Phil Höfer's avatar
Phil Höfer committed
498 499 500 501 502 503 504 505 506 507 508 509
            $result = json_decode(file_get_contents($link, false, $context));
            if ($result->{'api-access'} == true) {
                return true;
            } else {
                return false;
            }

        } catch (\ErrorException $e) {
            return false;
        }
    }

510 511
    public function createQuicktips()
    {
512 513 514 515 516
        # Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
        $quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime(), $this->getHashCode());
        return $quicktips;
    }

Karl's avatar
Karl committed
517 518 519 520
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

521 522
    public function createSearchEngines(Request $request)
    {
523
        # Wenn es kein Suchwort gibt
Dominik Hebeler's avatar
Dominik Hebeler committed
524
        if (!$request->filled("eingabe") || $this->q === "") {
525
            return;
526
        }
527

528 529
        $xml = simplexml_load_file($this->sumaFile);
        $sumas = $xml->xpath("suma");
530
        $enabledSearchengines = [];
531 532
        $overtureEnabled = false;
        $sumaCount = 0;
533

534 535 536 537 538 539 540 541 542 543 544 545 546 547
        /*
        * Erstellt eine Liste mit Foki, die verfügbar sind
        */
        $this->availableFoki = [];
        foreach ($sumas as $suma) {
            $foki = explode(",", trim($suma["type"]));
            foreach ($foki as $fokus) {
                if (!empty($fokus)) {
                    $this->availableFoki[$fokus] = "available";
                }

            }
        }

548
        $isCustomSearch = $this->startsWith($this->fokus, 'focus_');
549

550
        # Im Falle einer Custom-Suche ohne mindestens einer selektierter Suchmaschine wird eine Web-Suche durchgeführt
551
        if ($isCustomSearch && !$this->atLeastOneSearchengineSelected($request)) {
552 553 554
            $isCustomSearch = false;
            $this->fokus = 'web';
        }
555

Karl's avatar
Karl committed
556
        /* Erstellt die Liste der eingestellten Sumas
557 558 559
         * Der einzige Unterschied bei angepasstem Suchfokus ist,
         * dass nicht nach den Typen einer Suma,
         * sondern den im Request mitgegebenen Typen entschieden wird.
Karl's avatar
Karl committed
560 561 562 563 564 565
         * Ansonsten wird genau das selbe geprüft und gemacht:
         * Handelt es sich um spezielle Suchmaschinen die immer an sein müssen
         * Wenn es Overture ist vermerken dass Overture an ist
         * Suma Zähler erhöhen
         * Zu Liste hinzufügen
         */
566
        foreach ($sumas as $suma) {
567
            if (($this->sumaIsSelected($suma, $request, $isCustomSearch)
568
                || (!$this->isBildersuche()
569
                    && $this->sumaIsAdsuche($suma, $overtureEnabled)))
570 571 572
                && (!$this->sumaIsDisabled($suma))) {
                if ($this->sumaIsOverture($suma)) {
                    $overtureEnabled = true;
573
                }
574 575
                if ($this->sumaIsNotAdsuche($suma)) {
                    $sumaCount += 1;
576
                }
577
                $enabledSearchengines[] = $suma;
578 579 580 581
            }
        }

        # Sonderregelung für alle Suchmaschinen, die zu den Minisuchern gehören. Diese können alle gemeinsam über einen Link abgefragt werden
582
        $subcollections = [];
583 584

        $tmp = [];
585 586 587 588
        // Es gibt den Schalter "minism=on" Dieser soll bewirken, dass alle Minisucher angeschaltet werden.
        // Wenn also "minism=on" ist, dann durchsuchen wir statt den tatsächlich angeschalteten Suchmaschinen,
        // alle Suchmaschinen nach "minismCollection"
        if ($request->input("minism", "off") === "on") {
Dominik Hebeler's avatar
Dominik Hebeler committed
589 590 591
            // Wir laden alle Minisucher
            foreach ($sumas as $engine) {
                if (isset($engine["minismCollection"])) {
592
                    $subcollections[] = $engine["minismCollection"]->__toString();
Dominik Hebeler's avatar
Dominik Hebeler committed
593 594
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
595 596 597 598 599 600
            # Nur noch alle eventuell angeschalteten Minisucher deaktivieren
            foreach ($enabledSearchengines as $index => $engine) {
                if (!isset($engine["minismCollection"])) {
                    $tmp[] = $engine;
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
601 602 603 604
        } else {
            // Wir schalten eine Teilmenge, oder aber gar keine an
            foreach ($enabledSearchengines as $engine) {
                if (isset($engine['minismCollection'])) {
605
                    $subcollections[] = $engine['minismCollection']->__toString();
Dominik Hebeler's avatar
Dominik Hebeler committed
606 607
                } else {
                    $tmp[] = $engine;
Dominik Hebeler's avatar
Dominik Hebeler committed
608
                }
609
            }
610 611
        }
        $enabledSearchengines = $tmp;
612 613
        if (sizeof($subcollections) > 0) {
            $enabledSearchengines[] = $this->loadMiniSucher($xml, $subcollections);
614
        }
615
        if ($sumaCount <= 0) {
616
            $this->errors[] = trans('metaGer.settings.noneSelected');
617
        }
618
        $engines = [];
619
        # Wenn eine Sitesearch durchgeführt werden soll, überprüfen wir ob überhaupt eine der Suchmaschinen eine Sitesearch unterstützt
620
        $siteSearchFailed = $this->checkCanNotSitesearch($enabledSearchengines);
621 622

        $typeslist = [];
623
        $counter = 0;
624

625 626
        if ($this->requestIsCached($request)) {
            $engines = $this->getCachedEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
627
            # We need to edit some Options of the Cached Search Engines
Phil Höfer's avatar
Phil Höfer committed
628
            foreach ($engines as $engine) {
Dominik Hebeler's avatar
Dominik Hebeler committed
629 630
                $engine->setResultHash($this->getHashCode());
            }
631
        } else {
632
            $engines = $this->actuallyCreateSearchEngines($enabledSearchengines, $siteSearchFailed);
633
        }
634

635
        # Wir starten alle Suchen
636 637
        foreach ($engines as $engine) {
            $engine->startSearch($this);
638
        }
639

640 641 642 643 644
        /* Wir warten auf die Antwort der Suchmaschinen
         * Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
         * Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
         * $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
         * welche Suchmaschinen nicht rechtzeitig geantwortet haben.
645 646
         */

647
        $enginesToLoad = [];
648
        $canBreak = false;
649 650 651 652 653
        foreach ($engines as $engine) {
            if ($engine->cached) {
                if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
                    $canBreak = true;
                }
654 655
            } else {
                $enginesToLoad[$engine->name] = false;
656 657 658 659 660 661 662 663
            }
        }

        $this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);

        $this->retrieveResults($engines);
    }

664 665
    # Spezielle Suchen und Sumas

666
    public function sumaIsSelected($suma, $request, $custom)
667
    {
668
        if ($custom) {
669
            if ($request->filled("engine_" . strtolower($suma["name"]))) {
670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693
                return true;
            }
        } else {
            $types = explode(",", $suma["type"]);
            if (in_array($this->fokus, $types)) {
                return true;
            }
        }
        return false;
    }

    public function actuallyCreateSearchEngines($enabledSearchengines, $siteSearchFailed)
    {
        $engines = [];
        foreach ($enabledSearchengines as $engine) {

            # Wenn diese Suchmaschine gar nicht eingeschaltet sein soll
            if (!$siteSearchFailed
                && strlen($this->site) > 0
                && (!isset($engine['hasSiteSearch'])
                    || $engine['hasSiteSearch']->__toString() === "0")) {
                continue;
            }

Dominik Hebeler's avatar
Dominik Hebeler committed
694 695 696
            if (!isset($engine["package"])) {
                die(var_dump($engine));
            }
697 698 699 700 701
            # Setze Pfad zu Parser
            $path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString());

            # Prüfe ob Parser vorhanden
            if (!file_exists(app_path() . "/Models/parserSkripte/" . ucfirst($engine["package"]->__toString()) . ".php")) {
702
                Log::error("Konnte " . $engine["name"] . " nicht abfragen, da kein Parser existiert");
703
                $this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine["name"]]);
704 705 706 707 708 709 710 711
                continue;
            }

            # Es wird versucht die Suchengine zu erstellen
            $time = microtime();
            try {
                $tmp = new $path($engine, $this);
            } catch (\ErrorException $e) {
712
                Log::error("Konnte " . $engine["name"] . " nicht abfragen. " . var_dump($e));
713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730
                continue;
            }

            # Ausgabe bei Debug-Modus
            if ($tmp->enabled && isset($this->debug)) {
                $this->warnings[] = $tmp->service . "   Connection_Time: " . $tmp->connection_time . "    Write_Time: " . $tmp->write_time . " Insgesamt:" . ((microtime() - $time) / 1000);
            }

            # Wenn die neu erstellte Engine eingeschaltet ist, wird sie der Liste hinzugefügt
            if ($tmp->isEnabled()) {
                $engines[] = $tmp;
            }
        }
        return $engines;
    }

    public function isBildersuche()
    {
731
        return $this->fokus === "bilder";
732 733 734 735
    }

    public function sumaIsAdsuche($suma, $overtureEnabled)
    {
736
        $sumaName = $suma["name"]->__toString();
737
        return
738 739
            $sumaName === "qualigo"
            || $sumaName === "similar_product_ads"
Karl Hasselbring's avatar
Karl Hasselbring committed
740
            || (!$overtureEnabled && $sumaName === "overtureAds");
741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767
    }

    public function sumaIsDisabled($suma)
    {
        return
        isset($suma['disabled'])
        && $suma['disabled']->__toString() === "1";
    }

    public function sumaIsOverture($suma)
    {
        return
        $suma["name"]->__toString() === "overture"
        || $suma["name"]->__toString() === "overtureAds";
    }

    public function sumaIsNotAdsuche($suma)
    {
        return
        $suma["name"]->__toString() !== "qualigo"
        && $suma["name"]->__toString() !== "similar_product_ads"
        && $suma["name"]->__toString() !== "overtureAds";
    }

    public function requestIsCached($request)
    {
        return
Dominik Hebeler's avatar
Dominik Hebeler committed
768
        $request->filled('next')
769 770 771 772 773 774
        && Cache::has($request->input('next'))
        && unserialize(Cache::get($request->input('next')))['page'] > 1;
    }

    public function getCachedEngines($request)
    {
775
        $next = unserialize(Cache::get($request->input('next')));
776
        $this->page = $next['page'];
777
        $engines = $next['engines'];
778 779 780 781 782 783 784 785 786 787 788
        if (isset($next['startForwards'])) {
            $this->startForwards = $next['startForwards'];
        }
        if (isset($next['startBackwards'])) {
            $this->startBackwards = $next['startBackwards'];
        }
        return $engines;
    }

    public function loadMiniSucher($xml, $subcollections)
    {
789
        $minisucherEngine = $xml->xpath('suma[@name="minism"]')[0];
790
        $minisucherEngine["subcollections"] = implode(", ", $subcollections);
791
        $subcollectionsString = urlencode("(" . implode(" OR ", $subcollections) . ")");
792
        $minisucherEngine["formData"] = str_replace("<<SUBCOLLECTIONS>>", $subcollectionsString, $minisucherEngine["formData"]);
793 794 795 796
        $minisucherEngine["formData"] = str_replace("<<COUNT>>", sizeof($subcollections) * 10, $minisucherEngine["formData"]);
        return $minisucherEngine;
    }

797
    # Passt den Suchfokus an, falls für einen Fokus genau alle vorhandenen Sumas eingeschaltet sind
798 799
    public function adjustFocus($sumas, $enabledSearchengines)
    {
800 801
        # Findet für alle Foki die enthaltenen Sumas
        $foki = []; # [fokus][suma] => [suma]
802
        foreach ($sumas as $suma) {
803
            if ((!$this->sumaIsDisabled($suma)) && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
804
                if (isset($suma['type'])) {
805 806 807
                    # Wenn foki für diese Suchmaschine angegeben sind
                    $focuses = explode(",", $suma['type']->__toString());
                    foreach ($focuses as $foc) {
808 809 810 811 812
                        if (isset($suma['minismCollection'])) {
                            $foki[$foc][] = "minism";
                        } else {
                            $foki[$foc][] = $suma['name']->__toString();
                        }
813
                    }
814
                } else {
815
                    # Wenn keine foki für diese Suchmaschine angegeben sind
816 817 818 819 820
                    if (isset($suma['minismCollection'])) {
                        $foki["andere"][] = "minism";
                    } else {
                        $foki["andere"][] = $suma['name']->__toString();
                    }
821 822 823 824
                }
            }
        }

825
        # Findet die Namen der aktuell eingeschalteten Sumas
826
        $realEngNames = [];
827
        foreach ($enabledSearchengines as $realEng) {
828
            $nam = $realEng["name"]->__toString();
Karl Hasselbring's avatar
Karl Hasselbring committed
829
            if ($nam !== "qualigo" && $nam !== "overtureAds") {
830 831 832
                $realEngNames[] = $nam;
            }
        }
833

834
        # Anschließend werden diese beiden Listen verglichen (jeweils eine der Fokuslisten für jeden Fokus), um herauszufinden ob sie vielleicht identisch sind. Ist dies der Fall, so hat der Nutzer anscheinend Suchmaschinen eines kompletten Fokus eingestellt. Der Fokus wird dementsprechend angepasst.
835
        foreach ($foki as $fok => $engines) {
836
            $isFokus = true;
837
            $fokiEngNames = [];
838
            foreach ($engines as $eng) {
839 840
                $fokiEngNames[] = $eng;
            }
841
            # Jede eingeschaltete Engine ist für diesen Fokus geeignet
842
            foreach ($fokiEngNames as $fen) {
843 844
                # Bei Bildersuchen ist uns egal, ob alle Suchmaschinen aus dem Suchfokus eingeschaltet sind, da wir sie eh als Bildersuche anzeigen müssen
                if (!in_array($fen, $realEngNames) && $fok !== "bilder") {
845 846 847
                    $isFokus = false;
                }
            }
848
            # Jede im Fokus erwartete Engine ist auch eingeschaltet
849 850
            foreach ($realEngNames as $ren) {
                if (!in_array($ren, $fokiEngNames)) {
851 852 853
                    $isFokus = false;
                }
            }
854
            # Wenn die Listen identisch sind, setze den Fokus um
855
            if ($isFokus) {
856 857 858
                $this->fokus = $fok;
            }
        }
859
    }
860

861 862 863 864 865 866 867
    public function checkCanNotSitesearch($enabledSearchengines)
    {
        if (strlen($this->site) > 0) {
            $enginesWithSite = 0;
            foreach ($enabledSearchengines as $engine) {
                if (isset($engine['hasSiteSearch']) && $engine['hasSiteSearch']->__toString() === "1") {
                    $enginesWithSite++;
868
                }
869 870
            }
            if ($enginesWithSite === 0) {
871
                $this->errors[] = trans('metaGer.sitesearch.failed', ['site' => $this->site, 'searchLink' => $this->generateSearchLink("web", false)]);
872 873
                return true;
            } else {
874
                $this->warnings[] = trans('metaGer.sitesearch.success', ['site' => $this->site]);
875
                return false;
876 877
            }
        }
878
        return false;
879 880 881 882
    }

    public function waitForResults($enginesToLoad, $overtureEnabled, $canBreak)
    {
883

Phil Höfer's avatar
Phil Höfer committed
884
        $timeStart = microtime(true);
885
        $results = null;
886
        while (true) {