MetaGer.php 58 KB
Newer Older
1 2 3 4
<?php
namespace App;

use App;
5
use Cache;
6
use Carbon;
7
use Illuminate\Http\Request;
Phil Höfer's avatar
Phil Höfer committed
8
use Illuminate\Support\Facades\Redis;
9
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix  
Dominik Hebeler committed
10
use LaravelLocalization;
11
use Log;
12
use Predis\Connection\ConnectionException;
13 14 15

class MetaGer
{
16 17 18 19 20 21 22 23 24 25
    # Einstellungen für die Suche
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $category;
    protected $time;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
26
    protected $hostBlacklist = [];
27
    protected $domainBlacklist = [];
28 29 30 31 32 33 34 35 36
    private $urlBlacklist = [];
    protected $stopWords = [];
    protected $phrases = [];
    protected $engines = [];
    protected $results = [];
    protected $ads = [];
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
37
    protected $availableFoki = [];
38 39
    protected $startCount = 0;
    protected $canCache = false;
40
    # Daten über die Abfrage$
41
    protected $ip;
Dominik Hebeler's avatar
Dominik Hebeler committed
42
    protected $useragent;
43 44
    protected $language;
    protected $agent;
45
    protected $apiKey = "";
Phil Höfer's avatar
Phil Höfer committed
46
    protected $apiAuthorized = false;
47 48 49 50 51
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
52
    protected $maps;
53
    protected $newtab;
54
    protected $domainsBlacklisted = [];
55
    protected $urlsBlacklisted = [];
56
    protected $url;
57
    protected $fullUrl;
58
    protected $languageDetect;
Dominik Hebeler's avatar
Dominik Hebeler committed
59 60
    protected $verificationId;
    protected $verificationCount;
61

62 63
    public function __construct()
    {
64
        # Timer starten
65
        $this->starttime = microtime(true);
66
        # Versuchen Blacklists einzulesen
67
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
68
            $tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
69
            $this->domainsBlacklisted = explode("\n", $tmp);
70 71
            $tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted = explode("\n", $tmp);
72
        } else {
73
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
74 75
        }

76
        # Parser Skripte einhängen
77 78 79 80
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
81
                require_once $path;
82 83 84
            }
        }

85
        # Cachebarkeit testen
86 87 88 89 90 91
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
92
    }
93

94
    # Erstellt aus den gesammelten Ergebnissen den View
95
    public function createView($quicktipResults = null)
96
    {
97 98
        # Hiermit werden die evtl. ausgewählten SuMas extrahiert, damit die Input-Boxen richtig gesetzt werden können
        $focusPages = [];
Aria Givi's avatar
Aria Givi committed
99

100 101
        foreach ($this->request->all() as $key => $value) {
            if ($value === 'on') {
Aria Givi's avatar
Aria Givi committed
102
                $focusPages[] = str_replace('param_', '', str_replace('engine_', '', $key));
103 104 105
            }
        }

106
        $viewResults = [];
107
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
108
        foreach ($this->results as $result) {
109 110 111 112
            $viewResults[] = get_object_vars($result);
        }
        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();
113 114
        if ($this->fokus === "bilder") {
            switch ($this->out) {
115
                case 'results':
116
                    return view('resultpages.results_images')
117 118 119 120 121
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
122
                        ->with('apiAuthorized', $this->apiAuthorized)
123
                        ->with('metager', $this)
124
                        ->with('browser', (new Agent())->browser());
125
                default:
126
                    return view('resultpages.resultpage_images')
127 128 129 130 131
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
132
                        ->with('apiAuthorized', $this->apiAuthorized)
133
                        ->with('metager', $this)
134
                        ->with('browser', (new Agent())->browser());
135
            }
136 137 138
        } else {
            switch ($this->out) {
                case 'results':
139
                    return view('resultpages.results')
140 141 142 143 144
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
145
                        ->with('apiAuthorized', $this->apiAuthorized)
146
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
147 148
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
149 150
                    break;
                case 'results-with-style':
151
                    return view('resultpages.resultpage')
152 153 154 155 156
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
157
                        ->with('apiAuthorized', $this->apiAuthorized)
158 159
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
Dominik Hebeler's avatar
Dominik Hebeler committed
160 161
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
162
                    break;
Phil Höfer's avatar
Phil Höfer committed
163
                case 'rich':
164
                    return view('resultpages.metager3rich')
165
                        ->with('results', $viewResults)
Phil Höfer's avatar
Phil Höfer committed
166 167 168 169
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
170
                        ->with('apiAuthorized', $this->apiAuthorized)
Phil Höfer's avatar
Phil Höfer committed
171
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
172 173
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
Phil Höfer's avatar
Phil Höfer committed
174
                    break;
175
                case 'rss20':
176
                    return view('resultpages.metager3resultsrss20')
177 178
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
Phil Höfer's avatar
Phil Höfer committed
179
                        ->with('apiAuthorized', $this->apiAuthorized)
180
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
181 182
                        ->with('resultcount', sizeof($viewResults))
                        ->with('fokus', $this->fokus);
183
                    break;
Aria Givi's avatar
Aria Givi committed
184
                case 'atom10':
185 186
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults)])
                        ->header('Content-Type', 'application/xml');
Aria Givi's avatar
Aria Givi committed
187
                    break;
188
                case 'result-count':
189 190
                    # Wir geben die Ergebniszahl und die benötigte Zeit zurück:
                    return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
191
                    break;
192
                default:
193
                    return view('resultpages.resultpage')
194
                        ->with('eingabe', $this->eingabe)
195
                        ->with('focusPages', $focusPages)
196 197 198
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
199
                        ->with('apiAuthorized', $this->apiAuthorized)
200
                        ->with('metager', $this)
201
                        ->with('browser', (new Agent())->browser())
202
                        ->with('quicktips', $quicktipResults)
203 204
                        ->with('resultcount', count($this->results))
                        ->with('focus', $this->fokus);
205 206
                    break;
            }
207 208 209
        }
    }

Phil Höfer's avatar
Phil Höfer committed
210
    public function prepareResults()
211
    {
Phil Höfer's avatar
Phil Höfer committed
212
        $engines = $this->engines;
213

Phil Höfer's avatar
Phil Höfer committed
214 215
        // combine
        $combinedResults = $this->combineResults($engines);
216 217 218 219

        # Wir bestimmen die Sprache eines jeden Suchergebnisses
        $this->results = $this->addLangCodes($this->results);

Phil Höfer's avatar
Phil Höfer committed
220 221 222 223 224
        // sort
        //$sortedResults = $this->sortResults($engines);
        // filter
        // augment (boost&adgoal)
        // authorize
Phil Höfer's avatar
Phil Höfer committed
225 226 227
        if ($this->apiKey) {
            $this->apiAuthorized = $this->authorize($this->apiKey);
        }
Phil Höfer's avatar
Phil Höfer committed
228
        // misc (WiP)
229
        if ($this->fokus == "nachrichten") {
230 231 232
            $this->results = array_filter($this->results, function ($v, $k) {
                return !is_null($v->getRank());
            }, ARRAY_FILTER_USE_BOTH);
233 234 235 236 237 238 239 240 241 242
            uasort($this->results, function ($a, $b) {
                $datea = $a->getDate();
                $dateb = $b->getDate();
                return $dateb - $datea;
            });
        } else {
            uasort($this->results, function ($a, $b) {
                if ($a->getRank() == $b->getRank()) {
                    return 0;
                }
243

244 245 246
                return ($a->getRank() < $b->getRank()) ? 1 : -1;
            });
        }
247

248 249
        # Validate Results
        $newResults = [];
250 251
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
252
                $newResults[] = $result;
253 254
            }

255 256 257
        }
        $this->results = $newResults;

258 259 260
        #Adgoal Implementation
        $this->results = $this->parseAdgoal($this->results);

Dominik Hebeler's avatar
Dominik Hebeler committed
261 262
        # Human Verification
        $this->results = $this->humanVerification($this->results);
263
        $this->ads = $this->humanVerification($this->ads);
Dominik Hebeler's avatar
Dominik Hebeler committed
264

265
        $counter = 0;
266
        $firstRank = 0;
267

268
        if (isset($this->startForwards)) {
269
            $this->startCount = $this->startForwards;
270
        } elseif (isset($this->startBackwards)) {
271
            $this->startCount = $this->startBackwards - count($this->results) - 1;
272
        } else {
273 274 275
            $this->startCount = 0;
        }

276 277
        foreach ($this->results as $result) {
            if ($counter === 0) {
278
                $firstRank = $result->rank;
279 280
            }

281
            $counter++;
282
            $result->number = $counter + $this->startCount;
283
            $confidence = 0;
284 285 286
            if ($firstRank > 0) {
                $confidence = $result->rank / $firstRank;
            } else {
287
                $confidence = 0;
288 289 290
            }

            if ($confidence > 0.65) {
291
                $result->color = "#FF4000";
292
            } elseif ($confidence > 0.4) {
293
                $result->color = "#FF0080";
294
            } elseif ($confidence > 0.2) {
295
                $result->color = "#C000C0";
296
            } else {
297
                $result->color = "#000000";
298 299
            }

300 301
        }

302
        if ($this->validated) {
303 304
            $this->ads = [];
            $this->maps = false;
305
        }
306 307

        if (count($this->results) <= 0) {
308 309 310 311 312 313
            if (strlen($this->site) > 0) {
                $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
                $this->errors[] = trans('metaGer.results.failedSitesearch', ['altSearch' => $no_sitesearch_query]);
            } else {
                $this->errors[] = trans('metaGer.results.failed');
            }
314
        }
315

316
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
317
            $page = $this->page + 1;
318
            $this->next = [
319
                'page' => $page,
320
                'startForwards' => $this->results[count($this->results) - 1]->number,
321
                'engines' => $this->next,
322 323
            ];
            Cache::put(md5(serialize($this->next)), serialize($this->next), 60);
324 325
        } else {
            $this->next = [];
326 327
        }

328
    }
329

330 331
    private function addLangCodes($results)
    {
332 333 334 335 336
        # Wenn es keine Ergebnisse gibt, brauchen wir uns gar nicht erst zu bemühen
        if (sizeof($results) === 0) {
            return $results;
        }

337 338 339 340 341
        # Bei der Spracheinstellung "all" wird nicht gefiltert
        if ($this->getLang() === "all") {
            return $results;
        } else {
            # Ansonsten müssen wir jedem Result einen Sprachcode hinzufügen
342
            $id = 0;
343 344 345 346 347
            $langStrings = [];
            foreach ($results as $result) {
                # Wir geben jedem Ergebnis eine ID um später die Sprachcodes zuordnen zu können
                $result->id = $id;

Dominik Hebeler's avatar
Dominik Hebeler committed
348
                $langStrings["result_" . $id] = utf8_encode($result->getLangString());
349 350 351 352 353 354 355 356

                $id++;
            }
            # Wir schreiben die Strings in eine temporäre JSON-Datei,
            # Da das Array unter umständen zu groß ist für eine direkte Übergabe an das Skript
            $filename = "/tmp/" . getmypid();
            file_put_contents($filename, json_encode($langStrings));
            $langDetectorPath = app_path() . "/Models/lang.pl";
357 358
            $lang = exec("echo '$filename' | $langDetectorPath");
            $lang = json_decode($lang, true);
359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375

            # Wir haben nun die Sprachcodes der einzelnen Ergebnisse.
            # Diese müssen wir nur noch korrekt zuordnen, dann sind wir fertig.
            foreach ($lang as $key => $langCode) {
                # Prefix vom Key entfernen:
                $id = intval(str_replace("result_", "", $key));
                foreach ($this->results as $result) {
                    if ($result->id === $id) {
                        $result->langCode = $langCode;
                        break;
                    }
                }
            }
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395
    public function combineResults($engines)
    {
        foreach ($engines as $engine) {
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
            foreach ($engine->results as $result) {
                if ($result->valid) {
                    $this->results[] = $result;
                }
            }
            foreach ($engine->ads as $ad) {
                $this->ads[] = $ad;
            }
        }
    }

396 397
    public function parseAdgoal($results)
    {
398
        $publicKey = getenv('adgoal_public');
399
        $privateKey = getenv('adgoal_private');
400
        if ($publicKey === false) {
401 402 403
            return $results;
        }
        $tldList = "";
404 405
        try {
            foreach ($results as $result) {
406
                $link = $result->anzeigeLink;
407
                if (strpos($link, "http") !== 0) {
408 409 410 411 412 413 414 415 416 417
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

418
            # Query
419 420
            $query = urlencode($this->q);

421
            $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . $publicKey . "&k=" . $hash . "&tld=" . $tldList . "&q=" . $query;
422 423 424
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
425
            foreach ($answer as $el) {
426
                $hoster = $el[0];
427
                $hash = $el[1];
428

429
                foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
430
                    if ($hoster === $result->tld && !$result->partnershop) {
431 432
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
433
                        if ($result->image !== "") {
434
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . $hash . ".gif";
435
                        } else {
436
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . $hash . ".gif";
437 438
                        }

439 440 441
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
                        $targetUrl = $result->anzeigeLink;
442
                        if (strpos($targetUrl, "http") !== 0) {
443
                            $targetUrl = "http://" . $targetUrl;
444 445
                        }

446 447 448
                        $gateHash = md5($targetUrl . $privateKey);
                        $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . $publicKey . "&k=" . $gateHash . "&url=" . urlencode($targetUrl) . "&q=" . $query;
                        $result->link = $newLink;
449 450 451 452
                        $result->partnershop = true;
                    }
                }
            }
453
        } catch (\ErrorException $e) {
454 455 456 457 458
            return $results;
        }

        return $results;
    }
459

460 461
    public function humanVerification($results)
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
462
        # Let's check if we need to implement a redirect for human verification
463 464
        if ($this->verificationCount > 10) {
            foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
465 466 467
                $link = $result->link;
                $day = Carbon::now()->day;
                $pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
468 469 470
                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
Dominik Hebeler's avatar
Dominik Hebeler committed
471
                $result->link = $url;
472
                $result->proxyLink = $proxyUrl;
Dominik Hebeler's avatar
Dominik Hebeler committed
473 474
            }
            return $results;
475
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
476 477 478 479
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
480 481 482 483 484 485
    public function authorize($key)
    {
        $postdata = http_build_query(array(
            'dummy' => rand(),
        ));
        $opts = array('http' => array(
486 487
            'method' => 'POST',
            'header' => 'Content-type: application/x-www-form-urlencoded',
Phil Höfer's avatar
Phil Höfer committed
488 489 490 491 492 493 494
            'content' => $postdata,
        ),
        );

        $context = stream_context_create($opts);

        try {
495
            $link = "https://key.metager3.de/" . urlencode($key) . "/request-permission/api-access";
Phil Höfer's avatar
Phil Höfer committed
496 497 498 499 500 501 502 503 504 505 506 507
            $result = json_decode(file_get_contents($link, false, $context));
            if ($result->{'api-access'} == true) {
                return true;
            } else {
                return false;
            }

        } catch (\ErrorException $e) {
            return false;
        }
    }

508 509
    public function createQuicktips()
    {
510 511 512 513 514
        # Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
        $quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime(), $this->getHashCode());
        return $quicktips;
    }

Karl's avatar
Karl committed
515 516 517 518
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

519 520
    public function createSearchEngines(Request $request)
    {
521
        # Wenn es kein Suchwort gibt
Dominik Hebeler's avatar
Dominik Hebeler committed
522
        if (!$request->filled("eingabe") || $this->q === "") {
523
            return;
524
        }
525

526 527
        $xml = simplexml_load_file($this->sumaFile);
        $sumas = $xml->xpath("suma");
528
        $enabledSearchengines = [];
529 530
        $overtureEnabled = false;
        $sumaCount = 0;
531

532 533 534 535 536 537 538 539 540 541 542 543 544 545
        /*
        * Erstellt eine Liste mit Foki, die verfügbar sind
        */
        $this->availableFoki = [];
        foreach ($sumas as $suma) {
            $foki = explode(",", trim($suma["type"]));
            foreach ($foki as $fokus) {
                if (!empty($fokus)) {
                    $this->availableFoki[$fokus] = "available";
                }

            }
        }

546
        $isCustomSearch = $this->startsWith($this->fokus, 'focus_');
547

548
        # Im Falle einer Custom-Suche ohne mindestens einer selektierter Suchmaschine wird eine Web-Suche durchgeführt
549
        if ($isCustomSearch && !$this->atLeastOneSearchengineSelected($request)) {
550 551 552
            $isCustomSearch = false;
            $this->fokus = 'web';
        }
553

Karl's avatar
Karl committed
554
        /* Erstellt die Liste der eingestellten Sumas
555 556 557
         * Der einzige Unterschied bei angepasstem Suchfokus ist,
         * dass nicht nach den Typen einer Suma,
         * sondern den im Request mitgegebenen Typen entschieden wird.
Karl's avatar
Karl committed
558 559 560 561 562 563
         * Ansonsten wird genau das selbe geprüft und gemacht:
         * Handelt es sich um spezielle Suchmaschinen die immer an sein müssen
         * Wenn es Overture ist vermerken dass Overture an ist
         * Suma Zähler erhöhen
         * Zu Liste hinzufügen
         */
564
        foreach ($sumas as $suma) {
565
            if (($this->sumaIsSelected($suma, $request, $isCustomSearch)
566
                || (!$this->isBildersuche()
567
                    && $this->sumaIsAdsuche($suma, $overtureEnabled)))
568 569 570
                && (!$this->sumaIsDisabled($suma))) {
                if ($this->sumaIsOverture($suma)) {
                    $overtureEnabled = true;
571
                }
572 573
                if ($this->sumaIsNotAdsuche($suma)) {
                    $sumaCount += 1;
574
                }
575
                $enabledSearchengines[] = $suma;
576 577 578 579
            }
        }

        # Sonderregelung für alle Suchmaschinen, die zu den Minisuchern gehören. Diese können alle gemeinsam über einen Link abgefragt werden
580
        $subcollections = [];
581 582

        $tmp = [];
583 584 585 586
        // Es gibt den Schalter "minism=on" Dieser soll bewirken, dass alle Minisucher angeschaltet werden.
        // Wenn also "minism=on" ist, dann durchsuchen wir statt den tatsächlich angeschalteten Suchmaschinen,
        // alle Suchmaschinen nach "minismCollection"
        if ($request->input("minism", "off") === "on") {
Dominik Hebeler's avatar
Dominik Hebeler committed
587 588 589
            // Wir laden alle Minisucher
            foreach ($sumas as $engine) {
                if (isset($engine["minismCollection"])) {
590
                    $subcollections[] = $engine["minismCollection"]->__toString();
Dominik Hebeler's avatar
Dominik Hebeler committed
591 592
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
593 594 595 596 597 598
            # Nur noch alle eventuell angeschalteten Minisucher deaktivieren
            foreach ($enabledSearchengines as $index => $engine) {
                if (!isset($engine["minismCollection"])) {
                    $tmp[] = $engine;
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
599 600 601 602
        } else {
            // Wir schalten eine Teilmenge, oder aber gar keine an
            foreach ($enabledSearchengines as $engine) {
                if (isset($engine['minismCollection'])) {
603
                    $subcollections[] = $engine['minismCollection']->__toString();
Dominik Hebeler's avatar
Dominik Hebeler committed
604 605
                } else {
                    $tmp[] = $engine;
Dominik Hebeler's avatar
Dominik Hebeler committed
606
                }
607
            }
608 609
        }
        $enabledSearchengines = $tmp;
610 611
        if (sizeof($subcollections) > 0) {
            $enabledSearchengines[] = $this->loadMiniSucher($xml, $subcollections);
612
        }
613
        if ($sumaCount <= 0) {
614
            $this->errors[] = trans('metaGer.settings.noneSelected');
615
        }
616
        $engines = [];
617
        # Wenn eine Sitesearch durchgeführt werden soll, überprüfen wir ob überhaupt eine der Suchmaschinen eine Sitesearch unterstützt
618
        $siteSearchFailed = $this->checkCanNotSitesearch($enabledSearchengines);
619 620

        $typeslist = [];
621
        $counter = 0;
622

623 624
        if ($this->requestIsCached($request)) {
            $engines = $this->getCachedEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
625
            # We need to edit some Options of the Cached Search Engines
Phil Höfer's avatar
Phil Höfer committed
626
            foreach ($engines as $engine) {
Dominik Hebeler's avatar
Dominik Hebeler committed
627 628
                $engine->setResultHash($this->getHashCode());
            }
629
        } else {
630
            $engines = $this->actuallyCreateSearchEngines($enabledSearchengines, $siteSearchFailed);
631
        }
632

633
        # Wir starten alle Suchen
634 635
        foreach ($engines as $engine) {
            $engine->startSearch($this);
636
        }
637

638 639 640 641 642
        /* Wir warten auf die Antwort der Suchmaschinen
         * Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
         * Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
         * $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
         * welche Suchmaschinen nicht rechtzeitig geantwortet haben.
643 644
         */

645
        $enginesToLoad = [];
646
        $canBreak = false;
647 648 649 650 651
        foreach ($engines as $engine) {
            if ($engine->cached) {
                if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
                    $canBreak = true;
                }
652 653
            } else {
                $enginesToLoad[$engine->name] = false;
654 655 656 657 658 659 660 661
            }
        }

        $this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);

        $this->retrieveResults($engines);
    }

662 663
    # Spezielle Suchen und Sumas

664
    public function sumaIsSelected($suma, $request, $custom)
665
    {
666
        if ($custom) {
667
            if ($request->filled("engine_" . strtolower($suma["name"]))) {
668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691
                return true;
            }
        } else {
            $types = explode(",", $suma["type"]);
            if (in_array($this->fokus, $types)) {
                return true;
            }
        }
        return false;
    }

    public function actuallyCreateSearchEngines($enabledSearchengines, $siteSearchFailed)
    {
        $engines = [];
        foreach ($enabledSearchengines as $engine) {

            # Wenn diese Suchmaschine gar nicht eingeschaltet sein soll
            if (!$siteSearchFailed
                && strlen($this->site) > 0
                && (!isset($engine['hasSiteSearch'])
                    || $engine['hasSiteSearch']->__toString() === "0")) {
                continue;
            }

Dominik Hebeler's avatar
Dominik Hebeler committed
692 693 694
            if (!isset($engine["package"])) {
                die(var_dump($engine));
            }
695 696 697 698 699
            # Setze Pfad zu Parser
            $path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString());

            # Prüfe ob Parser vorhanden
            if (!file_exists(app_path() . "/Models/parserSkripte/" . ucfirst($engine["package"]->__toString()) . ".php")) {
700
                Log::error("Konnte " . $engine["name"] . " nicht abfragen, da kein Parser existiert");
701
                $this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine["name"]]);
702 703 704 705 706 707 708 709
                continue;
            }

            # Es wird versucht die Suchengine zu erstellen
            $time = microtime();
            try {
                $tmp = new $path($engine, $this);
            } catch (\ErrorException $e) {
710
                Log::error("Konnte " . $engine["name"] . " nicht abfragen. " . var_dump($e));
711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728
                continue;
            }

            # Ausgabe bei Debug-Modus
            if ($tmp->enabled && isset($this->debug)) {
                $this->warnings[] = $tmp->service . "   Connection_Time: " . $tmp->connection_time . "    Write_Time: " . $tmp->write_time . " Insgesamt:" . ((microtime() - $time) / 1000);
            }

            # Wenn die neu erstellte Engine eingeschaltet ist, wird sie der Liste hinzugefügt
            if ($tmp->isEnabled()) {
                $engines[] = $tmp;
            }
        }
        return $engines;
    }

    public function isBildersuche()
    {
729
        return $this->fokus === "bilder";
730 731 732 733
    }

    public function sumaIsAdsuche($suma, $overtureEnabled)
    {
734
        $sumaName = $suma["name"]->__toString();
735
        return
736 737
            $sumaName === "qualigo"
            || $sumaName === "similar_product_ads"
Karl Hasselbring's avatar
Karl Hasselbring committed
738
            || (!$overtureEnabled && $sumaName === "overtureAds");
739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765
    }

    public function sumaIsDisabled($suma)
    {
        return
        isset($suma['disabled'])
        && $suma['disabled']->__toString() === "1";
    }

    public function sumaIsOverture($suma)
    {
        return
        $suma["name"]->__toString() === "overture"
        || $suma["name"]->__toString() === "overtureAds";
    }

    public function sumaIsNotAdsuche($suma)
    {
        return
        $suma["name"]->__toString() !== "qualigo"
        && $suma["name"]->__toString() !== "similar_product_ads"
        && $suma["name"]->__toString() !== "overtureAds";
    }

    public function requestIsCached($request)
    {
        return
Dominik Hebeler's avatar
Dominik Hebeler committed
766
        $request->filled('next')
767 768 769 770 771 772
        && Cache::has($request->input('next'))
        && unserialize(Cache::get($request->input('next')))['page'] > 1;
    }

    public function getCachedEngines($request)
    {
773
        $next = unserialize(Cache::get($request->input('next')));
774
        $this->page = $next['page'];
775
        $engines = $next['engines'];
776 777 778 779 780 781 782 783 784 785 786
        if (isset($next['startForwards'])) {
            $this->startForwards = $next['startForwards'];
        }
        if (isset($next['startBackwards'])) {
            $this->startBackwards = $next['startBackwards'];
        }
        return $engines;
    }

    public function loadMiniSucher($xml, $subcollections)
    {
787
        $minisucherEngine = $xml->xpath('suma[@name="minism"]')[0];
788
        $minisucherEngine["subcollections"] = implode(", ", $subcollections);
789
        $subcollectionsString = urlencode("(" . implode(" OR ", $subcollections) . ")");
790
        $minisucherEngine["formData"] = str_replace("<<SUBCOLLECTIONS>>", $subcollectionsString, $minisucherEngine["formData"]);
791 792 793 794
        $minisucherEngine["formData"] = str_replace("<<COUNT>>", sizeof($subcollections) * 10, $minisucherEngine["formData"]);
        return $minisucherEngine;
    }

795
    # Passt den Suchfokus an, falls für einen Fokus genau alle vorhandenen Sumas eingeschaltet sind
796 797
    public function adjustFocus($sumas, $enabledSearchengines)
    {
798 799
        # Findet für alle Foki die enthaltenen Sumas
        $foki = []; # [fokus][suma] => [suma]
800
        foreach ($sumas as $suma) {
801
            if ((!$this->sumaIsDisabled($suma)) && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
802
                if (isset($suma['type'])) {
803 804 805
                    # Wenn foki für diese Suchmaschine angegeben sind
                    $focuses = explode(",", $suma['type']->__toString());
                    foreach ($focuses as $foc) {
806 807 808 809 810
                        if (isset($suma['minismCollection'])) {
                            $foki[$foc][] = "minism";
                        } else {
                            $foki[$foc][] = $suma['name']->__toString();
                        }
811
                    }
812
                } else {
813
                    # Wenn keine foki für diese Suchmaschine angegeben sind
814 815 816 817 818
                    if (isset($suma['minismCollection'])) {
                        $foki["andere"][] = "minism";
                    } else {
                        $foki["andere"][] = $suma['name']->__toString();
                    }
819 820 821 822
                }
            }
        }

823
        # Findet die Namen der aktuell eingeschalteten Sumas
824
        $realEngNames = [];
825
        foreach ($enabledSearchengines as $realEng) {
826
            $nam = $realEng["name"]->__toString();
Karl Hasselbring's avatar
Karl Hasselbring committed
827
            if ($nam !== "qualigo" && $nam !== "overtureAds") {
828 829 830
                $realEngNames[] = $nam;
            }
        }
831

832
        # Anschließend werden diese beiden Listen verglichen (jeweils eine der Fokuslisten für jeden Fokus), um herauszufinden ob sie vielleicht identisch sind. Ist dies der Fall, so hat der Nutzer anscheinend Suchmaschinen eines kompletten Fokus eingestellt. Der Fokus wird dementsprechend angepasst.
833
        foreach ($foki as $fok => $engines) {
834
            $isFokus = true;
835
            $fokiEngNames = [];
836
            foreach ($engines as $eng) {
837 838
                $fokiEngNames[] = $eng;
            }
839
            # Jede eingeschaltete Engine ist für diesen Fokus geeignet
840
            foreach ($fokiEngNames as $fen) {
841 842
                # Bei Bildersuchen ist uns egal, ob alle Suchmaschinen aus dem Suchfokus eingeschaltet sind, da wir sie eh als Bildersuche anzeigen müssen
                if (!in_array($fen, $realEngNames) && $fok !== "bilder") {
843 844 845
                    $isFokus = false;
                }
            }
846
            # Jede im Fokus erwartete Engine ist auch eingeschaltet
847 848
            foreach ($realEngNames as $ren) {
                if (!in_array($ren, $fokiEngNames)) {
849 850 851
                    $isFokus = false;
                }
            }
852
            # Wenn die Listen identisch sind, setze den Fokus um
853
            if ($isFokus) {
854 855 856
                $this->fokus = $fok;
            }
        }
857
    }
858

859 860 861 862 863 864 865
    public function checkCanNotSitesearch($enabledSearchengines)
    {
        if (strlen($this->site) > 0) {
            $enginesWithSite = 0;
            foreach ($enabledSearchengines as $engine) {
                if (isset($engine['hasSiteSearch']) && $engine['hasSiteSearch']->__toString() === "1") {
                    $enginesWithSite++;
866
                }
867 868
            }
            if ($enginesWithSite === 0) {
869
                $this->errors[] = trans('metaGer.sitesearch.failed', ['site' => $this->site, 'searchLink' => $this->generateSearchLink("web", false)]);
870 871
                return true;
            } else {
872
                $this->warnings[] = trans('metaGer.sitesearch.success', ['site' => $this->site]);
873
                return false;
874 875
            }
        }
876
        return false;
877 878 879 880
    }

    public function waitForResults($enginesToLoad, $overtureEnabled, $canBreak)
    {
881

Phil Höfer's avatar
Phil Höfer committed
882
        $timeStart = microtime(true);
883
        $results = null;
884
        while (true) {
885 886 887
            $results = Redis::hgetall('search.' . $this->getHashCode());

            $ready = true;
Phil Höfer's avatar
Phil Höfer committed
888
            // When every
Dominik Hebeler's avatar