MetaGer.php 58.3 KB
Newer Older
1 2 3 4
<?php
namespace App;

use App;
5
use Cache;
6
use Carbon;
7
use Illuminate\Http\Request;
Phil Höfer's avatar
Phil Höfer committed
8
use Illuminate\Support\Facades\Redis;
9
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix  
Dominik Hebeler committed
10
use LaravelLocalization;
11
use Log;
12
use Predis\Connection\ConnectionException;
13 14 15

class MetaGer
{
16 17 18 19 20 21 22 23 24 25
    # Einstellungen für die Suche
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $category;
    protected $time;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
26
    protected $hostBlacklist = [];
27
    protected $domainBlacklist = [];
28 29 30 31 32 33 34 35 36
    private $urlBlacklist = [];
    protected $stopWords = [];
    protected $phrases = [];
    protected $engines = [];
    protected $results = [];
    protected $ads = [];
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
37
    protected $availableFoki = [];
38 39
    protected $startCount = 0;
    protected $canCache = false;
40
    # Daten über die Abfrage$
41
    protected $ip;
Dominik Hebeler's avatar
Dominik Hebeler committed
42
    protected $useragent;
43 44
    protected $language;
    protected $agent;
45
    protected $apiKey = "";
Phil Höfer's avatar
Phil Höfer committed
46
    protected $apiAuthorized = false;
47 48 49 50 51
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
52
    protected $maps;
53
    protected $newtab;
54
    protected $domainsBlacklisted = [];
55
    protected $urlsBlacklisted = [];
56
    protected $url;
57
    protected $fullUrl;
58
    protected $languageDetect;
Dominik Hebeler's avatar
Dominik Hebeler committed
59 60
    protected $verificationId;
    protected $verificationCount;
61

62 63
    public function __construct()
    {
64
        # Timer starten
65
        $this->starttime = microtime(true);
66
        # Versuchen Blacklists einzulesen
67
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
68
            $tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
69
            $this->domainsBlacklisted = explode("\n", $tmp);
70 71
            $tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted = explode("\n", $tmp);
72
        } else {
73
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
74 75
        }

76
        # Parser Skripte einhängen
77 78 79 80
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
81
                require_once $path;
82 83 84
            }
        }

85
        # Cachebarkeit testen
86 87 88 89 90 91
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
92
    }
93

94
    # Erstellt aus den gesammelten Ergebnissen den View
95
    public function createView($quicktipResults = [])
96
    {
97 98
        # Hiermit werden die evtl. ausgewählten SuMas extrahiert, damit die Input-Boxen richtig gesetzt werden können
        $focusPages = [];
Aria Givi's avatar
Aria Givi committed
99

100
        foreach ($this->request->all() as $key => $value) {
101 102
            if (starts_with($key, 'engine_') && $value === 'on') {
                $focusPages[] = $key;
103 104 105
            }
        }

106
        $viewResults = [];
107
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
108
        foreach ($this->results as $result) {
109 110 111 112
            $viewResults[] = get_object_vars($result);
        }
        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();
113 114
        if ($this->fokus === "bilder") {
            switch ($this->out) {
115
                case 'results':
116
                    return view('resultpages.results_images')
117 118 119 120 121
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
122
                        ->with('apiAuthorized', $this->apiAuthorized)
123
                        ->with('metager', $this)
124
                        ->with('browser', (new Agent())->browser());
125
                default:
126
                    return view('resultpages.resultpage_images')
127 128 129 130 131
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
132
                        ->with('apiAuthorized', $this->apiAuthorized)
133
                        ->with('metager', $this)
134
                        ->with('browser', (new Agent())->browser());
135
            }
136 137 138
        } else {
            switch ($this->out) {
                case 'results':
139
                    return view('resultpages.results')
140 141 142 143 144
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
145
                        ->with('apiAuthorized', $this->apiAuthorized)
146
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
147 148
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
149 150
                    break;
                case 'results-with-style':
151
                    return view('resultpages.resultpage')
152 153 154 155 156
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
157
                        ->with('apiAuthorized', $this->apiAuthorized)
158 159
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
Dominik Hebeler's avatar
Dominik Hebeler committed
160 161
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
162
                    break;
Phil Höfer's avatar
Phil Höfer committed
163
                case 'rich':
164
                    return view('resultpages.metager3rich')
165
                        ->with('results', $viewResults)
Phil Höfer's avatar
Phil Höfer committed
166 167 168 169
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
170
                        ->with('apiAuthorized', $this->apiAuthorized)
Phil Höfer's avatar
Phil Höfer committed
171
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
172 173
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
Phil Höfer's avatar
Phil Höfer committed
174
                    break;
175
                case 'rss20':
176
                    return view('resultpages.metager3resultsrss20')
177 178
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
Phil Höfer's avatar
Phil Höfer committed
179
                        ->with('apiAuthorized', $this->apiAuthorized)
180
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
181 182
                        ->with('resultcount', sizeof($viewResults))
                        ->with('fokus', $this->fokus);
183
                    break;
184 185 186
                case 'api':
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'apiAuthorized' => $this->apiAuthorized])->header('Content-Type', 'application/xml');
                    break;
Aria Givi's avatar
Aria Givi committed
187
                case 'atom10':
188
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'apiAuthorized' => true])
189
                        ->header('Content-Type', 'application/xml');
Aria Givi's avatar
Aria Givi committed
190
                    break;
191
                case 'result-count':
192 193
                    # Wir geben die Ergebniszahl und die benötigte Zeit zurück:
                    return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
194
                    break;
195
                default:
196
                    return view('resultpages.resultpage')
197
                        ->with('eingabe', $this->eingabe)
198
                        ->with('focusPages', $focusPages)
199 200 201
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
202
                        ->with('apiAuthorized', $this->apiAuthorized)
203
                        ->with('metager', $this)
204
                        ->with('browser', (new Agent())->browser())
205
                        ->with('quicktips', $quicktipResults)
206 207
                        ->with('resultcount', count($this->results))
                        ->with('focus', $this->fokus);
208 209
                    break;
            }
210 211 212
        }
    }

Phil Höfer's avatar
Phil Höfer committed
213
    public function prepareResults()
214
    {
Phil Höfer's avatar
Phil Höfer committed
215
        $engines = $this->engines;
216

Phil Höfer's avatar
Phil Höfer committed
217 218
        // combine
        $combinedResults = $this->combineResults($engines);
219 220 221 222

        # Wir bestimmen die Sprache eines jeden Suchergebnisses
        $this->results = $this->addLangCodes($this->results);

Phil Höfer's avatar
Phil Höfer committed
223 224 225 226 227
        // sort
        //$sortedResults = $this->sortResults($engines);
        // filter
        // augment (boost&adgoal)
        // authorize
Phil Höfer's avatar
Phil Höfer committed
228 229 230
        if ($this->apiKey) {
            $this->apiAuthorized = $this->authorize($this->apiKey);
        }
Phil Höfer's avatar
Phil Höfer committed
231
        // misc (WiP)
232
        if ($this->fokus == "nachrichten") {
233 234 235
            $this->results = array_filter($this->results, function ($v, $k) {
                return !is_null($v->getRank());
            }, ARRAY_FILTER_USE_BOTH);
236 237 238 239 240 241 242 243 244 245
            uasort($this->results, function ($a, $b) {
                $datea = $a->getDate();
                $dateb = $b->getDate();
                return $dateb - $datea;
            });
        } else {
            uasort($this->results, function ($a, $b) {
                if ($a->getRank() == $b->getRank()) {
                    return 0;
                }
246

247 248 249
                return ($a->getRank() < $b->getRank()) ? 1 : -1;
            });
        }
250

251 252
        # Validate Results
        $newResults = [];
253 254
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
255
                $newResults[] = $result;
256 257
            }

258 259 260
        }
        $this->results = $newResults;

261
        #Adgoal Implementation
262 263 264
        if (!$this->apiAuthorized) {
            $this->results = $this->parseAdgoal($this->results);
        }
265

Dominik Hebeler's avatar
Dominik Hebeler committed
266 267
        # Human Verification
        $this->results = $this->humanVerification($this->results);
268
        $this->ads = $this->humanVerification($this->ads);
Dominik Hebeler's avatar
Dominik Hebeler committed
269

270
        $counter = 0;
271
        $firstRank = 0;
272

273
        if (isset($this->startForwards)) {
274
            $this->startCount = $this->startForwards;
275
        } elseif (isset($this->startBackwards)) {
276
            $this->startCount = $this->startBackwards - count($this->results) - 1;
277
        } else {
278 279 280
            $this->startCount = 0;
        }

281 282
        foreach ($this->results as $result) {
            if ($counter === 0) {
283
                $firstRank = $result->rank;
284 285
            }

286
            $counter++;
287
            $result->number = $counter + $this->startCount;
288
            $confidence = 0;
289 290 291
            if ($firstRank > 0) {
                $confidence = $result->rank / $firstRank;
            } else {
292
                $confidence = 0;
293 294 295
            }

            if ($confidence > 0.65) {
296
                $result->color = "#FF4000";
297
            } elseif ($confidence > 0.4) {
298
                $result->color = "#FF0080";
299
            } elseif ($confidence > 0.2) {
300
                $result->color = "#C000C0";
301
            } else {
302
                $result->color = "#000000";
303 304
            }

305 306
        }

307
        if ($this->validated) {
308 309
            $this->ads = [];
            $this->maps = false;
310
        }
311 312

        if (count($this->results) <= 0) {
313 314 315 316 317 318
            if (strlen($this->site) > 0) {
                $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
                $this->errors[] = trans('metaGer.results.failedSitesearch', ['altSearch' => $no_sitesearch_query]);
            } else {
                $this->errors[] = trans('metaGer.results.failed');
            }
319
        }
320

321
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
322
            $page = $this->page + 1;
323
            $this->next = [
324
                'page' => $page,
325
                'startForwards' => $this->results[count($this->results) - 1]->number,
326
                'engines' => $this->next,
327 328
            ];
            Cache::put(md5(serialize($this->next)), serialize($this->next), 60);
329 330
        } else {
            $this->next = [];
331 332
        }

333
    }
334

335 336
    private function addLangCodes($results)
    {
337 338 339 340 341
        # Wenn es keine Ergebnisse gibt, brauchen wir uns gar nicht erst zu bemühen
        if (sizeof($results) === 0) {
            return $results;
        }

342 343 344 345 346
        # Bei der Spracheinstellung "all" wird nicht gefiltert
        if ($this->getLang() === "all") {
            return $results;
        } else {
            # Ansonsten müssen wir jedem Result einen Sprachcode hinzufügen
347
            $id = 0;
348 349 350 351 352
            $langStrings = [];
            foreach ($results as $result) {
                # Wir geben jedem Ergebnis eine ID um später die Sprachcodes zuordnen zu können
                $result->id = $id;

Dominik Hebeler's avatar
Dominik Hebeler committed
353
                $langStrings["result_" . $id] = utf8_encode($result->getLangString());
354 355 356 357 358 359 360 361

                $id++;
            }
            # Wir schreiben die Strings in eine temporäre JSON-Datei,
            # Da das Array unter umständen zu groß ist für eine direkte Übergabe an das Skript
            $filename = "/tmp/" . getmypid();
            file_put_contents($filename, json_encode($langStrings));
            $langDetectorPath = app_path() . "/Models/lang.pl";
362 363
            $lang = exec("echo '$filename' | $langDetectorPath");
            $lang = json_decode($lang, true);
364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380

            # Wir haben nun die Sprachcodes der einzelnen Ergebnisse.
            # Diese müssen wir nur noch korrekt zuordnen, dann sind wir fertig.
            foreach ($lang as $key => $langCode) {
                # Prefix vom Key entfernen:
                $id = intval(str_replace("result_", "", $key));
                foreach ($this->results as $result) {
                    if ($result->id === $id) {
                        $result->langCode = $langCode;
                        break;
                    }
                }
            }
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400
    public function combineResults($engines)
    {
        foreach ($engines as $engine) {
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
            foreach ($engine->results as $result) {
                if ($result->valid) {
                    $this->results[] = $result;
                }
            }
            foreach ($engine->ads as $ad) {
                $this->ads[] = $ad;
            }
        }
    }

401 402
    public function parseAdgoal($results)
    {
403
        $publicKey = getenv('adgoal_public');
404
        $privateKey = getenv('adgoal_private');
405
        if ($publicKey === false) {
406 407 408
            return $results;
        }
        $tldList = "";
409 410
        try {
            foreach ($results as $result) {
411
                $link = $result->anzeigeLink;
412
                if (strpos($link, "http") !== 0) {
413 414 415 416 417 418 419 420 421 422
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

423
            # Query
424 425
            $query = urlencode($this->q);

426
            $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . $publicKey . "&k=" . $hash . "&tld=" . $tldList . "&q=" . $query;
427 428 429
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
430
            foreach ($answer as $el) {
431
                $hoster = $el[0];
432
                $hash = $el[1];
433

434
                foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
435
                    if ($hoster === $result->tld && !$result->partnershop) {
436 437
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
438
                        if ($result->image !== "") {
439
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . $hash . ".gif";
440
                        } else {
441
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . $hash . ".gif";
442 443
                        }

444 445 446
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
                        $targetUrl = $result->anzeigeLink;
447
                        if (strpos($targetUrl, "http") !== 0) {
448
                            $targetUrl = "http://" . $targetUrl;
449 450
                        }

451 452 453
                        $gateHash = md5($targetUrl . $privateKey);
                        $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . $publicKey . "&k=" . $gateHash . "&url=" . urlencode($targetUrl) . "&q=" . $query;
                        $result->link = $newLink;
454 455 456 457
                        $result->partnershop = true;
                    }
                }
            }
458
        } catch (\ErrorException $e) {
459 460 461 462 463
            return $results;
        }

        return $results;
    }
464

465 466
    public function humanVerification($results)
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
467
        # Let's check if we need to implement a redirect for human verification
468 469
        if ($this->verificationCount > 10) {
            foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
470 471 472
                $link = $result->link;
                $day = Carbon::now()->day;
                $pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
473 474 475
                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
Dominik Hebeler's avatar
Dominik Hebeler committed
476
                $result->link = $url;
477
                $result->proxyLink = $proxyUrl;
Dominik Hebeler's avatar
Dominik Hebeler committed
478 479
            }
            return $results;
480
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
481 482 483 484
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
485 486 487 488 489 490
    public function authorize($key)
    {
        $postdata = http_build_query(array(
            'dummy' => rand(),
        ));
        $opts = array('http' => array(
491 492
            'method' => 'POST',
            'header' => 'Content-type: application/x-www-form-urlencoded',
Phil Höfer's avatar
Phil Höfer committed
493 494 495 496 497 498 499
            'content' => $postdata,
        ),
        );

        $context = stream_context_create($opts);

        try {
500
            $link = "https://key.metager3.de/" . urlencode($key) . "/request-permission/api-access";
Phil Höfer's avatar
Phil Höfer committed
501 502 503 504 505 506 507 508 509 510 511 512
            $result = json_decode(file_get_contents($link, false, $context));
            if ($result->{'api-access'} == true) {
                return true;
            } else {
                return false;
            }

        } catch (\ErrorException $e) {
            return false;
        }
    }

513 514
    public function createQuicktips()
    {
515 516 517 518 519
        # Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
        $quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime(), $this->getHashCode());
        return $quicktips;
    }

Karl's avatar
Karl committed
520 521 522 523
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

524 525
    public function createSearchEngines(Request $request)
    {
526
        # Wenn es kein Suchwort gibt
Dominik Hebeler's avatar
Dominik Hebeler committed
527
        if (!$request->filled("eingabe") || $this->q === "") {
528
            return;
529
        }
530

531 532
        $xml = simplexml_load_file($this->sumaFile);
        $sumas = $xml->xpath("suma");
533
        $enabledSearchengines = [];
534 535
        $overtureEnabled = false;
        $sumaCount = 0;
536

537
        /*
538 539
         * Erstellt eine Liste mit Foki, die verfügbar sind
         */
540 541 542 543 544 545 546 547 548 549 550
        $this->availableFoki = [];
        foreach ($sumas as $suma) {
            $foki = explode(",", trim($suma["type"]));
            foreach ($foki as $fokus) {
                if (!empty($fokus)) {
                    $this->availableFoki[$fokus] = "available";
                }

            }
        }

551
        $isCustomSearch = $this->startsWith($this->fokus, 'focus_');
552

553
        # Im Falle einer Custom-Suche ohne mindestens einer selektierter Suchmaschine wird eine Web-Suche durchgeführt
554
        if ($isCustomSearch && !$this->atLeastOneSearchengineSelected($request)) {
555 556 557
            $isCustomSearch = false;
            $this->fokus = 'web';
        }
558

Karl's avatar
Karl committed
559
        /* Erstellt die Liste der eingestellten Sumas
560 561 562
         * Der einzige Unterschied bei angepasstem Suchfokus ist,
         * dass nicht nach den Typen einer Suma,
         * sondern den im Request mitgegebenen Typen entschieden wird.
Karl's avatar
Karl committed
563 564 565 566 567 568
         * Ansonsten wird genau das selbe geprüft und gemacht:
         * Handelt es sich um spezielle Suchmaschinen die immer an sein müssen
         * Wenn es Overture ist vermerken dass Overture an ist
         * Suma Zähler erhöhen
         * Zu Liste hinzufügen
         */
569
        foreach ($sumas as $suma) {
570
            if (($this->sumaIsSelected($suma, $request, $isCustomSearch)
571
                || (!$this->isBildersuche()
572
                    && $this->sumaIsAdsuche($suma, $overtureEnabled)))
573 574 575
                && (!$this->sumaIsDisabled($suma))) {
                if ($this->sumaIsOverture($suma)) {
                    $overtureEnabled = true;
576
                }
577 578
                if ($this->sumaIsNotAdsuche($suma)) {
                    $sumaCount += 1;
579
                }
580
                $enabledSearchengines[] = $suma;
581 582 583 584
            }
        }

        # Sonderregelung für alle Suchmaschinen, die zu den Minisuchern gehören. Diese können alle gemeinsam über einen Link abgefragt werden
585
        $subcollections = [];
586 587

        $tmp = [];
588 589 590 591
        // Es gibt den Schalter "minism=on" Dieser soll bewirken, dass alle Minisucher angeschaltet werden.
        // Wenn also "minism=on" ist, dann durchsuchen wir statt den tatsächlich angeschalteten Suchmaschinen,
        // alle Suchmaschinen nach "minismCollection"
        if ($request->input("minism", "off") === "on") {
Dominik Hebeler's avatar
Dominik Hebeler committed
592 593 594
            // Wir laden alle Minisucher
            foreach ($sumas as $engine) {
                if (isset($engine["minismCollection"])) {
595
                    $subcollections[] = $engine["minismCollection"]->__toString();
Dominik Hebeler's avatar
Dominik Hebeler committed
596 597
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
598 599 600 601 602 603
            # Nur noch alle eventuell angeschalteten Minisucher deaktivieren
            foreach ($enabledSearchengines as $index => $engine) {
                if (!isset($engine["minismCollection"])) {
                    $tmp[] = $engine;
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
604 605 606 607
        } else {
            // Wir schalten eine Teilmenge, oder aber gar keine an
            foreach ($enabledSearchengines as $engine) {
                if (isset($engine['minismCollection'])) {
608
                    $subcollections[] = $engine['minismCollection']->__toString();
Dominik Hebeler's avatar
Dominik Hebeler committed
609 610
                } else {
                    $tmp[] = $engine;
Dominik Hebeler's avatar
Dominik Hebeler committed
611
                }
612
            }
613 614
        }
        $enabledSearchengines = $tmp;
615 616
        if (sizeof($subcollections) > 0) {
            $enabledSearchengines[] = $this->loadMiniSucher($xml, $subcollections);
617
        }
618
        if ($sumaCount <= 0) {
619
            $this->errors[] = trans('metaGer.settings.noneSelected');
620
        }
621
        $engines = [];
622
        # Wenn eine Sitesearch durchgeführt werden soll, überprüfen wir ob überhaupt eine der Suchmaschinen eine Sitesearch unterstützt
623
        $siteSearchFailed = $this->checkCanNotSitesearch($enabledSearchengines);
624 625

        $typeslist = [];
626
        $counter = 0;
627

628 629
        if ($this->requestIsCached($request)) {
            $engines = $this->getCachedEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
630
            # We need to edit some Options of the Cached Search Engines
Phil Höfer's avatar
Phil Höfer committed
631
            foreach ($engines as $engine) {
Dominik Hebeler's avatar
Dominik Hebeler committed
632 633
                $engine->setResultHash($this->getHashCode());
            }
634
        } else {
635
            $engines = $this->actuallyCreateSearchEngines($enabledSearchengines, $siteSearchFailed);
636
        }
637

638
        # Wir starten alle Suchen
639 640
        foreach ($engines as $engine) {
            $engine->startSearch($this);
641
        }
642

643 644 645 646 647
        /* Wir warten auf die Antwort der Suchmaschinen
         * Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
         * Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
         * $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
         * welche Suchmaschinen nicht rechtzeitig geantwortet haben.
648 649
         */

650
        $enginesToLoad = [];
651
        $canBreak = false;
652 653 654 655 656
        foreach ($engines as $engine) {
            if ($engine->cached) {
                if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
                    $canBreak = true;
                }
657 658
            } else {
                $enginesToLoad[$engine->name] = false;
659 660 661 662 663 664 665 666
            }
        }

        $this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);

        $this->retrieveResults($engines);
    }

667 668
    # Spezielle Suchen und Sumas

669
    public function sumaIsSelected($suma, $request, $custom)
670
    {
671
        if ($custom) {
672
            if ($request->filled("engine_" . strtolower($suma["name"]))) {
673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696
                return true;
            }
        } else {
            $types = explode(",", $suma["type"]);
            if (in_array($this->fokus, $types)) {
                return true;
            }
        }
        return false;
    }

    public function actuallyCreateSearchEngines($enabledSearchengines, $siteSearchFailed)
    {
        $engines = [];
        foreach ($enabledSearchengines as $engine) {

            # Wenn diese Suchmaschine gar nicht eingeschaltet sein soll
            if (!$siteSearchFailed
                && strlen($this->site) > 0
                && (!isset($engine['hasSiteSearch'])
                    || $engine['hasSiteSearch']->__toString() === "0")) {
                continue;
            }

Dominik Hebeler's avatar
Dominik Hebeler committed
697 698 699
            if (!isset($engine["package"])) {
                die(var_dump($engine));
            }
700 701 702 703 704
            # Setze Pfad zu Parser
            $path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString());

            # Prüfe ob Parser vorhanden
            if (!file_exists(app_path() . "/Models/parserSkripte/" . ucfirst($engine["package"]->__toString()) . ".php")) {
705
                Log::error("Konnte " . $engine["name"] . " nicht abfragen, da kein Parser existiert");
706
                $this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine["name"]]);
707 708 709 710 711 712 713 714
                continue;
            }

            # Es wird versucht die Suchengine zu erstellen
            $time = microtime();
            try {
                $tmp = new $path($engine, $this);
            } catch (\ErrorException $e) {
715
                Log::error("Konnte " . $engine["name"] . " nicht abfragen. " . var_dump($e));
716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733
                continue;
            }

            # Ausgabe bei Debug-Modus
            if ($tmp->enabled && isset($this->debug)) {
                $this->warnings[] = $tmp->service . "   Connection_Time: " . $tmp->connection_time . "    Write_Time: " . $tmp->write_time . " Insgesamt:" . ((microtime() - $time) / 1000);
            }

            # Wenn die neu erstellte Engine eingeschaltet ist, wird sie der Liste hinzugefügt
            if ($tmp->isEnabled()) {
                $engines[] = $tmp;
            }
        }
        return $engines;
    }

    public function isBildersuche()
    {
734
        return $this->fokus === "bilder";
735 736 737 738
    }

    public function sumaIsAdsuche($suma, $overtureEnabled)
    {
739
        $sumaName = $suma["name"]->__toString();
740
        return
741 742
            $sumaName === "qualigo"
            || $sumaName === "similar_product_ads"
Karl Hasselbring's avatar
Karl Hasselbring committed
743
            || (!$overtureEnabled && $sumaName === "overtureAds");
744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770
    }

    public function sumaIsDisabled($suma)
    {
        return
        isset($suma['disabled'])
        && $suma['disabled']->__toString() === "1";
    }

    public function sumaIsOverture($suma)
    {
        return
        $suma["name"]->__toString() === "overture"
        || $suma["name"]->__toString() === "overtureAds";
    }

    public function sumaIsNotAdsuche($suma)
    {
        return
        $suma["name"]->__toString() !== "qualigo"
        && $suma["name"]->__toString() !== "similar_product_ads"
        && $suma["name"]->__toString() !== "overtureAds";
    }

    public function requestIsCached($request)
    {
        return
Dominik Hebeler's avatar
Dominik Hebeler committed
771
        $request->filled('next')
772 773 774 775 776 777
        && Cache::has($request->input('next'))
        && unserialize(Cache::get($request->input('next')))['page'] > 1;
    }

    public function getCachedEngines($request)
    {
778
        $next = unserialize(Cache::get($request->input('next')));
779
        $this->page = $next['page'];
780
        $engines = $next['engines'];
781 782 783 784 785 786 787 788 789 790 791
        if (isset($next['startForwards'])) {
            $this->startForwards = $next['startForwards'];
        }
        if (isset($next['startBackwards'])) {
            $this->startBackwards = $next['startBackwards'];
        }
        return $engines;
    }

    public function loadMiniSucher($xml, $subcollections)
    {
792
        $minisucherEngine = $xml->xpath('suma[@name="minism"]')[0];
793
        $minisucherEngine["subcollections"] = implode(", ", $subcollections);
794
        $subcollectionsString = urlencode("(" . implode(" OR ", $subcollections) . ")");
795
        $minisucherEngine["formData"] = str_replace("<<SUBCOLLECTIONS>>", $subcollectionsString, $minisucherEngine["formData"]);
796 797 798 799
        $minisucherEngine["formData"] = str_replace("<<COUNT>>", sizeof($subcollections) * 10, $minisucherEngine["formData"]);
        return $minisucherEngine;
    }

800
    # Passt den Suchfokus an, falls für einen Fokus genau alle vorhandenen Sumas eingeschaltet sind
801 802
    public function adjustFocus($sumas, $enabledSearchengines)
    {
803 804
        # Findet für alle Foki die enthaltenen Sumas
        $foki = []; # [fokus][suma] => [suma]
805
        foreach ($sumas as $suma) {
806
            if ((!$this->sumaIsDisabled($suma)) && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
807
                if (isset($suma['type'])) {
808 809 810
                    # Wenn foki für diese Suchmaschine angegeben sind
                    $focuses = explode(",", $suma['type']->__toString());
                    foreach ($focuses as $foc) {
811 812 813 814 815
                        if (isset($suma['minismCollection'])) {
                            $foki[$foc][] = "minism";
                        } else {
                            $foki[$foc][] = $suma['name']->__toString();
                        }
816
                    }
817
                } else {
818
                    # Wenn keine foki für diese Suchmaschine angegeben sind
819 820 821 822 823
                    if (isset($suma['minismCollection'])) {
                        $foki["andere"][] = "minism";
                    } else {
                        $foki["andere"][] = $suma['name']->__toString();
                    }
824 825 826 827
                }
            }
        }

828
        # Findet die Namen der aktuell eingeschalteten Sumas
829
        $realEngNames = [];
830
        foreach ($enabledSearchengines as $realEng) {
831
            $nam = $realEng["name"]->__toString();
Karl Hasselbring's avatar
Karl Hasselbring committed
832
            if ($nam !== "qualigo" && $nam !== "overtureAds") {
833 834 835
                $realEngNames[] = $nam;
            }
        }
836

837
        # Anschließend werden diese beiden Listen verglichen (jeweils eine der Fokuslisten für jeden Fokus), um herauszufinden ob sie vielleicht identisch sind. Ist dies der Fall, so hat der Nutzer anscheinend Suchmaschinen eines kompletten Fokus eingestellt. Der Fokus wird dementsprechend angepasst.
838
        foreach ($foki as $fok => $engines) {
839
            $isFokus = true;
840
            $fokiEngNames = [];
841
            foreach ($engines as $eng) {
842 843
                $fokiEngNames[] = $eng;
            }
844
            # Jede eingeschaltete Engine ist für diesen Fokus geeignet
845
            foreach ($fokiEngNames as $fen) {
846 847
                # Bei Bildersuchen ist uns egal, ob alle Suchmaschinen aus dem Suchfokus eingeschaltet sind, da wir sie eh als Bildersuche anzeigen müssen
                if (!in_array($fen, $realEngNames) && $fok !== "bilder") {
848 849 850
                    $isFokus = false;
                }
            }
851
            # Jede im Fokus erwartete Engine ist auch eingeschaltet
852 853
            foreach ($realEngNames as $ren) {
                if (!in_array($ren, $fokiEngNames)) {
854 855 856
                    $isFokus = false;
                }
            }
857
            # Wenn die Listen identisch sind, setze den Fokus um
858
            if ($isFokus) {
859 860 861
                $this->fokus = $fok;
            }
        }
862
    }
863

864 865 866 867 868 869 870
    public function checkCanNotSitesearch($enabledSearchengines)
    {
        if (strlen($this->site) > 0) {
            $enginesWithSite = 0;
            foreach ($enabledSearchengines as $engine) {
                if (isset($engine['hasSiteSearch']) && $engine['hasSiteSearch']->__toString() === "1") {
                    $enginesWithSite++;
871
                }
872 873
            }
            if ($enginesWithSite === 0) {
874
                $this->errors[] = trans('metaGer.sitesearch.failed', ['site' => $this->site, 'searchLink' => $this->generateSearchLink("web", false)]);
875 876
                return true;
            } else {
877
                $this->warnings[] = trans('metaGer.sitesearch.success', ['site' => $this->site]);
878
                return false;
879 880
            }
        }
881
        return false;
882 883 884 885
    }

    public function waitForResults($enginesToLoad, $overtureEnabled, $canBreak)
    {