MetaGer.php 61.6 KB
Newer Older
1 2 3 4
<?php
namespace App;

use App;
5
use Cache;
6
use Carbon;
7
use Illuminate\Http\Request;
Phil Höfer's avatar
Phil Höfer committed
8
use Illuminate\Support\Facades\Redis;
9
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix  
Dominik Hebeler committed
10
use LaravelLocalization;
11
use Log;
12
use Predis\Connection\ConnectionException;
13 14 15

class MetaGer
{
16
    # Einstellungen für die Suche
17 18
    public $alteredQuery = "";
    public $alterationOverrideQuery = "";
19 20 21 22 23 24 25
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
26
    protected $time = 2000;
27
    protected $hostBlacklist = [];
28
    protected $domainBlacklist = [];
29 30 31 32
    private $urlBlacklist = [];
    protected $stopWords = [];
    protected $phrases = [];
    protected $engines = [];
33
    protected $totalResults = 0;
34
    protected $results = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
35 36
    protected $queryFilter = [];
    protected $parameterFilter = [];
37
    protected $ads = [];
38
    protected $infos = [];
39 40 41
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
42
    protected $availableFoki = [];
43 44
    protected $startCount = 0;
    protected $canCache = false;
45
    # Daten über die Abfrage$
46
    protected $ip;
Dominik Hebeler's avatar
Dominik Hebeler committed
47
    protected $useragent;
48 49
    protected $language;
    protected $agent;
50
    protected $apiKey = "";
Phil Höfer's avatar
Phil Höfer committed
51
    protected $apiAuthorized = false;
52
    protected $next = [];
53 54 55 56 57
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
58
    protected $newtab;
59
    protected $domainsBlacklisted = [];
60
    protected $adDomainsBlacklisted = [];
61
    protected $urlsBlacklisted = [];
62
    protected $adUrlsBlacklisted = [];
63
    protected $url;
64
    protected $fullUrl;
65
    protected $languageDetect;
Dominik Hebeler's avatar
Dominik Hebeler committed
66 67
    protected $verificationId;
    protected $verificationCount;
68
    protected $searchUid;
69
    protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult, $redisCurrentResultList;
70

71
    public function __construct($hash = "")
72
    {
73
        # Timer starten
74
        $this->starttime = microtime(true);
75
        # Versuchen Blacklists einzulesen
76
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
77
            $tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
78
            $this->domainsBlacklisted = explode("\n", $tmp);
79 80
            $tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted = explode("\n", $tmp);
81
        } else {
82
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
83
        }
84 85 86 87 88 89 90 91 92
        # Versuchen Blacklists einzulesen
        if (file_exists(config_path() . "/adBlacklistDomains.txt") && file_exists(config_path() . "/adBlacklistUrl.txt")) {
            $tmp = file_get_contents(config_path() . "/adBlacklistDomains.txt");
            $this->adDomainsBlacklisted = explode("\n", $tmp);
            $tmp = file_get_contents(config_path() . "/adBlacklistUrl.txt");
            $this->adUrlsBlacklisted = explode("\n", $tmp);
        } else {
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
        }
93

94
        # Parser Skripte einhängen
95 96 97 98
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
99
                require_once $path;
100 101 102
            }
        }

103
        # Cachebarkeit testen
104 105 106 107 108 109
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
110 111 112 113 114
        if ($hash === "") {
            $this->searchUid = md5(uniqid());
        } else {
            $this->searchUid = $hash;
        }
115 116 117 118 119 120 121
        $redisPrefix = "search";
        # This is a list on which the MetaGer process can do a blocking pop to wait for new results
        $this->redisResultWaitingKey = $redisPrefix . "." . $this->searchUid . ".ready";
        # This is a list of searchengines which have delivered results for this search
        $this->redisResultEngineList = $redisPrefix . "." . $this->searchUid . ".engines";
        # This is the key where the results of the engine are stored as well as some statistical data
        $this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results.";
122 123 124
        # A list of all search results already delivered to the user (sorted of course)
        $this->redisCurrentResultList = $redisPrefix . "." . $this->searchUid . ".currentResults";

125
    }
126

127
    # Erstellt aus den gesammelten Ergebnissen den View
128
    public function createView($quicktipResults = [])
129
    {
130 131
        # Hiermit werden die evtl. ausgewählten SuMas extrahiert, damit die Input-Boxen richtig gesetzt werden können
        $focusPages = [];
Aria Givi's avatar
Aria Givi committed
132

133
        foreach ($this->request->all() as $key => $value) {
134 135
            if (starts_with($key, 'engine_') && $value === 'on') {
                $focusPages[] = $key;
136 137 138
            }
        }

139
        $viewResults = [];
140
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
141
        foreach ($this->results as $result) {
142 143 144 145
            $viewResults[] = get_object_vars($result);
        }
        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();
146 147
        if ($this->fokus === "bilder") {
            switch ($this->out) {
148
                case 'results':
149
                    return view('resultpages.results_images')
150 151 152 153 154
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
155
                        ->with('apiAuthorized', $this->apiAuthorized)
156
                        ->with('metager', $this)
157
                        ->with('browser', (new Agent())->browser());
158
                default:
159
                    return view('resultpages.resultpage_images')
160 161 162 163 164
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
165
                        ->with('apiAuthorized', $this->apiAuthorized)
166
                        ->with('metager', $this)
167 168 169 170
                        ->with('browser', (new Agent())->browser())
                        ->with('quicktips', $quicktipResults)
                        ->with('focus', $this->fokus)
                        ->with('resultcount', count($this->results));
171
            }
172 173 174
        } else {
            switch ($this->out) {
                case 'results':
175
                    return view('resultpages.results')
176 177 178 179 180
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
181
                        ->with('apiAuthorized', $this->apiAuthorized)
182
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
183 184
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
185 186
                    break;
                case 'results-with-style':
187
                    return view('resultpages.resultpage')
188 189 190 191 192
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
193
                        ->with('apiAuthorized', $this->apiAuthorized)
194 195
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
Dominik Hebeler's avatar
Dominik Hebeler committed
196 197
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
198
                    break;
Phil Höfer's avatar
Phil Höfer committed
199
                case 'rich':
200
                    return view('resultpages.metager3rich')
201
                        ->with('results', $viewResults)
Phil Höfer's avatar
Phil Höfer committed
202 203 204 205
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
206
                        ->with('apiAuthorized', $this->apiAuthorized)
Phil Höfer's avatar
Phil Höfer committed
207
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
208 209
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
Phil Höfer's avatar
Phil Höfer committed
210
                    break;
211
                case 'rss20':
212
                    return view('resultpages.metager3resultsrss20')
213 214
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
Phil Höfer's avatar
Phil Höfer committed
215
                        ->with('apiAuthorized', $this->apiAuthorized)
216
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
217 218
                        ->with('resultcount', sizeof($viewResults))
                        ->with('fokus', $this->fokus);
219
                    break;
220
                case 'api':
Phil Höfer's avatar
Phil Höfer committed
221
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'key' => $this->apiKey, 'apiAuthorized' => $this->apiAuthorized])->header('Content-Type', 'application/xml');
222
                    break;
Aria Givi's avatar
Aria Givi committed
223
                case 'atom10':
Phil Höfer's avatar
Phil Höfer committed
224
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'key' => $this->apiKey, 'apiAuthorized' => true])
225
                        ->header('Content-Type', 'application/xml');
Aria Givi's avatar
Aria Givi committed
226
                    break;
227
                case 'result-count':
228 229
                    # Wir geben die Ergebniszahl und die benötigte Zeit zurück:
                    return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
230
                    break;
231
                default:
232
                    return view('resultpages.resultpage')
233
                        ->with('eingabe', $this->eingabe)
234
                        ->with('focusPages', $focusPages)
235 236 237
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
238
                        ->with('apiAuthorized', $this->apiAuthorized)
239
                        ->with('metager', $this)
240
                        ->with('browser', (new Agent())->browser())
241
                        ->with('quicktips', $quicktipResults)
242 243
                        ->with('resultcount', count($this->results))
                        ->with('focus', $this->fokus);
244 245
                    break;
            }
246 247 248
        }
    }

Phil Höfer's avatar
Phil Höfer committed
249
    public function prepareResults()
250
    {
Phil Höfer's avatar
Phil Höfer committed
251 252
        $engines = $this->engines;
        // combine
253
        $this->combineResults($engines);
Phil Höfer's avatar
Phil Höfer committed
254
        // misc (WiP)
255
        if ($this->fokus == "nachrichten") {
256 257 258
            $this->results = array_filter($this->results, function ($v, $k) {
                return !is_null($v->getRank());
            }, ARRAY_FILTER_USE_BOTH);
259 260 261 262 263 264 265 266 267 268
            uasort($this->results, function ($a, $b) {
                $datea = $a->getDate();
                $dateb = $b->getDate();
                return $dateb - $datea;
            });
        } else {
            uasort($this->results, function ($a, $b) {
                if ($a->getRank() == $b->getRank()) {
                    return 0;
                }
269

270 271 272
                return ($a->getRank() < $b->getRank()) ? 1 : -1;
            });
        }
273

274 275
        # Validate Results
        $newResults = [];
276 277
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
278
                $newResults[] = $result;
279
            }
280 281 282
        }
        $this->results = $newResults;

283 284 285 286 287 288 289 290 291 292 293 294 295
        # Validate Advertisements
        $newResults = [];
        foreach ($this->ads as $ad) {
            if (($ad->strippedHost !== "" && (in_array($ad->strippedHost, $this->adDomainsBlacklisted) ||
                in_array($ad->strippedLink, $this->adUrlsBlacklisted))) ||
                ($ad->strippedHostAnzeige !== "" && (in_array($ad->strippedHostAnzeige, $this->adDomainsBlacklisted) ||
                    in_array($ad->strippedLinkAnzeige, $this->adUrlsBlacklisted)))) {
                continue;
            }
            $newResults[] = $ad;
        }
        $this->ads = $newResults;

296
        #Adgoal Implementation
297 298 299
        if (!$this->apiAuthorized) {
            $this->results = $this->parseAdgoal($this->results);
        }
300

Dominik Hebeler's avatar
Dominik Hebeler committed
301 302
        # Human Verification
        $this->results = $this->humanVerification($this->results);
303
        $this->ads = $this->humanVerification($this->ads);
Dominik Hebeler's avatar
Dominik Hebeler committed
304

305
        $counter = 0;
306
        $firstRank = 0;
307

308
        if (count($this->results) <= 0) {
309 310 311 312 313 314
            if (strlen($this->site) > 0) {
                $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
                $this->errors[] = trans('metaGer.results.failedSitesearch', ['altSearch' => $no_sitesearch_query]);
            } else {
                $this->errors[] = trans('metaGer.results.failed');
            }
315
        }
316

317
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
318
            $page = $this->page + 1;
319
            $this->next = [
320 321
                'page' => $page,
                'engines' => $this->next,
322
            ];
323
            Cache::put($this->getSearchUid(), serialize($this->next), 60);
324 325
        } else {
            $this->next = [];
326 327
        }

328
    }
329

Phil Höfer's avatar
Phil Höfer committed
330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349
    public function combineResults($engines)
    {
        foreach ($engines as $engine) {
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
            foreach ($engine->results as $result) {
                if ($result->valid) {
                    $this->results[] = $result;
                }
            }
            foreach ($engine->ads as $ad) {
                $this->ads[] = $ad;
            }
        }
    }

350 351
    public function parseAdgoal($results)
    {
352
        $publicKey = getenv('adgoal_public');
353
        $privateKey = getenv('adgoal_private');
354
        if ($publicKey === false) {
355 356 357
            return $results;
        }
        $tldList = "";
358 359
        try {
            foreach ($results as $result) {
360 361 362
                if (!$result->new) {
                    continue;
                }
363
                $link = $result->link;
364
                if (strpos($link, "http") !== 0) {
365 366 367 368 369 370 371 372 373 374
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

375
            # Query
376
            $query = $this->q;
377

378
            $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . urlencode($publicKey) . "&k=" . urlencode($hash) . "&tld=" . urlencode($tldList) . "&q=" . urlencode($query);
379 380 381
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
382
            foreach ($answer as $el) {
383
                $hoster = $el[0];
384
                $hash = $el[1];
385

386
                foreach ($results as $result) {
387
                    if ($result->new && $hoster === $result->tld && !$result->partnershop) {
388 389
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
390
                        if ($result->image !== "") {
391
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . urlencode($hash) . ".gif";
392
                        } else {
393
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . urlencode($hash) . ".gif";
394 395
                        }

396 397
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
398
                        $targetUrl = $result->link;
399

400
                        $gateHash = md5($targetUrl . $privateKey);
401
                        $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . urlencode($publicKey) . "&k=" . urlencode($gateHash) . "&url=" . urlencode($targetUrl) . "&q=" . urlencode($query);
402
                        $result->link = $newLink;
403 404 405 406
                        $result->partnershop = true;
                    }
                }
            }
407
        } catch (\ErrorException $e) {
408 409 410 411 412
            return $results;
        }

        return $results;
    }
413

414 415
    public function humanVerification($results)
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
416
        # Let's check if we need to implement a redirect for human verification
417 418
        if ($this->verificationCount > 10) {
            foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
419 420 421
                $link = $result->link;
                $day = Carbon::now()->day;
                $pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
422 423 424
                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
Dominik Hebeler's avatar
Dominik Hebeler committed
425
                $result->link = $url;
426
                $result->proxyLink = $proxyUrl;
Dominik Hebeler's avatar
Dominik Hebeler committed
427 428
            }
            return $results;
429
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
430 431 432 433
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
434 435 436 437 438 439
    public function authorize($key)
    {
        $postdata = http_build_query(array(
            'dummy' => rand(),
        ));
        $opts = array('http' => array(
440 441
            'method' => 'POST',
            'header' => 'Content-type: application/x-www-form-urlencoded',
Phil Höfer's avatar
Phil Höfer committed
442 443 444 445 446 447 448
            'content' => $postdata,
        ),
        );

        $context = stream_context_create($opts);

        try {
449
            $link = "https://key.metager3.de/" . urlencode($key) . "/request-permission/api-access";
Phil Höfer's avatar
Phil Höfer committed
450 451 452 453 454 455 456 457 458 459 460 461
            $result = json_decode(file_get_contents($link, false, $context));
            if ($result->{'api-access'} == true) {
                return true;
            } else {
                return false;
            }

        } catch (\ErrorException $e) {
            return false;
        }
    }

462 463
    public function createQuicktips()
    {
464
        # Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
465
        $quicktips = new \App\Models\Quicktips\Quicktips($this->q, LaravelLocalization::getCurrentLocale(), $this->getTime());
466 467 468
        return $quicktips;
    }

Karl's avatar
Karl committed
469 470 471 472
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

473 474
    public function createSearchEngines(Request $request)
    {
475
        # Wenn es kein Suchwort gibt
Dominik Hebeler's avatar
Dominik Hebeler committed
476
        if (!$request->filled("eingabe") || $this->q === "") {
477
            return;
478
        }
479

Dominik Hebeler's avatar
Dominik Hebeler committed
480
        $this->enabledSearchengines = [];
481
        $overtureEnabled = false;
482

483 484 485
        # Check if selected focus is valid
        if (empty($this->sumaFile->foki->{$this->fokus})) {
            $this->fokus = "web";
486
        }
487 488 489 490 491 492 493 494 495 496 497 498 499 500

        $sumaList = $this->sumaFile->foki->{$this->fokus}->sumas;

        # If the user is authorized to use adfree search we won't activate yahoo or yahoo-ads
        if ($this->apiAuthorized && ($key = array_search("yahoo", $sumaList)) !== false) {
            unset($sumaList[$key]);
            if ($this->fokus === "web") {
                $this->sumaFile->sumas->{"bing"}->{"filter-opt-in"} = false;
            }
        } elseif ($this->apiAuthorized && ($key = array_search("yahoo-ads", $sumaList)) !== false) {
            unset($sumaList[$key]);
        }

        foreach ($sumaList as $suma) {
501
            # Check if this engine is disabled and can't be used
502
            $disabled = empty($this->sumaFile->sumas->{$suma}->disabled) ? false : $this->sumaFile->sumas->{$suma}->disabled;
503
            $autoDisabled = empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) ? false : $this->sumaFile->sumas->{$suma}->{"auto-disabled"};
504 505 506
            if ($disabled || $autoDisabled
                || \Cookie::get($this->getFokus() . "_engine_" . $suma) === "off") { # Check if the user has disabled this engine
            continue;
507
            }
508 509 510
            # Check if this engine can use eventually defined query-filter
            $valid = true;
            foreach ($this->queryFilter as $queryFilter => $filter) {
Dominik Hebeler's avatar
Dominik Hebeler committed
511
                if (empty($this->sumaFile->filter->{"query-filter"}->$queryFilter->sumas->$suma)) {
512 513
                    $valid = false;
                    break;
Dominik Hebeler's avatar
Dominik Hebeler committed
514 515
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
516 517 518
            # Check if this engine can use eventually defined parameter-filter
            if ($valid) {
                foreach ($this->parameterFilter as $filterName => $filter) {
519
                    # We need to check if the searchengine supports the parameter value, too
520
                    if (empty($filter->sumas->$suma) || empty($filter->sumas->{$suma}->values->{$filter->value})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
521 522 523 524 525
                        $valid = false;
                        break;
                    }
                }
            }
526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
            # Check if this engine should only be active when filter is used
            if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"}) {
                # This search engine should only be used when a parameter filter of it is used
                $validTmp = false;
                foreach ($this->parameterFilter as $filterName => $filter) {
                    if (!empty($filter->sumas->{$suma})) {
                        $validTmp = true;
                        break;
                    }
                }
                if (!$validTmp) {
                    $valid = false;
                }

            }
541 542
            # If it can we add it
            if ($valid) {
Dominik Hebeler's avatar
Dominik Hebeler committed
543
                $this->enabledSearchengines[$suma] = $this->sumaFile->sumas->{$suma};
544
            }
545

546
        }
547

548
        # Implements Yahoo Ads if Yahoo is not enabled as a searchengine
549
        if (!$this->apiAuthorized && empty($this->enabledSearchengines["yahoo"]) && $this->fokus != "bilder" && !empty($this->sumaFile->sumas->{"yahoo-ads"})) {
550 551 552
            $this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
        }

553 554 555 556 557 558
        # Special case if search engines are disabled
        # Since bing is normally only active if a filter is set but it should be active, too if yahoo is disabled
        if ($this->getFokus() === "web" && empty($this->enabledSearchengines["yahoo"]) && \Cookie::get("web_engine_bing") !== "off") {
            $this->enabledSearchengines["bing"] = $this->sumaFile->sumas->{"bing"};
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
559
        if (sizeof($this->enabledSearchengines) === 0) {
560 561
            $filter = "";
            foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
Dominik Hebeler's avatar
Dominik Hebeler committed
562
                $filter .= trans($this->sumaFile->filter->{"query-filter"}->{$queryFilter}->name) . ",";
563 564 565 566 567
            }
            $filter = rtrim($filter, ",");
            $error = trans('metaGer.engines.noSpecialSearch', ['fokus' => trans($this->sumaFile->foki->{$this->fokus}->{"display-name"}),
                'filter' => $filter]);
            $this->errors[] = $error;
568
        }
569
        $engines = [];
570
        $typeslist = [];
571
        $counter = 0;
572 573
        $this->setEngines($request);
    }
574

575 576
    public function setEngines(Request $request, $enabledSearchengines = [])
    {
577
        if ($this->requestIsCached($request)) {
578
            # If this is a page other than 1 the request is "cached"
579
            $engines = $this->getCachedEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
580
            # We need to edit some Options of the Cached Search Engines
Phil Höfer's avatar
Phil Höfer committed
581
            foreach ($engines as $engine) {
582
                $engine->setResultHash($this->getSearchUid());
Dominik Hebeler's avatar
Dominik Hebeler committed
583
            }
584
            $this->engines = $engines;
585
        } else {
586 587 588
            if (sizeof($enabledSearchengines) > 0) {
                $this->enabledSearchengines = $enabledSearchengines;
            }
589
            $this->actuallyCreateSearchEngines($this->enabledSearchengines);
590
        }
591
    }
Dominik Hebeler's avatar
Dominik Hebeler committed
592

593 594
    public function startSearch()
    {
595
        # Wir starten alle Suchen
596
        foreach ($this->engines as $engine) {
597
            $engine->startSearch($this);
598
        }
599 600
    }

601 602
    # Spezielle Suchen und Sumas

603
    public function sumaIsSelected($suma, $request, $custom)
604
    {
605
        if ($custom) {
606
            if ($request->filled("engine_" . strtolower($suma["name"]))) {
607 608 609 610 611 612 613 614 615 616 617
                return true;
            }
        } else {
            $types = explode(",", $suma["type"]);
            if (in_array($this->fokus, $types)) {
                return true;
            }
        }
        return false;
    }

618
    public function actuallyCreateSearchEngines($enabledSearchengines)
619 620
    {
        $engines = [];
621
        foreach ($enabledSearchengines as $engineName => $engine) {
622

623
            if (!isset($engine->{"parser-class"})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
624 625
                die(var_dump($engine));
            }
626
            # Setze Pfad zu Parser
627
            $path = "App\\Models\\parserSkripte\\" . $engine->{"parser-class"};
628 629

            # Prüfe ob Parser vorhanden
630 631 632
            if (!file_exists(app_path() . "/Models/parserSkripte/" . $engine->{"parser-class"} . ".php")) {
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen, da kein Parser existiert");
                $this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine->{"display-name"}]);
633 634 635 636 637 638
                continue;
            }

            # Es wird versucht die Suchengine zu erstellen
            $time = microtime();
            try {
639
                $tmp = new $path($engineName, $engine, $this);
640
            } catch (\ErrorException $e) {
641
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen. " . var_dump($e));
642 643 644
                continue;
            }

645
            $engines[] = $tmp;
646
        }
647
        $this->engines = $engines;
648 649
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
650 651 652 653 654 655 656
    public function getAvailableParameterFilter()
    {
        $parameterFilter = $this->sumaFile->filter->{"parameter-filter"};

        $availableFilter = [];

        foreach ($parameterFilter as $filterName => $filter) {
657
            $values = $filter->values;
Dominik Hebeler's avatar
Dominik Hebeler committed
658 659 660
            # Check if any of the enabled search engines provide this filter
            foreach ($this->enabledSearchengines as $engineName => $engine) {
                if (!empty($filter->sumas->$engineName)) {
661 662 663 664 665 666 667 668 669 670
                    if (empty($availableFilter[$filterName])) {
                        $availableFilter[$filterName] = $filter;
                        unset($availableFilter[$filterName]->values);
                    }
                    if (empty($availableFilter[$filterName]->values)) {
                        $availableFilter[$filterName]->values = (object) ["" => $values->{""}];
                    }
                    foreach ($filter->sumas->{$engineName}->values as $key => $value) {
                        $availableFilter[$filterName]->values->$key = $values->$key;
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
671 672
                }
            }
673 674
            # We will also add the filter from the opt-in search engines (the searchengines that are only used when a filter of it is too)
            foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
675
                if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"} && \Cookie::get($this->getFokus() . "_engine_" . $suma) !== "off") {
676
                    if (!empty($filter->sumas->{$suma})) {
677 678 679 680 681
                        # If the searchengine is disabled this filter shouldn't be available
                        if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
                            || (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)) {
                            continue;
                        }
682 683 684 685 686 687 688 689 690 691
                        if (empty($availableFilter[$filterName])) {
                            $availableFilter[$filterName] = $filter;
                            unset($availableFilter[$filterName]->values);
                        }
                        if (empty($availableFilter[$filterName]->values)) {
                            $availableFilter[$filterName]->values = (object) ["" => $values->{""}];
                        }
                        foreach ($filter->sumas->{$suma}->values as $key => $value) {
                            $availableFilter[$filterName]->values->$key = $values->$key;
                        }
692 693 694
                    }
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
695 696
        }

697 698 699 700 701 702 703 704 705
        # Set the current values for the filters
        foreach ($availableFilter as $filterName => $filter) {
            if (\Request::filled($filter->{"get-parameter"})) {
                $filter->value = \Request::input($filter->{"get-parameter"});
            } else if (\Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"}) !== null) {
                $filter->value = \Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"});
            }
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
706 707 708
        return $availableFilter;
    }

709 710
    public function isBildersuche()
    {
711
        return $this->fokus === "bilder";
712 713 714 715
    }

    public function sumaIsAdsuche($suma, $overtureEnabled)
    {
716
        $sumaName = $suma["name"]->__toString();
717
        return
718 719
            $sumaName === "qualigo"
            || $sumaName === "similar_product_ads"
Karl Hasselbring's avatar
Karl Hasselbring committed
720
            || (!$overtureEnabled && $sumaName === "overtureAds");
721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747
    }

    public function sumaIsDisabled($suma)
    {
        return
        isset($suma['disabled'])
        && $suma['disabled']->__toString() === "1";
    }

    public function sumaIsOverture($suma)
    {
        return
        $suma["name"]->__toString() === "overture"
        || $suma["name"]->__toString() === "overtureAds";
    }

    public function sumaIsNotAdsuche($suma)
    {
        return
        $suma["name"]->__toString() !== "qualigo"
        && $suma["name"]->__toString() !== "similar_product_ads"
        && $suma["name"]->__toString() !== "overtureAds";
    }

    public function requestIsCached($request)
    {
        return
Dominik Hebeler's avatar
Dominik Hebeler committed
748
        $request->filled('next')
749 750 751 752 753 754
        && Cache::has($request->input('next'))
        && unserialize(Cache::get($request->input('next')))['page'] > 1;
    }

    public function getCachedEngines($request)
    {
755
        $next = unserialize(Cache::get($request->input('next')));
756
        $this->page = $next['page'];
757
        $engines = $next['engines'];
758 759 760
        return $engines;
    }

761
    public function waitForMainResults()
762
    {
763
        $redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
764
        $engines = $this->engines;
765
        $enginesToWaitFor = [];
766 767 768 769 770 771
        $mainEngines = $this->sumaFile->foki->{$this->fokus}->main;
        foreach ($mainEngines as $mainEngine) {
            foreach ($engines as $engine) {
                if (!$engine->cached && $engine->name === $mainEngine) {
                    $enginesToWaitFor[] = $engine;
                }
772 773
            }
        }
774

Phil Höfer's avatar
Phil Höfer committed
775
        $timeStart = microtime(true);
776
        $answered = [];
777
        $results = null;
778 779 780 781 782 783 784 785 786

        # If there is no main searchengine to wait for or if the only main engine is yahoo-ads we will define a timeout of 1s
        $forceTimeout = null;
        if (sizeof($enginesToWaitFor) === 0 || (sizeof($enginesToWaitFor) === 1 && $enginesToWaitFor[0]->name === "yahoo-ads")) {
            $forceTimeout = 1;
        }

        while (sizeof($enginesToWaitFor) > 0 || ($forceTimeout !== null && (microtime(true) - $timeStart) < $forceTimeout)) {
            $newEngine = $redis->blpop($this->redisResultWaitingKey, 1);
787 788 789 790 791 792 793 794 795
            if ($newEngine === null || sizeof($newEngine) !== 2) {
                continue;
            } else {
                $newEngine = $newEngine[1];
                foreach ($enginesToWaitFor as $index => $engine) {
                    if ($engine->name === $newEngine) {
                        unset($enginesToWaitFor[$index]);
                        break;
                    }
796
                }
797
                $answered[] = $newEngine;
798
            }
799
            if ((microtime(true) - $timeStart) >= 2) {
800 801
                break;
            }
802
        }
803 804

        # Now we can add an entry to Redis which defines the starting time and how many engines should answer this request
805

806 807 808 809
        $pipeline = $redis->pipeline();
        $pipeline->hset($this->getRedisEngineResult() . "status", "startTime", $timeStart);
        $pipeline->hset($this->getRedisEngineResult() . "status", "engineCount", sizeof($engines));
        $pipeline->hset($this->getRedisEngineResult() . "status", "engineDelivered", sizeof($answered));
810 811 812 813 814 815 816
        # Add the cached engines as answered
        foreach ($engines as $engine) {
            if ($engine->cached) {
                $pipeline->hincrby($this->getRedisEngineResult() . "status", "engineDelivered", 1);
                $pipeline->hincrby($this->getRedisEngineResult() . "status", "engineAnswered", 1);
            }
        }
817 818 819 820 821
        foreach ($answered as $engine) {
            $pipeline->hset($this->getRedisEngineResult() . $engine, "delivered", "1");
        }
        $pipeline->execute();

822
    }
823

824
    public function retrieveResults()
825
    {
826
        $engines = $this->engines;
827
        # Von geladenen Engines die Ergebnisse holen
828 829 830
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
                try {
831
                    $engine->retrieveResults($this);
832
                } catch (\ErrorException $e) {
833 834 835
                    Log::error($e);
                }
            }
836 837 838
            if (!empty($engine->totalResults) && $engine->totalResults > $this->totalResults) {
                $this->totalResults = $engine->totalResults;
            }
839 840 841 842
            if (!empty($engine->alteredQuery) && !empty($engine->alterationOverrideQuery)) {
                $this->alteredQuery = $engine->alteredQuery;
                $this->alterationOverrideQuery = $engine->alterationOverrideQuery;
            }
843
        }
844 845
    }

846 847 848
/*
 * Ende Suchmaschinenerstellung und Ergebniserhalt
 */
Karl's avatar
Karl committed
849

850 851
    public function parseFormData(Request $request)
    {
852

853
        # Sichert, dass der request in UTF-8 formatiert ist
854
        if ($request->input('encoding', 'utf8') !== "utf8") {
855 856 857 858 859 860 861
            # In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
            $input = $request->all();
            foreach ($input as $key => $value) {
                $input[$key] = mb_convert_encoding("$value", "UTF-8", "ISO-8859-1");
            }
            $request->replace($input);
        }
862
        $this->url = $request->url();
863
        $this->fullUrl = $request->fullUrl();
864
        # Zunächst überprüfen wir die eingegebenen Einstellungen: