MetaGer.php 52.6 KB
Newer Older
1 2 3 4
<?php
namespace App;

use App;
5
use Cache;
6
use Carbon;
7
use Illuminate\Http\Request;
Phil Höfer's avatar
Phil Höfer committed
8
use Illuminate\Support\Facades\Redis;
9
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix  
Dominik Hebeler committed
10
use LaravelLocalization;
11
use Log;
12
use Predis\Connection\ConnectionException;
13 14 15

class MetaGer
{
16 17 18 19 20 21 22 23
    # Einstellungen für die Suche
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
24
    protected $time = 2000;
25
    protected $hostBlacklist = [];
26
    protected $domainBlacklist = [];
27 28 29 30
    private $urlBlacklist = [];
    protected $stopWords = [];
    protected $phrases = [];
    protected $engines = [];
31
    protected $totalResults = 0;
32
    protected $results = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
33 34
    protected $queryFilter = [];
    protected $parameterFilter = [];
35 36 37 38
    protected $ads = [];
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
39
    protected $availableFoki = [];
40 41
    protected $startCount = 0;
    protected $canCache = false;
42
    # Daten über die Abfrage$
43
    protected $ip;
Dominik Hebeler's avatar
Dominik Hebeler committed
44
    protected $useragent;
45 46
    protected $language;
    protected $agent;
47
    protected $apiKey = "";
Phil Höfer's avatar
Phil Höfer committed
48
    protected $apiAuthorized = false;
49
    protected $next = [];
50 51 52 53 54
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
55
    protected $newtab;
56
    protected $domainsBlacklisted = [];
57
    protected $urlsBlacklisted = [];
58
    protected $url;
59
    protected $fullUrl;
60
    protected $languageDetect;
Dominik Hebeler's avatar
Dominik Hebeler committed
61 62
    protected $verificationId;
    protected $verificationCount;
63
    protected $searchUid;
64
    protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult, $redisCurrentResultList;
65

66
    public function __construct($hash = "")
67
    {
68
        # Timer starten
69
        $this->starttime = microtime(true);
70
        # Versuchen Blacklists einzulesen
71
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
72
            $tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
73
            $this->domainsBlacklisted = explode("\n", $tmp);
74 75
            $tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted = explode("\n", $tmp);
76
        } else {
77
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
78 79
        }

80
        # Parser Skripte einhängen
81 82 83 84
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
85
                require_once $path;
86 87 88
            }
        }

89
        # Cachebarkeit testen
90 91 92 93 94 95
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
96 97 98 99 100
        if ($hash === "") {
            $this->searchUid = md5(uniqid());
        } else {
            $this->searchUid = $hash;
        }
101 102 103 104 105 106 107
        $redisPrefix = "search";
        # This is a list on which the MetaGer process can do a blocking pop to wait for new results
        $this->redisResultWaitingKey = $redisPrefix . "." . $this->searchUid . ".ready";
        # This is a list of searchengines which have delivered results for this search
        $this->redisResultEngineList = $redisPrefix . "." . $this->searchUid . ".engines";
        # This is the key where the results of the engine are stored as well as some statistical data
        $this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results.";
108 109 110
        # A list of all search results already delivered to the user (sorted of course)
        $this->redisCurrentResultList = $redisPrefix . "." . $this->searchUid . ".currentResults";

111
    }
112

113
    # Erstellt aus den gesammelten Ergebnissen den View
114
    public function createView($quicktipResults = [])
115
    {
116 117
        # Hiermit werden die evtl. ausgewählten SuMas extrahiert, damit die Input-Boxen richtig gesetzt werden können
        $focusPages = [];
Aria Givi's avatar
Aria Givi committed
118

119
        foreach ($this->request->all() as $key => $value) {
120 121
            if (starts_with($key, 'engine_') && $value === 'on') {
                $focusPages[] = $key;
122 123 124
            }
        }

125
        $viewResults = [];
126
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
127
        foreach ($this->results as $result) {
128 129 130 131
            $viewResults[] = get_object_vars($result);
        }
        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();
132 133
        if ($this->fokus === "bilder") {
            switch ($this->out) {
134
                case 'results':
135
                    return view('resultpages.results_images')
136 137 138 139 140
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
141
                        ->with('apiAuthorized', $this->apiAuthorized)
142
                        ->with('metager', $this)
143
                        ->with('browser', (new Agent())->browser());
144
                default:
145
                    return view('resultpages.resultpage_images')
146 147 148 149 150
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
151
                        ->with('apiAuthorized', $this->apiAuthorized)
152
                        ->with('metager', $this)
153 154 155 156
                        ->with('browser', (new Agent())->browser())
                        ->with('quicktips', $quicktipResults)
                        ->with('focus', $this->fokus)
                        ->with('resultcount', count($this->results));
157
            }
158 159 160
        } else {
            switch ($this->out) {
                case 'results':
161
                    return view('resultpages.results')
162 163 164 165 166
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
167
                        ->with('apiAuthorized', $this->apiAuthorized)
168
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
169 170
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
171 172
                    break;
                case 'results-with-style':
173
                    return view('resultpages.resultpage')
174 175 176 177 178
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
179
                        ->with('apiAuthorized', $this->apiAuthorized)
180 181
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
Dominik Hebeler's avatar
Dominik Hebeler committed
182 183
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
184
                    break;
Phil Höfer's avatar
Phil Höfer committed
185
                case 'rich':
186
                    return view('resultpages.metager3rich')
187
                        ->with('results', $viewResults)
Phil Höfer's avatar
Phil Höfer committed
188 189 190 191
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
192
                        ->with('apiAuthorized', $this->apiAuthorized)
Phil Höfer's avatar
Phil Höfer committed
193
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
194 195
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
Phil Höfer's avatar
Phil Höfer committed
196
                    break;
197
                case 'rss20':
198
                    return view('resultpages.metager3resultsrss20')
199 200
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
Phil Höfer's avatar
Phil Höfer committed
201
                        ->with('apiAuthorized', $this->apiAuthorized)
202
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
203 204
                        ->with('resultcount', sizeof($viewResults))
                        ->with('fokus', $this->fokus);
205
                    break;
206 207 208
                case 'api':
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'apiAuthorized' => $this->apiAuthorized])->header('Content-Type', 'application/xml');
                    break;
Aria Givi's avatar
Aria Givi committed
209
                case 'atom10':
210
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'apiAuthorized' => true])
211
                        ->header('Content-Type', 'application/xml');
Aria Givi's avatar
Aria Givi committed
212
                    break;
213
                case 'result-count':
214 215
                    # Wir geben die Ergebniszahl und die benötigte Zeit zurück:
                    return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
216
                    break;
217
                default:
218
                    return view('resultpages.resultpage')
219
                        ->with('eingabe', $this->eingabe)
220
                        ->with('focusPages', $focusPages)
221 222 223
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
224
                        ->with('apiAuthorized', $this->apiAuthorized)
225
                        ->with('metager', $this)
226
                        ->with('browser', (new Agent())->browser())
227
                        ->with('quicktips', $quicktipResults)
228 229
                        ->with('resultcount', count($this->results))
                        ->with('focus', $this->fokus);
230 231
                    break;
            }
232 233 234
        }
    }

Phil Höfer's avatar
Phil Höfer committed
235
    public function prepareResults()
236
    {
Phil Höfer's avatar
Phil Höfer committed
237 238
        $engines = $this->engines;
        // combine
239
        $this->combineResults($engines);
Phil Höfer's avatar
Phil Höfer committed
240
        // misc (WiP)
241
        if ($this->fokus == "nachrichten") {
242 243 244
            $this->results = array_filter($this->results, function ($v, $k) {
                return !is_null($v->getRank());
            }, ARRAY_FILTER_USE_BOTH);
245 246 247 248 249 250 251 252 253 254
            uasort($this->results, function ($a, $b) {
                $datea = $a->getDate();
                $dateb = $b->getDate();
                return $dateb - $datea;
            });
        } else {
            uasort($this->results, function ($a, $b) {
                if ($a->getRank() == $b->getRank()) {
                    return 0;
                }
255

256 257 258
                return ($a->getRank() < $b->getRank()) ? 1 : -1;
            });
        }
259

260 261
        # Validate Results
        $newResults = [];
262 263
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
264
                $newResults[] = $result;
265 266
            }

267 268 269
        }
        $this->results = $newResults;

270
        #Adgoal Implementation
271 272 273
        if (!$this->apiAuthorized) {
            $this->results = $this->parseAdgoal($this->results);
        }
274

Dominik Hebeler's avatar
Dominik Hebeler committed
275 276
        # Human Verification
        $this->results = $this->humanVerification($this->results);
277
        $this->ads = $this->humanVerification($this->ads);
Dominik Hebeler's avatar
Dominik Hebeler committed
278

279
        $counter = 0;
280
        $firstRank = 0;
281

282
        if (count($this->results) <= 0) {
283 284 285 286 287 288
            if (strlen($this->site) > 0) {
                $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
                $this->errors[] = trans('metaGer.results.failedSitesearch', ['altSearch' => $no_sitesearch_query]);
            } else {
                $this->errors[] = trans('metaGer.results.failed');
            }
289
        }
290

291
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
292
            $page = $this->page + 1;
293
            $this->next = [
294 295
                'page' => $page,
                'engines' => $this->next,
296
            ];
297
            Cache::put($this->getSearchUid(), serialize($this->next), 60);
298 299
        } else {
            $this->next = [];
300 301
        }

302
    }
303

Phil Höfer's avatar
Phil Höfer committed
304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323
    public function combineResults($engines)
    {
        foreach ($engines as $engine) {
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
            foreach ($engine->results as $result) {
                if ($result->valid) {
                    $this->results[] = $result;
                }
            }
            foreach ($engine->ads as $ad) {
                $this->ads[] = $ad;
            }
        }
    }

324 325
    public function parseAdgoal($results)
    {
326
        $publicKey = getenv('adgoal_public');
327
        $privateKey = getenv('adgoal_private');
328
        if ($publicKey === false) {
329 330 331
            return $results;
        }
        $tldList = "";
332 333
        try {
            foreach ($results as $result) {
334 335 336
                if (!$result->new) {
                    continue;
                }
337
                $link = $result->anzeigeLink;
338
                if (strpos($link, "http") !== 0) {
339 340 341 342 343 344 345 346 347 348
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

349
            # Query
350 351
            $query = urlencode($this->q);

352
            $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . $publicKey . "&k=" . $hash . "&tld=" . $tldList . "&q=" . $query;
353 354 355
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
356
            foreach ($answer as $el) {
357
                $hoster = $el[0];
358
                $hash = $el[1];
359

360
                foreach ($results as $result) {
361
                    if ($result->new && $hoster === $result->tld && !$result->partnershop) {
362 363
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
364
                        if ($result->image !== "") {
365
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . $hash . ".gif";
366
                        } else {
367
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . $hash . ".gif";
368 369
                        }

370 371 372
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
                        $targetUrl = $result->anzeigeLink;
373
                        if (strpos($targetUrl, "http") !== 0) {
374
                            $targetUrl = "http://" . $targetUrl;
375 376
                        }

377 378 379
                        $gateHash = md5($targetUrl . $privateKey);
                        $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . $publicKey . "&k=" . $gateHash . "&url=" . urlencode($targetUrl) . "&q=" . $query;
                        $result->link = $newLink;
380 381 382 383
                        $result->partnershop = true;
                    }
                }
            }
384
        } catch (\ErrorException $e) {
385 386 387 388 389
            return $results;
        }

        return $results;
    }
390

391 392
    public function humanVerification($results)
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
393
        # Let's check if we need to implement a redirect for human verification
394 395
        if ($this->verificationCount > 10) {
            foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
396 397 398
                $link = $result->link;
                $day = Carbon::now()->day;
                $pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
399 400 401
                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
Dominik Hebeler's avatar
Dominik Hebeler committed
402
                $result->link = $url;
403
                $result->proxyLink = $proxyUrl;
Dominik Hebeler's avatar
Dominik Hebeler committed
404 405
            }
            return $results;
406
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
407 408 409 410
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
411 412 413 414 415 416
    public function authorize($key)
    {
        $postdata = http_build_query(array(
            'dummy' => rand(),
        ));
        $opts = array('http' => array(
417 418
            'method' => 'POST',
            'header' => 'Content-type: application/x-www-form-urlencoded',
Phil Höfer's avatar
Phil Höfer committed
419 420 421 422 423 424 425
            'content' => $postdata,
        ),
        );

        $context = stream_context_create($opts);

        try {
426
            $link = "https://key.metager3.de/" . urlencode($key) . "/request-permission/api-access";
Phil Höfer's avatar
Phil Höfer committed
427 428 429 430 431 432 433 434 435 436 437 438
            $result = json_decode(file_get_contents($link, false, $context));
            if ($result->{'api-access'} == true) {
                return true;
            } else {
                return false;
            }

        } catch (\ErrorException $e) {
            return false;
        }
    }

439 440
    public function createQuicktips()
    {
441
        # Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
442
        $quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime());
443 444 445
        return $quicktips;
    }

Karl's avatar
Karl committed
446 447 448 449
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

450 451
    public function createSearchEngines(Request $request)
    {
452
        # Wenn es kein Suchwort gibt
Dominik Hebeler's avatar
Dominik Hebeler committed
453
        if (!$request->filled("eingabe") || $this->q === "") {
454
            return;
455
        }
456

Dominik Hebeler's avatar
Dominik Hebeler committed
457
        $this->enabledSearchengines = [];
458
        $overtureEnabled = false;
459

460 461 462
        # Check if selected focus is valid
        if (empty($this->sumaFile->foki->{$this->fokus})) {
            $this->fokus = "web";
463
        }
464 465
        foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
            # Check if this engine is disabled and can't be used
466
            $disabled = empty($this->sumaFile->sumas->{$suma}->disabled) ? false : $this->sumaFile->sumas->{$suma}->disabled;
467 468
            $autoDisabled = empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) ? false : $this->sumaFile->sumas->{$suma}->{"auto-disabled"};
            if ($disabled || $autoDisabled) {
469
                continue;
470 471
            }

472 473 474
            # Check if this engine can use eventually defined query-filter
            $valid = true;
            foreach ($this->queryFilter as $queryFilter => $filter) {
Dominik Hebeler's avatar
Dominik Hebeler committed
475
                if (empty($this->sumaFile->filter->{"query-filter"}->$queryFilter->sumas->$suma)) {
476 477
                    $valid = false;
                    break;
Dominik Hebeler's avatar
Dominik Hebeler committed
478 479
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
480 481 482
            # Check if this engine can use eventually defined parameter-filter
            if ($valid) {
                foreach ($this->parameterFilter as $filterName => $filter) {
483 484 485
                    # We need to check if the searchengine supports the parameter value, too
                    $value = $request->input($filter->{"get-parameter"}, "");
                    if (empty($filter->sumas->$suma) || empty($filter->sumas->{$suma}->values->{$value})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
486 487 488 489 490
                        $valid = false;
                        break;
                    }
                }
            }
491 492 493 494 495 496 497 498 499 500 501 502 503 504 505
            # Check if this engine should only be active when filter is used
            if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"}) {
                # This search engine should only be used when a parameter filter of it is used
                $validTmp = false;
                foreach ($this->parameterFilter as $filterName => $filter) {
                    if (!empty($filter->sumas->{$suma})) {
                        $validTmp = true;
                        break;
                    }
                }
                if (!$validTmp) {
                    $valid = false;
                }

            }
506 507
            # If it can we add it
            if ($valid) {
Dominik Hebeler's avatar
Dominik Hebeler committed
508
                $this->enabledSearchengines[$suma] = $this->sumaFile->sumas->{$suma};
509
            }
510

511
        }
512

513 514 515 516 517
        # Implements Yahoo Ads if Yahoo is not enabled as a searchengine
        if (empty($this->enabledSearchengines["yahoo"]) && $this->fokus != "bilder" && !empty($this->sumaFile->sumas->{"yahoo-ads"})) {
            $this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
518
        if (sizeof($this->enabledSearchengines) === 0) {
519 520
            $filter = "";
            foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
Dominik Hebeler's avatar
Dominik Hebeler committed
521
                $filter .= trans($this->sumaFile->filter->{"query-filter"}->{$queryFilter}->name) . ",";
522 523 524 525 526
            }
            $filter = rtrim($filter, ",");
            $error = trans('metaGer.engines.noSpecialSearch', ['fokus' => trans($this->sumaFile->foki->{$this->fokus}->{"display-name"}),
                'filter' => $filter]);
            $this->errors[] = $error;
527
        }
528
        $engines = [];
529
        $typeslist = [];
530
        $counter = 0;
531 532
        $this->setEngines($request);
    }
533

534 535
    public function setEngines(Request $request, $enabledSearchengines = [])
    {
536
        if ($this->requestIsCached($request)) {
537
            # If this is a page other than 1 the request is "cached"
538
            $engines = $this->getCachedEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
539
            # We need to edit some Options of the Cached Search Engines
Phil Höfer's avatar
Phil Höfer committed
540
            foreach ($engines as $engine) {
541
                $engine->setResultHash($this->getSearchUid());
Dominik Hebeler's avatar
Dominik Hebeler committed
542
            }
543
            $this->engines = $engines;
544
        } else {
545 546 547
            if (sizeof($enabledSearchengines) > 0) {
                $this->enabledSearchengines = $enabledSearchengines;
            }
548
            $this->actuallyCreateSearchEngines($this->enabledSearchengines);
549
        }
550
    }
Dominik Hebeler's avatar
Dominik Hebeler committed
551

552 553
    public function startSearch()
    {
554
        # Wir starten alle Suchen
555
        foreach ($this->engines as $engine) {
556
            $engine->startSearch($this);
557
        }
558 559
    }

560 561
    # Spezielle Suchen und Sumas

562
    public function sumaIsSelected($suma, $request, $custom)
563
    {
564
        if ($custom) {
565
            if ($request->filled("engine_" . strtolower($suma["name"]))) {
566 567 568 569 570 571 572 573 574 575 576
                return true;
            }
        } else {
            $types = explode(",", $suma["type"]);
            if (in_array($this->fokus, $types)) {
                return true;
            }
        }
        return false;
    }

577
    public function actuallyCreateSearchEngines($enabledSearchengines)
578 579
    {
        $engines = [];
580
        foreach ($enabledSearchengines as $engineName => $engine) {
581

582
            if (!isset($engine->{"parser-class"})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
583 584
                die(var_dump($engine));
            }
585
            # Setze Pfad zu Parser
586
            $path = "App\\Models\\parserSkripte\\" . $engine->{"parser-class"};
587 588

            # Prüfe ob Parser vorhanden
589 590 591
            if (!file_exists(app_path() . "/Models/parserSkripte/" . $engine->{"parser-class"} . ".php")) {
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen, da kein Parser existiert");
                $this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine->{"display-name"}]);
592 593 594 595 596 597
                continue;
            }

            # Es wird versucht die Suchengine zu erstellen
            $time = microtime();
            try {
598
                $tmp = new $path($engineName, $engine, $this);
599
            } catch (\ErrorException $e) {
600
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen. " . var_dump($e));
601 602 603
                continue;
            }

604
            $engines[] = $tmp;
605
        }
606
        $this->engines = $engines;
607 608
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
609 610 611 612 613 614 615 616 617 618 619 620 621
    public function getAvailableParameterFilter()
    {
        $parameterFilter = $this->sumaFile->filter->{"parameter-filter"};

        $availableFilter = [];

        foreach ($parameterFilter as $filterName => $filter) {
            # Check if any of the enabled search engines provide this filter
            foreach ($this->enabledSearchengines as $engineName => $engine) {
                if (!empty($filter->sumas->$engineName)) {
                    $availableFilter[$filterName] = $filter;
                }
            }
622 623 624 625
            # We will also add the filter from the opt-in search engines (the searchengines that are only used when a filter of it is too)
            foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
                if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"}) {
                    if (!empty($filter->sumas->{$suma})) {
626 627 628 629 630
                        # If the searchengine is disabled this filter shouldn't be available
                        if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
                            || (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)) {
                            continue;
                        }
631 632 633 634
                        $availableFilter[$filterName] = $filter;
                    }
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
635 636 637 638 639
        }

        return $availableFilter;
    }

640 641
    public function isBildersuche()
    {
642
        return $this->fokus === "bilder";
643 644 645 646
    }

    public function sumaIsAdsuche($suma, $overtureEnabled)
    {
647
        $sumaName = $suma["name"]->__toString();
648
        return
649 650
            $sumaName === "qualigo"
            || $sumaName === "similar_product_ads"
Karl Hasselbring's avatar
Karl Hasselbring committed
651
            || (!$overtureEnabled && $sumaName === "overtureAds");
652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678
    }

    public function sumaIsDisabled($suma)
    {
        return
        isset($suma['disabled'])
        && $suma['disabled']->__toString() === "1";
    }

    public function sumaIsOverture($suma)
    {
        return
        $suma["name"]->__toString() === "overture"
        || $suma["name"]->__toString() === "overtureAds";
    }

    public function sumaIsNotAdsuche($suma)
    {
        return
        $suma["name"]->__toString() !== "qualigo"
        && $suma["name"]->__toString() !== "similar_product_ads"
        && $suma["name"]->__toString() !== "overtureAds";
    }

    public function requestIsCached($request)
    {
        return
Dominik Hebeler's avatar
Dominik Hebeler committed
679
        $request->filled('next')
680 681 682 683 684 685
        && Cache::has($request->input('next'))
        && unserialize(Cache::get($request->input('next')))['page'] > 1;
    }

    public function getCachedEngines($request)
    {
686
        $next = unserialize(Cache::get($request->input('next')));
687
        $this->page = $next['page'];
688
        $engines = $next['engines'];
689 690 691
        return $engines;
    }

692
    public function waitForMainResults()
693
    {
694
        $redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
695
        $engines = $this->engines;
696
        $enginesToWaitFor = [];
697 698 699 700 701 702
        $mainEngines = $this->sumaFile->foki->{$this->fokus}->main;
        foreach ($mainEngines as $mainEngine) {
            foreach ($engines as $engine) {
                if (!$engine->cached && $engine->name === $mainEngine) {
                    $enginesToWaitFor[] = $engine;
                }
703 704
            }
        }
705

Phil Höfer's avatar
Phil Höfer committed
706
        $timeStart = microtime(true);
707
        $answered = [];
708
        $results = null;
709
        while (sizeof($enginesToWaitFor) > 0) {
710
            $newEngine = $redis->blpop($this->redisResultWaitingKey, 5);
711 712 713 714 715 716 717 718 719
            if ($newEngine === null || sizeof($newEngine) !== 2) {
                continue;
            } else {
                $newEngine = $newEngine[1];
                foreach ($enginesToWaitFor as $index => $engine) {
                    if ($engine->name === $newEngine) {
                        unset($enginesToWaitFor[$index]);
                        break;
                    }
720
                }
721
                $answered[] = $newEngine;
722
            }
723
            if ((microtime(true) - $timeStart) >= 2) {
724 725
                break;
            }
726
        }
727 728

        # Now we can add an entry to Redis which defines the starting time and how many engines should answer this request
729

730 731 732 733 734 735 736 737 738
        $pipeline = $redis->pipeline();
        $pipeline->hset($this->getRedisEngineResult() . "status", "startTime", $timeStart);
        $pipeline->hset($this->getRedisEngineResult() . "status", "engineCount", sizeof($engines));
        $pipeline->hset($this->getRedisEngineResult() . "status", "engineDelivered", sizeof($answered));
        foreach ($answered as $engine) {
            $pipeline->hset($this->getRedisEngineResult() . $engine, "delivered", "1");
        }
        $pipeline->execute();

739
    }
740

741
    public function retrieveResults()
742
    {
743
        $engines = $this->engines;
744
        # Von geladenen Engines die Ergebnisse holen
745 746 747
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
                try {
748
                    $engine->retrieveResults($this);
749
                } catch (\ErrorException $e) {
750 751 752
                    Log::error($e);
                }
            }
753 754 755
            if (!empty($engine->totalResults) && $engine->totalResults > $this->totalResults) {
                $this->totalResults = $engine->totalResults;
            }
756
        }
757 758
    }

759 760 761
/*
 * Ende Suchmaschinenerstellung und Ergebniserhalt
 */
Karl's avatar
Karl committed
762

763 764
    public function parseFormData(Request $request)
    {
765

766
        # Sichert, dass der request in UTF-8 formatiert ist
767
        if ($request->input('encoding', 'utf8') !== "utf8") {
768 769 770 771 772 773 774
            # In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
            $input = $request->all();
            foreach ($input as $key => $value) {
                $input[$key] = mb_convert_encoding("$value", "UTF-8", "ISO-8859-1");
            }
            $request->replace($input);
        }
775
        $this->url = $request->url();
776
        $this->fullUrl = $request->fullUrl();
777
        # Zunächst überprüfen wir die eingegebenen Einstellungen:
778
        # Fokus
779
        $this->fokus = $request->input('focus', 'web');
780
        # Suma-File
781
        if (App::isLocale("en")) {
782
            $this->sumaFile = config_path() . "/sumasEn.json";
783
        } else {
784
            $this->sumaFile = config_path() . "/sumas.json";
785
        }
786
        if (!file_exists($this->sumaFile)) {
787
            die(trans('metaGer.formdata.cantLoad'));
788 789
        } else {
            $this->sumaFile = json_decode(file_get_contents($this->sumaFile));
790
        }
791
        # Sucheingabe
792
        $this->eingabe = trim($request->input('eingabe', ''));
793
        $this->q = $this->eingabe;
794
        # IP
795
        $this->ip = $request->ip();
796 797 798 799
        # Unser erster Schritt wird sein, IP-Adresse und USER-Agent zu anonymisieren, damit
        # nicht einmal wir selbst noch Zugriff auf die Daten haben:
        $this->ip = preg_replace("/(\d+)\.(\d+)\.\d+.\d+/s", "$1.$2.0.0", $this->ip);

Dominik Hebeler's avatar
Dominik Hebeler committed
800 801
        $this->useragent = $request->header('User-Agent');

802
        # Language
803
        if (isset($_SERVER['HTTP_LANGUAGE'])) {
804
            $this->language = $_SERVER['HTTP_LANGUAGE'];
805
        } else {
806 807
            $this->language = "";
        }
808

809
        # Page
810
        $this->page = 1;
811 812
        # Lang
        $this->lang = $request->input('lang', 'all');
813 814
        if ($this->lang !== "de" && $this->lang !== "en" && $this->lang !== "all") {
            $this->lang = "all";
815
        }
816

817
        $this->agent = new Agent();
818
        $this->mobile = $this->agent->isMobile();
819
        # Sprüche
820 821
        $this->sprueche = $request->input('sprueche', 'on');
        if ($this->sprueche === "on") {
822
            $this->sprueche = true;
823
        } else {
824
            $this->sprueche = false;
825
        }
826

827
        $this->newtab = $request->input('newtab', 'on');
828 829 830 831 832
        if ($this->newtab === "on") {
            $this->newtab = "_blank";
        } else {
            $this->newtab = "_self";
        }
Dominik Hebeler's avatar