MetaGer.php 61.8 KB
Newer Older
1
<?php
Karl Hasselbring's avatar
Karl Hasselbring committed
2

3
4
5
namespace App;

use App;
6
use Cache;
7
use Carbon;
8
use Illuminate\Http\Request;
9
use Illuminate\Support\Facades\Redis;
10
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix    
Dominik Hebeler committed
11
use LaravelLocalization;
12
use Log;
13
use Predis\Connection\ConnectionException;
14
15
16

class MetaGer
{
Dominik Hebeler's avatar
Dominik Hebeler committed
17
18
    const FETCHQUEUE_KEY = "fetcher.queue";

19
    # Einstellungen für die Suche
20
21
    public $alteredQuery = "";
    public $alterationOverrideQuery = "";
22
23
24
25
26
27
28
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
29
    protected $time = 2000;
30
    protected $hostBlacklist = [];
31
    protected $domainBlacklist = [];
32
33
34
35
    private $urlBlacklist = [];
    protected $stopWords = [];
    protected $phrases = [];
    protected $engines = [];
36
    protected $totalResults = 0;
37
    protected $results = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
38
39
    protected $queryFilter = [];
    protected $parameterFilter = [];
40
    protected $ads = [];
41
    protected $infos = [];
42
43
44
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
45
    protected $availableFoki = [];
46
47
    protected $startCount = 0;
    protected $canCache = false;
48
    # Daten über die Abfrage$
49
    protected $ip;
Dominik Hebeler's avatar
Dominik Hebeler committed
50
    protected $useragent;
51
52
    protected $language;
    protected $agent;
53
    protected $apiKey = "";
Phil Höfer's avatar
Phil Höfer committed
54
    protected $apiAuthorized = false;
55
    protected $next = [];
56
57
58
59
60
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
61
    protected $newtab;
62
    protected $domainsBlacklisted = [];
63
    protected $adDomainsBlacklisted = [];
64
    protected $urlsBlacklisted = [];
65
    protected $adUrlsBlacklisted = [];
66
    protected $url;
67
    protected $fullUrl;
Aria's avatar
Aria committed
68
    protected $enabledSearchengines = [];
69
    protected $languageDetect;
Dominik Hebeler's avatar
Dominik Hebeler committed
70
71
    protected $verificationId;
    protected $verificationCount;
72
    protected $searchUid;
73
    protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult, $redisCurrentResultList;
74

75
    public function __construct($hash = "")
76
    {
77
        # Timer starten
78
        $this->starttime = microtime(true);
79
        # Versuchen Blacklists einzulesen
80
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
81
            $tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
82
            $this->domainsBlacklisted = explode("\n", $tmp);
83
84
            $tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted = explode("\n", $tmp);
85
        } else {
86
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
87
        }
88
89
90
91
92
93
94
95
96
        # Versuchen Blacklists einzulesen
        if (file_exists(config_path() . "/adBlacklistDomains.txt") && file_exists(config_path() . "/adBlacklistUrl.txt")) {
            $tmp = file_get_contents(config_path() . "/adBlacklistDomains.txt");
            $this->adDomainsBlacklisted = explode("\n", $tmp);
            $tmp = file_get_contents(config_path() . "/adBlacklistUrl.txt");
            $this->adUrlsBlacklisted = explode("\n", $tmp);
        } else {
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
        }
97

98
        # Parser Skripte einhängen
99
100
101
102
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
103
                require_once $path;
104
105
106
            }
        }

107
        # Cachebarkeit testen
108
109
110
111
112
113
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
114
115
116
117
118
        if ($hash === "") {
            $this->searchUid = md5(uniqid());
        } else {
            $this->searchUid = $hash;
        }
119
120
121
122
123
124
125
        $redisPrefix = "search";
        # This is a list on which the MetaGer process can do a blocking pop to wait for new results
        $this->redisResultWaitingKey = $redisPrefix . "." . $this->searchUid . ".ready";
        # This is a list of searchengines which have delivered results for this search
        $this->redisResultEngineList = $redisPrefix . "." . $this->searchUid . ".engines";
        # This is the key where the results of the engine are stored as well as some statistical data
        $this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results.";
126
127
        # A list of all search results already delivered to the user (sorted of course)
        $this->redisCurrentResultList = $redisPrefix . "." . $this->searchUid . ".currentResults";
128
    }
129

130
    # Erstellt aus den gesammelten Ergebnissen den View
131
    public function createView($quicktipResults = [])
132
    {
133
134
        # Hiermit werden die evtl. ausgewählten SuMas extrahiert, damit die Input-Boxen richtig gesetzt werden können
        $focusPages = [];
Aria Givi's avatar
Aria Givi committed
135

136
        foreach ($this->request->all() as $key => $value) {
137
138
            if (starts_with($key, 'engine_') && $value === 'on') {
                $focusPages[] = $key;
139
140
141
            }
        }

142
        $viewResults = [];
143
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
144
        foreach ($this->results as $result) {
145
146
147
148
            $viewResults[] = get_object_vars($result);
        }
        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();
149
150
        if ($this->fokus === "bilder") {
            switch ($this->out) {
151
                case 'results':
152
                    return view('resultpages.results_images')
153
154
155
156
157
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
158
                        ->with('apiAuthorized', $this->apiAuthorized)
159
                        ->with('metager', $this)
160
                        ->with('browser', (new Agent())->browser());
161
                default:
162
                    return view('resultpages.resultpage_images')
163
164
165
166
167
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
168
                        ->with('apiAuthorized', $this->apiAuthorized)
169
                        ->with('metager', $this)
170
171
172
173
                        ->with('browser', (new Agent())->browser())
                        ->with('quicktips', $quicktipResults)
                        ->with('focus', $this->fokus)
                        ->with('resultcount', count($this->results));
174
            }
175
176
177
        } else {
            switch ($this->out) {
                case 'results':
178
                    return view('resultpages.results')
179
180
181
182
183
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
184
                        ->with('apiAuthorized', $this->apiAuthorized)
185
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
186
187
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
188
189
                    break;
                case 'results-with-style':
190
                    return view('resultpages.resultpage')
191
192
193
194
195
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
196
                        ->with('apiAuthorized', $this->apiAuthorized)
197
198
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
Dominik Hebeler's avatar
Dominik Hebeler committed
199
200
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
201
                    break;
Phil Höfer's avatar
Phil Höfer committed
202
                case 'rich':
203
                    return view('resultpages.metager3rich')
204
                        ->with('results', $viewResults)
Phil Höfer's avatar
Phil Höfer committed
205
206
207
208
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
209
                        ->with('apiAuthorized', $this->apiAuthorized)
Phil Höfer's avatar
Phil Höfer committed
210
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
211
212
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
Phil Höfer's avatar
Phil Höfer committed
213
                    break;
214
                case 'rss20':
215
                    return view('resultpages.metager3resultsrss20')
216
217
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
Phil Höfer's avatar
Phil Höfer committed
218
                        ->with('apiAuthorized', $this->apiAuthorized)
219
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
220
221
                        ->with('resultcount', sizeof($viewResults))
                        ->with('fokus', $this->fokus);
222
                    break;
223
                case 'api':
Phil Höfer's avatar
Phil Höfer committed
224
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'key' => $this->apiKey, 'apiAuthorized' => $this->apiAuthorized])->header('Content-Type', 'application/xml');
225
                    break;
Aria Givi's avatar
Aria Givi committed
226
                case 'atom10':
Phil Höfer's avatar
Phil Höfer committed
227
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'key' => $this->apiKey, 'apiAuthorized' => true])
228
                        ->header('Content-Type', 'application/xml');
Aria Givi's avatar
Aria Givi committed
229
                    break;
230
                case 'result-count':
231
232
                    # Wir geben die Ergebniszahl und die benötigte Zeit zurück:
                    return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
233
                    break;
234
                default:
235
                    return view('resultpages.resultpage')
236
                        ->with('eingabe', $this->eingabe)
237
                        ->with('focusPages', $focusPages)
238
239
240
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
241
                        ->with('apiAuthorized', $this->apiAuthorized)
242
                        ->with('metager', $this)
243
                        ->with('browser', (new Agent())->browser())
244
                        ->with('quicktips', $quicktipResults)
245
246
                        ->with('resultcount', count($this->results))
                        ->with('focus', $this->fokus);
247
248
                    break;
            }
249
250
251
        }
    }

Phil Höfer's avatar
Phil Höfer committed
252
    public function prepareResults()
253
    {
Phil Höfer's avatar
Phil Höfer committed
254
255
        $engines = $this->engines;
        // combine
256
        $this->combineResults($engines);
Phil Höfer's avatar
Phil Höfer committed
257
        // misc (WiP)
258
        if ($this->fokus == "nachrichten") {
259
260
261
            $this->results = array_filter($this->results, function ($v, $k) {
                return !is_null($v->getRank());
            }, ARRAY_FILTER_USE_BOTH);
262
263
264
265
266
267
268
269
270
271
            uasort($this->results, function ($a, $b) {
                $datea = $a->getDate();
                $dateb = $b->getDate();
                return $dateb - $datea;
            });
        } else {
            uasort($this->results, function ($a, $b) {
                if ($a->getRank() == $b->getRank()) {
                    return 0;
                }
272

273
274
275
                return ($a->getRank() < $b->getRank()) ? 1 : -1;
            });
        }
276

277
278
        # Validate Results
        $newResults = [];
279
280
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
281
                $newResults[] = $result;
282
            }
283
284
285
        }
        $this->results = $newResults;

286
287
288
289
        # Validate Advertisements
        $newResults = [];
        foreach ($this->ads as $ad) {
            if (($ad->strippedHost !== "" && (in_array($ad->strippedHost, $this->adDomainsBlacklisted) ||
290
291
                in_array($ad->strippedLink, $this->adUrlsBlacklisted))) || ($ad->strippedHostAnzeige !== "" && (in_array($ad->strippedHostAnzeige, $this->adDomainsBlacklisted) ||
                in_array($ad->strippedLinkAnzeige, $this->adUrlsBlacklisted)))
Karl Hasselbring's avatar
Karl Hasselbring committed
292
            ) {
293
294
295
296
297
298
                continue;
            }
            $newResults[] = $ad;
        }
        $this->ads = $newResults;

299
        #Adgoal Implementation
300
301
302
        if (!$this->apiAuthorized) {
            $this->results = $this->parseAdgoal($this->results);
        }
303

Dominik Hebeler's avatar
Dominik Hebeler committed
304
305
        # Human Verification
        $this->results = $this->humanVerification($this->results);
306
        $this->ads = $this->humanVerification($this->ads);
Dominik Hebeler's avatar
Dominik Hebeler committed
307

308
        $counter = 0;
309
        $firstRank = 0;
310

311
        if (count($this->results) <= 0) {
312
313
314
315
316
317
            if (strlen($this->site) > 0) {
                $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
                $this->errors[] = trans('metaGer.results.failedSitesearch', ['altSearch' => $no_sitesearch_query]);
            } else {
                $this->errors[] = trans('metaGer.results.failed');
            }
318
        }
319

320
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
321
            $page = $this->page + 1;
322
            $this->next = [
323
324
                'page' => $page,
                'engines' => $this->next,
325
            ];
326
            \App\CacheHelper::put($this->getSearchUid(), serialize($this->next), 60 * 60);
327
328
        } else {
            $this->next = [];
329
        }
330
    }
331

Phil Höfer's avatar
Phil Höfer committed
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
    public function combineResults($engines)
    {
        foreach ($engines as $engine) {
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
            foreach ($engine->results as $result) {
                if ($result->valid) {
                    $this->results[] = $result;
                }
            }
            foreach ($engine->ads as $ad) {
                $this->ads[] = $ad;
            }
        }
    }

352
353
    public function parseAdgoal($results)
    {
354
        $publicKey = getenv('adgoal_public');
355
        $privateKey = getenv('adgoal_private');
356
        if ($publicKey === false) {
357
358
359
            return $results;
        }
        $tldList = "";
360
361
        try {
            foreach ($results as $result) {
362
363
364
                if (!$result->new) {
                    continue;
                }
365
                $link = $result->link;
366
                if (strpos($link, "http") !== 0) {
367
368
369
370
371
372
373
374
375
376
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

377
            # Query
378
            $query = $this->q;
379

380
            $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . urlencode($publicKey) . "&k=" . urlencode($hash) . "&tld=" . urlencode($tldList) . "&q=" . urlencode($query);
381
382
383
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
384
            foreach ($answer as $el) {
385
                $hoster = $el[0];
386
                $hash = $el[1];
387

388
                foreach ($results as $result) {
389
                    if ($result->new && $hoster === $result->tld && !$result->partnershop) {
390
391
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
392
                        if ($result->image !== "") {
393
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . urlencode($hash) . ".gif";
394
                        } else {
395
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . urlencode($hash) . ".gif";
396
397
                        }

398
399
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
400
                        $targetUrl = $result->link;
401

402
                        $gateHash = md5($targetUrl . $privateKey);
403
                        $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . urlencode($publicKey) . "&k=" . urlencode($gateHash) . "&url=" . urlencode($targetUrl) . "&q=" . urlencode($query);
404
                        $result->link = $newLink;
405
406
407
408
                        $result->partnershop = true;
                    }
                }
            }
409
        } catch (\ErrorException $e) {
410
411
412
413
414
            return $results;
        }

        return $results;
    }
415

416
417
    public function humanVerification($results)
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
418
        # Let's check if we need to implement a redirect for human verification
419
420
        if ($this->verificationCount > 10) {
            foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
421
422
423
                $link = $result->link;
                $day = Carbon::now()->day;
                $pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
424
425
426
                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
Dominik Hebeler's avatar
Dominik Hebeler committed
427
                $result->link = $url;
428
                $result->proxyLink = $proxyUrl;
Dominik Hebeler's avatar
Dominik Hebeler committed
429
430
            }
            return $results;
431
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
432
433
434
435
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
436
437
438
439
440
    public function authorize($key)
    {
        $postdata = http_build_query(array(
            'dummy' => rand(),
        ));
Karl Hasselbring's avatar
Karl Hasselbring committed
441
442
443
444
445
446
        $opts = array(
            'http' => array(
                'method' => 'POST',
                'header' => 'Content-type: application/x-www-form-urlencoded',
                'content' => $postdata,
            ),
Phil Höfer's avatar
Phil Höfer committed
447
448
449
450
451
        );

        $context = stream_context_create($opts);

        try {
452
            $link = "https://key.metager3.de/" . urlencode($key) . "/request-permission/api-access";
Phil Höfer's avatar
Phil Höfer committed
453
454
455
456
457
458
459
460
461
462
463
            $result = json_decode(file_get_contents($link, false, $context));
            if ($result->{'api-access'} == true) {
                return true;
            } else {
                return false;
            }
        } catch (\ErrorException $e) {
            return false;
        }
    }

464
465
    public function createQuicktips()
    {
466
        # Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
467
        $quicktips = new \App\Models\Quicktips\Quicktips($this->q, LaravelLocalization::getCurrentLocale(), $this->getTime());
468
469
470
        return $quicktips;
    }

Karl's avatar
Karl committed
471
472
473
474
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

475
476
    public function createSearchEngines(Request $request)
    {
477
        # Wenn es kein Suchwort gibt
Dominik Hebeler's avatar
Dominik Hebeler committed
478
        if (!$request->filled("eingabe") || $this->q === "") {
479
            return;
480
        }
481

Dominik Hebeler's avatar
Dominik Hebeler committed
482
        $this->enabledSearchengines = [];
483
        $overtureEnabled = false;
484

485
486
487
        # Check if selected focus is valid
        if (empty($this->sumaFile->foki->{$this->fokus})) {
            $this->fokus = "web";
488
        }
489

Karl Hasselbring's avatar
Karl Hasselbring committed
490
        $sumaNames = $this->sumaFile->foki->{$this->fokus}->sumas;
491

Karl Hasselbring's avatar
Karl Hasselbring committed
492
493
494
        $sumas = [];
        foreach ($sumaNames as $sumaName) {
            $sumas[$sumaName] = $this->sumaFile->sumas->{$sumaName};
495
496
        }

Karl Hasselbring's avatar
Karl Hasselbring committed
497
498
499
        $this->removeAdsFromListIfAdfree($sumas);

        foreach ($sumas as $sumaName => $suma) {
500
            # Check if this engine is disabled and can't be used
Karl Hasselbring's avatar
Karl Hasselbring committed
501
502
503
504
505
506
507
            $disabled = empty($suma->disabled) ? false : $suma->disabled;
            $autoDisabled = empty($suma->{"auto-disabled"}) ? false : $suma->{"auto-disabled"};
            if (
                $disabled || $autoDisabled
                || \Cookie::get($this->getFokus() . "_engine_" . $sumaName) === "off"
            ) {
                continue;
508
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
509

510
            $valid = true;
Karl Hasselbring's avatar
Karl Hasselbring committed
511
512
513
514

            # Check if this engine can use potentially defined query-filter
            foreach ($this->queryFilter as $filterName => $filter) {
                if (empty($this->sumaFile->filter->{"query-filter"}->$filterName->sumas->$sumaName)) {
515
516
                    $valid = false;
                    break;
Dominik Hebeler's avatar
Dominik Hebeler committed
517
518
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
519
520

            # Check if this engine can use potentially defined parameter-filter
Dominik Hebeler's avatar
Dominik Hebeler committed
521
522
            if ($valid) {
                foreach ($this->parameterFilter as $filterName => $filter) {
523
                    # We need to check if the searchengine supports the parameter value, too
Karl Hasselbring's avatar
Karl Hasselbring committed
524
                    if (empty($filter->sumas->$sumaName) || empty($filter->sumas->{$sumaName}->values->{$filter->value})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
525
526
527
528
529
                        $valid = false;
                        break;
                    }
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
530

531
            # Check if this engine should only be active when filter is used
Karl Hasselbring's avatar
Karl Hasselbring committed
532
            if ($suma->{"filter-opt-in"}) {
533
534
535
                # This search engine should only be used when a parameter filter of it is used
                $validTmp = false;
                foreach ($this->parameterFilter as $filterName => $filter) {
Karl Hasselbring's avatar
Karl Hasselbring committed
536
                    if (!empty($filter->sumas->{$sumaName})) {
537
538
539
540
541
542
543
544
                        $validTmp = true;
                        break;
                    }
                }
                if (!$validTmp) {
                    $valid = false;
                }
            }
Karl Hasselbring's avatar
Karl Hasselbring committed
545
546

            # If the suma is still valid, we can add it
547
            if ($valid) {
Karl Hasselbring's avatar
Karl Hasselbring committed
548
                $this->enabledSearchengines[$sumaName] = $suma;
549
            }
550
        }
551

Karl Hasselbring's avatar
Karl Hasselbring committed
552
553
        # Include Yahoo Ads if Yahoo is not enabled as a searchengine
        if (!$this->apiAuthorized && $this->fokus != "bilder" && empty($this->enabledSearchengines["yahoo"]) && isset($this->sumaFile->sumas->{"yahoo-ads"})) {
554
555
556
            $this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
        }

557
558
559
560
561
562
        # Special case if search engines are disabled
        # Since bing is normally only active if a filter is set but it should be active, too if yahoo is disabled
        if ($this->getFokus() === "web" && empty($this->enabledSearchengines["yahoo"]) && \Cookie::get("web_engine_bing") !== "off") {
            $this->enabledSearchengines["bing"] = $this->sumaFile->sumas->{"bing"};
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
563
        if (sizeof($this->enabledSearchengines) === 0) {
564
565
            $filter = "";
            foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
Dominik Hebeler's avatar
Dominik Hebeler committed
566
                $filter .= trans($this->sumaFile->filter->{"query-filter"}->{$queryFilter}->name) . ",";
567
568
            }
            $filter = rtrim($filter, ",");
Karl Hasselbring's avatar
Karl Hasselbring committed
569
570
            $error = trans('metaGer.engines.noSpecialSearch', [
                'fokus' => trans($this->sumaFile->foki->{$this->fokus}->{"display-name"}),
571
                'filter' => $filter,
Karl Hasselbring's avatar
Karl Hasselbring committed
572
            ]);
573
            $this->errors[] = $error;
574
        }
575
576
        $this->setEngines($request);
    }
577

Karl Hasselbring's avatar
Karl Hasselbring committed
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
    private function removeAdsFromListIfAdfree(&$sumas)
    {
        if ($this->apiAuthorized) {
            foreach ($sumas as $sumaName => $suma) {
                $ads = $suma->ads ?? false;
                if ($ads) {
                    unset($sumas[$sumaName]);

                    $adBackups = $suma->{"ad-backups"} ?? [];
                    $adBackupName = $adBackups->{$this->fokus} ?? null;
                    if (isset($adBackupName)) {
                        $this->sumaFile->sumas->{$adBackupName}->{"filter-opt-in"} = false;
                    }
                }
            }
        }
    }

596
597
    public function setEngines(Request $request, $enabledSearchengines = [])
    {
598
        if ($this->requestIsCached($request)) {
599
            # If this is a page other than 1 the request is "cached"
600
            $engines = $this->getCachedEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
601
            # We need to edit some Options of the Cached Search Engines
Phil Höfer's avatar
Phil Höfer committed
602
            foreach ($engines as $engine) {
603
                $engine->setResultHash($this->getSearchUid());
Dominik Hebeler's avatar
Dominik Hebeler committed
604
            }
605
            $this->engines = $engines;
606
        } else {
607
608
609
            if (sizeof($enabledSearchengines) > 0) {
                $this->enabledSearchengines = $enabledSearchengines;
            }
610
            $this->actuallyCreateSearchEngines($this->enabledSearchengines);
611
        }
612
    }
Dominik Hebeler's avatar
Dominik Hebeler committed
613

614
615
    public function startSearch()
    {
616
        # Wir starten alle Suchen
617
        foreach ($this->engines as $engine) {
618
            $engine->startSearch($this);
619
        }
620
621
    }

622
623
    # Spezielle Suchen und Sumas

624
    public function sumaIsSelected($suma, $request, $custom)
625
    {
626
        if ($custom) {
627
            if ($request->filled("engine_" . strtolower($suma["name"]))) {
628
629
630
631
632
633
634
635
636
637
638
                return true;
            }
        } else {
            $types = explode(",", $suma["type"]);
            if (in_array($this->fokus, $types)) {
                return true;
            }
        }
        return false;
    }

639
    public function actuallyCreateSearchEngines($enabledSearchengines)
640
641
    {
        $engines = [];
642
        foreach ($enabledSearchengines as $engineName => $engine) {
643

644
            if (!isset($engine->{"parser-class"})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
645
646
                die(var_dump($engine));
            }
647
            # Setze Pfad zu Parser
648
            $path = "App\\Models\\parserSkripte\\" . $engine->{"parser-class"};
649
650

            # Prüfe ob Parser vorhanden
651
652
653
            if (!file_exists(app_path() . "/Models/parserSkripte/" . $engine->{"parser-class"} . ".php")) {
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen, da kein Parser existiert");
                $this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine->{"display-name"}]);
654
655
656
657
658
659
                continue;
            }

            # Es wird versucht die Suchengine zu erstellen
            $time = microtime();
            try {
660
                $tmp = new $path($engineName, $engine, $this);
661
            } catch (\ErrorException $e) {
662
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen. " . var_dump($e));
663
664
665
                continue;
            }

666
            $engines[] = $tmp;
667
        }
668
        $this->engines = $engines;
669
670
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
671
672
673
674
675
676
677
    public function getAvailableParameterFilter()
    {
        $parameterFilter = $this->sumaFile->filter->{"parameter-filter"};

        $availableFilter = [];

        foreach ($parameterFilter as $filterName => $filter) {
678
            $values = $filter->values;
Dominik Hebeler's avatar
Dominik Hebeler committed
679
680
681
            # Check if any of the enabled search engines provide this filter
            foreach ($this->enabledSearchengines as $engineName => $engine) {
                if (!empty($filter->sumas->$engineName)) {
682
683
684
685
686
687
688
689
690
691
                    if (empty($availableFilter[$filterName])) {
                        $availableFilter[$filterName] = $filter;
                        unset($availableFilter[$filterName]->values);
                    }
                    if (empty($availableFilter[$filterName]->values)) {
                        $availableFilter[$filterName]->values = (object) ["" => $values->{""}];
                    }
                    foreach ($filter->sumas->{$engineName}->values as $key => $value) {
                        $availableFilter[$filterName]->values->$key = $values->$key;
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
692
693
                }
            }
694
695
            # We will also add the filter from the opt-in search engines (the searchengines that are only used when a filter of it is too)
            foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
696
                if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"} && \Cookie::get($this->getFokus() . "_engine_" . $suma) !== "off") {
697
                    if (!empty($filter->sumas->{$suma})) {
698
699
                        # If the searchengine is disabled this filter shouldn't be available
                        if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
Karl Hasselbring's avatar
Karl Hasselbring committed
700
701
                            || (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)
                        ) {
702
703
                            continue;
                        }
704
705
706
707
708
709
710
711
712
713
                        if (empty($availableFilter[$filterName])) {
                            $availableFilter[$filterName] = $filter;
                            unset($availableFilter[$filterName]->values);
                        }
                        if (empty($availableFilter[$filterName]->values)) {
                            $availableFilter[$filterName]->values = (object) ["" => $values->{""}];
                        }
                        foreach ($filter->sumas->{$suma}->values as $key => $value) {
                            $availableFilter[$filterName]->values->$key = $values->$key;
                        }
714
715
716
                    }
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
717
718
        }

719
720
721
722
723
724
725
726
727
        # Set the current values for the filters
        foreach ($availableFilter as $filterName => $filter) {
            if (\Request::filled($filter->{"get-parameter"})) {
                $filter->value = \Request::input($filter->{"get-parameter"});
            } else if (\Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"}) !== null) {
                $filter->value = \Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"});
            }
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
728
729
730
        return $availableFilter;
    }

731
732
    public function isBildersuche()
    {
733
        return $this->fokus === "bilder";
734
735
736
737
    }

    public function sumaIsAdsuche($suma, $overtureEnabled)
    {
738
        $sumaName = $suma["name"]->__toString();
739
        return
740
741
            $sumaName === "qualigo"
            || $sumaName === "similar_product_ads"
Karl Hasselbring's avatar
Karl Hasselbring committed
742
            || (!$overtureEnabled && $sumaName === "overtureAds");
743
744
745
746
747
    }

    public function sumaIsDisabled($suma)
    {
        return
748
749
        isset($suma['disabled'])
        && $suma['disabled']->__toString() === "1";
750
751
752
753
754
    }

    public function sumaIsOverture($suma)
    {
        return
755
756
        $suma["name"]->__toString() === "overture"
        || $suma["name"]->__toString() === "overtureAds";
757
758
759
760
761
    }

    public function sumaIsNotAdsuche($suma)
    {
        return
762
763
764
        $suma["name"]->__toString() !== "qualigo"
        && $suma["name"]->__toString() !== "similar_product_ads"
        && $suma["name"]->__toString() !== "overtureAds";
765
766
767
768
769
    }

    public function requestIsCached($request)
    {
        return
770
771
772
        $request->filled('next')
        && Cache::has($request->input('next'))
        && unserialize(Cache::get($request->input('next')))['page'] > 1;
773
774
775
776
    }

    public function getCachedEngines($request)
    {
777
        $next = unserialize(Cache::get($request->input('next')));
778
        $this->page = $next['page'];
779
        $engines = $next['engines'];
780
781
782
        return $engines;
    }

783
    public function waitForMainResults()
784
    {
785
        $engines = $this->engines;
786
        $enginesToWaitFor = [];
787
788
789
        $mainEngines = $this->sumaFile->foki->{$this->fokus}->main;
        foreach ($mainEngines as $mainEngine) {
            foreach ($engines as $engine) {
Dominik Hebeler's avatar
Dominik Hebeler committed
790
                if ($engine->name === $mainEngine) {
791
792
                    $enginesToWaitFor[] = $engine;
                }
793
794
            }
        }
795

Phil Höfer's avatar
Phil Höfer committed
796
        $timeStart = microtime(true);
797
        $answered = [];
798
        $results = null;
799
800
801
802
803
804
805
806

        # If there is no main searchengine to wait for or if the only main engine is yahoo-ads we will define a timeout of 1s
        $forceTimeout = null;
        if (sizeof($enginesToWaitFor) === 0 || (sizeof($enginesToWaitFor) === 1 && $enginesToWaitFor[0]->name === "yahoo-ads")) {
            $forceTimeout = 1;
        }

        while (sizeof($enginesToWaitFor) > 0 || ($forceTimeout !== null && (microtime(true) - $timeStart) < $forceTimeout)) {
Dominik Hebeler's avatar
Dominik Hebeler committed
807
            foreach ($enginesToWaitFor as $index => $engine) {
808
                if (Redis::get($engine->hash) !== null) {
Dominik Hebeler's avatar
Dominik Hebeler committed
809
810
811
                    $answered[] = $engine;
                    unset($enginesToWaitFor[$index]);
                    break;
812
                }
813
            }
814
            if ((microtime(true) - $timeStart) >= 2) {
815
                break;
Dominik Hebeler's avatar
Dominik Hebeler committed
816
817
            } else {
                usleep(50 * 1000);
818
            }
819
        }
820
821

        # Now we can add an entry to Redis which defines the starting time and how many engines should answer this request
Dominik Hebeler's avatar
Dominik Hebeler committed
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
        /*
    $pipeline = $redis->pipeline();
    $pipeline->hset($this->getRedisEngineResult() . "status", "startTime", $timeStart);
    $pipeline->hset($this->getRedisEngineResult() . "status", "engineCount", sizeof($engines));
    $pipeline->hset($this->getRedisEngineResult() . "status", "engineDelivered", sizeof($answered));
    # Add the cached engines as answered
    foreach ($engines as $engine) {
    if ($engine->cached) {
    $pipeline->hincrby($this->getRedisEngineResult() . "status", "engineDelivered", 1);
    $pipeline->hincrby($this->getRedisEngineResult() . "status", "engineAnswered", 1);
    }
    }
    foreach ($answered as $engine) {
    $pipeline->hset($this->getRedisEngineResult() . $engine, "delivered", "1");
    }
    $pipeline->execute();*/
838
    }
839

840
    public function retrieveResults()
841
    {
842
        $engines = $this->engines;
843
        # Von geladenen Engines die Ergebnisse holen
844
845
846
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
                try {
847
                    $engine->retrieveResults($this);
848
                } catch (\ErrorException $e) {
849
850
851
                    Log::error($e);
                }
            }
852
853
854
            if (!empty($engine->totalResults) && $engine->totalResults > $this->totalResults) {
                $this->totalResults = $engine->totalResults;
            }
855
856
857
858
            if (!empty($engine->alteredQuery) && !empty($engine->alterationOverrideQuery)) {
                $this->alteredQuery = $engine->alteredQuery;
                $this->alterationOverrideQuery = $engine->alterationOverrideQuery;
            }
859
        }
860
861
    }

Karl Hasselbring's avatar
Karl Hasselbring committed
862
    /*
863
864
     * Ende Suchmaschinenerstellung und Ergebniserhalt
     */
Karl's avatar
Karl committed
865

866
867
    public function parseFormData(Request $request)
    {
868

869
        # Sichert, dass der request in UTF-8 formatiert ist
870
        if ($request->input('encoding', 'utf8') !== "utf8") {
871
872
873
874
875
876
877
            # In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
            $input = $request->all();
            foreach ($input as $key => $value) {
                $input[$key] = mb_convert_encoding("$value", "UTF-8", "ISO-8859-1");
            }
            $request->replace($input);
        }