MetaGer.php 54.7 KB
Newer Older
1
2
3
4
<?php
namespace App;

use App;
5
use Cache;
6
use Carbon;
7
use Illuminate\Http\Request;
Phil Höfer's avatar
Phil Höfer committed
8
use Illuminate\Support\Facades\Redis;
9
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix    
Dominik Hebeler committed
10
use LaravelLocalization;
11
use Log;
12
use Predis\Connection\ConnectionException;
13
14
15

class MetaGer
{
16
17
18
19
20
21
22
23
    # Einstellungen für die Suche
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
24
    protected $time = 2000;
25
    protected $hostBlacklist = [];
26
    protected $domainBlacklist = [];
27
28
29
30
    private $urlBlacklist = [];
    protected $stopWords = [];
    protected $phrases = [];
    protected $engines = [];
31
    protected $totalResults = 0;
32
    protected $results = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
33
34
    protected $queryFilter = [];
    protected $parameterFilter = [];
35
36
37
38
    protected $ads = [];
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
39
    protected $availableFoki = [];
40
41
    protected $startCount = 0;
    protected $canCache = false;
42
    # Daten über die Abfrage$
43
    protected $ip;
Dominik Hebeler's avatar
Dominik Hebeler committed
44
    protected $useragent;
45
46
    protected $language;
    protected $agent;
47
    protected $apiKey = "";
Phil Höfer's avatar
Phil Höfer committed
48
    protected $apiAuthorized = false;
49
50
51
52
53
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
54
    protected $newtab;
55
    protected $domainsBlacklisted = [];
56
    protected $urlsBlacklisted = [];
57
    protected $url;
58
    protected $fullUrl;
59
    protected $languageDetect;
Dominik Hebeler's avatar
Dominik Hebeler committed
60
61
    protected $verificationId;
    protected $verificationCount;
62
63
    protected $searchUid;
    protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult;
64

65
66
    public function __construct()
    {
67
        # Timer starten
68
        $this->starttime = microtime(true);
69
        # Versuchen Blacklists einzulesen
70
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
71
            $tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
72
            $this->domainsBlacklisted = explode("\n", $tmp);
73
74
            $tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted = explode("\n", $tmp);
75
        } else {
76
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
77
78
        }

79
        # Parser Skripte einhängen
80
81
82
83
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
84
                require_once $path;
85
86
87
            }
        }

88
        # Cachebarkeit testen
89
90
91
92
93
94
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
95
96
97
98
99
100
101
102
103
        $this->canCache = false;
        $this->searchUid = md5(uniqid());
        $redisPrefix = "search";
        # This is a list on which the MetaGer process can do a blocking pop to wait for new results
        $this->redisResultWaitingKey = $redisPrefix . "." . $this->searchUid . ".ready";
        # This is a list of searchengines which have delivered results for this search
        $this->redisResultEngineList = $redisPrefix . "." . $this->searchUid . ".engines";
        # This is the key where the results of the engine are stored as well as some statistical data
        $this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results.";
104
    }
105

106
    # Erstellt aus den gesammelten Ergebnissen den View
107
    public function createView($quicktipResults = [])
108
    {
109
110
        # Hiermit werden die evtl. ausgewählten SuMas extrahiert, damit die Input-Boxen richtig gesetzt werden können
        $focusPages = [];
Aria Givi's avatar
Aria Givi committed
111

112
        foreach ($this->request->all() as $key => $value) {
113
114
            if (starts_with($key, 'engine_') && $value === 'on') {
                $focusPages[] = $key;
115
116
117
            }
        }

118
        $viewResults = [];
119
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
120
        foreach ($this->results as $result) {
121
122
123
124
            $viewResults[] = get_object_vars($result);
        }
        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();
125
126
        if ($this->fokus === "bilder") {
            switch ($this->out) {
127
                case 'results':
128
                    return view('resultpages.results_images')
129
130
131
132
133
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
134
                        ->with('apiAuthorized', $this->apiAuthorized)
135
                        ->with('metager', $this)
136
                        ->with('browser', (new Agent())->browser());
137
                default:
138
                    return view('resultpages.resultpage_images')
139
140
141
142
143
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
144
                        ->with('apiAuthorized', $this->apiAuthorized)
145
                        ->with('metager', $this)
146
147
148
149
                        ->with('browser', (new Agent())->browser())
                        ->with('quicktips', $quicktipResults)
                        ->with('focus', $this->fokus)
                        ->with('resultcount', count($this->results));
150
            }
151
152
153
        } else {
            switch ($this->out) {
                case 'results':
154
                    return view('resultpages.results')
155
156
157
158
159
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
160
                        ->with('apiAuthorized', $this->apiAuthorized)
161
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
162
163
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
164
165
                    break;
                case 'results-with-style':
166
                    return view('resultpages.resultpage')
167
168
169
170
171
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
172
                        ->with('apiAuthorized', $this->apiAuthorized)
173
174
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
Dominik Hebeler's avatar
Dominik Hebeler committed
175
176
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
177
                    break;
Phil Höfer's avatar
Phil Höfer committed
178
                case 'rich':
179
                    return view('resultpages.metager3rich')
180
                        ->with('results', $viewResults)
Phil Höfer's avatar
Phil Höfer committed
181
182
183
184
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
185
                        ->with('apiAuthorized', $this->apiAuthorized)
Phil Höfer's avatar
Phil Höfer committed
186
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
187
188
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
Phil Höfer's avatar
Phil Höfer committed
189
                    break;
190
                case 'rss20':
191
                    return view('resultpages.metager3resultsrss20')
192
193
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
Phil Höfer's avatar
Phil Höfer committed
194
                        ->with('apiAuthorized', $this->apiAuthorized)
195
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
196
197
                        ->with('resultcount', sizeof($viewResults))
                        ->with('fokus', $this->fokus);
198
                    break;
199
200
201
                case 'api':
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'apiAuthorized' => $this->apiAuthorized])->header('Content-Type', 'application/xml');
                    break;
Aria Givi's avatar
Aria Givi committed
202
                case 'atom10':
203
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'apiAuthorized' => true])
204
                        ->header('Content-Type', 'application/xml');
Aria Givi's avatar
Aria Givi committed
205
                    break;
206
                case 'result-count':
207
208
                    # Wir geben die Ergebniszahl und die benötigte Zeit zurück:
                    return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
209
                    break;
210
                default:
211
                    return view('resultpages.resultpage')
212
                        ->with('eingabe', $this->eingabe)
213
                        ->with('focusPages', $focusPages)
214
215
216
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
217
                        ->with('apiAuthorized', $this->apiAuthorized)
218
                        ->with('metager', $this)
219
                        ->with('browser', (new Agent())->browser())
220
                        ->with('quicktips', $quicktipResults)
221
222
                        ->with('resultcount', count($this->results))
                        ->with('focus', $this->fokus);
223
224
                    break;
            }
225
226
227
        }
    }

Phil Höfer's avatar
Phil Höfer committed
228
    public function prepareResults()
229
    {
Phil Höfer's avatar
Phil Höfer committed
230
        $engines = $this->engines;
231

Phil Höfer's avatar
Phil Höfer committed
232
233
        // combine
        $combinedResults = $this->combineResults($engines);
234
235
236
237

        # Wir bestimmen die Sprache eines jeden Suchergebnisses
        $this->results = $this->addLangCodes($this->results);

Phil Höfer's avatar
Phil Höfer committed
238
239
240
241
242
        // sort
        //$sortedResults = $this->sortResults($engines);
        // filter
        // augment (boost&adgoal)
        // authorize
Phil Höfer's avatar
Phil Höfer committed
243
244
245
        if ($this->apiKey) {
            $this->apiAuthorized = $this->authorize($this->apiKey);
        }
Phil Höfer's avatar
Phil Höfer committed
246
        // misc (WiP)
247
        if ($this->fokus == "nachrichten") {
248
249
250
            $this->results = array_filter($this->results, function ($v, $k) {
                return !is_null($v->getRank());
            }, ARRAY_FILTER_USE_BOTH);
251
252
253
254
255
256
257
258
259
260
            uasort($this->results, function ($a, $b) {
                $datea = $a->getDate();
                $dateb = $b->getDate();
                return $dateb - $datea;
            });
        } else {
            uasort($this->results, function ($a, $b) {
                if ($a->getRank() == $b->getRank()) {
                    return 0;
                }
261

262
263
264
                return ($a->getRank() < $b->getRank()) ? 1 : -1;
            });
        }
265

266
267
        # Validate Results
        $newResults = [];
268
269
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
270
                $newResults[] = $result;
271
272
            }

273
274
275
        }
        $this->results = $newResults;

276
        #Adgoal Implementation
277
278
279
        if (!$this->apiAuthorized) {
            $this->results = $this->parseAdgoal($this->results);
        }
280

Dominik Hebeler's avatar
Dominik Hebeler committed
281
282
        # Human Verification
        $this->results = $this->humanVerification($this->results);
283
        $this->ads = $this->humanVerification($this->ads);
Dominik Hebeler's avatar
Dominik Hebeler committed
284

285
        $counter = 0;
286
        $firstRank = 0;
287

288
        if (isset($this->startForwards)) {
289
            $this->startCount = $this->startForwards;
290
        } elseif (isset($this->startBackwards)) {
291
            $this->startCount = $this->startBackwards - count($this->results) - 1;
292
        } else {
293
294
295
            $this->startCount = 0;
        }

296
297
        foreach ($this->results as $result) {
            if ($counter === 0) {
298
                $firstRank = $result->rank;
299
300
            }

301
            $counter++;
302
            $result->number = $counter + $this->startCount;
303
            $confidence = 0;
304
305
306
            if ($firstRank > 0) {
                $confidence = $result->rank / $firstRank;
            } else {
307
                $confidence = 0;
308
309
310
            }

            if ($confidence > 0.65) {
311
                $result->color = "#FF4000";
312
            } elseif ($confidence > 0.4) {
313
                $result->color = "#FF0080";
314
            } elseif ($confidence > 0.2) {
315
                $result->color = "#C000C0";
316
            } else {
317
                $result->color = "#000000";
318
319
            }

320
321
        }

322
        if (count($this->results) <= 0) {
323
324
325
326
327
328
            if (strlen($this->site) > 0) {
                $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
                $this->errors[] = trans('metaGer.results.failedSitesearch', ['altSearch' => $no_sitesearch_query]);
            } else {
                $this->errors[] = trans('metaGer.results.failed');
            }
329
        }
330

331
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
332
            $page = $this->page + 1;
333
            $this->next = [
334
                'page' => $page,
335
                'startForwards' => $this->results[count($this->results) - 1]->number,
336
                'engines' => $this->next,
337
338
            ];
            Cache::put(md5(serialize($this->next)), serialize($this->next), 60);
339
340
        } else {
            $this->next = [];
341
342
        }

343
    }
344

345
346
    private function addLangCodes($results)
    {
347
348
349
350
351
        # Wenn es keine Ergebnisse gibt, brauchen wir uns gar nicht erst zu bemühen
        if (sizeof($results) === 0) {
            return $results;
        }

352
353
354
355
356
        # Bei der Spracheinstellung "all" wird nicht gefiltert
        if ($this->getLang() === "all") {
            return $results;
        } else {
            # Ansonsten müssen wir jedem Result einen Sprachcode hinzufügen
357
            $id = 0;
358
359
360
361
362
            $langStrings = [];
            foreach ($results as $result) {
                # Wir geben jedem Ergebnis eine ID um später die Sprachcodes zuordnen zu können
                $result->id = $id;

Dominik Hebeler's avatar
Dominik Hebeler committed
363
                $langStrings["result_" . $id] = utf8_encode($result->getLangString());
364
365
366
367
368
369
370
371

                $id++;
            }
            # Wir schreiben die Strings in eine temporäre JSON-Datei,
            # Da das Array unter umständen zu groß ist für eine direkte Übergabe an das Skript
            $filename = "/tmp/" . getmypid();
            file_put_contents($filename, json_encode($langStrings));
            $langDetectorPath = app_path() . "/Models/lang.pl";
372
373
            $lang = exec("echo '$filename' | $langDetectorPath");
            $lang = json_decode($lang, true);
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390

            # Wir haben nun die Sprachcodes der einzelnen Ergebnisse.
            # Diese müssen wir nur noch korrekt zuordnen, dann sind wir fertig.
            foreach ($lang as $key => $langCode) {
                # Prefix vom Key entfernen:
                $id = intval(str_replace("result_", "", $key));
                foreach ($this->results as $result) {
                    if ($result->id === $id) {
                        $result->langCode = $langCode;
                        break;
                    }
                }
            }
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
    public function combineResults($engines)
    {
        foreach ($engines as $engine) {
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
            foreach ($engine->results as $result) {
                if ($result->valid) {
                    $this->results[] = $result;
                }
            }
            foreach ($engine->ads as $ad) {
                $this->ads[] = $ad;
            }
        }
    }

411
412
    public function parseAdgoal($results)
    {
413
        $publicKey = getenv('adgoal_public');
414
        $privateKey = getenv('adgoal_private');
415
        if ($publicKey === false) {
416
417
418
            return $results;
        }
        $tldList = "";
419
420
        try {
            foreach ($results as $result) {
421
                $link = $result->anzeigeLink;
422
                if (strpos($link, "http") !== 0) {
423
424
425
426
427
428
429
430
431
432
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

433
            # Query
434
435
            $query = urlencode($this->q);

436
            $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . $publicKey . "&k=" . $hash . "&tld=" . $tldList . "&q=" . $query;
437
438
439
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
440
            foreach ($answer as $el) {
441
                $hoster = $el[0];
442
                $hash = $el[1];
443

444
                foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
445
                    if ($hoster === $result->tld && !$result->partnershop) {
446
447
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
448
                        if ($result->image !== "") {
449
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . $hash . ".gif";
450
                        } else {
451
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . $hash . ".gif";
452
453
                        }

454
455
456
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
                        $targetUrl = $result->anzeigeLink;
457
                        if (strpos($targetUrl, "http") !== 0) {
458
                            $targetUrl = "http://" . $targetUrl;
459
460
                        }

461
462
463
                        $gateHash = md5($targetUrl . $privateKey);
                        $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . $publicKey . "&k=" . $gateHash . "&url=" . urlencode($targetUrl) . "&q=" . $query;
                        $result->link = $newLink;
464
465
466
467
                        $result->partnershop = true;
                    }
                }
            }
468
        } catch (\ErrorException $e) {
469
470
471
472
473
            return $results;
        }

        return $results;
    }
474

475
476
    public function humanVerification($results)
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
477
        # Let's check if we need to implement a redirect for human verification
478
479
        if ($this->verificationCount > 10) {
            foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
480
481
482
                $link = $result->link;
                $day = Carbon::now()->day;
                $pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
483
484
485
                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
Dominik Hebeler's avatar
Dominik Hebeler committed
486
                $result->link = $url;
487
                $result->proxyLink = $proxyUrl;
Dominik Hebeler's avatar
Dominik Hebeler committed
488
489
            }
            return $results;
490
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
491
492
493
494
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
495
496
497
498
499
500
    public function authorize($key)
    {
        $postdata = http_build_query(array(
            'dummy' => rand(),
        ));
        $opts = array('http' => array(
501
502
            'method' => 'POST',
            'header' => 'Content-type: application/x-www-form-urlencoded',
Phil Höfer's avatar
Phil Höfer committed
503
504
505
506
507
508
509
            'content' => $postdata,
        ),
        );

        $context = stream_context_create($opts);

        try {
510
            $link = "https://key.metager3.de/" . urlencode($key) . "/request-permission/api-access";
Phil Höfer's avatar
Phil Höfer committed
511
512
513
514
515
516
517
518
519
520
521
522
            $result = json_decode(file_get_contents($link, false, $context));
            if ($result->{'api-access'} == true) {
                return true;
            } else {
                return false;
            }

        } catch (\ErrorException $e) {
            return false;
        }
    }

523
524
    public function createQuicktips()
    {
525
        # Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
526
        $quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime());
527
528
529
        return $quicktips;
    }

Karl's avatar
Karl committed
530
531
532
533
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

534
535
    public function createSearchEngines(Request $request)
    {
536
        # Wenn es kein Suchwort gibt
Dominik Hebeler's avatar
Dominik Hebeler committed
537
        if (!$request->filled("eingabe") || $this->q === "") {
538
            return;
539
        }
540

Dominik Hebeler's avatar
Dominik Hebeler committed
541
        $this->enabledSearchengines = [];
542
        $overtureEnabled = false;
543

544
545
546
        # Check if selected focus is valid
        if (empty($this->sumaFile->foki->{$this->fokus})) {
            $this->fokus = "web";
547
        }
548
549
        foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
            # Check if this engine is disabled and can't be used
550
            $disabled = empty($this->sumaFile->sumas->{$suma}->disabled) ? false : $this->sumaFile->sumas->{$suma}->disabled;
551
552
            $autoDisabled = empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) ? false : $this->sumaFile->sumas->{$suma}->{"auto-disabled"};
            if ($disabled || $autoDisabled) {
553
                continue;
554
555
            }

556
557
558
            # Check if this engine can use eventually defined query-filter
            $valid = true;
            foreach ($this->queryFilter as $queryFilter => $filter) {
Dominik Hebeler's avatar
Dominik Hebeler committed
559
                if (empty($this->sumaFile->filter->{"query-filter"}->$queryFilter->sumas->$suma)) {
560
561
                    $valid = false;
                    break;
Dominik Hebeler's avatar
Dominik Hebeler committed
562
563
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
564
565
566
            # Check if this engine can use eventually defined parameter-filter
            if ($valid) {
                foreach ($this->parameterFilter as $filterName => $filter) {
567
568
569
                    # We need to check if the searchengine supports the parameter value, too
                    $value = $request->input($filter->{"get-parameter"}, "");
                    if (empty($filter->sumas->$suma) || empty($filter->sumas->{$suma}->values->{$value})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
570
571
572
573
574
                        $valid = false;
                        break;
                    }
                }
            }
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
            # Check if this engine should only be active when filter is used
            if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"}) {
                # This search engine should only be used when a parameter filter of it is used
                $validTmp = false;
                foreach ($this->parameterFilter as $filterName => $filter) {
                    if (!empty($filter->sumas->{$suma})) {
                        $validTmp = true;
                        break;
                    }
                }
                if (!$validTmp) {
                    $valid = false;
                }

            }
590
591
            # If it can we add it
            if ($valid) {
Dominik Hebeler's avatar
Dominik Hebeler committed
592
                $this->enabledSearchengines[$suma] = $this->sumaFile->sumas->{$suma};
593
            }
594

595
        }
596

597
598
599
600
601
        # Implements Yahoo Ads if Yahoo is not enabled as a searchengine
        if (empty($this->enabledSearchengines["yahoo"]) && $this->fokus != "bilder" && !empty($this->sumaFile->sumas->{"yahoo-ads"})) {
            $this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
602
        if (sizeof($this->enabledSearchengines) === 0) {
603
604
            $filter = "";
            foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
Dominik Hebeler's avatar
Dominik Hebeler committed
605
                $filter .= trans($this->sumaFile->filter->{"query-filter"}->{$queryFilter}->name) . ",";
606
607
608
609
610
            }
            $filter = rtrim($filter, ",");
            $error = trans('metaGer.engines.noSpecialSearch', ['fokus' => trans($this->sumaFile->foki->{$this->fokus}->{"display-name"}),
                'filter' => $filter]);
            $this->errors[] = $error;
611
        }
612

613
        $engines = [];
614
        $typeslist = [];
615
        $counter = 0;
616

617
        if ($this->requestIsCached($request)) {
618
            # If this is a page other than 1 the request is "cached"
619
            $engines = $this->getCachedEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
620
            # We need to edit some Options of the Cached Search Engines
Phil Höfer's avatar
Phil Höfer committed
621
            foreach ($engines as $engine) {
Dominik Hebeler's avatar
Dominik Hebeler committed
622
623
                $engine->setResultHash($this->getHashCode());
            }
624
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
625
            $engines = $this->actuallyCreateSearchEngines($this->enabledSearchengines);
626
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
627

628
        # Wir starten alle Suchen
629
630
        foreach ($engines as $engine) {
            $engine->startSearch($this);
631
        }
632

633
634
635
636
637
        /* Wir warten auf die Antwort der Suchmaschinen
         * Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
         * Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
         * $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
         * welche Suchmaschinen nicht rechtzeitig geantwortet haben.
638
639
         */

640
        $this->waitForResults($engines);
641
642

        $this->retrieveResults($engines);
643
644
645
646
647
        foreach ($engines as $engine) {
            if (!empty($engine->totalResults) && $engine->totalResults > $this->totalResults) {
                $this->totalResults = $engine->totalResults;
            }
        }
648
649
    }

650
651
    # Spezielle Suchen und Sumas

652
    public function sumaIsSelected($suma, $request, $custom)
653
    {
654
        if ($custom) {
655
            if ($request->filled("engine_" . strtolower($suma["name"]))) {
656
657
658
659
660
661
662
663
664
665
666
                return true;
            }
        } else {
            $types = explode(",", $suma["type"]);
            if (in_array($this->fokus, $types)) {
                return true;
            }
        }
        return false;
    }

667
    public function actuallyCreateSearchEngines($enabledSearchengines)
668
669
    {
        $engines = [];
670
        foreach ($enabledSearchengines as $engineName => $engine) {
671

672
            if (!isset($engine->{"parser-class"})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
673
674
                die(var_dump($engine));
            }
675
            # Setze Pfad zu Parser
676
            $path = "App\\Models\\parserSkripte\\" . $engine->{"parser-class"};
677
678

            # Prüfe ob Parser vorhanden
679
680
681
            if (!file_exists(app_path() . "/Models/parserSkripte/" . $engine->{"parser-class"} . ".php")) {
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen, da kein Parser existiert");
                $this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine->{"display-name"}]);
682
683
684
685
686
687
                continue;
            }

            # Es wird versucht die Suchengine zu erstellen
            $time = microtime();
            try {
688
                $tmp = new $path($engineName, $engine, $this);
689
            } catch (\ErrorException $e) {
690
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen. " . var_dump($e));
691
692
693
                continue;
            }

694
            $engines[] = $tmp;
695
696
697
698
        }
        return $engines;
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
699
700
701
702
703
704
705
706
707
708
709
710
711
    public function getAvailableParameterFilter()
    {
        $parameterFilter = $this->sumaFile->filter->{"parameter-filter"};

        $availableFilter = [];

        foreach ($parameterFilter as $filterName => $filter) {
            # Check if any of the enabled search engines provide this filter
            foreach ($this->enabledSearchengines as $engineName => $engine) {
                if (!empty($filter->sumas->$engineName)) {
                    $availableFilter[$filterName] = $filter;
                }
            }
712
713
714
715
            # We will also add the filter from the opt-in search engines (the searchengines that are only used when a filter of it is too)
            foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
                if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"}) {
                    if (!empty($filter->sumas->{$suma})) {
716
717
718
719
720
                        # If the searchengine is disabled this filter shouldn't be available
                        if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
                            || (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)) {
                            continue;
                        }
721
722
723
724
                        $availableFilter[$filterName] = $filter;
                    }
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
725
726
727
728
729
        }

        return $availableFilter;
    }

730
731
    public function isBildersuche()
    {
732
        return $this->fokus === "bilder";
733
734
735
736
    }

    public function sumaIsAdsuche($suma, $overtureEnabled)
    {
737
        $sumaName = $suma["name"]->__toString();
738
        return
739
740
            $sumaName === "qualigo"
            || $sumaName === "similar_product_ads"
Karl Hasselbring's avatar
Karl Hasselbring committed
741
            || (!$overtureEnabled && $sumaName === "overtureAds");
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
    }

    public function sumaIsDisabled($suma)
    {
        return
        isset($suma['disabled'])
        && $suma['disabled']->__toString() === "1";
    }

    public function sumaIsOverture($suma)
    {
        return
        $suma["name"]->__toString() === "overture"
        || $suma["name"]->__toString() === "overtureAds";
    }

    public function sumaIsNotAdsuche($suma)
    {
        return
        $suma["name"]->__toString() !== "qualigo"
        && $suma["name"]->__toString() !== "similar_product_ads"
        && $suma["name"]->__toString() !== "overtureAds";
    }

    public function requestIsCached($request)
    {
        return
Dominik Hebeler's avatar
Dominik Hebeler committed
769
        $request->filled('next')
770
771
772
773
774
775
        && Cache::has($request->input('next'))
        && unserialize(Cache::get($request->input('next')))['page'] > 1;
    }

    public function getCachedEngines($request)
    {
776
        $next = unserialize(Cache::get($request->input('next')));
777
        $this->page = $next['page'];
778
        $engines = $next['engines'];
779
780
781
782
783
784
785
786
787
        if (isset($next['startForwards'])) {
            $this->startForwards = $next['startForwards'];
        }
        if (isset($next['startBackwards'])) {
            $this->startBackwards = $next['startBackwards'];
        }
        return $engines;
    }

788
    public function waitForResults($engines)
789
    {
790
791
792
793
        $enginesToWaitFor = [];
        foreach ($engines as $engine) {
            if ($engine->cached || !isset($engine->engine->main) || !$engine->engine->main) {
                continue;
794
            }
795
            $enginesToWaitFor[] = $engine;
796
        }
797

Phil Höfer's avatar
Phil Höfer committed
798
        $timeStart = microtime(true);
799
        $results = null;
800
801
802
803
804
805
806
807
808
809
810
        while (sizeof($enginesToWaitFor) > 0) {
            $newEngine = Redis::blpop($this->redisResultWaitingKey, 5);
            if ($newEngine === null || sizeof($newEngine) !== 2) {
                continue;
            } else {
                $newEngine = $newEngine[1];
                foreach ($enginesToWaitFor as $index => $engine) {
                    if ($engine->name === $newEngine) {
                        unset($enginesToWaitFor[$index]);
                        break;
                    }
811
                }
812
            }
813
            if ((microtime(true) - $timeStart) >= 2) {
814
815
                break;
            }
816
        }
817
    }
818

819
820
821
    public function retrieveResults($engines)
    {
        # Von geladenen Engines die Ergebnisse holen
822
823
824
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
                try {
825
                    $engine->retrieveResults($this);
826
                } catch (\ErrorException $e) {
827
828
829
830
                    Log::error($e);
                }
            }
        }
831

832
        $this->engines = $engines;
833
834
    }

835
836
837
/*
 * Ende Suchmaschinenerstellung und Ergebniserhalt
 */
Karl's avatar
Karl committed
838

839
840
    public function parseFormData(Request $request)
    {
841

842
        # Sichert, dass der request in UTF-8 formatiert ist
843
        if ($request->input('encoding', 'utf8') !== "utf8") {
844
845
846
847
848
849
850
            # In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
            $input = $request->all();
            foreach ($input as $key => $value) {
                $input[$key] = mb_convert_encoding("$value", "UTF-8", "ISO-8859-1");
            }
            $request->replace($input);
        }
851
        $this->url = $request->url();
852
        $this->fullUrl = $request->fullUrl();
853
        # Zunächst überprüfen wir die eingegebenen Einstellungen:
854
        # Fokus
855
        $this->fokus = $request->input('focus', 'web');
856
        # Suma-File
857
        if (App::isLocale("en")) {
858
            $this->sumaFile = config_path() . "/sumasEn.json";
859
        } else {
860
            $this->sumaFile = config_path() . "/sumas.json";
861
        }
862
        if (!file_exists($this->sumaFile)) {
863
            die(trans('metaGer.formdata.cantLoad'));
864
865
        } else {
            $this->sumaFile = json_decode(file_get_contents($this->sumaFile));
866
        }
867
        # Sucheingabe
868
        $this->eingabe = trim($request->input('eingabe', ''));
869
        $this->q = $this->eingabe;
870
        # IP
871
        $this->ip = $request->ip();
872
873
874
875
        # Unser erster Schritt wird sein, IP-Adresse und USER-Agent zu anonymisieren, damit
        # nicht einmal wir selbst noch Zugriff auf die Daten haben:
        $this->ip = preg_replace("/(\d+)\.(\d+)\.\d+.\d+/s", "$1.$2.0.0", $this->ip);

Dominik Hebeler's avatar
Dominik Hebeler committed
876
877
        $this->useragent = $request->header('User-Agent');

878
        # Language
879
        if (isset($_SERVER['HTTP_LANGUAGE'])) {
880
            $this->language = $_SERVER['HTTP_LANGUAGE'];
881
        } else {
882
883
            $this->language = "";
        }
884

885
        # Page
886
        $this->page = 1;
887
888
        # Lang
        $this->lang = $request->input('lang', 'all');
889
890
        if ($this->lang !== "de" && $this->lang !== "en" && $this->lang !== "all") {
            $this->lang = "all";
891
        }
892

893
        $this->agent = new Agent();
894
        $this->mobile = $this->agent->isMobile();