MetaGer.php 52.6 KB
Newer Older
1
2
3
4
<?php
namespace App;

use App;
5
use Cache;
6
use Carbon;
7
use Illuminate\Http\Request;
Phil Höfer's avatar
Phil Höfer committed
8
use Illuminate\Support\Facades\Redis;
9
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix    
Dominik Hebeler committed
10
use LaravelLocalization;
11
use Log;
12
use Predis\Connection\ConnectionException;
13
14
15

class MetaGer
{
16
17
18
19
20
21
22
23
    # Einstellungen für die Suche
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
24
    protected $time = 2000;
25
    protected $hostBlacklist = [];
26
    protected $domainBlacklist = [];
27
28
29
30
    private $urlBlacklist = [];
    protected $stopWords = [];
    protected $phrases = [];
    protected $engines = [];
31
    protected $totalResults = 0;
32
    protected $results = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
33
34
    protected $queryFilter = [];
    protected $parameterFilter = [];
35
36
37
38
    protected $ads = [];
    protected $warnings = [];
    protected $errors = [];
    protected $addedHosts = [];
39
    protected $availableFoki = [];
40
41
    protected $startCount = 0;
    protected $canCache = false;
42
    # Daten über die Abfrage$
43
    protected $ip;
Dominik Hebeler's avatar
Dominik Hebeler committed
44
    protected $useragent;
45
46
    protected $language;
    protected $agent;
47
    protected $apiKey = "";
Phil Höfer's avatar
Phil Höfer committed
48
    protected $apiAuthorized = false;
49
    protected $next = [];
50
51
52
53
54
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
55
    protected $newtab;
56
    protected $domainsBlacklisted = [];
57
    protected $urlsBlacklisted = [];
58
    protected $url;
59
    protected $fullUrl;
60
    protected $languageDetect;
Dominik Hebeler's avatar
Dominik Hebeler committed
61
62
    protected $verificationId;
    protected $verificationCount;
63
    protected $searchUid;
64
    protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult, $redisCurrentResultList;
65

66
    public function __construct($hash = "")
67
    {
68
        # Timer starten
69
        $this->starttime = microtime(true);
70
        # Versuchen Blacklists einzulesen
71
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
72
            $tmp = file_get_contents(config_path() . "/blacklistDomains.txt");
73
            $this->domainsBlacklisted = explode("\n", $tmp);
74
75
            $tmp = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted = explode("\n", $tmp);
76
        } else {
77
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
78
79
        }

80
        # Parser Skripte einhängen
81
82
83
84
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
85
                require_once $path;
86
87
88
            }
        }

89
        # Cachebarkeit testen
90
91
92
93
94
95
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
96
97
98
99
100
        if ($hash === "") {
            $this->searchUid = md5(uniqid());
        } else {
            $this->searchUid = $hash;
        }
101
102
103
104
105
106
107
        $redisPrefix = "search";
        # This is a list on which the MetaGer process can do a blocking pop to wait for new results
        $this->redisResultWaitingKey = $redisPrefix . "." . $this->searchUid . ".ready";
        # This is a list of searchengines which have delivered results for this search
        $this->redisResultEngineList = $redisPrefix . "." . $this->searchUid . ".engines";
        # This is the key where the results of the engine are stored as well as some statistical data
        $this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results.";
108
109
110
        # A list of all search results already delivered to the user (sorted of course)
        $this->redisCurrentResultList = $redisPrefix . "." . $this->searchUid . ".currentResults";

111
    }
112

113
    # Erstellt aus den gesammelten Ergebnissen den View
114
    public function createView($quicktipResults = [])
115
    {
116
117
        # Hiermit werden die evtl. ausgewählten SuMas extrahiert, damit die Input-Boxen richtig gesetzt werden können
        $focusPages = [];
Aria Givi's avatar
Aria Givi committed
118

119
        foreach ($this->request->all() as $key => $value) {
120
121
            if (starts_with($key, 'engine_') && $value === 'on') {
                $focusPages[] = $key;
122
123
124
            }
        }

125
        $viewResults = [];
126
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
127
        foreach ($this->results as $result) {
128
129
130
131
            $viewResults[] = get_object_vars($result);
        }
        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();
132
133
        if ($this->fokus === "bilder") {
            switch ($this->out) {
134
                case 'results':
135
                    return view('resultpages.results_images')
136
137
138
139
140
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
141
                        ->with('apiAuthorized', $this->apiAuthorized)
142
                        ->with('metager', $this)
143
                        ->with('browser', (new Agent())->browser());
144
                default:
145
                    return view('resultpages.resultpage_images')
146
147
148
149
150
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
151
                        ->with('apiAuthorized', $this->apiAuthorized)
152
                        ->with('metager', $this)
153
154
155
156
                        ->with('browser', (new Agent())->browser())
                        ->with('quicktips', $quicktipResults)
                        ->with('focus', $this->fokus)
                        ->with('resultcount', count($this->results));
157
            }
158
159
160
        } else {
            switch ($this->out) {
                case 'results':
161
                    return view('resultpages.results')
162
163
164
165
166
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
167
                        ->with('apiAuthorized', $this->apiAuthorized)
168
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
169
170
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
171
172
                    break;
                case 'results-with-style':
173
                    return view('resultpages.resultpage')
174
175
176
177
178
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
179
                        ->with('apiAuthorized', $this->apiAuthorized)
180
181
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
Dominik Hebeler's avatar
Dominik Hebeler committed
182
183
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
184
                    break;
Phil Höfer's avatar
Phil Höfer committed
185
                case 'rich':
186
                    return view('resultpages.metager3rich')
187
                        ->with('results', $viewResults)
Phil Höfer's avatar
Phil Höfer committed
188
189
190
191
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
192
                        ->with('apiAuthorized', $this->apiAuthorized)
Phil Höfer's avatar
Phil Höfer committed
193
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
194
195
                        ->with('browser', (new Agent())->browser())
                        ->with('fokus', $this->fokus);
Phil Höfer's avatar
Phil Höfer committed
196
                    break;
197
                case 'rss20':
198
                    return view('resultpages.metager3resultsrss20')
199
200
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
Phil Höfer's avatar
Phil Höfer committed
201
                        ->with('apiAuthorized', $this->apiAuthorized)
202
                        ->with('metager', $this)
Dominik Hebeler's avatar
Dominik Hebeler committed
203
204
                        ->with('resultcount', sizeof($viewResults))
                        ->with('fokus', $this->fokus);
205
                    break;
206
207
208
                case 'api':
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'apiAuthorized' => $this->apiAuthorized])->header('Content-Type', 'application/xml');
                    break;
Aria Givi's avatar
Aria Givi committed
209
                case 'atom10':
210
                    return response()->view('resultpages.metager3resultsatom10', ['results' => $viewResults, 'eingabe' => $this->eingabe, 'metager' => $this, 'resultcount' => sizeof($viewResults), 'apiAuthorized' => true])
211
                        ->header('Content-Type', 'application/xml');
Aria Givi's avatar
Aria Givi committed
212
                    break;
213
                case 'result-count':
214
215
                    # Wir geben die Ergebniszahl und die benötigte Zeit zurück:
                    return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
216
                    break;
217
                default:
218
                    return view('resultpages.resultpage')
219
                        ->with('eingabe', $this->eingabe)
220
                        ->with('focusPages', $focusPages)
221
222
223
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
Phil Höfer's avatar
Phil Höfer committed
224
                        ->with('apiAuthorized', $this->apiAuthorized)
225
                        ->with('metager', $this)
226
                        ->with('browser', (new Agent())->browser())
227
                        ->with('quicktips', $quicktipResults)
228
229
                        ->with('resultcount', count($this->results))
                        ->with('focus', $this->fokus);
230
231
                    break;
            }
232
233
234
        }
    }

Phil Höfer's avatar
Phil Höfer committed
235
    public function prepareResults()
236
    {
Phil Höfer's avatar
Phil Höfer committed
237
238
        $engines = $this->engines;
        // combine
239
        $this->combineResults($engines);
Phil Höfer's avatar
Phil Höfer committed
240
        // misc (WiP)
241
        if ($this->fokus == "nachrichten") {
242
243
244
            $this->results = array_filter($this->results, function ($v, $k) {
                return !is_null($v->getRank());
            }, ARRAY_FILTER_USE_BOTH);
245
246
247
248
249
250
251
252
253
254
            uasort($this->results, function ($a, $b) {
                $datea = $a->getDate();
                $dateb = $b->getDate();
                return $dateb - $datea;
            });
        } else {
            uasort($this->results, function ($a, $b) {
                if ($a->getRank() == $b->getRank()) {
                    return 0;
                }
255

256
257
258
                return ($a->getRank() < $b->getRank()) ? 1 : -1;
            });
        }
259

260
261
        # Validate Results
        $newResults = [];
262
263
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
264
                $newResults[] = $result;
265
266
            }

267
268
269
        }
        $this->results = $newResults;

270
        #Adgoal Implementation
271
272
273
        if (!$this->apiAuthorized) {
            $this->results = $this->parseAdgoal($this->results);
        }
274

Dominik Hebeler's avatar
Dominik Hebeler committed
275
276
        # Human Verification
        $this->results = $this->humanVerification($this->results);
277
        $this->ads = $this->humanVerification($this->ads);
Dominik Hebeler's avatar
Dominik Hebeler committed
278

279
        $counter = 0;
280
        $firstRank = 0;
281

282
        if (count($this->results) <= 0) {
283
284
285
286
287
288
            if (strlen($this->site) > 0) {
                $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
                $this->errors[] = trans('metaGer.results.failedSitesearch', ['altSearch' => $no_sitesearch_query]);
            } else {
                $this->errors[] = trans('metaGer.results.failed');
            }
289
        }
290

291
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
292
            $page = $this->page + 1;
293
            $this->next = [
294
295
                'page' => $page,
                'engines' => $this->next,
296
            ];
297
            Cache::put($this->getSearchUid(), serialize($this->next), 60);
298
299
        } else {
            $this->next = [];
300
301
        }

302
    }
303

Phil Höfer's avatar
Phil Höfer committed
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
    public function combineResults($engines)
    {
        foreach ($engines as $engine) {
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
            foreach ($engine->results as $result) {
                if ($result->valid) {
                    $this->results[] = $result;
                }
            }
            foreach ($engine->ads as $ad) {
                $this->ads[] = $ad;
            }
        }
    }

324
325
    public function parseAdgoal($results)
    {
326
        $publicKey = getenv('adgoal_public');
327
        $privateKey = getenv('adgoal_private');
328
        if ($publicKey === false) {
329
330
331
            return $results;
        }
        $tldList = "";
332
333
        try {
            foreach ($results as $result) {
334
335
336
                if (!$result->new) {
                    continue;
                }
337
                $link = $result->anzeigeLink;
338
                if (strpos($link, "http") !== 0) {
339
340
341
342
343
344
345
346
347
348
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

349
            # Query
350
351
            $query = urlencode($this->q);

352
            $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . $publicKey . "&k=" . $hash . "&tld=" . $tldList . "&q=" . $query;
353
354
355
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
356
            foreach ($answer as $el) {
357
                $hoster = $el[0];
358
                $hash = $el[1];
359

360
                foreach ($results as $result) {
361
                    if ($result->new && $hoster === $result->tld && !$result->partnershop) {
362
363
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
364
                        if ($result->image !== "") {
365
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . $hash . ".gif";
366
                        } else {
367
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . $hash . ".gif";
368
369
                        }

370
371
372
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
                        $targetUrl = $result->anzeigeLink;
373
                        if (strpos($targetUrl, "http") !== 0) {
374
                            $targetUrl = "http://" . $targetUrl;
375
376
                        }

377
378
379
                        $gateHash = md5($targetUrl . $privateKey);
                        $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . $publicKey . "&k=" . $gateHash . "&url=" . urlencode($targetUrl) . "&q=" . $query;
                        $result->link = $newLink;
380
381
382
383
                        $result->partnershop = true;
                    }
                }
            }
384
        } catch (\ErrorException $e) {
385
386
387
388
389
            return $results;
        }

        return $results;
    }
390

391
392
    public function humanVerification($results)
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
393
        # Let's check if we need to implement a redirect for human verification
394
395
        if ($this->verificationCount > 10) {
            foreach ($results as $result) {
Dominik Hebeler's avatar
Dominik Hebeler committed
396
397
398
                $link = $result->link;
                $day = Carbon::now()->day;
                $pw = md5($this->verificationId . $day . $link . env("PROXY_PASSWORD"));
399
400
401
                $url = route('humanverification', ['mm' => $this->verificationId, 'pw' => $pw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($link)))]);
                $proxyPw = md5($this->verificationId . $day . $result->proxyLink . env("PROXY_PASSWORD"));
                $proxyUrl = route('humanverification', ['mm' => $this->verificationId, 'pw' => $proxyPw, "url" => urlencode(str_replace("/", "<<SLASH>>", base64_encode($result->proxyLink)))]);
Dominik Hebeler's avatar
Dominik Hebeler committed
402
                $result->link = $url;
403
                $result->proxyLink = $proxyUrl;
Dominik Hebeler's avatar
Dominik Hebeler committed
404
405
            }
            return $results;
406
        } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
407
408
409
410
            return $results;
        }
    }

Phil Höfer's avatar
Phil Höfer committed
411
412
413
414
415
416
    public function authorize($key)
    {
        $postdata = http_build_query(array(
            'dummy' => rand(),
        ));
        $opts = array('http' => array(
417
418
            'method' => 'POST',
            'header' => 'Content-type: application/x-www-form-urlencoded',
Phil Höfer's avatar
Phil Höfer committed
419
420
421
422
423
424
425
            'content' => $postdata,
        ),
        );

        $context = stream_context_create($opts);

        try {
426
            $link = "https://key.metager3.de/" . urlencode($key) . "/request-permission/api-access";
Phil Höfer's avatar
Phil Höfer committed
427
428
429
430
431
432
433
434
435
436
437
438
            $result = json_decode(file_get_contents($link, false, $context));
            if ($result->{'api-access'} == true) {
                return true;
            } else {
                return false;
            }

        } catch (\ErrorException $e) {
            return false;
        }
    }

439
440
    public function createQuicktips()
    {
441
        # Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
442
        $quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime());
443
444
445
        return $quicktips;
    }

Karl's avatar
Karl committed
446
447
448
449
    /*
     * Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
     */

450
451
    public function createSearchEngines(Request $request)
    {
452
        # Wenn es kein Suchwort gibt
Dominik Hebeler's avatar
Dominik Hebeler committed
453
        if (!$request->filled("eingabe") || $this->q === "") {
454
            return;
455
        }
456

Dominik Hebeler's avatar
Dominik Hebeler committed
457
        $this->enabledSearchengines = [];
458
        $overtureEnabled = false;
459

460
461
462
        # Check if selected focus is valid
        if (empty($this->sumaFile->foki->{$this->fokus})) {
            $this->fokus = "web";
463
        }
464
465
        foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
            # Check if this engine is disabled and can't be used
466
            $disabled = empty($this->sumaFile->sumas->{$suma}->disabled) ? false : $this->sumaFile->sumas->{$suma}->disabled;
467
468
            $autoDisabled = empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) ? false : $this->sumaFile->sumas->{$suma}->{"auto-disabled"};
            if ($disabled || $autoDisabled) {
469
                continue;
470
471
            }

472
473
474
            # Check if this engine can use eventually defined query-filter
            $valid = true;
            foreach ($this->queryFilter as $queryFilter => $filter) {
Dominik Hebeler's avatar
Dominik Hebeler committed
475
                if (empty($this->sumaFile->filter->{"query-filter"}->$queryFilter->sumas->$suma)) {
476
477
                    $valid = false;
                    break;
Dominik Hebeler's avatar
Dominik Hebeler committed
478
479
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
480
481
482
            # Check if this engine can use eventually defined parameter-filter
            if ($valid) {
                foreach ($this->parameterFilter as $filterName => $filter) {
483
484
485
                    # We need to check if the searchengine supports the parameter value, too
                    $value = $request->input($filter->{"get-parameter"}, "");
                    if (empty($filter->sumas->$suma) || empty($filter->sumas->{$suma}->values->{$value})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
486
487
488
489
490
                        $valid = false;
                        break;
                    }
                }
            }
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
            # Check if this engine should only be active when filter is used
            if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"}) {
                # This search engine should only be used when a parameter filter of it is used
                $validTmp = false;
                foreach ($this->parameterFilter as $filterName => $filter) {
                    if (!empty($filter->sumas->{$suma})) {
                        $validTmp = true;
                        break;
                    }
                }
                if (!$validTmp) {
                    $valid = false;
                }

            }
506
507
            # If it can we add it
            if ($valid) {
Dominik Hebeler's avatar
Dominik Hebeler committed
508
                $this->enabledSearchengines[$suma] = $this->sumaFile->sumas->{$suma};
509
            }
510

511
        }
512

513
514
515
516
517
        # Implements Yahoo Ads if Yahoo is not enabled as a searchengine
        if (empty($this->enabledSearchengines["yahoo"]) && $this->fokus != "bilder" && !empty($this->sumaFile->sumas->{"yahoo-ads"})) {
            $this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
518
        if (sizeof($this->enabledSearchengines) === 0) {
519
520
            $filter = "";
            foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
Dominik Hebeler's avatar
Dominik Hebeler committed
521
                $filter .= trans($this->sumaFile->filter->{"query-filter"}->{$queryFilter}->name) . ",";
522
523
524
525
526
            }
            $filter = rtrim($filter, ",");
            $error = trans('metaGer.engines.noSpecialSearch', ['fokus' => trans($this->sumaFile->foki->{$this->fokus}->{"display-name"}),
                'filter' => $filter]);
            $this->errors[] = $error;
527
        }
528
        $engines = [];
529
        $typeslist = [];
530
        $counter = 0;
531
532
        $this->setEngines($request);
    }
533

534
535
    public function setEngines(Request $request, $enabledSearchengines = [])
    {
536
        if ($this->requestIsCached($request)) {
537
            # If this is a page other than 1 the request is "cached"
538
            $engines = $this->getCachedEngines($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
539
            # We need to edit some Options of the Cached Search Engines
Phil Höfer's avatar
Phil Höfer committed
540
            foreach ($engines as $engine) {
541
                $engine->setResultHash($this->getSearchUid());
Dominik Hebeler's avatar
Dominik Hebeler committed
542
            }
543
            $this->engines = $engines;
544
        } else {
545
546
547
            if (sizeof($enabledSearchengines) > 0) {
                $this->enabledSearchengines = $enabledSearchengines;
            }
548
            $this->actuallyCreateSearchEngines($this->enabledSearchengines);
549
        }
550
    }
Dominik Hebeler's avatar
Dominik Hebeler committed
551

552
553
    public function startSearch()
    {
554
        # Wir starten alle Suchen
555
        foreach ($this->engines as $engine) {
556
            $engine->startSearch($this);
557
        }
558
559
    }

560
561
    # Spezielle Suchen und Sumas

562
    public function sumaIsSelected($suma, $request, $custom)
563
    {
564
        if ($custom) {
565
            if ($request->filled("engine_" . strtolower($suma["name"]))) {
566
567
568
569
570
571
572
573
574
575
576
                return true;
            }
        } else {
            $types = explode(",", $suma["type"]);
            if (in_array($this->fokus, $types)) {
                return true;
            }
        }
        return false;
    }

577
    public function actuallyCreateSearchEngines($enabledSearchengines)
578
579
    {
        $engines = [];
580
        foreach ($enabledSearchengines as $engineName => $engine) {
581

582
            if (!isset($engine->{"parser-class"})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
583
584
                die(var_dump($engine));
            }
585
            # Setze Pfad zu Parser
586
            $path = "App\\Models\\parserSkripte\\" . $engine->{"parser-class"};
587
588

            # Prüfe ob Parser vorhanden
589
590
591
            if (!file_exists(app_path() . "/Models/parserSkripte/" . $engine->{"parser-class"} . ".php")) {
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen, da kein Parser existiert");
                $this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine->{"display-name"}]);
592
593
594
595
596
597
                continue;
            }

            # Es wird versucht die Suchengine zu erstellen
            $time = microtime();
            try {
598
                $tmp = new $path($engineName, $engine, $this);
599
            } catch (\ErrorException $e) {
600
                Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen. " . var_dump($e));
601
602
603
                continue;
            }

604
            $engines[] = $tmp;
605
        }
606
        $this->engines = $engines;
607
608
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
609
610
611
612
613
614
615
616
617
618
619
620
621
    public function getAvailableParameterFilter()
    {
        $parameterFilter = $this->sumaFile->filter->{"parameter-filter"};

        $availableFilter = [];

        foreach ($parameterFilter as $filterName => $filter) {
            # Check if any of the enabled search engines provide this filter
            foreach ($this->enabledSearchengines as $engineName => $engine) {
                if (!empty($filter->sumas->$engineName)) {
                    $availableFilter[$filterName] = $filter;
                }
            }
622
623
624
625
            # We will also add the filter from the opt-in search engines (the searchengines that are only used when a filter of it is too)
            foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
                if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"}) {
                    if (!empty($filter->sumas->{$suma})) {
626
627
628
629
630
                        # If the searchengine is disabled this filter shouldn't be available
                        if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
                            || (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)) {
                            continue;
                        }
631
632
633
634
                        $availableFilter[$filterName] = $filter;
                    }
                }
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
635
636
637
638
639
        }

        return $availableFilter;
    }

640
641
    public function isBildersuche()
    {
642
        return $this->fokus === "bilder";
643
644
645
646
    }

    public function sumaIsAdsuche($suma, $overtureEnabled)
    {
647
        $sumaName = $suma["name"]->__toString();
648
        return
649
650
            $sumaName === "qualigo"
            || $sumaName === "similar_product_ads"
Karl Hasselbring's avatar
Karl Hasselbring committed
651
            || (!$overtureEnabled && $sumaName === "overtureAds");
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
    }

    public function sumaIsDisabled($suma)
    {
        return
        isset($suma['disabled'])
        && $suma['disabled']->__toString() === "1";
    }

    public function sumaIsOverture($suma)
    {
        return
        $suma["name"]->__toString() === "overture"
        || $suma["name"]->__toString() === "overtureAds";
    }

    public function sumaIsNotAdsuche($suma)
    {
        return
        $suma["name"]->__toString() !== "qualigo"
        && $suma["name"]->__toString() !== "similar_product_ads"
        && $suma["name"]->__toString() !== "overtureAds";
    }

    public function requestIsCached($request)
    {
        return
Dominik Hebeler's avatar
Dominik Hebeler committed
679
        $request->filled('next')
680
681
682
683
684
685
        && Cache::has($request->input('next'))
        && unserialize(Cache::get($request->input('next')))['page'] > 1;
    }

    public function getCachedEngines($request)
    {
686
        $next = unserialize(Cache::get($request->input('next')));
687
        $this->page = $next['page'];
688
        $engines = $next['engines'];
689
690
691
        return $engines;
    }

692
    public function waitForMainResults()
693
    {
694
        $redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
695
        $engines = $this->engines;
696
        $enginesToWaitFor = [];
697
698
699
700
701
702
        $mainEngines = $this->sumaFile->foki->{$this->fokus}->main;
        foreach ($mainEngines as $mainEngine) {
            foreach ($engines as $engine) {
                if (!$engine->cached && $engine->name === $mainEngine) {
                    $enginesToWaitFor[] = $engine;
                }
703
704
            }
        }
705

Phil Höfer's avatar
Phil Höfer committed
706
        $timeStart = microtime(true);
707
        $answered = [];
708
        $results = null;
709
        while (sizeof($enginesToWaitFor) > 0) {
710
            $newEngine = $redis->blpop($this->redisResultWaitingKey, 5);
711
712
713
714
715
716
717
718
719
            if ($newEngine === null || sizeof($newEngine) !== 2) {
                continue;
            } else {
                $newEngine = $newEngine[1];
                foreach ($enginesToWaitFor as $index => $engine) {
                    if ($engine->name === $newEngine) {
                        unset($enginesToWaitFor[$index]);
                        break;
                    }
720
                }
721
                $answered[] = $newEngine;
722
            }
723
            if ((microtime(true) - $timeStart) >= 2) {
724
725
                break;
            }
726
        }
727
728

        # Now we can add an entry to Redis which defines the starting time and how many engines should answer this request
729

730
731
732
733
734
735
736
737
738
        $pipeline = $redis->pipeline();
        $pipeline->hset($this->getRedisEngineResult() . "status", "startTime", $timeStart);
        $pipeline->hset($this->getRedisEngineResult() . "status", "engineCount", sizeof($engines));
        $pipeline->hset($this->getRedisEngineResult() . "status", "engineDelivered", sizeof($answered));
        foreach ($answered as $engine) {
            $pipeline->hset($this->getRedisEngineResult() . $engine, "delivered", "1");
        }
        $pipeline->execute();

739
    }
740

741
    public function retrieveResults()
742
    {
743
        $engines = $this->engines;
744
        # Von geladenen Engines die Ergebnisse holen
745
746
747
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
                try {
748
                    $engine->retrieveResults($this);
749
                } catch (\ErrorException $e) {
750
751
752
                    Log::error($e);
                }
            }
753
754
755
            if (!empty($engine->totalResults) && $engine->totalResults > $this->totalResults) {
                $this->totalResults = $engine->totalResults;
            }
756
        }
757
758
    }

759
760
761
/*
 * Ende Suchmaschinenerstellung und Ergebniserhalt
 */
Karl's avatar
Karl committed
762

763
764
    public function parseFormData(Request $request)
    {
765

766
        # Sichert, dass der request in UTF-8 formatiert ist
767
        if ($request->input('encoding', 'utf8') !== "utf8") {
768
769
770
771
772
773
774
            # In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
            $input = $request->all();
            foreach ($input as $key => $value) {
                $input[$key] = mb_convert_encoding("$value", "UTF-8", "ISO-8859-1");
            }
            $request->replace($input);
        }
775
        $this->url = $request->url();
776
        $this->fullUrl = $request->fullUrl();
777
        # Zunächst überprüfen wir die eingegebenen Einstellungen:
778
        # Fokus
779
        $this->fokus = $request->input('focus', 'web');
780
        # Suma-File
781
        if (App::isLocale("en")) {
782
            $this->sumaFile = config_path() . "/sumasEn.json";
783
        } else {
784
            $this->sumaFile = config_path() . "/sumas.json";
785
        }
786
        if (!file_exists($this->sumaFile)) {
787
            die(trans('metaGer.formdata.cantLoad'));
788
789
        } else {
            $this->sumaFile = json_decode(file_get_contents($this->sumaFile));
790
        }
791
        # Sucheingabe
792
        $this->eingabe = trim($request->input('eingabe', ''));
793
        $this->q = $this->eingabe;
794
        # IP
795
        $this->ip = $request->ip();
796
797
798
799
        # Unser erster Schritt wird sein, IP-Adresse und USER-Agent zu anonymisieren, damit
        # nicht einmal wir selbst noch Zugriff auf die Daten haben:
        $this->ip = preg_replace("/(\d+)\.(\d+)\.\d+.\d+/s", "$1.$2.0.0", $this->ip);

Dominik Hebeler's avatar
Dominik Hebeler committed
800
801
        $this->useragent = $request->header('User-Agent');

802
        # Language
803
        if (isset($_SERVER['HTTP_LANGUAGE'])) {
804
            $this->language = $_SERVER['HTTP_LANGUAGE'];
805
        } else {
806
807
            $this->language = "";
        }
808

809
        # Page
810
        $this->page = 1;
811
812
        # Lang
        $this->lang = $request->input('lang', 'all');
813
814
        if ($this->lang !== "de" && $this->lang !== "en" && $this->lang !== "all") {
            $this->lang = "all";
815
        }
816

817
        $this->agent = new Agent();
818
        $this->mobile = $this->agent->isMobile();
819
        # Sprüche
820
821
        $this->sprueche = $request->input('sprueche', 'on');
        if ($this->sprueche === "on") {
822
            $this->sprueche = true;
823
        } else {
824
            $this->sprueche = false;
825
        }
826

827
        $this->newtab = $request->input('newtab', 'on');
828
829
830
831
832
        if ($this->newtab === "on") {
            $this->newtab = "_blank";
        } else {
            $this->newtab = "_self";
        }
833
        if ($request->filled("key") && $request->input('key') === getenv("mainz_key")) {
834
            $this->newtab = "_blank";
835
        }
836
        # Theme
Phil Höfer's avatar
Phil Höfer committed
837
        $this->theme = preg_replace("/[^[:alnum:][:space:]]/u", '', $request->input('theme', 'default'));
838
839
        # Ergebnisse pro Seite:
        $this->resultCount = $request->input('resultCount', '20');
840

Dominik Hebeler's avatar
Dominik Hebeler committed
841
        if ($request->filled('minism') && ($request->filled('fportal') || $request->filled('harvest'))) {
842
            $input = $request->all();
843
844
845
846
847
848
849
            $newInput = [];
            foreach ($input as $key => $value) {
                if ($key !== "fportal" && $key !== "harvest") {
                    $newInput[$key] = $value;
                }
            }
            $request->replace($newInput);
850
        }
851
852
        if (App::isLocale("en")) {
            $this->sprueche = "off";
853
        }
854
855
        if ($this->resultCount <= 0 || $this->resultCount > 200) {
            $this->resultCount = 1000;
856
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
857
        if ($request->filled('onenewspageAll') || $request->filled('onenewspageGermanyAll')) {
858
            $this->time = 5000;
859