MetaGer.php 38.1 KB
Newer Older
1
2
3
4
<?php
namespace App;

use App;
5
use App\lib\TextLanguageDetect\TextLanguageDetect;
6
use Cache;
7
8
use Illuminate\Http\Request;
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix    
Dominik Hebeler committed
9
use LaravelLocalization;
10
11
use Log;
use Redis;
12
13
14

class MetaGer
{
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
    # Einstellungen für die Suche
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $category;
    protected $time;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
    protected $hostBlacklist   = [];
    protected $domainBlacklist = [];
    protected $stopWords       = [];
    protected $phrases         = [];
    protected $engines         = [];
    protected $results         = [];
    protected $ads             = [];
    protected $warnings        = [];
    protected $errors          = [];
    protected $addedHosts      = [];
35
    protected $startCount      = 0;
36
37
38
39
40
41
42
43
44
    # Daten über die Abfrage
    protected $ip;
    protected $language;
    protected $agent;
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
45
    protected $domainsBlacklisted = [];
46
    protected $urlsBlacklisted    = [];
47
48
49
    protected $url;
    protected $languageDetect;

50
51
52
53
    public function __construct()
    {
        $this->starttime = microtime(true);
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
54
            # Blacklists einlesen:
55
            $tmp                      = file_get_contents(config_path() . "/blacklistDomains.txt");
56
            $this->domainsBlacklisted = explode("\n", $tmp);
57
58
59
            $tmp                      = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted    = explode("\n", $tmp);
        } else {
60
61
62
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
        }

63
64
65
66
67
68
69
70
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
                require $path;
            }
        }

71
72
        $this->languageDetect = new TextLanguageDetect();
        $this->languageDetect->setNameMode("2");
73
    }
74

75
    public function getHashCode()
76
77
78
79
80
    {
        $string = url()->full();
        return md5($string);
    }

81
    public function rankAll()
82
    {
83
        foreach ($this->engines as $engine) {
84
85
86
            $engine->rank($this);
        }
    }
87

88
89
90
    public function createView()
    {
        $viewResults = [];
91

92
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
93
        foreach ($this->results as $result) {
94
95
96
97
98
99
            $viewResults[] = get_object_vars($result);
        }

        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();

100
101
        if ($this->fokus === "bilder") {
            switch ($this->out) {
102
103
104
105
106
107
108
109
                case 'results':
                    return view('metager3bilderresults')
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
110
                        ->with('browser', (new Agent())->browser());
111
112
113
114
115
116
117
118
                default:
                    return view('metager3bilder')
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
119
                        ->with('browser', (new Agent())->browser());
120
121
122
123
124
125
126
127
128
129
130
131
            }
        }

        switch ($this->out) {
            case 'results':
                return view('metager3results')
                    ->with('results', $viewResults)
                    ->with('eingabe', $this->eingabe)
                    ->with('mobile', $this->mobile)
                    ->with('warnings', $this->warnings)
                    ->with('errors', $this->errors)
                    ->with('metager', $this)
132
                    ->with('browser', (new Agent())->browser());
133
134
135
136
137
138
139
140
141
142
                break;
            case 'results-with-style':
                return view('metager3')
                    ->with('results', $viewResults)
                    ->with('eingabe', $this->eingabe)
                    ->with('mobile', $this->mobile)
                    ->with('warnings', $this->warnings)
                    ->with('errors', $this->errors)
                    ->with('metager', $this)
                    ->with('suspendheader', "yes")
143
                    ->with('browser', (new Agent())->browser());
144
145
146
147
148
149
150
151
                break;
            default:
                return view('metager3')
                    ->with('eingabe', $this->eingabe)
                    ->with('mobile', $this->mobile)
                    ->with('warnings', $this->warnings)
                    ->with('errors', $this->errors)
                    ->with('metager', $this)
152
                    ->with('browser', (new Agent())->browser());
153
154
                break;
        }
155
    }
156

157
158
159
160
161
162
    private function createLogs()
    {
        $redis = Redis::connection('redisLogs');
        try
        {
            $logEntry = "";
163
            $logEntry .= "[" . date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y"))) . "]";
164
165
166
167
168
169
            $logEntry .= " pid=" . getmypid();
            $logEntry .= " ref=" . $this->request->header('Referer');
            $useragent = $this->request->header('User-Agent');
            $useragent = str_replace("(", " ", $useragent);
            $useragent = str_replace(")", " ", $useragent);
            $useragent = str_replace(" ", "", $useragent);
170
            $logEntry .= " time=" . round((microtime(true) - $this->starttime), 2) . " serv=" . $this->fokus;
171
172
            $logEntry .= " search=" . $this->eingabe;
            $redis->rpush('logs.search', $logEntry);
173
        } catch (\Exception $e) {
174
175
176
177
            return;
        }
    }

178
    public function removeInvalids()
179
180
    {
        $results = [];
181
182
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
183
                $results[] = $result;
184
185
            }

186
187
188
        }
        #$this->results = $results;
    }
189

190
191
192
    public function combineResults()
    {
        foreach ($this->engines as $engine) {
193
194
195
196
197
198
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
199
200
            foreach ($engine->results as $result) {
                if ($result->valid) {
201
                    $this->results[] = $result;
202
                }
203
            }
204
            foreach ($engine->ads as $ad) {
205
206
                $this->ads[] = $ad;
            }
207
        }
208

209
210
        uasort($this->results, function ($a, $b) {
            if ($a->getRank() == $b->getRank()) {
211
                return 0;
212
213
            }

214
215
216
217
            return ($a->getRank() < $b->getRank()) ? 1 : -1;
        });
        # Validate Results
        $newResults = [];
218
219
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
220
                $newResults[] = $result;
221
222
            }

223
224
225
        }
        $this->results = $newResults;

226
227
228
229
230
231
        # Boost implementation
        $this->results = $this->parseBoost($this->results);

        #Adgoal Implementation
        $this->results = $this->parseAdgoal($this->results);

232
        $counter   = 0;
233
        $firstRank = 0;
234

235
        if (isset($this->startForwards)) {
236
            $this->startCount = $this->startForwards;
237
        } elseif (isset($this->startBackwards)) {
238
            $this->startCount = $this->startBackwards - count($this->results) - 1;
239
        } else {
240
241
242
            $this->startCount = 0;
        }

243
244
        foreach ($this->results as $result) {
            if ($counter === 0) {
245
                $firstRank = $result->rank;
246
247
            }

248
            $counter++;
249
            $result->number = $counter + $this->startCount;
250
251
252
253
            $confidence     = 0;
            if ($firstRank > 0) {
                $confidence = $result->rank / $firstRank;
            } else {
254
                $confidence = 0;
255
256
257
            }

            if ($confidence > 0.65) {
258
                $result->color = "#FF4000";
259
            } elseif ($confidence > 0.4) {
260
                $result->color = "#FF0080";
261
            } elseif ($confidence > 0.2) {
262
                $result->color = "#C000C0";
263
            } else {
264
                $result->color = "#000000";
265
266
            }

267
268
        }

269
        if (LaravelLocalization::getCurrentLocale() === "en") {
270
271
272
            $this->ads = [];
        }

273
        $this->validated = false;
274
        if (isset($this->password)) {
275
276
277
            # Wir bieten einen bezahlten API-Zugriff an, bei dem dementsprechend die Werbung ausgeblendet wurde:
            # Aktuell ist es nur die Uni-Mainz. Deshalb überprüfen wir auch nur diese.
            $password = getenv('mainz');
278
            $eingabe  = $this->eingabe;
279
            $password = md5($eingabe . $password);
280
281
            if ($this->password === $password) {
                $this->ads       = [];
282
283
284
                $this->validated = true;
            }
        }
285
286

        if (count($this->results) <= 0) {
Dominik Hebeler's avatar
Dominik Hebeler committed
287
            $this->errors[] = "Leider konnten wir zu Ihrer Sucheingabe keine passenden Ergebnisse finden.";
288
        }
289

290
291
        if (isset($this->last) && count($this->last) > 0) {
            $page       = $this->page - 1;
292
            $this->last = [
293
294
295
                'page'           => $page,
                'startBackwards' => $this->results[0]->number,
                'engines'        => $this->last,
296
297
298
299
            ];
            Cache::put(md5(serialize($this->last)), serialize($this->last), 60);
        }

300
301
        if (isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
            $page       = $this->page + 1;
302
            $this->next = [
303
304
305
                'page'          => $page,
                'startForwards' => $this->results[count($this->results) - 1]->number,
                'engines'       => $this->next,
306
307
308
309
            ];
            Cache::put(md5(serialize($this->next)), serialize($this->next), 60);
        }

310
    }
311

312
313
    public function parseBoost($results)
    {
314
315
316
317
318
319
320
321
322
323
324
325
        foreach ($results as $result) {
            if (preg_match('/^(http[s]?\:\/\/)?(www.)?amazon\.de/', $result->anzeigeLink)) {
                if (preg_match('/\?/', $result->anzeigeLink)) {
                    $result->link .= '&tag=boostmg01-21';
                } else {
                    $result->link .= '?tag=boostmg01-21';
                }
                $result->partnershop = true;

            }
        }
        return $results;
326
327
328
    }
    public function parseAdgoal($results)
    {
329
        $publicKey  = getenv('adgoal_public');
330
        $privateKey = getenv('adgoal_private');
331
        if ($publicKey === false) {
332
333
334
            return $results;
        }
        $tldList = "";
335
336
        try {
            foreach ($results as $result) {
337
                $link = $result->anzeigeLink;
338
                if (strpos($link, "http") !== 0) {
339
340
341
342
343
344
345
346
347
348
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

349
            # Query
350
351
            $query = urlencode($this->q);

352
            $link   = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . $publicKey . "&k=" . $hash . "&tld=" . $tldList . "&q=" . $query;
353
354
355
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
356
            foreach ($answer as $el) {
357
                $hoster = $el[0];
358
                $hash   = $el[1];
359

360
361
                foreach ($results as $result) {
                    if ($hoster === $result->tld) {
362
363
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
364
                        if ($result->image !== "") {
365
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . $hash . ".gif";
366
                        } else {
367
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . $hash . ".gif";
368
369
                        }

370
371
372
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
                        $targetUrl = $result->anzeigeLink;
373
                        if (strpos($targetUrl, "http") !== 0) {
374
                            $targetUrl = "http://" . $targetUrl;
375
376
377
378
379
                        }

                        $gateHash            = md5($targetUrl . $privateKey);
                        $newLink             = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . $publicKey . "&k=" . $gateHash . "&url=" . urlencode($targetUrl) . "&q=" . $query;
                        $result->link        = $newLink;
380
381
382
383
                        $result->partnershop = true;
                    }
                }
            }
384
        } catch (\ErrorException $e) {
385
386
387
388
389
            return $results;
        }

        return $results;
    }
390

391
392
    public function createSearchEngines(Request $request)
    {
393

394
        if (!$request->has("eingabe")) {
395
            return;
396
        }
397

398
399
        # Überprüfe, welche Sumas eingeschaltet sind
        $xml                  = simplexml_load_file($this->sumaFile);
400
        $enabledSearchengines = [];
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
        $overtureEnabled      = false;
        $countSumas           = 0;
        $sumas                = $xml->xpath("suma");
        if ($this->fokus === "angepasst") {
            foreach ($sumas as $suma) {
                if ($request->has($suma["name"])
                    || ($this->fokus !== "bilder"
                        && ($suma["name"]->__toString() === "qualigo"
                            || $suma["name"]->__toString() === "similar_product_ads"
                            || (!$overtureEnabled && $suma["name"]->__toString() === "overtureAds")
                        )
                    )
                ) {

                    if (!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1")) {
                        if ($suma["name"]->__toString() === "overture" || $suma["name"]->__toString() === "overtureAds") {
                            $overtureEnabled = true;
418
                        }
419
                        if ($suma["name"]->__toString() !== "qualigo" && $suma["name"]->__toString() !== "similar_product_ads" && $suma["name"]->__toString() !== "overtureAds") {
420
                            $countSumas += 1;
421
422
                        }

423
424
425
426
                        $enabledSearchengines[] = $suma;
                    }
                }
            }
427
428
429
430
431
432
433
434
435
436
437
438
439
440
        } else {
            foreach ($sumas as $suma) {
                $types = explode(",", $suma["type"]);
                if (in_array($this->fokus, $types)
                    || ($this->fokus !== "bilder"
                        && ($suma["name"]->__toString() === "qualigo"
                            || $suma["name"]->__toString() === "similar_product_ads"
                            || (!$overtureEnabled && $suma["name"]->__toString() === "overtureAds")
                        )
                    )
                ) {
                    if (!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1")) {
                        if ($suma["name"]->__toString() === "overture" || $suma["name"]->__toString() === "overtureAds") {
                            $overtureEnabled = true;
441
                        }
442
                        if ($suma["name"]->__toString() !== "qualigo" && $suma["name"]->__toString() !== "similar_product_ads" && $suma["name"]->__toString() !== "overtureAds") {
443
                            $countSumas += 1;
444
445
                        }

446
447
448
449
450
451
452
453
                        $enabledSearchengines[] = $suma;
                    }
                }
            }
        }

        # Sonderregelung für alle Suchmaschinen, die zu den Minisuchern gehören. Diese können alle gemeinsam über einen Link abgefragt werden
        $subcollections = [];
454
455
456
        $tmp            = [];
        foreach ($enabledSearchengines as $engine) {
            if (isset($engine['minismCollection'])) {
457
                $subcollections[] = $engine['minismCollection']->__toString();
458
            } else {
459
                $tmp[] = $engine;
460
461
            }

462
463
        }
        $enabledSearchengines = $tmp;
464
465
466
467
        if (sizeof($subcollections) > 0) {
            $count                        = sizeof($subcollections) * 10;
            $minisucherEngine             = $xml->xpath('suma[@name="minism"]')[0];
            $subcollections               = urlencode("(" . implode(" OR ", $subcollections) . ")");
468
469
            $minisucherEngine["formData"] = str_replace("<<SUBCOLLECTIONS>>", $subcollections, $minisucherEngine["formData"]);
            $minisucherEngine["formData"] = str_replace("<<COUNT>>", $count, $minisucherEngine["formData"]);
470
            $enabledSearchengines[]       = $minisucherEngine;
471
472
473
474
        }

        #die(var_dump($enabledSearchengines));

475
        if ($countSumas <= 0) {
476
477
            $this->errors[] = "Achtung: Sie haben in ihren Einstellungen keine Suchmaschine ausgewählt.";
        }
478
        $engines = [];
479

480
        $siteSearchFailed = false;
481
        if (strlen($this->site) > 0) {
482
483
            # Wenn eine Sitesearch durchgeführt werden soll, überprüfen wir ob eine der Suchmaschinen überhaupt eine Sitesearch unterstützt:
            $enginesWithSite = 0;
484
485
            foreach ($enabledSearchengines as $engine) {
                if (isset($engine['hasSiteSearch']) && $engine['hasSiteSearch']->__toString() === "1") {
486
487
488
                    $enginesWithSite++;
                }
            }
489
490
            if ($enginesWithSite === 0) {
                $this->errors[]   = "Sie wollten eine Sitesearch auf " . $this->site . " durchführen. Leider unterstützen die eingestellten Suchmaschinen diese nicht. Sie können <a href=\"" . $this->generateSearchLink("web", false) . "\">hier</a> die Sitesearch im Web-Fokus durchführen. Es werden ihnen Ergebnisse ohne Sitesearch angezeigt.";
491
                $siteSearchFailed = true;
492
            } else {
493
494
495
496
497
498
                $this->warnings[] = "Sie führen eine Sitesearch durch. Es werden nur Ergebnisse von der Seite: <a href=\"http://" . $this->site . "\" target=\"_blank\">\"" . $this->site . "\"</a> angezeigt.";
            }

        }

        $typeslist = [];
499
        $counter   = 0;
500

501
502
        if ($request->has('next') && Cache::has($request->input('next')) && unserialize(Cache::get($request->input('next')))['page'] > 1) {
            $next       = unserialize(Cache::get($request->input('next')));
503
            $this->page = $next['page'];
504
505
            $engines    = $next['engines'];
            if (isset($next['startForwards'])) {
506
                $this->startForwards = $next['startForwards'];
507
508
509
            }

            if (isset($next['startBackwards'])) {
510
                $this->startBackwards = $next['startBackwards'];
511
512
            }

513
514
        } else {
            foreach ($enabledSearchengines as $engine) {
515

516
                if (!$siteSearchFailed && strlen($this->site) > 0 && (!isset($engine['hasSiteSearch']) || $engine['hasSiteSearch']->__toString() === "0")) {
517

518
519
520
521
                    continue;
                }
                # Wenn diese Suchmaschine gar nicht eingeschaltet sein soll
                $path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString());
522

523
524
525
526
                if (!file_exists(app_path() . "/Models/parserSkripte/" . ucfirst($engine["package"]->__toString()) . ".php")) {
                    Log::error("Konnte " . $engine["name"] . " nicht abfragen, da kein Parser existiert");
                    continue;
                }
527

528
                $time = microtime();
529

530
531
532
533
534
535
536
                try
                {
                    $tmp = new $path($engine, $this);
                } catch (\ErrorException $e) {
                    Log::error("Konnte " . $engine["name"] . " nicht abfragen." . var_dump($e));
                    continue;
                }
537

538
539
540
541
542
543
544
                if ($tmp->enabled && isset($this->debug)) {
                    $this->warnings[] = $tmp->service . "   Connection_Time: " . $tmp->connection_time . "    Write_Time: " . $tmp->write_time . " Insgesamt:" . ((microtime() - $time) / 1000);
                }

                if ($tmp->isEnabled()) {
                    $engines[] = $tmp;
                }
545
546

            }
547
        }
548

549
550
551
        # Wir starten die Suche manuell:
        foreach ($engines as $engine) {
            $engine->startSearch($this);
552
        }
553

554
555
        # Jetzt werden noch alle Kategorien der Settings durchgegangen und die jeweils enthaltenen namen der Suchmaschinen gespeichert.
        $foki = [];
556
557
558
        foreach ($sumas as $suma) {
            if ((!isset($suma['disabled']) || $suma['disabled'] === "") && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
                if (isset($suma['type'])) {
559
                    $f = explode(",", $suma['type']->__toString());
560
561
                    foreach ($f as $tmp) {
                        $name                                    = $suma['name']->__toString();
562
563
                        $foki[$tmp][$suma['name']->__toString()] = $name;
                    }
564
565
                } else {
                    $name                                        = $suma['name']->__toString();
566
567
568
569
570
571
572
                    $foki["andere"][$suma['name']->__toString()] = $name;
                }
            }
        }

        # Es werden auch die Namen der aktuell aktiven Suchmaschinen abgespeichert.
        $realEngNames = [];
573
        foreach ($enabledSearchengines as $realEng) {
574
            $nam = $realEng["name"]->__toString();
575
            if ($nam !== "qualigo" && $nam !== "overtureAds") {
576
577
578
579
                $realEngNames[] = $nam;
            }
        }
        # Anschließend werden diese beiden Listen verglichen (jeweils eine der Fokuslisten für jeden Fokus), um herauszufinden ob sie vielleicht identisch sind. Ist dies der Fall, so hat der Nutzer anscheinend Suchmaschinen eines kompletten Fokus eingestellt. Der Fokus wird dementsprechend angepasst.
580
581
        foreach ($foki as $fok => $engs) {
            $isFokus      = true;
582
            $fokiEngNames = [];
583
            foreach ($engs as $eng) {
584
585
                $fokiEngNames[] = $eng;
            }
586
587
            foreach ($fokiEngNames as $fen) {
                if (!in_array($fen, $realEngNames)) {
588
589
590
                    $isFokus = false;
                }
            }
591
592
            foreach ($realEngNames as $ren) {
                if (!in_array($ren, $fokiEngNames)) {
593
594
595
                    $isFokus = false;
                }
            }
596
            if ($isFokus) {
597
598
599
600
601
602
603
604
605
                $this->fokus = $fok;
            }
        }

        # Nun passiert ein elementarer Schritt.
        # Wir warten auf die Antwort der Suchmaschinen, da wir vorher nicht weiter machen können.
        # aber natürlich nicht ewig.
        # Die Verbindung steht zu diesem Zeitpunkt und auch unsere Request wurde schon gesendet.
        # Wir geben der Suchmaschine nun bis zu 500ms Zeit zu antworten.
606
607
608

        # Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
        $enginesToLoad = 0;
609
610
611
        $canBreak      = false;
        foreach ($engines as $engine) {
            if ($engine->cached) {
612
                $enginesToLoad--;
613
                if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
614
                    $canBreak = true;
615
616
                }

617
618
619
            }
        }
        $enginesToLoad += count($engines);
620
        $loadedEngines = 0;
621
622
623
        $timeStart     = microtime(true);
        while (true) {
            $time          = (microtime(true) - $timeStart) * 1000;
624
            $loadedEngines = intval(Redis::hlen('search.' . $this->getHashCode()));
625
            if ($overtureEnabled && (Redis::hexists('search.' . $this->getHashCode(), 'overture') || Redis::hexists('search.' . $this->getHashCode(), 'overtureAds'))) {
626
                $canBreak = true;
627
            }
628
629

            # Abbruchbedingung
630
631
            if ($time < 500) {
                if (($enginesToLoad === 0 || $loadedEngines >= $enginesToLoad) && $canBreak) {
632
                    break;
633
634
635
636
                }

            } elseif ($time >= 500 && $time < $this->time) {
                if (($enginesToLoad === 0 || ($loadedEngines / ($enginesToLoad * 1.0)) >= 0.8) && $canBreak) {
637
                    break;
638
639
640
                }

            } else {
641
642
643
644
645
                break;
            }
            usleep(50000);
        }

646
647
648
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
                try {
649
                    $engine->retrieveResults($this);
650
                } catch (\ErrorException $e) {
651
                    Log::error($e);
652

653
654
655
                }
            }
        }
656

657
        # und verwerfen den Rest:
658
659
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
660
                $engine->shutdown();
661
662
            }

663
664
665
        }

        $this->engines = $engines;
666
667
668
669
670
671
672
673
674
675
676
677
    }

    public function parseFormData(Request $request)
    {
        if ($request->input('encoding', '') !== "utf8") {
            # In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
            $input = $request->all();
            foreach ($input as $key => $value) {
                $input[$key] = mb_convert_encoding("$value", "UTF-8", "ISO-8859-1");
            }
            $request->replace($input);
        }
678
        $this->url = $request->url();
679
        # Zunächst überprüfen wir die eingegebenen Einstellungen:
680
681
        # FOKUS
        $this->fokus = trans('fokiNames.'
682
683
            . $request->input('focus', 'web'));
        if (strpos($this->fokus, ".")) {
684
685
686
687
            $this->fokus = trans('fokiNames.web');
        }

        # SUMA-FILE
688
        if (App::isLocale("en")) {
689
            $this->sumaFile = config_path() . "/sumas.xml";
690
        } else {
691
692
            $this->sumaFile = config_path() . "/sumas.xml";
        }
693
        if (!file_exists($this->sumaFile)) {
694
695
696
697
698
            die("Suma-File konnte nicht gefunden werden");
        }

        # Sucheingabe:
        $this->eingabe = trim($request->input('eingabe', ''));
699
        if (strlen($this->eingabe) === 0) {
700
701
702
703
704
705
706
707
            $this->warnings[] = 'Achtung: Sie haben keinen Suchbegriff eingegeben. Sie können ihre Suchbegriffe oben eingeben und es erneut versuchen.';
        }
        $this->q = $this->eingabe;

        # IP:
        $this->ip = $request->ip();

        # Language:
708
        if (isset($_SERVER['HTTP_LANGUAGE'])) {
709
            $this->language = $_SERVER['HTTP_LANGUAGE'];
710
        } else {
711
712
713
714
715
716
            $this->language = "";
        }
        # Category
        $this->category = $request->input('category', '');
        # Request Times:
        $this->time = $request->input('time', 1000);
717

718
        # Page
719
        $this->page = 1;
720
721
        # Lang
        $this->lang = $request->input('lang', 'all');
722
723
        if ($this->lang !== "de" && $this->lang !== "en" && $this->lang !== "all") {
            $this->lang = "all";
724
        }
725
        $this->agent  = new Agent();
726
727
728
729
        $this->mobile = $this->agent->isMobile();

        #Sprüche
        $this->sprueche = $request->input('sprueche', 'off');
730
        if ($this->sprueche === "off") {
731
            $this->sprueche = true;
732
        } else {
733
            $this->sprueche = false;
734
735
        }

736
737
738
739
        # Ergebnisse pro Seite:
        $this->resultCount = $request->input('resultCount', '20');

        # Manchmal müssen wir Parameter anpassen um den Sucheinstellungen gerecht zu werden:
740
741
742
        if ($request->has('dart')) {
            $this->time       = 10000;
            $this->warnings[] = "Hinweis: Sie haben Dart-Europe aktiviert. Die Suche kann deshalb länger dauern und die maximale Suchzeit wurde auf 10 Sekunden hochgesetzt.";
743
        }
744
745
        if ($this->time <= 500 || $this->time > 20000) {
            $this->time = 1000;
746
        }
747
748
749
750
751
752
753
754
755
        if ($request->has('minism') && ($request->has('fportal') || $request->has('harvest'))) {
            $input    = $request->all();
            $newInput = [];
            foreach ($input as $key => $value) {
                if ($key !== "fportal" && $key !== "harvest") {
                    $newInput[$key] = $value;
                }
            }
            $request->replace($newInput);
756
        }
757
758
        if (App::isLocale("en")) {
            $this->sprueche = "off";
759
        }
760
761
        if ($this->resultCount <= 0 || $this->resultCount > 200) {
            $this->resultCount = 1000;
762
        }
763
764
765
766
767
768
        if ($request->has('onenewspageAll') || $request->has('onenewspageGermanyAll')) {
            $this->time  = 5000;
            $this->cache = "cache";
        }
        if ($request->has('tab')) {
            if ($request->input('tab') === "off") {
769
                $this->tab = "_blank";
770
            } else {
771
772
                $this->tab = "_self";
            }
773
        } else {
774
775
            $this->tab = "_blank";
        }
776
        if ($request->has('password')) {
777
            $this->password = $request->input('password');
778
779
780
        }

        if ($request->has('quicktips')) {
781
            $this->quicktips = false;
782
        } else {
783
            $this->quicktips = true;
784
        }
785
786

        $this->out = $request->input('out', "html");
787
        if ($this->out !== "html" && $this->out !== "json" && $this->out !== "results" && $this->out !== "results-with-style") {
788
            $this->out = "html";
789
790
        }

791
        $this->request = $request;
792
793
794
795
796
797
798
799
800
801
    }

    public function checkSpecialSearches(Request $request)
    {
        # Site Search:
        if (preg_match("/(.*)\bsite:(\S+)(.*)/si", $this->q, $match)) {
            $this->site = $match[2];
            $this->q    = $match[1] . $match[3];
        }
        if ($request->has('site')) {
802
803
            $this->site = $request->input('site');
        }
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
        # Wenn die Suchanfrage um das Schlüsselwort "-host:*" ergänzt ist, sollen bestimmte Hosts nicht eingeblendet werden
        # Wir prüfen, ob das hier der Fall ist:
        while (preg_match("/(.*)(^|\s)-host:(\S+)(.*)/si", $this->q, $match)) {
            $this->hostBlacklist[] = $match[3];
            $this->q               = $match[1] . $match[4];
        }
        if (sizeof($this->hostBlacklist) > 0) {
            $hostString = "";
            foreach ($this->hostBlacklist as $host) {
                $hostString .= $host . ", ";
            }
            $hostString       = rtrim($hostString, ", ");
            $this->warnings[] = "Ergebnisse von folgenden Hosts werden nicht angezeigt: \"" . $hostString . "\"";
        }
        # Wenn die Suchanfrage um das Schlüsselwort "-domain:*" ergänzt ist, sollen bestimmte Domains nicht eingeblendet werden
        # Wir prüfen, ob das hier der Fall ist:
        while (preg_match("/(.*)(^|\s)-domain:(\S+)(.*)/si", $this->q, $match)) {
            $this->domainBlacklist[] = $match[3];
            $this->q                 = $match[1] . $match[4];
        }
        if (sizeof($this->domainBlacklist) > 0) {
            $domainString = "";
            foreach ($this->domainBlacklist as $domain) {
                $domainString .= $domain . ", ";
            }
            $domainString     = rtrim($domainString, ", ");
            $this->warnings[] = "Ergebnisse von folgenden Domains werden nicht angezeigt: \"" . $domainString . "\"";
        }

        # Alle mit "-" gepräfixten Worte sollen aus der Suche ausgeschlossen werden.
        # Wir prüfen, ob das hier der Fall ist:
        while (preg_match("/(.*)(^|\s)-(\S+)(.*)/si", $this->q, $match)) {
            $this->stopWords[] = $match[3];
            $this->q           = $match[1] . $match[4];
        }
        if (sizeof($this->stopWords) > 0) {
            $stopwordsString = "";
            foreach ($this->stopWords as $stopword) {
                $stopwordsString .= $stopword . ", ";
            }
            $stopwordsString  = rtrim($stopwordsString, ", ");
            $this->warnings[] = "Sie machen eine Ausschlusssuche. Ergebnisse mit folgenden Wörtern werden nicht angezeigt: \"" . $stopwordsString . "\"";
        }

        # Meldung über eine Phrasensuche
        $p   = "";
850
        $tmp = $this->q;
851
852
        while (preg_match("/(.*)\"(.+)\"(.*)/si", $tmp, $match)) {
            $tmp             = $match[1] . $match[3];
853
            $this->phrases[] = strtolower($match[2]);
854
855
        }
        foreach ($this->phrases as $phrase) {
856
857
858
            $p .= "\"$phrase\", ";
        }
        $p = rtrim($p, ", ");
859
        if (sizeof($this->phrases) > 0) {
860
            $this->warnings[] = "Sie führen eine Phrasensuche durch: $p";
861
        }
862

863
864
865
    }

    public function getFokus()
866
867
868
869
    {
        return $this->fokus;
    }

870
    public function getIp()
871
872
873
874
    {
        return $this->ip;
    }

875
    public function getEingabe()
876
877
878
879
    {
        return $this->eingabe;
    }

880
    public function getQ()
881
    {
882
        return $this->q;
883
884
    }

885
    public function getUrl()
886
887
888
    {
        return $this->url;
    }
889
    public function getTime()
890
891
892
893
    {
        return $this->time;
    }

894
    public function getLanguage()
895
896
897
898
    {
        return $this->language;
    }

899
    public function getLang()
900
901
902
903
    {
        return $this->lang;
    }

904
    public function getSprueche()
905
906
907
908
    {
        return $this->sprueche;
    }

909
    public function getCategory()
910
911
912
913
    {
        return $this->category;
    }

914
    public function getPhrases()
915
916
917
    {
        return $this->phrases;
    }
918
919
920
921
    public function getPage()
    {
        return $this->page;
    }
922

923
    public function getSumaFile()
924
925
926
927
    {
        return $this->sumaFile;
    }

928
    public function getUserHostBlacklist()
929
930
931
932
    {
        return $this->hostBlacklist;
    }

933
    public function getUserDomainBlacklist()
934
935
936
937
    {
        return $this->domainBlacklist;
    }

938
    public function getDomainBlacklist()
939
940
941
942
    {
        return $this->domainsBlacklisted;
    }

943
    public function getUrlBlacklist()
944
945
946
    {
        return $this->urlsBlacklisted;
    }
947
    public function getLanguageDetect()
948
949
950
    {
        return $this->languageDetect;
    }
951
    public function getStopWords()
952
953
954
955
956
    {
        return $this->stopWords;
    }
    public function getHostCount($host)
    {
957
        if (isset($this->addedHosts[$host])) {
958
            return $this->addedHosts[$host];
959
        } else {
960
961
962
            return 0;
        }
    }
963
964
965
966
    public function getStartCount()
    {
        return $this->startCount;
    }
967
968
969
    public function addHostCount($host)
    {
        $hash = md5($host);
970
        if (isset($this->addedHosts[$hash])) {
971
            $this->addedHosts[$hash] += 1;
972
        } else {
973
974
975
976
977
978
979
980
981
            $this->addedHosts[$hash] = 1;
        }
    }
    public function getSite()
    {
        return $this->site;
    }
    public function addLink($link)
    {
982
        if (strpos($link, "http://") === 0) {
983
            $link = substr($link, 7);
984
985
986
        }

        if (strpos($link, "https://") === 0) {
987
            $link = substr($link, 8);
988
989
990
        }

        if (strpos($link, "www.") === 0) {
991
            $link = substr($link, 4);
992
993
        }

994
995
        $link = trim($link, "/");
        $hash = md5($link);
996
        if (isset($this->addedLinks[$hash])) {
997
            return false;
998
        } else {
999
1000
1001
1002
1003
1004
1005
1006
            $this->addedLinks[$hash] = 1;

            return true;
        }
    }

    public function generateSearchLink($fokus, $results = true)
    {
1007
        $requestData          = $this->request->except(['page', 'next']);
1008
        $requestData['focus'] = $fokus;