MetaGer.php 38.6 KB
Newer Older
1
2
3
4
<?php
namespace App;

use App;
5
use App\lib\TextLanguageDetect\TextLanguageDetect;
6
use Cache;
7
8
use Illuminate\Http\Request;
use Jenssegers\Agent\Agent;
Dominik Hebeler's avatar
Bugfix    
Dominik Hebeler committed
9
use LaravelLocalization;
10
use Log;
11
use Predis\Connection\ConnectionException;
12
use Redis;
13
14
15

class MetaGer
{
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
    # Einstellungen für die Suche
    protected $fokus;
    protected $eingabe;
    protected $q;
    protected $category;
    protected $time;
    protected $page;
    protected $lang;
    protected $cache = "";
    protected $site;
    protected $hostBlacklist   = [];
    protected $domainBlacklist = [];
    protected $stopWords       = [];
    protected $phrases         = [];
    protected $engines         = [];
    protected $results         = [];
    protected $ads             = [];
    protected $warnings        = [];
    protected $errors          = [];
    protected $addedHosts      = [];
36
    protected $startCount      = 0;
37
    protected $canCache        = false;
38
39
40
41
42
43
44
45
46
    # Daten über die Abfrage
    protected $ip;
    protected $language;
    protected $agent;
    # Konfigurationseinstellungen:
    protected $sumaFile;
    protected $mobile;
    protected $resultCount;
    protected $sprueche;
47
    protected $domainsBlacklisted = [];
48
    protected $urlsBlacklisted    = [];
49
50
51
    protected $url;
    protected $languageDetect;

52
53
    public function __construct()
    {
54
        # Timer starten
55
        $this->starttime = microtime(true);
56
57

        # Versuchen Blacklists einzulesen
58
59
        if (file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt")) {
            $tmp                      = file_get_contents(config_path() . "/blacklistDomains.txt");
60
            $this->domainsBlacklisted = explode("\n", $tmp);
61
62
63
            $tmp                      = file_get_contents(config_path() . "/blacklistUrl.txt");
            $this->urlsBlacklisted    = explode("\n", $tmp);
        } else {
64
65
66
            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
        }

67
        # Parser Skripte einhängen
68
69
70
71
72
73
74
75
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
                require $path;
            }
        }

76
        # Spracherkennung starten
77
78
        $this->languageDetect = new TextLanguageDetect();
        $this->languageDetect->setNameMode("2");
79

80
        # Cachebarkeit testen
81
82
83
84
85
86
        try {
            Cache::has('test');
            $this->canCache = true;
        } catch (ConnectionException $e) {
            $this->canCache = false;
        }
87
    }
88

89
    # Erstellt aus den gesammelten Ergebnissen den View
90
91
92
    public function createView()
    {
        $viewResults = [];
93

94
        # Wir extrahieren alle notwendigen Variablen und geben Sie an unseren View:
95
        foreach ($this->results as $result) {
96
97
98
99
100
101
            $viewResults[] = get_object_vars($result);
        }

        # Wir müssen natürlich noch den Log für die durchgeführte Suche schreiben:
        $this->createLogs();

102
103
        if ($this->fokus === "bilder") {
            switch ($this->out) {
104
105
106
107
108
109
110
111
                case 'results':
                    return view('metager3bilderresults')
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
112
                        ->with('browser', (new Agent())->browser());
113
114
115
116
117
118
119
120
                default:
                    return view('metager3bilder')
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
121
                        ->with('browser', (new Agent())->browser());
122
            }
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
        } else {
            switch ($this->out) {
                case 'results':
                    return view('metager3results')
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
                        ->with('browser', (new Agent())->browser());
                    break;
                case 'results-with-style':
                    return view('metager3')
                        ->with('results', $viewResults)
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
                        ->with('suspendheader', "yes")
                        ->with('browser', (new Agent())->browser());
                    break;
                default:
                    return view('metager3')
                        ->with('eingabe', $this->eingabe)
                        ->with('mobile', $this->mobile)
                        ->with('warnings', $this->warnings)
                        ->with('errors', $this->errors)
                        ->with('metager', $this)
                        ->with('browser', (new Agent())->browser());
                    break;
            }
156
157
158
        }
    }

159
    public function removeInvalids()
160
161
    {
        $results = [];
162
163
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
164
                $results[] = $result;
165
166
            }

167
168
        }
    }
169

170
171
172
    public function combineResults()
    {
        foreach ($this->engines as $engine) {
173
174
175
176
177
178
            if (isset($engine->next)) {
                $this->next[] = $engine->next;
            }
            if (isset($engine->last)) {
                $this->last[] = $engine->last;
            }
179
180
            foreach ($engine->results as $result) {
                if ($result->valid) {
181
                    $this->results[] = $result;
182
                }
183
            }
184
            foreach ($engine->ads as $ad) {
185
186
                $this->ads[] = $ad;
            }
187
        }
188

189
190
        uasort($this->results, function ($a, $b) {
            if ($a->getRank() == $b->getRank()) {
191
                return 0;
192
193
            }

194
195
            return ($a->getRank() < $b->getRank()) ? 1 : -1;
        });
196

197
198
        # Validate Results
        $newResults = [];
199
200
        foreach ($this->results as $result) {
            if ($result->isValid($this)) {
201
                $newResults[] = $result;
202
203
            }

204
205
206
        }
        $this->results = $newResults;

207
208
209
210
211
212
        # Boost implementation
        $this->results = $this->parseBoost($this->results);

        #Adgoal Implementation
        $this->results = $this->parseAdgoal($this->results);

213
        $counter   = 0;
214
        $firstRank = 0;
215

216
        if (isset($this->startForwards)) {
217
            $this->startCount = $this->startForwards;
218
        } elseif (isset($this->startBackwards)) {
219
            $this->startCount = $this->startBackwards - count($this->results) - 1;
220
        } else {
221
222
223
            $this->startCount = 0;
        }

224
225
        foreach ($this->results as $result) {
            if ($counter === 0) {
226
                $firstRank = $result->rank;
227
228
            }

229
            $counter++;
230
            $result->number = $counter + $this->startCount;
231
232
233
234
            $confidence     = 0;
            if ($firstRank > 0) {
                $confidence = $result->rank / $firstRank;
            } else {
235
                $confidence = 0;
236
237
238
            }

            if ($confidence > 0.65) {
239
                $result->color = "#FF4000";
240
            } elseif ($confidence > 0.4) {
241
                $result->color = "#FF0080";
242
            } elseif ($confidence > 0.2) {
243
                $result->color = "#C000C0";
244
            } else {
245
                $result->color = "#000000";
246
247
            }

248
249
        }

250
        if (LaravelLocalization::getCurrentLocale() === "en") {
251
252
253
            $this->ads = [];
        }

254
        $this->validated = false;
255
        if (isset($this->password)) {
256
257
258
            # Wir bieten einen bezahlten API-Zugriff an, bei dem dementsprechend die Werbung ausgeblendet wurde:
            # Aktuell ist es nur die Uni-Mainz. Deshalb überprüfen wir auch nur diese.
            $password = getenv('mainz');
259
            $eingabe  = $this->eingabe;
260
            $password = md5($eingabe . $password);
261
262
            if ($this->password === $password) {
                $this->ads       = [];
263
264
265
                $this->validated = true;
            }
        }
266
267

        if (count($this->results) <= 0) {
Dominik Hebeler's avatar
Dominik Hebeler committed
268
            $this->errors[] = "Leider konnten wir zu Ihrer Sucheingabe keine passenden Ergebnisse finden.";
269
        }
270

271
        if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
272
            $page       = $this->page + 1;
273
            $this->next = [
274
275
276
                'page'          => $page,
                'startForwards' => $this->results[count($this->results) - 1]->number,
                'engines'       => $this->next,
277
278
            ];
            Cache::put(md5(serialize($this->next)), serialize($this->next), 60);
279
280
        } else {
            $this->next = [];
281
282
        }

283
    }
284

285
286
    public function parseBoost($results)
    {
287
288
289
290
291
292
293
294
295
296
297
298
        foreach ($results as $result) {
            if (preg_match('/^(http[s]?\:\/\/)?(www.)?amazon\.de/', $result->anzeigeLink)) {
                if (preg_match('/\?/', $result->anzeigeLink)) {
                    $result->link .= '&tag=boostmg01-21';
                } else {
                    $result->link .= '?tag=boostmg01-21';
                }
                $result->partnershop = true;

            }
        }
        return $results;
299
    }
300

301
302
    public function parseAdgoal($results)
    {
303
        $publicKey  = getenv('adgoal_public');
304
        $privateKey = getenv('adgoal_private');
305
        if ($publicKey === false) {
306
307
308
            return $results;
        }
        $tldList = "";
309
310
        try {
            foreach ($results as $result) {
311
                $link = $result->anzeigeLink;
312
                if (strpos($link, "http") !== 0) {
313
314
315
316
317
318
319
320
321
322
                    $link = "http://" . $link;
                }
                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                $result->tld = parse_url($link, PHP_URL_HOST);
            }
            $tldList = rtrim($tldList, ",");

            # Hashwert
            $hash = md5("meta" . $publicKey . $tldList . "GER");

323
            # Query
324
325
            $query = urlencode($this->q);

326
            $link   = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . $publicKey . "&k=" . $hash . "&tld=" . $tldList . "&q=" . $query;
327
328
329
            $answer = json_decode(file_get_contents($link));

            # Nun müssen wir nur noch die Links für die Advertiser ändern:
330
            foreach ($answer as $el) {
331
                $hoster = $el[0];
332
                $hash   = $el[1];
333

334
335
                foreach ($results as $result) {
                    if ($hoster === $result->tld) {
336
337
                        # Hier ist ein Advertiser:
                        # Das Logo hinzufügen:
338
                        if ($result->image !== "") {
339
                            $result->logo = "https://img.smartredirect.de/logos_v2/60x30/" . $hash . ".gif";
340
                        } else {
341
                            $result->image = "https://img.smartredirect.de/logos_v2/120x60/" . $hash . ".gif";
342
343
                        }

344
345
346
                        # Den Link hinzufügen:
                        $publicKey = $publicKey;
                        $targetUrl = $result->anzeigeLink;
347
                        if (strpos($targetUrl, "http") !== 0) {
348
                            $targetUrl = "http://" . $targetUrl;
349
350
351
352
353
                        }

                        $gateHash            = md5($targetUrl . $privateKey);
                        $newLink             = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . $publicKey . "&k=" . $gateHash . "&url=" . urlencode($targetUrl) . "&q=" . $query;
                        $result->link        = $newLink;
354
355
356
357
                        $result->partnershop = true;
                    }
                }
            }
358
        } catch (\ErrorException $e) {
359
360
361
362
363
            return $results;
        }

        return $results;
    }
364

365
366
367
    public function createSearchEngines(Request $request)
    {
        if (!$request->has("eingabe")) {
368
            return;
369
        }
370

371
372
        # Überprüfe, welche Sumas eingeschaltet sind
        $xml                  = simplexml_load_file($this->sumaFile);
373
        $enabledSearchengines = [];
374
        $overtureEnabled      = false;
375
        $sumaCount            = 0;
376
        $sumas                = $xml->xpath("suma");
377
378
379

        foreach ($sumas as $suma) {
            if ($this->fokus === "angepasst") {
380
381
382
383
384
385
386
387
388
389
390
391
                if ($request->has($suma["name"])
                    || ($this->fokus !== "bilder"
                        && ($suma["name"]->__toString() === "qualigo"
                            || $suma["name"]->__toString() === "similar_product_ads"
                            || (!$overtureEnabled && $suma["name"]->__toString() === "overtureAds")
                        )
                    )
                ) {

                    if (!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1")) {
                        if ($suma["name"]->__toString() === "overture" || $suma["name"]->__toString() === "overtureAds") {
                            $overtureEnabled = true;
392
                        }
393
                        if ($suma["name"]->__toString() !== "qualigo" && $suma["name"]->__toString() !== "similar_product_ads" && $suma["name"]->__toString() !== "overtureAds") {
394
                            $sumaCount += 1;
395
                        }
396
397
398
                        $enabledSearchengines[] = $suma;
                    }
                }
399
            } else {
400
401
402
403
404
405
406
407
408
409
410
411
                $types = explode(",", $suma["type"]);
                if (in_array($this->fokus, $types)
                    || ($this->fokus !== "bilder"
                        && ($suma["name"]->__toString() === "qualigo"
                            || $suma["name"]->__toString() === "similar_product_ads"
                            || (!$overtureEnabled && $suma["name"]->__toString() === "overtureAds")
                        )
                    )
                ) {
                    if (!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1")) {
                        if ($suma["name"]->__toString() === "overture" || $suma["name"]->__toString() === "overtureAds") {
                            $overtureEnabled = true;
412
                        }
413
                        if ($suma["name"]->__toString() !== "qualigo" && $suma["name"]->__toString() !== "similar_product_ads" && $suma["name"]->__toString() !== "overtureAds") {
414
                            $sumaCount += 1;
415
                        }
416
417
418
419
420
421
422
423
                        $enabledSearchengines[] = $suma;
                    }
                }
            }
        }

        # Sonderregelung für alle Suchmaschinen, die zu den Minisuchern gehören. Diese können alle gemeinsam über einen Link abgefragt werden
        $subcollections = [];
424
425
426
        $tmp            = [];
        foreach ($enabledSearchengines as $engine) {
            if (isset($engine['minismCollection'])) {
427
                $subcollections[] = $engine['minismCollection']->__toString();
428
            } else {
429
                $tmp[] = $engine;
430
431
            }

432
433
        }
        $enabledSearchengines = $tmp;
434
435
436
437
        if (sizeof($subcollections) > 0) {
            $count                        = sizeof($subcollections) * 10;
            $minisucherEngine             = $xml->xpath('suma[@name="minism"]')[0];
            $subcollections               = urlencode("(" . implode(" OR ", $subcollections) . ")");
438
439
            $minisucherEngine["formData"] = str_replace("<<SUBCOLLECTIONS>>", $subcollections, $minisucherEngine["formData"]);
            $minisucherEngine["formData"] = str_replace("<<COUNT>>", $count, $minisucherEngine["formData"]);
440
            $enabledSearchengines[]       = $minisucherEngine;
441
442
        }

443
        if ($sumaCount <= 0) {
444
445
            $this->errors[] = "Achtung: Sie haben in ihren Einstellungen keine Suchmaschine ausgewählt.";
        }
446
        $engines = [];
447

448
449
        # Wenn eine Sitesearch durchgeführt werden soll, überprüfen wir ob eine der Suchmaschinen überhaupt eine Sitesearch unterstützt
        $siteSearchFailed = $this->checkCanNotSitesearch($enabledSearchengines);
450
451

        $typeslist = [];
452
        $counter   = 0;
453

454
455
        if ($request->has('next') && Cache::has($request->input('next')) && unserialize(Cache::get($request->input('next')))['page'] > 1) {
            $next       = unserialize(Cache::get($request->input('next')));
456
            $this->page = $next['page'];
457
458
            $engines    = $next['engines'];
            if (isset($next['startForwards'])) {
459
                $this->startForwards = $next['startForwards'];
460
461
462
            }

            if (isset($next['startBackwards'])) {
463
                $this->startBackwards = $next['startBackwards'];
464
465
            }

466
467
        } else {
            foreach ($enabledSearchengines as $engine) {
468

469
                if (!$siteSearchFailed && strlen($this->site) > 0 && (!isset($engine['hasSiteSearch']) || $engine['hasSiteSearch']->__toString() === "0")) {
470

471
472
473
474
                    continue;
                }
                # Wenn diese Suchmaschine gar nicht eingeschaltet sein soll
                $path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString());
475

476
477
478
479
                if (!file_exists(app_path() . "/Models/parserSkripte/" . ucfirst($engine["package"]->__toString()) . ".php")) {
                    Log::error("Konnte " . $engine["name"] . " nicht abfragen, da kein Parser existiert");
                    continue;
                }
480

481
                $time = microtime();
482

483
484
485
486
487
488
489
                try
                {
                    $tmp = new $path($engine, $this);
                } catch (\ErrorException $e) {
                    Log::error("Konnte " . $engine["name"] . " nicht abfragen." . var_dump($e));
                    continue;
                }
490

491
492
493
494
495
496
497
                if ($tmp->enabled && isset($this->debug)) {
                    $this->warnings[] = $tmp->service . "   Connection_Time: " . $tmp->connection_time . "    Write_Time: " . $tmp->write_time . " Insgesamt:" . ((microtime() - $time) / 1000);
                }

                if ($tmp->isEnabled()) {
                    $engines[] = $tmp;
                }
498
499

            }
500
        }
501

502
503
504
        # Wir starten die Suche manuell:
        foreach ($engines as $engine) {
            $engine->startSearch($this);
505
        }
506

507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
        $this->adjustFocus($sumas, $enabledSearchengines);

        /* Nun passiert ein elementarer Schritt.
         * Wir warten auf die Antwort der Suchmaschinen, da wir vorher nicht weiter machen können.
         * Aber natürlich nicht ewig.
         * Die Verbindung steht zu diesem Zeitpunkt und auch unsere Request wurde schon gesendet.
         * Wir geben der Suchmaschine nun bis zu 500ms Zeit zu antworten.
         */

        # Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
        $enginesToLoad = 0;
        $canBreak      = false;
        foreach ($engines as $engine) {
            if ($engine->cached) {
                $enginesToLoad--;
                if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
                    $canBreak = true;
                }
            }
        }

        $enginesToLoad += count($engines);

        $this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);

        $this->retrieveResults($engines);
    }

    public function adjustFocus($sumas, $enabledSearchengines)
    {
537
538
        # Jetzt werden noch alle Kategorien der Settings durchgegangen und die jeweils enthaltenen namen der Suchmaschinen gespeichert.
        $foki = [];
539
540
541
        foreach ($sumas as $suma) {
            if ((!isset($suma['disabled']) || $suma['disabled'] === "") && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
                if (isset($suma['type'])) {
542
                    $f = explode(",", $suma['type']->__toString());
543
544
                    foreach ($f as $tmp) {
                        $name                                    = $suma['name']->__toString();
545
546
                        $foki[$tmp][$suma['name']->__toString()] = $name;
                    }
547
548
                } else {
                    $name                                        = $suma['name']->__toString();
549
550
551
552
553
554
555
                    $foki["andere"][$suma['name']->__toString()] = $name;
                }
            }
        }

        # Es werden auch die Namen der aktuell aktiven Suchmaschinen abgespeichert.
        $realEngNames = [];
556
        foreach ($enabledSearchengines as $realEng) {
557
            $nam = $realEng["name"]->__toString();
558
            if ($nam !== "qualigo" && $nam !== "overtureAds") {
559
560
561
                $realEngNames[] = $nam;
            }
        }
562

563
        # Anschließend werden diese beiden Listen verglichen (jeweils eine der Fokuslisten für jeden Fokus), um herauszufinden ob sie vielleicht identisch sind. Ist dies der Fall, so hat der Nutzer anscheinend Suchmaschinen eines kompletten Fokus eingestellt. Der Fokus wird dementsprechend angepasst.
564
565
        foreach ($foki as $fok => $engs) {
            $isFokus      = true;
566
            $fokiEngNames = [];
567
            foreach ($engs as $eng) {
568
569
                $fokiEngNames[] = $eng;
            }
570
571
            foreach ($fokiEngNames as $fen) {
                if (!in_array($fen, $realEngNames)) {
572
573
574
                    $isFokus = false;
                }
            }
575
576
            foreach ($realEngNames as $ren) {
                if (!in_array($ren, $fokiEngNames)) {
577
578
579
                    $isFokus = false;
                }
            }
580
            if ($isFokus) {
581
582
583
                $this->fokus = $fok;
            }
        }
584
    }
585

586
587
588
589
590
591
592
    public function checkCanNotSitesearch($enabledSearchengines)
    {
        if (strlen($this->site) > 0) {
            $enginesWithSite = 0;
            foreach ($enabledSearchengines as $engine) {
                if (isset($engine['hasSiteSearch']) && $engine['hasSiteSearch']->__toString() === "1") {
                    $enginesWithSite++;
593
                }
594
595
596
597
598
599
600
            }
            if ($enginesWithSite === 0) {
                $this->errors[] = "Sie wollten eine Sitesearch auf " . $this->site . " durchführen. Leider unterstützen die eingestellten Suchmaschinen diese nicht. Sie können <a href=\"" . $this->generateSearchLink("web", false) . "\">hier</a> die Sitesearch im Web-Fokus durchführen. Es werden ihnen Ergebnisse ohne Sitesearch angezeigt.";
                return true;
            } else {
                $this->warnings[] = "Sie führen eine Sitesearch durch. Es werden nur Ergebnisse von der Seite: <a href=\"http://" . $this->site . "\" target=\"_blank\">\"" . $this->site . "\"</a> angezeigt.";
                return false;
601
602
            }
        }
603
604
605
606
    }

    public function waitForResults($enginesToLoad, $overtureEnabled, $canBreak)
    {
607
        $loadedEngines = 0;
608
609
610
        $timeStart     = microtime(true);
        while (true) {
            $time          = (microtime(true) - $timeStart) * 1000;
611
            $loadedEngines = intval(Redis::hlen('search.' . $this->getHashCode()));
612
            if ($overtureEnabled && (Redis::hexists('search.' . $this->getHashCode(), 'overture') || Redis::hexists('search.' . $this->getHashCode(), 'overtureAds'))) {
613
                $canBreak = true;
614
            }
615
616

            # Abbruchbedingung
617
618
            if ($time < 500) {
                if (($enginesToLoad === 0 || $loadedEngines >= $enginesToLoad) && $canBreak) {
619
                    break;
620
621
622
623
                }

            } elseif ($time >= 500 && $time < $this->time) {
                if (($enginesToLoad === 0 || ($loadedEngines / ($enginesToLoad * 1.0)) >= 0.8) && $canBreak) {
624
                    break;
625
626
627
                }

            } else {
628
629
630
631
                break;
            }
            usleep(50000);
        }
632
    }
633

634
635
636
    public function retrieveResults($engines)
    {
        # Von geladenen Engines die Ergebnisse holen
637
638
639
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
                try {
640
                    $engine->retrieveResults($this);
641
                } catch (\ErrorException $e) {
642
643
644
645
                    Log::error($e);
                }
            }
        }
646

647
        # Nicht fertige Engines verwefen
648
649
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
650
                $engine->shutdown();
651
            }
652
653
654
        }

        $this->engines = $engines;
655
656
657
658
    }

    public function parseFormData(Request $request)
    {
659
        # Sichert, dass der request in UTF-8 formatiert ist
660
661
662
663
664
665
666
667
        if ($request->input('encoding', '') !== "utf8") {
            # In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
            $input = $request->all();
            foreach ($input as $key => $value) {
                $input[$key] = mb_convert_encoding("$value", "UTF-8", "ISO-8859-1");
            }
            $request->replace($input);
        }
668
        $this->url = $request->url();
669
        # Zunächst überprüfen wir die eingegebenen Einstellungen:
670
671
        # Fokus
        $this->fokus = trans('fokiNames.' . $request->input('focus', 'web'));
672
        if (strpos($this->fokus, ".")) {
673
674
            $this->fokus = trans('fokiNames.web');
        }
675
        # Suma-File
676
        if (App::isLocale("en")) {
677
            $this->sumaFile = config_path() . "/sumas.xml";
678
        } else {
679
680
            $this->sumaFile = config_path() . "/sumas.xml";
        }
681
        if (!file_exists($this->sumaFile)) {
682
683
            die("Suma-File konnte nicht gefunden werden");
        }
684
        # Sucheingabe
685
        $this->eingabe = trim($request->input('eingabe', ''));
686
        if (strlen($this->eingabe) === 0) {
687
688
689
            $this->warnings[] = 'Achtung: Sie haben keinen Suchbegriff eingegeben. Sie können ihre Suchbegriffe oben eingeben und es erneut versuchen.';
        }
        $this->q = $this->eingabe;
690
        # IP
691
        $this->ip = $request->ip();
692
        # Language
693
        if (isset($_SERVER['HTTP_LANGUAGE'])) {
694
            $this->language = $_SERVER['HTTP_LANGUAGE'];
695
        } else {
696
697
698
699
            $this->language = "";
        }
        # Category
        $this->category = $request->input('category', '');
700
        # Request Times
701
702
        $this->time = $request->input('time', 1000);
        # Page
703
        $this->page = 1;
704
705
        # Lang
        $this->lang = $request->input('lang', 'all');
706
707
        if ($this->lang !== "de" && $this->lang !== "en" && $this->lang !== "all") {
            $this->lang = "all";
708
        }
709
        $this->agent  = new Agent();
710
        $this->mobile = $this->agent->isMobile();
711
        # Sprüche
712
        $this->sprueche = $request->input('sprueche', 'off');
713
        if ($this->sprueche === "off") {
714
            $this->sprueche = true;
715
        } else {
716
            $this->sprueche = false;
717
        }
718
719
720
        # Ergebnisse pro Seite:
        $this->resultCount = $request->input('resultCount', '20');
        # Manchmal müssen wir Parameter anpassen um den Sucheinstellungen gerecht zu werden:
721
722
723
        if ($request->has('dart')) {
            $this->time       = 10000;
            $this->warnings[] = "Hinweis: Sie haben Dart-Europe aktiviert. Die Suche kann deshalb länger dauern und die maximale Suchzeit wurde auf 10 Sekunden hochgesetzt.";
724
        }
725
726
        if ($this->time <= 500 || $this->time > 20000) {
            $this->time = 1000;
727
        }
728
729
730
731
732
733
734
735
736
        if ($request->has('minism') && ($request->has('fportal') || $request->has('harvest'))) {
            $input    = $request->all();
            $newInput = [];
            foreach ($input as $key => $value) {
                if ($key !== "fportal" && $key !== "harvest") {
                    $newInput[$key] = $value;
                }
            }
            $request->replace($newInput);
737
        }
738
739
        if (App::isLocale("en")) {
            $this->sprueche = "off";
740
        }
741
742
        if ($this->resultCount <= 0 || $this->resultCount > 200) {
            $this->resultCount = 1000;
743
        }
744
745
746
747
748
749
        if ($request->has('onenewspageAll') || $request->has('onenewspageGermanyAll')) {
            $this->time  = 5000;
            $this->cache = "cache";
        }
        if ($request->has('tab')) {
            if ($request->input('tab') === "off") {
750
                $this->tab = "_blank";
751
            } else {
752
753
                $this->tab = "_self";
            }
754
        } else {
755
756
            $this->tab = "_blank";
        }
757
        if ($request->has('password')) {
758
            $this->password = $request->input('password');
759
760
        }
        if ($request->has('quicktips')) {
761
            $this->quicktips = false;
762
        } else {
763
            $this->quicktips = true;
764
        }
765
        $this->out = $request->input('out', "html");
766
        # Standard output format html
767
        if ($this->out !== "html" && $this->out !== "json" && $this->out !== "results" && $this->out !== "results-with-style") {
768
            $this->out = "html";
769
        }
770
        $this->request = $request;
771
772
773
774
    }

    public function checkSpecialSearches(Request $request)
    {
775
        # Site Search
776
777
778
779
780
        if (preg_match("/(.*)\bsite:(\S+)(.*)/si", $this->q, $match)) {
            $this->site = $match[2];
            $this->q    = $match[1] . $match[3];
        }
        if ($request->has('site')) {
781
782
            $this->site = $request->input('site');
        }
783
784

        # Host Blacklisting
785
786
787
788
789
790
791
792
793
794
795
796
797
        # Wenn die Suchanfrage um das Schlüsselwort "-host:*" ergänzt ist, sollen bestimmte Hosts nicht eingeblendet werden
        while (preg_match("/(.*)(^|\s)-host:(\S+)(.*)/si", $this->q, $match)) {
            $this->hostBlacklist[] = $match[3];
            $this->q               = $match[1] . $match[4];
        }
        if (sizeof($this->hostBlacklist) > 0) {
            $hostString = "";
            foreach ($this->hostBlacklist as $host) {
                $hostString .= $host . ", ";
            }
            $hostString       = rtrim($hostString, ", ");
            $this->warnings[] = "Ergebnisse von folgenden Hosts werden nicht angezeigt: \"" . $hostString . "\"";
        }
798
799

        # Domain Blacklisting
800
801
802
803
804
805
806
807
808
809
810
811
812
813
        # Wenn die Suchanfrage um das Schlüsselwort "-domain:*" ergänzt ist, sollen bestimmte Domains nicht eingeblendet werden
        while (preg_match("/(.*)(^|\s)-domain:(\S+)(.*)/si", $this->q, $match)) {
            $this->domainBlacklist[] = $match[3];
            $this->q                 = $match[1] . $match[4];
        }
        if (sizeof($this->domainBlacklist) > 0) {
            $domainString = "";
            foreach ($this->domainBlacklist as $domain) {
                $domainString .= $domain . ", ";
            }
            $domainString     = rtrim($domainString, ", ");
            $this->warnings[] = "Ergebnisse von folgenden Domains werden nicht angezeigt: \"" . $domainString . "\"";
        }

814
        # Stopwords
815
816
817
818
819
820
821
822
823
824
825
826
827
828
        # Alle mit "-" gepräfixten Worte sollen aus der Suche ausgeschlossen werden.
        while (preg_match("/(.*)(^|\s)-(\S+)(.*)/si", $this->q, $match)) {
            $this->stopWords[] = $match[3];
            $this->q           = $match[1] . $match[4];
        }
        if (sizeof($this->stopWords) > 0) {
            $stopwordsString = "";
            foreach ($this->stopWords as $stopword) {
                $stopwordsString .= $stopword . ", ";
            }
            $stopwordsString  = rtrim($stopwordsString, ", ");
            $this->warnings[] = "Sie machen eine Ausschlusssuche. Ergebnisse mit folgenden Wörtern werden nicht angezeigt: \"" . $stopwordsString . "\"";
        }

829
        # Phrasensuche
830
        $p   = "";
831
        $tmp = $this->q;
832
833
        while (preg_match("/(.*)\"(.+)\"(.*)/si", $tmp, $match)) {
            $tmp             = $match[1] . $match[3];
834
            $this->phrases[] = strtolower($match[2]);
835
836
        }
        foreach ($this->phrases as $phrase) {
837
838
839
            $p .= "\"$phrase\", ";
        }
        $p = rtrim($p, ", ");
840
        if (sizeof($this->phrases) > 0) {
841
            $this->warnings[] = "Sie führen eine Phrasensuche durch: $p";
842
        }
843

844
845
    }

846
    public function nextSearchLink()
847
    {
848
849
850
851
852
853
854
855
        if (isset($this->next) && isset($this->next['engines']) && count($this->next['engines']) > 0) {
            $requestData         = $this->request->except(['page', 'out']);
            $requestData['next'] = md5(serialize($this->next));
            $link                = action('MetaGerSearch@search', $requestData);
        } else {
            $link = "#";
        }
        return $link;
856
857
    }

858
    public function rankAll()
859
    {
860
861
862
        foreach ($this->engines as $engine) {
            $engine->rank($this);
        }
863
864
    }

865
# Hilfsfunktionen
866

867
    public function showQuicktips()
868
    {
869
        return $this->quicktips;
870
871
    }

872
    public function popAd()
873
    {
874
875
        if (count($this->ads) > 0) {
            return get_object_vars(array_shift($this->ads));
876
        } else {
877
            return null;
878
        }
879

880
    }
881

882
883
884
885
    public function canCache()
    {
        return $this->canCache;
    }
886
887

    public function createLogs()
888
    {
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
        $redis = Redis::connection('redisLogs');
        try
        {
            $logEntry = "";
            $logEntry .= "[" . date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y"))) . "]";
            $logEntry .= " pid=" . getmypid();
            $logEntry .= " ref=" . $this->request->header('Referer');
            $useragent = $this->request->header('User-Agent');
            $useragent = str_replace("(", " ", $useragent);
            $useragent = str_replace(")", " ", $useragent);
            $useragent = str_replace(" ", "", $useragent);
            $logEntry .= " time=" . round((microtime(true) - $this->starttime), 2) . " serv=" . $this->fokus;
            $logEntry .= " search=" . $this->eingabe;
            $redis->rpush('logs.search', $logEntry);
        } catch (\Exception $e) {
            return;
        }
906
    }
907

908
909
    public function addLink($link)
    {
910
        if (strpos($link, "http://") === 0) {
911
            $link = substr($link, 7);
912
913
914
        }

        if (strpos($link, "https://") === 0) {
915
            $link = substr($link, 8);
916
917
918
        }

        if (strpos($link, "www.") === 0) {
919
            $link = substr($link, 4);
920
921
        }

922
923
        $link = trim($link, "/");
        $hash = md5($link);
924
        if (isset($this->addedLinks[$hash])) {
925
            return false;
926
        } else {
927
928
929
930
931
932
            $this->addedLinks[$hash] = 1;

            return true;
        }
    }

933
934
935
936
937
938
939
940
941
942
943
944
    public function addHostCount($host)
    {
        $hash = md5($host);
        if (isset($this->addedHosts[$hash])) {
            $this->addedHosts[$hash] += 1;
        } else {
            $this->addedHosts[$hash] = 1;
        }
    }

# Generators

945
946
    public function generateSearchLink($fokus, $results = true)
    {
947
        $requestData          = $this->request->except(['page', 'next']);
948
        $requestData['focus'] = $fokus;
949
        if ($results) {
950
            $requestData['out'] = "results";
951
        } else {
952
            $requestData['out'] = "";
953
954
        }

955
956
957
958
959
960
961
962
963
964
965
966
967
        $link = action('MetaGerSearch@search', $requestData);
        return $link;
    }

    public function generateQuicktipLink()
    {
        $link = action('MetaGerSearch@quicktips');

        return $link;
    }

    public function generateSiteSearchLink($host)
    {
968
        $host        = urlencode($host);
969
        $requestData = $this->request->except(['page', 'out', 'next']);