Searchengine.php 9.06 KB
Newer Older
1 2
<?php

3
namespace App\Models;
4

5
use App\MetaGer;
6
use Cache;
7
use Illuminate\Support\Facades\Redis;
8

9
abstract class Searchengine
10
{
Karl's avatar
Karl committed
11 12
    public $getString = ""; # Der String für die Get-Anfrage
    public $engine; # Die ursprüngliche Engine XML
13
    public $totalResults = 0; # How many Results the Searchengine has found
Dominik Hebeler's avatar
Dominik Hebeler committed
14 15
    public $results = []; # Die geladenen Ergebnisse
    public $ads = []; # Die geladenen Werbungen
Dominik Hebeler's avatar
Dominik Hebeler committed
16
    public $products = []; # Die geladenen Produkte
Dominik Hebeler's avatar
Dominik Hebeler committed
17 18
    public $loaded = false; # wahr, sobald die Ergebnisse geladen wurden
    public $cached = false;
Karl's avatar
Karl committed
19 20 21 22 23 24 25 26 27 28

    public $ip; # Die IP aus der metager
    public $uses; # Die Anzahl der Nutzungen dieser Suchmaschine
    public $homepage; # Die Homepage dieser Suchmaschine
    public $name; # Der Name dieser Suchmaschine
    public $disabled; # Ob diese Suchmaschine ausgeschaltet ist
    public $useragent; # Der HTTP Useragent
    public $startTime; # Die Zeit der Erstellung dieser Suchmaschine
    public $hash; # Der Hash-Wert dieser Suchmaschine

29
    private $username; # Username für HTTP-Auth (falls angegeben)
30 31
    private $password; # Passwort für HTTP-Auth (falls angegeben)

32 33
    private $headers; # Headers to add

Karl's avatar
Karl committed
34
    public $fp; # Wird für Artefakte benötigt
Dominik Hebeler's avatar
Dominik Hebeler committed
35 36 37
    public $socketNumber = null; # Wird für Artefakte benötigt
    public $counter = 0; # Wird eventuell für Artefakte benötigt
    public $write_time = 0; # Wird eventuell für Artefakte benötigt
Karl's avatar
Karl committed
38
    public $connection_time = 0; # Wird eventuell für Artefakte benötigt
Dominik Hebeler's avatar
Dominik Hebeler committed
39
    public $cacheDuration = 60; # Wie lange soll das Ergebnis im Cache bleiben (Minuten)
40
    public $new = true; # Important for loading results by JS
41

42
    public function __construct($name, \stdClass $engine, MetaGer $metager)
43
    {
44 45
        $this->engine = $engine;
        $this->name = $name;
46

47
        if (isset($engine->{"cache-duration"}) && $engine->{"cache-duration"} !== -1) {
Dominik Hebeler's avatar
Dominik Hebeler committed
48
            $this->cacheDuration = $engine->{"cache-duration"};
49
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
50
        $this->cacheDuration = max($this->cacheDuration, 5);
Dominik Hebeler's avatar
Dominik Hebeler committed
51

Dominik Hebeler's avatar
Dominik Hebeler committed
52
        $this->useragent = $metager->getUserAgent();
Dominik Hebeler's avatar
Dominik Hebeler committed
53
        $this->ip = $metager->getIp();
Dominik Hebeler's avatar
Dominik Hebeler committed
54
        $this->startTime = microtime(true);
55 56 57 58 59
        # check for http Auth
        if (!empty($this->engine->{"http-auth-credentials"}->username) && !empty($this->engine->{"http-auth-credentials"}->password)) {
            $this->username = $this->engine->{"http-auth-credentials"}->username;
            $this->password = $this->engine->{"http-auth-credentials"}->password;
        }
60

Dominik Hebeler's avatar
Dominik Hebeler committed
61
        if (!empty($this->engine->{"request-header"})) {
Dominik Hebeler's avatar
Dominik Hebeler committed
62
            $this->headers = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
63
            foreach ($this->engine->{"request-header"} as $key => $value) {
Dominik Hebeler's avatar
Dominik Hebeler committed
64 65
                $this->headers[$key] = $value;
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
66 67 68
            if (sizeof($this->headers) == 0) {
                $this->headers = null;
            }
Dominik Hebeler's avatar
Dominik Hebeler committed
69
        }
70

71 72 73 74
        # Suchstring generieren
        $q = $metager->getQ();
        $filters = $metager->getSumaFile()->filter;
        foreach ($metager->getQueryFilter() as $queryFilter => $filter) {
Dominik Hebeler's avatar
Dominik Hebeler committed
75
            $filterOptions = $filters->{"query-filter"}->$queryFilter;
76 77 78
            if (!$filterOptions->sumas->{$this->name}) {
                continue;
            }
79 80 81
            $filterOptionsEngine = $filterOptions->sumas->{$this->name};
            $query = $filterOptionsEngine->prefix . $filter . $filterOptionsEngine->suffix;
            $q = $query . " " . $q;
82
        }
83

Dominik Hebeler's avatar
Dominik Hebeler committed
84 85
        # Parse enabled Parameter-Filter
        foreach ($metager->getParameterFilter() as $filterName => $filter) {
86 87
            $inputParameter = $filter->value;

Dominik Hebeler's avatar
Dominik Hebeler committed
88 89 90 91 92 93 94 95
            if (empty($inputParameter) || empty($filter->sumas->{$name}->values->{$inputParameter})) {
                continue;
            }
            $engineParameterKey = $filter->sumas->{$name}->{"get-parameter"};
            $engineParameterValue = $filter->sumas->{$name}->values->{$inputParameter};
            $this->engine->{"get-parameter"}->{$engineParameterKey} = $engineParameterValue;
        }

96
        $this->getString = $this->generateGetString($q);
Karl Hasselbring's avatar
Karl Hasselbring committed
97
        $this->updateHash();
Dominik Hebeler's avatar
Dominik Hebeler committed
98
        $this->canCache = $metager->canCache();
99 100 101 102
    }

    abstract public function loadResults($result);

103
    # Standardimplementierung der getNext Funktion, damit diese immer verwendet werden kann
104
    public function getNext(MetaGer $metager, $result)
Dominik Hebeler's avatar
Dominik Hebeler committed
105
    {}
106

Karl's avatar
Karl committed
107
    # Prüft, ob die Suche bereits gecached ist, ansonsted wird sie als Job dispatched
108 109
    public function startSearch(\App\MetaGer $metager)
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
110
        if ($this->canCache && Cache::has($this->hash)) {
111
            $this->cached = true;
Dominik Hebeler's avatar
Dominik Hebeler committed
112
            $this->retrieveResults($metager, true);
113
        } else {
114 115 116 117 118
            // We need to submit a action that one of our workers can understand
            // The missions are submitted to a redis queue in the following string format
            // <ResultHash>;<URL to fetch>
            // With <ResultHash> being the Hash Value where the fetcher will store the result.
            // and <URL to fetch> being the full URL to the searchengine
119

120
            $url = "";
121
            if ($this->engine->port === 443) {
122
                $url = "https://";
Dominik Hebeler's avatar
Dominik Hebeler committed
123
            } else {
124 125
                $url = "http://";
            }
126 127 128
            $url .= $this->engine->host;
            if ($this->engine->port !== 80 && $this->engine->port !== 443) {
                $url .= ":" . $this->engine->port;
Dominik Hebeler's avatar
Dominik Hebeler committed
129 130
            }
            $url .= $this->getString;
131

Dominik Hebeler's avatar
Dominik Hebeler committed
132 133 134 135 136 137 138 139 140 141 142
            $mission = [
                "resulthash" => $this->hash,
                "url" => $url,
                "username" => $this->username,
                "password" => $this->password,
                "headers" => $this->headers,
                "cacheDuration" => $this->cacheDuration,
            ];

            $mission = json_encode($mission);

143 144 145
            // Submit this mission to the corresponding Redis Queue
            // Since each Searcher is dedicated to one specific search engine
            // each Searcher has it's own queue lying under the redis key <name>.queue
Dominik Hebeler's avatar
Dominik Hebeler committed
146
            Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
147 148 149 150 151
            // The request is not cached and will be submitted to the searchengine
            // We need to check if the number of requests to this engine are limited
            if (!empty($this->engine->{"monthly-requests"})) {
                Redis::incr("monthlyRequests:" . $this->name);
            }
152 153 154
        }
    }

Karl's avatar
Karl committed
155
    # Ruft die Ranking-Funktion aller Ergebnisse auf.
156
    public function rank($eingabe)
157 158
    {
        foreach ($this->results as $result) {
159
            $result->rank($eingabe);
160 161 162
        }
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
163 164
    public function setResultHash($hash)
    {
165 166 167
        $this->resultHash = $hash;
    }

Karl Hasselbring's avatar
Karl Hasselbring committed
168 169 170 171 172
    public function updateHash()
    {
        $this->hash = md5($this->engine->host . $this->getString . $this->engine->port . $this->name);
    }

Karl's avatar
Karl committed
173
    # Fragt die Ergebnisse von Redis ab und lädt Sie
174
    public function retrieveResults(MetaGer $metager)
175 176 177
    {
        if ($this->loaded) {
            return true;
178 179
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
180 181 182 183 184 185 186 187
        $body = null;
        if ($this->cached) {
            $body = Cache::get($this->hash);
            if ($body === "no-result") {
                $body = "";
            }
        } else {
            $body = Redis::get($this->hash);
188
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
189

Dominik Hebeler's avatar
Dominik Hebeler committed
190
        if ($body !== null) {
191
            $this->loadResults($body);
192
            $this->getNext($metager, $body);
193
            $this->markNew();
194 195 196 197 198 199 200
            $this->loaded = true;
            return true;
        } else {
            return false;
        }
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
201
    public function markNew()
202 203 204 205 206 207
    {
        foreach ($this->results as $result) {
            $result->new = $this->new;
        }
    }

Karl's avatar
Karl committed
208
    # Erstellt den für die Get-Anfrage genutzten String
209
    protected function generateGetString($query)
210 211 212 213
    {
        $getString = "";

        # Skript:
214 215
        if (!empty($this->engine->path)) {
            $getString .= $this->engine->path;
216 217 218 219
        } else {
            $getString .= "/";
        }

220 221 222 223
        $getString .= "?";
        $parameter = [];
        foreach ($this->engine->{"get-parameter"} as $key => $value) {
            $parameter[] = $this->urlEncode($key) . "=" . $this->urlEncode($value);
224
        }
225
        $getString .= implode("&", $parameter);
226

227 228
        # Append the Query String
        $getString .= "&" . $this->engine->{"query-parameter"} . "=" . $this->urlEncode($query);
229

Karl Hasselbring's avatar
Karl Hasselbring committed
230 231
        $getString .= $this->getDynamicParamsString();

232 233 234
        return $getString;
    }

Karl's avatar
Karl committed
235
    # Wandelt einen String nach aktuell gesetztem inputEncoding dieser Searchengine in URL-Format um
236 237 238 239 240 241 242 243
    protected function urlEncode($string)
    {
        if (isset($this->inputEncoding)) {
            return urlencode(mb_convert_encoding($string, $this->inputEncoding));
        } else {
            return urlencode($string);
        }
    }
Karl Hasselbring's avatar
Karl Hasselbring committed
244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260

    private function getDynamicParamsString()
    {
        $paramString = "";

        $params = $this->getDynamicParams();
        foreach ($params as $key => $value) {
            $paramString .= sprintf("&%s=%s", urlencode($key), urlencode($value));
        }

        return $paramString;
    }

    protected function getDynamicParams()
    {
        return [];
    }
261 262 263 264 265

    public function setNew($new)
    {
        $this->new = $new;
    }
266
}