Search.php 11.9 KB
Newer Older
1
2
3
4
<?php

namespace App\Jobs;

5
use Illuminate\Bus\Queueable;
6
use Illuminate\Contracts\Queue\ShouldQueue;
7
8
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
9
use Illuminate\Support\Facades\Redis;
10

11
class Search implements ShouldQueue
12
{
13
    use InteractsWithQueue, Queueable, SerializesModels;
14

Phil Höfer's avatar
Phil Höfer committed
15
    protected $hash, $host, $port, $name, $getString, $useragent, $fp, $additionalHeaders;
16
17
18
19
20
21
22
    protected $buffer_length = 8192;

    /**
     * Create a new job instance.
     *
     * @return void
     */
Phil Höfer's avatar
Phil Höfer committed
23
    public function __construct($hash, $host, $port, $name, $getString, $useragent, $additionalHeaders)
24
    {
Phil Höfer's avatar
Phil Höfer committed
25
26
27
28
29
30
31
        $this->hash              = $hash;
        $this->host              = $host;
        $this->port              = $port;
        $this->name              = $name;
        $this->getString         = $getString;
        $this->useragent         = $useragent;
        $this->additionalHeaders = $additionalHeaders;
32
33
34
35
36
37
38
    }

    /**
     * Execute the job.
     *
     * @return void
     */
39
    public function handle()
40
    {
41
42
43
44
45
        $url = "";
        if($this->port === "443"){
            $url = "https://";
        }else{
            $url = "http://";
46
        }
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
        $url .= $this->host . $this->getString;


        $ch = curl_init($url);
        curl_setopt_array($ch, array(
                CURLOPT_RETURNTRANSFER => 1,
                CURLOPT_URL => $url,
                CURLOPT_USERAGENT => $this->useragent,
                CURLOPT_FOLLOWLOCATION => TRUE,
                CURLOPT_CONNECTTIMEOUT => 10,
                CURLOPT_MAXCONNECTS => 50,
                CURLOPT_LOW_SPEED_LIMIT => 500,
                CURLOPT_LOW_SPEED_TIME => 5,
                CURLOPT_TIMEOUT => 10
        ));

        $result = curl_exec($ch);

        curl_close($ch);
        Redis::hset('search.' . $this->hash, $this->name, $result);
67
68
    }

69
    private function readAnswer()
70
    {
71
        $time    = microtime(true);
72
        $headers = '';
73
74
        $body    = '';
        $length  = 0;
75

76
        if (!$this->fp) {
77
78
79
80
81
82
            return;
        }

        // get headers FIRST
        $c = 0;
        stream_set_blocking($this->fp, 1);
83
        do {
84
85
86
87
            // use fgets() not fread(), fgets stops reading at first newline
            // or buffer which ever one is reached first
            $data = fgets($this->fp, 8192);
            // a sincle CRLF indicates end of headers
88
            if ($data === false || $data == "\r\n" || feof($this->fp)) {
89
90
91
                // break BEFORE OUTPUT
                break;
            }
92
            if (sizeof(($tmp = explode(": ", $data))) === 2) {
93
                $headers[strtolower(trim($tmp[0]))] = trim($tmp[1]);
94
            }
95
            $c++;
96
        } while (true);
97
98

        // end of headers
99
        if (sizeof($headers) > 1) {
100
            $bodySize = 0;
101
            if (isset($headers["transfer-encoding"]) && $headers["transfer-encoding"] === "chunked") {
102
                $body = $this->readChunked();
103
104

            } elseif (isset($headers['content-length'])) {
105
                $length = trim($headers['content-length']);
106
                if (is_numeric($length) && $length >= 1) {
107
                    $body = $this->readBody($length);
108
109
                }

110
                $bodySize = strlen($body);
111
112
113
            } elseif (isset($headers["connection"]) && strtolower($headers["connection"]) === "close") {
                $body = $this->readUntilClose();
            }else {
114
115
                exit;
            }
116
        } else {
117
118
119
120
            return;
        }

        Redis::del($this->host . "." . $this->socketNumber);
121
        if (isset($headers["content-encoding"]) && $headers['content-encoding'] === "gzip") {
122
123
124
125
126
            $body = $this->gunzip($body);
        }
        Redis::hset('search.' . $this->hash, $this->name, $body);
        Redis::expire('search.' . $this->hash, 5);
    }
127
128
129
130
131
132
133
134
135
136
137
138
139
140
    
    private function readUntilClose()
    {
        $data = '';
        stream_set_blocking($this->fp, 1);
        while (!feof($this->fp)) {
            $data .= fgets($this->fp, 8192);
        }
        # Bei dieser Funktion unterstützt der Host kein Keep-Alive:
        # Wir beenden die Verbindung:
        fclose($this->fp);
        Redis::del($this->host . "." . $this->socketNumber);
        return $data;
    }
141
142
143
144

    private function readBody($length)
    {
        $theData = '';
145
        $done    = false;
146
147
        stream_set_blocking($this->fp, 0);
        $startTime = time();
148
149
        $lastTime  = $startTime;
        while (!feof($this->fp) && !$done && (($startTime + 1) > time()) && $length !== 0) {
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
            usleep(100);
            $theNewData = fgets($this->fp, 8192);
            $theData .= $theNewData;
            $length -= strlen($theNewData);
            $done = (trim($theNewData) === '0');

        }
        return $theData;
    }

    private function readChunked()
    {
        $body = '';
        // read from chunked stream
        // loop though the stream
165
        do {
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
            // NOTE: for chunked encoding to work properly make sure
            // there is NOTHING (besides newlines) before the first hexlength

            // get the line which has the length of this chunk (use fgets here)
            $line = fgets($this->fp, 8192);

            // if it's only a newline this normally means it's read
            // the total amount of data requested minus the newline
            // continue to next loop to make sure we're done
            if ($line == "\r\n") {
                continue;
            }

            // the length of the block is sent in hex decode it then loop through
            // that much data get the length
            // NOTE: hexdec() ignores all non hexadecimal chars it finds
            $length = hexdec($line);

            if (!is_int($length)) {
                trigger_error('Most likely not chunked encoding', E_USER_ERROR);
            }

            // zero is sent when at the end of the chunks
            // or the end of the stream or error
            if ($line === false || $length < 1 || feof($this->fp)) {
191
192
193
194
                if ($length <= 0) {
                    fgets($this->fp, 8192);
                }

195
196
197
198
199
                // break out of the streams loop
                break;
            }

            // loop though the chunk
200
            do {
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
                // read $length amount of data
                // (use fread here)
                $data = fread($this->fp, $length);

                // remove the amount received from the total length on the next loop
                // it'll attempt to read that much less data
                $length -= strlen($data);

                // PRINT out directly
                // you could also save it directly to a file here

                // store in string for later use
                $body .= $data;

                // zero or less or end of connection break
216
                if ($length <= 0 || feof($this->fp)) {
217
                    // break out of the chunk loop
218
                    if ($length <= 0) {
219
                        fgets($this->fp, 8192);
220
221
                    }

222
223
                    break;
                }
224
            } while (true);
225
            // end of chunk loop
226
        } while (true);
227
228
229
230
        // end of stream loop
        return $body;
    }

231
232
233
234
235
236
237
238
239
240
241
242
243
244
    private function gunzip($zipped)
    {
        $offset = 0;
        if (substr($zipped, 0, 2) == "\x1f\x8b") {
            $offset = 2;
        }

        if (substr($zipped, $offset, 1) == "\x08") {
            try
            {
                return gzinflate(substr($zipped, $offset + 8));
            } catch (\Exception $e) {
                abort(500, "Fehler beim unzip des Ergebnisses von folgendem Anbieter: " . $this->name);
            }
245
        }
246
247
        return "Unknown Format";
    }
248

249
    private function writeRequest()
250
    {
251

252
253
254
255
256
257
        $out = "GET " . $this->getString . " HTTP/1.1\r\n";
        $out .= "Host: " . $this->host . "\r\n";
        $out .= "User-Agent: " . $this->useragent . "\r\n";
        $out .= "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n";
        $out .= "Accept-Language: de,en-US;q=0.7,en;q=0.3\r\n";
        $out .= "Accept-Encoding: gzip, deflate, br\r\n";
Phil Höfer's avatar
Phil Höfer committed
258
        $out .= str_replace("$#!#$", "\r\n", $this->additionalHeaders);
259
260
        $out .= "Connection: keep-alive\r\n\r\n";
        # Anfrage senden:
261
262
263
264
265
        $sent   = 0;
        $string = $out;
        $time   = microtime(true);
        while (true) {
            $timeElapsed = microtime(true) - $time;
266
            if ($timeElapsed > 1.0) {
267
                # Irgendwas ist mit unserem Socket passiert. Wir brauchen einen neuen:
268
                if ($this->fp && is_resource($this->fp)) {
269
270
271
272
273
274
275
                    fclose($this->fp);
                }

                Redis::del($this->name . "." . $this->socketNumber);
                $this->fp = $this->getFreeSocket();
                $sent     = 0;
                $string   = $out;
276
                continue;
277
278
            }
            try {
279
                $tmp = fwrite($this->fp, $string);
280
            } catch (\ErrorException $e) {
281
                # Irgendwas ist mit unserem Socket passiert. Wir brauchen einen neuen:
282
                if ($this->fp && is_resource($this->fp)) {
283
284
285
                    fclose($this->fp);
                }

286
287
                Redis::del($this->name . "." . $this->socketNumber);
                $this->fp = $this->getFreeSocket();
288
289
                $sent     = 0;
                $string   = $out;
290
291
                continue;
            }
292
            if ($tmp) {
293
294
295
296
                $sent += $tmp;
                $string = substr($string, $tmp);
            }

297
            if ($sent >= strlen($out)) {
298
                break;
299
300
            }

301
        }
302

303
        if ($sent === strlen($out)) {
304
305
            return true;
        }
306

307
308
309
        return false;
    }

310
    public function getFreeSocket()
311
312
    {
        # Je nach Auslastung des Servers ( gleichzeitige Abfragen ), kann es sein, dass wir mehrere Sockets benötigen um die Abfragen ohne Wartezeit beantworten zu können.
313
        # pfsockopen öffnet dabei einen persistenten Socket, der also auch zwischen den verschiedenen php Prozessen geteilt werden kann.
314
315
316
317
318
        # Wenn der Hostname mit einem bereits erstellten Socket übereinstimmt, wird die Verbindung also aufgegriffen und fortgeführt.
        # Allerdings dürfen wir diesen nur verwenden, wenn er nicht bereits von einem anderen Prozess zur Kommunikation verwendet wird.
        # Wenn dem so ist, probieren wir den nächsten Socket zu verwenden.
        # Dies festzustellen ist komplizierter, als man sich das vorstellt. Folgendes System sollte funktionieren:
        # 1. Stelle fest, ob dieser Socket neu erstellt wurde, oder ob ein existierender geöffnet wurde.
319
320
        $counter = 0;
        $fp      = null;
321
        $time    = microtime(true);
322
323
324
        do {

            if (intval(Redis::exists($this->host . ".$counter")) === 0) {
325
326
327
328
329
330
                Redis::set($this->host . ".$counter", 1);
                Redis::expire($this->host . ".$counter", 5);
                $this->socketNumber = $counter;

                try
                {
Dominik Hebeler's avatar
Dominik Hebeler committed
331
                    $fp = pfsockopen($this->getHost(), $this->port, $errstr, $errno, 1);
332
                } catch (\ErrorException $e) {
333
334
335
336
337
                    break;
                }
                # Wir gucken, ob der Lesepuffer leer ist:
                stream_set_blocking($fp, 0);
                $string = fgets($fp, 8192);
338
                if ($string !== false || feof($fp)) {
339
340
341
                    if ($this->fp && is_resource($this->fp)) {
                        fclose($fp);
                    }
342
343
                    $this->socketNumber = null;
                    Redis::del($this->host . ".$counter");
344
345
346
347
348
                    continue;
                }
                break;
            }
            $counter++;
349
        } while (true);
350
351
352
        return $fp;
    }

353
    public function getHost()
354
355
    {
        $return = "";
356
        if ($this->port === "443") {
357
            $return .= "tls://";
358
        } else {
359
360
361
362
363
364
            $return .= "tcp://";
        }
        $return .= $this->host;
        return $return;
    }
}