RequestFetcher.php 6 KB
Newer Older
1
2
3
4
<?php

namespace App\Console\Commands;

Dominik Hebeler's avatar
Dominik Hebeler committed
5
use Artisan;
6
7
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
Dominik Hebeler's avatar
Dominik Hebeler committed
8
use Log;
9

Dominik Hebeler's avatar
Dominik Hebeler committed
10
class RequestFetcher extends Command
11
12
13
14
15
16
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
Dominik Hebeler's avatar
Dominik Hebeler committed
17
    protected $signature = 'requests:fetcher';
18
19
20
21
22
23

    /**
     * The console command description.
     *
     * @var string
     */
Dominik Hebeler's avatar
Dominik Hebeler committed
24
    protected $description = 'This commands fetches requests to the installed search engines';
25
26

    protected $shouldRun = true;
Dominik Hebeler's avatar
Dominik Hebeler committed
27
28
    protected $multicurl = null;
    protected $proxyhost, $proxyuser, $proxypassword;
29
30
31
32
33
34
35
36
37

    /**
     * Create a new command instance.
     *
     * @return void
     */
    public function __construct()
    {
        parent::__construct();
Dominik Hebeler's avatar
Dominik Hebeler committed
38
39
40
41
42
43
        $this->multicurl = curl_multi_init();
        $this->proxyhost = env("PROXY_HOST", "");
        $this->proxyport = env("PROXY_PORT", "");
        $this->proxyuser = env("PROXY_USER", "");
        $this->proxypassword = env("PROXY_PASSWORD", "");

44
45
46
47
48
49
50
51
52
    }

    /**
     * Execute the console command.
     *
     * @return mixed
     */
    public function handle()
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
53
54
55
56
57
58
59
60
61
62
        $pid = \pcntl_fork();
        if ($pid === 0) {
            Artisan::call('requests:cacher');
            exit;
        } else {
            pcntl_async_signals(true);
            pcntl_signal(SIGINT, [$this, "sig_handler"]);
            pcntl_signal(SIGTERM, [$this, "sig_handler"]);
            pcntl_signal(SIGHUP, [$this, "sig_handler"]);
        }
63
64

        try {
Dominik Hebeler's avatar
Dominik Hebeler committed
65
            $blocking = false;
66
            while ($this->shouldRun) {
Dominik Hebeler's avatar
Dominik Hebeler committed
67
68
69
70
71
                $status = curl_multi_exec($this->multicurl, $active);
                $currentJob = null;
                if (!$blocking) {
                    $currentJob = Redis::lpop(\App\MetaGer::FETCHQUEUE_KEY);
                } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
72
                    $currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
Dominik Hebeler's avatar
Dominik Hebeler committed
73
74
                    if (!empty($currentJob)) {
                        $currentJob = $currentJob[1];
75
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
                }

                if (!empty($currentJob)) {
                    $currentJob = json_decode($currentJob, true);
                    $ch = $this->getCurlHandle($currentJob);
                    curl_multi_add_handle($this->multicurl, $ch);
                    $blocking = false;
                    $active = true;
                }

                $answerRead = false;
                while (($info = curl_multi_info_read($this->multicurl)) !== false) {
                    $answerRead = true;
                    $infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE);
                    $infos = explode(";", $infos);
                    $resulthash = $infos[0];
                    $cacheDuration = intval($infos[1]);
                    $responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
                    $body = "";

                    $error = curl_error($info["handle"]);
                    if (!empty($error)) {
                        Log::error($error);
99
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
100
101
102
103
104

                    if ($responseCode !== 200) {
                        Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
                    } else {
                        $body = \curl_multi_getcontent($info["handle"]);
105
                    }
106

Dominik Hebeler's avatar
Dominik Hebeler committed
107
                    Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDuration) {
108
109
                        $pipe->set($resulthash, $body);
                        $pipe->expire($resulthash, 60);
Dominik Hebeler's avatar
Dominik Hebeler committed
110
111
112
113
114
115
                        $cacherItem = [
                            'cacheDuration' => $cacheDuration,
                            'hash' => $resulthash,
                            'body' => $body,
                        ];
                        $pipe->rpush(\App\Console\Commands\RequestCacher::CACHER_QUEUE, json_encode($cacherItem));
116
117
                    });
                    #Cache::put($resulthash, $body, now()->addMinutes($cacheDuration));
Dominik Hebeler's avatar
Dominik Hebeler committed
118
                    \curl_multi_remove_handle($this->multicurl, $info["handle"]);
119
                }
Dominik Hebeler's avatar
Dominik Hebeler committed
120
121
                if (!$active && !$answerRead) {
                    $blocking = true;
122
123
124
                }
            }
        } finally {
Dominik Hebeler's avatar
Dominik Hebeler committed
125
            curl_multi_close($this->multicurl);
126
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
127
128

        \pcntl_waitpid($pid, $status, WNOHANG);
129
130
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
131
    private function getCurlHandle($job)
132
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
        $ch = curl_init();

        curl_setopt_array($ch, array(
            CURLOPT_URL => $job["url"],
            CURLOPT_PRIVATE => $job["resulthash"] . ";" . $job["cacheDuration"],
            CURLOPT_RETURNTRANSFER => 1,
            CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_CONNECTTIMEOUT => 10,
            CURLOPT_MAXCONNECTS => 500,
            CURLOPT_LOW_SPEED_LIMIT => 500,
            CURLOPT_LOW_SPEED_TIME => 5,
            CURLOPT_TIMEOUT => 10,
        ));

        if (!empty($this->proxyhost) && !empty($this->proxyport) && !empty($this->proxyuser) && !empty($this->proxypassword)) {
            curl_setopt($ch, CURLOPT_PROXY, $this->proxyhost);
            curl_setopt($ch, CURLOPT_PROXYUSERPWD, $this->proxyuser . ":" . $this->proxypassword);
            curl_setopt($ch, CURLOPT_PROXYPORT, $this->proxyport);
            curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
        }

        if (!empty($job["username"]) && !empty($job["password"])) {
            curl_setopt($ch, CURLOPT_USERPWD, $job["username"] . ":" . $job["password"]);
        }

        if (!empty($job["headers"])) {
            $headers = [];
            foreach ($job["headers"] as $key => $value) {
                $headers[] = $key . ":" . $value;
            }
            # Headers are in the Form:
            # <key>:<value>;<key>:<value>
            curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
167
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
168
169

        return $ch;
170
171
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
172
    public function sig_handler($sig)
173
174
175
176
177
178
    {
        $this->shouldRun = false;
        echo ("Terminating Process\n");
    }

}