RequestFetcher.php 6.17 KB
Newer Older
1
2
3
4
<?php

namespace App\Console\Commands;

Dominik Hebeler's avatar
Dominik Hebeler committed
5
use Artisan;
6
7
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
Dominik Hebeler's avatar
Dominik Hebeler committed
8
use Log;
9

Dominik Hebeler's avatar
Dominik Hebeler committed
10
class RequestFetcher extends Command
11
12
13
14
15
16
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
Dominik Hebeler's avatar
Dominik Hebeler committed
17
    protected $signature = 'requests:fetcher';
18
19
20
21
22
23

    /**
     * The console command description.
     *
     * @var string
     */
Dominik Hebeler's avatar
Dominik Hebeler committed
24
    protected $description = 'This commands fetches requests to the installed search engines';
25
26

    protected $shouldRun = true;
Dominik Hebeler's avatar
Dominik Hebeler committed
27
28
    protected $multicurl = null;
    protected $proxyhost, $proxyuser, $proxypassword;
29
30
31
32
33
34
35
36
37

    /**
     * Create a new command instance.
     *
     * @return void
     */
    public function __construct()
    {
        parent::__construct();
Dominik Hebeler's avatar
Dominik Hebeler committed
38
39
40
41
42
43
        $this->multicurl = curl_multi_init();
        $this->proxyhost = env("PROXY_HOST", "");
        $this->proxyport = env("PROXY_PORT", "");
        $this->proxyuser = env("PROXY_USER", "");
        $this->proxypassword = env("PROXY_PASSWORD", "");

44
45
46
47
48
49
50
51
52
    }

    /**
     * Execute the console command.
     *
     * @return mixed
     */
    public function handle()
    {
53
54
55
56
57
58
59
60
61
        $pids = [];
        $pid = null;
        for ($i = 0; $i < 5; $i++) {
            $pid = \pcntl_fork();
            $pids[] = $pid;
            if ($pid === 0) {
                break;
            }
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
62
63
64
65
66
67
68
69
70
        if ($pid === 0) {
            Artisan::call('requests:cacher');
            exit;
        } else {
            pcntl_async_signals(true);
            pcntl_signal(SIGINT, [$this, "sig_handler"]);
            pcntl_signal(SIGTERM, [$this, "sig_handler"]);
            pcntl_signal(SIGHUP, [$this, "sig_handler"]);
        }
71
72

        try {
Dominik Hebeler's avatar
Dominik Hebeler committed
73
            $blocking = false;
74
            while ($this->shouldRun) {
Dominik Hebeler's avatar
Dominik Hebeler committed
75
76
77
78
79
                $status = curl_multi_exec($this->multicurl, $active);
                $currentJob = null;
                if (!$blocking) {
                    $currentJob = Redis::lpop(\App\MetaGer::FETCHQUEUE_KEY);
                } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
80
                    $currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
Dominik Hebeler's avatar
Dominik Hebeler committed
81
82
                    if (!empty($currentJob)) {
                        $currentJob = $currentJob[1];
83
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
                }

                if (!empty($currentJob)) {
                    $currentJob = json_decode($currentJob, true);
                    $ch = $this->getCurlHandle($currentJob);
                    curl_multi_add_handle($this->multicurl, $ch);
                    $blocking = false;
                    $active = true;
                }

                $answerRead = false;
                while (($info = curl_multi_info_read($this->multicurl)) !== false) {
                    $answerRead = true;
                    $infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE);
                    $infos = explode(";", $infos);
                    $resulthash = $infos[0];
100
                    $cacheDurationMinutes = intval($infos[1]);
Dominik Hebeler's avatar
Dominik Hebeler committed
101
102
103
104
105
106
                    $responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
                    $body = "";

                    $error = curl_error($info["handle"]);
                    if (!empty($error)) {
                        Log::error($error);
107
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
108
109
110
111
112

                    if ($responseCode !== 200) {
                        Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
                    } else {
                        $body = \curl_multi_getcontent($info["handle"]);
113
                    }
114

115
                    Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
116
117
                        $pipe->set($resulthash, $body);
                        $pipe->expire($resulthash, 60);
Dominik Hebeler's avatar
Dominik Hebeler committed
118
                        $cacherItem = [
119
120
121
                            'timeSeconds' => $cacheDurationMinutes * 60,
                            'key' => $resulthash,
                            'value' => $body,
Dominik Hebeler's avatar
Dominik Hebeler committed
122
123
                        ];
                        $pipe->rpush(\App\Console\Commands\RequestCacher::CACHER_QUEUE, json_encode($cacherItem));
124
                    });
Dominik Hebeler's avatar
Dominik Hebeler committed
125
                    \curl_multi_remove_handle($this->multicurl, $info["handle"]);
126
                }
Dominik Hebeler's avatar
Dominik Hebeler committed
127
128
                if (!$active && !$answerRead) {
                    $blocking = true;
129
130
131
                }
            }
        } finally {
Dominik Hebeler's avatar
Dominik Hebeler committed
132
            curl_multi_close($this->multicurl);
133
        }
134
135
136
        foreach ($pids as $tmppid) {
            \pcntl_waitpid($tmppid, $status, WNOHANG);
        }
137
138
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
139
    private function getCurlHandle($job)
140
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
        $ch = curl_init();

        curl_setopt_array($ch, array(
            CURLOPT_URL => $job["url"],
            CURLOPT_PRIVATE => $job["resulthash"] . ";" . $job["cacheDuration"],
            CURLOPT_RETURNTRANSFER => 1,
            CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_CONNECTTIMEOUT => 10,
            CURLOPT_MAXCONNECTS => 500,
            CURLOPT_LOW_SPEED_LIMIT => 500,
            CURLOPT_LOW_SPEED_TIME => 5,
            CURLOPT_TIMEOUT => 10,
        ));

        if (!empty($this->proxyhost) && !empty($this->proxyport) && !empty($this->proxyuser) && !empty($this->proxypassword)) {
            curl_setopt($ch, CURLOPT_PROXY, $this->proxyhost);
            curl_setopt($ch, CURLOPT_PROXYUSERPWD, $this->proxyuser . ":" . $this->proxypassword);
            curl_setopt($ch, CURLOPT_PROXYPORT, $this->proxyport);
            curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
        }

        if (!empty($job["username"]) && !empty($job["password"])) {
            curl_setopt($ch, CURLOPT_USERPWD, $job["username"] . ":" . $job["password"]);
        }

        if (!empty($job["headers"])) {
            $headers = [];
            foreach ($job["headers"] as $key => $value) {
                $headers[] = $key . ":" . $value;
            }
            # Headers are in the Form:
            # <key>:<value>;<key>:<value>
            curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
175
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
176
177

        return $ch;
178
179
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
180
    public function sig_handler($sig)
181
182
183
184
185
186
    {
        $this->shouldRun = false;
        echo ("Terminating Process\n");
    }

}