RequestFetcher.php 6.24 KB
Newer Older
1
2
3
4
<?php

namespace App\Console\Commands;

Dominik Hebeler's avatar
Dominik Hebeler committed
5
use Artisan;
6
7
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
Dominik Hebeler's avatar
Dominik Hebeler committed
8
use Log;
9

Dominik Hebeler's avatar
Dominik Hebeler committed
10
class RequestFetcher extends Command
11
12
13
14
15
16
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
Dominik Hebeler's avatar
Dominik Hebeler committed
17
    protected $signature = 'requests:fetcher';
18
19
20
21
22
23

    /**
     * The console command description.
     *
     * @var string
     */
Dominik Hebeler's avatar
Dominik Hebeler committed
24
    protected $description = 'This commands fetches requests to the installed search engines';
25
26

    protected $shouldRun = true;
Dominik Hebeler's avatar
Dominik Hebeler committed
27
28
    protected $multicurl = null;
    protected $proxyhost, $proxyuser, $proxypassword;
29
30
31
32
33
34
35
36
37

    /**
     * Create a new command instance.
     *
     * @return void
     */
    public function __construct()
    {
        parent::__construct();
Dominik Hebeler's avatar
Dominik Hebeler committed
38
39
40
41
42
43
        $this->multicurl = curl_multi_init();
        $this->proxyhost = env("PROXY_HOST", "");
        $this->proxyport = env("PROXY_PORT", "");
        $this->proxyuser = env("PROXY_USER", "");
        $this->proxypassword = env("PROXY_PASSWORD", "");

44
45
46
47
48
49
50
51
52
    }

    /**
     * Execute the console command.
     *
     * @return mixed
     */
    public function handle()
    {
53
54
55
56
57
58
59
60
61
        $pids = [];
        $pid = null;
        for ($i = 0; $i < 5; $i++) {
            $pid = \pcntl_fork();
            $pids[] = $pid;
            if ($pid === 0) {
                break;
            }
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
62
63
64
65
66
67
68
69
70
        if ($pid === 0) {
            Artisan::call('requests:cacher');
            exit;
        } else {
            pcntl_async_signals(true);
            pcntl_signal(SIGINT, [$this, "sig_handler"]);
            pcntl_signal(SIGTERM, [$this, "sig_handler"]);
            pcntl_signal(SIGHUP, [$this, "sig_handler"]);
        }
71
72

        try {
Dominik Hebeler's avatar
Dominik Hebeler committed
73
            $blocking = false;
74
            while ($this->shouldRun) {
Dominik Hebeler's avatar
Dominik Hebeler committed
75
76
77
78
79
                $status = curl_multi_exec($this->multicurl, $active);
                $currentJob = null;
                if (!$blocking) {
                    $currentJob = Redis::lpop(\App\MetaGer::FETCHQUEUE_KEY);
                } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
80
                    $currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
Dominik Hebeler's avatar
Dominik Hebeler committed
81
82
                    if (!empty($currentJob)) {
                        $currentJob = $currentJob[1];
83
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
                }

                if (!empty($currentJob)) {
                    $currentJob = json_decode($currentJob, true);
                    $ch = $this->getCurlHandle($currentJob);
                    curl_multi_add_handle($this->multicurl, $ch);
                    $blocking = false;
                    $active = true;
                }

                $answerRead = false;
                while (($info = curl_multi_info_read($this->multicurl)) !== false) {
                    $answerRead = true;
                    $infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE);
                    $infos = explode(";", $infos);
                    $resulthash = $infos[0];
                    $cacheDuration = intval($infos[1]);
                    $responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
                    $body = "";

                    $error = curl_error($info["handle"]);
                    if (!empty($error)) {
                        Log::error($error);
107
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
108
109
110
111
112

                    if ($responseCode !== 200) {
                        Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
                    } else {
                        $body = \curl_multi_getcontent($info["handle"]);
113
                    }
114

Dominik Hebeler's avatar
Dominik Hebeler committed
115
                    Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDuration) {
116
117
                        $pipe->set($resulthash, $body);
                        $pipe->expire($resulthash, 60);
Dominik Hebeler's avatar
Dominik Hebeler committed
118
119
120
121
122
123
                        $cacherItem = [
                            'cacheDuration' => $cacheDuration,
                            'hash' => $resulthash,
                            'body' => $body,
                        ];
                        $pipe->rpush(\App\Console\Commands\RequestCacher::CACHER_QUEUE, json_encode($cacherItem));
124
125
                    });
                    #Cache::put($resulthash, $body, now()->addMinutes($cacheDuration));
Dominik Hebeler's avatar
Dominik Hebeler committed
126
                    \curl_multi_remove_handle($this->multicurl, $info["handle"]);
127
                }
Dominik Hebeler's avatar
Dominik Hebeler committed
128
129
                if (!$active && !$answerRead) {
                    $blocking = true;
130
131
132
                }
            }
        } finally {
Dominik Hebeler's avatar
Dominik Hebeler committed
133
            curl_multi_close($this->multicurl);
134
        }
135
136
137
        foreach ($pids as $tmppid) {
            \pcntl_waitpid($tmppid, $status, WNOHANG);
        }
138
139
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
140
    private function getCurlHandle($job)
141
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
        $ch = curl_init();

        curl_setopt_array($ch, array(
            CURLOPT_URL => $job["url"],
            CURLOPT_PRIVATE => $job["resulthash"] . ";" . $job["cacheDuration"],
            CURLOPT_RETURNTRANSFER => 1,
            CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_CONNECTTIMEOUT => 10,
            CURLOPT_MAXCONNECTS => 500,
            CURLOPT_LOW_SPEED_LIMIT => 500,
            CURLOPT_LOW_SPEED_TIME => 5,
            CURLOPT_TIMEOUT => 10,
        ));

        if (!empty($this->proxyhost) && !empty($this->proxyport) && !empty($this->proxyuser) && !empty($this->proxypassword)) {
            curl_setopt($ch, CURLOPT_PROXY, $this->proxyhost);
            curl_setopt($ch, CURLOPT_PROXYUSERPWD, $this->proxyuser . ":" . $this->proxypassword);
            curl_setopt($ch, CURLOPT_PROXYPORT, $this->proxyport);
            curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
        }

        if (!empty($job["username"]) && !empty($job["password"])) {
            curl_setopt($ch, CURLOPT_USERPWD, $job["username"] . ":" . $job["password"]);
        }

        if (!empty($job["headers"])) {
            $headers = [];
            foreach ($job["headers"] as $key => $value) {
                $headers[] = $key . ":" . $value;
            }
            # Headers are in the Form:
            # <key>:<value>;<key>:<value>
            curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
176
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
177
178

        return $ch;
179
180
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
181
    public function sig_handler($sig)
182
183
184
185
186
187
    {
        $this->shouldRun = false;
        echo ("Terminating Process\n");
    }

}