RequestFetcher.php 6.21 KB
Newer Older
1
2
3
4
<?php

namespace App\Console\Commands;

Dominik Hebeler's avatar
Dominik Hebeler committed
5
use Artisan;
6
7
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
Dominik Hebeler's avatar
Dominik Hebeler committed
8
use Log;
9

Dominik Hebeler's avatar
Dominik Hebeler committed
10
class RequestFetcher extends Command
11
12
13
14
15
16
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
Dominik Hebeler's avatar
Dominik Hebeler committed
17
    protected $signature = 'requests:fetcher';
18
19
20
21
22
23

    /**
     * The console command description.
     *
     * @var string
     */
Dominik Hebeler's avatar
Dominik Hebeler committed
24
    protected $description = 'This commands fetches requests to the installed search engines';
25
26

    protected $shouldRun = true;
Dominik Hebeler's avatar
Dominik Hebeler committed
27
28
    protected $multicurl = null;
    protected $proxyhost, $proxyuser, $proxypassword;
29
30
31
32
33
34
35
36
37

    /**
     * Create a new command instance.
     *
     * @return void
     */
    public function __construct()
    {
        parent::__construct();
Dominik Hebeler's avatar
Dominik Hebeler committed
38
39
40
41
42
43
        $this->multicurl = curl_multi_init();
        $this->proxyhost = env("PROXY_HOST", "");
        $this->proxyport = env("PROXY_PORT", "");
        $this->proxyuser = env("PROXY_USER", "");
        $this->proxypassword = env("PROXY_PASSWORD", "");

44
45
46
47
48
49
50
51
52
    }

    /**
     * Execute the console command.
     *
     * @return mixed
     */
    public function handle()
    {
53
54
55
56
57
58
59
60
61
        $pids = [];
        $pid = null;
        for ($i = 0; $i < 5; $i++) {
            $pid = \pcntl_fork();
            $pids[] = $pid;
            if ($pid === 0) {
                break;
            }
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
62
63
64
65
66
67
68
69
70
        if ($pid === 0) {
            Artisan::call('requests:cacher');
            exit;
        } else {
            pcntl_async_signals(true);
            pcntl_signal(SIGINT, [$this, "sig_handler"]);
            pcntl_signal(SIGTERM, [$this, "sig_handler"]);
            pcntl_signal(SIGHUP, [$this, "sig_handler"]);
        }
71
72

        try {
Dominik Hebeler's avatar
Dominik Hebeler committed
73
            $blocking = false;
74
            while ($this->shouldRun) {
Dominik Hebeler's avatar
Dominik Hebeler committed
75
76
77
78
79
                $status = curl_multi_exec($this->multicurl, $active);
                $currentJob = null;
                if (!$blocking) {
                    $currentJob = Redis::lpop(\App\MetaGer::FETCHQUEUE_KEY);
                } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
80
                    $currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
Dominik Hebeler's avatar
Dominik Hebeler committed
81
82
                    if (!empty($currentJob)) {
                        $currentJob = $currentJob[1];
83
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
                }

                if (!empty($currentJob)) {
                    $currentJob = json_decode($currentJob, true);
                    $ch = $this->getCurlHandle($currentJob);
                    curl_multi_add_handle($this->multicurl, $ch);
                    $blocking = false;
                    $active = true;
                }

                $answerRead = false;
                while (($info = curl_multi_info_read($this->multicurl)) !== false) {
                    $answerRead = true;
                    $infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE);
                    $infos = explode(";", $infos);
                    $resulthash = $infos[0];
100
                    $cacheDurationMinutes = intval($infos[1]);
Dominik Hebeler's avatar
Dominik Hebeler committed
101
102
103
104
105
106
                    $responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
                    $body = "";

                    $error = curl_error($info["handle"]);
                    if (!empty($error)) {
                        Log::error($error);
107
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
108
109
110
111
112

                    if ($responseCode !== 200) {
                        Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
                    } else {
                        $body = \curl_multi_getcontent($info["handle"]);
113
                    }
114

115
                    Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
116
117
                        $pipe->set($resulthash, $body);
                        $pipe->expire($resulthash, 60);
Dominik Hebeler's avatar
Dominik Hebeler committed
118
                        $cacherItem = [
119
120
121
                            'timeSeconds' => $cacheDurationMinutes * 60,
                            'key' => $resulthash,
                            'value' => $body,
Dominik Hebeler's avatar
Dominik Hebeler committed
122
                        ];
123
                        $pipe->rpush(\App\Console\Commands\RequestCacher::CACHER_QUEUE, base64_encode(serialize($cacherItem)));
124
                    });
Dominik Hebeler's avatar
Dominik Hebeler committed
125
                    \curl_multi_remove_handle($this->multicurl, $info["handle"]);
126
                }
Dominik Hebeler's avatar
Dominik Hebeler committed
127
128
                if (!$active && !$answerRead) {
                    $blocking = true;
129
130
131
                }
            }
        } finally {
Dominik Hebeler's avatar
Dominik Hebeler committed
132
            curl_multi_close($this->multicurl);
133
        }
134
135
136
        foreach ($pids as $tmppid) {
            \pcntl_waitpid($tmppid, $status, WNOHANG);
        }
137
138
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
139
    private function getCurlHandle($job)
140
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
        $ch = curl_init();

        curl_setopt_array($ch, array(
            CURLOPT_URL => $job["url"],
            CURLOPT_PRIVATE => $job["resulthash"] . ";" . $job["cacheDuration"],
            CURLOPT_RETURNTRANSFER => 1,
            CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_CONNECTTIMEOUT => 10,
            CURLOPT_MAXCONNECTS => 500,
            CURLOPT_LOW_SPEED_LIMIT => 500,
            CURLOPT_LOW_SPEED_TIME => 5,
            CURLOPT_TIMEOUT => 10,
        ));

        if (!empty($this->proxyhost) && !empty($this->proxyport) && !empty($this->proxyuser) && !empty($this->proxypassword)) {
            curl_setopt($ch, CURLOPT_PROXY, $this->proxyhost);
            curl_setopt($ch, CURLOPT_PROXYUSERPWD, $this->proxyuser . ":" . $this->proxypassword);
            curl_setopt($ch, CURLOPT_PROXYPORT, $this->proxyport);
160
            curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
Dominik Hebeler's avatar
Dominik Hebeler committed
161
162
163
164
165
166
        }

        if (!empty($job["username"]) && !empty($job["password"])) {
            curl_setopt($ch, CURLOPT_USERPWD, $job["username"] . ":" . $job["password"]);
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
167
        if (!empty($job["headers"]) && sizeof($job["headers"]) > 0) {
Dominik Hebeler's avatar
Dominik Hebeler committed
168
169
170
171
172
173
174
            $headers = [];
            foreach ($job["headers"] as $key => $value) {
                $headers[] = $key . ":" . $value;
            }
            # Headers are in the Form:
            # <key>:<value>;<key>:<value>
            curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
175
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
176
177

        return $ch;
178
179
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
180
    public function sig_handler($sig)
181
182
183
184
185
186
    {
        $this->shouldRun = false;
        echo ("Terminating Process\n");
    }

}