RequestFetcher.php 6.12 KB
Newer Older
1
2
3
4
<?php

namespace App\Console\Commands;

5
use Cache;
6
7
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
Dominik Hebeler's avatar
Dominik Hebeler committed
8
use Log;
9

Dominik Hebeler's avatar
Dominik Hebeler committed
10
class RequestFetcher extends Command
11
12
13
14
15
16
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
Dominik Hebeler's avatar
Dominik Hebeler committed
17
    protected $signature = 'requests:fetcher';
18
19
20
21
22
23

    /**
     * The console command description.
     *
     * @var string
     */
Dominik Hebeler's avatar
Dominik Hebeler committed
24
    protected $description = 'This commands fetches requests to the installed search engines';
25
26

    protected $shouldRun = true;
Dominik Hebeler's avatar
Dominik Hebeler committed
27
28
    protected $multicurl = null;
    protected $proxyhost, $proxyuser, $proxypassword;
29
30
31
32
33
34
35
36
37

    /**
     * Create a new command instance.
     *
     * @return void
     */
    public function __construct()
    {
        parent::__construct();
Dominik Hebeler's avatar
Dominik Hebeler committed
38
39
40
41
42
43
        $this->multicurl = curl_multi_init();
        $this->proxyhost = env("PROXY_HOST", "");
        $this->proxyport = env("PROXY_PORT", "");
        $this->proxyuser = env("PROXY_USER", "");
        $this->proxypassword = env("PROXY_PASSWORD", "");

44
45
46
47
48
49
50
51
52
    }

    /**
     * Execute the console command.
     *
     * @return mixed
     */
    public function handle()
    {
53

54
        $pidFile = "/tmp/fetcher";
55
56
57
        pcntl_signal(SIGINT, [$this, "sig_handler"]);
        pcntl_signal(SIGTERM, [$this, "sig_handler"]);
        pcntl_signal(SIGHUP, [$this, "sig_handler"]);
58

59
60
61
62
63
64
65
66
67
68
69
70
71
72
        // Redis might not be available now
        for ($count = 0; $count < 10; $count++) {
            try {
                Redis::connection();
                break;
            } catch (\Predis\Connection\ConnectionException $e) {
                if ($count >= 9) {
                    // If its not available after 10 seconds we will exit
                    return;
                }
                sleep(1);
            }
        }

73
74
75
76
77
78
        touch($pidFile);

        if (!file_exists($pidFile)) {
            return;
        }

79
        try {
Dominik Hebeler's avatar
Dominik Hebeler committed
80
            $blocking = false;
81
            while ($this->shouldRun) {
Dominik Hebeler's avatar
Dominik Hebeler committed
82
83
84
                $status = curl_multi_exec($this->multicurl, $active);
                $currentJob = null;
                if (!$blocking) {
85
                    $currentJob = Redis::lpop(\App\MetaGer::FETCHQUEUE_KEY);
Dominik Hebeler's avatar
Dominik Hebeler committed
86
                } else {
87
                    $currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
Dominik Hebeler's avatar
Dominik Hebeler committed
88
89
                    if (!empty($currentJob)) {
                        $currentJob = $currentJob[1];
90
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
                }

                if (!empty($currentJob)) {
                    $currentJob = json_decode($currentJob, true);
                    $ch = $this->getCurlHandle($currentJob);
                    curl_multi_add_handle($this->multicurl, $ch);
                    $blocking = false;
                    $active = true;
                }

                $answerRead = false;
                while (($info = curl_multi_info_read($this->multicurl)) !== false) {
                    $answerRead = true;
                    $infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE);
                    $infos = explode(";", $infos);
                    $resulthash = $infos[0];
107
                    $cacheDurationMinutes = intval($infos[1]);
Dominik Hebeler's avatar
Dominik Hebeler committed
108
109
110
111
112
113
                    $responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
                    $body = "";

                    $error = curl_error($info["handle"]);
                    if (!empty($error)) {
                        Log::error($error);
114
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
115
116
117
118
119

                    if ($responseCode !== 200) {
                        Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
                    } else {
                        $body = \curl_multi_getcontent($info["handle"]);
120
                    }
121

122
123
124
125
126
                    Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
                        $pipe->set($resulthash, $body);
                        $pipe->expire($resulthash, 60);
                        Cache::put($resulthash, $body, $cacheDurationMinutes * 60);
                    });
Dominik Hebeler's avatar
Dominik Hebeler committed
127
                    \curl_multi_remove_handle($this->multicurl, $info["handle"]);
128
                }
Dominik Hebeler's avatar
Dominik Hebeler committed
129
130
                if (!$active && !$answerRead) {
                    $blocking = true;
131
132
                } else {
                    usleep(50 * 1000);
133
134
135
                }
            }
        } finally {
136
            unlink($pidFile);
Dominik Hebeler's avatar
Dominik Hebeler committed
137
            curl_multi_close($this->multicurl);
138
139
140
        }
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
141
    private function getCurlHandle($job)
142
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
143
144
145
146
147
148
149
150
        $ch = curl_init();

        curl_setopt_array($ch, array(
            CURLOPT_URL => $job["url"],
            CURLOPT_PRIVATE => $job["resulthash"] . ";" . $job["cacheDuration"],
            CURLOPT_RETURNTRANSFER => 1,
            CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
            CURLOPT_FOLLOWLOCATION => true,
Dominik Hebeler's avatar
Dominik Hebeler committed
151
            CURLOPT_CONNECTTIMEOUT => 2,
Dominik Hebeler's avatar
Dominik Hebeler committed
152
            CURLOPT_MAXCONNECTS => 500,
Dominik Hebeler's avatar
Dominik Hebeler committed
153
154
155
            CURLOPT_LOW_SPEED_LIMIT => 50000,
            CURLOPT_LOW_SPEED_TIME => 2,
            CURLOPT_TIMEOUT => 3,
Dominik Hebeler's avatar
Dominik Hebeler committed
156
157
158
159
160
161
        ));

        if (!empty($this->proxyhost) && !empty($this->proxyport) && !empty($this->proxyuser) && !empty($this->proxypassword)) {
            curl_setopt($ch, CURLOPT_PROXY, $this->proxyhost);
            curl_setopt($ch, CURLOPT_PROXYUSERPWD, $this->proxyuser . ":" . $this->proxypassword);
            curl_setopt($ch, CURLOPT_PROXYPORT, $this->proxyport);
162
            curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
Dominik Hebeler's avatar
Dominik Hebeler committed
163
164
165
166
167
168
        }

        if (!empty($job["username"]) && !empty($job["password"])) {
            curl_setopt($ch, CURLOPT_USERPWD, $job["username"] . ":" . $job["password"]);
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
169
        if (!empty($job["headers"]) && sizeof($job["headers"]) > 0) {
Dominik Hebeler's avatar
Dominik Hebeler committed
170
171
172
173
174
175
176
            $headers = [];
            foreach ($job["headers"] as $key => $value) {
                $headers[] = $key . ":" . $value;
            }
            # Headers are in the Form:
            # <key>:<value>;<key>:<value>
            curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
177
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
178
179

        return $ch;
180
181
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
182
    public function sig_handler($sig)
183
184
185
186
187
188
    {
        $this->shouldRun = false;
        echo ("Terminating Process\n");
    }

}