RequestFetcher.php 6.02 KB
Newer Older
1
2
3
4
5
6
<?php

namespace App\Console\Commands;

use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
Dominik Hebeler's avatar
Dominik Hebeler committed
7
use Log;
8

Dominik Hebeler's avatar
Dominik Hebeler committed
9
class RequestFetcher extends Command
10
11
12
13
14
15
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
Dominik Hebeler's avatar
Dominik Hebeler committed
16
    protected $signature = 'requests:fetcher';
17
18
19
20
21
22

    /**
     * The console command description.
     *
     * @var string
     */
Dominik Hebeler's avatar
Dominik Hebeler committed
23
    protected $description = 'This commands fetches requests to the installed search engines';
24
25

    protected $shouldRun = true;
Dominik Hebeler's avatar
Dominik Hebeler committed
26
27
    protected $multicurl = null;
    protected $proxyhost, $proxyuser, $proxypassword;
28
29
30
31
32
33
34
35
36

    /**
     * Create a new command instance.
     *
     * @return void
     */
    public function __construct()
    {
        parent::__construct();
Dominik Hebeler's avatar
Dominik Hebeler committed
37
38
39
40
41
42
        $this->multicurl = curl_multi_init();
        $this->proxyhost = env("PROXY_HOST", "");
        $this->proxyport = env("PROXY_PORT", "");
        $this->proxyuser = env("PROXY_USER", "");
        $this->proxypassword = env("PROXY_PASSWORD", "");

43
44
45
46
47
48
49
50
51
    }

    /**
     * Execute the console command.
     *
     * @return mixed
     */
    public function handle()
    {
52

53
        $pidFile = "/tmp/fetcher";
54
55
56
        pcntl_signal(SIGINT, [$this, "sig_handler"]);
        pcntl_signal(SIGTERM, [$this, "sig_handler"]);
        pcntl_signal(SIGHUP, [$this, "sig_handler"]);
57

58
59
60
61
62
63
64
65
66
67
68
69
70
71
        // Redis might not be available now
        for ($count = 0; $count < 10; $count++) {
            try {
                Redis::connection();
                break;
            } catch (\Predis\Connection\ConnectionException $e) {
                if ($count >= 9) {
                    // If its not available after 10 seconds we will exit
                    return;
                }
                sleep(1);
            }
        }

72
73
74
75
76
77
        touch($pidFile);

        if (!file_exists($pidFile)) {
            return;
        }

78
        try {
Dominik Hebeler's avatar
Dominik Hebeler committed
79
            $blocking = false;
80
            while ($this->shouldRun) {
Dominik Hebeler's avatar
Dominik Hebeler committed
81
82
83
                $status = curl_multi_exec($this->multicurl, $active);
                $currentJob = null;
                if (!$blocking) {
84
                    $currentJob = Redis::lpop(\App\MetaGer::FETCHQUEUE_KEY);
Dominik Hebeler's avatar
Dominik Hebeler committed
85
                } else {
86
                    $currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
Dominik Hebeler's avatar
Dominik Hebeler committed
87
88
                    if (!empty($currentJob)) {
                        $currentJob = $currentJob[1];
89
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
                }

                if (!empty($currentJob)) {
                    $currentJob = json_decode($currentJob, true);
                    $ch = $this->getCurlHandle($currentJob);
                    curl_multi_add_handle($this->multicurl, $ch);
                    $blocking = false;
                    $active = true;
                }

                $answerRead = false;
                while (($info = curl_multi_info_read($this->multicurl)) !== false) {
                    $answerRead = true;
                    $infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE);
                    $infos = explode(";", $infos);
                    $resulthash = $infos[0];
106
                    $cacheDurationMinutes = intval($infos[1]);
Dominik Hebeler's avatar
Dominik Hebeler committed
107
108
109
110
111
112
                    $responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
                    $body = "";

                    $error = curl_error($info["handle"]);
                    if (!empty($error)) {
                        Log::error($error);
113
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
114
115
116
117
118

                    if ($responseCode !== 200) {
                        Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
                    } else {
                        $body = \curl_multi_getcontent($info["handle"]);
119
                    }
120

121
122
123
124
                    Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
                        $pipe->set($resulthash, $body);
                        $pipe->expire($resulthash, 60);
                    });
Dominik Hebeler's avatar
Dominik Hebeler committed
125
                    \curl_multi_remove_handle($this->multicurl, $info["handle"]);
126
                }
Dominik Hebeler's avatar
Dominik Hebeler committed
127
128
                if (!$active && !$answerRead) {
                    $blocking = true;
129
130
                } else {
                    usleep(50 * 1000);
131
132
133
                }
            }
        } finally {
134
            unlink($pidFile);
Dominik Hebeler's avatar
Dominik Hebeler committed
135
            curl_multi_close($this->multicurl);
136
137
138
        }
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
139
    private function getCurlHandle($job)
140
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
141
142
143
144
145
146
147
148
        $ch = curl_init();

        curl_setopt_array($ch, array(
            CURLOPT_URL => $job["url"],
            CURLOPT_PRIVATE => $job["resulthash"] . ";" . $job["cacheDuration"],
            CURLOPT_RETURNTRANSFER => 1,
            CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
            CURLOPT_FOLLOWLOCATION => true,
Dominik Hebeler's avatar
Dominik Hebeler committed
149
            CURLOPT_CONNECTTIMEOUT => 2,
Dominik Hebeler's avatar
Dominik Hebeler committed
150
            CURLOPT_MAXCONNECTS => 500,
Dominik Hebeler's avatar
Dominik Hebeler committed
151
152
153
            CURLOPT_LOW_SPEED_LIMIT => 50000,
            CURLOPT_LOW_SPEED_TIME => 2,
            CURLOPT_TIMEOUT => 3,
Dominik Hebeler's avatar
Dominik Hebeler committed
154
155
156
157
158
159
        ));

        if (!empty($this->proxyhost) && !empty($this->proxyport) && !empty($this->proxyuser) && !empty($this->proxypassword)) {
            curl_setopt($ch, CURLOPT_PROXY, $this->proxyhost);
            curl_setopt($ch, CURLOPT_PROXYUSERPWD, $this->proxyuser . ":" . $this->proxypassword);
            curl_setopt($ch, CURLOPT_PROXYPORT, $this->proxyport);
160
            curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
Dominik Hebeler's avatar
Dominik Hebeler committed
161
162
163
164
165
166
        }

        if (!empty($job["username"]) && !empty($job["password"])) {
            curl_setopt($ch, CURLOPT_USERPWD, $job["username"] . ":" . $job["password"]);
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
167
        if (!empty($job["headers"]) && sizeof($job["headers"]) > 0) {
Dominik Hebeler's avatar
Dominik Hebeler committed
168
169
170
171
172
173
174
            $headers = [];
            foreach ($job["headers"] as $key => $value) {
                $headers[] = $key . ":" . $value;
            }
            # Headers are in the Form:
            # <key>:<value>;<key>:<value>
            curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
175
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
176
177

        return $ch;
178
179
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
180
    public function sig_handler($sig)
181
182
183
184
185
186
    {
        $this->shouldRun = false;
        echo ("Terminating Process\n");
    }

}