RequestFetcher.php 6.22 KB
Newer Older
1 2 3 4
<?php

namespace App\Console\Commands;

Dominik Hebeler's avatar
Dominik Hebeler committed
5
use Artisan;
6 7
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
Dominik Hebeler's avatar
Dominik Hebeler committed
8
use Log;
9

Dominik Hebeler's avatar
Dominik Hebeler committed
10
class RequestFetcher extends Command
11 12 13 14 15 16
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
Dominik Hebeler's avatar
Dominik Hebeler committed
17
    protected $signature = 'requests:fetcher';
18 19 20 21 22 23

    /**
     * The console command description.
     *
     * @var string
     */
Dominik Hebeler's avatar
Dominik Hebeler committed
24
    protected $description = 'This commands fetches requests to the installed search engines';
25 26

    protected $shouldRun = true;
Dominik Hebeler's avatar
Dominik Hebeler committed
27 28
    protected $multicurl = null;
    protected $proxyhost, $proxyuser, $proxypassword;
29 30 31 32 33 34 35 36 37

    /**
     * Create a new command instance.
     *
     * @return void
     */
    public function __construct()
    {
        parent::__construct();
Dominik Hebeler's avatar
Dominik Hebeler committed
38 39 40 41 42 43
        $this->multicurl = curl_multi_init();
        $this->proxyhost = env("PROXY_HOST", "");
        $this->proxyport = env("PROXY_PORT", "");
        $this->proxyuser = env("PROXY_USER", "");
        $this->proxypassword = env("PROXY_PASSWORD", "");

44 45 46 47 48 49 50 51 52
    }

    /**
     * Execute the console command.
     *
     * @return mixed
     */
    public function handle()
    {
53 54 55 56 57 58 59 60 61
        $pids = [];
        $pid = null;
        for ($i = 0; $i < 5; $i++) {
            $pid = \pcntl_fork();
            $pids[] = $pid;
            if ($pid === 0) {
                break;
            }
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
62 63 64 65 66 67 68 69 70
        if ($pid === 0) {
            Artisan::call('requests:cacher');
            exit;
        } else {
            pcntl_async_signals(true);
            pcntl_signal(SIGINT, [$this, "sig_handler"]);
            pcntl_signal(SIGTERM, [$this, "sig_handler"]);
            pcntl_signal(SIGHUP, [$this, "sig_handler"]);
        }
71 72

        try {
Dominik Hebeler's avatar
Dominik Hebeler committed
73
            $blocking = false;
74
            while ($this->shouldRun) {
Dominik Hebeler's avatar
Dominik Hebeler committed
75 76 77 78 79
                $status = curl_multi_exec($this->multicurl, $active);
                $currentJob = null;
                if (!$blocking) {
                    $currentJob = Redis::lpop(\App\MetaGer::FETCHQUEUE_KEY);
                } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
80
                    $currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
Dominik Hebeler's avatar
Dominik Hebeler committed
81 82
                    if (!empty($currentJob)) {
                        $currentJob = $currentJob[1];
83
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
                }

                if (!empty($currentJob)) {
                    $currentJob = json_decode($currentJob, true);
                    $ch = $this->getCurlHandle($currentJob);
                    curl_multi_add_handle($this->multicurl, $ch);
                    $blocking = false;
                    $active = true;
                }

                $answerRead = false;
                while (($info = curl_multi_info_read($this->multicurl)) !== false) {
                    $answerRead = true;
                    $infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE);
                    $infos = explode(";", $infos);
                    $resulthash = $infos[0];
100
                    $cacheDurationMinutes = intval($infos[1]);
Dominik Hebeler's avatar
Dominik Hebeler committed
101 102 103 104 105 106
                    $responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
                    $body = "";

                    $error = curl_error($info["handle"]);
                    if (!empty($error)) {
                        Log::error($error);
107
                    }
Dominik Hebeler's avatar
Dominik Hebeler committed
108 109 110 111 112

                    if ($responseCode !== 200) {
                        Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
                    } else {
                        $body = \curl_multi_getcontent($info["handle"]);
113
                    }
114

115
                    Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
116 117
                        $pipe->set($resulthash, $body);
                        $pipe->expire($resulthash, 60);
Dominik Hebeler's avatar
Dominik Hebeler committed
118
                        $cacherItem = [
119 120 121
                            'timeSeconds' => $cacheDurationMinutes * 60,
                            'key' => $resulthash,
                            'value' => $body,
Dominik Hebeler's avatar
Dominik Hebeler committed
122
                        ];
123
                        $pipe->rpush(\App\Console\Commands\RequestCacher::CACHER_QUEUE, base64_encode(serialize($cacherItem)));
124
                    });
Dominik Hebeler's avatar
Dominik Hebeler committed
125
                    \curl_multi_remove_handle($this->multicurl, $info["handle"]);
126
                }
Dominik Hebeler's avatar
Dominik Hebeler committed
127 128
                if (!$active && !$answerRead) {
                    $blocking = true;
129 130 131
                }
            }
        } finally {
Dominik Hebeler's avatar
Dominik Hebeler committed
132
            curl_multi_close($this->multicurl);
133
        }
134 135 136
        foreach ($pids as $tmppid) {
            \pcntl_waitpid($tmppid, $status, WNOHANG);
        }
137 138
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
139
    private function getCurlHandle($job)
140
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
        $ch = curl_init();

        curl_setopt_array($ch, array(
            CURLOPT_URL => $job["url"],
            CURLOPT_PRIVATE => $job["resulthash"] . ";" . $job["cacheDuration"],
            CURLOPT_RETURNTRANSFER => 1,
            CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_CONNECTTIMEOUT => 10,
            CURLOPT_MAXCONNECTS => 500,
            CURLOPT_LOW_SPEED_LIMIT => 500,
            CURLOPT_LOW_SPEED_TIME => 5,
            CURLOPT_TIMEOUT => 10,
        ));

        if (!empty($this->proxyhost) && !empty($this->proxyport) && !empty($this->proxyuser) && !empty($this->proxypassword)) {
            curl_setopt($ch, CURLOPT_PROXY, $this->proxyhost);
            curl_setopt($ch, CURLOPT_PROXYUSERPWD, $this->proxyuser . ":" . $this->proxypassword);
            curl_setopt($ch, CURLOPT_PROXYPORT, $this->proxyport);
            curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
        }

        if (!empty($job["username"]) && !empty($job["password"])) {
            curl_setopt($ch, CURLOPT_USERPWD, $job["username"] . ":" . $job["password"]);
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
167
        if (!empty($job["headers"]) && sizeof($job["headers"]) > 0) {
Dominik Hebeler's avatar
Dominik Hebeler committed
168 169 170 171 172 173 174
            $headers = [];
            foreach ($job["headers"] as $key => $value) {
                $headers[] = $key . ":" . $value;
            }
            # Headers are in the Form:
            # <key>:<value>;<key>:<value>
            curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
175
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
176 177

        return $ch;
178 179
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
180
    public function sig_handler($sig)
181 182 183 184 185 186
    {
        $this->shouldRun = false;
        echo ("Terminating Process\n");
    }

}