Resolve "use multi curl to improve performance"

Closes #19 (closed)

app/Console/Commands/RequestFetcher.php

@@ -9,8 +9,8 @@ use Log;
 
 class RequestFetcher extends Command
 {
-    const FETCHQUEUE_KEY = "fetcher.queue";
-    
+    public const FETCHQUEUE_KEY = "fetcher.queue";
+
     /**
      * The name and signature of the console command.
      *
@@ -106,14 +106,14 @@ class RequestFetcher extends Command
     {
         $newJobs = [];
         if ($operationsRunning === 0 && $messagesLeft === -1) {
-            $newJob = Redis::blpop(FETCHQUEUE_KEY, 1);
+            $newJob = Redis::blpop($this::FETCHQUEUE_KEY, 1);
             if (!empty($newJob)) {
                 $newJobs[] = $newJob[1];
             }
         } else {
             $elements = Redis::pipeline(function ($redis) {
-                $redis->lrange(FETCHQUEUE_KEY, 0, -1);
-                $redis->del(FETCHQUEUE_KEY);
+                $redis->lrange($this::FETCHQUEUE_KEY, 0, -1);
+                $redis->del($this::FETCHQUEUE_KEY);
             });
             $newJobs = $elements[0];
         }
@@ -145,7 +145,6 @@ class RequestFetcher extends Command
                 $infos = explode(";", $infos);
                 $resulthash = $infos[0];
                 $cacheDurationMinutes = intval($infos[1]);
-                $name = $infos[2];
                 $responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
                 $body = "no-result";
 
@@ -156,13 +155,8 @@ class RequestFetcher extends Command
                     Log::error($error);
                 }
 
-                if ($responseCode !== 200) {
-                    Log::debug($resulthash);
-                    Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
-                } else {
-                    $body = \curl_multi_getcontent($info["handle"]);
-                }
-
+                $body = \curl_multi_getcontent($info["handle"]);
+                
                 Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
                     $pipe->lpush($resulthash, $body);
                     $pipe->expire($resulthash, 60);
@@ -175,6 +169,8 @@ class RequestFetcher extends Command
                         Log::error($e->getMessage());
                     }
                 }
+            } catch (\Exception $e) {
+                Log::error($e->getMessage());
             } finally {
                 \curl_multi_remove_handle($mc, $info["handle"]);
             }
@@ -188,7 +184,7 @@ class RequestFetcher extends Command
 
         curl_setopt_array($ch, array(
             CURLOPT_URL => $job["url"],
-            CURLOPT_PRIVATE => $job["resulthash"] . ";" . $job["cacheDuration"] . ";" . $job["name"],
+            CURLOPT_PRIVATE => $job["resulthash"] . ";" . $job["cacheDuration"],
             CURLOPT_RETURNTRANSFER => 1,
             CURLOPT_USERAGENT => $job["useragent"],
             CURLOPT_FOLLOWLOCATION => true,
@@ -197,6 +193,7 @@ class RequestFetcher extends Command
             CURLOPT_LOW_SPEED_LIMIT => 50000,
             CURLOPT_LOW_SPEED_TIME => 5,
             CURLOPT_TIMEOUT => 7,
+            CURLOPT_HEADER => true,
         ));
 
         if (!empty($job["curlopts"])) {