added reinitialization of multicurl after n requests

DockerfileDev

@@ -23,7 +23,6 @@ RUN apk add --update \
     php7-gd \
     php7-json \
     php7-pcntl \
-    php7-opcache \
     php7-fileinfo \
     && rm -rf /var/cache/apk/*
 
@@ -46,14 +45,6 @@ RUN sed -i 's/;error_log = log\/php7\/error.log/error_log = \/dev\/stderr/g' /et
     sed -i 's/group = www-data/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
     sed -i 's/;cgi.fix_pathinfo=1/cgi.fix_pathinfo=0/g' /etc/php7/php.ini && \
     sed -i 's/expose_php = On/expose_php = Off/g' /etc/php7/php.ini && \
-    # Opcache configuration
-    sed -i 's/;opcache.enable=1/opcache.enable=1/g' /etc/php7/php.ini && \
-    sed -i 's/;opcache.memory_consumption=128/opcache.memory_consumption=128/g' /etc/php7/php.ini && \
-    sed -i 's/;opcache.interned_strings_buffer=8/opcache.interned_strings_buffer=8/g' /etc/php7/php.ini && \
-    sed -i 's/;opcache.max_accelerated_files=10000/opcache.max_accelerated_files=10000/g' /etc/php7/php.ini && \
-    sed -i 's/;opcache.max_wasted_percentage=5/opcache.max_wasted_percentage=5/g' /etc/php7/php.ini && \
-    sed -i 's/;opcache.validate_timestamps=1/opcache.validate_timestamps=1/g' /etc/php7/php.ini && \
-    sed -i 's/;opcache.revalidate_freq=2/opcache.revalidate_freq=300/g' /etc/php7/php.ini && \
     echo "daemonize yes" >> /etc/redis.conf && \
     ln -s /dev/null /var/log/nginx/access.log && \
     ln -s /dev/stdout /var/log/nginx/error.log && \

app/Console/Commands/RequestFetcher.php

@@ -24,6 +24,9 @@ class RequestFetcher extends Command
 
     protected $shouldRun = true;
     protected $multicurl = null;
+    protected $oldMultiCurl = null;
+    protected $maxFetchedDocuments = 1000;
+    protected $fetchedDocuments = 0;
     protected $proxyhost, $proxyuser, $proxypassword;
 
     /**
@@ -93,37 +96,21 @@ class RequestFetcher extends Command
                     $currentJob = json_decode($currentJob, true);
                     $ch = $this->getCurlHandle($currentJob);
                     curl_multi_add_handle($this->multicurl, $ch);
+                    $this->fetchedDocuments++;
+                    if ($this->fetchedDocuments > $this->maxFetchedDocuments) {
+                        Log::info("Reinitializing Multicurl after " . $this->fetchedDocuments . " requests.");
+                        $this->oldMultiCurl = $this->multicurl;
+                        $this->multicurl = curl_multi_init();
+                    }
                     $blocking = false;
                     $active = true;
                 }
 
-                $answerRead = false;
-                while (($info = curl_multi_info_read($this->multicurl)) !== false) {
-                    $answerRead = true;
-                    $infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE);
-                    $infos = explode(";", $infos);
-                    $resulthash = $infos[0];
-                    $cacheDurationMinutes = intval($infos[1]);
-                    $responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
-                    $body = "";
-
-                    $error = curl_error($info["handle"]);
-                    if (!empty($error)) {
-                        Log::error($error);
-                    }
-
-                    if ($responseCode !== 200) {
-                        Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
-                    } else {
-                        $body = \curl_multi_getcontent($info["handle"]);
-                    }
-
-                    Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
-                        $pipe->set($resulthash, $body);
-                        $pipe->expire($resulthash, 60);
-                    });
-                    \curl_multi_remove_handle($this->multicurl, $info["handle"]);
+                $answerRead = $this->readMultiCurl($this->multicurl);
+                if ($this->oldMultiCurl != null) {
+                    $this->readMultiCurl($this->oldMultiCurl);
                 }
+
                 if (!$active && !$answerRead) {
                     $blocking = true;
                 } else {
@@ -136,6 +123,41 @@ class RequestFetcher extends Command
         }
     }
 
+    private function readMultiCurl($mc)
+    {
+        $answerRead = false;
+        while (($info = curl_multi_info_read($mc)) !== false) {
+            try {
+                $answerRead = true;
+                $infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE);
+                $infos = explode(";", $infos);
+                $resulthash = $infos[0];
+                $cacheDurationMinutes = intval($infos[1]);
+                $responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
+                $body = "";
+
+                $error = curl_error($info["handle"]);
+                if (!empty($error)) {
+                    Log::error($error);
+                }
+
+                if ($responseCode !== 200) {
+                    Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
+                } else {
+                    $body = \curl_multi_getcontent($info["handle"]);
+                }
+
+                Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
+                    $pipe->set($resulthash, $body);
+                    $pipe->expire($resulthash, 60);
+                });
+            } finally {
+                \curl_multi_remove_handle($mc, $info["handle"]);
+            }
+        }
+        return $answerRead;
+    }
+
     private function getCurlHandle($job)
     {
         $ch = curl_init();