Merge branch '909-current-development-version-is-too-slow' into 'development'

trying speed without pv

Closes #909

See merge request !1486

app/CacheHelper.php

+<?php
+
+namespace App;
+
+use Illuminate\Support\Facades\Redis;
+
+class CacheHelper
+{
+
+    /**
+     * MetaGer uses a pretty slow harddrive for the configured cache
+     * That's why we have some processes running to write cache to disk in parallel
+     */
+    public static function put($key, $value, $timeSeconds)
+    {
+        $cacherItem = [
+            'timeSeconds' => $timeSeconds,
+            'key' => $key,
+            'value' => $value,
+        ];
+        Redis::rpush(\App\Console\Commands\RequestCacher::CACHER_QUEUE, json_encode($cacherItem));
+
+    }
+}

app/Console/Commands/CacheGC.php

+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+
+class CacheGC extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'cache:gc';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Cleans up every expired cache File';
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return mixed
+     */
+    public function handle()
+    {
+        $cachedir = storage_path('framework/cache');
+
+        $lockfile = $cachedir . "/cache.gc";
+
+        if (file_exists($lockfile)) {
+            return;
+        } else {
+            touch($lockfile);
+        }
+
+        try {
+            foreach (new \DirectoryIterator($cachedir) as $fileInfo) {
+                if ($fileInfo->isDot()) {
+                    continue;
+                }
+                $file = $fileInfo->getPathname();
+                $basename = basename($file);
+                if (!is_dir($file) && $basename !== "cache.gc" && $basename !== ".gitignore") {
+                    $fp = fopen($file, 'r');
+                    $delete = false;
+                    try {
+                        $time = intval(fread($fp, 10));
+                        if ($time < time()) {
+                            $delete = true;
+                        }
+                    } finally {
+                        fclose($fp);
+                    }
+                    if ($delete) {
+                        unlink($file);
+                    }
+                } else if (is_dir($file)) {
+                    // Delete Directory if empty
+                    try {
+                        rmdir($file);
+                    } catch (\ErrorException $e) {
+
+                    }
+                }
+            }
+        } finally {
+            unlink($lockfile);
+        }
+
+    }
+}

app/Console/Commands/RequestCacher.php

+<?php
+
+namespace App\Console\Commands;
+
+use Cache;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Redis;
+
+class RequestCacher extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'requests:cacher';
+
+    const CACHER_QUEUE = 'cacher.queue';
+    protected $shouldRun = true;
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Listens to a buffer of fetched search results and writes them to the filesystem cache.';
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return mixed
+     */
+    public function handle()
+    {
+        pcntl_async_signals(true);
+        pcntl_signal(SIGINT, [$this, "sig_handler"]);
+        pcntl_signal(SIGTERM, [$this, "sig_handler"]);
+        pcntl_signal(SIGHUP, [$this, "sig_handler"]);
+
+        while ($this->shouldRun) {
+            $cacheItem = Redis::blpop(self::CACHER_QUEUE, 1);
+            if (!empty($cacheItem)) {
+                $cacheItem = json_decode($cacheItem[1], true);
+                if (empty($cacheItem["body"])) {
+                    $cacheItem["body"] = "no-result";
+                }
+                Cache::put($cacheItem["hash"], $cacheItem["body"], now()->addMinutes($cacheItem["cacheDuration"]));
+            }
+        }
+    }
+
+    public function sig_handler($sig)
+    {
+        $this->shouldRun = false;
+        echo ("Terminating Cacher Process\n");
+    }
+}

app/Console/Commands/WorkerSpawner.php → app/Console/Commands/RequestFetcher.php

@@ -2,12 +2,12 @@
 
 namespace App\Console\Commands;
 
-use Cache;
+use Artisan;
 use Illuminate\Console\Command;
 use Illuminate\Support\Facades\Redis;
 use Log;
 
-class WorkerSpawner extends Command
+class RequestFetcher extends Command
 {
     /**
      * The name and signature of the console command.
@@ -21,7 +21,7 @@ class WorkerSpawner extends Command
      *
      * @var string
      */
-    protected $description = 'This command makes sure that enough worker processes are spawned';
+    protected $description = 'This commands fetches requests to the installed search engines';
 
     protected $shouldRun = true;
     protected $multicurl = null;
@@ -50,10 +50,24 @@ class WorkerSpawner extends Command
      */
     public function handle()
     {
-        pcntl_async_signals(true);
-        pcntl_signal(SIGINT, [$this, "sig_handler"]);
-        pcntl_signal(SIGTERM, [$this, "sig_handler"]);
-        pcntl_signal(SIGHUP, [$this, "sig_handler"]);
+        $pids = [];
+        $pid = null;
+        for ($i = 0; $i < 5; $i++) {
+            $pid = \pcntl_fork();
+            $pids[] = $pid;
+            if ($pid === 0) {
+                break;
+            }
+        }
+        if ($pid === 0) {
+            Artisan::call('requests:cacher');
+            exit;
+        } else {
+            pcntl_async_signals(true);
+            pcntl_signal(SIGINT, [$this, "sig_handler"]);
+            pcntl_signal(SIGTERM, [$this, "sig_handler"]);
+            pcntl_signal(SIGHUP, [$this, "sig_handler"]);
+        }
 
         try {
             $blocking = false;
@@ -63,7 +77,7 @@ class WorkerSpawner extends Command
                 if (!$blocking) {
                     $currentJob = Redis::lpop(\App\MetaGer::FETCHQUEUE_KEY);
                 } else {
-                    $currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 10);
+                    $currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
                     if (!empty($currentJob)) {
                         $currentJob = $currentJob[1];
                     }
@@ -83,7 +97,7 @@ class WorkerSpawner extends Command
                     $infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE);
                     $infos = explode(";", $infos);
                     $resulthash = $infos[0];
-                    $cacheDuration = intval($infos[1]);
+                    $cacheDurationMinutes = intval($infos[1]);
                     $responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
                     $body = "";
 
@@ -97,7 +111,17 @@ class WorkerSpawner extends Command
                     } else {
                         $body = \curl_multi_getcontent($info["handle"]);
                     }
-                    Cache::put($resulthash, $body, now()->addMinutes($cacheDuration));
+
+                    Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
+                        $pipe->set($resulthash, $body);
+                        $pipe->expire($resulthash, 60);
+                        $cacherItem = [
+                            'timeSeconds' => $cacheDurationMinutes * 60,
+                            'key' => $resulthash,
+                            'value' => $body,
+                        ];
+                        $pipe->rpush(\App\Console\Commands\RequestCacher::CACHER_QUEUE, json_encode($cacherItem));
+                    });
                     \curl_multi_remove_handle($this->multicurl, $info["handle"]);
                 }
                 if (!$active && !$answerRead) {
@@ -107,6 +131,9 @@ class WorkerSpawner extends Command
         } finally {
             curl_multi_close($this->multicurl);
         }
+        foreach ($pids as $tmppid) {
+            \pcntl_waitpid($tmppid, $status, WNOHANG);
+        }
     }
 
     private function getCurlHandle($job)

app/Console/Kernel.php

@@ -27,6 +27,7 @@ class Kernel extends ConsoleKernel
     {
         $schedule->command('requests:gather')->everyFifteenMinutes();
         $schedule->command('requests:useragents')->everyFiveMinutes();
+        $schedule->command('cache:gc')->hourly();
 
         $schedule->call(function () {
             DB::table('monthlyrequests')->truncate();

app/Http/Middleware/HumanVerification.php

@@ -50,7 +50,7 @@ class HumanVerification
 
             # Get all Users of this IP
             $users = Cache::get($prefix . "." . $id, []);
-            $users = $this->removeOldUsers($users);
+            $users = $this->removeOldUsers($prefix, $users);
 
             $user = [];
             if (empty($users[$uid])) {
@@ -148,10 +148,10 @@ class HumanVerification
         // Lock must be acquired within 2 seconds
         $userList = Cache::get($prefix . "." . $user["id"], []);
         $userList[$user["uid"]] = $user;
-        Cache::put($prefix . "." . $user["id"], $userList, now()->addWeeks(2));
+        \App\CacheHelper::put($prefix . "." . $user["id"], $userList, 2 * 7 * 24 * 60 * 60);
     }
 
-    public function removeOldUsers($userList)
+    public function removeOldUsers($prefix, $userList)
     {
         $newUserlist = [];
         $now = now();
@@ -168,10 +168,7 @@ class HumanVerification
         }
 
         if ($changed) {
-            // Lock must be acquired within 2 seconds
-            Cache::lock($prefix . "." . $user["id"])->block(2, function () {
-                Cache::put($prefix . "." . $user["id"], $newUserlist, now()->addWeeks(2));
-            });
+            \App\CacheHelper::put($prefix . "." . $user["id"], $newUserlist, 2 * 7 * 24 * 60 * 60);
         }
 
         return $newUserlist;

app/MetaGer.php

@@ -6,6 +6,7 @@ use App;
 use Cache;
 use Carbon;
 use Illuminate\Http\Request;
+use Illuminate\Support\Facades\Redis;
 use Jenssegers\Agent\Agent;
 use LaravelLocalization;
 use Log;
@@ -322,7 +323,7 @@ class MetaGer
                 'page' => $page,
                 'engines' => $this->next,
             ];
-            Cache::put($this->getSearchUid(), serialize($this->next), 60);
+            \App\CacheHelper::put($this->getSearchUid(), serialize($this->next), 60 * 60);
         } else {
             $this->next = [];
         }
@@ -805,7 +806,7 @@ class MetaGer
         while (sizeof($enginesToWaitFor) > 0 || ($forceTimeout !== null && (microtime(true) - $timeStart) < $forceTimeout)) {
             Log::info(sizeof($enginesToWaitFor) . " " . sizeof($answered) . " " . $enginesToWaitFor[0]->hash);
             foreach ($enginesToWaitFor as $index => $engine) {
-                if (Cache::has($engine->hash)) {
+                if (Redis::get($engine->hash) !== null) {
                     $answered[] = $engine;
                     unset($enginesToWaitFor[$index]);
                     break;

app/Models/Searchengine.php

@@ -102,7 +102,7 @@ abstract class Searchengine
     {
         if ($this->canCache && Cache::has($this->hash)) {
             $this->cached = true;
-            $this->retrieveResults($metager);
+            $this->retrieveResults($metager, true);
         } else {
             // We need to submit a action that one of our workers can understand
             // The missions are submitted to a redis queue in the following string format
@@ -171,19 +171,15 @@ abstract class Searchengine
         }
 
         $body = null;
-
-        if (Cache::has($this->hash)) {
+        if ($this->cached) {
             $body = Cache::get($this->hash);
+            if ($body === "no-result") {
+                $body = "";
+            }
+        } else {
+            $body = Redis::get($this->hash);
         }
-        /*
-        if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash)) {
-        $body = Cache::get($this->hash);
-        } elseif ($redis->hexists($metager->getRedisEngineResult() . $this->name, "response")) {
-        $body = $redis->hget($metager->getRedisEngineResult() . $this->name, "response");
-        if ($this->canCache && $this->cacheDuration > 0) {
-        Cache::put($this->hash, $body, $this->cacheDuration);
-        }
-        }*/
+
         if ($body !== null) {
             $this->loadResults($body);
             $this->getNext($metager, $body);