Commit f3ef0a97 authored by Dominik Hebeler's avatar Dominik Hebeler

Merge branch '909-current-development-version-is-too-slow' into 'development'

trying speed without pv

Closes #909

See merge request !1486
parents 5b27c39b ea99a7a5
<?php
namespace App;
use Illuminate\Support\Facades\Redis;
class CacheHelper
{
/**
* MetaGer uses a pretty slow harddrive for the configured cache
* That's why we have some processes running to write cache to disk in parallel
*/
public static function put($key, $value, $timeSeconds)
{
$cacherItem = [
'timeSeconds' => $timeSeconds,
'key' => $key,
'value' => $value,
];
Redis::rpush(\App\Console\Commands\RequestCacher::CACHER_QUEUE, json_encode($cacherItem));
}
}
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
class CacheGC extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'cache:gc';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Cleans up every expired cache File';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$cachedir = storage_path('framework/cache');
$lockfile = $cachedir . "/cache.gc";
if (file_exists($lockfile)) {
return;
} else {
touch($lockfile);
}
try {
foreach (new \DirectoryIterator($cachedir) as $fileInfo) {
if ($fileInfo->isDot()) {
continue;
}
$file = $fileInfo->getPathname();
$basename = basename($file);
if (!is_dir($file) && $basename !== "cache.gc" && $basename !== ".gitignore") {
$fp = fopen($file, 'r');
$delete = false;
try {
$time = intval(fread($fp, 10));
if ($time < time()) {
$delete = true;
}
} finally {
fclose($fp);
}
if ($delete) {
unlink($file);
}
} else if (is_dir($file)) {
// Delete Directory if empty
try {
rmdir($file);
} catch (\ErrorException $e) {
}
}
}
} finally {
unlink($lockfile);
}
}
}
<?php
namespace App\Console\Commands;
use Cache;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
class RequestCacher extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'requests:cacher';
const CACHER_QUEUE = 'cacher.queue';
protected $shouldRun = true;
/**
* The console command description.
*
* @var string
*/
protected $description = 'Listens to a buffer of fetched search results and writes them to the filesystem cache.';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
pcntl_async_signals(true);
pcntl_signal(SIGINT, [$this, "sig_handler"]);
pcntl_signal(SIGTERM, [$this, "sig_handler"]);
pcntl_signal(SIGHUP, [$this, "sig_handler"]);
while ($this->shouldRun) {
$cacheItem = Redis::blpop(self::CACHER_QUEUE, 1);
if (!empty($cacheItem)) {
$cacheItem = json_decode($cacheItem[1], true);
if (empty($cacheItem["body"])) {
$cacheItem["body"] = "no-result";
}
Cache::put($cacheItem["hash"], $cacheItem["body"], now()->addMinutes($cacheItem["cacheDuration"]));
}
}
}
public function sig_handler($sig)
{
$this->shouldRun = false;
echo ("Terminating Cacher Process\n");
}
}
...@@ -2,12 +2,12 @@ ...@@ -2,12 +2,12 @@
namespace App\Console\Commands; namespace App\Console\Commands;
use Cache; use Artisan;
use Illuminate\Console\Command; use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis; use Illuminate\Support\Facades\Redis;
use Log; use Log;
class WorkerSpawner extends Command class RequestFetcher extends Command
{ {
/** /**
* The name and signature of the console command. * The name and signature of the console command.
...@@ -21,7 +21,7 @@ class WorkerSpawner extends Command ...@@ -21,7 +21,7 @@ class WorkerSpawner extends Command
* *
* @var string * @var string
*/ */
protected $description = 'This command makes sure that enough worker processes are spawned'; protected $description = 'This commands fetches requests to the installed search engines';
protected $shouldRun = true; protected $shouldRun = true;
protected $multicurl = null; protected $multicurl = null;
...@@ -50,10 +50,24 @@ class WorkerSpawner extends Command ...@@ -50,10 +50,24 @@ class WorkerSpawner extends Command
*/ */
public function handle() public function handle()
{ {
pcntl_async_signals(true); $pids = [];
pcntl_signal(SIGINT, [$this, "sig_handler"]); $pid = null;
pcntl_signal(SIGTERM, [$this, "sig_handler"]); for ($i = 0; $i < 5; $i++) {
pcntl_signal(SIGHUP, [$this, "sig_handler"]); $pid = \pcntl_fork();
$pids[] = $pid;
if ($pid === 0) {
break;
}
}
if ($pid === 0) {
Artisan::call('requests:cacher');
exit;
} else {
pcntl_async_signals(true);
pcntl_signal(SIGINT, [$this, "sig_handler"]);
pcntl_signal(SIGTERM, [$this, "sig_handler"]);
pcntl_signal(SIGHUP, [$this, "sig_handler"]);
}
try { try {
$blocking = false; $blocking = false;
...@@ -63,7 +77,7 @@ class WorkerSpawner extends Command ...@@ -63,7 +77,7 @@ class WorkerSpawner extends Command
if (!$blocking) { if (!$blocking) {
$currentJob = Redis::lpop(\App\MetaGer::FETCHQUEUE_KEY); $currentJob = Redis::lpop(\App\MetaGer::FETCHQUEUE_KEY);
} else { } else {
$currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 10); $currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
if (!empty($currentJob)) { if (!empty($currentJob)) {
$currentJob = $currentJob[1]; $currentJob = $currentJob[1];
} }
...@@ -83,7 +97,7 @@ class WorkerSpawner extends Command ...@@ -83,7 +97,7 @@ class WorkerSpawner extends Command
$infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE); $infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE);
$infos = explode(";", $infos); $infos = explode(";", $infos);
$resulthash = $infos[0]; $resulthash = $infos[0];
$cacheDuration = intval($infos[1]); $cacheDurationMinutes = intval($infos[1]);
$responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE); $responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
$body = ""; $body = "";
...@@ -97,7 +111,17 @@ class WorkerSpawner extends Command ...@@ -97,7 +111,17 @@ class WorkerSpawner extends Command
} else { } else {
$body = \curl_multi_getcontent($info["handle"]); $body = \curl_multi_getcontent($info["handle"]);
} }
Cache::put($resulthash, $body, now()->addMinutes($cacheDuration));
Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
$pipe->set($resulthash, $body);
$pipe->expire($resulthash, 60);
$cacherItem = [
'timeSeconds' => $cacheDurationMinutes * 60,
'key' => $resulthash,
'value' => $body,
];
$pipe->rpush(\App\Console\Commands\RequestCacher::CACHER_QUEUE, json_encode($cacherItem));
});
\curl_multi_remove_handle($this->multicurl, $info["handle"]); \curl_multi_remove_handle($this->multicurl, $info["handle"]);
} }
if (!$active && !$answerRead) { if (!$active && !$answerRead) {
...@@ -107,6 +131,9 @@ class WorkerSpawner extends Command ...@@ -107,6 +131,9 @@ class WorkerSpawner extends Command
} finally { } finally {
curl_multi_close($this->multicurl); curl_multi_close($this->multicurl);
} }
foreach ($pids as $tmppid) {
\pcntl_waitpid($tmppid, $status, WNOHANG);
}
} }
private function getCurlHandle($job) private function getCurlHandle($job)
......
...@@ -27,6 +27,7 @@ class Kernel extends ConsoleKernel ...@@ -27,6 +27,7 @@ class Kernel extends ConsoleKernel
{ {
$schedule->command('requests:gather')->everyFifteenMinutes(); $schedule->command('requests:gather')->everyFifteenMinutes();
$schedule->command('requests:useragents')->everyFiveMinutes(); $schedule->command('requests:useragents')->everyFiveMinutes();
$schedule->command('cache:gc')->hourly();
$schedule->call(function () { $schedule->call(function () {
DB::table('monthlyrequests')->truncate(); DB::table('monthlyrequests')->truncate();
......
...@@ -50,7 +50,7 @@ class HumanVerification ...@@ -50,7 +50,7 @@ class HumanVerification
# Get all Users of this IP # Get all Users of this IP
$users = Cache::get($prefix . "." . $id, []); $users = Cache::get($prefix . "." . $id, []);
$users = $this->removeOldUsers($users); $users = $this->removeOldUsers($prefix, $users);
$user = []; $user = [];
if (empty($users[$uid])) { if (empty($users[$uid])) {
...@@ -148,10 +148,10 @@ class HumanVerification ...@@ -148,10 +148,10 @@ class HumanVerification
// Lock must be acquired within 2 seconds // Lock must be acquired within 2 seconds
$userList = Cache::get($prefix . "." . $user["id"], []); $userList = Cache::get($prefix . "." . $user["id"], []);
$userList[$user["uid"]] = $user; $userList[$user["uid"]] = $user;
Cache::put($prefix . "." . $user["id"], $userList, now()->addWeeks(2)); \App\CacheHelper::put($prefix . "." . $user["id"], $userList, 2 * 7 * 24 * 60 * 60);
} }
public function removeOldUsers($userList) public function removeOldUsers($prefix, $userList)
{ {
$newUserlist = []; $newUserlist = [];
$now = now(); $now = now();
...@@ -168,10 +168,7 @@ class HumanVerification ...@@ -168,10 +168,7 @@ class HumanVerification
} }
if ($changed) { if ($changed) {
// Lock must be acquired within 2 seconds \App\CacheHelper::put($prefix . "." . $user["id"], $newUserlist, 2 * 7 * 24 * 60 * 60);
Cache::lock($prefix . "." . $user["id"])->block(2, function () {
Cache::put($prefix . "." . $user["id"], $newUserlist, now()->addWeeks(2));
});
} }
return $newUserlist; return $newUserlist;
......
...@@ -6,6 +6,7 @@ use App; ...@@ -6,6 +6,7 @@ use App;
use Cache; use Cache;
use Carbon; use Carbon;
use Illuminate\Http\Request; use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
use Jenssegers\Agent\Agent; use Jenssegers\Agent\Agent;
use LaravelLocalization; use LaravelLocalization;
use Log; use Log;
...@@ -322,7 +323,7 @@ class MetaGer ...@@ -322,7 +323,7 @@ class MetaGer
'page' => $page, 'page' => $page,
'engines' => $this->next, 'engines' => $this->next,
]; ];
Cache::put($this->getSearchUid(), serialize($this->next), 60); \App\CacheHelper::put($this->getSearchUid(), serialize($this->next), 60 * 60);
} else { } else {
$this->next = []; $this->next = [];
} }
...@@ -805,7 +806,7 @@ class MetaGer ...@@ -805,7 +806,7 @@ class MetaGer
while (sizeof($enginesToWaitFor) > 0 || ($forceTimeout !== null && (microtime(true) - $timeStart) < $forceTimeout)) { while (sizeof($enginesToWaitFor) > 0 || ($forceTimeout !== null && (microtime(true) - $timeStart) < $forceTimeout)) {
Log::info(sizeof($enginesToWaitFor) . " " . sizeof($answered) . " " . $enginesToWaitFor[0]->hash); Log::info(sizeof($enginesToWaitFor) . " " . sizeof($answered) . " " . $enginesToWaitFor[0]->hash);
foreach ($enginesToWaitFor as $index => $engine) { foreach ($enginesToWaitFor as $index => $engine) {
if (Cache::has($engine->hash)) { if (Redis::get($engine->hash) !== null) {
$answered[] = $engine; $answered[] = $engine;
unset($enginesToWaitFor[$index]); unset($enginesToWaitFor[$index]);
break; break;
......
...@@ -102,7 +102,7 @@ abstract class Searchengine ...@@ -102,7 +102,7 @@ abstract class Searchengine
{ {
if ($this->canCache && Cache::has($this->hash)) { if ($this->canCache && Cache::has($this->hash)) {
$this->cached = true; $this->cached = true;
$this->retrieveResults($metager); $this->retrieveResults($metager, true);
} else { } else {
// We need to submit a action that one of our workers can understand // We need to submit a action that one of our workers can understand
// The missions are submitted to a redis queue in the following string format // The missions are submitted to a redis queue in the following string format
...@@ -171,19 +171,15 @@ abstract class Searchengine ...@@ -171,19 +171,15 @@ abstract class Searchengine
} }
$body = null; $body = null;
if ($this->cached) {
if (Cache::has($this->hash)) {
$body = Cache::get($this->hash); $body = Cache::get($this->hash);
if ($body === "no-result") {
$body = "";
}
} else {
$body = Redis::get($this->hash);
} }
/*
if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash)) {
$body = Cache::get($this->hash);
} elseif ($redis->hexists($metager->getRedisEngineResult() . $this->name, "response")) {
$body = $redis->hget($metager->getRedisEngineResult() . $this->name, "response");
if ($this->canCache && $this->cacheDuration > 0) {
Cache::put($this->hash, $body, $this->cacheDuration);
}
}*/
if ($body !== null) { if ($body !== null) {
$this->loadResults($body); $this->loadResults($body);
$this->getNext($metager, $body); $this->getNext($metager, $body);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment