Commit fff0bc3e authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

caching should work now

parent 9a5ccb35
<?php
namespace App\Console\Commands;
use Cache;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
class RequestCacher extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'requests:cacher';
const CACHER_QUEUE = 'cacher.queue';
protected $shouldRun = true;
/**
* The console command description.
*
* @var string
*/
protected $description = 'Listens to a buffer of fetched search results and writes them to the filesystem cache.';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
pcntl_async_signals(true);
pcntl_signal(SIGINT, [$this, "sig_handler"]);
pcntl_signal(SIGTERM, [$this, "sig_handler"]);
pcntl_signal(SIGHUP, [$this, "sig_handler"]);
while ($this->shouldRun) {
$cacheItem = Redis::blpop(self::CACHER_QUEUE, 1);
if (!empty($cacheItem)) {
$cacheItem = json_decode($cacheItem[1], true);
if (empty($cacheItem["body"])) {
$cacheItem["body"] = "no-result";
}
Cache::put($cacheItem["hash"], $cacheItem["body"], now()->addMinutes($cacheItem["cacheDuration"]));
}
}
}
public function sig_handler($sig)
{
$this->shouldRun = false;
echo ("Terminating Cacher Process\n");
}
}
...@@ -2,11 +2,12 @@ ...@@ -2,11 +2,12 @@
namespace App\Console\Commands; namespace App\Console\Commands;
use Artisan;
use Illuminate\Console\Command; use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis; use Illuminate\Support\Facades\Redis;
use Log; use Log;
class WorkerSpawner extends Command class RequestFetcher extends Command
{ {
/** /**
* The name and signature of the console command. * The name and signature of the console command.
...@@ -20,7 +21,7 @@ class WorkerSpawner extends Command ...@@ -20,7 +21,7 @@ class WorkerSpawner extends Command
* *
* @var string * @var string
*/ */
protected $description = 'This command makes sure that enough worker processes are spawned'; protected $description = 'This commands fetches requests to the installed search engines';
protected $shouldRun = true; protected $shouldRun = true;
protected $multicurl = null; protected $multicurl = null;
...@@ -49,10 +50,16 @@ class WorkerSpawner extends Command ...@@ -49,10 +50,16 @@ class WorkerSpawner extends Command
*/ */
public function handle() public function handle()
{ {
pcntl_async_signals(true); $pid = \pcntl_fork();
pcntl_signal(SIGINT, [$this, "sig_handler"]); if ($pid === 0) {
pcntl_signal(SIGTERM, [$this, "sig_handler"]); Artisan::call('requests:cacher');
pcntl_signal(SIGHUP, [$this, "sig_handler"]); exit;
} else {
pcntl_async_signals(true);
pcntl_signal(SIGINT, [$this, "sig_handler"]);
pcntl_signal(SIGTERM, [$this, "sig_handler"]);
pcntl_signal(SIGHUP, [$this, "sig_handler"]);
}
try { try {
$blocking = false; $blocking = false;
...@@ -62,7 +69,7 @@ class WorkerSpawner extends Command ...@@ -62,7 +69,7 @@ class WorkerSpawner extends Command
if (!$blocking) { if (!$blocking) {
$currentJob = Redis::lpop(\App\MetaGer::FETCHQUEUE_KEY); $currentJob = Redis::lpop(\App\MetaGer::FETCHQUEUE_KEY);
} else { } else {
$currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 10); $currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
if (!empty($currentJob)) { if (!empty($currentJob)) {
$currentJob = $currentJob[1]; $currentJob = $currentJob[1];
} }
...@@ -97,9 +104,15 @@ class WorkerSpawner extends Command ...@@ -97,9 +104,15 @@ class WorkerSpawner extends Command
$body = \curl_multi_getcontent($info["handle"]); $body = \curl_multi_getcontent($info["handle"]);
} }
Redis::pipeline(function ($pipe) use ($resulthash, $body) { Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDuration) {
$pipe->set($resulthash, $body); $pipe->set($resulthash, $body);
$pipe->expire($resulthash, 60); $pipe->expire($resulthash, 60);
$cacherItem = [
'cacheDuration' => $cacheDuration,
'hash' => $resulthash,
'body' => $body,
];
$pipe->rpush(\App\Console\Commands\RequestCacher::CACHER_QUEUE, json_encode($cacherItem));
}); });
#Cache::put($resulthash, $body, now()->addMinutes($cacheDuration)); #Cache::put($resulthash, $body, now()->addMinutes($cacheDuration));
\curl_multi_remove_handle($this->multicurl, $info["handle"]); \curl_multi_remove_handle($this->multicurl, $info["handle"]);
...@@ -111,6 +124,8 @@ class WorkerSpawner extends Command ...@@ -111,6 +124,8 @@ class WorkerSpawner extends Command
} finally { } finally {
curl_multi_close($this->multicurl); curl_multi_close($this->multicurl);
} }
\pcntl_waitpid($pid, $status, WNOHANG);
} }
private function getCurlHandle($job) private function getCurlHandle($job)
......
...@@ -102,7 +102,7 @@ abstract class Searchengine ...@@ -102,7 +102,7 @@ abstract class Searchengine
{ {
if ($this->canCache && Cache::has($this->hash)) { if ($this->canCache && Cache::has($this->hash)) {
$this->cached = true; $this->cached = true;
$this->retrieveResults($metager); $this->retrieveResults($metager, true);
} else { } else {
// We need to submit a action that one of our workers can understand // We need to submit a action that one of our workers can understand
// The missions are submitted to a redis queue in the following string format // The missions are submitted to a redis queue in the following string format
...@@ -170,20 +170,16 @@ abstract class Searchengine ...@@ -170,20 +170,16 @@ abstract class Searchengine
return true; return true;
} }
$body = Redis::get($this->hash); $body = null;
if ($this->cached) {
/*if (Cache::has($this->hash)) { $body = Cache::get($this->hash);
$body = Cache::get($this->hash); if ($body === "no-result") {
}*/ $body = "";
/* }
if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash)) { } else {
$body = Cache::get($this->hash); $body = Redis::get($this->hash);
} elseif ($redis->hexists($metager->getRedisEngineResult() . $this->name, "response")) {
$body = $redis->hget($metager->getRedisEngineResult() . $this->name, "response");
if ($this->canCache && $this->cacheDuration > 0) {
Cache::put($this->hash, $body, $this->cacheDuration);
} }
}*/
if ($body !== null) { if ($body !== null) {
$this->loadResults($body); $this->loadResults($body);
$this->getNext($metager, $body); $this->getNext($metager, $body);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment