diff --git a/.env.example b/.env.example
index a657413fbab84166dc7b4b7f96af819494904f00..d0b8b81bcfa87727379ddd8c9db66e8032235303 100644
--- a/.env.example
+++ b/.env.example
@@ -15,7 +15,7 @@ REDIS_RESULT_CONNECTION=default
REDIS_RESULT_CACHE_DURATION=60
BROADCAST_DRIVER=log
-CACHE_DRIVER=file
+CACHE_DRIVER=database
SESSION_DRIVER=file
QUEUE_CONNECTION=sync
diff --git a/app/CacheHelper.php b/app/CacheHelper.php
deleted file mode 100644
index 3a4aeaae55015b4c26f40d5ae2ebd0ed927a4d05..0000000000000000000000000000000000000000
--- a/app/CacheHelper.php
+++ /dev/null
@@ -1,24 +0,0 @@
-<?php
-
-namespace App;
-
-use Illuminate\Support\Facades\Redis;
-
-class CacheHelper
-{
-
- /**
- * MetaGer uses a pretty slow harddrive for the configured cache
- * That's why we have some processes running to write cache to disk in parallel
- */
- public static function put($key, $value, $timeSeconds)
- {
- $cacherItem = [
- 'timeSeconds' => $timeSeconds,
- 'key' => $key,
- 'value' => $value,
- ];
- Redis::rpush(\App\Console\Commands\RequestCacher::CACHER_QUEUE, base64_encode(serialize($cacherItem)));
-
- }
-}
diff --git a/app/Console/Commands/RequestCacher.php b/app/Console/Commands/RequestCacher.php
deleted file mode 100644
index bf52f428edd9f6474fb1c89b79a43f1b584ce5b4..0000000000000000000000000000000000000000
--- a/app/Console/Commands/RequestCacher.php
+++ /dev/null
@@ -1,67 +0,0 @@
-<?php
-
-namespace App\Console\Commands;
-
-use Cache;
-use Illuminate\Console\Command;
-use Illuminate\Support\Facades\Redis;
-
-class RequestCacher extends Command
-{
- /**
- * The name and signature of the console command.
- *
- * @var string
- */
- protected $signature = 'requests:cacher';
-
- const CACHER_QUEUE = 'cacher.queue';
- protected $shouldRun = true;
-
- /**
- * The console command description.
- *
- * @var string
- */
- protected $description = 'Listens to a buffer of fetched search results and writes them to the filesystem cache.';
-
- /**
- * Create a new command instance.
- *
- * @return void
- */
- public function __construct()
- {
- parent::__construct();
- }
-
- /**
- * Execute the console command.
- *
- * @return mixed
- */
- public function handle()
- {
- pcntl_async_signals(true);
- pcntl_signal(SIGINT, [$this, "sig_handler"]);
- pcntl_signal(SIGTERM, [$this, "sig_handler"]);
- pcntl_signal(SIGHUP, [$this, "sig_handler"]);
-
- while ($this->shouldRun) {
- $cacheItem = Redis::blpop(self::CACHER_QUEUE, 1);
- if (!empty($cacheItem)) {
- $cacheItem = unserialize(base64_decode($cacheItem[1]));
- if (empty($cacheItem["value"])) {
- $cacheItem["value"] = "no-result";
- }
- Cache::put($cacheItem["key"], $cacheItem["value"], now()->addSeconds($cacheItem["timeSeconds"]));
- }
- }
- }
-
- public function sig_handler($sig)
- {
- $this->shouldRun = false;
- echo ("Terminating Cacher Process\n");
- }
-}
diff --git a/app/Console/Commands/RequestFetcher.php b/app/Console/Commands/RequestFetcher.php
index 3af939a7ea2fc5611ce71bd1842361cc4b745a17..9b986603af972b57e341d7327218940608b28bec 100644
--- a/app/Console/Commands/RequestFetcher.php
+++ b/app/Console/Commands/RequestFetcher.php
@@ -2,7 +2,7 @@
namespace App\Console\Commands;
-use Artisan;
+use Cache;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
use Log;
@@ -51,23 +51,11 @@ class RequestFetcher extends Command
public function handle()
{
$pids = [];
- $pid = null;
- for ($i = 0; $i < 5; $i++) {
- $pid = \pcntl_fork();
- $pids[] = $pid;
- if ($pid === 0) {
- break;
- }
- }
- if ($pid === 0) {
- Artisan::call('requests:cacher');
- exit;
- } else {
- pcntl_async_signals(true);
- pcntl_signal(SIGINT, [$this, "sig_handler"]);
- pcntl_signal(SIGTERM, [$this, "sig_handler"]);
- pcntl_signal(SIGHUP, [$this, "sig_handler"]);
- }
+
+ pcntl_async_signals(true);
+ pcntl_signal(SIGINT, [$this, "sig_handler"]);
+ pcntl_signal(SIGTERM, [$this, "sig_handler"]);
+ pcntl_signal(SIGHUP, [$this, "sig_handler"]);
try {
$blocking = false;
@@ -115,12 +103,7 @@ class RequestFetcher extends Command
Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
$pipe->set($resulthash, $body);
$pipe->expire($resulthash, 60);
- $cacherItem = [
- 'timeSeconds' => $cacheDurationMinutes * 60,
- 'key' => $resulthash,
- 'value' => $body,
- ];
- $pipe->rpush(\App\Console\Commands\RequestCacher::CACHER_QUEUE, base64_encode(serialize($cacherItem)));
+ Cache::put($resulthash, $body, $cacheDurationMinutes * 60);
});
\curl_multi_remove_handle($this->multicurl, $info["handle"]);
}
diff --git a/app/Http/Controllers/MetaGerSearch.php b/app/Http/Controllers/MetaGerSearch.php
index 134b92beda984904081629189b197779d9318470..191cbd0cf174d5513f5b547f82f571582555b949 100644
--- a/app/Http/Controllers/MetaGerSearch.php
+++ b/app/Http/Controllers/MetaGerSearch.php
@@ -11,8 +11,13 @@ use View;
class MetaGerSearch extends Controller
{
- public function search(Request $request, MetaGer $metager)
+
+ public function search(Request $request, MetaGer $metager, $timing = false)
{
+ $timings = null;
+ if ($timing) {
+ $timings = ['starttime' => microtime(true)];
+ }
$time = microtime(true);
$spamEntries = [];
if (file_exists(config_path('spam.txt'))) {
@@ -34,9 +39,15 @@ class MetaGerSearch extends Controller
# Mit gelieferte Formulardaten parsen und abspeichern:
$metager->parseFormData($request);
+ if (!empty($timings)) {
+ $timings["parseFormData"] = microtime(true) - $time;
+ }
# Nach Spezialsuchen überprüfen:
$metager->checkSpecialSearches($request);
+ if (!empty($timings)) {
+ $timings["checkSpecialSearches"] = microtime(true) - $time;
+ }
if (Cache::has('spam.' . $metager->getFokus() . "." . md5($metager->getQ()))) {
return response(Cache::get('spam.' . $metager->getFokus() . "." . md5($metager->getEingabe())));
@@ -44,24 +55,43 @@ class MetaGerSearch extends Controller
# Die Quicktips als Job erstellen
$quicktips = $metager->createQuicktips();
+ if (!empty($timings)) {
+ $timings["createQuicktips"] = microtime(true) - $time;
+ }
# Suche für alle zu verwendenden Suchmaschinen als Job erstellen,
# auf Ergebnisse warten und die Ergebnisse laden
- $metager->createSearchEngines($request);
+ $metager->createSearchEngines($request, $timings);
- $metager->startSearch();
+ $metager->startSearch($timings);
$metager->waitForMainResults();
+ if (!empty($timings)) {
+ $timings["waitForMainResults"] = microtime(true) - $time;
+ }
$metager->retrieveResults();
+ if (!empty($timings)) {
+ $timings["retrieveResults"] = microtime(true) - $time;
+ }
# Versuchen die Ergebnisse der Quicktips zu laden
$quicktipResults = $quicktips->loadResults();
+ if (!empty($timings)) {
+ $timings["loadResults"] = microtime(true) - $time;
+ }
+
# Alle Ergebnisse vor der Zusammenführung ranken:
$metager->rankAll();
+ if (!empty($timings)) {
+ $timings["rankAll"] = microtime(true) - $time;
+ }
# Ergebnisse der Suchmaschinen kombinieren:
$metager->prepareResults();
+ if (!empty($timings)) {
+ $timings["prepareResults"] = microtime(true) - $time;
+ }
$finished = true;
foreach ($metager->getEngines() as $engine) {
@@ -71,7 +101,10 @@ class MetaGerSearch extends Controller
}
}
- \App\CacheHelper::put("loader_" . $metager->getSearchUid(), $metager->getEngines(), 60 * 60);
+ Cache::put("loader_" . $metager->getSearchUid(), $metager->getEngines(), 60 * 60);
+ if (!empty($timings)) {
+ $timings["Filled resultloader Cache"] = microtime(true) - $time;
+ }
# Die Ausgabe erstellen:
$resultpage = $metager->createView($quicktipResults);
@@ -84,9 +117,25 @@ class MetaGerSearch extends Controller
Cache::put('spam.' . $metager->getFokus() . "." . md5($metager->getEingabe()), $resultpage->render(), 604800);
}
}
+ if (!empty($timings)) {
+ $timings["createView"] = microtime(true) - $time;
+ }
+
+ if ($timings) {
+ dd($timings);
+ }
+
return $resultpage;
}
+ public function searchTimings(Request $request, MetaGer $metager)
+ {
+ $request->merge([
+ 'eingabe' => "Hannover",
+ ]);
+ return $this->search($request, $metager, true);
+ }
+
public function loadMore(Request $request)
{
/**
@@ -166,7 +215,7 @@ class MetaGerSearch extends Controller
$result["finished"] = $finished;
// Update new Engines
- \App\CacheHelper::put("loader_" . $metager->getSearchUid(), $metager->getEngines(), 1 * 60);
+ Cache::put("loader_" . $metager->getSearchUid(), $metager->getEngines(), 1 * 60);
return response()->json($result);
}
diff --git a/app/Http/Middleware/HumanVerification.php b/app/Http/Middleware/HumanVerification.php
index f7c61dc15b4736cec50de395a483a38ebbcdeb16..d646e1fbd935c507be1482104067bf516e63ed97 100644
--- a/app/Http/Middleware/HumanVerification.php
+++ b/app/Http/Middleware/HumanVerification.php
@@ -2,11 +2,11 @@
namespace App\Http\Middleware;
+use Cache;
use Captcha;
use Closure;
use Cookie;
use Illuminate\Http\Response;
-use Illuminate\Support\Facades\Cache;
use URL;
class HumanVerification
@@ -148,7 +148,7 @@ class HumanVerification
// Lock must be acquired within 2 seconds
$userList = Cache::get($prefix . "." . $user["id"], []);
$userList[$user["uid"]] = $user;
- \App\CacheHelper::put($prefix . "." . $user["id"], $userList, 2 * 7 * 24 * 60 * 60);
+ Cache::put($prefix . "." . $user["id"], $userList, 2 * 7 * 24 * 60 * 60);
}
public function removeOldUsers($prefix, $userList)
@@ -168,7 +168,7 @@ class HumanVerification
}
if ($changed) {
- \App\CacheHelper::put($prefix . "." . $user["id"], $newUserlist, 2 * 7 * 24 * 60 * 60);
+ Cache::put($prefix . "." . $user["id"], $newUserlist, 2 * 7 * 24 * 60 * 60);
}
return $newUserlist;
diff --git a/app/MetaGer.php b/app/MetaGer.php
index ea175f09edd33537bb15603ad4dbe7451c524672..550755979e96c9f92d4bb54f56d7fc7187b650b1 100644
--- a/app/MetaGer.php
+++ b/app/MetaGer.php
@@ -324,7 +324,7 @@ class MetaGer
'page' => $page,
'engines' => $this->next,
];
- \App\CacheHelper::put($this->getSearchUid(), serialize($this->next), 60 * 60);
+ Cache::put($this->getSearchUid(), serialize($this->next), 60 * 60);
} else {
$this->next = [];
}
@@ -473,8 +473,12 @@ class MetaGer
* Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
*/
- public function createSearchEngines(Request $request)
+ public function createSearchEngines(Request $request, &$timings)
{
+ if (!empty($timings)) {
+ $timings["createSearchEngines"]["start"] = microtime(true) - $timings["starttime"];
+ }
+
# Wenn es kein Suchwort gibt
if (!$request->filled("eingabe") || $this->q === "") {
return;
@@ -495,8 +499,16 @@ class MetaGer
$sumas[$sumaName] = $this->sumaFile->sumas->{$sumaName};
}
+ if (!empty($timings)) {
+ $timings["createSearchEngines"]["created engine array"] = microtime(true) - $timings["starttime"];
+ }
+
$this->removeAdsFromListIfAdfree($sumas);
+ if (!empty($timings)) {
+ $timings["createSearchEngines"]["removed ads"] = microtime(true) - $timings["starttime"];
+ }
+
foreach ($sumas as $sumaName => $suma) {
# Check if this engine is disabled and can't be used
$disabled = empty($suma->disabled) ? false : $suma->disabled;
@@ -550,6 +562,10 @@ class MetaGer
}
}
+ if (!empty($timings)) {
+ $timings["createSearchEngines"]["filtered invalid engines"] = microtime(true) - $timings["starttime"];
+ }
+
# Include Yahoo Ads if Yahoo is not enabled as a searchengine
if (!$this->apiAuthorized && $this->fokus != "bilder" && empty($this->enabledSearchengines["yahoo"]) && isset($this->sumaFile->sumas->{"yahoo-ads"})) {
$this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
@@ -574,6 +590,10 @@ class MetaGer
$this->errors[] = $error;
}
$this->setEngines($request);
+ if (!empty($timings)) {
+ $timings["createSearchEngines"]["saved engines"] = microtime(true) - $timings["starttime"];
+ }
+
}
private function removeAdsFromListIfAdfree(&$sumas)
@@ -612,12 +632,38 @@ class MetaGer
}
}
- public function startSearch()
+ public function startSearch(&$timings)
{
+ if (!empty($timings)) {
+ $timings["startSearch"]["start"] = microtime(true) - $timings["starttime"];
+ }
+
+ # Check all engines for Cached responses
+ if ($this->canCache()) {
+ $keys = [];
+ foreach ($this->engines as $engine) {
+ $keys[] = $engine->hash;
+ }
+ $cacheValues = Cache::many($keys);
+ foreach ($this->engines as $engine) {
+ if ($cacheValues[$engine->hash] !== null) {
+ $engine->cached = true;
+ $engine->retrieveResults($this, $cacheValues[$engine->hash]);
+ }
+ }
+ }
+ if (!empty($timings)) {
+ $timings["startSearch"]["cache checked"] = microtime(true) - $timings["starttime"];
+ }
+
# Wir starten alle Suchen
foreach ($this->engines as $engine) {
- $engine->startSearch($this);
+ $engine->startSearch($this, $timings);
+ }
+ if (!empty($timings)) {
+ $timings["startSearch"]["searches started"] = microtime(true) - $timings["starttime"];
}
+
}
# Spezielle Suchen und Sumas
@@ -788,19 +834,20 @@ class MetaGer
$mainEngines = $this->sumaFile->foki->{$this->fokus}->main;
foreach ($mainEngines as $mainEngine) {
foreach ($engines as $engine) {
- if ($engine->name === $mainEngine) {
+ if ($engine->name === $mainEngine && !$engine->loaded) {
$enginesToWaitFor[] = $engine;
}
}
}
$timeStart = microtime(true);
+
$answered = [];
$results = null;
# If there is no main searchengine to wait for or if the only main engine is yahoo-ads we will define a timeout of 1s
$forceTimeout = null;
- if (sizeof($enginesToWaitFor) === 0 || (sizeof($enginesToWaitFor) === 1 && $enginesToWaitFor[0]->name === "yahoo-ads")) {
+ if (sizeof($enginesToWaitFor) === 1 && $enginesToWaitFor[0]->name === "yahoo-ads") {
$forceTimeout = 1;
}
@@ -812,30 +859,13 @@ class MetaGer
break;
}
}
+
if ((microtime(true) - $timeStart) >= 2) {
break;
} else {
usleep(50 * 1000);
}
}
-
- # Now we can add an entry to Redis which defines the starting time and how many engines should answer this request
- /*
- $pipeline = $redis->pipeline();
- $pipeline->hset($this->getRedisEngineResult() . "status", "startTime", $timeStart);
- $pipeline->hset($this->getRedisEngineResult() . "status", "engineCount", sizeof($engines));
- $pipeline->hset($this->getRedisEngineResult() . "status", "engineDelivered", sizeof($answered));
- # Add the cached engines as answered
- foreach ($engines as $engine) {
- if ($engine->cached) {
- $pipeline->hincrby($this->getRedisEngineResult() . "status", "engineDelivered", 1);
- $pipeline->hincrby($this->getRedisEngineResult() . "status", "engineAnswered", 1);
- }
- }
- foreach ($answered as $engine) {
- $pipeline->hset($this->getRedisEngineResult() . $engine, "delivered", "1");
- }
- $pipeline->execute();*/
}
public function retrieveResults()
diff --git a/app/Models/Searchengine.php b/app/Models/Searchengine.php
index 6584aa30f75043645f5015d3f0ddf36c457b54cd..ef364a50c2ae6794752508c67afbcf2c12c31418 100644
--- a/app/Models/Searchengine.php
+++ b/app/Models/Searchengine.php
@@ -3,7 +3,6 @@
namespace App\Models;
use App\MetaGer;
-use Cache;
use Illuminate\Support\Facades\Redis;
abstract class Searchengine
@@ -105,12 +104,17 @@ abstract class Searchengine
{}
# Prüft, ob die Suche bereits gecached ist, ansonsted wird sie als Job dispatched
- public function startSearch(\App\MetaGer $metager)
+ public function startSearch(\App\MetaGer $metager, &$timings)
{
- if ($this->canCache && Cache::has($this->hash)) {
- $this->cached = true;
- $this->retrieveResults($metager, true);
- } else {
+ if (!empty($timings)) {
+ $timings["startSearch"][$this->name]["start"] = microtime(true) - $timings["starttime"];
+ }
+
+ if (!$this->cached) {
+ if (!empty($timings)) {
+ $timings["startSearch"][$this->name]["checked cache"] = microtime(true) - $timings["starttime"];
+ }
+
// We need to submit a action that one of our workers can understand
// The missions are submitted to a redis queue in the following string format
// <ResultHash>;<URL to fetch>
@@ -144,10 +148,17 @@ abstract class Searchengine
// Since each Searcher is dedicated to one specific search engine
// each Searcher has it's own queue lying under the redis key <name>.queue
Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
+ if (!empty($timings)) {
+ $timings["startSearch"][$this->name]["pushed job"] = microtime(true) - $timings["starttime"];
+ }
+
// The request is not cached and will be submitted to the searchengine
// We need to check if the number of requests to this engine are limited
if (!empty($this->engine->{"monthly-requests"})) {
Redis::incr("monthlyRequests:" . $this->name);
+ if (!empty($timings)) {
+ $timings["startSearch"][$this->name]["increased monthly requests"] = microtime(true) - $timings["starttime"];
+ }
}
}
}
@@ -171,15 +182,13 @@ abstract class Searchengine
}
# Fragt die Ergebnisse von Redis ab und lädt Sie
- public function retrieveResults(MetaGer $metager)
+ public function retrieveResults(MetaGer $metager, $body = null)
{
if ($this->loaded) {
return true;
}
- $body = null;
if ($this->cached) {
- $body = Cache::get($this->hash);
if ($body === "no-result") {
$body = "";
}
diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml
index 0940498c1fbb03087901827449603c36c39b83fd..e619fd5a112375ee9aac7e21700acb2f1e2c2a60 100644
--- a/chart/templates/deployment.yaml
+++ b/chart/templates/deployment.yaml
@@ -38,9 +38,6 @@ spec:
- name: mglogs-persistent-storage
persistentVolumeClaim:
claimName: mglogs
- - name: mgcache-persistent-storage
- persistentVolumeClaim:
- claimName: mgcache
containers:
- name: {{ .Chart.Name }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
@@ -105,9 +102,6 @@ spec:
- name: mglogs-persistent-storage
mountPath: /html/storage/logs/metager
readOnly: false
- - name: mgcache-persistent-storage
- mountPath: /html/storage/framework/cache
- readOnly: false
resources:
{{ toYaml .Values.resources | indent 12 }}
{{- end -}}
diff --git a/database/migrations/2020_02_05_163522_create_cache_table.php b/database/migrations/2020_02_05_163522_create_cache_table.php
new file mode 100644
index 0000000000000000000000000000000000000000..058afe69c98645601183f7b4adea77cd191bf882
--- /dev/null
+++ b/database/migrations/2020_02_05_163522_create_cache_table.php
@@ -0,0 +1,32 @@
+<?php
+
+use Illuminate\Support\Facades\Schema;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Database\Migrations\Migration;
+
+class CreateCacheTable extends Migration
+{
+ /**
+ * Run the migrations.
+ *
+ * @return void
+ */
+ public function up()
+ {
+ Schema::create('cache', function (Blueprint $table) {
+ $table->string('key')->unique();
+ $table->mediumText('value');
+ $table->integer('expiration');
+ });
+ }
+
+ /**
+ * Reverse the migrations.
+ *
+ * @return void
+ */
+ public function down()
+ {
+ Schema::dropIfExists('cache');
+ }
+}
diff --git a/routes/web.php b/routes/web.php
index 90f0257318fc166ade1596c64106cceb64ed0bce..cae78b76b3cc2c9e289a96d8f166898db3ed8d3e 100644
--- a/routes/web.php
+++ b/routes/web.php
@@ -170,6 +170,7 @@ Route::group(
Route::group(['middleware' => ['auth.basic'], 'prefix' => 'admin'], function () {
Route::get('/', 'AdminInterface@index');
Route::match(['get', 'post'], 'count', 'AdminInterface@count');
+ Route::get('timings', 'MetaGerSearch@searchTimings');
Route::get('count/graphtoday.svg', 'AdminInterface@countGraphToday');
Route::get('engine/stats.json', 'AdminInterface@engineStats');
Route::get('check', 'AdminInterface@check');
@@ -181,6 +182,7 @@ Route::group(
});
Route::match(['get', 'post'], 'meta/meta.ger3', 'MetaGerSearch@search')->middleware('humanverification', 'useragentmaster');
+
Route::get('meta/loadMore', 'MetaGerSearch@loadMore');
Route::post('img/cat.jpg', 'HumanVerification@remove');
Route::get('r/metager/{mm}/{pw}/{url}', ['as' => 'humanverification', 'uses' => 'HumanVerification@removeGet']);