diff --git a/app/Http/Controllers/MetaGerSearch.php b/app/Http/Controllers/MetaGerSearch.php
index f6f9f434c8d1893cde7aa9fe0ba063d06db5f362..191cbd0cf174d5513f5b547f82f571582555b949 100644
--- a/app/Http/Controllers/MetaGerSearch.php
+++ b/app/Http/Controllers/MetaGerSearch.php
@@ -11,8 +11,13 @@ use View;
class MetaGerSearch extends Controller
{
- public function search(Request $request, MetaGer $metager)
+
+ public function search(Request $request, MetaGer $metager, $timing = false)
{
+ $timings = null;
+ if ($timing) {
+ $timings = ['starttime' => microtime(true)];
+ }
$time = microtime(true);
$spamEntries = [];
if (file_exists(config_path('spam.txt'))) {
@@ -34,9 +39,15 @@ class MetaGerSearch extends Controller
# Mit gelieferte Formulardaten parsen und abspeichern:
$metager->parseFormData($request);
+ if (!empty($timings)) {
+ $timings["parseFormData"] = microtime(true) - $time;
+ }
# Nach Spezialsuchen überprüfen:
$metager->checkSpecialSearches($request);
+ if (!empty($timings)) {
+ $timings["checkSpecialSearches"] = microtime(true) - $time;
+ }
if (Cache::has('spam.' . $metager->getFokus() . "." . md5($metager->getQ()))) {
return response(Cache::get('spam.' . $metager->getFokus() . "." . md5($metager->getEingabe())));
@@ -44,24 +55,43 @@ class MetaGerSearch extends Controller
# Die Quicktips als Job erstellen
$quicktips = $metager->createQuicktips();
+ if (!empty($timings)) {
+ $timings["createQuicktips"] = microtime(true) - $time;
+ }
# Suche für alle zu verwendenden Suchmaschinen als Job erstellen,
# auf Ergebnisse warten und die Ergebnisse laden
- $metager->createSearchEngines($request);
+ $metager->createSearchEngines($request, $timings);
- $metager->startSearch();
+ $metager->startSearch($timings);
$metager->waitForMainResults();
+ if (!empty($timings)) {
+ $timings["waitForMainResults"] = microtime(true) - $time;
+ }
$metager->retrieveResults();
+ if (!empty($timings)) {
+ $timings["retrieveResults"] = microtime(true) - $time;
+ }
# Versuchen die Ergebnisse der Quicktips zu laden
$quicktipResults = $quicktips->loadResults();
+ if (!empty($timings)) {
+ $timings["loadResults"] = microtime(true) - $time;
+ }
+
# Alle Ergebnisse vor der Zusammenführung ranken:
$metager->rankAll();
+ if (!empty($timings)) {
+ $timings["rankAll"] = microtime(true) - $time;
+ }
# Ergebnisse der Suchmaschinen kombinieren:
$metager->prepareResults();
+ if (!empty($timings)) {
+ $timings["prepareResults"] = microtime(true) - $time;
+ }
$finished = true;
foreach ($metager->getEngines() as $engine) {
@@ -72,6 +102,9 @@ class MetaGerSearch extends Controller
}
Cache::put("loader_" . $metager->getSearchUid(), $metager->getEngines(), 60 * 60);
+ if (!empty($timings)) {
+ $timings["Filled resultloader Cache"] = microtime(true) - $time;
+ }
# Die Ausgabe erstellen:
$resultpage = $metager->createView($quicktipResults);
@@ -84,9 +117,25 @@ class MetaGerSearch extends Controller
Cache::put('spam.' . $metager->getFokus() . "." . md5($metager->getEingabe()), $resultpage->render(), 604800);
}
}
+ if (!empty($timings)) {
+ $timings["createView"] = microtime(true) - $time;
+ }
+
+ if ($timings) {
+ dd($timings);
+ }
+
return $resultpage;
}
+ public function searchTimings(Request $request, MetaGer $metager)
+ {
+ $request->merge([
+ 'eingabe' => "Hannover",
+ ]);
+ return $this->search($request, $metager, true);
+ }
+
public function loadMore(Request $request)
{
/**
diff --git a/app/MetaGer.php b/app/MetaGer.php
index 2609e1a3152c7b1ef1b7bf648975436d2edbbb55..550755979e96c9f92d4bb54f56d7fc7187b650b1 100644
--- a/app/MetaGer.php
+++ b/app/MetaGer.php
@@ -473,8 +473,12 @@ class MetaGer
* Die Erstellung der Suchmaschinen bis die Ergebnisse da sind mit Unterfunktionen
*/
- public function createSearchEngines(Request $request)
+ public function createSearchEngines(Request $request, &$timings)
{
+ if (!empty($timings)) {
+ $timings["createSearchEngines"]["start"] = microtime(true) - $timings["starttime"];
+ }
+
# Wenn es kein Suchwort gibt
if (!$request->filled("eingabe") || $this->q === "") {
return;
@@ -495,8 +499,16 @@ class MetaGer
$sumas[$sumaName] = $this->sumaFile->sumas->{$sumaName};
}
+ if (!empty($timings)) {
+ $timings["createSearchEngines"]["created engine array"] = microtime(true) - $timings["starttime"];
+ }
+
$this->removeAdsFromListIfAdfree($sumas);
+ if (!empty($timings)) {
+ $timings["createSearchEngines"]["removed ads"] = microtime(true) - $timings["starttime"];
+ }
+
foreach ($sumas as $sumaName => $suma) {
# Check if this engine is disabled and can't be used
$disabled = empty($suma->disabled) ? false : $suma->disabled;
@@ -550,6 +562,10 @@ class MetaGer
}
}
+ if (!empty($timings)) {
+ $timings["createSearchEngines"]["filtered invalid engines"] = microtime(true) - $timings["starttime"];
+ }
+
# Include Yahoo Ads if Yahoo is not enabled as a searchengine
if (!$this->apiAuthorized && $this->fokus != "bilder" && empty($this->enabledSearchengines["yahoo"]) && isset($this->sumaFile->sumas->{"yahoo-ads"})) {
$this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
@@ -574,6 +590,10 @@ class MetaGer
$this->errors[] = $error;
}
$this->setEngines($request);
+ if (!empty($timings)) {
+ $timings["createSearchEngines"]["saved engines"] = microtime(true) - $timings["starttime"];
+ }
+
}
private function removeAdsFromListIfAdfree(&$sumas)
@@ -612,12 +632,38 @@ class MetaGer
}
}
- public function startSearch()
+ public function startSearch(&$timings)
{
+ if (!empty($timings)) {
+ $timings["startSearch"]["start"] = microtime(true) - $timings["starttime"];
+ }
+
+ # Check all engines for Cached responses
+ if ($this->canCache()) {
+ $keys = [];
+ foreach ($this->engines as $engine) {
+ $keys[] = $engine->hash;
+ }
+ $cacheValues = Cache::many($keys);
+ foreach ($this->engines as $engine) {
+ if ($cacheValues[$engine->hash] !== null) {
+ $engine->cached = true;
+ $engine->retrieveResults($this, $cacheValues[$engine->hash]);
+ }
+ }
+ }
+ if (!empty($timings)) {
+ $timings["startSearch"]["cache checked"] = microtime(true) - $timings["starttime"];
+ }
+
# Wir starten alle Suchen
foreach ($this->engines as $engine) {
- $engine->startSearch($this);
+ $engine->startSearch($this, $timings);
+ }
+ if (!empty($timings)) {
+ $timings["startSearch"]["searches started"] = microtime(true) - $timings["starttime"];
}
+
}
# Spezielle Suchen und Sumas
@@ -788,19 +834,20 @@ class MetaGer
$mainEngines = $this->sumaFile->foki->{$this->fokus}->main;
foreach ($mainEngines as $mainEngine) {
foreach ($engines as $engine) {
- if ($engine->name === $mainEngine) {
+ if ($engine->name === $mainEngine && !$engine->loaded) {
$enginesToWaitFor[] = $engine;
}
}
}
$timeStart = microtime(true);
+
$answered = [];
$results = null;
# If there is no main searchengine to wait for or if the only main engine is yahoo-ads we will define a timeout of 1s
$forceTimeout = null;
- if (sizeof($enginesToWaitFor) === 0 || (sizeof($enginesToWaitFor) === 1 && $enginesToWaitFor[0]->name === "yahoo-ads")) {
+ if (sizeof($enginesToWaitFor) === 1 && $enginesToWaitFor[0]->name === "yahoo-ads") {
$forceTimeout = 1;
}
@@ -812,30 +859,13 @@ class MetaGer
break;
}
}
+
if ((microtime(true) - $timeStart) >= 2) {
break;
} else {
usleep(50 * 1000);
}
}
-
- # Now we can add an entry to Redis which defines the starting time and how many engines should answer this request
- /*
- $pipeline = $redis->pipeline();
- $pipeline->hset($this->getRedisEngineResult() . "status", "startTime", $timeStart);
- $pipeline->hset($this->getRedisEngineResult() . "status", "engineCount", sizeof($engines));
- $pipeline->hset($this->getRedisEngineResult() . "status", "engineDelivered", sizeof($answered));
- # Add the cached engines as answered
- foreach ($engines as $engine) {
- if ($engine->cached) {
- $pipeline->hincrby($this->getRedisEngineResult() . "status", "engineDelivered", 1);
- $pipeline->hincrby($this->getRedisEngineResult() . "status", "engineAnswered", 1);
- }
- }
- foreach ($answered as $engine) {
- $pipeline->hset($this->getRedisEngineResult() . $engine, "delivered", "1");
- }
- $pipeline->execute();*/
}
public function retrieveResults()
diff --git a/app/Models/Searchengine.php b/app/Models/Searchengine.php
index 6584aa30f75043645f5015d3f0ddf36c457b54cd..ef364a50c2ae6794752508c67afbcf2c12c31418 100644
--- a/app/Models/Searchengine.php
+++ b/app/Models/Searchengine.php
@@ -3,7 +3,6 @@
namespace App\Models;
use App\MetaGer;
-use Cache;
use Illuminate\Support\Facades\Redis;
abstract class Searchengine
@@ -105,12 +104,17 @@ abstract class Searchengine
{}
# Prüft, ob die Suche bereits gecached ist, ansonsted wird sie als Job dispatched
- public function startSearch(\App\MetaGer $metager)
+ public function startSearch(\App\MetaGer $metager, &$timings)
{
- if ($this->canCache && Cache::has($this->hash)) {
- $this->cached = true;
- $this->retrieveResults($metager, true);
- } else {
+ if (!empty($timings)) {
+ $timings["startSearch"][$this->name]["start"] = microtime(true) - $timings["starttime"];
+ }
+
+ if (!$this->cached) {
+ if (!empty($timings)) {
+ $timings["startSearch"][$this->name]["checked cache"] = microtime(true) - $timings["starttime"];
+ }
+
// We need to submit a action that one of our workers can understand
// The missions are submitted to a redis queue in the following string format
// <ResultHash>;<URL to fetch>
@@ -144,10 +148,17 @@ abstract class Searchengine
// Since each Searcher is dedicated to one specific search engine
// each Searcher has it's own queue lying under the redis key <name>.queue
Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
+ if (!empty($timings)) {
+ $timings["startSearch"][$this->name]["pushed job"] = microtime(true) - $timings["starttime"];
+ }
+
// The request is not cached and will be submitted to the searchengine
// We need to check if the number of requests to this engine are limited
if (!empty($this->engine->{"monthly-requests"})) {
Redis::incr("monthlyRequests:" . $this->name);
+ if (!empty($timings)) {
+ $timings["startSearch"][$this->name]["increased monthly requests"] = microtime(true) - $timings["starttime"];
+ }
}
}
}
@@ -171,15 +182,13 @@ abstract class Searchengine
}
# Fragt die Ergebnisse von Redis ab und lädt Sie
- public function retrieveResults(MetaGer $metager)
+ public function retrieveResults(MetaGer $metager, $body = null)
{
if ($this->loaded) {
return true;
}
- $body = null;
if ($this->cached) {
- $body = Cache::get($this->hash);
if ($body === "no-result") {
$body = "";
}
diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml
index 0940498c1fbb03087901827449603c36c39b83fd..e619fd5a112375ee9aac7e21700acb2f1e2c2a60 100644
--- a/chart/templates/deployment.yaml
+++ b/chart/templates/deployment.yaml
@@ -38,9 +38,6 @@ spec:
- name: mglogs-persistent-storage
persistentVolumeClaim:
claimName: mglogs
- - name: mgcache-persistent-storage
- persistentVolumeClaim:
- claimName: mgcache
containers:
- name: {{ .Chart.Name }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
@@ -105,9 +102,6 @@ spec:
- name: mglogs-persistent-storage
mountPath: /html/storage/logs/metager
readOnly: false
- - name: mgcache-persistent-storage
- mountPath: /html/storage/framework/cache
- readOnly: false
resources:
{{ toYaml .Values.resources | indent 12 }}
{{- end -}}
diff --git a/routes/web.php b/routes/web.php
index 90f0257318fc166ade1596c64106cceb64ed0bce..cae78b76b3cc2c9e289a96d8f166898db3ed8d3e 100644
--- a/routes/web.php
+++ b/routes/web.php
@@ -170,6 +170,7 @@ Route::group(
Route::group(['middleware' => ['auth.basic'], 'prefix' => 'admin'], function () {
Route::get('/', 'AdminInterface@index');
Route::match(['get', 'post'], 'count', 'AdminInterface@count');
+ Route::get('timings', 'MetaGerSearch@searchTimings');
Route::get('count/graphtoday.svg', 'AdminInterface@countGraphToday');
Route::get('engine/stats.json', 'AdminInterface@engineStats');
Route::get('check', 'AdminInterface@check');
@@ -181,6 +182,7 @@ Route::group(
});
Route::match(['get', 'post'], 'meta/meta.ger3', 'MetaGerSearch@search')->middleware('humanverification', 'useragentmaster');
+
Route::get('meta/loadMore', 'MetaGerSearch@loadMore');
Route::post('img/cat.jpg', 'HumanVerification@remove');
Route::get('r/metager/{mm}/{pw}/{url}', ['as' => 'humanverification', 'uses' => 'HumanVerification@removeGet']);