From 196021fce05cfbd6dd85ba3978ca735c52b7c032 Mon Sep 17 00:00:00 2001
From: Dominik Hebeler <dominik@suma-ev.de>
Date: Tue, 5 Mar 2019 08:12:32 +0100
Subject: [PATCH] First step to optimized request times
---
app/Http/Controllers/MetaGerSearch.php | 3 +-
app/Jobs/Searcher.php | 19 ++-
app/MetaGer.php | 167 ++++++-------------
app/Models/Quicktips/Quicktips.php | 30 ++--
app/Models/Searchengine.php | 12 +-
resources/js/scriptResultPage.js | 6 +-
resources/views/layouts/resultPage.blade.php | 1 +
7 files changed, 86 insertions(+), 152 deletions(-)
diff --git a/app/Http/Controllers/MetaGerSearch.php b/app/Http/Controllers/MetaGerSearch.php
index 56566146f..66e8f8c29 100644
--- a/app/Http/Controllers/MetaGerSearch.php
+++ b/app/Http/Controllers/MetaGerSearch.php
@@ -25,7 +25,7 @@ class MetaGerSearch extends Controller
}*/
#die($request->header('User-Agent'));
- $time = microtime();
+ $time = microtime(true);
# Mit gelieferte Formulardaten parsen und abspeichern:
$metager->parseFormData($request);
@@ -41,7 +41,6 @@ class MetaGerSearch extends Controller
# Versuchen die Ergebnisse der Quicktips zu laden
$quicktipResults = $quicktips->loadResults();
-
# Alle Ergebnisse vor der Zusammenführung ranken:
$metager->rankAll();
diff --git a/app/Jobs/Searcher.php b/app/Jobs/Searcher.php
index ec82937b4..7a57025c4 100644
--- a/app/Jobs/Searcher.php
+++ b/app/Jobs/Searcher.php
@@ -85,8 +85,7 @@ class Searcher implements ShouldQueue
$url = base64_decode($mission[1]); // The url to fetch
$timeout = $mission[2]; // Timeout from the MetaGer process in ms
$medianFetchTime = $this->getFetchTime(); // The median Fetch time of the search engine in ms
- Redis::hset('search.' . $hashValue, $this->name, "connected");
-
+ Redis::hset('search.' . $hashValue . ".results." . $this->name, "status", "connected");
$result = $this->retrieveUrl($url);
$this->storeResult($result, $poptime, $hashValue);
@@ -99,7 +98,7 @@ class Searcher implements ShouldQueue
// In sync mode every Searcher may only retrieve one result because it would block
// the execution of the remaining code otherwise:
- if (getenv("QUEUE_DRIVER") === "sync"
+ if (getenv("QUEUE_CONNECTION") === "sync"
|| $this->counter > $this->MAX_REQUESTS
|| (microtime(true) - $this->startTime) > $this->MAX_TIME) {
break;
@@ -161,16 +160,24 @@ class Searcher implements ShouldQueue
// Set this URL to the Curl handle
curl_setopt($this->ch, CURLOPT_URL, $url);
$result = curl_exec($this->ch);
-
$this->connectionInfo = curl_getinfo($this->ch);
return $result;
}
private function storeResult($result, $poptime, $hashValue)
{
- Redis::hset('search.' . $hashValue, $this->name, $result);
+ if ($this->name === "zeitde") {
+ sleep(3);
+ }
+ $pipeline = Redis::pipeline();
+ $pipeline->hset('search.' . $hashValue . ".results." . $this->name, "response", $result);
// After 60 seconds the results should be read by the MetaGer Process and stored in the Cache instead
- Redis::expire('search.' . $hashValue, 60);
+ $pipeline->expire('search.' . $hashValue . ".results." . $this->name, 60);
+ $pipeline->rpush('search.' . $hashValue . ".ready", $this->name);
+ $pipeline->expire('search.' . $hashValue . ".ready", 60);
+ $pipeline->sadd('search.' . $hashValue . ".engines", $this->name);
+ $pipeline->expire('search.' . $hashValue . ".engines", 60);
+ $pipeline->execute();
$this->lastTime = microtime(true);
}
diff --git a/app/MetaGer.php b/app/MetaGer.php
index 0a80fce18..543674aee 100644
--- a/app/MetaGer.php
+++ b/app/MetaGer.php
@@ -59,6 +59,8 @@ class MetaGer
protected $languageDetect;
protected $verificationId;
protected $verificationCount;
+ protected $searchUid;
+ protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult;
public function __construct()
{
@@ -90,6 +92,15 @@ class MetaGer
} catch (ConnectionException $e) {
$this->canCache = false;
}
+ $this->canCache = false;
+ $this->searchUid = md5(uniqid());
+ $redisPrefix = "search";
+ # This is a list on which the MetaGer process can do a blocking pop to wait for new results
+ $this->redisResultWaitingKey = $redisPrefix . "." . $this->searchUid . ".ready";
+ # This is a list of searchengines which have delivered results for this search
+ $this->redisResultEngineList = $redisPrefix . "." . $this->searchUid . ".engines";
+ # This is the key where the results of the engine are stored as well as some statistical data
+ $this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results.";
}
# Erstellt aus den gesammelten Ergebnissen den View
@@ -512,7 +523,7 @@ class MetaGer
public function createQuicktips()
{
# Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
- $quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime(), $this->getHashCode());
+ $quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime());
return $quicktips;
}
@@ -534,7 +545,6 @@ class MetaGer
if (empty($this->sumaFile->foki->{$this->fokus})) {
$this->fokus = "web";
}
-
foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
# Check if this engine is disabled and can't be used
$disabled = empty($this->sumaFile->sumas->{$suma}->disabled) ? false : $this->sumaFile->sumas->{$suma}->disabled;
@@ -589,8 +599,6 @@ class MetaGer
$this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
}
- #die(var_dump($this->enabledSearchengines));
-
if (sizeof($this->enabledSearchengines) === 0) {
$filter = "";
foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
@@ -607,6 +615,7 @@ class MetaGer
$counter = 0;
if ($this->requestIsCached($request)) {
+ # If this is a page other than 1 the request is "cached"
$engines = $this->getCachedEngines($request);
# We need to edit some Options of the Cached Search Engines
foreach ($engines as $engine) {
@@ -628,19 +637,7 @@ class MetaGer
* welche Suchmaschinen nicht rechtzeitig geantwortet haben.
*/
- $enginesToLoad = [];
- $canBreak = false;
- foreach ($engines as $engine) {
- if ($engine->cached) {
- if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
- $canBreak = true;
- }
- } else {
- $enginesToLoad[$engine->name] = false;
- }
- }
-
- $this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);
+ $this->waitForResults($engines);
$this->retrieveResults($engines);
foreach ($engines as $engine) {
@@ -788,115 +785,35 @@ class MetaGer
return $engines;
}
- # Passt den Suchfokus an, falls für einen Fokus genau alle vorhandenen Sumas eingeschaltet sind
- public function adjustFocus($sumas, $enabledSearchengines)
+ public function waitForResults($engines)
{
- # Findet für alle Foki die enthaltenen Sumas
- $foki = []; # [fokus][suma] => [suma]
- foreach ($sumas as $suma) {
- if ((!$this->sumaIsDisabled($suma)) && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
- if (isset($suma['type'])) {
- # Wenn foki für diese Suchmaschine angegeben sind
- $focuses = explode(",", $suma['type']->__toString());
- foreach ($focuses as $foc) {
- if (isset($suma['minismCollection'])) {
- $foki[$foc][] = "minism";
- } else {
- $foki[$foc][] = $suma['name']->__toString();
- }
- }
- } else {
- # Wenn keine foki für diese Suchmaschine angegeben sind
- if (isset($suma['minismCollection'])) {
- $foki["andere"][] = "minism";
- } else {
- $foki["andere"][] = $suma['name']->__toString();
- }
- }
- }
- }
-
- # Findet die Namen der aktuell eingeschalteten Sumas
- $realEngNames = [];
- foreach ($enabledSearchengines as $realEng) {
- $nam = $realEng["name"]->__toString();
- if ($nam !== "qualigo" && $nam !== "overtureAds") {
- $realEngNames[] = $nam;
- }
- }
-
- # Anschließend werden diese beiden Listen verglichen (jeweils eine der Fokuslisten für jeden Fokus), um herauszufinden ob sie vielleicht identisch sind. Ist dies der Fall, so hat der Nutzer anscheinend Suchmaschinen eines kompletten Fokus eingestellt. Der Fokus wird dementsprechend angepasst.
- foreach ($foki as $fok => $engines) {
- $isFokus = true;
- $fokiEngNames = [];
- foreach ($engines as $eng) {
- $fokiEngNames[] = $eng;
- }
- # Jede eingeschaltete Engine ist für diesen Fokus geeignet
- foreach ($fokiEngNames as $fen) {
- # Bei Bildersuchen ist uns egal, ob alle Suchmaschinen aus dem Suchfokus eingeschaltet sind, da wir sie eh als Bildersuche anzeigen müssen
- if (!in_array($fen, $realEngNames) && $fok !== "bilder") {
- $isFokus = false;
- }
- }
- # Jede im Fokus erwartete Engine ist auch eingeschaltet
- foreach ($realEngNames as $ren) {
- if (!in_array($ren, $fokiEngNames)) {
- $isFokus = false;
- }
- }
- # Wenn die Listen identisch sind, setze den Fokus um
- if ($isFokus) {
- $this->fokus = $fok;
+ $enginesToWaitFor = [];
+ foreach ($engines as $engine) {
+ if ($engine->cached || !isset($engine->engine->main) || !$engine->engine->main) {
+ continue;
}
+ $enginesToWaitFor[] = $engine;
}
- }
-
- public function waitForResults($enginesToLoad, $overtureEnabled, $canBreak)
- {
$timeStart = microtime(true);
$results = null;
- while (true) {
- $results = Redis::hgetall('search.' . $this->getHashCode());
-
- $ready = true;
- // When every
- $connected = true;
- foreach ($results as $key => $value) {
- if ($value === "waiting" || $value === "connected") {
- $ready = false;
- }
- if ($value === "waiting") {
- $connected = false;
+ while (sizeof($enginesToWaitFor) > 0) {
+ $newEngine = Redis::blpop($this->redisResultWaitingKey, 5);
+ if ($newEngine === null || sizeof($newEngine) !== 2) {
+ continue;
+ } else {
+ $newEngine = $newEngine[1];
+ foreach ($enginesToWaitFor as $index => $engine) {
+ if ($engine->name === $newEngine) {
+ unset($enginesToWaitFor[$index]);
+ break;
+ }
}
}
-
- // If $ready is false at this point, we're waiting for more searchengines
- // But we have to check for the timeout, too
- if (!$connected) {
- $timeStart = microtime(true);
- }
-
- $time = (microtime(true) - $timeStart) * 1000;
- // We will apply the timeout only if it's not Yahoo we're waiting for since they are one the most
- // important search engines.
- $canTimeout = !((isset($results["overture"]) && $results["overture"] === "waiting") || (isset($results["overtureAds"]) && $results["overtureAds"] === "waiting"));
- if ($time > $this->time && $canTimeout) {
- $ready = true;
- }
-
- if ($ready) {
+ if ((microtime(true) - $timeStart) >= 2) {
break;
}
- usleep(50000);
- }
-
- # Wir haben nun so lange wie möglich gewartet. Wir registrieren nun noch die Suchmaschinen, die geanwortet haben.
- foreach ($results as $key => $value) {
- $enginesToLoad[$key] = true;
}
- $this->enginesToLoad = $enginesToLoad;
}
public function retrieveResults($engines)
@@ -1472,10 +1389,9 @@ class MetaGer
return $link;
}
- public function getHashCode()
+ public function getSearchUid()
{
- $string = url()->full();
- return md5($string);
+ return $this->searchUid;
}
# Einfache Getter
@@ -1628,4 +1544,19 @@ class MetaGer
{
return $this->startCount;
}
+
+ public function getRedisResultWaitingKey()
+ {
+ return $this->redisResultWaitingKey;
+ }
+
+ public function getRedisResultEngineList()
+ {
+ return $this->redisResultEngineList;
+ }
+
+ public function getRedisEngineResult()
+ {
+ return $this->redisEngineResult;
+ }
}
diff --git a/app/Models/Quicktips/Quicktips.php b/app/Models/Quicktips/Quicktips.php
index 6e5c053bc..4aca5151b 100644
--- a/app/Models/Quicktips/Quicktips.php
+++ b/app/Models/Quicktips/Quicktips.php
@@ -3,7 +3,6 @@
namespace App\Models\Quicktips;
use App\Jobs\Searcher;
-use Cache;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Illuminate\Support\Facades\Redis;
use Log;
@@ -12,8 +11,8 @@ class Quicktips
{
use DispatchesJobs;
- const QUICKTIP_URL = "http://localhost:63825/1.1/quicktips.xml";
- const QUICKTIP_NAME = "quicktips";
+ const QUICKTIP_URL = "http://localhost:63825/1.1/quicktips.xml";
+ const QUICKTIP_NAME = "quicktips";
const CACHE_DURATION = 60;
private $hash;
@@ -26,18 +25,15 @@ class Quicktips
public function startSearch($search, $locale, $max_time)
{
$url = self::QUICKTIP_URL . "?search=" . $this->normalize_search($search) . "&locale=" . $locale;
-
- $hash = md5($url);
-
# TODO anders weitergeben
- $this->hash = $hash;
+ $this->hash = md5($url);
# TODO cache wieder einbauen (eventuell)
if ( /*!Cache::has($hash)*/true) {
- Redis::hset("search." . $hash, self::QUICKTIP_NAME, "waiting");
+ Redis::hset("search." . $this->hash . ".results." . self::QUICKTIP_NAME, "status", "waiting");
// Queue this search
- $mission = $hash . ";" . base64_encode($url) . ";" . $max_time;
+ $mission = $this->hash . ";" . base64_encode($url) . ";" . $max_time;
Redis::rpush(self::QUICKTIP_NAME . ".queue", $mission);
// Check the current status of Searchers for QUICKTIP_NAME
@@ -85,14 +81,10 @@ class Quicktips
public function retrieveResults($hash)
{
$body = "";
- #if (Cache::has($hash)) {
- $body = Cache::get($hash);
- #} elseif (Redis::hexists('search.' . $hash, self::QUICKTIP_NAME)) {
- $body = Redis::hget('search.' . $hash, self::QUICKTIP_NAME);
- Redis::hdel('search.' . $hash, self::QUICKTIP_NAME);
- Cache::put($hash, $body, self::CACHE_DURATION);
- #}
+ $body = Redis::hget('search.' . $hash . ".results." . self::QUICKTIP_NAME, "response");
+ Redis::del('search.' . $hash . ".results." . self::QUICKTIP_NAME);
+ Redis::del('search.' . $hash . ".ready");
if ($body !== "") {
return $body;
} else {
@@ -157,14 +149,14 @@ class Quicktips
$descr = $quicktip_xml->content->__toString();
// Details
- $details = [];
+ $details = [];
$details_xpath = $quicktip_xml->xpath('mg:details');
if (sizeof($details_xpath) > 0) {
foreach ($details_xpath[0] as $detail_xml) {
$details_title = $detail_xml->title->__toString();
- $details_link = $detail_xml->url->__toString();
+ $details_link = $detail_xml->url->__toString();
$details_descr = $detail_xml->text->__toString();
- $details[] = new \App\Models\Quicktips\Quicktip_detail(
+ $details[] = new \App\Models\Quicktips\Quicktip_detail(
$details_title,
$details_link,
$details_descr
diff --git a/app/Models/Searchengine.php b/app/Models/Searchengine.php
index 03b584048..66c486295 100644
--- a/app/Models/Searchengine.php
+++ b/app/Models/Searchengine.php
@@ -87,7 +87,7 @@ abstract class Searchengine
$this->getString = $this->generateGetString($q);
$this->hash = md5($this->engine->host . $this->getString . $this->engine->port . $this->name);
- $this->resultHash = $metager->getHashCode();
+ $this->resultHash = $metager->getSearchUid();
$this->canCache = $metager->canCache();
}
@@ -102,13 +102,14 @@ abstract class Searchengine
# Prüft, ob die Suche bereits gecached ist, ansonsted wird sie als Job dispatched
public function startSearch(\App\MetaGer $metager)
{
-
if ($this->canCache && Cache::has($this->hash)) {
$this->cached = true;
$this->retrieveResults($metager);
} else {
// We will push the confirmation of the submission to the Result Hash
- Redis::hset('search.' . $this->resultHash, $this->name, "waiting");
+ Redis::hset($metager->getRedisEngineResult() . $this->name, "status", "waiting");
+ Redis::expire($metager->getRedisEngineResult() . $this->name, 60);
+
// We need to submit a action that one of our workers can understand
// The missions are submitted to a redis queue in the following string format
// <ResultHash>;<URL to fetch>
@@ -213,9 +214,8 @@ abstract class Searchengine
if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash)) {
$body = Cache::get($this->hash);
- } elseif (Redis::hexists('search.' . $this->resultHash, $this->name)) {
- $body = Redis::hget('search.' . $this->resultHash, $this->name);
- Redis::hdel('search.' . $this->resultHash, $this->name);
+ } elseif (Redis::hexists($metager->getRedisEngineResult() . $this->name, "response")) {
+ $body = Redis::hget($metager->getRedisEngineResult() . $this->name, "response");
if ($this->canCache && $this->cacheDuration > 0) {
Cache::put($this->hash, $body, $this->cacheDuration);
}
diff --git a/resources/js/scriptResultPage.js b/resources/js/scriptResultPage.js
index d2c1491b3..da858d8ec 100644
--- a/resources/js/scriptResultPage.js
+++ b/resources/js/scriptResultPage.js
@@ -1,7 +1,7 @@
$(document).ready(function () {
botProtection();
-
enableFormResetter();
+ loadMoreResults();
});
function botProtection() {
@@ -50,4 +50,8 @@ function enableFormResetter() {
timeout = null;
}, 500);
});
+}
+
+function loadMoreResults() {
+
}
\ No newline at end of file
diff --git a/resources/views/layouts/resultPage.blade.php b/resources/views/layouts/resultPage.blade.php
index 00a2cd207..3450762ba 100644
--- a/resources/views/layouts/resultPage.blade.php
+++ b/resources/views/layouts/resultPage.blade.php
@@ -11,6 +11,7 @@
<meta name="l" content="{{ LaravelLocalization::getCurrentLocale() }}" />
<meta name="mm" content="{{ $metager->getVerificationId() }}" />
<meta name="mn" content="{{ $metager->getVerificationCount() }}" />
+ <meta name="searchkey" content="{{ $metager->getSearchUid() }}" />
<link rel="search" type="application/opensearchdescription+xml" title="{!! trans('resultPage.opensearch') !!}" href="{{ LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), action('StartpageController@loadPlugin', ['params' => base64_encode(serialize(Request::all()))])) }}">
<link type="text/css" rel="stylesheet" href="{{ mix('css/fontawesome.css') }}" />
<link type="text/css" rel="stylesheet" href="{{ mix('css/fontawesome-solid.css') }}" />
--
GitLab