Commit 196021fc authored by Dominik Hebeler's avatar Dominik Hebeler

First step to optimized request times

parent 978f83ff
......@@ -25,7 +25,7 @@ class MetaGerSearch extends Controller
}*/
#die($request->header('User-Agent'));
$time = microtime();
$time = microtime(true);
# Mit gelieferte Formulardaten parsen und abspeichern:
$metager->parseFormData($request);
......@@ -41,7 +41,6 @@ class MetaGerSearch extends Controller
# Versuchen die Ergebnisse der Quicktips zu laden
$quicktipResults = $quicktips->loadResults();
# Alle Ergebnisse vor der Zusammenführung ranken:
$metager->rankAll();
......
......@@ -85,8 +85,7 @@ class Searcher implements ShouldQueue
$url = base64_decode($mission[1]); // The url to fetch
$timeout = $mission[2]; // Timeout from the MetaGer process in ms
$medianFetchTime = $this->getFetchTime(); // The median Fetch time of the search engine in ms
Redis::hset('search.' . $hashValue, $this->name, "connected");
Redis::hset('search.' . $hashValue . ".results." . $this->name, "status", "connected");
$result = $this->retrieveUrl($url);
$this->storeResult($result, $poptime, $hashValue);
......@@ -99,7 +98,7 @@ class Searcher implements ShouldQueue
// In sync mode every Searcher may only retrieve one result because it would block
// the execution of the remaining code otherwise:
if (getenv("QUEUE_DRIVER") === "sync"
if (getenv("QUEUE_CONNECTION") === "sync"
|| $this->counter > $this->MAX_REQUESTS
|| (microtime(true) - $this->startTime) > $this->MAX_TIME) {
break;
......@@ -161,16 +160,24 @@ class Searcher implements ShouldQueue
// Set this URL to the Curl handle
curl_setopt($this->ch, CURLOPT_URL, $url);
$result = curl_exec($this->ch);
$this->connectionInfo = curl_getinfo($this->ch);
return $result;
}
private function storeResult($result, $poptime, $hashValue)
{
Redis::hset('search.' . $hashValue, $this->name, $result);
if ($this->name === "zeitde") {
sleep(3);
}
$pipeline = Redis::pipeline();
$pipeline->hset('search.' . $hashValue . ".results." . $this->name, "response", $result);
// After 60 seconds the results should be read by the MetaGer Process and stored in the Cache instead
Redis::expire('search.' . $hashValue, 60);
$pipeline->expire('search.' . $hashValue . ".results." . $this->name, 60);
$pipeline->rpush('search.' . $hashValue . ".ready", $this->name);
$pipeline->expire('search.' . $hashValue . ".ready", 60);
$pipeline->sadd('search.' . $hashValue . ".engines", $this->name);
$pipeline->expire('search.' . $hashValue . ".engines", 60);
$pipeline->execute();
$this->lastTime = microtime(true);
}
......
......@@ -59,6 +59,8 @@ class MetaGer
protected $languageDetect;
protected $verificationId;
protected $verificationCount;
protected $searchUid;
protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult;
public function __construct()
{
......@@ -90,6 +92,15 @@ class MetaGer
} catch (ConnectionException $e) {
$this->canCache = false;
}
$this->canCache = false;
$this->searchUid = md5(uniqid());
$redisPrefix = "search";
# This is a list on which the MetaGer process can do a blocking pop to wait for new results
$this->redisResultWaitingKey = $redisPrefix . "." . $this->searchUid . ".ready";
# This is a list of searchengines which have delivered results for this search
$this->redisResultEngineList = $redisPrefix . "." . $this->searchUid . ".engines";
# This is the key where the results of the engine are stored as well as some statistical data
$this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results.";
}
# Erstellt aus den gesammelten Ergebnissen den View
......@@ -512,7 +523,7 @@ class MetaGer
public function createQuicktips()
{
# Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
$quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime(), $this->getHashCode());
$quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime());
return $quicktips;
}
......@@ -534,7 +545,6 @@ class MetaGer
if (empty($this->sumaFile->foki->{$this->fokus})) {
$this->fokus = "web";
}
foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
# Check if this engine is disabled and can't be used
$disabled = empty($this->sumaFile->sumas->{$suma}->disabled) ? false : $this->sumaFile->sumas->{$suma}->disabled;
......@@ -589,8 +599,6 @@ class MetaGer
$this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
}
#die(var_dump($this->enabledSearchengines));
if (sizeof($this->enabledSearchengines) === 0) {
$filter = "";
foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
......@@ -607,6 +615,7 @@ class MetaGer
$counter = 0;
if ($this->requestIsCached($request)) {
# If this is a page other than 1 the request is "cached"
$engines = $this->getCachedEngines($request);
# We need to edit some Options of the Cached Search Engines
foreach ($engines as $engine) {
......@@ -628,19 +637,7 @@ class MetaGer
* welche Suchmaschinen nicht rechtzeitig geantwortet haben.
*/
$enginesToLoad = [];
$canBreak = false;
foreach ($engines as $engine) {
if ($engine->cached) {
if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
$canBreak = true;
}
} else {
$enginesToLoad[$engine->name] = false;
}
}
$this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);
$this->waitForResults($engines);
$this->retrieveResults($engines);
foreach ($engines as $engine) {
......@@ -788,115 +785,35 @@ class MetaGer
return $engines;
}
# Passt den Suchfokus an, falls für einen Fokus genau alle vorhandenen Sumas eingeschaltet sind
public function adjustFocus($sumas, $enabledSearchengines)
public function waitForResults($engines)
{
# Findet für alle Foki die enthaltenen Sumas
$foki = []; # [fokus][suma] => [suma]
foreach ($sumas as $suma) {
if ((!$this->sumaIsDisabled($suma)) && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
if (isset($suma['type'])) {
# Wenn foki für diese Suchmaschine angegeben sind
$focuses = explode(",", $suma['type']->__toString());
foreach ($focuses as $foc) {
if (isset($suma['minismCollection'])) {
$foki[$foc][] = "minism";
} else {
$foki[$foc][] = $suma['name']->__toString();
}
}
} else {
# Wenn keine foki für diese Suchmaschine angegeben sind
if (isset($suma['minismCollection'])) {
$foki["andere"][] = "minism";
} else {
$foki["andere"][] = $suma['name']->__toString();
}
}
}
}
# Findet die Namen der aktuell eingeschalteten Sumas
$realEngNames = [];
foreach ($enabledSearchengines as $realEng) {
$nam = $realEng["name"]->__toString();
if ($nam !== "qualigo" && $nam !== "overtureAds") {
$realEngNames[] = $nam;
}
}
# Anschließend werden diese beiden Listen verglichen (jeweils eine der Fokuslisten für jeden Fokus), um herauszufinden ob sie vielleicht identisch sind. Ist dies der Fall, so hat der Nutzer anscheinend Suchmaschinen eines kompletten Fokus eingestellt. Der Fokus wird dementsprechend angepasst.
foreach ($foki as $fok => $engines) {
$isFokus = true;
$fokiEngNames = [];
foreach ($engines as $eng) {
$fokiEngNames[] = $eng;
}
# Jede eingeschaltete Engine ist für diesen Fokus geeignet
foreach ($fokiEngNames as $fen) {
# Bei Bildersuchen ist uns egal, ob alle Suchmaschinen aus dem Suchfokus eingeschaltet sind, da wir sie eh als Bildersuche anzeigen müssen
if (!in_array($fen, $realEngNames) && $fok !== "bilder") {
$isFokus = false;
}
}
# Jede im Fokus erwartete Engine ist auch eingeschaltet
foreach ($realEngNames as $ren) {
if (!in_array($ren, $fokiEngNames)) {
$isFokus = false;
}
}
# Wenn die Listen identisch sind, setze den Fokus um
if ($isFokus) {
$this->fokus = $fok;
$enginesToWaitFor = [];
foreach ($engines as $engine) {
if ($engine->cached || !isset($engine->engine->main) || !$engine->engine->main) {
continue;
}
$enginesToWaitFor[] = $engine;
}
}
public function waitForResults($enginesToLoad, $overtureEnabled, $canBreak)
{
$timeStart = microtime(true);
$results = null;
while (true) {
$results = Redis::hgetall('search.' . $this->getHashCode());
$ready = true;
// When every
$connected = true;
foreach ($results as $key => $value) {
if ($value === "waiting" || $value === "connected") {
$ready = false;
}
if ($value === "waiting") {
$connected = false;
while (sizeof($enginesToWaitFor) > 0) {
$newEngine = Redis::blpop($this->redisResultWaitingKey, 5);
if ($newEngine === null || sizeof($newEngine) !== 2) {
continue;
} else {
$newEngine = $newEngine[1];
foreach ($enginesToWaitFor as $index => $engine) {
if ($engine->name === $newEngine) {
unset($enginesToWaitFor[$index]);
break;
}
}
}
// If $ready is false at this point, we're waiting for more searchengines
// But we have to check for the timeout, too
if (!$connected) {
$timeStart = microtime(true);
}
$time = (microtime(true) - $timeStart) * 1000;
// We will apply the timeout only if it's not Yahoo we're waiting for since they are one the most
// important search engines.
$canTimeout = !((isset($results["overture"]) && $results["overture"] === "waiting") || (isset($results["overtureAds"]) && $results["overtureAds"] === "waiting"));
if ($time > $this->time && $canTimeout) {
$ready = true;
}
if ($ready) {
if ((microtime(true) - $timeStart) >= 2) {
break;
}
usleep(50000);
}
# Wir haben nun so lange wie möglich gewartet. Wir registrieren nun noch die Suchmaschinen, die geanwortet haben.
foreach ($results as $key => $value) {
$enginesToLoad[$key] = true;
}
$this->enginesToLoad = $enginesToLoad;
}
public function retrieveResults($engines)
......@@ -1472,10 +1389,9 @@ class MetaGer
return $link;
}
public function getHashCode()
public function getSearchUid()
{
$string = url()->full();
return md5($string);
return $this->searchUid;
}
# Einfache Getter
......@@ -1628,4 +1544,19 @@ class MetaGer
{
return $this->startCount;
}
public function getRedisResultWaitingKey()
{
return $this->redisResultWaitingKey;
}
public function getRedisResultEngineList()
{
return $this->redisResultEngineList;
}
public function getRedisEngineResult()
{
return $this->redisEngineResult;
}
}
......@@ -3,7 +3,6 @@
namespace App\Models\Quicktips;
use App\Jobs\Searcher;
use Cache;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Illuminate\Support\Facades\Redis;
use Log;
......@@ -12,8 +11,8 @@ class Quicktips
{
use DispatchesJobs;
const QUICKTIP_URL = "http://localhost:63825/1.1/quicktips.xml";
const QUICKTIP_NAME = "quicktips";
const QUICKTIP_URL = "http://localhost:63825/1.1/quicktips.xml";
const QUICKTIP_NAME = "quicktips";
const CACHE_DURATION = 60;
private $hash;
......@@ -26,18 +25,15 @@ class Quicktips
public function startSearch($search, $locale, $max_time)
{
$url = self::QUICKTIP_URL . "?search=" . $this->normalize_search($search) . "&locale=" . $locale;
$hash = md5($url);
# TODO anders weitergeben
$this->hash = $hash;
$this->hash = md5($url);
# TODO cache wieder einbauen (eventuell)
if ( /*!Cache::has($hash)*/true) {
Redis::hset("search." . $hash, self::QUICKTIP_NAME, "waiting");
Redis::hset("search." . $this->hash . ".results." . self::QUICKTIP_NAME, "status", "waiting");
// Queue this search
$mission = $hash . ";" . base64_encode($url) . ";" . $max_time;
$mission = $this->hash . ";" . base64_encode($url) . ";" . $max_time;
Redis::rpush(self::QUICKTIP_NAME . ".queue", $mission);
// Check the current status of Searchers for QUICKTIP_NAME
......@@ -85,14 +81,10 @@ class Quicktips
public function retrieveResults($hash)
{
$body = "";
#if (Cache::has($hash)) {
$body = Cache::get($hash);
#} elseif (Redis::hexists('search.' . $hash, self::QUICKTIP_NAME)) {
$body = Redis::hget('search.' . $hash, self::QUICKTIP_NAME);
Redis::hdel('search.' . $hash, self::QUICKTIP_NAME);
Cache::put($hash, $body, self::CACHE_DURATION);
#}
$body = Redis::hget('search.' . $hash . ".results." . self::QUICKTIP_NAME, "response");
Redis::del('search.' . $hash . ".results." . self::QUICKTIP_NAME);
Redis::del('search.' . $hash . ".ready");
if ($body !== "") {
return $body;
} else {
......@@ -157,14 +149,14 @@ class Quicktips
$descr = $quicktip_xml->content->__toString();
// Details
$details = [];
$details = [];
$details_xpath = $quicktip_xml->xpath('mg:details');
if (sizeof($details_xpath) > 0) {
foreach ($details_xpath[0] as $detail_xml) {
$details_title = $detail_xml->title->__toString();
$details_link = $detail_xml->url->__toString();
$details_link = $detail_xml->url->__toString();
$details_descr = $detail_xml->text->__toString();
$details[] = new \App\Models\Quicktips\Quicktip_detail(
$details[] = new \App\Models\Quicktips\Quicktip_detail(
$details_title,
$details_link,
$details_descr
......
......@@ -87,7 +87,7 @@ abstract class Searchengine
$this->getString = $this->generateGetString($q);
$this->hash = md5($this->engine->host . $this->getString . $this->engine->port . $this->name);
$this->resultHash = $metager->getHashCode();
$this->resultHash = $metager->getSearchUid();
$this->canCache = $metager->canCache();
}
......@@ -102,13 +102,14 @@ abstract class Searchengine
# Prüft, ob die Suche bereits gecached ist, ansonsted wird sie als Job dispatched
public function startSearch(\App\MetaGer $metager)
{
if ($this->canCache && Cache::has($this->hash)) {
$this->cached = true;
$this->retrieveResults($metager);
} else {
// We will push the confirmation of the submission to the Result Hash
Redis::hset('search.' . $this->resultHash, $this->name, "waiting");
Redis::hset($metager->getRedisEngineResult() . $this->name, "status", "waiting");
Redis::expire($metager->getRedisEngineResult() . $this->name, 60);
// We need to submit a action that one of our workers can understand
// The missions are submitted to a redis queue in the following string format
// <ResultHash>;<URL to fetch>
......@@ -213,9 +214,8 @@ abstract class Searchengine
if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash)) {
$body = Cache::get($this->hash);
} elseif (Redis::hexists('search.' . $this->resultHash, $this->name)) {
$body = Redis::hget('search.' . $this->resultHash, $this->name);
Redis::hdel('search.' . $this->resultHash, $this->name);
} elseif (Redis::hexists($metager->getRedisEngineResult() . $this->name, "response")) {
$body = Redis::hget($metager->getRedisEngineResult() . $this->name, "response");
if ($this->canCache && $this->cacheDuration > 0) {
Cache::put($this->hash, $body, $this->cacheDuration);
}
......
$(document).ready(function () {
botProtection();
enableFormResetter();
loadMoreResults();
});
function botProtection() {
......@@ -50,4 +50,8 @@ function enableFormResetter() {
timeout = null;
}, 500);
});
}
function loadMoreResults() {
}
\ No newline at end of file
......@@ -11,6 +11,7 @@
<meta name="l" content="{{ LaravelLocalization::getCurrentLocale() }}" />
<meta name="mm" content="{{ $metager->getVerificationId() }}" />
<meta name="mn" content="{{ $metager->getVerificationCount() }}" />
<meta name="searchkey" content="{{ $metager->getSearchUid() }}" />
<link rel="search" type="application/opensearchdescription+xml" title="{!! trans('resultPage.opensearch') !!}" href="{{ LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), action('StartpageController@loadPlugin', ['params' => base64_encode(serialize(Request::all()))])) }}">
<link type="text/css" rel="stylesheet" href="{{ mix('css/fontawesome.css') }}" />
<link type="text/css" rel="stylesheet" href="{{ mix('css/fontawesome-solid.css') }}" />
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment