Commit 40bc1b99 authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Merge branch '864-optimize-request-times' into 'development'

Resolve "Optimize request times"

Closes #864

See merge request !1386
parents c502f47f 11147134
...@@ -4,7 +4,10 @@ namespace App\Http\Controllers; ...@@ -4,7 +4,10 @@ namespace App\Http\Controllers;
use App; use App;
use App\MetaGer; use App\MetaGer;
use Cache;
use Illuminate\Http\Request; use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
use View;
const TIP_SERVER = 'http://metager3.de:63825/tips.xml'; const TIP_SERVER = 'http://metager3.de:63825/tips.xml';
...@@ -19,13 +22,6 @@ class MetaGerSearch extends Controller ...@@ -19,13 +22,6 @@ class MetaGerSearch extends Controller
return redirect()->to('https://maps.metager.de/map/' . $searchinput . '/1240908.5493525574,6638783.2192695495,6'); return redirect()->to('https://maps.metager.de/map/' . $searchinput . '/1240908.5493525574,6638783.2192695495,6');
} }
/*if ($focus !== "angepasst" && $this->startsWith($focus, "focus_")) {
$metager->parseFormData($request);
return $metager->createView();
}*/
#die($request->header('User-Agent'));
$time = microtime();
# Mit gelieferte Formulardaten parsen und abspeichern: # Mit gelieferte Formulardaten parsen und abspeichern:
$metager->parseFormData($request); $metager->parseFormData($request);
...@@ -39,19 +35,132 @@ class MetaGerSearch extends Controller ...@@ -39,19 +35,132 @@ class MetaGerSearch extends Controller
# auf Ergebnisse warten und die Ergebnisse laden # auf Ergebnisse warten und die Ergebnisse laden
$metager->createSearchEngines($request); $metager->createSearchEngines($request);
$metager->startSearch();
$metager->waitForMainResults();
$metager->retrieveResults();
# Versuchen die Ergebnisse der Quicktips zu laden # Versuchen die Ergebnisse der Quicktips zu laden
$quicktipResults = $quicktips->loadResults(); $quicktipResults = $quicktips->loadResults();
# Alle Ergebnisse vor der Zusammenführung ranken: # Alle Ergebnisse vor der Zusammenführung ranken:
$metager->rankAll(); $metager->rankAll();
# Ergebnisse der Suchmaschinen kombinieren: # Ergebnisse der Suchmaschinen kombinieren:
$metager->prepareResults(); $metager->prepareResults();
# Save the results in Redis
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$pipeline = $redis->pipeline();
foreach ($metager->getResults() as $result) {
$pipeline->rpush($metager->getRedisCurrentResultList(), base64_encode(serialize($result)));
}
$pipeline->expire($metager->getRedisCurrentResultList(), env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->execute();
# Die Ausgabe erstellen: # Die Ausgabe erstellen:
return $metager->createView($quicktipResults); return $metager->createView($quicktipResults);
} }
public function loadMore(Request $request)
{
/**
* There are three forms of requests to the resultpage
* 1. Initial Request: Loads the fastest searchengines and sends them to the user
* 2. Load more results (with JS): Loads new search engines that answered after the initial request was send
* 3. Load more results (without JS): Loads new search engines that answered within 1s timeout
*/
if ($request->filled('loadMore') && $request->filled('script') && $request->input('script') === "yes") {
return $this->loadMoreJS($request);
}
}
private function loadMoreJS(Request $request)
{
# Create a MetaGer Instance with the supplied hash
$hash = $request->input('loadMore', '');
$metager = new MetaGer($hash);
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$result = [];
# Check if there should be more results
$stats = $redis->hgetall($metager->getRedisEngineResult() . "status");
$stats["startTime"] = floatval($stats["startTime"]);
$stats["engineCount"] = intval($stats["engineCount"]);
$stats["engineAnswered"] = intval($stats["engineAnswered"]);
$stats["engineDelivered"] = intval($stats["engineDelivered"]);
$result["finished"] = true;
$result["engineCount"] = $stats["engineCount"];
$result["engineAnswered"] = $stats["engineAnswered"];
$result["engineDelivered"] = $stats["engineDelivered"];
$result["timeWaiting"] = microtime(true) - $stats["startTime"];
# Check if we can abort
if ($stats["engineAnswered"] > $stats["engineDelivered"]/*&& $result["timeWaiting"] <= 10 */) {
$metager->parseFormData($request);
# Nach Spezialsuchen überprüfen:
$metager->checkSpecialSearches($request);
# Read which search engines are new
$newEngines = [];
while (($engine = $redis->lpop($metager->getRedisResultWaitingKey())) != null) {
$result["engineDelivered"]++;
$newEngines[$engine] = $metager->getSumaFile()->sumas->{$engine};
}
$cache = Cache::get($hash);
if ($cache != null) {
$metager->setNext(unserialize($cache)["engines"]);
}
# Check if this request is not for page one
$metager->setEngines($request, $newEngines);
# Add the results already delivered to the user
$results = $redis->lrange($metager->getRedisCurrentResultList(), 0, -1);
foreach ($results as $index => $oldResult) {
$results[$index] = unserialize(base64_decode($oldResult));
$results[$index]->new = false;
}
$metager->setResults($results);
$metager->retrieveResults();
$metager->rankAll();
$metager->prepareResults();
$result["nextSearchLink"] = $metager->nextSearchLink();
$results = $metager->getResults();
foreach ($results as $index => $resultTmp) {
if ($resultTmp->new) {
if ($metager->getFokus() !== "bilder") {
$view = View::make('layouts.result', ['result' => $resultTmp, 'metager' => $metager]);
$html = $view->render();
$result['newResults'][$index] = $html;
$result["imagesearch"] = false;
} else {
$view = View::make('layouts.image_result', ['result' => $resultTmp, 'metager' => $metager]);
$html = $view->render();
$result['newResults'][$index] = $html;
$result["imagesearch"] = true;
}
}
}
# Save the results in Redis
$pipeline = $redis->pipeline();
$pipeline->hincrby($metager->getRedisEngineResult() . "status", "engineDelivered", sizeof($newEngines));
$pipeline->hset($metager->getRedisEngineResult() . "status", "nextSearchLink", $result["nextSearchLink"]);
foreach ($metager->getResults() as $resultTmp) {
$resultTmp->new = false;
$pipeline->rpush($metager->getRedisCurrentResultList(), base64_encode(serialize($resultTmp)));
}
$pipeline->expire($metager->getRedisCurrentResultList(), env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->execute();
}
return response()->json($result);
}
public function botProtection($redirect) public function botProtection($redirect)
{ {
$hash = md5(date('YmdHi')); $hash = md5(date('YmdHi'));
......
...@@ -85,8 +85,7 @@ class Searcher implements ShouldQueue ...@@ -85,8 +85,7 @@ class Searcher implements ShouldQueue
$url = base64_decode($mission[1]); // The url to fetch $url = base64_decode($mission[1]); // The url to fetch
$timeout = $mission[2]; // Timeout from the MetaGer process in ms $timeout = $mission[2]; // Timeout from the MetaGer process in ms
$medianFetchTime = $this->getFetchTime(); // The median Fetch time of the search engine in ms $medianFetchTime = $this->getFetchTime(); // The median Fetch time of the search engine in ms
Redis::hset('search.' . $hashValue, $this->name, "connected"); Redis::hset('search.' . $hashValue . ".results." . $this->name, "status", "connected");
$result = $this->retrieveUrl($url); $result = $this->retrieveUrl($url);
$this->storeResult($result, $poptime, $hashValue); $this->storeResult($result, $poptime, $hashValue);
...@@ -99,7 +98,7 @@ class Searcher implements ShouldQueue ...@@ -99,7 +98,7 @@ class Searcher implements ShouldQueue
// In sync mode every Searcher may only retrieve one result because it would block // In sync mode every Searcher may only retrieve one result because it would block
// the execution of the remaining code otherwise: // the execution of the remaining code otherwise:
if (getenv("QUEUE_DRIVER") === "sync" if (getenv("QUEUE_CONNECTION") === "sync"
|| $this->counter > $this->MAX_REQUESTS || $this->counter > $this->MAX_REQUESTS
|| (microtime(true) - $this->startTime) > $this->MAX_TIME) { || (microtime(true) - $this->startTime) > $this->MAX_TIME) {
break; break;
...@@ -161,16 +160,24 @@ class Searcher implements ShouldQueue ...@@ -161,16 +160,24 @@ class Searcher implements ShouldQueue
// Set this URL to the Curl handle // Set this URL to the Curl handle
curl_setopt($this->ch, CURLOPT_URL, $url); curl_setopt($this->ch, CURLOPT_URL, $url);
$result = curl_exec($this->ch); $result = curl_exec($this->ch);
$this->connectionInfo = curl_getinfo($this->ch); $this->connectionInfo = curl_getinfo($this->ch);
return $result; return $result;
} }
private function storeResult($result, $poptime, $hashValue) private function storeResult($result, $poptime, $hashValue)
{ {
Redis::hset('search.' . $hashValue, $this->name, $result); $redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$pipeline = $redis->pipeline();
$pipeline->hset('search.' . $hashValue . ".results." . $this->name, "response", $result);
$pipeline->hset('search.' . $hashValue . ".results." . $this->name, "delivered", "0");
$pipeline->hincrby('search.' . $hashValue . ".results.status", "engineAnswered", 1);
// After 60 seconds the results should be read by the MetaGer Process and stored in the Cache instead // After 60 seconds the results should be read by the MetaGer Process and stored in the Cache instead
Redis::expire('search.' . $hashValue, 60); $pipeline->expire('search.' . $hashValue . ".results." . $this->name, env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->rpush('search.' . $hashValue . ".ready", $this->name);
$pipeline->expire('search.' . $hashValue . ".ready", env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->sadd('search.' . $hashValue . ".engines", $this->name);
$pipeline->expire('search.' . $hashValue . ".engines", env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->execute();
$this->lastTime = microtime(true); $this->lastTime = microtime(true);
} }
......
...@@ -46,6 +46,7 @@ class MetaGer ...@@ -46,6 +46,7 @@ class MetaGer
protected $agent; protected $agent;
protected $apiKey = ""; protected $apiKey = "";
protected $apiAuthorized = false; protected $apiAuthorized = false;
protected $next = [];
# Konfigurationseinstellungen: # Konfigurationseinstellungen:
protected $sumaFile; protected $sumaFile;
protected $mobile; protected $mobile;
...@@ -59,8 +60,10 @@ class MetaGer ...@@ -59,8 +60,10 @@ class MetaGer
protected $languageDetect; protected $languageDetect;
protected $verificationId; protected $verificationId;
protected $verificationCount; protected $verificationCount;
protected $searchUid;
protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult, $redisCurrentResultList;
public function __construct() public function __construct($hash = "")
{ {
# Timer starten # Timer starten
$this->starttime = microtime(true); $this->starttime = microtime(true);
...@@ -90,6 +93,21 @@ class MetaGer ...@@ -90,6 +93,21 @@ class MetaGer
} catch (ConnectionException $e) { } catch (ConnectionException $e) {
$this->canCache = false; $this->canCache = false;
} }
if ($hash === "") {
$this->searchUid = md5(uniqid());
} else {
$this->searchUid = $hash;
}
$redisPrefix = "search";
# This is a list on which the MetaGer process can do a blocking pop to wait for new results
$this->redisResultWaitingKey = $redisPrefix . "." . $this->searchUid . ".ready";
# This is a list of searchengines which have delivered results for this search
$this->redisResultEngineList = $redisPrefix . "." . $this->searchUid . ".engines";
# This is the key where the results of the engine are stored as well as some statistical data
$this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results.";
# A list of all search results already delivered to the user (sorted of course)
$this->redisCurrentResultList = $redisPrefix . "." . $this->searchUid . ".currentResults";
} }
# Erstellt aus den gesammelten Ergebnissen den View # Erstellt aus den gesammelten Ergebnissen den View
...@@ -217,21 +235,8 @@ class MetaGer ...@@ -217,21 +235,8 @@ class MetaGer
public function prepareResults() public function prepareResults()
{ {
$engines = $this->engines; $engines = $this->engines;
// combine // combine
$combinedResults = $this->combineResults($engines); $this->combineResults($engines);
# Wir bestimmen die Sprache eines jeden Suchergebnisses
$this->results = $this->addLangCodes($this->results);
// sort
//$sortedResults = $this->sortResults($engines);
// filter
// augment (boost&adgoal)
// authorize
if ($this->apiKey) {
$this->apiAuthorized = $this->authorize($this->apiKey);
}
// misc (WiP) // misc (WiP)
if ($this->fokus == "nachrichten") { if ($this->fokus == "nachrichten") {
$this->results = array_filter($this->results, function ($v, $k) { $this->results = array_filter($this->results, function ($v, $k) {
...@@ -274,40 +279,6 @@ class MetaGer ...@@ -274,40 +279,6 @@ class MetaGer
$counter = 0; $counter = 0;
$firstRank = 0; $firstRank = 0;
if (isset($this->startForwards)) {
$this->startCount = $this->startForwards;
} elseif (isset($this->startBackwards)) {
$this->startCount = $this->startBackwards - count($this->results) - 1;
} else {
$this->startCount = 0;
}
foreach ($this->results as $result) {
if ($counter === 0) {
$firstRank = $result->rank;
}
$counter++;
$result->number = $counter + $this->startCount;
$confidence = 0;
if ($firstRank > 0) {
$confidence = $result->rank / $firstRank;
} else {
$confidence = 0;
}
if ($confidence > 0.65) {
$result->color = "#FF4000";
} elseif ($confidence > 0.4) {
$result->color = "#FF0080";
} elseif ($confidence > 0.2) {
$result->color = "#C000C0";
} else {
$result->color = "#000000";
}
}
if (count($this->results) <= 0) { if (count($this->results) <= 0) {
if (strlen($this->site) > 0) { if (strlen($this->site) > 0) {
$no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl); $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
...@@ -321,62 +292,15 @@ class MetaGer ...@@ -321,62 +292,15 @@ class MetaGer
$page = $this->page + 1; $page = $this->page + 1;
$this->next = [ $this->next = [
'page' => $page, 'page' => $page,
'startForwards' => $this->results[count($this->results) - 1]->number,
'engines' => $this->next, 'engines' => $this->next,
]; ];
Cache::put(md5(serialize($this->next)), serialize($this->next), 60); Cache::put($this->getSearchUid(), serialize($this->next), 60);
} else { } else {
$this->next = []; $this->next = [];
} }
} }
private function addLangCodes($results)
{
# Wenn es keine Ergebnisse gibt, brauchen wir uns gar nicht erst zu bemühen
if (sizeof($results) === 0) {
return $results;
}
# Bei der Spracheinstellung "all" wird nicht gefiltert
if ($this->getLang() === "all") {
return $results;
} else {
# Ansonsten müssen wir jedem Result einen Sprachcode hinzufügen
$id = 0;
$langStrings = [];
foreach ($results as $result) {
# Wir geben jedem Ergebnis eine ID um später die Sprachcodes zuordnen zu können
$result->id = $id;
$langStrings["result_" . $id] = utf8_encode($result->getLangString());
$id++;
}
# Wir schreiben die Strings in eine temporäre JSON-Datei,
# Da das Array unter umständen zu groß ist für eine direkte Übergabe an das Skript
$filename = "/tmp/" . getmypid();
file_put_contents($filename, json_encode($langStrings));
$langDetectorPath = app_path() . "/Models/lang.pl";
$lang = exec("echo '$filename' | $langDetectorPath");
$lang = json_decode($lang, true);
# Wir haben nun die Sprachcodes der einzelnen Ergebnisse.
# Diese müssen wir nur noch korrekt zuordnen, dann sind wir fertig.
foreach ($lang as $key => $langCode) {
# Prefix vom Key entfernen:
$id = intval(str_replace("result_", "", $key));
foreach ($this->results as $result) {
if ($result->id === $id) {
$result->langCode = $langCode;
break;
}
}
}
return $results;
}
}
public function combineResults($engines) public function combineResults($engines)
{ {
foreach ($engines as $engine) { foreach ($engines as $engine) {
...@@ -407,6 +331,9 @@ class MetaGer ...@@ -407,6 +331,9 @@ class MetaGer
$tldList = ""; $tldList = "";
try { try {
foreach ($results as $result) { foreach ($results as $result) {
if (!$result->new) {
continue;
}
$link = $result->anzeigeLink; $link = $result->anzeigeLink;
if (strpos($link, "http") !== 0) { if (strpos($link, "http") !== 0) {
$link = "http://" . $link; $link = "http://" . $link;
...@@ -431,7 +358,7 @@ class MetaGer ...@@ -431,7 +358,7 @@ class MetaGer
$hash = $el[1]; $hash = $el[1];
foreach ($results as $result) { foreach ($results as $result) {
if ($hoster === $result->tld && !$result->partnershop) { if ($result->new && $hoster === $result->tld && !$result->partnershop) {
# Hier ist ein Advertiser: # Hier ist ein Advertiser:
# Das Logo hinzufügen: # Das Logo hinzufügen:
if ($result->image !== "") { if ($result->image !== "") {
...@@ -512,7 +439,7 @@ class MetaGer ...@@ -512,7 +439,7 @@ class MetaGer
public function createQuicktips() public function createQuicktips()
{ {
# Die quicktips werden als job erstellt und zur Abarbeitung freigegeben # Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
$quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime(), $this->getHashCode()); $quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime());
return $quicktips; return $quicktips;
} }
...@@ -534,7 +461,6 @@ class MetaGer ...@@ -534,7 +461,6 @@ class MetaGer
if (empty($this->sumaFile->foki->{$this->fokus})) { if (empty($this->sumaFile->foki->{$this->fokus})) {
$this->fokus = "web"; $this->fokus = "web";
} }
foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) { foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
# Check if this engine is disabled and can't be used # Check if this engine is disabled and can't be used
$disabled = empty($this->sumaFile->sumas->{$suma}->disabled) ? false : $this->sumaFile->sumas->{$suma}->disabled; $disabled = empty($this->sumaFile->sumas->{$suma}->disabled) ? false : $this->sumaFile->sumas->{$suma}->disabled;
...@@ -589,8 +515,6 @@ class MetaGer ...@@ -589,8 +515,6 @@ class MetaGer
$this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"}; $this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
} }
#die(var_dump($this->enabledSearchengines));
if (sizeof($this->enabledSearchengines) === 0) { if (sizeof($this->enabledSearchengines) === 0) {
$filter = ""; $filter = "";
foreach ($this->queryFilter as $queryFilter => $filterPhrase) { foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
...@@ -601,53 +525,36 @@ class MetaGer ...@@ -601,53 +525,36 @@ class MetaGer
'filter' => $filter]); 'filter' => $filter]);
$this->errors[] = $error; $this->errors[] = $error;
} }
$engines = []; $engines = [];
$typeslist = []; $typeslist = [];
$counter = 0; $counter = 0;
$this->setEngines($request);
}
public function setEngines(Request $request, $enabledSearchengines = [])
{
if ($this->requestIsCached($request)) { if ($this->requestIsCached($request)) {
# If this is a page other than 1 the request is "cached"
$engines = $this->getCachedEngines($request); $engines = $this->getCachedEngines($request);
# We need to edit some Options of the Cached Search Engines # We need to edit some Options of the Cached Search Engines
foreach ($engines as $engine) { foreach ($engines as $engine) {
$engine->setResultHash($this->getHashCode()); $engine->setResultHash($this->getSearchUid());
} }
$this->engines = $engines;
} else { } else {
$engines = $this->actuallyCreateSearchEngines($this->enabledSearchengines); if (sizeof($enabledSearchengines) > 0) {
$this->enabledSearchengines = $enabledSearchengines;
}
$this->actuallyCreateSearchEngines($this->enabledSearchengines);
} }
}
public function startSearch()
{
# Wir starten alle Suchen # Wir starten alle Suchen
foreach ($engines as $engine) { foreach ($this->engines as $engine) {
$engine->startSearch($this); $engine->startSearch($this);
} }
/* Wir warten auf die Antwort der Suchmaschinen
* Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
* Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
* $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
* welche Suchmaschinen nicht rechtzeitig geantwortet haben.
*/
$enginesToLoad = [];
$canBreak = false;
foreach ($engines as $engine) {
if ($engine->cached) {
if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
$canBreak = true;
}
} else {
$enginesToLoad[$engine->name] = false;
}
}
$this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);
$this->retrieveResults($engines);
foreach ($engines as $engine) {
if (!empty($engine->totalResults) && $engine->totalResults > $this->totalResults) {
$this->totalResults = $engine->totalResults;
}
}
} }
# Spezielle Suchen und Sumas # Spezielle Suchen und Sumas
...@@ -696,7 +603,7 @@ class MetaGer ...@@ -696,7 +603,7 @@ class MetaGer
$engines[] = $tmp; $engines[] = $tmp;
} }
return $engines;