Skip to content
GitLab
Commit 40bc1b99 authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Merge branch '864-optimize-request-times' into 'development' 

Resolve "Optimize request times"

Closes #864

See merge request !1386
parents c502f47f 11147134
Show whitespace changes
Inline Side-by-side
app/Http/Controllers/MetaGerSearch.php
View file @ 40bc1b99
......@@ -4,7 +4,10 @@ namespace App\Http\Controllers;
use App;
use App\MetaGer;
use Cache;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
use View;
const TIP_SERVER = 'http://metager3.de:63825/tips.xml';
......@@ -19,13 +22,6 @@ class MetaGerSearch extends Controller
return redirect()->to('https://maps.metager.de/map/' . $searchinput . '/1240908.5493525574,6638783.2192695495,6');
}
/*if ($focus !== "angepasst" && $this->startsWith($focus, "focus_")) {
$metager->parseFormData($request);
return $metager->createView();
}*/
#die($request->header('User-Agent'));
$time = microtime();
# Mit gelieferte Formulardaten parsen und abspeichern:
$metager->parseFormData($request);
......@@ -39,19 +35,132 @@ class MetaGerSearch extends Controller
# auf Ergebnisse warten und die Ergebnisse laden
$metager->createSearchEngines($request);
$metager->startSearch();
$metager->waitForMainResults();
$metager->retrieveResults();
# Versuchen die Ergebnisse der Quicktips zu laden
$quicktipResults = $quicktips->loadResults();
# Alle Ergebnisse vor der Zusammenführung ranken:
$metager->rankAll();
# Ergebnisse der Suchmaschinen kombinieren:
$metager->prepareResults();
# Save the results in Redis
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$pipeline = $redis->pipeline();
foreach ($metager->getResults() as $result) {
$pipeline->rpush($metager->getRedisCurrentResultList(), base64_encode(serialize($result)));
}
$pipeline->expire($metager->getRedisCurrentResultList(), env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->execute();
# Die Ausgabe erstellen:
return $metager->createView($quicktipResults);
}
public function loadMore(Request $request)
{
/**
* There are three forms of requests to the resultpage
* 1. Initial Request: Loads the fastest searchengines and sends them to the user
* 2. Load more results (with JS): Loads new search engines that answered after the initial request was send
* 3. Load more results (without JS): Loads new search engines that answered within 1s timeout
*/
if ($request->filled('loadMore') && $request->filled('script') && $request->input('script') === "yes") {
return $this->loadMoreJS($request);
}
}
private function loadMoreJS(Request $request)
{
# Create a MetaGer Instance with the supplied hash
$hash = $request->input('loadMore', '');
$metager = new MetaGer($hash);
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$result = [];
# Check if there should be more results
$stats = $redis->hgetall($metager->getRedisEngineResult() . "status");
$stats["startTime"] = floatval($stats["startTime"]);
$stats["engineCount"] = intval($stats["engineCount"]);
$stats["engineAnswered"] = intval($stats["engineAnswered"]);
$stats["engineDelivered"] = intval($stats["engineDelivered"]);
$result["finished"] = true;
$result["engineCount"] = $stats["engineCount"];
$result["engineAnswered"] = $stats["engineAnswered"];
$result["engineDelivered"] = $stats["engineDelivered"];
$result["timeWaiting"] = microtime(true) - $stats["startTime"];
# Check if we can abort
if ($stats["engineAnswered"] > $stats["engineDelivered"]/*&& $result["timeWaiting"] <= 10 */) {
$metager->parseFormData($request);
# Nach Spezialsuchen überprüfen:
$metager->checkSpecialSearches($request);
# Read which search engines are new
$newEngines = [];
while (($engine = $redis->lpop($metager->getRedisResultWaitingKey())) != null) {
$result["engineDelivered"]++;
$newEngines[$engine] = $metager->getSumaFile()->sumas->{$engine};
}
$cache = Cache::get($hash);
if ($cache != null) {
$metager->setNext(unserialize($cache)["engines"]);
}
# Check if this request is not for page one
$metager->setEngines($request, $newEngines);
# Add the results already delivered to the user
$results = $redis->lrange($metager->getRedisCurrentResultList(), 0, -1);
foreach ($results as $index => $oldResult) {
$results[$index] = unserialize(base64_decode($oldResult));
$results[$index]->new = false;
}
$metager->setResults($results);
$metager->retrieveResults();
$metager->rankAll();
$metager->prepareResults();
$result["nextSearchLink"] = $metager->nextSearchLink();
$results = $metager->getResults();
foreach ($results as $index => $resultTmp) {
if ($resultTmp->new) {
if ($metager->getFokus() !== "bilder") {
$view = View::make('layouts.result', ['result' => $resultTmp, 'metager' => $metager]);
$html = $view->render();
$result['newResults'][$index] = $html;
$result["imagesearch"] = false;
} else {
$view = View::make('layouts.image_result', ['result' => $resultTmp, 'metager' => $metager]);
$html = $view->render();
$result['newResults'][$index] = $html;
$result["imagesearch"] = true;
}
}
}
# Save the results in Redis
$pipeline = $redis->pipeline();
$pipeline->hincrby($metager->getRedisEngineResult() . "status", "engineDelivered", sizeof($newEngines));
$pipeline->hset($metager->getRedisEngineResult() . "status", "nextSearchLink", $result["nextSearchLink"]);
foreach ($metager->getResults() as $resultTmp) {
$resultTmp->new = false;
$pipeline->rpush($metager->getRedisCurrentResultList(), base64_encode(serialize($resultTmp)));
}
$pipeline->expire($metager->getRedisCurrentResultList(), env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->execute();
}
return response()->json($result);
}
public function botProtection($redirect)
{
$hash = md5(date('YmdHi'));
......
app/Jobs/Searcher.php
View file @ 40bc1b99
......@@ -85,8 +85,7 @@ class Searcher implements ShouldQueue
$url = base64_decode($mission[1]); // The url to fetch
$timeout = $mission[2]; // Timeout from the MetaGer process in ms
$medianFetchTime = $this->getFetchTime(); // The median Fetch time of the search engine in ms
Redis::hset('search.' . $hashValue, $this->name, "connected");
Redis::hset('search.' . $hashValue . ".results." . $this->name, "status", "connected");
$result = $this->retrieveUrl($url);
$this->storeResult($result, $poptime, $hashValue);
......@@ -99,7 +98,7 @@ class Searcher implements ShouldQueue
// In sync mode every Searcher may only retrieve one result because it would block
// the execution of the remaining code otherwise:
if (getenv("QUEUE_DRIVER") === "sync"
if (getenv("QUEUE_CONNECTION") === "sync"
|| $this->counter > $this->MAX_REQUESTS
|| (microtime(true) - $this->startTime) > $this->MAX_TIME) {
break;
......@@ -161,16 +160,24 @@ class Searcher implements ShouldQueue
// Set this URL to the Curl handle
curl_setopt($this->ch, CURLOPT_URL, $url);
$result = curl_exec($this->ch);
$this->connectionInfo = curl_getinfo($this->ch);
return $result;
}
private function storeResult($result, $poptime, $hashValue)
{
Redis::hset('search.' . $hashValue, $this->name, $result);
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$pipeline = $redis->pipeline();
$pipeline->hset('search.' . $hashValue . ".results." . $this->name, "response", $result);
$pipeline->hset('search.' . $hashValue . ".results." . $this->name, "delivered", "0");
$pipeline->hincrby('search.' . $hashValue . ".results.status", "engineAnswered", 1);
// After 60 seconds the results should be read by the MetaGer Process and stored in the Cache instead
Redis::expire('search.' . $hashValue, 60);
$pipeline->expire('search.' . $hashValue . ".results." . $this->name, env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->rpush('search.' . $hashValue . ".ready", $this->name);
$pipeline->expire('search.' . $hashValue . ".ready", env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->sadd('search.' . $hashValue . ".engines", $this->name);
$pipeline->expire('search.' . $hashValue . ".engines", env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->execute();
$this->lastTime = microtime(true);
}
......
app/MetaGer.php
View file @ 40bc1b99
......@@ -46,6 +46,7 @@ class MetaGer
protected $agent;
protected $apiKey = "";
protected $apiAuthorized = false;
protected $next = [];
# Konfigurationseinstellungen:
protected $sumaFile;
protected $mobile;
......@@ -59,8 +60,10 @@ class MetaGer
protected $languageDetect;
protected $verificationId;
protected $verificationCount;
protected $searchUid;
protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult, $redisCurrentResultList;
public function __construct()
public function __construct($hash = "")
{
# Timer starten
$this->starttime = microtime(true);
......@@ -90,6 +93,21 @@ class MetaGer
} catch (ConnectionException $e) {
$this->canCache = false;
}
if ($hash === "") {
$this->searchUid = md5(uniqid());
} else {
$this->searchUid = $hash;
}
$redisPrefix = "search";
# This is a list on which the MetaGer process can do a blocking pop to wait for new results
$this->redisResultWaitingKey = $redisPrefix . "." . $this->searchUid . ".ready";
# This is a list of searchengines which have delivered results for this search
$this->redisResultEngineList = $redisPrefix . "." . $this->searchUid . ".engines";
# This is the key where the results of the engine are stored as well as some statistical data
$this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results.";
# A list of all search results already delivered to the user (sorted of course)
$this->redisCurrentResultList = $redisPrefix . "." . $this->searchUid . ".currentResults";
}
# Erstellt aus den gesammelten Ergebnissen den View
......@@ -217,21 +235,8 @@ class MetaGer
public function prepareResults()
{
$engines = $this->engines;
// combine
$combinedResults = $this->combineResults($engines);
# Wir bestimmen die Sprache eines jeden Suchergebnisses
$this->results = $this->addLangCodes($this->results);
// sort
//$sortedResults = $this->sortResults($engines);
// filter
// augment (boost&adgoal)
// authorize
if ($this->apiKey) {
$this->apiAuthorized = $this->authorize($this->apiKey);
}
$this->combineResults($engines);
// misc (WiP)
if ($this->fokus == "nachrichten") {
$this->results = array_filter($this->results, function ($v, $k) {
......@@ -274,40 +279,6 @@ class MetaGer
$counter = 0;
$firstRank = 0;
if (isset($this->startForwards)) {
$this->startCount = $this->startForwards;
} elseif (isset($this->startBackwards)) {
$this->startCount = $this->startBackwards - count($this->results) - 1;
} else {
$this->startCount = 0;
}
foreach ($this->results as $result) {
if ($counter === 0) {
$firstRank = $result->rank;
}
$counter++;
$result->number = $counter + $this->startCount;
$confidence = 0;
if ($firstRank > 0) {
$confidence = $result->rank / $firstRank;
} else {
$confidence = 0;
}
if ($confidence > 0.65) {
$result->color = "#FF4000";
} elseif ($confidence > 0.4) {
$result->color = "#FF0080";
} elseif ($confidence > 0.2) {
$result->color = "#C000C0";
} else {
$result->color = "#000000";
}
}
if (count($this->results) <= 0) {
if (strlen($this->site) > 0) {
$no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
......@@ -321,62 +292,15 @@ class MetaGer
$page = $this->page + 1;
$this->next = [
'page' => $page,
'startForwards' => $this->results[count($this->results) - 1]->number,
'engines' => $this->next,
];
Cache::put(md5(serialize($this->next)), serialize($this->next), 60);
Cache::put($this->getSearchUid(), serialize($this->next), 60);
} else {
$this->next = [];
}
}
private function addLangCodes($results)
{
# Wenn es keine Ergebnisse gibt, brauchen wir uns gar nicht erst zu bemühen
if (sizeof($results) === 0) {
return $results;
}
# Bei der Spracheinstellung "all" wird nicht gefiltert
if ($this->getLang() === "all") {
return $results;
} else {
# Ansonsten müssen wir jedem Result einen Sprachcode hinzufügen
$id = 0;
$langStrings = [];
foreach ($results as $result) {
# Wir geben jedem Ergebnis eine ID um später die Sprachcodes zuordnen zu können
$result->id = $id;
$langStrings["result_" . $id] = utf8_encode($result->getLangString());
$id++;
}
# Wir schreiben die Strings in eine temporäre JSON-Datei,
# Da das Array unter umständen zu groß ist für eine direkte Übergabe an das Skript
$filename = "/tmp/" . getmypid();
file_put_contents($filename, json_encode($langStrings));
$langDetectorPath = app_path() . "/Models/lang.pl";
$lang = exec("echo '$filename' | $langDetectorPath");
$lang = json_decode($lang, true);
# Wir haben nun die Sprachcodes der einzelnen Ergebnisse.
# Diese müssen wir nur noch korrekt zuordnen, dann sind wir fertig.
foreach ($lang as $key => $langCode) {
# Prefix vom Key entfernen:
$id = intval(str_replace("result_", "", $key));
foreach ($this->results as $result) {
if ($result->id === $id) {
$result->langCode = $langCode;
break;
}
}
}
return $results;
}
}
public function combineResults($engines)
{
foreach ($engines as $engine) {
......@@ -407,6 +331,9 @@ class MetaGer
$tldList = "";
try {
foreach ($results as $result) {
if (!$result->new) {
continue;
}
$link = $result->anzeigeLink;
if (strpos($link, "http") !== 0) {
$link = "http://" . $link;
......@@ -431,7 +358,7 @@ class MetaGer
$hash = $el[1];
foreach ($results as $result) {
if ($hoster === $result->tld && !$result->partnershop) {
if ($result->new && $hoster === $result->tld && !$result->partnershop) {
# Hier ist ein Advertiser:
# Das Logo hinzufügen:
if ($result->image !== "") {
......@@ -512,7 +439,7 @@ class MetaGer
public function createQuicktips()
{
# Die quicktips werden als job erstellt und zur Abarbeitung freigegeben
$quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime(), $this->getHashCode());
$quicktips = new \App\Models\Quicktips\Quicktips($this->q, $this->lang, $this->getTime());
return $quicktips;
}
......@@ -534,7 +461,6 @@ class MetaGer
if (empty($this->sumaFile->foki->{$this->fokus})) {
$this->fokus = "web";
}
foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
# Check if this engine is disabled and can't be used
$disabled = empty($this->sumaFile->sumas->{$suma}->disabled) ? false : $this->sumaFile->sumas->{$suma}->disabled;
......@@ -589,8 +515,6 @@ class MetaGer
$this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
}
#die(var_dump($this->enabledSearchengines));
if (sizeof($this->enabledSearchengines) === 0) {
$filter = "";
foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
......@@ -601,52 +525,35 @@ class MetaGer
'filter' => $filter]);
$this->errors[] = $error;
}
$engines = [];
$typeslist = [];
$counter = 0;
$this->setEngines($request);
}
public function setEngines(Request $request, $enabledSearchengines = [])
{
if ($this->requestIsCached($request)) {
# If this is a page other than 1 the request is "cached"
$engines = $this->getCachedEngines($request);
# We need to edit some Options of the Cached Search Engines
foreach ($engines as $engine) {
$engine->setResultHash($this->getHashCode());
$engine->setResultHash($this->getSearchUid());
}
$this->engines = $engines;
} else {
$engines = $this->actuallyCreateSearchEngines($this->enabledSearchengines);
}
# Wir starten alle Suchen
foreach ($engines as $engine) {
$engine->startSearch($this);
if (sizeof($enabledSearchengines) > 0) {
$this->enabledSearchengines = $enabledSearchengines;
}
/* Wir warten auf die Antwort der Suchmaschinen
* Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
* Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
* $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
* welche Suchmaschinen nicht rechtzeitig geantwortet haben.
*/
$enginesToLoad = [];
$canBreak = false;
foreach ($engines as $engine) {
if ($engine->cached) {
if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
$canBreak = true;
}
} else {
$enginesToLoad[$engine->name] = false;
$this->actuallyCreateSearchEngines($this->enabledSearchengines);
}
}
$this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);
$this->retrieveResults($engines);
foreach ($engines as $engine) {
if (!empty($engine->totalResults) && $engine->totalResults > $this->totalResults) {
$this->totalResults = $engine->totalResults;
}
public function startSearch()
{
# Wir starten alle Suchen
foreach ($this->engines as $engine) {
$engine->startSearch($this);
}
}
......@@ -696,7 +603,7 @@ class MetaGer
$engines[] = $tmp;
}
return $engines;
$this->engines = $engines;
}
public function getAvailableParameterFilter()
......@@ -779,128 +686,75 @@ class MetaGer
$next = unserialize(Cache::get($request->input('next')));
$this->page = $next['page'];
$engines = $next['engines'];
if (isset($next['startForwards'])) {
$this->startForwards = $next['startForwards'];
}
if (isset($next['startBackwards'])) {
$this->startBackwards = $next['startBackwards'];
}
return $engines;
}
# Passt den Suchfokus an, falls für einen Fokus genau alle vorhandenen Sumas eingeschaltet sind
public function adjustFocus($sumas, $enabledSearchengines)
public function waitForMainResults()
{
# Findet für alle Foki die enthaltenen Sumas
$foki = []; # [fokus][suma] => [suma]
foreach ($sumas as $suma) {
if ((!$this->sumaIsDisabled($suma)) && (!isset($suma['userSelectable']) || $suma['userSelectable']->__toString() === "1")) {
if (isset($suma['type'])) {
# Wenn foki für diese Suchmaschine angegeben sind
$focuses = explode(",", $suma['type']->__toString());
foreach ($focuses as $foc) {
if (isset($suma['minismCollection'])) {
$foki[$foc][] = "minism";
} else {
$foki[$foc][] = $suma['name']->__toString();
}
}
} else {
# Wenn keine foki für diese Suchmaschine angegeben sind
if (isset($suma['minismCollection'])) {
$foki["andere"][] = "minism";
} else {
$foki["andere"][] = $suma['name']->__toString();
}
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$engines = $this->engines;
$enginesToWaitFor = [];
$mainEngines = $this->sumaFile->foki->{$this->fokus}->main;
foreach ($mainEngines as $mainEngine) {
foreach ($engines as $engine) {
if (!$engine->cached && $engine->name === $mainEngine) {
$enginesToWaitFor[] = $engine;
}
}
}
# Findet die Namen der aktuell eingeschalteten Sumas
$realEngNames = [];
foreach ($enabledSearchengines as $realEng) {
$nam = $realEng["name"]->__toString();
if ($nam !== "qualigo" && $nam !== "overtureAds") {
$realEngNames[] = $nam;
}
}
$timeStart = microtime(true);
$answered = [];
$results = null;
# Anschließend werden diese beiden Listen verglichen (jeweils eine der Fokuslisten für jeden Fokus), um herauszufinden ob sie vielleicht identisch sind. Ist dies der Fall, so hat der Nutzer anscheinend Suchmaschinen eines kompletten Fokus eingestellt. Der Fokus wird dementsprechend angepasst.
foreach ($foki as $fok => $engines) {
$isFokus = true;
$fokiEngNames = [];
foreach ($engines as $eng) {
$fokiEngNames[] = $eng;
}
# Jede eingeschaltete Engine ist für diesen Fokus geeignet
foreach ($fokiEngNames as $fen) {
# Bei Bildersuchen ist uns egal, ob alle Suchmaschinen aus dem Suchfokus eingeschaltet sind, da wir sie eh als Bildersuche anzeigen müssen
if (!in_array($fen, $realEngNames) && $fok !== "bilder") {
$isFokus = false;
}
# If there is no main searchengine to wait for or if the only main engine is yahoo-ads we will define a timeout of 1s
$forceTimeout = null;
if (sizeof($enginesToWaitFor) === 0 || (sizeof($enginesToWaitFor) === 1 && $enginesToWaitFor[0]->name === "yahoo-ads")) {
$forceTimeout = 1;
}
# Jede im Fokus erwartete Engine ist auch eingeschaltet
foreach ($realEngNames as $ren) {
if (!in_array($ren, $fokiEngNames)) {
$isFokus = false;
}
}
# Wenn die Listen identisch sind, setze den Fokus um
if ($isFokus) {
$this->fokus = $fok;
while (sizeof($enginesToWaitFor) > 0 || ($forceTimeout !== null && (microtime(true) - $timeStart) < $forceTimeout)) {
$newEngine = $redis->blpop($this->redisResultWaitingKey, 1);
if ($newEngine === null || sizeof($newEngine) !== 2) {
continue;
} else {
$newEngine = $newEngine[1];
foreach ($enginesToWaitFor as $index => $engine) {
if ($engine->name === $newEngine) {
unset($enginesToWaitFor[$index]);
break;
}
}
$answered[] = $newEngine;
}
public function waitForResults($enginesToLoad, $overtureEnabled, $canBreak)
{
$timeStart = microtime(true);
$results = null;
while (true) {
$results = Redis::hgetall('search.' . $this->getHashCode());
$ready = true;
// When every
$connected = true;
foreach ($results as $key => $value) {
if ($value === "waiting" || $value === "connected") {
$ready =