Commit 40bc1b99 authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Merge branch '864-optimize-request-times' into 'development'

Resolve "Optimize request times"

Closes #864

See merge request !1386
parents c502f47f 11147134
...@@ -4,7 +4,10 @@ namespace App\Http\Controllers; ...@@ -4,7 +4,10 @@ namespace App\Http\Controllers;
use App; use App;
use App\MetaGer; use App\MetaGer;
use Cache;
use Illuminate\Http\Request; use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
use View;
const TIP_SERVER = 'http://metager3.de:63825/tips.xml'; const TIP_SERVER = 'http://metager3.de:63825/tips.xml';
...@@ -19,13 +22,6 @@ class MetaGerSearch extends Controller ...@@ -19,13 +22,6 @@ class MetaGerSearch extends Controller
return redirect()->to('https://maps.metager.de/map/' . $searchinput . '/1240908.5493525574,6638783.2192695495,6'); return redirect()->to('https://maps.metager.de/map/' . $searchinput . '/1240908.5493525574,6638783.2192695495,6');
} }
/*if ($focus !== "angepasst" && $this->startsWith($focus, "focus_")) {
$metager->parseFormData($request);
return $metager->createView();
}*/
#die($request->header('User-Agent'));
$time = microtime();
# Mit gelieferte Formulardaten parsen und abspeichern: # Mit gelieferte Formulardaten parsen und abspeichern:
$metager->parseFormData($request); $metager->parseFormData($request);
...@@ -39,19 +35,132 @@ class MetaGerSearch extends Controller ...@@ -39,19 +35,132 @@ class MetaGerSearch extends Controller
# auf Ergebnisse warten und die Ergebnisse laden # auf Ergebnisse warten und die Ergebnisse laden
$metager->createSearchEngines($request); $metager->createSearchEngines($request);
$metager->startSearch();
$metager->waitForMainResults();
$metager->retrieveResults();
# Versuchen die Ergebnisse der Quicktips zu laden # Versuchen die Ergebnisse der Quicktips zu laden
$quicktipResults = $quicktips->loadResults(); $quicktipResults = $quicktips->loadResults();
# Alle Ergebnisse vor der Zusammenführung ranken: # Alle Ergebnisse vor der Zusammenführung ranken:
$metager->rankAll(); $metager->rankAll();
# Ergebnisse der Suchmaschinen kombinieren: # Ergebnisse der Suchmaschinen kombinieren:
$metager->prepareResults(); $metager->prepareResults();
# Save the results in Redis
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$pipeline = $redis->pipeline();
foreach ($metager->getResults() as $result) {
$pipeline->rpush($metager->getRedisCurrentResultList(), base64_encode(serialize($result)));
}
$pipeline->expire($metager->getRedisCurrentResultList(), env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->execute();
# Die Ausgabe erstellen: # Die Ausgabe erstellen:
return $metager->createView($quicktipResults); return $metager->createView($quicktipResults);
} }
public function loadMore(Request $request)
{
/**
* There are three forms of requests to the resultpage
* 1. Initial Request: Loads the fastest searchengines and sends them to the user
* 2. Load more results (with JS): Loads new search engines that answered after the initial request was send
* 3. Load more results (without JS): Loads new search engines that answered within 1s timeout
*/
if ($request->filled('loadMore') && $request->filled('script') && $request->input('script') === "yes") {
return $this->loadMoreJS($request);
}
}
private function loadMoreJS(Request $request)
{
# Create a MetaGer Instance with the supplied hash
$hash = $request->input('loadMore', '');
$metager = new MetaGer($hash);
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$result = [];
# Check if there should be more results
$stats = $redis->hgetall($metager->getRedisEngineResult() . "status");
$stats["startTime"] = floatval($stats["startTime"]);
$stats["engineCount"] = intval($stats["engineCount"]);
$stats["engineAnswered"] = intval($stats["engineAnswered"]);
$stats["engineDelivered"] = intval($stats["engineDelivered"]);
$result["finished"] = true;
$result["engineCount"] = $stats["engineCount"];
$result["engineAnswered"] = $stats["engineAnswered"];
$result["engineDelivered"] = $stats["engineDelivered"];
$result["timeWaiting"] = microtime(true) - $stats["startTime"];
# Check if we can abort
if ($stats["engineAnswered"] > $stats["engineDelivered"]/*&& $result["timeWaiting"] <= 10 */) {
$metager->parseFormData($request);
# Nach Spezialsuchen überprüfen:
$metager->checkSpecialSearches($request);
# Read which search engines are new
$newEngines = [];
while (($engine = $redis->lpop($metager->getRedisResultWaitingKey())) != null) {
$result["engineDelivered"]++;
$newEngines[$engine] = $metager->getSumaFile()->sumas->{$engine};
}
$cache = Cache::get($hash);
if ($cache != null) {
$metager->setNext(unserialize($cache)["engines"]);
}
# Check if this request is not for page one
$metager->setEngines($request, $newEngines);
# Add the results already delivered to the user
$results = $redis->lrange($metager->getRedisCurrentResultList(), 0, -1);
foreach ($results as $index => $oldResult) {
$results[$index] = unserialize(base64_decode($oldResult));
$results[$index]->new = false;
}
$metager->setResults($results);
$metager->retrieveResults();
$metager->rankAll();
$metager->prepareResults();
$result["nextSearchLink"] = $metager->nextSearchLink();
$results = $metager->getResults();
foreach ($results as $index => $resultTmp) {
if ($resultTmp->new) {
if ($metager->getFokus() !== "bilder") {
$view = View::make('layouts.result', ['result' => $resultTmp, 'metager' => $metager]);
$html = $view->render();
$result['newResults'][$index] = $html;
$result["imagesearch"] = false;
} else {
$view = View::make('layouts.image_result', ['result' => $resultTmp, 'metager' => $metager]);
$html = $view->render();
$result['newResults'][$index] = $html;
$result["imagesearch"] = true;
}
}
}
# Save the results in Redis
$pipeline = $redis->pipeline();
$pipeline->hincrby($metager->getRedisEngineResult() . "status", "engineDelivered", sizeof($newEngines));
$pipeline->hset($metager->getRedisEngineResult() . "status", "nextSearchLink", $result["nextSearchLink"]);
foreach ($metager->getResults() as $resultTmp) {
$resultTmp->new = false;
$pipeline->rpush($metager->getRedisCurrentResultList(), base64_encode(serialize($resultTmp)));
}
$pipeline->expire($metager->getRedisCurrentResultList(), env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->execute();
}
return response()->json($result);
}
public function botProtection($redirect) public function botProtection($redirect)
{ {
$hash = md5(date('YmdHi')); $hash = md5(date('YmdHi'));
......
...@@ -85,8 +85,7 @@ class Searcher implements ShouldQueue ...@@ -85,8 +85,7 @@ class Searcher implements ShouldQueue
$url = base64_decode($mission[1]); // The url to fetch $url = base64_decode($mission[1]); // The url to fetch
$timeout = $mission[2]; // Timeout from the MetaGer process in ms $timeout = $mission[2]; // Timeout from the MetaGer process in ms
$medianFetchTime = $this->getFetchTime(); // The median Fetch time of the search engine in ms $medianFetchTime = $this->getFetchTime(); // The median Fetch time of the search engine in ms
Redis::hset('search.' . $hashValue, $this->name, "connected"); Redis::hset('search.' . $hashValue . ".results." . $this->name, "status", "connected");
$result = $this->retrieveUrl($url); $result = $this->retrieveUrl($url);
$this->storeResult($result, $poptime, $hashValue); $this->storeResult($result, $poptime, $hashValue);
...@@ -99,7 +98,7 @@ class Searcher implements ShouldQueue ...@@ -99,7 +98,7 @@ class Searcher implements ShouldQueue
// In sync mode every Searcher may only retrieve one result because it would block // In sync mode every Searcher may only retrieve one result because it would block
// the execution of the remaining code otherwise: // the execution of the remaining code otherwise:
if (getenv("QUEUE_DRIVER") === "sync" if (getenv("QUEUE_CONNECTION") === "sync"
|| $this->counter > $this->MAX_REQUESTS || $this->counter > $this->MAX_REQUESTS
|| (microtime(true) - $this->startTime) > $this->MAX_TIME) { || (microtime(true) - $this->startTime) > $this->MAX_TIME) {
break; break;
...@@ -161,16 +160,24 @@ class Searcher implements ShouldQueue ...@@ -161,16 +160,24 @@ class Searcher implements ShouldQueue
// Set this URL to the Curl handle // Set this URL to the Curl handle
curl_setopt($this->ch, CURLOPT_URL, $url); curl_setopt($this->ch, CURLOPT_URL, $url);
$result = curl_exec($this->ch); $result = curl_exec($this->ch);
$this->connectionInfo = curl_getinfo($this->ch); $this->connectionInfo = curl_getinfo($this->ch);
return $result; return $result;
} }
private function storeResult($result, $poptime, $hashValue) private function storeResult($result, $poptime, $hashValue)
{ {
Redis::hset('search.' . $hashValue, $this->name, $result); $redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$pipeline = $redis->pipeline();
$pipeline->hset('search.' . $hashValue . ".results." . $this->name, "response", $result);
$pipeline->hset('search.' . $hashValue . ".results." . $this->name, "delivered", "0");
$pipeline->hincrby('search.' . $hashValue . ".results.status", "engineAnswered", 1);
// After 60 seconds the results should be read by the MetaGer Process and stored in the Cache instead // After 60 seconds the results should be read by the MetaGer Process and stored in the Cache instead
Redis::expire('search.' . $hashValue, 60); $pipeline->expire('search.' . $hashValue . ".results." . $this->name, env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->rpush('search.' . $hashValue . ".ready", $this->name);
$pipeline->expire('search.' . $hashValue . ".ready", env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->sadd('search.' . $hashValue . ".engines", $this->name);
$pipeline->expire('search.' . $hashValue . ".engines", env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->execute();
$this->lastTime = microtime(true); $this->lastTime = microtime(true);
} }
......
This diff is collapsed.
...@@ -3,7 +3,6 @@ ...@@ -3,7 +3,6 @@
namespace App\Models\Quicktips; namespace App\Models\Quicktips;
use App\Jobs\Searcher; use App\Jobs\Searcher;
use Cache;
use Illuminate\Foundation\Bus\DispatchesJobs; use Illuminate\Foundation\Bus\DispatchesJobs;
use Illuminate\Support\Facades\Redis; use Illuminate\Support\Facades\Redis;
use Log; use Log;
...@@ -12,8 +11,8 @@ class Quicktips ...@@ -12,8 +11,8 @@ class Quicktips
{ {
use DispatchesJobs; use DispatchesJobs;
const QUICKTIP_URL = "http://localhost:63825/1.1/quicktips.xml"; const QUICKTIP_URL = "http://localhost:63825/1.1/quicktips.xml";
const QUICKTIP_NAME = "quicktips"; const QUICKTIP_NAME = "quicktips";
const CACHE_DURATION = 60; const CACHE_DURATION = 60;
private $hash; private $hash;
...@@ -26,18 +25,17 @@ class Quicktips ...@@ -26,18 +25,17 @@ class Quicktips
public function startSearch($search, $locale, $max_time) public function startSearch($search, $locale, $max_time)
{ {
$url = self::QUICKTIP_URL . "?search=" . $this->normalize_search($search) . "&locale=" . $locale; $url = self::QUICKTIP_URL . "?search=" . $this->normalize_search($search) . "&locale=" . $locale;
$hash = md5($url);
# TODO anders weitergeben # TODO anders weitergeben
$this->hash = $hash; $this->hash = md5($url);
# TODO cache wieder einbauen (eventuell) # TODO cache wieder einbauen (eventuell)
if ( /*!Cache::has($hash)*/true) { if ( /*!Cache::has($hash)*/true) {
Redis::hset("search." . $hash, self::QUICKTIP_NAME, "waiting"); $redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$redis->hset("search." . $this->hash . ".results." . self::QUICKTIP_NAME, "status", "waiting");
// Queue this search // Queue this search
$mission = $hash . ";" . base64_encode($url) . ";" . $max_time; $mission = $this->hash . ";" . base64_encode($url) . ";" . $max_time;
Redis::rpush(self::QUICKTIP_NAME . ".queue", $mission); Redis::rpush(self::QUICKTIP_NAME . ".queue", $mission);
// Check the current status of Searchers for QUICKTIP_NAME // Check the current status of Searchers for QUICKTIP_NAME
...@@ -85,14 +83,11 @@ class Quicktips ...@@ -85,14 +83,11 @@ class Quicktips
public function retrieveResults($hash) public function retrieveResults($hash)
{ {
$body = ""; $body = "";
#if (Cache::has($hash)) { $redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$body = Cache::get($hash); $body = $redis->hget('search.' . $hash . ".results." . self::QUICKTIP_NAME, "response");
#} elseif (Redis::hexists('search.' . $hash, self::QUICKTIP_NAME)) {
$body = Redis::hget('search.' . $hash, self::QUICKTIP_NAME);
Redis::hdel('search.' . $hash, self::QUICKTIP_NAME);
Cache::put($hash, $body, self::CACHE_DURATION);
#}
$redis->del('search.' . $hash . ".results." . self::QUICKTIP_NAME);
$redis->del('search.' . $hash . ".ready");
if ($body !== "") { if ($body !== "") {
return $body; return $body;
} else { } else {
...@@ -157,14 +152,14 @@ class Quicktips ...@@ -157,14 +152,14 @@ class Quicktips
$descr = $quicktip_xml->content->__toString(); $descr = $quicktip_xml->content->__toString();
// Details // Details
$details = []; $details = [];
$details_xpath = $quicktip_xml->xpath('mg:details'); $details_xpath = $quicktip_xml->xpath('mg:details');
if (sizeof($details_xpath) > 0) { if (sizeof($details_xpath) > 0) {
foreach ($details_xpath[0] as $detail_xml) { foreach ($details_xpath[0] as $detail_xml) {
$details_title = $detail_xml->title->__toString(); $details_title = $detail_xml->title->__toString();
$details_link = $detail_xml->url->__toString(); $details_link = $detail_xml->url->__toString();
$details_descr = $detail_xml->text->__toString(); $details_descr = $detail_xml->text->__toString();
$details[] = new \App\Models\Quicktips\Quicktip_detail( $details[] = new \App\Models\Quicktips\Quicktip_detail(
$details_title, $details_title,
$details_link, $details_link,
$details_descr $details_descr
......
...@@ -27,6 +27,7 @@ class Result ...@@ -27,6 +27,7 @@ class Result
public $strippedDomain; # Die Domain in Form "bar.de" public $strippedDomain; # Die Domain in Form "bar.de"
public $strippedLink; # Der Link in Form "foo.bar.de/test" public $strippedLink; # Der Link in Form "foo.bar.de/test"
public $rank; # Das Ranking für das Ergebnis public $rank; # Das Ranking für das Ergebnis
public $new = true;
# Erstellt ein neues Ergebnis # Erstellt ein neues Ergebnis
public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $gefVonLink, $sourceRank, $additionalInformation = []) public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $gefVonLink, $sourceRank, $additionalInformation = [])
...@@ -67,6 +68,7 @@ class Result ...@@ -67,6 +68,7 @@ class Result
$this->price = isset($additionalInformation["price"]) ? $additionalInformation["price"] : 0; $this->price = isset($additionalInformation["price"]) ? $additionalInformation["price"] : 0;
$this->price_text = $this->price_to_text($this->price); $this->price_text = $this->price_to_text($this->price);
$this->additionalInformation = $additionalInformation; $this->additionalInformation = $additionalInformation;
$this->hash = spl_object_hash($this);
} }
private function price_to_text($price) private function price_to_text($price)
......
...@@ -87,7 +87,7 @@ abstract class Searchengine ...@@ -87,7 +87,7 @@ abstract class Searchengine
$this->getString = $this->generateGetString($q); $this->getString = $this->generateGetString($q);
$this->hash = md5($this->engine->host . $this->getString . $this->engine->port . $this->name); $this->hash = md5($this->engine->host . $this->getString . $this->engine->port . $this->name);
$this->resultHash = $metager->getHashCode(); $this->resultHash = $metager->getSearchUid();
$this->canCache = $metager->canCache(); $this->canCache = $metager->canCache();
} }
...@@ -102,13 +102,15 @@ abstract class Searchengine ...@@ -102,13 +102,15 @@ abstract class Searchengine
# Prüft, ob die Suche bereits gecached ist, ansonsted wird sie als Job dispatched # Prüft, ob die Suche bereits gecached ist, ansonsted wird sie als Job dispatched
public function startSearch(\App\MetaGer $metager) public function startSearch(\App\MetaGer $metager)
{ {
if ($this->canCache && Cache::has($this->hash)) { if ($this->canCache && Cache::has($this->hash)) {
$this->cached = true; $this->cached = true;
$this->retrieveResults($metager); $this->retrieveResults($metager);
} else { } else {
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
// We will push the confirmation of the submission to the Result Hash // We will push the confirmation of the submission to the Result Hash
Redis::hset('search.' . $this->resultHash, $this->name, "waiting"); $redis->hset($metager->getRedisEngineResult() . $this->name, "status", "waiting");
$redis->expire($metager->getRedisEngineResult() . $this->name, env('REDIS_RESULT_CACHE_DURATION'));
// We need to submit a action that one of our workers can understand // We need to submit a action that one of our workers can understand
// The missions are submitted to a redis queue in the following string format // The missions are submitted to a redis queue in the following string format
// <ResultHash>;<URL to fetch> // <ResultHash>;<URL to fetch>
...@@ -190,10 +192,6 @@ abstract class Searchengine ...@@ -190,10 +192,6 @@ abstract class Searchengine
{ {
foreach ($this->results as $result) { foreach ($this->results as $result) {
$result->rank($eingabe); $result->rank($eingabe);
if (str_contains($this->engine->{"display-name"}, "Yahoo")) {
#die(var_dump($result));
}
} }
} }
...@@ -210,12 +208,12 @@ abstract class Searchengine ...@@ -210,12 +208,12 @@ abstract class Searchengine
} }
$body = ""; $body = "";
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash)) { if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash)) {
$body = Cache::get($this->hash); $body = Cache::get($this->hash);
} elseif (Redis::hexists('search.' . $this->resultHash, $this->name)) { } elseif ($redis->hexists($metager->getRedisEngineResult() . $this->name, "response")) {
$body = Redis::hget('search.' . $this->resultHash, $this->name); $body = $redis->hget($metager->getRedisEngineResult() . $this->name, "response");
Redis::hdel('search.' . $this->resultHash, $this->name);
if ($this->canCache && $this->cacheDuration > 0) { if ($this->canCache && $this->cacheDuration > 0) {
Cache::put($this->hash, $body, $this->cacheDuration); Cache::put($this->hash, $body, $this->cacheDuration);
} }
...@@ -251,13 +249,7 @@ abstract class Searchengine ...@@ -251,13 +249,7 @@ abstract class Searchengine
# Append the Query String # Append the Query String
$getString .= "&" . $this->engine->{"query-parameter"} . "=" . $this->urlEncode($query); $getString .= "&" . $this->engine->{"query-parameter"} . "=" . $this->urlEncode($query);
/*
die(var_dump($getString));
# Affildata
if (strpos($getString, "<<AFFILDATA>>")) {
$getString = str_replace("<<AFFILDATA>>", $this->getOvertureAffilData($url), $getString);
}*/
return $getString; return $getString;
} }
......
#!/usr/bin/perl
use Lingua::Identify qw(:language_identification);
use JSON;
use warnings;
use strict;
binmode STDOUT, ":utf8";
binmode STDIN, ":utf8";
use utf8;
chomp(my $filename = <STDIN>);
# Lets open the given file:
open(my $fh, "<", $filename)
or die "Can't open < $filename: $!";
my $json = <$fh>;
close $fh;
# Decode the JSON String
my $data = JSON->new->utf8->decode($json);
# Wir durchlaufen den Hash:
foreach my $key (keys %{$data}){
$data->{$key} = langof($data->{$key});
}
$data = encode_json($data);
# Nur noch die temporäre Datei löschen:
unlink($filename);
print $data;
...@@ -18,9 +18,6 @@ ...@@ -18,9 +18,6 @@
* php-gd * php-gd
* sqlite3 * sqlite3
* redis-server * redis-server
* Die Perl-Pakete
* Lingua::Identify (http://search.cpan.org/~ambs/Lingua-Identify-0.56/lib/Lingua/Identify.pm)
* JSON (http://search.cpan.org/~makamaka/JSON-2.90/lib/JSON.pm)
--- ---
[<img src="public/img/Browserstack-logo_2x.png" width="250px" alt="Browserstack Logo" />](https://www.browserstack.com) <br /> [<img src="public/img/Browserstack-logo_2x.png" width="250px" alt="Browserstack Logo" />](https://www.browserstack.com) <br />
......
$(document).ready(function () { $(document).ready(function () {
botProtection(); botProtection();
enableFormResetter(); enableFormResetter();
loadMoreResults();
}); });
function botProtection() { function botProtection() {
...@@ -50,4 +50,72 @@ function enableFormResetter() { ...@@ -50,4 +50,72 @@ function enableFormResetter() {
timeout = null; timeout = null;
}, 500); }, 500);
}); });
}
function loadMoreResults() {
var searchKey = $("meta[name=searchkey]").attr("content");
var updateUrl = document.location.href;
updateUrl += "&loadMore=" + searchKey + "&script=yes";
updateUrl = updateUrl.replace("/meta.ger3", "/loadMore");
var expectedEngines = -1;
var deliveredEngines = -1;
var currentlyLoading = false;
// Regularily check for not yet delivered Results
var resultLoader = window.setInterval(function () {
if (!currentlyLoading) {
currentlyLoading = true;
$.getJSON(updateUrl, function (data) {
// Check if we can clear the interval (once every searchengine has answered)
if (data.engineDelivered == data.engineCount || data.timeWaiting > 15) {
clearInterval(resultLoader);
}
// If there are new results we can add them
if (typeof data.newResults != "undefined") {
for (var key in data.newResults) {
var value = data.newResults[key];
// If there are more results than the given index we will prepend otherwise we will append the result
if (!data.imagesearch) {
var results = $(".result:not(.ad)");
if (key == 0) {
if ($(".result.ad").length > 0) {
$(value).insertAfter($($(".result.ad")[$(".result.ad").length - 1]));
} else {
$("#results").prepend(value);
}
} else if (typeof results[key] != "undefined") {
$(value).insertBefore($(results[key]));
} else if (typeof results[key - 1] != "undefined") {
$(value).insertAfter($(results[key - 1]));
}
} else {
var results = $(".image-container > .image");
if (key == 0) {
$(".image-container").prepend(value);