Commit e642b172 authored by Dominik Hebeler's avatar Dominik Hebeler

Merge branch '913-resultloader-needs-to-be-fixed' into 'development'

Resolve "ResultLoader needs to be fixed"

Closes #913

See merge request !1488
parents 9bfd9c16 2cab70c7
...@@ -36,7 +36,9 @@ COPY config/nginx-default.conf /etc/nginx/conf.d/default.conf ...@@ -36,7 +36,9 @@ COPY config/nginx-default.conf /etc/nginx/conf.d/default.conf
COPY --chown=root:nginx . /html COPY --chown=root:nginx . /html
RUN chmod -R g+w storage bootstrap/cache RUN chmod -R g+w storage bootstrap/cache
CMD /etc/init.d/cron start && \ CMD chown -R root:nginx storage/logs/metager bootstrap/cache && \
chmod -R g+w storage/logs/metager bootstrap/cache && \
/etc/init.d/cron start && \
/etc/init.d/php7.3-fpm start && \ /etc/init.d/php7.3-fpm start && \
/etc/init.d/nginx start && \ /etc/init.d/nginx start && \
/etc/init.d/redis-server start && \ /etc/init.d/redis-server start && \
......
...@@ -18,7 +18,7 @@ class CacheHelper ...@@ -18,7 +18,7 @@ class CacheHelper
'key' => $key, 'key' => $key,
'value' => $value, 'value' => $value,
]; ];
Redis::rpush(\App\Console\Commands\RequestCacher::CACHER_QUEUE, json_encode($cacherItem)); Redis::rpush(\App\Console\Commands\RequestCacher::CACHER_QUEUE, base64_encode(serialize($cacherItem)));
} }
} }
...@@ -50,11 +50,11 @@ class RequestCacher extends Command ...@@ -50,11 +50,11 @@ class RequestCacher extends Command
while ($this->shouldRun) { while ($this->shouldRun) {
$cacheItem = Redis::blpop(self::CACHER_QUEUE, 1); $cacheItem = Redis::blpop(self::CACHER_QUEUE, 1);
if (!empty($cacheItem)) { if (!empty($cacheItem)) {
$cacheItem = json_decode($cacheItem[1], true); $cacheItem = unserialize(base64_decode($cacheItem[1]));
if (empty($cacheItem["body"])) { if (empty($cacheItem["value"])) {
$cacheItem["body"] = "no-result"; $cacheItem["value"] = "no-result";
} }
Cache::put($cacheItem["hash"], $cacheItem["body"], now()->addMinutes($cacheItem["cacheDuration"])); Cache::put($cacheItem["key"], $cacheItem["value"], now()->addSeconds($cacheItem["timeSeconds"]));
} }
} }
} }
......
...@@ -120,7 +120,7 @@ class RequestFetcher extends Command ...@@ -120,7 +120,7 @@ class RequestFetcher extends Command
'key' => $resulthash, 'key' => $resulthash,
'value' => $body, 'value' => $body,
]; ];
$pipe->rpush(\App\Console\Commands\RequestCacher::CACHER_QUEUE, json_encode($cacherItem)); $pipe->rpush(\App\Console\Commands\RequestCacher::CACHER_QUEUE, base64_encode(serialize($cacherItem)));
}); });
\curl_multi_remove_handle($this->multicurl, $info["handle"]); \curl_multi_remove_handle($this->multicurl, $info["handle"]);
} }
......
...@@ -6,7 +6,6 @@ use App; ...@@ -6,7 +6,6 @@ use App;
use App\MetaGer; use App\MetaGer;
use Cache; use Cache;
use Illuminate\Http\Request; use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
use LaravelLocalization; use LaravelLocalization;
use View; use View;
...@@ -64,14 +63,15 @@ class MetaGerSearch extends Controller ...@@ -64,14 +63,15 @@ class MetaGerSearch extends Controller
# Ergebnisse der Suchmaschinen kombinieren: # Ergebnisse der Suchmaschinen kombinieren:
$metager->prepareResults(); $metager->prepareResults();
# Save the results in Redis $finished = true;
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION')); foreach ($metager->getEngines() as $engine) {
$pipeline = $redis->pipeline(); if ($engine->loaded) {
foreach ($metager->getResults() as $result) { $engine->setNew(false);
$pipeline->rpush($metager->getRedisCurrentResultList(), base64_encode(serialize($result))); $engine->markNew();
}
} }
$pipeline->expire($metager->getRedisCurrentResultList(), env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->execute(); \App\CacheHelper::put("loader_" . $metager->getSearchUid(), $metager->getEngines(), 60 * 60);
# Die Ausgabe erstellen: # Die Ausgabe erstellen:
$resultpage = $metager->createView($quicktipResults); $resultpage = $metager->createView($quicktipResults);
...@@ -106,83 +106,67 @@ class MetaGerSearch extends Controller ...@@ -106,83 +106,67 @@ class MetaGerSearch extends Controller
# Create a MetaGer Instance with the supplied hash # Create a MetaGer Instance with the supplied hash
$hash = $request->input('loadMore', ''); $hash = $request->input('loadMore', '');
$metager = new MetaGer($hash); # Parser Skripte einhängen
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION')); $dir = app_path() . "/Models/parserSkripte/";
foreach (scandir($dir) as $filename) {
$result = []; $path = $dir . $filename;
# Check if there should be more results if (is_file($path)) {
$stats = $redis->hgetall($metager->getRedisEngineResult() . "status"); require_once $path;
$stats["startTime"] = floatval($stats["startTime"]);
$stats["engineCount"] = intval($stats["engineCount"]);
$stats["engineAnswered"] = intval($stats["engineAnswered"]);
$stats["engineDelivered"] = intval($stats["engineDelivered"]);
$result["finished"] = true;
$result["engineCount"] = $stats["engineCount"];
$result["engineAnswered"] = $stats["engineAnswered"];
$result["engineDelivered"] = $stats["engineDelivered"];
$result["timeWaiting"] = microtime(true) - $stats["startTime"];
# Check if we can abort
if ($stats["engineAnswered"] > $stats["engineDelivered"]/*&& $result["timeWaiting"] <= 10 */) {
$metager->parseFormData($request);
# Nach Spezialsuchen überprüfen:
$metager->checkSpecialSearches($request);
# Read which search engines are new
$newEngines = [];
while (($engine = $redis->lpop($metager->getRedisResultWaitingKey())) != null) {
$result["engineDelivered"]++;
$newEngines[$engine] = $metager->getSumaFile()->sumas->{$engine};
}
$cache = Cache::get($hash);
if ($cache != null) {
$metager->setNext(unserialize($cache)["engines"]);
} }
}
$engines = Cache::get($hash);
if ($engines === null) {
return response()->json(['finished' => true]);
}
# Check if this request is not for page one $metager = new MetaGer(substr($hash, strpos($hash, "loader_") + 7));
$metager->setEngines($request, $newEngines);
# Add the results already delivered to the user $metager->parseFormData($request);
$results = $redis->lrange($metager->getRedisCurrentResultList(), 0, -1); # Nach Spezialsuchen überprüfen:
foreach ($results as $index => $oldResult) { $metager->checkSpecialSearches($request);
$results[$index] = unserialize(base64_decode($oldResult)); $metager->restoreEngines($engines);
$results[$index]->new = false;
} $metager->retrieveResults();
$metager->setResults($results); $metager->rankAll();
$metager->retrieveResults(); $metager->prepareResults();
$metager->rankAll();
$metager->prepareResults(); $result = [
$result["nextSearchLink"] = $metager->nextSearchLink(); 'finished' => true,
$results = $metager->getResults(); 'newResults' => [],
foreach ($results as $index => $resultTmp) { ];
if ($resultTmp->new) { $result["nextSearchLink"] = $metager->nextSearchLink();
if ($metager->getFokus() !== "bilder") {
$view = View::make('layouts.result', ['index' => $index, 'result' => $resultTmp, 'metager' => $metager]); foreach ($metager->getResults() as $index => $resultTmp) {
$html = $view->render(); if ($resultTmp->new) {
$result['newResults'][$index] = $html; if ($metager->getFokus() !== "bilder") {
$result["imagesearch"] = false; $view = View::make('layouts.result', ['index' => $index, 'result' => $resultTmp, 'metager' => $metager]);
} else { $html = $view->render();
$view = View::make('layouts.image_result', ['index' => $index, 'result' => $resultTmp, 'metager' => $metager]); $result['newResults'][$index] = $html;
$html = $view->render(); $result["imagesearch"] = false;
$result['newResults'][$index] = $html; } else {
$result["imagesearch"] = true; $view = View::make('layouts.image_result', ['index' => $index, 'result' => $resultTmp, 'metager' => $metager]);
} $html = $view->render();
$result['newResults'][$index] = $html;
$result["imagesearch"] = true;
} }
} }
# Save the results in Redis }
$pipeline = $redis->pipeline();
$pipeline->hincrby($metager->getRedisEngineResult() . "status", "engineDelivered", sizeof($newEngines));
$pipeline->hset($metager->getRedisEngineResult() . "status", "nextSearchLink", $result["nextSearchLink"]);
foreach ($metager->getResults() as $resultTmp) {
$resultTmp->new = false;
$pipeline->rpush($metager->getRedisCurrentResultList(), base64_encode(serialize($resultTmp)));
}
$pipeline->expire($metager->getRedisCurrentResultList(), env('REDIS_RESULT_CACHE_DURATION'));
$pipeline->execute();
$finished = true;
foreach ($engines as $engine) {
if (!$engine->loaded) {
$finished = false;
} else {
$engine->setNew(false);
$engine->markNew();
}
} }
$result["finished"] = $finished;
// Update new Engines
\App\CacheHelper::put("loader_" . $metager->getSearchUid(), $metager->getEngines(), 1 * 60);
return response()->json($result); return response()->json($result);
} }
......
...@@ -71,6 +71,7 @@ class MetaGer ...@@ -71,6 +71,7 @@ class MetaGer
protected $verificationCount; protected $verificationCount;
protected $searchUid; protected $searchUid;
protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult, $redisCurrentResultList; protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult, $redisCurrentResultList;
public $starttime;
public function __construct($hash = "") public function __construct($hash = "")
{ {
...@@ -804,7 +805,6 @@ class MetaGer ...@@ -804,7 +805,6 @@ class MetaGer
} }
while (sizeof($enginesToWaitFor) > 0 || ($forceTimeout !== null && (microtime(true) - $timeStart) < $forceTimeout)) { while (sizeof($enginesToWaitFor) > 0 || ($forceTimeout !== null && (microtime(true) - $timeStart) < $forceTimeout)) {
Log::info(sizeof($enginesToWaitFor) . " " . sizeof($answered) . " " . $enginesToWaitFor[0]->hash);
foreach ($enginesToWaitFor as $index => $engine) { foreach ($enginesToWaitFor as $index => $engine) {
if (Redis::get($engine->hash) !== null) { if (Redis::get($engine->hash) !== null) {
$answered[] = $engine; $answered[] = $engine;
...@@ -1734,4 +1734,16 @@ class MetaGer ...@@ -1734,4 +1734,16 @@ class MetaGer
{ {
return $this->redisCurrentResultList; return $this->redisCurrentResultList;
} }
public function getEngines()
{
return $this->engines;
}
/**
* Used by JS result loader to restore MetaGer Object of previous request
*/
public function restoreEngines($engines)
{
$this->engines = $engines;
}
} }
...@@ -37,6 +37,7 @@ abstract class Searchengine ...@@ -37,6 +37,7 @@ abstract class Searchengine
public $write_time = 0; # Wird eventuell für Artefakte benötigt public $write_time = 0; # Wird eventuell für Artefakte benötigt
public $connection_time = 0; # Wird eventuell für Artefakte benötigt public $connection_time = 0; # Wird eventuell für Artefakte benötigt
public $cacheDuration = 60; # Wie lange soll das Ergebnis im Cache bleiben (Minuten) public $cacheDuration = 60; # Wie lange soll das Ergebnis im Cache bleiben (Minuten)
public $new = true; # Important for loading results by JS
public function __construct($name, \stdClass $engine, MetaGer $metager) public function __construct($name, \stdClass $engine, MetaGer $metager)
{ {
...@@ -50,7 +51,7 @@ abstract class Searchengine ...@@ -50,7 +51,7 @@ abstract class Searchengine
$this->useragent = $metager->getUserAgent(); $this->useragent = $metager->getUserAgent();
$this->ip = $metager->getIp(); $this->ip = $metager->getIp();
$this->startTime = microtime(); $this->startTime = microtime(true);
# check for http Auth # check for http Auth
if (!empty($this->engine->{"http-auth-credentials"}->username) && !empty($this->engine->{"http-auth-credentials"}->password)) { if (!empty($this->engine->{"http-auth-credentials"}->username) && !empty($this->engine->{"http-auth-credentials"}->password)) {
$this->username = $this->engine->{"http-auth-credentials"}->username; $this->username = $this->engine->{"http-auth-credentials"}->username;
...@@ -183,6 +184,7 @@ abstract class Searchengine ...@@ -183,6 +184,7 @@ abstract class Searchengine
if ($body !== null) { if ($body !== null) {
$this->loadResults($body); $this->loadResults($body);
$this->getNext($metager, $body); $this->getNext($metager, $body);
$this->markNew();
$this->loaded = true; $this->loaded = true;
return true; return true;
} else { } else {
...@@ -190,6 +192,13 @@ abstract class Searchengine ...@@ -190,6 +192,13 @@ abstract class Searchengine
} }
} }
public function markNew()
{
foreach ($this->results as $result) {
$result->new = $this->new;
}
}
# Erstellt den für die Get-Anfrage genutzten String # Erstellt den für die Get-Anfrage genutzten String
protected function generateGetString($query) protected function generateGetString($query)
{ {
...@@ -243,4 +252,9 @@ abstract class Searchengine ...@@ -243,4 +252,9 @@ abstract class Searchengine
{ {
return []; return [];
} }
public function setNew($new)
{
$this->new = $new;
}
} }
...@@ -104,8 +104,10 @@ spec: ...@@ -104,8 +104,10 @@ spec:
volumeMounts: volumeMounts:
- name: mglogs-persistent-storage - name: mglogs-persistent-storage
mountPath: /html/storage/logs/metager mountPath: /html/storage/logs/metager
readOnly: false
- name: mgcache-persistent-storage - name: mgcache-persistent-storage
mountPath: /html/storage/framework/cache mountPath: /html/storage/framework/cache
readOnly: false
resources: resources:
{{ toYaml .Values.resources | indent 12 }} {{ toYaml .Values.resources | indent 12 }}
{{- end -}} {{- end -}}
...@@ -55,13 +55,10 @@ function enableFormResetter() { ...@@ -55,13 +55,10 @@ function enableFormResetter() {
function loadMoreResults() { function loadMoreResults() {
var searchKey = $("meta[name=searchkey]").attr("content"); var searchKey = $("meta[name=searchkey]").attr("content");
var updateUrl = document.location.href; var updateUrl = document.location.href;
updateUrl += "&loadMore=" + searchKey + "&script=yes"; updateUrl += "&loadMore=loader_" + searchKey + "&script=yes";
updateUrl = updateUrl.replace("/meta.ger3", "/loadMore"); updateUrl = updateUrl.replace("/meta.ger3", "/loadMore");
var expectedEngines = -1;
var deliveredEngines = -1;
var currentlyLoading = false; var currentlyLoading = false;
// Regularily check for not yet delivered Results // Regularily check for not yet delivered Results
...@@ -70,7 +67,7 @@ function loadMoreResults() { ...@@ -70,7 +67,7 @@ function loadMoreResults() {
currentlyLoading = true; currentlyLoading = true;
$.getJSON(updateUrl, function (data) { $.getJSON(updateUrl, function (data) {
// Check if we can clear the interval (once every searchengine has answered) // Check if we can clear the interval (once every searchengine has answered)
if (data.engineDelivered == data.engineCount || data.timeWaiting > 15) { if (!data || data.finished) {
clearInterval(resultLoader); clearInterval(resultLoader);
} }
// If there are new results we can add them // If there are new results we can add them
...@@ -109,13 +106,9 @@ function loadMoreResults() { ...@@ -109,13 +106,9 @@ function loadMoreResults() {
$(".alert.alert-danger").remove(); $(".alert.alert-danger").remove();
} }
} }
console.log(data);
} }
currentlyLoading = false; currentlyLoading = false;
}); });
} }
}, 1000); }, 1000);
//clearInterval(resultLoader);
console.log(updateUrl);
} }
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment