Skip to content
Snippets Groups Projects
Commit 4dab18eb authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Merge branch '485-ux-bei-such-timeout' into 'development' 

Bugfix

Closes #485

See merge request !840
parents da71e3d8 450e9d36
No related branches found
No related tags found
2 merge requests!859Development,!840Bugfix
Hide whitespace changes
Inline Side-by-side
app/Exceptions/Handler.php
+ 2
− 0
View file @ 4dab18eb
......@@ -5,6 +5,8 @@ namespace App\Exceptions;
use Exception;
use Illuminate\Auth\AuthenticationException;
use Illuminate\Foundation\Exceptions\Handler as ExceptionHandler;
use Illuminate\Http\Response;
use Predis\Connection\ConnectionException;
class Handler extends ExceptionHandler
{
......
app/Jobs/Searcher.php
+ 7
− 0
View file @ 4dab18eb
......@@ -8,6 +8,7 @@ use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Illuminate\Support\Facades\Redis;
use Predis\Connection\ConnectionException;
use Log;
class Searcher implements ShouldQueue
......@@ -106,6 +107,12 @@ class Searcher implements ShouldQueue
$result = curl_exec($this->ch);
if(!empty(curl_error($this->ch))){
// Es gab einen Fehler beim Abruf des Ergebnisses
// diesen sollten wir protokollieren
Log::error("Fehler in der Searcher Klasse, beim Abruf des Ergebnisses: \n" . curl_error($this->ch));
}
return $result;
}
......
app/MetaGer.php
+ 37
− 25
View file @ 4dab18eb
......@@ -509,6 +509,7 @@ class MetaGer
$sumaCount += 1;
}
$enabledSearchengines[] = $suma;
break;
}
}
......@@ -563,35 +564,42 @@ class MetaGer
}
# Wir starten alle Suchen
$success = true;
foreach ($engines as $engine) {
$engine->startSearch($this);
if(!$engine->startSearch($this)){
$success = false;
}
}
// Derzeit deaktiviert, da es die eigene Suche gibt
// $this->adjustFocus($sumas, $enabledSearchengines);
/* Wir warten auf die Antwort der Suchmaschinen
* Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
* Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
* $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
* welche Suchmaschinen nicht rechtzeitig geantwortet haben.
*/
$enginesToLoad = [];
$canBreak = false;
foreach ($engines as $engine) {
if ($engine->cached) {
if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
$canBreak = true;
if($success){
// Derzeit deaktiviert, da es die eigene Suche gibt
// $this->adjustFocus($sumas, $enabledSearchengines);
/* Wir warten auf die Antwort der Suchmaschinen
* Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
* Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
* $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
* welche Suchmaschinen nicht rechtzeitig geantwortet haben.
*/
$enginesToLoad = [];
$canBreak = false;
foreach ($engines as $engine) {
if ($engine->cached) {
if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
$canBreak = true;
}
} else {
$enginesToLoad[$engine->name] = false;
}
} else {
$enginesToLoad[$engine->name] = false;
}
}
$this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);
$this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);
$this->retrieveResults($engines);
$this->retrieveResults($engines);
}else{
Log::error('Fehler beim Verbindungsaufbau zum lokalen Redis Server!');
$this->errors[] = trans('metager.redis.error');
}
}
# Spezielle Suchen und Sumas
......@@ -815,9 +823,13 @@ class MetaGer
{
$timeStart = microtime(true);
$results = null;
$results = array();
while (true) {
$results = Redis::hgetall('search.' . $this->getHashCode());
try{
$results = Redis::hgetall('search.' . $this->getHashCode());
}catch(ConnectionException $e){
break;
}
$ready = true;
// When every
......
app/Models/Searchengine.php
+ 88
− 76
View file @ 4dab18eb
......@@ -8,6 +8,7 @@ use Cache;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Log;
use Illuminate\Support\Facades\Redis;
use Predis\Connection\ConnectionException;
abstract class Searchengine
{
......@@ -109,67 +110,73 @@ abstract class Searchengine
$this->cached = true;
$this->retrieveResults($metager);
} else {
// We will push the confirmation of the submission to the Result Hash
Redis::hset('search.' . $this->resultHash, $this->name, "waiting");
// We need to submit a action that one of our workers can understand
// The missions are submitted to a redis queue in the following string format
// <ResultHash>;<URL to fetch>
// With <ResultHash> being the Hash Value where the fetcher will store the result.
// and <URL to fetch> being the full URL to the searchengine
$url = "";
if($this->port === "443"){
$url = "https://";
}else{
$url = "http://";
}
$url .= $this->host . $this->getString;
$mission = $this->resultHash . ";" . $url;
// Submit this mission to the corresponding Redis Queue
// Since each Searcher is dedicated to one specific search engine
// each Searcher has it's own queue lying under the redis key <name>.queue
Redis::rpush($this->name . ".queue", $mission);
/**
* We have Searcher processes running for MetaGer
* Each Searcher is dedicated to one specific Searchengine and fetches it's results.
* We can have multiple Searchers for each engine, if needed.
* At this point we need to decide, whether we need to start a new Searcher process or
* if we have enough of them running.
* The information for that is provided through the redis system. Each running searcher
* gives information how long it has waited to be given the last fetcher job.
* The longer this time value is, the less frequent the search engine is used and the less
* searcher of that type we need.
* But if it's too low, i.e. 100ms, then the searcher is near to it's full workload and needs assistence.
**/
$needSearcher = false;
$searcherData = Redis::hgetall($this->name . ".stats");
// We now have an array of statistical data from the searchers
// Each searcher has one entry in it.
// So if it's empty, then we have currently no searcher running and
// of course need to spawn a new one.
if(sizeof($searcherData) === 0){
$needSearcher = true;
}else{
// There we go:
// There's at least one Fetcher running for this search engine.
// Now we have to check if the current count is enough to fetch all the
// searches or if it needs help.
// Let's hardcode a minimum of 100ms between every search job.
// First calculate the median of all Times
$median = 0;
foreach($searcherData as $pid => $data){
$data = explode(";", $data);
$median += floatval($data[1]);
try{
// We will push the confirmation of the submission to the Result Hash
Redis::hset('search.' . $this->resultHash, $this->name, "waiting");
// We need to submit a action that one of our workers can understand
// The missions are submitted to a redis queue in the following string format
// <ResultHash>;<URL to fetch>
// With <ResultHash> being the Hash Value where the fetcher will store the result.
// and <URL to fetch> being the full URL to the searchengine
$url = "";
if($this->port === "443"){
$url = "https://";
}else{
$url = "http://";
}
$median /= sizeof($searcherData);
if($median < 100){
$url .= $this->host . $this->getString;
$mission = $this->resultHash . ";" . $url;
// Submit this mission to the corresponding Redis Queue
// Since each Searcher is dedicated to one specific search engine
// each Searcher has it's own queue lying under the redis key <name>.queue
Redis::rpush($this->name . ".queue", $mission);
/**
* We have Searcher processes running for MetaGer
* Each Searcher is dedicated to one specific Searchengine and fetches it's results.
* We can have multiple Searchers for each engine, if needed.
* At this point we need to decide, whether we need to start a new Searcher process or
* if we have enough of them running.
* The information for that is provided through the redis system. Each running searcher
* gives information how long it has waited to be given the last fetcher job.
* The longer this time value is, the less frequent the search engine is used and the less
* searcher of that type we need.
* But if it's too low, i.e. 100ms, then the searcher is near to it's full workload and needs assistence.
**/
$needSearcher = false;
$searcherData = Redis::hgetall($this->name . ".stats");
// We now have an array of statistical data from the searchers
// Each searcher has one entry in it.
// So if it's empty, then we have currently no searcher running and
// of course need to spawn a new one.
if(sizeof($searcherData) === 0){
$needSearcher = true;
}else{
// There we go:
// There's at least one Fetcher running for this search engine.
// Now we have to check if the current count is enough to fetch all the
// searches or if it needs help.
// Let's hardcode a minimum of 100ms between every search job.
// First calculate the median of all Times
$median = 0;
foreach($searcherData as $pid => $data){
$data = explode(";", $data);
$median += floatval($data[1]);
}
$median /= sizeof($searcherData);
if($median < .1){ // Median is given as a float in seconds
$needSearcher = true;
}
}
}
if($needSearcher && Redis::get($this->name) !== "locked"){
Redis::set($this->name, "locked");
$this->dispatch(new Searcher($this->name));
if($needSearcher && Redis::get($this->name) !== "locked"){
Redis::set($this->name, "locked");
$this->dispatch(new Searcher($this->name));
}
return true;
}catch(ConnectionException $e){
return false;
}
}
}
......@@ -222,28 +229,33 @@ abstract class Searchengine
# Fragt die Ergebnisse von Redis ab und lädt Sie
public function retrieveResults(MetaGer $metager)
{
if ($this->loaded) {
return true;
}
$body = "";
if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash) && 0 === 1) {
$body = Cache::get($this->hash);
} elseif (Redis::hexists('search.' . $this->resultHash, $this->name)) {
$body = Redis::hget('search.' . $this->resultHash, $this->name);
if ($this->canCache && $this->cacheDuration > 0 && 0 === 1) {
Cache::put($this->hash, $body, $this->cacheDuration);
try{
if ($this->loaded) {
return true;
}
}
if ($body !== "") {
$this->loadResults($body);
$this->getNext($metager, $body);
$body = "";
if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash) && 0 === 1) {
$body = Cache::get($this->hash);
} elseif (Redis::hexists('search.' . $this->resultHash, $this->name)) {
$body = Redis::hget('search.' . $this->resultHash, $this->name);
if ($this->canCache && $this->cacheDuration > 0 && 0 === 1) {
Cache::put($this->hash, $body, $this->cacheDuration);
}
}
if ($body !== "") {
$this->loadResults($body);
$this->getNext($metager, $body);
$this->loaded = true;
Redis::hdel('search.' . $this->hash, $this->name);
return true;
} else {
return false;
}
}catch(ConnectionException $e){
$this->loaded = true;
Redis::hdel('search.' . $this->hash, $this->name);
return true;
} else {
return false;
}
}
......
app/Providers/AppServiceProvider.php
+ 1
− 34
View file @ 4dab18eb
......@@ -17,40 +17,7 @@ class AppServiceProvider extends ServiceProvider
*/
public function boot()
{
# Wir loggen im Redis-System für jede Sekunde des Tages, wie viele Worker aktiv am Laufen waren.
# Dies ist notwendig, damit wir mitbekommen können, ab welchem Zeitpunkt wir zu wenig Worker zur Verfügung haben.
Queue::before(function (JobProcessing $event) {
$this->begin = strtotime(date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y"))));
});
Queue::after(function (JobProcessed $event) {
$today = strtotime(date(DATE_RFC822, mktime(0, 0, 0, date("m"), date("d"), date("Y"))));
$end = strtotime(date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y")))) - $today;
$expireAt = strtotime(date(DATE_RFC822, mktime(0, 0, 0, date("m"), date("d") + 1, date("Y"))));
try {
$redis = Redis::connection('redisLogs');
if (!$redis) {
return;
}
$p = getmypid();
$host = gethostname();
$begin = $this->begin - $today;
$redis->pipeline(function ($pipe) use ($p, $expireAt, $host, $begin, $end) {
for ($i = $begin; $i <= $end; $i++) {
$pipe->sadd("logs.worker.$host.$i", strval($p));
$pipe->expire("logs.worker.$host.$i", 10);
$pipe->eval("redis.call('hset', 'logs.worker.$host', '$i', redis.call('scard', 'logs.worker.$host.$i'))", 0);
$pipe->sadd("logs.worker", $host);
if (date("H") !== 0) {
$pipe->expire("logs.worker.$host", $expireAt);
$pipe->expire("logs.worker", $expireAt);
}
}
});
} catch (\Exception $e) {
return;
}
});
//
}
/**
......
resources/lang/de/metaGer.php
+ 1
− 0
View file @ 4dab18eb
......@@ -18,4 +18,5 @@ return [
'sitesearch.failed' => 'Sie wollten eine Sitesearch auf :site durchführen. Leider unterstützen die eingestellten Suchmaschinen diese nicht. Sie können <a href=":searchLink">hier</a> die Sitesearch im Web-Fokus durchführen. Es werden ihnen Ergebnisse ohne Sitesearch angezeigt.',
'sitesearch.success' => 'Sie führen eine Sitesearch durch. Es werden nur Ergebnisse von der Seite: <a href="http://:site" target="_blank" rel="noopener">":site"</a> angezeigt.',
'redis.error' => 'Aktuell treten bei uns leider technische Probleme auf. Bitte versuchen Sie es zu einem späteren Zeitpunkt erneut.',
];
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment