diff --git a/app/Exceptions/Handler.php b/app/Exceptions/Handler.php
index 21c9784cbd9a1dcbcef500101649a878978ba76f..415cb73a7a94207aa549703409b741bb65bc83b3 100644
--- a/app/Exceptions/Handler.php
+++ b/app/Exceptions/Handler.php
@@ -5,6 +5,8 @@ namespace App\Exceptions;
use Exception;
use Illuminate\Auth\AuthenticationException;
use Illuminate\Foundation\Exceptions\Handler as ExceptionHandler;
+use Illuminate\Http\Response;
+use Predis\Connection\ConnectionException;
class Handler extends ExceptionHandler
{
diff --git a/app/Jobs/Searcher.php b/app/Jobs/Searcher.php
index 194a075362f13d5c12cb297e144855be9bd2e6e2..4ca6fda1d42de023a351c8c2310c967b9108732a 100644
--- a/app/Jobs/Searcher.php
+++ b/app/Jobs/Searcher.php
@@ -8,6 +8,7 @@ use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Illuminate\Support\Facades\Redis;
+use Predis\Connection\ConnectionException;
use Log;
class Searcher implements ShouldQueue
@@ -106,6 +107,12 @@ class Searcher implements ShouldQueue
$result = curl_exec($this->ch);
+ if(!empty(curl_error($this->ch))){
+ // Es gab einen Fehler beim Abruf des Ergebnisses
+ // diesen sollten wir protokollieren
+ Log::error("Fehler in der Searcher Klasse, beim Abruf des Ergebnisses: \n" . curl_error($this->ch));
+ }
+
return $result;
}
diff --git a/app/MetaGer.php b/app/MetaGer.php
index 3730cf814275760142ac9ac63752b4fecc58607e..0cc5269a60f1b5b8300701f1c862275b089548dc 100644
--- a/app/MetaGer.php
+++ b/app/MetaGer.php
@@ -509,6 +509,7 @@ class MetaGer
$sumaCount += 1;
}
$enabledSearchengines[] = $suma;
+ break;
}
}
@@ -563,35 +564,42 @@ class MetaGer
}
# Wir starten alle Suchen
+ $success = true;
foreach ($engines as $engine) {
- $engine->startSearch($this);
+ if(!$engine->startSearch($this)){
+ $success = false;
+ }
}
-
- // Derzeit deaktiviert, da es die eigene Suche gibt
- // $this->adjustFocus($sumas, $enabledSearchengines);
-
- /* Wir warten auf die Antwort der Suchmaschinen
- * Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
- * Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
- * $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
- * welche Suchmaschinen nicht rechtzeitig geantwortet haben.
- */
-
- $enginesToLoad = [];
- $canBreak = false;
- foreach ($engines as $engine) {
- if ($engine->cached) {
- if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
- $canBreak = true;
+ if($success){
+ // Derzeit deaktiviert, da es die eigene Suche gibt
+ // $this->adjustFocus($sumas, $enabledSearchengines);
+
+ /* Wir warten auf die Antwort der Suchmaschinen
+ * Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
+ * Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
+ * $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
+ * welche Suchmaschinen nicht rechtzeitig geantwortet haben.
+ */
+
+ $enginesToLoad = [];
+ $canBreak = false;
+ foreach ($engines as $engine) {
+ if ($engine->cached) {
+ if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
+ $canBreak = true;
+ }
+ } else {
+ $enginesToLoad[$engine->name] = false;
}
- } else {
- $enginesToLoad[$engine->name] = false;
}
- }
- $this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);
+ $this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);
- $this->retrieveResults($engines);
+ $this->retrieveResults($engines);
+ }else{
+ Log::error('Fehler beim Verbindungsaufbau zum lokalen Redis Server!');
+ $this->errors[] = trans('metager.redis.error');
+ }
}
# Spezielle Suchen und Sumas
@@ -815,9 +823,13 @@ class MetaGer
{
$timeStart = microtime(true);
- $results = null;
+ $results = array();
while (true) {
- $results = Redis::hgetall('search.' . $this->getHashCode());
+ try{
+ $results = Redis::hgetall('search.' . $this->getHashCode());
+ }catch(ConnectionException $e){
+ break;
+ }
$ready = true;
// When every
diff --git a/app/Models/Searchengine.php b/app/Models/Searchengine.php
index eb26154c16f98ee096647e57f5b7568d7b94fb3c..f65fd92f51dcda62fe2157736694d747f830a73a 100644
--- a/app/Models/Searchengine.php
+++ b/app/Models/Searchengine.php
@@ -8,6 +8,7 @@ use Cache;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Log;
use Illuminate\Support\Facades\Redis;
+use Predis\Connection\ConnectionException;
abstract class Searchengine
{
@@ -109,67 +110,73 @@ abstract class Searchengine
$this->cached = true;
$this->retrieveResults($metager);
} else {
- // We will push the confirmation of the submission to the Result Hash
- Redis::hset('search.' . $this->resultHash, $this->name, "waiting");
- // We need to submit a action that one of our workers can understand
- // The missions are submitted to a redis queue in the following string format
- // <ResultHash>;<URL to fetch>
- // With <ResultHash> being the Hash Value where the fetcher will store the result.
- // and <URL to fetch> being the full URL to the searchengine
- $url = "";
- if($this->port === "443"){
- $url = "https://";
- }else{
- $url = "http://";
- }
- $url .= $this->host . $this->getString;
- $mission = $this->resultHash . ";" . $url;
- // Submit this mission to the corresponding Redis Queue
- // Since each Searcher is dedicated to one specific search engine
- // each Searcher has it's own queue lying under the redis key <name>.queue
- Redis::rpush($this->name . ".queue", $mission);
-
- /**
- * We have Searcher processes running for MetaGer
- * Each Searcher is dedicated to one specific Searchengine and fetches it's results.
- * We can have multiple Searchers for each engine, if needed.
- * At this point we need to decide, whether we need to start a new Searcher process or
- * if we have enough of them running.
- * The information for that is provided through the redis system. Each running searcher
- * gives information how long it has waited to be given the last fetcher job.
- * The longer this time value is, the less frequent the search engine is used and the less
- * searcher of that type we need.
- * But if it's too low, i.e. 100ms, then the searcher is near to it's full workload and needs assistence.
- **/
- $needSearcher = false;
- $searcherData = Redis::hgetall($this->name . ".stats");
-
- // We now have an array of statistical data from the searchers
- // Each searcher has one entry in it.
- // So if it's empty, then we have currently no searcher running and
- // of course need to spawn a new one.
- if(sizeof($searcherData) === 0){
- $needSearcher = true;
- }else{
- // There we go:
- // There's at least one Fetcher running for this search engine.
- // Now we have to check if the current count is enough to fetch all the
- // searches or if it needs help.
- // Let's hardcode a minimum of 100ms between every search job.
- // First calculate the median of all Times
- $median = 0;
- foreach($searcherData as $pid => $data){
- $data = explode(";", $data);
- $median += floatval($data[1]);
+ try{
+ // We will push the confirmation of the submission to the Result Hash
+ Redis::hset('search.' . $this->resultHash, $this->name, "waiting");
+
+ // We need to submit a action that one of our workers can understand
+ // The missions are submitted to a redis queue in the following string format
+ // <ResultHash>;<URL to fetch>
+ // With <ResultHash> being the Hash Value where the fetcher will store the result.
+ // and <URL to fetch> being the full URL to the searchengine
+ $url = "";
+ if($this->port === "443"){
+ $url = "https://";
+ }else{
+ $url = "http://";
}
- $median /= sizeof($searcherData);
- if($median < 100){
+ $url .= $this->host . $this->getString;
+ $mission = $this->resultHash . ";" . $url;
+ // Submit this mission to the corresponding Redis Queue
+ // Since each Searcher is dedicated to one specific search engine
+ // each Searcher has it's own queue lying under the redis key <name>.queue
+ Redis::rpush($this->name . ".queue", $mission);
+
+ /**
+ * We have Searcher processes running for MetaGer
+ * Each Searcher is dedicated to one specific Searchengine and fetches it's results.
+ * We can have multiple Searchers for each engine, if needed.
+ * At this point we need to decide, whether we need to start a new Searcher process or
+ * if we have enough of them running.
+ * The information for that is provided through the redis system. Each running searcher
+ * gives information how long it has waited to be given the last fetcher job.
+ * The longer this time value is, the less frequent the search engine is used and the less
+ * searcher of that type we need.
+ * But if it's too low, i.e. 100ms, then the searcher is near to it's full workload and needs assistence.
+ **/
+ $needSearcher = false;
+ $searcherData = Redis::hgetall($this->name . ".stats");
+
+ // We now have an array of statistical data from the searchers
+ // Each searcher has one entry in it.
+ // So if it's empty, then we have currently no searcher running and
+ // of course need to spawn a new one.
+ if(sizeof($searcherData) === 0){
$needSearcher = true;
+ }else{
+ // There we go:
+ // There's at least one Fetcher running for this search engine.
+ // Now we have to check if the current count is enough to fetch all the
+ // searches or if it needs help.
+ // Let's hardcode a minimum of 100ms between every search job.
+ // First calculate the median of all Times
+ $median = 0;
+ foreach($searcherData as $pid => $data){
+ $data = explode(";", $data);
+ $median += floatval($data[1]);
+ }
+ $median /= sizeof($searcherData);
+ if($median < .1){ // Median is given as a float in seconds
+ $needSearcher = true;
+ }
}
- }
- if($needSearcher && Redis::get($this->name) !== "locked"){
- Redis::set($this->name, "locked");
- $this->dispatch(new Searcher($this->name));
+ if($needSearcher && Redis::get($this->name) !== "locked"){
+ Redis::set($this->name, "locked");
+ $this->dispatch(new Searcher($this->name));
+ }
+ return true;
+ }catch(ConnectionException $e){
+ return false;
}
}
}
@@ -222,28 +229,33 @@ abstract class Searchengine
# Fragt die Ergebnisse von Redis ab und lädt Sie
public function retrieveResults(MetaGer $metager)
{
- if ($this->loaded) {
- return true;
- }
-
- $body = "";
- if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash) && 0 === 1) {
- $body = Cache::get($this->hash);
- } elseif (Redis::hexists('search.' . $this->resultHash, $this->name)) {
- $body = Redis::hget('search.' . $this->resultHash, $this->name);
- if ($this->canCache && $this->cacheDuration > 0 && 0 === 1) {
- Cache::put($this->hash, $body, $this->cacheDuration);
+ try{
+ if ($this->loaded) {
+ return true;
}
- }
- if ($body !== "") {
- $this->loadResults($body);
- $this->getNext($metager, $body);
+ $body = "";
+ if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash) && 0 === 1) {
+ $body = Cache::get($this->hash);
+ } elseif (Redis::hexists('search.' . $this->resultHash, $this->name)) {
+ $body = Redis::hget('search.' . $this->resultHash, $this->name);
+ if ($this->canCache && $this->cacheDuration > 0 && 0 === 1) {
+ Cache::put($this->hash, $body, $this->cacheDuration);
+ }
+
+ }
+ if ($body !== "") {
+ $this->loadResults($body);
+ $this->getNext($metager, $body);
+ $this->loaded = true;
+ Redis::hdel('search.' . $this->hash, $this->name);
+ return true;
+ } else {
+ return false;
+ }
+ }catch(ConnectionException $e){
$this->loaded = true;
- Redis::hdel('search.' . $this->hash, $this->name);
return true;
- } else {
- return false;
}
}
diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php
index 1e34cf8cd303acfb3b23198aeaa94dcf9ba75883..448e39dbe076e1298accf6d0676b96cf0096e415 100644
--- a/app/Providers/AppServiceProvider.php
+++ b/app/Providers/AppServiceProvider.php
@@ -17,40 +17,7 @@ class AppServiceProvider extends ServiceProvider
*/
public function boot()
{
- # Wir loggen im Redis-System für jede Sekunde des Tages, wie viele Worker aktiv am Laufen waren.
- # Dies ist notwendig, damit wir mitbekommen können, ab welchem Zeitpunkt wir zu wenig Worker zur Verfügung haben.
- Queue::before(function (JobProcessing $event) {
- $this->begin = strtotime(date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y"))));
- });
- Queue::after(function (JobProcessed $event) {
- $today = strtotime(date(DATE_RFC822, mktime(0, 0, 0, date("m"), date("d"), date("Y"))));
- $end = strtotime(date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y")))) - $today;
- $expireAt = strtotime(date(DATE_RFC822, mktime(0, 0, 0, date("m"), date("d") + 1, date("Y"))));
- try {
- $redis = Redis::connection('redisLogs');
- if (!$redis) {
- return;
- }
-
- $p = getmypid();
- $host = gethostname();
- $begin = $this->begin - $today;
- $redis->pipeline(function ($pipe) use ($p, $expireAt, $host, $begin, $end) {
- for ($i = $begin; $i <= $end; $i++) {
- $pipe->sadd("logs.worker.$host.$i", strval($p));
- $pipe->expire("logs.worker.$host.$i", 10);
- $pipe->eval("redis.call('hset', 'logs.worker.$host', '$i', redis.call('scard', 'logs.worker.$host.$i'))", 0);
- $pipe->sadd("logs.worker", $host);
- if (date("H") !== 0) {
- $pipe->expire("logs.worker.$host", $expireAt);
- $pipe->expire("logs.worker", $expireAt);
- }
- }
- });
- } catch (\Exception $e) {
- return;
- }
- });
+ //
}
/**
diff --git a/resources/lang/de/metaGer.php b/resources/lang/de/metaGer.php
index e2baeb7c7bc4ae9de74c85f5ecd5559ee9ee8912..0339c2e790ba998ace737135ddda58e9a7d2fb18 100644
--- a/resources/lang/de/metaGer.php
+++ b/resources/lang/de/metaGer.php
@@ -18,4 +18,5 @@ return [
'sitesearch.failed' => 'Sie wollten eine Sitesearch auf :site durchführen. Leider unterstützen die eingestellten Suchmaschinen diese nicht. Sie können <a href=":searchLink">hier</a> die Sitesearch im Web-Fokus durchführen. Es werden ihnen Ergebnisse ohne Sitesearch angezeigt.',
'sitesearch.success' => 'Sie führen eine Sitesearch durch. Es werden nur Ergebnisse von der Seite: <a href="http://:site" target="_blank" rel="noopener">":site"</a> angezeigt.',
+ 'redis.error' => 'Aktuell treten bei uns leider technische Probleme auf. Bitte versuchen Sie es zu einem späteren Zeitpunkt erneut.',
];