Skip to content
Snippets Groups Projects
Commit 179ea0bf authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Bugfix 

Habe bei der Zeitmessung Sekunden mit Millisekunden verwechselt.
parent b228e9ac
No related branches found
No related tags found
1 merge request!1365Resolve "Filter Options for MetaGer"
Hide whitespace changes
Inline Side-by-side
app/Exceptions/Handler.php
+ 2
− 0
View file @ 179ea0bf
......@@ -5,6 +5,8 @@ namespace App\Exceptions;
use Exception;
use Illuminate\Auth\AuthenticationException;
use Illuminate\Foundation\Exceptions\Handler as ExceptionHandler;
use Illuminate\Http\Response;
use Predis\Connection\ConnectionException;
class Handler extends ExceptionHandler
{
......
app/Jobs/Searcher.php
+ 7
− 0
View file @ 179ea0bf
......@@ -8,6 +8,7 @@ use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Illuminate\Support\Facades\Redis;
use Predis\Connection\ConnectionException;
use Log;
class Searcher implements ShouldQueue
......@@ -106,6 +107,12 @@ class Searcher implements ShouldQueue
$result = curl_exec($this->ch);
if(!empty(curl_error($this->ch))){
// Es gab einen Fehler beim Abruf des Ergebnisses
// diesen sollten wir protokollieren
Log::error("Fehler in der Searcher Klasse, beim Abruf des Ergebnisses: \n" . curl_error($this->ch));
}
return $result;
}
......
app/MetaGer.php
+ 37
− 25
View file @ 179ea0bf
......@@ -509,6 +509,7 @@ class MetaGer
$sumaCount += 1;
}
$enabledSearchengines[] = $suma;
break;
}
}
......@@ -563,35 +564,42 @@ class MetaGer
}
# Wir starten alle Suchen
$success = true;
foreach ($engines as $engine) {
$engine->startSearch($this);
if(!$engine->startSearch($this)){
$success = false;
}
}
// Derzeit deaktiviert, da es die eigene Suche gibt
// $this->adjustFocus($sumas, $enabledSearchengines);
/* Wir warten auf die Antwort der Suchmaschinen
* Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
* Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
* $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
* welche Suchmaschinen nicht rechtzeitig geantwortet haben.
*/
$enginesToLoad = [];
$canBreak = false;
foreach ($engines as $engine) {
if ($engine->cached) {
if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
$canBreak = true;
if($success){
// Derzeit deaktiviert, da es die eigene Suche gibt
// $this->adjustFocus($sumas, $enabledSearchengines);
/* Wir warten auf die Antwort der Suchmaschinen
* Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
* Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
* $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
* welche Suchmaschinen nicht rechtzeitig geantwortet haben.
*/
$enginesToLoad = [];
$canBreak = false;
foreach ($engines as $engine) {
if ($engine->cached) {
if ($overtureEnabled && ($engine->name === "overture" || $engine->name === "overtureAds")) {
$canBreak = true;
}
} else {
$enginesToLoad[$engine->name] = false;
}
} else {
$enginesToLoad[$engine->name] = false;
}
}
$this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);
$this->waitForResults($enginesToLoad, $overtureEnabled, $canBreak);
$this->retrieveResults($engines);
$this->retrieveResults($engines);
}else{
Log::error('Fehler beim Verbindungsaufbau zum lokalen Redis Server!');
$this->errors[] = trans('metager.redis.error');
}
}
# Spezielle Suchen und Sumas
......@@ -815,9 +823,13 @@ class MetaGer
{
$timeStart = microtime(true);
$results = null;
$results = array();
while (true) {
$results = Redis::hgetall('search.' . $this->getHashCode());
try{
$results = Redis::hgetall('search.' . $this->getHashCode());
}catch(ConnectionException $e){
break;
}
$ready = true;
// When every
......
app/Models/Searchengine.php
+ 88
− 76
View file @ 179ea0bf
......@@ -8,6 +8,7 @@ use Cache;
use Illuminate\Foundation\Bus\DispatchesJobs;
use Log;
use Illuminate\Support\Facades\Redis;
use Predis\Connection\ConnectionException;
abstract class Searchengine
{
......@@ -109,67 +110,73 @@ abstract class Searchengine
$this->cached = true;
$this->retrieveResults($metager);
} else {
// We will push the confirmation of the submission to the Result Hash
Redis::hset('search.' . $this->resultHash, $this->name, "waiting");
// We need to submit a action that one of our workers can understand
// The missions are submitted to a redis queue in the following string format
// <ResultHash>;<URL to fetch>
// With <ResultHash> being the Hash Value where the fetcher will store the result.
// and <URL to fetch> being the full URL to the searchengine
$url = "";
if($this->port === "443"){
$url = "https://";
}else{
$url = "http://";
}
$url .= $this->host . $this->getString;
$mission = $this->resultHash . ";" . $url;
// Submit this mission to the corresponding Redis Queue
// Since each Searcher is dedicated to one specific search engine
// each Searcher has it's own queue lying under the redis key <name>.queue
Redis::rpush($this->name . ".queue", $mission);
/**
* We have Searcher processes running for MetaGer
* Each Searcher is dedicated to one specific Searchengine and fetches it's results.
* We can have multiple Searchers for each engine, if needed.
* At this point we need to decide, whether we need to start a new Searcher process or
* if we have enough of them running.
* The information for that is provided through the redis system. Each running searcher
* gives information how long it has waited to be given the last fetcher job.
* The longer this time value is, the less frequent the search engine is used and the less
* searcher of that type we need.
* But if it's too low, i.e. 100ms, then the searcher is near to it's full workload and needs assistence.
**/
$needSearcher = false;
$searcherData = Redis::hgetall($this->name . ".stats");
// We now have an array of statistical data from the searchers
// Each searcher has one entry in it.
// So if it's empty, then we have currently no searcher running and
// of course need to spawn a new one.
if(sizeof($searcherData) === 0){
$needSearcher = true;
}else{
// There we go:
// There's at least one Fetcher running for this search engine.
// Now we have to check if the current count is enough to fetch all the
// searches or if it needs help.
// Let's hardcode a minimum of 100ms between every search job.
// First calculate the median of all Times
$median = 0;
foreach($searcherData as $pid => $data){
$data = explode(";", $data);
$median += floatval($data[1]);
try{
// We will push the confirmation of the submission to the Result Hash
Redis::hset('search.' . $this->resultHash, $this->name, "waiting");
// We need to submit a action that one of our workers can understand
// The missions are submitted to a redis queue in the following string format
// <ResultHash>;<URL to fetch>
// With <ResultHash> being the Hash Value where the fetcher will store the result.
// and <URL to fetch> being the full URL to the searchengine
$url = "";
if($this->port === "443"){
$url = "https://";
}else{
$url = "http://";
}
$median /= sizeof($searcherData);
if($median < 100){
$url .= $this->host . $this->getString;
$mission = $this->resultHash . ";" . $url;
// Submit this mission to the corresponding Redis Queue
// Since each Searcher is dedicated to one specific search engine
// each Searcher has it's own queue lying under the redis key <name>.queue
Redis::rpush($this->name . ".queue", $mission);
/**
* We have Searcher processes running for MetaGer
* Each Searcher is dedicated to one specific Searchengine and fetches it's results.
* We can have multiple Searchers for each engine, if needed.
* At this point we need to decide, whether we need to start a new Searcher process or
* if we have enough of them running.
* The information for that is provided through the redis system. Each running searcher
* gives information how long it has waited to be given the last fetcher job.
* The longer this time value is, the less frequent the search engine is used and the less
* searcher of that type we need.
* But if it's too low, i.e. 100ms, then the searcher is near to it's full workload and needs assistence.
**/
$needSearcher = false;
$searcherData = Redis::hgetall($this->name . ".stats");
// We now have an array of statistical data from the searchers
// Each searcher has one entry in it.
// So if it's empty, then we have currently no searcher running and
// of course need to spawn a new one.
if(sizeof($searcherData) === 0){
$needSearcher = true;
}else{
// There we go:
// There's at least one Fetcher running for this search engine.
// Now we have to check if the current count is enough to fetch all the
// searches or if it needs help.
// Let's hardcode a minimum of 100ms between every search job.
// First calculate the median of all Times
$median = 0;
foreach($searcherData as $pid => $data){
$data = explode(";", $data);
$median += floatval($data[1]);
}
$median /= sizeof($searcherData);
if($median < .1){ // Median is given as a float in seconds
$needSearcher = true;
}
}
}
if($needSearcher && Redis::get($this->name) !== "locked"){
Redis::set($this->name, "locked");
$this->dispatch(new Searcher($this->name));
if($needSearcher && Redis::get($this->name) !== "locked"){
Redis::set($this->name, "locked");
$this->dispatch(new Searcher($this->name));
}
return true;
}catch(ConnectionException $e){
return false;
}
}
}
......@@ -222,28 +229,33 @@ abstract class Searchengine
# Fragt die Ergebnisse von Redis ab und lädt Sie
public function retrieveResults(MetaGer $metager)
{
if ($this->loaded) {
return true;
}
$body = "";
if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash) && 0 === 1) {
$body = Cache::get($this->hash);
} elseif (Redis::hexists('search.' . $this->resultHash, $this->name)) {
$body = Redis::hget('search.' . $this->resultHash, $this->name);
if ($this->canCache && $this->cacheDuration > 0 && 0 === 1) {
Cache::put($this->hash, $body, $this->cacheDuration);
try{
if ($this->loaded) {
return true;
}
}
if ($body !== "") {
$this->loadResults($body);
$this->getNext($metager, $body);
$body = "";
if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash) && 0 === 1) {
$body = Cache::get($this->hash);
} elseif (Redis::hexists('search.' . $this->resultHash, $this->name)) {
$body = Redis::hget('search.' . $this->resultHash, $this->name);
if ($this->canCache && $this->cacheDuration > 0 && 0 === 1) {
Cache::put($this->hash, $body, $this->cacheDuration);
}
}
if ($body !== "") {
$this->loadResults($body);
$this->getNext($metager, $body);
$this->loaded = true;
Redis::hdel('search.' . $this->hash, $this->name);
return true;
} else {
return false;
}
}catch(ConnectionException $e){
$this->loaded = true;
Redis::hdel('search.' . $this->hash, $this->name);
return true;
} else {
return false;
}
}
......
app/Providers/AppServiceProvider.php
+ 1
− 34
View file @ 179ea0bf
......@@ -17,40 +17,7 @@ class AppServiceProvider extends ServiceProvider
*/
public function boot()
{
# Wir loggen im Redis-System für jede Sekunde des Tages, wie viele Worker aktiv am Laufen waren.
# Dies ist notwendig, damit wir mitbekommen können, ab welchem Zeitpunkt wir zu wenig Worker zur Verfügung haben.
Queue::before(function (JobProcessing $event) {
$this->begin = strtotime(date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y"))));
});
Queue::after(function (JobProcessed $event) {
$today = strtotime(date(DATE_RFC822, mktime(0, 0, 0, date("m"), date("d"), date("Y"))));
$end = strtotime(date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y")))) - $today;
$expireAt = strtotime(date(DATE_RFC822, mktime(0, 0, 0, date("m"), date("d") + 1, date("Y"))));
try {
$redis = Redis::connection('redisLogs');
if (!$redis) {
return;
}
$p = getmypid();
$host = gethostname();
$begin = $this->begin - $today;
$redis->pipeline(function ($pipe) use ($p, $expireAt, $host, $begin, $end) {
for ($i = $begin; $i <= $end; $i++) {
$pipe->sadd("logs.worker.$host.$i", strval($p));
$pipe->expire("logs.worker.$host.$i", 10);
$pipe->eval("redis.call('hset', 'logs.worker.$host', '$i', redis.call('scard', 'logs.worker.$host.$i'))", 0);
$pipe->sadd("logs.worker", $host);
if (date("H") !== 0) {
$pipe->expire("logs.worker.$host", $expireAt);
$pipe->expire("logs.worker", $expireAt);
}
}
});
} catch (\Exception $e) {
return;
}
});
//
}
/**
......
resources/lang/de/metaGer.php
+ 1
− 0
View file @ 179ea0bf
......@@ -18,4 +18,5 @@ return [
'sitesearch.failed' => 'Sie wollten eine Sitesearch auf :site durchführen. Leider unterstützen die eingestellten Suchmaschinen diese nicht. Sie können <a href=":searchLink">hier</a> die Sitesearch im Web-Fokus durchführen. Es werden ihnen Ergebnisse ohne Sitesearch angezeigt.',
'sitesearch.success' => 'Sie führen eine Sitesearch durch. Es werden nur Ergebnisse von der Seite: <a href="http://:site" target="_blank" rel="noopener">":site"</a> angezeigt.',
'redis.error' => 'Aktuell treten bei uns leider technische Probleme auf. Bitte versuchen Sie es zu einem späteren Zeitpunkt erneut.',
];
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment