Skip to content
GitLab
Commit 6f5f5583 authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Merge branch... 

Merge branch '207-wenn-der-cache-server-nicht-erreichbar-ist-sollte-die-suche-trotzdem-funktionieren' into 'development'

Wenn keine Verbindung zum Cache Server aufgebaut werden kann (z.B. bei einem Upd…

…ate), benutzen wir den Cache nun einfach nicht mehr anstatt abzustürzen.
Closes #207

See merge request !329
parents ba6d00b2 9d65f4e8
Hide whitespace changes
Inline Side-by-side
app/Jobs/Search.php
View file @ 6f5f5583
...@@ -3,16 +3,16 @@ ...@@ -3,16 +3,16 @@
namespace App\Jobs; namespace App\Jobs;
use App\Jobs\Job; use App\Jobs\Job;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Http\Request; use Illuminate\Http\Request;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Redis; use Redis;
class Search extends Job implements ShouldQueue class Search extends Job implements ShouldQueue
{ {
use InteractsWithQueue, SerializesModels; use InteractsWithQueue, SerializesModels;
protected $hash, $host, $port, $name, $getString, $useragent, $fp; protected $hash, $host, $port, $name, $getString, $useragent, $fp;
protected $buffer_length = 8192; protected $buffer_length = 8192;
...@@ -23,10 +23,10 @@ class Search extends Job implements ShouldQueue ...@@ -23,10 +23,10 @@ class Search extends Job implements ShouldQueue
*/ */
public function __construct($hash, $host, $port, $name, $getString, $useragent) public function __construct($hash, $host, $port, $name, $getString, $useragent)
{ {
$this->hash = $hash; $this->hash = $hash;
$this->host = $host; $this->host = $host;
$this->port = $port; $this->port = $port;
$this->name = $name; $this->name = $name;
$this->getString = $getString; $this->getString = $getString;
$this->useragent = $useragent; $this->useragent = $useragent;
} }
...@@ -39,72 +39,66 @@ class Search extends Job implements ShouldQueue ...@@ -39,72 +39,66 @@ class Search extends Job implements ShouldQueue
public function handle(Request $request) public function handle(Request $request)
{ {
$this->fp = $this->getFreeSocket(); $this->fp = $this->getFreeSocket();
if($this->fp) if ($this->fp) {
{ if ($this->writeRequest()) {
if($this->writeRequest())
{
$this->readAnswer(); $this->readAnswer();
} }
} }
} }
private function readAnswer () private function readAnswer()
{ {
$time = microtime(true); $time = microtime(true);
$headers = ''; $headers = '';
$body = ''; $body = '';
$length = 0; $length = 0;
if(!$this->fp) if (!$this->fp) {
{
return; return;
} }
// get headers FIRST // get headers FIRST
$c = 0; $c = 0;
stream_set_blocking($this->fp, 1); stream_set_blocking($this->fp, 1);
do do {
{
// use fgets() not fread(), fgets stops reading at first newline // use fgets() not fread(), fgets stops reading at first newline
// or buffer which ever one is reached first // or buffer which ever one is reached first
$data = fgets($this->fp, 8192); $data = fgets($this->fp, 8192);
// a sincle CRLF indicates end of headers // a sincle CRLF indicates end of headers
if ($data === false || $data == "\r\n" || feof($this->fp) ) { if ($data === false || $data == "\r\n" || feof($this->fp)) {
// break BEFORE OUTPUT // break BEFORE OUTPUT
break; break;
} }
if( sizeof(($tmp = explode(": ", $data))) === 2 ) if (sizeof(($tmp = explode(": ", $data))) === 2) {
$headers[strtolower(trim($tmp[0]))] = trim($tmp[1]); $headers[strtolower(trim($tmp[0]))] = trim($tmp[1]);
}
$c++; $c++;
} } while (true);
while (true);
// end of headers // end of headers
if(sizeof($headers) > 1){ if (sizeof($headers) > 1) {
$bodySize = 0; $bodySize = 0;
if( isset($headers["transfer-encoding"]) && $headers["transfer-encoding"] === "chunked" ) if (isset($headers["transfer-encoding"]) && $headers["transfer-encoding"] === "chunked") {
{
$body = $this->readChunked(); $body = $this->readChunked();
}elseif( isset($headers['content-length']) ) } elseif (isset($headers['content-length'])) {
{
$length = trim($headers['content-length']); $length = trim($headers['content-length']);
if(is_numeric($length) && $length >= 1) if (is_numeric($length) && $length >= 1) {
$body = $this->readBody($length); $body = $this->readBody($length);
}
$bodySize = strlen($body); $bodySize = strlen($body);
}else } else {
{
exit; exit;
} }
}else } else {
{
return; return;
} }
Redis::del($this->host . "." . $this->socketNumber); Redis::del($this->host . "." . $this->socketNumber);
if( isset($headers["content-encoding"]) && $headers['content-encoding'] === "gzip") if (isset($headers["content-encoding"]) && $headers['content-encoding'] === "gzip") {
{
$body = $this->gunzip($body); $body = $this->gunzip($body);
} }
Redis::hset('search.' . $this->hash, $this->name, $body); Redis::hset('search.' . $this->hash, $this->name, $body);
...@@ -114,12 +108,11 @@ class Search extends Job implements ShouldQueue ...@@ -114,12 +108,11 @@ class Search extends Job implements ShouldQueue
private function readBody($length) private function readBody($length)
{ {
$theData = ''; $theData = '';
$done = false; $done = false;
stream_set_blocking($this->fp, 0); stream_set_blocking($this->fp, 0);
$startTime = time(); $startTime = time();
$lastTime = $startTime; $lastTime = $startTime;
while (!feof($this->fp) && !$done && (($startTime + 1) > time()) && $length !== 0) while (!feof($this->fp) && !$done && (($startTime + 1) > time()) && $length !== 0) {
{
usleep(100); usleep(100);
$theNewData = fgets($this->fp, 8192); $theNewData = fgets($this->fp, 8192);
$theData .= $theNewData; $theData .= $theNewData;
...@@ -135,8 +128,7 @@ class Search extends Job implements ShouldQueue ...@@ -135,8 +128,7 @@ class Search extends Job implements ShouldQueue
$body = ''; $body = '';
// read from chunked stream // read from chunked stream
// loop though the stream // loop though the stream
do do {
{
// NOTE: for chunked encoding to work properly make sure // NOTE: for chunked encoding to work properly make sure
// there is NOTHING (besides newlines) before the first hexlength // there is NOTHING (besides newlines) before the first hexlength
...@@ -162,15 +154,16 @@ class Search extends Job implements ShouldQueue ...@@ -162,15 +154,16 @@ class Search extends Job implements ShouldQueue
// zero is sent when at the end of the chunks // zero is sent when at the end of the chunks
// or the end of the stream or error // or the end of the stream or error
if ($line === false || $length < 1 || feof($this->fp)) { if ($line === false || $length < 1 || feof($this->fp)) {
if($length <= 0) if ($length <= 0) {
fgets($this->fp, 8192); fgets($this->fp, 8192);
}
// break out of the streams loop // break out of the streams loop
break; break;
} }
// loop though the chunk // loop though the chunk
do do {
{
// read $length amount of data // read $length amount of data
// (use fread here) // (use fread here)
$data = fread($this->fp, $length); $data = fread($this->fp, $length);
...@@ -186,42 +179,42 @@ class Search extends Job implements ShouldQueue ...@@ -186,42 +179,42 @@ class Search extends Job implements ShouldQueue
$body .= $data; $body .= $data;
// zero or less or end of connection break // zero or less or end of connection break
if ($length <= 0 || feof($this->fp)) if ($length <= 0 || feof($this->fp)) {
{
// break out of the chunk loop // break out of the chunk loop
if($length <= 0) if ($length <= 0) {
fgets($this->fp, 8192); fgets($this->fp, 8192);
}
break; break;
} }
} } while (true);
while (true);
// end of chunk loop // end of chunk loop
} } while (true);
while (true);
// end of stream loop // end of stream loop
return $body; return $body;
} }
private function gunzip($zipped) { private function gunzip($zipped)
$offset = 0; {
if (substr($zipped,0,2) == "\x1f\x8b") $offset = 0;
$offset = 2; if (substr($zipped, 0, 2) == "\x1f\x8b") {
if (substr($zipped,$offset,1) == "\x08") $offset = 2;
{ }
try
{ if (substr($zipped, $offset, 1) == "\x08") {
return gzinflate(substr($zipped, $offset + 8)); try
} catch (\Exception $e) {
{ return gzinflate(substr($zipped, $offset + 8));
abort(500, "Fehler beim unzip des Ergebnisses von folgendem Anbieter: " . $this->name); } catch (\Exception $e) {
abort(500, "Fehler beim unzip des Ergebnisses von folgendem Anbieter: " . $this->name);
}
} }
} return "Unknown Format";
return "Unknown Format"; }
}
private function writeRequest () private function writeRequest()
{ {
$out = "GET " . $this->getString . " HTTP/1.1\r\n"; $out = "GET " . $this->getString . " HTTP/1.1\r\n";
$out .= "Host: " . $this->host . "\r\n"; $out .= "Host: " . $this->host . "\r\n";
$out .= "User-Agent: " . $this->useragent . "\r\n"; $out .= "User-Agent: " . $this->useragent . "\r\n";
...@@ -230,51 +223,69 @@ class Search extends Job implements ShouldQueue ...@@ -230,51 +223,69 @@ class Search extends Job implements ShouldQueue
$out .= "Accept-Encoding: gzip, deflate, br\r\n"; $out .= "Accept-Encoding: gzip, deflate, br\r\n";
$out .= "Connection: keep-alive\r\n\r\n"; $out .= "Connection: keep-alive\r\n\r\n";
# Anfrage senden: # Anfrage senden:
$sent = 0; $string = $out; $time = microtime(true); $sent = 0;
while(true) $string = $out;
{ $time = microtime(true);
try{ while (true) {
$timeElapsed = microtime(true) - $time;
if ($timeElapsed > 0.5) {
# Irgendwas ist mit unserem Socket passiert. Wir brauchen einen neuen:
if ($this->fp) {
fclose($this->fp);
}
Redis::del($this->name . "." . $this->socketNumber);
$this->fp = $this->getFreeSocket();
$sent = 0;
$string = $out;
break;
}
try {
$tmp = fwrite($this->fp, $string); $tmp = fwrite($this->fp, $string);
}catch(\ErrorException $e) } catch (\ErrorException $e) {
{
# Irgendwas ist mit unserem Socket passiert. Wir brauchen einen neuen: # Irgendwas ist mit unserem Socket passiert. Wir brauchen einen neuen:
fclose($this->fp); try {
fclose($this->fp);
} catch (\ErrorException $e) {
}
Redis::del($this->name . "." . $this->socketNumber); Redis::del($this->name . "." . $this->socketNumber);
$this->fp = $this->getFreeSocket(); $this->fp = $this->getFreeSocket();
$sent = 0; $sent = 0;
$string = $out; $string = $out;
continue; continue;
} }
if($tmp){ if ($tmp) {
$sent += $tmp; $sent += $tmp;
$string = substr($string, $tmp); $string = substr($string, $tmp);
} }
if($sent >= strlen($out)) if ($sent >= strlen($out)) {
break; break;
}
} }
if( $sent === strlen($out) ) if ($sent === strlen($out)) {
{
return true; return true;
} }
return false; return false;
} }
private function getFreeSocket() public function getFreeSocket()
{ {
# Je nach Auslastung des Servers ( gleichzeitige Abfragen ), kann es sein, dass wir mehrere Sockets benötigen um die Abfragen ohne Wartezeit beantworten zu können. # Je nach Auslastung des Servers ( gleichzeitige Abfragen ), kann es sein, dass wir mehrere Sockets benötigen um die Abfragen ohne Wartezeit beantworten zu können.
# pfsockopen öffnet dabei einen persistenten Socket, der also auch zwischen den verschiedenen php Prozessen geteilt werden kann. # pfsockopen öffnet dabei einen persistenten Socket, der also auch zwischen den verschiedenen php Prozessen geteilt werden kann.
# Wenn der Hostname mit einem bereits erstellten Socket übereinstimmt, wird die Verbindung also aufgegriffen und fortgeführt. # Wenn der Hostname mit einem bereits erstellten Socket übereinstimmt, wird die Verbindung also aufgegriffen und fortgeführt.
# Allerdings dürfen wir diesen nur verwenden, wenn er nicht bereits von einem anderen Prozess zur Kommunikation verwendet wird. # Allerdings dürfen wir diesen nur verwenden, wenn er nicht bereits von einem anderen Prozess zur Kommunikation verwendet wird.
# Wenn dem so ist, probieren wir den nächsten Socket zu verwenden. # Wenn dem so ist, probieren wir den nächsten Socket zu verwenden.
# Dies festzustellen ist komplizierter, als man sich das vorstellt. Folgendes System sollte funktionieren: # Dies festzustellen ist komplizierter, als man sich das vorstellt. Folgendes System sollte funktionieren:
# 1. Stelle fest, ob dieser Socket neu erstellt wurde, oder ob ein existierender geöffnet wurde. # 1. Stelle fest, ob dieser Socket neu erstellt wurde, oder ob ein existierender geöffnet wurde.
$counter = 0; $fp = null; $counter = 0;
do $fp = null;
{ do {
if( intval(Redis::exists($this->host . ".$counter")) === 0 ) if (intval(Redis::exists($this->host . ".$counter")) === 0) {
{
Redis::set($this->host . ".$counter", 1); Redis::set($this->host . ".$counter", 1);
Redis::expire($this->host . ".$counter", 5); Redis::expire($this->host . ".$counter", 5);
$this->socketNumber = $counter; $this->socketNumber = $counter;
...@@ -282,33 +293,29 @@ class Search extends Job implements ShouldQueue ...@@ -282,33 +293,29 @@ class Search extends Job implements ShouldQueue
try try
{ {
$fp = pfsockopen($this->getHost() . ":" . $this->port . "/$counter", $this->port, $errstr, $errno, 1); $fp = pfsockopen($this->getHost() . ":" . $this->port . "/$counter", $this->port, $errstr, $errno, 1);
}catch(\ErrorException $e) } catch (\ErrorException $e) {
{
break; break;
} }
# Wir gucken, ob der Lesepuffer leer ist: # Wir gucken, ob der Lesepuffer leer ist:
stream_set_blocking($fp, 0); stream_set_blocking($fp, 0);
$string = fgets($fp, 8192); $string = fgets($fp, 8192);
if( $string !== false || feof($fp) ) if ($string !== false || feof($fp)) {
{
fclose($fp); fclose($fp);
continue; continue;
} }
break; break;
} }
$counter++; $counter++;
}while(true); } while (true);
return $fp; return $fp;
} }
private function getHost() public function getHost()
{ {
$return = ""; $return = "";
if( $this->port === "443" ) if ($this->port === "443") {
{
$return .= "tls://"; $return .= "tls://";
}else } else {
{
$return .= "tcp://"; $return .= "tcp://";
} }
$return .= $this->host; $return .= $this->host;
......
app/MetaGer.php
View file @ 6f5f5583
...@@ -8,6 +8,7 @@ use Illuminate\Http\Request; ...@@ -8,6 +8,7 @@ use Illuminate\Http\Request;
use Jenssegers\Agent\Agent; use Jenssegers\Agent\Agent;
use LaravelLocalization; use LaravelLocalization;
use Log; use Log;
use Predis\Connection\ConnectionException;
use Redis; use Redis;
class MetaGer class MetaGer
...@@ -33,6 +34,7 @@ class MetaGer ...@@ -33,6 +34,7 @@ class MetaGer
protected $errors = []; protected $errors = [];
protected $addedHosts = []; protected $addedHosts = [];
protected $startCount = 0; protected $startCount = 0;
protected $canCache = false;
# Daten über die Abfrage # Daten über die Abfrage
protected $ip; protected $ip;
protected $language; protected $language;
...@@ -70,6 +72,13 @@ class MetaGer ...@@ -70,6 +72,13 @@ class MetaGer
$this->languageDetect = new TextLanguageDetect(); $this->languageDetect = new TextLanguageDetect();
$this->languageDetect->setNameMode("2"); $this->languageDetect->setNameMode("2");
try {
Cache::has('test');
$this->canCache = true;
} catch (ConnectionException $e) {
$this->canCache = false;
}
} }
public function getHashCode() public function getHashCode()
...@@ -287,17 +296,7 @@ class MetaGer ...@@ -287,17 +296,7 @@ class MetaGer
$this->errors[] = "Leider konnten wir zu Ihrer Sucheingabe keine passenden Ergebnisse finden."; $this->errors[] = "Leider konnten wir zu Ihrer Sucheingabe keine passenden Ergebnisse finden.";
} }
if (isset($this->last) && count($this->last) > 0) { if ($this->canCache() && isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
$page = $this->page - 1;
$this->last = [
'page' => $page,
'startBackwards' => $this->results[0]->number,
'engines' => $this->last,
];
Cache::put(md5(serialize($this->last)), serialize($this->last), 60);
}
if (isset($this->next) && count($this->next) > 0 && count($this->results) > 0) {
$page = $this->page + 1; $page = $this->page + 1;
$this->next = [ $this->next = [
'page' => $page, 'page' => $page,
...@@ -305,6 +304,8 @@ class MetaGer ...@@ -305,6 +304,8 @@ class MetaGer
'engines' => $this->next, 'engines' => $this->next,
]; ];
Cache::put(md5(serialize($this->next)), serialize($this->next), 60); Cache::put(md5(serialize($this->next)), serialize($this->next), 60);
} else {
$this->next = [];
} }
} }
...@@ -973,6 +974,10 @@ class MetaGer ...@@ -973,6 +974,10 @@ class MetaGer
$this->addedHosts[$hash] = 1; $this->addedHosts[$hash] = 1;
} }
} }
public function canCache()
{
return $this->canCache;
}
public function getSite() public function getSite()
{ {
return $this->site; return $this->site;
......
app/Models/Searchengine.php
View file @ 6f5f5583
...@@ -71,6 +71,7 @@ abstract class Searchengine ...@@ -71,6 +71,7 @@ abstract class Searchengine
$this->getString = $this->generateGetString($q, $metager->getUrl(), $metager->getLanguage(), $metager->getCategory()); $this->getString = $this->generateGetString($q, $metager->getUrl(), $metager->getLanguage(), $metager->getCategory());
$this->hash = md5($this->host . $this->getString . $this->port . $this->name); $this->hash = md5($this->host . $this->getString . $this->port . $this->name);
$this->resultHash = $metager->getHashCode(); $this->resultHash = $metager->getHashCode();
$this->canCache = $metager->canCache();
} }
abstract public function loadResults($result); abstract public function loadResults($result);
...@@ -82,7 +83,7 @@ abstract class Searchengine ...@@ -82,7 +83,7 @@ abstract class Searchengine
public function startSearch(\App\MetaGer $metager) public function startSearch(\App\MetaGer $metager)
{ {
if (Cache::has($this->hash)) { if ($this->canCache && Cache::has($this->hash)) {
$this->cached = true; $this->cached = true;
$this->retrieveResults($metager); $this->retrieveResults($metager);
} else { } else {
...@@ -142,11 +143,11 @@ abstract class Searchengine ...@@ -142,11 +143,11 @@ abstract class Searchengine
} }
$body = ""; $body = "";
if ($this->cacheDuration > 0 && Cache::has($this->hash)) { if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash)) {
$body = Cache::get($this->hash); $body = Cache::get($this->hash);
} elseif (Redis::hexists('search.' . $this->resultHash, $this->name)) { } elseif (Redis::hexists('search.' . $this->resultHash, $this->name)) {
$body = Redis::hget('search.' . $this->resultHash, $this->name);