Commit d52875a7 authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Merge branch 'development' into '943-optimize-deployment'

Development

See merge request !1540
parents d0496140 301b9974
...@@ -63,14 +63,13 @@ class RequestFetcher extends Command ...@@ -63,14 +63,13 @@ class RequestFetcher extends Command
try { try {
$blocking = false; $blocking = false;
$redis = Redis::connection("cache");
while ($this->shouldRun) { while ($this->shouldRun) {
$status = curl_multi_exec($this->multicurl, $active); $status = curl_multi_exec($this->multicurl, $active);
$currentJob = null; $currentJob = null;
if (!$blocking) { if (!$blocking) {
$currentJob = $redis->lpop(\App\MetaGer::FETCHQUEUE_KEY); $currentJob = Redis::lpop(\App\MetaGer::FETCHQUEUE_KEY);
} else { } else {
$currentJob = $redis->blpop(\App\MetaGer::FETCHQUEUE_KEY, 1); $currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
if (!empty($currentJob)) { if (!empty($currentJob)) {
$currentJob = $currentJob[1]; $currentJob = $currentJob[1];
} }
...@@ -105,8 +104,11 @@ class RequestFetcher extends Command ...@@ -105,8 +104,11 @@ class RequestFetcher extends Command
$body = \curl_multi_getcontent($info["handle"]); $body = \curl_multi_getcontent($info["handle"]);
} }
Cache::put($resulthash, $body, $cacheDurationMinutes * 60); Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
$pipe->set($resulthash, $body);
$pipe->expire($resulthash, 60);
Cache::put($resulthash, $body, $cacheDurationMinutes * 60);
});
\curl_multi_remove_handle($this->multicurl, $info["handle"]); \curl_multi_remove_handle($this->multicurl, $info["handle"]);
} }
if (!$active && !$answerRead) { if (!$active && !$answerRead) {
...@@ -131,11 +133,11 @@ class RequestFetcher extends Command ...@@ -131,11 +133,11 @@ class RequestFetcher extends Command
CURLOPT_RETURNTRANSFER => 1, CURLOPT_RETURNTRANSFER => 1,
CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1", CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
CURLOPT_FOLLOWLOCATION => true, CURLOPT_FOLLOWLOCATION => true,
CURLOPT_CONNECTTIMEOUT => 10, CURLOPT_CONNECTTIMEOUT => 2,
CURLOPT_MAXCONNECTS => 500, CURLOPT_MAXCONNECTS => 500,
CURLOPT_LOW_SPEED_LIMIT => 500, CURLOPT_LOW_SPEED_LIMIT => 50000,
CURLOPT_LOW_SPEED_TIME => 5, CURLOPT_LOW_SPEED_TIME => 2,
CURLOPT_TIMEOUT => 10, CURLOPT_TIMEOUT => 3,
)); ));
if (!empty($this->proxyhost) && !empty($this->proxyport) && !empty($this->proxyuser) && !empty($this->proxypassword)) { if (!empty($this->proxyhost) && !empty($this->proxyport) && !empty($this->proxyuser) && !empty($this->proxypassword)) {
......
...@@ -6,6 +6,7 @@ use App; ...@@ -6,6 +6,7 @@ use App;
use Cache; use Cache;
use Carbon; use Carbon;
use Illuminate\Http\Request; use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
use Jenssegers\Agent\Agent; use Jenssegers\Agent\Agent;
use LaravelLocalization; use LaravelLocalization;
use Log; use Log;
...@@ -853,7 +854,7 @@ class MetaGer ...@@ -853,7 +854,7 @@ class MetaGer
while (sizeof($enginesToWaitFor) > 0 || ($forceTimeout !== null && (microtime(true) - $timeStart) < $forceTimeout)) { while (sizeof($enginesToWaitFor) > 0 || ($forceTimeout !== null && (microtime(true) - $timeStart) < $forceTimeout)) {
foreach ($enginesToWaitFor as $index => $engine) { foreach ($enginesToWaitFor as $index => $engine) {
if (Cache::get($engine->hash) !== null) { if (Redis::get($engine->hash) !== null) {
$answered[] = $engine; $answered[] = $engine;
unset($enginesToWaitFor[$index]); unset($enginesToWaitFor[$index]);
break; break;
......
...@@ -46,7 +46,7 @@ class Quicktips ...@@ -46,7 +46,7 @@ class Quicktips
$mission = json_encode($mission); $mission = json_encode($mission);
Redis::connection('cache')->rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission); Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
} }
} }
......
...@@ -3,7 +3,6 @@ ...@@ -3,7 +3,6 @@
namespace App\Models; namespace App\Models;
use App\MetaGer; use App\MetaGer;
use Cache;
use Illuminate\Support\Facades\Redis; use Illuminate\Support\Facades\Redis;
abstract class Searchengine abstract class Searchengine
...@@ -148,7 +147,7 @@ abstract class Searchengine ...@@ -148,7 +147,7 @@ abstract class Searchengine
// Submit this mission to the corresponding Redis Queue // Submit this mission to the corresponding Redis Queue
// Since each Searcher is dedicated to one specific search engine // Since each Searcher is dedicated to one specific search engine
// each Searcher has it's own queue lying under the redis key <name>.queue // each Searcher has it's own queue lying under the redis key <name>.queue
Redis::connection('cache')->rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission); Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
if (!empty($timings)) { if (!empty($timings)) {
$timings["startSearch"][$this->name]["pushed job"] = microtime(true) - $timings["starttime"]; $timings["startSearch"][$this->name]["pushed job"] = microtime(true) - $timings["starttime"];
} }
...@@ -194,7 +193,7 @@ abstract class Searchengine ...@@ -194,7 +193,7 @@ abstract class Searchengine
$body = ""; $body = "";
} }
} else { } else {
$body = Cache::get($this->hash); $body = Redis::get($this->hash);
} }
if ($body !== null) { if ($body !== null) {
......
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
namespace App\Models\parserSkripte; namespace App\Models\parserSkripte;
use App\Models\XmlSearchengine; use App\Models\XmlSearchengine;
use Log;
class Qualigo extends XmlSearchengine class Qualigo extends XmlSearchengine
{ {
...@@ -14,6 +15,10 @@ class Qualigo extends XmlSearchengine ...@@ -14,6 +15,10 @@ class Qualigo extends XmlSearchengine
protected function loadXmlResults($resultsXml) protected function loadXmlResults($resultsXml)
{ {
if(empty($resultsXml) || !$resultsXml){
Log::error("A problem occurred parsing results from $this->name:");
return;
}
try { try {
$results = $resultsXml->xpath('//RL/RANK'); $results = $resultsXml->xpath('//RL/RANK');
foreach ($results as $result) { foreach ($results as $result) {
......
...@@ -21,9 +21,7 @@ Route::group( ...@@ -21,9 +21,7 @@ Route::group(
/** ADD ALL LOCALIZED ROUTES INSIDE THIS GROUP **/ /** ADD ALL LOCALIZED ROUTES INSIDE THIS GROUP **/
Route::get('/', 'StartpageController@loadStartPage'); Route::get('/', 'StartpageController@loadStartPage');
Route::get('sand', function () {
abort(500, "test");
});
Route::get('asso', function () { Route::get('asso', function () {
return view('assoziator.asso') return view('assoziator.asso')
->with('title', trans('titles.asso')) ->with('title', trans('titles.asso'))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment