Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • sebastian/MetaGer
  • cpietsch/MetaGer
  • kossow/MetaGer
  • jens/MetaGer
  • open-source/MetaGer
5 results
Show changes
Commits on Source (5)
...@@ -63,14 +63,13 @@ class RequestFetcher extends Command ...@@ -63,14 +63,13 @@ class RequestFetcher extends Command
try { try {
$blocking = false; $blocking = false;
$redis = Redis::connection("cache");
while ($this->shouldRun) { while ($this->shouldRun) {
$status = curl_multi_exec($this->multicurl, $active); $status = curl_multi_exec($this->multicurl, $active);
$currentJob = null; $currentJob = null;
if (!$blocking) { if (!$blocking) {
$currentJob = $redis->lpop(\App\MetaGer::FETCHQUEUE_KEY); $currentJob = Redis::lpop(\App\MetaGer::FETCHQUEUE_KEY);
} else { } else {
$currentJob = $redis->blpop(\App\MetaGer::FETCHQUEUE_KEY, 1); $currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
if (!empty($currentJob)) { if (!empty($currentJob)) {
$currentJob = $currentJob[1]; $currentJob = $currentJob[1];
} }
...@@ -105,8 +104,11 @@ class RequestFetcher extends Command ...@@ -105,8 +104,11 @@ class RequestFetcher extends Command
$body = \curl_multi_getcontent($info["handle"]); $body = \curl_multi_getcontent($info["handle"]);
} }
Cache::put($resulthash, $body, $cacheDurationMinutes * 60); Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
$pipe->set($resulthash, $body);
$pipe->expire($resulthash, 60);
Cache::put($resulthash, $body, $cacheDurationMinutes * 60);
});
\curl_multi_remove_handle($this->multicurl, $info["handle"]); \curl_multi_remove_handle($this->multicurl, $info["handle"]);
} }
if (!$active && !$answerRead) { if (!$active && !$answerRead) {
...@@ -131,11 +133,11 @@ class RequestFetcher extends Command ...@@ -131,11 +133,11 @@ class RequestFetcher extends Command
CURLOPT_RETURNTRANSFER => 1, CURLOPT_RETURNTRANSFER => 1,
CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1", CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
CURLOPT_FOLLOWLOCATION => true, CURLOPT_FOLLOWLOCATION => true,
CURLOPT_CONNECTTIMEOUT => 10, CURLOPT_CONNECTTIMEOUT => 2,
CURLOPT_MAXCONNECTS => 500, CURLOPT_MAXCONNECTS => 500,
CURLOPT_LOW_SPEED_LIMIT => 500, CURLOPT_LOW_SPEED_LIMIT => 50000,
CURLOPT_LOW_SPEED_TIME => 5, CURLOPT_LOW_SPEED_TIME => 2,
CURLOPT_TIMEOUT => 10, CURLOPT_TIMEOUT => 3,
)); ));
if (!empty($this->proxyhost) && !empty($this->proxyport) && !empty($this->proxyuser) && !empty($this->proxypassword)) { if (!empty($this->proxyhost) && !empty($this->proxyport) && !empty($this->proxyuser) && !empty($this->proxypassword)) {
......
...@@ -6,6 +6,7 @@ use App; ...@@ -6,6 +6,7 @@ use App;
use Cache; use Cache;
use Carbon; use Carbon;
use Illuminate\Http\Request; use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
use Jenssegers\Agent\Agent; use Jenssegers\Agent\Agent;
use LaravelLocalization; use LaravelLocalization;
use Log; use Log;
...@@ -853,7 +854,7 @@ class MetaGer ...@@ -853,7 +854,7 @@ class MetaGer
while (sizeof($enginesToWaitFor) > 0 || ($forceTimeout !== null && (microtime(true) - $timeStart) < $forceTimeout)) { while (sizeof($enginesToWaitFor) > 0 || ($forceTimeout !== null && (microtime(true) - $timeStart) < $forceTimeout)) {
foreach ($enginesToWaitFor as $index => $engine) { foreach ($enginesToWaitFor as $index => $engine) {
if (Cache::get($engine->hash) !== null) { if (Redis::get($engine->hash) !== null) {
$answered[] = $engine; $answered[] = $engine;
unset($enginesToWaitFor[$index]); unset($enginesToWaitFor[$index]);
break; break;
......
...@@ -46,7 +46,7 @@ class Quicktips ...@@ -46,7 +46,7 @@ class Quicktips
$mission = json_encode($mission); $mission = json_encode($mission);
Redis::connection('cache')->rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission); Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
} }
} }
......
...@@ -3,7 +3,6 @@ ...@@ -3,7 +3,6 @@
namespace App\Models; namespace App\Models;
use App\MetaGer; use App\MetaGer;
use Cache;
use Illuminate\Support\Facades\Redis; use Illuminate\Support\Facades\Redis;
abstract class Searchengine abstract class Searchengine
...@@ -148,7 +147,7 @@ abstract class Searchengine ...@@ -148,7 +147,7 @@ abstract class Searchengine
// Submit this mission to the corresponding Redis Queue // Submit this mission to the corresponding Redis Queue
// Since each Searcher is dedicated to one specific search engine // Since each Searcher is dedicated to one specific search engine
// each Searcher has it's own queue lying under the redis key <name>.queue // each Searcher has it's own queue lying under the redis key <name>.queue
Redis::connection('cache')->rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission); Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
if (!empty($timings)) { if (!empty($timings)) {
$timings["startSearch"][$this->name]["pushed job"] = microtime(true) - $timings["starttime"]; $timings["startSearch"][$this->name]["pushed job"] = microtime(true) - $timings["starttime"];
} }
...@@ -194,7 +193,7 @@ abstract class Searchengine ...@@ -194,7 +193,7 @@ abstract class Searchengine
$body = ""; $body = "";
} }
} else { } else {
$body = Cache::get($this->hash); $body = Redis::get($this->hash);
} }
if ($body !== null) { if ($body !== null) {
......
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
namespace App\Models\parserSkripte; namespace App\Models\parserSkripte;
use App\Models\XmlSearchengine; use App\Models\XmlSearchengine;
use Log;
class Qualigo extends XmlSearchengine class Qualigo extends XmlSearchengine
{ {
...@@ -14,6 +15,10 @@ class Qualigo extends XmlSearchengine ...@@ -14,6 +15,10 @@ class Qualigo extends XmlSearchengine
protected function loadXmlResults($resultsXml) protected function loadXmlResults($resultsXml)
{ {
if(empty($resultsXml) || !$resultsXml){
Log::error("A problem occurred parsing results from $this->name:");
return;
}
try { try {
$results = $resultsXml->xpath('//RL/RANK'); $results = $resultsXml->xpath('//RL/RANK');
foreach ($results as $result) { foreach ($results as $result) {
......
...@@ -21,9 +21,7 @@ Route::group( ...@@ -21,9 +21,7 @@ Route::group(
/** ADD ALL LOCALIZED ROUTES INSIDE THIS GROUP **/ /** ADD ALL LOCALIZED ROUTES INSIDE THIS GROUP **/
Route::get('/', 'StartpageController@loadStartPage'); Route::get('/', 'StartpageController@loadStartPage');
Route::get('sand', function () {
abort(500, "test");
});
Route::get('asso', function () { Route::get('asso', function () {
return view('assoziator.asso') return view('assoziator.asso')
->with('title', trans('titles.asso')) ->with('title', trans('titles.asso'))
......