Commit cb6e97de authored by Dominik Hebeler's avatar Dominik Hebeler

Die Suche wird nun mit Hilfe der Laravel Queue in den entsprechenden...

Die Suche wird nun mit Hilfe der Laravel Queue in den entsprechenden Einstellungen parallel durchgeführt.
Hier für muss in der .env Datei der Queue-Driver auf redis gesetzt werden und entsprechende Worker am laufen sein:
z.B. php artisan queue:listen, oder php artisan queue:work Prozesse im Supervisor
parent 37e857e3
This diff is collapsed.
......@@ -52,8 +52,6 @@ class MetaGer
function __construct()
{
$this->starttime = microtime(true);
define('CRLF', "\r\n");
define('BUFFER_LENGTH', 8192);
if( file_exists(config_path() . "/blacklistDomains.txt") && file_exists(config_path() . "/blacklistUrl.txt") )
{
# Blacklists einlesen:
......@@ -70,6 +68,12 @@ class MetaGer
$this->languageDetect->setNameMode("2");
}
public function getHashCode ()
{
$string = url()->full();
return md5($string);
}
public function rankAll ()
{
foreach( $this->engines as $engine )
......@@ -278,7 +282,6 @@ class MetaGer
public function createSearchEngines (Request $request)
{
#die(SocketRocket::get("tls", "dominik-pfennig.de", "", 443));
# Überprüfe, welche Sumas eingeschaltet sind
......@@ -312,7 +315,7 @@ class MetaGer
if(!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1"))
{
if($suma["name"]->__toString() === "overture")
if($suma["name"]->__toString() === "overture" || $suma["name"]->__toString() === "overtureAds")
{
$overtureEnabled = TRUE;
}
......@@ -336,7 +339,7 @@ class MetaGer
){
if(!(isset($suma['disabled']) && $suma['disabled']->__toString() === "1"))
{
if($suma["name"]->__toString() === "overture")
if($suma["name"]->__toString() === "overture" || $suma["name"]->__toString() === "overtureAds")
{
$overtureEnabled = TRUE;
}
......@@ -353,6 +356,8 @@ class MetaGer
$this->errors[] = "Achtung: Sie haben in ihren Einstellungen keine Suchmaschine ausgewählt.";
}
$engines = [];
foreach($enabledSearchengines as $engine){
......@@ -360,8 +365,8 @@ class MetaGer
{
continue;
}
# Wenn diese Suchmaschine gar nicht eingeschaltet sein soll
# Wenn diese Suchmaschine gar nicht eingeschaltet sein soll
$path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString());
$time = microtime();
......@@ -378,43 +383,51 @@ class MetaGer
$this->sockets[$tmp->name] = $tmp->fp;
}
}
# Nun passiert ein elementarer Schritt.
# Wir warten auf die Antwort der Suchmaschinen, da wir vorher nicht weiter machen können.
# aber natürlich nicht ewig.
# Die Verbindung steht zu diesem Zeitpunkt und auch unsere Request wurde schon gesendet.
# Wir geben der Suchmaschine nun bis zu 500ms Zeit zu antworten.
# Jetzt lesen wir alles aus, was da ist und verwerfen den Rest:
$enginesToLoad = count($engines);
$loadedEngines = 0;
$time = 0;
$timeStart = microtime(true);
while( true )
{
$time = (microtime(true) - $timeStart) * 1000;
$loadedEngines = intval(Redis::hlen('search.' . $this->getHashCode()));
$canBreak = true;
if( $overtureEnabled && !Redis::hexists('search.' . $this->getHashCode(), 'overture') && !Redis::hexists('search.' . $this->getHashCode(), 'overtureAds'))
$canBreak = false;
# Abbruchbedingung
if($time < 500)
{
if($loadedEngines >= $enginesToLoad)
if($loadedEngines >= $enginesToLoad && $canBreak)
break;
}elseif( $time >= 500 && $time < $this->time)
{
if( ($loadedEngines / ($enginesToLoad * 1.0)) >= 0.8 )
if( ($loadedEngines / ($enginesToLoad * 1.0)) >= 0.8 && $canBreak )
break;
}else
{
break;
}
foreach($engines as $engine)
usleep(50000);
}
foreach($engines as $engine)
{
if(!$engine->loaded)
{
if(!$engine->loaded)
{
$success = $engine->retrieveResults();
if($engine->loaded)
$loadedEngines += 1;
}
$engine->retrieveResults();
}
usleep(50000);
$time += 50;
}
# und verwerfen den Rest:
foreach( $engines as $engine )
{
if( !$engine->loaded )
......
This diff is collapsed.
......@@ -48,7 +48,7 @@ return [
'sqlite' => [
'driver' => 'sqlite',
'database' => env('DB_DATABASE', database_path('database.sqlite')),
'database' => database_path(env('DB_DATABASE', 'database.sqlite')),
'prefix' => '',
],
......
<?php
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
class CreateJobsTable extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('jobs', function (Blueprint $table) {
$table->bigIncrements('id');
$table->string('queue');
$table->longText('payload');
$table->tinyInteger('attempts')->unsigned();
$table->tinyInteger('reserved')->unsigned();
$table->unsignedInteger('reserved_at')->nullable();
$table->unsignedInteger('available_at');
$table->unsignedInteger('created_at');
$table->index(['queue', 'reserved', 'reserved_at']);
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::drop('jobs');
}
}
<?php
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
class CreateFailedJobsTable extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('failed_jobs', function (Blueprint $table) {
$table->increments('id');
$table->text('connection');
$table->text('queue');
$table->longText('payload');
$table->timestamp('failed_at')->useCurrent();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::drop('failed_jobs');
}
}
<?php
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
class CreateFailedJobsTable extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('failed_jobs', function (Blueprint $table) {
$table->increments('id');
$table->text('connection');
$table->text('queue');
$table->longText('payload');
$table->timestamp('failed_at')->useCurrent();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::drop('failed_jobs');
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment