Skip to content

GitLab

MetaGer
MetaGer
Commit 29aafe9f authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files
Options

Added the option to auto disable search engines

parent 54e2b9d5
Hide whitespace changes
Inline Side-by-side
Showing with 175 additions and 133 deletions
app/Console/Commands/LogRotate.php deleted 100644 → 0
View file @ 54e2b9d5
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
class LogRotate extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'log:rotate';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Dieses Kommando bezieht alle Log-Einträge aus dem Redis System und exportiert diese in entsprechende Dateien.';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
# Wir extrahieren die Logs mit den Daten der einzelnen Suchmaschinen:
# In den Daten ist festgehalten, wie oft eine jede Suchmaschine abgefragt wurde
# und wie häufig diese geantwortet hat:
$this->extractSearchEngineLogs();
}
private function extractSearchEngineLogs()
{
$redis = Redis::connection('redisLogs');
# Hier legen wir das Ergebnis ab:
# [
# 'fastbot' => [
# ''
# ]
# ]
$searchEngineLogs = ['recent' => [], 'overall' => []];
try {
# Wir benötigen zunächst die Namen aller Suchmaschinen:
$sumasFile = config_path() . "/sumas.xml";
$xml = simplexml_load_file($sumasFile);
$xml = $xml->xpath("suma");
$sumaNames = [];
foreach ($xml as $suma) {
$searchEngineLogs['recent'][$suma["name"]->__toString()] = ["requests" => 0, "answered" => 0];
}
#Auslesen/hinzufügen der Daten:
foreach ($searchEngineLogs['recent'] as $name => $values) {
$searchEngineLogs['recent'][$name]["requests"] = $redis->getset('logs.engines.requests.' . $name, 0);
$searchEngineLogs['recent'][$name]["answered"] = $redis->getset('logs.engines.answered.' . $name, 0);
}
$filePath = "/var/log/metager/";
# Wir haben nun die aktuellen daten und müssen diese noch mit den aktuellen Kombinieren:
# Hierbei behalten wir die Daten, seid dem letzten Rotate so wie sie sind und verknüpfen diese mit einer gesamt Statistik.
if (file_exists($filePath . "engine.log") && is_readable($filePath . "engine.log")) {
$oldData = file_get_contents($filePath . "engine.log");
# JSON Decode
$oldData = json_decode($oldData, true);
if (isset($oldData['overall'])) {
$searchEngineLogs['overall'] = $oldData['overall'];
}
}
# Jetzt fügen wir zu den Gesamtdaten die zuletzt erfassten hinzu:
foreach ($searchEngineLogs['recent'] as $name => $values) {
if (isset($searchEngineLogs['overall'][$name]["requests"])) {
$searchEngineLogs['overall'][$name]["requests"] += $values["requests"];
} else {
$searchEngineLogs['overall'][$name]["requests"] = $values["requests"];
}
if (isset($searchEngineLogs['overall'][$name]["answered"])) {
$searchEngineLogs['overall'][$name]["answered"] += $values["answered"];
} else {
$searchEngineLogs['overall'][$name]["answered"] = $values["answered"];
}
}
# Ins JSON Format umwandeln:
$searchEngineLogs = json_encode($searchEngineLogs, JSON_PRETTY_PRINT);
# Und in die Datei sichern
if (((file_exists($filePath . "engine.log") && is_writable($filePath . "engine.log")) || (!file_exists($filePath . "engine.log") && is_writable($filePath))) && file_put_contents($filePath . "engine.log", $searchEngineLogs) !== false) {
$this->info("Logs der Suchmaschinen erfolgreich exportiert.");
return;
} else {
# Der Schreibvorgang war nicht erfolgreich. Wir packen die Daten zurück
foreach (json_decode($searchEngineLogs, true) as $name => $values) {
$redis->incrby('logs.engines.requests.' . $name, $values["requests"]);
$redis->incrby('logs.engines.answered.' . $name, $values["answered"]);
}
$this->info("Konnte die Datei \"$filePath" . "engine.log\" nicht erstellen. Keine Rechte.");
return;
}
} catch (\ErrorException $e) {
return;
}
}
}
app/Console/Commands/MonthlyRequestsGather.php 0 → 100644
View file @ 29aafe9f
<?php
namespace App\Console\Commands;
use DB;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
class MonthlyRequestsGather extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'requests:gather';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Sends the gathered monthly requests from the redis to the central Database';
private $values = [];
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
# Read in the suma Files
$sumaFile = config_path('sumas.json');
$sumaEnFile = config_path('sumasEn.json');
if (file_exists($sumaFile) && file_exists($sumaEnFile)) {
$sumas = json_decode(file_get_contents($sumaFile));
$sumasEn = json_decode(file_get_contents($sumaEnFile));
$this->gatherLogs($sumas);
$this->gatherLogs($sumasEn);
foreach ($this->values as $name => $value) {
$entry = DB::table('monthlyrequests')->where(['name' => $name])->first();
$newCount = $value;
if ($entry === null) {
DB::table('monthlyrequests')->insert(['name' => $name, 'count' => $newCount]);
} else {
$newCount = $value + $entry->count;
DB::table('monthlyrequests')->where(['name' => $name])->update(['count' => $newCount]);
}
}
$this->disableOverusedEngines($sumaFile, $sumas);
$this->disableOverusedEngines($sumaEnFile, $sumasEn);
}
}
private function disableOverusedEngines($sumasFile, $sumas)
{
$currentValues = DB::table('monthlyrequests')->get();
foreach ($sumas->sumas as $sumaName => $suma) {
if (!empty($suma->disabled) && $suma->disabled === true) {
continue;
}
if (empty($suma->{"monthly-requests"})) {
continue;
}
$currentValue = 0;
foreach ($currentValues as $value) {
if ($value->name === $sumaName) {
$currentValue = $value->count;
}
}
$monthlyRequests = $suma->{"monthly-requests"};
if (!empty($suma->{"auto-disabled"}) && $suma->{"auto-disabled"} === true) {
# Maybe this engine should be enabled again
if ($currentValue < $monthlyRequests) {
unset($sumas->sumas->{$sumaName}->{"auto-disabled"});
file_put_contents($sumasFile, json_encode($sumas, JSON_PRETTY_PRINT));
}
} else if ($currentValue >= $monthlyRequests) {
$sumas->sumas->{$sumaName}->{"auto-disabled"} = true;
file_put_contents($sumasFile, json_encode($sumas, JSON_PRETTY_PRINT));
}
}
}
private function gatherLogs($sumas)
{
foreach ($sumas->sumas as $sumaName => $suma) {
if (!empty($suma->{"monthly-requests"})) {
$monthlyLimit = $suma->{"monthly-requests"};
$currentValue = Redis::get('monthlyRequests:' . $sumaName);
Redis::del('monthlyRequests:' . $sumaName);
if (empty($currentValue)) {
$currentValue = 0;
}
if (empty($this->values[$sumaName]) && $currentValue > 0) {
$this->values[$sumaName] = intval($currentValue);
}
}
}
}
}
app/Console/Kernel.php
View file @ 29aafe9f
......@@ -25,10 +25,14 @@ class Kernel extends ConsoleKernel
*/
protected function schedule(Schedule $schedule)
{
$schedule->command('log:rotate')->everyTenMinutes();
$schedule->command('requests:gather')->everyFifteenMinutes();
$schedule->call(function () {
DB::table('monthlyrequests')->truncate();
})->monthlyOn(1, '00:00');
// Delete all of the old humanverification entries
$schedule->call(function() {
$schedule->call(function () {
DB::delete('DELETE FROM humanverification WHERE updated_at < (now() - interval 72 hour) AND whitelist = 0 ORDER BY updated_at DESC');
DB::delete('DELETE FROM humanverification WHERE updated_at < (now() - interval 2 week) AND whitelist = 1 ORDER BY updated_at DESC');
})->everyThirtyMinutes();
......@@ -42,6 +46,6 @@ class Kernel extends ConsoleKernel
protected function commands()
{
require base_path('routes/console.php');
$this->load(__DIR__.'/Commands');
$this->load(__DIR__ . '/Commands');
}
}
app/MetaGer.php
View file @ 29aafe9f
......@@ -542,7 +542,8 @@ class MetaGer
foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
# Check if this engine is disabled and can't be used
$disabled = empty($this->sumaFile->sumas->{$suma}->disabled) ? false : $this->sumaFile->sumas->{$suma}->disabled;
if ($disabled) {
$autoDisabled = empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) ? false : $this->sumaFile->sumas->{$suma}->{"auto-disabled"};
if ($disabled || $autoDisabled) {
continue;
}
......@@ -710,6 +711,11 @@ class MetaGer
foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"}) {
if (!empty($filter->sumas->{$suma})) {
# If the searchengine is disabled this filter shouldn't be available
if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
|| (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)) {
continue;
}
$availableFilter[$filterName] = $filter;
}
}
......@@ -1343,16 +1349,7 @@ class MetaGer
# 2 Arten von Logs in einem wird die Anzahl der Abfragen an eine Suchmaschine gespeichert und in der anderen
# die Anzahl, wie häufig diese Ergebnisse geliefert hat.
$enginesToLoad = $this->enginesToLoad;
$redis->pipeline(function ($pipe) use ($enginesToLoad, $logEntry) {
$pipe->rpush('logs.search', $logEntry);
foreach ($this->enginesToLoad as $name => $answered) {
$pipe->incr('logs.engines.requests.' . $name);
if ($answered) {
$pipe->incr('logs.engines.answered.' . $name);
}
}
});
$redis->rpush('logs.search', $logEntry);
} catch (\Exception $e) {
return;
}
......
app/Models/Searchengine.php
View file @ 29aafe9f
......@@ -134,6 +134,12 @@ abstract class Searchengine
// each Searcher has it's own queue lying under the redis key <name>.queue
Redis::rpush($this->name . ".queue", $mission);
// The request is not cached and will be submitted to the searchengine
// We need to check if the number of requests to this engine are limited
if (!empty($this->engine->{"monthly-requests"})) {
Redis::incr("monthlyRequests:" . $this->name);
}
/**
* We have Searcher processes running for MetaGer
* Each Searcher is dedicated to one specific Searchengine and fetches it's results.
......
database/migrations/2019_02_19_094337_monthlyrequests.php 0 → 100644
View file @ 29aafe9f
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class Monthlyrequests extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('monthlyrequests', function (Blueprint $table) {
$table->string('name')->unique();
$table->integer('count');
$table->timestamp('updated_at');
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::drop('sponsorenlinks');
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment