Skip to content

GitLab

Commit 29aafe9f authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Added the option to auto disable search engines

parent 54e2b9d5
Hide whitespace changes
Inline Side-by-side
app/Console/Commands/LogRotate.php deleted 100644 → 0
View file @ 54e2b9d5
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
class LogRotate extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'log:rotate';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Dieses Kommando bezieht alle Log-Einträge aus dem Redis System und exportiert diese in entsprechende Dateien.';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
# Wir extrahieren die Logs mit den Daten der einzelnen Suchmaschinen:
# In den Daten ist festgehalten, wie oft eine jede Suchmaschine abgefragt wurde
# und wie häufig diese geantwortet hat:
$this->extractSearchEngineLogs();
}
private function extractSearchEngineLogs()
{
$redis = Redis::connection('redisLogs');
# Hier legen wir das Ergebnis ab:
# [
# 'fastbot' => [
# ''
# ]
# ]
$searchEngineLogs = ['recent' => [], 'overall' => []];
try {
# Wir benötigen zunächst die Namen aller Suchmaschinen:
$sumasFile = config_path() . "/sumas.xml";
$xml = simplexml_load_file($sumasFile);
$xml = $xml->xpath("suma");
$sumaNames = [];
foreach ($xml as $suma) {
$searchEngineLogs['recent'][$suma["name"]->__toString()] = ["requests" => 0, "answered" => 0];
}
#Auslesen/hinzufügen der Daten:
foreach ($searchEngineLogs['recent'] as $name => $values) {
$searchEngineLogs['recent'][$name]["requests"] = $redis->getset('logs.engines.requests.' . $name, 0);
$searchEngineLogs['recent'][$name]["answered"] = $redis->getset('logs.engines.answered.' . $name, 0);
}
$filePath = "/var/log/metager/";
# Wir haben nun die aktuellen daten und müssen diese noch mit den aktuellen Kombinieren:
# Hierbei behalten wir die Daten, seid dem letzten Rotate so wie sie sind und verknüpfen diese mit einer gesamt Statistik.
if (file_exists($filePath . "engine.log") && is_readable($filePath . "engine.log")) {
$oldData = file_get_contents($filePath . "engine.log");
# JSON Decode
$oldData = json_decode($oldData, true);
if (isset($oldData['overall'])) {
$searchEngineLogs['overall'] = $oldData['overall'];
}
}
# Jetzt fügen wir zu den Gesamtdaten die zuletzt erfassten hinzu:
foreach ($searchEngineLogs['recent'] as $name => $values) {
if (isset($searchEngineLogs['overall'][$name]["requests"])) {
$searchEngineLogs['overall'][$name]["requests"] += $values["requests"];
} else {
$searchEngineLogs['overall'][$name]["requests"] = $values["requests"];
}
if (isset($searchEngineLogs['overall'][$name]["answered"])) {
$searchEngineLogs['overall'][$name]["answered"] += $values["answered"];
} else {
$searchEngineLogs['overall'][$name]["answered"] = $values["answered"];
}
}
# Ins JSON Format umwandeln:
$searchEngineLogs = json_encode($searchEngineLogs, JSON_PRETTY_PRINT);
# Und in die Datei sichern
if (((file_exists($filePath . "engine.log") && is_writable($filePath . "engine.log")) || (!file_exists($filePath . "engine.log") && is_writable($filePath))) && file_put_contents($filePath . "engine.log", $searchEngineLogs) !== false) {
$this->info("Logs der Suchmaschinen erfolgreich exportiert.");
return;
} else {
# Der Schreibvorgang war nicht erfolgreich. Wir packen die Daten zurück
foreach (json_decode($searchEngineLogs, true) as $name => $values) {
$redis->incrby('logs.engines.requests.' . $name, $values["requests"]);
$redis->incrby('logs.engines.answered.' . $name, $values["answered"]);
}
$this->info("Konnte die Datei \"$filePath" . "engine.log\" nicht erstellen. Keine Rechte.");
return;
}
} catch (\ErrorException $e) {
return;
}
}
}
app/Console/Commands/MonthlyRequestsGather.php 0 → 100644
View file @ 29aafe9f
<?php
namespace App\Console\Commands;
use DB;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
class MonthlyRequestsGather extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'requests:gather';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Sends the gathered monthly requests from the redis to the central Database';
private $values = [];
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
# Read in the suma Files
$sumaFile = config_path('sumas.json');
$sumaEnFile = config_path('sumasEn.json');
if (file_exists($sumaFile) && file_exists($sumaEnFile)) {
$sumas = json_decode(file_get_contents($sumaFile));
$sumasEn = json_decode(file_get_contents($sumaEnFile));
$this->gatherLogs($sumas);
$this->gatherLogs($sumasEn);
foreach ($this->values as $name => $value) {
$entry = DB::table('monthlyrequests')->where(['name' => $name])->first();
$newCount = $value;
if ($entry === null) {
DB::table('monthlyrequests')->insert(['name' => $name, 'count' => $newCount]);
} else {
$newCount = $value + $entry->count;
DB::table('monthlyrequests')->where(['name' => $name])->update(['count' => $newCount]);
}
}
$this->disableOverusedEngines($sumaFile, $sumas);
$this->disableOverusedEngines($sumaEnFile, $sumasEn);
}
}
private function disableOverusedEngines($sumasFile, $sumas)
{
$currentValues = DB::table('monthlyrequests')->get();
foreach ($sumas->sumas as $sumaName => $suma) {
if (!empty($suma->disabled) && $suma->disabled === true) {
continue;
}
if (empty($suma->{"monthly-requests"})) {
continue;
}
$currentValue = 0;
foreach ($currentValues as $value) {
if ($value->name === $sumaName) {
$currentValue = $value->count;
}
}
$monthlyRequests = $suma->{"monthly-requests"};
if (!empty($suma->{"auto-disabled"}) && $suma->{"auto-disabled"} === true) {
# Maybe this engine should be enabled again
if ($currentValue < $monthlyRequests) {
unset($sumas->sumas->{$sumaName}->{"auto-disabled"});
file_put_contents($sumasFile, json_encode($sumas, JSON_PRETTY_PRINT));
}
} else if ($currentValue >= $monthlyRequests) {
$sumas->sumas->{$sumaName}->{"auto-disabled"} = true;
file_put_contents($sumasFile, json_encode($sumas, JSON_PRETTY_PRINT));
}
}
}
private function gatherLogs($sumas)
{
foreach ($sumas->sumas as $sumaName => $suma) {
if (!empty($suma->{"monthly-requests"})) {
$monthlyLimit = $suma->{"monthly-requests"};
$currentValue = Redis::get('monthlyRequests:' . $sumaName);
Redis::del('monthlyRequests:' . $sumaName);
if (empty($currentValue)) {
$currentValue = 0;
}
if (empty($this->values[$sumaName]) && $currentValue > 0) {
$this->values[$sumaName] = intval($currentValue);
}
}
}
}
}
app/Console/Kernel.php
View file @ 29aafe9f
......@@ -25,10 +25,14 @@ class Kernel extends ConsoleKernel
*/
protected function schedule(Schedule $schedule)
{
$schedule->command('log:rotate')->everyTenMinutes();
$schedule->command('requests:gather')->everyFifteenMinutes();
$schedule->call(function () {
DB::table('monthlyrequests')->truncate();
})->monthlyOn(1, '00:00');
// Delete all of the old humanverification entries
$schedule->call(function() {
$schedule->call(function () {
DB::delete('DELETE FROM humanverification WHERE updated_at < (now() - interval 72 hour) AND whitelist = 0 ORDER BY updated_at DESC');
DB::delete('DELETE FROM humanverification WHERE updated_at < (now() - interval 2 week) AND whitelist = 1 ORDER BY updated_at DESC');
})->everyThirtyMinutes();
......@@ -42,6 +46,6 @@ class Kernel extends ConsoleKernel
protected function commands()
{
require base_path('routes/console.php');
$this->load(__DIR__.'/Commands');
$this->load(__DIR__ . '/Commands');
}
}
app/MetaGer.php
View file @ 29aafe9f
......@@ -542,7 +542,8 @@ class MetaGer
foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
# Check if this engine is disabled and can't be used
$disabled = empty($this->sumaFile->sumas->{$suma}->disabled) ? false : $this->sumaFile->sumas->{$suma}->disabled;
if ($disabled) {
$autoDisabled = empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) ? false : $this->sumaFile->sumas->{$suma}->{"auto-disabled"};
if ($disabled || $autoDisabled) {
continue;
}
......@@ -710,6 +711,11 @@ class MetaGer
foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"}) {
if (!empty($filter->sumas->{$suma})) {
# If the searchengine is disabled this filter shouldn't be available
if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
|| (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)) {
continue;
}
$availableFilter[$filterName] = $filter;
}
}
......@@ -1343,16 +1349,7 @@ class MetaGer
# 2 Arten von Logs in einem wird die Anzahl der Abfragen an eine Suchmaschine gespeichert und in der anderen
# die Anzahl, wie häufig diese Ergebnisse geliefert hat.
$enginesToLoad = $this->enginesToLoad;
$redis->pipeline(function ($pipe) use ($enginesToLoad, $logEntry) {
$pipe->rpush('logs.search', $logEntry);
foreach ($this->enginesToLoad as $name => $answered) {
$pipe->incr('logs.engines.requests.' . $name);
if ($answered) {
$pipe->incr('logs.engines.answered.' . $name);
}
}
});
$redis->rpush('logs.search', $logEntry);
} catch (\Exception $e) {
return;
}
......
app/Models/Searchengine.php
View file @ 29aafe9f
......@@ -134,6 +134,12 @@ abstract class Searchengine
// each Searcher has it's own queue lying under the redis key <name>.queue
Redis::rpush($this->name . ".queue", $mission);
// The request is not cached and will be submitted to the searchengine
// We need to check if the number of requests to this engine are limited
if (!empty($this->engine->{"monthly-requests"})) {
Redis::incr("monthlyRequests:" . $this->name);
}
/**
* We have Searcher processes running for MetaGer
* Each Searcher is dedicated to one specific Searchengine and fetches it's results.
......
database/migrations/2019_02_19_094337_monthlyrequests.php 0 → 100644
View file @ 29aafe9f
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class Monthlyrequests extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('monthlyrequests', function (Blueprint $table) {
$table->string('name')->unique();
$table->integer('count');
$table->timestamp('updated_at');
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::drop('sponsorenlinks');
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment