From 29aafe9f75fa8765b8f15a0fbea8a8fe2e33a5e4 Mon Sep 17 00:00:00 2001
From: Dominik Hebeler <dominik@suma-ev.de>
Date: Tue, 19 Feb 2019 12:10:04 +0100
Subject: [PATCH] Added the option to auto disable search engines
---
app/Console/Commands/LogRotate.php | 119 -----------------
.../Commands/MonthlyRequestsGather.php | 121 ++++++++++++++++++
app/Console/Kernel.php | 10 +-
app/MetaGer.php | 19 ++-
app/Models/Searchengine.php | 6 +
.../2019_02_19_094337_monthlyrequests.php | 33 +++++
6 files changed, 175 insertions(+), 133 deletions(-)
delete mode 100644 app/Console/Commands/LogRotate.php
create mode 100644 app/Console/Commands/MonthlyRequestsGather.php
create mode 100644 database/migrations/2019_02_19_094337_monthlyrequests.php
diff --git a/app/Console/Commands/LogRotate.php b/app/Console/Commands/LogRotate.php
deleted file mode 100644
index c539881ff..000000000
--- a/app/Console/Commands/LogRotate.php
+++ /dev/null
@@ -1,119 +0,0 @@
-<?php
-
-namespace App\Console\Commands;
-
-use Illuminate\Console\Command;
-use Illuminate\Support\Facades\Redis;
-
-class LogRotate extends Command
-{
- /**
- * The name and signature of the console command.
- *
- * @var string
- */
- protected $signature = 'log:rotate';
-
- /**
- * The console command description.
- *
- * @var string
- */
- protected $description = 'Dieses Kommando bezieht alle Log-Einträge aus dem Redis System und exportiert diese in entsprechende Dateien.';
-
- /**
- * Create a new command instance.
- *
- * @return void
- */
- public function __construct()
- {
- parent::__construct();
- }
-
- /**
- * Execute the console command.
- *
- * @return mixed
- */
- public function handle()
- {
- # Wir extrahieren die Logs mit den Daten der einzelnen Suchmaschinen:
- # In den Daten ist festgehalten, wie oft eine jede Suchmaschine abgefragt wurde
- # und wie häufig diese geantwortet hat:
- $this->extractSearchEngineLogs();
- }
-
- private function extractSearchEngineLogs()
- {
- $redis = Redis::connection('redisLogs');
- # Hier legen wir das Ergebnis ab:
- # [
- # 'fastbot' => [
- # ''
- # ]
- # ]
- $searchEngineLogs = ['recent' => [], 'overall' => []];
- try {
- # Wir benötigen zunächst die Namen aller Suchmaschinen:
- $sumasFile = config_path() . "/sumas.xml";
- $xml = simplexml_load_file($sumasFile);
- $xml = $xml->xpath("suma");
- $sumaNames = [];
- foreach ($xml as $suma) {
- $searchEngineLogs['recent'][$suma["name"]->__toString()] = ["requests" => 0, "answered" => 0];
- }
- #Auslesen/hinzufügen der Daten:
- foreach ($searchEngineLogs['recent'] as $name => $values) {
- $searchEngineLogs['recent'][$name]["requests"] = $redis->getset('logs.engines.requests.' . $name, 0);
- $searchEngineLogs['recent'][$name]["answered"] = $redis->getset('logs.engines.answered.' . $name, 0);
- }
-
- $filePath = "/var/log/metager/";
- # Wir haben nun die aktuellen daten und müssen diese noch mit den aktuellen Kombinieren:
- # Hierbei behalten wir die Daten, seid dem letzten Rotate so wie sie sind und verknüpfen diese mit einer gesamt Statistik.
- if (file_exists($filePath . "engine.log") && is_readable($filePath . "engine.log")) {
- $oldData = file_get_contents($filePath . "engine.log");
- # JSON Decode
- $oldData = json_decode($oldData, true);
- if (isset($oldData['overall'])) {
- $searchEngineLogs['overall'] = $oldData['overall'];
- }
- }
-
- # Jetzt fügen wir zu den Gesamtdaten die zuletzt erfassten hinzu:
- foreach ($searchEngineLogs['recent'] as $name => $values) {
- if (isset($searchEngineLogs['overall'][$name]["requests"])) {
- $searchEngineLogs['overall'][$name]["requests"] += $values["requests"];
- } else {
- $searchEngineLogs['overall'][$name]["requests"] = $values["requests"];
- }
- if (isset($searchEngineLogs['overall'][$name]["answered"])) {
- $searchEngineLogs['overall'][$name]["answered"] += $values["answered"];
- } else {
- $searchEngineLogs['overall'][$name]["answered"] = $values["answered"];
- }
- }
-
- # Ins JSON Format umwandeln:
- $searchEngineLogs = json_encode($searchEngineLogs, JSON_PRETTY_PRINT);
-
- # Und in die Datei sichern
- if (((file_exists($filePath . "engine.log") && is_writable($filePath . "engine.log")) || (!file_exists($filePath . "engine.log") && is_writable($filePath))) && file_put_contents($filePath . "engine.log", $searchEngineLogs) !== false) {
- $this->info("Logs der Suchmaschinen erfolgreich exportiert.");
- return;
- } else {
- # Der Schreibvorgang war nicht erfolgreich. Wir packen die Daten zurück
- foreach (json_decode($searchEngineLogs, true) as $name => $values) {
- $redis->incrby('logs.engines.requests.' . $name, $values["requests"]);
- $redis->incrby('logs.engines.answered.' . $name, $values["answered"]);
- }
- $this->info("Konnte die Datei \"$filePath" . "engine.log\" nicht erstellen. Keine Rechte.");
- return;
- }
-
- } catch (\ErrorException $e) {
- return;
- }
- }
-}
diff --git a/app/Console/Commands/MonthlyRequestsGather.php b/app/Console/Commands/MonthlyRequestsGather.php
new file mode 100644
index 000000000..f2cbb06c0
--- /dev/null
+++ b/app/Console/Commands/MonthlyRequestsGather.php
@@ -0,0 +1,121 @@
+<?php
+
+namespace App\Console\Commands;
+
+use DB;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Redis;
+
+class MonthlyRequestsGather extends Command
+{
+ /**
+ * The name and signature of the console command.
+ *
+ * @var string
+ */
+ protected $signature = 'requests:gather';
+
+ /**
+ * The console command description.
+ *
+ * @var string
+ */
+ protected $description = 'Sends the gathered monthly requests from the redis to the central Database';
+
+ private $values = [];
+
+ /**
+ * Create a new command instance.
+ *
+ * @return void
+ */
+ public function __construct()
+ {
+ parent::__construct();
+ }
+
+ /**
+ * Execute the console command.
+ *
+ * @return mixed
+ */
+ public function handle()
+ {
+ # Read in the suma Files
+ $sumaFile = config_path('sumas.json');
+ $sumaEnFile = config_path('sumasEn.json');
+ if (file_exists($sumaFile) && file_exists($sumaEnFile)) {
+ $sumas = json_decode(file_get_contents($sumaFile));
+ $sumasEn = json_decode(file_get_contents($sumaEnFile));
+
+ $this->gatherLogs($sumas);
+ $this->gatherLogs($sumasEn);
+
+ foreach ($this->values as $name => $value) {
+ $entry = DB::table('monthlyrequests')->where(['name' => $name])->first();
+ $newCount = $value;
+ if ($entry === null) {
+ DB::table('monthlyrequests')->insert(['name' => $name, 'count' => $newCount]);
+ } else {
+ $newCount = $value + $entry->count;
+ DB::table('monthlyrequests')->where(['name' => $name])->update(['count' => $newCount]);
+ }
+ }
+
+ $this->disableOverusedEngines($sumaFile, $sumas);
+ $this->disableOverusedEngines($sumaEnFile, $sumasEn);
+
+ }
+ }
+
+ private function disableOverusedEngines($sumasFile, $sumas)
+ {
+ $currentValues = DB::table('monthlyrequests')->get();
+ foreach ($sumas->sumas as $sumaName => $suma) {
+ if (!empty($suma->disabled) && $suma->disabled === true) {
+ continue;
+ }
+
+ if (empty($suma->{"monthly-requests"})) {
+ continue;
+ }
+ $currentValue = 0;
+ foreach ($currentValues as $value) {
+ if ($value->name === $sumaName) {
+ $currentValue = $value->count;
+ }
+ }
+
+ $monthlyRequests = $suma->{"monthly-requests"};
+
+ if (!empty($suma->{"auto-disabled"}) && $suma->{"auto-disabled"} === true) {
+ # Maybe this engine should be enabled again
+ if ($currentValue < $monthlyRequests) {
+ unset($sumas->sumas->{$sumaName}->{"auto-disabled"});
+ file_put_contents($sumasFile, json_encode($sumas, JSON_PRETTY_PRINT));
+ }
+ } else if ($currentValue >= $monthlyRequests) {
+ $sumas->sumas->{$sumaName}->{"auto-disabled"} = true;
+ file_put_contents($sumasFile, json_encode($sumas, JSON_PRETTY_PRINT));
+ }
+ }
+ }
+
+ private function gatherLogs($sumas)
+ {
+ foreach ($sumas->sumas as $sumaName => $suma) {
+ if (!empty($suma->{"monthly-requests"})) {
+ $monthlyLimit = $suma->{"monthly-requests"};
+ $currentValue = Redis::get('monthlyRequests:' . $sumaName);
+ Redis::del('monthlyRequests:' . $sumaName);
+ if (empty($currentValue)) {
+ $currentValue = 0;
+ }
+ if (empty($this->values[$sumaName]) && $currentValue > 0) {
+ $this->values[$sumaName] = intval($currentValue);
+ }
+ }
+
+ }
+ }
+}
diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php
index d3655e50e..6290dc5c2 100644
--- a/app/Console/Kernel.php
+++ b/app/Console/Kernel.php
@@ -25,10 +25,14 @@ class Kernel extends ConsoleKernel
*/
protected function schedule(Schedule $schedule)
{
- $schedule->command('log:rotate')->everyTenMinutes();
+ $schedule->command('requests:gather')->everyFifteenMinutes();
+
+ $schedule->call(function () {
+ DB::table('monthlyrequests')->truncate();
+ })->monthlyOn(1, '00:00');
// Delete all of the old humanverification entries
- $schedule->call(function() {
+ $schedule->call(function () {
DB::delete('DELETE FROM humanverification WHERE updated_at < (now() - interval 72 hour) AND whitelist = 0 ORDER BY updated_at DESC');
DB::delete('DELETE FROM humanverification WHERE updated_at < (now() - interval 2 week) AND whitelist = 1 ORDER BY updated_at DESC');
})->everyThirtyMinutes();
@@ -42,6 +46,6 @@ class Kernel extends ConsoleKernel
protected function commands()
{
require base_path('routes/console.php');
- $this->load(__DIR__.'/Commands');
+ $this->load(__DIR__ . '/Commands');
}
}
diff --git a/app/MetaGer.php b/app/MetaGer.php
index 65a1e4a69..f1eca712b 100644
--- a/app/MetaGer.php
+++ b/app/MetaGer.php
@@ -542,7 +542,8 @@ class MetaGer
foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
# Check if this engine is disabled and can't be used
$disabled = empty($this->sumaFile->sumas->{$suma}->disabled) ? false : $this->sumaFile->sumas->{$suma}->disabled;
- if ($disabled) {
+ $autoDisabled = empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) ? false : $this->sumaFile->sumas->{$suma}->{"auto-disabled"};
+ if ($disabled || $autoDisabled) {
continue;
}
@@ -710,6 +711,11 @@ class MetaGer
foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"}) {
if (!empty($filter->sumas->{$suma})) {
+ # If the searchengine is disabled this filter shouldn't be available
+ if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
+ || (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)) {
+ continue;
+ }
$availableFilter[$filterName] = $filter;
}
}
@@ -1343,16 +1349,7 @@ class MetaGer
# 2 Arten von Logs in einem wird die Anzahl der Abfragen an eine Suchmaschine gespeichert und in der anderen
# die Anzahl, wie häufig diese Ergebnisse geliefert hat.
- $enginesToLoad = $this->enginesToLoad;
- $redis->pipeline(function ($pipe) use ($enginesToLoad, $logEntry) {
- $pipe->rpush('logs.search', $logEntry);
- foreach ($this->enginesToLoad as $name => $answered) {
- $pipe->incr('logs.engines.requests.' . $name);
- if ($answered) {
- $pipe->incr('logs.engines.answered.' . $name);
- }
- }
- });
+ $redis->rpush('logs.search', $logEntry);
} catch (\Exception $e) {
return;
}
diff --git a/app/Models/Searchengine.php b/app/Models/Searchengine.php
index 686268a28..03b584048 100644
--- a/app/Models/Searchengine.php
+++ b/app/Models/Searchengine.php
@@ -134,6 +134,12 @@ abstract class Searchengine
// each Searcher has it's own queue lying under the redis key <name>.queue
Redis::rpush($this->name . ".queue", $mission);
+ // The request is not cached and will be submitted to the searchengine
+ // We need to check if the number of requests to this engine are limited
+ if (!empty($this->engine->{"monthly-requests"})) {
+ Redis::incr("monthlyRequests:" . $this->name);
+ }
+
/**
* We have Searcher processes running for MetaGer
* Each Searcher is dedicated to one specific Searchengine and fetches it's results.
diff --git a/database/migrations/2019_02_19_094337_monthlyrequests.php b/database/migrations/2019_02_19_094337_monthlyrequests.php
new file mode 100644
index 000000000..bb1491894
--- /dev/null
+++ b/database/migrations/2019_02_19_094337_monthlyrequests.php
@@ -0,0 +1,33 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class Monthlyrequests extends Migration
+{
+ /**
+ * Run the migrations.
+ *
+ * @return void
+ */
+ public function up()
+ {
+ Schema::create('monthlyrequests', function (Blueprint $table) {
+ $table->string('name')->unique();
+ $table->integer('count');
+ $table->timestamp('updated_at');
+ });
+
+ }
+
+ /**
+ * Reverse the migrations.
+ *
+ * @return void
+ */
+ public function down()
+ {
+ Schema::drop('sponsorenlinks');
+ }
+}
--
GitLab