Skip to content
Snippets Groups Projects
Commit 1651e670 authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Merge branch '991-include-dynamic-spam-rules' into 'development' 

Resolve "Include Dynamic Spam rules"

Closes #991

See merge request !1632
parents e8b064cc 29c3c29c
No related branches found
No related tags found
5 merge requests!1645Development,!1641Development,!1635Development,!1634Development,!1632Resolve "Include Dynamic Spam rules"
Hide whitespace changes
Inline Side-by-side
app/Console/Commands/LoadSpam.php 0 → 100644
+ 64
− 0
View file @ 1651e670
<?php
namespace App\Console\Commands;
use Carbon;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
class LoadSpam extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'spam:load';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Loads a list of current Spams into redis';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$filePath = \storage_path('logs/metager/ban.txt');
$bans = [];
if (\file_exists($filePath)) {
$bans = json_decode(file_get_contents($filePath), true);
}
$bansToLoad = [];
foreach ($bans as $ban) {
$bannedUntil = Carbon::createFromFormat("Y-m-d H:i:s", $ban["banned-until"]);
if ($bannedUntil->isAfter(Carbon::now())) {
$bansToLoad[] = $ban["regexp"];
}
}
Redis::pipeline(function ($redis) use ($bansToLoad) {
$redis->del("spam");
foreach ($bansToLoad as $ban) {
$redis->rpush("spam", $ban);
}
});
}
}
app/Console/Kernel.php
+ 1
− 1
View file @ 1651e670
...@@ -28,7 +28,7 @@ class Kernel extends ConsoleKernel ...@@ -28,7 +28,7 @@ class Kernel extends ConsoleKernel
$schedule->command('requests:gather')->everyFifteenMinutes(); $schedule->command('requests:gather')->everyFifteenMinutes();
$schedule->command('requests:useragents')->everyFiveMinutes(); $schedule->command('requests:useragents')->everyFiveMinutes();
$schedule->command('logs:gather')->everyMinute(); $schedule->command('logs:gather')->everyMinute();
$schedule->command('spam:load')->everyFiveMinutes();
$schedule->call(function () { $schedule->call(function () {
DB::table('monthlyrequests')->truncate(); DB::table('monthlyrequests')->truncate();
DB::disconnect('mysql'); DB::disconnect('mysql');
......
app/Http/Controllers/AdminSpamController.php 0 → 100644
+ 145
− 0
View file @ 1651e670
<?php
namespace App\Http\Controllers;
use Carbon;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
class AdminSpamController extends Controller
{
public function index()
{
$queries = $this->getQueries();
$currentBans = $this->getBans();
$loadedBans = Redis::lrange("spam", 0, -1);
return view("admin.spam")
->with('title', "Spam Konfiguration - MetaGer")
->with('queries', $queries)
->with('bans', $currentBans)
->with('loadedBans', $loadedBans);
}
public function ban(Request $request)
{
$banTime = $request->input('ban-time');
$banRegexp = $request->input('regexp');
$file = storage_path('logs/metager/ban.txt');
$bans = [];
if (file_exists($file)) {
$bans = json_decode(file_get_contents($file), true);
}
$bans[] = ["banned-until" => Carbon::now()->add($banTime)->format("Y-m-d H:i:s"), "regexp" => $banRegexp];
\file_put_contents($file, json_encode($bans));
return redirect(url('admin/spam'));
}
public function jsonQueries()
{
$queries = $this->getQueries();
return response()->json($queries);
}
public function queryregexp(Request $request)
{
$data = json_decode($request->getContent(), true);
$queries = $data["queries"];
$regexps = [$data["regexp"]];
$bans = $this->getBans();
foreach ($bans as $ban) {
$regexps[] = $ban["regexp"];
}
$resultData = [];
foreach ($queries as $query) {
$matches = false;
foreach ($regexps as $regexp) {
try {
if (preg_match($regexp, $query)) {
$matches = true;
}
} catch (\Exception $e) {
// Exceptions are expected when no valid regexp is given
}
}
$resultData[] = [
"query" => $query,
"matches" => $matches,
];
}
return response()->json($resultData);
}
private function getQueries()
{
$minuteToFetch = Carbon::now()->subMinutes(2);
$logFile = storage_path("logs/metager/" . $minuteToFetch->format("Y/m/d") . ".log");
$result = shell_exec("cat $logFile | grep " . $minuteToFetch->format("H:i:"));
$result = explode(PHP_EOL, $result);
$queries = array();
foreach ($result as $line) {
if ($query = \preg_match("/.*eingabe=(.*)$/", $line, $matches)) {
$queries[] = $matches[1];
}
}
return $queries;
}
public function getBans()
{
$file = \storage_path('logs/metager/ban.txt');
$bans = [];
if (file_exists($file)) {
$tmpBans = json_decode(file_get_contents($file), true);
foreach ($tmpBans as $ban) {
#dd($ban["banned-until"]);
$bannedUntil = Carbon::createFromFormat('Y-m-d H:i:s', $ban["banned-until"]);
if ($bannedUntil->isAfter(Carbon::now())) {
$bans[] = $ban;
}
}
}
file_put_contents($file, json_encode($bans));
return $bans;
}
public function deleteRegexp(Request $request)
{
$file = \storage_path('logs/metager/ban.txt');
$bans = [];
if (file_exists($file)) {
$bans = json_decode(file_get_contents($file), true);
}
$regexpToDelete = $request->input('regexp');
$newBans = [];
foreach ($bans as $ban) {
if ($ban["regexp"] !== $regexpToDelete) {
$newBans[] = $ban;
}
}
file_put_contents($file, json_encode($newBans));
return redirect(url('admin/spam'));
}
}
app/Http/Controllers/HumanVerification.php
+ 7
− 10
View file @ 1651e670
...@@ -7,6 +7,7 @@ use Carbon; ...@@ -7,6 +7,7 @@ use Carbon;
use Illuminate\Hashing\BcryptHasher as Hasher; use Illuminate\Hashing\BcryptHasher as Hasher;
use Illuminate\Http\Request; use Illuminate\Http\Request;
use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Redis;
use Input; use Input;
class HumanVerification extends Controller class HumanVerification extends Controller
...@@ -198,20 +199,16 @@ class HumanVerification extends Controller ...@@ -198,20 +199,16 @@ class HumanVerification extends Controller
public static function couldBeSpammer($ip) public static function couldBeSpammer($ip)
{ {
$possibleSpammer = false;
# Check for recent Spams # Check for recent Spams
$eingabe = \Request::input('eingabe'); $eingabe = \Request::input('eingabe');
if (\preg_match("/^susimail\s+-site:[^\s]+\s-site:/si", $eingabe)) { $spams = Redis::lrange("spam", 0, -1);
return true; foreach ($spams as $spam) {
} else if (\preg_match("/^\s*site:\"linkedin\.com[^\"]*\"\s+/si", $eingabe)) { if (\preg_match($spam, $eingabe)) {
return true; return true;
} else if (\preg_match("/^\d+.(php|asp)\s+\?.*=.*/si", $eingabe)) { }
return true;
} }
return $possibleSpammer; return false;
} }
public function botOverview(Request $request) public function botOverview(Request $request)
......
resources/views/admin/spam.blade.php 0 → 100644
+ 198
− 0
View file @ 1651e670
@extends('layouts.subPages')
@section('title', $title )
@section('content')
<style>
#head {
display: flex;
align-items: center;
margin-bottom: 16px;
}
#head > button {
margin-left: 16px;
}
#head > h1 {
margin: 0;
}
#queries {
display: flex;
flex-wrap: wrap;
justify-content: space-between;
}
.matches {
background-color: #c9f4c9;
}
#block-requests {
margin-bottom: 16px;
}
#regexp {
margin-bottom: 8px;
}
#ban-time {
margin-bottom: 8px;
}
</style>
<div id="block-requests">
<form method="post">
<input class="form-control" type="text" name="regexp" id="regexp" placeholder="Type in regexp to match queries...">
<select name="ban-time" id="ban-time" class="form-control">
<option value="1 day">Einen Tag</option>
<option value="1 week">Eine Woche</option>
<option value="2 weeks">Zwei Wochen</option>
<option value="1 month" selected>Einen Monat</option>
</select>
<button type="submit" class="btn btn-default btn-sm">Sperren</button>
</form>
</div>
<div id="head">
<h1>Letzte Suchanfragen</h1>
<button type="button" class="btn btn-success btn-sm">Aktualisierung stoppen (60)</button>
</div>
<input class="form-control" type="text" name="" id="check-against" placeholder="Match against...">
<div id="queries">
@foreach($queries as $query)
<div class="query card">{{$query}}</div>
@endforeach
</div>
<div id="bans">
<h1>Current Bans</h1>
<table class="table table-striped">
<thead>
<tr>
<td>Regexp</td>
<td>Banned until</td>
<td>Actions</td>
</tr>
</thead>
<tbody>
@foreach($bans as $ban)
<tr>
<td>{{ $ban["regexp"] }}</td>
<td>{{ Carbon::createFromFormat("Y-m-d H:i:s", $ban["banned-until"])->format("d.m.Y H:i:s")}} ({{ Carbon::createFromFormat("Y-m-d H:i:s", $ban["banned-until"])->diffInDays(Carbon::now()) }} Days)</td>
<td>
<form action="{{ url("admin/spam/deleteRegexp") }}" method="post">
<input type="hidden" name="regexp" value="{{ $ban["regexp"] }}">
<button type="submit">&#128465;</button>
</form>
</td>
</tr>
@endforeach
</tbody>
</table>
</div>
<div id="loadedbans">
<h1>Loaded Bans</h1>
<table class="table table-striped">
<thead>
<tr>
<td>Regexp</td>
</tr>
</thead>
<tbody>
@foreach($loadedBans as $ban)
<tr>
<td>{{ $ban }}</td>
</tr>
@endforeach
</tbody>
</table>
</div>
<script>
var lastUpdate = Date.now();
var updating = true;
var buttonText = "Aktualisierung stoppen";
var interval = setInterval(updateQueries, 1000);
$("#regexp").on("input", checkRegexp);
$("#check-against").on("input", checkRegexp);
$(document).ready(function(){
checkRegexp();
});
$("#head > button").click(function() {
if(!updating) {
$("#head > button").removeClass("btn-danger");
$("#head > button").addClass("btn-success");
buttonText = "Aktualisierung stoppen";
interval = setInterval(updateQueries, 1000);
}
var updateAt = lastUpdate + 60000;
var updateIn = Math.round((updateAt - Date.now()) / 1000);
$("#head > button").html(buttonText + " (" + updateIn + ")");
updating = !updating;
});
function updateQueries() {
var updateAt = lastUpdate + 60000;
var updateIn = Math.round((updateAt - Date.now()) / 1000);
if(!updating){
$("#head > button").removeClass("btn-success");
$("#head > button").addClass("btn-danger");
buttonText = "Aktualisierung starten";
clearInterval(interval);
}
$("#head > button").html(buttonText + " (" + updateIn + ")");
if(updateAt > Date.now()){
return;
}
fetch("{{ url('admin/spam/jsonQueries') }}")
.then(response => response.json())
.then(data => {
$("#queries").html("");
$(data).each(function(index, el){
$("#queries").append("<div class=\"query card\">" + el + "</div>");
});
lastUpdate = Date.now();
checkRegexp();
});
}
function checkRegexp() {
var val = $("#regexp").val();
var queries = [];
$("#queries > .query").each(function(index, el){
queries.push($(el).html());
});
queries.push($("#check-against").val());
var url = "{{ url('admin/spam/queryregexp') }}";
var options = {
method: 'POST',
body: JSON.stringify({
"queries": queries,
"regexp": val
}),
headers: {
'Content-Type': 'application/json'
}
};
fetch(url, options)
.then(response => response.json())
.then(data => {
$("#queries > .query").each(function(index, el){
if(data[index]["matches"]){
$(el).addClass("matches");
}else{
$(el).removeClass("matches");
}
});
if(data[data.length-1]["matches"]){
$("#check-against").addClass("matches");
}else{
$("#check-against").removeClass("matches");
}
});
}
</script>
@endsection
routes/web.php
+ 7
− 0
View file @ 1651e670
...@@ -182,6 +182,13 @@ Route::group( ...@@ -182,6 +182,13 @@ Route::group(
}); });
Route::get('bot', 'HumanVerification@botOverview'); Route::get('bot', 'HumanVerification@botOverview');
Route::post('bot', 'HumanVerification@botOverviewChange'); Route::post('bot', 'HumanVerification@botOverviewChange');
Route::group(['prefix' => 'spam'], function () {
Route::get('/', 'AdminSpamController@index');
Route::post('/', 'AdminSpamController@ban');
Route::get('jsonQueries', 'AdminSpamController@jsonQueries');
Route::post('queryregexp', 'AdminSpamController@queryregexp');
Route::post('deleteRegexp', 'AdminSpamController@deleteRegexp');
});
}); });
Route::get('settings', function () { Route::get('settings', function () {
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment