Merge branch '1004-browserverification' into 'development'

Resolve "Browserverification"

Closes #1004

See merge request !1658

app/Http/Controllers/HumanVerification.php

@@ -4,6 +4,7 @@ namespace App\Http\Controllers;
 
 use Captcha;
 use Carbon;
+use Cookie;
 use Illuminate\Hashing\BcryptHasher as Hasher;
 use Illuminate\Http\Request;
 use Illuminate\Support\Facades\Cache;
@@ -270,4 +271,84 @@ class HumanVerification extends Controller
         HumanVerification::saveUser($user);
         return redirect('admin/bot');
     }
+
+    public function browserVerification(Request $request)
+    {
+        $key = $request->input("id", "");
+
+        // Verify that key is a md5 checksum
+        if (!preg_match("/^[a-f0-9]{32}$/", $key)) {
+            abort(404);
+        }
+
+        Redis::connection("cache")->pipeline(function ($redis) use ($key) {
+            $redis->rpush($key, true);
+            $redis->expire($key, 30);
+        });
+
+        return response("", 200)->header("Content-Type", "text/css");
+    }
+
+    public static function block(Request $request)
+    {
+        $prefix = "humanverification";
+
+        $ip = $request->ip();
+        $id = "";
+        $uid = "";
+        if (\App\Http\Controllers\HumanVerification::couldBeSpammer($ip)) {
+            $id = hash("sha1", "999.999.999.999");
+            $uid = hash("sha1", "999.999.999.999" . $ip . $_SERVER["AGENT"] . "uid");
+        } else {
+            $id = hash("sha1", $ip);
+            $uid = hash("sha1", $ip . $_SERVER["AGENT"] . "uid");
+        }
+
+        /**
+         * If the user sends a Password or a key
+         * We will not verificate the user.
+         * If someone that uses a bot finds this out we
+         * might have to change it at some point.
+         */
+        if ($request->filled('password') || $request->filled('key') || Cookie::get('key') !== null || $request->filled('appversion') || !env('BOT_PROTECTION', false)) {
+            $update = false;
+            return $next($request);
+        }
+
+        # Get all Users of this IP
+        $users = Cache::get($prefix . "." . $id, []);
+
+        $user = [];
+        $changed = false;
+        if (empty($users[$uid])) {
+            $user = [
+                'uid' => $uid,
+                'id' => $id,
+                'unusedResultPages' => 0,
+                'whitelist' => false,
+                'locked' => true,
+                "lockedKey" => "",
+                "expiration" => now()->addWeeks(2),
+            ];
+            $changed = true;
+        } else {
+            $user = $users[$uid];
+            if (!$user["locked"]) {
+                $user["locked"] = true;
+                $changed = true;
+            }
+        }
+
+        if ($user["whitelist"]) {
+            $user["expiration"] = now()->addWeeks(2);
+        } else {
+            $user["expiration"] = now()->addHours(72);
+        }
+        if ($changed) {
+            $userList = Cache::get($prefix . "." . $user["id"], []);
+            $userList[$user["uid"]] = $user;
+            Cache::put($prefix . "." . $user["id"], $userList, 2 * 7 * 24 * 60 * 60);
+        }
+        return [$id, $uid];
+    }
 }

app/Http/Kernel.php

@@ -62,5 +62,6 @@ class Kernel extends HttpKernel
         'referer.check' => \App\Http\Middleware\RefererCheck::class,
         'humanverification' => \App\Http\Middleware\HumanVerification::class,
         'useragentmaster' => \App\Http\Middleware\UserAgentMaster::class,
+        'browserverification' => \App\Http\Middleware\BrowserVerification::class,
     ];
 }

app/Http/Middleware/BrowserVerification.php

+<?php
+
+namespace App\Http\Middleware;
+
+use Closure;
+use GrahamCampbell\Throttle\Facades\Throttle;
+use Illuminate\Support\Facades\Redis;
+use \App\Http\Controllers\HumanVerification;
+
+class BrowserVerification
+{
+    /**
+     * Handle an incoming request.
+     *
+     * @param  \Illuminate\Http\Request  $request
+     * @param  \Closure  $next
+     * @return mixed
+     */
+    public function handle($request, Closure $next)
+    {
+        $bvEnabled = config("metager.metager.browserverification_enabled");
+        if (empty($bvEnabled) || !$bvEnabled) {
+            return $next($request);
+        }
+
+        // Check if throttled
+        $accept = Throttle::check($request, 8, 1);
+        if (!$accept) {
+            Throttle::hit($request, 8, 1);
+            abort(429);
+        }
+        header('Content-type: text/html; charset=utf-8');
+        header('X-Accel-Buffering: no');
+        ini_set('zlib.output_compression', 'Off');
+        ini_set('output_buffering', 'Off');
+        ini_set('output_handler', '');
+
+        ob_end_clean();
+
+        $key = md5($request->ip() . microtime(true));
+
+        echo (view('layouts.resultpage.verificationHeader')->with('key', $key)->render());
+        flush();
+
+        $answer = boolval(Redis::connection("cache")->blpop($key, 5));
+
+        if ($answer === true) {
+            return $next($request);
+        } else {
+            $accept = Throttle::attempt($request, 8, 1);
+            if (!$accept) {
+                abort(429);
+            }
+
+            # Lockout
+            $ids = HumanVerification::block($request);
+        }
+
+        return redirect()->route('captcha', ["id" => $ids[0], "uid" => $ids[1], "url" => url()->full()]);
+
+    }
+}

chart/templates/deployment.yaml

@@ -63,6 +63,9 @@ spec:
       - name: blacklist-ad
         secret:
           secretName: metager-ad-blacklist
+      - name: metager-config
+        configMap:
+          name: metager
       containers:
       - name: {{ .Chart.Name }}-phpfpm
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
@@ -93,6 +96,9 @@ spec:
           initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
           timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
         volumeMounts:
+        - name: metager-config
+          mountPath: /html/config/metager
+          readOnly: true
         - name: mglogs-persistent-storage
           mountPath: /html/storage/logs/metager
           readOnly: false

composer.json

@@ -8,17 +8,18 @@
     ],
     "license": "MIT",
     "require": {
-        "laravel/framework": "5.8.*",
         "php": "^7.1.3",
+        "endclothing/prometheus_client_php": "^1.0",
         "fideloper/proxy": "^4.0",
-        "laravel/tinker": "^1.0",
         "globalcitizen/php-iban": "^2.6",
+        "graham-campbell/throttle": "^7.5",
         "jenssegers/agent": "^2.6",
+        "laravel/framework": "5.8.*",
+        "laravel/tinker": "^1.0",
         "mcamara/laravel-localization": "dev-master#13f418e481ed06f482e4fca87ec5ff67c2949373",
         "mews/captcha": "^2.2",
         "predis/predis": "^1.1",
-        "symfony/dom-crawler": "^4.1",
-        "endclothing/prometheus_client_php": "^1.0"
+        "symfony/dom-crawler": "^4.1"
     },
     "require-dev": {
         "beyondcode/laravel-dump-server": "^1.0",

config/metager/metager.php

+<?php
+
+return [
+    "browserverification_enabled" => true,
+];

resources/lang/de/429.php

+<?php
+
+return [
+    'title' => '429 - Zu viele Anfragen',
+    'text' => '',
+];

resources/lang/en/429.php

+<?php
+
+return [
+    'title' => '429 - Too many Requests',
+    'text' => '',
+];

resources/views/errors/404.blade.php

@@ -3,6 +3,12 @@
 @section('title', 'Fehler 404 - Seite nicht gefunden')
 
 @section('content')
+	<style>
+		main#main-content {
+			align-items: center;
+			justify-content: center;
+		}
+	</style>
 	<h1>{{ trans('404.title') }}</h1>
 	<p>{{ trans('404.text') }}</p>
 @endsection

resources/views/errors/429.blade.php

+@extends('layouts.subPages')
+
+@section('title', trans('429.title'))
+
+@section('content')
+	<style>
+		main#main-content {
+			align-items: center;
+			justify-content: center;
+		}
+	</style>
+	<h1>{{ trans('429.title') }}</h1>
+	<p>{{ trans('429.text') }}</p>
+@endsection

resources/views/layouts/resultPage.blade.php

+@if(empty(config("metager.metager.browserverification_enabled")) || !config("metager.metager.browserverification_enabled"))
 <!DOCTYPE html>
 <html lang="{!! trans('staticPages.meta.language') !!}">
 	<head>
 		<meta charset="utf-8">
+@endif
 		<title>{{ $eingabe }} - MetaGer</title>
 		<link href="/favicon.ico" rel="icon" type="image/x-icon" />
 		<link href="/favicon.ico" rel="shortcut icon" type="image/x-icon" />

resources/views/layouts/resultpage/verificationHeader.blade.php

+<!DOCTYPE html>
+<html lang="{!! trans('staticPages.meta.language') !!}">
+<head>
+    <meta charset="UTF-8">
+    <link rel="stylesheet" href="/index.css?id={{ $key }}">

routes/web.php

@@ -195,13 +195,14 @@ Route::group(
             return redirect(LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), '/'));
         });
 
-        Route::match(['get', 'post'], 'meta/meta.ger3', 'MetaGerSearch@search')->middleware('humanverification', 'useragentmaster');
+        Route::match(['get', 'post'], 'meta/meta.ger3', 'MetaGerSearch@search')->middleware('browserverification', 'humanverification', 'useragentmaster');
 
         Route::get('meta/loadMore', 'MetaGerSearch@loadMore');
         Route::post('img/cat.jpg', 'HumanVerification@remove');
         Route::get('verify/metager/{id}/{uid}', ['as' => 'captcha', 'uses' => 'HumanVerification@captcha', 'middleware' => 'throttle:12,1']);
         Route::get('r/metager/{mm}/{pw}/{url}', ['as' => 'humanverification', 'uses' => 'HumanVerification@removeGet']);
         Route::post('img/dog.jpg', 'HumanVerification@whitelist');
+        Route::get('index.css', 'HumanVerification@browserVerification');
 
         Route::get('meta/picture', 'Pictureproxy@get');
         Route::get('clickstats', 'LogController@clicklog');