Merge branch '794-botschutz-uberarbeiten' into 'development'

Resolve "Botschutz überarbeiten"

Closes #794

See merge request !1304

app/Http/Controllers/HumanVerification.php

@@ -20,23 +20,25 @@ class HumanVerification extends Controller
         }
 
         if ($request->getMethod() == 'POST') {
-            $user = DB::table('humanverification')->where('id', $id)->first();
+            $user = DB::table('humanverification')->where('uid', $id)->first();
 
             $lockedKey = $user->lockedKey;
             $key = $request->input('captcha');
             $key = strtolower($key);
+
             if (!$hasher->check($key, $lockedKey)) {
                 $captcha = Captcha::create("default", true);
-                DB::table('humanverification')->where('id', $id)->update(['lockedKey' => $captcha["key"]]);
-                return view('captcha')->with('title', 'Bestätigung notwendig')
+                DB::table('humanverification')->where('uid', $id)->update(['lockedKey' => $captcha["key"]]);
+                return view('humanverification.captcha')->with('title', 'Bestätigung notwendig')
                     ->with('id', $id)
                     ->with('url', $url)
                     ->with('image', $captcha["img"])
-                    ->with('errorMessage', 'Bitte Captcha eingeben:');
+                    ->with('errorMessage', 'Fehler: Falsches Captcha eingegeben!');
             } else {
                 # If we can unlock the Account of this user we will redirect him to the result page
                 if ($user !== null && $user->locked === 1) {
-                    DB::table('humanverification')->where('id', $id)->update(['locked' => false]);
+                    # The Captcha was correct. We can remove the key from the user
+                    DB::table('humanverification')->where('uid', $id)->update(['locked' => false, 'lockedKey' => "", 'whitelist' => 1]);
                     return redirect($url);
                 } else {
                     return redirect('/');
@@ -44,8 +46,8 @@ class HumanVerification extends Controller
             }
         }
         $captcha = Captcha::create("default", true);
-        DB::table('humanverification')->where('id', $id)->update(['lockedKey' => $captcha["key"]]);
-        return view('captcha')->with('title', 'Bestätigung notwendig')
+        DB::table('humanverification')->where('uid', $id)->update(['lockedKey' => $captcha["key"]]);
+        return view('humanverification.captcha')->with('title', 'Bestätigung notwendig')
             ->with('id', $id)
             ->with('url', $url)
             ->with('image', $captcha["img"]);
@@ -56,10 +58,9 @@ class HumanVerification extends Controller
         if (!$request->has('mm')) {
             abort(404, "Keine Katze gefunden.");
         }
-        $id = md5($request->ip());
+
         if (HumanVerification::checkId($request, $request->input('mm'))) {
-            # Remove the entry from the database
-            DB::table('humanverification')->where('id', $id)->where('updated_at', '<', Carbon::NOW()->subSeconds(2))->delete();
+            HumanVerification::removeUser($request, $request->input('mm'));
         }
         return response(hex2bin('89504e470d0a1a0a0000000d494844520000000100000001010300000025db56ca00000003504c5445000000a77a3dda0000000174524e530040e6d8660000000a4944415408d76360000000020001e221bc330000000049454e44ae426082'), 200)
             ->header('Content-Type', 'image/png');
@@ -72,15 +73,47 @@ class HumanVerification extends Controller
         # If the user is correct and the password is we will delete any entry in the database
         $requiredPass = md5($mm . Carbon::NOW()->day . $url . env("PROXY_PASSWORD"));
         if (HumanVerification::checkId($request, $mm) && $requiredPass === $password) {
-            # Remove the entry from the database
-            DB::table('humanverification')->where('id', $mm)->where('updated_at', '<', Carbon::NOW()->subSeconds(2))->delete();
+            HumanVerification::removeUser($request, $mm);
         }
         return redirect($url);
     }
 
+    private static function removeUser($request, $uid)
+    {
+        $id = hash("sha512", $request->ip());
+
+        $sum = DB::table('humanverification')->where('id', $id)->where('whitelist', false)->sum('unusedResultPages');
+        $user = DB::table('humanverification')->where('uid', $uid)->first();
+
+        if ($user === null) {
+            return;
+        }
+
+        # Check if we have to whitelist the user or if we can simply delete the data
+        if ($user->unusedResultPages < $sum && $user->whitelist === 0) {
+            # Whitelist
+            DB::table('humanverification')->where('uid', $uid)->update(['whitelist' => true, 'whitelistCounter' => 0]);
+            $user->whitelist = 1;
+            $user->whitelistCounter = 0;
+        }
+
+        if ($user->whitelist === 1) {
+            if (
+                DB::table('humanverification')->where('uid', $uid)->update(['unusedResultPages' => 0])
+                === 1
+            ) {
+                DB::table('usedurls')->where('uid', $uid)->delete();
+            }
+        } else {
+            DB::table('humanverification')->where('uid', $uid)->where('updated_at', '<', Carbon::NOW()->subSeconds(2))->delete();
+
+        }
+
+    }
+
     private static function checkId($request, $id)
     {
-        if (md5($request->ip()) === $id) {
+        if (hash("sha512", $request->ip() . $_SERVER["AGENT"]) === $id) {
             return true;
         } else {
             return false;

app/Http/Middleware/HumanVerification.php

@@ -7,6 +7,7 @@ use Carbon;
 use Closure;
 use DB;
 use Illuminate\Http\Response;
+use URL;
 
 class HumanVerification
 {
@@ -20,7 +21,9 @@ class HumanVerification
     public function handle($request, Closure $next)
     {
         try {
-            $id = md5($request->ip());
+            $id = hash("sha512", $request->ip());
+            $uid = hash("sha512", $request->ip() . $_SERVER["AGENT"]);
+            unset($_SERVER["AGENT"]);
 
             /**
              * If the user sends a Password or a key
@@ -28,57 +31,102 @@ class HumanVerification
              * If someone that uses a bot finds this out we
              * might have to change it at some point.
              */
-            if ($request->has('password') || $request->filled('key') || $request->filled('appversion') || !env('BOT_PROTECTION', false)) {
+            if ($request->filled('password') || $request->filled('key') || $request->filled('appversion') || !env('BOT_PROTECTION', false)) {
                 return $next($request);
             }
 
-            $user = DB::table('humanverification')->where('id', $id)->first();
+            // The specific user
+            $user = DB::table('humanverification')->where('uid', $uid)->first();
+
             $createdAt = Carbon::now();
             $unusedResultPages = 1;
             $locked = false;
             # If this user doesn't have an entry we will create one
+
             if ($user === null) {
                 DB::table('humanverification')->insert(
-                    ['id' => $id, 'unusedResultPages' => 1, 'locked' => false, "lockedKey" => "", 'updated_at' => Carbon::now()]
+                    [
+                        'uid' => $uid,
+                        'id' => $id,
+                        'unusedResultPages' => 0,
+                        'whitelist' => false,
+                        'whitelistCounter' => 0,
+                        'locked' => false,
+                        "lockedKey" => "",
+                        'updated_at' => Carbon::now(),
+                    ]
                 );
                 # Insert the URL the user tries to reach
                 $url = url()->full();
-                DB::table('usedurls')->insert(['user_id' => $id, 'url' => $url]);
-                $user = DB::table('humanverification')->where('id', $id)->first();
-            } else if ($user->locked !== 1) {
-                $unusedResultPages = intval($user->unusedResultPages);
-                $unusedResultPages++;
-                # We have different security gates:
-                #   50, 75, 85, >=90 => Captcha validated Result Pages
-                # If the user shows activity on our result page the counter will be deleted
-                # Maybe I'll add a ban if the user reaches 100
-                if ($unusedResultPages === 50 || $unusedResultPages === 75 || $unusedResultPages === 85 || $unusedResultPages >= 90) {
-                    $locked = true;
+                DB::table('usedurls')->insert(['uid' => $uid, 'id' => $id, 'eingabe' => $request->input('eingabe', '')]);
+                $user = DB::table('humanverification')->where('uid', $uid)->first();
+            }
+
+            # Lock out everyone in a Bot network
+            # Find out how many requests this IP has made
+            $sum = DB::table('humanverification')->where('id', $id)->where('whitelist', false)->sum('unusedResultPages');
+
+            # A lot of automated requests are from websites that redirect users to our result page.
+            # We will detect those requests and put a captcha
+            $referer = URL::previous();
+            # Just the URL-Parameter
+            $refererLock = false;
+            if (stripos($referer, "?") !== false) {
+                $referer = substr($referer, stripos($referer, "?") + 1);
+                $referer = urldecode($referer);
+                if (preg_match("/http[s]{0,1}:\/\/metager\.de\/meta\/meta.ger3\?.*?eingabe=([\w\d]+\.){1,2}[\w\d]+/si", $referer) === 1) {
+                    $refererLock = true;
                 }
-                DB::table('humanverification')->where('id', $id)->update(['unusedResultPages' => $unusedResultPages, 'locked' => $locked, 'updated_at' => $createdAt]);
-                # Insert the URL the user tries to reach
-                DB::table('usedurls')->insert(['user_id' => $id, 'url' => url()->full()]);
+
             }
 
-            $request->request->add(['verification_id' => $id, 'verification_count' => $unusedResultPages]);
+            // Defines if this is the only user using that IP Adress
+            $alone = DB::table('humanverification')->where('id', $id)->count() === 1;
+            if ((!$alone && $sum >= 50 && $user->whitelist !== 1) || $refererLock) {
+                DB::table('humanverification')->where('uid', $uid)->update(['locked' => true]);
+                $user->locked = 1;
+            }
 
             # If the user is locked we will force a Captcha validation
             if ($user->locked === 1) {
                 $captcha = Captcha::create("default", true);
-                DB::table('humanverification')->where('id', $id)->update(['lockedKey' => $captcha["key"]]);
+                DB::table('humanverification')->where('uid', $uid)->update(['lockedKey' => $captcha["key"]]);
                 return
                 new Response(
-                    view('captcha')
+                    view('humanverification.captcha')
                         ->with('title', "Bestätigung erforderlich")
-                        ->with('id', $id)
+                        ->with('id', $uid)
                         ->with('url', url()->full())
                         ->with('image', $captcha["img"])
                 );
             }
+
+            $unusedResultPages = intval($user->unusedResultPages);
+            $unusedResultPages++;
+            $locked = false;
+
+            if ($alone || $user->whitelist === 1) {
+                # This IP doesn't need verification yet
+                # The user currently isn't locked
+
+                # We have different security gates:
+                #   50, 75, 85, >=90 => Captcha validated Result Pages
+                # If the user shows activity on our result page the counter will be deleted
+                # Maybe I'll add a ban if the user reaches 100
+
+                if ($unusedResultPages === 50 || $unusedResultPages === 75 || $unusedResultPages === 85 || $unusedResultPages >= 90) {
+                    $locked = true;
+                }
+
+            }
+            DB::table('humanverification')->where('uid', $uid)->update(['unusedResultPages' => $unusedResultPages, 'locked' => $locked]);
+            # Insert the URL the user tries to reach
+            DB::table('usedurls')->insert(['uid' => $uid, 'id' => $id, 'eingabe' => $request->input('eingabe', '')]);
+
         } catch (\Illuminate\Database\QueryException $e) {
             // Failure in contacting metager3.de
         }
-
+        $request->request->add(['verification_id' => $uid, 'verification_count' => $unusedResultPages]);
         return $next($request);
     }
 }

app/MetaGer.php

@@ -934,7 +934,7 @@ class MetaGer
 
     public function parseFormData(Request $request)
     {
-        $this->request = $request;
+        
         # Sichert, dass der request in UTF-8 formatiert ist
         if ($request->input('encoding', 'utf8') !== "utf8") {
             # In früheren Versionen, als es den Encoding Parameter noch nicht gab, wurden die Daten in ISO-8859-1 übertragen
@@ -1051,6 +1051,8 @@ class MetaGer
         $this->verificationId = $request->input('verification_id', null);
         $this->verificationCount = intval($request->input('verification_count', '0'));
         $this->apiKey = $request->input('key', '');
+        // Remove Inputs that are not used
+        $this->request = $request->replace($request->except(['verification_id', 'uid', 'verification_count']));
 
         $this->validated = false;
         if (isset($this->password)) {
@@ -1492,6 +1494,14 @@ class MetaGer
 
 # Einfache Getter
 
+    public function getVerificationId() {
+        return $this->verificationId;
+    }
+
+    public function getVerificationCount() {
+        return $this->verificationCount;
+    }
+
     public function getSite()
     {
         return $this->site;

database/migrations/2018_04_26_114745_create_humanverification_table.php

 <?php
 
-use Illuminate\Support\Facades\Schema;
-use Illuminate\Database\Schema\Blueprint;
 use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
 
 class CreateHumanverificationTable extends Migration
 {
@@ -14,9 +14,13 @@ class CreateHumanverificationTable extends Migration
     public function up()
     {
         Schema::create('humanverification', function (Blueprint $table) {
-            $table->string('id')->unique();
+            $table->string('uid')->unique();
+            $table->string('id');
             $table->integer('unusedResultPages');
+            $table->boolean('whitelist');
+            $table->integer('whitelistCounter');
             $table->boolean('locked');
+            $table->string('lockedKey');
             $table->timestamp('updated_at');
         });
     }

database/migrations/2018_05_03_101301_usedurls.php

 <?php
 
-use Illuminate\Support\Facades\Schema;
-use Illuminate\Database\Schema\Blueprint;
 use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
 
 class Usedurls extends Migration
 {
@@ -14,11 +14,12 @@ class Usedurls extends Migration
     public function up()
     {
         Schema::create('usedurls', function (Blueprint $table) {
-            $table->increments('id')->unique();
-            $table->string('user_id');
-            $table->text('url');
+            $table->increments('number')->unique();
+            $table->string('uid');
+            $table->string('id');
+            $table->text('eingabe');
             $table->timestamp('created_at');
-            $table->foreign('user_id')->references('id')->on('humanverification')->onDelete('cascade');
+            $table->foreign('uid')->references('uid')->on('humanverification')->onDelete('cascade');
         });
     }
 

public/index.php

@@ -13,6 +13,8 @@ if (isset($_SERVER["HTTP_FORWARDED"]) && isset($_SERVER["HTTP_X_FORWARDED_FOR"])
     unset($_SERVER["HTTP_FORWARDED"]);
 }
 
+$_SERVER["AGENT"] = $_SERVER["HTTP_USER_AGENT"];
+
 if (isset($_SERVER['HTTP_USER_AGENT'])) {
     $agentPieces = explode(" ", $_SERVER['HTTP_USER_AGENT']);
 

resources/views/captcha.blade.php → resources/views/humanverification/captcha.blade.php

@@ -4,11 +4,10 @@
 
 @section('content')
     <h1>Entschuldigen Sie die Störung</h1>
-    <p>Wir haben Grund zur Annahme, dass von Ihrem Anschluss verstärkt automatisierte Abfragen abgeschickt wurden.
-    Deshalb bitten wir Sie, die nachfolgende Captcha Abfrage zu beantworten.</p>
-    <p>Sollten Sie diese Nachricht häufiger sehen oder handelt es sich dabei um einen Irrtum, schicken Sie uns gerne eine Nachricht über unser <a href="/kontakt">Kontaktformular</a>.</p>
-    <p>Nennen Sie uns in diesem Fall bitte unbedingt folgende Vorgangsnummer: {{ $id }}
-    <p>Wir schauen uns den Vorgang dann gerne im Detail an.</p>
+    <p>Sie befinden sich in einem Netzwerk aus dem wir verstärkt automatisierte Anfragen erhalten. Keine Sorge: Das bedeutet nicht unbedingt, dass diese Anfragen von Ihrem PC kommen.</p>
+    <p>Allerdings können wir Ihre Anfragen nicht von denen des "Robots" unterscheiden. Zum Schutz der von uns abgefragten Suchmaschinen müssen wir aber sicherstellen, dass diese nicht mit (automatisierten) Abfragen überflutet werden.</p>
+
+    <p>Bitte geben Sie deshalb die Zeichen aus dem Bild in die Eingabebox ein und bestätigen Sie mit "OK" um zur Ergebnisseite zu gelangen.</p>
     <form method="post" action="{{ route('verification', ['id' => $id]) }}">
         <input type="hidden" name="url" value="{!! $url !!}">
         <input type="hidden" name="id" value="{{ $id }}">
@@ -16,7 +15,9 @@
         @if(isset($errorMessage))
         <p><font color="red">{{$errorMessage}}</font></p>
         @endif
-        <p><input type="text" name="captcha"></p>
-        <p><button type="submit" name="check">OK</button></p>
+        <p><input type="text" class="form-control" name="captcha" placeholder="Captcha eingeben"  autofocus></p>
+        <p><button type="submit" class="btn btn-success" name="check">OK</button></p>
     </form>
+    <p>Sollten Sie diese Nachricht häufiger sehen oder handelt es sich dabei um einen Irrtum, schicken Sie uns gerne eine Nachricht über unser <a href="/kontakt">Kontaktformular</a>.</p>
+    <p>Nennen Sie uns in diesem Fall bitte unbedingt folgende Vorgangsnummer: {{ $id }}
 @endsection

resources/views/layouts/resultPage.blade.php

@@ -9,8 +9,8 @@
 		<meta name="p" content="{{ getmypid() }}" />
 		<meta name="q" content="{{ $eingabe }}" />
 		<meta name="l" content="{{ LaravelLocalization::getCurrentLocale() }}" />
-		<meta name="mm" content="{{ Request::input('verification_id') }}" />
-		<meta name="mn" content="{{ Request::input('verification_count') }}" />
+		<meta name="mm" content="{{ $metager->getVerificationId() }}" />
+		<meta name="mn" content="{{ $metager->getVerificationCount() }}" />
 		<link rel="search" type="application/opensearchdescription+xml" title="{!! trans('resultPage.opensearch') !!}" href="{{  LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), action('StartpageController@loadPlugin', ['params' => base64_encode(serialize(Request::all()))])) }}">
 		<link type="text/css" rel="stylesheet" href="{{ mix('css/themes/default.css') }}" />
 		<link type="text/css" rel="stylesheet" href="/font-awesome/css/font-awesome.min.css" />

routes/web.php

@@ -164,6 +164,7 @@ Route::group(
         Route::match(['get', 'post'], 'meta/meta.ger3', 'MetaGerSearch@search')->middleware('humanverification');
         Route::post('img/cat.jpg', 'HumanVerification@remove');
         Route::get('r/metager/{mm}/{pw}/{url}', ['as' => 'humanverification', 'uses' => 'HumanVerification@removeGet']);
+        Route::post('img/dog.jpg', 'HumanVerification@whitelist');
 
         Route::get('meta/picture', 'Pictureproxy@get');
         Route::get('clickstats', 'LogController@clicklog');