Merge branch '865-optimize-human-verification' into 'development'

Resolve "Optimize Human Verification"

Closes #865

See merge request open-source/MetaGer!1387

app/Console/Kernel.php

@@ -31,13 +31,6 @@ class Kernel extends ConsoleKernel
             DB::table('monthlyrequests')->truncate();
             DB::disconnect('mysql');
         })->monthlyOn(1, '00:00');
-
-        // Delete all of the old humanverification entries
-        $schedule->call(function () {
-            DB::delete('DELETE FROM humanverification WHERE updated_at < (now() - interval 72 hour) AND whitelist = 0 ORDER BY updated_at DESC');
-            DB::delete('DELETE FROM humanverification WHERE updated_at < (now() - interval 2 week) AND whitelist = 1 ORDER BY updated_at DESC');
-            DB::disconnect('mysql');
-        })->everyThirtyMinutes();
     }
 
     /**

app/Http/Controllers/HumanVerification.php

@@ -4,15 +4,21 @@ namespace App\Http\Controllers;
 
 use Captcha;
 use Carbon;
-use DB;
 use Illuminate\Hashing\BcryptHasher as Hasher;
 use Illuminate\Http\Request;
+use Illuminate\Support\Facades\Redis;
 use Input;
 
 class HumanVerification extends Controller
 {
+    const PREFIX = "humanverification";
+    const EXPIRELONG = 60 * 60 * 24 * 14;
+    const EXPIRESHORT = 60 * 60 * 72;
+
     public static function captcha(Request $request, Hasher $hasher, $id, $url = null)
     {
+        $redis = Redis::connection('redisCache');
+
         if ($url != null) {
             $url = base64_decode(str_replace("<<SLASH>>", "/", $url));
         } else {
@@ -20,15 +26,26 @@ class HumanVerification extends Controller
         }
 
         if ($request->getMethod() == 'POST') {
-            $user = DB::table('humanverification')->where('uid', $id)->first();
 
-            $lockedKey = $user->lockedKey;
+            $user = $redis->hgetall(HumanVerification::PREFIX . "." . $id);
+            $user = ['uid' => $user["uid"],
+                'id' => $user["id"],
+                'unusedResultPages' => intval($user["unusedResultPages"]),
+                'whitelist' => filter_var($user["whitelist"], FILTER_VALIDATE_BOOLEAN),
+                'locked' => filter_var($user["locked"], FILTER_VALIDATE_BOOLEAN),
+                "lockedKey" => $user["lockedKey"],
+            ];
+
+            $lockedKey = $user["lockedKey"];
             $key = $request->input('captcha');
             $key = strtolower($key);
 
             if (!$hasher->check($key, $lockedKey)) {
                 $captcha = Captcha::create("default", true);
-                DB::table('humanverification')->where('uid', $id)->update(['lockedKey' => $captcha["key"]]);
+                $pipeline = $redis->pipeline();
+                $pipeline->hset(HumanVerification::PREFIX . "." . $id, 'lockedKey', $captcha["key"]);
+                $pipeline->expire(HumanVerification::PREFIX . "." . $id, $user["whitelist"] ? HumanVerification::EXPIRELONG : HumanVerification::EXPIRESHORT);
+                $pipeline->execute();
                 return view('humanverification.captcha')->with('title', 'Bestätigung notwendig')
                     ->with('id', $id)
                     ->with('url', $url)
@@ -36,9 +53,15 @@ class HumanVerification extends Controller
                     ->with('errorMessage', 'Fehler: Falsches Captcha eingegeben!');
             } else {
                 # If we can unlock the Account of this user we will redirect him to the result page
-                if ($user !== null && $user->locked === 1) {
+                if ($user !== null && $user["locked"]) {
                     # The Captcha was correct. We can remove the key from the user
-                    DB::table('humanverification')->where('uid', $id)->update(['locked' => false, 'lockedKey' => "", 'whitelist' => 1]);
+                    # If the sum of all users with that ip is too high we need to whitelist the user or they will receive a captcha again on the next request
+                    $sum = 0;
+                    $users = [];
+                    $pipeline = $redis->pipeline();
+                    $pipeline->hmset(HumanVerification::PREFIX . "." . $id, ['locked' => "0", 'lockedKey' => "", 'whitelist' => '1']);
+                    $pipeline->expire(HumanVerification::PREFIX . "." . $id, $user["whitelist"] ? HumanVerification::EXPIRELONG : HumanVerification::EXPIRESHORT);
+                    $pipeline->execute();
                     return redirect($url);
                 } else {
                     return redirect('/');
@@ -46,7 +69,10 @@ class HumanVerification extends Controller
             }
         }
         $captcha = Captcha::create("default", true);
-        DB::table('humanverification')->where('uid', $id)->update(['lockedKey' => $captcha["key"]]);
+        $pipeline = $redis->pipeline();
+        $pipeline->hset(HumanVerification::PREFIX . "." . $id, 'lockedKey', $captcha["key"]);
+        $pipeline->expire(HumanVerification::PREFIX . "." . $id, $user["whitelist"] ? HumanVerification::EXPIRELONG : HumanVerification::EXPIRESHORT);
+        $pipeline->execute();
         return view('humanverification.captcha')->with('title', 'Bestätigung notwendig')
             ->with('id', $id)
             ->with('url', $url)
@@ -81,30 +107,63 @@ class HumanVerification extends Controller
 
     private static function removeUser($request, $uid)
     {
+        $redis = Redis::conection('redisCache');
         $id = hash("sha512", $request->ip());
 
-        $sum = DB::table('humanverification')->where('id', $id)->where('whitelist', false)->sum('unusedResultPages');
-        $user = DB::table('humanverification')->where('uid', $uid)->first();
+        $userList = $redis->smembers(HumanVerification::PREFIX . "." . $id);
+        $pipe = $redis->pipeline();
+        foreach ($userList as $userid) {
+            $pipe->hgetall(HumanVerification::PREFIX . "." . $userid);
+        }
+        $usersData = $pipe->execute();
+
+        $user = [];
+        $users = [];
+        $sum = 0;
+        foreach ($usersData as $userTmp) {
+            if (empty($userTmp)) {
+                continue;
+            }
+            $userNew = ['uid' => $userTmp["uid"],
+                'id' => $userTmp["id"],
+                'unusedResultPages' => intval($userTmp["unusedResultPages"]),
+                'whitelist' => filter_var($userTmp["whitelist"], FILTER_VALIDATE_BOOLEAN),
+                'locked' => filter_var($userTmp["locked"], FILTER_VALIDATE_BOOLEAN),
+                "lockedKey" => $userTmp["lockedKey"],
+            ];
+
+            if ($uid === $userTmp["uid"]) {
+                $user = $userNew;
+            } else {
+                $users[] = $userNew;
+            }
+            if ($userNew["whitelist"]) {
+                $sum += intval($userTmp["unusedResultPages"]);
+            }
+
+        }
 
-        if ($user === null) {
+        if (empty($user)) {
             return;
         }
 
+        $pipeline = $redis->pipeline();
         # Check if we have to whitelist the user or if we can simply delete the data
-        if ($user->unusedResultPages < $sum && $user->whitelist === 0) {
+        if ($user["unusedResultPages"] < $sum && !$user["whitelist"]) {
             # Whitelist
-            DB::table('humanverification')->where('uid', $uid)->update(['whitelist' => true, 'whitelistCounter' => 0]);
-            $user->whitelist = 1;
-            $user->whitelistCounter = 0;
+            $pipeline->hset(HumanVerification::PREFIX . "." . $uid, 'whitelist', "1");
+            $user["whitelist"] = true;
         }
 
-        if ($user->whitelist === 1) {
-            DB::table('humanverification')->where('uid', $uid)->update(['unusedResultPages' => 0]);
+        if ($user["whitelist"]) {
+            $pipeline->hset(HumanVerification::PREFIX . "." . $uid, 'unusedResultPages', "0");
         } else {
-            DB::table('humanverification')->where('uid', $uid)->where('updated_at', '<', Carbon::NOW()->subSeconds(2))->delete();
-
+            $pipeline->hdel(HumanVerification::PREFIX . "." . $uid);
+            $pipeline->srem(HumanVerification::PREFIX . "." . $id, $uid);
         }
-
+        $pipeline->expire(HumanVerification::PREFIX . "." . $uid, $user["whitelist"] ? HumanVerification::EXPIRELONG : HumanVerification::EXPIRESHORT);
+        $pipeline->expire(HumanVerification::PREFIX . "." . $id, HumanVerification::EXPIRELONG);
+        $pipeline->execute();
     }
 
     private static function checkId($request, $id)

app/Http/Controllers/MetaGerSearch.php

@@ -12,7 +12,6 @@ class MetaGerSearch extends Controller
 {
     public function search(Request $request, MetaGer $metager)
     {
-
         $focus = $request->input("focus", "web");
 
         if ($focus === "maps") {

app/Http/Middleware/HumanVerification.php

@@ -3,10 +3,10 @@
 namespace App\Http\Middleware;
 
 use Captcha;
-use Carbon;
 use Closure;
-use DB;
+use Cookie;
 use Illuminate\Http\Response;
+use Illuminate\Support\Facades\Redis;
 use URL;
 
 class HumanVerification
@@ -22,8 +22,9 @@ class HumanVerification
     {
         // The specific user
         $user = null;
-        $newUser = true;
         $update = true;
+        $prefix = "humanverification";
+        $redis = Redis::connection('redisCache');
         try {
             $id = hash("sha512", $request->ip());
             $uid = hash("sha512", $request->ip() . $_SERVER["AGENT"]);
@@ -35,47 +36,61 @@ class HumanVerification
              * If someone that uses a bot finds this out we
              * might have to change it at some point.
              */
-            if ($request->filled('password') || $request->filled('key') || $request->filled('appversion') || !env('BOT_PROTECTION', false)) {
+            if ($request->filled('password') || $request->filled('key') || Cookie::get('key') !== null || $request->filled('appversion') || !env('BOT_PROTECTION', false)) {
                 $update = false;
                 return $next($request);
             }
 
-            $users = DB::select('select * from humanverification where id = ?', [$id]);
+            # Get all Users of this IP
+            $userList = $redis->smembers($prefix . "." . $id);
+            $pipe = $redis->pipeline();
 
+            foreach ($userList as $userid) {
+                $pipe->hgetall($prefix . "." . $userid);
+            }
+
+            $usersData = $pipe->execute();
+
+            $user = [];
+            $users = [];
             # Lock out everyone in a Bot network
             # Find out how many requests this IP has made
             $sum = 0;
-            foreach ($users as $userTmp) {
-                if ($uid == $userTmp->uid) {
-                    $user = ['uid' => $userTmp->uid,
-                        'id' => $userTmp->id,
-                        'unusedResultPages' => intval($userTmp->unusedResultPages),
-                        'whitelist' => filter_var($userTmp->whitelist, FILTER_VALIDATE_BOOLEAN),
-                        'whitelistCounter' => $userTmp->whitelistCounter,
-                        'locked' => filter_var($userTmp->locked, FILTER_VALIDATE_BOOLEAN),
-                        "lockedKey" => $userTmp->lockedKey,
-                        'updated_at' => Carbon::now(),
+            foreach ($usersData as $index => $userTmp) {
+                if (empty($userTmp)) {
+                    // This is a key that has been expired and should be deleted
+                    $redis->srem($prefix . "." . $id, $userList[$index]);
+                    continue;
+                }
+                $userNew = ['uid' => $userTmp["uid"],
+                    'id' => $userTmp["id"],
+                    'unusedResultPages' => intval($userTmp["unusedResultPages"]),
+                    'whitelist' => filter_var($userTmp["whitelist"], FILTER_VALIDATE_BOOLEAN),
+                    'locked' => filter_var($userTmp["locked"], FILTER_VALIDATE_BOOLEAN),
+                    "lockedKey" => $userTmp["lockedKey"],
                 ];
-                    $newUser = false;
+
+                if ($uid === $userTmp["uid"]) {
+                    $user = $userNew;
+                } else {
+                    $users[] = $userNew;
                 }
-                if ($userTmp->whitelist === 0) {
-                    $sum += $userTmp->unusedResultPages;
+                if (!$userNew["whitelist"]) {
+                    $sum += intval($userTmp["unusedResultPages"]);
                 }
 
             }
-            # If this user doesn't have an entry we will create one
 
-            if ($user === null) {
+            # If this user doesn't have an entry we will create one
+            if (empty($user)) {
                 $user =
                     [
                     'uid' => $uid,
                     'id' => $id,
                     'unusedResultPages' => 0,
                     'whitelist' => false,
-                    'whitelistCounter' => 0,
                     'locked' => false,
                     "lockedKey" => "",
-                    'updated_at' => Carbon::now(),
                 ];
             }
 
@@ -96,7 +111,7 @@ class HumanVerification
             // Defines if this is the only user using that IP Adress
             $alone = true;
             foreach ($users as $userTmp) {
-                if ($userTmp->uid != $uid && !$userTmp->whitelist) {
+                if ($userTmp["uid"] != $uid && !$userTmp["whitelist"]) {
                     $alone = false;
                 }
 
@@ -133,41 +148,34 @@ class HumanVerification
                 }
 
             }
-        } catch (\Illuminate\Database\QueryException $e) {
-            // Failure in contacting metager3.de
+        } catch (\Predis\Connection\ConnectionException $e) {
+            $update = false;
         } finally {
             if ($update) {
+
                 // Update the user in the database
-                if ($newUser) {
-                    DB::table('humanverification')->insert(
-                        [
-                            'uid' => $user["uid"],
-                            'id' => $user["id"],
-                            'unusedResultPages' => $user['unusedResultPages'],
-                            'whitelist' => $user["whitelist"],
-                            'whitelistCounter' => $user["whitelistCounter"],
-                            'locked' => $user["locked"],
-                            "lockedKey" => $user["lockedKey"],
-                            'updated_at' => $user["updated_at"],
-                        ]
-                    );
+                $pipeline = $redis->pipeline();
+
+                $pipeline->hmset($prefix . "." . $user['uid'], $user);
+                $pipeline->sadd($prefix . "." . $user["id"], $user["uid"]);
+
+                // Expire in two weeks
+                $expireLong = 60 * 60 * 24 * 14;
+                // Expire in 72h
+                $expireShort = 60 * 60 * 72;
+
+                if ($user["whitelist"]) {
+                    $pipeline->expire($prefix . "." . $user['uid'], $expireLong);
                 } else {
-                    DB::table('humanverification')->where('uid', $uid)->update(
-                        [
-                            'uid' => $user["uid"],
-                            'id' => $user["id"],
-                            'unusedResultPages' => $user['unusedResultPages'],
-                            'whitelist' => $user["whitelist"],
-                            'whitelistCounter' => $user["whitelistCounter"],
-                            'locked' => $user["locked"],
-                            "lockedKey" => $user["lockedKey"],
-                            'updated_at' => $user["updated_at"],
-                        ]
-                    );
+                    $pipeline->expire($prefix . "." . $user['uid'], $expireShort);
                 }
+
+                $pipeline->expire($prefix . "." . $user["id"], $expireLong);
+
+                $pipeline->execute();
             }
-            DB::disconnect('mysql');
         }
+
         $request->request->add(['verification_id' => $user["uid"], 'verification_count' => $user["unusedResultPages"]]);
         return $next($request);