Merge branch 'development' into '864-optimize-request-times'

Development

See merge request !1393

app/Console/Kernel.php

@@ -31,13 +31,6 @@ class Kernel extends ConsoleKernel
             DB::table('monthlyrequests')->truncate();
             DB::disconnect('mysql');
         })->monthlyOn(1, '00:00');
-
-        // Delete all of the old humanverification entries
-        $schedule->call(function () {
-            DB::delete('DELETE FROM humanverification WHERE updated_at < (now() - interval 72 hour) AND whitelist = 0 ORDER BY updated_at DESC');
-            DB::delete('DELETE FROM humanverification WHERE updated_at < (now() - interval 2 week) AND whitelist = 1 ORDER BY updated_at DESC');
-            DB::disconnect('mysql');
-        })->everyThirtyMinutes();
     }
 
     /**

app/Http/Controllers/HumanVerification.php

@@ -4,15 +4,21 @@ namespace App\Http\Controllers;
 
 use Captcha;
 use Carbon;
-use DB;
 use Illuminate\Hashing\BcryptHasher as Hasher;
 use Illuminate\Http\Request;
+use Illuminate\Support\Facades\Redis;
 use Input;
 
 class HumanVerification extends Controller
 {
+    const PREFIX = "humanverification";
+    const EXPIRELONG = 60 * 60 * 24 * 14;
+    const EXPIRESHORT = 60 * 60 * 72;
+
     public static function captcha(Request $request, Hasher $hasher, $id, $url = null)
     {
+        $redis = Redis::connection('redisCache');
+
         if ($url != null) {
             $url = base64_decode(str_replace("<<SLASH>>", "/", $url));
         } else {
@@ -20,15 +26,26 @@ class HumanVerification extends Controller
         }
 
         if ($request->getMethod() == 'POST') {
-            $user = DB::table('humanverification')->where('uid', $id)->first();
 
-            $lockedKey = $user->lockedKey;
+            $user = $redis->hgetall(HumanVerification::PREFIX . "." . $id);
+            $user = ['uid' => $user["uid"],
+                'id' => $user["id"],
+                'unusedResultPages' => intval($user["unusedResultPages"]),
+                'whitelist' => filter_var($user["whitelist"], FILTER_VALIDATE_BOOLEAN),
+                'locked' => filter_var($user["locked"], FILTER_VALIDATE_BOOLEAN),
+                "lockedKey" => $user["lockedKey"],
+            ];
+
+            $lockedKey = $user["lockedKey"];
             $key = $request->input('captcha');
             $key = strtolower($key);
 
             if (!$hasher->check($key, $lockedKey)) {
                 $captcha = Captcha::create("default", true);
-                DB::table('humanverification')->where('uid', $id)->update(['lockedKey' => $captcha["key"]]);
+                $pipeline = $redis->pipeline();
+                $pipeline->hset(HumanVerification::PREFIX . "." . $id, 'lockedKey', $captcha["key"]);
+                $pipeline->expire(HumanVerification::PREFIX . "." . $id, $user["whitelist"] ? HumanVerification::EXPIRELONG : HumanVerification::EXPIRESHORT);
+                $pipeline->execute();
                 return view('humanverification.captcha')->with('title', 'Bestätigung notwendig')
                     ->with('id', $id)
                     ->with('url', $url)
@@ -36,9 +53,15 @@ class HumanVerification extends Controller
                     ->with('errorMessage', 'Fehler: Falsches Captcha eingegeben!');
             } else {
                 # If we can unlock the Account of this user we will redirect him to the result page
-                if ($user !== null && $user->locked === 1) {
+                if ($user !== null && $user["locked"]) {
                     # The Captcha was correct. We can remove the key from the user
-                    DB::table('humanverification')->where('uid', $id)->update(['locked' => false, 'lockedKey' => "", 'whitelist' => 1]);
+                    # If the sum of all users with that ip is too high we need to whitelist the user or they will receive a captcha again on the next request
+                    $sum = 0;
+                    $users = [];
+                    $pipeline = $redis->pipeline();
+                    $pipeline->hmset(HumanVerification::PREFIX . "." . $id, ['locked' => "0", 'lockedKey' => "", 'whitelist' => '1']);
+                    $pipeline->expire(HumanVerification::PREFIX . "." . $id, $user["whitelist"] ? HumanVerification::EXPIRELONG : HumanVerification::EXPIRESHORT);
+                    $pipeline->execute();
                     return redirect($url);
                 } else {
                     return redirect('/');
@@ -46,7 +69,10 @@ class HumanVerification extends Controller
             }
         }
         $captcha = Captcha::create("default", true);
-        DB::table('humanverification')->where('uid', $id)->update(['lockedKey' => $captcha["key"]]);
+        $pipeline = $redis->pipeline();
+        $pipeline->hset(HumanVerification::PREFIX . "." . $id, 'lockedKey', $captcha["key"]);
+        $pipeline->expire(HumanVerification::PREFIX . "." . $id, $user["whitelist"] ? HumanVerification::EXPIRELONG : HumanVerification::EXPIRESHORT);
+        $pipeline->execute();
         return view('humanverification.captcha')->with('title', 'Bestätigung notwendig')
             ->with('id', $id)
             ->with('url', $url)
@@ -81,30 +107,63 @@ class HumanVerification extends Controller
 
     private static function removeUser($request, $uid)
     {
+        $redis = Redis::connection('redisCache');
         $id = hash("sha512", $request->ip());
 
-        $sum = DB::table('humanverification')->where('id', $id)->where('whitelist', false)->sum('unusedResultPages');
-        $user = DB::table('humanverification')->where('uid', $uid)->first();
+        $userList = $redis->smembers(HumanVerification::PREFIX . "." . $id);
+        $pipe = $redis->pipeline();
+        foreach ($userList as $userid) {
+            $pipe->hgetall(HumanVerification::PREFIX . "." . $userid);
+        }
+        $usersData = $pipe->execute();
+
+        $user = [];
+        $users = [];
+        $sum = 0;
+        foreach ($usersData as $userTmp) {
+            if (empty($userTmp)) {
+                continue;
+            }
+            $userNew = ['uid' => $userTmp["uid"],
+                'id' => $userTmp["id"],
+                'unusedResultPages' => intval($userTmp["unusedResultPages"]),
+                'whitelist' => filter_var($userTmp["whitelist"], FILTER_VALIDATE_BOOLEAN),
+                'locked' => filter_var($userTmp["locked"], FILTER_VALIDATE_BOOLEAN),
+                "lockedKey" => $userTmp["lockedKey"],
+            ];
+
+            if ($uid === $userTmp["uid"]) {
+                $user = $userNew;
+            } else {
+                $users[] = $userNew;
+            }
+            if ($userNew["whitelist"]) {
+                $sum += intval($userTmp["unusedResultPages"]);
+            }
+
+        }
 
-        if ($user === null) {
+        if (empty($user)) {
             return;
         }
 
+        $pipeline = $redis->pipeline();
         # Check if we have to whitelist the user or if we can simply delete the data
-        if ($user->unusedResultPages < $sum && $user->whitelist === 0) {
+        if ($user["unusedResultPages"] < $sum && !$user["whitelist"]) {
             # Whitelist
-            DB::table('humanverification')->where('uid', $uid)->update(['whitelist' => true, 'whitelistCounter' => 0]);
-            $user->whitelist = 1;
-            $user->whitelistCounter = 0;
+            $pipeline->hset(HumanVerification::PREFIX . "." . $uid, 'whitelist', "1");
+            $user["whitelist"] = true;
         }
 
-        if ($user->whitelist === 1) {
-            DB::table('humanverification')->where('uid', $uid)->update(['unusedResultPages' => 0]);
+        if ($user["whitelist"]) {
+            $pipeline->hset(HumanVerification::PREFIX . "." . $uid, 'unusedResultPages', "0");
         } else {
-            DB::table('humanverification')->where('uid', $uid)->where('updated_at', '<', Carbon::NOW()->subSeconds(2))->delete();
-
+            $pipeline->del(HumanVerification::PREFIX . "." . $uid);
+            $pipeline->srem(HumanVerification::PREFIX . "." . $id, $uid);
         }
-
+        $pipeline->expire(HumanVerification::PREFIX . "." . $uid, $user["whitelist"] ? HumanVerification::EXPIRELONG : HumanVerification::EXPIRESHORT);
+        $pipeline->expire(HumanVerification::PREFIX . "." . $id, HumanVerification::EXPIRELONG);
+        $pipeline->execute();
     }
 
     private static function checkId($request, $id)

app/Http/Controllers/MetaGerSearch.php

@@ -12,7 +12,6 @@ class MetaGerSearch extends Controller
 {
     public function search(Request $request, MetaGer $metager)
     {
-
         $focus = $request->input("focus", "web");
 
         if ($focus === "maps") {

app/Http/Middleware/HumanVerification.php

@@ -3,10 +3,10 @@
 namespace App\Http\Middleware;
 
 use Captcha;
-use Carbon;
 use Closure;
-use DB;
+use Cookie;
 use Illuminate\Http\Response;
+use Illuminate\Support\Facades\Redis;
 use URL;
 
 class HumanVerification
@@ -22,8 +22,9 @@ class HumanVerification
     {
         // The specific user
         $user = null;
-        $newUser = true;
         $update = true;
+        $prefix = "humanverification";
+        $redis = Redis::connection('redisCache');
         try {
             $id = hash("sha512", $request->ip());
             $uid = hash("sha512", $request->ip() . $_SERVER["AGENT"]);
@@ -35,47 +36,61 @@ class HumanVerification
              * If someone that uses a bot finds this out we
              * might have to change it at some point.
              */
-            if ($request->filled('password') || $request->filled('key') || $request->filled('appversion') || !env('BOT_PROTECTION', false)) {
+            if ($request->filled('password') || $request->filled('key') || Cookie::get('key') !== null || $request->filled('appversion') || !env('BOT_PROTECTION', false)) {
                 $update = false;
                 return $next($request);
             }
 
-            $users = DB::select('select * from humanverification where id = ?', [$id]);
+            # Get all Users of this IP
+            $userList = $redis->smembers($prefix . "." . $id);
+            $pipe = $redis->pipeline();
 
+            foreach ($userList as $userid) {
+                $pipe->hgetall($prefix . "." . $userid);
+            }
+
+            $usersData = $pipe->execute();
+
+            $user = [];
+            $users = [];
             # Lock out everyone in a Bot network
             # Find out how many requests this IP has made
             $sum = 0;
-            foreach ($users as $userTmp) {
-                if ($uid == $userTmp->uid) {
-                    $user = ['uid' => $userTmp->uid,
-                        'id' => $userTmp->id,
-                        'unusedResultPages' => intval($userTmp->unusedResultPages),
-                        'whitelist' => filter_var($userTmp->whitelist, FILTER_VALIDATE_BOOLEAN),
-                        'whitelistCounter' => $userTmp->whitelistCounter,
-                        'locked' => filter_var($userTmp->locked, FILTER_VALIDATE_BOOLEAN),
-                        "lockedKey" => $userTmp->lockedKey,
-                        'updated_at' => Carbon::now(),
-                    ];
-                    $newUser = false;
+            foreach ($usersData as $index => $userTmp) {
+                if (empty($userTmp)) {
+                    // This is a key that has been expired and should be deleted
+                    $redis->srem($prefix . "." . $id, $userList[$index]);
+                    continue;
                 }
-                if ($userTmp->whitelist === 0) {
-                    $sum += $userTmp->unusedResultPages;
+                $userNew = ['uid' => $userTmp["uid"],
+                    'id' => $userTmp["id"],
+                    'unusedResultPages' => intval($userTmp["unusedResultPages"]),
+                    'whitelist' => filter_var($userTmp["whitelist"], FILTER_VALIDATE_BOOLEAN),
+                    'locked' => filter_var($userTmp["locked"], FILTER_VALIDATE_BOOLEAN),
+                    "lockedKey" => $userTmp["lockedKey"],
+                ];
+
+                if ($uid === $userTmp["uid"]) {
+                    $user = $userNew;
+                } else {
+                    $users[] = $userNew;
+                }
+                if (!$userNew["whitelist"]) {
+                    $sum += intval($userTmp["unusedResultPages"]);
                 }
 
             }
-            # If this user doesn't have an entry we will create one
 
-            if ($user === null) {
+            # If this user doesn't have an entry we will create one
+            if (empty($user)) {
                 $user =
                     [
                     'uid' => $uid,
                     'id' => $id,
                     'unusedResultPages' => 0,
                     'whitelist' => false,
-                    'whitelistCounter' => 0,
                     'locked' => false,
                     "lockedKey" => "",
-                    'updated_at' => Carbon::now(),
                 ];
             }
 
@@ -96,7 +111,7 @@ class HumanVerification
             // Defines if this is the only user using that IP Adress
             $alone = true;
             foreach ($users as $userTmp) {
-                if ($userTmp->uid != $uid && !$userTmp->whitelist) {
+                if ($userTmp["uid"] != $uid && !$userTmp["whitelist"]) {
                     $alone = false;
                 }
 
@@ -133,41 +148,34 @@ class HumanVerification
                 }
 
             }
-        } catch (\Illuminate\Database\QueryException $e) {
-            // Failure in contacting metager3.de
+        } catch (\Predis\Connection\ConnectionException $e) {
+            $update = false;
         } finally {
             if ($update) {
+
                 // Update the user in the database
-                if ($newUser) {
-                    DB::table('humanverification')->insert(
-                        [
-                            'uid' => $user["uid"],
-                            'id' => $user["id"],
-                            'unusedResultPages' => $user['unusedResultPages'],
-                            'whitelist' => $user["whitelist"],
-                            'whitelistCounter' => $user["whitelistCounter"],
-                            'locked' => $user["locked"],
-                            "lockedKey" => $user["lockedKey"],
-                            'updated_at' => $user["updated_at"],
-                        ]
-                    );
+                $pipeline = $redis->pipeline();
+
+                $pipeline->hmset($prefix . "." . $user['uid'], $user);
+                $pipeline->sadd($prefix . "." . $user["id"], $user["uid"]);
+
+                // Expire in two weeks
+                $expireLong = 60 * 60 * 24 * 14;
+                // Expire in 72h
+                $expireShort = 60 * 60 * 72;
+
+                if ($user["whitelist"]) {
+                    $pipeline->expire($prefix . "." . $user['uid'], $expireLong);
                 } else {
-                    DB::table('humanverification')->where('uid', $uid)->update(
-                        [
-                            'uid' => $user["uid"],
-                            'id' => $user["id"],
-                            'unusedResultPages' => $user['unusedResultPages'],
-                            'whitelist' => $user["whitelist"],
-                            'whitelistCounter' => $user["whitelistCounter"],
-                            'locked' => $user["locked"],
-                            "lockedKey" => $user["lockedKey"],
-                            'updated_at' => $user["updated_at"],
-                        ]
-                    );
+                    $pipeline->expire($prefix . "." . $user['uid'], $expireShort);
                 }
+
+                $pipeline->expire($prefix . "." . $user["id"], $expireLong);
+
+                $pipeline->execute();
             }
-            DB::disconnect('mysql');
         }
+
         $request->request->add(['verification_id' => $user["uid"], 'verification_count' => $user["unusedResultPages"]]);
         return $next($request);
 

resources/lang/de/index.php

@@ -32,7 +32,7 @@ return [
     'slogan.2' => 'Mit MetaGer bewahren Sie einen neutralen Blick auf’s Web!',
     'preredesign' => '<a href="https://klassik.metager.org">Rückblick auf die vorige MetaGer Version</a>',
 
-    'sponsors.head' => 'Sponsoren',
+    'sponsors.head' => 'Partner',
     'sponsors.woxikon' => 'SEO Agentur',
     'sponsors.gutscheine' => 'STERN.de: Günstige Kredite im Kreditvergleich',
     'sponsors.seo' => 'Weihnachtsfeier',

resources/lang/en/index.php

@@ -32,7 +32,7 @@ return [
     'slogan.2' => 'You keep a neutral view on the web by using MetaGer',
     'preredesign' => '<a href="https://klassik.metager.org/en">Flashback to the previous MetaGer version</a>',
 
-    "sponsors.head" => "Sponsors",
+    "sponsors.head" => "Partners",
     'sponsors.woxikon' => 'Tagesgeld jetzt!',
     'sponsors.gutscheine' => 'Aktuelle Gutscheine auf Gutschein-Magazin.de',
     'sponsors.seo' => 'Suchmaschinenoptimierung',

resources/lang/fr/index.php

 <?php
 
 return [
-    "foki.web"          => "Web",
-    "foki.bilder"       => "Images",
-    "foki.nachrichten"  => "News/Politique",
+    "foki.web" => "Web",
+    "foki.bilder" => "Images",
+    "foki.nachrichten" => "News/Politique",
     "foki.wissenschaft" => "Sciènces",
-    "foki.produkte"     => "Produits",
-    "foki.angepasst"    => "ajusté(e)",
-    "foki.maps"         => "Maps.metager.de",
-    "design"            => "Choisir un design personnel",
-    "partnertitle"      => "Soutenir MetaGer, sans frais supplémentaires pour vous",
-    "plugin"            => "Ajouter un plug-in MetaGer",
-    "plugin-title"      => "Ajouter MetaGer à votre browser",
-    "sponsors.head"     => "Sponsors",
+    "foki.produkte" => "Produits",
+    "foki.angepasst" => "ajusté(e)",
+    "foki.maps" => "Maps.metager.de",
+    "design" => "Choisir un design personnel",
+    "partnertitle" => "Soutenir MetaGer, sans frais supplémentaires pour vous",
+    "plugin" => "Ajouter un plug-in MetaGer",
+    "plugin-title" => "Ajouter MetaGer à votre browser",
+    "sponsors.head" => "Partners",
 ];

resources/less/metager/pages/start-page.less

@@ -69,6 +69,7 @@
             margin: 0px;
             max-width: 100%;
             .card-medium;
+            order: 2;
             p {
                 text-align: justify;
             }
@@ -92,6 +93,7 @@
             list-style-type: none;
             padding: 0px;
             text-align: left;
+            margin-bottom: 0;
             li.sr {
                 a {
                     color: inherit;

resources/views/ad-info.blade.php

@@ -6,7 +6,7 @@
 	<h1 class="page-title">Werbung bei MetaGer</h1>
 	<div class="card-heavy">
 		<h2>Warum wir Werbung zeigen</h2>
-		<p>MetaGer ist nicht einfach nur eine Webseite, sondern ein ganzer Service der gepflegt, gewartet und ständig verbessert  werden muss. Dabei entstehen laufende Kosten, die sich derzeit nicht alleine durch Mitgliedsbeiträge und Spenden decken lassen. Deshalb zeigen wir zusätzlich zu unseren Suchergebnissen auch Werbeergebnisse an, die möglichst gut zur Suche passen. Zusätzlich finden sich auf unserer Startseite Sponsorenlinks von ausgewählten Firmen.</p>
+		<p>MetaGer ist nicht einfach nur eine Webseite, sondern ein ganzer Service der gepflegt, gewartet und ständig verbessert  werden muss. Dabei entstehen laufende Kosten, die sich derzeit nicht alleine durch Mitgliedsbeiträge und Spenden decken lassen. Deshalb zeigen wir zusätzlich zu unseren Suchergebnissen auch Werbeergebnisse an, die möglichst gut zur Suche passen. Zusätzlich finden sich auf unserer Startseite Partnerlinks von ausgewählten Firmen.</p>
         <p>MetaGer wird betrieben und stetig weiterentwickelt vom SUMA-EV - Verein für freien Wissenszugang. Der SUMA-EV ist ein als gemeinnützig anerkannter Verein und erhält keine öffentlichen Fördergelder. Deshalb sind wir auf ihre Mithilfe angewiesen.</p>
         {{-- <span>~ Hier grafik mit benötigten Mitgliedern / Spenden für Werbefreiheit einfügen ~</span> --}}
         {{-- <p>Helfen Sie uns jetzt dieses Ziel zu erreichen:<p> --}}