Merge branch '901-avoid-more-of-the-yahoo-filtering' into 'development'

Resolve "Avoid more of the Yahoo Filtering"

Closes #901

See merge request !1467

app/Console/Commands/SaveUseragents.php

+<?php
+
+namespace App\Console\Commands;
+
+use Carbon\Carbon;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Redis;
+
+class SaveUseragents extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'requests:useragents';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Saves a list of recent User-Agents from Redis into a sqlite database';
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return mixed
+     */
+    public function handle()
+    {
+        $agents = [];
+        $agent = null;
+        $now = Carbon::now('utc')->toDateTimeString();
+
+        while (($agent = Redis::lpop("useragents")) !== null) {
+            $newEntry = json_decode($agent, true);
+            $newEntry["created_at"] = $now;
+            $newEntry["updated_at"] = $now;
+            $agents[] = $newEntry;
+        }
+
+        \App\UserAgent::insert($agents);
+
+        // Delete old entries (older than 24h)
+        $expiration = Carbon::now('utc')->subDays(1);
+        \App\UserAgent::where('created_at', '<', $expiration)->delete();
+    }
+}

app/Console/Kernel.php

@@ -26,6 +26,7 @@ class Kernel extends ConsoleKernel
     protected function schedule(Schedule $schedule)
     {
         $schedule->command('requests:gather')->everyFifteenMinutes();
+        $schedule->command('requests:useragents')->everyFiveMinutes();
 
         $schedule->call(function () {
             DB::table('monthlyrequests')->truncate();

app/Http/Kernel.php

@@ -60,5 +60,6 @@ class Kernel extends HttpKernel
         'throttle' => \Illuminate\Routing\Middleware\ThrottleRequests::class,
         'referer.check' => \App\Http\Middleware\RefererCheck::class,
         'humanverification' => \App\Http\Middleware\HumanVerification::class,
+        'useragentmaster' => \App\Http\Middleware\UserAgentMaster::class,
     ];
 }

app/Http/Middleware/UserAgentMaster.php

+<?php
+
+namespace App\Http\Middleware;
+
+use Closure;
+use Illuminate\Support\Facades\Redis;
+use Jenssegers\Agent\Agent;
+
+class UserAgentMaster
+{
+    /**
+     * This Middleware takes the User-Agent of the user and saves it into a Database
+     * It will also take a random User Agent of a matching device and replace the current User-Agent with it.
+     */
+    /**
+     * Handle an incoming request.
+     *
+     * @param  \Illuminate\Http\Request  $request
+     * @param  \Closure  $next
+     * @return mixed
+     */
+    public function handle($request, Closure $next)
+    {
+        /**
+         * Categorize the User-Agents by
+         * 1. Platform (i.e. Ubuntu)
+         * 2. Browser (i.e. Firefox)
+         * 3. Device (desktop|tablet|mobile)
+         */
+        $agent = new Agent();
+        $device = "desktop";
+        if ($agent->isTablet()) {
+            $device = "tablet";
+        } else if ($agent->isPhone()) {
+            $device = "mobile";
+        }
+        // Push an entry to a list in Redis
+        // App\Console\Commands\SaveUseragents.php is called regulary to save the list into a sqlite database
+        Redis::rpush('useragents', json_encode(["platform" => $agent->platform(), "browser" => $agent->browser(), "device" => $device, "useragent" => $_SERVER['HTTP_USER_AGENT']]));
+        Redis::expire('useragents', 301);
+
+        // Try to retrieve a random User-Agent of the same category from the sqlite database
+        $newAgent = \App\UserAgent::where("platform", $agent->platform())
+            ->where("browser", $agent->browser())
+            ->where("device", $device)
+            ->inRandomOrder()
+            ->limit(1)
+            ->get();
+
+        if (sizeof($newAgent) >= 1) {
+            // If there is an Entry in the database use it
+            $newAgent = $newAgent[0]->useragent;
+        } else {
+            // Else anonymize the version of the current User-Agent
+            $agentPieces = explode(" ", $_SERVER['HTTP_USER_AGENT']);
+            for ($i = 0; $i < count($agentPieces); $i++) {
+                $agentPieces[$i] = preg_replace("/(\d+\.\d+)/s", "0.0", $agentPieces[$i]);
+                $agentPieces[$i] = preg_replace("/([^\/]*)\/\w+/s", "$1/0.0", $agentPieces[$i]);
+            }
+            $newAgent = implode(" ", $agentPieces);
+        }
+        // Replace the User-Agent
+        $_SERVER['HTTP_USER_AGENT'] = $newAgent;
+
+        return $next($request);
+    }
+}

app/UserAgent.php

+<?php
+
+namespace App;
+
+use Illuminate\Database\Eloquent\Model;
+
+class UserAgent extends Model
+{
+    protected $connection = 'useragents';
+}

config/database.php

@@ -51,7 +51,11 @@ return [
             'database' => database_path(env('SQLITE_DATABASE', 'database.sqlite')),
             'prefix' => '',
         ],
-
+        'useragents' => [
+            'driver' => 'sqlite',
+            'database' => database_path('useragents.sqlite'),
+            'prefix' => '',
+        ],
         'mysql' => [
             'driver' => 'mysql',
             'host' => env('DB_HOST', 'localhost'),

database/migrations/2019_10_15_103139_create_user_agents_table.php

+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateUserAgentsTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::connection('useragents')->create('user_agents', function (Blueprint $table) {
+            $table->increments('id');
+            $table->string('platform');
+            $table->string('browser');
+            $table->enum('device', ["desktop", "tablet", "mobile"]);
+            $table->string('useragent', 300);
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::connection('useragents')->dropIfExists('user_agents');
+    }
+}

public/index.php

@@ -15,16 +15,6 @@ if (isset($_SERVER["HTTP_FORWARDED"]) && isset($_SERVER["HTTP_X_FORWARDED_FOR"])
 
 $_SERVER["AGENT"] = $_SERVER["HTTP_USER_AGENT"];
 
-if (isset($_SERVER['HTTP_USER_AGENT'])) {
-    $agentPieces = explode(" ", $_SERVER['HTTP_USER_AGENT']);
-
-    for ($i = 0; $i < count($agentPieces); $i++) {
-        $agentPieces[$i] = preg_replace("/(\d+\.\d+)/s", "0.0", $agentPieces[$i]);
-        $agentPieces[$i] = preg_replace("/([^\/]*)\/\w+/s", "$1/0.0", $agentPieces[$i]);
-    }
-    $_SERVER['HTTP_USER_AGENT'] = implode(" ", $agentPieces);
-}
-
 /*
 |--------------------------------------------------------------------------
 | Register The Auto Loader

resources/js/keyboardNavigation.js

@@ -7,30 +7,29 @@ $(document).ready(function () {
     // Add entry point for tabbing to the first result
     $('.result, .ad').first().attr("id", "results-entry");
     // Initially focus the searchbar
-    $('div.search-input input').focus();
 });
 
 /**
  * Simulate a click on enter keypress when focused on labels
  */
-$('label').on('keydown', function(e) {
-    if(e.keyCode == '13') {
+$('label').on('keydown', function (e) {
+    if (e.keyCode == '13') {
         $(this).click();
         $('a', this)
     }
-}); 
+});
 
 /**
  * Handles tab keypress and escape keypress
  */
-$(document).on('keydown', function(e) {
+$(document).on('keydown', function (e) {
     e = e || window.event;
     // On first tab keypress there is special behaviour and the ctrlInfo flag is set
-    if(!ctrlInfo && e.keyCode == '9') {
+    if (!ctrlInfo && e.keyCode == '9') {
         focusNavBox();
         e.preventDefault();
         ctrlInfo = true;
-    } else if(e.keyCode == '27') {
+    } else if (e.keyCode == '27') {
         escKeyPressed();
     }
 });

resources/views/parts/searchbar.blade.php

@@ -8,7 +8,7 @@
 					</a>
 				</div>
 				<div class="search-input">
-					<input type="search" name="eingabe" value="@if(isset($eingabe)){{$eingabe}}@endif" required=""  @if(\Request::is('/') && stripos(\Request::header('User-Agent'), "MetaGer App") !== 0) autofocus @endif autocomplete="off" class="form-control" placeholder="{{ trans('index.placeholder') }}" tabindex="0">
+					<input type="search" name="eingabe" value="@if(isset($eingabe)){{$eingabe}}@endif" required=""  @if(\Request::is('/') && !\Request::filled('mgapp') autofocus @endif autocomplete="off" class="form-control" placeholder="{{ trans('index.placeholder') }}" tabindex="0">
 					<button id="search-delete-btn" name="delete-search-input" type="button" tabindex="-1">
 						&#xd7;
 					</button>

routes/web.php

@@ -180,7 +180,7 @@ Route::group(
             return redirect(LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), '/'));
         });
 
-        Route::match(['get', 'post'], 'meta/meta.ger3', 'MetaGerSearch@search')->middleware('humanverification');
+        Route::match(['get', 'post'], 'meta/meta.ger3', 'MetaGerSearch@search')->middleware('humanverification', 'useragentmaster');
         Route::get('meta/loadMore', 'MetaGerSearch@loadMore');
         Route::post('img/cat.jpg', 'HumanVerification@remove');
         Route::get('r/metager/{mm}/{pw}/{url}', ['as' => 'humanverification', 'uses' => 'HumanVerification@removeGet']);