From c5c5c3467ce51ec2b964c237fe49ec3892ad3c51 Mon Sep 17 00:00:00 2001
From: Dominik Hebeler <dominik@suma-ev.de>
Date: Tue, 15 Oct 2019 14:00:23 +0200
Subject: [PATCH] Users will now get a random user agent string instead of a
modified one
---
app/Console/Commands/SaveUseragents.php | 59 ++++++++++++++++
app/Console/Kernel.php | 1 +
app/Http/Kernel.php | 1 +
app/Http/Middleware/UserAgentMaster.php | 67 +++++++++++++++++++
app/UserAgent.php | 10 +++
config/database.php | 6 +-
..._10_15_103139_create_user_agents_table.php | 35 ++++++++++
public/index.php | 10 ---
routes/web.php | 2 +-
9 files changed, 179 insertions(+), 12 deletions(-)
create mode 100644 app/Console/Commands/SaveUseragents.php
create mode 100644 app/Http/Middleware/UserAgentMaster.php
create mode 100644 app/UserAgent.php
create mode 100644 database/migrations/2019_10_15_103139_create_user_agents_table.php
diff --git a/app/Console/Commands/SaveUseragents.php b/app/Console/Commands/SaveUseragents.php
new file mode 100644
index 000000000..a6e82947e
--- /dev/null
+++ b/app/Console/Commands/SaveUseragents.php
@@ -0,0 +1,59 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Carbon\Carbon;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Redis;
+
+class SaveUseragents extends Command
+{
+ /**
+ * The name and signature of the console command.
+ *
+ * @var string
+ */
+ protected $signature = 'requests:useragents';
+
+ /**
+ * The console command description.
+ *
+ * @var string
+ */
+ protected $description = 'Saves a list of recent User-Agents from Redis into a sqlite database';
+
+ /**
+ * Create a new command instance.
+ *
+ * @return void
+ */
+ public function __construct()
+ {
+ parent::__construct();
+ }
+
+ /**
+ * Execute the console command.
+ *
+ * @return mixed
+ */
+ public function handle()
+ {
+ $agents = [];
+ $agent = null;
+ $now = Carbon::now('utc')->toDateTimeString();
+
+ while (($agent = Redis::lpop("useragents")) !== null) {
+ $newEntry = json_decode($agent, true);
+ $newEntry["created_at"] = $now;
+ $newEntry["updated_at"] = $now;
+ $agents[] = $newEntry;
+ }
+
+ \App\UserAgent::insert($agents);
+
+ // Delete old entries (older than 24h)
+ $expiration = Carbon::now('utc')->subDays(1);
+ \App\UserAgent::where('created_at', '<', $expiration)->delete();
+ }
+}
diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php
index 4489d002d..decf0ba0c 100644
--- a/app/Console/Kernel.php
+++ b/app/Console/Kernel.php
@@ -26,6 +26,7 @@ class Kernel extends ConsoleKernel
protected function schedule(Schedule $schedule)
{
$schedule->command('requests:gather')->everyFifteenMinutes();
+ $schedule->command('requests:useragents')->everyFiveMinutes();
$schedule->call(function () {
DB::table('monthlyrequests')->truncate();
diff --git a/app/Http/Kernel.php b/app/Http/Kernel.php
index 1071b8326..c9b5b3406 100644
--- a/app/Http/Kernel.php
+++ b/app/Http/Kernel.php
@@ -60,5 +60,6 @@ class Kernel extends HttpKernel
'throttle' => \Illuminate\Routing\Middleware\ThrottleRequests::class,
'referer.check' => \App\Http\Middleware\RefererCheck::class,
'humanverification' => \App\Http\Middleware\HumanVerification::class,
+ 'useragentmaster' => \App\Http\Middleware\UserAgentMaster::class,
];
}
diff --git a/app/Http/Middleware/UserAgentMaster.php b/app/Http/Middleware/UserAgentMaster.php
new file mode 100644
index 000000000..64b1c75e9
--- /dev/null
+++ b/app/Http/Middleware/UserAgentMaster.php
@@ -0,0 +1,67 @@
+<?php
+
+namespace App\Http\Middleware;
+
+use Closure;
+use Illuminate\Support\Facades\Redis;
+use Jenssegers\Agent\Agent;
+
+class UserAgentMaster
+{
+ /**
+ * This Middleware takes the User-Agent of the user and saves it into a Database
+ * It will also take a random User Agent of a matching device and replace the current User-Agent with it.
+ */
+ /**
+ * Handle an incoming request.
+ *
+ * @param \Illuminate\Http\Request $request
+ * @param \Closure $next
+ * @return mixed
+ */
+ public function handle($request, Closure $next)
+ {
+ /**
+ * Categorize the User-Agents by
+ * 1. Platform (i.e. Ubuntu)
+ * 2. Browser (i.e. Firefox)
+ * 3. Device (desktop|tablet|mobile)
+ */
+ $agent = new Agent();
+ $device = "desktop";
+ if ($agent->isTablet()) {
+ $device = "tablet";
+ } else if ($agent->isPhone()) {
+ $device = "mobile";
+ }
+ // Push an entry to a list in Redis
+ // App\Console\Commands\SaveUseragents.php is called regulary to save the list into a sqlite database
+ Redis::rpush('useragents', json_encode(["platform" => $agent->platform(), "browser" => $agent->browser(), "device" => $device, "useragent" => $_SERVER['HTTP_USER_AGENT']]));
+ Redis::expire('useragents', 301);
+
+ // Try to retrieve a random User-Agent of the same category from the sqlite database
+ $newAgent = \App\UserAgent::where("platform", $agent->platform())
+ ->where("browser", $agent->browser())
+ ->where("device", $device)
+ ->inRandomOrder()
+ ->limit(1)
+ ->get();
+
+ if (sizeof($newAgent) >= 1) {
+ // If there is an Entry in the database use it
+ $newAgent = $newAgent[0]->useragent;
+ } else {
+ // Else anonymize the version of the current User-Agent
+ $agentPieces = explode(" ", $_SERVER['HTTP_USER_AGENT']);
+ for ($i = 0; $i < count($agentPieces); $i++) {
+ $agentPieces[$i] = preg_replace("/(\d+\.\d+)/s", "0.0", $agentPieces[$i]);
+ $agentPieces[$i] = preg_replace("/([^\/]*)\/\w+/s", "$1/0.0", $agentPieces[$i]);
+ }
+ $newAgent = implode(" ", $agentPieces);
+ }
+ // Replace the User-Agent
+ $_SERVER['HTTP_USER_AGENT'] = $newAgent;
+
+ return $next($request);
+ }
+}
diff --git a/app/UserAgent.php b/app/UserAgent.php
new file mode 100644
index 000000000..81f3ee620
--- /dev/null
+++ b/app/UserAgent.php
@@ -0,0 +1,10 @@
+<?php
+
+namespace App;
+
+use Illuminate\Database\Eloquent\Model;
+
+class UserAgent extends Model
+{
+ protected $connection = 'useragents';
+}
diff --git a/config/database.php b/config/database.php
index 891fb9238..23e8c70e9 100644
--- a/config/database.php
+++ b/config/database.php
@@ -51,7 +51,11 @@ return [
'database' => database_path(env('SQLITE_DATABASE', 'database.sqlite')),
'prefix' => '',
],
-
+ 'useragents' => [
+ 'driver' => 'sqlite',
+ 'database' => database_path('useragents.sqlite'),
+ 'prefix' => '',
+ ],
'mysql' => [
'driver' => 'mysql',
'host' => env('DB_HOST', 'localhost'),
diff --git a/database/migrations/2019_10_15_103139_create_user_agents_table.php b/database/migrations/2019_10_15_103139_create_user_agents_table.php
new file mode 100644
index 000000000..bf18a8a0e
--- /dev/null
+++ b/database/migrations/2019_10_15_103139_create_user_agents_table.php
@@ -0,0 +1,35 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateUserAgentsTable extends Migration
+{
+ /**
+ * Run the migrations.
+ *
+ * @return void
+ */
+ public function up()
+ {
+ Schema::connection('useragents')->create('user_agents', function (Blueprint $table) {
+ $table->increments('id');
+ $table->string('platform');
+ $table->string('browser');
+ $table->enum('device', ["desktop", "tablet", "mobile"]);
+ $table->string('useragent', 300);
+ $table->timestamps();
+ });
+ }
+
+ /**
+ * Reverse the migrations.
+ *
+ * @return void
+ */
+ public function down()
+ {
+ Schema::connection('useragents')->dropIfExists('user_agents');
+ }
+}
diff --git a/public/index.php b/public/index.php
index 4334ce537..0db12691b 100644
--- a/public/index.php
+++ b/public/index.php
@@ -15,16 +15,6 @@ if (isset($_SERVER["HTTP_FORWARDED"]) && isset($_SERVER["HTTP_X_FORWARDED_FOR"])
$_SERVER["AGENT"] = $_SERVER["HTTP_USER_AGENT"];
-if (isset($_SERVER['HTTP_USER_AGENT'])) {
- $agentPieces = explode(" ", $_SERVER['HTTP_USER_AGENT']);
-
- for ($i = 0; $i < count($agentPieces); $i++) {
- $agentPieces[$i] = preg_replace("/(\d+\.\d+)/s", "0.0", $agentPieces[$i]);
- $agentPieces[$i] = preg_replace("/([^\/]*)\/\w+/s", "$1/0.0", $agentPieces[$i]);
- }
- $_SERVER['HTTP_USER_AGENT'] = implode(" ", $agentPieces);
-}
-
/*
|--------------------------------------------------------------------------
| Register The Auto Loader
diff --git a/routes/web.php b/routes/web.php
index ffc44c408..90f025731 100644
--- a/routes/web.php
+++ b/routes/web.php
@@ -180,7 +180,7 @@ Route::group(
return redirect(LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), '/'));
});
- Route::match(['get', 'post'], 'meta/meta.ger3', 'MetaGerSearch@search')->middleware('humanverification');
+ Route::match(['get', 'post'], 'meta/meta.ger3', 'MetaGerSearch@search')->middleware('humanverification', 'useragentmaster');
Route::get('meta/loadMore', 'MetaGerSearch@loadMore');
Route::post('img/cat.jpg', 'HumanVerification@remove');
Route::get('r/metager/{mm}/{pw}/{url}', ['as' => 'humanverification', 'uses' => 'HumanVerification@removeGet']);
--
GitLab