Commit aa72ae3a authored by Dominik Hebeler's avatar Dominik Hebeler

Merge branch '901-avoid-more-of-the-yahoo-filtering' into 'development'

Resolve "Avoid more of the Yahoo Filtering"

Closes #901

See merge request !1467
parents 7be4985b c715408e
<?php
namespace App\Console\Commands;
use Carbon\Carbon;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
class SaveUseragents extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'requests:useragents';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Saves a list of recent User-Agents from Redis into a sqlite database';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$agents = [];
$agent = null;
$now = Carbon::now('utc')->toDateTimeString();
while (($agent = Redis::lpop("useragents")) !== null) {
$newEntry = json_decode($agent, true);
$newEntry["created_at"] = $now;
$newEntry["updated_at"] = $now;
$agents[] = $newEntry;
}
\App\UserAgent::insert($agents);
// Delete old entries (older than 24h)
$expiration = Carbon::now('utc')->subDays(1);
\App\UserAgent::where('created_at', '<', $expiration)->delete();
}
}
......@@ -26,6 +26,7 @@ class Kernel extends ConsoleKernel
protected function schedule(Schedule $schedule)
{
$schedule->command('requests:gather')->everyFifteenMinutes();
$schedule->command('requests:useragents')->everyFiveMinutes();
$schedule->call(function () {
DB::table('monthlyrequests')->truncate();
......
......@@ -60,5 +60,6 @@ class Kernel extends HttpKernel
'throttle' => \Illuminate\Routing\Middleware\ThrottleRequests::class,
'referer.check' => \App\Http\Middleware\RefererCheck::class,
'humanverification' => \App\Http\Middleware\HumanVerification::class,
'useragentmaster' => \App\Http\Middleware\UserAgentMaster::class,
];
}
<?php
namespace App\Http\Middleware;
use Closure;
use Illuminate\Support\Facades\Redis;
use Jenssegers\Agent\Agent;
class UserAgentMaster
{
/**
* This Middleware takes the User-Agent of the user and saves it into a Database
* It will also take a random User Agent of a matching device and replace the current User-Agent with it.
*/
/**
* Handle an incoming request.
*
* @param \Illuminate\Http\Request $request
* @param \Closure $next
* @return mixed
*/
public function handle($request, Closure $next)
{
/**
* Categorize the User-Agents by
* 1. Platform (i.e. Ubuntu)
* 2. Browser (i.e. Firefox)
* 3. Device (desktop|tablet|mobile)
*/
$agent = new Agent();
$device = "desktop";
if ($agent->isTablet()) {
$device = "tablet";
} else if ($agent->isPhone()) {
$device = "mobile";
}
// Push an entry to a list in Redis
// App\Console\Commands\SaveUseragents.php is called regulary to save the list into a sqlite database
Redis::rpush('useragents', json_encode(["platform" => $agent->platform(), "browser" => $agent->browser(), "device" => $device, "useragent" => $_SERVER['HTTP_USER_AGENT']]));
Redis::expire('useragents', 301);
// Try to retrieve a random User-Agent of the same category from the sqlite database
$newAgent = \App\UserAgent::where("platform", $agent->platform())
->where("browser", $agent->browser())
->where("device", $device)
->inRandomOrder()
->limit(1)
->get();
if (sizeof($newAgent) >= 1) {
// If there is an Entry in the database use it
$newAgent = $newAgent[0]->useragent;
} else {
// Else anonymize the version of the current User-Agent
$agentPieces = explode(" ", $_SERVER['HTTP_USER_AGENT']);
for ($i = 0; $i < count($agentPieces); $i++) {
$agentPieces[$i] = preg_replace("/(\d+\.\d+)/s", "0.0", $agentPieces[$i]);
$agentPieces[$i] = preg_replace("/([^\/]*)\/\w+/s", "$1/0.0", $agentPieces[$i]);
}
$newAgent = implode(" ", $agentPieces);
}
// Replace the User-Agent
$_SERVER['HTTP_USER_AGENT'] = $newAgent;
return $next($request);
}
}
<?php
namespace App;
use Illuminate\Database\Eloquent\Model;
class UserAgent extends Model
{
protected $connection = 'useragents';
}
......@@ -51,7 +51,11 @@ return [
'database' => database_path(env('SQLITE_DATABASE', 'database.sqlite')),
'prefix' => '',
],
'useragents' => [
'driver' => 'sqlite',
'database' => database_path('useragents.sqlite'),
'prefix' => '',
],
'mysql' => [
'driver' => 'mysql',
'host' => env('DB_HOST', 'localhost'),
......
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class CreateUserAgentsTable extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::connection('useragents')->create('user_agents', function (Blueprint $table) {
$table->increments('id');
$table->string('platform');
$table->string('browser');
$table->enum('device', ["desktop", "tablet", "mobile"]);
$table->string('useragent', 300);
$table->timestamps();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::connection('useragents')->dropIfExists('user_agents');
}
}
......@@ -15,16 +15,6 @@ if (isset($_SERVER["HTTP_FORWARDED"]) && isset($_SERVER["HTTP_X_FORWARDED_FOR"])
$_SERVER["AGENT"] = $_SERVER["HTTP_USER_AGENT"];
if (isset($_SERVER['HTTP_USER_AGENT'])) {
$agentPieces = explode(" ", $_SERVER['HTTP_USER_AGENT']);
for ($i = 0; $i < count($agentPieces); $i++) {
$agentPieces[$i] = preg_replace("/(\d+\.\d+)/s", "0.0", $agentPieces[$i]);
$agentPieces[$i] = preg_replace("/([^\/]*)\/\w+/s", "$1/0.0", $agentPieces[$i]);
}
$_SERVER['HTTP_USER_AGENT'] = implode(" ", $agentPieces);
}
/*
|--------------------------------------------------------------------------
| Register The Auto Loader
......
......@@ -7,30 +7,29 @@ $(document).ready(function () {
// Add entry point for tabbing to the first result
$('.result, .ad').first().attr("id", "results-entry");
// Initially focus the searchbar
$('div.search-input input').focus();
});
/**
* Simulate a click on enter keypress when focused on labels
*/
$('label').on('keydown', function(e) {
if(e.keyCode == '13') {
$('label').on('keydown', function (e) {
if (e.keyCode == '13') {
$(this).click();
$('a', this)
}
});
});
/**
* Handles tab keypress and escape keypress
*/
$(document).on('keydown', function(e) {
$(document).on('keydown', function (e) {
e = e || window.event;
// On first tab keypress there is special behaviour and the ctrlInfo flag is set
if(!ctrlInfo && e.keyCode == '9') {
if (!ctrlInfo && e.keyCode == '9') {
focusNavBox();
e.preventDefault();
ctrlInfo = true;
} else if(e.keyCode == '27') {
} else if (e.keyCode == '27') {
escKeyPressed();
}
});
......
......@@ -8,7 +8,7 @@
</a>
</div>
<div class="search-input">
<input type="search" name="eingabe" value="@if(isset($eingabe)){{$eingabe}}@endif" required="" @if(\Request::is('/') && stripos(\Request::header('User-Agent'), "MetaGer App") !== 0) autofocus @endif autocomplete="off" class="form-control" placeholder="{{ trans('index.placeholder') }}" tabindex="0">
<input type="search" name="eingabe" value="@if(isset($eingabe)){{$eingabe}}@endif" required="" @if(\Request::is('/') && !\Request::filled('mgapp') autofocus @endif autocomplete="off" class="form-control" placeholder="{{ trans('index.placeholder') }}" tabindex="0">
<button id="search-delete-btn" name="delete-search-input" type="button" tabindex="-1">
&#xd7;
</button>
......
......@@ -180,7 +180,7 @@ Route::group(
return redirect(LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), '/'));
});
Route::match(['get', 'post'], 'meta/meta.ger3', 'MetaGerSearch@search')->middleware('humanverification');
Route::match(['get', 'post'], 'meta/meta.ger3', 'MetaGerSearch@search')->middleware('humanverification', 'useragentmaster');
Route::get('meta/loadMore', 'MetaGerSearch@loadMore');
Route::post('img/cat.jpg', 'HumanVerification@remove');
Route::get('r/metager/{mm}/{pw}/{url}', ['as' => 'humanverification', 'uses' => 'HumanVerification@removeGet']);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment