Skip to content
GitLab
Commit 792462ea authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Merge branch '1225-reduce-server-wait-times-for-bv' into 'development' 

Resolve "Reduce server wait times for BV"

Closes #1225

See merge request !2014
parents 17d9a8da ced89b85
Hide whitespace changes
Inline Side-by-side
metager/app/Console/Commands/AppendLogs.php
View file @ 792462ea
......@@ -2,6 +2,7 @@
namespace App\Console\Commands;
use App\Http\Middleware\BrowserVerification;
use App\QueryLogger;
use Illuminate\Console\Command;
......@@ -44,5 +45,6 @@ class AppendLogs extends Command
private function handleMGLogs()
{
QueryLogger::flushLogs();
BrowserVerification::FLUSH_LOGS();
}
}
}
\ No newline at end of file
metager/app/Http/Controllers/HumanVerification.php
View file @ 792462ea
......@@ -87,7 +87,7 @@ class HumanVerification extends Controller
$lockedKey = $request->post("c", "");
$rules = ['captcha' => 'required|captcha_api:' . $lockedKey . ',math'];
$rules = ['captcha' => 'required|captcha_api:' . $lockedKey . ',math'];
$validator = validator()->make(request()->all(), $rules);
if (empty($lockedKey) || $validator->fails() || !$request->has("key") || !Cache::has($request->input("key"))) {
......@@ -134,7 +134,7 @@ class HumanVerification extends Controller
self::logCaptchaSolve($query, $time, $request->has("dnaa"));
$params['token'] = $token; // Overwrite if exists
$params['token'] = $token; // Overwrite if exists
// Note that this will url_encode all values
$url_parts['query'] = http_build_query($params);
......@@ -366,10 +366,7 @@ class HumanVerification extends Controller
Cache::put($key, $bvData, now()->addMinutes(self::BV_DATA_EXPIRATION_MINUTES));
});
return response()->file(\public_path("img/1px.png", ["Content-Type" => "image/png"]));
return response()->file(\public_path("img/1px.png"), ["Content-Type" => "image/png"]);
}
public function verificationCSP(Request $request, string $mgv)
......@@ -431,4 +428,4 @@ class HumanVerification extends Controller
Cache::put($mgv, $bvData, now()->addMinutes(self::BV_DATA_EXPIRATION_MINUTES));
});
}
}
}
\ No newline at end of file
metager/app/Http/Middleware/BrowserVerification.php
View file @ 792462ea
......@@ -2,7 +2,9 @@
namespace App\Http\Middleware;
use App\Http\Controllers\HumanVerification;
use Closure;
use Illuminate\Support\Facades\Redis;
use Jenssegers\Agent\Agent;
use Illuminate\Http\Request;
use App\QueryTimer;
......@@ -12,6 +14,7 @@ use Response;
class BrowserVerification
{
const LOG_KEY = "bv_logs";
/**
* Handle an incoming request.
......@@ -24,8 +27,6 @@ class BrowserVerification
{
\app()->make(QueryTimer::class)->observeStart(self::class);
$this->verification_key = \hash("sha512", $request->ip() . $_SERVER["AGENT"]);
$bvEnabled = config("metager.metager.browserverification_enabled");
if (empty($bvEnabled) || !$bvEnabled) {
\app()->make(QueryTimer::class)->observeEnd(self::class);
......@@ -41,26 +42,18 @@ class BrowserVerification
}
}
if (\in_array($request->input("out", ""), ["api", "atom10", "rss20"]) && (app('App\Models\Key')->getStatus() || ($request->filled("key") && $request->input('key') === config("metager.metager.keys.uni_mainz")))) {
header('Content-type: application/xml; charset=utf-8');
} elseif (($request->input("out", "") === "api" || $request->input("out", "") === "atom10") && !app('App\Models\Key')->getStatus()) {
if (($request->input("out", "") === "api" || $request->input("out", "") === "atom10") && !app('App\Models\Key')->getStatus()) {
\app()->make(QueryTimer::class)->observeEnd(self::class);
self::logBrowserverification($request);
abort(403);
} else {
header('Content-type: text/html; charset=utf-8');
}
header('X-Accel-Buffering: no');
ini_set('zlib.output_compression', 'Off');
ini_set('output_buffering', 'Off');
ini_set('output_handler', '');
ob_end_clean();
//use parameter for middleware to skip this when using associator
if (($request->filled("loadMore") && Cache::has($request->input("loadMore"))) || app('App\Models\Key')->getStatus() ||
if (
($request->filled("loadMore") && Cache::has($request->input("loadMore"))) || app('App\Models\Key')->getStatus() ||
($request->filled("key") && $request->input('key') === config("metager.metager.keys.uni_mainz"))
) {
\app()->make(QueryTimer::class)->observeEnd(self::class);
\app()->make(SearchSettings::class)->header_printed = false;
return $next($request);
}
......@@ -69,114 +62,114 @@ class BrowserVerification
// Verify that key is a md5 checksum
if (!preg_match("/^[a-f0-9]{32}$/", $key)) {
\app()->make(QueryTimer::class)->observeEnd(self::class);
self::logBrowserverification($request);
abort(404);
}
$js_enabled = false;
if ($request->filled("js") && $request->input("js") === "true") {
$js_enabled = true;
$bvData = Cache::get($key);
if ($bvData === null) {
// Key does not exist if this is called in a frame abort. if not redirect to the page without mgv parameter
// Check if request header "Sec-Fetch-Dest" is set
$framed = false;
if ($request->header("Sec-Fetch-Dest") === "iframe") {
$framed = true;
} elseif ($request->input("iframe", "0") === "1") {
$framed = true;
}
if ($framed) {
self::logBrowserverification($request);
abort(429);
} else {
$params = $request->all();
unset($params["mgv"]);
return redirect(route($route, $params));
}
}
$bv_result = $this->waitForBV($key, false, $js_enabled);
if ($bv_result) {
\app()->make(SearchSettings::class)->header_printed = false;
\app()->make(QueryTimer::class)->observeEnd(self::class);
return $next($request);
} elseif ($bv_result === null) {
$params = request()->except("mgv");
$url = route("resultpage", $params);
self::logBrowserverification($request);
return redirect($url);
} else {
self::logBrowserverification($request);
return redirect(url("/"));
// The css key has to be present in order to continue
if (!array_key_exists("css", $bvData)) {
if (sizeof($bvData["tries"]) < 5) {
$time_since_last_try = now()->diffInMilliseconds($bvData["tries"][sizeof($bvData["tries"]) - 1]);
// Redirect the user to make him refresh (up to 5 times)
Cache::lock($key . "_lock", 10)->block(5, function () use ($key) {
$bvData = Cache::get($key);
if ($bvData === null) {
$bvData = [];
}
$bvData["tries"][] = now();
Cache::put($key, $bvData, now()->addMinutes(HumanVerification::BV_DATA_EXPIRATION_MINUTES));
});
if ($time_since_last_try < 100) {
// Make sure there are at least 100ms between each try
usleep((100 - $time_since_last_try) * 1000);
}
$params = $request->all();
$params["mgv"] = $key;
$url = route($route, $params);
\app()->make(QueryTimer::class)->observeEnd(self::class);
return redirect($url);
} else {
\app()->make(QueryTimer::class)->observeEnd(self::class);
self::logBrowserverification($request);
abort(429);
}
}
}
$key = md5($request->ip() . microtime(true));
Cache::put($key, [
"start" => now()
], now()->addMinutes(30));
// CSS/JS Data is loaded we need to wait for CSP
$csp_result = $this->waitForCSP($bvData, $key);
$report_to = route("csp_verification", ["mgv" => $key]);
return response()->stream(function () use ($next, $request, $route, $key) {
echo (view('layouts.resultpage.verificationHeader')->with('key', $key)->render());
flush();
// Save state in Search settings
$search_settings = \app()->make(SearchSettings::class);
$search_settings->bv_key = $key;
if ($this->supportsInlineVerification() && $this->waitForBV($key, true, null)) {
echo (view('layouts.resultpage.resources')->render());
flush();
\app()->make(QueryTimer::class)->observeEnd(self::class);
\app()->make(SearchSettings::class)->header_printed = true;
$next($request);
return;
// Check if Javascript was loaded
// No after waiting load a fresh copy of bvData
$bvData = Cache::get($key);
if (array_key_exists("js", $bvData)) {
$search_settings->javascript_enabled = true;
}
\app()->make(QueryTimer::class)->observeEnd(self::class);
return $next($request);
} else {
// The verification key
$key = md5($request->ip() . microtime(true));
Cache::put($key, [
"start" => now(),
"tries" => [
now()
]
], now()->addMinutes(30));
$report_to = route("csp_verification", ["mgv" => $key]);
$params = $request->all();
$params["mgv"] = $key;
$url = route($route, $params);
echo (view('layouts.resultpage.unverifiedResultPage')
->with('url', $url)
->with('mgv', $key)
->render());
flush();
$js_url = route($route, $params);
$params["iframe"] = "1";
$frame_url = route($route, $params);
\app()->make(QueryTimer::class)->observeEnd(self::class);
}, 200, ["Content-Security-Policy" => "default-src 'self'; script-src 'self' 'nonce-$key'; script-src-elem 'self' 'nonce-$key'; script-src-attr 'self'; style-src 'self'; style-src-elem 'self'; style-src-attr 'self'; img-src 'self' data:; font-src 'self'; connect-src 'self'; frame-src 'self'; frame-ancestors 'self' https://scripts.zdv.uni-mainz.de; form-action 'self' www.paypal.com; report-uri " . $report_to . "; report_to " . $report_to]);
return response(
view('layouts.resultpage.framedResultPage', ["frame_url" => $frame_url, "js_url" => $js_url, "mgv" => $key]),
200,
["Content-Security-Policy" => "default-src 'self'; script-src 'self' 'nonce-$key'; script-src-elem 'self' 'nonce-$key'; script-src-attr 'self'; style-src 'self' 'nonce-$key'; style-src-elem 'self' 'nonce-$key'; style-src-attr 'self' 'nonce-$key'; img-src 'self' data:; font-src 'self'; connect-src 'self'; frame-src 'self'; frame-ancestors 'self' https://scripts.zdv.uni-mainz.de; form-action 'self' www.paypal.com; report-uri " . $report_to . "; report_to " . $report_to]
);
}
}
private function waitForBV($key, $inline = false, $js_enabled = false)
private function waitForCSP(&$bvData, $key)
{
$bvData = null;
$max_wait_time_ms = 5000;
if ($inline) {
$max_wait_time_ms = 2000;
}
$wait_time_ms = 250;
$wait_start = now();
$wait_start = $bvData["start"];
$css_loaded = false;
$js_loaded = false;
$csp_loaded = null;
$search_settings = \app()->make(SearchSettings::class);
$search_settings->bv_key = $key;
do {
// Calculate Sleep Time
// Sleeptime gradually increases with the current wait time
// Min 10ms and max 1s
$sleep_time_milliseconds = round(now()->diffInMilliseconds($wait_start) / 10);
$sleep_time_milliseconds = max(10, $sleep_time_milliseconds);
$sleep_time_milliseconds = min(1000, $sleep_time_milliseconds);
usleep($sleep_time_milliseconds * 1000);
$bvData = Cache::get($key);
if ($bvData === null) {
return null;
}
if ($css_loaded !== true) {
if (\array_key_exists("css", $bvData) && \array_key_exists("loaded", $bvData["css"])) {
$css_loaded = true;
}
}
if ($js_loaded !== true) {
if (\array_key_exists("js", $bvData) && \array_key_exists("loaded", $bvData["js"])) {
$js_loaded = true;
$search_settings = \app()->make(SearchSettings::class);
$search_settings->javascript_enabled = true;
} elseif ($css_loaded && $js_enabled && now()->diffInSeconds($bvData["css"]["loaded"]) >= 10) {
$js_loaded = true;
} elseif ($js_enabled === false) {
$js_loaded = true;
} elseif ($js_enabled === null && $css_loaded && now()->diffInMilliseconds($bvData["css"]["loaded"]) > $wait_time_ms) {
break;
}
}
if ($csp_loaded !== true) {
if (\array_key_exists("csp", $bvData) && \array_key_exists("loaded", $bvData["csp"])) {
if (now()->diffInMilliseconds($bvData["csp"]["loaded"]) > $wait_time_ms) {
......@@ -184,86 +177,76 @@ class BrowserVerification
} else {
$csp_loaded = false;
}
} elseif ($css_loaded && $js_loaded && $csp_loaded !== false) {
} else {
// If css and javascript is both loaded we will wait a few more moments
/** @var \Carbon\Carbon $stop_waiting_for_csp */
$stop_waiting_for_csp = $bvData["css"]["loaded"]->addMilliseconds($wait_time_ms);
if (\array_key_exists("js", $bvData) && \array_key_exists("loaded", $bvData["js"])) {
$diff_css_js_milliseconds = $bvData["css"]["loaded"]->diffInMilliseconds($bvData["js"]["loaded"]);
$stop_waiting_for_csp = $bvData["css"]["loaded"]->addMilliseconds($diff_css_js_milliseconds * 2);
/** @var \Carbon\Carbon $stop_waiting_for_csp_js */
$stop_waiting_for_csp_js = $bvData["css"]["loaded"]->addMilliseconds($wait_time_ms);
if ($stop_waiting_for_csp_js->isAfter($stop_waiting_for_csp)) {
$stop_waiting_for_csp = $stop_waiting_for_csp_js;
}
}
if (now() > $stop_waiting_for_csp) {
if (now()->isAfter($stop_waiting_for_csp)) {
$csp_loaded = true;
}
}
}
if (
$css_loaded &&
$js_loaded &&
$csp_loaded
) {
if ($csp_loaded) {
return true;
}
// Calculate Sleep Time
// Sleeptime gradually increases with the current wait time
// Min 10ms and max 1s
$sleep_time_milliseconds = round(now()->diffInMilliseconds($wait_start) / 10);
$sleep_time_milliseconds = max(10, $sleep_time_milliseconds);
$sleep_time_milliseconds = min(1000, $sleep_time_milliseconds);
usleep($sleep_time_milliseconds * 1000);
} while (now()->diffInMilliseconds($wait_start) < $max_wait_time_ms);
return false;
}
private function supportsInlineVerification()
{
$agent = new Agent();
$agent->setUserAgent($_SERVER["AGENT"]);
$browser = $agent->browser();
$version = $agent->version($browser);
// IE and Opera doesn't work at all
if ($browser === "IE") {
return false;
}
// Edge Browser up to and including version 16 doesn't support it
if ($browser === "Edge" && \version_compare($version, 17) === -1) {
return false;
}
// Safari Browser up to and including version 7 doesn't support it
if ($browser === "Safari" && \version_compare($version, 8) === -1) {
return false;
}
return true;
}
public static function logBrowserverification(Request $request)
{
$log = [
now()->format("Y-m-d H:i:s"),
$request->input("eingabe"),
"js=" . \app()->make(SearchSettings::class)->javascript_enabled,
];
$file_path = \storage_path("logs/metager/bv_fail.csv");
$fh = fopen($file_path, "a");
try {
\fputcsv($fh, $log);
} finally {
fclose($fh);
}
Redis::rpush(self::LOG_KEY, $log);
}
public static function logCSP()
public static function FLUSH_LOGS()
{
$request = request();
$log = [
now()->format("Y-m-d H:i:s"),
$request->input("eingabe"),
"ua=" . $_SERVER["AGENT"],
];
$file_path = \storage_path("logs/metager/csp_fail.csv");
$max_entries = 250;
$file_path = \storage_path("logs/metager/bv_fail.csv");
$fh = fopen($file_path, "a");
$logs = [];
try {
\fputcsv($fh, $log);
while (true) {
$entry = Redis::lpop(self::LOG_KEY);
if (!empty($entry)) {
$logs[] = $entry;
}
if (sizeof($logs) >= $max_entries || empty($entry)) {
if (sizeof($logs) > 0) {
foreach ($logs as $log) {
\fputcsv($fh, $log);
$logs = [];
}
}
if (empty($entry)) {
break;
}
}
}
} finally {
fclose($fh);
}
}
}
}
\ No newline at end of file
metager/app/MetaGer.php
View file @ 792462ea
......@@ -62,6 +62,7 @@ class MetaGer
# Konfigurationseinstellungen:
protected $sumaFile;
protected $mobile;
protected $framed;
protected $resultCount;
protected $sprueche;
protected $newtab;
......@@ -300,7 +301,8 @@ class MetaGer
# Validate Advertisements
$newResults = [];
foreach ($this->ads as $ad) {
if (($ad->strippedHost !== "" && (in_array($ad->strippedHost, $this->adDomainsBlacklisted) ||
if (
($ad->strippedHost !== "" && (in_array($ad->strippedHost, $this->adDomainsBlacklisted) ||
in_array($ad->strippedLink, $this->adUrlsBlacklisted))) || ($ad->strippedHostAnzeige !== "" && (in_array($ad->strippedHostAnzeige, $this->adDomainsBlacklisted) ||
in_array($ad->strippedLinkAnzeige, $this->adUrlsBlacklisted)))
) {
......@@ -403,7 +405,7 @@ class MetaGer
$arr[$link]->changed = true;
}
} else {
$arr[$link] = &$this->results[$i];
$arr[$link] = & $this->results[$i];
}
}
}
......@@ -448,7 +450,7 @@ class MetaGer
* every so often. ~33% in this case
* ToDo set back to 5 once we do not want to advertise donations as much anymore
*/
if (/*sizeof($this->ads) === 0 &&*/rand(1, 100) >= 34) {
if ( /*sizeof($this->ads) === 0 &&*/rand(1, 100) >= 34) {
return;
}
......@@ -488,7 +490,7 @@ class MetaGer
$url = route('humanverification', $params);
$proxyPw = md5($day . $result->proxyLink . config("metager.metager.proxy.password"));
$params["pw"] = $proxyPw;
$params["url"] = \bin2hex($result->proxyLink);
$params["url"] = \bin2hex($result->proxyLink);
$proxyUrl = route('humanverification', $params);
$result->link = $url;
$result->proxyLink = $proxyUrl;
......@@ -579,7 +581,7 @@ class MetaGer
# and no other search engine can provide this filter
$validTmp = false;
foreach ($this->parameterFilter as $filterName => $filter) {
if (count((array)$filter->sumas) === 1 && !empty($filter->sumas->{$sumaName})) {
if (count((array) $filter->sumas) === 1 && !empty($filter->sumas->{$sumaName})) {
$validTmp = true;
break;
}
......@@ -602,7 +604,7 @@ class MetaGer
# Special case if search engines are disabled
# Since bing is normally only active if a filter is set but it should be active, too if yahoo is disabled
if ($this->getFokus() === "web" && empty($this->enabledSearchengines["yahoo"]) && Cookie::get("web_engine_bing") !== "off" && isset($this->sumaFile->sumas->{"bing"})) {
if ($this->getFokus() === "web" && empty($this->enabledSearchengines["yahoo"]) && Cookie::get("web_engine_bing") !== "off" && isset($this->sumaFile->sumas->{"bing"})) {
$this->enabledSearchengines["bing"] = $this->sumaFile->sumas->{"bing"};
}
......@@ -764,7 +766,8 @@ class MetaGer
if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"} && Cookie::get($this->getFokus() . "_engine_" . $suma) !== "off") {
if (!empty($filter->sumas->{$suma})) {
# If the searchengine is disabled this filter shouldn't be available
if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
if (
(!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
|| (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)
) {
continue;
......@@ -851,14 +854,14 @@ class MetaGer
public function sumaIsOverture($suma)
{
return
$suma["name"]->__toString() === "overture"
$suma["name"]->__toString() === "overture"
|| $suma["name"]->__toString() === "overtureAds";
}
public function sumaIsNotAdsuche($suma)
{
return
$suma["name"]->__toString() !== "qualigo"
$suma["name"]->__toString() !== "qualigo"
&& $suma["name"]->__toString() !== "similar_product_ads"
&& $suma["name"]->__toString() !== "overtureAds";
}
......@@ -998,11 +1001,16 @@ class MetaGer
$this->out = $request->input("out", "");
if ($request->filled("mgv") || $request->input("out", "") === "results-with-style") {
// Check if request header "Sec-Fetch-Dest" is set
$this->framed = false;
if ($request->header("Sec-Fetch-Dest") === "iframe") {
$this->framed = true;
} elseif ($request->input("out", "") === "results-with-style") {
$this->framed = true;
} elseif ($request->input("iframe", "false") === "true") {
$this->framed = true;
} else {
$this->framed = false;
}
unset($request["iframe"]);
# IP
$this->ip = $this->anonymizeIp($request->ip());
......@@ -1269,7 +1277,8 @@ class MetaGer
$usedParameters[$filter->{"get-parameter"}] = true;
}
if (($request->filled($filter->{"get-parameter"}) && $request->input($filter->{"get-parameter"}) !== "off") ||
if (
($request->filled($filter->{"get-parameter"}) && $request->input($filter->{"get-parameter"}) !== "off") ||
Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"}) !== null
) { # If the filter is set via Cookie
$this->parameterFilter[$filterName] = $filter;
......@@ -1951,4 +1960,4 @@ class MetaGer
{
$this->dummy = $dummy;
}
}
}
\ No newline at end of file
metager/app/SearchSettings.php
View file @ 792462ea
......@@ -7,9 +7,8 @@ class SearchSettings
public $bv_key = null; // Cache Key where data of BV is temporarily stored
public $javascript_enabled = false;
public $header_printed = false;
public function __construct()
{
}
}
}
\ No newline at end of file
metager/config/metager/metager.php
View file @ 792462ea
......@@ -3,7 +3,6 @@
return [
"browserverification_enabled" => true,
"browserverification_whitelist" => [
"w3m\/",
],
"admin" => [
"user" => env("ADMIN_USER", "admin"),
......@@ -73,4 +72,4 @@ return [
"selenium" => [
"host" => env("SELENIUM_HOST", "localhost"),
],
];
];
\ No newline at end of file
metager/resources/js/verify.js
View file @ 792462ea
require('es6-promise').polyfill();
require('fetch-ie8');
require("es6-promise").polyfill();
require("fetch-ie8");