Skip to content
GitLab
Commit 7f4d8175 authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

do not block while waiting for bv

parent 17d9a8da
Hide whitespace changes
Inline Side-by-side
metager/app/Http/Controllers/HumanVerification.php
View file @ 7f4d8175
......@@ -87,7 +87,7 @@ class HumanVerification extends Controller
$lockedKey = $request->post("c", "");
$rules = ['captcha' => 'required|captcha_api:' . $lockedKey . ',math'];
$rules = ['captcha' => 'required|captcha_api:' . $lockedKey . ',math'];
$validator = validator()->make(request()->all(), $rules);
if (empty($lockedKey) || $validator->fails() || !$request->has("key") || !Cache::has($request->input("key"))) {
......@@ -134,7 +134,7 @@ class HumanVerification extends Controller
self::logCaptchaSolve($query, $time, $request->has("dnaa"));
$params['token'] = $token; // Overwrite if exists
$params['token'] = $token; // Overwrite if exists
// Note that this will url_encode all values
$url_parts['query'] = http_build_query($params);
......@@ -366,10 +366,7 @@ class HumanVerification extends Controller
Cache::put($key, $bvData, now()->addMinutes(self::BV_DATA_EXPIRATION_MINUTES));
});
return response()->file(\public_path("img/1px.png", ["Content-Type" => "image/png"]));
return response()->file(\public_path("img/1px.png"), ["Content-Type" => "image/png"]);
}
public function verificationCSP(Request $request, string $mgv)
......@@ -431,4 +428,4 @@ class HumanVerification extends Controller
Cache::put($mgv, $bvData, now()->addMinutes(self::BV_DATA_EXPIRATION_MINUTES));
});
}
}
}
\ No newline at end of file
metager/app/Http/Middleware/BrowserVerification.php
View file @ 7f4d8175
......@@ -2,6 +2,7 @@
namespace App\Http\Middleware;
use App\Http\Controllers\HumanVerification;
use Closure;
use Jenssegers\Agent\Agent;
use Illuminate\Http\Request;
......@@ -24,8 +25,6 @@ class BrowserVerification
{
\app()->make(QueryTimer::class)->observeStart(self::class);
$this->verification_key = \hash("sha512", $request->ip() . $_SERVER["AGENT"]);
$bvEnabled = config("metager.metager.browserverification_enabled");
if (empty($bvEnabled) || !$bvEnabled) {
\app()->make(QueryTimer::class)->observeEnd(self::class);
......@@ -41,26 +40,17 @@ class BrowserVerification
}
}
if (\in_array($request->input("out", ""), ["api", "atom10", "rss20"]) && (app('App\Models\Key')->getStatus() || ($request->filled("key") && $request->input('key') === config("metager.metager.keys.uni_mainz")))) {
header('Content-type: application/xml; charset=utf-8');
} elseif (($request->input("out", "") === "api" || $request->input("out", "") === "atom10") && !app('App\Models\Key')->getStatus()) {
if (($request->input("out", "") === "api" || $request->input("out", "") === "atom10") && !app('App\Models\Key')->getStatus()) {
\app()->make(QueryTimer::class)->observeEnd(self::class);
abort(403);
} else {
header('Content-type: text/html; charset=utf-8');
}
header('X-Accel-Buffering: no');
ini_set('zlib.output_compression', 'Off');
ini_set('output_buffering', 'Off');
ini_set('output_handler', '');
ob_end_clean();
//use parameter for middleware to skip this when using associator
if (($request->filled("loadMore") && Cache::has($request->input("loadMore"))) || app('App\Models\Key')->getStatus() ||
if (
($request->filled("loadMore") && Cache::has($request->input("loadMore"))) || app('App\Models\Key')->getStatus() ||
($request->filled("key") && $request->input('key') === config("metager.metager.keys.uni_mainz"))
) {
\app()->make(QueryTimer::class)->observeEnd(self::class);
\app()->make(SearchSettings::class)->header_printed = false;
return $next($request);
}
......@@ -71,112 +61,111 @@ class BrowserVerification
\app()->make(QueryTimer::class)->observeEnd(self::class);
abort(404);
}
$js_enabled = false;
if ($request->filled("js") && $request->input("js") === "true") {
$js_enabled = true;
$bvData = Cache::get($key);
if ($bvData === null) {
// Key does not exist if this is called in a frame abort. if not redirect to the page without mgv parameter
// Check if request header "Sec-Fetch-Dest" is set
$framed = false;
if ($request->header("Sec-Fetch-Dest") === "iframe") {
$framed = true;
} elseif ($request->input("iframe", "0") === "1") {
$framed = true;
}
if ($framed) {
self::logBrowserverification($request);
abort(429);
} else {
$params = $request->all();
unset($params["mgv"]);
return redirect(route($route, $params));
}
}
$bv_result = $this->waitForBV($key, false, $js_enabled);
if ($bv_result) {
\app()->make(SearchSettings::class)->header_printed = false;
\app()->make(QueryTimer::class)->observeEnd(self::class);
return $next($request);
} elseif ($bv_result === null) {
$params = request()->except("mgv");
$url = route("resultpage", $params);
self::logBrowserverification($request);
return redirect($url);
} else {
self::logBrowserverification($request);
return redirect(url("/"));
// The css key has to be present in order to continue
if (!array_key_exists("css", $bvData)) {
if (sizeof($bvData["tries"]) < 5) {
$time_since_last_try = now()->diffInMilliseconds($bvData["tries"][sizeof($bvData["tries"]) - 1]);
// Redirect the user to make him refresh (up to 5 times)
Cache::lock($key . "_lock", 10)->block(5, function () use ($key) {
$bvData = Cache::get($key);
if ($bvData === null) {
$bvData = [];
}
$bvData["tries"][] = now();
Cache::put($key, $bvData, now()->addMinutes(HumanVerification::BV_DATA_EXPIRATION_MINUTES));
});
if ($time_since_last_try < 100) {
// Make sure there are at least 100ms between each try
usleep((100 - $time_since_last_try) * 1000);
}
$params = $request->all();
$params["mgv"] = $key;
$url = route($route, $params);
\app()->make(QueryTimer::class)->observeEnd(self::class);
return redirect($url);
} else {
\app()->make(QueryTimer::class)->observeEnd(self::class);
self::logBrowserverification($request);
abort(429);
}
}
}
$key = md5($request->ip() . microtime(true));
Cache::put($key, [
"start" => now()
], now()->addMinutes(30));
// CSS/JS Data is loaded we need to wait for CSP
$csp_result = $this->waitForCSP($bvData, $key);
$report_to = route("csp_verification", ["mgv" => $key]);
return response()->stream(function () use ($next, $request, $route, $key) {
echo (view('layouts.resultpage.verificationHeader')->with('key', $key)->render());
flush();
// Save state in Search settings
$search_settings = \app()->make(SearchSettings::class);
$search_settings->bv_key = $key;
if ($this->supportsInlineVerification() && $this->waitForBV($key, true, null)) {
echo (view('layouts.resultpage.resources')->render());
flush();
\app()->make(QueryTimer::class)->observeEnd(self::class);
\app()->make(SearchSettings::class)->header_printed = true;
$next($request);
return;
// Check if Javascript was loaded
// No after waiting load a fresh copy of bvData
$bvData = Cache::get($key);
if (array_key_exists("js", $bvData)) {
$search_settings->javascript_enabled = true;
}
\app()->make(QueryTimer::class)->observeEnd(self::class);
return $next($request);
} else {
// The verification key
$key = md5($request->ip() . microtime(true));
Cache::put($key, [
"start" => now(),
"tries" => [
now()
]
], now()->addMinutes(30));
$report_to = route("csp_verification", ["mgv" => $key]);
$params = $request->all();
$params["mgv"] = $key;
$url = route($route, $params);
echo (view('layouts.resultpage.unverifiedResultPage')
->with('url', $url)
->with('mgv', $key)
->render());
flush();
$js_url = route($route, $params);
$params["iframe"] = "1";
$frame_url = route($route, $params);
\app()->make(QueryTimer::class)->observeEnd(self::class);
}, 200, ["Content-Security-Policy" => "default-src 'self'; script-src 'self' 'nonce-$key'; script-src-elem 'self' 'nonce-$key'; script-src-attr 'self'; style-src 'self'; style-src-elem 'self'; style-src-attr 'self'; img-src 'self' data:; font-src 'self'; connect-src 'self'; frame-src 'self'; frame-ancestors 'self' https://scripts.zdv.uni-mainz.de; form-action 'self' www.paypal.com; report-uri " . $report_to . "; report_to " . $report_to]);
return response(
view('layouts.resultpage.framedResultPage', ["frame_url" => $frame_url, "js_url" => $js_url, "mgv" => $key]),
200,
["Content-Security-Policy" => "default-src 'self'; script-src 'self' 'nonce-$key'; script-src-elem 'self' 'nonce-$key'; script-src-attr 'self'; style-src 'self' 'nonce-$key'; style-src-elem 'self' 'nonce-$key'; style-src-attr 'self' 'nonce-$key'; img-src 'self' data:; font-src 'self'; connect-src 'self'; frame-src 'self'; frame-ancestors 'self' https://scripts.zdv.uni-mainz.de; form-action 'self' www.paypal.com; report-uri " . $report_to . "; report_to " . $report_to]
);
}
}
private function waitForBV($key, $inline = false, $js_enabled = false)
private function waitForCSP(&$bvData, $key)
{
$bvData = null;
$max_wait_time_ms = 5000;
if ($inline) {
$max_wait_time_ms = 2000;
}
$wait_time_ms = 250;
$wait_start = now();
$wait_start = $bvData["start"];
$css_loaded = false;
$js_loaded = false;
$csp_loaded = null;
$search_settings = \app()->make(SearchSettings::class);
$search_settings->bv_key = $key;
do {
// Calculate Sleep Time
// Sleeptime gradually increases with the current wait time
// Min 10ms and max 1s
$sleep_time_milliseconds = round(now()->diffInMilliseconds($wait_start) / 10);
$sleep_time_milliseconds = max(10, $sleep_time_milliseconds);
$sleep_time_milliseconds = min(1000, $sleep_time_milliseconds);
usleep($sleep_time_milliseconds * 1000);
$bvData = Cache::get($key);
if ($bvData === null) {
return null;
}
if ($css_loaded !== true) {
if (\array_key_exists("css", $bvData) && \array_key_exists("loaded", $bvData["css"])) {
$css_loaded = true;
}
}
if ($js_loaded !== true) {
if (\array_key_exists("js", $bvData) && \array_key_exists("loaded", $bvData["js"])) {
$js_loaded = true;
$search_settings = \app()->make(SearchSettings::class);
$search_settings->javascript_enabled = true;
} elseif ($css_loaded && $js_enabled && now()->diffInSeconds($bvData["css"]["loaded"]) >= 10) {
$js_loaded = true;
} elseif ($js_enabled === false) {
$js_loaded = true;
} elseif ($js_enabled === null && $css_loaded && now()->diffInMilliseconds($bvData["css"]["loaded"]) > $wait_time_ms) {
break;
}
}
if ($csp_loaded !== true) {
if (\array_key_exists("csp", $bvData) && \array_key_exists("loaded", $bvData["csp"])) {
if (now()->diffInMilliseconds($bvData["csp"]["loaded"]) > $wait_time_ms) {
......@@ -184,26 +173,33 @@ class BrowserVerification
} else {
$csp_loaded = false;
}
} elseif ($css_loaded && $js_loaded && $csp_loaded !== false) {
} else {
// If css and javascript is both loaded we will wait a few more moments
/** @var \Carbon\Carbon $stop_waiting_for_csp */
$stop_waiting_for_csp = $bvData["css"]["loaded"]->addMilliseconds($wait_time_ms);
if (\array_key_exists("js", $bvData) && \array_key_exists("loaded", $bvData["js"])) {
$diff_css_js_milliseconds = $bvData["css"]["loaded"]->diffInMilliseconds($bvData["js"]["loaded"]);
$stop_waiting_for_csp = $bvData["css"]["loaded"]->addMilliseconds($diff_css_js_milliseconds * 2);
/** @var \Carbon\Carbon $stop_waiting_for_csp_js */
$stop_waiting_for_csp_js = $bvData["css"]["loaded"]->addMilliseconds($wait_time_ms);
if ($stop_waiting_for_csp_js->isAfter($stop_waiting_for_csp)) {
$stop_waiting_for_csp = $stop_waiting_for_csp_js;
}
}
if (now() > $stop_waiting_for_csp) {
if (now()->isAfter($stop_waiting_for_csp)) {
$csp_loaded = true;
}
}
}
if (
$css_loaded &&
$js_loaded &&
$csp_loaded
) {
if ($csp_loaded) {
return true;
}
// Calculate Sleep Time
// Sleeptime gradually increases with the current wait time
// Min 10ms and max 1s
$sleep_time_milliseconds = round(now()->diffInMilliseconds($wait_start) / 10);
$sleep_time_milliseconds = max(10, $sleep_time_milliseconds);
$sleep_time_milliseconds = min(1000, $sleep_time_milliseconds);
usleep($sleep_time_milliseconds * 1000);
} while (now()->diffInMilliseconds($wait_start) < $max_wait_time_ms);
return false;
}
......@@ -266,4 +262,4 @@ class BrowserVerification
fclose($fh);
}
}
}
}
\ No newline at end of file
metager/app/MetaGer.php
View file @ 7f4d8175
......@@ -62,6 +62,7 @@ class MetaGer
# Konfigurationseinstellungen:
protected $sumaFile;
protected $mobile;
protected $framed;
protected $resultCount;
protected $sprueche;
protected $newtab;
......@@ -300,7 +301,8 @@ class MetaGer
# Validate Advertisements
$newResults = [];
foreach ($this->ads as $ad) {
if (($ad->strippedHost !== "" && (in_array($ad->strippedHost, $this->adDomainsBlacklisted) ||
if (
($ad->strippedHost !== "" && (in_array($ad->strippedHost, $this->adDomainsBlacklisted) ||
in_array($ad->strippedLink, $this->adUrlsBlacklisted))) || ($ad->strippedHostAnzeige !== "" && (in_array($ad->strippedHostAnzeige, $this->adDomainsBlacklisted) ||
in_array($ad->strippedLinkAnzeige, $this->adUrlsBlacklisted)))
) {
......@@ -403,7 +405,7 @@ class MetaGer
$arr[$link]->changed = true;
}
} else {
$arr[$link] = &$this->results[$i];
$arr[$link] = & $this->results[$i];
}
}
}
......@@ -448,7 +450,7 @@ class MetaGer
* every so often. ~33% in this case
* ToDo set back to 5 once we do not want to advertise donations as much anymore
*/
if (/*sizeof($this->ads) === 0 &&*/rand(1, 100) >= 34) {
if ( /*sizeof($this->ads) === 0 &&*/rand(1, 100) >= 34) {
return;
}
......@@ -488,7 +490,7 @@ class MetaGer
$url = route('humanverification', $params);
$proxyPw = md5($day . $result->proxyLink . config("metager.metager.proxy.password"));
$params["pw"] = $proxyPw;
$params["url"] = \bin2hex($result->proxyLink);
$params["url"] = \bin2hex($result->proxyLink);
$proxyUrl = route('humanverification', $params);
$result->link = $url;
$result->proxyLink = $proxyUrl;
......@@ -579,7 +581,7 @@ class MetaGer
# and no other search engine can provide this filter
$validTmp = false;
foreach ($this->parameterFilter as $filterName => $filter) {
if (count((array)$filter->sumas) === 1 && !empty($filter->sumas->{$sumaName})) {
if (count((array) $filter->sumas) === 1 && !empty($filter->sumas->{$sumaName})) {
$validTmp = true;
break;
}
......@@ -602,7 +604,7 @@ class MetaGer
# Special case if search engines are disabled
# Since bing is normally only active if a filter is set but it should be active, too if yahoo is disabled
if ($this->getFokus() === "web" && empty($this->enabledSearchengines["yahoo"]) && Cookie::get("web_engine_bing") !== "off" && isset($this->sumaFile->sumas->{"bing"})) {
if ($this->getFokus() === "web" && empty($this->enabledSearchengines["yahoo"]) && Cookie::get("web_engine_bing") !== "off" && isset($this->sumaFile->sumas->{"bing"})) {
$this->enabledSearchengines["bing"] = $this->sumaFile->sumas->{"bing"};
}
......@@ -764,7 +766,8 @@ class MetaGer
if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"} && Cookie::get($this->getFokus() . "_engine_" . $suma) !== "off") {
if (!empty($filter->sumas->{$suma})) {
# If the searchengine is disabled this filter shouldn't be available
if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
if (
(!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
|| (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)
) {
continue;
......@@ -851,14 +854,14 @@ class MetaGer
public function sumaIsOverture($suma)
{
return
$suma["name"]->__toString() === "overture"
$suma["name"]->__toString() === "overture"
|| $suma["name"]->__toString() === "overtureAds";
}
public function sumaIsNotAdsuche($suma)
{
return
$suma["name"]->__toString() !== "qualigo"
$suma["name"]->__toString() !== "qualigo"
&& $suma["name"]->__toString() !== "similar_product_ads"
&& $suma["name"]->__toString() !== "overtureAds";
}
......@@ -998,11 +1001,16 @@ class MetaGer
$this->out = $request->input("out", "");
if ($request->filled("mgv") || $request->input("out", "") === "results-with-style") {
// Check if request header "Sec-Fetch-Dest" is set
$this->framed = false;
if ($request->header("Sec-Fetch-Dest") === "iframe") {
$this->framed = true;
} elseif ($request->input("out", "") === "results-with-style") {
$this->framed = true;
} elseif ($request->input("iframe", "false") === "true") {
$this->framed = true;
} else {
$this->framed = false;
}
unset($request["iframe"]);
# IP
$this->ip = $this->anonymizeIp($request->ip());
......@@ -1269,7 +1277,8 @@ class MetaGer
$usedParameters[$filter->{"get-parameter"}] = true;
}
if (($request->filled($filter->{"get-parameter"}) && $request->input($filter->{"get-parameter"}) !== "off") ||
if (
($request->filled($filter->{"get-parameter"}) && $request->input($filter->{"get-parameter"}) !== "off") ||
Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"}) !== null
) { # If the filter is set via Cookie
$this->parameterFilter[$filterName] = $filter;
......@@ -1951,4 +1960,4 @@ class MetaGer
{
$this->dummy = $dummy;
}
}
}
\ No newline at end of file
metager/app/SearchSettings.php
View file @ 7f4d8175
......@@ -7,9 +7,8 @@ class SearchSettings
public $bv_key = null; // Cache Key where data of BV is temporarily stored
public $javascript_enabled = false;
public $header_printed = false;
public function __construct()
{
}
}
}
\ No newline at end of file
metager/config/metager/metager.php
View file @ 7f4d8175
......@@ -3,7 +3,6 @@
return [
"browserverification_enabled" => true,
"browserverification_whitelist" => [
"w3m\/",
],
"admin" => [
"user" => env("ADMIN_USER", "admin"),
......@@ -73,4 +72,4 @@ return [
"selenium" => [
"host" => env("SELENIUM_HOST", "localhost"),
],
];
];
\ No newline at end of file
metager/resources/js/verify.js
View file @ 7f4d8175
require('es6-promise').polyfill();
require('fetch-ie8');
require("es6-promise").polyfill();
require("fetch-ie8");
// Find the key id for the browser-verification
document.querySelectorAll("link").forEach(element => {
let href = element.href;
let matches = href.match(/http[s]{0,1}:\/\/[^\/]+\/index\.css\?id=(.+)/i);
if (!matches) {
return true;
}
try {
// Should get blocked by csp
eval("window.sp = 1;");
} catch (err) { }
document.querySelectorAll("link").forEach((element) => {
let href = element.href;
let matches = href.match(/http[s]{0,1}:\/\/[^\/]+\/index\.css\?id=(.+)/i);
if (!matches) {
return true;
}
try {
// Should get blocked by csp
eval("window.sp = 1;");
} catch (err) {}
let key = matches[1];
let url = "/img/logo.png?id=" + key;
if (window.sp == 1) {
url += "&sp"
}
let key = matches[1];
let url = "/img/logo.png?id=" + key;
if (window.sp == 1) {
url += "&sp";
}
if (navigator.webdriver) {
url += "&wd"
}
if (navigator.webdriver) {
url += "&wd";
}
return fetch(url);
});
fetch(url).then((res) => {
if (res.status === 200) {
let url = document.querySelector("meta[name=url]").content;
let nonce = document.querySelector("meta[name=nonce]").content;
let check = `/index.css?id=${nonce}`;
console.log(url);
console.log(nonce);
console.log(check);
let interval = setInterval(function () {
let links = document.querySelectorAll("link");
for (let i = 0; i < links.length; i++) {
if (links[i].href.includes(check)) {
clearInterval(interval);
window.location = url;
}
}
}, 100);
}
});
});
metager/resources/views/humanverification/captcha.blade.php
View file @ 7f4d8175
......@@ -5,7 +5,7 @@
@section('content')
<h1>@lang('captcha.1')</h1>
<p>@lang('captcha.2')</p>
<form method="post" action="{{ route('captcha_solve') }}">
<form method="post" action="{{ route('captcha_solve') }}" target="_top">
<input type="hidden" name="url" value="{!! $url !!}">
<input type="hidden" name="key" value="{{ $key }}">
<input type="hidden" name="begin" value="{{ \microtime(true) }}">
......
metager/resources/views/layouts/resultPage.blade.php
View file @ 7f4d8175
@if(!\app()->make(\App\SearchSettings::class)->header_printed)
<!DOCTYPE html>
<html lang="{{ LaravelLocalization::getCurrentLocale() }}">
......@@ -45,7 +44,6 @@
<link type="text/css" rel="stylesheet" media="(prefers-color-scheme:dark)" href="{{ mix('css/themes/metager-dark.css') }}" />
@endif
@endif
<title>{{ $eingabe }} - MetaGer</title>
<meta content="width=device-width, initial-scale=1.0, user-scalable=no" name="viewport" />
<meta name="p" content="{{ getmypid() }}" />
......
metager/resources/views/layouts/resultpage/verificationHeader.blade.php → metager/resources/views/layouts/resultpage/framedResultPage.blade.php
View file @ 7f4d8175
......@@ -2,10 +2,40 @@
<html lang="{{ LaravelLocalization::getCurrentLocale() }}">
<head>
<meta charset="UTF-8">
<link rel="stylesheet" href="/index.css?id={{ $key }}">
<meta name="nonce" content="{{ $mgv }}">
<meta name="url" content="{!! $js_url !!}">
<link rel="stylesheet" href="/index.css?id={{ $mgv }}">
<script src="{{ mix('js/index.js') }}"></script>
@foreach(LaravelLocalization::getSupportedLocales() as $locale => $locale_data)
@if(LaravelLocalization::getCurrentLocale() !== $locale)
<link rel="alternate" hreflang="{{ $locale }}" href="{{ LaravelLocalization::getLocalizedUrl($locale, null, [], true) }}">
@endif
@endforeach
\ No newline at end of file
@endforeach
<title>{{ Request::input('eingabe', '') }} - MetaGer</title>
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" />
<style nonce="{{ $mgv }}">
html {