Commit 7c6b1670 authored by Dominik Hebeler's avatar Dominik Hebeler

Merge branch 'development' into '994-remove-bootsrap'

Development

See merge request !1667
parents e235d3f5 85b514b3
......@@ -97,6 +97,8 @@ stop_review:
- if: '$REVIEW_DISABLED'
when: never
- if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
when: manual
.development: &development_template
......
......@@ -2,12 +2,20 @@ service:
externalPort: 80
internalPort: 80
hpa:
enabled: true
minReplicas: 1
maxReplicas: 5
resources:
limits:
cpu: 500m
memory: 1Gi
requests:
cpu: 500m
memory: 1Gi
podDisruptionBudget:
enabled: true
minAvailable: 1
maxUnavailable:
minAvailable:
maxUnavailable: 0
podAnnotations:
prometheus.io/scrape: "true"
prometheus.io/path: /metrics
......@@ -17,6 +25,14 @@ ingress:
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
nginx.ingress.kubernetes.io/configuration-snippet: |
more_set_headers "Content-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline'; script-src-elem 'self' 'unsafe-inline'; script-src-attr 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; style-src-elem 'self' 'unsafe-inline'; style-src-attr 'self' 'unsafe-inline'; img-src 'self'; font-src 'self'; connect-src 'self'; media-src; object-src; prefetch-src; child-src; frame-src 'self'; worker-src; frame-ancestors 'self' https://scripts.zdv.uni-mainz.de; form-action 'self'; base-uri; manifest-src; plugin-types; report-uri; report-to";
more_set_headers "X-Frame-Options: sameorigin";
more_set_headers "X-Content-Type-Options: nosniff";
more_set_headers "ReferrerPolicy: origin";
more_set_headers "X-XSS-Protection: 1; mode=block";
if ($arg_out = "results-with-style") {
more_set_headers "X-Frame-Options: allow-from https://scripts.zdv.uni-mainz.de/";
}
if ($host = "www.metager3.de") {
return 301 https://metager3.de$request_uri;
}
......@@ -2,7 +2,8 @@ service:
externalPort: 80
internalPort: 80
hpa:
minReplicas: 5
enabled: true
minReplicas: 1
maxReplicas: 100
resources:
limits:
......@@ -13,8 +14,8 @@ resources:
memory: 1Gi
podDisruptionBudget:
enabled: true
minAvailable: 4
maxUnavailable:
minAvailable:
maxUnavailable: 0
podAnnotations:
prometheus.io/scrape: "true"
prometheus.io/path: /metrics
......@@ -24,6 +25,14 @@ ingress:
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
nginx.ingress.kubernetes.io/configuration-snippet: |
more_set_headers "Content-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline'; script-src-elem 'self' 'unsafe-inline'; script-src-attr 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; style-src-elem 'self' 'unsafe-inline'; style-src-attr 'self' 'unsafe-inline'; img-src 'self'; font-src 'self'; connect-src 'self'; media-src; object-src; prefetch-src; child-src; frame-src 'self'; worker-src; frame-ancestors 'self' https://scripts.zdv.uni-mainz.de; form-action 'self'; base-uri; manifest-src; plugin-types; report-uri; report-to";
more_set_headers "X-Frame-Options: sameorigin";
more_set_headers "X-Content-Type-Options: nosniff";
more_set_headers "ReferrerPolicy: origin";
more_set_headers "X-XSS-Protection: 1; mode=block";
if ($arg_out = "results-with-style") {
more_set_headers "X-Frame-Options: allow-from https://scripts.zdv.uni-mainz.de/";
}
if ($host = "www.metager.de") {
return 301 https://metager.de$request_uri;
}
......
......@@ -5,6 +5,15 @@ ingress:
annotations:
kubernetes.io/tls-acme: "false"
nginx.ingress.kubernetes.io/ssl-redirect: "false"
nginx.ingress.kubernetes.io/configuration-snippet: |
more_set_headers "Content-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline'; script-src-elem 'self' 'unsafe-inline'; script-src-attr 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; style-src-elem 'self' 'unsafe-inline'; style-src-attr 'self' 'unsafe-inline'; img-src 'self'; font-src 'self'; connect-src 'self'; media-src; object-src; prefetch-src; child-src; frame-src 'self'; worker-src; frame-ancestors 'self' https://scripts.zdv.uni-mainz.de; form-action 'self'; base-uri; manifest-src; plugin-types; report-uri; report-to";
more_set_headers "X-Frame-Options: sameorigin";
more_set_headers "X-Content-Type-Options: nosniff";
more_set_headers "ReferrerPolicy: origin";
more_set_headers "X-XSS-Protection: 1; mode=block";
if ($arg_out = "results-with-style") {
more_set_headers "X-Frame-Options: allow-from https://scripts.zdv.uni-mainz.de/";
}
tls:
enabled: false
service:
......
......@@ -7,6 +7,7 @@ RUN apk add --update \
dcron \
zip \
redis \
libpng \
php7 \
php7-fpm \
php7-common \
......@@ -38,10 +39,10 @@ RUN sed -i 's/;error_log = log\/php7\/error.log/error_log = \/dev\/stderr/g' /et
sed -i 's/;catch_workers_output = yes/catch_workers_output = yes/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = nobody/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_children = 5/pm.max_children = 100/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.start_servers = 2/pm.start_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_children = 5/pm.max_children = 1024/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.start_servers = 2/pm.start_servers = 50/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.min_spare_servers = 1/pm.min_spare_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_spare_servers = 3/pm.max_spare_servers = 25/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_spare_servers = 3/pm.max_spare_servers = 50/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/user = www-data/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = www-data/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;cgi.fix_pathinfo=1/cgi.fix_pathinfo=0/g' /etc/php7/php.ini && \
......
......@@ -4,6 +4,7 @@ namespace App\Http\Controllers;
use Captcha;
use Carbon;
use Cookie;
use Illuminate\Hashing\BcryptHasher as Hasher;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Cache;
......@@ -21,7 +22,17 @@ class HumanVerification extends Controller
if ($url != null) {
$url = base64_decode(str_replace("<<SLASH>>", "/", $url));
} else {
$url = $request->input('url');
$url = $request->input('url', url("/"));
}
$protocol = "http://";
if ($request->secure()) {
$protocol = "https://";
}
if (stripos($url, $protocol . $request->getHttpHost()) !== 0) {
$url = url("/");
}
$userlist = Cache::get(HumanVerification::PREFIX . "." . $id, []);
......@@ -40,7 +51,6 @@ class HumanVerification extends Controller
$key = strtolower($key);
if (!$hasher->check($key, $lockedKey)) {
sleep(\random_int(1, 8));
$captcha = Captcha::create("default", true);
$user["lockedKey"] = $captcha["key"];
HumanVerification::saveUser($user);
......@@ -67,7 +77,7 @@ class HumanVerification extends Controller
}
}
}
sleep(\random_int(1, 8));
$captcha = Captcha::create("default", true);
$user["lockedKey"] = $captcha["key"];
HumanVerification::saveUser($user);
......@@ -146,9 +156,9 @@ class HumanVerification extends Controller
$ip = $request->ip();
$id = "";
if (HumanVerification::couldBeSpammer($ip)) {
$id = hash("sha512", "999.999.999.999");
$id = hash("sha1", "999.999.999.999");
} else {
$id = hash("sha512", $ip);
$id = hash("sha1", $ip);
}
$userlist = Cache::get(HumanVerification::PREFIX . "." . $id, []);
......@@ -185,9 +195,9 @@ class HumanVerification extends Controller
$uid = "";
$ip = $request->ip();
if (HumanVerification::couldBeSpammer($ip)) {
$uid = hash("sha512", "999.999.999.999" . $ip . $_SERVER["AGENT"] . "uid");
$uid = hash("sha1", "999.999.999.999" . $ip . $_SERVER["AGENT"] . "uid");
} else {
$uid = hash("sha512", $ip . $_SERVER["AGENT"] . "uid");
$uid = hash("sha1", $ip . $_SERVER["AGENT"] . "uid");
}
if ($uid === $id) {
......@@ -217,11 +227,11 @@ class HumanVerification extends Controller
$uid = "";
$ip = $request->ip();
if (\App\Http\Controllers\HumanVerification::couldBeSpammer($ip)) {
$id = hash("sha512", "999.999.999.999");
$uid = hash("sha512", "999.999.999.999" . $ip . $_SERVER["AGENT"] . "uid");
$id = hash("sha1", "999.999.999.999");
$uid = hash("sha1", "999.999.999.999" . $ip . $_SERVER["AGENT"] . "uid");
} else {
$id = hash("sha512", $ip);
$uid = hash("sha512", $ip . $_SERVER["AGENT"] . "uid");
$id = hash("sha1", $ip);
$uid = hash("sha1", $ip . $_SERVER["AGENT"] . "uid");
}
$userList = Cache::get(HumanVerification::PREFIX . "." . $id);
......@@ -240,11 +250,11 @@ class HumanVerification extends Controller
$uid = "";
$ip = $request->ip();
if (\App\Http\Controllers\HumanVerification::couldBeSpammer($ip)) {
$id = hash("sha512", "999.999.999.999");
$uid = hash("sha512", "999.999.999.999" . $ip . $_SERVER["AGENT"] . "uid");
$id = hash("sha1", "999.999.999.999");
$uid = hash("sha1", "999.999.999.999" . $ip . $_SERVER["AGENT"] . "uid");
} else {
$id = hash("sha512", $ip);
$uid = hash("sha512", $ip . $_SERVER["AGENT"] . "uid");
$id = hash("sha1", $ip);
$uid = hash("sha1", $ip . $_SERVER["AGENT"] . "uid");
}
$userList = Cache::get(HumanVerification::PREFIX . "." . $id);
......@@ -261,4 +271,84 @@ class HumanVerification extends Controller
HumanVerification::saveUser($user);
return redirect('admin/bot');
}
public function browserVerification(Request $request)
{
$key = $request->input("id", "");
// Verify that key is a md5 checksum
if (!preg_match("/^[a-f0-9]{32}$/", $key)) {
abort(404);
}
Redis::connection("cache")->pipeline(function ($redis) use ($key) {
$redis->rpush($key, true);
$redis->expire($key, 30);
});
return response(view('layouts.resultpage.verificationCss'), 200)->header("Content-Type", "text/css");
}
public static function block(Request $request)
{
$prefix = "humanverification";
$ip = $request->ip();
$id = "";
$uid = "";
if (\App\Http\Controllers\HumanVerification::couldBeSpammer($ip)) {
$id = hash("sha1", "999.999.999.999");
$uid = hash("sha1", "999.999.999.999" . $ip . $_SERVER["AGENT"] . "uid");
} else {
$id = hash("sha1", $ip);
$uid = hash("sha1", $ip . $_SERVER["AGENT"] . "uid");
}
/**
* If the user sends a Password or a key
* We will not verificate the user.
* If someone that uses a bot finds this out we
* might have to change it at some point.
*/
if ($request->filled('password') || $request->filled('key') || Cookie::get('key') !== null || $request->filled('appversion') || !env('BOT_PROTECTION', false)) {
$update = false;
return $next($request);
}
# Get all Users of this IP
$users = Cache::get($prefix . "." . $id, []);
$user = [];
$changed = false;
if (empty($users[$uid])) {
$user = [
'uid' => $uid,
'id' => $id,
'unusedResultPages' => 0,
'whitelist' => false,
'locked' => true,
"lockedKey" => "",
"expiration" => now()->addWeeks(2),
];
$changed = true;
} else {
$user = $users[$uid];
if (!$user["locked"]) {
$user["locked"] = true;
$changed = true;
}
}
if ($user["whitelist"]) {
$user["expiration"] = now()->addWeeks(2);
} else {
$user["expiration"] = now()->addHours(72);
}
if ($changed) {
$userList = Cache::get($prefix . "." . $user["id"], []);
$userList[$user["uid"]] = $user;
Cache::put($prefix . "." . $user["id"], $userList, 2 * 7 * 24 * 60 * 60);
}
return [$id, $uid];
}
}
......@@ -62,5 +62,6 @@ class Kernel extends HttpKernel
'referer.check' => \App\Http\Middleware\RefererCheck::class,
'humanverification' => \App\Http\Middleware\HumanVerification::class,
'useragentmaster' => \App\Http\Middleware\UserAgentMaster::class,
'browserverification' => \App\Http\Middleware\BrowserVerification::class,
];
}
<?php
namespace App\Http\Middleware;
use Closure;
use Illuminate\Support\Facades\Redis;
class BrowserVerification
{
/**
* Handle an incoming request.
*
* @param \Illuminate\Http\Request $request
* @param \Closure $next
* @return mixed
*/
public function handle($request, Closure $next)
{
$bvEnabled = config("metager.metager.browserverification_enabled");
if (empty($bvEnabled) || !$bvEnabled) {
return $next($request);
}
$mgv = $request->input('mgv', "");
if (!empty($mgv)) {
// Verify that key is a md5 checksum
if (!preg_match("/^[a-f0-9]{32}$/", $mgv)) {
abort(404);
}
$result = boolval(Redis::connection("cache")->blpop($mgv, 5));
if ($result === true) {
return $next($request);
} else {
return redirect("/");
}
}
header('Content-type: text/html; charset=utf-8');
header('X-Accel-Buffering: no');
ini_set('zlib.output_compression', 'Off');
ini_set('output_buffering', 'Off');
ini_set('output_handler', '');
ob_end_clean();
$key = md5($request->ip() . microtime(true));
echo (view('layouts.resultpage.verificationHeader')->with('key', $key)->render());
flush();
$answer = boolval(Redis::connection("cache")->blpop($key, 2));
if ($answer === true) {
return $next($request);
}
$params = $request->all();
$params["mgv"] = $key;
$url = route("resultpage", $params);
echo (view('layouts.resultpage.unverifiedResultPage')
->with('url', $url)
->render());
}
}
......@@ -3,10 +3,8 @@
namespace App\Http\Middleware;
use Cache;
use Captcha;
use Closure;
use Cookie;
use Illuminate\Http\Response;
use Log;
use URL;
......@@ -30,11 +28,11 @@ class HumanVerification
$id = "";
$uid = "";
if (\App\Http\Controllers\HumanVerification::couldBeSpammer($ip)) {
$id = hash("sha512", "999.999.999.999");
$uid = hash("sha512", "999.999.999.999" . $ip . $_SERVER["AGENT"] . "uid");
$id = hash("sha1", "999.999.999.999");
$uid = hash("sha1", "999.999.999.999" . $ip . $_SERVER["AGENT"] . "uid");
} else {
$id = hash("sha512", $ip);
$uid = hash("sha512", $ip . $_SERVER["AGENT"] . "uid");
$id = hash("sha1", $ip);
$uid = hash("sha1", $ip . $_SERVER["AGENT"] . "uid");
}
unset($_SERVER["AGENT"]);
......@@ -101,19 +99,7 @@ class HumanVerification
# If the user is locked we will force a Captcha validation
if ($user["locked"]) {
sleep(\random_int(1, 8));
$captcha = Captcha::create("default", true);
$user["lockedKey"] = $captcha["key"];
\App\PrometheusExporter::CaptchaShown();
return
new Response(
view('humanverification.captcha')
->with('title', "Bestätigung erforderlich")
->with('uid', $uid)
->with('id', $id)
->with('url', url()->full())
->with('image', $captcha["img"])
);
return redirect()->route('captcha', ["id" => $id, "uid" => $uid, "url" => url()->full()]);
}
$user["unusedResultPages"]++;
......
......@@ -19,27 +19,31 @@ class LocalizationRedirect
$locale = LaravelLocalization::getCurrentLocale();
$host = $request->getHttpHost();
// We only redirect to the TLDs in the production version and exclude our onion domain
if(env("APP_ENV", "") !== "production" || $host === "b7cxf4dkdsko6ah2.onion" || $request->is('metrics')){
if (env("APP_ENV", "") !== "production" || $host === "metagerv65pwclop2rsfzg4jwowpavpwd6grhhlvdgsswvo6ii4akgyd.onion" || $request->is('metrics')) {
return $next($request);
}
// Redirect from v2 onion to v3 onion
if ($host === "b7cxf4dkdsko6ah2.onion") {
return redirect("http://metagerv65pwclop2rsfzg4jwowpavpwd6grhhlvdgsswvo6ii4akgyd.onion");
}
$url = url()->full();
$url = preg_replace("/^http:\/\//", "https://", $url);
if($host !== "metager.de" && $locale == "de"){
if ($host !== "metager.de" && $locale == "de") {
$url = str_replace($host, "metager.de", $url);
$url = preg_replace("/^(https:\/\/[^\/]+)\/de/", "$1", $url);
return redirect($url);
}
if($host !== "metager.es" && $locale == "es"){
if ($host !== "metager.es" && $locale == "es") {
$url = str_replace($host, "metager.es", $url);
$url = preg_replace("/^(https:\/\/[^\/]+)\/es/", "$1", $url);
return redirect($url);
}
if($host !== "metager.org" && $locale == "en"){
if ($host !== "metager.org" && $locale == "en") {
$url = str_replace($host, "metager.org", $url);
$url = preg_replace("/^(https:\/\/[^\/]+)\/en/", "$1", $url);
return redirect($url);
......
......@@ -723,8 +723,8 @@ class MetaGer
if (!empty($filter->sumas->$engineName)) {
if (empty($availableFilter[$filterName])) {
$availableFilter[$filterName] = $filter;
foreach($availableFilter[$filterName]->values as $key => $value){
if($key !== "nofilter"){
foreach ($availableFilter[$filterName]->values as $key => $value) {
if ($key !== "nofilter") {
unset($availableFilter[$filterName]->values->{$key});
}
}
......@@ -749,8 +749,8 @@ class MetaGer
}
if (empty($availableFilter[$filterName])) {
$availableFilter[$filterName] = $filter;
foreach($availableFilter[$filterName]->values as $key => $value){
if($key !== "nofilter"){
foreach ($availableFilter[$filterName]->values as $key => $value) {
if ($key !== "nofilter") {
unset($availableFilter[$filterName]->values->{$key});
}
}
......@@ -928,6 +928,13 @@ class MetaGer
# Sucheingabe
$this->eingabe = trim($request->input('eingabe', ''));
$this->q = $this->eingabe;
if ($request->filled("mgv")) {
$this->framed = true;
} else {
$this->framed = false;
}
# IP
$this->ip = $this->anonymizeIp($request->ip());
......@@ -953,16 +960,18 @@ class MetaGer
# Sprüche
if (!App::isLocale("de") || (\Cookie::has($this->getFokus() . '_setting_zitate') && \Cookie::get($this->getFokus() . '_setting_zitate') === "off")) {
$this->sprueche = "off";
}else{
} else {
$this->sprueche = "on";
}
if($request->filled("zitate") && $request->input('zitate') === "on" || $request->input('zitate') === "off"){
if ($request->filled("zitate") && $request->input('zitate') === "on" || $request->input('zitate') === "off") {
$this->sprueche = $request->input('quotes');
}
$this->newtab = $request->input('newtab', 'on');
if ($this->newtab === "on") {
$this->newtab = "_blank";
} else if ($this->framed) {
$this->newtab = "_top";
} else {
$this->newtab = "_self";
}
......@@ -1020,19 +1029,19 @@ class MetaGer
$this->request = $request->replace($request->except(['verification_id', 'uid', 'verification_count']));
// Disable freshness filter if custom freshness filter isset
if($this->request->filled("ff") && $this->request->filled("f")){
if ($this->request->filled("ff") && $this->request->filled("f")) {
$this->request = $this->request->replace($this->request->except(["f"]));
}
// Remove custom time filter if either of the dates isn't set or is not a date
if($this->request->input("fc") === "on"){
if(!$this->request->filled("ff") || !$this->request->filled("ft")){
if ($this->request->input("fc") === "on") {
if (!$this->request->filled("ff") || !$this->request->filled("ft")) {
$this->request = $this->request->replace($this->request->except(["fc", "ff", "ft"]));
}else{
} else {
$ff = $this->request->input("ff");
$ft = $this->request->input("ft");
if(!preg_match("/^\d{4}-\d{2}-\d{2}$/", $ff) || !preg_match("/^\d{4}-\d{2}-\d{2}$/", $ft)){
if (!preg_match("/^\d{4}-\d{2}-\d{2}$/", $ff) || !preg_match("/^\d{4}-\d{2}-\d{2}$/", $ft)) {
$this->request = $this->request->replace($this->request->except(["fc", "ff", "ft"]));
}else{
} else {
// Now Check if there is something wrong with the dates
$from = $this->request->input("ff");
$to = $this->request->input("ft");
......@@ -1040,21 +1049,21 @@ class MetaGer
$from = Carbon::createFromFormat("Y-m-d H:i:s", $from . " 00:00:00");
$to = Carbon::createFromFormat("Y-m-d H:i:s", $to . " 00:00:00");
if($from > Carbon::now()){
if ($from > Carbon::now()) {
$from = Carbon::now();
$changed = true;
}
if($to > Carbon::now()){
if ($to > Carbon::now()) {
$to = Carbon::now();
$changed = true;
}
if($from > $to){
if ($from > $to) {
$tmp = $to;
$to = $from;
$from = $tmp;
$changed = true;
}
if($changed){
if ($changed) {
$oldParameters = $this->request->all();
$oldParameters["ff"] = $from->format("Y-m-d");
$oldParameters["ft"] = $to->format("Y-m-d");
......@@ -1062,7 +1071,7 @@ class MetaGer
}
}
}
}else if($this->request->filled("ff") || $this->request->filled("ft")){
} else if ($this->request->filled("ff") || $this->request->filled("ft")) {
$this->request = $this->request->replace($this->request->except(["fc", "ff", "ft"]));
}
......@@ -1170,7 +1179,7 @@ class MetaGer
if (($request->filled($filter->{"get-parameter"}) && $request->input($filter->{"get-parameter"}) !== "off") ||
\Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"}) !== null
) { # If the filter is set via Cookie
$this->parameterFilter[$filterName] = $filter;
$this->parameterFilter[$filterName] = $filter;
$this->parameterFilter[$filterName]->value = $request->input($filter->{"get-parameter"}, '');
if (empty($this->parameterFilter[$filterName]->value)) {
$this->parameterFilter[$filterName]->value = \Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"});
......@@ -1349,7 +1358,7 @@ class MetaGer
public function nextSearchLink()
{
if (isset($this->next) && isset($this->next['engines']) && count($this->next['engines']) > 0) {
$requestData = $this->request->except(['page', 'out']);
$requestData = $this->request->except(['page', 'out', 'submit-query', 'mgv']);
if ($this->request->input('out', '') !== "results" && $this->request->input('out', '') !== '') {
$requestData["out"] = $this->request->input('out');
}
......@@ -1495,7 +1504,7 @@ class MetaGer
public function generateSearchLink($fokus, $results = true)
{
$except = ['page', 'next', 'out'];
$except = ['page', 'next', 'out', 'submit-query', 'mgv'];
# Remove every Filter
foreach ($this->su