Commit 58f85ea1 authored by Dominik Hebeler's avatar Dominik Hebeler

Merge branch 'development' into '990-make-the-cache-connection-optional'

Development

See merge request !1641
parents 9b5b4ada 0985e888
<?php
namespace App\Console\Commands;
use Carbon;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
class LoadSpam extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'spam:load';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Loads a list of current Spams into redis';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$filePath = \storage_path('logs/metager/ban.txt');
$bans = [];
if (\file_exists($filePath)) {
$bans = json_decode(file_get_contents($filePath), true);
}
$bansToLoad = [];
foreach ($bans as $ban) {
$bannedUntil = Carbon::createFromFormat("Y-m-d H:i:s", $ban["banned-until"]);
if ($bannedUntil->isAfter(Carbon::now())) {
$bansToLoad[] = $ban["regexp"];
}
}
Redis::pipeline(function ($redis) use ($bansToLoad) {
$redis->del("spam");
foreach ($bansToLoad as $ban) {
$redis->rpush("spam", $ban);
}
});
}
}
......@@ -28,7 +28,7 @@ class Kernel extends ConsoleKernel
$schedule->command('requests:gather')->everyFifteenMinutes();
$schedule->command('requests:useragents')->everyFiveMinutes();
$schedule->command('logs:gather')->everyMinute();
$schedule->command('spam:load')->everyFiveMinutes();
$schedule->call(function () {
DB::table('monthlyrequests')->truncate();
DB::disconnect('mysql');
......
<?php
namespace App\Http\Controllers;
use Carbon;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
class AdminSpamController extends Controller
{
public function index()
{
$queries = $this->getQueries();
$currentBans = $this->getBans();
$loadedBans = Redis::lrange("spam", 0, -1);
return view("admin.spam")
->with('title', "Spam Konfiguration - MetaGer")
->with('queries', $queries)
->with('bans', $currentBans)
->with('loadedBans', $loadedBans);
}
public function ban(Request $request)
{
$banTime = $request->input('ban-time');
$banRegexp = $request->input('regexp');
$file = storage_path('logs/metager/ban.txt');
$bans = [];
if (file_exists($file)) {
$bans = json_decode(file_get_contents($file), true);
}
$bans[] = ["banned-until" => Carbon::now()->add($banTime)->format("Y-m-d H:i:s"), "regexp" => $banRegexp];
\file_put_contents($file, json_encode($bans));
return redirect(url('admin/spam'));
}
public function jsonQueries()
{
$queries = $this->getQueries();
return response()->json($queries);
}
public function queryregexp(Request $request)
{
$data = json_decode($request->getContent(), true);
$queries = $data["queries"];
$regexps = [$data["regexp"]];
$bans = $this->getBans();
foreach ($bans as $ban) {
$regexps[] = $ban["regexp"];
}
$resultData = [];
foreach ($queries as $query) {
$matches = false;
foreach ($regexps as $regexp) {
try {
if (preg_match($regexp, $query)) {
$matches = true;
}
} catch (\Exception $e) {
// Exceptions are expected when no valid regexp is given
}
}
$resultData[] = [
"query" => $query,
"matches" => $matches,
];
}
return response()->json($resultData);
}
private function getQueries()
{
$minuteToFetch = Carbon::now()->subMinutes(2);
$logFile = storage_path("logs/metager/" . $minuteToFetch->format("Y/m/d") . ".log");
$result = shell_exec("cat $logFile | grep " . $minuteToFetch->format("H:i:"));
$result = explode(PHP_EOL, $result);
$queries = array();
foreach ($result as $line) {
if ($query = \preg_match("/.*eingabe=(.*)$/", $line, $matches)) {
$queries[] = $matches[1];
}
}
return $queries;
}
public function getBans()
{
$file = \storage_path('logs/metager/ban.txt');
$bans = [];
if (file_exists($file)) {
$tmpBans = json_decode(file_get_contents($file), true);
foreach ($tmpBans as $ban) {
#dd($ban["banned-until"]);
$bannedUntil = Carbon::createFromFormat('Y-m-d H:i:s', $ban["banned-until"]);
if ($bannedUntil->isAfter(Carbon::now())) {
$bans[] = $ban;
}
}
}
file_put_contents($file, json_encode($bans));
return $bans;
}
public function deleteRegexp(Request $request)
{
$file = \storage_path('logs/metager/ban.txt');
$bans = [];
if (file_exists($file)) {
$bans = json_decode(file_get_contents($file), true);
}
$regexpToDelete = $request->input('regexp');
$newBans = [];
foreach ($bans as $ban) {
if ($ban["regexp"] !== $regexpToDelete) {
$newBans[] = $ban;
}
}
file_put_contents($file, json_encode($newBans));
return redirect(url('admin/spam'));
}
}
......@@ -7,6 +7,7 @@ use Carbon;
use Illuminate\Hashing\BcryptHasher as Hasher;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Redis;
use Input;
class HumanVerification extends Controller
......@@ -39,6 +40,7 @@ class HumanVerification extends Controller
$key = strtolower($key);
if (!$hasher->check($key, $lockedKey)) {
sleep(\random_int(1, 8));
$captcha = Captcha::create("default", true);
$user["lockedKey"] = $captcha["key"];
HumanVerification::saveUser($user);
......@@ -65,6 +67,7 @@ class HumanVerification extends Controller
}
}
}
sleep(\random_int(1, 8));
$captcha = Captcha::create("default", true);
$user["lockedKey"] = $captcha["key"];
HumanVerification::saveUser($user);
......@@ -105,7 +108,7 @@ class HumanVerification extends Controller
private static function saveUser($user)
{
$userList = Cache::get(HumanVerification::PREFIX . "." . $user["id"], []);
if ($user["whitelist"]) {
$user["expiration"] = now()->addWeeks(2);
} else {
......@@ -196,21 +199,20 @@ class HumanVerification extends Controller
public static function couldBeSpammer($ip)
{
$possibleSpammer = false;
# Check for recent Spams
$eingabe = \Request::input('eingabe');
if (\preg_match("/^susimail\s+-site:[^\s]+\s-site:/si", $eingabe)) {
return true;
} else if (\preg_match("/^\s*site:\"linkedin\.com[^\"]*\"\s+/si", $eingabe)) {
return true;
$spams = Redis::lrange("spam", 0, -1);
foreach ($spams as $spam) {
if (\preg_match($spam, $eingabe)) {
return true;
}
}
return $possibleSpammer;
return false;
}
public function botOverview(Request $request){
public function botOverview(Request $request)
{
$id = "";
$uid = "";
$ip = $request->ip();
......@@ -232,7 +234,8 @@ class HumanVerification extends Controller
->with('user', $user);
}
public function botOverviewChange(Request $request) {
public function botOverviewChange(Request $request)
{
$id = "";
$uid = "";
$ip = $request->ip();
......@@ -247,11 +250,11 @@ class HumanVerification extends Controller
$userList = Cache::get(HumanVerification::PREFIX . "." . $id);
$user = $userList[$uid];
if($request->filled("locked")){
if ($request->filled("locked")) {
$user["locked"] = boolval($request->input('locked'));
}elseif($request->filled("whitelist")) {
} elseif ($request->filled("whitelist")) {
$user["whitelist"] = boolval($request->input('whitelist'));
}elseif($request->filled("unusedResultPages")) {
} elseif ($request->filled("unusedResultPages")) {
$user["unusedResultPages"] = intval($request->input('unusedResultPages'));
}
......
......@@ -80,7 +80,7 @@ class HumanVerification
}
}
}
# A lot of automated requests are from websites that redirect users to our result page.
# We will detect those requests and put a captcha
$referer = URL::previous();
......@@ -98,9 +98,10 @@ class HumanVerification
if ((!$alone && $sum >= 50 && !$user["whitelist"]) || $refererLock) {
$user["locked"] = true;
}
# If the user is locked we will force a Captcha validation
if ($user["locked"]) {
sleep(\random_int(1, 8));
$captcha = Captcha::create("default", true);
$user["lockedKey"] = $captcha["key"];
\App\PrometheusExporter::CaptchaShown();
......
# Contributor Agreement
### Entity Contributor Non-Exclusive License Agreement (including the Traditional Patent License OPTION)
Thank you for your interest in contributing to SUMA-EV – Verein für freien Wissenszugang's MetaGer ("We" or "Us").
The purpose of this contributor agreement ("Agreement") is to clarify and document the rights granted by contributors to Us.
## 1. Definitions
"You" means the individual Copyright owner who Submits a Contribution to Us.
"Legal Entity" means an entity that is not a natural person.
"Affiliate" means any other Legal Entity that controls, is controlled by, or under common control with that Legal Entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such Legal Entity, whether by contract or otherwise, (ii) ownership of fifty percent (50%) or more of the outstanding shares or securities that vote to elect the management or other persons who direct such Legal Entity or (iii) beneficial ownership of such entity.
"Contribution" means any original work of authorship, including any original modifications or additions to an existing work of authorship, Submitted by You to Us, in which You own the Copyright.
"Copyright" means all rights protecting works of authorship, including copyright, moral and neighboring rights, as appropriate, for the full term of their existence.
"Material" means the software or documentation made available by Us to third parties. When this Agreement covers more than one software project, the Material means the software or documentation to which the Contribution was Submitted. After You Submit the Contribution, it may be included in the Material.
"Submit" means any act by which a Contribution is transferred to Us by You by means of tangible or intangible media, including but not limited to electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, Us, but excluding any transfer that is conspicuously marked or otherwise designated in writing by You as "Not a Contribution."
"Documentation" means any non-software portion of a Contribution.
## 2. License grant
### 2.1 Copyright license to Us
Subject to the terms and conditions of this Agreement, You hereby grant to Us a worldwide, royalty-free, NON-exclusive, perpetual and irrevocable (except as stated in Section 8.2) license, with the right to transfer an unlimited number of non-exclusive licenses or to grant sublicenses to third parties, under the Copyright covering the Contribution to use the Contribution by all means, including, but not limited to:
- publish the Contribution,
- modify the Contribution,
- prepare derivative works based upon or containing the Contribution and/or to combine the Contribution with other Materials,
- reproduce the Contribution in original or modified form,
- distribute, to make the Contribution available to the public, display and publicly perform the Contribution in original or modified form.
## 2.2 Moral rights
Moral Rights remain unaffected to the extent they are recognized and not waivable by applicable law. Notwithstanding, You may add your name to the attribution mechanism customary used in the Materials you Contribute to, such as the header of the source code files of Your Contribution, and We will respect this attribution when using Your Contribution.
## 3. Patents
### 3.1 Patent license
Subject to the terms and conditions of this Agreement You hereby grant to Us and to recipients of Materials distributed by Us a worldwide, royalty-free, non-exclusive, perpetual and irrevocable (except as stated in Section 3.2) patent license, with the right to transfer an unlimited number of non-exclusive licenses or to grant sublicenses to third parties, to make, have made, use, sell, offer for sale, import and otherwise transfer the Contribution and the Contribution in combination with any Material (and portions of such combination). This license applies to all patents owned or controlled by You, whether already acquired or hereafter acquired, that would be infringed by making, having made, using, selling, offering for sale, importing or otherwise transferring of Your Contribution(s) alone or by combination of Your Contribution(s) with any Material.
3.2 Revocation of patent license
You reserve the right to revoke the patent license stated in section 3.1 if We make any infringement claim that is targeted at your Contribution and not asserted for a Defensive Purpose. An assertion of claims of the Patents shall be considered for a "Defensive Purpose" if the claims are asserted against an entity that has filed, maintained, threatened, or voluntarily participated in a patent infringement lawsuit against Us or any of Our licensees.
## 4. License obligations by Us
We agree to (sub)license the Contribution or any Materials containing, based on or derived from your Contribution under the terms of any licenses the Free Software Foundation classifies as Free Software License and which are approved by the Open Source Initiative as Open Source licenses.
We agree to license patents owned or controlled by You only to the extent necessary to (sub)license Your Contribution(s) and the combination of Your Contribution(s) with the Material under the terms of any licenses the Free Software Foundation classifies as Free Software licenses and which are approved by the Open Source Initiative as Open Source licenses.
## 5. Disclaimer
THE CONTRIBUTION IS PROVIDED "AS IS". MORE PARTICULARLY, ALL EXPRESS OR IMPLIED WARRANTIES INCLUDING, WITHOUT LIMITATION, ANY IMPLIED WARRANTY OF SATISFACTORY QUALITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT ARE EXPRESSLY DISCLAIMED BY YOU TO US AND BY US TO YOU. TO THE EXTENT THAT ANY SUCH WARRANTIES CANNOT BE DISCLAIMED, SUCH WARRANTY IS LIMITED IN DURATION AND EXTENT TO THE MINIMUM PERIOD AND EXTENT PERMITTED BY LAW.
## 6. Consequential damage waiver
TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT WILL YOU OR WE BE LIABLE FOR ANY LOSS OF PROFITS, LOSS OF ANTICIPATED SAVINGS, LOSS OF DATA, INDIRECT, SPECIAL, INCIDENTAL, CONSEQUENTIAL AND EXEMPLARY DAMAGES ARISING OUT OF THIS AGREEMENT REGARDLESS OF THE LEGAL OR EQUITABLE THEORY (CONTRACT, TORT OR OTHERWISE) UPON WHICH THE CLAIM IS BASED.
## 7. Approximation of disclaimer and damage waiver
IF THE DISCLAIMER AND DAMAGE WAIVER MENTIONED IN SECTION 5. AND SECTION 6. CANNOT BE GIVEN LEGAL EFFECT UNDER APPLICABLE LOCAL LAW, REVIEWING COURTS SHALL APPLY LOCAL LAW THAT MOST CLOSELY APPROXIMATES AN ABSOLUTE WAIVER OF ALL CIVIL OR CONTRACTUAL LIABILITY IN CONNECTION WITH THE CONTRIBUTION.
## 8. Term
- 8.1 This Agreement shall come into effect upon commiting your contribution to our software repository.
- 8.2 This Agreement shall apply for the term of the copyright and patents licensed here. However, You shall have the right to terminate the Agreement if We do not fulfill the obligations as set forth in Section 4. Such termination must be made in writing.
- 8.3 In the event of a termination of this Agreement Sections 5, 6, 7, 8 and 9 shall survive such termination and shall remain in full force thereafter. For the avoidance of doubt, Free and Open Source Software (sub)licenses that have already been granted for Contributions at the date of the termination shall remain in full force after the termination of this Agreement.
## 9 Miscellaneous
- 9.1 This Agreement and all disputes, claims, actions, suits or other proceedings arising out of this agreement or relating in any way to it shall be governed by the laws of Germany excluding its private international law provisions.
- 9.2 This Agreement sets out the entire agreement between You and Us for Your Contributions to Us and overrides all other agreements or understandings.
- 9.3 In case of Your death, this agreement shall continue with Your heirs. In case of more than one heir, all heirs must exercise their rights through a commonly authorized person.
- 9.4 If any provision of this Agreement is found void and unenforceable, such provision will be replaced to the extent possible with a provision that comes closest to the meaning of the original provision and that is enforceable. The terms and conditions set forth in this Agreement shall apply notwithstanding any failure of essential purpose of this Agreement or any limited remedy to the maximum extent possible under law.
- 9.5 You agree to notify Us of any facts or circumstances of which you become aware that would make this Agreement inaccurate in any respect.
\ No newline at end of file
@extends('layouts.subPages')
@section('title', $title )
@section('content')
<style>
#head {
display: flex;
align-items: center;
margin-bottom: 16px;
}
#head > button {
margin-left: 16px;
}
#head > h1 {
margin: 0;
}
#queries {
display: flex;
flex-wrap: wrap;
justify-content: space-between;
}
.matches {
background-color: #c9f4c9;
}
#block-requests {
margin-bottom: 16px;
}
#regexp {
margin-bottom: 8px;
}
#ban-time {
margin-bottom: 8px;
}
</style>
<div id="block-requests">
<form method="post">
<input class="form-control" type="text" name="regexp" id="regexp" placeholder="Type in regexp to match queries...">
<select name="ban-time" id="ban-time" class="form-control">
<option value="1 day">Einen Tag</option>
<option value="1 week">Eine Woche</option>
<option value="2 weeks">Zwei Wochen</option>
<option value="1 month" selected>Einen Monat</option>
</select>
<button type="submit" class="btn btn-default btn-sm">Sperren</button>
</form>
</div>
<div id="head">
<h1>Letzte Suchanfragen</h1>
<button type="button" class="btn btn-success btn-sm">Aktualisierung stoppen (60)</button>
</div>
<input class="form-control" type="text" name="" id="check-against" placeholder="Match against...">
<div id="queries">
@foreach($queries as $query)
<div class="query card">{{$query}}</div>
@endforeach
</div>
<div id="bans">
<h1>Current Bans</h1>
<table class="table table-striped">
<thead>
<tr>
<td>Regexp</td>
<td>Banned until</td>
<td>Actions</td>
</tr>
</thead>
<tbody>
@foreach($bans as $ban)
<tr>
<td>{{ $ban["regexp"] }}</td>
<td>{{ Carbon::createFromFormat("Y-m-d H:i:s", $ban["banned-until"])->format("d.m.Y H:i:s")}} ({{ Carbon::createFromFormat("Y-m-d H:i:s", $ban["banned-until"])->diffInDays(Carbon::now()) }} Days)</td>
<td>
<form action="{{ url("admin/spam/deleteRegexp") }}" method="post">
<input type="hidden" name="regexp" value="{{ $ban["regexp"] }}">
<button type="submit">&#128465;</button>
</form>
</td>
</tr>
@endforeach
</tbody>
</table>
</div>
<div id="loadedbans">
<h1>Loaded Bans</h1>
<table class="table table-striped">
<thead>
<tr>
<td>Regexp</td>
</tr>
</thead>
<tbody>
@foreach($loadedBans as $ban)
<tr>
<td>{{ $ban }}</td>
</tr>
@endforeach
</tbody>
</table>
</div>
<script>
var lastUpdate = Date.now();
var updating = true;
var buttonText = "Aktualisierung stoppen";
var interval = setInterval(updateQueries, 1000);
$("#regexp").on("input", checkRegexp);
$("#check-against").on("input", checkRegexp);
$(document).ready(function(){
checkRegexp();
});
$("#head > button").click(function() {
if(!updating) {
$("#head > button").removeClass("btn-danger");
$("#head > button").addClass("btn-success");
buttonText = "Aktualisierung stoppen";
interval = setInterval(updateQueries, 1000);
}
var updateAt = lastUpdate + 60000;
var updateIn = Math.round((updateAt - Date.now()) / 1000);
$("#head > button").html(buttonText + " (" + updateIn + ")");
updating = !updating;
});
function updateQueries() {
var updateAt = lastUpdate + 60000;
var updateIn = Math.round((updateAt - Date.now()) / 1000);
if(!updating){
$("#head > button").removeClass("btn-success");
$("#head > button").addClass("btn-danger");
buttonText = "Aktualisierung starten";
clearInterval(interval);
}
$("#head > button").html(buttonText + " (" + updateIn + ")");
if(updateAt > Date.now()){
return;
}
fetch("{{ url('admin/spam/jsonQueries') }}")
.then(response => response.json())
.then(data => {
$("#queries").html("");
$(data).each(function(index, el){
$("#queries").append("<div class=\"query card\">" + el + "</div>");
});
lastUpdate = Date.now();
checkRegexp();
});
}
function checkRegexp() {
var val = $("#regexp").val();
var queries = [];
$("#queries > .query").each(function(index, el){
queries.push($(el).html());
});
queries.push($("#check-against").val());
var url = "{{ url('admin/spam/queryregexp') }}";
var options = {
method: 'POST',
body: JSON.stringify({
"queries": queries,
"regexp": val
}),
headers: {
'Content-Type': 'application/json'
}
};
fetch(url, options)
.then(response => response.json())
.then(data => {
$("#queries > .query").each(function(index, el){
if(data[index]["matches"]){
$(el).addClass("matches");
}else{
$(el).removeClass("matches");
}
});
if(data[data.length-1]["matches"]){
$("#check-against").addClass("matches");
}else{
$("#check-against").removeClass("matches");
}
});
}
</script>
@endsection
......@@ -270,8 +270,7 @@
</div>
<div class="section">
<h1>Hosting</h1>
The websites under the domain "suma-ev.de" are hosted and administered by Intares GmbH. The remaining services
are administrated by us, the SUMA-EV, and operated on hired hardware at Hetzner Online GmbH.
Our services are administrated by us, the SUMA-EV, and operated on rented hardware at Hetzner Online GmbH.
</div>
<div class="section">
<h1>Legal basis for processing</h1>
......@@ -345,4 +344,4 @@
Like our offers, this privacy policy is subject to constant change. You should therefore read it again
regularly.
<br />This version of our Privacy Policy is dated 2018-05-24.
</div>
\ No newline at end of file
</div>
......@@ -286,8 +286,7 @@
</div>
<div class="section">
<h1>Hosting</h1>
Die Webseiten unter der Domain „suma-ev.de“ werden bei der Intares GmbH gehostet und administriert. Die übrigen
Dienste werden von uns, dem SUMA-EV, administriert und auf angemieteter Hardware bei der Hetzner Online GmbH
Unsere Dienste werden von uns, dem SUMA-EV, administriert und auf angemieteter Hardware bei der Hetzner Online GmbH
betrieben.