Commit de0af5bb authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Merge branch 'development' into 1142-include-key-for-paypal-donations

parents b668fa1b 98436c52
......@@ -22,6 +22,7 @@ CACHE_DRIVER=redis
SESSION_DRIVER=file
QUEUE_CONNECTION=sync
REDIS_CACHE_CONNECTION=cache
REDIS_CACHE_HOST=redis
REDIS_HOST=redis
REDIS_PASSWORD=null
......
......@@ -92,7 +92,6 @@ stop_review:
- auto-deploy delete rollout
- auto-deploy persist_environment_url
variables:
ADDITIONAL_HOSTS: "www.metager3.de,test.metager.de"
HELM_UPGRADE_VALUES_FILE: .gitlab/development-values.yaml
ROLLOUT_RESOURCE_TYPE: deployment
environment:
......@@ -120,7 +119,7 @@ development:
production:
variables:
ADDITIONAL_HOSTS: "www.metager.de,metager.org,www.metager.org,metager.es,www.metager.es,klassik.metager.org"
ADDITIONAL_HOSTS: "metager.org,metager.es"
HELM_UPGRADE_VALUES_FILE: .gitlab/production-values.yaml
ROLLOUT_RESOURCE_TYPE: deployment
environment:
......
......@@ -50,8 +50,11 @@ podAnnotations:
prometheus.io/port: "80"
deploymentApiVersion: apps/v1
ingress:
tls:
enabled: true
acme: false
secretName: "metager-tls"
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
nginx.ingress.kubernetes.io/client-body-buffer-size: "30m"
nginx.ingress.kubernetes.io/proxy-body-size: "30m"
nginx.ingress.kubernetes.io/configuration-snippet: |
......@@ -64,6 +67,3 @@ ingress:
if ($arg_out = "results-with-style") {
more_set_headers "X-Frame-Options: allow-from https://scripts.zdv.uni-mainz.de/";
}
if ($host = "www.metager3.de") {
return 301 https://metager3.de$request_uri;
}
......@@ -11,11 +11,8 @@ podDisruptionBudget:
enabled: true
minAvailable: 1
maxUnavailable:
podAnnotations:
prometheus.io/scrape: "true"
prometheus.io/path: /metrics
prometheus.io/port: "80"
deploymentApiVersion: apps/v1
prometheus:
metrics: true
livenessProbe:
initialDelaySeconds: 0
readinessProbe:
......@@ -55,23 +52,9 @@ resources:
ingress:
tls:
enabled: true
custom:
tls:
- hosts:
- metager.de
- www.metager.de
secretName: metager-de-tls
- hosts:
- metager.org
- www.metager.org
- klassik.metager.org
secretName: metager-org-tls
- hosts:
- metager.es
- www.metager.es
secretName: production-auto-deploy-tls
acme: false
secretName: "metager-tls"
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
nginx.ingress.kubernetes.io/client-body-buffer-size: "30m"
nginx.ingress.kubernetes.io/proxy-body-size: "30m"
nginx.ingress.kubernetes.io/configuration-snippet: |
......@@ -83,16 +66,4 @@ ingress:
more_set_headers "Permissions-Policy: interest-cohort=()";
if ($arg_out = "results-with-style") {
more_set_headers "X-Frame-Options: allow-from https://scripts.zdv.uni-mainz.de/";
}
if ($host = "www.metager.de") {
return 301 https://metager.de$request_uri;
}
if ($host = "www.metager.org") {
return 301 https://metager.org$request_uri;
}
if ($host = "www.metager.es") {
return 301 https://metager.es$request_uri;
}
if ($host = "klassik.metager.org") {
return 301 https://metager.de$request_uri;
}
}
\ No newline at end of file
......@@ -4,7 +4,6 @@ namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
use Monospice\LaravelRedisSentinel\RedisSentinel;
class AppendLogs extends Command
{
......@@ -47,22 +46,17 @@ class AppendLogs extends Command
{
$redis = null;
if (config("database.redis.cache.driver", "redis") === "redis") {
$redis = Redis::connection('cache');
} elseif (config("database.redis.cache.driver", "redis") === "redis-sentinel") {
$redis = RedisSentinel::connection('cache');
}
$redis = Redis::connection(config('cache.stores.redis.connection'));
if ($redis === null) {
$this->error("No valid Redis Connection specified");
return;
}
$elements = [];
$reply = $redis->pipeline(function ($pipe) use ($elements) {
$pipe->lrange(\App\Console\Commands\AppendLogs::LOGKEY, 0, -1);
$pipe->del(\App\Console\Commands\AppendLogs::LOGKEY);
});
$elements = $reply[0];
$elementCount = $redis->llen(\App\Console\Commands\AppendLogs::LOGKEY);
$elements = $redis->lpop(\App\Console\Commands\AppendLogs::LOGKEY, $elementCount);
if (!is_array($elements) || sizeof($elements) <= 0) {
return;
}
......
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Redis;
class LoadAffiliateBlacklist extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'load:affiliate-blacklist';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Loads the Affiliate Blacklist from DB into Redis';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
$blacklistItems = DB::table("affiliate_blacklist", "b")
->select("hostname")
->where("blacklist", true)
->get();
Redis::pipeline(function ($redis) use ($blacklistItems) {
$redisKey = \App\Http\Controllers\AdgoalController::REDIS_BLACKLIST_KEY;
$redis->del($redisKey);
foreach ($blacklistItems as $item) {
$hostname = $item->hostname;
$redis->hset(\App\Http\Controllers\AdgoalController::REDIS_BLACKLIST_KEY, $hostname, true);
}
});
return 0;
}
}
......@@ -82,7 +82,7 @@ class RequestFetcher extends Command
$answersRead = $status[0];
$messagesLeft = $status[1];
$newJobs = $this->checkNewJobs($operationsRunning, $messagesLeft);
if ($newJobs === 0 && $answersRead === 0) {
usleep(10 * 1000);
}
......@@ -188,11 +188,11 @@ class RequestFetcher extends Command
CURLOPT_RETURNTRANSFER => 1,
CURLOPT_USERAGENT => $job["useragent"],
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_CONNECTTIMEOUT => 2,
CURLOPT_CONNECTTIMEOUT => 8,
CURLOPT_MAXCONNECTS => 500,
CURLOPT_LOW_SPEED_LIMIT => 50000,
CURLOPT_LOW_SPEED_TIME => 5,
CURLOPT_TIMEOUT => 7,
CURLOPT_LOW_SPEED_TIME => 10,
CURLOPT_TIMEOUT => 10,
));
if (!empty($job["curlopts"])) {
......@@ -201,7 +201,7 @@ class RequestFetcher extends Command
if (!empty($this->proxyhost) && !empty($this->proxyport)) {
curl_setopt($ch, CURLOPT_PROXY, $this->proxyhost);
if(!empty($this->proxyuser) && !empty($this->proxypassword)){
if (!empty($this->proxyuser) && !empty($this->proxypassword)) {
curl_setopt($ch, CURLOPT_PROXYUSERPWD, $this->proxyuser . ":" . $this->proxypassword);
}
curl_setopt($ch, CURLOPT_PROXYPORT, $this->proxyport);
......@@ -228,6 +228,6 @@ class RequestFetcher extends Command
public function sig_handler($sig)
{
$this->shouldRun = false;
echo("Terminating Process\n");
echo ("Terminating Process\n");
}
}
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use \App\Http\Controllers\AdgoalController;
use Illuminate\Support\Facades\DB;
class StorePartnerCalls extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'affilliates:store';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Stores cached clicks on affiliate links into DB.';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
AdgoalController::storePartnerCalls();
# Remove old entries
# The duration in hours for entries to last is defined as constant in AdgoalController
DB::delete("delete from affiliate_clicks where created_at < DATE_SUB(NOW(), INTERVAL ? HOUR);", [\App\Http\Controllers\AdgoalController::STORAGE_DURATION_HOURS]);
return 0;
}
}
......@@ -13,9 +13,7 @@ class Kernel extends ConsoleKernel
*
* @var array
*/
protected $commands = [
];
protected $commands = [];
/**
* Define the application's command schedule.
......@@ -30,6 +28,9 @@ class Kernel extends ConsoleKernel
$schedule->command('requests:useragents')->everyFiveMinutes();
$schedule->command('logs:gather')->everyMinute();
$schedule->command('spam:load')->everyMinute();
$schedule->command('load:affiliate-blacklist')->everyMinute();
$schedule->command('affilliates:store')->everyMinute()
->onOneServer();
$schedule->call(function () {
DB::table('monthlyrequests')->truncate();
DB::disconnect('mysql');
......
<?php
namespace App\Http\Controllers;
use Illuminate\Http\Request;
use LaravelLocalization;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Redis;
use Illuminate\Support\Facades\Validator;
/**
* Before we redirect users to the affiliate shops we will track the clicks ourself.
* Reason is that many Affiliate shops redirect to invalid URLs (404, 500, ... Errors) which leads to bad user experience.
* We will store the clicked Affiliate Link together with the final URL the user should land on.
* No userdata or other metadata will be stored together with that information.
*
* That way we can do manual validation of affiliate links and exclude bad partnershop links to be shown in the future. Since this will be
* a lot of links we count the clicks so we can validate the most used ones first.
*/
class AdgoalController extends Controller
{
# Data will be stored for 24 hours
const STORAGE_DURATION_HOURS = 24;
const REDIS_STORAGE_KEY = "affiliate_click";
const REDIS_BLACKLIST_KEY = "affiliate_blacklist";
/**
* This function is called when a user clicks on a affiliate link. It will first validate that the URL
* was generated by us to prevent random redirect links to be manually created for our domains.
* After that we will store the necessary information (link and affiliate link) into our database.
* After that we will redirect the user to the affiliate shop
*/
public function forward(Request $request)
{
// $link = "https://metager.de";
// $affillink = "https://test.de";
// $password = self::generatePassword($affillink, $link);
// dd(route('adgoal-redirect', ["link" => $link, "affillink" => $affillink, "password" => $password]));
/**
* Get Parameters (Result informations)
* 1. affillink (With Affiliate Redirect)
* 2. link
* 5. Password (hmac with adgoal private key and the two parameters)
*/
$request->validate([
'affillink' => ['required', 'url', 'active_url'],
'link' => ['required', 'url', 'active_url'],
# Validation of redirect request so that one cannot generate random redirect URLs pointing to our domains
'password' => function ($attribute, $value, $fail) use ($request) {
// Check if hmac matches
$correctPassword = self::generatePassword($request->input('affillink'), $request->input('link'));
if (!hash_equals($correctPassword, $value)) {
$fail('The given password is incorrect!');
}
}
]);
$this->storePartnerCallFast($request->input('affillink'), $request->input('link'));
return redirect($request->input('affillink'));
}
/**
* Stores Click information into Redis Cache for fast execution since this is synchronous call
* at search time.
* A Cronjob will pick the data up and store it into Mariadb later (see self::storePartnerCall)
*/
private function storePartnerCallFast($affillink, $link)
{
# Generate Data to store
$host = parse_url($link, PHP_URL_HOST);
if (empty($host)) {
return;
}
$storeObject = [
"host" => $host,
"affillink" => $affillink,
"link" => $link,
];
# Store Data in Redis
$redis = Redis::connection(config('cache.stores.redis.connection'));
$redis->rpush($this::REDIS_STORAGE_KEY, json_encode($storeObject));
}
public static function storePartnerCalls()
{
$redis = Redis::connection(config('cache.stores.redis.connection'));
DB::transaction(function () use ($redis) {
while (!empty($data = $redis->lpop(self::REDIS_STORAGE_KEY))) {
$data = json_decode($data, true);
# Insert data into mariadb table
DB::insert(
'insert into affiliate_clicks (hostname, affillink, link) values (?, ?, ?)',
[$data["host"], $data["affillink"], $data["link"]]
);
}
});
}
/**
* Generates a Redirect URL for our partnershops
*/
public static function generateRedirectUrl($affillink, $link)
{
$password = self::generatePassword($affillink, $link);
return LaravelLocalization::getLocalizedURL(
LaravelLocalization::getCurrentLocale(),
route('adgoal-redirect', ["link" => $link, "affillink" => $affillink, "password" => $password])
);
}
/**
* Generates hmac password to validate redirect URLs
*/
public static function generatePassword($affillink, $link)
{
return hash_hmac("sha256", $affillink . $link, config('metager.metager.adgoal.private_key'));
}
/**
* Routes for the Admin Interface
*/
public function adminIndex(Request $request)
{
return view('admin.affiliates.index')
->with('title', "Affilliates Overview - MetaGer")
->with('css', [
mix('/css/admin/affilliates/index.css')
])
->with('darkcss', [
mix('/css/admin/affilliates/index-dark.css')
])
->with('js', [
mix('/js/admin/affilliates.js')
]);
}
public function blacklistJson(Request $request)
{
$validator = Validator::make($request->all(), [
"blacklist" => 'boolean'
]);
if ($validator->fails()) {
return response()->json("Invalid Request Data", 422);
}
$count = 5; # How Many results to return
$skip = 0; # How many results to skip
$blacklist = $request->input('blacklist', true);
$total = DB::select("select count(*) as total_rows from affiliate_blacklist where blacklist = ?", [$blacklist]);
$total = intval($total[0]->{"total_rows"});
$blacklistItems = DB::select('select * from affiliate_blacklist where blacklist = ? order by created_at desc limit ? offset ?', [$blacklist, $count, $skip]);
$result = [
"count" => $count,
"skip" => $skip,
"total" => $total,
"results" => $blacklistItems
];
return response()->json($result);
}
public function addblacklistJson(Request $request)
{
$validator = Validator::make($request->all(), [
"hostname" => [
"required",
function ($attribute, $value, $fail) use ($request) {
# Validate that this is indeed a hostname which is resolvable
if (!filter_var(gethostbyname($request->input("hostname")), FILTER_VALIDATE_IP)) {
$fail("The selected entry is not a valid hostname");
}
},
function ($attribute, $value, $fail) use ($request) {
# Validate that entry does not already exist in database
$entry = DB::select("select * from metager.affiliate_blacklist where hostname = ?", [$request->input("hostname")]);
if (sizeof($entry) !== 0) {
$fail("The selected entry does already exist in database");
}
}
],
]);
if ($validator->fails()) {
return response()->json([
"message" => "Invalid Request Data"
], 422);
}
$hostname = $validator->validated()["hostname"];
$rowsInserted = DB::insert("insert into metager.affiliate_blacklist (hostname, blacklist) values (?, 1)", [$hostname]);
if ($rowsInserted === TRUE) {
return response()->json([
"message" => "Entry added."
]);
} else {
return response()->json([
"message" => "Error inserting entry."
], 422);
}
}
public function deleteblacklistJson(Request $request)
{
$validator = Validator::make($request->all(), [
"id" => ["required", "integer", "min:1", function ($attribute, $value, $fail) use ($request) {
$entry = DB::select("select * from metager.affiliate_blacklist where id = ? and blacklist = ?", [$request->input("id"), true]);
if (sizeof($entry) !== 1) {
$fail("The selected entry does not exist in database");
}
}],
]);
if ($validator->fails()) {
return response()->json("Invalid Request Data", 422);
}
$id = intval($validator->validated()["id"]);
$rowsDeleted = DB::delete("delete from metager.affiliate_blacklist where id = ?", [$id]);
if ($rowsDeleted > 0) {
return response()->json([
"message" => "$rowsDeleted entries deleted."
]);
} else {
return response()->json([
"message" => "Error deleting entry."
], 422);
}
}
public function whitelistJson(Request $request)
{
$input = $request->all();
$input["blacklist"] = false;
$request->replace($input);
return $this->blacklistJson($request);
}
public function addwhitelistJson(Request $request)
{
$validator = Validator::make($request->all(), [
"hostname" => [
"required",
function ($attribute, $value, $fail) use ($request) {
# Validate that this is indeed a hostname which is resolvable
if (!filter_var(gethostbyname($request->input("hostname")), FILTER_VALIDATE_IP)) {
$fail("The selected entry is not a valid hostname");
}
},
function ($attribute, $value, $fail) use ($request) {
# Validate that entry does not already exist in database
$entry = DB::select("select * from metager.affiliate_blacklist where hostname = ?", [$request->input("hostname")]);
if (sizeof($entry) !== 0) {
$fail("The selected entry does already exist in database");
}
}
],
]);
if ($validator->fails()) {
return response()->json([
"message" => "Invalid Request Data"
], 422);
}
$hostname = $validator->validated()["hostname"];
$rowsInserted = DB::insert("insert into metager.affiliate_blacklist (hostname, blacklist) values (?, 0)", [$hostname]);
if ($rowsInserted === TRUE) {
return response()->json([
"message" => "Entry added."
]);
} else {
return response()->json([
"message" => "Error inserting entry."
], 422);
}
}
public function deletewhitelistJson(Request $request)
{
$validator = Validator::make($request->all(), [
"id" => ["required", "integer", "min:1", function ($attribute, $value, $fail) use ($request) {
$entry = DB::select("select * from metager.affiliate_blacklist where id = ? and blacklist = ?", [$request->input("id"), false]);
if (sizeof($entry) !== 1) {
$fail("The selected entry does not exist in database");
}
}],
]);
if ($validator->fails()) {
return response()->json("Invalid Request Data", 422);