Merge branch 'development' into 1142-include-key-for-paypal-donations

.env.example

@@ -22,6 +22,7 @@ CACHE_DRIVER=redis
 SESSION_DRIVER=file
 QUEUE_CONNECTION=sync
 
+REDIS_CACHE_CONNECTION=cache
 REDIS_CACHE_HOST=redis
 REDIS_HOST=redis
 REDIS_PASSWORD=null

.gitlab-ci.yml

@@ -92,7 +92,6 @@ stop_review:
     - auto-deploy delete rollout
     - auto-deploy persist_environment_url
   variables:
-    ADDITIONAL_HOSTS: "www.metager3.de,test.metager.de"
     HELM_UPGRADE_VALUES_FILE: .gitlab/development-values.yaml
     ROLLOUT_RESOURCE_TYPE: deployment
   environment:
@@ -120,7 +119,7 @@ development:
 
 production:
   variables:
-    ADDITIONAL_HOSTS: "www.metager.de,metager.org,www.metager.org,metager.es,www.metager.es,klassik.metager.org"
+    ADDITIONAL_HOSTS: "metager.org,metager.es"
     HELM_UPGRADE_VALUES_FILE: .gitlab/production-values.yaml
     ROLLOUT_RESOURCE_TYPE: deployment
   environment:

.gitlab/development-values.yaml

@@ -50,8 +50,11 @@ podAnnotations:
   prometheus.io/port: "80"
 deploymentApiVersion: apps/v1
 ingress:
+  tls:
+    enabled: true
+    acme: false
+    secretName: "metager-tls"
   annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-prod
     nginx.ingress.kubernetes.io/client-body-buffer-size: "30m"
     nginx.ingress.kubernetes.io/proxy-body-size: "30m"
     nginx.ingress.kubernetes.io/configuration-snippet: |
@@ -64,6 +67,3 @@ ingress:
       if ($arg_out = "results-with-style") {
         more_set_headers "X-Frame-Options: allow-from https://scripts.zdv.uni-mainz.de/";
       }
-      if ($host = "www.metager3.de") {
-          return 301 https://metager3.de$request_uri;
-      }

.gitlab/production-values.yaml

@@ -11,11 +11,8 @@ podDisruptionBudget:
   enabled: true
   minAvailable: 1
   maxUnavailable:
-podAnnotations:
-  prometheus.io/scrape: "true"
-  prometheus.io/path: /metrics
-  prometheus.io/port: "80"
-deploymentApiVersion: apps/v1
+prometheus:
+  metrics: true
 livenessProbe:
   initialDelaySeconds: 0
 readinessProbe:
@@ -55,23 +52,9 @@ resources:
 ingress:
   tls:
     enabled: true
-    custom:
-      tls:
-      - hosts:
-        - metager.de
-        - www.metager.de
-        secretName: metager-de-tls
-      - hosts:
-        - metager.org
-        - www.metager.org
-        - klassik.metager.org
-        secretName: metager-org-tls
-      - hosts:
-        - metager.es
-        - www.metager.es
-        secretName: production-auto-deploy-tls
+    acme: false
+    secretName: "metager-tls"
   annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-prod
     nginx.ingress.kubernetes.io/client-body-buffer-size: "30m"
     nginx.ingress.kubernetes.io/proxy-body-size: "30m"
     nginx.ingress.kubernetes.io/configuration-snippet: |
@@ -84,15 +67,3 @@ ingress:
       if ($arg_out = "results-with-style") {
         more_set_headers "X-Frame-Options: allow-from https://scripts.zdv.uni-mainz.de/";
       }
\ No newline at end of file
-      if ($host = "www.metager.de") {
-          return 301 https://metager.de$request_uri;
-      }
-      if ($host = "www.metager.org") {
-          return 301 https://metager.org$request_uri;
-      }
-      if ($host = "www.metager.es") {
-          return 301 https://metager.es$request_uri;
-      }
-      if ($host = "klassik.metager.org") {
-          return 301 https://metager.de$request_uri;
-      }

app/Console/Commands/AppendLogs.php

@@ -4,7 +4,6 @@ namespace App\Console\Commands;
 
 use Illuminate\Console\Command;
 use Illuminate\Support\Facades\Redis;
-use Monospice\LaravelRedisSentinel\RedisSentinel;
 
 class AppendLogs extends Command
 {
@@ -47,22 +46,17 @@ class AppendLogs extends Command
     {
         $redis = null;
         
-        if (config("database.redis.cache.driver", "redis") === "redis") {
-            $redis = Redis::connection('cache');
-        } elseif (config("database.redis.cache.driver", "redis") === "redis-sentinel") {
-            $redis = RedisSentinel::connection('cache');
-        }
+        $redis = Redis::connection(config('cache.stores.redis.connection'));
+
         if ($redis === null) {
             $this->error("No valid Redis Connection specified");
             return;
         }
 
         $elements = [];
-        $reply = $redis->pipeline(function ($pipe) use ($elements) {
-            $pipe->lrange(\App\Console\Commands\AppendLogs::LOGKEY, 0, -1);
-            $pipe->del(\App\Console\Commands\AppendLogs::LOGKEY);
-        });
-        $elements = $reply[0];
+        $elementCount = $redis->llen(\App\Console\Commands\AppendLogs::LOGKEY);
+        $elements = $redis->lpop(\App\Console\Commands\AppendLogs::LOGKEY, $elementCount);
+
         if (!is_array($elements) || sizeof($elements) <= 0) {
             return;
         }

app/Console/Commands/LoadAffiliateBlacklist.php

+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Redis;
+
+class LoadAffiliateBlacklist extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'load:affiliate-blacklist';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Loads the Affiliate Blacklist from DB into Redis';
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return int
+     */
+    public function handle()
+    {
+        $blacklistItems = DB::table("affiliate_blacklist", "b")
+            ->select("hostname")
+            ->where("blacklist", true)
+            ->get();
+
+        Redis::pipeline(function ($redis) use ($blacklistItems) {
+            $redisKey = \App\Http\Controllers\AdgoalController::REDIS_BLACKLIST_KEY;
+            $redis->del($redisKey);
+            foreach ($blacklistItems as $item) {
+                $hostname = $item->hostname;
+                $redis->hset(\App\Http\Controllers\AdgoalController::REDIS_BLACKLIST_KEY, $hostname, true);
+            }
+        });
+
+        return 0;
+    }
+}

app/Console/Commands/RequestFetcher.php

@@ -188,11 +188,11 @@ class RequestFetcher extends Command
             CURLOPT_RETURNTRANSFER => 1,
             CURLOPT_USERAGENT => $job["useragent"],
             CURLOPT_FOLLOWLOCATION => true,
-            CURLOPT_CONNECTTIMEOUT => 2,
+            CURLOPT_CONNECTTIMEOUT => 8,
             CURLOPT_MAXCONNECTS => 500,
             CURLOPT_LOW_SPEED_LIMIT => 50000,
-            CURLOPT_LOW_SPEED_TIME => 5,
-            CURLOPT_TIMEOUT => 7,
+            CURLOPT_LOW_SPEED_TIME => 10,
+            CURLOPT_TIMEOUT => 10,
         ));
 
         if (!empty($job["curlopts"])) {
@@ -201,7 +201,7 @@ class RequestFetcher extends Command
 
         if (!empty($this->proxyhost) && !empty($this->proxyport)) {
             curl_setopt($ch, CURLOPT_PROXY, $this->proxyhost);
-            if(!empty($this->proxyuser) && !empty($this->proxypassword)){
+            if (!empty($this->proxyuser) && !empty($this->proxypassword)) {
                 curl_setopt($ch, CURLOPT_PROXYUSERPWD, $this->proxyuser . ":" . $this->proxypassword);
             }
             curl_setopt($ch, CURLOPT_PROXYPORT, $this->proxyport);
@@ -228,6 +228,6 @@ class RequestFetcher extends Command
     public function sig_handler($sig)
     {
         $this->shouldRun = false;
-        echo("Terminating Process\n");
+        echo ("Terminating Process\n");
     }
 }

app/Console/Commands/StorePartnerCalls.php

+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use \App\Http\Controllers\AdgoalController;
+use Illuminate\Support\Facades\DB;
+
+class StorePartnerCalls extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'affilliates:store';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Stores cached clicks on affiliate links into DB.';
+
+    /**
+     * Create a new command instance.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return int
+     */
+    public function handle()
+    {
+        AdgoalController::storePartnerCalls();
+
+        # Remove old entries
+        # The duration in hours for entries to last is defined as constant in AdgoalController
+        DB::delete("delete from affiliate_clicks where created_at < DATE_SUB(NOW(), INTERVAL ? HOUR);", [\App\Http\Controllers\AdgoalController::STORAGE_DURATION_HOURS]);
+
+        return 0;
+    }
+}

app/Console/Kernel.php

@@ -13,9 +13,7 @@ class Kernel extends ConsoleKernel
      *
      * @var array
      */
-    protected $commands = [
-
-    ];
+    protected $commands = [];
 
     /**
      * Define the application's command schedule.
@@ -30,6 +28,9 @@ class Kernel extends ConsoleKernel
         $schedule->command('requests:useragents')->everyFiveMinutes();
         $schedule->command('logs:gather')->everyMinute();
         $schedule->command('spam:load')->everyMinute();
+        $schedule->command('load:affiliate-blacklist')->everyMinute();
+        $schedule->command('affilliates:store')->everyMinute()
+            ->onOneServer();
         $schedule->call(function () {
             DB::table('monthlyrequests')->truncate();
             DB::disconnect('mysql');

app/Http/Controllers/AdgoalController.php

+<?php
+
+namespace App\Http\Controllers;
+
+use Illuminate\Http\Request;
+use LaravelLocalization;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Redis;
+use Illuminate\Support\Facades\Validator;
+
+/**
+ * Before we redirect users to the affiliate shops we will track the clicks ourself.
+ * Reason is that many Affiliate shops redirect to invalid URLs (404, 500, ... Errors) which leads to bad user experience.
+ * We will store the clicked Affiliate Link together with the final URL the user should land on.
+ * No userdata or other metadata will be stored together with that information.
+ * 
+ * That way we can do manual validation of affiliate links and exclude bad partnershop links to be shown in the future. Since this will be
+ * a lot of links we count the clicks so we can validate the most used ones first.
+ */
+
+class AdgoalController extends Controller
+{
+    # Data will be stored for 24 hours
+    const STORAGE_DURATION_HOURS = 24;
+    const REDIS_STORAGE_KEY = "affiliate_click";
+    const REDIS_BLACKLIST_KEY = "affiliate_blacklist";
+
+    /**
+     * This function is called when a user clicks on a affiliate link. It will first validate that the URL
+     * was generated by us to prevent random redirect links to be manually created for our domains.
+     * After that we will store the necessary information (link and affiliate link) into our database.
+     * After that we will redirect the user to the affiliate shop
+     */
+    public function forward(Request $request)
+    {
+        // $link = "https://metager.de";
+        // $affillink = "https://test.de";
+        // $password = self::generatePassword($affillink, $link);
+        // dd(route('adgoal-redirect', ["link" => $link, "affillink" => $affillink, "password" => $password]));
+        /**
+         * Get Parameters (Result informations)
+         * 1. affillink (With Affiliate Redirect)
+         * 2. link
+         * 5. Password (hmac with adgoal private key and the two parameters)
+         */
+        $request->validate([
+            'affillink' => ['required', 'url', 'active_url'],
+            'link' => ['required', 'url', 'active_url'],
+            # Validation of redirect request so that one cannot generate random redirect URLs pointing to our domains
+            'password' => function ($attribute, $value, $fail) use ($request) {
+                // Check if hmac matches
+                $correctPassword = self::generatePassword($request->input('affillink'), $request->input('link'));
+                if (!hash_equals($correctPassword, $value)) {
+                    $fail('The given password is incorrect!');
+                }
+            }
+        ]);
+
+        $this->storePartnerCallFast($request->input('affillink'), $request->input('link'));
+
+        return redirect($request->input('affillink'));
+    }
+
+    /**
+     * Stores Click information into Redis Cache for fast execution since this is synchronous call
+     * at search time.
+     * A Cronjob will pick the data up and store it into Mariadb later (see self::storePartnerCall)
+     */
+    private function storePartnerCallFast($affillink, $link)
+    {
+        # Generate Data to store
+        $host = parse_url($link, PHP_URL_HOST);
+        if (empty($host)) {
+            return;
+        }
+        $storeObject = [
+            "host" => $host,
+            "affillink" => $affillink,
+            "link" => $link,
+        ];
+
+
+        # Store Data in Redis
+        $redis = Redis::connection(config('cache.stores.redis.connection'));
+        $redis->rpush($this::REDIS_STORAGE_KEY, json_encode($storeObject));
+    }
+
+    public static function storePartnerCalls()
+    {
+        $redis = Redis::connection(config('cache.stores.redis.connection'));
+        DB::transaction(function () use ($redis) {
+            while (!empty($data = $redis->lpop(self::REDIS_STORAGE_KEY))) {
+                $data = json_decode($data, true);
+                # Insert data into mariadb table
+                DB::insert(
+                    'insert into affiliate_clicks (hostname, affillink, link) values (?, ?, ?)',
+                    [$data["host"], $data["affillink"], $data["link"]]
+                );
+            }
+        });
+    }
+
+    /**
+     * Generates a Redirect URL for our partnershops
+     */
+    public static function generateRedirectUrl($affillink, $link)
+    {
+        $password = self::generatePassword($affillink, $link);
+        return LaravelLocalization::getLocalizedURL(
+            LaravelLocalization::getCurrentLocale(),
+            route('adgoal-redirect', ["link" => $link, "affillink" => $affillink, "password" => $password])
+        );
+    }
+
+    /**
+     * Generates hmac password to validate redirect URLs
+     */
+    public static function generatePassword($affillink, $link)
+    {
+        return hash_hmac("sha256", $affillink . $link, config('metager.metager.adgoal.private_key'));
+    }
+
+
+    /**
+     * Routes for the Admin Interface
+     */
+    public function adminIndex(Request $request)
+    {
+        return view('admin.affiliates.index')
+            ->with('title', "Affilliates Overview - MetaGer")
+            ->with('css', [
+                mix('/css/admin/affilliates/index.css')
+            ])
+            ->with('darkcss', [
+                mix('/css/admin/affilliates/index-dark.css')
+            ])
+            ->with('js', [
+                mix('/js/admin/affilliates.js')
+            ]);
+    }
+
+    public function blacklistJson(Request $request)
+    {
+        $validator = Validator::make($request->all(), [
+            "blacklist" => 'boolean'
+        ]);
+
+        if ($validator->fails()) {
+            return response()->json("Invalid Request Data", 422);
+        }
+
+        $count = 5; # How Many results to return
+        $skip = 0; # How many results to skip
+        $blacklist = $request->input('blacklist', true);
+
+        $total = DB::select("select count(*) as total_rows from affiliate_blacklist where blacklist = ?", [$blacklist]);
+        $total = intval($total[0]->{"total_rows"});
+        $blacklistItems = DB::select('select * from affiliate_blacklist where blacklist = ? order by created_at desc limit ? offset ?', [$blacklist, $count, $skip]);
+
+        $result = [
+            "count" => $count,
+            "skip" => $skip,
+            "total" => $total,
+            "results" => $blacklistItems
+        ];
+
+        return response()->json($result);
+    }
+
+    public function addblacklistJson(Request $request)
+    {
+        $validator = Validator::make($request->all(), [
+            "hostname" => [
+                "required",
+                function ($attribute, $value, $fail) use ($request) {
+                    # Validate that this is indeed a hostname which is resolvable
+                    if (!filter_var(gethostbyname($request->input("hostname")), FILTER_VALIDATE_IP)) {
+                        $fail("The selected entry is not a valid hostname");
+                    }
+                },
+                function ($attribute, $value, $fail) use ($request) {
+                    # Validate that entry does not already exist in database
+                    $entry = DB::select("select * from metager.affiliate_blacklist where hostname = ?", [$request->input("hostname")]);
+                    if (sizeof($entry) !== 0) {
+                        $fail("The selected entry does already exist in database");
+                    }
+                }
+            ],
+        ]);
+
+        if ($validator->fails()) {
+            return response()->json([
+                "message" => "Invalid Request Data"
+            ], 422);
+        }
+
+        $hostname = $validator->validated()["hostname"];
+
+        $rowsInserted = DB::insert("insert into metager.affiliate_blacklist (hostname, blacklist) values (?, 1)", [$hostname]);
+        if ($rowsInserted === TRUE) {
+            return response()->json([
+                "message" => "Entry added."
+            ]);
+        } else {
+            return response()->json([
+                "message" => "Error inserting entry."
+            ], 422);
+        }
+    }
+
+    public function deleteblacklistJson(Request $request)
+    {
+        $validator = Validator::make($request->all(), [
+            "id" => ["required", "integer", "min:1", function ($attribute, $value, $fail) use ($request) {
+                $entry = DB::select("select * from metager.affiliate_blacklist where id = ? and blacklist = ?", [$request->input("id"), true]);
+                if (sizeof($entry) !== 1) {
+                    $fail("The selected entry does not exist in database");
+                }
+            }],
+        ]);
+
+        if ($validator->fails()) {
+            return response()->json("Invalid Request Data", 422);
+        }
+
+        $id = intval($validator->validated()["id"]);
+        $rowsDeleted = DB::delete("delete from metager.affiliate_blacklist where id = ?", [$id]);
+        if ($rowsDeleted > 0) {
+            return response()->json([
+                "message" => "$rowsDeleted entries deleted."
+            ]);
+        } else {
+            return response()->json([
+                "message" => "Error deleting entry."
+            ], 422);
+        }
+    }
+
+    public function whitelistJson(Request $request)
+    {
+        $input = $request->all();
+        $input["blacklist"] = false;
+        $request->replace($input);
+        return $this->blacklistJson($request);
+    }
+
+    public function addwhitelistJson(Request $request)
+    {
+        $validator = Validator::make($request->all(), [
+            "hostname" => [
+                "required",
+                function ($attribute, $value, $fail) use ($request) {
+                    # Validate that this is indeed a hostname which is resolvable
+                    if (!filter_var(gethostbyname($request->input("hostname")), FILTER_VALIDATE_IP)) {
+                        $fail("The selected entry is not a valid hostname");
+                    }
+                },
+                function ($attribute, $value, $fail) use ($request) {
+                    # Validate that entry does not already exist in database
+                    $entry = DB::select("select * from metager.affiliate_blacklist where hostname = ?", [$request->input("hostname")]);
+                    if (sizeof($entry) !== 0) {
+                        $fail("The selected entry does already exist in database");
+                    }
+                }
+            ],
+        ]);
+
+        if ($validator->fails()) {
+            return response()->json([
+                "message" => "Invalid Request Data"
+            ], 422);
+        }
+
+        $hostname = $validator->validated()["hostname"];
+
+        $rowsInserted = DB::insert("insert into metager.affiliate_blacklist (hostname, blacklist) values (?, 0)", [$hostname]);
+        if ($rowsInserted === TRUE) {
+            return response()->json([
+                "message" => "Entry added."
+            ]);
+        } else {
+            return response()->json([
+                "message" => "Error inserting entry."
+            ], 422);
+        }
+    }
+
+    public function deletewhitelistJson(Request $request)
+    {
+        $validator = Validator::make($request->all(), [
+            "id" => ["required", "integer", "min:1", function ($attribute, $value, $fail) use ($request) {
+                $entry = DB::select("select * from metager.affiliate_blacklist where id = ? and blacklist = ?", [$request->input("id"), false]);
+                if (sizeof($entry) !== 1) {
+                    $fail("The selected entry does not exist in database");
+                }
+            }],
+        ]);
+
+        if ($validator->fails()) {
+            return response()->json("Invalid Request Data", 422);
+        }
+
+        $id = intval($validator->validated()["id"]);
+        $rowsDeleted = DB::delete("delete from metager.affiliate_blacklist where id = ?", [$id]);
+        if ($rowsDeleted > 0) {
+            return response()->json([
+                "message" => "$rowsDeleted entries deleted."