Commit d016590b authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Loading and using the blacklist

parent bca4c88b
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Redis;
class LoadAffiliateBlacklist extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'load:affiliate-blacklist';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Loads the Affiliate Blacklist from DB into Redis';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
$blacklistItems = DB::table("affiliate_blacklist", "b")
->select("hostname")
->where("blacklist", true)
->get();
Redis::pipeline(function ($redis) use ($blacklistItems) {
$redisKey = \App\Http\Controllers\AdgoalController::REDIS_BLACKLIST_KEY;
$redis->del($redisKey);
foreach ($blacklistItems as $item) {
$hostname = $item->hostname;
$redis->hset(\App\Http\Controllers\AdgoalController::REDIS_BLACKLIST_KEY, $hostname, true);
}
});
return 0;
}
}
...@@ -13,9 +13,7 @@ class Kernel extends ConsoleKernel ...@@ -13,9 +13,7 @@ class Kernel extends ConsoleKernel
* *
* @var array * @var array
*/ */
protected $commands = [ protected $commands = [];
];
/** /**
* Define the application's command schedule. * Define the application's command schedule.
...@@ -30,6 +28,7 @@ class Kernel extends ConsoleKernel ...@@ -30,6 +28,7 @@ class Kernel extends ConsoleKernel
$schedule->command('requests:useragents')->everyFiveMinutes(); $schedule->command('requests:useragents')->everyFiveMinutes();
$schedule->command('logs:gather')->everyMinute(); $schedule->command('logs:gather')->everyMinute();
$schedule->command('spam:load')->everyMinute(); $schedule->command('spam:load')->everyMinute();
$schedule->command('load:affiliate-blacklist')->everyMinute();
$schedule->command('affilliates:store')->everyMinute() $schedule->command('affilliates:store')->everyMinute()
->onOneServer(); ->onOneServer();
$schedule->call(function () { $schedule->call(function () {
......
...@@ -23,6 +23,7 @@ class AdgoalController extends Controller ...@@ -23,6 +23,7 @@ class AdgoalController extends Controller
# Data will be stored for 24 hours # Data will be stored for 24 hours
const STORAGE_DURATION_HOURS = 24; const STORAGE_DURATION_HOURS = 24;
const REDIS_STORAGE_KEY = "affiliate_click"; const REDIS_STORAGE_KEY = "affiliate_click";
const REDIS_BLACKLIST_KEY = "affiliate_blacklist";
/** /**
* This function is called when a user clicks on a affiliate link. It will first validate that the URL * This function is called when a user clicks on a affiliate link. It will first validate that the URL
......
...@@ -10,17 +10,19 @@ use LaravelLocalization; ...@@ -10,17 +10,19 @@ use LaravelLocalization;
class Adgoal class Adgoal
{ {
const COUNTRIES = ["af","al","dz","um","as","vi","ad","ao","ai","ag","ar","am","aw","az","au","eg","gq","et","bs", const COUNTRIES = [
"bh","bd","bb","be","bz","bj","bm","bt","bo","ba","bw","bv","br","vg","io","bn","bg","bf","bi","cl","cn","ck", "af", "al", "dz", "um", "as", "vi", "ad", "ao", "ai", "ag", "ar", "am", "aw", "az", "au", "eg", "gq", "et", "bs",
"cr","ci","dk","de","dm","do","dj","ec","sv","er","ee","eu","fk","fo","fj","fi","fr","gf","pf","tf","ga","gm", "bh", "bd", "bb", "be", "bz", "bj", "bm", "bt", "bo", "ba", "bw", "bv", "br", "vg", "io", "bn", "bg", "bf", "bi", "cl", "cn", "ck",
"ge","gh","gi","gd","gr","gb","uk","gl","gp","gu","gt","gn","gw","gy","ht","hm","hn","hk","in","id","iq","ir", "cr", "ci", "dk", "de", "dm", "do", "dj", "ec", "sv", "er", "ee", "eu", "fk", "fo", "fj", "fi", "fr", "gf", "pf", "tf", "ga", "gm",
"ie","is","il","it","jm","sj","jp","ye","jo","yu","ky","kh","cm","ca","cv","kz","qa","ke","kg","ki","cc","co", "ge", "gh", "gi", "gd", "gr", "gb", "uk", "gl", "gp", "gu", "gt", "gn", "gw", "gy", "ht", "hm", "hn", "hk", "in", "id", "iq", "ir",
"km","cg","cd","hr","cu","kw","la","ls","lv","lb","lr","ly","li","lt","lu","mo","mg","mw","my","mv","ml","mt", "ie", "is", "il", "it", "jm", "sj", "jp", "ye", "jo", "yu", "ky", "kh", "cm", "ca", "cv", "kz", "qa", "ke", "kg", "ki", "cc", "co",
"mp","ma","mh","mq","mr","mu","yt","mk","mx","fm","md","mc","mn","ms","mz","mm","na","nr","np","nc","nz","ni", "km", "cg", "cd", "hr", "cu", "kw", "la", "ls", "lv", "lb", "lr", "ly", "li", "lt", "lu", "mo", "mg", "mw", "my", "mv", "ml", "mt",
"nl","an","ne","ng","nu","kp","nf","no","om","tp","at","pk","pw","ps","pa","pg","py","pe","ph","pn","pl","pt", "mp", "ma", "mh", "mq", "mr", "mu", "yt", "mk", "mx", "fm", "md", "mc", "mn", "ms", "mz", "mm", "na", "nr", "np", "nc", "nz", "ni",
"pr","re","rw","ro","ru","st","sb","zm","ws","sm","sa","se","ch","sn","sc","sl","zw","sg","sk","si","so","es", "nl", "an", "ne", "ng", "nu", "kp", "nf", "no", "om", "tp", "at", "pk", "pw", "ps", "pa", "pg", "py", "pe", "ph", "pn", "pl", "pt",
"lk","sh","kn","lc","pm","vc","sd","sr","za","kr","sz","sy","tj","tw","tz","th","tg","to","tt","td","cz","tn", "pr", "re", "rw", "ro", "ru", "st", "sb", "zm", "ws", "sm", "sa", "se", "ch", "sn", "sc", "sl", "zw", "sg", "sk", "si", "so", "es",
"tm","tc","tv","tr","us","ug","ua","xx","hu","uy","uz","vu","va","ve","ae","vn","wf","cx","by","eh","ww","zr","cf","cy",]; "lk", "sh", "kn", "lc", "pm", "vc", "sd", "sr", "za", "kr", "sz", "sy", "tj", "tw", "tz", "th", "tg", "to", "tt", "td", "cz", "tn",
"tm", "tc", "tv", "tr", "us", "ug", "ua", "xx", "hu", "uy", "uz", "vu", "va", "ve", "ae", "vn", "wf", "cx", "by", "eh", "ww", "zr", "cf", "cy",
];
public $hash; public $hash;
...@@ -55,17 +57,17 @@ class Adgoal ...@@ -55,17 +57,17 @@ class Adgoal
} }
$linkList .= $link . ","; $linkList .= $link . ",";
} }
if(empty($linkList)){ if (empty($linkList)) {
return; return;
} }
$linkList = rtrim($linkList, ","); $linkList = rtrim($linkList, ",");
# Hashwert # Hashwert
$this->hash = md5($linkList . $privateKey); $this->hash = md5($linkList . $privateKey);
$link = "https://xf.gdprvalidate.de/v4/check"; $link = "https://xf.gdprvalidate.de/v4/check";
# Which country to use # Which country to use
# Will be de for metager.de and en for metager.org # Will be de for metager.de and en for metager.org
$country = "de"; $country = "de";
...@@ -75,9 +77,9 @@ class Adgoal ...@@ -75,9 +77,9 @@ class Adgoal
$preferredLanguage = Request::getPreferredLanguage(); $preferredLanguage = Request::getPreferredLanguage();
if (!empty($preferredLanguage)) { if (!empty($preferredLanguage)) {
if (str_contains($preferredLanguage, "_")) { if (str_contains($preferredLanguage, "_")) {
$preferredLanguage = substr($preferredLanguage, stripos($preferredLanguage, "_")+1); $preferredLanguage = substr($preferredLanguage, stripos($preferredLanguage, "_") + 1);
} elseif (str_contains($preferredLanguage, "-")) { } elseif (str_contains($preferredLanguage, "-")) {
$preferredLanguage = substr($preferredLanguage, stripos($preferredLanguage, "-")+1); $preferredLanguage = substr($preferredLanguage, stripos($preferredLanguage, "-") + 1);
} }
$preferredLanguage = strtolower($preferredLanguage); $preferredLanguage = strtolower($preferredLanguage);
...@@ -86,45 +88,46 @@ class Adgoal ...@@ -86,45 +88,46 @@ class Adgoal
$country = $preferredLanguage; $country = $preferredLanguage;
} }
} }
$postfields = [ $postfields = [
"key" => $publicKey, "key" => $publicKey,
"panel" => "ZMkW9eSKJS", "panel" => "ZMkW9eSKJS",
"member" => "338b9Bnm", "member" => "338b9Bnm",
"signature" => $this->hash, "signature" => $this->hash,
"links" => $linkList, "links" => $linkList,
"country" => $country, "country" => $country,
]; ];
// Submit fetch job to worker // Submit fetch job to worker
$mission = [ $mission = [
"resulthash" => $this->hash, "resulthash" => $this->hash,
"url" => $link, "url" => $link,
"useragent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0", "useragent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0",
"username" => null, "username" => null,
"password" => null, "password" => null,
"headers" => [ "headers" => [
"Content-Type" => "application/x-www-form-urlencoded" "Content-Type" => "application/x-www-form-urlencoded"
], ],
"cacheDuration" => 60, "cacheDuration" => 60,
"name" => "Adgoal", "name" => "Adgoal",
"curlopts" => [ "curlopts" => [
CURLOPT_POST => true, CURLOPT_POST => true,
CURLOPT_POSTFIELDS => \http_build_query($postfields) CURLOPT_POSTFIELDS => \http_build_query($postfields)
] ]
]; ];
$mission = json_encode($mission); $mission = json_encode($mission);
Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission); Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
} }
public function fetchAffiliates($wait = false){ public function fetchAffiliates($wait = false)
if($this->affiliates !== null){ {
if ($this->affiliates !== null) {
return; return;
} }
$answer = null; $answer = null;
$startTime = microtime(true); $startTime = microtime(true);
if($wait){ if ($wait) {
while (microtime(true) - $startTime < 5) { while (microtime(true) - $startTime < 5) {
$answer = Cache::get($this->hash); $answer = Cache::get($this->hash);
if ($answer === null) { if ($answer === null) {
...@@ -133,24 +136,24 @@ class Adgoal ...@@ -133,24 +136,24 @@ class Adgoal
break; break;
} }
} }
}else{ } else {
$answer = Cache::get($this->hash); $answer = Cache::get($this->hash);
} }
$answer = json_decode($answer, true); $answer = json_decode($answer, true);
// If the fetcher had an Error // If the fetcher had an Error
if($answer === "no-result"){ if ($answer === "no-result") {
$this->affiliates = []; $this->affiliates = [];
return; return;
} }
if(empty($answer) && !is_array($answer)){ if (empty($answer) && !is_array($answer)) {
return; return;
} }
$this->affiliates = $answer; $this->affiliates = $answer;
} }
/** /**
* Converts all Affiliate Links. * Converts all Affiliate Links.
* *
...@@ -158,17 +161,17 @@ class Adgoal ...@@ -158,17 +161,17 @@ class Adgoal
*/ */
public function parseAffiliates(&$results) public function parseAffiliates(&$results)
{ {
if($this->finished || $this->affiliates === null){ if ($this->finished || $this->affiliates === null) {
return; return;
} }
foreach ($this->affiliates as $partnershop) { foreach ($this->affiliates as $partnershop) {
$targetUrl = $partnershop["url"]; $targetUrl = $partnershop["url"];
$tld = $partnershop["tld"]; $tld = $partnershop["tld"];
// Sometimes TLD is null // Sometimes TLD is null
if(empty($tld)){ if (empty($tld)) {
continue; continue;
} }
...@@ -178,13 +181,18 @@ class Adgoal ...@@ -178,13 +181,18 @@ class Adgoal
Adgoal sometimes returns affiliate Links for every URL Adgoal sometimes returns affiliate Links for every URL
That's why we check if the corresponding TLD matches the orginial URL That's why we check if the corresponding TLD matches the orginial URL
*/ */
if($targetHost !== false && stripos($targetHost, $tld) === false){ if ($targetHost !== false && stripos($targetHost, $tld) === false) {
continue; continue;
} }
$preference = config("metager.metager.affiliate_preference", "adgoal"); $preference = config("metager.metager.affiliate_preference", "adgoal");
foreach ($results as $result) { foreach ($results as $result) {
if ($result->originalLink === $targetUrl && (config("metager.metager.affiliate_preference", "adgoal") === "adgoal" || !$result->partnershop)) { if ($result->originalLink === $targetUrl && (config("metager.metager.affiliate_preference", "adgoal") === "adgoal" || !$result->partnershop)) {
# Ein Advertiser gefunden # Ein Advertiser gefunden
# Check ob er auf der Blacklist steht
if (Redis::hexists(\App\Http\Controllers\AdgoalController::REDIS_BLACKLIST_KEY, $targetHost)) {
continue;
}
if ($result->image !== "" && !$result->partnershop) { if ($result->image !== "" && !$result->partnershop) {
$result->logo = $partnershop["logo"]; $result->logo = $partnershop["logo"];
} else { } else {
...@@ -201,7 +209,7 @@ class Adgoal ...@@ -201,7 +209,7 @@ class Adgoal
} }
} }
} }
$requestTime = microtime(true) - $this->startTime; $requestTime = microtime(true) - $this->startTime;
\App\PrometheusExporter::Duration($requestTime, "adgoal"); \App\PrometheusExporter::Duration($requestTime, "adgoal");
$this->finished = true; $this->finished = true;
......
...@@ -5,5 +5,6 @@ set -e ...@@ -5,5 +5,6 @@ set -e
/bin/sh -c "/html/helpers/entrypoint.sh" /bin/sh -c "/html/helpers/entrypoint.sh"
php artisan spam:load php artisan spam:load
php artisan load:affiliate-blacklist
php-fpm7.4 -F -R php-fpm7.4 -F -R
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment