Skip to content
Snippets Groups Projects
Commit d016590b authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Loading and using the blacklist

parent bca4c88b
No related branches found
No related tags found
3 merge requests!1937Development,!1915Development,!1914Resolve "Create Blacklist and Whitelist for affiliate links"
Hide whitespace changes
Inline Side-by-side
app/Console/Commands/LoadAffiliateBlacklist.php 0 → 100644
+ 58
− 0
View file @ d016590b
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Redis;
class LoadAffiliateBlacklist extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'load:affiliate-blacklist';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Loads the Affiliate Blacklist from DB into Redis';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
$blacklistItems = DB::table("affiliate_blacklist", "b")
->select("hostname")
->where("blacklist", true)
->get();
Redis::pipeline(function ($redis) use ($blacklistItems) {
$redisKey = \App\Http\Controllers\AdgoalController::REDIS_BLACKLIST_KEY;
$redis->del($redisKey);
foreach ($blacklistItems as $item) {
$hostname = $item->hostname;
$redis->hset(\App\Http\Controllers\AdgoalController::REDIS_BLACKLIST_KEY, $hostname, true);
}
});
return 0;
}
}
app/Console/Kernel.php
+ 2
− 3
View file @ d016590b
......@@ -13,9 +13,7 @@ class Kernel extends ConsoleKernel
*
* @var array
*/
protected $commands = [
];
protected $commands = [];
/**
* Define the application's command schedule.
......@@ -30,6 +28,7 @@ class Kernel extends ConsoleKernel
$schedule->command('requests:useragents')->everyFiveMinutes();
$schedule->command('logs:gather')->everyMinute();
$schedule->command('spam:load')->everyMinute();
$schedule->command('load:affiliate-blacklist')->everyMinute();
$schedule->command('affilliates:store')->everyMinute()
->onOneServer();
$schedule->call(function () {
......
app/Http/Controllers/AdgoalController.php
+ 1
− 0
View file @ d016590b
......@@ -23,6 +23,7 @@ class AdgoalController extends Controller
# Data will be stored for 24 hours
const STORAGE_DURATION_HOURS = 24;
const REDIS_STORAGE_KEY = "affiliate_click";
const REDIS_BLACKLIST_KEY = "affiliate_blacklist";
/**
* This function is called when a user clicks on a affiliate link. It will first validate that the URL
......
app/Models/Adgoal.php
+ 63
− 55
View file @ d016590b
......@@ -10,17 +10,19 @@ use LaravelLocalization;
class Adgoal
{
const COUNTRIES = ["af","al","dz","um","as","vi","ad","ao","ai","ag","ar","am","aw","az","au","eg","gq","et","bs",
"bh","bd","bb","be","bz","bj","bm","bt","bo","ba","bw","bv","br","vg","io","bn","bg","bf","bi","cl","cn","ck",
"cr","ci","dk","de","dm","do","dj","ec","sv","er","ee","eu","fk","fo","fj","fi","fr","gf","pf","tf","ga","gm",
"ge","gh","gi","gd","gr","gb","uk","gl","gp","gu","gt","gn","gw","gy","ht","hm","hn","hk","in","id","iq","ir",
"ie","is","il","it","jm","sj","jp","ye","jo","yu","ky","kh","cm","ca","cv","kz","qa","ke","kg","ki","cc","co",
"km","cg","cd","hr","cu","kw","la","ls","lv","lb","lr","ly","li","lt","lu","mo","mg","mw","my","mv","ml","mt",
"mp","ma","mh","mq","mr","mu","yt","mk","mx","fm","md","mc","mn","ms","mz","mm","na","nr","np","nc","nz","ni",
"nl","an","ne","ng","nu","kp","nf","no","om","tp","at","pk","pw","ps","pa","pg","py","pe","ph","pn","pl","pt",
"pr","re","rw","ro","ru","st","sb","zm","ws","sm","sa","se","ch","sn","sc","sl","zw","sg","sk","si","so","es",
"lk","sh","kn","lc","pm","vc","sd","sr","za","kr","sz","sy","tj","tw","tz","th","tg","to","tt","td","cz","tn",
"tm","tc","tv","tr","us","ug","ua","xx","hu","uy","uz","vu","va","ve","ae","vn","wf","cx","by","eh","ww","zr","cf","cy",];
const COUNTRIES = [
"af", "al", "dz", "um", "as", "vi", "ad", "ao", "ai", "ag", "ar", "am", "aw", "az", "au", "eg", "gq", "et", "bs",
"bh", "bd", "bb", "be", "bz", "bj", "bm", "bt", "bo", "ba", "bw", "bv", "br", "vg", "io", "bn", "bg", "bf", "bi", "cl", "cn", "ck",
"cr", "ci", "dk", "de", "dm", "do", "dj", "ec", "sv", "er", "ee", "eu", "fk", "fo", "fj", "fi", "fr", "gf", "pf", "tf", "ga", "gm",
"ge", "gh", "gi", "gd", "gr", "gb", "uk", "gl", "gp", "gu", "gt", "gn", "gw", "gy", "ht", "hm", "hn", "hk", "in", "id", "iq", "ir",
"ie", "is", "il", "it", "jm", "sj", "jp", "ye", "jo", "yu", "ky", "kh", "cm", "ca", "cv", "kz", "qa", "ke", "kg", "ki", "cc", "co",
"km", "cg", "cd", "hr", "cu", "kw", "la", "ls", "lv", "lb", "lr", "ly", "li", "lt", "lu", "mo", "mg", "mw", "my", "mv", "ml", "mt",
"mp", "ma", "mh", "mq", "mr", "mu", "yt", "mk", "mx", "fm", "md", "mc", "mn", "ms", "mz", "mm", "na", "nr", "np", "nc", "nz", "ni",
"nl", "an", "ne", "ng", "nu", "kp", "nf", "no", "om", "tp", "at", "pk", "pw", "ps", "pa", "pg", "py", "pe", "ph", "pn", "pl", "pt",
"pr", "re", "rw", "ro", "ru", "st", "sb", "zm", "ws", "sm", "sa", "se", "ch", "sn", "sc", "sl", "zw", "sg", "sk", "si", "so", "es",
"lk", "sh", "kn", "lc", "pm", "vc", "sd", "sr", "za", "kr", "sz", "sy", "tj", "tw", "tz", "th", "tg", "to", "tt", "td", "cz", "tn",
"tm", "tc", "tv", "tr", "us", "ug", "ua", "xx", "hu", "uy", "uz", "vu", "va", "ve", "ae", "vn", "wf", "cx", "by", "eh", "ww", "zr", "cf", "cy",
];
public $hash;
......@@ -55,17 +57,17 @@ class Adgoal
}
$linkList .= $link . ",";
}
if(empty($linkList)){
if (empty($linkList)) {
return;
}
$linkList = rtrim($linkList, ",");
# Hashwert
$this->hash = md5($linkList . $privateKey);
$link = "https://xf.gdprvalidate.de/v4/check";
# Which country to use
# Will be de for metager.de and en for metager.org
$country = "de";
......@@ -75,9 +77,9 @@ class Adgoal
$preferredLanguage = Request::getPreferredLanguage();
if (!empty($preferredLanguage)) {
if (str_contains($preferredLanguage, "_")) {
$preferredLanguage = substr($preferredLanguage, stripos($preferredLanguage, "_")+1);
$preferredLanguage = substr($preferredLanguage, stripos($preferredLanguage, "_") + 1);
} elseif (str_contains($preferredLanguage, "-")) {
$preferredLanguage = substr($preferredLanguage, stripos($preferredLanguage, "-")+1);
$preferredLanguage = substr($preferredLanguage, stripos($preferredLanguage, "-") + 1);
}
$preferredLanguage = strtolower($preferredLanguage);
......@@ -86,45 +88,46 @@ class Adgoal
$country = $preferredLanguage;
}
}
$postfields = [
"key" => $publicKey,
"panel" => "ZMkW9eSKJS",
"member" => "338b9Bnm",
"signature" => $this->hash,
"links" => $linkList,
"country" => $country,
];
"key" => $publicKey,
"panel" => "ZMkW9eSKJS",
"member" => "338b9Bnm",
"signature" => $this->hash,
"links" => $linkList,
"country" => $country,
];
// Submit fetch job to worker
$mission = [
"resulthash" => $this->hash,
"url" => $link,
"useragent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0",
"username" => null,
"password" => null,
"headers" => [
"Content-Type" => "application/x-www-form-urlencoded"
],
"cacheDuration" => 60,
"name" => "Adgoal",
"curlopts" => [
CURLOPT_POST => true,
CURLOPT_POSTFIELDS => \http_build_query($postfields)
]
];
"resulthash" => $this->hash,
"url" => $link,
"useragent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0",
"username" => null,
"password" => null,
"headers" => [
"Content-Type" => "application/x-www-form-urlencoded"
],
"cacheDuration" => 60,
"name" => "Adgoal",
"curlopts" => [
CURLOPT_POST => true,
CURLOPT_POSTFIELDS => \http_build_query($postfields)
]
];
$mission = json_encode($mission);
Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
}
public function fetchAffiliates($wait = false){
if($this->affiliates !== null){
public function fetchAffiliates($wait = false)
{
if ($this->affiliates !== null) {
return;
}
$answer = null;
$startTime = microtime(true);
if($wait){
if ($wait) {
while (microtime(true) - $startTime < 5) {
$answer = Cache::get($this->hash);
if ($answer === null) {
......@@ -133,24 +136,24 @@ class Adgoal
break;
}
}
}else{
} else {
$answer = Cache::get($this->hash);
}
$answer = json_decode($answer, true);
// If the fetcher had an Error
if($answer === "no-result"){
if ($answer === "no-result") {
$this->affiliates = [];
return;
}
if(empty($answer) && !is_array($answer)){
if (empty($answer) && !is_array($answer)) {
return;
}
$this->affiliates = $answer;
}
/**
* Converts all Affiliate Links.
*
......@@ -158,17 +161,17 @@ class Adgoal
*/
public function parseAffiliates(&$results)
{
if($this->finished || $this->affiliates === null){
if ($this->finished || $this->affiliates === null) {
return;
}
foreach ($this->affiliates as $partnershop) {
$targetUrl = $partnershop["url"];
$tld = $partnershop["tld"];
// Sometimes TLD is null
if(empty($tld)){
if (empty($tld)) {
continue;
}
......@@ -178,13 +181,18 @@ class Adgoal
Adgoal sometimes returns affiliate Links for every URL
That's why we check if the corresponding TLD matches the orginial URL
*/
if($targetHost !== false && stripos($targetHost, $tld) === false){
if ($targetHost !== false && stripos($targetHost, $tld) === false) {
continue;
}
$preference = config("metager.metager.affiliate_preference", "adgoal");
foreach ($results as $result) {
if ($result->originalLink === $targetUrl && (config("metager.metager.affiliate_preference", "adgoal") === "adgoal" || !$result->partnershop)) {
# Ein Advertiser gefunden
# Check ob er auf der Blacklist steht
if (Redis::hexists(\App\Http\Controllers\AdgoalController::REDIS_BLACKLIST_KEY, $targetHost)) {
continue;
}
if ($result->image !== "" && !$result->partnershop) {
$result->logo = $partnershop["logo"];
} else {
......@@ -201,7 +209,7 @@ class Adgoal
}
}
}
$requestTime = microtime(true) - $this->startTime;
\App\PrometheusExporter::Duration($requestTime, "adgoal");
$this->finished = true;
......
helpers/entrypointProduction.sh
+ 1
− 0
View file @ d016590b
......@@ -5,5 +5,6 @@ set -e
/bin/sh -c "/html/helpers/entrypoint.sh"
php artisan spam:load
php artisan load:affiliate-blacklist
php-fpm7.4 -F -R
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment