Commit 34d9981e authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Merge branch '1152-create-blacklist-and-whitelist-for-affiliate-links' into 'development'

Resolve "Create Blacklist and Whitelist for affiliate links"

Closes #1152

See merge request !1914
parents 708b5803 d016590b
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Redis;
class LoadAffiliateBlacklist extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'load:affiliate-blacklist';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Loads the Affiliate Blacklist from DB into Redis';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
$blacklistItems = DB::table("affiliate_blacklist", "b")
->select("hostname")
->where("blacklist", true)
->get();
Redis::pipeline(function ($redis) use ($blacklistItems) {
$redisKey = \App\Http\Controllers\AdgoalController::REDIS_BLACKLIST_KEY;
$redis->del($redisKey);
foreach ($blacklistItems as $item) {
$hostname = $item->hostname;
$redis->hset(\App\Http\Controllers\AdgoalController::REDIS_BLACKLIST_KEY, $hostname, true);
}
});
return 0;
}
}
......@@ -13,9 +13,7 @@ class Kernel extends ConsoleKernel
*
* @var array
*/
protected $commands = [
];
protected $commands = [];
/**
* Define the application's command schedule.
......@@ -30,6 +28,7 @@ class Kernel extends ConsoleKernel
$schedule->command('requests:useragents')->everyFiveMinutes();
$schedule->command('logs:gather')->everyMinute();
$schedule->command('spam:load')->everyMinute();
$schedule->command('load:affiliate-blacklist')->everyMinute();
$schedule->command('affilliates:store')->everyMinute()
->onOneServer();
$schedule->call(function () {
......
......@@ -6,6 +6,7 @@ use Illuminate\Http\Request;
use LaravelLocalization;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Redis;
use Illuminate\Support\Facades\Validator;
/**
* Before we redirect users to the affiliate shops we will track the clicks ourself.
......@@ -22,6 +23,7 @@ class AdgoalController extends Controller
# Data will be stored for 24 hours
const STORAGE_DURATION_HOURS = 24;
const REDIS_STORAGE_KEY = "affiliate_click";
const REDIS_BLACKLIST_KEY = "affiliate_blacklist";
/**
* This function is called when a user clicks on a affiliate link. It will first validate that the URL
......@@ -29,7 +31,8 @@ class AdgoalController extends Controller
* After that we will store the necessary information (link and affiliate link) into our database.
* After that we will redirect the user to the affiliate shop
*/
public function forward(Request $request){
public function forward(Request $request)
{
// $link = "https://metager.de";
// $affillink = "https://test.de";
// $password = self::generatePassword($affillink, $link);
......@@ -44,10 +47,10 @@ class AdgoalController extends Controller
'affillink' => ['required', 'url', 'active_url'],
'link' => ['required', 'url', 'active_url'],
# Validation of redirect request so that one cannot generate random redirect URLs pointing to our domains
'password' => function($attribute, $value, $fail) use($request) {
'password' => function ($attribute, $value, $fail) use ($request) {
// Check if hmac matches
$correctPassword = self::generatePassword($request->input('affillink'), $request->input('link'));
if(!hash_equals($correctPassword, $value)){
if (!hash_equals($correctPassword, $value)) {
$fail('The given password is incorrect!');
}
}
......@@ -63,10 +66,11 @@ class AdgoalController extends Controller
* at search time.
* A Cronjob will pick the data up and store it into Mariadb later (see self::storePartnerCall)
*/
private function storePartnerCallFast($affillink, $link) {
private function storePartnerCallFast($affillink, $link)
{
# Generate Data to store
$host = parse_url($link, PHP_URL_HOST);
if(empty($host)){
if (empty($host)) {
return;
}
$storeObject = [
......@@ -81,14 +85,17 @@ class AdgoalController extends Controller
$redis->rpush($this::REDIS_STORAGE_KEY, json_encode($storeObject));
}
public static function storePartnerCalls() {
public static function storePartnerCalls()
{
$redis = Redis::connection(config('cache.stores.redis.connection'));
DB::transaction(function() use($redis){
while(!empty($data = $redis->lpop(self::REDIS_STORAGE_KEY))){
DB::transaction(function () use ($redis) {
while (!empty($data = $redis->lpop(self::REDIS_STORAGE_KEY))) {
$data = json_decode($data, true);
# Insert data into mariadb table
DB::insert('insert into affiliate_clicks (hostname, affillink, link) values (?, ?, ?)',
[$data["host"], $data["affillink"], $data["link"]]);
DB::insert(
'insert into affiliate_clicks (hostname, affillink, link) values (?, ?, ?)',
[$data["host"], $data["affillink"], $data["link"]]
);
}
});
}
......@@ -96,7 +103,8 @@ class AdgoalController extends Controller
/**
* Generates a Redirect URL for our partnershops
*/
public static function generateRedirectUrl($affillink, $link){
public static function generateRedirectUrl($affillink, $link)
{
$password = self::generatePassword($affillink, $link);
return LaravelLocalization::getLocalizedURL(
LaravelLocalization::getCurrentLocale(),
......@@ -107,7 +115,319 @@ class AdgoalController extends Controller
/**
* Generates hmac password to validate redirect URLs
*/
public static function generatePassword($affillink, $link){
public static function generatePassword($affillink, $link)
{
return hash_hmac("sha256", $affillink . $link, config('metager.metager.adgoal.private_key'));
}
/**
* Routes for the Admin Interface
*/
public function adminIndex(Request $request)
{
return view('admin.affiliates.index')
->with('title', "Affilliates Overview - MetaGer")
->with('css', [
mix('/css/admin/affilliates/index.css')
])
->with('darkcss', [
mix('/css/admin/affilliates/index-dark.css')
])
->with('js', [
mix('/js/admin/affilliates.js')
]);
}
public function blacklistJson(Request $request)
{
$validator = Validator::make($request->all(), [
"blacklist" => 'boolean'
]);
if ($validator->fails()) {
return response()->json("Invalid Request Data", 422);
}
$count = 5; # How Many results to return
$skip = 0; # How many results to skip
$blacklist = $request->input('blacklist', true);
$total = DB::select("select count(*) as total_rows from affiliate_blacklist where blacklist = ?", [$blacklist]);
$total = intval($total[0]->{"total_rows"});
$blacklistItems = DB::select('select * from affiliate_blacklist where blacklist = ? order by created_at desc limit ? offset ?', [$blacklist, $count, $skip]);
$result = [
"count" => $count,
"skip" => $skip,
"total" => $total,
"results" => $blacklistItems
];
return response()->json($result);
}
public function addblacklistJson(Request $request)
{
$validator = Validator::make($request->all(), [
"hostname" => [
"required",
function ($attribute, $value, $fail) use ($request) {
# Validate that this is indeed a hostname which is resolvable
if (!filter_var(gethostbyname($request->input("hostname")), FILTER_VALIDATE_IP)) {
$fail("The selected entry is not a valid hostname");
}
},
function ($attribute, $value, $fail) use ($request) {
# Validate that entry does not already exist in database
$entry = DB::select("select * from metager.affiliate_blacklist where hostname = ?", [$request->input("hostname")]);
if (sizeof($entry) !== 0) {
$fail("The selected entry does already exist in database");
}
}
],
]);
if ($validator->fails()) {
return response()->json([
"message" => "Invalid Request Data"
], 422);
}
$hostname = $validator->validated()["hostname"];
$rowsInserted = DB::insert("insert into metager.affiliate_blacklist (hostname, blacklist) values (?, 1)", [$hostname]);
if ($rowsInserted === TRUE) {
return response()->json([
"message" => "Entry added."
]);
} else {
return response()->json([
"message" => "Error inserting entry."
], 422);
}
}
public function deleteblacklistJson(Request $request)
{
$validator = Validator::make($request->all(), [
"id" => ["required", "integer", "min:1", function ($attribute, $value, $fail) use ($request) {
$entry = DB::select("select * from metager.affiliate_blacklist where id = ? and blacklist = ?", [$request->input("id"), true]);
if (sizeof($entry) !== 1) {
$fail("The selected entry does not exist in database");
}
}],
]);
if ($validator->fails()) {
return response()->json("Invalid Request Data", 422);
}
$id = intval($validator->validated()["id"]);
$rowsDeleted = DB::delete("delete from metager.affiliate_blacklist where id = ?", [$id]);
if ($rowsDeleted > 0) {
return response()->json([
"message" => "$rowsDeleted entries deleted."
]);
} else {
return response()->json([
"message" => "Error deleting entry."
], 422);
}
}
public function whitelistJson(Request $request)
{
$input = $request->all();
$input["blacklist"] = false;
$request->replace($input);
return $this->blacklistJson($request);
}
public function addwhitelistJson(Request $request)
{
$validator = Validator::make($request->all(), [
"hostname" => [
"required",
function ($attribute, $value, $fail) use ($request) {
# Validate that this is indeed a hostname which is resolvable
if (!filter_var(gethostbyname($request->input("hostname")), FILTER_VALIDATE_IP)) {
$fail("The selected entry is not a valid hostname");
}
},
function ($attribute, $value, $fail) use ($request) {
# Validate that entry does not already exist in database
$entry = DB::select("select * from metager.affiliate_blacklist where hostname = ?", [$request->input("hostname")]);
if (sizeof($entry) !== 0) {
$fail("The selected entry does already exist in database");
}
}
],
]);
if ($validator->fails()) {
return response()->json([
"message" => "Invalid Request Data"
], 422);
}
$hostname = $validator->validated()["hostname"];
$rowsInserted = DB::insert("insert into metager.affiliate_blacklist (hostname, blacklist) values (?, 0)", [$hostname]);
if ($rowsInserted === TRUE) {
return response()->json([
"message" => "Entry added."
]);
} else {
return response()->json([
"message" => "Error inserting entry."
], 422);
}
}
public function deletewhitelistJson(Request $request)
{
$validator = Validator::make($request->all(), [
"id" => ["required", "integer", "min:1", function ($attribute, $value, $fail) use ($request) {
$entry = DB::select("select * from metager.affiliate_blacklist where id = ? and blacklist = ?", [$request->input("id"), false]);
if (sizeof($entry) !== 1) {
$fail("The selected entry does not exist in database");
}
}],
]);
if ($validator->fails()) {
return response()->json("Invalid Request Data", 422);
}
$id = intval($validator->validated()["id"]);
$rowsDeleted = DB::delete("delete from metager.affiliate_blacklist where id = ?", [$id]);
if ($rowsDeleted > 0) {
return response()->json([
"message" => "$rowsDeleted entries deleted."
]);
} else {
return response()->json([
"message" => "Error deleting entry."
], 422);
}
}
public function hostsJson(Request $request)
{
$validator = Validator::make($request->all(), [
"count" => ["integer", "min:1", "max:50"],
"skip" => ["integer", "min:0"],
]);
if ($validator->fails()) {
return response()->json("Invalid Request Data", 422);
}
$count = intval($request->input("count", 10));
$skip = intval($request->input("skip", 0));
$filter = $request->input("filter", "");
$hostCount = DB::table("metager.affiliate_clicks", "c")
->select(DB::raw("count(distinct c.hostname) as total_hosts"))
->leftJoin("metager.affiliate_blacklist", function ($join) {
$join->on("c.hostname", "=", "metager.affiliate_blacklist.hostname");
})
->where("c.hostname", 'like', "%$filter%")
->whereNull("metager.affiliate_blacklist.hostname")
->get();
$hostCount = $hostCount[0]->{"total_hosts"};
$clickCount = DB::table("metager.affiliate_clicks", "c")
->select(DB::raw("count(*) as click_count"))
->leftJoin("metager.affiliate_blacklist", function ($join) {
$join->on("c.hostname", "=", "metager.affiliate_blacklist.hostname");
})
->where("c.hostname", 'like', "%$filter%")
->whereNull("metager.affiliate_blacklist.hostname")
->get();
$clickCount = $clickCount[0]->{"click_count"};
$hosts = DB::table("metager.affiliate_clicks", "c")
->select("c.hostname", DB::raw('count(c.hostname) as clicks'))
->leftJoin("metager.affiliate_blacklist", function ($join) {
$join->on("c.hostname", "=", "metager.affiliate_blacklist.hostname");
})
->where("c.hostname", 'like', "%$filter%")
->whereNull("metager.affiliate_blacklist.hostname")
->groupBy("c.hostname")
->orderByDesc("clicks")
->limit($count)
->offset($skip)
->get();
$result = [
"count" => $count,
"skip" => $skip,
"total_hosts" => $hostCount,
"total_clicks" => $clickCount,
"hosts" => $hosts
];
return response()->json($result);
}
public function hostClicksJson(Request $request)
{
$validator = Validator::make($request->all(), [
"count" => ["integer", "min:1", "max:50"],
"skip" => ["integer", "min:0"],
"hostname" => [
"required",
function ($attribute, $value, $fail) use ($request) {
# Validate that this is indeed a hostname which is resolvable
if (!filter_var(gethostbyname($request->input("hostname")), FILTER_VALIDATE_IP)) {
$fail("The selected entry is not a valid hostname");
}
},
function ($attribute, $value, $fail) use ($request) {
# Validate that entry does not already exist in database
$entry = DB::table("affiliate_clicks", "c")
->where("hostname", "=", $request->input("hostname"))
->limit(1)
->get();
if (sizeof($entry) !== 1) {
$fail("The selected entry does not exist in database");
}
}
],
]);
if ($validator->fails()) {
return response()->json([
"message" => "Invalid Request Data"
], 422);
}
$count = intval($request->input("count", 10));
$skip = intval($request->input("skip", 0));
$total = DB::table("affiliate_clicks", "c")
->select(DB::raw("count(*) as count"))
->where("hostname", "=", $request->input("hostname"))
->get();
$total = $total[0]->count;
// Query Data
$clicks = DB::table("affiliate_clicks", "c")
->where("hostname", "=", $request->input("hostname"))
->orderByDesc("c.created_at")
->limit($count)
->offset($skip)
->get();
$result = [
"count" => $count,
"skip" => $skip,
"total" => $total,
"results" => $clicks
];
return response()->json($result);
}
}
......@@ -10,17 +10,19 @@ use LaravelLocalization;
class Adgoal
{
const COUNTRIES = ["af","al","dz","um","as","vi","ad","ao","ai","ag","ar","am","aw","az","au","eg","gq","et","bs",
"bh","bd","bb","be","bz","bj","bm","bt","bo","ba","bw","bv","br","vg","io","bn","bg","bf","bi","cl","cn","ck",
"cr","ci","dk","de","dm","do","dj","ec","sv","er","ee","eu","fk","fo","fj","fi","fr","gf","pf","tf","ga","gm",
"ge","gh","gi","gd","gr","gb","uk","gl","gp","gu","gt","gn","gw","gy","ht","hm","hn","hk","in","id","iq","ir",
"ie","is","il","it","jm","sj","jp","ye","jo","yu","ky","kh","cm","ca","cv","kz","qa","ke","kg","ki","cc","co",
"km","cg","cd","hr","cu","kw","la","ls","lv","lb","lr","ly","li","lt","lu","mo","mg","mw","my","mv","ml","mt",
"mp","ma","mh","mq","mr","mu","yt","mk","mx","fm","md","mc","mn","ms","mz","mm","na","nr","np","nc","nz","ni",
"nl","an","ne","ng","nu","kp","nf","no","om","tp","at","pk","pw","ps","pa","pg","py","pe","ph","pn","pl","pt",
"pr","re","rw","ro","ru","st","sb","zm","ws","sm","sa","se","ch","sn","sc","sl","zw","sg","sk","si","so","es",
"lk","sh","kn","lc","pm","vc","sd","sr","za","kr","sz","sy","tj","tw","tz","th","tg","to","tt","td","cz","tn",
"tm","tc","tv","tr","us","ug","ua","xx","hu","uy","uz","vu","va","ve","ae","vn","wf","cx","by","eh","ww","zr","cf","cy",];
const COUNTRIES = [
"af", "al", "dz", "um", "as", "vi", "ad", "ao", "ai", "ag", "ar", "am", "aw", "az", "au", "eg", "gq", "et", "bs",
"bh", "bd", "bb", "be", "bz", "bj", "bm", "bt", "bo", "ba", "bw", "bv", "br", "vg", "io", "bn", "bg", "bf", "bi", "cl", "cn", "ck",
"cr", "ci", "dk", "de", "dm", "do", "dj", "ec", "sv", "er", "ee", "eu", "fk", "fo", "fj", "fi", "fr", "gf", "pf", "tf", "ga", "gm",
"ge", "gh", "gi", "gd", "gr", "gb", "uk", "gl", "gp", "gu", "gt", "gn", "gw", "gy", "ht", "hm", "hn", "hk", "in", "id", "iq", "ir",
"ie", "is", "il", "it", "jm", "sj", "jp", "ye", "jo", "yu", "ky", "kh", "cm", "ca", "cv", "kz", "qa", "ke", "kg", "ki", "cc", "co",
"km", "cg", "cd", "hr", "cu", "kw", "la", "ls", "lv", "lb", "lr", "ly", "li", "lt", "lu", "mo", "mg", "mw", "my", "mv", "ml", "mt",
"mp", "ma", "mh", "mq", "mr", "mu", "yt", "mk", "mx", "fm", "md", "mc", "mn", "ms", "mz", "mm", "na", "nr", "np", "nc", "nz", "ni",
"nl", "an", "ne", "ng", "nu", "kp", "nf", "no", "om", "tp", "at", "pk", "pw", "ps", "pa", "pg", "py", "pe", "ph", "pn", "pl", "pt",
"pr", "re", "rw", "ro", "ru", "st", "sb", "zm", "ws", "sm", "sa", "se", "ch", "sn", "sc", "sl", "zw", "sg", "sk", "si", "so", "es",
"lk", "sh", "kn", "lc", "pm", "vc", "sd", "sr", "za", "kr", "sz", "sy", "tj", "tw", "tz", "th", "tg", "to", "tt", "td", "cz", "tn",
"tm", "tc", "tv", "tr", "us", "ug", "ua", "xx", "hu", "uy", "uz", "vu", "va", "ve", "ae", "vn", "wf", "cx", "by", "eh", "ww", "zr", "cf", "cy",
];
public $hash;
......@@ -55,7 +57,7 @@ class Adgoal
}
$linkList .= $link . ",";
}
if(empty($linkList)){
if (empty($linkList)) {
return;
}
......@@ -75,9 +77,9 @@ class Adgoal
$preferredLanguage = Request::getPreferredLanguage();
if (!empty($preferredLanguage)) {
if (str_contains($preferredLanguage, "_")) {
$preferredLanguage = substr($preferredLanguage, stripos($preferredLanguage, "_")+1);
$preferredLanguage = substr($preferredLanguage, stripos($preferredLanguage, "_") + 1);
} elseif (str_contains($preferredLanguage, "-")) {
$preferredLanguage = substr($preferredLanguage, stripos($preferredLanguage, "-")+1);
$preferredLanguage = substr($preferredLanguage, stripos($preferredLanguage, "-") + 1);
}
$preferredLanguage = strtolower($preferredLanguage);
......@@ -117,14 +119,15 @@ class Adgoal
Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
}
public function fetchAffiliates($wait = false){
if($this->affiliates !== null){
public function fetchAffiliates($wait = false)
{
if ($this->affiliates !== null) {
return;
}
$answer = null;
$startTime = microtime(true);
if($wait){
if ($wait) {
while (microtime(true) - $startTime < 5) {
$answer = Cache::get($this->hash);
if ($answer === null) {
......@@ -133,18 +136,18 @@ class Adgoal
break;
}
}
}else{
} else {
$answer = Cache::get($this->hash);
}
$answer = json_decode($answer, true);
// If the fetcher had an Error
if($answer === "no-result"){
if ($answer === "no-result") {
$this->affiliates = [];
return;
}
if(empty($answer) && !is_array($answer)){
if (empty($answer) && !is_array($answer)) {
return;
}
......@@ -158,7 +161,7 @@ class Adgoal
*/
public function parseAffiliates(&$results)
{
if($this->finished || $this->affiliates === null){
if ($this->finished || $this->affiliates === null) {
return;
}
......@@ -168,7 +171,7 @@ class Adgoal
$tld = $partnershop["tld"];
// Sometimes TLD is null
if(empty($tld)){
if (empty($tld)) {
continue;
}
......@@ -178,13 +181,18 @@ class Adgoal
Adgoal sometimes returns affiliate Links for every URL