Commit 34d9981e authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Merge branch '1152-create-blacklist-and-whitelist-for-affiliate-links' into 'development'

Resolve "Create Blacklist and Whitelist for affiliate links"

Closes #1152

See merge request !1914
parents 708b5803 d016590b
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Redis;
class LoadAffiliateBlacklist extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'load:affiliate-blacklist';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Loads the Affiliate Blacklist from DB into Redis';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
$blacklistItems = DB::table("affiliate_blacklist", "b")
->select("hostname")
->where("blacklist", true)
->get();
Redis::pipeline(function ($redis) use ($blacklistItems) {
$redisKey = \App\Http\Controllers\AdgoalController::REDIS_BLACKLIST_KEY;
$redis->del($redisKey);
foreach ($blacklistItems as $item) {
$hostname = $item->hostname;
$redis->hset(\App\Http\Controllers\AdgoalController::REDIS_BLACKLIST_KEY, $hostname, true);
}
});
return 0;
}
}
...@@ -13,9 +13,7 @@ class Kernel extends ConsoleKernel ...@@ -13,9 +13,7 @@ class Kernel extends ConsoleKernel
* *
* @var array * @var array
*/ */
protected $commands = [ protected $commands = [];
];
/** /**
* Define the application's command schedule. * Define the application's command schedule.
...@@ -30,6 +28,7 @@ class Kernel extends ConsoleKernel ...@@ -30,6 +28,7 @@ class Kernel extends ConsoleKernel
$schedule->command('requests:useragents')->everyFiveMinutes(); $schedule->command('requests:useragents')->everyFiveMinutes();
$schedule->command('logs:gather')->everyMinute(); $schedule->command('logs:gather')->everyMinute();
$schedule->command('spam:load')->everyMinute(); $schedule->command('spam:load')->everyMinute();
$schedule->command('load:affiliate-blacklist')->everyMinute();
$schedule->command('affilliates:store')->everyMinute() $schedule->command('affilliates:store')->everyMinute()
->onOneServer(); ->onOneServer();
$schedule->call(function () { $schedule->call(function () {
......
...@@ -6,6 +6,7 @@ use Illuminate\Http\Request; ...@@ -6,6 +6,7 @@ use Illuminate\Http\Request;
use LaravelLocalization; use LaravelLocalization;
use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Redis; use Illuminate\Support\Facades\Redis;
use Illuminate\Support\Facades\Validator;
/** /**
* Before we redirect users to the affiliate shops we will track the clicks ourself. * Before we redirect users to the affiliate shops we will track the clicks ourself.
...@@ -22,6 +23,7 @@ class AdgoalController extends Controller ...@@ -22,6 +23,7 @@ class AdgoalController extends Controller
# Data will be stored for 24 hours # Data will be stored for 24 hours
const STORAGE_DURATION_HOURS = 24; const STORAGE_DURATION_HOURS = 24;
const REDIS_STORAGE_KEY = "affiliate_click"; const REDIS_STORAGE_KEY = "affiliate_click";
const REDIS_BLACKLIST_KEY = "affiliate_blacklist";
/** /**
* This function is called when a user clicks on a affiliate link. It will first validate that the URL * This function is called when a user clicks on a affiliate link. It will first validate that the URL
...@@ -29,7 +31,8 @@ class AdgoalController extends Controller ...@@ -29,7 +31,8 @@ class AdgoalController extends Controller
* After that we will store the necessary information (link and affiliate link) into our database. * After that we will store the necessary information (link and affiliate link) into our database.
* After that we will redirect the user to the affiliate shop * After that we will redirect the user to the affiliate shop
*/ */
public function forward(Request $request){ public function forward(Request $request)
{
// $link = "https://metager.de"; // $link = "https://metager.de";
// $affillink = "https://test.de"; // $affillink = "https://test.de";
// $password = self::generatePassword($affillink, $link); // $password = self::generatePassword($affillink, $link);
...@@ -44,10 +47,10 @@ class AdgoalController extends Controller ...@@ -44,10 +47,10 @@ class AdgoalController extends Controller
'affillink' => ['required', 'url', 'active_url'], 'affillink' => ['required', 'url', 'active_url'],
'link' => ['required', 'url', 'active_url'], 'link' => ['required', 'url', 'active_url'],
# Validation of redirect request so that one cannot generate random redirect URLs pointing to our domains # Validation of redirect request so that one cannot generate random redirect URLs pointing to our domains
'password' => function($attribute, $value, $fail) use($request) { 'password' => function ($attribute, $value, $fail) use ($request) {
// Check if hmac matches // Check if hmac matches
$correctPassword = self::generatePassword($request->input('affillink'), $request->input('link')); $correctPassword = self::generatePassword($request->input('affillink'), $request->input('link'));
if(!hash_equals($correctPassword, $value)){ if (!hash_equals($correctPassword, $value)) {
$fail('The given password is incorrect!'); $fail('The given password is incorrect!');
} }
} }
...@@ -63,10 +66,11 @@ class AdgoalController extends Controller ...@@ -63,10 +66,11 @@ class AdgoalController extends Controller
* at search time. * at search time.
* A Cronjob will pick the data up and store it into Mariadb later (see self::storePartnerCall) * A Cronjob will pick the data up and store it into Mariadb later (see self::storePartnerCall)
*/ */
private function storePartnerCallFast($affillink, $link) { private function storePartnerCallFast($affillink, $link)
{
# Generate Data to store # Generate Data to store
$host = parse_url($link, PHP_URL_HOST); $host = parse_url($link, PHP_URL_HOST);
if(empty($host)){ if (empty($host)) {
return; return;
} }
$storeObject = [ $storeObject = [
...@@ -81,14 +85,17 @@ class AdgoalController extends Controller ...@@ -81,14 +85,17 @@ class AdgoalController extends Controller
$redis->rpush($this::REDIS_STORAGE_KEY, json_encode($storeObject)); $redis->rpush($this::REDIS_STORAGE_KEY, json_encode($storeObject));
} }
public static function storePartnerCalls() { public static function storePartnerCalls()
{
$redis = Redis::connection(config('cache.stores.redis.connection')); $redis = Redis::connection(config('cache.stores.redis.connection'));
DB::transaction(function() use($redis){ DB::transaction(function () use ($redis) {
while(!empty($data = $redis->lpop(self::REDIS_STORAGE_KEY))){ while (!empty($data = $redis->lpop(self::REDIS_STORAGE_KEY))) {
$data = json_decode($data, true); $data = json_decode($data, true);
# Insert data into mariadb table # Insert data into mariadb table
DB::insert('insert into affiliate_clicks (hostname, affillink, link) values (?, ?, ?)', DB::insert(
[$data["host"], $data["affillink"], $data["link"]]); 'insert into affiliate_clicks (hostname, affillink, link) values (?, ?, ?)',
[$data["host"], $data["affillink"], $data["link"]]
);
} }
}); });
} }
...@@ -96,10 +103,11 @@ class AdgoalController extends Controller ...@@ -96,10 +103,11 @@ class AdgoalController extends Controller
/** /**
* Generates a Redirect URL for our partnershops * Generates a Redirect URL for our partnershops
*/ */
public static function generateRedirectUrl($affillink, $link){ public static function generateRedirectUrl($affillink, $link)
{
$password = self::generatePassword($affillink, $link); $password = self::generatePassword($affillink, $link);
return LaravelLocalization::getLocalizedURL( return LaravelLocalization::getLocalizedURL(
LaravelLocalization::getCurrentLocale(), LaravelLocalization::getCurrentLocale(),
route('adgoal-redirect', ["link" => $link, "affillink" => $affillink, "password" => $password]) route('adgoal-redirect', ["link" => $link, "affillink" => $affillink, "password" => $password])
); );
} }
...@@ -107,7 +115,319 @@ class AdgoalController extends Controller ...@@ -107,7 +115,319 @@ class AdgoalController extends Controller
/** /**
* Generates hmac password to validate redirect URLs * Generates hmac password to validate redirect URLs
*/ */
public static function generatePassword($affillink, $link){ public static function generatePassword($affillink, $link)
{
return hash_hmac("sha256", $affillink . $link, config('metager.metager.adgoal.private_key')); return hash_hmac("sha256", $affillink . $link, config('metager.metager.adgoal.private_key'));
} }
/**
* Routes for the Admin Interface
*/
public function adminIndex(Request $request)
{
return view('admin.affiliates.index')
->with('title', "Affilliates Overview - MetaGer")
->with('css', [
mix('/css/admin/affilliates/index.css')
])
->with('darkcss', [
mix('/css/admin/affilliates/index-dark.css')
])
->with('js', [
mix('/js/admin/affilliates.js')
]);
}
public function blacklistJson(Request $request)
{
$validator = Validator::make($request->all(), [
"blacklist" => 'boolean'
]);
if ($validator->fails()) {
return response()->json("Invalid Request Data", 422);
}
$count = 5; # How Many results to return
$skip = 0; # How many results to skip
$blacklist = $request->input('blacklist', true);
$total = DB::select("select count(*) as total_rows from affiliate_blacklist where blacklist = ?", [$blacklist]);
$total = intval($total[0]->{"total_rows"});
$blacklistItems = DB::select('select * from affiliate_blacklist where blacklist = ? order by created_at desc limit ? offset ?', [$blacklist, $count, $skip]);
$result = [
"count" => $count,
"skip" => $skip,
"total" => $total,
"results" => $blacklistItems
];
return response()->json($result);
}
public function addblacklistJson(Request $request)
{
$validator = Validator::make($request->all(), [
"hostname" => [
"required",
function ($attribute, $value, $fail) use ($request) {
# Validate that this is indeed a hostname which is resolvable
if (!filter_var(gethostbyname($request->input("hostname")), FILTER_VALIDATE_IP)) {
$fail("The selected entry is not a valid hostname");
}
},
function ($attribute, $value, $fail) use ($request) {
# Validate that entry does not already exist in database
$entry = DB::select("select * from metager.affiliate_blacklist where hostname = ?", [$request->input("hostname")]);
if (sizeof($entry) !== 0) {
$fail("The selected entry does already exist in database");
}
}
],
]);
if ($validator->fails()) {
return response()->json([
"message" => "Invalid Request Data"
], 422);
}
$hostname = $validator->validated()["hostname"];
$rowsInserted = DB::insert("insert into metager.affiliate_blacklist (hostname, blacklist) values (?, 1)", [$hostname]);
if ($rowsInserted === TRUE) {
return response()->json([
"message" => "Entry added."
]);
} else {
return response()->json([
"message" => "Error inserting entry."
], 422);
}
}
public function deleteblacklistJson(Request $request)
{
$validator = Validator::make($request->all(), [
"id" => ["required", "integer", "min:1", function ($attribute, $value, $fail) use ($request) {
$entry = DB::select("select * from metager.affiliate_blacklist where id = ? and blacklist = ?", [$request->input("id"), true]);
if (sizeof($entry) !== 1) {
$fail("The selected entry does not exist in database");
}
}],
]);
if ($validator->fails()) {
return response()->json("Invalid Request Data", 422);
}
$id = intval($validator->validated()["id"]);
$rowsDeleted = DB::delete("delete from metager.affiliate_blacklist where id = ?", [$id]);
if ($rowsDeleted > 0) {
return response()->json([
"message" => "$rowsDeleted entries deleted."
]);
} else {
return response()->json([
"message" => "Error deleting entry."
], 422);
}
}
public function whitelistJson(Request $request)
{
$input = $request->all();
$input["blacklist"] = false;
$request->replace($input);
return $this->blacklistJson($request);
}
public function addwhitelistJson(Request $request)
{
$validator = Validator::make($request->all(), [
"hostname" => [
"required",
function ($attribute, $value, $fail) use ($request) {
# Validate that this is indeed a hostname which is resolvable
if (!filter_var(gethostbyname($request->input("hostname")), FILTER_VALIDATE_IP)) {
$fail("The selected entry is not a valid hostname");
}
},
function ($attribute, $value, $fail) use ($request) {
# Validate that entry does not already exist in database
$entry = DB::select("select * from metager.affiliate_blacklist where hostname = ?", [$request->input("hostname")]);
if (sizeof($entry) !== 0) {
$fail("The selected entry does already exist in database");
}
}
],
]);
if ($validator->fails()) {
return response()->json([
"message" => "Invalid Request Data"
], 422);
}
$hostname = $validator->validated()["hostname"];
$rowsInserted = DB::insert("insert into metager.affiliate_blacklist (hostname, blacklist) values (?, 0)", [$hostname]);
if ($rowsInserted === TRUE) {
return response()->json([
"message" => "Entry added."
]);
} else {
return response()->json([
"message" => "Error inserting entry."
], 422);
}
}
public function deletewhitelistJson(Request $request)
{
$validator = Validator::make($request->all(), [
"id" => ["required", "integer", "min:1", function ($attribute, $value, $fail) use ($request) {
$entry = DB::select("select * from metager.affiliate_blacklist where id = ? and blacklist = ?", [$request->input("id"), false]);
if (sizeof($entry) !== 1) {
$fail("The selected entry does not exist in database");
}
}],
]);
if ($validator->fails()) {
return response()->json("Invalid Request Data", 422);
}
$id = intval($validator->validated()["id"]);
$rowsDeleted = DB::delete("delete from metager.affiliate_blacklist where id = ?", [$id]);
if ($rowsDeleted > 0) {
return response()->json([
"message" => "$rowsDeleted entries deleted."
]);
} else {
return response()->json([
"message" => "Error deleting entry."
], 422);
}
}
public function hostsJson(Request $request)
{
$validator = Validator::make($request->all(), [
"count" => ["integer", "min:1", "max:50"],
"skip" => ["integer", "min:0"],
]);
if ($validator->fails()) {
return response()->json("Invalid Request Data", 422);
}
$count = intval($request->input("count", 10));
$skip = intval($request->input("skip", 0));
$filter = $request->input("filter", "");
$hostCount = DB::table("metager.affiliate_clicks", "c")
->select(DB::raw("count(distinct c.hostname) as total_hosts"))
->leftJoin("metager.affiliate_blacklist", function ($join) {
$join->on("c.hostname", "=", "metager.affiliate_blacklist.hostname");
})
->where("c.hostname", 'like', "%$filter%")
->whereNull("metager.affiliate_blacklist.hostname")
->get();
$hostCount = $hostCount[0]->{"total_hosts"};
$clickCount = DB::table("metager.affiliate_clicks", "c")
->select(DB::raw("count(*) as click_count"))
->leftJoin("metager.affiliate_blacklist", function ($join) {
$join->on("c.hostname", "=", "metager.affiliate_blacklist.hostname");
})
->where("c.hostname", 'like', "%$filter%")
->whereNull("metager.affiliate_blacklist.hostname")
->get();
$clickCount = $clickCount[0]->{"click_count"};
$hosts = DB::table("metager.affiliate_clicks", "c")
->select("c.hostname", DB::raw('count(c.hostname) as clicks'))
->leftJoin("metager.affiliate_blacklist", function ($join) {
$join->on("c.hostname", "=", "metager.affiliate_blacklist.hostname");
})
->where("c.hostname", 'like', "%$filter%")
->whereNull("metager.affiliate_blacklist.hostname")
->groupBy("c.hostname")
->orderByDesc("clicks")
->limit($count)
->offset($skip)
->get();
$result = [
"count" => $count,
"skip" => $skip,
"total_hosts" => $hostCount,
"total_clicks" => $clickCount,
"hosts" => $hosts
];
return response()->json($result);
}
public function hostClicksJson(Request $request)
{
$validator = Validator::make($request->all(), [
"count" => ["integer", "min:1", "max:50"],
"skip" => ["integer", "min:0"],
"hostname" => [
"required",
function ($attribute, $value, $fail) use ($request) {
# Validate that this is indeed a hostname which is resolvable
if (!filter_var(gethostbyname($request->input("hostname")), FILTER_VALIDATE_IP)) {
$fail("The selected entry is not a valid hostname");
}
},
function ($attribute, $value, $fail) use ($request) {
# Validate that entry does not already exist in database
$entry = DB::table("affiliate_clicks", "c")
->where("hostname", "=", $request->input("hostname"))
->limit(1)
->get();
if (sizeof($entry) !== 1) {
$fail("The selected entry does not exist in database");
}
}
],
]);
if ($validator->fails()) {
return response()->json([
"message" => "Invalid Request Data"
], 422);
}
$count = intval($request->input("count", 10));
$skip = intval($request->input("skip", 0));
$total = DB::table("affiliate_clicks", "c")
->select(DB::raw("count(*) as count"))
->where("hostname", "=", $request->input("hostname"))
->get();
$total = $total[0]->count;
// Query Data
$clicks = DB::table("affiliate_clicks", "c")
->where("hostname", "=", $request->input("hostname"))
->orderByDesc("c.created_at")
->limit($count)
->offset($skip)
->get();
$result = [
"count" => $count,
"skip" => $skip,
"total" => $total,
"results" => $clicks
];
return response()->json($result);
}
} }
...@@ -10,17 +10,19 @@ use LaravelLocalization; ...@@ -10,17 +10,19 @@ use LaravelLocalization;
class Adgoal class Adgoal
{ {
const COUNTRIES = ["af","al","dz","um","as","vi","ad","ao","ai","ag","ar","am","aw","az","au","eg","gq","et","bs", const COUNTRIES = [
"bh","bd","bb","be","bz","bj","bm","bt","bo","ba","bw","bv","br","vg","io","bn","bg","bf","bi","cl","cn","ck", "af", "al", "dz", "um", "as", "vi", "ad", "ao", "ai", "ag", "ar", "am", "aw", "az", "au", "eg", "gq", "et", "bs",
"cr","ci","dk","de","dm","do","dj","ec","sv","er","ee","eu","fk","fo","fj","fi","fr","gf","pf","tf","ga","gm", "bh", "bd", "bb", "be", "bz", "bj", "bm", "bt", "bo", "ba", "bw", "bv", "br", "vg", "io", "bn", "bg", "bf", "bi", "cl", "cn", "ck",
"ge","gh","gi","gd","gr","gb","uk","gl","gp","gu","gt","gn","gw","gy","ht","hm","hn","hk","in","id","iq","ir", "cr", "ci", "dk", "de", "dm", "do", "dj", "ec", "sv", "er", "ee", "eu", "fk", "fo", "fj", "fi", "fr", "gf", "pf", "tf", "ga", "gm",
"ie","is","il","it","jm","sj","jp","ye","jo","yu","ky","kh","cm","ca","cv","kz","qa","ke","kg","ki","cc","co", "ge", "gh", "gi", "gd", "gr", "gb", "uk", "gl", "gp", "gu", "gt", "gn", "gw", "gy", "ht", "hm", "hn", "hk", "in", "id", "iq", "ir",
"km","cg","cd","hr","cu","kw","la","ls","lv","lb","lr","ly","li","lt","lu","mo","mg","mw","my","mv","ml","mt", "ie", "is", "il", "it", "jm", "sj", "jp", "ye", "jo", "yu", "ky", "kh", "cm", "ca", "cv", "kz", "qa", "ke", "kg", "ki", "cc", "co",
"mp","ma","mh","mq","mr","mu","yt","mk","mx","fm","md","mc","mn","ms","mz","mm","na","nr","np","nc","nz","ni", "km", "cg", "cd", "hr", "cu", "kw", "la", "ls", "lv", "lb", "lr", "ly", "li", "lt", "lu", "mo", "mg", "mw", "my", "mv", "ml", "mt",
"nl","an","ne","ng","nu","kp","nf","no","om","tp","at","pk","pw","ps","pa","pg","py","pe","ph","pn","pl","pt", "mp", "ma", "mh", "mq", "mr", "mu", "yt", "mk", "mx", "fm", "md", "mc", "mn", "ms", "mz", "mm", "na", "nr", "np", "nc", "nz", "ni",
"pr","re","rw","ro","ru","st","sb","zm","ws","sm","sa","se","ch","sn","sc","sl","zw","sg","sk","si","so","es", "nl", "an", "ne", "ng", "nu", "kp", "nf", "no", "om", "tp", "at", "pk", "pw", "ps", "pa", "pg", "py", "pe", "ph", "pn", "pl", "pt",
"lk","sh","kn","lc","pm","vc","sd","sr","za","kr","sz","sy","tj","tw","tz","th","tg","to","tt","td","cz","tn", "pr", "re", "rw", "ro", "ru", "st", "sb", "zm", "ws", "sm", "sa", "se", "ch", "sn", "sc", "sl", "zw", "sg", "sk", "si", "so", "es",
"tm","tc","tv","tr","us","ug","ua","xx","hu","uy","uz","vu","va","ve","ae","vn","wf","cx","by","eh","ww","zr","cf","cy",]; "lk", "sh", "kn", "lc", "pm", "vc", "sd", "sr", "za", "kr", "sz", "sy", "tj", "tw", "tz", "th", "tg", "to", "tt", "td", "cz", "tn",
"tm", "tc", "tv", "tr", "us", "ug", "ua", "xx", "hu", "uy", "uz", "vu", "va", "ve", "ae", "vn", "wf", "cx", "by", "eh", "ww", "zr", "cf", "cy",
];
public $hash; public $hash;
...@@ -55,17 +57,17 @@ class Adgoal ...@@ -55,17 +57,17 @@ class Adgoal
} }
$linkList .= $link . ","; $linkList .= $link . ",";
} }
if(empty($linkList)){ if (empty($linkList)) {
return; return;
} }
$linkList = rtrim($linkList, ","); $linkList = rtrim($linkList, ",");
# Hashwert # Hashwert
$this->hash = md5($linkList . $privateKey); $this->hash = md5($linkList . $privateKey);