Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
open-source
MetaGer
Commits
d016590b
Commit
d016590b
authored
Oct 06, 2021
by
Dominik Hebeler
Browse files
Loading and using the blacklist
parent
bca4c88b
Changes
5
Pipelines
1
Show whitespace changes
Inline
Side-by-side
app/Console/Commands/LoadAffiliateBlacklist.php
0 → 100644
View file @
d016590b
<?php
namespace
App\Console\Commands
;
use
Illuminate\Console\Command
;
use
Illuminate\Support\Facades\DB
;
use
Illuminate\Support\Facades\Redis
;
class
LoadAffiliateBlacklist
extends
Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected
$signature
=
'load:affiliate-blacklist'
;
/**
* The console command description.
*
* @var string
*/
protected
$description
=
'Loads the Affiliate Blacklist from DB into Redis'
;
/**
* Create a new command instance.
*
* @return void
*/
public
function
__construct
()
{
parent
::
__construct
();
}
/**
* Execute the console command.
*
* @return int
*/
public
function
handle
()
{
$blacklistItems
=
DB
::
table
(
"affiliate_blacklist"
,
"b"
)
->
select
(
"hostname"
)
->
where
(
"blacklist"
,
true
)
->
get
();
Redis
::
pipeline
(
function
(
$redis
)
use
(
$blacklistItems
)
{
$redisKey
=
\
App\Http\Controllers\AdgoalController
::
REDIS_BLACKLIST_KEY
;
$redis
->
del
(
$redisKey
);
foreach
(
$blacklistItems
as
$item
)
{
$hostname
=
$item
->
hostname
;
$redis
->
hset
(
\
App\Http\Controllers\AdgoalController
::
REDIS_BLACKLIST_KEY
,
$hostname
,
true
);
}
});
return
0
;
}
}
app/Console/Kernel.php
View file @
d016590b
...
...
@@ -13,9 +13,7 @@ class Kernel extends ConsoleKernel
*
* @var array
*/
protected
$commands
=
[
];
protected
$commands
=
[];
/**
* Define the application's command schedule.
...
...
@@ -30,6 +28,7 @@ class Kernel extends ConsoleKernel
$schedule
->
command
(
'requests:useragents'
)
->
everyFiveMinutes
();
$schedule
->
command
(
'logs:gather'
)
->
everyMinute
();
$schedule
->
command
(
'spam:load'
)
->
everyMinute
();
$schedule
->
command
(
'load:affiliate-blacklist'
)
->
everyMinute
();
$schedule
->
command
(
'affilliates:store'
)
->
everyMinute
()
->
onOneServer
();
$schedule
->
call
(
function
()
{
...
...
app/Http/Controllers/AdgoalController.php
View file @
d016590b
...
...
@@ -23,6 +23,7 @@ class AdgoalController extends Controller
# Data will be stored for 24 hours
const
STORAGE_DURATION_HOURS
=
24
;
const
REDIS_STORAGE_KEY
=
"affiliate_click"
;
const
REDIS_BLACKLIST_KEY
=
"affiliate_blacklist"
;
/**
* This function is called when a user clicks on a affiliate link. It will first validate that the URL
...
...
app/Models/Adgoal.php
View file @
d016590b
...
...
@@ -10,17 +10,19 @@ use LaravelLocalization;
class
Adgoal
{
const
COUNTRIES
=
[
"af"
,
"al"
,
"dz"
,
"um"
,
"as"
,
"vi"
,
"ad"
,
"ao"
,
"ai"
,
"ag"
,
"ar"
,
"am"
,
"aw"
,
"az"
,
"au"
,
"eg"
,
"gq"
,
"et"
,
"bs"
,
"bh"
,
"bd"
,
"bb"
,
"be"
,
"bz"
,
"bj"
,
"bm"
,
"bt"
,
"bo"
,
"ba"
,
"bw"
,
"bv"
,
"br"
,
"vg"
,
"io"
,
"bn"
,
"bg"
,
"bf"
,
"bi"
,
"cl"
,
"cn"
,
"ck"
,
"cr"
,
"ci"
,
"dk"
,
"de"
,
"dm"
,
"do"
,
"dj"
,
"ec"
,
"sv"
,
"er"
,
"ee"
,
"eu"
,
"fk"
,
"fo"
,
"fj"
,
"fi"
,
"fr"
,
"gf"
,
"pf"
,
"tf"
,
"ga"
,
"gm"
,
"ge"
,
"gh"
,
"gi"
,
"gd"
,
"gr"
,
"gb"
,
"uk"
,
"gl"
,
"gp"
,
"gu"
,
"gt"
,
"gn"
,
"gw"
,
"gy"
,
"ht"
,
"hm"
,
"hn"
,
"hk"
,
"in"
,
"id"
,
"iq"
,
"ir"
,
"ie"
,
"is"
,
"il"
,
"it"
,
"jm"
,
"sj"
,
"jp"
,
"ye"
,
"jo"
,
"yu"
,
"ky"
,
"kh"
,
"cm"
,
"ca"
,
"cv"
,
"kz"
,
"qa"
,
"ke"
,
"kg"
,
"ki"
,
"cc"
,
"co"
,
"km"
,
"cg"
,
"cd"
,
"hr"
,
"cu"
,
"kw"
,
"la"
,
"ls"
,
"lv"
,
"lb"
,
"lr"
,
"ly"
,
"li"
,
"lt"
,
"lu"
,
"mo"
,
"mg"
,
"mw"
,
"my"
,
"mv"
,
"ml"
,
"mt"
,
"mp"
,
"ma"
,
"mh"
,
"mq"
,
"mr"
,
"mu"
,
"yt"
,
"mk"
,
"mx"
,
"fm"
,
"md"
,
"mc"
,
"mn"
,
"ms"
,
"mz"
,
"mm"
,
"na"
,
"nr"
,
"np"
,
"nc"
,
"nz"
,
"ni"
,
"nl"
,
"an"
,
"ne"
,
"ng"
,
"nu"
,
"kp"
,
"nf"
,
"no"
,
"om"
,
"tp"
,
"at"
,
"pk"
,
"pw"
,
"ps"
,
"pa"
,
"pg"
,
"py"
,
"pe"
,
"ph"
,
"pn"
,
"pl"
,
"pt"
,
"pr"
,
"re"
,
"rw"
,
"ro"
,
"ru"
,
"st"
,
"sb"
,
"zm"
,
"ws"
,
"sm"
,
"sa"
,
"se"
,
"ch"
,
"sn"
,
"sc"
,
"sl"
,
"zw"
,
"sg"
,
"sk"
,
"si"
,
"so"
,
"es"
,
"lk"
,
"sh"
,
"kn"
,
"lc"
,
"pm"
,
"vc"
,
"sd"
,
"sr"
,
"za"
,
"kr"
,
"sz"
,
"sy"
,
"tj"
,
"tw"
,
"tz"
,
"th"
,
"tg"
,
"to"
,
"tt"
,
"td"
,
"cz"
,
"tn"
,
"tm"
,
"tc"
,
"tv"
,
"tr"
,
"us"
,
"ug"
,
"ua"
,
"xx"
,
"hu"
,
"uy"
,
"uz"
,
"vu"
,
"va"
,
"ve"
,
"ae"
,
"vn"
,
"wf"
,
"cx"
,
"by"
,
"eh"
,
"ww"
,
"zr"
,
"cf"
,
"cy"
,];
const
COUNTRIES
=
[
"af"
,
"al"
,
"dz"
,
"um"
,
"as"
,
"vi"
,
"ad"
,
"ao"
,
"ai"
,
"ag"
,
"ar"
,
"am"
,
"aw"
,
"az"
,
"au"
,
"eg"
,
"gq"
,
"et"
,
"bs"
,
"bh"
,
"bd"
,
"bb"
,
"be"
,
"bz"
,
"bj"
,
"bm"
,
"bt"
,
"bo"
,
"ba"
,
"bw"
,
"bv"
,
"br"
,
"vg"
,
"io"
,
"bn"
,
"bg"
,
"bf"
,
"bi"
,
"cl"
,
"cn"
,
"ck"
,
"cr"
,
"ci"
,
"dk"
,
"de"
,
"dm"
,
"do"
,
"dj"
,
"ec"
,
"sv"
,
"er"
,
"ee"
,
"eu"
,
"fk"
,
"fo"
,
"fj"
,
"fi"
,
"fr"
,
"gf"
,
"pf"
,
"tf"
,
"ga"
,
"gm"
,
"ge"
,
"gh"
,
"gi"
,
"gd"
,
"gr"
,
"gb"
,
"uk"
,
"gl"
,
"gp"
,
"gu"
,
"gt"
,
"gn"
,
"gw"
,
"gy"
,
"ht"
,
"hm"
,
"hn"
,
"hk"
,
"in"
,
"id"
,
"iq"
,
"ir"
,
"ie"
,
"is"
,
"il"
,
"it"
,
"jm"
,
"sj"
,
"jp"
,
"ye"
,
"jo"
,
"yu"
,
"ky"
,
"kh"
,
"cm"
,
"ca"
,
"cv"
,
"kz"
,
"qa"
,
"ke"
,
"kg"
,
"ki"
,
"cc"
,
"co"
,
"km"
,
"cg"
,
"cd"
,
"hr"
,
"cu"
,
"kw"
,
"la"
,
"ls"
,
"lv"
,
"lb"
,
"lr"
,
"ly"
,
"li"
,
"lt"
,
"lu"
,
"mo"
,
"mg"
,
"mw"
,
"my"
,
"mv"
,
"ml"
,
"mt"
,
"mp"
,
"ma"
,
"mh"
,
"mq"
,
"mr"
,
"mu"
,
"yt"
,
"mk"
,
"mx"
,
"fm"
,
"md"
,
"mc"
,
"mn"
,
"ms"
,
"mz"
,
"mm"
,
"na"
,
"nr"
,
"np"
,
"nc"
,
"nz"
,
"ni"
,
"nl"
,
"an"
,
"ne"
,
"ng"
,
"nu"
,
"kp"
,
"nf"
,
"no"
,
"om"
,
"tp"
,
"at"
,
"pk"
,
"pw"
,
"ps"
,
"pa"
,
"pg"
,
"py"
,
"pe"
,
"ph"
,
"pn"
,
"pl"
,
"pt"
,
"pr"
,
"re"
,
"rw"
,
"ro"
,
"ru"
,
"st"
,
"sb"
,
"zm"
,
"ws"
,
"sm"
,
"sa"
,
"se"
,
"ch"
,
"sn"
,
"sc"
,
"sl"
,
"zw"
,
"sg"
,
"sk"
,
"si"
,
"so"
,
"es"
,
"lk"
,
"sh"
,
"kn"
,
"lc"
,
"pm"
,
"vc"
,
"sd"
,
"sr"
,
"za"
,
"kr"
,
"sz"
,
"sy"
,
"tj"
,
"tw"
,
"tz"
,
"th"
,
"tg"
,
"to"
,
"tt"
,
"td"
,
"cz"
,
"tn"
,
"tm"
,
"tc"
,
"tv"
,
"tr"
,
"us"
,
"ug"
,
"ua"
,
"xx"
,
"hu"
,
"uy"
,
"uz"
,
"vu"
,
"va"
,
"ve"
,
"ae"
,
"vn"
,
"wf"
,
"cx"
,
"by"
,
"eh"
,
"ww"
,
"zr"
,
"cf"
,
"cy"
,
];
public
$hash
;
...
...
@@ -55,7 +57,7 @@ class Adgoal
}
$linkList
.
=
$link
.
","
;
}
if
(
empty
(
$linkList
)){
if
(
empty
(
$linkList
))
{
return
;
}
...
...
@@ -75,9 +77,9 @@ class Adgoal
$preferredLanguage
=
Request
::
getPreferredLanguage
();
if
(
!
empty
(
$preferredLanguage
))
{
if
(
str_contains
(
$preferredLanguage
,
"_"
))
{
$preferredLanguage
=
substr
(
$preferredLanguage
,
stripos
(
$preferredLanguage
,
"_"
)
+
1
);
$preferredLanguage
=
substr
(
$preferredLanguage
,
stripos
(
$preferredLanguage
,
"_"
)
+
1
);
}
elseif
(
str_contains
(
$preferredLanguage
,
"-"
))
{
$preferredLanguage
=
substr
(
$preferredLanguage
,
stripos
(
$preferredLanguage
,
"-"
)
+
1
);
$preferredLanguage
=
substr
(
$preferredLanguage
,
stripos
(
$preferredLanguage
,
"-"
)
+
1
);
}
$preferredLanguage
=
strtolower
(
$preferredLanguage
);
...
...
@@ -117,14 +119,15 @@ class Adgoal
Redis
::
rpush
(
\
App\MetaGer
::
FETCHQUEUE_KEY
,
$mission
);
}
public
function
fetchAffiliates
(
$wait
=
false
){
if
(
$this
->
affiliates
!==
null
){
public
function
fetchAffiliates
(
$wait
=
false
)
{
if
(
$this
->
affiliates
!==
null
)
{
return
;
}
$answer
=
null
;
$startTime
=
microtime
(
true
);
if
(
$wait
){
if
(
$wait
)
{
while
(
microtime
(
true
)
-
$startTime
<
5
)
{
$answer
=
Cache
::
get
(
$this
->
hash
);
if
(
$answer
===
null
)
{
...
...
@@ -133,18 +136,18 @@ class Adgoal
break
;
}
}
}
else
{
}
else
{
$answer
=
Cache
::
get
(
$this
->
hash
);
}
$answer
=
json_decode
(
$answer
,
true
);
// If the fetcher had an Error
if
(
$answer
===
"no-result"
){
if
(
$answer
===
"no-result"
)
{
$this
->
affiliates
=
[];
return
;
}
if
(
empty
(
$answer
)
&&
!
is_array
(
$answer
)){
if
(
empty
(
$answer
)
&&
!
is_array
(
$answer
))
{
return
;
}
...
...
@@ -158,7 +161,7 @@ class Adgoal
*/
public
function
parseAffiliates
(
&
$results
)
{
if
(
$this
->
finished
||
$this
->
affiliates
===
null
){
if
(
$this
->
finished
||
$this
->
affiliates
===
null
)
{
return
;
}
...
...
@@ -168,7 +171,7 @@ class Adgoal
$tld
=
$partnershop
[
"tld"
];
// Sometimes TLD is null
if
(
empty
(
$tld
)){
if
(
empty
(
$tld
))
{
continue
;
}
...
...
@@ -178,13 +181,18 @@ class Adgoal
Adgoal sometimes returns affiliate Links for every URL
That's why we check if the corresponding TLD matches the orginial URL
*/
if
(
$targetHost
!==
false
&&
stripos
(
$targetHost
,
$tld
)
===
false
){
if
(
$targetHost
!==
false
&&
stripos
(
$targetHost
,
$tld
)
===
false
)
{
continue
;
}
$preference
=
config
(
"metager.metager.affiliate_preference"
,
"adgoal"
);
foreach
(
$results
as
$result
)
{
if
(
$result
->
originalLink
===
$targetUrl
&&
(
config
(
"metager.metager.affiliate_preference"
,
"adgoal"
)
===
"adgoal"
||
!
$result
->
partnershop
))
{
# Ein Advertiser gefunden
# Check ob er auf der Blacklist steht
if
(
Redis
::
hexists
(
\
App\Http\Controllers\AdgoalController
::
REDIS_BLACKLIST_KEY
,
$targetHost
))
{
continue
;
}
if
(
$result
->
image
!==
""
&&
!
$result
->
partnershop
)
{
$result
->
logo
=
$partnershop
[
"logo"
];
}
else
{
...
...
helpers/entrypointProduction.sh
View file @
d016590b
...
...
@@ -5,5 +5,6 @@ set -e
/bin/sh
-c
"/html/helpers/entrypoint.sh"
php artisan spam:load
php artisan load:affiliate-blacklist
php-fpm7.4
-F
-R
\ No newline at end of file
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment