Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
open-source
MetaGer
Commits
7a343603
Commit
7a343603
authored
Apr 15, 2019
by
Dominik Hebeler
Browse files
added blacklist for advertisements
parent
72edb302
Changes
5
Hide whitespace changes
Inline
Side-by-side
.gitlab-ci.yml
View file @
7a343603
...
...
@@ -22,6 +22,8 @@ update(144.76.113.134):
-
scp -P 63824 metager@metager3.de:~/sumasEn.json config/
-
scp -P 63824 metager@metager3.de:~/blacklistUrl.txt config/
-
scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
-
scp -P 63824 metager@metager3.de:~/adBlacklistUrl.txt config/
-
scp -P 63824 metager@metager3.de:~/adBlacklistDomains.txt config/
-
touch storage/logs/laravel.log
-
touch storage/logs/worker.log
-
touch database/metager.sqlite
...
...
app/MetaGer.php
View file @
7a343603
...
...
@@ -57,7 +57,9 @@ class MetaGer
protected
$sprueche
;
protected
$newtab
;
protected
$domainsBlacklisted
=
[];
protected
$adDomainsBlacklisted
=
[];
protected
$urlsBlacklisted
=
[];
protected
$adUrlsBlacklisted
=
[];
protected
$url
;
protected
$fullUrl
;
protected
$languageDetect
;
...
...
@@ -79,6 +81,15 @@ class MetaGer
}
else
{
Log
::
warning
(
"Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden"
);
}
# Versuchen Blacklists einzulesen
if
(
file_exists
(
config_path
()
.
"/adBlacklistDomains.txt"
)
&&
file_exists
(
config_path
()
.
"/adBlacklistUrl.txt"
))
{
$tmp
=
file_get_contents
(
config_path
()
.
"/adBlacklistDomains.txt"
);
$this
->
adDomainsBlacklisted
=
explode
(
"
\n
"
,
$tmp
);
$tmp
=
file_get_contents
(
config_path
()
.
"/adBlacklistUrl.txt"
);
$this
->
adUrlsBlacklisted
=
explode
(
"
\n
"
,
$tmp
);
}
else
{
Log
::
warning
(
"Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden"
);
}
# Parser Skripte einhängen
$dir
=
app_path
()
.
"/Models/parserSkripte/"
;
...
...
@@ -266,10 +277,22 @@ class MetaGer
if
(
$result
->
isValid
(
$this
))
{
$newResults
[]
=
$result
;
}
}
$this
->
results
=
$newResults
;
# Validate Advertisements
$newResults
=
[];
foreach
(
$this
->
ads
as
$ad
)
{
if
((
$ad
->
strippedHost
!==
""
&&
(
in_array
(
$ad
->
strippedHost
,
$this
->
adDomainsBlacklisted
)
||
in_array
(
$ad
->
strippedLink
,
$this
->
adUrlsBlacklisted
)))
||
(
$ad
->
strippedHostAnzeige
!==
""
&&
(
in_array
(
$ad
->
strippedHostAnzeige
,
$this
->
adDomainsBlacklisted
)
||
in_array
(
$ad
->
strippedLinkAnzeige
,
$this
->
adUrlsBlacklisted
))))
{
continue
;
}
$newResults
[]
=
$ad
;
}
$this
->
ads
=
$newResults
;
#Adgoal Implementation
if
(
!
$this
->
apiAuthorized
)
{
$this
->
results
=
$this
->
parseAdgoal
(
$this
->
results
);
...
...
app/Models/Result.php
View file @
7a343603
...
...
@@ -26,6 +26,9 @@ class Result
public
$strippedHost
;
# Der Host in Form "foo.bar.de"
public
$strippedDomain
;
# Die Domain in Form "bar.de"
public
$strippedLink
;
# Der Link in Form "foo.bar.de/test"
public
$strippedHostAnzeige
;
# Der Host in Form "foo.bar.de"
public
$strippedDomainAnzeige
;
# Die Domain in Form "bar.de"
public
$strippedLinkAnzeige
;
# Der Link in Form "foo.bar.de/test"
public
$rank
;
# Das Ranking für das Ergebnis
public
$new
=
true
;
...
...
@@ -62,6 +65,9 @@ class Result
$this
->
strippedHost
=
$this
->
getStrippedHost
(
$this
->
link
);
$this
->
strippedDomain
=
$this
->
getStrippedDomain
(
$this
->
link
);
$this
->
strippedLink
=
$this
->
getStrippedLink
(
$this
->
link
);
$this
->
strippedHostAnzeige
=
$this
->
getStrippedHost
(
$this
->
anzeigeLink
);
$this
->
strippedDomainAnzeige
=
$this
->
getStrippedDomain
(
$this
->
anzeigeLink
);
$this
->
strippedLinkAnzeige
=
$this
->
getStrippedLink
(
$this
->
anzeigeLink
);
$this
->
rank
=
0
;
$this
->
partnershop
=
isset
(
$additionalInformation
[
"partnershop"
])
?
$additionalInformation
[
"partnershop"
]
:
false
;
$this
->
image
=
isset
(
$additionalInformation
[
"image"
])
?
$additionalInformation
[
"image"
]
:
""
;
...
...
@@ -247,7 +253,7 @@ class Result
}
# Allgemeine URL und Domain Blacklist
if
(
$this
->
s
trippedHost
!==
""
&&
(
in_array
(
$this
->
strippedHost
,
$metager
->
getDomain
Black
l
ist
())
||
in_array
(
$this
->
strippedLink
,
$metager
->
getUrlBlacklist
())
))
{
if
(
$this
->
i
sBlack
L
ist
ed
(
$metager
))
{
return
false
;
}
...
...
@@ -280,6 +286,19 @@ class Result
}
}
public
function
isBlackListed
(
\
App\MetaGer
$metager
)
{
if
((
$this
->
strippedHost
!==
""
&&
(
in_array
(
$this
->
strippedHost
,
$metager
->
getDomainBlacklist
())
||
in_array
(
$this
->
strippedLink
,
$metager
->
getUrlBlacklist
())))
||
(
$this
->
strippedHostAnzeige
!==
""
&&
(
in_array
(
$this
->
strippedHostAnzeige
,
$metager
->
getDomainBlacklist
())
||
in_array
(
$this
->
strippedLinkAnzeige
,
$metager
->
getUrlBlacklist
()))))
{
return
true
;
}
else
{
return
false
;
}
}
/* Liest aus einem Link den Host.
* Dieser wird dabei in die Form:
* "http://www.foo.bar.de/test?ja=1" -> "foo.bar.de"
...
...
@@ -362,6 +381,10 @@ return "https://proxy.suma-ev.de/mger/nph-proxy.cgi/en/w0/" . $tmp;
*/
public
function
getUrlElements
(
$url
)
{
if
(
!
starts_with
(
$url
,
"http"
))
{
$url
=
"http://"
.
$url
;
}
$parts
=
parse_url
(
$url
);
$re
=
[];
...
...
config/adBlacklistDomains.txt
0 → 100644
View file @
7a343603
kopp-verlag.de
\ No newline at end of file
config/adBlacklistUrl.txt
0 → 100644
View file @
7a343603
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment