added blacklist for advertisements

.gitlab-ci.yml

@@ -22,6 +22,8 @@ update(144.76.113.134):
     - scp -P 63824 metager@metager3.de:~/sumasEn.json config/
     - scp -P 63824 metager@metager3.de:~/blacklistUrl.txt config/
     - scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
+    - scp -P 63824 metager@metager3.de:~/adBlacklistUrl.txt config/
+    - scp -P 63824 metager@metager3.de:~/adBlacklistDomains.txt config/
     - touch storage/logs/laravel.log
     - touch storage/logs/worker.log
     - touch database/metager.sqlite

app/MetaGer.php

@@ -57,7 +57,9 @@ class MetaGer
     protected $sprueche;
     protected $newtab;
     protected $domainsBlacklisted = [];
+    protected $adDomainsBlacklisted = [];
     protected $urlsBlacklisted = [];
+    protected $adUrlsBlacklisted = [];
     protected $url;
     protected $fullUrl;
     protected $languageDetect;
@@ -79,6 +81,15 @@ class MetaGer
         } else {
             Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
         }
+        # Versuchen Blacklists einzulesen
+        if (file_exists(config_path() . "/adBlacklistDomains.txt") && file_exists(config_path() . "/adBlacklistUrl.txt")) {
+            $tmp = file_get_contents(config_path() . "/adBlacklistDomains.txt");
+            $this->adDomainsBlacklisted = explode("\n", $tmp);
+            $tmp = file_get_contents(config_path() . "/adBlacklistUrl.txt");
+            $this->adUrlsBlacklisted = explode("\n", $tmp);
+        } else {
+            Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
+        }
 
         # Parser Skripte einhängen
         $dir = app_path() . "/Models/parserSkripte/";
@@ -266,10 +277,22 @@ class MetaGer
             if ($result->isValid($this)) {
                 $newResults[] = $result;
             }
-
         }
         $this->results = $newResults;
 
+        # Validate Advertisements
+        $newResults = [];
+        foreach ($this->ads as $ad) {
+            if (($ad->strippedHost !== "" && (in_array($ad->strippedHost, $this->adDomainsBlacklisted) ||
+                in_array($ad->strippedLink, $this->adUrlsBlacklisted))) ||
+                ($ad->strippedHostAnzeige !== "" && (in_array($ad->strippedHostAnzeige, $this->adDomainsBlacklisted) ||
+                    in_array($ad->strippedLinkAnzeige, $this->adUrlsBlacklisted)))) {
+                continue;
+            }
+            $newResults[] = $ad;
+        }
+        $this->ads = $newResults;
+
         #Adgoal Implementation
         if (!$this->apiAuthorized) {
             $this->results = $this->parseAdgoal($this->results);

app/Models/Result.php

@@ -26,6 +26,9 @@ class Result
     public $strippedHost; # Der Host in Form "foo.bar.de"
     public $strippedDomain; # Die Domain in Form "bar.de"
     public $strippedLink; # Der Link in Form "foo.bar.de/test"
+    public $strippedHostAnzeige; # Der Host in Form "foo.bar.de"
+    public $strippedDomainAnzeige; # Die Domain in Form "bar.de"
+    public $strippedLinkAnzeige; # Der Link in Form "foo.bar.de/test"
     public $rank; # Das Ranking für das Ergebnis
     public $new = true;
 
@@ -62,6 +65,9 @@ class Result
         $this->strippedHost = $this->getStrippedHost($this->link);
         $this->strippedDomain = $this->getStrippedDomain($this->link);
         $this->strippedLink = $this->getStrippedLink($this->link);
+        $this->strippedHostAnzeige = $this->getStrippedHost($this->anzeigeLink);
+        $this->strippedDomainAnzeige = $this->getStrippedDomain($this->anzeigeLink);
+        $this->strippedLinkAnzeige = $this->getStrippedLink($this->anzeigeLink);
         $this->rank = 0;
         $this->partnershop = isset($additionalInformation["partnershop"]) ? $additionalInformation["partnershop"] : false;
         $this->image = isset($additionalInformation["image"]) ? $additionalInformation["image"] : "";
@@ -247,7 +253,7 @@ class Result
         }
 
         # Allgemeine URL und Domain Blacklist
-        if ($this->strippedHost !== "" && (in_array($this->strippedHost, $metager->getDomainBlacklist()) || in_array($this->strippedLink, $metager->getUrlBlacklist()))) {
+        if ($this->isBlackListed($metager)) {
             return false;
         }
 
@@ -280,6 +286,19 @@ class Result
         }
     }
 
+    public function isBlackListed(\App\MetaGer $metager)
+    {
+        if (($this->strippedHost !== "" && (in_array($this->strippedHost, $metager->getDomainBlacklist()) ||
+            in_array($this->strippedLink, $metager->getUrlBlacklist()))) ||
+            ($this->strippedHostAnzeige !== "" && (in_array($this->strippedHostAnzeige, $metager->getDomainBlacklist()) ||
+                in_array($this->strippedLinkAnzeige, $metager->getUrlBlacklist())))) {
+            return true;
+        } else {
+            return false;
+        }
+
+    }
+
     /* Liest aus einem Link den Host.
      *  Dieser wird dabei in die Form:
      *  "http://www.foo.bar.de/test?ja=1" -> "foo.bar.de"
@@ -362,6 +381,10 @@ return "https://proxy.suma-ev.de/mger/nph-proxy.cgi/en/w0/" . $tmp;
      */
     public function getUrlElements($url)
     {
+        if (!starts_with($url, "http")) {
+            $url = "http://" . $url;
+        }
+
         $parts = parse_url($url);
         $re = [];
 

config/adBlacklistDomains.txt

+kopp-verlag.de
\ No newline at end of file