Merge branch '1119-implement-admitad' into 'development'

Resolve "Implement Admitad"

Closes #1119

See merge request !1847

app/Http/Controllers/MetaGerSearch.php

@@ -93,6 +93,21 @@ class MetaGerSearch extends Controller
 
         # Ergebnisse der Suchmaschinen kombinieren:
         $metager->prepareResults($timings);
+        $admitad = [];
+        $adgoal = [];
+        if(!$metager->isApiAuthorized() && !$metager->isDummy()){
+            $newAdmitad = new \App\Models\Admitad($metager);
+            if(!empty($newAdmitad->hash)){
+                $admitad[] = $newAdmitad;
+            }
+            $newAdgoal = new \App\Models\Adgoal($metager);
+            if(!empty($newAdgoal->hash)){
+                $adgoal[] = $newAdgoal;
+            }
+        }
+
+        $metager->parseAffiliates($admitad);
+        $metager->parseAffiliates($adgoal);
 
         $finished = true;
         foreach ($metager->getEngines() as $engine) {
@@ -106,10 +121,8 @@ class MetaGerSearch extends Controller
                 "metager" => [
                     "apiAuthorized" => $metager->isApiAuthorized(),
                 ],
-                "adgoal" => [
-                    "loaded" => $metager->isAdgoalLoaded(),
-                    "adgoalHash" => $metager->getAdgoalHash(),
-                ],
+                "admitad" => $admitad,
+                "adgoal" => $adgoal,
                 "engines" => $metager->getEngines(),
             ], 60 * 60);
         } catch (\Exception $e) {
@@ -190,14 +203,14 @@ class MetaGerSearch extends Controller
 
         $engines = $cached["engines"];
         $adgoal = $cached["adgoal"];
+        $admitad = $cached["admitad"];
         $mg = $cached["metager"];
 
         $metager = new MetaGer(substr($hash, strpos($hash, "loader_") + 7));
         $metager->setApiAuthorized($mg["apiAuthorized"]);
-        $metager->setAdgoalLoaded($adgoal["loaded"]);
-        $metager->setAdgoalHash($adgoal["adgoalHash"]);
 
         $metager->parseFormData($request, false);
+        $metager->setJsEnabled(true);
         # Nach Spezialsuchen überprüfen:
         $metager->checkSpecialSearches($request);
         $metager->restoreEngines($engines);
@@ -210,6 +223,20 @@ class MetaGerSearch extends Controller
         $metager->rankAll();
         $metager->prepareResults();
 
+        if(!$metager->isApiAuthorized() && !$metager->isDummy()){
+            $newAdmitad = new \App\Models\Admitad($metager);
+            if(!empty($newAdmitad->hash)){
+                $admitad[] = $newAdmitad;
+            }
+            $newAdgoal = new \App\Models\Adgoal($metager);
+            if(!empty($newAdgoal->hash)){
+                $adgoal[] = $newAdgoal;
+            }
+        }
+
+        $admitadFinished = $metager->parseAffiliates($admitad);
+        $adgoalFinished = $metager->parseAffiliates($adgoal);
+
         $result = [
             'finished' => true,
             'newResults' => [],
@@ -256,7 +283,7 @@ class MetaGerSearch extends Controller
             }
         }
 
-        if (!$metager->isAdgoalLoaded()) {
+        if (!$adgoalFinished || !$admitadFinished) {
             $finished = false;
         }
 
@@ -273,10 +300,8 @@ class MetaGerSearch extends Controller
             "metager" => [
                 "apiAuthorized" => $metager->isApiAuthorized(),
             ],
-            "adgoal" => [
-                "loaded" => $metager->isAdgoalLoaded(),
-                "adgoalHash" => $metager->getAdgoalHash(),
-            ],
+            "admitad" => $admitad,
+            "adgoal" => $adgoal,
             "engines" => $metager->getEngines(),
         ], 1 * 60);
 

app/Http/Controllers/Stresstest.php

@@ -24,7 +24,6 @@ class Stresstest extends MetaGerSearch
 
         # deactivates adgoal
         $metager->setDummy(true);
-        $metager->setAdgoalHash(true);
 
         parent::search($request, $metager, $timing);
     }

app/MetaGer.php

@@ -48,6 +48,7 @@ class MetaGer
     protected $availableFoki = [];
     protected $startCount = 0;
     protected $canCache = false;
+    protected $javascript = false;
     # Daten über die Abfrage$
     protected $ip;
     protected $useragent;
@@ -307,33 +308,12 @@ class MetaGer
         if (empty($this->adgoalLoaded)) {
             $this->adgoalLoaded = false;
         }
-        if (!$this->apiAuthorized && !$this->adgoalLoaded && !$this->dummy) {
-            if (empty($this->adgoalHash)) {
-                if (!empty($this->jskey)) {
-                    $js = Redis::connection('cache')->lpop("js" . $this->jskey);
-                    if ($js !== null && boolval($js)) {
-                        $this->javascript = true;
-                    }
-                }
-                $this->adgoalHash = \App\Models\Adgoal::startAdgoal($this->results);
-                if (!empty($timings)) {
-                    $timings["prepareResults"]["started adgoal"] = microtime(true) - $timings["starttime"];
-                }
-            }
-        
-            if (!$this->javascript) {
-                $this->adgoalLoaded = \App\Models\Adgoal::parseAdgoal($this->results, $this->adgoalHash, true);
-                if (!empty($timings)) {
-                    $timings["prepareResults"]["parsed adgoal"] = microtime(true) - $timings["starttime"];
-                }
-            } else {
-                $this->adgoalLoaded = \App\Models\Adgoal::parseAdgoal($this->results, $this->adgoalHash, false);
-                if (!empty($timings)) {
-                    $timings["prepareResults"]["parsed adgoal"] = microtime(true) - $timings["starttime"];
-                }
+
+        if (!empty($this->jskey)) {
+            $js = Redis::connection('cache')->lpop("js" . $this->jskey);
+            if ($js !== null && boolval($js)) {
+                $this->javascript = true;
             }
-        } else {
-            $this->adgoalLoaded = true;
         }
 
         # Human Verification
@@ -435,7 +415,26 @@ class MetaGer
         }
     }
 
-    
+    /**
+     * @param \App\Models\Admitad[] $admitads
+     * @param Boolean $wait Wait for Results?
+     * @return Boolean whether or not all Admitad Objects are finished
+     */
+    public function parseAffiliates(&$affiliates){
+        $wait = false;
+        $finished = true;
+        if(!$this->javascript){
+            $wait = true;
+        }
+        foreach ($affiliates as $affiliate) {
+            $affiliate->fetchAffiliates($wait);
+            $affiliate->parseAffiliates($this->results);
+            if(!$affiliate->finished){
+                $finished = false;
+            }
+        }
+        return $finished;
+    }
 
     public function humanVerification(&$results)
     {
@@ -1718,6 +1717,9 @@ class MetaGer
         $this->results = $results;
     }
 
+    /**
+     * @return \App\Models\Result[]
+     */
     public function getResults()
     {
         return $this->results;
@@ -1891,26 +1893,6 @@ class MetaGer
         return $this->engines;
     }
 
-    public function setAdgoalHash($hash)
-    {
-        $this->adgoalHash = $hash;
-    }
-
-    public function getAdgoalHash()
-    {
-        return $this->adgoalHash;
-    }
-
-    public function isAdgoalLoaded()
-    {
-        return $this->adgoalLoaded;
-    }
-
-    public function setAdgoalLoaded($adgoalLoaded)
-    {
-        $this->adgoalLoaded = $adgoalLoaded;
-    }
-
     public function isApiAuthorized()
     {
         return $this->apiAuthorized;
@@ -1931,6 +1913,17 @@ class MetaGer
         return $this->headerPrinted;
     }
 
+    public function isDummy(){
+        return $this->dummy;
+    }
+
+    public function jsEnabled() {
+        return $this->javascript;
+    }
+    
+    public function setJsEnabled(bool $bool){
+        $this->javascript = $bool;
+    }
     /**
      * Used by JS result loader to restore MetaGer Object of previous request
      */

app/Models/Adgoal.php

@@ -22,29 +22,47 @@ class Adgoal
         "lk","sh","kn","lc","pm","vc","sd","sr","za","kr","sz","sy","tj","tw","tz","th","tg","to","tt","td","cz","tn",
         "tm","tc","tv","tr","us","ug","ua","xx","hu","uy","uz","vu","va","ve","ae","vn","wf","cx","by","eh","ww","zr","cf","cy",];
 
-    public static function startAdgoal(&$results)
+
+    public $hash;
+    public $finished = false; // Is true when the Request was sent to and read from Admitad App
+    private $affiliates = null;
+    private $startTime;
+
+    /**
+     * Creates a new Adgoal object which will start a request for affiliate links
+     * based on a result List from MetaGer.
+     * It will parse the Links of the results and query any affiliate shops.
+     * 
+     * @param \App\MetaGer $metager
+     */
+    public function __construct(&$metager)
     {
+        $this->startTime = microtime(true);
         $publicKey = getenv('adgoal_public');
         $privateKey = getenv('adgoal_private');
         if ($publicKey === false) {
             return true;
         }
+        $results = $metager->getResults();
         $linkList = "";
         foreach ($results as $result) {
             if (!$result->new) {
                 continue;
             }
-            $link = $result->link;
+            $link = $result->originalLink;
             if (strpos($link, "http") !== 0) {
                 $link = "http://" . $link;
             }
             $linkList .= $link . ",";
         }
+        if(empty($linkList)){
+            return;
+        }
     
         $linkList = rtrim($linkList, ",");
     
         # Hashwert
-        $hash = md5($linkList . $privateKey);
+        $this->hash = md5($linkList . $privateKey);
     
         $link = "https://xf.gdprvalidate.de/v4/check";
     
@@ -73,14 +91,14 @@ class Adgoal
                 "key" => $publicKey,
                 "panel" => "ZMkW9eSKJS",
                 "member" => "338b9Bnm",
-                "signature" => $hash,
+                "signature" => $this->hash,
                 "links" => $linkList,
                 "country" => $country,
             ];
     
         // Submit fetch job to worker
         $mission = [
-                "resulthash" => $hash,
+                "resulthash" => $this->hash,
                 "url" => $link,
                 "useragent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0",
                 "username" => null,
@@ -97,76 +115,86 @@ class Adgoal
             ];
         $mission = json_encode($mission);
         Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
-    
-        return $hash;
     }
-    
-    public static function parseAdgoal(&$results, $hash, $waitForResult)
-    {
-        # Wait for result
-        $startTime = microtime(true);
-        $answer = null;
-    
-        # Hash is true if Adgoal request wasn't started in the first place
-        if ($hash === true) {
-            return true;
+
+    public function fetchAffiliates($wait = false){
+        if($this->affiliates !== null){
+            return;
         }
-            
-        if ($waitForResult) {
+
+        $answer = null;
+        $startTime = microtime(true);
+        if($wait){
             while (microtime(true) - $startTime < 5) {
-                $answer = Cache::get($hash);
+                $answer = Cache::get($this->hash);
                 if ($answer === null) {
                     usleep(50 * 1000);
                 } else {
                     break;
                 }
             }
-        } else {
-            $answer = Cache::get($hash);
+        }else{
+            $answer = Cache::get($this->hash);
         }
-        if ($answer === null) {
-            return false;
+        $answer = json_decode($answer, true);
+        
+        // If the fetcher had an Error
+        if($answer === "no-result"){
+            $this->affiliates = [];
+            return;
+        }
+
+        if(empty($answer) || !is_array($answer)){
+            return;
         }
-            
-        try {
-            $answer = json_decode($answer, true);
 
-            foreach ($answer as $partnershop) {
-                $targetUrl = $partnershop["url"];
+        $this->affiliates = $answer;
+    }
     
-                $tld = $partnershop["tld"];
-                $targetHost = parse_url($targetUrl, PHP_URL_HOST);
+    /**
+     * Converts all Affiliate Links.
+     * 
+     * @param \App\Models\Result[] $results
+     */
+    public function parseAffiliates(&$results)
+    {
+        if($this->finished || $this->affiliates === null){
+            return;
+        }
+            
+        foreach ($this->affiliates as $partnershop) {
+            $targetUrl = $partnershop["url"];
 
-                /*
-                    Adgoal sometimes returns affiliate Links for every URL
-                    That's why we check if the corresponding TLD matches the orginial URL
-                */
-                if($targetHost !== false && stripos($targetHost, $tld) === false){
-                    continue;
-                }
+            $tld = $partnershop["tld"];
+            $targetHost = parse_url($targetUrl, PHP_URL_HOST);
 
-                foreach ($results as $result) {
-                    if ($result->link === $targetUrl && !$result->partnershop) {
-                        # Ein Advertiser gefunden
-                        if ($result->image !== "") {
-                            $result->logo = $partnershop["logo"];
-                        } else {
-                            $result->image = $partnershop["logo"];
-                        }
-    
-                        # Den Link hinzufügen:
-                        $result->link = $partnershop["click_url"];
-                        $result->partnershop = true;
-                        $result->changed = true;
+            /*
+                Adgoal sometimes returns affiliate Links for every URL
+                That's why we check if the corresponding TLD matches the orginial URL
+            */
+            if($targetHost !== false && stripos($targetHost, $tld) === false){
+                continue;
+            }
+            $preference = config("metager.metager.affiliate_preference", "adgoal");
+            foreach ($results as $result) {
+                if ($result->originalLink === $targetUrl && (config("metager.metager.affiliate_preference", "adgoal") === "adgoal" || !$result->partnershop)) {
+                    # Ein Advertiser gefunden
+                    if ($result->image !== "" && !$result->partnershop) {
+                        $result->logo = $partnershop["logo"];
+                    } else {
+                        $result->image = $partnershop["logo"];
                     }
+
+                    # Den Link hinzufügen:
+                    $result->link = $partnershop["click_url"];
+                    $result->partnershop = true;
+                    $result->changed = true;
                 }
             }
-        } catch (\ErrorException $e) {
-            Log::error($e->getMessage());
-        } finally {
-            $requestTime = microtime(true) - $startTime;
-            \App\PrometheusExporter::Duration($requestTime, "adgoal");
         }
-        return true;
+    
+        $requestTime = microtime(true) - $this->startTime;
+        \App\PrometheusExporter::Duration($requestTime, "adgoal");
+        $this->finished = true;
     }
 }

app/Models/Admitad.php

+<?php
+
+namespace App\Models;
+
+use Cache;
+use Log;
+use LaravelLocalization;
+use Illuminate\Support\Facades\Redis;
+
+class Admitad
+{
+    const VALID_LANGS = [
+        "de",
+        "en"
+    ];
+
+    public $hash;
+    public $finished = false; // Is true when the Request was sent to and read from Admitad App
+    private $affiliates = null;
+
+    /**
+     * Creates a new Admitad object which will start a request for affiliate links
+     * based on a result List from MetaGer.
+     * It will parse the Links of the results and query any affiliate shops.
+     * 
+     * @param \App\MetaGer $metager
+     */
+    public function __construct(&$metager)
+    {
+        $results = $metager->getResults();
+        // Generate a list of URLs
+        $resultLinks = [];
+        foreach($results as $result){
+            if ($result->new) {
+                $resultLinks[] = $result->originalLink;
+            }
+        }
+
+        if(empty($resultLinks)){
+            return;
+        }
+
+        $lang = LaravelLocalization::getCurrentLocale();
+        if(!in_array($lang, self::VALID_LANGS)){
+            $lang = "de";
+        }
+
+        $requestData = [
+            "lang" => $lang,
+            "urls" => $resultLinks,
+        ];
+        $requestData = json_encode($requestData);
+        $this->hash = md5($requestData);
+
+        $url = "https://direct.metager.de/check";
+        $token = env("ADMITAD_TOKEN", "");
+
+        // Submit fetch job to worker
+        $mission = [
+            "resulthash" => $this->hash,
+            "url" => $url,
+            "useragent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0",
+            "username" => null,
+            "password" => null,
+            "headers" => [
+                "Authorization" => "Bearer $token"
+            ],
+            "cacheDuration" => 60,
+            "name" => "Admitad",
+            "curlopts" => [
+                CURLOPT_POST => true,
+                CURLOPT_POSTFIELDS => $requestData
+            ]
+        ];
+        $mission = json_encode($mission);
+        Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
+    }
+
+    /**
+     * Fetches the Admitad Response from Redis
+     * @param Boolean $wait Whether or not to wait for a response
+     */
+    public function fetchAffiliates($wait = false) {
+        if($this->affiliates !== null){
+            return;
+        }
+
+        $answer = null;
+        $startTime = microtime(true);
+        if($wait){
+            while (microtime(true) - $startTime < 5) {
+                $answer = Cache::get($this->hash);
+                if ($answer === null) {
+                    usleep(50 * 1000);
+                } else {
+                    break;
+                }
+            }
+        }else{
+            $answer = Cache::get($this->hash);
+        }
+        $answer = json_decode($answer, true);
+        
+        // If the fetcher had an Error
+        if($answer === "no-result"){
+            $this->affiliates = [];
+            return;
+        }
+
+        if(empty($answer) || !isset($answer["error"]) || $answer["error"] || !is_array($answer["result"])){
+            return;
+        }
+
+        $this->affiliates = $answer["result"];
+    }
+
+    /**
+     * Converts all Affiliate Links.
+     * 
+     * @param \App\Models\Result[] $results
+     */
+    public function parseAffiliates(&$results){
+        if($this->finished || $this->affiliates === null){
+            return;
+        }
+        foreach($this->affiliates as $linkResult){
+            $originalUrl = $linkResult["originalUrl"];
+            $redirUrl = $linkResult["redirUrl"];
+            $image = $linkResult["image"];
+
+            if(empty($redirUrl)){
+                // No Partnershop
+                continue;
+            }
+
+            foreach ($results as $result) {
+                if ($result->originalLink === $originalUrl && (config("metager.metager.affiliate_preference", "adgoal") === "admitad" || !$result->partnershop)) {
+                    # Ein Advertiser gefunden
+                    if ($result->image !== "" && !$result->partnershop) {
+                        $result->logo = $image;
+                    } else {
+                        $result->image = $image;
+                    }
+
+                    # Den Link hinzufügen:
+                    $result->link = $redirUrl;
+                    $result->partnershop = true;
+                    $result->changed = true;
+                }
+            }
+        }
+        $this->finished = true;
+    }
+}

app/Models/Result.php

@@ -9,6 +9,7 @@ class Result
 {
     public $provider; # Die Engine von der das Suchergebnis kommt
     public $titel; # Der Groß Angezeigte Name für das Suchergebnis
+    public $originalLink;
     public $link; # Der Link auf die Ergebnisseite
     public $anzeigeLink; # Der tatsächlich angezeigte Link (rein optisch)
     public $descr; # Die eventuell gekürzte Beschreibung des Suchergebnisses
@@ -41,6 +42,7 @@ class Result
         $this->provider = $provider;
         $this->titel = $this->sanitizeText(strip_tags(trim($titel)));
         $this->link = trim($link);
+        $this->originalLink = trim($link);
         $this->anzeigeLink = trim($anzeigeLink);
         $this->anzeigeLink = preg_replace("/(http[s]{0,1}:\/\/){0,1}(www\.){0,1}/si", "", $this->anzeigeLink);
         $this->descr = $this->sanitizeText(strip_tags(trim($descr), '<p>'));

config/metager/metager.php

@@ -6,4 +6,5 @@ return [
         "w3m\/",
     ],
     "fail2ban_enabled" => true,
+    "affiliate_preference" => "adgoal",
 ];

resources/lang/de/partnershops.php

@@ -3,6 +3,6 @@
 return [
     'heading'     => 'MetaGer-Partnershop',
     'paragraph.1' => 'Wenn in der MetaGer-Ergebnisliste Links erscheinen, bei denen das Wort "Partnershop" steht, dann sind das zunächst normale Ergebnisse einer MetaGer-Suche. Sie werden im Ranking nicht bevorzugt oder höher eingestuft, sondern wie alle anderen Suchergebnisse auch behandelt.',
-    'paragraph.2' => 'Hinter diesem Link steht dann eine Firma, von der wir dann, wenn der Link angeklickt wird, einen geringen Betrag zur Finanzierung von MetaGer erhalten. Beim Ergebnis wird das zugehörige Firmenlogo angezeigt. Auch daran erkennen Sie immer unsere Partnershops. Beim Anklicken (und nur dann) eines solchen Ergebnisses werden auch Nutzerdaten an die Klickadresse weitergegeben. Zu diesen Nutzerdaten gehören auch die Suchbegriffe, die Sie auf MetaGer eingegeben haben, in Verbindung mit Ihrer IP-Adresse. Wenn Sie dies vermeiden wollen, dann können Sie be