loading adgoal async when javascript is enabled

app/Http/Controllers/MetaGerSearch.php

@@ -111,7 +111,16 @@ class MetaGerSearch extends Controller
         }
 
         try {
-            Cache::put("loader_" . $metager->getSearchUid(), $metager->getEngines(), 60 * 60);
+            Cache::put("loader_" . $metager->getSearchUid(), [
+                "metager" => [
+                    "apiAuthorized" => $metager->isApiAuthorized(),
+                ],
+                "adgoal" => [
+                    "loaded" => $metager->isAdgoalLoaded(),
+                    "adgoalHash" => $metager->getAdgoalHash(),
+                ],
+                "engines" => $metager->getEngines(),
+            ], 60 * 60);
         } catch (\Exception $e) {
             Log::error($e->getMessage());
         }
@@ -190,12 +199,19 @@ class MetaGerSearch extends Controller
             }
         }
 
-        $engines = Cache::get($hash);
-        if ($engines === null) {
+        $cached = Cache::get($hash);
+        if ($cached === null) {
             return response()->json(['finished' => true]);
         }
 
+        $engines = $cached["engines"];
+        $adgoal = $cached["adgoal"];
+        $mg = $cached["metager"];
+
         $metager = new MetaGer(substr($hash, strpos($hash, "loader_") + 7));
+        $metager->setApiAuthorized($mg["apiAuthorized"]);
+        $metager->setAdgoalLoaded($adgoal["loaded"]);
+        $metager->setAdgoalHash($adgoal["adgoalHash"]);
 
         $metager->parseFormData($request);
         # Nach Spezialsuchen überprüfen:
@@ -203,27 +219,37 @@ class MetaGerSearch extends Controller
         $metager->restoreEngines($engines);
 
         $metager->retrieveResults();
+
         $metager->rankAll();
         $metager->prepareResults();
 
         $result = [
             'finished' => true,
             'newResults' => [],
+            'changedResults' => [],
         ];
         $result["nextSearchLink"] = $metager->nextSearchLink();
 
         $newResults = 0;
         foreach ($metager->getResults() as $index => $resultTmp) {
-            if ($resultTmp->new) {
+            if ($resultTmp->new || $resultTmp->adgoalChanged) {
                 if ($metager->getFokus() !== "bilder") {
                     $view = View::make('layouts.result', ['index' => $index, 'result' => $resultTmp, 'metager' => $metager]);
                     $html = $view->render();
-                    $result['newResults'][$index] = $html;
+                    if (!$resultTmp->new && $resultTmp->adgoalChanged) {
+                        $result['changedResults'][$index] = $html;
+                    } else {
+                        $result['newResults'][$index] = $html;
+                    }
                     $result["imagesearch"] = false;
                 } else {
                     $view = View::make('layouts.image_result', ['index' => $index, 'result' => $resultTmp, 'metager' => $metager]);
                     $html = $view->render();
-                    $result['newResults'][$index] = $html;
+                    if (!$resultTmp->new && $resultTmp->adgoalChanged) {
+                        $result['changedResults'][$index] = $html;
+                    } else {
+                        $result['newResults'][$index] = $html;
+                    }
                     $result["imagesearch"] = true;
                 }
                 $newResults++;
@@ -239,6 +265,9 @@ class MetaGerSearch extends Controller
                 $engine->markNew();
             }
         }
+        if (!$metager->isAdgoalLoaded()) {
+            $finished = false;
+        }
 
         $result["finished"] = $finished;
 
@@ -248,7 +277,16 @@ class MetaGerSearch extends Controller
             $counter->incBy($newResults);
         }
         // Update new Engines
-        Cache::put("loader_" . $metager->getSearchUid(), $metager->getEngines(), 1 * 60);
+        Cache::put("loader_" . $metager->getSearchUid(), [
+            "metager" => [
+                "apiAuthorized" => $metager->isApiAuthorized(),
+            ],
+            "adgoal" => [
+                "loaded" => $metager->isAdgoalLoaded(),
+                "adgoalHash" => $metager->getAdgoalHash(),
+            ],
+            "engines" => $metager->getEngines(),
+        ], 1 * 60);
         return response()->json($result);
     }
 

app/MetaGer.php

@@ -300,13 +300,25 @@ class MetaGer
         $this->ads = $newResults;
 
         #Adgoal Implementation
-        if (!$this->apiAuthorized) {
-            $this->results = $this->parseAdgoal($this->results);
+        if (empty($this->adgoalLoaded)) {
+            $this->adgoalLoaded = false;
+        }
+        if (!$this->apiAuthorized && !$this->adgoalLoaded) {
+            if (empty($this->adgoalHash)) {
+                $this->adgoalHash = $this->startAdgoal($this->results);
+            }
+            if (!$this->javascript) {
+                $this->adgoalLoaded = $this->parseAdgoal($this->results, $this->adgoalHash, true);
+            } else {
+                $this->adgoalLoaded = $this->parseAdgoal($this->results, $this->adgoalHash, false);
+            }
+        } else {
+            $this->adgoalLoaded = true;
         }
 
         # Human Verification
-        $this->results = $this->humanVerification($this->results);
-        $this->ads = $this->humanVerification($this->ads);
+        $this->humanVerification($this->results);
+        $this->humanVerification($this->ads);
 
         $counter = 0;
         $firstRank = 0;
@@ -343,54 +355,98 @@ class MetaGer
             }
             foreach ($engine->results as $result) {
                 if ($result->valid) {
-                    $this->results[] = $result;
+                    $this->results[] = clone $result;
                 }
             }
             foreach ($engine->ads as $ad) {
-                $this->ads[] = $ad;
+                $this->ads[] = clone $ad;
             }
         }
     }
 
-    public function parseAdgoal($results)
+    public function startAdgoal(&$results)
     {
-        $time = microtime(true);
         $publicKey = getenv('adgoal_public');
         $privateKey = getenv('adgoal_private');
         if ($publicKey === false) {
             return $results;
         }
         $tldList = "";
+        foreach ($results as $result) {
+            if (!$result->new) {
+                continue;
+            }
+            $link = $result->link;
+            if (strpos($link, "http") !== 0) {
+                $link = "http://" . $link;
+            }
+            $tldList .= parse_url($link, PHP_URL_HOST) . ",";
+            $result->tld = parse_url($link, PHP_URL_HOST);
+        }
+        $tldList = rtrim($tldList, ",");
+
+        # Hashwert
+        $hash = md5("meta" . $publicKey . $tldList . "GER");
+        Redis::del($hash); # TODO remove
+        # Query
+        $query = $this->q;
+
+        $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . urlencode($publicKey) . "&k=" . urlencode($hash) . "&tld=" . urlencode($tldList) . "&q=" . urlencode($query);
+
+        // Submit fetch job to worker
+        $mission = [
+            "resulthash" => $hash,
+            "url" => $link,
+            "useragent" => "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:81.0) Gecko/20100101 Firefox/81.0",
+            "username" => null,
+            "password" => null,
+            "headers" => null,
+            "cacheDuration" => 60,
+        ];
+        $mission = json_encode($mission);
+        Redis::rpush(\App\MetaGer::FETCHQUEUE_KEY, $mission);
+
+        return $hash;
+    }
+
+    public function parseAdgoal(&$results, $hash, $waitForResult)
+    {
+        # Wait for result
+        $startTime = microtime(true);
+        $answer = null;
+
+        if ($waitForResult) {
+            while (microtime(true) - $startTime < 5) {
+                $answer = Redis::get($hash);
+                if ($answer === null) {
+                    usleep(50 * 1000);
+                } else {
+                    break;
+                }
+            }
+        } else {
+            $answer = Redis::get($hash);
+        }
+        if ($answer === null) {
+            return false;
+        }
         try {
+            $answer = json_decode($answer);
+            $publicKey = getenv('adgoal_public');
+            $privateKey = getenv('adgoal_private');
+
+            # Nun müssen wir nur noch die Links für die Advertiser ändern:
             foreach ($results as $result) {
-                if (!$result->new) {
-                    continue;
-                }
                 $link = $result->link;
-                if (strpos($link, "http") !== 0) {
-                    $link = "http://" . $link;
-                }
-                $tldList .= parse_url($link, PHP_URL_HOST) . ",";
                 $result->tld = parse_url($link, PHP_URL_HOST);
             }
-            $tldList = rtrim($tldList, ",");
-
-            # Hashwert
-            $hash = md5("meta" . $publicKey . $tldList . "GER");
 
-            # Query
-            $query = $this->q;
-
-            $link = "https://api.smartredirect.de/api_v2/CheckForAffiliateUniversalsearchMetager.php?p=" . urlencode($publicKey) . "&k=" . urlencode($hash) . "&tld=" . urlencode($tldList) . "&q=" . urlencode($query);
-            $answer = json_decode(file_get_contents($link));
-
-            # Nun müssen wir nur noch die Links für die Advertiser ändern:
             foreach ($answer as $el) {
                 $hoster = $el[0];
                 $hash = $el[1];
 
                 foreach ($results as $result) {
-                    if ($result->new && $hoster === $result->tld && !$result->partnershop) {
+                    if ($hoster === $result->tld && !$result->partnershop) {
                         # Hier ist ein Advertiser:
                         # Das Logo hinzufügen:
                         if ($result->image !== "") {
@@ -400,27 +456,29 @@ class MetaGer
                         }
 
                         # Den Link hinzufügen:
-                        $publicKey = $publicKey;
                         $targetUrl = $result->link;
+                        # Query
+                        $query = $this->q;
 
                         $gateHash = md5($targetUrl . $privateKey);
                         $newLink = "https://api.smartredirect.de/api_v2/ClickGate.php?p=" . urlencode($publicKey) . "&k=" . urlencode($gateHash) . "&url=" . urlencode($targetUrl) . "&q=" . urlencode($query);
                         $result->link = $newLink;
                         $result->partnershop = true;
+                        $result->adgoalChanged = true;
                     }
                 }
             }
         } catch (\ErrorException $e) {
-            return $results;
+            Log::error($e->getMessage());
         } finally {
-            $requestTime = microtime(true) - $time;
+            $requestTime = microtime(true) - $startTime;
             \App\PrometheusExporter::Duration($requestTime, "adgoal");
         }
+        return true;
 
-        return $results;
     }
 
-    public function humanVerification($results)
+    public function humanVerification(&$results)
     {
         # Let's check if we need to implement a redirect for human verification
         if ($this->verificationCount > 10) {
@@ -434,9 +492,6 @@ class MetaGer
                 $result->link = $url;
                 $result->proxyLink = $proxyUrl;
             }
-            return $results;
-        } else {
-            return $results;
         }
     }
 
@@ -1833,6 +1888,36 @@ class MetaGer
         return $this->engines;
     }
 
+    public function setAdgoalHash($hash)
+    {
+        $this->adgoalHash = $hash;
+    }
+
+    public function getAdgoalHash()
+    {
+        return $this->adgoalHash;
+    }
+
+    public function isAdgoalLoaded()
+    {
+        return $this->adgoalLoaded;
+    }
+
+    public function setAdgoalLoaded($adgoalLoaded)
+    {
+        $this->adgoalLoaded = $adgoalLoaded;
+    }
+
+    public function isApiAuthorized()
+    {
+        return $this->apiAuthorized;
+    }
+
+    public function setApiAuthorized($authorized)
+    {
+        $this->apiAuthorized = $authorized;
+    }
+
     public function isFramed()
     {
         return $this->framed;
@@ -1849,5 +1934,8 @@ class MetaGer
     public function restoreEngines($engines)
     {
         $this->engines = $engines;
+        foreach ($this->engines as $engine) {
+            $engine->setCached(true);
+        }
     }
 }

app/Models/Result.php

@@ -31,6 +31,7 @@ class Result
     public $strippedLinkAnzeige; # Der Link in Form "foo.bar.de/test"
     public $rank; # Das Ranking für das Ergebnis
     public $new = true;
+    public $adgoalChanged = false;
 
     # Erstellt ein neues Ergebnis
     public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $gefVonLink, $sourceRank, $additionalInformation = [])

app/Models/Searchengine.php

@@ -208,10 +208,12 @@ abstract class Searchengine
         }
 
         if ($body !== null) {
-            try {
-                Cache::put($this->hash, $body, $this->cacheDuration * 60);
-            } catch (\Exception $e) {
-                Log::error($e->getMessage());
+            if (!$this->cached) {
+                try {
+                    Cache::put($this->hash, $body, $this->cacheDuration * 60);
+                } catch (\Exception $e) {
+                    Log::error($e->getMessage());
+                }
             }
             $this->loadResults($body);
             $this->getNext($metager, $body);
@@ -288,4 +290,9 @@ abstract class Searchengine
     {
         $this->new = $new;
     }
+
+    public function setCached($cached)
+    {
+        $this->cached = $cached;
+    }
 }

resources/js/scriptResultPage.js

@@ -116,6 +116,17 @@ function loadMoreResults() {
             }
           }
         }
+        if (typeof data.changedResults != "undefined") {
+          for (var key in data.changedResults) {
+            var value = data.changedResults[key];
+            // If there are more results than the given index we will prepend otherwise we will append the result
+            if (!data.imagesearch) {
+              $($(".result:not(.ad)")[key]).replaceWith($(value));
+            } else {
+              $($(".image-container > .image")[key]).replaceWith($(value));
+            }
+          }
+        }
         currentlyLoading = false;
       });
     }