Merge branch '864-optimize-request-times' into 'development'

Resolve "Optimize request times"

Closes #864

See merge request !1386

app/Http/Controllers/MetaGerSearch.php

@@ -4,7 +4,10 @@ namespace App\Http\Controllers;
 
 use App;
 use App\MetaGer;
+use Cache;
 use Illuminate\Http\Request;
+use Illuminate\Support\Facades\Redis;
+use View;
 
 const TIP_SERVER = 'http://metager3.de:63825/tips.xml';
 
@@ -19,13 +22,6 @@ class MetaGerSearch extends Controller
             return redirect()->to('https://maps.metager.de/map/' . $searchinput . '/1240908.5493525574,6638783.2192695495,6');
         }
 
-        /*if ($focus !== "angepasst" && $this->startsWith($focus, "focus_")) {
-        $metager->parseFormData($request);
-        return $metager->createView();
-        }*/
-
-        #die($request->header('User-Agent'));
-        $time = microtime();
         # Mit gelieferte Formulardaten parsen und abspeichern:
         $metager->parseFormData($request);
 
@@ -39,19 +35,132 @@ class MetaGerSearch extends Controller
         # auf Ergebnisse warten und die Ergebnisse laden
         $metager->createSearchEngines($request);
 
+        $metager->startSearch();
+
+        $metager->waitForMainResults();
+
+        $metager->retrieveResults();
+
         # Versuchen die Ergebnisse der Quicktips zu laden
         $quicktipResults = $quicktips->loadResults();
-
         # Alle Ergebnisse vor der Zusammenführung ranken:
         $metager->rankAll();
 
         # Ergebnisse der Suchmaschinen kombinieren:
         $metager->prepareResults();
 
+        # Save the results in Redis
+        $redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
+        $pipeline = $redis->pipeline();
+        foreach ($metager->getResults() as $result) {
+            $pipeline->rpush($metager->getRedisCurrentResultList(), base64_encode(serialize($result)));
+        }
+        $pipeline->expire($metager->getRedisCurrentResultList(), env('REDIS_RESULT_CACHE_DURATION'));
+        $pipeline->execute();
+
         # Die Ausgabe erstellen:
         return $metager->createView($quicktipResults);
     }
 
+    public function loadMore(Request $request)
+    {
+        /**
+         * There are three forms of requests to the resultpage
+         * 1. Initial Request: Loads the fastest searchengines and sends them to the user
+         * 2. Load more results (with JS): Loads new search engines that answered after the initial request was send
+         * 3. Load more results (without JS): Loads new search engines that answered within 1s timeout
+         */
+        if ($request->filled('loadMore') && $request->filled('script') && $request->input('script') === "yes") {
+            return $this->loadMoreJS($request);
+        }
+
+    }
+
+    private function loadMoreJS(Request $request)
+    {
+        # Create a MetaGer Instance with the supplied hash
+        $hash = $request->input('loadMore', '');
+
+        $metager = new MetaGer($hash);
+        $redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
+
+        $result = [];
+        # Check if there should be more results
+        $stats = $redis->hgetall($metager->getRedisEngineResult() . "status");
+        $stats["startTime"] = floatval($stats["startTime"]);
+        $stats["engineCount"] = intval($stats["engineCount"]);
+        $stats["engineAnswered"] = intval($stats["engineAnswered"]);
+        $stats["engineDelivered"] = intval($stats["engineDelivered"]);
+
+        $result["finished"] = true;
+        $result["engineCount"] = $stats["engineCount"];
+        $result["engineAnswered"] = $stats["engineAnswered"];
+        $result["engineDelivered"] = $stats["engineDelivered"];
+        $result["timeWaiting"] = microtime(true) - $stats["startTime"];
+
+        # Check if we can abort
+        if ($stats["engineAnswered"] > $stats["engineDelivered"]/*&& $result["timeWaiting"] <= 10 */) {
+            $metager->parseFormData($request);
+            # Nach Spezialsuchen überprüfen:
+            $metager->checkSpecialSearches($request);
+
+            # Read which search engines are new
+            $newEngines = [];
+
+            while (($engine = $redis->lpop($metager->getRedisResultWaitingKey())) != null) {
+                $result["engineDelivered"]++;
+                $newEngines[$engine] = $metager->getSumaFile()->sumas->{$engine};
+            }
+            $cache = Cache::get($hash);
+            if ($cache != null) {
+                $metager->setNext(unserialize($cache)["engines"]);
+            }
+
+            # Check if this request is not for page one
+            $metager->setEngines($request, $newEngines);
+
+            # Add the results already delivered to the user
+            $results = $redis->lrange($metager->getRedisCurrentResultList(), 0, -1);
+            foreach ($results as $index => $oldResult) {
+                $results[$index] = unserialize(base64_decode($oldResult));
+                $results[$index]->new = false;
+            }
+            $metager->setResults($results);
+            $metager->retrieveResults();
+            $metager->rankAll();
+            $metager->prepareResults();
+            $result["nextSearchLink"] = $metager->nextSearchLink();
+            $results = $metager->getResults();
+            foreach ($results as $index => $resultTmp) {
+                if ($resultTmp->new) {
+                    if ($metager->getFokus() !== "bilder") {
+                        $view = View::make('layouts.result', ['result' => $resultTmp, 'metager' => $metager]);
+                        $html = $view->render();
+                        $result['newResults'][$index] = $html;
+                        $result["imagesearch"] = false;
+                    } else {
+                        $view = View::make('layouts.image_result', ['result' => $resultTmp, 'metager' => $metager]);
+                        $html = $view->render();
+                        $result['newResults'][$index] = $html;
+                        $result["imagesearch"] = true;
+                    }
+                }
+            }
+            # Save the results in Redis
+            $pipeline = $redis->pipeline();
+            $pipeline->hincrby($metager->getRedisEngineResult() . "status", "engineDelivered", sizeof($newEngines));
+            $pipeline->hset($metager->getRedisEngineResult() . "status", "nextSearchLink", $result["nextSearchLink"]);
+            foreach ($metager->getResults() as $resultTmp) {
+                $resultTmp->new = false;
+                $pipeline->rpush($metager->getRedisCurrentResultList(), base64_encode(serialize($resultTmp)));
+            }
+            $pipeline->expire($metager->getRedisCurrentResultList(), env('REDIS_RESULT_CACHE_DURATION'));
+            $pipeline->execute();
+
+        }
+        return response()->json($result);
+    }
+
     public function botProtection($redirect)
     {
         $hash = md5(date('YmdHi'));

app/Jobs/Searcher.php

@@ -85,8 +85,7 @@ class Searcher implements ShouldQueue
                     $url = base64_decode($mission[1]); // The url to fetch
                     $timeout = $mission[2]; // Timeout from the MetaGer process in ms
                     $medianFetchTime = $this->getFetchTime(); // The median Fetch time of the search engine in ms
-                    Redis::hset('search.' . $hashValue, $this->name, "connected");
-
+                    Redis::hset('search.' . $hashValue . ".results." . $this->name, "status", "connected");
                     $result = $this->retrieveUrl($url);
 
                     $this->storeResult($result, $poptime, $hashValue);
@@ -99,7 +98,7 @@ class Searcher implements ShouldQueue
 
                 // In sync mode every Searcher may only retrieve one result because it would block
                 // the execution of the remaining code otherwise:
-                if (getenv("QUEUE_DRIVER") === "sync"
+                if (getenv("QUEUE_CONNECTION") === "sync"
                     || $this->counter > $this->MAX_REQUESTS
                     || (microtime(true) - $this->startTime) > $this->MAX_TIME) {
                     break;
@@ -161,16 +160,24 @@ class Searcher implements ShouldQueue
         // Set this URL to the Curl handle
         curl_setopt($this->ch, CURLOPT_URL, $url);
         $result = curl_exec($this->ch);
-
         $this->connectionInfo = curl_getinfo($this->ch);
         return $result;
     }
 
     private function storeResult($result, $poptime, $hashValue)
     {
-        Redis::hset('search.' . $hashValue, $this->name, $result);
+        $redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
+        $pipeline = $redis->pipeline();
+        $pipeline->hset('search.' . $hashValue . ".results." . $this->name, "response", $result);
+        $pipeline->hset('search.' . $hashValue . ".results." . $this->name, "delivered", "0");
+        $pipeline->hincrby('search.' . $hashValue . ".results.status", "engineAnswered", 1);
         // After 60 seconds the results should be read by the MetaGer Process and stored in the Cache instead
-        Redis::expire('search.' . $hashValue, 60);
+        $pipeline->expire('search.' . $hashValue . ".results." . $this->name, env('REDIS_RESULT_CACHE_DURATION'));
+        $pipeline->rpush('search.' . $hashValue . ".ready", $this->name);
+        $pipeline->expire('search.' . $hashValue . ".ready", env('REDIS_RESULT_CACHE_DURATION'));
+        $pipeline->sadd('search.' . $hashValue . ".engines", $this->name);
+        $pipeline->expire('search.' . $hashValue . ".engines", env('REDIS_RESULT_CACHE_DURATION'));
+        $pipeline->execute();
         $this->lastTime = microtime(true);
     }
 

app/Models/Quicktips/Quicktips.php

@@ -3,7 +3,6 @@
 namespace App\Models\Quicktips;
 
 use App\Jobs\Searcher;
-use Cache;
 use Illuminate\Foundation\Bus\DispatchesJobs;
 use Illuminate\Support\Facades\Redis;
 use Log;
@@ -26,18 +25,17 @@ class Quicktips
     public function startSearch($search, $locale, $max_time)
     {
         $url = self::QUICKTIP_URL . "?search=" . $this->normalize_search($search) . "&locale=" . $locale;
-
-        $hash = md5($url);
-
         # TODO anders weitergeben
-        $this->hash = $hash;
+        $this->hash = md5($url);
 
         # TODO cache wieder einbauen (eventuell)
         if ( /*!Cache::has($hash)*/true) {
-            Redis::hset("search." . $hash, self::QUICKTIP_NAME, "waiting");
+            $redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
+
+            $redis->hset("search." . $this->hash . ".results." . self::QUICKTIP_NAME, "status", "waiting");
 
             // Queue this search
-            $mission = $hash . ";" . base64_encode($url) . ";" . $max_time;
+            $mission = $this->hash . ";" . base64_encode($url) . ";" . $max_time;
             Redis::rpush(self::QUICKTIP_NAME . ".queue", $mission);
 
             // Check the current status of Searchers for QUICKTIP_NAME
@@ -85,14 +83,11 @@ class Quicktips
     public function retrieveResults($hash)
     {
         $body = "";
-        #if (Cache::has($hash)) {
-        $body = Cache::get($hash);
-        #} elseif (Redis::hexists('search.' . $hash, self::QUICKTIP_NAME)) {
-        $body = Redis::hget('search.' . $hash, self::QUICKTIP_NAME);
-        Redis::hdel('search.' . $hash, self::QUICKTIP_NAME);
-        Cache::put($hash, $body, self::CACHE_DURATION);
-        #}
+        $redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
+        $body = $redis->hget('search.' . $hash . ".results." . self::QUICKTIP_NAME, "response");
 
+        $redis->del('search.' . $hash . ".results." . self::QUICKTIP_NAME);
+        $redis->del('search.' . $hash . ".ready");
         if ($body !== "") {
             return $body;
         } else {

app/Models/Result.php

@@ -27,6 +27,7 @@ class Result
     public $strippedDomain; # Die Domain in Form "bar.de"
     public $strippedLink; # Der Link in Form "foo.bar.de/test"
     public $rank; # Das Ranking für das Ergebnis
+    public $new = true;
 
     # Erstellt ein neues Ergebnis
     public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $gefVonLink, $sourceRank, $additionalInformation = [])
@@ -67,6 +68,7 @@ class Result
         $this->price = isset($additionalInformation["price"]) ? $additionalInformation["price"] : 0;
         $this->price_text = $this->price_to_text($this->price);
         $this->additionalInformation = $additionalInformation;
+        $this->hash = spl_object_hash($this);
     }
 
     private function price_to_text($price)

app/Models/Searchengine.php

@@ -87,7 +87,7 @@ abstract class Searchengine
 
         $this->getString = $this->generateGetString($q);
         $this->hash = md5($this->engine->host . $this->getString . $this->engine->port . $this->name);
-        $this->resultHash = $metager->getHashCode();
+        $this->resultHash = $metager->getSearchUid();
         $this->canCache = $metager->canCache();
     }
 
@@ -102,13 +102,15 @@ abstract class Searchengine
     # Prüft, ob die Suche bereits gecached ist, ansonsted wird sie als Job dispatched
     public function startSearch(\App\MetaGer $metager)
     {
-
         if ($this->canCache && Cache::has($this->hash)) {
             $this->cached = true;
             $this->retrieveResults($metager);
         } else {
+            $redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
             // We will push the confirmation of the submission to the Result Hash
-            Redis::hset('search.' . $this->resultHash, $this->name, "waiting");
+            $redis->hset($metager->getRedisEngineResult() . $this->name, "status", "waiting");
+            $redis->expire($metager->getRedisEngineResult() . $this->name, env('REDIS_RESULT_CACHE_DURATION'));
+
             // We need to submit a action that one of our workers can understand
             // The missions are submitted to a redis queue in the following string format
             // <ResultHash>;<URL to fetch>
@@ -190,10 +192,6 @@ abstract class Searchengine
     {
         foreach ($this->results as $result) {
             $result->rank($eingabe);
-            if (str_contains($this->engine->{"display-name"}, "Yahoo")) {
-                #die(var_dump($result));
-            }
-
         }
     }
 
@@ -210,12 +208,12 @@ abstract class Searchengine
         }
 
         $body = "";
+        $redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
 
         if ($this->canCache && $this->cacheDuration > 0 && Cache::has($this->hash)) {
             $body = Cache::get($this->hash);
-        } elseif (Redis::hexists('search.' . $this->resultHash, $this->name)) {
-            $body = Redis::hget('search.' . $this->resultHash, $this->name);
-            Redis::hdel('search.' . $this->resultHash, $this->name);
+        } elseif ($redis->hexists($metager->getRedisEngineResult() . $this->name, "response")) {
+            $body = $redis->hget($metager->getRedisEngineResult() . $this->name, "response");
             if ($this->canCache && $this->cacheDuration > 0) {
                 Cache::put($this->hash, $body, $this->cacheDuration);
             }
@@ -251,13 +249,7 @@ abstract class Searchengine
 
         # Append the Query String
         $getString .= "&" . $this->engine->{"query-parameter"} . "=" . $this->urlEncode($query);
-/*
-die(var_dump($getString));
 
-# Affildata
-if (strpos($getString, "<<AFFILDATA>>")) {
-$getString = str_replace("<<AFFILDATA>>", $this->getOvertureAffilData($url), $getString);
-}*/
         return $getString;
     }
 

app/Models/lang.pl

-#!/usr/bin/perl
-
-use Lingua::Identify qw(:language_identification);
-use JSON;
-use warnings;
-use strict;
-binmode STDOUT, ":utf8";
-binmode STDIN, ":utf8";
-use utf8;
-
-chomp(my $filename = <STDIN>);
-
-# Lets open the given file:
-open(my $fh, "<", $filename)
-	or die "Can't open < $filename: $!";
-my $json = <$fh>;
-close $fh;
-
-# Decode the JSON String
-my $data = JSON->new->utf8->decode($json);
-
-# Wir durchlaufen den Hash:
-foreach my $key (keys %{$data}){
-	$data->{$key} = langof($data->{$key});
-}
-
-$data = encode_json($data);
-
-# Nur noch die temporäre Datei löschen:
-unlink($filename);
-
-print $data;

readme.md

@@ -18,9 +18,6 @@
   * php-gd
 * sqlite3
 * redis-server
-* Die Perl-Pakete
-  * Lingua::Identify (http://search.cpan.org/~ambs/Lingua-Identify-0.56/lib/Lingua/Identify.pm)
-  * JSON (http://search.cpan.org/~makamaka/JSON-2.90/lib/JSON.pm)
 
 ---
 [<img src="public/img/Browserstack-logo_2x.png" width="250px" alt="Browserstack Logo" />](https://www.browserstack.com) <br /> 

resources/js/scriptResultPage.js

 $(document).ready(function () {
   botProtection();
-
   enableFormResetter();
+  loadMoreResults();
 });
 
 function botProtection() {
@@ -51,3 +51,71 @@ function enableFormResetter() {
     }, 500);
   });
 }
+
+function loadMoreResults() {
+  var searchKey = $("meta[name=searchkey]").attr("content");
+  var updateUrl = document.location.href;
+  updateUrl += "&loadMore=" + searchKey + "&script=yes";
+
+  updateUrl = updateUrl.replace("/meta.ger3", "/loadMore");
+
+  var expectedEngines = -1;
+  var deliveredEngines = -1;
+
+  var currentlyLoading = false;
+
+  // Regularily check for not yet delivered Results
+  var resultLoader = window.setInterval(function () {
+    if (!currentlyLoading) {
+      currentlyLoading = true;
+      $.getJSON(updateUrl, function (data) {
+        // Check if we can clear the interval (once every searchengine has answered)
+        if (data.engineDelivered == data.engineCount || data.timeWaiting > 15) {
+          clearInterval(resultLoader);
+        }
+        // If there are new results we can add them
+        if (typeof data.newResults != "undefined") {
+          for (var key in data.newResults) {
+            var value = data.newResults[key];
+
+            // If there are more results than the given index we will prepend otherwise we will append the result
+            if (!data.imagesearch) {
+              var results = $(".result:not(.ad)");
+              if (key == 0) {
+                if ($(".result.ad").length > 0) {
+                  $(value).insertAfter($($(".result.ad")[$(".result.ad").length - 1]));
+                } else {
+                  $("#results").prepend(value);
+                }
+              } else if (typeof results[key] != "undefined") {
+                $(value).insertBefore($(results[key]));
+              } else if (typeof results[key - 1] != "undefined") {
+                $(value).insertAfter($(results[key - 1]));
+              }
+            } else {
+              var results = $(".image-container > .image");
+              if (key == 0) {
+                $(".image-container").prepend(value);
+              } else if (typeof results[key] != "undefined") {
+                $(value).insertBefore($(results[key]));
+              } else if (typeof results[key - 1] != "undefined") {
+                $(value).insertAfter($(results[key - 1]));
+              }
+            }
+          }
+          if ($(".no-results-error").length > 0 && ($(".image-container > .image").length > 0) || $(".result:not(.ad)").length > 0) {
+            $(".no-results-error").remove();
+            if ($(".alert.alert-danger > ul").children().length == 0) {
+              $(".alert.alert-danger").remove();
+            }
+          }
+          console.log(data);
+        }
+        currentlyLoading = false;
+      });
+    }
+  }, 1000);
+  //clearInterval(resultLoader);
+  console.log(updateUrl);
+
+}
\ No newline at end of file

resources/views/layouts/image_result.blade.php

+<div class="image">
+	<a href="{{ $result->link }}" target="_blank">
+		<div title="{{ $result->titel }}">
+			<img src="{{ $metager->getImageProxyLink($result->image)}}" alt="{{ $result->titel }}"/>
+			<div>{{ $result->gefVon }}</div>
+		</div>
+	</a>
+</div>

resources/views/layouts/result.blade.php

-<div class="result" data-count="{{ $result->number }}">
+<div class="result" data-count="{{ $result->hash }}">
 	<div class="result-header">
 		<div class="result-headline">
 			<h2 class="result-title" title="{{ $result->titel }}">