We can now generate json responses for missing results

app/Http/Controllers/MetaGerSearch.php

@@ -5,6 +5,8 @@ namespace App\Http\Controllers;
 use App;
 use App\MetaGer;
 use Illuminate\Http\Request;
+use Illuminate\Support\Facades\Redis;
+use View;
 
 const TIP_SERVER = 'http://metager3.de:63825/tips.xml';
 
@@ -19,13 +21,6 @@ class MetaGerSearch extends Controller
             return redirect()->to('https://maps.metager.de/map/' . $searchinput . '/1240908.5493525574,6638783.2192695495,6');
         }
 
-        /*if ($focus !== "angepasst" && $this->startsWith($focus, "focus_")) {
-        $metager->parseFormData($request);
-        return $metager->createView();
-        }*/
-
-        #die($request->header('User-Agent'));
-        $time = microtime(true);
         # Mit gelieferte Formulardaten parsen und abspeichern:
         $metager->parseFormData($request);
 
@@ -39,6 +34,12 @@ class MetaGerSearch extends Controller
         # auf Ergebnisse warten und die Ergebnisse laden
         $metager->createSearchEngines($request);
 
+        $metager->startSearch();
+
+        $metager->waitForMainResults();
+
+        $metager->retrieveResults();
+
         # Versuchen die Ergebnisse der Quicktips zu laden
         $quicktipResults = $quicktips->loadResults();
         # Alle Ergebnisse vor der Zusammenführung ranken:
@@ -47,10 +48,93 @@ class MetaGerSearch extends Controller
         # Ergebnisse der Suchmaschinen kombinieren:
         $metager->prepareResults();
 
+        # Save the results in Redis
+        $redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
+        $pipeline = $redis->pipeline();
+        foreach ($metager->getResults() as $result) {
+            $pipeline->rpush($metager->getRedisCurrentResultList(), base64_encode(serialize($result)));
+        }
+        $pipeline->expire($metager->getRedisCurrentResultList(), 6000);
+        $pipeline->execute();
+
         # Die Ausgabe erstellen:
         return $metager->createView($quicktipResults);
     }
 
+    public function loadMore(Request $request)
+    {
+        /**
+         * There are three forms of requests to the resultpage
+         * 1. Initial Request: Loads the fastest searchengines and sends them to the user
+         * 2. Load more results (with JS): Loads new search engines that answered after the initial request was send
+         * 3. Load more results (without JS): Loads new search engines that answered within 1s timeout
+         */
+        if ($request->filled('loadMore') && $request->filled('script') && $request->input('script') === "yes") {
+            return $this->loadMoreJS($request);
+        }
+
+    }
+
+    private function loadMoreJS(Request $request)
+    {
+        # Create a MetaGer Instance with the supplied hash
+        $hash = $request->input('loadMore', '');
+
+        $metager = new MetaGer($hash);
+        $redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
+
+        $result = [];
+        # Check if there should be more results
+        $stats = $redis->hgetall($metager->getRedisEngineResult() . "status");
+        $stats["startTime"] = floatval($stats["startTime"]);
+        $stats["engineCount"] = intval($stats["engineCount"]);
+        $stats["engineAnswered"] = intval($stats["engineAnswered"]);
+        $stats["engineDelivered"] = intval($stats["engineDelivered"]);
+
+        $result["finished"] = true;
+        $result["engineCount"] = $stats["engineCount"];
+        $result["engineAnswered"] = $stats["engineAnswered"];
+        $result["engineDelivered"] = $stats["engineDelivered"];
+        $result["timeWaiting"] = microtime(true) - $stats["startTime"];
+
+        # Check if we can abort
+        if ($stats["engineAnswered"] > $stats["engineDelivered"]/*&& $result["timeWaiting"] <= 10 */) {
+            $metager->parseFormData($request);
+            # Nach Spezialsuchen überprüfen:
+            $metager->checkSpecialSearches($request);
+
+            # Read which search engines are new
+            $newEngines = [];
+
+            foreach ($redis->lrange($metager->getRedisResultWaitingKey(), 0, -1) as $engine) {
+                $result["engineDelivered"]++;
+                $newEngines[$engine] = $metager->getSumaFile()->sumas->{$engine};
+            }
+            $metager->actuallyCreateSearchEngines($newEngines);
+            # Add the results already delivered to the suer
+            $results = $redis->lrange($metager->getRedisCurrentResultList(), 0, -1);
+            foreach ($results as $index => $oldResult) {
+                $results[$index] = unserialize(base64_decode($oldResult));
+                $results[$index]->new = false;
+            }
+            $metager->setResults($results);
+            $metager->retrieveResults();
+            $metager->rankAll();
+            $metager->prepareResults();
+
+            $results = $metager->getResults();
+            foreach ($results as $index => $resultTmp) {
+                if ($resultTmp->new) {
+                    $view = View::make('layouts.result', ['result' => $resultTmp, 'metager' => $metager]);
+                    $html = $view->render();
+                    $result['newResults'][$index] = $html;
+                }
+            }
+
+        }
+        return response()->json($result);
+    }
+
     public function botProtection($redirect)
     {
         $hash = md5(date('YmdHi'));

app/Jobs/Searcher.php

@@ -171,12 +171,14 @@ class Searcher implements ShouldQueue
         }
         $pipeline = Redis::pipeline();
         $pipeline->hset('search.' . $hashValue . ".results." . $this->name, "response", $result);
+        $pipeline->hset('search.' . $hashValue . ".results." . $this->name, "delivered", "0");
+        $pipeline->hincrby('search.' . $hashValue . ".results.status", "engineAnswered", 1);
         // After 60 seconds the results should be read by the MetaGer Process and stored in the Cache instead
-        $pipeline->expire('search.' . $hashValue . ".results." . $this->name, 60);
+        $pipeline->expire('search.' . $hashValue . ".results." . $this->name, 6000);
         $pipeline->rpush('search.' . $hashValue . ".ready", $this->name);
-        $pipeline->expire('search.' . $hashValue . ".ready", 60);
+        $pipeline->expire('search.' . $hashValue . ".ready", 6000);
         $pipeline->sadd('search.' . $hashValue . ".engines", $this->name);
-        $pipeline->expire('search.' . $hashValue . ".engines", 60);
+        $pipeline->expire('search.' . $hashValue . ".engines", 6000);
         $pipeline->execute();
         $this->lastTime = microtime(true);
     }

app/Models/Result.php

@@ -27,6 +27,7 @@ class Result
     public $strippedDomain; # Die Domain in Form "bar.de"
     public $strippedLink; # Der Link in Form "foo.bar.de/test"
     public $rank; # Das Ranking für das Ergebnis
+    public $new = true;
 
     # Erstellt ein neues Ergebnis
     public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $gefVonLink, $sourceRank, $additionalInformation = [])
@@ -67,6 +68,7 @@ class Result
         $this->price = isset($additionalInformation["price"]) ? $additionalInformation["price"] : 0;
         $this->price_text = $this->price_to_text($this->price);
         $this->additionalInformation = $additionalInformation;
+        $this->hash = spl_object_hash($this);
     }
 
     private function price_to_text($price)

app/Models/lang.pl

-#!/usr/bin/perl
-
-use Lingua::Identify qw(:language_identification);
-use JSON;
-use warnings;
-use strict;
-binmode STDOUT, ":utf8";
-binmode STDIN, ":utf8";
-use utf8;
-
-chomp(my $filename = <STDIN>);
-
-# Lets open the given file:
-open(my $fh, "<", $filename)
-	or die "Can't open < $filename: $!";
-my $json = <$fh>;
-close $fh;
-
-# Decode the JSON String
-my $data = JSON->new->utf8->decode($json);
-
-# Wir durchlaufen den Hash:
-foreach my $key (keys %{$data}){
-	$data->{$key} = langof($data->{$key});
-}
-
-$data = encode_json($data);
-
-# Nur noch die temporäre Datei löschen:
-unlink($filename);
-
-print $data;

readme.md

@@ -18,9 +18,6 @@
   * php-gd
 * sqlite3
 * redis-server
-* Die Perl-Pakete
-  * Lingua::Identify (http://search.cpan.org/~ambs/Lingua-Identify-0.56/lib/Lingua/Identify.pm)
-  * JSON (http://search.cpan.org/~makamaka/JSON-2.90/lib/JSON.pm)
 
 ---
 [<img src="public/img/Browserstack-logo_2x.png" width="250px" alt="Browserstack Logo" />](https://www.browserstack.com) <br /> 

resources/js/scriptResultPage.js

@@ -53,5 +53,48 @@ function enableFormResetter() {
 }
 
 function loadMoreResults() {
+  var searchKey = $("meta[name=searchkey]").attr("content");
+  var updateUrl = document.location.href;
+  updateUrl += "&loadMore=" + searchKey + "&script=yes";
+
+  updateUrl = updateUrl.replace("/meta.ger3", "/loadMore");
+
+  var expectedEngines = -1;
+  var deliveredEngines = -1;
+
+  var currentlyLoading = false;
+
+  // Regularily check for not yet delivered Results
+  var resultLoader = window.setInterval(function () {
+    if (!currentlyLoading) {
+      currentlyLoading = true;
+      $.getJSON(updateUrl, function (data) {
+        // Check if we can clear the interval (once every searchengine has answered)
+        if (data.engineDelivered == data.engineCount || data.timeWaiting > 5) {
+          clearInterval(resultLoader);
+        }
+        // If there are new results we can add them
+        if (typeof data.newResults != "undefined") {
+          for (var key in data.newResults) {
+            var value = data.newResults[key];
+
+            // If there are more results than the given index we will prepend otherwise we will append the result
+            var results = $(".result:not(.ad)");
+            if (typeof results[key] != "undefined") {
+              console.log("inserting before " + key);
+              $(results[key]).insertBefore(value);
+            } else {
+              $(results[key - 1]).insertAfter(value);
+            }
+
+            console.log(key + "=>" + value);
+          }
+        }
+        currentlyLoading = false;
+      });
+    }
+  }, 1000);
+
+  console.log(updateUrl);
 
 }
\ No newline at end of file

resources/views/layouts/result.blade.php

-<div class="result" data-count="{{ $result->number }}">
+<div class="result" data-count="{{ $result->hash }}">
 	<div class="result-header">
 		<div class="result-headline">
 			<h2 class="result-title" title="{{ $result->titel }}">
@@ -47,7 +47,7 @@
 			</div>
 		@endif
 	</div>
-	<input type="checkbox" id="result-toggle-{{$result->number}}" class="result-toggle" style="display: none">
+	<input type="checkbox" id="result-toggle-{{$result->hash}}" class="result-toggle" style="display: none">
 	<div class="result-footer">
 		<a class="result-open" href="{{ $result->link }}" target="_self" rel="noopener">
 			{!! trans('result.options.7') !!}
@@ -58,10 +58,10 @@
 		<a class="result-open-proxy" onmouseover="$(this).popover('show');" onmouseout="$(this).popover('hide');" data-toggle="popover" data-placement="auto right" data-container="body" data-content="@lang('result.proxytext')" href="{{ $result->proxyLink }}" target="{{ $metager->getNewtab() }}" rel="noopener">
 			{!! trans('result.options.5') !!}
 		</a>
-		<label class="open-result-options navigation-element" for="result-toggle-{{$result->number}}">
+		<label class="open-result-options navigation-element" for="result-toggle-{{$result->hash}}">
 			{{ trans('result.options.more')}}
 		</label>
-		<label class="close-result-options navigation-element" for="result-toggle-{{$result->number}}">
+		<label class="close-result-options navigation-element" for="result-toggle-{{$result->hash}}">
 			{{ trans('result.options.less')}}
 		</label>
 	</div>
@@ -69,7 +69,7 @@
 		<div class="options">
 				<ul class="option-list list-unstyled small">
 					<li class="js-only">
-						<a href="javascript:resultSaver({{ $result->number }});" class="saver">
+						<a href="javascript:resultSaver("{{ $result->hash }}");" class="saver">
 							<nobr><i class="fa fa-floppy-o"></i> {!! trans('result.options.savetab') !!}</nobr>
 						</a>
 					</li>

resources/views/resultpages/results.blade.php

@@ -7,17 +7,17 @@
 		@endfor
 	@endif
 	{{-- Create results and ongoing ads --}}
-	@foreach($metager->getResults() as $result)
+	@foreach($metager->getResults() as $index => $result)
 		@if($mobile)
-			@if($result->number % 4 === 0)
+			@if($index % 4 === 0)
 				@include('layouts.ad', ['ad' => $metager->popAd()])
 			@endif
 		@else
-			@if($result->number % 7 === 0)
+			@if($index % 7 === 0)
 				@include('layouts.ad', ['ad' => $metager->popAd()])
 			@endif
 		@endif
 		@include('layouts.result', ['result' => $result])
 	@endforeach
 	@include('parts.pager')
-</div>
\ No newline at end of file
+</div>

routes/web.php

@@ -179,6 +179,7 @@ Route::group(
         Route::get('settings', 'StartpageController@loadSettings');
 
         Route::match(['get', 'post'], 'meta/meta.ger3', 'MetaGerSearch@search')->middleware('humanverification');
+        Route::get('meta/loadMore', 'MetaGerSearch@loadMore');
         Route::post('img/cat.jpg', 'HumanVerification@remove');
         Route::get('r/metager/{mm}/{pw}/{url}', ['as' => 'humanverification', 'uses' => 'HumanVerification@removeGet']);
         Route::post('img/dog.jpg', 'HumanVerification@whitelist');