diff --git a/app/Http/Controllers/MetaGerSearch.php b/app/Http/Controllers/MetaGerSearch.php index 66e8f8c29671c5bdaba8e00689fd07a810abda57..9b992a76398a00b04c993b97b0c8dfcc3a0a3c83 100644 --- a/app/Http/Controllers/MetaGerSearch.php +++ b/app/Http/Controllers/MetaGerSearch.php @@ -5,6 +5,8 @@ namespace App\Http\Controllers; use App; use App\MetaGer; use Illuminate\Http\Request; +use Illuminate\Support\Facades\Redis; +use View; const TIP_SERVER = 'http://metager3.de:63825/tips.xml'; @@ -19,13 +21,6 @@ class MetaGerSearch extends Controller return redirect()->to('https://maps.metager.de/map/' . $searchinput . '/1240908.5493525574,6638783.2192695495,6'); } - /*if ($focus !== "angepasst" && $this->startsWith($focus, "focus_")) { - $metager->parseFormData($request); - return $metager->createView(); - }*/ - - #die($request->header('User-Agent')); - $time = microtime(true); # Mit gelieferte Formulardaten parsen und abspeichern: $metager->parseFormData($request); @@ -39,6 +34,12 @@ class MetaGerSearch extends Controller # auf Ergebnisse warten und die Ergebnisse laden $metager->createSearchEngines($request); + $metager->startSearch(); + + $metager->waitForMainResults(); + + $metager->retrieveResults(); + # Versuchen die Ergebnisse der Quicktips zu laden $quicktipResults = $quicktips->loadResults(); # Alle Ergebnisse vor der Zusammenführung ranken: @@ -47,10 +48,93 @@ class MetaGerSearch extends Controller # Ergebnisse der Suchmaschinen kombinieren: $metager->prepareResults(); + # Save the results in Redis + $redis = Redis::connection(env('REDIS_RESULT_CONNECTION')); + $pipeline = $redis->pipeline(); + foreach ($metager->getResults() as $result) { + $pipeline->rpush($metager->getRedisCurrentResultList(), base64_encode(serialize($result))); + } + $pipeline->expire($metager->getRedisCurrentResultList(), 6000); + $pipeline->execute(); + # Die Ausgabe erstellen: return $metager->createView($quicktipResults); } + public function loadMore(Request $request) + { + /** + * There are three forms of requests to the resultpage + * 1. Initial Request: Loads the fastest searchengines and sends them to the user + * 2. Load more results (with JS): Loads new search engines that answered after the initial request was send + * 3. Load more results (without JS): Loads new search engines that answered within 1s timeout + */ + if ($request->filled('loadMore') && $request->filled('script') && $request->input('script') === "yes") { + return $this->loadMoreJS($request); + } + + } + + private function loadMoreJS(Request $request) + { + # Create a MetaGer Instance with the supplied hash + $hash = $request->input('loadMore', ''); + + $metager = new MetaGer($hash); + $redis = Redis::connection(env('REDIS_RESULT_CONNECTION')); + + $result = []; + # Check if there should be more results + $stats = $redis->hgetall($metager->getRedisEngineResult() . "status"); + $stats["startTime"] = floatval($stats["startTime"]); + $stats["engineCount"] = intval($stats["engineCount"]); + $stats["engineAnswered"] = intval($stats["engineAnswered"]); + $stats["engineDelivered"] = intval($stats["engineDelivered"]); + + $result["finished"] = true; + $result["engineCount"] = $stats["engineCount"]; + $result["engineAnswered"] = $stats["engineAnswered"]; + $result["engineDelivered"] = $stats["engineDelivered"]; + $result["timeWaiting"] = microtime(true) - $stats["startTime"]; + + # Check if we can abort + if ($stats["engineAnswered"] > $stats["engineDelivered"]/*&& $result["timeWaiting"] <= 10 */) { + $metager->parseFormData($request); + # Nach Spezialsuchen überprüfen: + $metager->checkSpecialSearches($request); + + # Read which search engines are new + $newEngines = []; + + foreach ($redis->lrange($metager->getRedisResultWaitingKey(), 0, -1) as $engine) { + $result["engineDelivered"]++; + $newEngines[$engine] = $metager->getSumaFile()->sumas->{$engine}; + } + $metager->actuallyCreateSearchEngines($newEngines); + # Add the results already delivered to the suer + $results = $redis->lrange($metager->getRedisCurrentResultList(), 0, -1); + foreach ($results as $index => $oldResult) { + $results[$index] = unserialize(base64_decode($oldResult)); + $results[$index]->new = false; + } + $metager->setResults($results); + $metager->retrieveResults(); + $metager->rankAll(); + $metager->prepareResults(); + + $results = $metager->getResults(); + foreach ($results as $index => $resultTmp) { + if ($resultTmp->new) { + $view = View::make('layouts.result', ['result' => $resultTmp, 'metager' => $metager]); + $html = $view->render(); + $result['newResults'][$index] = $html; + } + } + + } + return response()->json($result); + } + public function botProtection($redirect) { $hash = md5(date('YmdHi')); diff --git a/app/Jobs/Searcher.php b/app/Jobs/Searcher.php index 7a57025c461d62da92b1fa15bc438ea3c9a0df92..e911713affd56e259de814d2815fce254824ec35 100644 --- a/app/Jobs/Searcher.php +++ b/app/Jobs/Searcher.php @@ -171,12 +171,14 @@ class Searcher implements ShouldQueue } $pipeline = Redis::pipeline(); $pipeline->hset('search.' . $hashValue . ".results." . $this->name, "response", $result); + $pipeline->hset('search.' . $hashValue . ".results." . $this->name, "delivered", "0"); + $pipeline->hincrby('search.' . $hashValue . ".results.status", "engineAnswered", 1); // After 60 seconds the results should be read by the MetaGer Process and stored in the Cache instead - $pipeline->expire('search.' . $hashValue . ".results." . $this->name, 60); + $pipeline->expire('search.' . $hashValue . ".results." . $this->name, 6000); $pipeline->rpush('search.' . $hashValue . ".ready", $this->name); - $pipeline->expire('search.' . $hashValue . ".ready", 60); + $pipeline->expire('search.' . $hashValue . ".ready", 6000); $pipeline->sadd('search.' . $hashValue . ".engines", $this->name); - $pipeline->expire('search.' . $hashValue . ".engines", 60); + $pipeline->expire('search.' . $hashValue . ".engines", 6000); $pipeline->execute(); $this->lastTime = microtime(true); } diff --git a/app/MetaGer.php b/app/MetaGer.php index 543674aeec3edfae456ff8c7f0b838bec0915343..ac12cfbb4eae8251304540e9d88d1054e4bb1cc4 100644 --- a/app/MetaGer.php +++ b/app/MetaGer.php @@ -60,9 +60,9 @@ class MetaGer protected $verificationId; protected $verificationCount; protected $searchUid; - protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult; + protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult, $redisCurrentResultList; - public function __construct() + public function __construct($hash = "") { # Timer starten $this->starttime = microtime(true); @@ -93,7 +93,11 @@ class MetaGer $this->canCache = false; } $this->canCache = false; - $this->searchUid = md5(uniqid()); + if ($hash === "") { + $this->searchUid = md5(uniqid()); + } else { + $this->searchUid = $hash; + } $redisPrefix = "search"; # This is a list on which the MetaGer process can do a blocking pop to wait for new results $this->redisResultWaitingKey = $redisPrefix . "." . $this->searchUid . ".ready"; @@ -101,6 +105,9 @@ class MetaGer $this->redisResultEngineList = $redisPrefix . "." . $this->searchUid . ".engines"; # This is the key where the results of the engine are stored as well as some statistical data $this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results."; + # A list of all search results already delivered to the user (sorted of course) + $this->redisCurrentResultList = $redisPrefix . "." . $this->searchUid . ".currentResults"; + } # Erstellt aus den gesammelten Ergebnissen den View @@ -230,19 +237,7 @@ class MetaGer $engines = $this->engines; // combine - $combinedResults = $this->combineResults($engines); - - # Wir bestimmen die Sprache eines jeden Suchergebnisses - $this->results = $this->addLangCodes($this->results); - - // sort - //$sortedResults = $this->sortResults($engines); - // filter - // augment (boost&adgoal) - // authorize - if ($this->apiKey) { - $this->apiAuthorized = $this->authorize($this->apiKey); - } + $this->combineResults($engines); // misc (WiP) if ($this->fokus == "nachrichten") { $this->results = array_filter($this->results, function ($v, $k) { @@ -293,32 +288,6 @@ class MetaGer $this->startCount = 0; } - foreach ($this->results as $result) { - if ($counter === 0) { - $firstRank = $result->rank; - } - - $counter++; - $result->number = $counter + $this->startCount; - $confidence = 0; - if ($firstRank > 0) { - $confidence = $result->rank / $firstRank; - } else { - $confidence = 0; - } - - if ($confidence > 0.65) { - $result->color = "#FF4000"; - } elseif ($confidence > 0.4) { - $result->color = "#FF0080"; - } elseif ($confidence > 0.2) { - $result->color = "#C000C0"; - } else { - $result->color = "#000000"; - } - - } - if (count($this->results) <= 0) { if (strlen($this->site) > 0) { $no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl); @@ -342,52 +311,6 @@ class MetaGer } - private function addLangCodes($results) - { - # Wenn es keine Ergebnisse gibt, brauchen wir uns gar nicht erst zu bemühen - if (sizeof($results) === 0) { - return $results; - } - - # Bei der Spracheinstellung "all" wird nicht gefiltert - if ($this->getLang() === "all") { - return $results; - } else { - # Ansonsten müssen wir jedem Result einen Sprachcode hinzufügen - $id = 0; - $langStrings = []; - foreach ($results as $result) { - # Wir geben jedem Ergebnis eine ID um später die Sprachcodes zuordnen zu können - $result->id = $id; - - $langStrings["result_" . $id] = utf8_encode($result->getLangString()); - - $id++; - } - # Wir schreiben die Strings in eine temporäre JSON-Datei, - # Da das Array unter umständen zu groß ist für eine direkte Übergabe an das Skript - $filename = "/tmp/" . getmypid(); - file_put_contents($filename, json_encode($langStrings)); - $langDetectorPath = app_path() . "/Models/lang.pl"; - $lang = exec("echo '$filename' | $langDetectorPath"); - $lang = json_decode($lang, true); - - # Wir haben nun die Sprachcodes der einzelnen Ergebnisse. - # Diese müssen wir nur noch korrekt zuordnen, dann sind wir fertig. - foreach ($lang as $key => $langCode) { - # Prefix vom Key entfernen: - $id = intval(str_replace("result_", "", $key)); - foreach ($this->results as $result) { - if ($result->id === $id) { - $result->langCode = $langCode; - break; - } - } - } - return $results; - } - } - public function combineResults($engines) { foreach ($engines as $engine) { @@ -418,6 +341,9 @@ class MetaGer $tldList = ""; try { foreach ($results as $result) { + if (!$result->new) { + continue; + } $link = $result->anzeigeLink; if (strpos($link, "http") !== 0) { $link = "http://" . $link; @@ -442,7 +368,7 @@ class MetaGer $hash = $el[1]; foreach ($results as $result) { - if ($hoster === $result->tld && !$result->partnershop) { + if ($result->new && $hoster === $result->tld && !$result->partnershop) { # Hier ist ein Advertiser: # Das Logo hinzufügen: if ($result->image !== "") { @@ -609,7 +535,6 @@ class MetaGer 'filter' => $filter]); $this->errors[] = $error; } - $engines = []; $typeslist = []; $counter = 0; @@ -622,29 +547,16 @@ class MetaGer $engine->setResultHash($this->getHashCode()); } } else { - $engines = $this->actuallyCreateSearchEngines($this->enabledSearchengines); + $this->actuallyCreateSearchEngines($this->enabledSearchengines); } + } + public function startSearch() + { # Wir starten alle Suchen - foreach ($engines as $engine) { + foreach ($this->engines as $engine) { $engine->startSearch($this); } - - /* Wir warten auf die Antwort der Suchmaschinen - * Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet. - * Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden: - * $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits - * welche Suchmaschinen nicht rechtzeitig geantwortet haben. - */ - - $this->waitForResults($engines); - - $this->retrieveResults($engines); - foreach ($engines as $engine) { - if (!empty($engine->totalResults) && $engine->totalResults > $this->totalResults) { - $this->totalResults = $engine->totalResults; - } - } } # Spezielle Suchen und Sumas @@ -693,7 +605,7 @@ class MetaGer $engines[] = $tmp; } - return $engines; + $this->engines = $engines; } public function getAvailableParameterFilter() @@ -785,8 +697,9 @@ class MetaGer return $engines; } - public function waitForResults($engines) + public function waitForMainResults() { + $engines = $this->engines; $enginesToWaitFor = []; foreach ($engines as $engine) { if ($engine->cached || !isset($engine->engine->main) || !$engine->engine->main) { @@ -796,6 +709,7 @@ class MetaGer } $timeStart = microtime(true); + $answered = []; $results = null; while (sizeof($enginesToWaitFor) > 0) { $newEngine = Redis::blpop($this->redisResultWaitingKey, 5); @@ -809,15 +723,29 @@ class MetaGer break; } } + $answered[] = $newEngine; } if ((microtime(true) - $timeStart) >= 2) { break; } } + + # Now we can add an entry to Redis which defines the starting time and how many engines should answer this request + $redis = Redis::connection(env('REDIS_RESULT_CONNECTION')); + $pipeline = $redis->pipeline(); + $pipeline->hset($this->getRedisEngineResult() . "status", "startTime", $timeStart); + $pipeline->hset($this->getRedisEngineResult() . "status", "engineCount", sizeof($engines)); + $pipeline->hset($this->getRedisEngineResult() . "status", "engineDelivered", sizeof($answered)); + foreach ($answered as $engine) { + $pipeline->hset($this->getRedisEngineResult() . $engine, "delivered", "1"); + } + $pipeline->execute(); + } - public function retrieveResults($engines) + public function retrieveResults() { + $engines = $this->engines; # Von geladenen Engines die Ergebnisse holen foreach ($engines as $engine) { if (!$engine->loaded) { @@ -827,9 +755,10 @@ class MetaGer Log::error($e); } } + if (!empty($engine->totalResults) && $engine->totalResults > $this->totalResults) { + $this->totalResults = $engine->totalResults; + } } - - $this->engines = $engines; } /* @@ -954,6 +883,9 @@ class MetaGer $this->apiKey = ""; } } + if ($this->apiKey) { + $this->apiAuthorized = $this->authorize($this->apiKey); + } // Remove Inputs that are not used $this->request = $request->replace($request->except(['verification_id', 'uid', 'verification_count'])); @@ -1416,6 +1348,11 @@ class MetaGer return $this->newtab; } + public function setResults($results) + { + $this->results = $results; + } + public function getResults() { return $this->results; @@ -1559,4 +1496,8 @@ class MetaGer { return $this->redisEngineResult; } + public function getRedisCurrentResultList() + { + return $this->redisCurrentResultList; + } } diff --git a/app/Models/Result.php b/app/Models/Result.php index 529788464d4021f4f3739c7181e594243059e58d..127626bb9241bf7d6d4cb4dc86b381d9a2b12472 100644 --- a/app/Models/Result.php +++ b/app/Models/Result.php @@ -27,6 +27,7 @@ class Result public $strippedDomain; # Die Domain in Form "bar.de" public $strippedLink; # Der Link in Form "foo.bar.de/test" public $rank; # Das Ranking für das Ergebnis + public $new = true; # Erstellt ein neues Ergebnis public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $gefVonLink, $sourceRank, $additionalInformation = []) @@ -67,6 +68,7 @@ class Result $this->price = isset($additionalInformation["price"]) ? $additionalInformation["price"] : 0; $this->price_text = $this->price_to_text($this->price); $this->additionalInformation = $additionalInformation; + $this->hash = spl_object_hash($this); } private function price_to_text($price) diff --git a/app/Models/lang.pl b/app/Models/lang.pl deleted file mode 100755 index 2be7f502cb792c45d2ff8c9a34602b7cfbc9a9e3..0000000000000000000000000000000000000000 --- a/app/Models/lang.pl +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/perl - -use Lingua::Identify qw(:language_identification); -use JSON; -use warnings; -use strict; -binmode STDOUT, ":utf8"; -binmode STDIN, ":utf8"; -use utf8; - -chomp(my $filename = <STDIN>); - -# Lets open the given file: -open(my $fh, "<", $filename) - or die "Can't open < $filename: $!"; -my $json = <$fh>; -close $fh; - -# Decode the JSON String -my $data = JSON->new->utf8->decode($json); - -# Wir durchlaufen den Hash: -foreach my $key (keys %{$data}){ - $data->{$key} = langof($data->{$key}); -} - -$data = encode_json($data); - -# Nur noch die temporäre Datei löschen: -unlink($filename); - -print $data; diff --git a/readme.md b/readme.md index b2ac2b86c6cb647420b9f9a0fec8cf60832cc7a5..26920c06bbc130723e345b62d23b8f86db019eda 100644 --- a/readme.md +++ b/readme.md @@ -18,9 +18,6 @@ * php-gd * sqlite3 * redis-server -* Die Perl-Pakete - * Lingua::Identify (http://search.cpan.org/~ambs/Lingua-Identify-0.56/lib/Lingua/Identify.pm) - * JSON (http://search.cpan.org/~makamaka/JSON-2.90/lib/JSON.pm) --- [<img src="public/img/Browserstack-logo_2x.png" width="250px" alt="Browserstack Logo" />](https://www.browserstack.com) <br /> diff --git a/resources/js/scriptResultPage.js b/resources/js/scriptResultPage.js index da858d8ec7c4c6810f030dd76efd056cbd94193b..9bfd5fc6549dc6c6974cc09b1c650855ea051909 100644 --- a/resources/js/scriptResultPage.js +++ b/resources/js/scriptResultPage.js @@ -53,5 +53,48 @@ function enableFormResetter() { } function loadMoreResults() { + var searchKey = $("meta[name=searchkey]").attr("content"); + var updateUrl = document.location.href; + updateUrl += "&loadMore=" + searchKey + "&script=yes"; + + updateUrl = updateUrl.replace("/meta.ger3", "/loadMore"); + + var expectedEngines = -1; + var deliveredEngines = -1; + + var currentlyLoading = false; + + // Regularily check for not yet delivered Results + var resultLoader = window.setInterval(function () { + if (!currentlyLoading) { + currentlyLoading = true; + $.getJSON(updateUrl, function (data) { + // Check if we can clear the interval (once every searchengine has answered) + if (data.engineDelivered == data.engineCount || data.timeWaiting > 5) { + clearInterval(resultLoader); + } + // If there are new results we can add them + if (typeof data.newResults != "undefined") { + for (var key in data.newResults) { + var value = data.newResults[key]; + + // If there are more results than the given index we will prepend otherwise we will append the result + var results = $(".result:not(.ad)"); + if (typeof results[key] != "undefined") { + console.log("inserting before " + key); + $(results[key]).insertBefore(value); + } else { + $(results[key - 1]).insertAfter(value); + } + + console.log(key + "=>" + value); + } + } + currentlyLoading = false; + }); + } + }, 1000); + + console.log(updateUrl); } \ No newline at end of file diff --git a/resources/views/layouts/result.blade.php b/resources/views/layouts/result.blade.php index a8f0eb2da387b305580d694f661d4bfa2ff4616e..347fb056b5a98dbf12a68c25ab96601faaadfd28 100644 --- a/resources/views/layouts/result.blade.php +++ b/resources/views/layouts/result.blade.php @@ -1,4 +1,4 @@ -<div class="result" data-count="{{ $result->number }}"> +<div class="result" data-count="{{ $result->hash }}"> <div class="result-header"> <div class="result-headline"> <h2 class="result-title" title="{{ $result->titel }}"> @@ -47,7 +47,7 @@ </div> @endif </div> - <input type="checkbox" id="result-toggle-{{$result->number}}" class="result-toggle" style="display: none"> + <input type="checkbox" id="result-toggle-{{$result->hash}}" class="result-toggle" style="display: none"> <div class="result-footer"> <a class="result-open" href="{{ $result->link }}" target="_self" rel="noopener"> {!! trans('result.options.7') !!} @@ -58,10 +58,10 @@ <a class="result-open-proxy" onmouseover="$(this).popover('show');" onmouseout="$(this).popover('hide');" data-toggle="popover" data-placement="auto right" data-container="body" data-content="@lang('result.proxytext')" href="{{ $result->proxyLink }}" target="{{ $metager->getNewtab() }}" rel="noopener"> {!! trans('result.options.5') !!} </a> - <label class="open-result-options navigation-element" for="result-toggle-{{$result->number}}"> + <label class="open-result-options navigation-element" for="result-toggle-{{$result->hash}}"> {{ trans('result.options.more')}} </label> - <label class="close-result-options navigation-element" for="result-toggle-{{$result->number}}"> + <label class="close-result-options navigation-element" for="result-toggle-{{$result->hash}}"> {{ trans('result.options.less')}} </label> </div> @@ -69,7 +69,7 @@ <div class="options"> <ul class="option-list list-unstyled small"> <li class="js-only"> - <a href="javascript:resultSaver({{ $result->number }});" class="saver"> + <a href="javascript:resultSaver("{{ $result->hash }}");" class="saver"> <nobr><i class="fa fa-floppy-o"></i> {!! trans('result.options.savetab') !!}</nobr> </a> </li> diff --git a/resources/views/resultpages/results.blade.php b/resources/views/resultpages/results.blade.php index e9e717e563763926b7577a6e060a484c737c05c7..8b242029225455afe9dbd2f59015fd5c841efb84 100644 --- a/resources/views/resultpages/results.blade.php +++ b/resources/views/resultpages/results.blade.php @@ -7,17 +7,17 @@ @endfor @endif {{-- Create results and ongoing ads --}} - @foreach($metager->getResults() as $result) + @foreach($metager->getResults() as $index => $result) @if($mobile) - @if($result->number % 4 === 0) + @if($index % 4 === 0) @include('layouts.ad', ['ad' => $metager->popAd()]) @endif @else - @if($result->number % 7 === 0) + @if($index % 7 === 0) @include('layouts.ad', ['ad' => $metager->popAd()]) @endif @endif @include('layouts.result', ['result' => $result]) @endforeach @include('parts.pager') -</div> \ No newline at end of file +</div> diff --git a/routes/web.php b/routes/web.php index 913101e1db25268d425378ba067d2976a13ea8e3..d687370404ad06b941f91e6a8666f28bf5900191 100644 --- a/routes/web.php +++ b/routes/web.php @@ -179,6 +179,7 @@ Route::group( Route::get('settings', 'StartpageController@loadSettings'); Route::match(['get', 'post'], 'meta/meta.ger3', 'MetaGerSearch@search')->middleware('humanverification'); + Route::get('meta/loadMore', 'MetaGerSearch@loadMore'); Route::post('img/cat.jpg', 'HumanVerification@remove'); Route::get('r/metager/{mm}/{pw}/{url}', ['as' => 'humanverification', 'uses' => 'HumanVerification@removeGet']); Route::post('img/dog.jpg', 'HumanVerification@whitelist');