Commit 37fe69f8 authored by Dominik Hebeler's avatar Dominik Hebeler

We can now generate json responses for missing results

parent 7d16f746
......@@ -5,6 +5,8 @@ namespace App\Http\Controllers;
use App;
use App\MetaGer;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
use View;
const TIP_SERVER = 'http://metager3.de:63825/tips.xml';
......@@ -19,13 +21,6 @@ class MetaGerSearch extends Controller
return redirect()->to('https://maps.metager.de/map/' . $searchinput . '/1240908.5493525574,6638783.2192695495,6');
}
/*if ($focus !== "angepasst" && $this->startsWith($focus, "focus_")) {
$metager->parseFormData($request);
return $metager->createView();
}*/
#die($request->header('User-Agent'));
$time = microtime(true);
# Mit gelieferte Formulardaten parsen und abspeichern:
$metager->parseFormData($request);
......@@ -39,6 +34,12 @@ class MetaGerSearch extends Controller
# auf Ergebnisse warten und die Ergebnisse laden
$metager->createSearchEngines($request);
$metager->startSearch();
$metager->waitForMainResults();
$metager->retrieveResults();
# Versuchen die Ergebnisse der Quicktips zu laden
$quicktipResults = $quicktips->loadResults();
# Alle Ergebnisse vor der Zusammenführung ranken:
......@@ -47,10 +48,93 @@ class MetaGerSearch extends Controller
# Ergebnisse der Suchmaschinen kombinieren:
$metager->prepareResults();
# Save the results in Redis
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$pipeline = $redis->pipeline();
foreach ($metager->getResults() as $result) {
$pipeline->rpush($metager->getRedisCurrentResultList(), base64_encode(serialize($result)));
}
$pipeline->expire($metager->getRedisCurrentResultList(), 6000);
$pipeline->execute();
# Die Ausgabe erstellen:
return $metager->createView($quicktipResults);
}
public function loadMore(Request $request)
{
/**
* There are three forms of requests to the resultpage
* 1. Initial Request: Loads the fastest searchengines and sends them to the user
* 2. Load more results (with JS): Loads new search engines that answered after the initial request was send
* 3. Load more results (without JS): Loads new search engines that answered within 1s timeout
*/
if ($request->filled('loadMore') && $request->filled('script') && $request->input('script') === "yes") {
return $this->loadMoreJS($request);
}
}
private function loadMoreJS(Request $request)
{
# Create a MetaGer Instance with the supplied hash
$hash = $request->input('loadMore', '');
$metager = new MetaGer($hash);
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$result = [];
# Check if there should be more results
$stats = $redis->hgetall($metager->getRedisEngineResult() . "status");
$stats["startTime"] = floatval($stats["startTime"]);
$stats["engineCount"] = intval($stats["engineCount"]);
$stats["engineAnswered"] = intval($stats["engineAnswered"]);
$stats["engineDelivered"] = intval($stats["engineDelivered"]);
$result["finished"] = true;
$result["engineCount"] = $stats["engineCount"];
$result["engineAnswered"] = $stats["engineAnswered"];
$result["engineDelivered"] = $stats["engineDelivered"];
$result["timeWaiting"] = microtime(true) - $stats["startTime"];
# Check if we can abort
if ($stats["engineAnswered"] > $stats["engineDelivered"]/*&& $result["timeWaiting"] <= 10 */) {
$metager->parseFormData($request);
# Nach Spezialsuchen überprüfen:
$metager->checkSpecialSearches($request);
# Read which search engines are new
$newEngines = [];
foreach ($redis->lrange($metager->getRedisResultWaitingKey(), 0, -1) as $engine) {
$result["engineDelivered"]++;
$newEngines[$engine] = $metager->getSumaFile()->sumas->{$engine};
}
$metager->actuallyCreateSearchEngines($newEngines);
# Add the results already delivered to the suer
$results = $redis->lrange($metager->getRedisCurrentResultList(), 0, -1);
foreach ($results as $index => $oldResult) {
$results[$index] = unserialize(base64_decode($oldResult));
$results[$index]->new = false;
}
$metager->setResults($results);
$metager->retrieveResults();
$metager->rankAll();
$metager->prepareResults();
$results = $metager->getResults();
foreach ($results as $index => $resultTmp) {
if ($resultTmp->new) {
$view = View::make('layouts.result', ['result' => $resultTmp, 'metager' => $metager]);
$html = $view->render();
$result['newResults'][$index] = $html;
}
}
}
return response()->json($result);
}
public function botProtection($redirect)
{
$hash = md5(date('YmdHi'));
......
......@@ -171,12 +171,14 @@ class Searcher implements ShouldQueue
}
$pipeline = Redis::pipeline();
$pipeline->hset('search.' . $hashValue . ".results." . $this->name, "response", $result);
$pipeline->hset('search.' . $hashValue . ".results." . $this->name, "delivered", "0");
$pipeline->hincrby('search.' . $hashValue . ".results.status", "engineAnswered", 1);
// After 60 seconds the results should be read by the MetaGer Process and stored in the Cache instead
$pipeline->expire('search.' . $hashValue . ".results." . $this->name, 60);
$pipeline->expire('search.' . $hashValue . ".results." . $this->name, 6000);
$pipeline->rpush('search.' . $hashValue . ".ready", $this->name);
$pipeline->expire('search.' . $hashValue . ".ready", 60);
$pipeline->expire('search.' . $hashValue . ".ready", 6000);
$pipeline->sadd('search.' . $hashValue . ".engines", $this->name);
$pipeline->expire('search.' . $hashValue . ".engines", 60);
$pipeline->expire('search.' . $hashValue . ".engines", 6000);
$pipeline->execute();
$this->lastTime = microtime(true);
}
......
......@@ -60,9 +60,9 @@ class MetaGer
protected $verificationId;
protected $verificationCount;
protected $searchUid;
protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult;
protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult, $redisCurrentResultList;
public function __construct()
public function __construct($hash = "")
{
# Timer starten
$this->starttime = microtime(true);
......@@ -93,7 +93,11 @@ class MetaGer
$this->canCache = false;
}
$this->canCache = false;
$this->searchUid = md5(uniqid());
if ($hash === "") {
$this->searchUid = md5(uniqid());
} else {
$this->searchUid = $hash;
}
$redisPrefix = "search";
# This is a list on which the MetaGer process can do a blocking pop to wait for new results
$this->redisResultWaitingKey = $redisPrefix . "." . $this->searchUid . ".ready";
......@@ -101,6 +105,9 @@ class MetaGer
$this->redisResultEngineList = $redisPrefix . "." . $this->searchUid . ".engines";
# This is the key where the results of the engine are stored as well as some statistical data
$this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results.";
# A list of all search results already delivered to the user (sorted of course)
$this->redisCurrentResultList = $redisPrefix . "." . $this->searchUid . ".currentResults";
}
# Erstellt aus den gesammelten Ergebnissen den View
......@@ -230,19 +237,7 @@ class MetaGer
$engines = $this->engines;
// combine
$combinedResults = $this->combineResults($engines);
# Wir bestimmen die Sprache eines jeden Suchergebnisses
$this->results = $this->addLangCodes($this->results);
// sort
//$sortedResults = $this->sortResults($engines);
// filter
// augment (boost&adgoal)
// authorize
if ($this->apiKey) {
$this->apiAuthorized = $this->authorize($this->apiKey);
}
$this->combineResults($engines);
// misc (WiP)
if ($this->fokus == "nachrichten") {
$this->results = array_filter($this->results, function ($v, $k) {
......@@ -293,32 +288,6 @@ class MetaGer
$this->startCount = 0;
}
foreach ($this->results as $result) {
if ($counter === 0) {
$firstRank = $result->rank;
}
$counter++;
$result->number = $counter + $this->startCount;
$confidence = 0;
if ($firstRank > 0) {
$confidence = $result->rank / $firstRank;
} else {
$confidence = 0;
}
if ($confidence > 0.65) {
$result->color = "#FF4000";
} elseif ($confidence > 0.4) {
$result->color = "#FF0080";
} elseif ($confidence > 0.2) {
$result->color = "#C000C0";
} else {
$result->color = "#000000";
}
}
if (count($this->results) <= 0) {
if (strlen($this->site) > 0) {
$no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
......@@ -342,52 +311,6 @@ class MetaGer
}
private function addLangCodes($results)
{
# Wenn es keine Ergebnisse gibt, brauchen wir uns gar nicht erst zu bemühen
if (sizeof($results) === 0) {
return $results;
}
# Bei der Spracheinstellung "all" wird nicht gefiltert
if ($this->getLang() === "all") {
return $results;
} else {
# Ansonsten müssen wir jedem Result einen Sprachcode hinzufügen
$id = 0;
$langStrings = [];
foreach ($results as $result) {
# Wir geben jedem Ergebnis eine ID um später die Sprachcodes zuordnen zu können
$result->id = $id;
$langStrings["result_" . $id] = utf8_encode($result->getLangString());
$id++;
}
# Wir schreiben die Strings in eine temporäre JSON-Datei,
# Da das Array unter umständen zu groß ist für eine direkte Übergabe an das Skript
$filename = "/tmp/" . getmypid();
file_put_contents($filename, json_encode($langStrings));
$langDetectorPath = app_path() . "/Models/lang.pl";
$lang = exec("echo '$filename' | $langDetectorPath");
$lang = json_decode($lang, true);
# Wir haben nun die Sprachcodes der einzelnen Ergebnisse.
# Diese müssen wir nur noch korrekt zuordnen, dann sind wir fertig.
foreach ($lang as $key => $langCode) {
# Prefix vom Key entfernen:
$id = intval(str_replace("result_", "", $key));
foreach ($this->results as $result) {
if ($result->id === $id) {
$result->langCode = $langCode;
break;
}
}
}
return $results;
}
}
public function combineResults($engines)
{
foreach ($engines as $engine) {
......@@ -418,6 +341,9 @@ class MetaGer
$tldList = "";
try {
foreach ($results as $result) {
if (!$result->new) {
continue;
}
$link = $result->anzeigeLink;
if (strpos($link, "http") !== 0) {
$link = "http://" . $link;
......@@ -442,7 +368,7 @@ class MetaGer
$hash = $el[1];
foreach ($results as $result) {
if ($hoster === $result->tld && !$result->partnershop) {
if ($result->new && $hoster === $result->tld && !$result->partnershop) {
# Hier ist ein Advertiser:
# Das Logo hinzufügen:
if ($result->image !== "") {
......@@ -609,7 +535,6 @@ class MetaGer
'filter' => $filter]);
$this->errors[] = $error;
}
$engines = [];
$typeslist = [];
$counter = 0;
......@@ -622,29 +547,16 @@ class MetaGer
$engine->setResultHash($this->getHashCode());
}
} else {
$engines = $this->actuallyCreateSearchEngines($this->enabledSearchengines);
$this->actuallyCreateSearchEngines($this->enabledSearchengines);
}
}
public function startSearch()
{
# Wir starten alle Suchen
foreach ($engines as $engine) {
foreach ($this->engines as $engine) {
$engine->startSearch($this);
}
/* Wir warten auf die Antwort der Suchmaschinen
* Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
* Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
* $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
* welche Suchmaschinen nicht rechtzeitig geantwortet haben.
*/
$this->waitForResults($engines);
$this->retrieveResults($engines);
foreach ($engines as $engine) {
if (!empty($engine->totalResults) && $engine->totalResults > $this->totalResults) {
$this->totalResults = $engine->totalResults;
}
}
}
# Spezielle Suchen und Sumas
......@@ -693,7 +605,7 @@ class MetaGer
$engines[] = $tmp;
}
return $engines;
$this->engines = $engines;
}
public function getAvailableParameterFilter()
......@@ -785,8 +697,9 @@ class MetaGer
return $engines;
}
public function waitForResults($engines)
public function waitForMainResults()
{
$engines = $this->engines;
$enginesToWaitFor = [];
foreach ($engines as $engine) {
if ($engine->cached || !isset($engine->engine->main) || !$engine->engine->main) {
......@@ -796,6 +709,7 @@ class MetaGer
}
$timeStart = microtime(true);
$answered = [];
$results = null;
while (sizeof($enginesToWaitFor) > 0) {
$newEngine = Redis::blpop($this->redisResultWaitingKey, 5);
......@@ -809,15 +723,29 @@ class MetaGer
break;
}
}
$answered[] = $newEngine;
}
if ((microtime(true) - $timeStart) >= 2) {
break;
}
}
# Now we can add an entry to Redis which defines the starting time and how many engines should answer this request
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$pipeline = $redis->pipeline();
$pipeline->hset($this->getRedisEngineResult() . "status", "startTime", $timeStart);
$pipeline->hset($this->getRedisEngineResult() . "status", "engineCount", sizeof($engines));
$pipeline->hset($this->getRedisEngineResult() . "status", "engineDelivered", sizeof($answered));
foreach ($answered as $engine) {
$pipeline->hset($this->getRedisEngineResult() . $engine, "delivered", "1");
}
$pipeline->execute();
}
public function retrieveResults($engines)
public function retrieveResults()
{
$engines = $this->engines;
# Von geladenen Engines die Ergebnisse holen
foreach ($engines as $engine) {
if (!$engine->loaded) {
......@@ -827,9 +755,10 @@ class MetaGer
Log::error($e);
}
}
if (!empty($engine->totalResults) && $engine->totalResults > $this->totalResults) {
$this->totalResults = $engine->totalResults;
}
}
$this->engines = $engines;
}
/*
......@@ -954,6 +883,9 @@ class MetaGer
$this->apiKey = "";
}
}
if ($this->apiKey) {
$this->apiAuthorized = $this->authorize($this->apiKey);
}
// Remove Inputs that are not used
$this->request = $request->replace($request->except(['verification_id', 'uid', 'verification_count']));
......@@ -1416,6 +1348,11 @@ class MetaGer
return $this->newtab;
}
public function setResults($results)
{
$this->results = $results;
}
public function getResults()
{
return $this->results;
......@@ -1559,4 +1496,8 @@ class MetaGer
{
return $this->redisEngineResult;
}
public function getRedisCurrentResultList()
{
return $this->redisCurrentResultList;
}
}
......@@ -27,6 +27,7 @@ class Result
public $strippedDomain; # Die Domain in Form "bar.de"
public $strippedLink; # Der Link in Form "foo.bar.de/test"
public $rank; # Das Ranking für das Ergebnis
public $new = true;
# Erstellt ein neues Ergebnis
public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $gefVonLink, $sourceRank, $additionalInformation = [])
......@@ -67,6 +68,7 @@ class Result
$this->price = isset($additionalInformation["price"]) ? $additionalInformation["price"] : 0;
$this->price_text = $this->price_to_text($this->price);
$this->additionalInformation = $additionalInformation;
$this->hash = spl_object_hash($this);
}
private function price_to_text($price)
......
#!/usr/bin/perl
use Lingua::Identify qw(:language_identification);
use JSON;
use warnings;
use strict;
binmode STDOUT, ":utf8";
binmode STDIN, ":utf8";
use utf8;
chomp(my $filename = <STDIN>);
# Lets open the given file:
open(my $fh, "<", $filename)
or die "Can't open < $filename: $!";
my $json = <$fh>;
close $fh;
# Decode the JSON String
my $data = JSON->new->utf8->decode($json);
# Wir durchlaufen den Hash:
foreach my $key (keys %{$data}){
$data->{$key} = langof($data->{$key});
}
$data = encode_json($data);
# Nur noch die temporäre Datei löschen:
unlink($filename);
print $data;
......@@ -18,9 +18,6 @@
* php-gd
* sqlite3
* redis-server
* Die Perl-Pakete
* Lingua::Identify (http://search.cpan.org/~ambs/Lingua-Identify-0.56/lib/Lingua/Identify.pm)
* JSON (http://search.cpan.org/~makamaka/JSON-2.90/lib/JSON.pm)
---
[<img src="public/img/Browserstack-logo_2x.png" width="250px" alt="Browserstack Logo" />](https://www.browserstack.com) <br />
......
......@@ -53,5 +53,48 @@ function enableFormResetter() {
}
function loadMoreResults() {
var searchKey = $("meta[name=searchkey]").attr("content");
var updateUrl = document.location.href;
updateUrl += "&loadMore=" + searchKey + "&script=yes";
updateUrl = updateUrl.replace("/meta.ger3", "/loadMore");
var expectedEngines = -1;
var deliveredEngines = -1;
var currentlyLoading = false;
// Regularily check for not yet delivered Results
var resultLoader = window.setInterval(function () {
if (!currentlyLoading) {
currentlyLoading = true;
$.getJSON(updateUrl, function (data) {
// Check if we can clear the interval (once every searchengine has answered)
if (data.engineDelivered == data.engineCount || data.timeWaiting > 5) {
clearInterval(resultLoader);
}
// If there are new results we can add them
if (typeof data.newResults != "undefined") {
for (var key in data.newResults) {
var value = data.newResults[key];
// If there are more results than the given index we will prepend otherwise we will append the result
var results = $(".result:not(.ad)");
if (typeof results[key] != "undefined") {
console.log("inserting before " + key);
$(results[key]).insertBefore(value);
} else {
$(results[key - 1]).insertAfter(value);
}
console.log(key + "=>" + value);
}
}
currentlyLoading = false;
});
}
}, 1000);
console.log(updateUrl);
}
\ No newline at end of file
<div class="result" data-count="{{ $result->number }}">
<div class="result" data-count="{{ $result->hash }}">
<div class="result-header">
<div class="result-headline">
<h2 class="result-title" title="{{ $result->titel }}">
......@@ -47,7 +47,7 @@
</div>
@endif
</div>
<input type="checkbox" id="result-toggle-{{$result->number}}" class="result-toggle" style="display: none">
<input type="checkbox" id="result-toggle-{{$result->hash}}" class="result-toggle" style="display: none">
<div class="result-footer">
<a class="result-open" href="{{ $result->link }}" target="_self" rel="noopener">
{!! trans('result.options.7') !!}
......@@ -58,10 +58,10 @@
<a class="result-open-proxy" onmouseover="$(this).popover('show');" onmouseout="$(this).popover('hide');" data-toggle="popover" data-placement="auto right" data-container="body" data-content="@lang('result.proxytext')" href="{{ $result->proxyLink }}" target="{{ $metager->getNewtab() }}" rel="noopener">
{!! trans('result.options.5') !!}
</a>
<label class="open-result-options navigation-element" for="result-toggle-{{$result->number}}">
<label class="open-result-options navigation-element" for="result-toggle-{{$result->hash}}">
{{ trans('result.options.more')}}
</label>
<label class="close-result-options navigation-element" for="result-toggle-{{$result->number}}">
<label class="close-result-options navigation-element" for="result-toggle-{{$result->hash}}">
{{ trans('result.options.less')}}
</label>
</div>
......@@ -69,7 +69,7 @@
<div class="options">
<ul class="option-list list-unstyled small">
<li class="js-only">
<a href="javascript:resultSaver({{ $result->number }});" class="saver">
<a href="javascript:resultSaver("{{ $result->hash }}");" class="saver">
<nobr><i class="fa fa-floppy-o"></i> {!! trans('result.options.savetab') !!}</nobr>
</a>
</li>
......
......@@ -7,17 +7,17 @@
@endfor
@endif
{{-- Create results and ongoing ads --}}
@foreach($metager->getResults() as $result)
@foreach($metager->getResults() as $index => $result)
@if($mobile)
@if($result->number % 4 === 0)
@if($index % 4 === 0)
@include('layouts.ad', ['ad' => $metager->popAd()])
@endif
@else
@if($result->number % 7 === 0)
@if($index % 7 === 0)
@include('layouts.ad', ['ad' => $metager->popAd()])
@endif
@endif
@include('layouts.result', ['result' => $result])
@endforeach
@include('parts.pager')
</div>
\ No newline at end of file
</div>
......@@ -179,6 +179,7 @@ Route::group(
Route::get('settings', 'StartpageController@loadSettings');
Route::match(['get', 'post'], 'meta/meta.ger3', 'MetaGerSearch@search')->middleware('humanverification');
Route::get('meta/loadMore', 'MetaGerSearch@loadMore');
Route::post('img/cat.jpg', 'HumanVerification@remove');
Route::get('r/metager/{mm}/{pw}/{url}', ['as' => 'humanverification', 'uses' => 'HumanVerification@removeGet']);
Route::post('img/dog.jpg', 'HumanVerification@whitelist');
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment