Skip to content

GitLab

Commit 37fe69f8 authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

We can now generate json responses for missing results

parent 7d16f746
Hide whitespace changes
Inline Side-by-side
Showing with 205 additions and 167 deletions
app/Http/Controllers/MetaGerSearch.php
View file @ 37fe69f8
......@@ -5,6 +5,8 @@ namespace App\Http\Controllers;
use App;
use App\MetaGer;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
use View;
const TIP_SERVER = 'http://metager3.de:63825/tips.xml';
......@@ -19,13 +21,6 @@ class MetaGerSearch extends Controller
return redirect()->to('https://maps.metager.de/map/' . $searchinput . '/1240908.5493525574,6638783.2192695495,6');
}
/*if ($focus !== "angepasst" && $this->startsWith($focus, "focus_")) {
$metager->parseFormData($request);
return $metager->createView();
}*/
#die($request->header('User-Agent'));
$time = microtime(true);
# Mit gelieferte Formulardaten parsen und abspeichern:
$metager->parseFormData($request);
......@@ -39,6 +34,12 @@ class MetaGerSearch extends Controller
# auf Ergebnisse warten und die Ergebnisse laden
$metager->createSearchEngines($request);
$metager->startSearch();
$metager->waitForMainResults();
$metager->retrieveResults();
# Versuchen die Ergebnisse der Quicktips zu laden
$quicktipResults = $quicktips->loadResults();
# Alle Ergebnisse vor der Zusammenführung ranken:
......@@ -47,10 +48,93 @@ class MetaGerSearch extends Controller
# Ergebnisse der Suchmaschinen kombinieren:
$metager->prepareResults();
# Save the results in Redis
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$pipeline = $redis->pipeline();
foreach ($metager->getResults() as $result) {
$pipeline->rpush($metager->getRedisCurrentResultList(), base64_encode(serialize($result)));
}
$pipeline->expire($metager->getRedisCurrentResultList(), 6000);
$pipeline->execute();
# Die Ausgabe erstellen:
return $metager->createView($quicktipResults);
}
public function loadMore(Request $request)
{
/**
* There are three forms of requests to the resultpage
* 1. Initial Request: Loads the fastest searchengines and sends them to the user
* 2. Load more results (with JS): Loads new search engines that answered after the initial request was send
* 3. Load more results (without JS): Loads new search engines that answered within 1s timeout
*/
if ($request->filled('loadMore') && $request->filled('script') && $request->input('script') === "yes") {
return $this->loadMoreJS($request);
}
}
private function loadMoreJS(Request $request)
{
# Create a MetaGer Instance with the supplied hash
$hash = $request->input('loadMore', '');
$metager = new MetaGer($hash);
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$result = [];
# Check if there should be more results
$stats = $redis->hgetall($metager->getRedisEngineResult() . "status");
$stats["startTime"] = floatval($stats["startTime"]);
$stats["engineCount"] = intval($stats["engineCount"]);
$stats["engineAnswered"] = intval($stats["engineAnswered"]);
$stats["engineDelivered"] = intval($stats["engineDelivered"]);
$result["finished"] = true;
$result["engineCount"] = $stats["engineCount"];
$result["engineAnswered"] = $stats["engineAnswered"];
$result["engineDelivered"] = $stats["engineDelivered"];
$result["timeWaiting"] = microtime(true) - $stats["startTime"];
# Check if we can abort
if ($stats["engineAnswered"] > $stats["engineDelivered"]/*&& $result["timeWaiting"] <= 10 */) {
$metager->parseFormData($request);
# Nach Spezialsuchen überprüfen:
$metager->checkSpecialSearches($request);
# Read which search engines are new
$newEngines = [];
foreach ($redis->lrange($metager->getRedisResultWaitingKey(), 0, -1) as $engine) {
$result["engineDelivered"]++;
$newEngines[$engine] = $metager->getSumaFile()->sumas->{$engine};
}
$metager->actuallyCreateSearchEngines($newEngines);
# Add the results already delivered to the suer
$results = $redis->lrange($metager->getRedisCurrentResultList(), 0, -1);
foreach ($results as $index => $oldResult) {
$results[$index] = unserialize(base64_decode($oldResult));
$results[$index]->new = false;
}
$metager->setResults($results);
$metager->retrieveResults();
$metager->rankAll();
$metager->prepareResults();
$results = $metager->getResults();
foreach ($results as $index => $resultTmp) {
if ($resultTmp->new) {
$view = View::make('layouts.result', ['result' => $resultTmp, 'metager' => $metager]);
$html = $view->render();
$result['newResults'][$index] = $html;
}
}
}
return response()->json($result);
}
public function botProtection($redirect)
{
$hash = md5(date('YmdHi'));
......
app/Jobs/Searcher.php
View file @ 37fe69f8
......@@ -171,12 +171,14 @@ class Searcher implements ShouldQueue
}
$pipeline = Redis::pipeline();
$pipeline->hset('search.' . $hashValue . ".results." . $this->name, "response", $result);
$pipeline->hset('search.' . $hashValue . ".results." . $this->name, "delivered", "0");
$pipeline->hincrby('search.' . $hashValue . ".results.status", "engineAnswered", 1);
// After 60 seconds the results should be read by the MetaGer Process and stored in the Cache instead
$pipeline->expire('search.' . $hashValue . ".results." . $this->name, 60);
$pipeline->expire('search.' . $hashValue . ".results." . $this->name, 6000);
$pipeline->rpush('search.' . $hashValue . ".ready", $this->name);
$pipeline->expire('search.' . $hashValue . ".ready", 60);
$pipeline->expire('search.' . $hashValue . ".ready", 6000);
$pipeline->sadd('search.' . $hashValue . ".engines", $this->name);
$pipeline->expire('search.' . $hashValue . ".engines", 60);
$pipeline->expire('search.' . $hashValue . ".engines", 6000);
$pipeline->execute();
$this->lastTime = microtime(true);
}
......
app/MetaGer.php
View file @ 37fe69f8
......@@ -60,9 +60,9 @@ class MetaGer
protected $verificationId;
protected $verificationCount;
protected $searchUid;
protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult;
protected $redisResultWaitingKey, $redisResultEngineList, $redisEngineResult, $redisCurrentResultList;
public function __construct()
public function __construct($hash = "")
{
# Timer starten
$this->starttime = microtime(true);
......@@ -93,7 +93,11 @@ class MetaGer
$this->canCache = false;
}
$this->canCache = false;
$this->searchUid = md5(uniqid());
if ($hash === "") {
$this->searchUid = md5(uniqid());
} else {
$this->searchUid = $hash;
}
$redisPrefix = "search";
# This is a list on which the MetaGer process can do a blocking pop to wait for new results
$this->redisResultWaitingKey = $redisPrefix . "." . $this->searchUid . ".ready";
......@@ -101,6 +105,9 @@ class MetaGer
$this->redisResultEngineList = $redisPrefix . "." . $this->searchUid . ".engines";
# This is the key where the results of the engine are stored as well as some statistical data
$this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results.";
# A list of all search results already delivered to the user (sorted of course)
$this->redisCurrentResultList = $redisPrefix . "." . $this->searchUid . ".currentResults";
}
# Erstellt aus den gesammelten Ergebnissen den View
......@@ -230,19 +237,7 @@ class MetaGer
$engines = $this->engines;
// combine
$combinedResults = $this->combineResults($engines);
# Wir bestimmen die Sprache eines jeden Suchergebnisses
$this->results = $this->addLangCodes($this->results);
// sort
//$sortedResults = $this->sortResults($engines);
// filter
// augment (boost&adgoal)
// authorize
if ($this->apiKey) {
$this->apiAuthorized = $this->authorize($this->apiKey);
}
$this->combineResults($engines);
// misc (WiP)
if ($this->fokus == "nachrichten") {
$this->results = array_filter($this->results, function ($v, $k) {
......@@ -293,32 +288,6 @@ class MetaGer
$this->startCount = 0;
}
foreach ($this->results as $result) {
if ($counter === 0) {
$firstRank = $result->rank;
}
$counter++;
$result->number = $counter + $this->startCount;
$confidence = 0;
if ($firstRank > 0) {
$confidence = $result->rank / $firstRank;
} else {
$confidence = 0;
}
if ($confidence > 0.65) {
$result->color = "#FF4000";
} elseif ($confidence > 0.4) {
$result->color = "#FF0080";
} elseif ($confidence > 0.2) {
$result->color = "#C000C0";
} else {
$result->color = "#000000";
}
}
if (count($this->results) <= 0) {
if (strlen($this->site) > 0) {
$no_sitesearch_query = str_replace(urlencode("site:" . $this->site), "", $this->fullUrl);
......@@ -342,52 +311,6 @@ class MetaGer
}
private function addLangCodes($results)
{
# Wenn es keine Ergebnisse gibt, brauchen wir uns gar nicht erst zu bemühen
if (sizeof($results) === 0) {
return $results;
}
# Bei der Spracheinstellung "all" wird nicht gefiltert
if ($this->getLang() === "all") {
return $results;
} else {
# Ansonsten müssen wir jedem Result einen Sprachcode hinzufügen
$id = 0;
$langStrings = [];
foreach ($results as $result) {
# Wir geben jedem Ergebnis eine ID um später die Sprachcodes zuordnen zu können
$result->id = $id;
$langStrings["result_" . $id] = utf8_encode($result->getLangString());
$id++;
}
# Wir schreiben die Strings in eine temporäre JSON-Datei,
# Da das Array unter umständen zu groß ist für eine direkte Übergabe an das Skript
$filename = "/tmp/" . getmypid();
file_put_contents($filename, json_encode($langStrings));
$langDetectorPath = app_path() . "/Models/lang.pl";
$lang = exec("echo '$filename' | $langDetectorPath");
$lang = json_decode($lang, true);
# Wir haben nun die Sprachcodes der einzelnen Ergebnisse.
# Diese müssen wir nur noch korrekt zuordnen, dann sind wir fertig.
foreach ($lang as $key => $langCode) {
# Prefix vom Key entfernen:
$id = intval(str_replace("result_", "", $key));
foreach ($this->results as $result) {
if ($result->id === $id) {
$result->langCode = $langCode;
break;
}
}
}
return $results;
}
}
public function combineResults($engines)
{
foreach ($engines as $engine) {
......@@ -418,6 +341,9 @@ class MetaGer
$tldList = "";
try {
foreach ($results as $result) {
if (!$result->new) {
continue;
}
$link = $result->anzeigeLink;
if (strpos($link, "http") !== 0) {
$link = "http://" . $link;
......@@ -442,7 +368,7 @@ class MetaGer
$hash = $el[1];
foreach ($results as $result) {
if ($hoster === $result->tld && !$result->partnershop) {
if ($result->new && $hoster === $result->tld && !$result->partnershop) {
# Hier ist ein Advertiser:
# Das Logo hinzufügen:
if ($result->image !== "") {
......@@ -609,7 +535,6 @@ class MetaGer
'filter' => $filter]);
$this->errors[] = $error;
}
$engines = [];
$typeslist = [];
$counter = 0;
......@@ -622,29 +547,16 @@ class MetaGer
$engine->setResultHash($this->getHashCode());
}
} else {
$engines = $this->actuallyCreateSearchEngines($this->enabledSearchengines);
$this->actuallyCreateSearchEngines($this->enabledSearchengines);
}
}
public function startSearch()
{
# Wir starten alle Suchen
foreach ($engines as $engine) {
foreach ($this->engines as $engine) {
$engine->startSearch($this);
}
/* Wir warten auf die Antwort der Suchmaschinen
* Die Verbindung steht zu diesem Zeitpunkt und auch unsere Requests wurden schon gesendet.
* Wir zählen die Suchmaschinen, die durch den Cache beantwortet wurden:
* $enginesToLoad zählt einerseits die Suchmaschinen auf die wir warten und andererseits
* welche Suchmaschinen nicht rechtzeitig geantwortet haben.
*/
$this->waitForResults($engines);
$this->retrieveResults($engines);
foreach ($engines as $engine) {
if (!empty($engine->totalResults) && $engine->totalResults > $this->totalResults) {
$this->totalResults = $engine->totalResults;
}
}
}
# Spezielle Suchen und Sumas
......@@ -693,7 +605,7 @@ class MetaGer
$engines[] = $tmp;
}
return $engines;
$this->engines = $engines;
}
public function getAvailableParameterFilter()
......@@ -785,8 +697,9 @@ class MetaGer
return $engines;
}
public function waitForResults($engines)
public function waitForMainResults()
{
$engines = $this->engines;
$enginesToWaitFor = [];
foreach ($engines as $engine) {
if ($engine->cached || !isset($engine->engine->main) || !$engine->engine->main) {
......@@ -796,6 +709,7 @@ class MetaGer
}
$timeStart = microtime(true);
$answered = [];
$results = null;
while (sizeof($enginesToWaitFor) > 0) {
$newEngine = Redis::blpop($this->redisResultWaitingKey, 5);
......@@ -809,15 +723,29 @@ class MetaGer
break;
}
}
$answered[] = $newEngine;
}
if ((microtime(true) - $timeStart) >= 2) {
break;
}
}
# Now we can add an entry to Redis which defines the starting time and how many engines should answer this request
$redis = Redis::connection(env('REDIS_RESULT_CONNECTION'));
$pipeline = $redis->pipeline();
$pipeline->hset($this->getRedisEngineResult() . "status", "startTime", $timeStart);
$pipeline->hset($this->getRedisEngineResult() . "status", "engineCount", sizeof($engines));
$pipeline->hset($this->getRedisEngineResult() . "status", "engineDelivered", sizeof($answered));
foreach ($answered as $engine) {
$pipeline->hset($this->getRedisEngineResult() . $engine, "delivered", "1");
}
$pipeline->execute();
}
public function retrieveResults($engines)
public function retrieveResults()
{
$engines = $this->engines;
# Von geladenen Engines die Ergebnisse holen
foreach ($engines as $engine) {
if (!$engine->loaded) {
......@@ -827,9 +755,10 @@ class MetaGer
Log::error($e);
}
}
if (!empty($engine->totalResults) && $engine->totalResults > $this->totalResults) {
$this->totalResults = $engine->totalResults;
}
}
$this->engines = $engines;
}
/*
......@@ -954,6 +883,9 @@ class MetaGer
$this->apiKey = "";
}
}
if ($this->apiKey) {
$this->apiAuthorized = $this->authorize($this->apiKey);
}
// Remove Inputs that are not used
$this->request = $request->replace($request->except(['verification_id', 'uid', 'verification_count']));
......@@ -1416,6 +1348,11 @@ class MetaGer
return $this->newtab;
}
public function setResults($results)
{
$this->results = $results;
}
public function getResults()
{
return $this->results;
......@@ -1559,4 +1496,8 @@ class MetaGer
{
return $this->redisEngineResult;
}
public function getRedisCurrentResultList()
{
return $this->redisCurrentResultList;
}
}
app/Models/Result.php
View file @ 37fe69f8
......@@ -27,6 +27,7 @@ class Result
public $strippedDomain; # Die Domain in Form "bar.de"
public $strippedLink; # Der Link in Form "foo.bar.de/test"
public $rank; # Das Ranking für das Ergebnis
public $new = true;
# Erstellt ein neues Ergebnis
public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $gefVonLink, $sourceRank, $additionalInformation = [])
......@@ -67,6 +68,7 @@ class Result
$this->price = isset($additionalInformation["price"]) ? $additionalInformation["price"] : 0;
$this->price_text = $this->price_to_text($this->price);
$this->additionalInformation = $additionalInformation;
$this->hash = spl_object_hash($this);
}
private function price_to_text($price)
......
app/Models/lang.pl deleted 100755 → 0
View file @ 7d16f746
#!/usr/bin/perl
use Lingua::Identify qw(:language_identification);
use JSON;
use warnings;
use strict;
binmode STDOUT, ":utf8";
binmode STDIN, ":utf8";
use utf8;
chomp(my $filename = <STDIN>);
# Lets open the given file:
open(my $fh, "<", $filename)
or die "Can't open < $filename: $!";
my $json = <$fh>;
close $fh;
# Decode the JSON String
my $data = JSON->new->utf8->decode($json);
# Wir durchlaufen den Hash:
foreach my $key (keys %{$data}){
$data->{$key} = langof($data->{$key});
}
$data = encode_json($data);
# Nur noch die temporäre Datei löschen:
unlink($filename);
print $data;
readme.md
View file @ 37fe69f8
......@@ -18,9 +18,6 @@
* php-gd
* sqlite3
* redis-server
* Die Perl-Pakete
* Lingua::Identify (http://search.cpan.org/~ambs/Lingua-Identify-0.56/lib/Lingua/Identify.pm)
* JSON (http://search.cpan.org/~makamaka/JSON-2.90/lib/JSON.pm)
---
[<img src="public/img/Browserstack-logo_2x.png" width="250px" alt="Browserstack Logo" />](https://www.browserstack.com) <br />
......
resources/js/scriptResultPage.js
View file @ 37fe69f8
......@@ -53,5 +53,48 @@ function enableFormResetter() {
}
function loadMoreResults() {
var searchKey = $("meta[name=searchkey]").attr("content");
var updateUrl = document.location.href;
updateUrl += "&loadMore=" + searchKey + "&script=yes";
updateUrl = updateUrl.replace("/meta.ger3", "/loadMore");
var expectedEngines = -1;
var deliveredEngines = -1;
var currentlyLoading = false;
// Regularily check for not yet delivered Results
var resultLoader = window.setInterval(function () {
if (!currentlyLoading) {
currentlyLoading = true;
$.getJSON(updateUrl, function (data) {
// Check if we can clear the interval (once every searchengine has answered)
if (data.engineDelivered == data.engineCount || data.timeWaiting > 5) {
clearInterval(resultLoader);
}
// If there are new results we can add them
if (typeof data.newResults != "undefined") {
for (var key in data.newResults) {
var value = data.newResults[key];
// If there are more results than the given index we will prepend otherwise we will append the result
var results = $(".result:not(.ad)");
if (typeof results[key] != "undefined") {
console.log("inserting before " + key);
$(results[key]).insertBefore(value);
} else {
$(results[key - 1]).insertAfter(value);
}
console.log(key + "=>" + value);
}
}
currentlyLoading = false;
});
}
}, 1000);
console.log(updateUrl);
}
\ No newline at end of file
resources/views/layouts/result.blade.php
View file @ 37fe69f8
<div class="result" data-count="{{ $result->number }}">
<div class="result" data-count="{{ $result->hash }}">
<div class="result-header">
<div class="result-headline">
<h2 class="result-title" title="{{ $result->titel }}">
......@@ -47,7 +47,7 @@
</div>
@endif
</div>
<input type="checkbox" id="result-toggle-{{$result->number}}" class="result-toggle" style="display: none">
<input type="checkbox" id="result-toggle-{{$result->hash}}" class="result-toggle" style="display: none">
<div class="result-footer">
<a class="result-open" href="{{ $result->link }}" target="_self" rel="noopener">
{!! trans('result.options.7') !!}
......@@ -58,10 +58,10 @@
<a class="result-open-proxy" onmouseover="$(this).popover('show');" onmouseout="$(this).popover('hide');" data-toggle="popover" data-placement="auto right" data-container="body" data-content="@lang('result.proxytext')" href="{{ $result->proxyLink }}" target="{{ $metager->getNewtab() }}" rel="noopener">
{!! trans('result.options.5') !!}
</a>
<label class="open-result-options navigation-element" for="result-toggle-{{$result->number}}">
<label class="open-result-options navigation-element" for="result-toggle-{{$result->hash}}">
{{ trans('result.options.more')}}
</label>
<label class="close-result-options navigation-element" for="result-toggle-{{$result->number}}">
<label class="close-result-options navigation-element" for="result-toggle-{{$result->hash}}">
{{ trans('result.options.less')}}
</label>
</div>
......@@ -69,7 +69,7 @@
<div class="options">
<ul class="option-list list-unstyled small">
<li class="js-only">
<a href="javascript:resultSaver({{ $result->number }});" class="saver">
<a href="javascript:resultSaver("{{ $result->hash }}");" class="saver">
<nobr><i class="fa fa-floppy-o"></i> {!! trans('result.options.savetab') !!}</nobr>
</a>
</li>
......
resources/views/resultpages/results.blade.php
View file @ 37fe69f8
......@@ -7,17 +7,17 @@
@endfor
@endif
{{-- Create results and ongoing ads --}}
@foreach($metager->getResults() as $result)
@foreach($metager->getResults() as $index => $result)
@if($mobile)
@if($result->number % 4 === 0)
@if($index % 4 === 0)
@include('layouts.ad', ['ad' => $metager->popAd()])
@endif
@else
@if($result->number % 7 === 0)
@if($index % 7 === 0)
@include('layouts.ad', ['ad' => $metager->popAd()])
@endif
@endif
@include('layouts.result', ['result' => $result])
@endforeach
@include('parts.pager')
</div>
\ No newline at end of file
</div>
routes/web.php
View file @ 37fe69f8
......@@ -179,6 +179,7 @@ Route::group(
Route::get('settings', 'StartpageController@loadSettings');
Route::match(['get', 'post'], 'meta/meta.ger3', 'MetaGerSearch@search')->middleware('humanverification');