diff --git a/app/Jobs/Search.php b/app/Jobs/Search.php
index 62fc6b289a31bca00015f1d3e6a7397b3f9ef57b..d6da7086f05a0133b409a56b53e5d9aff4226997 100644
--- a/app/Jobs/Search.php
+++ b/app/Jobs/Search.php
@@ -13,7 +13,7 @@ class Search extends Job implements ShouldQueue
{
use InteractsWithQueue, SerializesModels;
- protected $hash, $host, $port, $name, $getString, $useragent, $fp, $sumaFile;
+ protected $hash, $host, $port, $name, $getString, $useragent, $fp;
protected $buffer_length = 8192;
/**
@@ -21,7 +21,7 @@ class Search extends Job implements ShouldQueue
*
* @return void
*/
- public function __construct($hash, $host, $port, $name, $getString, $useragent, $sumaFile)
+ public function __construct($hash, $host, $port, $name, $getString, $useragent)
{
$this->hash = $hash;
$this->host = $host;
@@ -29,7 +29,6 @@ class Search extends Job implements ShouldQueue
$this->name = $name;
$this->getString = $getString;
$this->useragent = $useragent;
- $this->sumaFile = $sumaFile;
}
/**
@@ -50,13 +49,6 @@ class Search extends Job implements ShouldQueue
}
}
- public function disable($sumaFile, $message)
- {
- $xml = simplexml_load_file($sumaFile);
- $xml->xpath("//sumas/suma[@name='" . $this->name . "']")['0']['disabled'] = date(DATE_RFC822, mktime(date("H")+1,date("i"), date("s"), date("m"), date("d"), date("Y")));
- $xml->saveXML($sumaFile);
- }
-
private function readAnswer ()
{
$time = microtime(true);
diff --git a/app/MetaGer.php b/app/MetaGer.php
index bcb98cdbe5e50ebad90fe8fb5a062f77386ab654..61f735571cbfc1d523f367dfcb14d7995fb96670 100644
--- a/app/MetaGer.php
+++ b/app/MetaGer.php
@@ -3,6 +3,7 @@ namespace App;
use App;
use App\lib\TextLanguageDetect\TextLanguageDetect;
+use Cache;
use Illuminate\Http\Request;
use Jenssegers\Agent\Agent;
use LaravelLocalization;
@@ -31,6 +32,7 @@ class MetaGer
protected $warnings = [];
protected $errors = [];
protected $addedHosts = [];
+ protected $startCount = 0;
# Daten über die Abfrage
protected $ip;
protected $language;
@@ -58,6 +60,14 @@ class MetaGer
Log::warning("Achtung: Eine, oder mehrere Blacklist Dateien, konnten nicht geöffnet werden");
}
+ $dir = app_path() . "/Models/parserSkripte/";
+ foreach (scandir($dir) as $filename) {
+ $path = $dir . $filename;
+ if (is_file($path)) {
+ require $path;
+ }
+ }
+
$this->languageDetect = new TextLanguageDetect();
$this->languageDetect->setNameMode("2");
}
@@ -180,16 +190,22 @@ class MetaGer
public function combineResults()
{
foreach ($this->engines as $engine) {
+ if (isset($engine->next)) {
+ $this->next[] = $engine->next;
+ }
+ if (isset($engine->last)) {
+ $this->last[] = $engine->last;
+ }
foreach ($engine->results as $result) {
if ($result->valid) {
$this->results[] = $result;
}
-
}
foreach ($engine->ads as $ad) {
$this->ads[] = $ad;
}
}
+
uasort($this->results, function ($a, $b) {
if ($a->getRank() == $b->getRank()) {
return 0;
@@ -209,13 +225,23 @@ class MetaGer
$counter = 0;
$firstRank = 0;
+
+ if(isset($this->startForwards))
+ {
+ $this->startCount = $this->startForwards;
+ }elseif (isset($this->startBackwards)) {
+ $this->startCount = $this->startBackwards - count($this->results) - 1;
+ }else{
+ $this->startCount = 0;
+ }
+
foreach ($this->results as $result) {
if ($counter === 0) {
$firstRank = $result->rank;
}
$counter++;
- $result->number = $counter;
+ $result->number = $counter + $this->startCount;
$confidence = 0;
if ($firstRank > 0) {
$confidence = $result->rank / $firstRank;
@@ -256,6 +282,28 @@ class MetaGer
$this->errors[] = "Leider konnten wir zu Ihrer Sucheingabe keine passenden Ergebnisse finden.";
}
+ if( isset($this->last) && count($this->last) > 0 )
+ {
+ $page = $this->page - 1;
+ $this->last = [
+ 'page' => $page,
+ 'startBackwards' => $this->results[0]->number,
+ 'engines' => $this->last,
+ ];
+ Cache::put(md5(serialize($this->last)), serialize($this->last), 60);
+ }
+
+ if( isset($this->next) && count($this->next) > 0 && count($this->results) > 0)
+ {
+ $page = $this->page + 1;
+ $this->next = [
+ 'page' => $page,
+ 'startForwards' => $this->results[count($this->results)-1]->number,
+ 'engines' => $this->next,
+ ];
+ Cache::put(md5(serialize($this->next)), serialize($this->next), 60);
+ }
+
}
public function parseBoost($results)
@@ -447,39 +495,53 @@ class MetaGer
$typeslist = [];
$counter = 0;
- foreach ($enabledSearchengines as $engine) {
+ if ($request->has('next') && Cache::has($request->input('next')) && unserialize(Cache::get($request->input('next')))['page'] > 1 ) {
+ $next = unserialize(Cache::get($request->input('next')));
+ $this->page = $next['page'];
+ $engines = $next['engines'];
+ if(isset($next['startForwards']))
+ $this->startForwards = $next['startForwards'];
+ if(isset($next['startBackwards']))
+ $this->startBackwards = $next['startBackwards'];
+ } else {
+ foreach ($enabledSearchengines as $engine) {
- if (!$siteSearchFailed && strlen($this->site) > 0 && (!isset($engine['hasSiteSearch']) || $engine['hasSiteSearch']->__toString() === "0")) {
+ if (!$siteSearchFailed && strlen($this->site) > 0 && (!isset($engine['hasSiteSearch']) || $engine['hasSiteSearch']->__toString() === "0")) {
- continue;
- }
- # Wenn diese Suchmaschine gar nicht eingeschaltet sein soll
- $path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString());
+ continue;
+ }
+ # Wenn diese Suchmaschine gar nicht eingeschaltet sein soll
+ $path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString());
- if (!file_exists(app_path() . "/Models/parserSkripte/" . ucfirst($engine["package"]->__toString()) . ".php")) {
- Log::error("Konnte " . $engine["name"] . " nicht abfragen, da kein Parser existiert");
- continue;
- }
+ if (!file_exists(app_path() . "/Models/parserSkripte/" . ucfirst($engine["package"]->__toString()) . ".php")) {
+ Log::error("Konnte " . $engine["name"] . " nicht abfragen, da kein Parser existiert");
+ continue;
+ }
- $time = microtime();
+ $time = microtime();
- try
- {
- $tmp = new $path($engine, $this);
- } catch (\ErrorException $e) {
- Log::error("Konnte " . $engine["name"] . " nicht abfragen." . var_dump($e));
- continue;
- }
+ try
+ {
+ $tmp = new $path($engine, $this);
+ } catch (\ErrorException $e) {
+ Log::error("Konnte " . $engine["name"] . " nicht abfragen." . var_dump($e));
+ continue;
+ }
- if ($tmp->enabled && isset($this->debug)) {
- $this->warnings[] = $tmp->service . " Connection_Time: " . $tmp->connection_time . " Write_Time: " . $tmp->write_time . " Insgesamt:" . ((microtime() - $time) / 1000);
- }
+ if ($tmp->enabled && isset($this->debug)) {
+ $this->warnings[] = $tmp->service . " Connection_Time: " . $tmp->connection_time . " Write_Time: " . $tmp->write_time . " Insgesamt:" . ((microtime() - $time) / 1000);
+ }
+
+ if ($tmp->isEnabled()) {
+ $engines[] = $tmp;
+ }
- if ($tmp->isEnabled()) {
- $engines[] = $tmp;
- $this->sockets[$tmp->name] = $tmp->fp;
}
+ }
+ # Wir starten die Suche manuell:
+ foreach ($engines as $engine) {
+ $engine->startSearch($this);
}
# Jetzt werden noch alle Kategorien der Settings durchgegangen und die jeweils enthaltenen namen der Suchmaschinen gespeichert.
@@ -574,11 +636,10 @@ class MetaGer
usleep(50000);
}
- #exit;
foreach ($engines as $engine) {
if (!$engine->loaded) {
try {
- $engine->retrieveResults();
+ $engine->retrieveResults($this);
} catch (\ErrorException $e) {
Log::error($e);
@@ -648,7 +709,7 @@ class MetaGer
$this->time = $request->input('time', 1000);
# Page
- $this->page = $request->input('page', 1);
+ $this->page = 1;
# Lang
$this->lang = $request->input('lang', 'all');
if ($this->lang !== "de" && $this->lang !== "en" && $this->lang !== "all") {
@@ -847,6 +908,10 @@ class MetaGer
{
return $this->phrases;
}
+ public function getPage()
+ {
+ return $this->page;
+ }
public function getSumaFile()
{
@@ -888,6 +953,10 @@ class MetaGer
return 0;
}
}
+ public function getStartCount()
+ {
+ return $this->startCount;
+ }
public function addHostCount($host)
{
$hash = md5($host);
@@ -928,7 +997,7 @@ class MetaGer
public function generateSearchLink($fokus, $results = true)
{
- $requestData = $this->request->except('page');
+ $requestData = $this->request->except(['page', 'next']);
$requestData['focus'] = $fokus;
if ($results) {
$requestData['out'] = "results";
@@ -950,7 +1019,7 @@ class MetaGer
public function generateSiteSearchLink($host)
{
$host = urlencode($host);
- $requestData = $this->request->except(['page', 'out']);
+ $requestData = $this->request->except(['page', 'out', 'next']);
$requestData['eingabe'] .= " site:$host";
$requestData['focus'] = "web";
$link = action('MetaGerSearch@search', $requestData);
@@ -960,7 +1029,7 @@ class MetaGer
public function generateRemovedHostLink($host)
{
$host = urlencode($host);
- $requestData = $this->request->except(['page', 'out']);
+ $requestData = $this->request->except(['page', 'out', 'next']);
$requestData['eingabe'] .= " -host:$host";
$link = action('MetaGerSearch@search', $requestData);
return $link;
@@ -969,12 +1038,36 @@ class MetaGer
public function generateRemovedDomainLink($domain)
{
$domain = urlencode($domain);
- $requestData = $this->request->except(['page', 'out']);
+ $requestData = $this->request->except(['page', 'out', 'next']);
$requestData['eingabe'] .= " -domain:$domain";
$link = action('MetaGerSearch@search', $requestData);
return $link;
}
+ public function lastSearchLink()
+ {
+ if( isset($this->last) && count($this->last['engines']) > 0){
+ $requestData = $this->request->except(['page', 'out']);
+ $requestData['next'] = md5(serialize($this->last));
+ $link = action('MetaGerSearch@search', $requestData);
+ }else{
+ $link = "#";
+ }
+ return $link;
+ }
+
+ public function nextSearchLink()
+ {
+ if( isset($this->next) && count($this->next['engines']) > 0){
+ $requestData = $this->request->except(['page', 'out']);
+ $requestData['next'] = md5(serialize($this->next));
+ $link = action('MetaGerSearch@search', $requestData);
+ }else{
+ $link = "#";
+ }
+ return $link;
+ }
+
public function getTab()
{
return $this->tab;
diff --git a/app/Models/Result.php b/app/Models/Result.php
index 62bd865d094e1a6bb41b4900d08afb7f1b50ff2e..ab0e43c8c1969c6c5238bf8e25830c9d6136c13f 100644
--- a/app/Models/Result.php
+++ b/app/Models/Result.php
@@ -5,8 +5,9 @@ namespace App\Models;
class Result
{
- public function __construct(\SimpleXMLElement $provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $sourceRank, $partnershop = false, $image = "", $price = 0)
+ public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $sourceRank, $partnershop = false, $image = "", $price = 0)
{
+ $provider = simplexml_load_string($provider);
$this->titel = strip_tags(trim($titel));
$this->link = trim($link);
$this->anzeigeLink = trim($anzeigeLink);
diff --git a/app/Models/Searchengine.php b/app/Models/Searchengine.php
index a9c2c3638b81ad9f34d4860b3606a9ae32bca335..c93d8ecbde39a2aa143c625f2460b1d884e0f3c3 100644
--- a/app/Models/Searchengine.php
+++ b/app/Models/Searchengine.php
@@ -36,16 +36,12 @@ abstract class Searchengine
$this->homepage = "https://metager.de";
}
- $this->engine = $engine;
+ $this->engine = $engine->asXML();
if (!isset($this->cacheDuration)) {
$this->cacheDuration = 60;
}
- # Wir registrieren die Benutzung dieser Suchmaschine
- $this->uses = intval(Redis::hget($this->name, "uses")) + 1;
- Redis::hset($this->name, "uses", $this->uses);
-
# Eine Suchmaschine kann automatisch temporär deaktiviert werden, wenn es Verbindungsprobleme gab:
if (isset($this->disabled) && strtotime($this->disabled) <= time()) {
# In diesem Fall ist der Timeout der Suchmaschine abgelaufen.
@@ -75,21 +71,32 @@ abstract class Searchengine
$this->getString = $this->generateGetString($q, $metager->getUrl(), $metager->getLanguage(), $metager->getCategory());
$this->hash = md5($this->host . $this->getString . $this->port . $this->name);
$this->resultHash = $metager->getHashCode();
+ }
+
+ abstract public function loadResults($result);
+
+ public function getLast(MetaGer $metager, $result){
+
+ }
+ public function getNext(MetaGer $metager, $result){
+
+ }
+
+ public function startSearch(\App\MetaGer $metager)
+ {
if (Cache::has($this->hash)) {
$this->cached = true;
- $this->retrieveResults();
+ $this->retrieveResults($metager);
} else {
# Die Anfragen an die Suchmaschinen werden nun von der Laravel-Queue bearbeitet:
# Hinweis: solange in der .env der QUEUE_DRIVER auf "sync" gestellt ist, werden die Abfragen
# nacheinander abgeschickt.
# Sollen diese Parallel verarbeitet werden, muss ein anderer QUEUE_DRIVER verwendet werden.
# siehe auch: https://laravel.com/docs/5.2/queues
- $this->dispatch(new Search($this->resultHash, $this->host, $this->port, $this->name, $this->getString, $this->useragent, $metager->getSumaFile()));
+ $this->dispatch(new Search($this->resultHash, $this->host, $this->port, $this->name, $this->getString, $this->useragent));
}
}
- abstract public function loadResults($result);
-
public function rank(\App\MetaGer $metager)
{
foreach ($this->results as $result) {
@@ -130,7 +137,7 @@ abstract class Searchengine
}
}
- public function retrieveResults()
+ public function retrieveResults(MetaGer $metager)
{
if ($this->loaded) {
return true;
@@ -149,6 +156,8 @@ abstract class Searchengine
if ($body !== "") {
$this->loadResults($body);
+ $this->getNext($metager, $body);
+ $this->getLast($metager, $body);
$this->loaded = true;
Redis::hdel('search.' . $this->hash, $this->name);
return true;
diff --git a/app/Models/parserSkripte/Overture.php b/app/Models/parserSkripte/Overture.php
index f806c083212a6f36e5f15e6ea7711be739d1c3f2..2770f57aac802389757f7367b43324f364de6390 100644
--- a/app/Models/parserSkripte/Overture.php
+++ b/app/Models/parserSkripte/Overture.php
@@ -62,4 +62,58 @@ class Overture extends Searchengine
);
}
}
+
+ public function getLast(\App\MetaGer $metager, $result)
+ {
+ # Auslesen der Argumente für die nächste Suchseite:
+ $result = preg_replace("/\r\n/si", "", $result);
+ try {
+ $content = simplexml_load_string($result);
+ } catch (\Exception $e) {
+ abort(500, "$result is not a valid xml string");
+ }
+ $lastArgs = $content->xpath('//Results/PrevArgs');
+ if (isset($lastArgs[0])) {
+ $lastArgs = $lastArgs[0]->__toString();
+ } else {
+ $lastArgs = $content->xpath('//Results/ResultSet[@id="inktomi"]/PrevArgs');
+ if (isset($lastArgs[0])) {
+ $lastArgs = $lastArgs[0]->__toString();
+ } else {
+ return;
+ }
+ }
+
+ # Erstellen des neuen Suchmaschinenobjekts und anpassen des GetStrings:
+ $last = new Overture(simplexml_load_string($this->engine), $metager);
+ $last->getString = preg_replace("/&Keywords=.*?&/si", "&", $last->getString) . "&" . $lastArgs;
+ $this->last = $last;
+ }
+
+ public function getNext(\App\MetaGer $metager, $result)
+ {
+ # Auslesen der Argumente für die nächste Suchseite:
+ $result = preg_replace("/\r\n/si", "", $result);
+ try {
+ $content = simplexml_load_string($result);
+ } catch (\Exception $e) {
+ abort(500, "$result is not a valid xml string");
+ }
+ $nextArgs = $content->xpath('//Results/NextArgs');
+ if (isset($nextArgs[0])) {
+ $nextArgs = $nextArgs[0]->__toString();
+ } else {
+ $nextArgs = $content->xpath('//Results/ResultSet[@id="inktomi"]/NextArgs');
+ if (isset($nextArgs[0])) {
+ $nextArgs = $nextArgs[0]->__toString();
+ } else {
+ return;
+ }
+ }
+
+ # Erstellen des neuen Suchmaschinenobjekts und anpassen des GetStrings:
+ $next = new Overture(simplexml_load_string($this->engine), $metager);
+ $next->getString = preg_replace("/&Keywords=.*?&/si", "&", $next->getString) . "&" . $nextArgs;
+ $this->next = $next;
+ }
}
diff --git a/app/Models/parserSkripte/Yandex.php b/app/Models/parserSkripte/Yandex.php
index 2de70900a280932cd772222baffdbfbbd5ae22f9..d264802cbc58c77a1588e4ce51ff1f0b3ca7bf0d 100644
--- a/app/Models/parserSkripte/Yandex.php
+++ b/app/Models/parserSkripte/Yandex.php
@@ -51,4 +51,22 @@ class Yandex extends Searchengine
);
}
}
+
+ public function getLast(\App\MetaGer $metager, $result)
+ {
+ if( $metager->getPage() <= 1 )
+ return;
+ $next = new Yandex(simplexml_load_string($this->engine), $metager);
+ $next->getString .= "page=" . ($metager->getPage() -1 );
+ $this->next = $next;
+ }
+
+ public function getNext(\App\MetaGer $metager, $result)
+ {
+ if( count($this->results) <= 0 )
+ return;
+ $next = new Yandex(simplexml_load_string($this->engine), $metager);
+ $next->getString .= "page=" . $metager->getPage();
+ $this->next = $next;
+ }
}
diff --git a/resources/views/layouts/researchandtabs.blade.php b/resources/views/layouts/researchandtabs.blade.php
index 5518dc5b96011d386d85655910c043fe5baf80fb..e8eee40cbe84958f98976b2e105f5bf8a0d56da9 100644
--- a/resources/views/layouts/researchandtabs.blade.php
+++ b/resources/views/layouts/researchandtabs.blade.php
@@ -21,7 +21,7 @@
</div>
@foreach( $metager->request->all() as $key => $value)
- @if($key !== "eingabe" && $key !== "page")
+ @if($key !== "eingabe" && $key !== "page" && $key !== "next")
<input type='hidden' name='{{ $key }}' value='{{ $value }}' form='submitForm' />
@endif
@endforeach
diff --git a/resources/views/layouts/resultPage.blade.php b/resources/views/layouts/resultPage.blade.php
index e4a1b2571922fc4501e4e0848da1fa4f95460ee8..17d7ea743f4f13186f48e41576043831624f5b0d 100644
--- a/resources/views/layouts/resultPage.blade.php
+++ b/resources/views/layouts/resultPage.blade.php
@@ -15,7 +15,6 @@
<meta content="{{ getmypid() }}" name="p" />
<meta content="{{ $eingabe }}" name="q" />
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
- <meta name="referrer" content="no-referrer" />
<link rel="search" type="application/opensearchdescription+xml" title="MetaGer: Sicher suchen & finden, Privatsphäre schützen" href="{{ LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), action('StartpageController@loadPlugin', ['params' => base64_encode(serialize(Request::all()))])) }}">
</head>
@@ -24,9 +23,15 @@
@include('layouts.researchandtabs')
@else
<div class="tab-content container-fluid">
- @yield('results')
+ @yield('results')
</div>
@endif
+ <nav aria-label="...">
+ <ul class="pager">
+ <li @if($metager->lastSearchLink() === "#") class="disabled" @endif><a href="{{ $metager->lastSearchLink() }}">Zurück</a></li>
+ <li @if($metager->nextSearchLink() === "#") class="disabled" @endif><a href="{{ $metager->nextSearchLink() }}">Weiter Suchen</a></li>
+ </ul>
+ </nav>
<footer>
<div class="row">
<div class="col-xs-6">