Commit f8f4cb38 authored by Karl's avatar Karl
Browse files

Merge branch 'development' into 363-spanische-version-vorbereiten

parents 534e2a69 632598f1
......@@ -28,7 +28,7 @@ class StartpageController extends Controller
$focusPages = [];
$theme = "default";
foreach ($request->all() as $key => $value) {
if ($value === 'on' && $key != 'param_sprueche' && $key != 'param_tab' && $key !== 'param_maps') {
if ($value === 'on' && $key != 'param_sprueche' && $key != 'param_tab' && $key !== 'param_maps' && $key !== 'param_autocomplete') {
$focusPages[] = str_replace('param_', '', $key);
}
if ($key === 'param_theme') {
......@@ -54,7 +54,8 @@ class StartpageController extends Controller
->with('browser', $browser)
->with('navbarFocus', 'suche')
->with('theme', $theme)
->with('maps', $maps);
->with('maps', $maps)
->with('autocomplete', $request->input('param_autocomplete', 'on'));
}
public function loadPage($subpage)
......
......@@ -140,6 +140,10 @@ class MetaGer
->with('suspendheader', "yes")
->with('browser', (new Agent())->browser());
break;
case 'result-count':
# Wir geben die Ergebniszahl und die benötigte Zeit zurück:
return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
break;
default:
return view('metager3')
->with('eingabe', $this->eingabe)
......@@ -287,7 +291,7 @@ class MetaGer
# Wir geben jedem Ergebnis eine ID um später die Sprachcodes zuordnen zu können
$result->id = $id;
$langStrings["result_" . $id] = utf8_encode($result->getLangString());
$langStrings["result_" . $id] = $result->getLangString();
$id++;
}
......@@ -854,10 +858,7 @@ class MetaGer
}
# Sucheingabe
$this->eingabe = trim($request->input('eingabe', ''));
if (strlen($this->eingabe) === 0) {
$this->warnings[] = trans('metaGer.formdata.noSearch');
}
$this->q = $this->eingabe;
$this->q = strtolower($this->eingabe);
# IP
$this->ip = $request->ip();
# Language
......@@ -941,9 +942,21 @@ class MetaGer
}
$this->out = $request->input('out', "html");
# Standard output format html
if ($this->out !== "html" && $this->out !== "json" && $this->out !== "results" && $this->out !== "results-with-style") {
if ($this->out !== "html" && $this->out !== "json" && $this->out !== "results" && $this->out !== "results-with-style" && $this->out !== "result-count") {
$this->out = "html";
}
# Wir schalten den Cache aus, wenn die Ergebniszahl überprüft werden soll
# => out=result-count
# Ist dieser Parameter gesetzt, so soll überprüft werden, wie viele Ergebnisse wir liefern.
# Wenn wir gecachte Ergebnisse zurück liefern würden, wäre das nicht sonderlich klug, da es dann keine Aussagekraft hätte
# ob MetaGer funktioniert (bzw. die Fetcher laufen)
# Auch ein Log sollte nicht geschrieben werden, da es am Ende ziemlich viele Logs werden könnten.
if ($this->out === "result-count") {
$this->canCache = false;
$this->shouldLog = false;
} else {
$this->shouldLog = true;
}
}
public function checkSpecialSearches(Request $request)
......@@ -959,15 +972,13 @@ class MetaGer
$this->searchCheckDomainBlacklist();
$this->searchCheckPhrase();
$this->searchCheckStopwords();
if ($this->q === "") {
$this->warnings[] = trans('metaGer.formdata.noSearch');
}
$this->searchCheckNoSearch();
}
public function searchCheckSitesearch($site)
{
if (preg_match("/(.*)\bsite:(\S+)(.*)/si", $this->q, $match)) {
// matches '[... ]site:test.de[ ...]'
while (preg_match("/(^|.+\s)site:(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->site = $match[2];
$this->q = $match[1] . $match[3];
}
......@@ -978,9 +989,10 @@ class MetaGer
public function searchCheckHostBlacklist()
{
while (preg_match("/(.*)(^|\s)-host:(\S+)(.*)/si", $this->q, $match)) {
$this->hostBlacklist[] = $match[3];
$this->q = $match[1] . $match[4];
// matches '[... ]-site:test.de[ ...]'
while (preg_match("/(^|.+\s)-site:([^\s\*]\S*)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->hostBlacklist[] = $match[2];
$this->q = $match[1] . $match[3];
}
if (sizeof($this->hostBlacklist) > 0) {
$hostString = "";
......@@ -994,9 +1006,10 @@ class MetaGer
public function searchCheckDomainBlacklist()
{
while (preg_match("/(.*)(^|\s)-domain:(\S+)(.*)/si", $this->q, $match)) {
$this->domainBlacklist[] = $match[3];
$this->q = $match[1] . $match[4];
// matches '[... ]-site:*.test.de[ ...]'
while (preg_match("/(^|.+\s)-site:\*\.(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->domainBlacklist[] = $match[2];
$this->q = $match[1] . $match[3];
}
if (sizeof($this->domainBlacklist) > 0) {
$domainString = "";
......@@ -1010,9 +1023,10 @@ class MetaGer
public function searchCheckStopwords()
{
while (preg_match("/(.*)(^|\s)-(\S+)(.*)/si", $this->q, $match)) {
$this->stopWords[] = $match[3];
$this->q = $match[1] . $match[4];
// matches '[... ]-test[ ...]'
while (preg_match("/(^|.+\s)-(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->stopWords[] = $match[2];
$this->q = $match[1] . $match[3];
}
if (sizeof($this->stopWords) > 0) {
$stopwordsString = "";
......@@ -1028,7 +1042,8 @@ class MetaGer
{
$p = "";
$tmp = $this->q;
while (preg_match("/(.*)\"(.+)\"(.*)/si", $tmp, $match)) {
// matches '[... ]"test satz"[ ...]'
while (preg_match("/(^|.+\s)\"(.+)\"(?:\s(.+)|($))/si", $tmp, $match)) {
$tmp = $match[1] . $match[3];
$this->phrases[] = strtolower($match[2]);
}
......@@ -1041,6 +1056,13 @@ class MetaGer
}
}
public function searchCheckNoSearch()
{
if ($this->q === "") {
$this->warnings[] = trans('metaGer.formdata.noSearch');
}
}
public function nextSearchLink()
{
if (isset($this->next) && isset($this->next['engines']) && count($this->next['engines']) > 0) {
......@@ -1112,6 +1134,7 @@ class MetaGer
public function createLogs()
{
if ($this->shouldLog) {
$redis = Redis::connection('redisLogs');
try
{
......@@ -1142,6 +1165,7 @@ class MetaGer
return;
}
}
}
public function addLink($link)
{
......@@ -1214,7 +1238,7 @@ class MetaGer
{
$host = urlencode($host);
$requestData = $this->request->except(['page', 'out', 'next']);
$requestData['eingabe'] .= " -host:$host";
$requestData['eingabe'] .= " -site:$host";
$link = action('MetaGerSearch@search', $requestData);
return $link;
}
......@@ -1223,7 +1247,7 @@ class MetaGer
{
$domain = urlencode($domain);
$requestData = $this->request->except(['page', 'out', 'next']);
$requestData['eingabe'] .= " -domain:$domain";
$requestData['eingabe'] .= " -site:*.$domain";
$link = action('MetaGerSearch@search', $requestData);
return $link;
}
......
......@@ -17,7 +17,7 @@ my $json = <$fh>;
close $fh;
# Decode the JSON String
my $data = decode_json($json);
my $data = JSON->new->utf8->decode($json);
# Wir durchlaufen den Hash:
foreach my $key (keys %{$data}){
......
......@@ -3,6 +3,7 @@
namespace App\Models\parserSkripte;
use App\Models\Searchengine;
use Log;
use Symfony\Component\DomCrawler\Crawler;
class Allesklar extends Searchengine
......@@ -46,10 +47,10 @@ class Allesklar extends Searchengine
$this->gefVon,
$this->counter
);
} catch (\InvalidArgumentException $e) {
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
});
}
......
......@@ -15,20 +15,6 @@ class BASE extends Searchengine
public function loadResults($result)
{
$title = "";
$link = "";
$anzeigeLink = $link;
$descr = "";
/*$this->counter++;
$this->results[] = new \App\Models\Result(
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter
);*/
return;
}
}
......@@ -16,9 +16,7 @@ class Bing extends Searchengine
public function loadResults($result)
{
try
{
try {
$crawler = new Crawler($result);
$crawler->filter('ol#b_results > li.b_algo')->each(function (Crawler $node, $i) {
$title = $node->filter('li h2 > a')->text();
......@@ -37,7 +35,8 @@ class Bing extends Searchengine
$this->counter
);
});
} catch (\ErrorException $e) {
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
......
......@@ -16,18 +16,13 @@ class Blogsearch extends Searchengine
public function loadResults($result)
{
$result = preg_replace("/\r\n/si", "", $result);
try {
$content = simplexml_load_string($result);
} catch (\Exception $e) {
abort(500, "$result is not a valid xml string");
}
if (!$content) {
return;
}
$results = $content->xpath('//xml/docs/doc');
# die(var_dump($results));
foreach ($results as $result) {
$title = $result->{"title"}->__toString();
$link = $result->{"url"}->__toString();
......@@ -44,6 +39,9 @@ class Blogsearch extends Searchengine
$this->counter
);
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
}
}
......@@ -18,10 +18,6 @@ class Dailymotion extends Searchengine
$result = preg_replace("/\r\n/si", "", $result);
try {
$content = json_decode($result);
} catch (\Exception $e) {
abort(500, "$result is not a valid xml string");
}
if (!$content) {
return;
}
......@@ -46,5 +42,9 @@ class Dailymotion extends Searchengine
$image
);
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
}
}
......@@ -15,18 +15,14 @@ class Ebay extends Searchengine
public function loadResults($result)
{
$result = preg_replace("/\r\n/si", "", $result);
try {
$content = simplexml_load_string($result);
} catch (\Exception $e) {
abort(500, "$result is not a valid xml string");
}
if (!$content) {
return;
}
$results = $content;
$results = $results->{"searchResult"};
$results = $content->{"searchResult"};
$count = 0;
foreach ($results->{"item"} as $result) {
$title = $result->{"title"}->__toString();
......@@ -61,8 +57,10 @@ class Ebay extends Searchengine
$price
);
$count++;
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
}
}
......@@ -16,16 +16,13 @@ class Ecoshopper extends Searchengine
public function loadResults($result)
{
$result = preg_replace("/\r\n/si", "", $result);
try {
$content = simplexml_load_string($result);
} catch (\Exception $e) {
Log::error("Ergebnisse von Ecoshopper konnten nicht eingelesen werden");
return;
}
if (!$content) {
return;
}
$results = $content->xpath('//response/result[@name="response"]/doc');
foreach ($results as $result) {
$result = simplexml_load_string($result->saveXML());
......@@ -58,5 +55,9 @@ class Ecoshopper extends Searchengine
$image
);
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
}
}
......@@ -3,6 +3,7 @@
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
use Log;
class Europeana extends Searchengine
{
......@@ -18,10 +19,6 @@ class Europeana extends Searchengine
$result = preg_replace("/\r\n/si", "", $result);
try {
$content = json_decode($result);
} catch (\Exception $e) {
abort(500, "$result is not a valid json string");
}
if (!$content) {
return;
}
......@@ -52,12 +49,21 @@ class Europeana extends Searchengine
);
}
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
}
public function getNext(\App\MetaGer $metager, $result)
{
$start = ($metager->getPage()) * 10 + 1;
try {
$content = json_decode($result);
if (!$content) {
return;
}
$start = ($metager->getPage()) * 10 + 1;
if ($start > $content->totalResults) {
return;
}
......@@ -65,5 +71,9 @@ class Europeana extends Searchengine
$next->getString .= "&start=" . $start;
$next->hash = md5($next->host . $next->getString . $next->port . $next->name);
$this->next = $next;
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
}
}
......@@ -18,13 +18,10 @@ class Exalead extends Searchengine
$result = preg_replace("/\r\n/si", "", $result);
try {
$content = simplexml_load_string($result);
} catch (\Exception $e) {
return;
}
if (!$content) {
return;
}
$results = $content;
$prefix = "";
$namespace = "";
......@@ -72,5 +69,9 @@ class Exalead extends Searchengine
continue;
}
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
}
}
......@@ -18,10 +18,6 @@ class Fernsehsuche extends Searchengine
$result = preg_replace("/\r\n/si", "", $result);
try {
$content = json_decode($result);
} catch (\Exception $e) {
abort(500, "$result is not a valid xml string");
}
if (!$content) {
return;
}
......@@ -50,5 +46,9 @@ class Fernsehsuche extends Searchengine
}
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
}
}
......@@ -3,6 +3,7 @@
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
use Log;
class Flickr extends Searchengine
{
......@@ -18,13 +19,10 @@ class Flickr extends Searchengine
$result = preg_replace("/\r\n/si", "", $result);
try {
$content = simplexml_load_string($result);
} catch (\Exception $e) {
abort(500, "$result is not a valid xml string");
}
if (!$content) {
return;
}
$results = $content->xpath('//photos/photo');
foreach ($results as $result) {
$title = $result["title"]->__toString();
......@@ -45,22 +43,23 @@ class Flickr extends Searchengine
$image
);
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
}
public function getNext(\App\MetaGer $metager, $result)
{
$page = $metager->getPage() + 1;
$result = preg_replace("/\r\n/si", "", $result);
$content = simplexml_load_string($result);
$results = $content->xpath('//photos')[0];
try {
$content = simplexml_load_string($result);
} catch (\Exception $e) {
abort(500, "$result is not a valid xml string");
}
if (!$content) {
return;
}
$page = $metager->getPage() + 1;
$results = $content->xpath('//photos')[0];
if ($page >= intval($results["pages"]->__toString())) {
return;
}
......@@ -68,5 +67,9 @@ class Flickr extends Searchengine
$next->getString .= "&page=" . $page;
$next->hash = md5($next->host . $next->getString . $next->port . $next->name);
$this->next = $next;
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
}
}
......@@ -19,22 +19,20 @@ class Mg_produkt2 extends Searchengine
$result = preg_replace("/\r\n/si", "", $result);
try {
$content = simplexml_load_string($result);
} catch (\Exception $e) {
Log::error("MG_Produkt konnte keine Ergebnisse bekommen");
return;
}
if (!$content) {
return;
}
$results = $content->xpath('//response/result[@name="response"]/doc');
foreach ($results as $result) {
$result = simplexml_load_string($result->saveXML());
$title = $result->xpath('/doc/arr[@name="artikelName"]')[0]->{"str"}->__toString();
$link = $result->xpath('/doc/arr[@name="artikelDeeplink"]')[0]->{"str"}->__toString();
$anzeigeLink = parse_url($link);
parse_str($anzeigeLink['query'], $query);
$anzeigeLink = $query['diurl'];
$anzeigeLink = $query['url'];
$descr = $result->xpath('/doc/arr[@name="artikelBeschreibung"]')[0]->{"str"}->__toString();
$image = $result->xpath('/doc/arr[@name="artikelImageurl"]')[0]->{"str"}->__toString();
$this->counter++;
......@@ -50,5 +48,9 @@ class Mg_produkt2 extends Searchengine
$image
);
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
}
}
......@@ -3,6 +3,7 @@
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
use Log;
class Openclipart extends Searchengine
{
......@@ -18,10 +19,6 @@ class Openclipart extends Searchengine
$result = preg_replace("/\r\n/si", "", $result);
try {
$content = json_decode($result);
} catch (\Exception $e) {
abort(500, "$result is not a valid json string");
}
if (!$content) {
return;
}
......@@ -46,11 +43,20 @@ class Openclipart extends Searchengine
$image
);
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
}
public function getNext(\App\MetaGer $metager, $result)
{
try {
$content = json_decode($result);
if (!$content) {
return;
}
if ($content->info->current_page > $content->info->pages) {
return;
}
......@@ -58,5 +64,9 @@ class Openclipart extends Searchengine
$next->getString .= "&page=" . ($metager->getPage()