Commit f8f4cb38 authored by Karl's avatar Karl
Browse files

Merge branch 'development' into 363-spanische-version-vorbereiten

parents 534e2a69 632598f1
......@@ -28,7 +28,7 @@ class StartpageController extends Controller
$focusPages = [];
$theme = "default";
foreach ($request->all() as $key => $value) {
if ($value === 'on' && $key != 'param_sprueche' && $key != 'param_tab' && $key !== 'param_maps') {
if ($value === 'on' && $key != 'param_sprueche' && $key != 'param_tab' && $key !== 'param_maps' && $key !== 'param_autocomplete') {
$focusPages[] = str_replace('param_', '', $key);
}
if ($key === 'param_theme') {
......@@ -54,7 +54,8 @@ class StartpageController extends Controller
->with('browser', $browser)
->with('navbarFocus', 'suche')
->with('theme', $theme)
->with('maps', $maps);
->with('maps', $maps)
->with('autocomplete', $request->input('param_autocomplete', 'on'));
}
public function loadPage($subpage)
......
......@@ -140,6 +140,10 @@ class MetaGer
->with('suspendheader', "yes")
->with('browser', (new Agent())->browser());
break;
case 'result-count':
# Wir geben die Ergebniszahl und die benötigte Zeit zurück:
return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
break;
default:
return view('metager3')
->with('eingabe', $this->eingabe)
......@@ -287,7 +291,7 @@ class MetaGer
# Wir geben jedem Ergebnis eine ID um später die Sprachcodes zuordnen zu können
$result->id = $id;
$langStrings["result_" . $id] = utf8_encode($result->getLangString());
$langStrings["result_" . $id] = $result->getLangString();
$id++;
}
......@@ -854,10 +858,7 @@ class MetaGer
}
# Sucheingabe
$this->eingabe = trim($request->input('eingabe', ''));
if (strlen($this->eingabe) === 0) {
$this->warnings[] = trans('metaGer.formdata.noSearch');
}
$this->q = $this->eingabe;
$this->q = strtolower($this->eingabe);
# IP
$this->ip = $request->ip();
# Language
......@@ -941,9 +942,21 @@ class MetaGer
}
$this->out = $request->input('out', "html");
# Standard output format html
if ($this->out !== "html" && $this->out !== "json" && $this->out !== "results" && $this->out !== "results-with-style") {
if ($this->out !== "html" && $this->out !== "json" && $this->out !== "results" && $this->out !== "results-with-style" && $this->out !== "result-count") {
$this->out = "html";
}
# Wir schalten den Cache aus, wenn die Ergebniszahl überprüft werden soll
# => out=result-count
# Ist dieser Parameter gesetzt, so soll überprüft werden, wie viele Ergebnisse wir liefern.
# Wenn wir gecachte Ergebnisse zurück liefern würden, wäre das nicht sonderlich klug, da es dann keine Aussagekraft hätte
# ob MetaGer funktioniert (bzw. die Fetcher laufen)
# Auch ein Log sollte nicht geschrieben werden, da es am Ende ziemlich viele Logs werden könnten.
if ($this->out === "result-count") {
$this->canCache = false;
$this->shouldLog = false;
} else {
$this->shouldLog = true;
}
}
public function checkSpecialSearches(Request $request)
......@@ -959,15 +972,13 @@ class MetaGer
$this->searchCheckDomainBlacklist();
$this->searchCheckPhrase();
$this->searchCheckStopwords();
if ($this->q === "") {
$this->warnings[] = trans('metaGer.formdata.noSearch');
}
$this->searchCheckNoSearch();
}
public function searchCheckSitesearch($site)
{
if (preg_match("/(.*)\bsite:(\S+)(.*)/si", $this->q, $match)) {
// matches '[... ]site:test.de[ ...]'
while (preg_match("/(^|.+\s)site:(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->site = $match[2];
$this->q = $match[1] . $match[3];
}
......@@ -978,9 +989,10 @@ class MetaGer
public function searchCheckHostBlacklist()
{
while (preg_match("/(.*)(^|\s)-host:(\S+)(.*)/si", $this->q, $match)) {
$this->hostBlacklist[] = $match[3];
$this->q = $match[1] . $match[4];
// matches '[... ]-site:test.de[ ...]'
while (preg_match("/(^|.+\s)-site:([^\s\*]\S*)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->hostBlacklist[] = $match[2];
$this->q = $match[1] . $match[3];
}
if (sizeof($this->hostBlacklist) > 0) {
$hostString = "";
......@@ -994,9 +1006,10 @@ class MetaGer
public function searchCheckDomainBlacklist()
{
while (preg_match("/(.*)(^|\s)-domain:(\S+)(.*)/si", $this->q, $match)) {
$this->domainBlacklist[] = $match[3];
$this->q = $match[1] . $match[4];
// matches '[... ]-site:*.test.de[ ...]'
while (preg_match("/(^|.+\s)-site:\*\.(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->domainBlacklist[] = $match[2];
$this->q = $match[1] . $match[3];
}
if (sizeof($this->domainBlacklist) > 0) {
$domainString = "";
......@@ -1010,9 +1023,10 @@ class MetaGer
public function searchCheckStopwords()
{
while (preg_match("/(.*)(^|\s)-(\S+)(.*)/si", $this->q, $match)) {
$this->stopWords[] = $match[3];
$this->q = $match[1] . $match[4];
// matches '[... ]-test[ ...]'
while (preg_match("/(^|.+\s)-(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->stopWords[] = $match[2];
$this->q = $match[1] . $match[3];
}
if (sizeof($this->stopWords) > 0) {
$stopwordsString = "";
......@@ -1028,7 +1042,8 @@ class MetaGer
{
$p = "";
$tmp = $this->q;
while (preg_match("/(.*)\"(.+)\"(.*)/si", $tmp, $match)) {
// matches '[... ]"test satz"[ ...]'
while (preg_match("/(^|.+\s)\"(.+)\"(?:\s(.+)|($))/si", $tmp, $match)) {
$tmp = $match[1] . $match[3];
$this->phrases[] = strtolower($match[2]);
}
......@@ -1041,6 +1056,13 @@ class MetaGer
}
}
public function searchCheckNoSearch()
{
if ($this->q === "") {
$this->warnings[] = trans('metaGer.formdata.noSearch');
}
}
public function nextSearchLink()
{
if (isset($this->next) && isset($this->next['engines']) && count($this->next['engines']) > 0) {
......@@ -1112,34 +1134,36 @@ class MetaGer
public function createLogs()
{
$redis = Redis::connection('redisLogs');
try
{
$logEntry = "";
$logEntry .= "[" . date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y"))) . "]";
$logEntry .= " pid=" . getmypid();
$logEntry .= " ref=" . $this->request->header('Referer');
$useragent = $this->request->header('User-Agent');
$useragent = str_replace("(", " ", $useragent);
$useragent = str_replace(")", " ", $useragent);
$useragent = str_replace(" ", "", $useragent);
$logEntry .= " time=" . round((microtime(true) - $this->starttime), 2) . " serv=" . $this->fokus;
$logEntry .= " search=" . $this->eingabe;
# 2 Arten von Logs in einem wird die Anzahl der Abfragen an eine Suchmaschine gespeichert und in der anderen
# die Anzahl, wie häufig diese Ergebnisse geliefert hat.
$enginesToLoad = $this->enginesToLoad;
$redis->pipeline(function ($pipe) use ($enginesToLoad, $logEntry) {
$pipe->rpush('logs.search', $logEntry);
foreach ($this->enginesToLoad as $name => $answered) {
$pipe->incr('logs.engines.requests.' . $name);
if ($answered) {
$pipe->incr('logs.engines.answered.' . $name);
if ($this->shouldLog) {
$redis = Redis::connection('redisLogs');
try
{
$logEntry = "";
$logEntry .= "[" . date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y"))) . "]";
$logEntry .= " pid=" . getmypid();
$logEntry .= " ref=" . $this->request->header('Referer');
$useragent = $this->request->header('User-Agent');
$useragent = str_replace("(", " ", $useragent);
$useragent = str_replace(")", " ", $useragent);
$useragent = str_replace(" ", "", $useragent);
$logEntry .= " time=" . round((microtime(true) - $this->starttime), 2) . " serv=" . $this->fokus;
$logEntry .= " search=" . $this->eingabe;
# 2 Arten von Logs in einem wird die Anzahl der Abfragen an eine Suchmaschine gespeichert und in der anderen
# die Anzahl, wie häufig diese Ergebnisse geliefert hat.
$enginesToLoad = $this->enginesToLoad;
$redis->pipeline(function ($pipe) use ($enginesToLoad, $logEntry) {
$pipe->rpush('logs.search', $logEntry);
foreach ($this->enginesToLoad as $name => $answered) {
$pipe->incr('logs.engines.requests.' . $name);
if ($answered) {
$pipe->incr('logs.engines.answered.' . $name);
}
}
}
});
} catch (\Exception $e) {
return;
});
} catch (\Exception $e) {
return;
}
}
}
......@@ -1214,7 +1238,7 @@ class MetaGer
{
$host = urlencode($host);
$requestData = $this->request->except(['page', 'out', 'next']);
$requestData['eingabe'] .= " -host:$host";
$requestData['eingabe'] .= " -site:$host";
$link = action('MetaGerSearch@search', $requestData);
return $link;
}
......@@ -1223,7 +1247,7 @@ class MetaGer
{
$domain = urlencode($domain);
$requestData = $this->request->except(['page', 'out', 'next']);
$requestData['eingabe'] .= " -domain:$domain";
$requestData['eingabe'] .= " -site:*.$domain";
$link = action('MetaGerSearch@search', $requestData);
return $link;
}
......
......@@ -17,7 +17,7 @@ my $json = <$fh>;
close $fh;
# Decode the JSON String
my $data = decode_json($json);
my $data = JSON->new->utf8->decode($json);
# Wir durchlaufen den Hash:
foreach my $key (keys %{$data}){
......
......@@ -3,6 +3,7 @@
namespace App\Models\parserSkripte;
use App\Models\Searchengine;
use Log;
use Symfony\Component\DomCrawler\Crawler;
class Allesklar extends Searchengine
......@@ -46,10 +47,10 @@ class Allesklar extends Searchengine
$this->gefVon,
$this->counter
);
} catch (\InvalidArgumentException $e) {
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
});
}
......
......@@ -15,20 +15,6 @@ class BASE extends Searchengine
public function loadResults($result)
{
$title = "";
$link = "";
$anzeigeLink = $link;
$descr = "";
/*$this->counter++;
$this->results[] = new \App\Models\Result(
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter
);*/
return;
}
}
......@@ -16,9 +16,7 @@ class Bing extends Searchengine
public function loadResults($result)
{
try
{
try {
$crawler = new Crawler($result);
$crawler->filter('ol#b_results > li.b_algo')->each(function (Crawler $node, $i) {
$title = $node->filter('li h2 > a')->text();
......@@ -37,7 +35,8 @@ class Bing extends Searchengine
$this->counter
);
});
} catch (\ErrorException $e) {
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
......
......@@ -16,34 +16,32 @@ class Blogsearch extends Searchengine
public function loadResults($result)
{
$result = preg_replace("/\r\n/si", "", $result);
try {
$content = simplexml_load_string($result);
if (!$content) {
return;
}
$results = $content->xpath('//xml/docs/doc');
foreach ($results as $result) {
$title = $result->{"title"}->__toString();
$link = $result->{"url"}->__toString();
$anzeigeLink = $link;
$descr = $result->{"content"}->__toString();
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter
);
}
} catch (\Exception $e) {
abort(500, "$result is not a valid xml string");
}
if (!$content) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
$results = $content->xpath('//xml/docs/doc');
# die(var_dump($results));
foreach ($results as $result) {
$title = $result->{"title"}->__toString();
$link = $result->{"url"}->__toString();
$anzeigeLink = $link;
$descr = $result->{"content"}->__toString();
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter
);
}
}
}
......@@ -18,33 +18,33 @@ class Dailymotion extends Searchengine
$result = preg_replace("/\r\n/si", "", $result);
try {
$content = json_decode($result);
} catch (\Exception $e) {
abort(500, "$result is not a valid xml string");
}
if (!$content) {
return;
}
if (!$content) {
$results = $content->list;
foreach ($results as $result) {
$title = $result->title;
$link = $result->url;
$anzeigeLink = $link;
$descr = $result->description;
$image = $result->thumbnail_240_url;
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter,
false,
$image
);
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
$results = $content->list;
foreach ($results as $result) {
$title = $result->title;
$link = $result->url;
$anzeigeLink = $link;
$descr = $result->description;
$image = $result->thumbnail_240_url;
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter,
false,
$image
);
}
}
}
......@@ -15,54 +15,52 @@ class Ebay extends Searchengine
public function loadResults($result)
{
$result = preg_replace("/\r\n/si", "", $result);
try {
$content = simplexml_load_string($result);
} catch (\Exception $e) {
abort(500, "$result is not a valid xml string");
}
if (!$content) {
return;
}
$results = $content;
$results = $results->{"searchResult"};
$count = 0;
foreach ($results->{"item"} as $result) {
$title = $result->{"title"}->__toString();
$link = $result->{"viewItemURL"}->__toString();
$anzeigeLink = $link;
$time = $result->{"listingInfo"}->{"endTime"}->__toString();
$time = date(DATE_RFC2822, strtotime($time));
$price = intval($result->{"sellingStatus"}->{"convertedCurrentPrice"}->__toString()) * 100;
$descr = "<p>Preis: " . $result->{"sellingStatus"}->{"convertedCurrentPrice"}->__toString() . " €</p>";
$descr .= "<p>Versandkosten: " . $result->{"shippingInfo"}->{"shippingServiceCost"}->__toString() . " €</p>";
if (isset($result->{"listingInfo"}->{"listingType"})) {
$descr .= "<p>Auktionsart: " . $result->{"listingInfo"}->{"listingType"}->__toString() . "</p>";
if (!$content) {
return;
}
$descr .= "<p>Auktionsende: " . $time . "</p>";
if (isset($result->{"primaryCategory"}->{"categoryName"})) {
$descr .= "<p class=\"text-muted\">Kategorie: " . $result->{"primaryCategory"}->{"categoryName"}->__toString() . "</p>";
}
$results = $content->{"searchResult"};
$count = 0;
foreach ($results->{"item"} as $result) {
$title = $result->{"title"}->__toString();
$link = $result->{"viewItemURL"}->__toString();
$anzeigeLink = $link;
$time = $result->{"listingInfo"}->{"endTime"}->__toString();
$time = date(DATE_RFC2822, strtotime($time));
$price = intval($result->{"sellingStatus"}->{"convertedCurrentPrice"}->__toString()) * 100;
$descr = "<p>Preis: " . $result->{"sellingStatus"}->{"convertedCurrentPrice"}->__toString() . " €</p>";
$descr .= "<p>Versandkosten: " . $result->{"shippingInfo"}->{"shippingServiceCost"}->__toString() . " €</p>";
if (isset($result->{"listingInfo"}->{"listingType"})) {
$descr .= "<p>Auktionsart: " . $result->{"listingInfo"}->{"listingType"}->__toString() . "</p>";
}
$image = $result->{"galleryURL"}->__toString();
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter,
false,
$image,
$price
);
$count++;
$descr .= "<p>Auktionsende: " . $time . "</p>";
if (isset($result->{"primaryCategory"}->{"categoryName"})) {
$descr .= "<p class=\"text-muted\">Kategorie: " . $result->{"primaryCategory"}->{"categoryName"}->__toString() . "</p>";
}
$image = $result->{"galleryURL"}->__toString();
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter,
false,
$image,
$price
);
$count++;
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
}
}
}
......@@ -16,47 +16,48 @@ class Ecoshopper extends Searchengine
public function loadResults($result)
{
$result = preg_replace("/\r\n/si", "", $result);
try {
$content = simplexml_load_string($result);
} catch (\Exception $e) {
Log::error("Ergebnisse von Ecoshopper konnten nicht eingelesen werden");
return;
}
if (!$content) {
return;
}
$results = $content->xpath('//response/result[@name="response"]/doc');
foreach ($results as $result) {
$result = simplexml_load_string($result->saveXML());
$title = $result->xpath('//doc/str[@name="artikelName"]')[0]->__toString();
$link = $result->xpath('//doc/str[@name="artikelDeeplink"]')[0]->__toString();
$anzeigeLink = parse_url($link);
if (isset($anzeigeLink['query'])) {
parse_str($anzeigeLink['query'], $query);
if (isset($query['diurl'])) {
$anzeigeLink = $query['diurl'];
} else {
$anzeigeLink = $link;
}
if (!$content) {
return;
}
$results = $content->xpath('//response/result[@name="response"]/doc');
foreach ($results as $result) {
$result = simplexml_load_string($result->saveXML());
$title = $result->xpath('//doc/str[@name="artikelName"]')[0]->__toString();
$link = $result->xpath('//doc/str[@name="artikelDeeplink"]')[0]->__toString();
$anzeigeLink = parse_url($link);
if (isset($anzeigeLink['query'])) {
parse_str($anzeigeLink['query'], $query);
if (isset($query['diurl'])) {
$anzeigeLink = $query['diurl'];
} else {
$anzeigeLink = $link;
}
} else {
$anzeigeLink = $link;
$descr = $result->xpath('//doc/str[@name="artikelBeschreibung"]')[0]->__toString();
$image = $result->xpath('//doc/str[@name="artikelImageurl"]')[0]->__toString();
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter,
false,
$image
);
}
$descr = $result->xpath('//doc/str[@name="artikelBeschreibung"]')[0]->__toString();
$image = $result->xpath('//doc/str[@name="artikelImageurl"]')[0]->__toString();
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,