Skip to content

GitLab

Commit f8f4cb38 authored by Karl's avatar Karl
Browse files

Merge branch 'development' into 363-spanische-version-vorbereiten

parents 534e2a69 632598f1
Hide whitespace changes
Inline Side-by-side
app/Http/Controllers/StartpageController.php
View file @ f8f4cb38
...@@ -28,7 +28,7 @@ class StartpageController extends Controller ...@@ -28,7 +28,7 @@ class StartpageController extends Controller
$focusPages = []; $focusPages = [];
$theme = "default"; $theme = "default";
foreach ($request->all() as $key => $value) { foreach ($request->all() as $key => $value) {
if ($value === 'on' && $key != 'param_sprueche' && $key != 'param_tab' && $key !== 'param_maps') { if ($value === 'on' && $key != 'param_sprueche' && $key != 'param_tab' && $key !== 'param_maps' && $key !== 'param_autocomplete') {
$focusPages[] = str_replace('param_', '', $key); $focusPages[] = str_replace('param_', '', $key);
} }
if ($key === 'param_theme') { if ($key === 'param_theme') {
...@@ -54,7 +54,8 @@ class StartpageController extends Controller ...@@ -54,7 +54,8 @@ class StartpageController extends Controller
->with('browser', $browser) ->with('browser', $browser)
->with('navbarFocus', 'suche') ->with('navbarFocus', 'suche')
->with('theme', $theme) ->with('theme', $theme)
->with('maps', $maps); ->with('maps', $maps)
->with('autocomplete', $request->input('param_autocomplete', 'on'));
} }
public function loadPage($subpage) public function loadPage($subpage)
......
app/MetaGer.php
View file @ f8f4cb38
...@@ -140,6 +140,10 @@ class MetaGer ...@@ -140,6 +140,10 @@ class MetaGer
->with('suspendheader', "yes") ->with('suspendheader', "yes")
->with('browser', (new Agent())->browser()); ->with('browser', (new Agent())->browser());
break; break;
case 'result-count':
# Wir geben die Ergebniszahl und die benötigte Zeit zurück:
return sizeof($viewResults) . ";" . round((microtime(true) - $this->starttime), 2);
break;
default: default:
return view('metager3') return view('metager3')
->with('eingabe', $this->eingabe) ->with('eingabe', $this->eingabe)
...@@ -287,7 +291,7 @@ class MetaGer ...@@ -287,7 +291,7 @@ class MetaGer
# Wir geben jedem Ergebnis eine ID um später die Sprachcodes zuordnen zu können # Wir geben jedem Ergebnis eine ID um später die Sprachcodes zuordnen zu können
$result->id = $id; $result->id = $id;
$langStrings["result_" . $id] = utf8_encode($result->getLangString()); $langStrings["result_" . $id] = $result->getLangString();
$id++; $id++;
} }
...@@ -854,10 +858,7 @@ class MetaGer ...@@ -854,10 +858,7 @@ class MetaGer
} }
# Sucheingabe # Sucheingabe
$this->eingabe = trim($request->input('eingabe', '')); $this->eingabe = trim($request->input('eingabe', ''));
if (strlen($this->eingabe) === 0) { $this->q = strtolower($this->eingabe);
$this->warnings[] = trans('metaGer.formdata.noSearch');
}
$this->q = $this->eingabe;
# IP # IP
$this->ip = $request->ip(); $this->ip = $request->ip();
# Language # Language
...@@ -941,9 +942,21 @@ class MetaGer ...@@ -941,9 +942,21 @@ class MetaGer
} }
$this->out = $request->input('out', "html"); $this->out = $request->input('out', "html");
# Standard output format html # Standard output format html
if ($this->out !== "html" && $this->out !== "json" && $this->out !== "results" && $this->out !== "results-with-style") { if ($this->out !== "html" && $this->out !== "json" && $this->out !== "results" && $this->out !== "results-with-style" && $this->out !== "result-count") {
$this->out = "html"; $this->out = "html";
} }
# Wir schalten den Cache aus, wenn die Ergebniszahl überprüft werden soll
# => out=result-count
# Ist dieser Parameter gesetzt, so soll überprüft werden, wie viele Ergebnisse wir liefern.
# Wenn wir gecachte Ergebnisse zurück liefern würden, wäre das nicht sonderlich klug, da es dann keine Aussagekraft hätte
# ob MetaGer funktioniert (bzw. die Fetcher laufen)
# Auch ein Log sollte nicht geschrieben werden, da es am Ende ziemlich viele Logs werden könnten.
if ($this->out === "result-count") {
$this->canCache = false;
$this->shouldLog = false;
} else {
$this->shouldLog = true;
}
} }
public function checkSpecialSearches(Request $request) public function checkSpecialSearches(Request $request)
...@@ -959,15 +972,13 @@ class MetaGer ...@@ -959,15 +972,13 @@ class MetaGer
$this->searchCheckDomainBlacklist(); $this->searchCheckDomainBlacklist();
$this->searchCheckPhrase(); $this->searchCheckPhrase();
$this->searchCheckStopwords(); $this->searchCheckStopwords();
$this->searchCheckNoSearch();
if ($this->q === "") {
$this->warnings[] = trans('metaGer.formdata.noSearch');
}
} }
public function searchCheckSitesearch($site) public function searchCheckSitesearch($site)
{ {
if (preg_match("/(.*)\bsite:(\S+)(.*)/si", $this->q, $match)) { // matches '[... ]site:test.de[ ...]'
while (preg_match("/(^|.+\s)site:(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->site = $match[2]; $this->site = $match[2];
$this->q = $match[1] . $match[3]; $this->q = $match[1] . $match[3];
} }
...@@ -978,9 +989,10 @@ class MetaGer ...@@ -978,9 +989,10 @@ class MetaGer
public function searchCheckHostBlacklist() public function searchCheckHostBlacklist()
{ {
while (preg_match("/(.*)(^|\s)-host:(\S+)(.*)/si", $this->q, $match)) { // matches '[... ]-site:test.de[ ...]'
$this->hostBlacklist[] = $match[3]; while (preg_match("/(^|.+\s)-site:([^\s\*]\S*)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->q = $match[1] . $match[4]; $this->hostBlacklist[] = $match[2];
$this->q = $match[1] . $match[3];
} }
if (sizeof($this->hostBlacklist) > 0) { if (sizeof($this->hostBlacklist) > 0) {
$hostString = ""; $hostString = "";
...@@ -994,9 +1006,10 @@ class MetaGer ...@@ -994,9 +1006,10 @@ class MetaGer
public function searchCheckDomainBlacklist() public function searchCheckDomainBlacklist()
{ {
while (preg_match("/(.*)(^|\s)-domain:(\S+)(.*)/si", $this->q, $match)) { // matches '[... ]-site:*.test.de[ ...]'
$this->domainBlacklist[] = $match[3]; while (preg_match("/(^|.+\s)-site:\*\.(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->q = $match[1] . $match[4]; $this->domainBlacklist[] = $match[2];
$this->q = $match[1] . $match[3];
} }
if (sizeof($this->domainBlacklist) > 0) { if (sizeof($this->domainBlacklist) > 0) {
$domainString = ""; $domainString = "";
...@@ -1010,9 +1023,10 @@ class MetaGer ...@@ -1010,9 +1023,10 @@ class MetaGer
public function searchCheckStopwords() public function searchCheckStopwords()
{ {
while (preg_match("/(.*)(^|\s)-(\S+)(.*)/si", $this->q, $match)) { // matches '[... ]-test[ ...]'
$this->stopWords[] = $match[3]; while (preg_match("/(^|.+\s)-(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->q = $match[1] . $match[4]; $this->stopWords[] = $match[2];
$this->q = $match[1] . $match[3];
} }
if (sizeof($this->stopWords) > 0) { if (sizeof($this->stopWords) > 0) {
$stopwordsString = ""; $stopwordsString = "";
...@@ -1028,7 +1042,8 @@ class MetaGer ...@@ -1028,7 +1042,8 @@ class MetaGer
{ {
$p = ""; $p = "";
$tmp = $this->q; $tmp = $this->q;
while (preg_match("/(.*)\"(.+)\"(.*)/si", $tmp, $match)) { // matches '[... ]"test satz"[ ...]'
while (preg_match("/(^|.+\s)\"(.+)\"(?:\s(.+)|($))/si", $tmp, $match)) {
$tmp = $match[1] . $match[3]; $tmp = $match[1] . $match[3];
$this->phrases[] = strtolower($match[2]); $this->phrases[] = strtolower($match[2]);
} }
...@@ -1041,6 +1056,13 @@ class MetaGer ...@@ -1041,6 +1056,13 @@ class MetaGer
} }
} }
public function searchCheckNoSearch()
{
if ($this->q === "") {
$this->warnings[] = trans('metaGer.formdata.noSearch');
}
}
public function nextSearchLink() public function nextSearchLink()
{ {
if (isset($this->next) && isset($this->next['engines']) && count($this->next['engines']) > 0) { if (isset($this->next) && isset($this->next['engines']) && count($this->next['engines']) > 0) {
...@@ -1112,34 +1134,36 @@ class MetaGer ...@@ -1112,34 +1134,36 @@ class MetaGer
public function createLogs() public function createLogs()
{ {
$redis = Redis::connection('redisLogs'); if ($this->shouldLog) {
try $redis = Redis::connection('redisLogs');
{ try
$logEntry = ""; {
$logEntry .= "[" . date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y"))) . "]"; $logEntry = "";
$logEntry .= " pid=" . getmypid(); $logEntry .= "[" . date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y"))) . "]";
$logEntry .= " ref=" . $this->request->header('Referer'); $logEntry .= " pid=" . getmypid();
$useragent = $this->request->header('User-Agent'); $logEntry .= " ref=" . $this->request->header('Referer');
$useragent = str_replace("(", " ", $useragent); $useragent = $this->request->header('User-Agent');
$useragent = str_replace(")", " ", $useragent); $useragent = str_replace("(", " ", $useragent);
$useragent = str_replace(" ", "", $useragent); $useragent = str_replace(")", " ", $useragent);
$logEntry .= " time=" . round((microtime(true) - $this->starttime), 2) . " serv=" . $this->fokus; $useragent = str_replace(" ", "", $useragent);
$logEntry .= " search=" . $this->eingabe; $logEntry .= " time=" . round((microtime(true) - $this->starttime), 2) . " serv=" . $this->fokus;
$logEntry .= " search=" . $this->eingabe;
# 2 Arten von Logs in einem wird die Anzahl der Abfragen an eine Suchmaschine gespeichert und in der anderen
# die Anzahl, wie häufig diese Ergebnisse geliefert hat. # 2 Arten von Logs in einem wird die Anzahl der Abfragen an eine Suchmaschine gespeichert und in der anderen
$enginesToLoad = $this->enginesToLoad; # die Anzahl, wie häufig diese Ergebnisse geliefert hat.
$redis->pipeline(function ($pipe) use ($enginesToLoad, $logEntry) { $enginesToLoad = $this->enginesToLoad;
$pipe->rpush('logs.search', $logEntry); $redis->pipeline(function ($pipe) use ($enginesToLoad, $logEntry) {
foreach ($this->enginesToLoad as $name => $answered) { $pipe->rpush('logs.search', $logEntry);
$pipe->incr('logs.engines.requests.' . $name); foreach ($this->enginesToLoad as $name => $answered) {
if ($answered) { $pipe->incr('logs.engines.requests.' . $name);
$pipe->incr('logs.engines.answered.' . $name); if ($answered) {
$pipe->incr('logs.engines.answered.' . $name);
}
} }
} });
}); } catch (\Exception $e) {
} catch (\Exception $e) { return;
return; }
} }
} }
...@@ -1214,7 +1238,7 @@ class MetaGer ...@@ -1214,7 +1238,7 @@ class MetaGer
{ {
$host = urlencode($host); $host = urlencode($host);
$requestData = $this->request->except(['page', 'out', 'next']); $requestData = $this->request->except(['page', 'out', 'next']);
$requestData['eingabe'] .= " -host:$host"; $requestData['eingabe'] .= " -site:$host";
$link = action('MetaGerSearch@search', $requestData); $link = action('MetaGerSearch@search', $requestData);
return $link; return $link;
} }
...@@ -1223,7 +1247,7 @@ class MetaGer ...@@ -1223,7 +1247,7 @@ class MetaGer
{ {
$domain = urlencode($domain); $domain = urlencode($domain);
$requestData = $this->request->except(['page', 'out', 'next']); $requestData = $this->request->except(['page', 'out', 'next']);
$requestData['eingabe'] .= " -domain:$domain"; $requestData['eingabe'] .= " -site:*.$domain";
$link = action('MetaGerSearch@search', $requestData); $link = action('MetaGerSearch@search', $requestData);
return $link; return $link;
} }
......
app/Models/lang.pl
View file @ f8f4cb38
...@@ -17,7 +17,7 @@ my $json = <$fh>; ...@@ -17,7 +17,7 @@ my $json = <$fh>;
close $fh; close $fh;
# Decode the JSON String # Decode the JSON String
my $data = decode_json($json); my $data = JSON->new->utf8->decode($json);
# Wir durchlaufen den Hash: # Wir durchlaufen den Hash:
foreach my $key (keys %{$data}){ foreach my $key (keys %{$data}){
......
app/Models/parserSkripte/Allesklar.php
View file @ f8f4cb38
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
namespace App\Models\parserSkripte; namespace App\Models\parserSkripte;
use App\Models\Searchengine; use App\Models\Searchengine;
use Log;
use Symfony\Component\DomCrawler\Crawler; use Symfony\Component\DomCrawler\Crawler;
class Allesklar extends Searchengine class Allesklar extends Searchengine
...@@ -46,10 +47,10 @@ class Allesklar extends Searchengine ...@@ -46,10 +47,10 @@ class Allesklar extends Searchengine
$this->gefVon, $this->gefVon,
$this->counter $this->counter
); );
} catch (\InvalidArgumentException $e) { } catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return;
} }
}); });
} }
......
app/Models/parserSkripte/BASE.php
View file @ f8f4cb38
...@@ -15,20 +15,6 @@ class BASE extends Searchengine ...@@ -15,20 +15,6 @@ class BASE extends Searchengine
public function loadResults($result) public function loadResults($result)
{ {
return;
$title = "";
$link = "";
$anzeigeLink = $link;
$descr = "";
/*$this->counter++;
$this->results[] = new \App\Models\Result(
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter
);*/
} }
} }
app/Models/parserSkripte/Bing.php
View file @ f8f4cb38
...@@ -16,9 +16,7 @@ class Bing extends Searchengine ...@@ -16,9 +16,7 @@ class Bing extends Searchengine
public function loadResults($result) public function loadResults($result)
{ {
try {
try
{
$crawler = new Crawler($result); $crawler = new Crawler($result);
$crawler->filter('ol#b_results > li.b_algo')->each(function (Crawler $node, $i) { $crawler->filter('ol#b_results > li.b_algo')->each(function (Crawler $node, $i) {
$title = $node->filter('li h2 > a')->text(); $title = $node->filter('li h2 > a')->text();
...@@ -37,7 +35,8 @@ class Bing extends Searchengine ...@@ -37,7 +35,8 @@ class Bing extends Searchengine
$this->counter $this->counter
); );
}); });
} catch (\ErrorException $e) { } catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return; return;
} }
......
app/Models/parserSkripte/Blogsearch.php
View file @ f8f4cb38
...@@ -16,34 +16,32 @@ class Blogsearch extends Searchengine ...@@ -16,34 +16,32 @@ class Blogsearch extends Searchengine
public function loadResults($result) public function loadResults($result)
{ {
$result = preg_replace("/\r\n/si", "", $result); $result = preg_replace("/\r\n/si", "", $result);
try { try {
$content = simplexml_load_string($result); $content = simplexml_load_string($result);
if (!$content) {
return;
}
$results = $content->xpath('//xml/docs/doc');
foreach ($results as $result) {
$title = $result->{"title"}->__toString();
$link = $result->{"url"}->__toString();
$anzeigeLink = $link;
$descr = $result->{"content"}->__toString();
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter
);
}
} catch (\Exception $e) { } catch (\Exception $e) {
abort(500, "$result is not a valid xml string"); Log::error("A problem occurred parsing results from $this->name");
}
if (!$content) {
return; return;
} }
$results = $content->xpath('//xml/docs/doc');
# die(var_dump($results));
foreach ($results as $result) {
$title = $result->{"title"}->__toString();
$link = $result->{"url"}->__toString();
$anzeigeLink = $link;
$descr = $result->{"content"}->__toString();
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter
);
}
} }
} }
app/Models/parserSkripte/Dailymotion.php
View file @ f8f4cb38
...@@ -18,33 +18,33 @@ class Dailymotion extends Searchengine ...@@ -18,33 +18,33 @@ class Dailymotion extends Searchengine
$result = preg_replace("/\r\n/si", "", $result); $result = preg_replace("/\r\n/si", "", $result);
try { try {
$content = json_decode($result); $content = json_decode($result);
} catch (\Exception $e) { if (!$content) {
abort(500, "$result is not a valid xml string"); return;
} }
if (!$content) { $results = $content->list;
foreach ($results as $result) {
$title = $result->title;
$link = $result->url;
$anzeigeLink = $link;
$descr = $result->description;
$image = $result->thumbnail_240_url;
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter,
false,
$image
);
}
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name");
return; return;
} }
$results = $content->list;
foreach ($results as $result) {
$title = $result->title;
$link = $result->url;
$anzeigeLink = $link;
$descr = $result->description;
$image = $result->thumbnail_240_url;
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->counter,
false,
$image
);
}
} }
} }
app/Models/parserSkripte/Ebay.php
View file @ f8f4cb38
...@@ -15,54 +15,52 @@ class Ebay extends Searchengine ...@@ -15,54 +15,52 @@ class Ebay extends Searchengine
public function loadResults($result) public function loadResults($result)
{ {
$result = preg_replace("/\r\n/si", "", $result);
try { try {
$content = simplexml_load_string($result); $content = simplexml_load_string($result);
} catch (\Exception $e) { if (!$content) {
abort(500, "$result is not a valid xml string"); return;
}
if (!$content) {
return;
}
$results = $content;
$results = $results->{"searchResult"};
$count = 0;
foreach ($results->{"item"} as $result) {
$title = $result->{"title"}->__toString();
$link = $result->{"viewItemURL"}->__toString();
$anzeigeLink = $link;
$time = $result->{"listingInfo"}->{"endTime"}->__toString();
$time = date(DATE_RFC2822, strtotime($time));
$price = intval($result->{"sellingStatus"}->{"convertedCurrentPrice"}->__toString()) * 100;
$descr = "<p>Preis: " . $result->{"sellingStatus"}->{"convertedCurrentPrice"}->__toString() . " €</p>";
$descr .= "<p>Versandkosten: " . $result->{"shippingInfo"}->{"shippingServiceCost"}->__toString() . " €</p>";
if (isset($result->{"listingInfo"}->{"listingType"})) {
$descr .= "<p>Auktionsart: " . $result->{"listingInfo"}->{"listingType"}->__toString() . "</p>";
} }
$descr .= "<p>Auktionsende: " . $time . "</p>"; $results = $content->{"searchResult"};
if (isset($result->{"primaryCategory"}->{"categoryName"})) { $count = 0;
$descr .= "<p class=\"text-muted\">Kategorie: " . $result->{"primaryCategory"}->{"categoryName"}->__toString() . "</p>"; foreach ($results->{"item"} as $result) {
} $title = $result->{"title"}->__toString();
$link = $result->{"viewItemURL"}->__toString();
$anzeigeLink = $link;
$time = $result->{"listingInfo"}->{"endTime"}->__toString();
$time = date(DATE_RFC2822, strtotime($time));
$price = intval($result->{"sellingStatus"}->{"convertedCurrentPrice"}->__toString()) * 100;
$descr = "<p>Preis: " . $result->{"sellingStatus"}->{"convertedCurrentPrice"}->__toString() . " €</p>";
$descr .= "<p>Versandkosten: " . $result->{"shippingInfo"}->{"shippingServiceCost"}->__toString() . " €</p>";
if (isset($result->{"listingInfo"}->{"listingType"})) {
$descr .= "<p>Auktionsart: " . $result->{"listingInfo"}->{"listingType"}->__toString() . "</p>";
}
$image = $result->{"galleryURL"}->__toString(); $descr .= "<p>Auktionsende: " . $time . "</p>";
$this->counter++;