Skip to content

GitLab

Commit 12a13ff5 authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Query Filter are working switched to sumas.json

parent 81c74a85
Hide whitespace changes
Inline Side-by-side
app/Jobs/Searcher.php
View file @ 12a13ff5
......@@ -161,6 +161,7 @@ class Searcher implements ShouldQueue
// Set this URL to the Curl handle
curl_setopt($this->ch, CURLOPT_URL, $url);
$result = curl_exec($this->ch);
$this->connectionInfo = curl_getinfo($this->ch);
return $result;
}
......@@ -203,10 +204,13 @@ class Searcher implements ShouldQueue
}
if ($this->headers !== null) {
$headers = [];
foreach ($this->headers as $key => $value) {
$headers[] = $key . ":" . $value;
}
# Headers are in the Form:
# <key>:<value>;<key>:<value>
$headerArray = explode(";", $this->headers);
curl_setopt($ch, CURLOPT_HTTPHEADER, $headerArray);
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
}
return $ch;
......
app/MetaGer.php
View file @ 12a13ff5
......@@ -17,12 +17,11 @@ class MetaGer
protected $fokus;
protected $eingabe;
protected $q;
protected $category;
protected $time;
protected $page;
protected $lang;
protected $cache = "";
protected $site;
protected $time = 2000;
protected $hostBlacklist = [];
protected $domainBlacklist = [];
private $urlBlacklist = [];
......@@ -49,7 +48,6 @@ class MetaGer
protected $mobile;
protected $resultCount;
protected $sprueche;
protected $maps;
protected $newtab;
protected $domainsBlacklisted = [];
protected $urlsBlacklisted = [];
......@@ -309,7 +307,6 @@ class MetaGer
if ($this->validated) {
$this->ads = [];
$this->maps = false;
}
if (count($this->results) <= 0) {
......@@ -531,99 +528,48 @@ class MetaGer
return;
}
$xml = simplexml_load_file($this->sumaFile);
$sumas = $xml->xpath("suma");
$enabledSearchengines = [];
$overtureEnabled = false;
$sumaCount = 0;
/*
* Erstellt eine Liste mit Foki, die verfügbar sind
*/
$this->availableFoki = [];
foreach ($sumas as $suma) {
$foki = explode(",", trim($suma["type"]));
foreach ($foki as $fokus) {
if (!empty($fokus)) {
$this->availableFoki[$fokus] = "available";
}
}
# Check if selected focus is valid
if (empty($this->sumaFile->foki->{$this->fokus})) {
$this->fokus = "web";
}
$isCustomSearch = $this->startsWith($this->fokus, 'focus_');
# Im Falle einer Custom-Suche ohne mindestens einer selektierter Suchmaschine wird eine Web-Suche durchgeführt
if ($isCustomSearch && !$this->atLeastOneSearchengineSelected($request)) {
$isCustomSearch = false;
$this->fokus = 'web';
}
/* Erstellt die Liste der eingestellten Sumas
* Der einzige Unterschied bei angepasstem Suchfokus ist,
* dass nicht nach den Typen einer Suma,
* sondern den im Request mitgegebenen Typen entschieden wird.
* Ansonsten wird genau das selbe geprüft und gemacht:
* Handelt es sich um spezielle Suchmaschinen die immer an sein müssen
* Wenn es Overture ist vermerken dass Overture an ist
* Suma Zähler erhöhen
* Zu Liste hinzufügen
*/
foreach ($sumas as $suma) {
if (($this->sumaIsSelected($suma, $request, $isCustomSearch)
|| (!$this->isBildersuche()
&& $this->sumaIsAdsuche($suma, $overtureEnabled)))
&& (!$this->sumaIsDisabled($suma))) {
if ($this->sumaIsOverture($suma)) {
$overtureEnabled = true;
}
if ($this->sumaIsNotAdsuche($suma)) {
$sumaCount += 1;
}
$enabledSearchengines[] = $suma;
foreach ($this->sumaFile->foki->{$this->fokus}->sumas as $suma) {
# Check if this engine is disabled and can't be used
$disabled = empty($suma->disabled) ? false : $suma->disabled;
if ($disabled) {
continue;
}
}
# Sonderregelung für alle Suchmaschinen, die zu den Minisuchern gehören. Diese können alle gemeinsam über einen Link abgefragt werden
$subcollections = [];
$tmp = [];
// Es gibt den Schalter "minism=on" Dieser soll bewirken, dass alle Minisucher angeschaltet werden.
// Wenn also "minism=on" ist, dann durchsuchen wir statt den tatsächlich angeschalteten Suchmaschinen,
// alle Suchmaschinen nach "minismCollection"
if ($request->input("minism", "off") === "on") {
// Wir laden alle Minisucher
foreach ($sumas as $engine) {
if (isset($engine["minismCollection"])) {
$subcollections[] = $engine["minismCollection"]->__toString();
}
}
# Nur noch alle eventuell angeschalteten Minisucher deaktivieren
foreach ($enabledSearchengines as $index => $engine) {
if (!isset($engine["minismCollection"])) {
$tmp[] = $engine;
# Check if this engine can use eventually defined query-filter
$valid = true;
foreach ($this->queryFilter as $queryFilter => $filter) {
if (empty($this->sumaFile->filter->$queryFilter->sumas->$suma)) {
$valid = false;
break;
}
}
} else {
// Wir schalten eine Teilmenge, oder aber gar keine an
foreach ($enabledSearchengines as $engine) {
if (isset($engine['minismCollection'])) {
$subcollections[] = $engine['minismCollection']->__toString();
} else {
$tmp[] = $engine;
}
# If it can we add it
if ($valid) {
$enabledSearchengines[$suma] = $this->sumaFile->sumas->{$suma};
}
}
$enabledSearchengines = $tmp;
if (sizeof($subcollections) > 0) {
$enabledSearchengines[] = $this->loadMiniSucher($xml, $subcollections);
}
if ($sumaCount <= 0) {
$this->errors[] = trans('metaGer.settings.noneSelected');
if (sizeof($enabledSearchengines) === 0) {
$filter = "";
foreach ($this->queryFilter as $queryFilter => $filterPhrase) {
$filter .= trans($this->sumaFile->filter->{$queryFilter}->name) . ",";
}
$filter = rtrim($filter, ",");
$error = trans('metaGer.engines.noSpecialSearch', ['fokus' => trans($this->sumaFile->foki->{$this->fokus}->{"display-name"}),
'filter' => $filter]);
$this->errors[] = $error;
}
$engines = [];
# Wenn eine Sitesearch durchgeführt werden soll, überprüfen wir ob überhaupt eine der Suchmaschinen eine Sitesearch unterstützt
$siteSearchFailed = $this->checkCanNotSitesearch($enabledSearchengines);
$typeslist = [];
$counter = 0;
......@@ -635,9 +581,8 @@ class MetaGer
$engine->setResultHash($this->getHashCode());
}
} else {
$engines = $this->actuallyCreateSearchEngines($enabledSearchengines, $siteSearchFailed);
$engines = $this->actuallyCreateSearchEngines($enabledSearchengines);
}
# Wir starten alle Suchen
foreach ($engines as $engine) {
$engine->startSearch($this);
......@@ -684,50 +629,34 @@ class MetaGer
return false;
}
public function actuallyCreateSearchEngines($enabledSearchengines, $siteSearchFailed)
public function actuallyCreateSearchEngines($enabledSearchengines)
{
$engines = [];
foreach ($enabledSearchengines as $engine) {
foreach ($enabledSearchengines as $engineName => $engine) {
# Wenn diese Suchmaschine gar nicht eingeschaltet sein soll
if (!$siteSearchFailed
&& strlen($this->site) > 0
&& (!isset($engine['hasSiteSearch'])
|| $engine['hasSiteSearch']->__toString() === "0")) {
continue;
}
if (!isset($engine["package"])) {
if (!isset($engine->{"parser-class"})) {
die(var_dump($engine));
}
# Setze Pfad zu Parser
$path = "App\Models\parserSkripte\\" . ucfirst($engine["package"]->__toString());
$path = "App\\Models\\parserSkripte\\" . $engine->{"parser-class"};
# Prüfe ob Parser vorhanden
if (!file_exists(app_path() . "/Models/parserSkripte/" . ucfirst($engine["package"]->__toString()) . ".php")) {
Log::error("Konnte " . $engine["name"] . " nicht abfragen, da kein Parser existiert");
$this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine["name"]]);
if (!file_exists(app_path() . "/Models/parserSkripte/" . $engine->{"parser-class"} . ".php")) {
Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen, da kein Parser existiert");
$this->errors[] = trans('metaGer.engines.noParser', ['engine' => $engine->{"display-name"}]);
continue;
}
# Es wird versucht die Suchengine zu erstellen
$time = microtime();
try {
$tmp = new $path($engine, $this);
$tmp = new $path($engineName, $engine, $this);
} catch (\ErrorException $e) {
Log::error("Konnte " . $engine["name"] . " nicht abfragen. " . var_dump($e));
Log::error("Konnte " . $engine->{"display-name"} . " nicht abfragen. " . var_dump($e));
continue;
}
# Ausgabe bei Debug-Modus
if ($tmp->enabled && isset($this->debug)) {
$this->warnings[] = $tmp->service . " Connection_Time: " . $tmp->connection_time . " Write_Time: " . $tmp->write_time . " Insgesamt:" . ((microtime() - $time) / 1000);
}
# Wenn die neu erstellte Engine eingeschaltet ist, wird sie der Liste hinzugefügt
if ($tmp->isEnabled()) {
$engines[] = $tmp;
}
$engines[] = $tmp;
}
return $engines;
}
......@@ -790,16 +719,6 @@ class MetaGer
return $engines;
}
public function loadMiniSucher($xml, $subcollections)
{
$minisucherEngine = $xml->xpath('suma[@name="minism"]')[0];
$minisucherEngine["subcollections"] = implode(", ", $subcollections);
$subcollectionsString = urlencode("(" . implode(" OR ", $subcollections) . ")");
$minisucherEngine["formData"] = str_replace("<<SUBCOLLECTIONS>>", $subcollectionsString, $minisucherEngine["formData"]);
$minisucherEngine["formData"] = str_replace("<<COUNT>>", sizeof($subcollections) * 10, $minisucherEngine["formData"]);
return $minisucherEngine;
}
# Passt den Suchfokus an, falls für einen Fokus genau alle vorhandenen Sumas eingeschaltet sind
public function adjustFocus($sumas, $enabledSearchengines)
{
......@@ -864,26 +783,6 @@ class MetaGer
}
}
public function checkCanNotSitesearch($enabledSearchengines)
{
if (strlen($this->site) > 0) {
$enginesWithSite = 0;
foreach ($enabledSearchengines as $engine) {
if (isset($engine['hasSiteSearch']) && $engine['hasSiteSearch']->__toString() === "1") {
$enginesWithSite++;
}
}
if ($enginesWithSite === 0) {
$this->errors[] = trans('metaGer.sitesearch.failed', ['site' => $this->site, 'searchLink' => $this->generateSearchLink("web", false)]);
return true;
} else {
$this->warnings[] = trans('metaGer.sitesearch.success', ['site' => $this->site]);
return false;
}
}
return false;
}
public function waitForResults($enginesToLoad, $overtureEnabled, $canBreak)
{
......@@ -977,12 +876,14 @@ class MetaGer
$this->fokus = $request->input('focus', 'web');
# Suma-File
if (App::isLocale("en")) {
$this->sumaFile = config_path() . "/sumasEn.xml";
$this->sumaFile = config_path() . "/sumasEn.json";
} else {
$this->sumaFile = config_path() . "/sumas.xml";
$this->sumaFile = config_path() . "/sumas.json";
}
if (!file_exists($this->sumaFile)) {
die(trans('metaGer.formdata.cantLoad'));
} else {
$this->sumaFile = json_decode(file_get_contents($this->sumaFile));
}
# Sucheingabe
$this->eingabe = trim($request->input('eingabe', ''));
......@@ -1001,10 +902,7 @@ class MetaGer
} else {
$this->language = "";
}
# Category
$this->category = $request->input('category', '');
# Request Times
$this->time = $request->input('time', 1500);
# Page
$this->page = 1;
# Lang
......@@ -1022,12 +920,7 @@ class MetaGer
} else {
$this->sprueche = false;
}
$this->maps = $request->input('maps', 'off');
if ($this->maps === "on") {
$this->maps = true;
} else {
$this->maps = false;
}
$this->newtab = $request->input('newtab', 'on');
if ($this->newtab === "on") {
$this->newtab = "_blank";
......@@ -1038,14 +931,7 @@ class MetaGer
$this->theme = preg_replace("/[^[:alnum:][:space:]]/u", '', $request->input('theme', 'default'));
# Ergebnisse pro Seite:
$this->resultCount = $request->input('resultCount', '20');
# Manchmal müssen wir Parameter anpassen um den Sucheinstellungen gerecht zu werden:
if ($request->filled('dart')) {
$this->time = 10000;
$this->warnings[] = trans('metaGer.formdata.dartEurope');
}
if ($this->time <= 500 || $this->time > 20000) {
$this->time = 1000;
}
if ($request->filled('minism') && ($request->filled('fportal') || $request->filled('harvest'))) {
$input = $request->all();
$newInput = [];
......@@ -1075,6 +961,7 @@ class MetaGer
$this->quicktips = true;
}
$this->queryFilter = [];
$this->verificationId = $request->input('verification_id', null);
$this->verificationCount = intval($request->input('verification_count', '0'));
$this->apiKey = $request->input('key', '');
......@@ -1117,7 +1004,29 @@ class MetaGer
public function checkSpecialSearches(Request $request)
{
$this->searchCheckPhrase();
$this->searchCheckSitesearch($request);
# Check for query-filter (i.e. Sitesearch, etc.):
foreach ($this->sumaFile->filter as $filterName => $filter) {
if ($filter->type !== "query-filter") {
continue;
}
if (!empty($filter->{"optional-parameter"}) && $request->filled($filter->{"optional-parameter"})) {
$this->queryFilter[$filterName] = $request->input($filter->{"optional-parameter"});
} else if (preg_match_all("/" . $filter->regex . "/si", $this->q, $matches) > 0) {
switch ($filter->match) {
case "last":
$this->queryFilter[$filterName] = $matches[$filter->save][sizeof($matches[$filter->save]) - 1];
$toDelete = preg_quote($matches[$filter->delete][sizeof($matches[$filter->delete]) - 1], "/");
$this->q = preg_replace('/(' . $toDelete . '(?!.*' . $toDelete . '))/si', '', $this->q);
break;
default: # First occurence
$this->queryFilter[$filterName] = $matches[$filter->save][0];
$toDelete = preg_quote($matches[$filter->delete][0], "/");
$this->q = preg_replace('/' . $toDelete . '/si', '', $this->q, 1);
}
}
}
$this->searchCheckHostBlacklist($request);
$this->searchCheckDomainBlacklist($request);
$this->searchCheckUrlBlacklist();
......@@ -1143,19 +1052,6 @@ class MetaGer
}
}
private function searchCheckSitesearch($request)
{
// matches '[... ]site:test.de[ ...]'
while (preg_match("/(^|.*?\s)site:(\S+)(\s.*|$)/si", $this->q, $match)) {
$this->site = $match[2];
$this->q = $match[1] . $match[3];
}
# Overwrite Setting if it's submitted via Parameter
if ($request->has('site')) {
$this->site = $request->input('site');
}
}
private function searchCheckHostBlacklist($request)
{
// matches '[... ]-site:test.de[ ...]'
......@@ -1573,10 +1469,6 @@ class MetaGer
{
return $this->url;
}
public function getTime()
{
return $this->time;
}
public function getLanguage()
{
......@@ -1598,16 +1490,6 @@ class MetaGer
return $this->sprueche;
}
public function getMaps()
{
return $this->maps;
}
public function getCategory()
{
return $this->category;
}
public function getPhrases()
{
return $this->phrases;
......@@ -1622,6 +1504,16 @@ class MetaGer
return $this->sumaFile;
}
public function getQueryFilter()
{
return $this->queryFilter;
}
public function getTime()
{
return $this->time;
}
public function getUserHostBlacklist()
{
return $this->hostBlacklist;
......
app/Models/Result.php
View file @ 12a13ff5
......@@ -31,7 +31,6 @@ class Result
# Erstellt ein neues Ergebnis
public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $gefVonLink, $sourceRank, $additionalInformation = [])
{
$provider = simplexml_load_string($provider);
$this->titel = strip_tags(trim($titel));
$this->link = trim($link);
$this->anzeigeLink = trim($anzeigeLink);
......@@ -52,8 +51,8 @@ class Result
$this->sourceRank = 20;
}
$this->sourceRank = 20 - $this->sourceRank;
if (isset($provider["engineBoost"])) {
$this->engineBoost = floatval($provider["engineBoost"]->__toString());
if (isset($provider->{"engine-boost"})) {
$this->engineBoost = floatval($provider->{"engine-boost"});
} else {
$this->engineBoost = 1;
}
......@@ -275,22 +274,6 @@ class Result
}
}
*/
/* Der Host-Filter der sicherstellt,
* dass von jedem Host maximal 3 Links angezeigt werden.
* Diese Überprüfung führen wir unter bestimmten Bedingungen nicht durch.
*/
if (($metager->getSite() === "" || $metager->getSite() === null) &&
strpos($this->strippedHost, "ncbi.nlm.nih.gov") === false &&
strpos($this->strippedHost, "twitter.com") === false &&
strpos($this->strippedHost, "www.ladenpreis.net") === false &&
strpos($this->strippedHost, "ncbi.nlm.nih.gov") === false &&
strpos($this->strippedHost, "www.onenewspage.com") === false &&
$this->gefVon !== "Shopzilla") {
$count = $metager->getHostCount($this->strippedHost);
if ($count >= 3) {
return false;
}
}
/* Der Dublettenfilter, der sicher stellt,
* dass wir nach Möglichkeit keinen Link doppelt in der Ergebnisliste haben.
......
app/Models/Searchengine.php
View file @ 12a13ff5
......@@ -12,10 +12,8 @@ abstract class Searchengine
{
use DispatchesJobs;
public $ch; # Curl Handle zum erhalten der Ergebnisse
public $getString = ""; # Der String für die Get-Anfrage
public $engine; # Die ursprüngliche Engine XML
public $enabled = true; # true, wenn die Suchmaschine nicht explizit disabled ist
public $results = []; # Die geladenen Ergebnisse
public $ads = []; # Die geladenen Werbungen
public $products = []; # Die geladenen Produkte
......@@ -31,7 +29,7 @@ abstract class Searchengine
public $startTime; # Die Zeit der Erstellung dieser Suchmaschine
public $hash; # Der Hash-Wert dieser Suchmaschine
private $user; # Username für HTTP-Auth (falls angegeben)
private $username; # Username für HTTP-Auth (falls angegeben)
private $password; # Passwort für HTTP-Auth (falls angegeben)
private $headers; # Headers to add
......@@ -42,58 +40,40 @@ abstract class Searchengine
public $write_time = 0; # Wird eventuell für Artefakte benötigt
public $connection_time = 0; # Wird eventuell für Artefakte benötigt
public function __construct(\SimpleXMLElement $engine, MetaGer $metager)
public function __construct($name, \stdClass $engine, MetaGer $metager)
{
# Versucht möglichst viele attribute aus dem engine XML zu laden
foreach ($engine->attributes() as $key => $value) {
$this->$key = $value->__toString();
}
# Standardhomepage metager.de
if (!isset($this->homepage)) {
$this->homepage = "https://metager.de";
}
# Speichert die XML der Engine
$this->engine = $engine->asXML();
$this->engine = $engine;
$this->name = $name;
# Cache Standarddauer 60
if (!isset($this->cacheDuration)) {
$this->cacheDuration = 60;
}
$this->enabled = true;
# Eine Suchmaschine kann automatisch temporär deaktiviert werden, wenn es Verbindungsprobleme gab:
if (isset($this->disabled) && strtotime($this->disabled) <= time()) {
# In diesem Fall ist der Timeout der Suchmaschine abgelaufen.
$this->enable($metager->getSumaFile(), "Die Suchmaschine " . $this->name . " wurde wieder eingeschaltet.");
} elseif (isset($this->disabled) && strtotime($this->disabled) > time()) {
$this->enabled = false;
return;
}
$this->useragent = $metager->getUserAgent();
$this->ip = $metager->getIp();
$this->startTime = microtime();
# check for http Auth
if (!empty($this->engine->{"http-auth-credentials"}->username) && !empty($this->engine->{"http-auth-credentials"}->password)) {
$this->username = $this->engine->{"http-auth-credentials"}->username;
$this->password = $this->engine->{"http-auth-credentials"}->password;
}
# Suchstring generieren
$q = "";
if (isset($this->hasSiteSearch) && $this->hasSiteSearch === "1") {
if (strlen($metager->getSite()) === 0) {
$q = $metager->getQ();
} else {
$q = $metager->getQ() . " site:" . $metager->getSite();
}
$this->headers = $this->engine->{"request-header"};
} else {
$q = $metager->getQ();
# Suchstring generieren
$q = $metager->getQ();
$filters = $metager->getSumaFile()->filter;
foreach ($metager->getQueryFilter() as $queryFilter => $filter) {
$filterOptions = $filters->$queryFilter;
$filterOptionsEngine = $filterOptions->sumas->{$this->name};
$query = $filterOptionsEngine->prefix . $filter . $filterOptionsEngine->suffix;
$q = $query . " " . $q;
}
$this->getString = $this->generateGetString($q, $metager->getUrl(), $metager->getLanguage(), $metager->getCategory());
$this->hash = md5($this->host . $this->getString . $this->port . $this->name);
$this->getString = $this->generateGetString($q);
$this->hash = md5($this->engine->host . $this->getString . $this->engine->port . $this->name);
$this->resultHash = $metager->getHashCode();
$this->canCache = $metager->canCache();
if (!isset($this->additionalHeaders)) {$this->additionalHeaders = "";}
}
abstract public function loadResults($result);
......@@ -107,6 +87,7 @@ abstract class Searchengine
# Prüft, ob die Suche bereits gecached ist, ansonsted wird sie als Job dispatched
public function startSearch(\App\MetaGer $metager)
{
if ($this->canCache && Cache::has($this->hash)) {
$this->cached = true;
$this->retrieveResults($metager);
......@@ -118,18 +99,20 @@ abstract class Searchengine
// <ResultHash>;<URL to fetch>
// With <ResultHash> being the Hash Value where the fetcher will store the result.
// and <URL to fetch> being the full URL to the searchengine
$url = "";
if ($this->port === "443") {
if ($this->engine->port === 443) {
$url = "https://";
} else {
$url = "http://";
}
$url .= $this->host;
if ($this->port !== 80 && $this->port !== 443) {
$url .= ":" . $this->port;