Commit 73e1eef7 authored by Dominik Hebeler's avatar Dominik Hebeler

Merge branch '897-add-qualigo-as-product-search' into 'development'

Resolve "Add Qualigo as Product Search"

Closes #897

See merge request !1458
parents 0a17d5ec 429749f8
<?php <?php
namespace App; namespace App;
use App; use App;
...@@ -121,7 +122,6 @@ class MetaGer ...@@ -121,7 +122,6 @@ class MetaGer
$this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results."; $this->redisEngineResult = $redisPrefix . "." . $this->searchUid . ".results.";
# A list of all search results already delivered to the user (sorted of course) # A list of all search results already delivered to the user (sorted of course)
$this->redisCurrentResultList = $redisPrefix . "." . $this->searchUid . ".currentResults"; $this->redisCurrentResultList = $redisPrefix . "." . $this->searchUid . ".currentResults";
} }
# Erstellt aus den gesammelten Ergebnissen den View # Erstellt aus den gesammelten Ergebnissen den View
...@@ -284,9 +284,9 @@ class MetaGer ...@@ -284,9 +284,9 @@ class MetaGer
$newResults = []; $newResults = [];
foreach ($this->ads as $ad) { foreach ($this->ads as $ad) {
if (($ad->strippedHost !== "" && (in_array($ad->strippedHost, $this->adDomainsBlacklisted) || if (($ad->strippedHost !== "" && (in_array($ad->strippedHost, $this->adDomainsBlacklisted) ||
in_array($ad->strippedLink, $this->adUrlsBlacklisted))) || in_array($ad->strippedLink, $this->adUrlsBlacklisted))) || ($ad->strippedHostAnzeige !== "" && (in_array($ad->strippedHostAnzeige, $this->adDomainsBlacklisted) ||
($ad->strippedHostAnzeige !== "" && (in_array($ad->strippedHostAnzeige, $this->adDomainsBlacklisted) || in_array($ad->strippedLinkAnzeige, $this->adUrlsBlacklisted)))
in_array($ad->strippedLinkAnzeige, $this->adUrlsBlacklisted)))) { ) {
continue; continue;
} }
$newResults[] = $ad; $newResults[] = $ad;
...@@ -324,7 +324,6 @@ class MetaGer ...@@ -324,7 +324,6 @@ class MetaGer
} else { } else {
$this->next = []; $this->next = [];
} }
} }
public function combineResults($engines) public function combineResults($engines)
...@@ -436,11 +435,12 @@ class MetaGer ...@@ -436,11 +435,12 @@ class MetaGer
$postdata = http_build_query(array( $postdata = http_build_query(array(
'dummy' => rand(), 'dummy' => rand(),
)); ));
$opts = array('http' => array( $opts = array(
'method' => 'POST', 'http' => array(
'header' => 'Content-type: application/x-www-form-urlencoded', 'method' => 'POST',
'content' => $postdata, 'header' => 'Content-type: application/x-www-form-urlencoded',
), 'content' => $postdata,
),
); );
$context = stream_context_create($opts); $context = stream_context_create($opts);
...@@ -453,7 +453,6 @@ class MetaGer ...@@ -453,7 +453,6 @@ class MetaGer
} else { } else {
return false; return false;
} }
} catch (\ErrorException $e) { } catch (\ErrorException $e) {
return false; return false;
} }
...@@ -485,50 +484,53 @@ class MetaGer ...@@ -485,50 +484,53 @@ class MetaGer
$this->fokus = "web"; $this->fokus = "web";
} }
$sumaList = $this->sumaFile->foki->{$this->fokus}->sumas; $sumaNames = $this->sumaFile->foki->{$this->fokus}->sumas;
# If the user is authorized to use adfree search we won't activate yahoo or yahoo-ads $sumas = [];
if ($this->apiAuthorized && ($key = array_search("yahoo", $sumaList)) !== false) { foreach ($sumaNames as $sumaName) {
unset($sumaList[$key]); $sumas[$sumaName] = $this->sumaFile->sumas->{$sumaName};
if ($this->fokus === "web") {
$this->sumaFile->sumas->{"bing"}->{"filter-opt-in"} = false;
}
} elseif ($this->apiAuthorized && ($key = array_search("yahoo-ads", $sumaList)) !== false) {
unset($sumaList[$key]);
} }
foreach ($sumaList as $suma) { $this->removeAdsFromListIfAdfree($sumas);
foreach ($sumas as $sumaName => $suma) {
# Check if this engine is disabled and can't be used # Check if this engine is disabled and can't be used
$disabled = empty($this->sumaFile->sumas->{$suma}->disabled) ? false : $this->sumaFile->sumas->{$suma}->disabled; $disabled = empty($suma->disabled) ? false : $suma->disabled;
$autoDisabled = empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) ? false : $this->sumaFile->sumas->{$suma}->{"auto-disabled"}; $autoDisabled = empty($suma->{"auto-disabled"}) ? false : $suma->{"auto-disabled"};
if ($disabled || $autoDisabled if (
|| \Cookie::get($this->getFokus() . "_engine_" . $suma) === "off") { # Check if the user has disabled this engine $disabled || $autoDisabled
continue; || \Cookie::get($this->getFokus() . "_engine_" . $sumaName) === "off"
) {
continue;
} }
# Check if this engine can use eventually defined query-filter
$valid = true; $valid = true;
foreach ($this->queryFilter as $queryFilter => $filter) {
if (empty($this->sumaFile->filter->{"query-filter"}->$queryFilter->sumas->$suma)) { # Check if this engine can use potentially defined query-filter
foreach ($this->queryFilter as $filterName => $filter) {
if (empty($this->sumaFile->filter->{"query-filter"}->$filterName->sumas->$sumaName)) {
$valid = false; $valid = false;
break; break;
} }
} }
# Check if this engine can use eventually defined parameter-filter
# Check if this engine can use potentially defined parameter-filter
if ($valid) { if ($valid) {
foreach ($this->parameterFilter as $filterName => $filter) { foreach ($this->parameterFilter as $filterName => $filter) {
# We need to check if the searchengine supports the parameter value, too # We need to check if the searchengine supports the parameter value, too
if (empty($filter->sumas->$suma) || empty($filter->sumas->{$suma}->values->{$filter->value})) { if (empty($filter->sumas->$sumaName) || empty($filter->sumas->{$sumaName}->values->{$filter->value})) {
$valid = false; $valid = false;
break; break;
} }
} }
} }
# Check if this engine should only be active when filter is used # Check if this engine should only be active when filter is used
if ($this->sumaFile->sumas->{$suma}->{"filter-opt-in"}) { if ($suma->{"filter-opt-in"}) {
# This search engine should only be used when a parameter filter of it is used # This search engine should only be used when a parameter filter of it is used
$validTmp = false; $validTmp = false;
foreach ($this->parameterFilter as $filterName => $filter) { foreach ($this->parameterFilter as $filterName => $filter) {
if (!empty($filter->sumas->{$suma})) { if (!empty($filter->sumas->{$sumaName})) {
$validTmp = true; $validTmp = true;
break; break;
} }
...@@ -536,17 +538,16 @@ class MetaGer ...@@ -536,17 +538,16 @@ class MetaGer
if (!$validTmp) { if (!$validTmp) {
$valid = false; $valid = false;
} }
} }
# If it can we add it
# If the suma is still valid, we can add it
if ($valid) { if ($valid) {
$this->enabledSearchengines[$suma] = $this->sumaFile->sumas->{$suma}; $this->enabledSearchengines[$sumaName] = $suma;
} }
} }
# Implements Yahoo Ads if Yahoo is not enabled as a searchengine # Include Yahoo Ads if Yahoo is not enabled as a searchengine
if (!$this->apiAuthorized && empty($this->enabledSearchengines["yahoo"]) && $this->fokus != "bilder" && !empty($this->sumaFile->sumas->{"yahoo-ads"})) { if (!$this->apiAuthorized && $this->fokus != "bilder" && empty($this->enabledSearchengines["yahoo"]) && isset($this->sumaFile->sumas->{"yahoo-ads"})) {
$this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"}; $this->enabledSearchengines["yahoo-ads"] = $this->sumaFile->sumas->{"yahoo-ads"};
} }
...@@ -562,16 +563,33 @@ class MetaGer ...@@ -562,16 +563,33 @@ class MetaGer
$filter .= trans($this->sumaFile->filter->{"query-filter"}->{$queryFilter}->name) . ","; $filter .= trans($this->sumaFile->filter->{"query-filter"}->{$queryFilter}->name) . ",";
} }
$filter = rtrim($filter, ","); $filter = rtrim($filter, ",");
$error = trans('metaGer.engines.noSpecialSearch', ['fokus' => trans($this->sumaFile->foki->{$this->fokus}->{"display-name"}), $error = trans('metaGer.engines.noSpecialSearch', [
'filter' => $filter]); 'fokus' => trans($this->sumaFile->foki->{$this->fokus}->{"display-name"}),
'filter' => $filter
]);
$this->errors[] = $error; $this->errors[] = $error;
} }
$engines = [];
$typeslist = [];
$counter = 0;
$this->setEngines($request); $this->setEngines($request);
} }
private function removeAdsFromListIfAdfree(&$sumas)
{
if ($this->apiAuthorized) {
foreach ($sumas as $sumaName => $suma) {
$ads = $suma->ads ?? false;
if ($ads) {
unset($sumas[$sumaName]);
$adBackups = $suma->{"ad-backups"} ?? [];
$adBackupName = $adBackups->{$this->fokus} ?? null;
if (isset($adBackupName)) {
$this->sumaFile->sumas->{$adBackupName}->{"filter-opt-in"} = false;
}
}
}
}
}
public function setEngines(Request $request, $enabledSearchengines = []) public function setEngines(Request $request, $enabledSearchengines = [])
{ {
if ($this->requestIsCached($request)) { if ($this->requestIsCached($request)) {
...@@ -676,7 +694,8 @@ class MetaGer ...@@ -676,7 +694,8 @@ class MetaGer
if (!empty($filter->sumas->{$suma})) { if (!empty($filter->sumas->{$suma})) {
# If the searchengine is disabled this filter shouldn't be available # If the searchengine is disabled this filter shouldn't be available
if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true) if ((!empty($this->sumaFile->sumas->{$suma}->disabled) && $this->sumaFile->sumas->{$suma}->disabled === true)
|| (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)) { || (!empty($this->sumaFile->sumas->{$suma}->{"auto-disabled"}) && $this->sumaFile->sumas->{$suma}->{"auto-disabled"} === true)
) {
continue; continue;
} }
if (empty($availableFilter[$filterName])) { if (empty($availableFilter[$filterName])) {
...@@ -723,31 +742,31 @@ class MetaGer ...@@ -723,31 +742,31 @@ class MetaGer
public function sumaIsDisabled($suma) public function sumaIsDisabled($suma)
{ {
return return
isset($suma['disabled']) isset($suma['disabled'])
&& $suma['disabled']->__toString() === "1"; && $suma['disabled']->__toString() === "1";
} }
public function sumaIsOverture($suma) public function sumaIsOverture($suma)
{ {
return return
$suma["name"]->__toString() === "overture" $suma["name"]->__toString() === "overture"
|| $suma["name"]->__toString() === "overtureAds"; || $suma["name"]->__toString() === "overtureAds";
} }
public function sumaIsNotAdsuche($suma) public function sumaIsNotAdsuche($suma)
{ {
return return
$suma["name"]->__toString() !== "qualigo" $suma["name"]->__toString() !== "qualigo"
&& $suma["name"]->__toString() !== "similar_product_ads" && $suma["name"]->__toString() !== "similar_product_ads"
&& $suma["name"]->__toString() !== "overtureAds"; && $suma["name"]->__toString() !== "overtureAds";
} }
public function requestIsCached($request) public function requestIsCached($request)
{ {
return return
$request->filled('next') $request->filled('next')
&& Cache::has($request->input('next')) && Cache::has($request->input('next'))
&& unserialize(Cache::get($request->input('next')))['page'] > 1; && unserialize(Cache::get($request->input('next')))['page'] > 1;
} }
public function getCachedEngines($request) public function getCachedEngines($request)
...@@ -818,7 +837,6 @@ class MetaGer ...@@ -818,7 +837,6 @@ class MetaGer
$pipeline->hset($this->getRedisEngineResult() . $engine, "delivered", "1"); $pipeline->hset($this->getRedisEngineResult() . $engine, "delivered", "1");
} }
$pipeline->execute(); $pipeline->execute();
} }
public function retrieveResults() public function retrieveResults()
...@@ -843,7 +861,7 @@ class MetaGer ...@@ -843,7 +861,7 @@ class MetaGer
} }
} }
/* /*
* Ende Suchmaschinenerstellung und Ergebniserhalt * Ende Suchmaschinenerstellung und Ergebniserhalt
*/ */
...@@ -1060,7 +1078,6 @@ class MetaGer ...@@ -1060,7 +1078,6 @@ class MetaGer
$this->q = preg_replace('/' . $toDelete . '/si', '', $this->q, 1); $this->q = preg_replace('/' . $toDelete . '/si', '', $this->q, 1);
} }
} }
} }
# Check for parameter-filter (i.e. SafeSearch) # Check for parameter-filter (i.e. SafeSearch)
$this->parameterFilter = []; $this->parameterFilter = [];
...@@ -1073,8 +1090,9 @@ class MetaGer ...@@ -1073,8 +1090,9 @@ class MetaGer
} }
if (($request->filled($filter->{"get-parameter"}) && $request->input($filter->{"get-parameter"}) !== "off") || if (($request->filled($filter->{"get-parameter"}) && $request->input($filter->{"get-parameter"}) !== "off") ||
\Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"}) !== null) { # If the filter is set via Cookie \Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"}) !== null
$this->parameterFilter[$filterName] = $filter; ) { # If the filter is set via Cookie
$this->parameterFilter[$filterName] = $filter;
$this->parameterFilter[$filterName]->value = $request->input($filter->{"get-parameter"}, ''); $this->parameterFilter[$filterName]->value = $request->input($filter->{"get-parameter"}, '');
if (empty($this->parameterFilter[$filterName]->value)) { if (empty($this->parameterFilter[$filterName]->value)) {
$this->parameterFilter[$filterName]->value = \Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"}); $this->parameterFilter[$filterName]->value = \Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"});
...@@ -1263,7 +1281,7 @@ class MetaGer ...@@ -1263,7 +1281,7 @@ class MetaGer
} }
} }
# Hilfsfunktionen # Hilfsfunktionen
public function startsWith($haystack, $needle) public function startsWith($haystack, $needle)
{ {
$length = strlen($needle); $length = strlen($needle);
...@@ -1314,8 +1332,7 @@ class MetaGer ...@@ -1314,8 +1332,7 @@ class MetaGer
{ {
if ($this->shouldLog) { if ($this->shouldLog) {
$redis = Redis::connection('redisLogs'); $redis = Redis::connection('redisLogs');
try try {
{
$logEntry = ""; $logEntry = "";
$logEntry .= "[" . date("D M d H:i:s") . "]"; $logEntry .= "[" . date("D M d H:i:s") . "]";
/* /*
...@@ -1385,7 +1402,7 @@ class MetaGer ...@@ -1385,7 +1402,7 @@ class MetaGer
} }
} }
# Generators # Generators
public function generateSearchLink($fokus, $results = true) public function generateSearchLink($fokus, $results = true)
{ {
...@@ -1455,7 +1472,7 @@ class MetaGer ...@@ -1455,7 +1472,7 @@ class MetaGer
return $link; return $link;
} }
# Komplexe Getter # Komplexe Getter
public function getHostCount($host) public function getHostCount($host)
{ {
...@@ -1484,8 +1501,10 @@ class MetaGer ...@@ -1484,8 +1501,10 @@ class MetaGer
{ {
$filters = $this->sumaFile->filter->{"parameter-filter"}; $filters = $this->sumaFile->filter->{"parameter-filter"};
foreach ($filters as $filterName => $filter) { foreach ($filters as $filterName => $filter) {
if (\Request::filled($filter->{"get-parameter"}) if (
&& \Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"}) !== \Request::input($filter->{"get-parameter"})) { \Request::filled($filter->{"get-parameter"})
&& \Cookie::get($this->getFokus() . "_setting_" . $filter->{"get-parameter"}) !== \Request::input($filter->{"get-parameter"})
) {
return true; return true;
} }
} }
...@@ -1505,7 +1524,7 @@ class MetaGer ...@@ -1505,7 +1524,7 @@ class MetaGer
return $count; return $count;
} }
# Einfache Getter # Einfache Getter
public function getVerificationId() public function getVerificationId()
{ {
......
...@@ -86,7 +86,7 @@ abstract class Searchengine ...@@ -86,7 +86,7 @@ abstract class Searchengine
} }
$this->getString = $this->generateGetString($q); $this->getString = $this->generateGetString($q);
$this->hash = md5($this->engine->host . $this->getString . $this->engine->port . $this->name); $this->updateHash();
$this->resultHash = $metager->getSearchUid(); $this->resultHash = $metager->getSearchUid();
$this->canCache = $metager->canCache(); $this->canCache = $metager->canCache();
} }
...@@ -95,9 +95,7 @@ abstract class Searchengine ...@@ -95,9 +95,7 @@ abstract class Searchengine
# Standardimplementierung der getNext Funktion, damit diese immer verwendet werden kann # Standardimplementierung der getNext Funktion, damit diese immer verwendet werden kann
public function getNext(MetaGer $metager, $result) public function getNext(MetaGer $metager, $result)
{ { }
}
# Prüft, ob die Suche bereits gecached ist, ansonsted wird sie als Job dispatched # Prüft, ob die Suche bereits gecached ist, ansonsted wird sie als Job dispatched
public function startSearch(\App\MetaGer $metager) public function startSearch(\App\MetaGer $metager)
...@@ -200,6 +198,11 @@ abstract class Searchengine ...@@ -200,6 +198,11 @@ abstract class Searchengine
$this->resultHash = $hash; $this->resultHash = $hash;
} }
public function updateHash()
{
$this->hash = md5($this->engine->host . $this->getString . $this->engine->port . $this->name);
}
# Fragt die Ergebnisse von Redis ab und lädt Sie # Fragt die Ergebnisse von Redis ab und lädt Sie
public function retrieveResults(MetaGer $metager) public function retrieveResults(MetaGer $metager)
{ {
...@@ -250,6 +253,8 @@ abstract class Searchengine ...@@ -250,6 +253,8 @@ abstract class Searchengine
# Append the Query String # Append the Query String
$getString .= "&" . $this->engine->{"query-parameter"} . "=" . $this->urlEncode($query); $getString .= "&" . $this->engine->{"query-parameter"} . "=" . $this->urlEncode($query);
$getString .= $this->getDynamicParamsString();
return $getString; return $getString;
} }
...@@ -262,4 +267,21 @@ abstract class Searchengine ...@@ -262,4 +267,21 @@ abstract class Searchengine
return urlencode($string); return urlencode($string);
} }
} }
private function getDynamicParamsString()
{
$paramString = "";
$params = $this->getDynamicParams();
foreach ($params as $key => $value) {
$paramString .= sprintf("&%s=%s", urlencode($key), urlencode($value));
}
return $paramString;
}
protected function getDynamicParams()
{
return [];
}
} }
<?php
namespace App\Models;
abstract class XmlSearchengine extends Searchengine
{
public function loadresults($results)
{
try {
$resultsXml = simplexml_load_string($results);
$this->loadXmlResults($resultsXml);
} catch (\Exception $e) {
abort(500, "\n~~~~~~~~\n$results\n~~~~~~~~\nis not a valid xml string");
}
}
protected abstract function loadXmlResults($resultsXml);
}
...@@ -14,7 +14,7 @@ class Overture extends Searchengine ...@@ -14,7 +14,7 @@ class Overture extends Searchengine
parent::__construct($name, $engine, $metager); parent::__construct($name, $engine, $metager);
# We need some Affil-Data for the advertisements # We need some Affil-Data for the advertisements
$this->getString .= $this->getOvertureAffilData($metager->getUrl()); $this->getString .= $this->getOvertureAffilData($metager->getUrl());
$this->hash = md5($this->engine->host . $this->getString . $this->engine->port . $this->name); $this->updateHash();
} }
public function loadResults($result) public function loadResults($result)
......
...@@ -2,9 +2,9 @@ ...@@ -2,9 +2,9 @@
namespace App\Models\parserSkripte; namespace App\Models\parserSkripte;
use App\Models\Searchengine; use App\Models\XmlSearchengine;
class Qualigo extends Searchengine class Qualigo extends XmlSearchengine
{ {
public function __construct($name, \StdClass $engine, \App\MetaGer $metager) public function __construct($name, \StdClass $engine, \App\MetaGer $metager)
...@@ -12,18 +12,9 @@ class Qualigo extends Searchengine ...@@ -12,18 +12,9 @@ class Qualigo extends Searchengine
parent::__construct($name, $engine, $metager); parent::__construct($name, $engine, $metager);
} }
public function loadResults($results) protected function loadXmlResults($resultsXml)
{ {
try { $results = $resultsXml->xpath('//RL/RANK');
$content = simplexml_load_string($results);
} catch (\Exception $e) {
abort(500, "$result is not a valid xml string");