Commit a19e3e93 authored by Karl Hasselbring's avatar Karl Hasselbring

Merge mit MetaGer-Redesign

parents f5dd4d92 fadb537e
......@@ -24,3 +24,5 @@ langfiles.zip
**/*.map
/.buildpath
/.project
composer.lock
\ No newline at end of file
......@@ -7,7 +7,7 @@ update(144.76.113.134):
# Abhängigkeiten überprüfen
- which composer
- which git
- which php7.0
- which php
- which sqlite3
script:
# Pfad zum neu geklonten Repo
......@@ -24,16 +24,17 @@ update(144.76.113.134):
- scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
- touch storage/logs/laravel.log
- touch storage/logs/worker.log
- touch database/metager.sqlite
- chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
- chmod -R 777 storage
- chmod -R 777 bootstrap/cache
- npm install
- npm run prodution
- php artisan migrate --force
- if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
- cd ~/
- while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
- mv MetaGer_neu MetaGer
- php ~/MetaGer/artisan migrate --force
- sudo pkill --signal SIGHUP supervisord
- php ~/MetaGer/artisan up
......@@ -46,7 +47,7 @@ update(metager2):
# Abhängigkeiten überprüfen
- which composer
- which git
- which php7.0
- which php
- which sqlite3
script:
# Pfad zum neu geklonten Repo
......@@ -63,16 +64,17 @@ update(metager2):
- scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
- touch storage/logs/laravel.log
- touch storage/logs/worker.log
- touch database/metager.sqlite
- chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
- chmod -R 777 storage
- chmod -R 777 bootstrap/cache
- npm install
- npm run prodution
- php artisan migrate --force
- if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
- cd ~/
- while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
- mv MetaGer_neu MetaGer
- php ~/MetaGer/artisan migrate --force
- sudo pkill --signal SIGHUP supervisord
- php ~/MetaGer/artisan up
......@@ -85,7 +87,7 @@ update(metager3.de):
# Abhängigkeiten überprüfen
- which composer
- which git
- which php7.0
- which php
- which sqlite3
script:
# Pfad zum neu geklonten Repo
......@@ -102,15 +104,16 @@ update(metager3.de):
- scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
- touch storage/logs/laravel.log
- touch storage/logs/worker.log
- touch database/metager.sqlite
- chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
- chmod -R 777 storage
- chmod -R 777 bootstrap/cache
- npm install
- ./gulpbuild.sh
- npm run prodution
- php artisan migrate --force
- if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
- cd ~/
- while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
- mv MetaGer_neu MetaGer
- php ~/MetaGer/artisan migrate --force
- sudo pkill --signal SIGHUP supervisord
- php ~/MetaGer/artisan up
\ No newline at end of file
......@@ -15,7 +15,11 @@ class Searcher implements ShouldQueue
use InteractsWithQueue, Queueable, SerializesModels;
protected $name, $ch, $pid, $counter, $lastTime, $connectionInfo;
# Each Searcher will shutdown after a specified time(s) or number of requests
protected $MAX_REQUESTS = 100;
# This value should always be below the retry_after value in config/queue.php
protected $MAX_TIME = 240;
protected $startTime = null;
protected $importantEngines = array("Fastbot", "overture", "overtureAds");
protected $recheck;
......@@ -35,6 +39,7 @@ class Searcher implements ShouldQueue
$this->name = $name;
$this->pid = getmypid();
$this->recheck = false;
$this->startTime = microtime(true);
// Submit this worker to the Redis System
Redis::expire($this->name, 5);
}
......@@ -92,7 +97,9 @@ class Searcher implements ShouldQueue
// In sync mode every Searcher may only retrieve one result because it would block
// the execution of the remaining code otherwise:
if(getenv("QUEUE_DRIVER") === "sync" || $this->counter > $this->MAX_REQUESTS){
if(getenv("QUEUE_DRIVER") === "sync"
|| $this->counter > $this->MAX_REQUESTS
|| (microtime(true)-$this->startTime) > $this->MAX_TIME){
break;
}
}
......
......@@ -30,7 +30,6 @@ class MetaGer
protected $engines = [];
protected $results = [];
protected $ads = [];
protected $products = [];
protected $warnings = [];
protected $errors = [];
protected $addedHosts = [];
......@@ -38,6 +37,7 @@ class MetaGer
protected $canCache = false;
# Daten über die Abfrage$
protected $ip;
protected $useragent;
protected $language;
protected $agent;
protected $apiKey = "";
......@@ -295,7 +295,6 @@ class MetaGer
if ($this->validated) {
$this->ads = [];
$this->products = [];
$this->maps = false;
}
......@@ -380,9 +379,6 @@ class MetaGer
foreach ($engine->ads as $ad) {
$this->ads[] = $ad;
}
foreach ($engine->products as $product) {
$this->products[] = $product;
}
}
}
......@@ -694,8 +690,7 @@ class MetaGer
return
$sumaName === "qualigo"
|| $sumaName === "similar_product_ads"
|| (!$overtureEnabled && $sumaName === "overtureAds")
|| $sumaName == "rlvproduct";
|| (!$overtureEnabled && $sumaName === "overtureAds");
}
public function sumaIsDisabled($suma)
......@@ -784,7 +779,7 @@ class MetaGer
$realEngNames = [];
foreach ($enabledSearchengines as $realEng) {
$nam = $realEng["name"]->__toString();
if ($nam !== "qualigo" && $nam !== "overtureAds" && $nam !== "rlvproduct") {
if ($nam !== "qualigo" && $nam !== "overtureAds") {
$realEngNames[] = $nam;
}
}
......@@ -944,6 +939,8 @@ class MetaGer
# nicht einmal wir selbst noch Zugriff auf die Daten haben:
$this->ip = preg_replace("/(\d+)\.(\d+)\.\d+.\d+/s", "$1.$2.0.0", $this->ip);
$this->useragent = $request->header('User-Agent');
# Language
if (isset($_SERVER['HTTP_LANGUAGE'])) {
$this->language = $_SERVER['HTTP_LANGUAGE'];
......@@ -1061,40 +1058,52 @@ class MetaGer
public function checkSpecialSearches(Request $request)
{
if ($request->filled('site')) {
$site = $request->input('site');
} else {
$site = "";
}
$this->searchCheckSitesearch($site);
$this->searchCheckHostBlacklist();
$this->searchCheckDomainBlacklist();
$this->searchCheckSitesearch($request);
$this->searchCheckHostBlacklist($request);
$this->searchCheckDomainBlacklist($request);
$this->searchCheckUrlBlacklist();
$this->searchCheckPhrase();
$this->searchCheckStopwords();
$this->searchCheckStopwords($request);
$this->searchCheckNoSearch();
}
private function searchCheckSitesearch($site)
private function searchCheckSitesearch($request)
{
// matches '[... ]site:test.de[ ...]'
while (preg_match("/(^|.+\s)site:(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->site = $match[2];
$this->q = $match[1] . $match[3];
}
if ($site !== "") {
$this->site = $site;
# Overwrite Setting if it's submitted via Parameter
if ($request->has('site')) {
$this->site = $request->input('site');
}
}
private function searchCheckHostBlacklist()
private function searchCheckHostBlacklist($request)
{
// matches '[... ]-site:test.de[ ...]'
while (preg_match("/(^|.+\s)-site:([^\s\*]\S*)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->hostBlacklist[] = $match[2];
$this->q = $match[1] . $match[3];
}
# Overwrite Setting if it's submitted via Parameter
if($request->has('blacklist')){
$this->hostBlacklist = [];
$blacklistString = trim($request->input('blacklist'));
if(strpos($blacklistString, ",") !== FALSE){
$blacklistArray = explode(',', $blacklistString);
foreach($blacklistArray as $blacklistElement){
$blacklistElement = trim($blacklistElement);
if(strpos($blacklistElement, "*") !== 0){
$this->hostBlacklist[] = $blacklistElement;
}
}
}else if(strpos($blacklistString, "*") !== 0){
$this->hostBlacklist[] = $blacklistString;
}
}
// print the host blacklist as a user warning
if (sizeof($this->hostBlacklist) > 0) {
$hostString = "";
......@@ -1106,13 +1115,29 @@ class MetaGer
}
}
private function searchCheckDomainBlacklist()
private function searchCheckDomainBlacklist($request)
{
// matches '[... ]-site:*.test.de[ ...]'
while (preg_match("/(^|.+\s)-site:\*\.(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->domainBlacklist[] = $match[2];
$this->q = $match[1] . $match[3];
}
# Overwrite Setting if it's submitted via Parameter
if($request->has('blacklist')){
$this->domainBlacklist = [];
$blacklistString = trim($request->input('blacklist'));
if(strpos($blacklistString, ",") !== FALSE){
$blacklistArray = explode(',', $blacklistString);
foreach($blacklistArray as $blacklistElement){
$blacklistElement = trim($blacklistElement);
if(strpos($blacklistElement, "*.") === 0){
$this->domainBlacklist[] = substr($blacklistElement, strpos($blacklistElement, "*.")+2);
}
}
}else if(strpos($blacklistString, "*.") === 0){
$this->domainBlacklist[] = substr($blacklistString, strpos($blacklistString, "*.")+2);
}
}
// print the domain blacklist as a user warning
if (sizeof($this->domainBlacklist) > 0) {
$domainString = "";
......@@ -1142,13 +1167,27 @@ class MetaGer
}
}
private function searchCheckStopwords()
private function searchCheckStopwords($request)
{
// matches '[... ]-test[ ...]'
while (preg_match("/(^|.+\s)-(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->stopWords[] = $match[2];
$this->q = $match[1] . $match[3];
}
# Overwrite Setting if submitted via Parameter
if($request->has('stop')){
$this->stopWords = [];
$stop = trim($request->input('stop'));
if(strpos($stop, ',') !== FALSE){
$stopArray = explode(',', $stop);
foreach($stopArray as $stopElement){
$stopElement = trim($stopElement);
$this->stopWords[] = $stopElement;
}
}else{
$this->stopWords[] = $stop;
}
}
// print the stopwords as a user warning
if (sizeof($this->stopWords) > 0) {
$stopwordsString = "";
......@@ -1248,24 +1287,6 @@ class MetaGer
}
}
public function hasProducts()
{
if (count($this->products) > 0) {
return true;
} else {
return false;
}
}
public function getProducts()
{
$return = [];
foreach ($this->products as $product) {
$return[] = get_object_vars($product);
}
return $return;
}
public function canCache()
{
return $this->canCache;
......@@ -1457,6 +1478,11 @@ class MetaGer
{
return $this->ip;
}
public function getUserAgent()
{
return $this->useragent;
}
public function getEingabe()
{
......
......@@ -13,7 +13,8 @@ class Result
public $anzeigeLink; # Der tatsächlich angezeigte Link (rein optisch)
public $descr; # Die eventuell gekürzte Beschreibung des Suchergebnisses
public $longDescr; # Die ungekürzte Beschreibung des Suchergebnisses
public $gefVon; # Das bei Suchergebnissen angezeigte von ... mitsamt Verlinkung
public $gefVon; # Die Suchmaschine von der dieses Ergebnis stammt
public $gefVonLink;
public $sourceRank; # Das Ranking für dieses Suchergebnis von der Seite, die es geliefert hat (implizit durch Ergebnisreihenfolge: 20 - Position in Ergebnisliste)
public $partnershop; # Ist das Ergebnis von einem Partnershop? (bool)
public $image; # Ein Vorschaubild für das Suchergebnis (als URL)
......@@ -28,7 +29,7 @@ class Result
public $rank; # Das Ranking für das Ergebnis
# Erstellt ein neues Ergebnis
public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $sourceRank, $additionalInformation = [])
public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $gefVonLink, $sourceRank, $additionalInformation = [])
{
$provider = simplexml_load_string($provider);
$this->titel = strip_tags(trim($titel));
......@@ -46,6 +47,7 @@ class Result
}
$this->gefVon = trim($gefVon);
$this->gefVonLink = trim($gefVonLink);
$this->proxyLink = $this->generateProxyLink($this->link);
$this->sourceRank = $sourceRank;
if ($this->sourceRank <= 0 || $this->sourceRank > 20) {
......
......@@ -23,7 +23,6 @@ abstract class Searchengine
public $cached = false;
public $ip; # Die IP aus der metager
public $gefVon; # Der HTML-Code für die Verlinkung des Suchanbieters
public $uses; # Die Anzahl der Nutzungen dieser Suchmaschine
public $homepage; # Die Homepage dieser Suchmaschine
public $name; # Der Name dieser Suchmaschine
......@@ -69,9 +68,8 @@ abstract class Searchengine
return;
}
$this->useragent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1";
$this->useragent = $metager->getUserAgent();
$this->ip = $metager->getIp();
$this->gefVon = $this->displayName;
$this->startTime = microtime();
# Suchstring generieren
......@@ -242,7 +240,7 @@ abstract class Searchengine
Cache::put($this->hash, $body, $this->cacheDuration);
}
}
if ($body !== "") {
if ($body !== "" && $body !== "connected" && $body !== "waiting") {
$this->loadResults($body);
$this->getNext($metager, $body);
$this->loaded = true;
......@@ -335,9 +333,6 @@ abstract class Searchengine
{
$affil_data = 'ip=' . $this->ip;
$affil_data .= '&ua=' . $this->useragent;
if (isset($_SERVER['HTTP_X_FORWARDED_FOR'])) {
$affil_data .= '&xfip=' . $_SERVER['HTTP_X_FORWARDED_FOR'];
}
$affilDataValue = $this->urlEncode($affil_data);
# Wir benötigen die ServeUrl:
$serveUrl = $this->urlEncode($url);
......
......@@ -43,7 +43,7 @@ class Allesklar extends Searchengine
$link,
$link,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter
);
} catch (\Exception $e) {
......
......@@ -49,7 +49,7 @@ class BASE extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter
);
}
......
......@@ -34,7 +34,7 @@ class Beammachine extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter
);
}
......
......@@ -32,7 +32,7 @@ class Bing extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter
);
});
......
......@@ -31,7 +31,7 @@ class Bing_bilder extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter,
['image' => $image]
);
......
......@@ -36,7 +36,7 @@ class Blogsearch extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter
);
}
......
......@@ -37,7 +37,7 @@ class Dailymotion extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter,
['partnershop' => false]
);
......
......@@ -57,7 +57,7 @@ class Dart extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter
);
}
......
......@@ -36,7 +36,7 @@ class Dmoznebel extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter
);
}
......
......@@ -28,7 +28,7 @@ class Dmoznebel_int extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter
);
}
......
......@@ -51,7 +51,7 @@ class Ebay extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter,
['partnershop' => false,
'price' => $price,
......
......@@ -49,7 +49,7 @@ class Ecoshopper extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter,
['partnershop' => false,
'image' => $image]
......
......@@ -42,7 +42,7 @@ class Europeana extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter,
['image' => $image]
);
......
......@@ -61,7 +61,7 @@ class Exalead extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter
);
} catch (\ErrorException $e) {
......
......@@ -42,7 +42,7 @@ class Fairmondo extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter,
[
'price' => $price,
......
......@@ -38,7 +38,7 @@ class Fastbot extends Searchengine
$link,
$result[3],
$result[2],
$this->gefVon,
$this->displayName,$this->homepage,
$counter
);
}
......
......@@ -38,7 +38,7 @@ class Fernsehsuche extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter,
['image' => $image]
);
......
......@@ -37,7 +37,7 @@ class Fess extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter
);
} catch (\ErrorException $e) {
......
......@@ -37,7 +37,7 @@ class Flickr extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter,
['image' => $image]
);
......
......@@ -31,7 +31,7 @@ class Goyax extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter
);
});
......
......@@ -35,7 +35,7 @@ class Loklak extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->gefVon,
$this->displayName,$this->homepage,
$this->counter
);
}
......
......@@ -34,7 +34,7 @@ class Mg_hochsch_de extends Searchengine