Commit e5aa5b61 authored by Karl Hasselbring's avatar Karl Hasselbring

Merge branch 'development' into MetaGer-Redesign

Inklusive Folgereparaturen
parents 25a4d3b7 7166dbfb
......@@ -7,7 +7,7 @@ update(144.76.113.134):
# Abhängigkeiten überprüfen
- which composer
- which git
- which php7.0
- which php
- which sqlite3
script:
# Pfad zum neu geklonten Repo
......@@ -24,16 +24,17 @@ update(144.76.113.134):
- scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
- touch storage/logs/laravel.log
- touch storage/logs/worker.log
- touch database/metager.sqlite
- chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
- chmod -R 777 storage
- chmod -R 777 bootstrap/cache
- npm install
- npm run prodution
- php artisan migrate --force
- if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
- cd ~/
- while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
- mv MetaGer_neu MetaGer
- php ~/MetaGer/artisan migrate --force
- sudo pkill --signal SIGHUP supervisord
- php ~/MetaGer/artisan up
......@@ -46,7 +47,7 @@ update(metager2):
# Abhängigkeiten überprüfen
- which composer
- which git
- which php7.0
- which php
- which sqlite3
script:
# Pfad zum neu geklonten Repo
......@@ -63,16 +64,17 @@ update(metager2):
- scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
- touch storage/logs/laravel.log
- touch storage/logs/worker.log
- touch database/metager.sqlite
- chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
- chmod -R 777 storage
- chmod -R 777 bootstrap/cache
- npm install
- npm run prodution
- php artisan migrate --force
- if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
- cd ~/
- while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
- mv MetaGer_neu MetaGer
- php ~/MetaGer/artisan migrate --force
- sudo pkill --signal SIGHUP supervisord
- php ~/MetaGer/artisan up
......@@ -85,7 +87,7 @@ update(metager3.de):
# Abhängigkeiten überprüfen
- which composer
- which git
- which php7.0
- which php
- which sqlite3
script:
# Pfad zum neu geklonten Repo
......@@ -102,15 +104,16 @@ update(metager3.de):
- scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
- touch storage/logs/laravel.log
- touch storage/logs/worker.log
- touch database/metager.sqlite
- chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
- chmod -R 777 storage
- chmod -R 777 bootstrap/cache
- npm install
- ./gulpbuild.sh
- npm run prodution
- php artisan migrate --force
- if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
- cd ~/
- while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
- mv MetaGer_neu MetaGer
- php ~/MetaGer/artisan migrate --force
- sudo pkill --signal SIGHUP supervisord
- php ~/MetaGer/artisan up
\ No newline at end of file
......@@ -15,7 +15,11 @@ class Searcher implements ShouldQueue
use InteractsWithQueue, Queueable, SerializesModels;
protected $name, $ch, $pid, $counter, $lastTime, $connectionInfo;
# Each Searcher will shutdown after a specified time(s) or number of requests
protected $MAX_REQUESTS = 100;
# This value should always be below the retry_after value in config/queue.php
protected $MAX_TIME = 240;
protected $startTime = null;
protected $importantEngines = array("Fastbot", "overture", "overtureAds");
protected $recheck;
......@@ -35,6 +39,7 @@ class Searcher implements ShouldQueue
$this->name = $name;
$this->pid = getmypid();
$this->recheck = false;
$this->startTime = microtime(true);
// Submit this worker to the Redis System
Redis::expire($this->name, 5);
}
......@@ -92,7 +97,9 @@ class Searcher implements ShouldQueue
// In sync mode every Searcher may only retrieve one result because it would block
// the execution of the remaining code otherwise:
if(getenv("QUEUE_DRIVER") === "sync" || $this->counter > $this->MAX_REQUESTS){
if(getenv("QUEUE_DRIVER") === "sync"
|| $this->counter > $this->MAX_REQUESTS
|| (microtime(true)-$this->startTime) > $this->MAX_TIME){
break;
}
}
......
......@@ -30,7 +30,6 @@ class MetaGer
protected $engines = [];
protected $results = [];
protected $ads = [];
protected $products = [];
protected $warnings = [];
protected $errors = [];
protected $addedHosts = [];
......@@ -38,6 +37,7 @@ class MetaGer
protected $canCache = false;
# Daten über die Abfrage$
protected $ip;
protected $useragent;
protected $language;
protected $agent;
protected $apiKey = "";
......@@ -295,7 +295,6 @@ class MetaGer
if ($this->validated) {
$this->ads = [];
$this->products = [];
$this->maps = false;
}
......@@ -380,9 +379,6 @@ class MetaGer
foreach ($engine->ads as $ad) {
$this->ads[] = $ad;
}
foreach ($engine->products as $product) {
$this->products[] = $product;
}
}
}
......@@ -944,6 +940,8 @@ class MetaGer
# nicht einmal wir selbst noch Zugriff auf die Daten haben:
$this->ip = preg_replace("/(\d+)\.(\d+)\.\d+.\d+/s", "$1.$2.0.0", $this->ip);
$this->useragent = $request->header('User-Agent');
# Language
if (isset($_SERVER['HTTP_LANGUAGE'])) {
$this->language = $_SERVER['HTTP_LANGUAGE'];
......@@ -1061,40 +1059,52 @@ class MetaGer
public function checkSpecialSearches(Request $request)
{
if ($request->filled('site')) {
$site = $request->input('site');
} else {
$site = "";
}
$this->searchCheckSitesearch($site);
$this->searchCheckHostBlacklist();
$this->searchCheckDomainBlacklist();
$this->searchCheckSitesearch($request);
$this->searchCheckHostBlacklist($request);
$this->searchCheckDomainBlacklist($request);
$this->searchCheckUrlBlacklist();
$this->searchCheckPhrase();
$this->searchCheckStopwords();
$this->searchCheckStopwords($request);
$this->searchCheckNoSearch();
}
private function searchCheckSitesearch($site)
private function searchCheckSitesearch($request)
{
// matches '[... ]site:test.de[ ...]'
while (preg_match("/(^|.+\s)site:(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->site = $match[2];
$this->q = $match[1] . $match[3];
}
if ($site !== "") {
$this->site = $site;
# Overwrite Setting if it's submitted via Parameter
if ($request->has('site')) {
$this->site = $request->input('site');
}
}
private function searchCheckHostBlacklist()
private function searchCheckHostBlacklist($request)
{
// matches '[... ]-site:test.de[ ...]'
while (preg_match("/(^|.+\s)-site:([^\s\*]\S*)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->hostBlacklist[] = $match[2];
$this->q = $match[1] . $match[3];
}
# Overwrite Setting if it's submitted via Parameter
if($request->has('blacklist')){
$this->hostBlacklist = [];
$blacklistString = trim($request->input('blacklist'));
if(strpos($blacklistString, ",") !== FALSE){
$blacklistArray = explode(',', $blacklistString);
foreach($blacklistArray as $blacklistElement){
$blacklistElement = trim($blacklistElement);
if(strpos($blacklistElement, "*") !== 0){
$this->hostBlacklist[] = $blacklistElement;
}
}
}else if(strpos($blacklistString, "*") !== 0){
$this->hostBlacklist[] = $blacklistString;
}
}
// print the host blacklist as a user warning
if (sizeof($this->hostBlacklist) > 0) {
$hostString = "";
......@@ -1106,13 +1116,29 @@ class MetaGer
}
}
private function searchCheckDomainBlacklist()
private function searchCheckDomainBlacklist($request)
{
// matches '[... ]-site:*.test.de[ ...]'
while (preg_match("/(^|.+\s)-site:\*\.(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->domainBlacklist[] = $match[2];
$this->q = $match[1] . $match[3];
}
# Overwrite Setting if it's submitted via Parameter
if($request->has('blacklist')){
$this->domainBlacklist = [];
$blacklistString = trim($request->input('blacklist'));
if(strpos($blacklistString, ",") !== FALSE){
$blacklistArray = explode(',', $blacklistString);
foreach($blacklistArray as $blacklistElement){
$blacklistElement = trim($blacklistElement);
if(strpos($blacklistElement, "*.") === 0){
$this->domainBlacklist[] = substr($blacklistElement, strpos($blacklistElement, "*.")+2);
}
}
}else if(strpos($blacklistString, "*.") === 0){
$this->domainBlacklist[] = substr($blacklistString, strpos($blacklistString, "*.")+2);
}
}
// print the domain blacklist as a user warning
if (sizeof($this->domainBlacklist) > 0) {
$domainString = "";
......@@ -1142,13 +1168,27 @@ class MetaGer
}
}
private function searchCheckStopwords()
private function searchCheckStopwords($request)
{
// matches '[... ]-test[ ...]'
while (preg_match("/(^|.+\s)-(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->stopWords[] = $match[2];
$this->q = $match[1] . $match[3];
}
# Overwrite Setting if submitted via Parameter
if($request->has('stop')){
$this->stopWords = [];
$stop = trim($request->input('stop'));
if(strpos($stop, ',') !== FALSE){
$stopArray = explode(',', $stop);
foreach($stopArray as $stopElement){
$stopElement = trim($stopElement);
$this->stopWords[] = $stopElement;
}
}else{
$this->stopWords[] = $stop;
}
}
// print the stopwords as a user warning
if (sizeof($this->stopWords) > 0) {
$stopwordsString = "";
......@@ -1248,24 +1288,6 @@ class MetaGer
}
}
public function hasProducts()
{
if (count($this->products) > 0) {
return true;
} else {
return false;
}
}
public function getProducts()
{
$return = [];
foreach ($this->products as $product) {
$return[] = get_object_vars($product);
}
return $return;
}
public function canCache()
{
return $this->canCache;
......@@ -1457,6 +1479,11 @@ class MetaGer
{
return $this->ip;
}
public function getUserAgent()
{
return $this->useragent;
}
public function getEingabe()
{
......
......@@ -68,7 +68,7 @@ abstract class Searchengine
return;
}
$this->useragent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1";
$this->useragent = $metager->getUserAgent();
$this->ip = $metager->getIp();
$this->startTime = microtime();
......@@ -240,7 +240,7 @@ abstract class Searchengine
Cache::put($this->hash, $body, $this->cacheDuration);
}
}
if ($body !== "") {
if ($body !== "" && $body !== "connected" && $body !== "waiting") {
$this->loadResults($body);
$this->getNext($metager, $body);
$this->loaded = true;
......@@ -333,9 +333,6 @@ abstract class Searchengine
{
$affil_data = 'ip=' . $this->ip;
$affil_data .= '&ua=' . $this->useragent;
if (isset($_SERVER['HTTP_X_FORWARDED_FOR'])) {
$affil_data .= '&xfip=' . $_SERVER['HTTP_X_FORWARDED_FOR'];
}
$affilDataValue = $this->urlEncode($affil_data);
# Wir benötigen die ServeUrl:
$serveUrl = $this->urlEncode($url);
......
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
class RlvProduct extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
}
public function loadResults($result)
{
# try
# {
$results = json_decode($result, true);
$counter = 0;
foreach ($results["products"] as $result) {
$counter++;
$image = $result["productImage"];
$image = str_replace("//", "https://", $image);
$this->products[] = new \App\Models\Result(
$this->engine,
$result["productTitle"],
$result["shopLink"],
$result["shopLink"],
"",
$result["shopTitle"],
$counter,
['partnershop' => false,
'price' => $result["price"],
'image' => $image,
'shipping' => $result["distribution"]]
);
}
}
}
{
"_readme": [
"This file locks the dependencies of your project to a known state",
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"This file is @generated automatically"
],
"content-hash": "27d7554c4cf4ca03f97a076d1628bd02",
"packages": [
{
"name": "doctrine/inflector",
"version": "v1.3.0",
"source": {
"type": "git",
"url": "https://github.com/doctrine/inflector.git",
"reference": "5527a48b7313d15261292c149e55e26eae771b0a"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/doctrine/inflector/zipball/5527a48b7313d15261292c149e55e26eae771b0a",
"reference": "5527a48b7313d15261292c149e55e26eae771b0a",
"shasum": ""
},
"require": {
"php": "^7.1"
},
"require-dev": {
"phpunit/phpunit": "^6.2"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.3.x-dev"
}
},
"autoload": {
"psr-4": {
"Doctrine\\Common\\Inflector\\": "lib/Doctrine/Common/Inflector"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Roman Borschel",
"email": "roman@code-factory.org"
},
{
"name": "Benjamin Eberlei",
"email": "kontakt@beberlei.de"
},
{
"name": "Guilherme Blanco",
"email": "guilhermeblanco@gmail.com"
},
{
"name": "Jonathan Wage",
"email": "jonwage@gmail.com"
},
{
"name": "Johannes Schmitt",
"email": "schmittjoh@gmail.com"
}
],
"description": "Common String Manipulations with regard to casing and singular/plural rules.",
"homepage": "http://www.doctrine-project.org",
"keywords": [
"inflection",
"pluralize",
"singularize",
"string"
],
"time": "2018-01-09T20:05:19+00:00"
},
{
"name": "doctrine/lexer",
"version": "v1.0.1",
"source": {
"type": "git",
"url": "https://github.com/doctrine/lexer.git",
"reference": "83893c552fd2045dd78aef794c31e694c37c0b8c"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/doctrine/lexer/zipball/83893c552fd2045dd78aef794c31e694c37c0b8c",
"reference": "83893c552fd2045dd78aef794c31e694c37c0b8c",
"shasum": ""
},
"require": {
"php": ">=5.3.2"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.0.x-dev"
}
},
"autoload": {
"psr-0": {
"Doctrine\\Common\\Lexer\\": "lib/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Roman Borschel",
"email": "roman@code-factory.org"
},
{
"name": "Guilherme Blanco",
"email": "guilhermeblanco@gmail.com"
},
{
"name": "Johannes Schmitt",
"email": "schmittjoh@gmail.com"
}
],
"description": "Base library for a lexer that can be used in Top-Down, Recursive Descent Parsers.",
"homepage": "http://www.doctrine-project.org",
"keywords": [
"lexer",
"parser"
],
"time": "2014-09-09T13:34:57+00:00"
},
{
"name": "egulias/email-validator",
"version": "2.1.3",
"source": {
"type": "git",
"url": "https://github.com/egulias/EmailValidator.git",
"reference": "1bec00a10039b823cc94eef4eddd47dcd3b2ca04"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/egulias/EmailValidator/zipball/1bec00a10039b823cc94eef4eddd47dcd3b2ca04",
"reference": "1bec00a10039b823cc94eef4eddd47dcd3b2ca04",
"shasum": ""
},
"require": {
"doctrine/lexer": "^1.0.1",
"php": ">= 5.5"
},
"require-dev": {
"dominicsayers/isemail": "dev-master",
"phpunit/phpunit": "^4.8.35",
"satooshi/php-coveralls": "^1.0.1"
},
"suggest": {
"ext-intl": "PHP Internationalization Libraries are required to use the SpoofChecking validation"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "2.0.x-dev"
}
},
"autoload": {
"psr-4": {
"Egulias\\EmailValidator\\": "EmailValidator"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Eduardo Gulias Davis"
}
],
"description": "A library for validating emails against several RFCs",
"homepage": "https://github.com/egulias/EmailValidator",
"keywords": [
"email",
"emailvalidation",
"emailvalidator",
"validation",
"validator"
],
"time": "2017-11-15T23:40:40+00:00"
},
{
"name": "erusev/parsedown",
"version": "1.6.4",
"source": {
"type": "git",
"url": "https://github.com/erusev/parsedown.git",
"reference": "fbe3fe878f4fe69048bb8a52783a09802004f548"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/erusev/parsedown/zipball/fbe3fe878f4fe69048bb8a52783a09802004f548",
"reference": "fbe3fe878f4fe69048bb8a52783a09802004f548",
"shasum": ""
},
"require": {
"php": ">=5.3.0"
},
"require-dev": {
"phpunit/phpunit": "^4.8.35"
},
"type": "library",
"autoload": {
"psr-0": {
"Parsedown": ""
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Emanuil Rusev",
"email": "hello@erusev.com",
"homepage": "http://erusev.com"
}
],
"description": "Parser for Markdown.",
"homepage": "http://parsedown.org",
"keywords": [
"markdown",
"parser"
],
"time": "2017-11-14T20:44:03+00:00"
},
{
"name": "fideloper/proxy",
"version": "3.3.4",
"source": {
"type": "git",
"url": "https://github.com/fideloper/TrustedProxy.git",
"reference": "9cdf6f118af58d89764249bbcc7bb260c132924f"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/fideloper/TrustedProxy/zipball/9cdf6f118af58d89764249bbcc7bb260c132924f",
"reference": "9cdf6f118af58d89764249bbcc7bb260c132924f",
"shasum": ""
},
"require": {
"illuminate/contracts": "~5.0",
"php": ">=5.4.0"
},
"require-dev": {
"illuminate/http": "~5.0",
"mockery/mockery": "~0.9.3",
"phpunit/phpunit": "^5.7"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "3.3-dev"
},
"laravel": {
"providers": [
"Fideloper\\Proxy\\TrustedProxyServiceProvider"
]
}
},
"autoload": {
"psr-4": {