Commit e5aa5b61 authored by Karl Hasselbring's avatar Karl Hasselbring

Merge branch 'development' into MetaGer-Redesign

Inklusive Folgereparaturen
parents 25a4d3b7 7166dbfb
...@@ -7,7 +7,7 @@ update(144.76.113.134): ...@@ -7,7 +7,7 @@ update(144.76.113.134):
# Abhängigkeiten überprüfen # Abhängigkeiten überprüfen
- which composer - which composer
- which git - which git
- which php7.0 - which php
- which sqlite3 - which sqlite3
script: script:
# Pfad zum neu geklonten Repo # Pfad zum neu geklonten Repo
...@@ -24,16 +24,17 @@ update(144.76.113.134): ...@@ -24,16 +24,17 @@ update(144.76.113.134):
- scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/ - scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
- touch storage/logs/laravel.log - touch storage/logs/laravel.log
- touch storage/logs/worker.log - touch storage/logs/worker.log
- touch database/metager.sqlite
- chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite - chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
- chmod -R 777 storage - chmod -R 777 storage
- chmod -R 777 bootstrap/cache - chmod -R 777 bootstrap/cache
- npm install - npm install
- npm run prodution - npm run prodution
- php artisan migrate --force
- if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi - if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
- cd ~/ - cd ~/
- while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done - while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
- mv MetaGer_neu MetaGer - mv MetaGer_neu MetaGer
- php ~/MetaGer/artisan migrate --force
- sudo pkill --signal SIGHUP supervisord - sudo pkill --signal SIGHUP supervisord
- php ~/MetaGer/artisan up - php ~/MetaGer/artisan up
...@@ -46,7 +47,7 @@ update(metager2): ...@@ -46,7 +47,7 @@ update(metager2):
# Abhängigkeiten überprüfen # Abhängigkeiten überprüfen
- which composer - which composer
- which git - which git
- which php7.0 - which php
- which sqlite3 - which sqlite3
script: script:
# Pfad zum neu geklonten Repo # Pfad zum neu geklonten Repo
...@@ -63,16 +64,17 @@ update(metager2): ...@@ -63,16 +64,17 @@ update(metager2):
- scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/ - scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
- touch storage/logs/laravel.log - touch storage/logs/laravel.log
- touch storage/logs/worker.log - touch storage/logs/worker.log
- touch database/metager.sqlite
- chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite - chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
- chmod -R 777 storage - chmod -R 777 storage
- chmod -R 777 bootstrap/cache - chmod -R 777 bootstrap/cache
- npm install - npm install
- npm run prodution - npm run prodution
- php artisan migrate --force
- if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi - if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
- cd ~/ - cd ~/
- while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done - while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
- mv MetaGer_neu MetaGer - mv MetaGer_neu MetaGer
- php ~/MetaGer/artisan migrate --force
- sudo pkill --signal SIGHUP supervisord - sudo pkill --signal SIGHUP supervisord
- php ~/MetaGer/artisan up - php ~/MetaGer/artisan up
...@@ -85,7 +87,7 @@ update(metager3.de): ...@@ -85,7 +87,7 @@ update(metager3.de):
# Abhängigkeiten überprüfen # Abhängigkeiten überprüfen
- which composer - which composer
- which git - which git
- which php7.0 - which php
- which sqlite3 - which sqlite3
script: script:
# Pfad zum neu geklonten Repo # Pfad zum neu geklonten Repo
...@@ -102,15 +104,16 @@ update(metager3.de): ...@@ -102,15 +104,16 @@ update(metager3.de):
- scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/ - scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
- touch storage/logs/laravel.log - touch storage/logs/laravel.log
- touch storage/logs/worker.log - touch storage/logs/worker.log
- touch database/metager.sqlite
- chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite - chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
- chmod -R 777 storage - chmod -R 777 storage
- chmod -R 777 bootstrap/cache - chmod -R 777 bootstrap/cache
- npm install - npm install
- ./gulpbuild.sh - npm run prodution
- php artisan migrate --force
- if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi - if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
- cd ~/ - cd ~/
- while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done - while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
- mv MetaGer_neu MetaGer - mv MetaGer_neu MetaGer
- php ~/MetaGer/artisan migrate --force
- sudo pkill --signal SIGHUP supervisord - sudo pkill --signal SIGHUP supervisord
- php ~/MetaGer/artisan up - php ~/MetaGer/artisan up
\ No newline at end of file
...@@ -15,7 +15,11 @@ class Searcher implements ShouldQueue ...@@ -15,7 +15,11 @@ class Searcher implements ShouldQueue
use InteractsWithQueue, Queueable, SerializesModels; use InteractsWithQueue, Queueable, SerializesModels;
protected $name, $ch, $pid, $counter, $lastTime, $connectionInfo; protected $name, $ch, $pid, $counter, $lastTime, $connectionInfo;
# Each Searcher will shutdown after a specified time(s) or number of requests
protected $MAX_REQUESTS = 100; protected $MAX_REQUESTS = 100;
# This value should always be below the retry_after value in config/queue.php
protected $MAX_TIME = 240;
protected $startTime = null;
protected $importantEngines = array("Fastbot", "overture", "overtureAds"); protected $importantEngines = array("Fastbot", "overture", "overtureAds");
protected $recheck; protected $recheck;
...@@ -35,6 +39,7 @@ class Searcher implements ShouldQueue ...@@ -35,6 +39,7 @@ class Searcher implements ShouldQueue
$this->name = $name; $this->name = $name;
$this->pid = getmypid(); $this->pid = getmypid();
$this->recheck = false; $this->recheck = false;
$this->startTime = microtime(true);
// Submit this worker to the Redis System // Submit this worker to the Redis System
Redis::expire($this->name, 5); Redis::expire($this->name, 5);
} }
...@@ -92,7 +97,9 @@ class Searcher implements ShouldQueue ...@@ -92,7 +97,9 @@ class Searcher implements ShouldQueue
// In sync mode every Searcher may only retrieve one result because it would block // In sync mode every Searcher may only retrieve one result because it would block
// the execution of the remaining code otherwise: // the execution of the remaining code otherwise:
if(getenv("QUEUE_DRIVER") === "sync" || $this->counter > $this->MAX_REQUESTS){ if(getenv("QUEUE_DRIVER") === "sync"
|| $this->counter > $this->MAX_REQUESTS
|| (microtime(true)-$this->startTime) > $this->MAX_TIME){
break; break;
} }
} }
......
...@@ -30,7 +30,6 @@ class MetaGer ...@@ -30,7 +30,6 @@ class MetaGer
protected $engines = []; protected $engines = [];
protected $results = []; protected $results = [];
protected $ads = []; protected $ads = [];
protected $products = [];
protected $warnings = []; protected $warnings = [];
protected $errors = []; protected $errors = [];
protected $addedHosts = []; protected $addedHosts = [];
...@@ -38,6 +37,7 @@ class MetaGer ...@@ -38,6 +37,7 @@ class MetaGer
protected $canCache = false; protected $canCache = false;
# Daten über die Abfrage$ # Daten über die Abfrage$
protected $ip; protected $ip;
protected $useragent;
protected $language; protected $language;
protected $agent; protected $agent;
protected $apiKey = ""; protected $apiKey = "";
...@@ -295,7 +295,6 @@ class MetaGer ...@@ -295,7 +295,6 @@ class MetaGer
if ($this->validated) { if ($this->validated) {
$this->ads = []; $this->ads = [];
$this->products = [];
$this->maps = false; $this->maps = false;
} }
...@@ -380,9 +379,6 @@ class MetaGer ...@@ -380,9 +379,6 @@ class MetaGer
foreach ($engine->ads as $ad) { foreach ($engine->ads as $ad) {
$this->ads[] = $ad; $this->ads[] = $ad;
} }
foreach ($engine->products as $product) {
$this->products[] = $product;
}
} }
} }
...@@ -944,6 +940,8 @@ class MetaGer ...@@ -944,6 +940,8 @@ class MetaGer
# nicht einmal wir selbst noch Zugriff auf die Daten haben: # nicht einmal wir selbst noch Zugriff auf die Daten haben:
$this->ip = preg_replace("/(\d+)\.(\d+)\.\d+.\d+/s", "$1.$2.0.0", $this->ip); $this->ip = preg_replace("/(\d+)\.(\d+)\.\d+.\d+/s", "$1.$2.0.0", $this->ip);
$this->useragent = $request->header('User-Agent');
# Language # Language
if (isset($_SERVER['HTTP_LANGUAGE'])) { if (isset($_SERVER['HTTP_LANGUAGE'])) {
$this->language = $_SERVER['HTTP_LANGUAGE']; $this->language = $_SERVER['HTTP_LANGUAGE'];
...@@ -1061,40 +1059,52 @@ class MetaGer ...@@ -1061,40 +1059,52 @@ class MetaGer
public function checkSpecialSearches(Request $request) public function checkSpecialSearches(Request $request)
{ {
if ($request->filled('site')) { $this->searchCheckSitesearch($request);
$site = $request->input('site'); $this->searchCheckHostBlacklist($request);
} else { $this->searchCheckDomainBlacklist($request);
$site = "";
}
$this->searchCheckSitesearch($site);
$this->searchCheckHostBlacklist();
$this->searchCheckDomainBlacklist();
$this->searchCheckUrlBlacklist(); $this->searchCheckUrlBlacklist();
$this->searchCheckPhrase(); $this->searchCheckPhrase();
$this->searchCheckStopwords(); $this->searchCheckStopwords($request);
$this->searchCheckNoSearch(); $this->searchCheckNoSearch();
} }
private function searchCheckSitesearch($site) private function searchCheckSitesearch($request)
{ {
// matches '[... ]site:test.de[ ...]' // matches '[... ]site:test.de[ ...]'
while (preg_match("/(^|.+\s)site:(\S+)(?:\s(.+)|($))/si", $this->q, $match)) { while (preg_match("/(^|.+\s)site:(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->site = $match[2]; $this->site = $match[2];
$this->q = $match[1] . $match[3]; $this->q = $match[1] . $match[3];
} }
if ($site !== "") { # Overwrite Setting if it's submitted via Parameter
$this->site = $site; if ($request->has('site')) {
$this->site = $request->input('site');
} }
} }
private function searchCheckHostBlacklist() private function searchCheckHostBlacklist($request)
{ {
// matches '[... ]-site:test.de[ ...]' // matches '[... ]-site:test.de[ ...]'
while (preg_match("/(^|.+\s)-site:([^\s\*]\S*)(?:\s(.+)|($))/si", $this->q, $match)) { while (preg_match("/(^|.+\s)-site:([^\s\*]\S*)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->hostBlacklist[] = $match[2]; $this->hostBlacklist[] = $match[2];
$this->q = $match[1] . $match[3]; $this->q = $match[1] . $match[3];
} }
# Overwrite Setting if it's submitted via Parameter
if($request->has('blacklist')){
$this->hostBlacklist = [];
$blacklistString = trim($request->input('blacklist'));
if(strpos($blacklistString, ",") !== FALSE){
$blacklistArray = explode(',', $blacklistString);
foreach($blacklistArray as $blacklistElement){
$blacklistElement = trim($blacklistElement);
if(strpos($blacklistElement, "*") !== 0){
$this->hostBlacklist[] = $blacklistElement;
}
}
}else if(strpos($blacklistString, "*") !== 0){
$this->hostBlacklist[] = $blacklistString;
}
}
// print the host blacklist as a user warning // print the host blacklist as a user warning
if (sizeof($this->hostBlacklist) > 0) { if (sizeof($this->hostBlacklist) > 0) {
$hostString = ""; $hostString = "";
...@@ -1106,13 +1116,29 @@ class MetaGer ...@@ -1106,13 +1116,29 @@ class MetaGer
} }
} }
private function searchCheckDomainBlacklist() private function searchCheckDomainBlacklist($request)
{ {
// matches '[... ]-site:*.test.de[ ...]' // matches '[... ]-site:*.test.de[ ...]'
while (preg_match("/(^|.+\s)-site:\*\.(\S+)(?:\s(.+)|($))/si", $this->q, $match)) { while (preg_match("/(^|.+\s)-site:\*\.(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->domainBlacklist[] = $match[2]; $this->domainBlacklist[] = $match[2];
$this->q = $match[1] . $match[3]; $this->q = $match[1] . $match[3];
} }
# Overwrite Setting if it's submitted via Parameter
if($request->has('blacklist')){
$this->domainBlacklist = [];
$blacklistString = trim($request->input('blacklist'));
if(strpos($blacklistString, ",") !== FALSE){
$blacklistArray = explode(',', $blacklistString);
foreach($blacklistArray as $blacklistElement){
$blacklistElement = trim($blacklistElement);
if(strpos($blacklistElement, "*.") === 0){
$this->domainBlacklist[] = substr($blacklistElement, strpos($blacklistElement, "*.")+2);
}
}
}else if(strpos($blacklistString, "*.") === 0){
$this->domainBlacklist[] = substr($blacklistString, strpos($blacklistString, "*.")+2);
}
}
// print the domain blacklist as a user warning // print the domain blacklist as a user warning
if (sizeof($this->domainBlacklist) > 0) { if (sizeof($this->domainBlacklist) > 0) {
$domainString = ""; $domainString = "";
...@@ -1142,13 +1168,27 @@ class MetaGer ...@@ -1142,13 +1168,27 @@ class MetaGer
} }
} }
private function searchCheckStopwords() private function searchCheckStopwords($request)
{ {
// matches '[... ]-test[ ...]' // matches '[... ]-test[ ...]'
while (preg_match("/(^|.+\s)-(\S+)(?:\s(.+)|($))/si", $this->q, $match)) { while (preg_match("/(^|.+\s)-(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
$this->stopWords[] = $match[2]; $this->stopWords[] = $match[2];
$this->q = $match[1] . $match[3]; $this->q = $match[1] . $match[3];
} }
# Overwrite Setting if submitted via Parameter
if($request->has('stop')){
$this->stopWords = [];
$stop = trim($request->input('stop'));
if(strpos($stop, ',') !== FALSE){
$stopArray = explode(',', $stop);
foreach($stopArray as $stopElement){
$stopElement = trim($stopElement);
$this->stopWords[] = $stopElement;
}
}else{
$this->stopWords[] = $stop;
}
}
// print the stopwords as a user warning // print the stopwords as a user warning
if (sizeof($this->stopWords) > 0) { if (sizeof($this->stopWords) > 0) {
$stopwordsString = ""; $stopwordsString = "";
...@@ -1248,24 +1288,6 @@ class MetaGer ...@@ -1248,24 +1288,6 @@ class MetaGer
} }
} }
public function hasProducts()
{
if (count($this->products) > 0) {
return true;
} else {
return false;
}
}
public function getProducts()
{
$return = [];
foreach ($this->products as $product) {
$return[] = get_object_vars($product);
}
return $return;
}
public function canCache() public function canCache()
{ {
return $this->canCache; return $this->canCache;
...@@ -1457,6 +1479,11 @@ class MetaGer ...@@ -1457,6 +1479,11 @@ class MetaGer
{ {
return $this->ip; return $this->ip;
} }
public function getUserAgent()
{
return $this->useragent;
}
public function getEingabe() public function getEingabe()
{ {
......
...@@ -68,7 +68,7 @@ abstract class Searchengine ...@@ -68,7 +68,7 @@ abstract class Searchengine
return; return;
} }
$this->useragent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1"; $this->useragent = $metager->getUserAgent();
$this->ip = $metager->getIp(); $this->ip = $metager->getIp();
$this->startTime = microtime(); $this->startTime = microtime();
...@@ -240,7 +240,7 @@ abstract class Searchengine ...@@ -240,7 +240,7 @@ abstract class Searchengine
Cache::put($this->hash, $body, $this->cacheDuration); Cache::put($this->hash, $body, $this->cacheDuration);
} }
} }
if ($body !== "") { if ($body !== "" && $body !== "connected" && $body !== "waiting") {
$this->loadResults($body); $this->loadResults($body);
$this->getNext($metager, $body); $this->getNext($metager, $body);
$this->loaded = true; $this->loaded = true;
...@@ -333,9 +333,6 @@ abstract class Searchengine ...@@ -333,9 +333,6 @@ abstract class Searchengine
{ {
$affil_data = 'ip=' . $this->ip; $affil_data = 'ip=' . $this->ip;
$affil_data .= '&ua=' . $this->useragent; $affil_data .= '&ua=' . $this->useragent;
if (isset($_SERVER['HTTP_X_FORWARDED_FOR'])) {
$affil_data .= '&xfip=' . $_SERVER['HTTP_X_FORWARDED_FOR'];
}
$affilDataValue = $this->urlEncode($affil_data); $affilDataValue = $this->urlEncode($affil_data);
# Wir benötigen die ServeUrl: # Wir benötigen die ServeUrl:
$serveUrl = $this->urlEncode($url); $serveUrl = $this->urlEncode($url);
......
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
class RlvProduct extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
}
public function loadResults($result)
{
# try
# {
$results = json_decode($result, true);
$counter = 0;
foreach ($results["products"] as $result) {
$counter++;
$image = $result["productImage"];
$image = str_replace("//", "https://", $image);
$this->products[] = new \App\Models\Result(
$this->engine,
$result["productTitle"],
$result["shopLink"],
$result["shopLink"],
"",
$result["shopTitle"],
$counter,
['partnershop' => false,
'price' => $result["price"],
'image' => $image,
'shipping' => $result["distribution"]]
);
}
}
}
This diff is collapsed.
...@@ -45,7 +45,7 @@ return [ ...@@ -45,7 +45,7 @@ return [
'driver' => 'beanstalkd', 'driver' => 'beanstalkd',
'host' => 'localhost', 'host' => 'localhost',
'queue' => 'default', 'queue' => 'default',
'retry_after' => 90, 'retry_after' => 90,
], ],
'sqs' => [ 'sqs' => [
...@@ -61,7 +61,7 @@ return [ ...@@ -61,7 +61,7 @@ return [
'driver' => 'redis', 'driver' => 'redis',
'connection' => 'default', 'connection' => 'default',
'queue' => 'default', 'queue' => 'default',
'retry_after' => 90, 'retry_after' => 300,
], ],
], ],
......
<?php
use Illuminate\Support\Facades\Schema;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
class CreateFailedJobsTable extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('failed_jobs', function (Blueprint $table) {
$table->bigIncrements('id');
$table->text('connection');
$table->text('queue');
$table->longText('payload');
$table->longText('exception');
$table->timestamp('failed_at')->useCurrent();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::dropIfExists('failed_jobs');
}
}
google-site-verification: google1d0130450de230c1.html
\ No newline at end of file
public/img/Firefox.png

47.8 KB | W: | H:

public/img/Firefox.png

47.8 KB | W: | H:

public/img/Firefox.png
public/img/Firefox.png
public/img/Firefox.png
public/img/Firefox.png
  • 2-up
  • Swipe
  • Onion skin
This diff is collapsed.
...@@ -27,31 +27,11 @@ $(document).ready(function () { ...@@ -27,31 +27,11 @@ $(document).ready(function () {
}); });
$('#unten').click(function () { $('#unten').click(function () {
$('#settings-form').append('<input type="hidden" name="usage" value="once">'); $('#settings-form').append('<input type="hidden" name="usage" value="once">');
switch (getLanguage()) { alert(t('saved-settings'));
case 'de':
alert('Auf der folgenden Startseite sind Ihre Einstellungen nun einmalig gespeichert. Nach Ihrer ersten Suche sind diese wieder verloren. Wenn Sie diese speichern möchten, können Sie sich allerdings ein Lesezeichen für die generierte Startseite einrichten.');
break;
case 'en':