Merge branch 'development' into MetaGer-Redesign

Inklusive Folgereparaturen

.gitlab-ci.yml

@@ -7,7 +7,7 @@ update(144.76.113.134):
     # Abhängigkeiten überprüfen
     - which composer
     - which git
-    - which php7.0
+    - which php
     - which sqlite3
   script:
     # Pfad zum neu geklonten Repo
@@ -24,16 +24,17 @@ update(144.76.113.134):
     - scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
     - touch storage/logs/laravel.log
     - touch storage/logs/worker.log
+    - touch database/metager.sqlite
     - chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
     - chmod -R 777 storage
     - chmod -R 777 bootstrap/cache
     - npm install
     - npm run prodution
+    - php artisan migrate --force
     - if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
     - cd ~/
     - while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
     - mv MetaGer_neu MetaGer
-    - php ~/MetaGer/artisan migrate --force
     - sudo pkill --signal SIGHUP supervisord
     - php ~/MetaGer/artisan up
    
@@ -46,7 +47,7 @@ update(metager2):
     # Abhängigkeiten überprüfen
     - which composer
     - which git
-    - which php7.0
+    - which php
     - which sqlite3
   script:
     # Pfad zum neu geklonten Repo
@@ -63,16 +64,17 @@ update(metager2):
     - scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
     - touch storage/logs/laravel.log
     - touch storage/logs/worker.log
+    - touch database/metager.sqlite
     - chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
     - chmod -R 777 storage
     - chmod -R 777 bootstrap/cache
     - npm install
     - npm run prodution
+    - php artisan migrate --force
     - if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
     - cd ~/
     - while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
     - mv MetaGer_neu MetaGer
-    - php ~/MetaGer/artisan migrate --force
     - sudo pkill --signal SIGHUP supervisord
     - php ~/MetaGer/artisan up
     
@@ -85,7 +87,7 @@ update(metager3.de):
     # Abhängigkeiten überprüfen
     - which composer
     - which git
-    - which php7.0
+    - which php
     - which sqlite3
   script:
     # Pfad zum neu geklonten Repo
@@ -102,15 +104,16 @@ update(metager3.de):
     - scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
     - touch storage/logs/laravel.log
     - touch storage/logs/worker.log
+    - touch database/metager.sqlite
     - chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
     - chmod -R 777 storage
     - chmod -R 777 bootstrap/cache
     - npm install
-    - ./gulpbuild.sh
+    - npm run prodution
+    - php artisan migrate --force
     - if [ -f ~/MetaGer/artisan ]; then php ~/MetaGer/artisan down;fi
     - cd ~/
     - while [ -d ~/MetaGer ]; do rm -rf ~/MetaGer;done
     - mv MetaGer_neu MetaGer
-    - php ~/MetaGer/artisan migrate --force
     - sudo pkill --signal SIGHUP supervisord
     - php ~/MetaGer/artisan up
\ No newline at end of file

app/Jobs/Searcher.php

@@ -15,7 +15,11 @@ class Searcher implements ShouldQueue
     use InteractsWithQueue, Queueable, SerializesModels;
 
     protected $name, $ch, $pid, $counter, $lastTime, $connectionInfo;
+    # Each Searcher will shutdown after a specified time(s) or number of requests
     protected $MAX_REQUESTS = 100;
+    # This value should always be below the retry_after value in config/queue.php
+    protected $MAX_TIME = 240;
+    protected $startTime = null;
     protected $importantEngines = array("Fastbot", "overture", "overtureAds");
     protected $recheck;
 
@@ -35,6 +39,7 @@ class Searcher implements ShouldQueue
         $this->name = $name;
         $this->pid = getmypid();
         $this->recheck = false;
+        $this->startTime = microtime(true);
         // Submit this worker to the Redis System
         Redis::expire($this->name, 5);
     }
@@ -92,7 +97,9 @@ class Searcher implements ShouldQueue
 
             // In sync mode every Searcher may only retrieve one result because it would block
             // the execution of the remaining code otherwise:
-            if(getenv("QUEUE_DRIVER") === "sync" || $this->counter > $this->MAX_REQUESTS){
+            if(getenv("QUEUE_DRIVER") === "sync" 
+                || $this->counter > $this->MAX_REQUESTS 
+                || (microtime(true)-$this->startTime) > $this->MAX_TIME){
                break;
             } 
         }

app/MetaGer.php

@@ -30,7 +30,6 @@ class MetaGer
     protected $engines         = [];
     protected $results         = [];
     protected $ads             = [];
-    protected $products        = [];
     protected $warnings        = [];
     protected $errors          = [];
     protected $addedHosts      = [];
@@ -38,6 +37,7 @@ class MetaGer
     protected $canCache        = false;
     # Daten über die Abfrage$
     protected $ip;
+    protected $useragent;
     protected $language;
     protected $agent;
     protected $apiKey        = "";
@@ -295,7 +295,6 @@ class MetaGer
         
         if ($this->validated) {
             $this->ads       = [];
-            $this->products  = [];
             $this->maps      = false;
         }
 
@@ -380,9 +379,6 @@ class MetaGer
             foreach ($engine->ads as $ad) {
                 $this->ads[] = $ad;
             }
-            foreach ($engine->products as $product) {
-                $this->products[] = $product;
-            }
         }
 
     }
@@ -944,6 +940,8 @@ class MetaGer
         # nicht einmal wir selbst noch Zugriff auf die Daten haben:
         $this->ip = preg_replace("/(\d+)\.(\d+)\.\d+.\d+/s", "$1.$2.0.0", $this->ip);
 
+        $this->useragent = $request->header('User-Agent');
+
         # Language
         if (isset($_SERVER['HTTP_LANGUAGE'])) {
             $this->language = $_SERVER['HTTP_LANGUAGE'];
@@ -1061,40 +1059,52 @@ class MetaGer
 
     public function checkSpecialSearches(Request $request)
     {
-        if ($request->filled('site')) {
-            $site = $request->input('site');
-        } else {
-            $site = "";
-        }
-
-        $this->searchCheckSitesearch($site);
-        $this->searchCheckHostBlacklist();
-        $this->searchCheckDomainBlacklist();
+        $this->searchCheckSitesearch($request);
+        $this->searchCheckHostBlacklist($request);
+        $this->searchCheckDomainBlacklist($request);
         $this->searchCheckUrlBlacklist();
         $this->searchCheckPhrase();
-        $this->searchCheckStopwords();
+        $this->searchCheckStopwords($request);
         $this->searchCheckNoSearch();
     }
 
-    private function searchCheckSitesearch($site)
+    private function searchCheckSitesearch($request)
     {
         // matches '[... ]site:test.de[ ...]'
         while (preg_match("/(^|.+\s)site:(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
             $this->site = $match[2];
             $this->q    = $match[1] . $match[3];
         }
-        if ($site !== "") {
-            $this->site = $site;
+        # Overwrite Setting if it's submitted via Parameter
+        if ($request->has('site')) {
+            $this->site = $request->input('site');
         }
     }
 
-    private function searchCheckHostBlacklist()
+    private function searchCheckHostBlacklist($request)
     {
         // matches '[... ]-site:test.de[ ...]'
         while (preg_match("/(^|.+\s)-site:([^\s\*]\S*)(?:\s(.+)|($))/si", $this->q, $match)) {
             $this->hostBlacklist[] = $match[2];
             $this->q               = $match[1] . $match[3];
         }
+        # Overwrite Setting if it's submitted via Parameter
+        if($request->has('blacklist')){
+            $this->hostBlacklist = [];
+            $blacklistString = trim($request->input('blacklist'));
+            if(strpos($blacklistString, ",") !== FALSE){
+                $blacklistArray = explode(',', $blacklistString);
+                foreach($blacklistArray as $blacklistElement){
+                    $blacklistElement = trim($blacklistElement);
+                    if(strpos($blacklistElement, "*") !== 0){
+                        $this->hostBlacklist[] = $blacklistElement;
+                    }
+                }
+            }else if(strpos($blacklistString, "*") !== 0){
+                $this->hostBlacklist[] = $blacklistString;
+            }
+        }
+
         // print the host blacklist as a user warning
         if (sizeof($this->hostBlacklist) > 0) {
             $hostString = "";
@@ -1106,13 +1116,29 @@ class MetaGer
         }
     }
 
-    private function searchCheckDomainBlacklist()
+    private function searchCheckDomainBlacklist($request)
     {
         // matches '[... ]-site:*.test.de[ ...]'
         while (preg_match("/(^|.+\s)-site:\*\.(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
             $this->domainBlacklist[] = $match[2];
             $this->q                 = $match[1] . $match[3];
         }
+        # Overwrite Setting if it's submitted via Parameter
+        if($request->has('blacklist')){
+            $this->domainBlacklist = [];
+            $blacklistString = trim($request->input('blacklist'));
+            if(strpos($blacklistString, ",") !== FALSE){
+                $blacklistArray = explode(',', $blacklistString);
+                foreach($blacklistArray as $blacklistElement){
+                    $blacklistElement = trim($blacklistElement);
+                    if(strpos($blacklistElement, "*.") === 0){
+                        $this->domainBlacklist[] = substr($blacklistElement, strpos($blacklistElement, "*.")+2);
+                    }
+                }
+            }else if(strpos($blacklistString, "*.") === 0){
+                $this->domainBlacklist[] = substr($blacklistString, strpos($blacklistString, "*.")+2);
+            }
+        }
         // print the domain blacklist as a user warning
         if (sizeof($this->domainBlacklist) > 0) {
             $domainString = "";
@@ -1142,13 +1168,27 @@ class MetaGer
         }
     }
 
-    private function searchCheckStopwords()
+    private function searchCheckStopwords($request)
     {
         // matches '[... ]-test[ ...]'
         while (preg_match("/(^|.+\s)-(\S+)(?:\s(.+)|($))/si", $this->q, $match)) {
             $this->stopWords[] = $match[2];
             $this->q           = $match[1] . $match[3];
         }
+        # Overwrite Setting if submitted via Parameter
+        if($request->has('stop')){
+            $this->stopWords = [];
+            $stop = trim($request->input('stop'));
+            if(strpos($stop, ',') !== FALSE){ 
+                $stopArray = explode(',', $stop);
+                foreach($stopArray as $stopElement){
+                    $stopElement = trim($stopElement);
+                    $this->stopWords[] = $stopElement;
+                }
+            }else{
+                $this->stopWords[] = $stop;
+            }
+        }
         // print the stopwords as a user warning
         if (sizeof($this->stopWords) > 0) {
             $stopwordsString = "";
@@ -1248,24 +1288,6 @@ class MetaGer
         }
     }
 
-    public function hasProducts()
-    {
-        if (count($this->products) > 0) {
-            return true;
-        } else {
-            return false;
-        }
-    }
-
-    public function getProducts()
-    {
-        $return = [];
-        foreach ($this->products as $product) {
-            $return[] = get_object_vars($product);
-        }
-        return $return;
-    }
-
     public function canCache()
     {
         return $this->canCache;
@@ -1457,6 +1479,11 @@ class MetaGer
     {
         return $this->ip;
     }
+    
+    public function getUserAgent()
+    {
+        return $this->useragent;
+    }
 
     public function getEingabe()
     {

app/Models/Searchengine.php

@@ -68,7 +68,7 @@ abstract class Searchengine
             return;
         }
 
-        $this->useragent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1";
+        $this->useragent = $metager->getUserAgent();
         $this->ip        = $metager->getIp();
         $this->startTime = microtime();
 
@@ -240,7 +240,7 @@ abstract class Searchengine
                 Cache::put($this->hash, $body, $this->cacheDuration);
             }
         }
-        if ($body !== "") {
+        if ($body !== "" && $body !== "connected" && $body !== "waiting") {
             $this->loadResults($body);
             $this->getNext($metager, $body);
             $this->loaded = true;
@@ -333,9 +333,6 @@ abstract class Searchengine
     {
         $affil_data = 'ip=' . $this->ip;
         $affil_data .= '&ua=' . $this->useragent;
-        if (isset($_SERVER['HTTP_X_FORWARDED_FOR'])) {
-            $affil_data .= '&xfip=' . $_SERVER['HTTP_X_FORWARDED_FOR'];
-        }
         $affilDataValue = $this->urlEncode($affil_data);
         # Wir benötigen die ServeUrl:
         $serveUrl = $this->urlEncode($url);

app/Models/parserSkripte/Rlvproduct.php

-<?php
-
-namespace app\Models\parserSkripte;
-
-use App\Models\Searchengine;
-
-class RlvProduct extends Searchengine
-{
-    public $results = [];
-
-    public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
-    {
-        parent::__construct($engine, $metager);
-    }
-
-    public function loadResults($result)
-    {
-        # try
-        # {
-        $results = json_decode($result, true);
-        $counter = 0;
-        foreach ($results["products"] as $result) {
-            $counter++;
-            $image            = $result["productImage"];
-            $image            = str_replace("//", "https://", $image);
-            $this->products[] = new \App\Models\Result(
-                $this->engine,
-                $result["productTitle"],
-                $result["shopLink"],
-                $result["shopLink"],
-                "",
-                $result["shopTitle"],
-                $counter,
-                ['partnershop' => false,
-                    'price'        => $result["price"],
-                    'image'        => $image,
-                    'shipping'     => $result["distribution"]]
-            );
-        }
-    }
-}

config/queue.php

@@ -45,7 +45,7 @@ return [
             'driver' => 'beanstalkd',
             'host' => 'localhost',
             'queue' => 'default',
-            'retry_after' => 90,
+            'retry_after' => 90, 
         ],
 
         'sqs' => [
@@ -61,7 +61,7 @@ return [
             'driver' => 'redis',
             'connection' => 'default',
             'queue' => 'default',
-            'retry_after' => 90,
+            'retry_after' => 300,
         ],
 
     ],

database/migrations/2018_02_22_122047_create_failed_jobs_table.php

+<?php
+
+use Illuminate\Support\Facades\Schema;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Database\Migrations\Migration;
+
+class CreateFailedJobsTable extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('failed_jobs', function (Blueprint $table) {
+            $table->bigIncrements('id');
+            $table->text('connection');
+            $table->text('queue');
+            $table->longText('payload');
+            $table->longText('exception');
+            $table->timestamp('failed_at')->useCurrent();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::dropIfExists('failed_jobs');
+    }
+}

public/google1d0130450de230c1.html

+google-site-verification: google1d0130450de230c1.html
\ No newline at end of file

public/js/settings.js

@@ -27,31 +27,11 @@ $(document).ready(function () {
   });
   $('#unten').click(function () {
     $('#settings-form').append('<input type="hidden" name="usage" value="once">');
-    switch (getLanguage()) {
-      case 'de':
-        alert('Auf der folgenden Startseite sind Ihre Einstellungen nun einmalig gespeichert. Nach Ihrer ersten Suche sind diese wieder verloren. Wenn Sie diese speichern möchten, können Sie sich allerdings ein Lesezeichen für die generierte Startseite einrichten.');
-        break;
-      case 'en':
+      alert(t('saved-settings'));