Merge branch 'development' into 'master'

Development

See merge request !1589

.gitlab-ci.yml

@@ -193,6 +193,7 @@ development:
       - $INCREMENTAL_ROLLOUT_ENABLED
       - $INCREMENTAL_ROLLOUT_MODE
 
+
 production:
   variables:
     ADDITIONAL_HOSTS: "www.metager.de,metager.org,www.metager.org,metager.es,www.metager.es,klassik.metager.org"

app/Console/Commands/RequestFetcher.php

@@ -180,12 +180,12 @@ class RequestFetcher extends Command
     private function getCurlHandle($job)
     {
         $ch = curl_init();
-
+        
         curl_setopt_array($ch, array(
             CURLOPT_URL => $job["url"],
             CURLOPT_PRIVATE => $job["resulthash"] . ";" . $job["cacheDuration"],
             CURLOPT_RETURNTRANSFER => 1,
-            CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
+            CURLOPT_USERAGENT => $job["useragent"],
             CURLOPT_FOLLOWLOCATION => true,
             CURLOPT_CONNECTTIMEOUT => 2,
             CURLOPT_MAXCONNECTS => 500,

app/Models/Searchengine.php

@@ -49,6 +49,8 @@ abstract class Searchengine
         }
         $this->cacheDuration = max($this->cacheDuration, 5);
 
+        // Thanks to our Middleware this is a almost completely random useragent
+        // which matches the correct device type
         $this->useragent = $metager->getUserAgent();
         $this->ip = $metager->getIp();
         $this->startTime = microtime(true);
@@ -143,6 +145,7 @@ abstract class Searchengine
             $mission = [
                 "resulthash" => $this->hash,
                 "url" => $url,
+                "useragent" => $this->useragent,
                 "username" => $this->username,
                 "password" => $this->password,
                 "headers" => $this->headers,