diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9da99fdfeabda26d7ca8c1f5aa0e0242fab3b100..25b4ec6555deaab359447460e1df94863a7e28c6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -193,6 +193,7 @@ development: - $INCREMENTAL_ROLLOUT_ENABLED - $INCREMENTAL_ROLLOUT_MODE + production: variables: ADDITIONAL_HOSTS: "www.metager.de,metager.org,www.metager.org,metager.es,www.metager.es,klassik.metager.org" diff --git a/app/Console/Commands/RequestFetcher.php b/app/Console/Commands/RequestFetcher.php index 113617ad732009e04b5101dc0dc27c8b058c3d45..13bb5e71cbb35b9ce522157955fe424f0b327506 100644 --- a/app/Console/Commands/RequestFetcher.php +++ b/app/Console/Commands/RequestFetcher.php @@ -180,12 +180,12 @@ class RequestFetcher extends Command private function getCurlHandle($job) { $ch = curl_init(); - + curl_setopt_array($ch, array( CURLOPT_URL => $job["url"], CURLOPT_PRIVATE => $job["resulthash"] . ";" . $job["cacheDuration"], CURLOPT_RETURNTRANSFER => 1, - CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1", + CURLOPT_USERAGENT => $job["useragent"], CURLOPT_FOLLOWLOCATION => true, CURLOPT_CONNECTTIMEOUT => 2, CURLOPT_MAXCONNECTS => 500, diff --git a/app/Models/Searchengine.php b/app/Models/Searchengine.php index 8b612d551b0a19111875b2ce51643cc9483ffee0..eba7c83bd86e4a2104b17fd5a165de4c43fc3570 100644 --- a/app/Models/Searchengine.php +++ b/app/Models/Searchengine.php @@ -49,6 +49,8 @@ abstract class Searchengine } $this->cacheDuration = max($this->cacheDuration, 5); + // Thanks to our Middleware this is a almost completely random useragent + // which matches the correct device type $this->useragent = $metager->getUserAgent(); $this->ip = $metager->getIp(); $this->startTime = microtime(true); @@ -143,6 +145,7 @@ abstract class Searchengine $mission = [ "resulthash" => $this->hash, "url" => $url, + "useragent" => $this->useragent, "username" => $this->username, "password" => $this->password, "headers" => $this->headers,