Merge branch 'master' of https://gitlab.metager.de/open-source/Proxy

Dockerfile

@@ -21,6 +21,7 @@ RUN apk add --update \
     php7-dom \
     php7-fileinfo \
     php7-redis \
+    && apk upgrade -u -a \
     && rm -rf /var/cache/apk/* \
     && update-ca-certificates
 

app/Console/Commands/RequestFetcher.php

@@ -10,7 +10,7 @@ use Log;
 class RequestFetcher extends Command
 {
     public const FETCHQUEUE_KEY = "fetcher.queue";
-    public const SIZE_LIMIT = 2 * 1024 * 1024; // Limit Document Size to 2MB
+    public const SIZE_LIMIT = 20 * 1024 * 1024; // Limit Document Size to 2MB
 
     /**
      * The name and signature of the console command.
@@ -146,16 +146,11 @@ class RequestFetcher extends Command
                 $infos = explode(";", $infos);
                 $resulthash = $infos[0];
                 $cacheDurationMinutes = intval($infos[1]);
-                $responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
-                $body = "no-result";
-
-                $totalTime = curl_getinfo($info["handle"], CURLINFO_TOTAL_TIME);
-
-                $error = curl_error($info["handle"]);
-                if (!empty($error)) {
-                    Log::error($error);
+                
+                if(curl_errno($info["handle"])){
+                    Log::error(curl_error($info["handle"]));
                 }
-
+                
                 $result = $this->parseResponse($info["handle"]);
                 
                 Redis::pipeline(function ($pipe) use ($resulthash, $result, $cacheDurationMinutes) {
@@ -243,11 +238,11 @@ class RequestFetcher extends Command
             CURLOPT_RETURNTRANSFER => 1,
             CURLOPT_USERAGENT => $job["useragent"],
             CURLOPT_FOLLOWLOCATION => true,
-            CURLOPT_CONNECTTIMEOUT => 2,
+            CURLOPT_CONNECTTIMEOUT => 10,
             CURLOPT_MAXCONNECTS => 500,
             CURLOPT_LOW_SPEED_LIMIT => 50000,
             CURLOPT_LOW_SPEED_TIME => 5,
-            CURLOPT_TIMEOUT => 7,
+            CURLOPT_TIMEOUT => 20,
             CURLOPT_HEADER => true,
             CURLOPT_NOPROGRESS => false,
             CURLOPT_PROGRESSFUNCTION => 'self::progress',

app/Http/Controllers/ProxyController.php

@@ -156,7 +156,7 @@ class ProxyController extends Controller
 
             $mission = [
                 "resulthash" => $hash,
-                "url" => $this->sanitizeUrl($targetUrl),
+                "url" => self::sanitizeUrl($targetUrl),
                 "useragent" => $useragent,
                 "cacheDuration" => $this::PROXY_CACHE,
             ];
@@ -373,18 +373,21 @@ class ProxyController extends Controller
      */
     public static function generateProxyWrapperUrl($url){
         $password = self::generatePassword($url, null);
+        $sanitizedUrl = self::sanitizeUrl($url);
 
-        $parts = parse_url($url);
+        $sanitizedParts = parse_url($sanitizedUrl);
         $host = null;
         $path = null;
 
-        if(!empty($parts["host"])){
-            $host = $parts["host"];
+        if(!empty($sanitizedParts["host"])){
+            $host = $sanitizedParts["host"];
         }
-        if(!empty($parts["path"])){
-            $path = trim($parts["path"], "/");
+        if(!empty($sanitizedParts["path"])){
+            $path = trim($sanitizedParts["path"], "/");
         }
 
+
+
         $parameters = [
             "host" => $host,
             "path" => $path,
@@ -504,7 +507,7 @@ class ProxyController extends Controller
         }
     }
 
-    private function sanitizeUrl($url){
+    private static function sanitizeUrl($url){
         $parts = parse_url($url);
 
         // Optional but we only sanitize URLs with scheme and host defined
@@ -525,7 +528,7 @@ class ProxyController extends Controller
         // Build the url
         $targetUrl = $parts["scheme"] . "://" .
             ((!empty($parts["user"]) && !empty($parts["pass"])) ? $parts["user"] . ":" . $parts["pass"] . "@" : "") .
-            $parts["host"] .
+            rtrim($parts["host"], ".") . "." .
             (!empty($parts["port"]) ? ":" . $parts["port"] : "") .
             (!empty($sanitizedPath) ? $sanitizedPath : "") .
             (!empty($parts["query"]) ? "?" . $parts["query"] : "") .