Skip to content
Snippets Groups Projects
Commit 68324caa authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files
parents 2ac0a04e 67994b7f
No related branches found
No related tags found
No related merge requests found
......@@ -21,6 +21,7 @@ RUN apk add --update \
php7-dom \
php7-fileinfo \
php7-redis \
&& apk upgrade -u -a \
&& rm -rf /var/cache/apk/* \
&& update-ca-certificates
......
......@@ -10,7 +10,7 @@ use Log;
class RequestFetcher extends Command
{
public const FETCHQUEUE_KEY = "fetcher.queue";
public const SIZE_LIMIT = 2 * 1024 * 1024; // Limit Document Size to 2MB
public const SIZE_LIMIT = 20 * 1024 * 1024; // Limit Document Size to 2MB
/**
* The name and signature of the console command.
......@@ -146,16 +146,11 @@ class RequestFetcher extends Command
$infos = explode(";", $infos);
$resulthash = $infos[0];
$cacheDurationMinutes = intval($infos[1]);
$responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
$body = "no-result";
$totalTime = curl_getinfo($info["handle"], CURLINFO_TOTAL_TIME);
$error = curl_error($info["handle"]);
if (!empty($error)) {
Log::error($error);
if(curl_errno($info["handle"])){
Log::error(curl_error($info["handle"]));
}
$result = $this->parseResponse($info["handle"]);
Redis::pipeline(function ($pipe) use ($resulthash, $result, $cacheDurationMinutes) {
......@@ -243,11 +238,11 @@ class RequestFetcher extends Command
CURLOPT_RETURNTRANSFER => 1,
CURLOPT_USERAGENT => $job["useragent"],
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_CONNECTTIMEOUT => 2,
CURLOPT_CONNECTTIMEOUT => 10,
CURLOPT_MAXCONNECTS => 500,
CURLOPT_LOW_SPEED_LIMIT => 50000,
CURLOPT_LOW_SPEED_TIME => 5,
CURLOPT_TIMEOUT => 7,
CURLOPT_TIMEOUT => 20,
CURLOPT_HEADER => true,
CURLOPT_NOPROGRESS => false,
CURLOPT_PROGRESSFUNCTION => 'self::progress',
......
......@@ -156,7 +156,7 @@ class ProxyController extends Controller
$mission = [
"resulthash" => $hash,
"url" => $this->sanitizeUrl($targetUrl),
"url" => self::sanitizeUrl($targetUrl),
"useragent" => $useragent,
"cacheDuration" => $this::PROXY_CACHE,
];
......@@ -373,18 +373,21 @@ class ProxyController extends Controller
*/
public static function generateProxyWrapperUrl($url){
$password = self::generatePassword($url, null);
$sanitizedUrl = self::sanitizeUrl($url);
$parts = parse_url($url);
$sanitizedParts = parse_url($sanitizedUrl);
$host = null;
$path = null;
if(!empty($parts["host"])){
$host = $parts["host"];
if(!empty($sanitizedParts["host"])){
$host = $sanitizedParts["host"];
}
if(!empty($parts["path"])){
$path = trim($parts["path"], "/");
if(!empty($sanitizedParts["path"])){
$path = trim($sanitizedParts["path"], "/");
}
$parameters = [
"host" => $host,
"path" => $path,
......@@ -504,7 +507,7 @@ class ProxyController extends Controller
}
}
private function sanitizeUrl($url){
private static function sanitizeUrl($url){
$parts = parse_url($url);
// Optional but we only sanitize URLs with scheme and host defined
......@@ -525,7 +528,7 @@ class ProxyController extends Controller
// Build the url
$targetUrl = $parts["scheme"] . "://" .
((!empty($parts["user"]) && !empty($parts["pass"])) ? $parts["user"] . ":" . $parts["pass"] . "@" : "") .
$parts["host"] .
rtrim($parts["host"], ".") . "." .
(!empty($parts["port"]) ? ":" . $parts["port"] : "") .
(!empty($sanitizedPath) ? $sanitizedPath : "") .
(!empty($parts["query"]) ? "?" . $parts["query"] : "") .
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment