Skip to content

GitLab

MetaGer
MetaGer
Commit 9d8ffbb3 authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files
Options

Merge branch 'development' into 998-fix-donation-form-and-implement-integrationtest

parents d64e35ec 28ca0eef
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 7135 additions and 19535 deletions
.gitignore
View file @ 9d8ffbb3
......@@ -21,4 +21,7 @@ npm-debug.log
/.project
composer.lock
local.log
\ No newline at end of file
package-lock.json
local.log
browserstack.err
.gitlab-ci.yml
View file @ 9d8ffbb3
......@@ -33,6 +33,8 @@ stages:
- integrationtest
- cleanup
.auto-deploy:
image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.6"
build:
services:
......@@ -75,20 +77,40 @@ review:
variables:
HELM_UPGRADE_VALUES_FILE: .gitlab/review-apps-values.yaml
ROLLOUT_RESOURCE_TYPE: deployment
except:
refs:
- master
- development
variables:
- $REVIEW_DISABLED
environment:
name: review/$CI_COMMIT_REF_NAME
on_stop: stop_review
auto_stop_in: 2 days
rules:
- if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
when: never
- if: '$CI_COMMIT_BRANCH == "master"'
when: never
- if: '$CI_COMMIT_BRANCH == "development"'
when: never
- if: '$REVIEW_DISABLED'
when: never
- if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
stop_review:
except:
refs:
- master
- development
variables:
- $REVIEW_DISABLED
variables:
GIT_STRATEGY: none
environment:
name: review/$CI_COMMIT_REF_NAME
action: stop
rules:
- if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
when: never
- if: '$CI_COMMIT_BRANCH == "master"'
when: never
- if: '$CI_COMMIT_BRANCH == "development"'
when: never
- if: '$REVIEW_DISABLED'
when: never
- if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
when: manual
.development: &development_template
extends: .auto-deploy
......@@ -152,4 +174,4 @@ integrationtest:
- sed -i "s#^APP_URL=.*#APP_URL=$URL#g" .env
- sed -i "s#^BRANCH_NAME=.*#BRANCH_NAME=$CI_COMMIT_REF_NAME#g" .env
- sed -i "s#^COMMIT_NAME=.*#COMMIT_NAME=$CI_COMMIT_REF_SLUG#g" .env
- php artisan dusk
\ No newline at end of file
- php artisan dusk
.gitlab/development-values.yaml
View file @ 9d8ffbb3
postgresql:
enabled: false
service:
externalPort: 80
internalPort: 80
hpa:
enabled: true
minReplicas: 1
maxReplicas: 5
resourcesPhpfpm:
requests:
cpu: 500m
memory: 500M
limits:
resourcesNginx:
requests:
cpu: 100m
memory: 100M
limits:
resourcesRedis:
requests:
cpu: 100m
memory: 1Gi
limits:
resourcesFetcher:
requests:
cpu: 500m
memory: 100M
limits:
podDisruptionBudget:
enabled: true
minAvailable: 1
maxUnavailable:
minAvailable:
maxUnavailable: 0
podAnnotations:
prometheus.io/scrape: "true"
prometheus.io/path: /metrics
prometheus.io/port: "80"
deploymentApiVersion: apps/v1
ingress:
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
nginx.ingress.kubernetes.io/configuration-snippet: |
more_set_headers "Content-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline'; script-src-elem 'self' 'unsafe-inline'; script-src-attr 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; style-src-elem 'self' 'unsafe-inline'; style-src-attr 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self'; connect-src 'self'; media-src; object-src; prefetch-src; child-src; frame-src 'self'; worker-src; frame-ancestors 'self' https://scripts.zdv.uni-mainz.de; form-action 'self' www.paypal.com; base-uri; manifest-src; plugin-types; report-uri; report-to";
more_set_headers "X-Frame-Options: sameorigin";
more_set_headers "X-Content-Type-Options: nosniff";
more_set_headers "ReferrerPolicy: origin";
more_set_headers "X-XSS-Protection: 1; mode=block";
if ($arg_out = "results-with-style") {
more_set_headers "X-Frame-Options: allow-from https://scripts.zdv.uni-mainz.de/";
}
if ($host = "www.metager3.de") {
return 301 https://metager3.de$request_uri;
}
.gitlab/production-values.yaml
View file @ 9d8ffbb3
postgresql:
enabled: false
service:
externalPort: 80
internalPort: 80
hpa:
minReplicas: 5
enabled: true
minReplicas: 3
maxReplicas: 100
resources:
limits:
cpu: 500m
memory: 1Gi
requests:
cpu: 500m
memory: 1Gi
podDisruptionBudget:
enabled: true
minAvailable: 4
maxUnavailable:
minAvailable:
maxUnavailable: 0
podAnnotations:
prometheus.io/scrape: "true"
prometheus.io/path: /metrics
prometheus.io/port: "80"
deploymentApiVersion: apps/v1
resources:
requests:
limits:
resourcesPhpfpm:
requests:
cpu: 500m
memory: 500M
limits:
resourcesNginx:
requests:
cpu: 100m
memory: 100M
limits:
resourcesRedis:
requests:
cpu: 100m
memory: 1Gi
limits:
resourcesFetcher:
requests:
cpu: 500m
memory: 100M
limits:
ingress:
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
nginx.ingress.kubernetes.io/configuration-snippet: |
more_set_headers "Content-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline'; script-src-elem 'self' 'unsafe-inline'; script-src-attr 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; style-src-elem 'self' 'unsafe-inline'; style-src-attr 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self'; connect-src 'self'; media-src; object-src; prefetch-src; child-src; frame-src 'self'; worker-src; frame-ancestors 'self' https://scripts.zdv.uni-mainz.de; form-action 'self' www.paypal.com; base-uri; manifest-src; plugin-types; report-uri; report-to";
more_set_headers "X-Frame-Options: sameorigin";
more_set_headers "X-Content-Type-Options: nosniff";
more_set_headers "ReferrerPolicy: origin";
more_set_headers "X-XSS-Protection: 1; mode=block";
if ($arg_out = "results-with-style") {
more_set_headers "X-Frame-Options: allow-from https://scripts.zdv.uni-mainz.de/";
}
if ($host = "www.metager.de") {
return 301 https://metager.de$request_uri;
}
......
.gitlab/review-apps-values.yaml
View file @ 9d8ffbb3
---
postgresql:
enabled: false
hpa:
enabled: false
resources:
requests:
limits:
ingress:
annotations:
kubernetes.io/tls-acme: "false"
nginx.ingress.kubernetes.io/ssl-redirect: "false"
nginx.ingress.kubernetes.io/configuration-snippet: |
more_set_headers "Content-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline'; script-src-elem 'self' 'unsafe-inline'; script-src-attr 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; style-src-elem 'self' 'unsafe-inline'; style-src-attr 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self'; connect-src 'self'; media-src; object-src; prefetch-src; child-src; frame-src 'self'; worker-src; frame-ancestors 'self' https://scripts.zdv.uni-mainz.de; form-action 'self'; base-uri; manifest-src; plugin-types; report-uri; report-to";
more_set_headers "X-Frame-Options: sameorigin";
more_set_headers "X-Content-Type-Options: nosniff";
more_set_headers "ReferrerPolicy: origin";
more_set_headers "X-XSS-Protection: 1; mode=block";
if ($arg_out = "results-with-style") {
more_set_headers "X-Frame-Options: allow-from https://scripts.zdv.uni-mainz.de/";
}
tls:
enabled: false
service:
commonName: ""
externalPort: 80
internalPort: 80
deploymentApiVersion: apps/v1
resourcesPhpfpm:
requests:
limits:
resourcesNginx:
requests:
limits:
resourcesRedis:
requests:
limits:
resourcesFetcher:
requests:
limits:
\ No newline at end of file
Dockerfile
View file @ 9d8ffbb3
......@@ -7,6 +7,7 @@ RUN apk add --update \
dcron \
zip \
redis \
libpng \
php7 \
php7-fpm \
php7-common \
......@@ -38,10 +39,10 @@ RUN sed -i 's/;error_log = log\/php7\/error.log/error_log = \/dev\/stderr/g' /et
sed -i 's/;catch_workers_output = yes/catch_workers_output = yes/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = nobody/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_children = 5/pm.max_children = 100/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.start_servers = 2/pm.start_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_children = 5/pm.max_children = 1024/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.start_servers = 2/pm.start_servers = 50/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.min_spare_servers = 1/pm.min_spare_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_spare_servers = 3/pm.max_spare_servers = 25/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_spare_servers = 3/pm.max_spare_servers = 50/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/user = www-data/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = www-data/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;cgi.fix_pathinfo=1/cgi.fix_pathinfo=0/g' /etc/php7/php.ini && \
......
app/Console/Commands/RequestFetcher.php
View file @ 9d8ffbb3
......@@ -2,6 +2,7 @@
namespace App\Console\Commands;
use Cache;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
use Log;
......@@ -24,10 +25,9 @@ class RequestFetcher extends Command
protected $shouldRun = true;
protected $multicurl = null;
protected $oldMultiCurl = null;
protected $maxFetchedDocuments = 10000;
protected $fetchedDocuments = 0;
protected $proxyhost, $proxyuser, $proxypassword;
protected $proxyhost;
protected $proxyuser;
protected $proxypassword;
/**
* Create a new command instance.
......@@ -42,7 +42,6 @@ class RequestFetcher extends Command
$this->proxyport = env("PROXY_PORT", "");
$this->proxyuser = env("PROXY_USER", "");
$this->proxypassword = env("PROXY_PASSWORD", "");
}
/**
......@@ -52,7 +51,6 @@ class RequestFetcher extends Command
*/
public function handle()
{
$pidFile = "/tmp/fetcher";
pcntl_signal(SIGINT, [$this, "sig_handler"]);
pcntl_signal(SIGTERM, [$this, "sig_handler"]);
......@@ -79,61 +77,16 @@ class RequestFetcher extends Command
}
try {
$blocking = false;
while ($this->shouldRun) {
$status = curl_multi_exec($this->multicurl, $active);
$currentJobs = [];
if (!$blocking) {
$elements = Redis::pipeline(function($redis){
$redis->lrange(\App\MetaGer::FETCHQUEUE_KEY, 0, -1);
$redis->del(\App\MetaGer::FETCHQUEUE_KEY);
});
$currentJobs = $elements[0];
} else {
$currentJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
if (!empty($currentJob)) {
$currentJobs[] = $currentJob[1];
}
}
if (sizeof($currentJobs) > 0) {
foreach($currentJobs as $currentJob){
$currentJob = json_decode($currentJob, true);
$ch = $this->getCurlHandle($currentJob);
if (curl_multi_add_handle($this->multicurl, $ch) !== 0) {
$this->shouldRun = false;
Log::error("Couldn't add Handle to multicurl");
break;
}
$this->fetchedDocuments++;
if ($this->fetchedDocuments > $this->maxFetchedDocuments) {
Log::info("Reinitializing Multicurl after " . $this->fetchedDocuments . " requests.");
$this->oldMultiCurl = $this->multicurl;
$this->multicurl = curl_multi_init();
$this->fetchedDocuments = 0;
}
$blocking = false;
$active = true;
}
}
$answerRead = $this->readMultiCurl($this->multicurl);
if ($this->oldMultiCurl != null) {
$this->readMultiCurl($this->oldMultiCurl);
$messagesLeft = -1;
if (curl_multi_info_read($this->oldMultiCurl, $messagesLeft) === false) {
if ($messagesLeft = 0) {
Log::debug("Removing finished multicurl handle");
curl_multi_close($this->oldMultiCurl);
$this->oldMultiCurl = null;
}
}
}
if (!$active && !$answerRead) {
$blocking = true;
} else {
usleep(50 * 1000);
$operationsRunning = true;
curl_multi_exec($this->multicurl, $operationsRunning);
$status = $this->readMultiCurl($this->multicurl);
$answersRead = $status[0];
$messagesLeft = $status[1];
$newJobs = $this->checkNewJobs($operationsRunning, $messagesLeft);
if ($newJobs === 0 && $answersRead === 0) {
usleep(10 * 1000);
}
}
} finally {
......@@ -142,18 +95,60 @@ class RequestFetcher extends Command
}
}
/**
* Checks the Redis queue if any new fetch jobs where submitted
* and adds them to multicurl if there are.
* Will be blocking call to redis if there are no running jobs in multicurl
*/
private function checkNewJobs($operationsRunning, $messagesLeft)
{
$newJobs = [];
if ($operationsRunning === 0 && $messagesLeft === -1) {
$newJob = Redis::blpop(\App\MetaGer::FETCHQUEUE_KEY, 1);
if (!empty($newJob)) {
$newJobs[] = $newJob[1];
}
} else {
$elements = Redis::pipeline(function ($redis) {
$redis->lrange(\App\MetaGer::FETCHQUEUE_KEY, 0, -1);
$redis->del(\App\MetaGer::FETCHQUEUE_KEY);
});
$newJobs = $elements[0];
}
$addedJobs = 0;
foreach ($newJobs as $newJob) {
$newJob = json_decode($newJob, true);
$ch = $this->getCurlHandle($newJob);
if (curl_multi_add_handle($this->multicurl, $ch) !== 0) {
$this->shouldRun = false;
Log::error("Couldn't add Handle to multicurl");
break;
} else {
$addedJobs++;
}
}
return $addedJobs;
}
private function readMultiCurl($mc)
{
$answerRead = false;
while (($info = curl_multi_info_read($mc)) !== false) {
$messagesLeft = -1;
$answersRead = 0;
while (($info = curl_multi_info_read($mc, $messagesLeft)) !== false) {
try {
$answerRead = true;
$answersRead++;
$infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE);
$infos = explode(";", $infos);
$resulthash = $infos[0];
$cacheDurationMinutes = intval($infos[1]);
$name = $infos[2];
$responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
$body = "";
$body = "no-result";
$totalTime = curl_getinfo($info["handle"], CURLINFO_TOTAL_TIME);
\App\PrometheusExporter::Duration($totalTime, $name);
$error = curl_error($info["handle"]);
if (!empty($error)) {
......@@ -161,29 +156,38 @@ class RequestFetcher extends Command
}
if ($responseCode !== 200) {
Log::debug($resulthash);
Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
} else {
$body = \curl_multi_getcontent($info["handle"]);
}
Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
$pipe->set($resulthash, $body);
$pipe->lpush($resulthash, $body);
$pipe->expire($resulthash, 60);
});
if ($cacheDurationMinutes > 0) {
try {
Cache::put($resulthash, $body, $cacheDurationMinutes * 60);
} catch (\Exception $e) {
Log::error($e->getMessage());
}
}
} finally {
\curl_multi_remove_handle($mc, $info["handle"]);
}
}
return $answerRead;
return [$answersRead, $messagesLeft];
}
private function getCurlHandle($job)
{
$ch = curl_init();
curl_setopt_array($ch, array(
CURLOPT_URL => $job["url"],
CURLOPT_PRIVATE => $job["resulthash"] . ";" . $job["cacheDuration"],
CURLOPT_PRIVATE => $job["resulthash"] . ";" . $job["cacheDuration"] . ";" . $job["name"],
CURLOPT_RETURNTRANSFER => 1,
CURLOPT_USERAGENT => $job["useragent"],
CURLOPT_FOLLOWLOCATION => true,
......@@ -221,7 +225,6 @@ class RequestFetcher extends Command
public function sig_handler($sig)
{
$this->shouldRun = false;
echo ("Terminating Process\n");
echo("Terminating Process\n");
}
}
app/Http/Controllers/AdminInterface.php
View file @ 9d8ffbb3
......@@ -31,7 +31,6 @@ class AdminInterface extends Controller
if (strstr($key, "_time")) {
$stati[$name]["fetcher"][$pid]["connection"][$key] = $val;
}
}
$stati[$name]["fetcher"][$pid]["poptime"] = $value[1];
}
......@@ -157,7 +156,7 @@ class AdminInterface extends Controller
$now->minute = 0;
$now->second = 0;
while($now->lessThanOrEqualTo(Carbon::now())){
while ($now->lessThanOrEqualTo(Carbon::now())) {
$sameTime += empty($stats->time->{$now->format('H:i')}->{$interface}) ? 0 : $stats->time->{$now->format('H:i')}->{$interface};
$now->addMinutes(5);
}
......@@ -178,19 +177,38 @@ class AdminInterface extends Controller
if ($size > 0<