Commit 8828d367 authored by Phil Höfer's avatar Phil Höfer
Browse files

Merge branch 'development' into '932-dark-stylesheet-alternative'

# Conflicts:
#   resources/less/metager/pages/resultpage/result-page.less
#   resources/less/metager/pages/resultpage/result.less
#   resources/less/metager/variables.less
#   webpack.mix.js
parents 1987d109 ed7b4825
README.md README.md
CHANGELOG.md CHANGELOG.md
docker-compose.yml docker-compose.yml
Dockerfile Dockerfile
\ No newline at end of file
APP_ENV=local APP_ENV=local
APP_DEBUG=true APP_DEBUG=true
APP_LOG_LEVEL=debug APP_LOG_LEVEL=debug
LOG_CHANNEL=stderr
APP_KEY= APP_KEY=
APP_URL=http://localhost APP_URL=http://localhost
BOT_PROTECTION=true
DB_CONNECTION=mysql DB_CONNECTION=mysql
DB_HOST=mgdb DB_HOST=mgdb
DB_PORT=3306 DB_PORT=3306
...@@ -15,11 +18,12 @@ REDIS_RESULT_CONNECTION=default ...@@ -15,11 +18,12 @@ REDIS_RESULT_CONNECTION=default
REDIS_RESULT_CACHE_DURATION=60 REDIS_RESULT_CACHE_DURATION=60
BROADCAST_DRIVER=log BROADCAST_DRIVER=log
CACHE_DRIVER=database CACHE_DRIVER=redis
SESSION_DRIVER=file SESSION_DRIVER=file
QUEUE_CONNECTION=sync QUEUE_CONNECTION=sync
REDIS_HOST=127.0.0.1 REDIS_CACHE_HOST=redis
REDIS_HOST=redis
REDIS_PASSWORD=null REDIS_PASSWORD=null
REDIS_PORT=6379 REDIS_PORT=6379
......
variables: variables:
DOCKER_HOST: "tcp://docker-dind.gitlab:2375" DOCKER_HOST: "tcp://docker-dind.gitlab:2375"
AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS: "--network host"
POSTGRES_ENABLED: "false" POSTGRES_ENABLED: "false"
CODE_QUALITY_DISABLED: "true" CODE_QUALITY_DISABLED: "true"
CONTAINER_SCANNING_DISABLED: "true" CONTAINER_SCANNING_DISABLED: "true"
...@@ -156,11 +157,10 @@ development: ...@@ -156,11 +157,10 @@ development:
- $INCREMENTAL_ROLLOUT_ENABLED - $INCREMENTAL_ROLLOUT_ENABLED
- $INCREMENTAL_ROLLOUT_MODE - $INCREMENTAL_ROLLOUT_MODE
production: production:
variables: variables:
ADDITIONAL_HOSTS: "www.metager.de,metager.org,www.metager.org,metager.es,www.metager.es" ADDITIONAL_HOSTS: "www.metager.de,metager.org,www.metager.org,metager.es,www.metager.es,klassik.metager.org"
HELM_UPGRADE_VALUES_FILE: .gitlab/production-values.yaml HELM_UPGRADE_VALUES_FILE: .gitlab/production-values.yaml
ROLLOUT_RESOURCE_TYPE: deployment ROLLOUT_RESOURCE_TYPE: deployment
environment: environment:
url: https://metager.de url: https://metager.de
\ No newline at end of file
service: service:
externalPort: 80 externalPort: 80
internalPort: 80 internalPort: 80
hpa:
minReplicas: 1
maxReplicas: 5
podDisruptionBudget:
enabled: true
minAvailable: 1
maxUnavailable:
ingress: ingress:
annotations: annotations:
certmanager.k8s.io/cluster-issuer: letsencrypt-prod certmanager.k8s.io/cluster-issuer: letsencrypt-prod
......
...@@ -2,11 +2,16 @@ service: ...@@ -2,11 +2,16 @@ service:
externalPort: 80 externalPort: 80
internalPort: 80 internalPort: 80
hpa: hpa:
minReplicas: 10 minReplicas: 5
maxReplicas: 40 maxReplicas: 25
podDisruptionBudget:
enabled: true
minAvailable: 5
maxUnavailable:
ingress: ingress:
annotations: annotations:
certmanager.k8s.io/cluster-issuer: letsencrypt-prod certmanager.k8s.io/cluster-issuer: letsencrypt-prod
nginx.ingress.kubernetes.io/ssl-redirect: "false"
nginx.ingress.kubernetes.io/configuration-snippet: | nginx.ingress.kubernetes.io/configuration-snippet: |
if ($host = "www.metager.de") { if ($host = "www.metager.de") {
return 301 https://metager.de$request_uri; return 301 https://metager.de$request_uri;
...@@ -16,4 +21,7 @@ ingress: ...@@ -16,4 +21,7 @@ ingress:
} }
if ($host = "www.metager.es") { if ($host = "www.metager.es") {
return 301 https://metager.es$request_uri; return 301 https://metager.es$request_uri;
}
if ($host = "klassik.metager.org") {
return 301 https://metager.de$request_uri;
} }
\ No newline at end of file
FROM nginx FROM alpine:3.11.3
RUN apt -y update && apt -y install php-fpm \ RUN apk add --update \
nginx \
tzdata \
ca-certificates \ ca-certificates \
cron \ dcron \
zip \ zip \
php7.3-common \ redis \
php7.3-curl \ php7 \
php7.3-mbstring \ php7-fpm \
php7.3-sqlite3 \ php7-common \
php7.3-mysql \ php7-curl \
php7.3-xml \ php7-mbstring \
php7.3-zip \ php7-sqlite3 \
php7.3-redis \ php7-pdo_mysql \
php7.3-gd \ php7-pdo_sqlite \
redis-server php7-dom \
php7-simplexml \
RUN sed -i 's/listen.owner = www-data/listen.owner = nginx/g' /etc/php/7.3/fpm/pool.d/www.conf && \ php7-tokenizer \
sed -i 's/listen.group = www-data/listen.group = nginx/g' /etc/php/7.3/fpm/pool.d/www.conf && \ php7-zip \
sed -i 's/pm.max_children = 5/pm.max_children = 100/g' /etc/php/7.3/fpm/pool.d/www.conf && \ php7-redis \
sed -i 's/pm.start_servers = 2/pm.start_servers = 25/g' /etc/php/7.3/fpm/pool.d/www.conf && \ php7-gd \
sed -i 's/pm.min_spare_servers = 1/pm.min_spare_servers = 5/g' /etc/php/7.3/fpm/pool.d/www.conf && \ php7-json \
sed -i 's/pm.max_spare_servers = 3/pm.max_spare_servers = 25/g' /etc/php/7.3/fpm/pool.d/www.conf && \ php7-pcntl \
sed -i 's/user = www-data/user = nginx/g' /etc/php/7.3/fpm/pool.d/www.conf && \ php7-opcache \
sed -i 's/group = www-data/group = nginx/g' /etc/php/7.3/fpm/pool.d/www.conf && \ php7-fileinfo \
sed -i 's/;cgi.fix_pathinfo=1/cgi.fix_pathinfo=0/g' /etc/php/7.3/fpm/php.ini && \ && rm -rf /var/cache/apk/*
mkdir /html
# Set correct timezone
RUN ln -fs /usr/share/zoneinfo/Europe/Berlin /etc/localtime && dpkg-reconfigure -f noninteractive tzdata
# Add Cronjob for Laravel
RUN (crontab -l ; echo "* * * * * php /html/artisan schedule:run >> /dev/null 2>&1") | crontab
WORKDIR /html WORKDIR /html
EXPOSE 80
RUN sed -i 's/;error_log = log\/php7\/error.log/error_log = \/dev\/stderr/g' /etc/php7/php-fpm.conf && \
sed -i 's/;daemonize = yes/daemonize = no/g' /etc/php7/php-fpm.conf && \
sed -i 's/listen = 127.0.0.1:9000/listen = 9000/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;request_terminate_timeout = 0/request_terminate_timeout = 30/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;request_terminate_timeout_track_finished = no/request_terminate_timeout_track_finished = yes/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;decorate_workers_output = no/decorate_workers_output = no/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;catch_workers_output = yes/catch_workers_output = yes/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = nobody/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_children = 5/pm.max_children = 100/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.start_servers = 2/pm.start_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.min_spare_servers = 1/pm.min_spare_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_spare_servers = 3/pm.max_spare_servers = 25/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/user = www-data/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = www-data/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;cgi.fix_pathinfo=1/cgi.fix_pathinfo=0/g' /etc/php7/php.ini && \
sed -i 's/expose_php = On/expose_php = Off/g' /etc/php7/php.ini && \
# Opcache configuration
sed -i 's/;opcache.enable=1/opcache.enable=1/g' /etc/php7/php.ini && \
sed -i 's/;opcache.memory_consumption=128/opcache.memory_consumption=128/g' /etc/php7/php.ini && \
sed -i 's/;opcache.interned_strings_buffer=8/opcache.interned_strings_buffer=8/g' /etc/php7/php.ini && \
sed -i 's/;opcache.max_accelerated_files=10000/opcache.max_accelerated_files=10000/g' /etc/php7/php.ini && \
sed -i 's/;opcache.max_wasted_percentage=5/opcache.max_wasted_percentage=5/g' /etc/php7/php.ini && \
sed -i 's/;opcache.validate_timestamps=1/opcache.validate_timestamps=1/g' /etc/php7/php.ini && \
sed -i 's/;opcache.revalidate_freq=2/opcache.revalidate_freq=300/g' /etc/php7/php.ini && \
echo "daemonize yes" >> /etc/redis.conf && \
ln -s /dev/null /var/log/nginx/access.log && \
ln -s /dev/stdout /var/log/nginx/error.log && \
cp /usr/share/zoneinfo/Europe/Berlin /etc/localtime && \
echo "Europe/Berlin" > /etc/timezone && \
(crontab -l ; echo "* * * * * php /html/artisan schedule:run >> /dev/null 2>&1") | crontab -
COPY config/nginx.conf /etc/nginx/nginx.conf COPY config/nginx.conf /etc/nginx/nginx.conf
COPY config/nginx-default.conf /etc/nginx/conf.d/default.conf COPY config/nginx-default.conf /etc/nginx/conf.d/default.conf
RUN sed -i 's/fastcgi_pass phpfpm:9000;/fastcgi_pass localhost:9000;/g' /etc/nginx/conf.d/default.conf
COPY --chown=root:nginx . /html COPY --chown=root:nginx . /html
WORKDIR /html
EXPOSE 80
CMD chown -R root:nginx storage/logs/metager bootstrap/cache && \ CMD chown -R root:nginx storage/logs/metager bootstrap/cache && \
chmod -R g+w storage/logs/metager bootstrap/cache && \ chmod -R g+w storage/logs/metager bootstrap/cache && \
/etc/init.d/cron start && \ crond -L /dev/stdout && \
/etc/init.d/php7.3-fpm start && \ php-fpm7
/etc/init.d/nginx start && \
/etc/init.d/redis-server start && \
su -s /bin/bash -c 'php artisan requests:fetcher' nginx
FROM alpine:3.11.3
RUN apk add --update \
nginx \
tzdata \
ca-certificates \
dcron \
zip \
redis \
php7 \
php7-fpm \
php7-common \
php7-curl \
php7-mbstring \
php7-sqlite3 \
php7-pdo_mysql \
php7-pdo_sqlite \
php7-dom \
php7-simplexml \
php7-tokenizer \
php7-zip \
php7-redis \
php7-gd \
php7-json \
php7-pcntl \
php7-fileinfo \
&& rm -rf /var/cache/apk/*
WORKDIR /html
RUN sed -i 's/;error_log = log\/php7\/error.log/error_log = \/dev\/stderr/g' /etc/php7/php-fpm.conf && \
sed -i 's/;daemonize = yes/daemonize = no/g' /etc/php7/php-fpm.conf && \
sed -i 's/listen = 127.0.0.1:9000/listen = 9000/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;request_terminate_timeout = 0/request_terminate_timeout = 30/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;request_terminate_timeout_track_finished = no/request_terminate_timeout_track_finished = yes/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;decorate_workers_output = no/decorate_workers_output = no/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;catch_workers_output = yes/catch_workers_output = yes/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = nobody/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_children = 5/pm.max_children = 100/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.start_servers = 2/pm.start_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.min_spare_servers = 1/pm.min_spare_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_spare_servers = 3/pm.max_spare_servers = 25/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/user = www-data/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = www-data/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;cgi.fix_pathinfo=1/cgi.fix_pathinfo=0/g' /etc/php7/php.ini && \
sed -i 's/expose_php = On/expose_php = Off/g' /etc/php7/php.ini && \
echo "daemonize yes" >> /etc/redis.conf && \
ln -s /dev/null /var/log/nginx/access.log && \
ln -s /dev/stdout /var/log/nginx/error.log && \
cp /usr/share/zoneinfo/Europe/Berlin /etc/localtime && \
echo "Europe/Berlin" > /etc/timezone && \
(crontab -l ; echo "* * * * * php /html/artisan schedule:run >> /dev/null 2>&1") | crontab -
WORKDIR /html
EXPOSE 80
CMD chown -R root:nginx storage/logs/metager bootstrap/cache && \
chmod -R g+w storage/logs/metager bootstrap/cache && \
crond -L /dev/stdout && \
php-fpm7
README.md
CHANGELOG.md
vendor/*
node_modules/*
storage/logs/*
\ No newline at end of file
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
use Log;
use Monospice\LaravelRedisSentinel\RedisSentinel;
class AppendLogs extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'logs:gather';
const LOGKEY = "metager.logs";
/**
* The console command description.
*
* @var string
*/
protected $description = 'Retrieves all Log Entries from Redis and writes them to file';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$redis = null;
if (env("REDIS_CACHE_DRIVER", "redis") === "redis") {
$redis = Redis::connection('cache');
} elseif (env("REDIS_CACHE_DRIVER", "redis") === "redis-sentinel") {
$redis = RedisSentinel::connection('cache');
}
if ($redis === null) {
Log::error("No valid Redis Connection specified");
return;
}
$elements = [];
$reply = $redis->pipeline(function ($pipe) use ($elements) {
$pipe->lrange(\App\Console\Commands\AppendLogs::LOGKEY, 0, -1);
$pipe->del(\App\Console\Commands\AppendLogs::LOGKEY);
});
$elements = $reply[0];
if (!is_array($elements) || sizeof($elements) <= 0) {
return;
}
if (file_put_contents(\App\MetaGer::getMGLogFile(), implode(PHP_EOL, $elements) . PHP_EOL, FILE_APPEND) === false) {
Log::error("Konnte Log Zeile(n) nicht schreiben");
$redis->lpush(array_reverse($elements));
} else {
Log::info("Added " . sizeof($elements) . " lines to todays log!");
}
}
}
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
class CacheGC extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'cache:gc';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Cleans up every expired cache File';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$cachedir = storage_path('framework/cache');
$lockfile = $cachedir . "/cache.gc";
if (file_exists($lockfile)) {
return;
} else {
touch($lockfile);
}
try {
foreach (new \DirectoryIterator($cachedir) as $fileInfo) {
if ($fileInfo->isDot()) {
continue;
}
$file = $fileInfo->getPathname();
$basename = basename($file);
if (!is_dir($file) && $basename !== "cache.gc" && $basename !== ".gitignore") {
$fp = fopen($file, 'r');
$delete = false;
try {
$time = intval(fread($fp, 10));
if ($time < time()) {
$delete = true;
}
} finally {
fclose($fp);
}
if ($delete) {
unlink($file);
}
} else if (is_dir($file)) {
// Delete Directory if empty
try {
rmdir($file);
} catch (\ErrorException $e) {
}
}
}
} finally {
unlink($lockfile);
}
}
}
...@@ -2,7 +2,6 @@ ...@@ -2,7 +2,6 @@
namespace App\Console\Commands; namespace App\Console\Commands;
use Cache;
use Illuminate\Console\Command; use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis; use Illuminate\Support\Facades\Redis;
use Log; use Log;
...@@ -25,6 +24,9 @@ class RequestFetcher extends Command ...@@ -25,6 +24,9 @@ class RequestFetcher extends Command
protected $shouldRun = true; protected $shouldRun = true;
protected $multicurl = null; protected $multicurl = null;
protected $oldMultiCurl = null;
protected $maxFetchedDocuments = 10000;
protected $fetchedDocuments = 0;
protected $proxyhost, $proxyuser, $proxypassword; protected $proxyhost, $proxyuser, $proxypassword;
/** /**
...@@ -50,13 +52,32 @@ class RequestFetcher extends Command ...@@ -50,13 +52,32 @@ class RequestFetcher extends Command
*/ */
public function handle() public function handle()
{ {
$pids = [];
pcntl_async_signals(true); $pidFile = "/tmp/fetcher";
pcntl_signal(SIGINT, [$this, "sig_handler"]); pcntl_signal(SIGINT, [$this, "sig_handler"]);
pcntl_signal(SIGTERM, [$this, "sig_handler"]); pcntl_signal(SIGTERM, [$this, "sig_handler"]);
pcntl_signal(SIGHUP, [$this, "sig_handler"]); pcntl_signal(SIGHUP, [$this, "sig_handler"]);
// Redis might not be available now
for ($count = 0; $count < 10; $count++) {
try {
Redis::connection();
break;
} catch (\Predis\Connection\ConnectionException $e) {
if ($count >= 9) {
// If its not available after 10 seconds we will exit
return;
}
sleep(1);
}
}
touch($pidFile);
if (!file_exists($pidFile)) {
return;
}
try { try {
$blocking = false; $blocking = false;
while ($this->shouldRun) { while ($this->shouldRun) {
...@@ -74,49 +95,80 @@ class RequestFetcher extends Command ...@@ -74,49 +95,80 @@ class RequestFetcher extends Command
if (!empty($currentJob)) { if (!empty($currentJob)) {
$currentJob = json_decode($currentJob, true); $currentJob = json_decode($currentJob, true);
$ch = $this->getCurlHandle($currentJob); $ch = $this->getCurlHandle($currentJob);
curl_multi_add_handle($this->multicurl, $ch); if (curl_multi_add_handle($this->multicurl, $ch) !== 0) {
$this->shouldRun = false;
Log::error("Couldn't add Handle to multicurl");
break;
}
$this->fetchedDocuments++;
if ($this->fetchedDocuments > $this->maxFetchedDocuments) {
Log::info("Reinitializing Multicurl after " . $this->fetchedDocuments . " requests.");
$this->oldMultiCurl = $this->multicurl;
$this->multicurl = curl_multi_init();
$this->fetchedDocuments = 0;
}
$blocking = false; $blocking = false;
$active = true; $active = true;
} }
$answerRead = false; $answerRead = $this->readMultiCurl($this->multicurl);
while (($info = curl_multi_info_read($this->multicurl)) !== false) { if ($this->oldMultiCurl != null) {
$answerRead = true; $this->readMultiCurl($this->oldMultiCurl);
$infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE); $messagesLeft = -1;
$infos = explode(";", $infos); if (curl_multi_info_read($this->oldMultiCurl, $messagesLeft) === false) {
$resulthash = $infos[0]; if ($messagesLeft = 0) {
$cacheDurationMinutes = intval($infos[1]); Log::debug("Removing finished multicurl handle");
$responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE); curl_multi_close($this->oldMultiCurl);
$body = ""; $this->oldMultiCurl = null;
}
$error = curl_error($info["handle"]);
if (!empty($error)) {
Log::error($error);
} }
if ($responseCode !== 200) {
Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
} else {
$body = \curl_multi_getcontent($info["handle"]);
}
Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
$pipe->set($resulthash, $body);
$pipe->expire($resulthash, 60);
Cache::put($resulthash, $body, $cacheDurationMinutes * 60);
});
\curl_multi_remove_handle($this->multicurl, $info["handle"]);
} }
if (!$active && !$answerRead) { if (!$active && !$answerRead) {
$blocking = true; $blocking = true;
} else {
usleep(50 * 1000);
} }
} }
} finally { } finally {
unlink($pidFile);
curl_multi_close($this->multicurl); curl_multi_close($this->multicurl);