Commit 8828d367 authored by Phil Höfer's avatar Phil Höfer
Browse files

Merge branch 'development' into '932-dark-stylesheet-alternative'

# Conflicts:
#   resources/less/metager/pages/resultpage/result-page.less
#   resources/less/metager/pages/resultpage/result.less
#   resources/less/metager/variables.less
#   webpack.mix.js
parents 1987d109 ed7b4825
README.md
CHANGELOG.md
docker-compose.yml
Dockerfile
Dockerfile
\ No newline at end of file
APP_ENV=local
APP_DEBUG=true
APP_LOG_LEVEL=debug
LOG_CHANNEL=stderr
APP_KEY=
APP_URL=http://localhost
BOT_PROTECTION=true
DB_CONNECTION=mysql
DB_HOST=mgdb
DB_PORT=3306
......@@ -15,11 +18,12 @@ REDIS_RESULT_CONNECTION=default
REDIS_RESULT_CACHE_DURATION=60
BROADCAST_DRIVER=log
CACHE_DRIVER=database
CACHE_DRIVER=redis
SESSION_DRIVER=file
QUEUE_CONNECTION=sync
REDIS_HOST=127.0.0.1
REDIS_CACHE_HOST=redis
REDIS_HOST=redis
REDIS_PASSWORD=null
REDIS_PORT=6379
......
variables:
DOCKER_HOST: "tcp://docker-dind.gitlab:2375"
AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS: "--network host"
POSTGRES_ENABLED: "false"
CODE_QUALITY_DISABLED: "true"
CONTAINER_SCANNING_DISABLED: "true"
......@@ -156,11 +157,10 @@ development:
- $INCREMENTAL_ROLLOUT_ENABLED
- $INCREMENTAL_ROLLOUT_MODE
production:
variables:
ADDITIONAL_HOSTS: "www.metager.de,metager.org,www.metager.org,metager.es,www.metager.es"
ADDITIONAL_HOSTS: "www.metager.de,metager.org,www.metager.org,metager.es,www.metager.es,klassik.metager.org"
HELM_UPGRADE_VALUES_FILE: .gitlab/production-values.yaml
ROLLOUT_RESOURCE_TYPE: deployment
environment:
url: https://metager.de
url: https://metager.de
\ No newline at end of file
service:
externalPort: 80
internalPort: 80
hpa:
minReplicas: 1
maxReplicas: 5
podDisruptionBudget:
enabled: true
minAvailable: 1
maxUnavailable:
ingress:
annotations:
certmanager.k8s.io/cluster-issuer: letsencrypt-prod
......
......@@ -2,11 +2,16 @@ service:
externalPort: 80
internalPort: 80
hpa:
minReplicas: 10
maxReplicas: 40
minReplicas: 5
maxReplicas: 25
podDisruptionBudget:
enabled: true
minAvailable: 5
maxUnavailable:
ingress:
annotations:
certmanager.k8s.io/cluster-issuer: letsencrypt-prod
nginx.ingress.kubernetes.io/ssl-redirect: "false"
nginx.ingress.kubernetes.io/configuration-snippet: |
if ($host = "www.metager.de") {
return 301 https://metager.de$request_uri;
......@@ -16,4 +21,7 @@ ingress:
}
if ($host = "www.metager.es") {
return 301 https://metager.es$request_uri;
}
if ($host = "klassik.metager.org") {
return 301 https://metager.de$request_uri;
}
\ No newline at end of file
FROM nginx
FROM alpine:3.11.3
RUN apt -y update && apt -y install php-fpm \
RUN apk add --update \
nginx \
tzdata \
ca-certificates \
cron \
dcron \
zip \
php7.3-common \
php7.3-curl \
php7.3-mbstring \
php7.3-sqlite3 \
php7.3-mysql \
php7.3-xml \
php7.3-zip \
php7.3-redis \
php7.3-gd \
redis-server
RUN sed -i 's/listen.owner = www-data/listen.owner = nginx/g' /etc/php/7.3/fpm/pool.d/www.conf && \
sed -i 's/listen.group = www-data/listen.group = nginx/g' /etc/php/7.3/fpm/pool.d/www.conf && \
sed -i 's/pm.max_children = 5/pm.max_children = 100/g' /etc/php/7.3/fpm/pool.d/www.conf && \
sed -i 's/pm.start_servers = 2/pm.start_servers = 25/g' /etc/php/7.3/fpm/pool.d/www.conf && \
sed -i 's/pm.min_spare_servers = 1/pm.min_spare_servers = 5/g' /etc/php/7.3/fpm/pool.d/www.conf && \
sed -i 's/pm.max_spare_servers = 3/pm.max_spare_servers = 25/g' /etc/php/7.3/fpm/pool.d/www.conf && \
sed -i 's/user = www-data/user = nginx/g' /etc/php/7.3/fpm/pool.d/www.conf && \
sed -i 's/group = www-data/group = nginx/g' /etc/php/7.3/fpm/pool.d/www.conf && \
sed -i 's/;cgi.fix_pathinfo=1/cgi.fix_pathinfo=0/g' /etc/php/7.3/fpm/php.ini && \
mkdir /html
# Set correct timezone
RUN ln -fs /usr/share/zoneinfo/Europe/Berlin /etc/localtime && dpkg-reconfigure -f noninteractive tzdata
# Add Cronjob for Laravel
RUN (crontab -l ; echo "* * * * * php /html/artisan schedule:run >> /dev/null 2>&1") | crontab
redis \
php7 \
php7-fpm \
php7-common \
php7-curl \
php7-mbstring \
php7-sqlite3 \
php7-pdo_mysql \
php7-pdo_sqlite \
php7-dom \
php7-simplexml \
php7-tokenizer \
php7-zip \
php7-redis \
php7-gd \
php7-json \
php7-pcntl \
php7-opcache \
php7-fileinfo \
&& rm -rf /var/cache/apk/*
WORKDIR /html
EXPOSE 80
RUN sed -i 's/;error_log = log\/php7\/error.log/error_log = \/dev\/stderr/g' /etc/php7/php-fpm.conf && \
sed -i 's/;daemonize = yes/daemonize = no/g' /etc/php7/php-fpm.conf && \
sed -i 's/listen = 127.0.0.1:9000/listen = 9000/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;request_terminate_timeout = 0/request_terminate_timeout = 30/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;request_terminate_timeout_track_finished = no/request_terminate_timeout_track_finished = yes/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;decorate_workers_output = no/decorate_workers_output = no/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;catch_workers_output = yes/catch_workers_output = yes/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = nobody/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_children = 5/pm.max_children = 100/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.start_servers = 2/pm.start_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.min_spare_servers = 1/pm.min_spare_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_spare_servers = 3/pm.max_spare_servers = 25/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/user = www-data/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = www-data/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;cgi.fix_pathinfo=1/cgi.fix_pathinfo=0/g' /etc/php7/php.ini && \
sed -i 's/expose_php = On/expose_php = Off/g' /etc/php7/php.ini && \
# Opcache configuration
sed -i 's/;opcache.enable=1/opcache.enable=1/g' /etc/php7/php.ini && \
sed -i 's/;opcache.memory_consumption=128/opcache.memory_consumption=128/g' /etc/php7/php.ini && \
sed -i 's/;opcache.interned_strings_buffer=8/opcache.interned_strings_buffer=8/g' /etc/php7/php.ini && \
sed -i 's/;opcache.max_accelerated_files=10000/opcache.max_accelerated_files=10000/g' /etc/php7/php.ini && \
sed -i 's/;opcache.max_wasted_percentage=5/opcache.max_wasted_percentage=5/g' /etc/php7/php.ini && \
sed -i 's/;opcache.validate_timestamps=1/opcache.validate_timestamps=1/g' /etc/php7/php.ini && \
sed -i 's/;opcache.revalidate_freq=2/opcache.revalidate_freq=300/g' /etc/php7/php.ini && \
echo "daemonize yes" >> /etc/redis.conf && \
ln -s /dev/null /var/log/nginx/access.log && \
ln -s /dev/stdout /var/log/nginx/error.log && \
cp /usr/share/zoneinfo/Europe/Berlin /etc/localtime && \
echo "Europe/Berlin" > /etc/timezone && \
(crontab -l ; echo "* * * * * php /html/artisan schedule:run >> /dev/null 2>&1") | crontab -
COPY config/nginx.conf /etc/nginx/nginx.conf
COPY config/nginx-default.conf /etc/nginx/conf.d/default.conf
RUN sed -i 's/fastcgi_pass phpfpm:9000;/fastcgi_pass localhost:9000;/g' /etc/nginx/conf.d/default.conf
COPY --chown=root:nginx . /html
WORKDIR /html
EXPOSE 80
CMD chown -R root:nginx storage/logs/metager bootstrap/cache && \
chmod -R g+w storage/logs/metager bootstrap/cache && \
/etc/init.d/cron start && \
/etc/init.d/php7.3-fpm start && \
/etc/init.d/nginx start && \
/etc/init.d/redis-server start && \
su -s /bin/bash -c 'php artisan requests:fetcher' nginx
crond -L /dev/stdout && \
php-fpm7
FROM alpine:3.11.3
RUN apk add --update \
nginx \
tzdata \
ca-certificates \
dcron \
zip \
redis \
php7 \
php7-fpm \
php7-common \
php7-curl \
php7-mbstring \
php7-sqlite3 \
php7-pdo_mysql \
php7-pdo_sqlite \
php7-dom \
php7-simplexml \
php7-tokenizer \
php7-zip \
php7-redis \
php7-gd \
php7-json \
php7-pcntl \
php7-fileinfo \
&& rm -rf /var/cache/apk/*
WORKDIR /html
RUN sed -i 's/;error_log = log\/php7\/error.log/error_log = \/dev\/stderr/g' /etc/php7/php-fpm.conf && \
sed -i 's/;daemonize = yes/daemonize = no/g' /etc/php7/php-fpm.conf && \
sed -i 's/listen = 127.0.0.1:9000/listen = 9000/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;request_terminate_timeout = 0/request_terminate_timeout = 30/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;request_terminate_timeout_track_finished = no/request_terminate_timeout_track_finished = yes/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;decorate_workers_output = no/decorate_workers_output = no/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;catch_workers_output = yes/catch_workers_output = yes/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/user = nobody/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = nobody/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_children = 5/pm.max_children = 100/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.start_servers = 2/pm.start_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.min_spare_servers = 1/pm.min_spare_servers = 5/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/pm.max_spare_servers = 3/pm.max_spare_servers = 25/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/user = www-data/user = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/group = www-data/group = nginx/g' /etc/php7/php-fpm.d/www.conf && \
sed -i 's/;cgi.fix_pathinfo=1/cgi.fix_pathinfo=0/g' /etc/php7/php.ini && \
sed -i 's/expose_php = On/expose_php = Off/g' /etc/php7/php.ini && \
echo "daemonize yes" >> /etc/redis.conf && \
ln -s /dev/null /var/log/nginx/access.log && \
ln -s /dev/stdout /var/log/nginx/error.log && \
cp /usr/share/zoneinfo/Europe/Berlin /etc/localtime && \
echo "Europe/Berlin" > /etc/timezone && \
(crontab -l ; echo "* * * * * php /html/artisan schedule:run >> /dev/null 2>&1") | crontab -
WORKDIR /html
EXPOSE 80
CMD chown -R root:nginx storage/logs/metager bootstrap/cache && \
chmod -R g+w storage/logs/metager bootstrap/cache && \
crond -L /dev/stdout && \
php-fpm7
README.md
CHANGELOG.md
vendor/*
node_modules/*
storage/logs/*
\ No newline at end of file
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
use Log;
use Monospice\LaravelRedisSentinel\RedisSentinel;
class AppendLogs extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'logs:gather';
const LOGKEY = "metager.logs";
/**
* The console command description.
*
* @var string
*/
protected $description = 'Retrieves all Log Entries from Redis and writes them to file';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$redis = null;
if (env("REDIS_CACHE_DRIVER", "redis") === "redis") {
$redis = Redis::connection('cache');
} elseif (env("REDIS_CACHE_DRIVER", "redis") === "redis-sentinel") {
$redis = RedisSentinel::connection('cache');
}
if ($redis === null) {
Log::error("No valid Redis Connection specified");
return;
}
$elements = [];
$reply = $redis->pipeline(function ($pipe) use ($elements) {
$pipe->lrange(\App\Console\Commands\AppendLogs::LOGKEY, 0, -1);
$pipe->del(\App\Console\Commands\AppendLogs::LOGKEY);
});
$elements = $reply[0];
if (!is_array($elements) || sizeof($elements) <= 0) {
return;
}
if (file_put_contents(\App\MetaGer::getMGLogFile(), implode(PHP_EOL, $elements) . PHP_EOL, FILE_APPEND) === false) {
Log::error("Konnte Log Zeile(n) nicht schreiben");
$redis->lpush(array_reverse($elements));
} else {
Log::info("Added " . sizeof($elements) . " lines to todays log!");
}
}
}
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
class CacheGC extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'cache:gc';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Cleans up every expired cache File';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$cachedir = storage_path('framework/cache');
$lockfile = $cachedir . "/cache.gc";
if (file_exists($lockfile)) {
return;
} else {
touch($lockfile);
}
try {
foreach (new \DirectoryIterator($cachedir) as $fileInfo) {
if ($fileInfo->isDot()) {
continue;
}
$file = $fileInfo->getPathname();
$basename = basename($file);
if (!is_dir($file) && $basename !== "cache.gc" && $basename !== ".gitignore") {
$fp = fopen($file, 'r');
$delete = false;
try {
$time = intval(fread($fp, 10));
if ($time < time()) {
$delete = true;
}
} finally {
fclose($fp);
}
if ($delete) {
unlink($file);
}
} else if (is_dir($file)) {
// Delete Directory if empty
try {
rmdir($file);
} catch (\ErrorException $e) {
}
}
}
} finally {
unlink($lockfile);
}
}
}
......@@ -2,7 +2,6 @@
namespace App\Console\Commands;
use Cache;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
use Log;
......@@ -25,6 +24,9 @@ class RequestFetcher extends Command
protected $shouldRun = true;
protected $multicurl = null;
protected $oldMultiCurl = null;
protected $maxFetchedDocuments = 10000;
protected $fetchedDocuments = 0;
protected $proxyhost, $proxyuser, $proxypassword;
/**
......@@ -50,13 +52,32 @@ class RequestFetcher extends Command
*/
public function handle()
{
$pids = [];
pcntl_async_signals(true);
$pidFile = "/tmp/fetcher";
pcntl_signal(SIGINT, [$this, "sig_handler"]);
pcntl_signal(SIGTERM, [$this, "sig_handler"]);
pcntl_signal(SIGHUP, [$this, "sig_handler"]);
// Redis might not be available now
for ($count = 0; $count < 10; $count++) {
try {
Redis::connection();
break;
} catch (\Predis\Connection\ConnectionException $e) {
if ($count >= 9) {
// If its not available after 10 seconds we will exit
return;
}
sleep(1);
}
}
touch($pidFile);
if (!file_exists($pidFile)) {
return;
}
try {
$blocking = false;
while ($this->shouldRun) {
......@@ -74,49 +95,80 @@ class RequestFetcher extends Command
if (!empty($currentJob)) {
$currentJob = json_decode($currentJob, true);
$ch = $this->getCurlHandle($currentJob);
curl_multi_add_handle($this->multicurl, $ch);
if (curl_multi_add_handle($this->multicurl, $ch) !== 0) {
$this->shouldRun = false;
Log::error("Couldn't add Handle to multicurl");
break;
}
$this->fetchedDocuments++;
if ($this->fetchedDocuments > $this->maxFetchedDocuments) {
Log::info("Reinitializing Multicurl after " . $this->fetchedDocuments . " requests.");
$this->oldMultiCurl = $this->multicurl;
$this->multicurl = curl_multi_init();
$this->fetchedDocuments = 0;
}
$blocking = false;
$active = true;
}
$answerRead = false;
while (($info = curl_multi_info_read($this->multicurl)) !== false) {
$answerRead = true;
$infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE);
$infos = explode(";", $infos);
$resulthash = $infos[0];
$cacheDurationMinutes = intval($infos[1]);
$responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
$body = "";
$error = curl_error($info["handle"]);
if (!empty($error)) {
Log::error($error);
$answerRead = $this->readMultiCurl($this->multicurl);
if ($this->oldMultiCurl != null) {
$this->readMultiCurl($this->oldMultiCurl);
$messagesLeft = -1;
if (curl_multi_info_read($this->oldMultiCurl, $messagesLeft) === false) {
if ($messagesLeft = 0) {
Log::debug("Removing finished multicurl handle");
curl_multi_close($this->oldMultiCurl);
$this->oldMultiCurl = null;
}
}
if ($responseCode !== 200) {
Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
} else {
$body = \curl_multi_getcontent($info["handle"]);
}
Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
$pipe->set($resulthash, $body);
$pipe->expire($resulthash, 60);
Cache::put($resulthash, $body, $cacheDurationMinutes * 60);
});
\curl_multi_remove_handle($this->multicurl, $info["handle"]);
}
if (!$active && !$answerRead) {
$blocking = true;
} else {
usleep(50 * 1000);
}
}
} finally {
unlink($pidFile);
curl_multi_close($this->multicurl);
}
foreach ($pids as $tmppid) {
\pcntl_waitpid($tmppid, $status, WNOHANG);
}
private function readMultiCurl($mc)
{
$answerRead = false;
while (($info = curl_multi_info_read($mc)) !== false) {
try {
$answerRead = true;
$infos = curl_getinfo($info["handle"], CURLINFO_PRIVATE);
$infos = explode(";", $infos);
$resulthash = $infos[0];
$cacheDurationMinutes = intval($infos[1]);
$responseCode = curl_getinfo($info["handle"], CURLINFO_HTTP_CODE);
$body = "";
$error = curl_error($info["handle"]);
if (!empty($error)) {
Log::error($error);
}
if ($responseCode !== 200) {
Log::debug("Got responsecode " . $responseCode . " fetching \"" . curl_getinfo($info["handle"], CURLINFO_EFFECTIVE_URL) . "\n");
} else {
$body = \curl_multi_getcontent($info["handle"]);
}
Redis::pipeline(function ($pipe) use ($resulthash, $body, $cacheDurationMinutes) {
$pipe->set($resulthash, $body);
$pipe->expire($resulthash, 60);
});
} finally {
\curl_multi_remove_handle($mc, $info["handle"]);
}
}
return $answerRead;
}
private function getCurlHandle($job)
......@@ -129,11 +181,11 @@ class RequestFetcher extends Command
CURLOPT_RETURNTRANSFER => 1,
CURLOPT_USERAGENT => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_CONNECTTIMEOUT => 10,
CURLOPT_CONNECTTIMEOUT => 2,
CURLOPT_MAXCONNECTS => 500,
CURLOPT_LOW_SPEED_LIMIT => 500,
CURLOPT_LOW_SPEED_LIMIT => 50000,
CURLOPT_LOW_SPEED_TIME => 5,
CURLOPT_TIMEOUT => 10,
CURLOPT_TIMEOUT => 7,
));
if (!empty($this->proxyhost) && !empty($this->proxyport) && !empty($this->proxyuser) && !empty($this->proxypassword)) {
......
......@@ -27,7 +27,7 @@ class Kernel extends ConsoleKernel
{
$schedule->command('requests:gather')->everyFifteenMinutes();
$schedule->command('requests:useragents')->everyFiveMinutes();
$schedule->command('cache:gc')->hourly();
$schedule->command('logs:gather')->everyMinute();
$schedule->call(function () {
DB::table('monthlyrequests')->truncate();
......
......@@ -77,7 +77,7 @@ class AdminInterface extends Controller