Commit b21d7ddf authored by Dominik Hebeler's avatar Dominik Hebeler

Merge branch '907-put-metager-into-our-kubernetes-cluster' into 'development'

Resolve "Put MetaGer into our Kubernetes Cluster"

Closes #907

See merge request !1481
parents 7e621590 67951c98
update(144.76.113.134):
tags:
- 144.76.113.134
variables:
DOCKER_HOST: "tcp://docker-dind.gitlab:2375"
POSTGRES_ENABLED: "false"
CODE_QUALITY_DISABLED: "true"
CONTAINER_SCANNING_DISABLED: "true"
DAST_DISABLED: "true"
DEPENDENCY_SCANNING_DISABLED: "true"
LICENSE_MANAGEMENT_DISABLED: "true"
PERFORMANCE_DISABLED: "true"
SAST_DISABLED: "true"
TEST_DISABLED: "true"
include:
- template: Jobs/Build.gitlab-ci.yml
- template: Jobs/Deploy.gitlab-ci.yml
stages:
- prepare
- build
- deploy # dummy stage to follow the template guidelines
- review
- dast
- staging
- canary
- development
- production
- incremental rollout 10%
- incremental rollout 25%
- incremental rollout 50%
- incremental rollout 100%
- performance
- cleanup
build:
services:
# Prepares the secret files that we cannot or don't want to share with public
prepare_secrets:
stage: prepare
image: alpine:latest
script:
- cp $ENVFILE .env
- cp $SUMAS config/sumas.json
- cp $SUMASEN config/sumasEn.json
- cp $BLACKLISTURL config/blacklistUrl.txt
- cp $BLACKLISTDOMAINS config/blacklistDomains.txt
- cp $ADBLACKLISTURL config/adBlacklistUrl.txt
- cp $ADBLACKLISTDOMAINS config/adBlacklistDomains.txt
- cp $SPAM config/spam.txt
- cp $USERSSEEDER database/seeds/UsersSeeder.php
- cp database/useragents.sqlite.example database/useragents.sqlite
- sed -i 's/^REDIS_PASSWORD=.*/REDIS_PASSWORD=null/g' .env
artifacts:
paths:
- .env
- config/sumas.json
- config/sumasEn.json
- config/blacklistUrl.txt
- config/blacklistDomains.txt
- config/adBlacklistUrl.txt
- config/adBlacklistDomains.txt
- config/spam.txt
- database/seeds/UsersSeeder.php
- database/useragents.sqlite
only:
- master@open-source/MetaGer
- branches
- tags
prepare_node:
stage: prepare
image: node:10
before_script:
# Abhängigkeiten überprüfen
- which composer
- which git
- which php
- which sqlite3
- npm install
script:
- sh build.sh
variables:
STAGE: production
update(metager2):
tags:
- metager2
- npm run prod
artifacts:
paths:
- public/js/
- public/css/
- public/mix-manifest.json
cache:
# Cache per Branch
key: "node-$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG"
paths:
- node_modules
only:
- master@open-source/MetaGer
before_script:
# Abhängigkeiten überprüfen
- which composer
- which git
- which php
- which sqlite3
- branches
- tags
prepare_composer:
stage: prepare
image: prooph/composer:7.3
script:
- sh build.sh
- composer install
artifacts:
paths:
- vendor
cache:
key: "composer-$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG"
paths:
- vendor
review:
variables:
STAGE: production
update(metager3.de):
tags:
- metager3
only:
- development@open-source/MetaGer
before_script:
# Abhängigkeiten überprüfen
- which composer
- which git
- which php
- which sqlite3
HELM_UPGRADE_EXTRA_ARGS: --set service.externalPort=80 --set service.internalPort=80 --set service.commonName= --set ingress.tls.enabled=false --set ingress.annotations.kubernetes\.io/tls-acme="false" --set ingress.annotations.nginx\.ingress\.kubernetes\.io/ssl-redirect="false"
ROLLOUT_RESOURCE_TYPE: deployment
.development: &development_template
extends: .auto-deploy
stage: development
script:
- sh build.sh
- auto-deploy check_kube_domain
- auto-deploy download_chart
- auto-deploy ensure_namespace
- auto-deploy initialize_tiller
- auto-deploy create_secret
- auto-deploy deploy
- auto-deploy delete canary
- auto-deploy delete rollout
- auto-deploy persist_environment_url
environment:
name: development
url: https://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN
artifacts:
paths: [environment_url.txt]
development:
<<: *development_template
only:
refs:
- development
kubernetes: active
variables:
HELM_UPGRADE_EXTRA_ARGS: --set service.externalPort=80 --set service.internalPort=80 --set service.commonName= --set ingress.annotations.certmanager\.k8s\.io/cluster-issuer=letsencrypt-prod
ROLLOUT_RESOURCE_TYPE: deployment
environment:
name: development
url: https://metager3.de
except:
variables:
- $STAGING_ENABLED
- $CANARY_ENABLED
- $INCREMENTAL_ROLLOUT_ENABLED
- $INCREMENTAL_ROLLOUT_MODE
production:
variables:
STAGE: development
\ No newline at end of file
HELM_UPGRADE_EXTRA_ARGS: --set service.externalPort=80 --set service.internalPort=80 --set service.commonName= --set ingress.annotations.certmanager\.k8s\.io/cluster-issuer=letsencrypt-prod
ROLLOUT_RESOURCE_TYPE: deployment
environment:
url: https://metager.de
FROM debian:buster
FROM nginx
RUN apt-get update && apt-get install -y \
composer \
php7.2 \
php-mbstring \
php7.2-xml\
php-zip \
php-gd \
php-sqlite3 \
php-mysql \
php-curl \
redis-server \
sqlite3 \
nodejs \
libpng-dev \
unzip \
npm
RUN npm install gulp -g
RUN apt -y update && apt -y install php-fpm \
ca-certificates \
cron \
zip \
php7.3-common \
php7.3-curl \
php7.3-mbstring \
php7.3-sqlite3 \
php7.3-mysql \
php7.3-xml \
php7.3-zip \
php7.3-redis \
php7.3-gd \
redis-server
COPY . /app
WORKDIR app
RUN mv config/sumas.xml.example config/sumas.xml && mv .env.example .env
RUN composer install --no-plugins --no-scripts
RUN npm install
RUN npm run dev
RUN sed -i 's/listen.owner = www-data/listen.owner = nginx/g' /etc/php/7.3/fpm/pool.d/www.conf && \
sed -i 's/listen.group = www-data/listen.group = nginx/g' /etc/php/7.3/fpm/pool.d/www.conf && \
sed -i 's/user = www-data/user = nginx/g' /etc/php/7.3/fpm/pool.d/www.conf && \
sed -i 's/group = www-data/group = nginx/g' /etc/php/7.3/fpm/pool.d/www.conf && \
sed -i 's/;cgi.fix_pathinfo=1/cgi.fix_pathinfo=0/g' /etc/php/7.3/fpm/php.ini && \
mkdir /html
RUN php artisan key:generate
# Set correct timezone
RUN ln -fs /usr/share/zoneinfo/Europe/Berlin /etc/localtime && dpkg-reconfigure -f noninteractive tzdata
CMD redis-server --daemonize yes && php artisan serve --host=0.0.0.0
# Add Cronjob for Laravel
RUN (crontab -l ; echo "* * * * * php /html/artisan schedule:run >> /dev/null 2>&1") | crontab
EXPOSE 8000
WORKDIR /html
EXPOSE 80
COPY config/nginx.conf /etc/nginx/nginx.conf
COPY config/nginx-default.conf /etc/nginx/conf.d/default.conf
COPY . /html
CMD /etc/init.d/cron start && \
/etc/init.d/php7.3-fpm start && \
/etc/init.d/nginx start && \
/etc/init.d/redis-server start && \
chmod -R 0777 /html/storage && \
chmod -R 0777 /html/bootstrap/cache && \
php artisan worker:spawner
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
class WorkerSpawner extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'worker:spawner';
/**
* The console command description.
*
* @var string
*/
protected $description = 'This command makes sure that enough worker processes are spawned';
protected $shouldRun = true;
protected $processes = [];
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
pcntl_async_signals(true);
pcntl_signal(SIGINT, [$this, "sig_handler"]);
pcntl_signal(SIGTERM, [$this, "sig_handler"]);
pcntl_signal(SIGHUP, [$this, "sig_handler"]);
try {
$counter = 0;
while ($this->shouldRun) {
$counter++;
$counter = $counter % 10;
$length = Redis::llen("queues:default");
if ($length > 0) {
while (true) {
usleep(50 * 1000);
if (Redis::llen("queues:default") !== $length) {
$length = Redis::llen("queues:default");
} else {
break;
}
}
$jobs = Redis::lrange("queues:default", 0, -1);
$length = sizeof($jobs) + 5;
$ids = $this->getJobIds($jobs);
for ($i = 0; $i <= $length; $i++) {
$this->processes[] = $this->spawnWorker();
}
while (sizeof($ids) > 0) {
$jobs = Redis::lrange("queues:default", 0, -1);
$newIds = $this->getJobIds($jobs);
foreach ($ids as $index => $id) {
foreach ($newIds as $newId) {
if ($id === $newId) {
continue 2;
}
}
unset($ids[$index]);
break;
}
}
} else {
usleep(100 * 1000); // Sleep for 100ms
}
if ($counter === 0) {
$newProcs = [];
foreach ($this->processes as $process) {
$infos = proc_get_status($process["process"]);
if (!$infos["running"]) {
fclose($process["pipes"][1]);
proc_close($process["process"]);
} else {
$newProcs[] = $process;
}
}
$this->processes = $newProcs;
}
}
} finally {
foreach ($this->processes as $process) {
fclose($process["pipes"][1]);
proc_close($process["process"]);
}
}
}
private function getJobIds($jobs)
{
$result = [];
foreach ($jobs as $job) {
$result[] = json_decode($job, true)["id"];
}
return $result;
}
private function sig_handler($sig)
{
$this->shouldRun = false;
echo ("Terminating Process\n");
}
private function spawnWorker()
{
$descriptorspec = array(
0 => array("pipe", "r"), // STDIN ist eine Pipe, von der das Child liest
1 => array("pipe", "w"), // STDOUT ist eine Pipe, in die das Child schreibt
2 => array("file", "/tmp/worker-error.txt", "a"), // STDERR ist eine Datei,
// in die geschrieben wird
);
$cwd = getcwd();
$env = array();
$process = proc_open('php artisan queue:work --stop-when-empty --sleep=1', $descriptorspec, $pipes, $cwd, $env);
if (is_resource($process)) {
fclose($pipes[0]);
\stream_set_blocking($pipes[1], 0);
return [
"process" => $process,
"pipes" => $pipes,
"working" => false,
];
}
}
}
......@@ -331,10 +331,17 @@ class AdminInterface extends Controller
public function check()
{
$q = "";
$redis = Redis::connection('redisLogs');
if ($redis) {
$q = $redis->lrange("logs.search", -1, -1)[0];
$q = substr($q, strpos($q, "search=") + 7);
$logpath = \App\MetaGer::getMGLogFile();
if (file_exists($logpath)) {
$fp = @fopen($logpath, "r");
while (($line = fgets($fp)) !== false) {
if (!empty($line)) {
$q = $line;
}
}
fclose($fp);
$q = substr($q, strpos($q, "eingabe=") + 8);
}
return view('admin.check')
->with('title', 'Wer sucht was? - MetaGer')
......
......@@ -6,7 +6,7 @@ use Captcha;
use Carbon;
use Illuminate\Hashing\BcryptHasher as Hasher;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
use Illuminate\Support\Facades\Cache;
use Input;
class HumanVerification extends Controller
......@@ -15,25 +15,24 @@ class HumanVerification extends Controller
const EXPIRELONG = 60 * 60 * 24 * 14;
const EXPIRESHORT = 60 * 60 * 72;
public static function captcha(Request $request, Hasher $hasher, $id, $url = null)
public static function captcha(Request $request, Hasher $hasher, $id, $uid, $url = null)
{
$redis = Redis::connection('redisCache');
if ($url != null) {
$url = base64_decode(str_replace("<<SLASH>>", "/", $url));
} else {
$url = $request->input('url');
}
$userlist = Cache::get(HumanVerification::PREFIX . "." . $id, []);
$user = null;
if (sizeof($userlist) === 0 || empty($userlist[$uid])) {
return redirect('/');
} else {
$user = $userlist[$uid];
}
if ($request->getMethod() == 'POST') {
$user = $redis->hgetall(HumanVerification::PREFIX . "." . $id);
$user = ['uid' => $user["uid"],
'id' => $user["id"],
'unusedResultPages' => intval($user["unusedResultPages"]),
'whitelist' => filter_var($user["whitelist"], FILTER_VALIDATE_BOOLEAN),
'locked' => filter_var($user["locked"], FILTER_VALIDATE_BOOLEAN),
"lockedKey" => $user["lockedKey"],
];
$lockedKey = $user["lockedKey"];
$key = $request->input('captcha');
......@@ -41,11 +40,11 @@ class HumanVerification extends Controller
if (!$hasher->check($key, $lockedKey)) {
$captcha = Captcha::create("default", true);
$pipeline = $redis->pipeline();
$pipeline->hset(HumanVerification::PREFIX . "." . $id, 'lockedKey', $captcha["key"]);
$pipeline->expire(HumanVerification::PREFIX . "." . $id, $user["whitelist"] ? HumanVerification::EXPIRELONG : HumanVerification::EXPIRESHORT);
$pipeline->execute();
$user["lockedKey"] = $captcha["key"];
HumanVerification::saveUser($user);
return view('humanverification.captcha')->with('title', 'Bestätigung notwendig')
->with('uid', $user["uid"])
->with('id', $id)
->with('url', $url)
->with('image', $captcha["img"])
......@@ -54,13 +53,11 @@ class HumanVerification extends Controller
# If we can unlock the Account of this user we will redirect him to the result page
if ($user !== null && $user["locked"]) {
# The Captcha was correct. We can remove the key from the user
# If the sum of all users with that ip is too high we need to whitelist the user or they will receive a captcha again on the next request
$sum = 0;
$users = [];
$pipeline = $redis->pipeline();
$pipeline->hmset(HumanVerification::PREFIX . "." . $id, ['locked' => "0", 'lockedKey' => "", 'whitelist' => '1']);
$pipeline->expire(HumanVerification::PREFIX . "." . $id, $user["whitelist"] ? HumanVerification::EXPIRELONG : HumanVerification::EXPIRESHORT);
$pipeline->execute();
# Additionally we will whitelist him so he is not counted towards botnetwork
$user["locked"] = false;
$user["lockedKey"] = "";
$user["whitelist"] = true;
HumanVerification::saveUser($user);
return redirect($url);
} else {
return redirect('/');
......@@ -68,11 +65,11 @@ class HumanVerification extends Controller
}
}
$captcha = Captcha::create("default", true);
$pipeline = $redis->pipeline();
$pipeline->hset(HumanVerification::PREFIX . "." . $id, 'lockedKey', $captcha["key"]);
$pipeline->expire(HumanVerification::PREFIX . "." . $id, $user["whitelist"] ? HumanVerification::EXPIRELONG : HumanVerification::EXPIRESHORT);
$pipeline->execute();
$user["lockedKey"] = $captcha["key"];
HumanVerification::saveUser($user);
return view('humanverification.captcha')->with('title', 'Bestätigung notwendig')
->with('uid', $user["uid"])
->with('id', $id)
->with('url', $url)
->with('image', $captcha["img"]);
......@@ -104,9 +101,38 @@ class HumanVerification extends Controller
return redirect($url);
}
private static function saveUser($user)
{
$userList = Cache::get(HumanVerification::PREFIX . "." . $user["id"], []);
$userList[$user["uid"]] = $user;
if ($user["whitelist"]) {
$user["expiration"] = now()->addWeeks(2);
} else {
$user["expiration"] = now()->addHours(72);
}
Cache::put(HumanVerification::PREFIX . "." . $user["id"], $userList, now()->addWeeks(2));
}
private static function deleteUser($user)
{
$userList = Cache::get(HumanVerification::PREFIX . "." . $user["id"], []);
$newUserList = [];
$changed = false;
foreach ($userList as $uid => $userTmp) {
if ($userTmp["uid"] !== $user["uid"]) {
$newUserList[$userTmp["uid"]] = $userTmp;
} else {
$changed = true;
}
}
if ($changed) {
Cache::put(HumanVerification::PREFIX . "." . $user["id"], $userList, now()->addWeeks(2));
}
}
private static function removeUser($request, $uid)
{
$redis = Redis::connection('redisCache');
$ip = $request->ip();
$id = "";
if (HumanVerification::couldBeSpammer($ip)) {
......@@ -115,60 +141,34 @@ class HumanVerification extends Controller
$id = hash("sha512", $ip);
}
$userList = $redis->smembers(HumanVerification::PREFIX . "." . $id);
$pipe = $redis->pipeline();
foreach ($userList as $userid) {
$pipe->hgetall(HumanVerification::PREFIX . "." . $userid);
$userlist = Cache::get(HumanVerification::PREFIX . "." . $id, []);
$user = null;
if (sizeof($userlist) === 0 || empty($userlist[$uid])) {
return;
} else {
$user = $userlist[$uid];
}
$usersData = $pipe->execute();
$user = [];
$users = [];
$sum = 0;
foreach ($usersData as $userTmp) {
if (empty($userTmp)) {
continue;
}
$userNew = ['uid' => $userTmp["uid"],
'id' => $userTmp["id"],
'unusedResultPages' => intval($userTmp["unusedResultPages"]),
'whitelist' => filter_var($userTmp["whitelist"], FILTER_VALIDATE_BOOLEAN),
'locked' => filter_var($userTmp["locked"], FILTER_VALIDATE_BOOLEAN),
"lockedKey" => $userTmp["lockedKey"],
];
if ($uid === $userTmp["uid"]) {
$user = $userNew;
} else {
$users[] = $userNew;
}
if ($userNew["whitelist"]) {
foreach ($userlist as $uidTmp => $userTmp) {
if (!empty($userTmp) && !empty($userTmp["whitelist"]) && !$userTmp["whitelist"]) {
$sum += intval($userTmp["unusedResultPages"]);
}
}
if (empty($user)) {
return;
}
$pipeline = $redis->pipeline();
# Check if we have to whitelist the user or if we can simply delete the data
if ($user["unusedResultPages"] < $sum && !$user["whitelist"]) {
# Whitelist
$pipeline->hset(HumanVerification::PREFIX . "." . $uid, 'whitelist', "1");
$user["whitelist"] = true;
}
if ($user["whitelist"]) {
$pipeline->hset(HumanVerification::PREFIX . "." . $uid, 'unusedResultPages', "0");
$user["unusedResultPages"] = 0;
HumanVerification::saveUser($user);
} else {
$pipeline->del(HumanVerification::PREFIX . "." . $uid);
$pipeline->srem(HumanVerification::PREFIX . "." . $id, $uid);
HumanVerification::deleteUser($user);
}
$pipeline->expire(HumanVerification::PREFIX . "." . $uid, $user["whitelist"] ? HumanVerification::EXPIRELONG : HumanVerification::EXPIRESHORT);
$pipeline->expire(HumanVerification::PREFIX . "." . $id, HumanVerification::EXPIRELONG);
$pipeline->execute();
}
private static function checkId($request, $id)
......
<?php
namespace App\Http\Controllers;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
class LogController extends Controller
{
public function clicklog(Request $request)
{
$redis = Redis::connection('redisLogs');
if ($redis) {
$logEntry = "";
$logEntry .= "[" . date(DATE_RFC822, mktime(date("H"), date("i"), date("s"), date("m"), date("d"), date("Y"))) . "]";
$logEntry .= " " . $request->input('i');
$logEntry .= " " . $request->input('s');
$logEntry .= " " . $request->input('q');
$logEntry .= " " . $request->input('p');
$logEntry .= " " . $request->input('url');
$redis->rpush('logs.clicks', $logEntry);
}
return '';
}
public function pluginClose()
{
$redis = Redis::connection('redisLogs');
if ($redis) {
$redis->incr('logs.plugin.close');
}
}
public function pluginInstall()
{
$redis = Redis::connection('redisLogs');
if ($redis) {
$redis->incr('logs.plugin.install');
}
}
}
......@@ -4,7 +4,6 @@ namespace App\Http\Controllers;
use App\Mail\Kontakt;
use App\Mail\Sprachdatei;
use DB;
use Illuminate\Http\Request;
use Illuminate\Http\Response;
use LaravelLocalization;
......@@ -143,23 +142,6 @@ class MailController extends Controller
$messageType = "success";
$messageToUser = "Herzlichen Dank!! Wir haben Ihre Spendenbenachrichtigung erhalten.";
try {
// Add the donation to our database
$spenden = DB::connection('spenden')->table('debits')->insert(
['name' => $name,
'iban' => $iban->MachineFormat(),
'bic' => $bic,
'amount' => $betrag,
'message' => $nachricht,
]
);
DB::disconnect('spenden');
} catch (\Illuminate\Database\QueryException $e) {
}