Commit c5b4df09 authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

Merge branch '1212-make-metager-pods-shutdown-gracefully' into 'development'

Resolve "Make MetaGer Pods shutdown gracefully"

Closes #1212

See merge request !1995
parents ec956642 cd1a74c8
......@@ -5,6 +5,7 @@ variables:
DOCKER_FPM_IMAGE_NAME: fpm
DOCKER_NGINX_IMAGE_NAME: nginx
DOCKER_NODE_IMAGE_NAME: node
DOCKER_REDIS_IMAGE_NAME: redis
KUBE_NAMESPACE: metager-2
workflow:
......@@ -18,6 +19,7 @@ workflow:
DOCKER_FPM_IMAGE_TAG: $DOCKER_IMAGE_TAG_PREFIX-$CI_COMMIT_SHA
DOCKER_NGINX_IMAGE_TAG: $DOCKER_IMAGE_TAG_PREFIX-$CI_COMMIT_SHA
DOCKER_NODE_IMAGE_TAG: $DOCKER_IMAGE_TAG_PREFIX-$CI_COMMIT_SHA
DOCKER_REDIS_IMAGE_TAG: $DOCKER_IMAGE_TAG_PREFIX-$CI_COMMIT_SHA
HELM_RELEASE_NAME: review-$DOCKER_IMAGE_TAG_PREFIX
- if: $CI_COMMIT_BRANCH == "master"
variables:
......@@ -28,6 +30,7 @@ workflow:
DOCKER_FPM_IMAGE_TAG: $DOCKER_IMAGE_TAG_PREFIX-$CI_COMMIT_SHA
DOCKER_NGINX_IMAGE_TAG: $DOCKER_IMAGE_TAG_PREFIX-$CI_COMMIT_SHA
DOCKER_NODE_IMAGE_TAG: $DOCKER_IMAGE_TAG_PREFIX-$CI_COMMIT_SHA
DOCKER_REDIS_IMAGE_TAG: $DOCKER_IMAGE_TAG_PREFIX-$CI_COMMIT_SHA
HELM_RELEASE_NAME: $DOCKER_IMAGE_TAG_PREFIX
- if: $CI_COMMIT_BRANCH == "development"
variables:
......@@ -38,6 +41,7 @@ workflow:
DOCKER_FPM_IMAGE_TAG: $DOCKER_IMAGE_TAG_PREFIX-$CI_COMMIT_SHA
DOCKER_NGINX_IMAGE_TAG: $DOCKER_IMAGE_TAG_PREFIX-$CI_COMMIT_SHA
DOCKER_NODE_IMAGE_TAG: $DOCKER_IMAGE_TAG_PREFIX-$CI_COMMIT_SHA
DOCKER_REDIS_IMAGE_TAG: $DOCKER_IMAGE_TAG_PREFIX-$CI_COMMIT_SHA
HELM_RELEASE_NAME: $DOCKER_IMAGE_TAG_PREFIX
stages:
......
......@@ -33,6 +33,19 @@ nginx:
after_script:
- docker logout $CI_REGISTRY
redis:
stage: build_docker_images
image: $BUILD_DOCKER_IMAGE
before_script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
script:
- cd build/redis
- docker build --network=host
-t ${CI_REGISTRY_IMAGE}/$DOCKER_REDIS_IMAGE_NAME:$DOCKER_REDIS_IMAGE_TAG .
- docker push ${CI_REGISTRY_IMAGE}/$DOCKER_REDIS_IMAGE_NAME:$DOCKER_REDIS_IMAGE_TAG
after_script:
- docker logout $CI_REGISTRY
.cleanup_revision_images:
stage: build_docker_images
image: $DEPLOY_KUBERNETES_IMAGE
......@@ -43,6 +56,7 @@ nginx:
FPM_REPOSITORY_ID: 418
NGINX_REPOSITORY_ID: 416
NODE_REPOSITORY_ID: 419
REDIS_REPOSITORY_ID: 425
KEEP_N: 9 # Trim to the latest 9 revisions as the 10th will be deleted in the next stage
before_script:
- kubectl config get-contexts
......
......@@ -73,6 +73,7 @@ stop_review:
KEEP_N: 0 # Environment gets deleted. No Image Tags to keep
FPM_REPOSITORY_ID: 418
NGINX_REPOSITORY_ID: 416
REDIS_REPOSITORY_ID: 425
before_script:
- kubectl config get-contexts
- kubectl config use-context open-source/MetaGer:metager
......
......@@ -104,16 +104,51 @@ done
echo "Got ${#existing_tags_node[@]} tags."
echo ""
# Get All existing tags for the redis repo
echo "Fetching existing Redis tags..."
declare -A existing_tags_redis
get_tags_url=$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$REDIS_REPOSITORY_ID/tags
page=1
counter=1
while [[ "$page" != "" && $counter -le 50 ]]
do
tags=$(curl --fail --silent -D headers.txt "${get_tags_url}?page=$page" | jq -r ".[][\"name\"]")
for tag in $tags
do
if [[ $tag = ${DOCKER_IMAGE_TAG_PREFIX}-* && "$tag" != $DOCKER_IMAGE_TAG_PREFIX && $tag != $DOCKER_REDIS_IMAGE_TAG ]]
then
existing_tags_redis[$tag]=1
fi
done
while read header
do
header=$(echo $header | sed -r 's/\s+//g')
key=$(echo $header | cut -d':' -f1 )
value=$(echo $header | cut -d':' -f2 )
case "$key" in
x-next-page)
page="$value"
sleep 1
;;
esac
done < headers.txt
counter=$((counter + 1))
done
echo "Got ${#existing_tags_redis[@]} tags."
echo ""
# Get List of existing revisions
echo "Fetching Tags from helm revision history to not be deleted..."
declare -A revision_tags_fpm
declare -A revision_tags_nginx
declare -A revision_tags_redis
helm_release_revisions=$(helm -n $KUBE_NAMESPACE history ${HELM_RELEASE_NAME} -o json | jq -r '.[]["revision"]')
for revision in $helm_release_revisions
do
revision_values=$(helm -n $KUBE_NAMESPACE get values ${HELM_RELEASE_NAME} --revision=$revision -o json | jq -r '.')
revision_tags_fpm[$(echo $revision_values | jq -r '.image.fpm.tag')]=1
revision_tags_nginx[$(echo $revision_values | jq -r '.image.nginx.tag')]=1
revision_tags_redis[$(echo $revision_values | jq -r '.image.redis.tag')]=1
done
echo "Got ${#revision_tags_fpm[@]} tags for fpm."
echo ${!revision_tags_fpm[@]}
......@@ -121,6 +156,9 @@ echo ""
echo "Got ${#revision_tags_nginx[@]} tags for nginx."
echo ${!revision_tags_nginx[@]}
echo ""
echo "Got ${#revision_tags_redis[@]} tags for redis."
echo ${!revision_tags_redis[@]}
echo ""
# Delete FPM Tags that are in no revision
echo "Deleting unused FPM Tags..."
......@@ -147,6 +185,18 @@ do
fi
done
# Delete Redis Tags that are in no revision
echo "Deleting unused Redis Tags..."
for redis_tag in ${!existing_tags_redis[@]}
do
if [[ ! -v revision_tags_nginx["$redis_tag"] ]]
then
echo $redis_tag
curl --fail --silent -X DELETE -H "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$REDIS_REPOSITORY_ID/tags/$redis_tag"
echo ""
fi
done
# Delete Node Tags
echo "Deleting unused Node Tags..."
for node_tag in ${!existing_tags_node[@]}
......
......@@ -23,6 +23,7 @@ expired_revisions=$(helm -n $KUBE_NAMESPACE history ${HELM_RELEASE_NAME} -o json
# Loop through those revisions
declare -A expired_fpm_tags
declare -A expired_nginx_tags
declare -A expired_redis_tags
for revision in $expired_revisions
do
# Get Values for this revision
......@@ -30,12 +31,14 @@ do
# Get Image Tags for this revision
revision_fpm_tag=$(echo $revision_values | jq -r '.image.fpm.tag')
revision_nginx_tag=$(echo $revision_values | jq -r '.image.nginx.tag')
revision_redis_tag=$(echo $revision_values | jq -r '.image.redis.tag')
# Add Tags to the arrays
if [[ $revision_fpm_tag = ${DOCKER_IMAGE_TAG_PREFIX}-* ]]
then
expired_fpm_tags[$revision_fpm_tag]=0
expired_nginx_tags[$revision_nginx_tag]=0
expired_redis_tags[$revision_redis_tag]=0
fi
done
......@@ -52,4 +55,11 @@ do
echo "Deleting nginx tag $nginx_tag"
curl --fail --silent -X DELETE -H "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$NGINX_REPOSITORY_ID/tags/$nginx_tag"
echo ""
done
# Delete all gathered redis tags
for redis_tag in ${!expired_redis_tags[@]}
do
echo "Deleting redis tag $redis_tag"
curl --fail --silent -X DELETE -H "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$REDIS_REPOSITORY_ID/tags/$redis_tag"
echo ""
done
\ No newline at end of file
......@@ -6,8 +6,8 @@ HELM_RELEASE_NAME=${HELM_RELEASE_NAME%%*(-)}
echo "Removing Image Tags..."
.gitlab/deployment_scripts/cleanup_tags_revision.sh
# For some reason an empty image tag gets created for this. We need to delete it until we find out why that is
'curl --fail --silent -X DELETE -H "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$FPM_REPOSITORY_ID/tags/$DOCKER_IMAGE_TAG_PREFIX"'
'curl --fail --silent -X DELETE -H "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$NGINX_REPOSITORY_ID/tags/$DOCKER_IMAGE_TAG_PREFIX"'
#'curl --fail --silent -X DELETE -H "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$FPM_REPOSITORY_ID/tags/$DOCKER_IMAGE_TAG_PREFIX"'
#'curl --fail --silent -X DELETE -H "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$NGINX_REPOSITORY_ID/tags/$DOCKER_IMAGE_TAG_PREFIX"'
echo "Stopping Deployment..."
kubectl -n $KUBE_NAMESPACE delete secret $HELM_RELEASE_NAME
helm -n $KUBE_NAMESPACE delete $HELM_RELEASE_NAME
\ No newline at end of file
......@@ -11,5 +11,6 @@ helm -n $KUBE_NAMESPACE upgrade --install \
--set ingress.hosts[0].host="$DEPLOYMENT_URL" \
--set image.fpm.tag=$DOCKER_FPM_IMAGE_TAG \
--set image.nginx.tag=$DOCKER_NGINX_IMAGE_TAG \
--set image.redis.tag=$DOCKER_REDIS_IMAGE_TAG \
--set app_url=$APP_URL \
--wait
\ No newline at end of file
#!/bin/sh
#!/bin/bash
set -e
_trap() {
echo "Waiting for child processes to finish"
php artisan fpm:graceful-stop
echo "Stopping FPM"
kill -s SIGQUIT $FPM_PID
}
trap _trap SIGQUIT
validate_laravel
if [ ! -f .env ];
......@@ -22,4 +31,6 @@ php artisan db:seed
php artisan ide-helper:generate
php artisan ide-helper:meta
docker-php-entrypoint php-fpm
\ No newline at end of file
docker-php-entrypoint php-fpm &
FPM_PID=$!
wait
\ No newline at end of file
#!/bin/sh
#!/bin/bash
set -e
_trap() {
echo "Waiting for child processes to finish"
php artisan fpm:graceful-stop
echo "Stopping FPM"
kill -s SIGQUIT $FPM_PID
}
trap _trap SIGQUIT
validate_laravel
# Production version will have the .env file mounted at /home/metager/.env
......@@ -19,4 +28,6 @@ php artisan route:trans:cache
php artisan spam:load
php artisan load:affiliate-blacklist
docker-php-entrypoint php-fpm
\ No newline at end of file
docker-php-entrypoint php-fpm &
FPM_PID=$!
wait
\ No newline at end of file
......@@ -12,4 +12,6 @@ ENTRYPOINT [ "/usr/local/bin/entrypoint" ]
FROM development as production
ADD build/node/entrypoint_production.sh /usr/local/bin/entrypoint
\ No newline at end of file
ADD build/node/entrypoint_production.sh /usr/local/bin/entrypoint
STOPSIGNAL SIGKILL
\ No newline at end of file
FROM redis:6
COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint-mg.sh
ENTRYPOINT [ "docker-entrypoint-mg.sh" ]
CMD ["redis-server"]
\ No newline at end of file
#!/bin/bash
_term() {
echo -n "Waiting for clients to disconnect before stopping"
while [ "$(redis-cli info clients | grep "connected_clients" | cut -d ":" -f 2 | tr -dc '0-9')" -gt 1 ];
do
echo -n "."
sleep 1;
done
echo ""
echo "Stopping Redis Server with PID $REDIS_PID"
kill -s SIGKILL $REDIS_PID
exit 1
}
trap _term SIGTERM
echo "Starting Redis Server"
docker-entrypoint.sh "$@" &
REDIS_PID=$!
wait
\ No newline at end of file
......@@ -73,6 +73,14 @@ Create the name of the service account to use
{{- end -}}
{{- end -}}
{{- define "redis_image" -}}
{{- if eq .Values.image.redis.tag "" -}}
{{- .Values.image.redis.repository -}}
{{- else -}}
{{- printf "%s:%s" .Values.image.redis.repository .Values.image.redis.tag -}}
{{- end -}}
{{- end -}}
{{- define "secret_name" -}}
{{- printf "%s" .Release.Name }}
{{- end -}}
\ No newline at end of file
......@@ -110,8 +110,8 @@ spec:
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ template "fpm_image" . }}"
command: ["/bin/bash", "-c"]
args: ["/usr/local/bin/php artisan schedule:run && /usr/local/bin/php artisan schedule:work"]
command: ["/usr/local/bin/php"]
args: ["artisan", "schedule:work-mg"]
imagePullPolicy: {{ .Values.image.fpm.pullPolicy }}
env:
- name: APP_ENV
......@@ -214,9 +214,9 @@ spec:
memory: 100M
limits:
- name: redis
image: "redis:6"
image: "{{ template "redis_image" . }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
command: ["redis-server", "/usr/local/etc/redis/redis.conf"]
args: ["redis-server", "/usr/local/etc/redis/redis.conf"]
volumeMounts:
- name: redis-config
mountPath: /usr/local/etc/redis/redis.conf
......
......@@ -16,6 +16,11 @@ image:
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: ""
redis:
repository: registry.metager.de/open-source/metager/redis
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: ""
imagePullSecrets: []
nameOverride: ""
......
version: "3"
version: "3.8"
# Volumes
volumes:
......@@ -37,7 +37,7 @@ services:
<<: *fpm_build
restart: unless-stopped
entrypoint: /usr/local/bin/php
command: artisan schedule:work
command: artisan schedule:work-mg
volumes:
- ./metager:/metager/metager_app
healthcheck:
......@@ -75,13 +75,15 @@ services:
dockerfile: build/node/Dockerfile
target: $APP_ENV
restart: unless-stopped
stop_signal: SIGKILL
depends_on:
- nginx
volumes:
- ./metager:/home/node/metager
- node_cache:/home/node/.npm
redis:
image: redis:6
build:
context: ./build/redis
restart: unless-stopped
user: "redis:redis"
healthcheck:
......
<?php
namespace App\Console\Commands;
use ErrorException;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
class FPMGracefulStop extends Command
{
const REDIS_FPM_STOPPED_KEY = "fpm_stopped";
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'fpm:graceful-stop';
/**
* The console command description.
*
* @var string
*/
protected $description = 'This command will wait until there are no active fpm processes anymore and then return.';
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
do {
$active_fpm_processes = $this->getActiveProcessCount();
usleep(500 * 1000); // Check every half second
} while ($active_fpm_processes === null || $active_fpm_processes > 1);
$this->info("Only one FPM process left. Ready to stop fpm...");
// The Request fetcher won't stop before FPM is not stopped with processing requests
// because there could be last jobs flying in
// This Redis Value will tell him that it's good to stop
Redis::set(self::REDIS_FPM_STOPPED_KEY, "true");
$this->info("Set Redis Key");
return 0;
}
/**
* Returns the number of active fpm processes
*
* @return int|null Number of active fpm processes; Causes one active process by itself; Null if an error occured
*/
private function getActiveProcessCount()
{
$url = route("fpm-status");
$auth = \base64_encode(config("metager.metager.admin.user") . ":" . config("metager.metager.admin.password"));
$context = \stream_context_create([
"http" => [
"header" => "Authorization: Basic $auth",
],
]);
try {
$fpm_info = \file_get_contents($url, false, $context);
} catch (ErrorException $e) {
// Webserver could not be reached. Probably already shut down so there are no active connections anyways
return 0;
}
if ($fpm_info !== false) {
$fpm_info = \json_decode($fpm_info);
} else {
return null;
}
if (!\is_object($fpm_info) || !\property_exists($fpm_info, "active-processes")) {
return null;
}
return $fpm_info->{"active-processes"};
}
}
......@@ -55,9 +55,7 @@ class RequestFetcher extends Command
*/
public function handle()
{
pcntl_signal(SIGINT, [$this, "sig_handler"]);
pcntl_signal(SIGTERM, [$this, "sig_handler"]);
pcntl_signal(SIGHUP, [$this, "sig_handler"]);
pcntl_signal(SIGQUIT, [$this, "sig_handler"]);
// Redis might not be available now
for ($count = 0; $count < 10; $count++) {
......@@ -74,7 +72,7 @@ class RequestFetcher extends Command
}
try {
while ($this->shouldRun) {
while (true) {
Redis::set(self::HEALTHCHECK_KEY, Carbon::now()->format(self::HEALTHCHECK_FORMAT));
$operationsRunning = true;
curl_multi_exec($this->multicurl, $operationsRunning);
......@@ -86,6 +84,10 @@ class RequestFetcher extends Command
if ($newJobs === 0 && $answersRead === 0) {
usleep(10 * 1000);
}
if (!$this->shouldRun && $operationsRunning === 0 && Redis::get(FPMGracefulStop::REDIS_FPM_STOPPED_KEY) !== NULL) {
break;
}
}
} finally {
curl_multi_close($this->multicurl);
......@@ -234,6 +236,6 @@ class RequestFetcher extends Command
public function sig_handler($sig)
{
$this->shouldRun = false;
echo ("Terminating Process\n");
$this->info("Terminating Process\n");
}
}
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
class ScheduleWorker extends Command
{
private $should_exit = false;
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'schedule:work-mg';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Starts the schedule worker with correct signal handling and graceful shutdown.';
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
pcntl_signal(SIGQUIT, array(&$this, "onExit"));
$this->info("Starting Scheduler");
$this->call('schedule:run');
do {
sleep(60);
$this->call('schedule:run');
} while (!$this->should_exit);
return 0;
}
public function onExit()
{
$this->info("Stopping Scheduler on SIGQUIT");
$this->should_exit = true;
}
}
......@@ -205,4 +205,10 @@ class AdminInterface extends Controller
->with('title', 'Wer sucht was? - MetaGer')
->with('q', $q);
}
public function getFPMStatus()
{
$status = \fpm_get_status();
return response()->json($status);
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment