From cea137bd9cdf7661ea4584887d2cfe8af81ae820 Mon Sep 17 00:00:00 2001
From: Dominik Hebeler <dominik@suma-ev.de>
Date: Fri, 19 Aug 2022 14:44:00 +0200
Subject: [PATCH] cleaning up images
---
.gitlab/ci/build_docker_images.yml | 1 +
.gitlab/ci/deploy.yml | 1 +
.../cleanup_tags_non_revision.sh | 50 +++++++++++++++++++
.../cleanup_tags_revision.sh | 10 ++++
.../deployment_scripts/delete_deployment.sh | 4 +-
5 files changed, 64 insertions(+), 2 deletions(-)
diff --git a/.gitlab/ci/build_docker_images.yml b/.gitlab/ci/build_docker_images.yml
index 3a197d01d..ef900cf73 100644
--- a/.gitlab/ci/build_docker_images.yml
+++ b/.gitlab/ci/build_docker_images.yml
@@ -56,6 +56,7 @@ redis:
FPM_REPOSITORY_ID: 418
NGINX_REPOSITORY_ID: 416
NODE_REPOSITORY_ID: 419
+ REDIS_REPOSITORY_ID: 425
KEEP_N: 9 # Trim to the latest 9 revisions as the 10th will be deleted in the next stage
before_script:
- kubectl config get-contexts
diff --git a/.gitlab/ci/deploy.yml b/.gitlab/ci/deploy.yml
index 01cc6d9e7..b899436c5 100644
--- a/.gitlab/ci/deploy.yml
+++ b/.gitlab/ci/deploy.yml
@@ -73,6 +73,7 @@ stop_review:
KEEP_N: 0 # Environment gets deleted. No Image Tags to keep
FPM_REPOSITORY_ID: 418
NGINX_REPOSITORY_ID: 416
+ REDIS_REPOSITORY_ID: 425
before_script:
- kubectl config get-contexts
- kubectl config use-context open-source/MetaGer:metager
diff --git a/.gitlab/deployment_scripts/cleanup_tags_non_revision.sh b/.gitlab/deployment_scripts/cleanup_tags_non_revision.sh
index 9fdefd705..85a9f400e 100755
--- a/.gitlab/deployment_scripts/cleanup_tags_non_revision.sh
+++ b/.gitlab/deployment_scripts/cleanup_tags_non_revision.sh
@@ -104,16 +104,51 @@ done
echo "Got ${#existing_tags_node[@]} tags."
echo ""
+# Get All existing tags for the redis repo
+echo "Fetching existing Redis tags..."
+declare -A existing_tags_redis
+get_tags_url=$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$REDIS_REPOSITORY_ID/tags
+page=1
+counter=1
+while [[ "$page" != "" && $counter -le 50 ]]
+do
+ tags=$(curl --fail --silent -D headers.txt "${get_tags_url}?page=$page" | jq -r ".[][\"name\"]")
+ for tag in $tags
+ do
+ if [[ $tag = ${DOCKER_IMAGE_TAG_PREFIX}-* && "$tag" != $DOCKER_IMAGE_TAG_PREFIX && $tag != $DOCKER_REDIS_IMAGE_TAG ]]
+ then
+ existing_tags_redis[$tag]=1
+ fi
+ done
+ while read header
+ do
+ header=$(echo $header | sed -r 's/\s+//g')
+ key=$(echo $header | cut -d':' -f1 )
+ value=$(echo $header | cut -d':' -f2 )
+ case "$key" in
+ x-next-page)
+ page="$value"
+ sleep 1
+ ;;
+ esac
+ done < headers.txt
+ counter=$((counter + 1))
+done
+echo "Got ${#existing_tags_redis[@]} tags."
+echo ""
+
# Get List of existing revisions
echo "Fetching Tags from helm revision history to not be deleted..."
declare -A revision_tags_fpm
declare -A revision_tags_nginx
+declare -A revision_tags_redis
helm_release_revisions=$(helm -n $KUBE_NAMESPACE history ${HELM_RELEASE_NAME} -o json | jq -r '.[]["revision"]')
for revision in $helm_release_revisions
do
revision_values=$(helm -n $KUBE_NAMESPACE get values ${HELM_RELEASE_NAME} --revision=$revision -o json | jq -r '.')
revision_tags_fpm[$(echo $revision_values | jq -r '.image.fpm.tag')]=1
revision_tags_nginx[$(echo $revision_values | jq -r '.image.nginx.tag')]=1
+ revision_tags_redis[$(echo $revision_values | jq -r '.image.redis.tag')]=1
done
echo "Got ${#revision_tags_fpm[@]} tags for fpm."
echo ${!revision_tags_fpm[@]}
@@ -121,6 +156,9 @@ echo ""
echo "Got ${#revision_tags_nginx[@]} tags for nginx."
echo ${!revision_tags_nginx[@]}
echo ""
+echo "Got ${#revision_tags_redis[@]} tags for redis."
+echo ${!revision_tags_redis[@]}
+echo ""
# Delete FPM Tags that are in no revision
echo "Deleting unused FPM Tags..."
@@ -147,6 +185,18 @@ do
fi
done
+# Delete Redis Tags that are in no revision
+echo "Deleting unused Redis Tags..."
+for redis_tag in ${!existing_tags_redis[@]}
+do
+ if [[ ! -v revision_tags_nginx["$redis_tag"] ]]
+ then
+ echo $redis_tag
+ curl --fail --silent -X DELETE -H "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$REDIS_REPOSITORY_ID/tags/$redis_tag"
+ echo ""
+ fi
+done
+
# Delete Node Tags
echo "Deleting unused Node Tags..."
for node_tag in ${!existing_tags_node[@]}
diff --git a/.gitlab/deployment_scripts/cleanup_tags_revision.sh b/.gitlab/deployment_scripts/cleanup_tags_revision.sh
index 39f164dec..c16936e14 100755
--- a/.gitlab/deployment_scripts/cleanup_tags_revision.sh
+++ b/.gitlab/deployment_scripts/cleanup_tags_revision.sh
@@ -23,6 +23,7 @@ expired_revisions=$(helm -n $KUBE_NAMESPACE history ${HELM_RELEASE_NAME} -o json
# Loop through those revisions
declare -A expired_fpm_tags
declare -A expired_nginx_tags
+declare -A expired_redis_tags
for revision in $expired_revisions
do
# Get Values for this revision
@@ -30,12 +31,14 @@ do
# Get Image Tags for this revision
revision_fpm_tag=$(echo $revision_values | jq -r '.image.fpm.tag')
revision_nginx_tag=$(echo $revision_values | jq -r '.image.nginx.tag')
+ revision_redis_tag=$(echo $revision_values | jq -r '.image.redis.tag')
# Add Tags to the arrays
if [[ $revision_fpm_tag = ${DOCKER_IMAGE_TAG_PREFIX}-* ]]
then
expired_fpm_tags[$revision_fpm_tag]=0
expired_nginx_tags[$revision_nginx_tag]=0
+ expired_redis_tags[$revision_redis_tag]=0
fi
done
@@ -52,4 +55,11 @@ do
echo "Deleting nginx tag $nginx_tag"
curl --fail --silent -X DELETE -H "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$NGINX_REPOSITORY_ID/tags/$nginx_tag"
echo ""
+done
+# Delete all gathered redis tags
+for redis_tag in ${!expired_redis_tags[@]}
+do
+ echo "Deleting redis tag $redis_tag"
+ curl --fail --silent -X DELETE -H "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$REDIS_REPOSITORY_ID/tags/$redis_tag"
+ echo ""
done
\ No newline at end of file
diff --git a/.gitlab/deployment_scripts/delete_deployment.sh b/.gitlab/deployment_scripts/delete_deployment.sh
index 3a02daa0b..50f4fdcab 100755
--- a/.gitlab/deployment_scripts/delete_deployment.sh
+++ b/.gitlab/deployment_scripts/delete_deployment.sh
@@ -6,8 +6,8 @@ HELM_RELEASE_NAME=${HELM_RELEASE_NAME%%*(-)}
echo "Removing Image Tags..."
.gitlab/deployment_scripts/cleanup_tags_revision.sh
# For some reason an empty image tag gets created for this. We need to delete it until we find out why that is
-'curl --fail --silent -X DELETE -H "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$FPM_REPOSITORY_ID/tags/$DOCKER_IMAGE_TAG_PREFIX"'
-'curl --fail --silent -X DELETE -H "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$NGINX_REPOSITORY_ID/tags/$DOCKER_IMAGE_TAG_PREFIX"'
+#'curl --fail --silent -X DELETE -H "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$FPM_REPOSITORY_ID/tags/$DOCKER_IMAGE_TAG_PREFIX"'
+#'curl --fail --silent -X DELETE -H "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/registry/repositories/$NGINX_REPOSITORY_ID/tags/$DOCKER_IMAGE_TAG_PREFIX"'
echo "Stopping Deployment..."
kubectl -n $KUBE_NAMESPACE delete secret $HELM_RELEASE_NAME
helm -n $KUBE_NAMESPACE delete $HELM_RELEASE_NAME
\ No newline at end of file
--
GitLab