From 1394a250e67332fcc112a7ea79fc10d942d04434 Mon Sep 17 00:00:00 2001
From: Dominik Hebeler <dominik@hebeler.club>
Date: Sun, 5 Mar 2023 21:37:30 +0100
Subject: [PATCH] added cronjob
---
chart/templates/deployment.yaml | 26 +++++++++++++++++++
docker-compose.yml | 12 +++++++++
pass/app.js | 45 +++++++++++++++++++++++----------
pass/bin/cron | 33 +++++++++++++++---------
4 files changed, 91 insertions(+), 25 deletions(-)
diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml
index a68d247..58dd2e0 100644
--- a/chart/templates/deployment.yaml
+++ b/chart/templates/deployment.yaml
@@ -63,6 +63,32 @@ spec:
port: http
resources:
{{- toYaml .Values.resources | nindent 12 }}
+ - name: {{ .Chart.Name }}-cron
+ securityContext:
+ {{- toYaml .Values.securityContext | nindent 12 }}
+ image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ command: ["/usr/local/bin/node"]
+ args: ["./bin/cron"]
+ volumeMounts:
+ - name: order-data
+ mountPath: /data
+ {{- if .Values.application.secretName }}
+ - name: application-secret
+ readOnly: true
+ mountPath: /app/config/production.json
+ subPath: production.json
+ {{- end }}
+ livenessProbe:
+ httpGet:
+ path: /healthz/cron
+ port: 3000
+ readinessProbe:
+ httpGet:
+ path: /healthz/cron
+ port: 3000
+ resources:
+ {{- toYaml .Values.resources | nindent 12 }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
diff --git a/docker-compose.yml b/docker-compose.yml
index 9afce28..efd54ee 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -27,6 +27,18 @@ services:
ports:
- 8085:3000
- 9229:9229
+ express_cron:
+ build:
+ context: ./build/pass
+ target: development
+ entrypoint: "/usr/local/bin/node"
+ command: "./bin/cron"
+ networks:
+ - metager
+ working_dir: /app
+ volumes:
+ - ./pass:/app
+ - mgpassdata:/data
express_redis:
build:
context: ./build/redis
diff --git a/pass/app.js b/pass/app.js
index 1aeec96..4beb019 100644
--- a/pass/app.js
+++ b/pass/app.js
@@ -5,6 +5,7 @@ var cookieParser = require("cookie-parser");
var logger = require("morgan");
var indexRouter = require("./routes/index");
+const RedisClient = require("./app/RedisClient");
var app = express();
@@ -17,9 +18,13 @@ if (process.env.NODE_ENV === "development") {
app.use(logger("dev"));
} else {
// Do not log successful requests in production
- app.use(logger("dev", {
- skip: (req, res) => { return res.statusCode < 400 }
- }));
+ app.use(
+ logger("dev", {
+ skip: (req, res) => {
+ return res.statusCode < 400;
+ },
+ })
+ );
}
app.use(express.json());
app.use(express.urlencoded({ extended: false, limit: "1mb" }));
@@ -28,24 +33,38 @@ app.use(cookieParser());
// Healthcheck URL
app.get("/healthz", (req, res) => {
res.status(200);
- res.header({ "Content-Type": "application/json" })
- res.send(JSON.stringify({ "status": "OK" }));
-})
+ res.header({ "Content-Type": "application/json" });
+ res.send(JSON.stringify({ status: "OK" }));
+});
+
+app.get("/healthz/cron", async (req, res) => {
+ let redis_client = RedisClient.CLIENT();
+ let live = await redis_client.get("cron_liveness");
+
+ if (live !== null) {
+ res.json({ message: "OK" });
+ } else {
+ res.status(404).json({ message: "DOWN" });
+ }
+});
app.use((req, res, next) => {
res.locals.baseDir = "";
let subPath = req.url.match(/^((\/.*)?\/keys)/);
- let allowed_hosts = [
- "localhost",
- "metager.de",
- "metager.org"
- ];
+ let allowed_hosts = ["localhost", "metager.de", "metager.org"];
- if (allowed_hosts.includes(req.hostname) || (process.env.NODE_ENV === "development" && req.hostname.match(/^(localhost|.*\.ngrok\.io|.*\.review\.metager\.de)$/))) {
+ if (
+ allowed_hosts.includes(req.hostname) ||
+ (process.env.NODE_ENV === "development" &&
+ req.hostname.match(/^(localhost|.*\.ngrok\.io|.*\.review\.metager\.de)$/))
+ ) {
let proto = req.get("x-forwarded-proto") ?? req.protocol;
let host = req.get("x-forwarded-host") ?? req.get("host");
let port = req.get("x-forwarded-port");
- res.locals.baseDir = `${proto}://${host}` + (port ? `:${port}` : '') + (subPath ? subPath[1] : "");
+ res.locals.baseDir =
+ `${proto}://${host}` +
+ (port ? `:${port}` : "") +
+ (subPath ? subPath[1] : "");
}
next();
});
diff --git a/pass/bin/cron b/pass/bin/cron
index 81e89bc..763c829 100644
--- a/pass/bin/cron
+++ b/pass/bin/cron
@@ -1,6 +1,7 @@
+const dayjs = require("dayjs");
const Order = require("../app/Order");
const RedisClient = require("../app/RedisClient");
-
+let redis_client;
/**
* Will call every cron script every minute.
* The cron script itself will make sure it executes in the desired
@@ -10,19 +11,30 @@ const RedisClient = require("../app/RedisClient");
*/
let cronjobs = async () => {
- await writeLogsToOrder();
+ let now = dayjs();
+ console.log(`[${now.format("YYYY-MM-DD HH:mm:ss")}] Start`);
+ redis_client = RedisClient.CLIENT();
+ await redis_client.setex("cron_liveness", 70, 1);
+ let written_logs = await writeLogsToOrder();
+ console.log(
+ `[${now.format(
+ "YYYY-MM-DD HH:mm:ss"
+ )}] Written ${written_logs} key changes.`
+ );
+ await redis_client.quit();
+ console.log(`[${now.format("YYYY-MM-DD HH:mm:ss")}] Finish`);
};
+console.log("Start");
let interval = setInterval(cronjobs, 60000);
cronjobs();
async function writeLogsToOrder() {
- let redis_client = RedisClient.CLIENT();
let redis_lock_key = "cron:writeLogsToOrder";
let interval_seconds = 60; // Will execute every minute
let lock = await tryToGetLock(redis_lock_key, interval_seconds);
if (!lock) {
- return;
+ return 0;
}
// Start gathering changes in keys; do it in packets of 1000
return redis_client
@@ -45,24 +57,25 @@ async function writeLogsToOrder() {
return key_changes;
})
.then(async (key_changes) => {
+ let written_changes = 0;
for (let order_id in key_changes) {
/**
* @type {Order}
*/
let loaded_order = await Order.LOAD_ORDER_FROM_ID(order_id);
await loaded_order.getWriteLock();
+ written_changes += key_changes[order_id].length;
loaded_order.addOrderKeyChanges(key_changes[order_id]);
await loaded_order.save();
await loaded_order.releaseWriteLock().catch((reason) => {
console.error(reason);
});
}
- })
- .then(() => redis_client.quit());
+ return written_changes;
+ });
}
async function tryToGetLock(redis_lock_key, interval_seconds) {
- let redis_client = RedisClient.CLIENT();
let redis_lock = await redis_client
.pipeline()
.setnx(redis_lock_key, 1)
@@ -76,19 +89,15 @@ async function tryToGetLock(redis_lock_key, interval_seconds) {
redis_lock[2][0] !== null
) {
console.error("Error while getting redis lock for cronjob");
- await redis_client.quit();
return false;
}
if (redis_lock[0][1] === 1) {
// Got Lock successfully
- await redis_client.quit();
return true;
} else {
// Did not get lock. Reset expiretime to previous value
- await redis_client
- .expireat(redis_lock_key, redis_lock[1][1])
- .then(() => redis_client.quit());
+ await redis_client.expireat(redis_lock_key, redis_lock[1][1]);
return false;
}
}
--
GitLab