Commit 6ee87e0d authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

fixed liveness probes

parent 1980491e
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
use Carbon;
class Heartbeat extends Command
{
const REDIS_KEY = "heartbeat";
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'heartbeat';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Stores in local Redis when it last ran. Provides a heartbeat for liveness probes to check whether scheduler is running or not.';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
try{
$now = Carbon::now();
Redis::set(self::REDIS_KEY, $now->format('Y-m-d H:i:s'));
} catch (\Exception $e){
echo $e->getTraceAsString();
return 1;
}
return 0;
}
}
......@@ -6,9 +6,13 @@ use Cache;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Redis;
use Log;
use Carbon;
class RequestFetcher extends Command
{
const HEALTHCHECK_KEY = "fetcher_healthcheck";
const HEALTHCHECK_FORMAT = "Y-m-d H:i:s";
/**
* The name and signature of the console command.
*
......@@ -51,7 +55,6 @@ class RequestFetcher extends Command
*/
public function handle()
{
$pidFile = "/tmp/fetcher";
pcntl_signal(SIGINT, [$this, "sig_handler"]);
pcntl_signal(SIGTERM, [$this, "sig_handler"]);
pcntl_signal(SIGHUP, [$this, "sig_handler"]);
......@@ -70,14 +73,9 @@ class RequestFetcher extends Command
}
}
touch($pidFile);
if (!file_exists($pidFile)) {
return;
}
try {
while ($this->shouldRun) {
Redis::set(self::HEALTHCHECK_KEY, Carbon::now()->format(self::HEALTHCHECK_FORMAT));
$operationsRunning = true;
curl_multi_exec($this->multicurl, $operationsRunning);
$status = $this->readMultiCurl($this->multicurl);
......@@ -90,7 +88,6 @@ class RequestFetcher extends Command
}
}
} finally {
unlink($pidFile);
curl_multi_close($this->multicurl);
}
}
......
......@@ -25,6 +25,7 @@ class Kernel extends ConsoleKernel
*/
protected function schedule(Schedule $schedule)
{
$schedule->command('heartbeat')->everyMinute();
$schedule->command('requests:gather')->everyFifteenMinutes();
$schedule->command('requests:useragents')->everyFiveMinutes();
$schedule->command('logs:gather')->everyMinute();
......
<?php
namespace App\Http\Controllers;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
use Carbon;
class HealthcheckController extends Controller
{
/**
* Check if the server is ready
*/
public function liveness() {
return response('ok', 200);
}
public function livenessScheduler() {
$lastSchedule = Redis::get(\App\Console\Commands\Heartbeat::REDIS_KEY);
$lastSchedule = Carbon::createFromFormat('Y-m-d H:i:s', $lastSchedule);
if(Carbon::now()->diffInMinutes($lastSchedule) > 1){
abort(500, "Last heartbeat too long ago");
}else{
return response('ok', 200);
}
}
public function livenessWorker() {
$lastSchedule = Redis::get(\App\Console\Commands\RequestFetcher::HEALTHCHECK_KEY);
$lastSchedule = Carbon::createFromFormat(\App\Console\Commands\RequestFetcher::HEALTHCHECK_FORMAT, $lastSchedule);
if(Carbon::now()->diffInMinutes($lastSchedule) > 1){
abort(500, "Last heartbeat too long ago");
}else{
return response('ok', 200);
}
}
}
......@@ -262,17 +262,15 @@ spec:
readOnly: true
imagePullPolicy: {{ .Values.image.pullPolicy }}
livenessProbe:
exec:
command:
- cat
- /tmp/fetcher
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
httpGet:
path: "/health-check/liveness-worker"
scheme: "HTTP"
port: 8080
readinessProbe:
exec:
command:
- cat
- /tmp/fetcher
httpGet:
path: "/health-check/liveness-worker"
scheme: "HTTP"
port: 8080
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
resources:
......@@ -289,17 +287,17 @@ spec:
readOnly: true
imagePullPolicy: {{ .Values.image.pullPolicy }}
livenessProbe:
exec:
command:
- cat
- /tmp/fetcher
httpGet:
path: "/health-check/liveness-scheduler"
scheme: "HTTP"
port: 8080
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
readinessProbe:
exec:
command:
- cat
- /tmp/fetcher
httpGet:
path: "/health-check/liveness-scheduler"
scheme: "HTTP"
port: 8080
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
resources:
......
......@@ -26,6 +26,8 @@ services:
image: metager/development:latest
volumes:
- .:/html
healthcheck:
test: "curl -f http://nginx:8080/health-check/liveness"
nginx:
depends_on:
- "nodejs"
......@@ -38,14 +40,18 @@ services:
- ./config/nginx-default-dev.conf:/etc/nginx/sites-available/default
ports:
- "8080:8080"
healthcheck:
test: "curl -f http://nginx:8080/health-check/liveness"
scheduler:
depends_on:
- "nodejs"
restart: always
image: metager/development:latest
command: /bin/sh -c "/html/helpers/scheduler.sh"
command: /bin/sh -c "php artisan schedule:work"
volumes:
- .:/html
healthcheck:
test: "curl -f http://nginx:8080/health-check/liveness-scheduler"
worker:
depends_on:
- "nodejs"
......@@ -54,6 +60,8 @@ services:
volumes:
- .:/html
command: "php artisan requests:fetcher"
healthcheck:
test: "curl -f http://nginx:8080/health-check/liveness-worker"
# dummy-phpfpm:
# restart: on-failure
# image: registry.metager.de/open-source/dummy-engine/master
......@@ -74,6 +82,11 @@ services:
- MYSQL_USER=metager
- MYSQL_PASSWORD=metager
- MYSQL_DATABASE=metager
healthcheck:
test: ["CMD", "mysqladmin", "ping", "--silent"]
redis:
restart: on-failure
image: redis:6
\ No newline at end of file
user: "redis:redis"
image: redis:6
healthcheck:
test: "redis-cli ping"
while true;
do
php artisan schedule:run
sleep 60
done
\ No newline at end of file
......@@ -356,5 +356,11 @@ Route::group(
return response($result, 200)
->header('Content-Type', RenderTextFormat::MIME_TYPE);
});
Route::group(['prefix' => 'health-check'], function () {
Route::get('liveness', 'HealthcheckController@liveness');
Route::get('liveness-scheduler', 'HealthcheckController@livenessScheduler');
Route::get('liveness-worker', 'HealthcheckController@livenessWorker');
});
}
);
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment