Skip to content
GitLab
Commit d1d5f74d authored by Dominik Hebeler's avatar Dominik Hebeler
Browse files

fixed memory consumption on sqlite connection

parent 0642cf57
Hide whitespace changes
Inline Side-by-side
metager/app/Http/Controllers/AdminInterface.php
View file @ d1d5f74d
......@@ -32,7 +32,7 @@ class AdminInterface extends Controller
}
}
public function getCountDataTotal(Request $request)
public function getCountData(Request $request)
{
$date = $request->input('date', '');
$date = Carbon::createFromFormat("Y-m-d H:i:s", "$date 00:00:00");
......@@ -45,99 +45,63 @@ class AdminInterface extends Controller
$year = $date->format("Y");
$month = $date->format("m");
$day = $date->format("d");
$cache_key = "admin_count_total_${interface}_${year}_${month}_${day}";
$total_count = Cache::get($cache_key);
$now = now();
$cache_key = "admin_count_data_${interface}_${year}_${month}_${day}";
$result = Cache::get($cache_key);
if ($total_count === null) {
$database_file = $this->getDatabaseLogFile($date);
if ($result === null) {
$result = [
"total" => 0,
"until_now" => 0
];
$database_file = \storage_path("logs/metager/" . $date->format("Y/m/d") . ".sqlite");
if ($database_file === null) {
abort(404);
}
$connection = new SQLiteConnection(new PDO("sqlite:$database_file", null, null, [PDO::SQLITE_ATTR_OPEN_FLAGS => PDO::SQLITE_OPEN_READONLY]));
$connection->disableQueryLog();
try {
if (!$connection->getSchemaBuilder()->hasTable("logs")) {
abort(404);
}
$total_count = $connection->table("logs");
$data = $connection->table("logs")->selectRaw("COUNT(*) AS count, time");
if ($interface !== "all") {
$total_count = $total_count->where("interface", "=", $interface);
$data = $data->where("interface", "=", $interface);
}
$total_count = $total_count->count('*');
$data->groupByRaw("STRFTIME('%s', time) / 300");
$data = $data->get();
} finally {
$connection->disconnect();
}
// No Cache for today
if ($date->isToday()) {
Cache::put($cache_key, $total_count, now()->addMinutes(5));
} else {
Cache::put($cache_key, $total_count, now()->addWeek());
}
}
$result = [
"status" => 200,
"error" => false,
"data" => [
"date" => "$year-$month-$day",
"total" => $total_count,
]
];
return \response()->json($result);
}
public function getCountDataUntil(Request $request)
{
$date = $request->input('date', '');
$date = Carbon::createFromFormat("Y-m-d", $date);
if ($date === false) {
abort(404);
}
$interface = $request->input('interface', 'all');
$year = $date->format("Y");
$month = $date->format("m");
$day = $date->format("d");
$time = now()->format("H:i:s");
$cache_key = "admin_count_until_${interface}_${year}_${month}_${day}";
$total_count = Cache::get($cache_key);
if ($total_count === null) {
$database_file = $this->getDatabaseLogFile($date);
if ($database_file === null) {
abort(404);
}
$connection = new SQLiteConnection(new PDO("sqlite:$database_file", null, null, [PDO::SQLITE_ATTR_OPEN_FLAGS => PDO::SQLITE_OPEN_READONLY]));
try {
if (!$connection->getSchemaBuilder()->hasTable("logs")) {
abort(404);
}
$total_count = $connection->table("logs")->whereTime("time", "<", $time);
if ($interface !== "all") {
$total_count = $total_count->where("interface", "=", $interface);
foreach ($data as $entry) {
$time = Carbon::createFromFormat("Y-m-d H:i:s", $entry->time);
$time->year($now->year);
$time->month($now->month);
$time->day($now->day);
$result["total"] += $entry->count;
if ($time->isBefore($now)) {
$result["until_now"] += $entry->count;
}
$total_count = $total_count->count('*');
} finally {
$connection->disconnect();
}
Cache::put($cache_key, $total_count, now()->addMinutes(5));
Cache::put($cache_key, $result, now()->addMinutes(5));
}
$result = [
"status" => 200,
"error" => false,
"data" => [
"date" => "$year-$month-$day",
"total" => $total_count,
"time" => $now->format("H:i:s"),
"total" => $result["total"],
"until_now" => $result["until_now"],
]
];
return \response()->json($result);
}
......@@ -210,47 +174,4 @@ class AdminInterface extends Controller
$status = \fpm_get_status();
return response()->json($status);
}
private function getDatabaseLogFile(Carbon $date)
{
$original_file = \storage_path("logs/metager/" . $date->format("Y/m/d") . ".sqlite");
$fast_file = \storage_path("logs/metager/fast_dir/" . $date->format("Y/m/d") . ".sqlite");
if (!\file_exists($original_file)) {
return null;
}
if ($date->isToday()) {
return $original_file;
}
$lock_hash = \md5($fast_file);
if (\file_exists($fast_file) && Redis::get($lock_hash) === null) {
return $fast_file;
}
// Verify that the target directory exists
$fast_file_dir = dirname($fast_file);
if (!\file_exists($fast_file_dir) && !\mkdir($fast_file_dir, 0777, true)) {
return null;
}
// Acquire a lock for this file to make sure we're the only process copying it
do {
$lock = Redis::setnx($lock_hash, "lock");
if ($lock === 1) {
Redis::expire($lock_hash, 15);
} else {
\usleep(350 * 1000);
}
} while ($lock !== 1);
try {
if (\copy($original_file, $fast_file)) {
return $fast_file;
}
} finally {
Redis::del($lock_hash);
}
return null;
}
}
\ No newline at end of file
metager/resources/js/admin/count.js
View file @ d1d5f74d
......@@ -13,18 +13,23 @@ load();
function load() {
let parallel = Math.floor(parallel_fetches / 2);
let fetches = [];
let fetches_same_time = loadSameTimes(parallel);
let fetches_total = loadTotals(parallel);
fetches = fetches.concat(fetches_same_time, fetches_total);
let fetches = loadData(parallel);
if (fetches.length > 0) {
let allData = Promise.all(fetches);
allData.then((res) => {
updateTable();
updateChart();
updateRecord();
load();
});
allData
.then((res) => {
updateTable();
updateChart();
updateRecord();
load();
})
.catch((reason) => {
if (reason === "Unauthorized") {
// Not logged in anymore. Reload
history.go();
}
});
} else {
updateTable();
updateChart();
......@@ -125,49 +130,7 @@ function updateTable() {
}
}
function loadTotals(parallel) {
let loading_elements = document.querySelectorAll("tr > td.total.loading");
let fetches = [];
for (let i = 0; i < loading_elements.length; i++) {
let element = loading_elements[i];
let date = element.parentNode.querySelector(".date").dataset.date;
let days_ago = parseInt(element.parentNode.dataset.days_ago);
if (fetches.length < parallel) {
fetches.push(
fetch(
"/admin/count/count-data-total?date=" + date + "&interface=" + lang,
{ redirect: "error" }
)
.then((response) => {
if (response.status === 302) {
// We are not logged in anymore
history.go();
} else {
return response.json();
}
})
.then((response) => {
let total_requests = parseInt(response.data.total);
if (!data[days_ago]) {
data[days_ago] = {};
}
data[days_ago]["total"] = total_requests;
})
.catch((reason) => {
// We are not logged in anymore
history.go();
})
);
} else {
break;
}
}
return fetches;
}
function loadSameTimes(parallel) {
function loadData(parallel) {
let loading_elements = document.querySelectorAll("tr > td.same-time.loading");
let fetches = [];
......@@ -179,30 +142,30 @@ function loadSameTimes(parallel) {
if (fetches.length < parallel) {
fetches.push(
fetch(
"/admin/count/count-data-until?date=" + date + "&interface=" + lang,
{ redirect: "error" }
)
fetch("/admin/count/count-data?date=" + date + "&interface=" + lang)
.then((response) => {
if (response.status === 302) {
// We are not logged in anymore
history.go();
if (response.status === 404) {
// File Could not be found
return {
data: {
total: 0,
until_now: 0,
},
};
} else if (response.redirected) {
throw "Unauthorized";
} else {
return response.json();
}
})
.then((response) => {
let total_requests = parseInt(response.data.total);
let until_now = parseInt(response.data.until_now);
if (!data[days_ago]) {
data[days_ago] = {};
}
data[days_ago]["same_time"] = total_requests;
})
.catch((reason) => {
if (!data[days_ago]) {
data[days_ago] = {};
}
data[days_ago]["same_time"] = 0;
data[days_ago]["total"] = total_requests;
data[days_ago]["same_time"] = until_now;
})
);
} else {
......
metager/routes/session.php
View file @ d1d5f74d
......@@ -7,11 +7,10 @@ Route::get('login', [Vizir\KeycloakWebGuard\Controllers\AuthController::class, "
Route::get('logout', [Vizir\KeycloakWebGuard\Controllers\AuthController::class, "logout"])->name('keycloak.logout');
Route::get('callback', [Vizir\KeycloakWebGuard\Controllers\AuthController::class, "callback"])->name('keycloak.callback');
Route::group(['middleware' => ['keycloak-web'], 'prefix' => 'admin'], function () {
Route::group(['middleware' => ["keycloak-web"], 'prefix' => 'admin'], function () {
Route::get('fpm-status', [AdminInterface::class, "getFPMStatus"])->name("fpm-status");
Route::get('count', 'AdminInterface@count');
Route::get('count/count-data-total', [AdminInterface::class, 'getCountDataTotal']);
Route::get('count/count-data-until', [AdminInterface::class, 'getCountDataUntil']);
Route::get('count/count-data', [AdminInterface::class, 'getCountData']);
Route::get('timings', 'MetaGerSearch@searchTimings');
Route::get('engine/stats.json', 'AdminInterface@engineStats');
Route::get('check', 'AdminInterface@check');
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment