From 775b09f4a5c9f12b1d3a470d09adbea9b5470e66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Phil=20H=C3=B6fer?= <phil.hoefer@suma-ev.de>
Date: Mon, 22 Jul 2024 14:34:17 +0200
Subject: [PATCH] Add Basic Incremental Startup
---
src/main.rs | 4 ++--
src/predictors/basic_markov.rs | 12 ++++++++++--
src/predictors/basic_set.rs | 11 ++++++++++-
src/predictors/composite.rs | 3 +++
src/predictors/mod.rs | 1 +
5 files changed, 26 insertions(+), 5 deletions(-)
diff --git a/src/main.rs b/src/main.rs
index 8b75aab..0c1125b 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -143,7 +143,7 @@ fn process_request(request: tiny_http::Request, config: HashMap<String, String>,
request.respond(response);
let now = std::time::SystemTime::now();
let elapsed = now.duration_since(last_update).expect("Time went backwards");
- if elapsed >= std::time::Duration::from_secs(12 * 60 * 60) { //every 12h
+ if elapsed >= std::time::Duration::from_secs(2 * 60 * 60) || (markov_chain.update_count()<1800000) { //every 2h
markov_chain.decay();
read_from_db(config.clone(), markov_chain);
last_update = now;
@@ -180,7 +180,7 @@ fn read_from_db(config: HashMap<String, String>, predictor: &mut CompositePredic
match client {
Ok(mut c) => {
let mut count = 0;
- match c.query("SELECT query FROM public.logs_partitioned ORDER BY time DESC LIMIT 10000", &[]) {
+ match c.query("SELECT query FROM public.logs_partitioned ORDER BY time DESC LIMIT 20000", &[]) {
Ok(rows) => {
for row in rows {
let query: &str = row.get(0);
diff --git a/src/predictors/basic_markov.rs b/src/predictors/basic_markov.rs
index ed4bcc3..40ad615 100644
--- a/src/predictors/basic_markov.rs
+++ b/src/predictors/basic_markov.rs
@@ -11,6 +11,7 @@ pub type MarkovChain = HashMap<String, HashMap<String, usize>>;
pub struct MarkovChainPredictor {
chain: MarkovChain,
configuration: HashMap<String, String>,
+ update_count: usize
}
impl Predictor for MarkovChainPredictor {
@@ -18,7 +19,8 @@ impl Predictor for MarkovChainPredictor {
{
MarkovChainPredictor {
chain: HashMap::new(),
- configuration: HashMap::new()
+ configuration: HashMap::new(),
+ update_count: 0
}
}
@@ -32,9 +34,14 @@ impl Predictor for MarkovChainPredictor {
MarkovChainPredictor {
chain: HashMap::new(),
configuration,
+ update_count: 0
}
}
+ fn update_count(&self) -> usize {
+ return self.update_count;
+ }
+
fn update_from_importer<I: Importer>(&mut self, importer: &mut I, count: usize) -> Result<usize,Box<dyn std::error::Error>> {let mut raw_list = String::new();
let blocklist: Vec<&str> = match self.configuration.get("blocklist") {
Some(list) => {
@@ -80,7 +87,8 @@ impl Predictor for MarkovChainPredictor {
.or_insert(1);
}
}
-
+
+ self.update_count += 1;
Ok(())
}
diff --git a/src/predictors/basic_set.rs b/src/predictors/basic_set.rs
index 05edb13..27f0af3 100644
--- a/src/predictors/basic_set.rs
+++ b/src/predictors/basic_set.rs
@@ -9,6 +9,7 @@ use super::Predictor;
pub struct SetPredictor {
set: HashMap<String,usize>,
configuration: HashMap<String, String>,
+ update_count: usize
}
impl Predictor for SetPredictor {
@@ -16,7 +17,8 @@ impl Predictor for SetPredictor {
{
SetPredictor {
set: HashMap::new(),
- configuration: HashMap::new()
+ configuration: HashMap::new(),
+ update_count: 0
}
}
@@ -30,9 +32,15 @@ impl Predictor for SetPredictor {
SetPredictor {
set: HashMap::new(),
configuration,
+ update_count: 0
}
}
+
+ fn update_count(&self) -> usize {
+ return self.update_count;
+ }
+
fn update_from_importer<I: Importer>(&mut self, importer: &mut I, count: usize) -> Result<usize,Box<dyn std::error::Error>> {
let mut raw_list = String::new();
@@ -85,6 +93,7 @@ impl Predictor for SetPredictor {
let counter = self.set.entry(word.to_string()).or_insert(0);
*counter += 1;
}
+ self.update_count += 1;
Ok(())
}
diff --git a/src/predictors/composite.rs b/src/predictors/composite.rs
index 10494ac..ad43241 100644
--- a/src/predictors/composite.rs
+++ b/src/predictors/composite.rs
@@ -34,6 +34,9 @@ impl Predictor for CompositePredictor {
}
}
+ fn update_count(&self) -> usize {
+ self.markov_predictor.update_count()
+ }
fn update_from_importer<I: Importer>(&mut self, importer: &mut I, count: usize) -> Result<usize,Box<dyn std::error::Error>> {
diff --git a/src/predictors/mod.rs b/src/predictors/mod.rs
index e8d1def..09ce077 100644
--- a/src/predictors/mod.rs
+++ b/src/predictors/mod.rs
@@ -11,6 +11,7 @@ pub trait Predictor {
fn update_from_query(&mut self, query: &str, blocklist: &Vec<&str>) -> Result<(),Box<dyn std::error::Error>>;
fn update_from_importer<I: Importer>(&mut self, importer: &mut I, count: usize) -> Result<usize,Box<dyn std::error::Error>>;
fn decay(&mut self) -> ();
+ fn update_count(&self) -> usize;
fn new() -> Self where Self: Sized;
fn new_from_config(config: HashMap<String, impl Into<String>>) -> Self where Self:Sized;
--
GitLab