Add Basic Incremental Startup

src/main.rs

@@ -143,7 +143,7 @@ fn process_request(request: tiny_http::Request, config: HashMap<String, String>,
                 request.respond(response);
                 let now = std::time::SystemTime::now();
                 let elapsed = now.duration_since(last_update).expect("Time went backwards");
-                if elapsed >= std::time::Duration::from_secs(12 * 60 * 60) { //every 12h
+                if elapsed >= std::time::Duration::from_secs(2 * 60 * 60) || (markov_chain.update_count()<1800000) { //every 2h
                     markov_chain.decay();
                     read_from_db(config.clone(), markov_chain);
                     last_update = now;
@@ -180,7 +180,7 @@ fn read_from_db(config: HashMap<String, String>, predictor: &mut CompositePredic
             match client {
                 Ok(mut c) => {
                     let mut count = 0;
-                    match c.query("SELECT query FROM public.logs_partitioned ORDER BY time DESC LIMIT 10000", &[]) {
+                    match c.query("SELECT query FROM public.logs_partitioned ORDER BY time DESC LIMIT 20000", &[]) {
                         Ok(rows) => {
                             for row in rows {
                                 let query: &str = row.get(0);

src/predictors/basic_markov.rs

@@ -11,6 +11,7 @@ pub type MarkovChain = HashMap<String, HashMap<String, usize>>;
 pub struct MarkovChainPredictor {
     chain: MarkovChain,
     configuration: HashMap<String, String>,
+    update_count: usize
 }
 
 impl Predictor for MarkovChainPredictor {
@@ -18,7 +19,8 @@ impl Predictor for MarkovChainPredictor {
     {
         MarkovChainPredictor {
             chain: HashMap::new(),
-            configuration: HashMap::new()
+            configuration: HashMap::new(),
+            update_count: 0
         }
     }
 
@@ -32,9 +34,14 @@ impl Predictor for MarkovChainPredictor {
         MarkovChainPredictor {
             chain: HashMap::new(),
             configuration,
+            update_count: 0
         }
     }
 
+    fn update_count(&self) -> usize {
+        return self.update_count;
+    }
+
     fn update_from_importer<I: Importer>(&mut self, importer:  &mut I, count: usize) -> Result<usize,Box<dyn std::error::Error>> {let mut raw_list = String::new();
         let blocklist: Vec<&str> = match self.configuration.get("blocklist") {
             Some(list) => {
@@ -80,7 +87,8 @@ impl Predictor for MarkovChainPredictor {
                     .or_insert(1);
             }
         }
-
+        
+        self.update_count += 1;
         Ok(())
     }
 

src/predictors/basic_set.rs

@@ -9,6 +9,7 @@ use super::Predictor;
 pub struct SetPredictor {
     set: HashMap<String,usize>,
     configuration: HashMap<String, String>,
+    update_count: usize
 }
 
 impl Predictor for SetPredictor {
@@ -16,7 +17,8 @@ impl Predictor for SetPredictor {
     {
         SetPredictor {
             set: HashMap::new(),
-            configuration: HashMap::new()
+            configuration: HashMap::new(),
+            update_count: 0
         }
     }
 
@@ -30,9 +32,15 @@ impl Predictor for SetPredictor {
         SetPredictor {
             set: HashMap::new(),
             configuration,
+            update_count: 0
         }
     }
 
+
+    fn update_count(&self) -> usize {
+        return self.update_count;
+    }
+
     fn update_from_importer<I: Importer>(&mut self, importer:  &mut I, count: usize) -> Result<usize,Box<dyn std::error::Error>> {
 
         let mut raw_list = String::new();
@@ -85,6 +93,7 @@ impl Predictor for SetPredictor {
             let counter = self.set.entry(word.to_string()).or_insert(0);
             *counter += 1;
         }
+        self.update_count += 1;
 
         Ok(())
     }

src/predictors/composite.rs

@@ -34,6 +34,9 @@ impl Predictor for CompositePredictor {
         }
     }
 
+    fn update_count(&self) -> usize {
+        self.markov_predictor.update_count()
+    }
 
     fn update_from_importer<I: Importer>(&mut self, importer:  &mut I, count: usize) -> Result<usize,Box<dyn std::error::Error>> {
 

src/predictors/mod.rs

@@ -11,6 +11,7 @@ pub trait Predictor {
     fn update_from_query(&mut self, query: &str, blocklist: &Vec<&str>) -> Result<(),Box<dyn std::error::Error>>;
     fn update_from_importer<I: Importer>(&mut self, importer: &mut I, count: usize) -> Result<usize,Box<dyn std::error::Error>>;
     fn decay(&mut self) -> ();
+    fn update_count(&self) -> usize;
 
     fn new() -> Self where Self: Sized;
     fn new_from_config(config: HashMap<String, impl Into<String>>) -> Self where Self:Sized;