From cc083ec2847ae9d7e7c2c69569bf5697d7db4efa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Phil=20H=C3=B6fer?= <phil.hoefer@suma-ev.de>
Date: Mon, 22 Jul 2024 12:03:02 +0200
Subject: [PATCH] Start Implementing New Blocklist
---
src/predictors/basic_markov.rs | 21 +++++++++++++++++----
src/predictors/basic_set.rs | 21 +++++++++++++++++----
2 files changed, 34 insertions(+), 8 deletions(-)
diff --git a/src/predictors/basic_markov.rs b/src/predictors/basic_markov.rs
index be08257..963a7fb 100644
--- a/src/predictors/basic_markov.rs
+++ b/src/predictors/basic_markov.rs
@@ -1,4 +1,4 @@
-use std::{collections::HashMap, fs::File};
+use std::{collections::HashMap, fs::{self, File}};
use csv::ReaderBuilder;
@@ -46,13 +46,26 @@ impl Predictor for MarkovChainPredictor {
}
fn update_from_query(&mut self, query: &str) -> Result<(), Box<dyn std::error::Error>> {
- let blocklist: Vec<&str> = match self.configuration.get("blocked_words") {
+ let mut raw_list = String::new();
+ let blocklist: Vec<&str> = match self.configuration.get("blocklist") {
Some(list) => {
- list.split_whitespace().collect()
+ match fs::read_to_string(list) {
+ Ok(l) => {
+ raw_list = l.clone();
+ raw_list.lines().collect()
+ },
+ _ => Vec::new()
+ }
},
_ => Vec::new()
- };
+ };
+
+ for word in blocklist.clone() {
+ if query.contains(word) {
+ return Ok(());
+ }
+ }
//println!("blocklist:{:?}",self.configuration);
diff --git a/src/predictors/basic_set.rs b/src/predictors/basic_set.rs
index a5eb354..d208039 100644
--- a/src/predictors/basic_set.rs
+++ b/src/predictors/basic_set.rs
@@ -1,4 +1,4 @@
-use std::{collections::HashMap, f32::consts::E, fs::File};
+use std::{collections::HashMap, f32::consts::E, fs::{self, File}};
use crate::importers::Importer;
@@ -45,13 +45,26 @@ impl Predictor for SetPredictor {
}
fn update_from_query(&mut self, query: &str) -> Result<(), Box<dyn std::error::Error>> {
- let blocklist: Vec<&str> = match self.configuration.get("blocked_words") {
+ let mut raw_list = String::new();
+ let blocklist: Vec<&str> = match self.configuration.get("blocklist") {
Some(list) => {
- list.split_whitespace().collect()
+ match fs::read_to_string(list) {
+ Ok(l) => {
+ raw_list = l.clone();
+ raw_list.lines().collect()
+ },
+ _ => Vec::new()
+ }
},
_ => Vec::new()
- };
+ };
+
+ for word in blocklist.clone() {
+ if query.contains(word) {
+ return Ok(());
+ }
+ }
//println!("blocklist:{:?}",self.configuration);
--
GitLab