Merge Embedded Novelty Detection Feature into Main

owlyshield_predict/Cargo.toml

@@ -8,6 +8,29 @@ license-file = "LICENSE.txt"
 moonfire-tflite = { path = "moonfire-tflite" }
 slc-paths = { path = "slc-paths" }
 win-pe-inspection = { path = "win-pe-inspection" }
+<<<<<<< HEAD
+wchar = "0.11.0"
+registry = "1.2.0"
+strum = "0.24.1"
+strum_macros = "0.24.2"
+byteorder = "1.4.3"
+chrono = { version = "0.4.19", features = ["serde"] }
+num = "0.4.0"
+num-derive = "0.3.3"
+num-traits = "0.2.15"
+serde = { version = "1.0.140", features = ["derive"] }
+serde_json = "1.0.82"
+serde_yaml = "0.9.17"
+log = "0.4.17"
+rmp-serde = "1.1.0"
+hostname = "0.3.1"
+curl = "0.4.43"
+jsonrpc-http-server = "18.0.0"
+configparser = "3.0.2"
+rumqtt = "0.31.0"
+lru = "0.9.0"
+sysinfo = "0.27.7"
+=======
 wchar = "0.11"
 registry = "1.2"
 strum = "0.24"
@@ -28,6 +51,7 @@ configparser = "3.0"
 rumqtt = "0.31"
 lru = "^0.11"
 sysinfo = "0.28"
+>>>>>>> 611eb295336686ce16d056e2f0c12193efefb68a
 
 #Windows
 [target.'cfg(target_os = "windows")'.dependencies]
@@ -71,3 +95,4 @@ record = []
 replay = []
 jsonrpc = []
 mqtt = []
+novelty = []

owlyshield_predict/slc-paths/Cargo.toml

@@ -6,8 +6,14 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+<<<<<<< HEAD
+kodama = { git = "https://github.com/SubconsciousCompute/kodama" } # uses a nightly feature
+serde = { version = "1.0.140", features = ["derive"] }
+serde_json = "1.0.82"
+=======
 #kodama = { git = "https://github.com/SubconsciousCompute/kodama" } # uses a nightly feature
 kodama = "0.2.0"
+>>>>>>> 611eb295336686ce16d056e2f0c12193efefb68a
 
 [dev-dependencies]
 criterion = "0.4.0"

owlyshield_predict/slc-paths/src/lib.rs

@@ -107,12 +107,13 @@ pub mod clustering {
     use std::io::BufRead;
     use std::io::BufReader;
     use std::path::Path;
+    use serde::{Serialize, Deserialize};
 
-    #[derive(Debug)]
+    #[derive(Serialize, Deserialize, Debug)]
     pub struct Cluster {
-        root: String,
-        size: usize,
-        step: usize,
+        pub root: String,
+        pub size: usize,
+        pub step: usize,
     }
 
     pub type Clusters = Vec<Cluster>;

owlyshield_predict/src/config.rs

@@ -3,8 +3,7 @@ use std::ops::Index;
 use std::path::{Path, PathBuf};
 use configparser::ini::Ini;
 #[cfg(target_os = "windows")]
-use registry::*;
-use strum::IntoEnumIterator;
+use registry::{Hive, Security};
 use strum_macros::EnumIter;
 
 use crate::extensions::ExtensionList;
@@ -20,6 +19,7 @@ pub enum Param {
     Language,
     Telemetry,
     MqttServer,
+    NoveltyPath,
     Unknown,
 }
 
@@ -43,6 +43,7 @@ impl Param {
             Param::Language => "LANGUAGE",    // Language used at installation
             Param::Telemetry => "TELEMETRY",  // 1 if telemetry is active, 0 if not
             Param::MqttServer => "MQTT_SERVER",
+            Param::NoveltyPath => "NOVELTY_PATH",
             _ => "UNKNOWN"
         }
     }
@@ -59,6 +60,7 @@ impl Param {
             Param::Language => "language",    // Language used at installation
             Param::Telemetry => "telemetry",  // 1 if telemetry is active, 0 if not
             Param::MqttServer => "mqtt_server",
+            Param::NoveltyPath => "novelty_path",
             _ => "unknown"
         }
     }
@@ -83,6 +85,10 @@ impl Param {
             params.push(Param::MqttServer);
         }
 
+        if cfg!(feature = "novelty") {
+            params.push(Param::NoveltyPath);
+        }
+
         let mut ret = Vec::new();
         for param in params {
             let val = Self::convert_to_str(&param).to_string();
@@ -103,6 +109,7 @@ impl Param {
             "LANGUAGE" => Param::Language,    // Language used at installation
             "TELEMETRY" => Param::Telemetry,  // 1 if telemetry is active, 0 if not
             "MQTT_SERVER" => Param::MqttServer,
+            "NOVELTY_PATH" => Param::NoveltyPath,
             _ => Param::Unknown,
         }
     }
@@ -119,6 +126,7 @@ impl Param {
             "language" => Param::Language,    // Language used at installation
             "telemetry" => Param::Telemetry,  // 1 if telemetry is active, 0 if not
             "mqtt_server" => Param::MqttServer,
+            "novelty_path" => Param::NoveltyPath,
             _ => Param::Unknown,
         }
     }

owlyshield_predict/src/linux/run.rs

@@ -13,7 +13,14 @@ use crate::threathandling::LinuxThreatHandler;
 use crate::whitelist;
 use crate::Connectors;
 use crate::ExepathLive;
+<<<<<<< HEAD
+use crate::ProcessRecordHandlerLive;
+use crate::ProcessRecordHandlerNovelty;
+use crate::IOMsgPostProcessorWriter;
+use crate::IOMsgPostProcessorRPC;
+=======
 use crate::IOMessage;
+>>>>>>> 611eb295336686ce16d056e2f0c12193efefb68a
 use crate::IOMsgPostProcessorMqtt;
 use crate::IOMsgPostProcessorRPC;
 use crate::IOMsgPostProcessorWriter;
@@ -26,6 +33,21 @@ use std::io::{Read, Seek, SeekFrom};
 use std::path::Path;
 use std::sync::mpsc::channel;
 use std::thread;
+<<<<<<< HEAD
+use crate::config::Param;
+use crate::driver_com::Buf;
+use crate::watchlist::WatchList;
+
+fn probe_code() -> &'static [u8] {
+    include_bytes!(
+        concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/target/bpf/programs/openmonitor/openmonitor.elf"
+        )
+        // "/home/fedora/redbpf_test/target/bpf/programs/openmonitor/openmonitor.elf"
+    )
+}
+=======
 
 use aya::maps::perf::AsyncPerfEventArray;
 use aya::programs::KProbe;
@@ -37,6 +59,7 @@ use ebpf_monitor_common::*;
 use log::{debug, info, warn};
 use tokio::signal;
 use tokio::task;
+>>>>>>> 611eb295336686ce16d056e2f0c12193efefb68a
 
 #[tokio::main(flavor = "current_thread")]
 pub async fn run() -> Result<(), anyhow::Error> {
@@ -53,13 +76,18 @@ pub async fn run() -> Result<(), anyhow::Error> {
         println!("Replay Driver Messages");
         let config = config::Config::new();
         let whitelist = whitelist::WhiteList::from(
-            &Path::new(&config[config::Param::ConfigPath]).join(Path::new("exclusions.txt")),
+            &Path::new(&config[Param::ConfigPath]).join(Path::new("exclusions.txt")),
         )
             .unwrap();
         let mut worker = Worker::new_replay(&config, &whitelist);
 
+<<<<<<< HEAD
+        let filename =
+            &Path::new(&config[Param::ProcessActivityLogPath]).join(Path::new("drivermessages.txt"));
+=======
         let filename = &Path::new(&config[config::Param::ProcessActivityLogPath])
             .join(Path::new("drivermessages.txt"));
+>>>>>>> 611eb295336686ce16d056e2f0c12193efefb68a
         let mut file = File::open(Path::new(filename)).unwrap();
         let file_len = file.metadata().unwrap().len() as usize;
 
@@ -116,12 +144,18 @@ pub async fn run() -> Result<(), anyhow::Error> {
             //NEW
             thread::spawn(move || {
                 let whitelist = whitelist::WhiteList::from(
-                    &Path::new(&config[config::Param::ConfigPath])
+                    &Path::new(&config[Param::ConfigPath])
                         .join(Path::new("exclusions.txt")),
                 )
                     .expect("Cannot open exclusions.txt");
                 whitelist.refresh_periodically();
 
+                let watchlist = WatchList::from(
+                    &Path::new(&config[Param::NoveltyPath])
+                        .join(Path::new("to_analyze.yml")),
+                ).expect("Cannot open to_analyze.yml");
+                watchlist.refresh_periodically();
+
                 let mut worker = Worker::new();
 
                 worker = worker.exepath_handler(Box::new(ExepathLive::default()));
@@ -135,6 +169,13 @@ pub async fn run() -> Result<(), anyhow::Error> {
                         )));
                 }
 
+                if cfg!(feature = "novelty") {
+                    worker = worker
+                        .process_record_handler(Box::new(ProcessRecordHandlerNovelty::new(
+                            &config, watchlist,
+                        )));
+                }
+
                 if cfg!(feature = "record") {
                     worker = worker.register_iomsg_postprocessor(Box::new(
                         IOMsgPostProcessorWriter::from(&config),

owlyshield_predict/src/logging.rs

@@ -14,6 +14,7 @@ enum Status {
     Alert, // Program detected a malware
     Warning, // Warning in program execution
     Error, // Error in program execution
+    Novelty, // Notice a novelty
 }
 
 impl Status {
@@ -24,6 +25,7 @@ impl Status {
             Status::Alert => "ALERT",
             Status::Warning => "WARNING",
             Status::Error => "ERROR",
+            Status::Novelty => "NOVELTY",
         }
     }
 }
@@ -69,12 +71,17 @@ impl Logging {
         Logging::log(Status::Error, message);
     }
 
+    /// Notice a novelty
+    pub fn novelty(message: &str) {
+        Logging::log(Status::Novelty, message);
+    }
+
     #[cfg(target_os = "windows")]
     fn log(status: Status, message: &str) {
         Self::log_in_file(status, message, ConfigReader::read_param_from_registry("LOG_PATH", r"SOFTWARE\Owlyshield").as_str());
 
         match status.clone() {
-            Status::Alert | Status::Warning => { warn!("{}: {}", status.to_str(), message); },
+            Status::Alert | Status::Warning | Status::Novelty => { warn!("{}: {}", status.to_str(), message); },
             Status::Error => error!("{}: {}", status.to_str(), message),
             _ => {
                 if message.is_empty() {

owlyshield_predict/src/main.rs

@@ -40,7 +40,7 @@ use crate::driver_com::CDriverMsgs;
 use crate::driver_com::LDriverMsg;
 use crate::shared_def::IOMessage;
 use crate::logging::Logging;
-use crate::worker::process_record_handling::{ExepathLive, ProcessRecordHandlerLive};
+use crate::worker::process_record_handling::{ExepathLive, ProcessRecordHandlerLive, ProcessRecordHandlerNovelty};
 use crate::worker::worker_instance::{IOMsgPostProcessorMqtt, IOMsgPostProcessorRPC, IOMsgPostProcessorWriter, Worker};
 
 mod actions_on_kill;
@@ -72,14 +72,19 @@ mod run;
 mod run;
 mod shared_def;
 mod utils;
+mod watchlist;
 mod whitelist;
 mod worker;
+<<<<<<< HEAD
+mod novelty;
+=======
 #[cfg(target_os = "windows")]
 #[path = "windows/threathandling.rs"]
 mod threathandling;
 #[cfg(target_os = "linux")]
 #[path = "linux/threathandling.rs"]
 mod threathandling;
+>>>>>>> 611eb295336686ce16d056e2f0c12193efefb68a
 
 #[cfg(feature = "service")]
 const SERVICE_NAME: &str = "Owlyshield Service";