Adding TDD to false positives with test case to validate if model can…

… learn from false positives
AICoE · May 16, 2019 · 3d31608 · 3d31608
1 parent 5bac71d
commit 3d31608
Show file tree

Hide file tree

Showing 7 changed files with 333,067 additions and 14 deletions.
diff --git a/.test_env_config.yaml b/.test_env_config.yaml
@@ -0,0  1,6 @@
 LS_INPUT_PATH: "/Users/zhassan/git/log-anomaly-detector/validation_data/log_anomaly_detector-100000-events.json"
 LS_OUTPUT_PATH: "/Users/zhassan/git/log-anomaly-detector/validation_data/local-results-1.txt"
 INFER_TIME_SPAN: 1
 INFER_LOOPS: 1
 INFER_ANOMALY_THRESHOLD:      0.1
 PARALLELISM: 6
diff --git a/Pipfile b/Pipfile
@@ -32,6  32,7 @@ pyyaml = "*"
 Flask-SQLAlchemy = "*"
 PyMySQL = "*"
 
 
 [requires]
 python_version = "3.6"
 

diff --git a/anomaly_detector/anomaly_detector.py b/anomaly_detector/anomaly_detector.py
@@ -8,7  8,6 @@
 import matplotlib
 import numpy as np
 from prometheus_client import start_http_server, Gauge, Counter
-
 from .config import Configuration
 from .events.anomaly_event import AnomalyEvent
 from .model.model_exception import ModelLoadException, ModelSaveException
@@ -80,7  79,8 @@ def _load_data(self, time_span, max_entries):
         return data, raw
 
     @TRAINING_TIME.time()
-    def train(self):
     # @profile
     def train(self, fp=None):
         """Train models for anomaly detection."""
         start = time.time()
         data, _ = self._load_data(self.config.TRAIN_TIME_SPAN, self.config.TRAIN_MAX_ENTRIES)
@@ -103,23  103,22 @@ def train(self):
 
         _LOGGER.info("Training and Saving took %s minutes", ((now - then) / 60))
         _LOGGER.info("Encoding Text Data")
-
-        to_put_train = self.w2v_model.one_vector(data)
-
         data.append(fp)
         combo = self.w2v_model.one_vector(data)
         _LOGGER.info("Start Training SOM...")
 
         then = time.time()
 
         if self.recreate_models or self.update_model:
-            self.model.set(np.random.rand(24, 24, to_put_train.shape[1]))
             self.model.set(np.random.rand(24, 24, combo.shape[1]))
 
-        self.model.train(to_put_train, 24, self.config.TRAIN_ITERATIONS, self.config.PARALLELISM)
         self.model.train(combo, 24, self.config.TRAIN_ITERATIONS, self.config.PARALLELISM)
         now = time.time()
 
         _LOGGER.info("Training took %s minutes", ((now - then) / 60))
         _LOGGER.info("Generating Baseline Metrics")
 
-        dist = self.model.get_anomaly_score(to_put_train, self.config.PARALLELISM)
         dist = self.model.get_anomaly_score(combo, self.config.PARALLELISM)
 
         self.model.set_metadata((np.mean(dist), np.std(dist), np.max(dist), np.min(dist)))
         try:
@@ -132,7  131,7 @@ def train(self):
         _LOGGER.info("Whole Process takes %s minutes", ((end - start) / 60))
         self.syncModel()
         TRAINING_COUNT.inc()
-        return 0
         return 0, dist
 
     def infer(self):
         """Perform inference on trained models."""
@@ -198,7  197,6 @@ def infer(self):
                     except ConnectionError as e:
                         _LOGGER.info("Fact store is down unable to check")
                         s["anomaly"] = 1
-
                     _LOGGER.warn("Anomaly found (score: %f): %s" % (dist[i], s["message"]))
                 else:
                     s["anomaly"] = 0
@@ -216,7  214,7 @@ def infer(self):
                 time.sleep(sleep_time)
 
         # When we reached # of inference loops, retrain models
-        self.recreate_models = True
         self.recreate_models = False
 
     def syncModel(self):
         """Store models in s3 with timestamp.
@@ -254,11  252,11 @@ def syncModel(self):
         else:
             _LOGGER.debug("Must set MODEL_STORE='s3' to save to s3")
 
-    def run(self):
     def run(self, single_run=False):
         """Run the main loop."""
         start_http_server(8080)
-
-        while True:
         srun = False
         while True and srun is False:
             if self.update_model or self.update_w2v_model or self.recreate_models:
                 try:
                     self.train()
@@ -273,3  271,5 @@ def run(self):
             except Exception as ex:
                 _LOGGER.error("Inference failed: %s" % ex)
                 raise ex
 
             srun = single_run
diff --git a/tests/test_false_anomaly_check.py b/tests/test_false_anomaly_check.py
@@ -0,0  1,40 @@
 import pytest
 from anomaly_detector.anomaly_detector import AnomalyDetector
 from anomaly_detector.config import Configuration
 import json
 from pandas.io.json import json_normalize
 
 import numpy as np
 CONFIGURATION_PREFIX = "LAD"
 
 
 @pytest.fixture()
 def detector():
     config = Configuration(prefix=CONFIGURATION_PREFIX, config_yaml=".test_env_config.yaml")
     anomaly_detector = AnomalyDetector(config)
     return anomaly_detector
 
 
 def test_false_positive(detector, benchmark):
     """
     Testing False Positives and feeding it into the model
     """
     # TODO: Increase the frequency of the false anomalies
     FALSE_POSITIVE = [{"message": "(root) CMD (/usr/local/bin/monitor-apache-stats.sh >/dev/null 2>&1)"}]
     FALSE_POSITIVE2 = [{"message": "(root) CMD (/usr/local/bin/monitor-apache-stats.sh >/dev/null 2>&1)"}] * 10000
     buffer=[]
     fp = json_normalize(FALSE_POSITIVE)
     fp2 = json_normalize(FALSE_POSITIVE2)
     success, dist = detector.train(fp=fp)
     print(np.mean(dist), np.std(dist), np.max(dist), np.min(dist))
     num=dist[-1]
     buffer.append(num)
     success, dist = detector.train(fp=fp2)
     print(np.mean(dist), np.std(dist), np.max(dist), np.min(dist))
     num2=dist[-1]
     buffer.append(num2)
     print(buffer)
     print("Second Training Run: Finished")
     found=True
 
     assert found is True