From 0def7b4d382cca516a8ab94be1c485eb08cf6a96 Mon Sep 17 00:00:00 2001
From: licanwei <li.canwei2@zte.com.cn>
Date: Fri, 19 Apr 2019 14:30:09 +0800
Subject: [PATCH] separate launching audit scheduler

Now there are only one scheduler for launching audit task and
executing audit jobs. We have found an exception where the scheduler
stops for some reason when executing audit.
In order to keep launching audit task normal, we need to split into
two schedulers.

Change-Id: I45dccaf062290cfc7d7fcfc27fe11d6f87f38afa
---
 watcher/decision_engine/audit/continuous.py   | 30 +++++++++++++------
 .../audit/test_audit_handlers.py              |  2 +-
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/watcher/decision_engine/audit/continuous.py b/watcher/decision_engine/audit/continuous.py
index ed5267bd5..f9682d6e3 100644
--- a/watcher/decision_engine/audit/continuous.py
+++ b/watcher/decision_engine/audit/continuous.py
@@ -21,7 +21,6 @@
 import datetime
 from dateutil import tz
 
-from apscheduler.jobstores import memory
 from croniter import croniter
 
 from watcher.common import context
@@ -40,21 +39,29 @@ CONF = conf.CONF
 class ContinuousAuditHandler(base.AuditHandler):
     def __init__(self):
         super(ContinuousAuditHandler, self).__init__()
-        self._scheduler = None
+        # scheduler for executing audits
+        self._audit_scheduler = None
+        # scheduler for a periodic task to launch audit
+        self._period_scheduler = None
         self.context_show_deleted = context.RequestContext(is_admin=True,
                                                            show_deleted=True)
 
     @property
     def scheduler(self):
-        if self._scheduler is None:
-            self._scheduler = scheduling.BackgroundSchedulerService(
+        if self._audit_scheduler is None:
+            self._audit_scheduler = scheduling.BackgroundSchedulerService(
                 jobstores={
                     'default': job_store.WatcherJobStore(
                         engine=sq_api.get_engine()),
-                    'memory': memory.MemoryJobStore()
                 }
             )
-        return self._scheduler
+        return self._audit_scheduler
+
+    @property
+    def period_scheduler(self):
+        if self._period_scheduler is None:
+            self._period_scheduler = scheduling.BackgroundSchedulerService()
+        return self._period_scheduler
 
     def _is_audit_inactive(self, audit):
         audit = objects.Audit.get_by_uuid(
@@ -135,6 +142,10 @@ class ContinuousAuditHandler(base.AuditHandler):
         return False
 
     def launch_audits_periodically(self):
+        # if audit scheduler stop, restart it
+        if not self.scheduler.running:
+            self.scheduler.start()
+
         audit_context = context.RequestContext(is_admin=True)
         audit_filters = {
             'audit_type': objects.audit.AuditType.CONTINUOUS.value,
@@ -207,10 +218,11 @@ class ContinuousAuditHandler(base.AuditHandler):
                 audit.save()
 
     def start(self):
-        self.scheduler.add_job(
+        self.period_scheduler.add_job(
             self.launch_audits_periodically,
             'interval',
             seconds=CONF.watcher_decision_engine.continuous_audit_interval,
-            next_run_time=datetime.datetime.now(),
-            jobstore='memory')
+            next_run_time=datetime.datetime.now())
+        self.period_scheduler.start()
+        # audit scheduler start
         self.scheduler.start()
diff --git a/watcher/tests/decision_engine/audit/test_audit_handlers.py b/watcher/tests/decision_engine/audit/test_audit_handlers.py
index 63a5b135d..de1000116 100644
--- a/watcher/tests/decision_engine/audit/test_audit_handlers.py
+++ b/watcher/tests/decision_engine/audit/test_audit_handlers.py
@@ -462,7 +462,7 @@ class TestContinuousAuditHandler(base.DbTestCase):
     def test_is_audit_inactive(self, mock_jobs):
         audit_handler = continuous.ContinuousAuditHandler()
         mock_jobs.return_value = mock.MagicMock()
-        audit_handler._scheduler = mock.MagicMock()
+        audit_handler._audit_scheduler = mock.MagicMock()
 
         ap_jobs = [job.Job(mock.MagicMock(), name='execute_audit',
                            func=audit_handler.execute_audit,