Merge "add start and end time for continuous audit"

releasenotes/notes/add-start-end-time-for-continuous-audit-52c45052cb06d153.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add start_time and end_time fields in audits table. User can set the start
+    time and/or end time when creating CONTINUOUS audit.

watcher/api/controllers/v1/audit.py

@@ -30,11 +30,13 @@ states, visit :ref:`the Audit State machine <audit_state_machine>`.
 """
 
 import datetime
+from dateutil import tz
 
 import pecan
 from pecan import rest
 import wsme
 from wsme import types as wtypes
+from wsme import utils as wutils
 import wsmeext.pecan as wsme_pecan
 
 from oslo_log import log
@@ -86,6 +88,10 @@ class AuditPostType(wtypes.Base):
 
     hostname = wtypes.wsattr(wtypes.text, readonly=True, mandatory=False)
 
+    start_time = wsme.wsattr(datetime.datetime, mandatory=False)
+
+    end_time = wsme.wsattr(datetime.datetime, mandatory=False)
+
     def as_audit(self, context):
         audit_type_values = [val.value for val in objects.audit.AuditType]
         if self.audit_type not in audit_type_values:
@@ -104,6 +110,12 @@ class AuditPostType(wtypes.Base):
             raise exception.Invalid('Either audit_template_uuid '
                                     'or goal should be provided.')
 
+        if (self.audit_type == objects.audit.AuditType.ONESHOT.value and
+                (self.start_time not in (wtypes.Unset, None)
+                    or self.end_time not in (wtypes.Unset, None))):
+            raise exception.AuditStartEndTimeNotAllowed(
+                audit_type=self.audit_type)
+
         # If audit_template_uuid was provided, we will provide any
         # variables not included in the request, but not override
         # those variables that were included.
@@ -161,7 +173,9 @@ class AuditPostType(wtypes.Base):
             strategy_id=self.strategy,
             interval=self.interval,
             scope=self.scope,
-            auto_trigger=self.auto_trigger)
+            auto_trigger=self.auto_trigger,
+            start_time=self.start_time,
+            end_time=self.end_time)
 
 
 class AuditPatchType(types.JsonPatchType):
@@ -322,6 +336,12 @@ class Audit(base.APIBase):
     hostname = wsme.wsattr(wtypes.text, mandatory=False)
     """Hostname the audit is running on"""
 
+    start_time = wsme.wsattr(datetime.datetime, mandatory=False)
+    """The start time for continuous audit launch"""
+
+    end_time = wsme.wsattr(datetime.datetime, mandatory=False)
+    """The end time that stopping continuous audit"""
+
     def __init__(self, **kwargs):
         self.fields = []
         fields = list(objects.Audit.fields)
@@ -382,7 +402,9 @@ class Audit(base.APIBase):
                      interval='7200',
                      scope=[],
                      auto_trigger=False,
-                     next_run_time=datetime.datetime.utcnow())
+                     next_run_time=datetime.datetime.utcnow(),
+                     start_time=datetime.datetime.utcnow(),
+                     end_time=datetime.datetime.utcnow())
 
         sample.goal_id = '7ae81bb3-dec3-4289-8d6c-da80bd8001ae'
         sample.strategy_id = '7ae81bb3-dec3-4289-8d6c-da80bd8001ff'
@@ -584,6 +606,17 @@ class AuditsController(rest.RestController):
                                       'parameter spec in predefined strategy'))
 
         audit_dict = audit.as_dict()
+        # convert local time to UTC time
+        start_time_value = audit_dict.get('start_time')
+        end_time_value = audit_dict.get('end_time')
+        if start_time_value:
+            audit_dict['start_time'] = start_time_value.replace(
+                tzinfo=tz.tzlocal()).astimezone(
+                    tz.tzutc()).replace(tzinfo=None)
+        if end_time_value:
+            audit_dict['end_time'] = end_time_value.replace(
+                tzinfo=tz.tzlocal()).astimezone(
+                    tz.tzutc()).replace(tzinfo=None)
 
         new_audit = objects.Audit(context, **audit_dict)
         new_audit.create()
@@ -628,6 +661,16 @@ class AuditsController(rest.RestController):
                     reason=error_message % dict(
                         initial_state=initial_state, new_state=new_state))
 
+            patch_path = api_utils.get_patch_key(patch, 'path')
+            if patch_path in ('start_time', 'end_time'):
+                patch_value = api_utils.get_patch_value(patch, patch_path)
+                # convert string format to UTC time
+                new_patch_value = wutils.parse_isodatetime(
+                    patch_value).replace(
+                        tzinfo=tz.tzlocal()).astimezone(
+                            tz.tzutc()).replace(tzinfo=None)
+                api_utils.set_patch_value(patch, patch_path, new_patch_value)
+
             audit = Audit(**api_utils.apply_jsonpatch(audit_dict, patch))
         except api_utils.JSONPATCH_EXCEPTIONS as e:
             raise exception.PatchError(patch=patch, reason=e)

watcher/api/controllers/v1/utils.py

@@ -101,6 +101,18 @@ def get_patch_value(patch, key):
             return p['value']
 
 
+def set_patch_value(patch, key, value):
+    for p in patch:
+        if p['op'] == 'replace' and p['path'] == '/%s' % key:
+            p['value'] = value
+
+
+def get_patch_key(patch, key):
+    for p in patch:
+        if p['op'] == 'replace' and key in p.keys():
+            return p[key][1:]
+
+
 def check_audit_state_transition(patch, initial):
     is_transition_valid = True
     state_value = get_patch_value(patch, "state")

watcher/common/exception.py

@@ -260,6 +260,11 @@ class AuditIntervalNotAllowed(Invalid):
     msg_fmt = _("Interval of audit must not be set for %(audit_type)s.")
 
 
+class AuditStartEndTimeNotAllowed(Invalid):
+    msg_fmt = _("Start or End time of audit must not be set for "
+                "%(audit_type)s.")
+
+
 class AuditReferenced(Invalid):
     msg_fmt = _("Audit %(audit)s is referenced by one or multiple action "
                 "plans")

watcher/db/sqlalchemy/alembic/versions/4b16194c56bc_add_start_end_time.py

@@ -0,0 +1,24 @@
+"""add_start_end_time
+
+Revision ID: 4b16194c56bc
+Revises: 52804f2498c4
+Create Date: 2018-03-23 00:36:29.031259
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '4b16194c56bc'
+down_revision = '52804f2498c4'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column('audits', sa.Column('start_time', sa.DateTime(), nullable=True))
+    op.add_column('audits', sa.Column('end_time', sa.DateTime(), nullable=True))
+
+
+def downgrade():
+    op.drop_column('audits', 'start_time')
+    op.drop_column('audits', 'end_time')

watcher/db/sqlalchemy/models.py

@@ -184,6 +184,8 @@ class Audit(Base):
     auto_trigger = Column(Boolean, nullable=False)
     next_run_time = Column(DateTime, nullable=True)
     hostname = Column(String(255), nullable=True)
+    start_time = Column(DateTime, nullable=True)
+    end_time = Column(DateTime, nullable=True)
 
     goal = orm.relationship(Goal, foreign_keys=goal_id, lazy=None)
     strategy = orm.relationship(Strategy, foreign_keys=strategy_id, lazy=None)

watcher/decision_engine/audit/continuous.py

@@ -58,9 +58,10 @@ class ContinuousAuditHandler(base.AuditHandler):
 
     def _is_audit_inactive(self, audit):
         audit = objects.Audit.get_by_uuid(
-            self.context_show_deleted, audit.uuid)
+            self.context_show_deleted, audit.uuid, eager=True)
         if (objects.audit.AuditStateTransitionManager().is_inactive(audit) or
-                audit.hostname != CONF.host):
+                (audit.hostname != CONF.host) or
+                (self.check_audit_expired(audit))):
             # if audit isn't in active states, audit's job must be removed to
             # prevent using of inactive audit in future.
             jobs = [job for job in self.scheduler.get_jobs()
@@ -119,13 +120,26 @@ class ContinuousAuditHandler(base.AuditHandler):
                                name='execute_audit',
                                **trigger_args)
 
+    def check_audit_expired(self, audit):
+        current = datetime.datetime.utcnow()
+        # Note: if audit still didn't get into the timeframe,
+        #       skip it
+        if audit.start_time and audit.start_time > current:
+            return True
+        if audit.end_time and audit.end_time < current:
+            if audit.state != objects.audit.State.SUCCEEDED:
+                audit.state = objects.audit.State.SUCCEEDED
+                audit.save()
+            return True
+
+        return False
+
     def launch_audits_periodically(self):
         audit_context = context.RequestContext(is_admin=True)
         audit_filters = {
             'audit_type': objects.audit.AuditType.CONTINUOUS.value,
             'state__in': (objects.audit.State.PENDING,
-                          objects.audit.State.ONGOING,
+                          objects.audit.State.ONGOING),
-                          objects.audit.State.SUCCEEDED),
         }
         audit_filters['hostname'] = None
         unscheduled_audits = objects.Audit.list(
@@ -152,6 +166,8 @@ class ContinuousAuditHandler(base.AuditHandler):
         audits = objects.Audit.list(
             audit_context, filters=audit_filters, eager=True)
         for audit in audits:
+            if self.check_audit_expired(audit):
+                continue
             existing_job = scheduler_jobs.get(audit.uuid, None)
             # if audit is not presented in scheduled audits yet,
             # just add a new audit job.

watcher/objects/audit.py

@@ -88,7 +88,8 @@ class Audit(base.WatcherPersistentObject, base.WatcherObject,
     #              'interval' type has been changed from Integer to String
     # Version 1.4: Added 'name' string field
     # Version 1.5: Added 'hostname' field
-    VERSION = '1.5'
+    # Version 1.6: Added 'start_time' and 'end_time' DateTime fields
+    VERSION = '1.6'
 
     dbapi = db_api.get_instance()
 
@@ -107,6 +108,8 @@ class Audit(base.WatcherPersistentObject, base.WatcherObject,
         'next_run_time': wfields.DateTimeField(nullable=True,
                                                tzinfo_aware=False),
         'hostname': wfields.StringField(nullable=True),
+        'start_time': wfields.DateTimeField(nullable=True, tzinfo_aware=False),
+        'end_time': wfields.DateTimeField(nullable=True, tzinfo_aware=False),
 
         'goal': wfields.ObjectField('Goal', nullable=True),
         'strategy': wfields.ObjectField('Strategy', nullable=True),

watcher/tests/api/v1/test_audits.py

@@ -11,6 +11,7 @@
 #    limitations under the License.
 
 import datetime
+from dateutil import tz
 import itertools
 import mock
 
@@ -883,6 +884,41 @@ class TestPost(api_base.FunctionalTest):
         self.assertEqual(201, response.status_int)
         self.assertNotEqual(long_name, response.json['name'])
 
+    @mock.patch.object(deapi.DecisionEngineAPI, 'trigger_audit')
+    def test_create_continuous_audit_with_start_end_time(
+            self, mock_trigger_audit):
+        mock_trigger_audit.return_value = mock.ANY
+        start_time = datetime.datetime(2018, 3, 1, 0, 0)
+        end_time = datetime.datetime(2018, 4, 1, 0, 0)
+
+        audit_dict = post_get_test_audit(
+            params_to_exclude=['uuid', 'state', 'scope',
+                               'next_run_time', 'hostname', 'goal']
+        )
+        audit_dict['audit_type'] = objects.audit.AuditType.CONTINUOUS.value
+        audit_dict['interval'] = '1200'
+        audit_dict['start_time'] = str(start_time)
+        audit_dict['end_time'] = str(end_time)
+
+        response = self.post_json('/audits', audit_dict)
+        self.assertEqual('application/json', response.content_type)
+        self.assertEqual(201, response.status_int)
+        self.assertEqual(objects.audit.State.PENDING,
+                         response.json['state'])
+        self.assertEqual(audit_dict['interval'], response.json['interval'])
+        self.assertTrue(utils.is_uuid_like(response.json['uuid']))
+        return_start_time = timeutils.parse_isotime(
+            response.json['start_time'])
+        return_end_time = timeutils.parse_isotime(
+            response.json['end_time'])
+        iso_start_time = start_time.replace(
+            tzinfo=tz.tzlocal()).astimezone(tz.tzutc())
+        iso_end_time = end_time.replace(
+            tzinfo=tz.tzlocal()).astimezone(tz.tzutc())
+
+        self.assertEqual(iso_start_time, return_start_time)
+        self.assertEqual(iso_end_time, return_end_time)
+
 
 class TestDelete(api_base.FunctionalTest):
 

watcher/tests/db/utils.py

@@ -97,6 +97,9 @@ def get_test_audit(**kwargs):
         'auto_trigger': kwargs.get('auto_trigger', False),
         'next_run_time': kwargs.get('next_run_time'),
         'hostname': kwargs.get('hostname', 'host_1'),
+        'start_time': kwargs.get('start_time'),
+        'end_time': kwargs.get('end_time')
+
     }
     # ObjectField doesn't allow None nor dict, so if we want to simulate a
     # non-eager object loading, the field should not be referenced at all.

watcher/tests/decision_engine/audit/test_audit_handlers.py

@@ -468,3 +468,31 @@ class TestContinuousAuditHandler(base.DbTestCase):
         self.assertTrue(is_inactive)
         is_inactive = audit_handler._is_audit_inactive(self.audits[0])
         self.assertFalse(is_inactive)
+
+    def test_check_audit_expired(self):
+        current = datetime.datetime.utcnow()
+
+        # start_time and end_time are None
+        audit_handler = continuous.ContinuousAuditHandler()
+        result = audit_handler.check_audit_expired(self.audits[0])
+        self.assertFalse(result)
+        self.assertIsNone(self.audits[0].start_time)
+        self.assertIsNone(self.audits[0].end_time)
+
+        # current time < start_time and end_time is None
+        self.audits[0].start_time = current+datetime.timedelta(days=1)
+        result = audit_handler.check_audit_expired(self.audits[0])
+        self.assertTrue(result)
+        self.assertIsNone(self.audits[0].end_time)
+
+        # current time is between start_time and end_time
+        self.audits[0].start_time = current-datetime.timedelta(days=1)
+        self.audits[0].end_time = current+datetime.timedelta(days=1)
+        result = audit_handler.check_audit_expired(self.audits[0])
+        self.assertFalse(result)
+
+        # current time > end_time
+        self.audits[0].end_time = current-datetime.timedelta(days=1)
+        result = audit_handler.check_audit_expired(self.audits[0])
+        self.assertTrue(result)
+        self.assertEqual(objects.audit.State.SUCCEEDED, self.audits[0].state)

watcher/tests/objects/test_objects.py

@@ -412,7 +412,7 @@ expected_object_fingerprints = {
     'Goal': '1.0-93881622db05e7b67a65ca885b4a022e',
     'Strategy': '1.1-73f164491bdd4c034f48083a51bdeb7b',
     'AuditTemplate': '1.1-b291973ffc5efa2c61b24fe34fdccc0b',
-    'Audit': '1.5-e4229dee89e669d1aff0805f5c665bee',
+    'Audit': '1.6-fc4abd6f133a8b419e42e05729ed0f8b',
     'ActionPlan': '2.2-3331270cb3666c93408934826d03c08d',
     'Action': '2.0-1dd4959a7e7ac30c62ef170fe08dd935',
     'EfficacyIndicator': '1.0-655b71234a82bc7478aff964639c4bb0',