From c2550e534e5013f11b776b924a83675ef6d8959c Mon Sep 17 00:00:00 2001 From: licanwei Date: Tue, 27 Mar 2018 02:23:56 -0700 Subject: [PATCH] add start and end time for continuous audit Add new start_time and end_time fields in the audit table Partially Implements: blueprint add-start-end-time-for-continuous-audit Change-Id: I6bb838d777b2c7aa799a70485980e5dc87838456 --- ...for-continuous-audit-52c45052cb06d153.yaml | 5 ++ watcher/api/controllers/v1/audit.py | 47 ++++++++++++++++++- watcher/api/controllers/v1/utils.py | 12 +++++ watcher/common/exception.py | 5 ++ .../4b16194c56bc_add_start_end_time.py | 24 ++++++++++ watcher/db/sqlalchemy/models.py | 2 + watcher/decision_engine/audit/continuous.py | 24 ++++++++-- watcher/objects/audit.py | 5 +- watcher/tests/api/v1/test_audits.py | 36 ++++++++++++++ watcher/tests/db/utils.py | 3 ++ .../audit/test_audit_handlers.py | 28 +++++++++++ watcher/tests/objects/test_objects.py | 2 +- 12 files changed, 185 insertions(+), 8 deletions(-) create mode 100644 releasenotes/notes/add-start-end-time-for-continuous-audit-52c45052cb06d153.yaml create mode 100644 watcher/db/sqlalchemy/alembic/versions/4b16194c56bc_add_start_end_time.py diff --git a/releasenotes/notes/add-start-end-time-for-continuous-audit-52c45052cb06d153.yaml b/releasenotes/notes/add-start-end-time-for-continuous-audit-52c45052cb06d153.yaml new file mode 100644 index 000000000..4660fa10c --- /dev/null +++ b/releasenotes/notes/add-start-end-time-for-continuous-audit-52c45052cb06d153.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add start_time and end_time fields in audits table. User can set the start + time and/or end time when creating CONTINUOUS audit. diff --git a/watcher/api/controllers/v1/audit.py b/watcher/api/controllers/v1/audit.py index 220f73cd8..e5c721ae0 100644 --- a/watcher/api/controllers/v1/audit.py +++ b/watcher/api/controllers/v1/audit.py @@ -30,11 +30,13 @@ states, visit :ref:`the Audit State machine `. """ import datetime +from dateutil import tz import pecan from pecan import rest import wsme from wsme import types as wtypes +from wsme import utils as wutils import wsmeext.pecan as wsme_pecan from oslo_log import log @@ -86,6 +88,10 @@ class AuditPostType(wtypes.Base): hostname = wtypes.wsattr(wtypes.text, readonly=True, mandatory=False) + start_time = wsme.wsattr(datetime.datetime, mandatory=False) + + end_time = wsme.wsattr(datetime.datetime, mandatory=False) + def as_audit(self, context): audit_type_values = [val.value for val in objects.audit.AuditType] if self.audit_type not in audit_type_values: @@ -104,6 +110,12 @@ class AuditPostType(wtypes.Base): raise exception.Invalid('Either audit_template_uuid ' 'or goal should be provided.') + if (self.audit_type == objects.audit.AuditType.ONESHOT.value and + (self.start_time not in (wtypes.Unset, None) + or self.end_time not in (wtypes.Unset, None))): + raise exception.AuditStartEndTimeNotAllowed( + audit_type=self.audit_type) + # If audit_template_uuid was provided, we will provide any # variables not included in the request, but not override # those variables that were included. @@ -161,7 +173,9 @@ class AuditPostType(wtypes.Base): strategy_id=self.strategy, interval=self.interval, scope=self.scope, - auto_trigger=self.auto_trigger) + auto_trigger=self.auto_trigger, + start_time=self.start_time, + end_time=self.end_time) class AuditPatchType(types.JsonPatchType): @@ -322,6 +336,12 @@ class Audit(base.APIBase): hostname = wsme.wsattr(wtypes.text, mandatory=False) """Hostname the audit is running on""" + start_time = wsme.wsattr(datetime.datetime, mandatory=False) + """The start time for continuous audit launch""" + + end_time = wsme.wsattr(datetime.datetime, mandatory=False) + """The end time that stopping continuous audit""" + def __init__(self, **kwargs): self.fields = [] fields = list(objects.Audit.fields) @@ -382,7 +402,9 @@ class Audit(base.APIBase): interval='7200', scope=[], auto_trigger=False, - next_run_time=datetime.datetime.utcnow()) + next_run_time=datetime.datetime.utcnow(), + start_time=datetime.datetime.utcnow(), + end_time=datetime.datetime.utcnow()) sample.goal_id = '7ae81bb3-dec3-4289-8d6c-da80bd8001ae' sample.strategy_id = '7ae81bb3-dec3-4289-8d6c-da80bd8001ff' @@ -584,6 +606,17 @@ class AuditsController(rest.RestController): 'parameter spec in predefined strategy')) audit_dict = audit.as_dict() + # convert local time to UTC time + start_time_value = audit_dict.get('start_time') + end_time_value = audit_dict.get('end_time') + if start_time_value: + audit_dict['start_time'] = start_time_value.replace( + tzinfo=tz.tzlocal()).astimezone( + tz.tzutc()).replace(tzinfo=None) + if end_time_value: + audit_dict['end_time'] = end_time_value.replace( + tzinfo=tz.tzlocal()).astimezone( + tz.tzutc()).replace(tzinfo=None) new_audit = objects.Audit(context, **audit_dict) new_audit.create() @@ -628,6 +661,16 @@ class AuditsController(rest.RestController): reason=error_message % dict( initial_state=initial_state, new_state=new_state)) + patch_path = api_utils.get_patch_key(patch, 'path') + if patch_path in ('start_time', 'end_time'): + patch_value = api_utils.get_patch_value(patch, patch_path) + # convert string format to UTC time + new_patch_value = wutils.parse_isodatetime( + patch_value).replace( + tzinfo=tz.tzlocal()).astimezone( + tz.tzutc()).replace(tzinfo=None) + api_utils.set_patch_value(patch, patch_path, new_patch_value) + audit = Audit(**api_utils.apply_jsonpatch(audit_dict, patch)) except api_utils.JSONPATCH_EXCEPTIONS as e: raise exception.PatchError(patch=patch, reason=e) diff --git a/watcher/api/controllers/v1/utils.py b/watcher/api/controllers/v1/utils.py index 956edbfed..e9e8fadfd 100644 --- a/watcher/api/controllers/v1/utils.py +++ b/watcher/api/controllers/v1/utils.py @@ -101,6 +101,18 @@ def get_patch_value(patch, key): return p['value'] +def set_patch_value(patch, key, value): + for p in patch: + if p['op'] == 'replace' and p['path'] == '/%s' % key: + p['value'] = value + + +def get_patch_key(patch, key): + for p in patch: + if p['op'] == 'replace' and key in p.keys(): + return p[key][1:] + + def check_audit_state_transition(patch, initial): is_transition_valid = True state_value = get_patch_value(patch, "state") diff --git a/watcher/common/exception.py b/watcher/common/exception.py index e64595123..22de6bd32 100644 --- a/watcher/common/exception.py +++ b/watcher/common/exception.py @@ -260,6 +260,11 @@ class AuditIntervalNotAllowed(Invalid): msg_fmt = _("Interval of audit must not be set for %(audit_type)s.") +class AuditStartEndTimeNotAllowed(Invalid): + msg_fmt = _("Start or End time of audit must not be set for " + "%(audit_type)s.") + + class AuditReferenced(Invalid): msg_fmt = _("Audit %(audit)s is referenced by one or multiple action " "plans") diff --git a/watcher/db/sqlalchemy/alembic/versions/4b16194c56bc_add_start_end_time.py b/watcher/db/sqlalchemy/alembic/versions/4b16194c56bc_add_start_end_time.py new file mode 100644 index 000000000..2c1c1af86 --- /dev/null +++ b/watcher/db/sqlalchemy/alembic/versions/4b16194c56bc_add_start_end_time.py @@ -0,0 +1,24 @@ +"""add_start_end_time + +Revision ID: 4b16194c56bc +Revises: 52804f2498c4 +Create Date: 2018-03-23 00:36:29.031259 + +""" + +# revision identifiers, used by Alembic. +revision = '4b16194c56bc' +down_revision = '52804f2498c4' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('audits', sa.Column('start_time', sa.DateTime(), nullable=True)) + op.add_column('audits', sa.Column('end_time', sa.DateTime(), nullable=True)) + + +def downgrade(): + op.drop_column('audits', 'start_time') + op.drop_column('audits', 'end_time') diff --git a/watcher/db/sqlalchemy/models.py b/watcher/db/sqlalchemy/models.py index 2c86281d6..d014b676f 100644 --- a/watcher/db/sqlalchemy/models.py +++ b/watcher/db/sqlalchemy/models.py @@ -184,6 +184,8 @@ class Audit(Base): auto_trigger = Column(Boolean, nullable=False) next_run_time = Column(DateTime, nullable=True) hostname = Column(String(255), nullable=True) + start_time = Column(DateTime, nullable=True) + end_time = Column(DateTime, nullable=True) goal = orm.relationship(Goal, foreign_keys=goal_id, lazy=None) strategy = orm.relationship(Strategy, foreign_keys=strategy_id, lazy=None) diff --git a/watcher/decision_engine/audit/continuous.py b/watcher/decision_engine/audit/continuous.py index 50e00c054..ed5267bd5 100644 --- a/watcher/decision_engine/audit/continuous.py +++ b/watcher/decision_engine/audit/continuous.py @@ -58,9 +58,10 @@ class ContinuousAuditHandler(base.AuditHandler): def _is_audit_inactive(self, audit): audit = objects.Audit.get_by_uuid( - self.context_show_deleted, audit.uuid) + self.context_show_deleted, audit.uuid, eager=True) if (objects.audit.AuditStateTransitionManager().is_inactive(audit) or - audit.hostname != CONF.host): + (audit.hostname != CONF.host) or + (self.check_audit_expired(audit))): # if audit isn't in active states, audit's job must be removed to # prevent using of inactive audit in future. jobs = [job for job in self.scheduler.get_jobs() @@ -119,13 +120,26 @@ class ContinuousAuditHandler(base.AuditHandler): name='execute_audit', **trigger_args) + def check_audit_expired(self, audit): + current = datetime.datetime.utcnow() + # Note: if audit still didn't get into the timeframe, + # skip it + if audit.start_time and audit.start_time > current: + return True + if audit.end_time and audit.end_time < current: + if audit.state != objects.audit.State.SUCCEEDED: + audit.state = objects.audit.State.SUCCEEDED + audit.save() + return True + + return False + def launch_audits_periodically(self): audit_context = context.RequestContext(is_admin=True) audit_filters = { 'audit_type': objects.audit.AuditType.CONTINUOUS.value, 'state__in': (objects.audit.State.PENDING, - objects.audit.State.ONGOING, - objects.audit.State.SUCCEEDED), + objects.audit.State.ONGOING), } audit_filters['hostname'] = None unscheduled_audits = objects.Audit.list( @@ -152,6 +166,8 @@ class ContinuousAuditHandler(base.AuditHandler): audits = objects.Audit.list( audit_context, filters=audit_filters, eager=True) for audit in audits: + if self.check_audit_expired(audit): + continue existing_job = scheduler_jobs.get(audit.uuid, None) # if audit is not presented in scheduled audits yet, # just add a new audit job. diff --git a/watcher/objects/audit.py b/watcher/objects/audit.py index bd3b1c58b..d040a8063 100644 --- a/watcher/objects/audit.py +++ b/watcher/objects/audit.py @@ -88,7 +88,8 @@ class Audit(base.WatcherPersistentObject, base.WatcherObject, # 'interval' type has been changed from Integer to String # Version 1.4: Added 'name' string field # Version 1.5: Added 'hostname' field - VERSION = '1.5' + # Version 1.6: Added 'start_time' and 'end_time' DateTime fields + VERSION = '1.6' dbapi = db_api.get_instance() @@ -107,6 +108,8 @@ class Audit(base.WatcherPersistentObject, base.WatcherObject, 'next_run_time': wfields.DateTimeField(nullable=True, tzinfo_aware=False), 'hostname': wfields.StringField(nullable=True), + 'start_time': wfields.DateTimeField(nullable=True, tzinfo_aware=False), + 'end_time': wfields.DateTimeField(nullable=True, tzinfo_aware=False), 'goal': wfields.ObjectField('Goal', nullable=True), 'strategy': wfields.ObjectField('Strategy', nullable=True), diff --git a/watcher/tests/api/v1/test_audits.py b/watcher/tests/api/v1/test_audits.py index bccb1d294..9fff442f1 100644 --- a/watcher/tests/api/v1/test_audits.py +++ b/watcher/tests/api/v1/test_audits.py @@ -11,6 +11,7 @@ # limitations under the License. import datetime +from dateutil import tz import itertools import mock @@ -883,6 +884,41 @@ class TestPost(api_base.FunctionalTest): self.assertEqual(201, response.status_int) self.assertNotEqual(long_name, response.json['name']) + @mock.patch.object(deapi.DecisionEngineAPI, 'trigger_audit') + def test_create_continuous_audit_with_start_end_time( + self, mock_trigger_audit): + mock_trigger_audit.return_value = mock.ANY + start_time = datetime.datetime(2018, 3, 1, 0, 0) + end_time = datetime.datetime(2018, 4, 1, 0, 0) + + audit_dict = post_get_test_audit( + params_to_exclude=['uuid', 'state', 'scope', + 'next_run_time', 'hostname', 'goal'] + ) + audit_dict['audit_type'] = objects.audit.AuditType.CONTINUOUS.value + audit_dict['interval'] = '1200' + audit_dict['start_time'] = str(start_time) + audit_dict['end_time'] = str(end_time) + + response = self.post_json('/audits', audit_dict) + self.assertEqual('application/json', response.content_type) + self.assertEqual(201, response.status_int) + self.assertEqual(objects.audit.State.PENDING, + response.json['state']) + self.assertEqual(audit_dict['interval'], response.json['interval']) + self.assertTrue(utils.is_uuid_like(response.json['uuid'])) + return_start_time = timeutils.parse_isotime( + response.json['start_time']) + return_end_time = timeutils.parse_isotime( + response.json['end_time']) + iso_start_time = start_time.replace( + tzinfo=tz.tzlocal()).astimezone(tz.tzutc()) + iso_end_time = end_time.replace( + tzinfo=tz.tzlocal()).astimezone(tz.tzutc()) + + self.assertEqual(iso_start_time, return_start_time) + self.assertEqual(iso_end_time, return_end_time) + class TestDelete(api_base.FunctionalTest): diff --git a/watcher/tests/db/utils.py b/watcher/tests/db/utils.py index 85c9614bc..840007a09 100644 --- a/watcher/tests/db/utils.py +++ b/watcher/tests/db/utils.py @@ -97,6 +97,9 @@ def get_test_audit(**kwargs): 'auto_trigger': kwargs.get('auto_trigger', False), 'next_run_time': kwargs.get('next_run_time'), 'hostname': kwargs.get('hostname', 'host_1'), + 'start_time': kwargs.get('start_time'), + 'end_time': kwargs.get('end_time') + } # ObjectField doesn't allow None nor dict, so if we want to simulate a # non-eager object loading, the field should not be referenced at all. diff --git a/watcher/tests/decision_engine/audit/test_audit_handlers.py b/watcher/tests/decision_engine/audit/test_audit_handlers.py index 142e2bc7c..72a352e35 100644 --- a/watcher/tests/decision_engine/audit/test_audit_handlers.py +++ b/watcher/tests/decision_engine/audit/test_audit_handlers.py @@ -468,3 +468,31 @@ class TestContinuousAuditHandler(base.DbTestCase): self.assertTrue(is_inactive) is_inactive = audit_handler._is_audit_inactive(self.audits[0]) self.assertFalse(is_inactive) + + def test_check_audit_expired(self): + current = datetime.datetime.utcnow() + + # start_time and end_time are None + audit_handler = continuous.ContinuousAuditHandler() + result = audit_handler.check_audit_expired(self.audits[0]) + self.assertFalse(result) + self.assertIsNone(self.audits[0].start_time) + self.assertIsNone(self.audits[0].end_time) + + # current time < start_time and end_time is None + self.audits[0].start_time = current+datetime.timedelta(days=1) + result = audit_handler.check_audit_expired(self.audits[0]) + self.assertTrue(result) + self.assertIsNone(self.audits[0].end_time) + + # current time is between start_time and end_time + self.audits[0].start_time = current-datetime.timedelta(days=1) + self.audits[0].end_time = current+datetime.timedelta(days=1) + result = audit_handler.check_audit_expired(self.audits[0]) + self.assertFalse(result) + + # current time > end_time + self.audits[0].end_time = current-datetime.timedelta(days=1) + result = audit_handler.check_audit_expired(self.audits[0]) + self.assertTrue(result) + self.assertEqual(objects.audit.State.SUCCEEDED, self.audits[0].state) diff --git a/watcher/tests/objects/test_objects.py b/watcher/tests/objects/test_objects.py index 2225cc989..237d483d5 100644 --- a/watcher/tests/objects/test_objects.py +++ b/watcher/tests/objects/test_objects.py @@ -412,7 +412,7 @@ expected_object_fingerprints = { 'Goal': '1.0-93881622db05e7b67a65ca885b4a022e', 'Strategy': '1.1-73f164491bdd4c034f48083a51bdeb7b', 'AuditTemplate': '1.1-b291973ffc5efa2c61b24fe34fdccc0b', - 'Audit': '1.5-e4229dee89e669d1aff0805f5c665bee', + 'Audit': '1.6-fc4abd6f133a8b419e42e05729ed0f8b', 'ActionPlan': '2.2-3331270cb3666c93408934826d03c08d', 'Action': '2.0-1dd4959a7e7ac30c62ef170fe08dd935', 'EfficacyIndicator': '1.0-655b71234a82bc7478aff964639c4bb0',