diff --git a/watcher/decision_engine/sync.py b/watcher/decision_engine/sync.py index a1086464a..3bb2a903b 100644 --- a/watcher/decision_engine/sync.py +++ b/watcher/decision_engine/sync.py @@ -14,22 +14,28 @@ # See the License for the specific language governing permissions and # limitations under the License. +import collections + from oslo_log import log -from watcher._i18n import _LI +from watcher._i18n import _LE, _LI from watcher.common import context from watcher.decision_engine.strategy.loading import default from watcher import objects LOG = log.getLogger(__name__) +GoalMapping = collections.namedtuple('GoalMapping', ['name', 'display_name']) +StrategyMapping = collections.namedtuple( + 'StrategyMapping', ['name', 'goal_name', 'display_name']) + class Syncer(object): """Syncs all available goals and strategies with the Watcher DB""" def __init__(self): self.ctx = context.make_context() - self._discovered_map = None + self.discovered_map = None self._available_goals = None self._available_goals_map = None @@ -42,6 +48,8 @@ class Syncer(object): # This strategy mapping maps stale strategy IDs to the synced goal self.strategy_mapping = dict() + self.stale_audit_templates_map = {} + @property def available_goals(self): if self._available_goals is None: @@ -58,7 +66,8 @@ class Syncer(object): def available_goals_map(self): if self._available_goals_map is None: self._available_goals_map = { - g: {"name": g.name, "display_name": g.display_name} + GoalMapping( + name=g.name, display_name=g.display_name): g for g in self.available_goals } return self._available_goals_map @@ -68,41 +77,45 @@ class Syncer(object): if self._available_strategies_map is None: goals_map = {g.id: g.name for g in self.available_goals} self._available_strategies_map = { - s: {"name": s.name, "goal_name": goals_map[s.goal_id], - "display_name": s.display_name} + StrategyMapping( + name=s.name, goal_name=goals_map[s.goal_id], + display_name=s.display_name): s for s in self.available_strategies } return self._available_strategies_map def sync(self): - discovered_map = self.discover() - goals_map = discovered_map["goals"] - strategies_map = discovered_map["strategies"] + self.discovered_map = self._discover() + goals_map = self.discovered_map["goals"] + strategies_map = self.discovered_map["strategies"] for goal_name, goal_map in goals_map.items(): - if goal_map in self.available_goals_map.values(): + if goal_map in self.available_goals_map: LOG.info(_LI("Goal %s already exists"), goal_name) continue self.goal_mapping.update(self._sync_goal(goal_map)) for strategy_name, strategy_map in strategies_map.items(): - if strategy_map in self.available_strategies_map.values(): + if (strategy_map in self.available_strategies_map and + strategy_map.goal_name not in + [g.name for g in self.goal_mapping.values()]): LOG.info(_LI("Strategy %s already exists"), strategy_name) continue self.strategy_mapping.update(self._sync_strategy(strategy_map)) - # TODO(v-francoise): Sync the audit templates + self._sync_audit_templates() def _sync_goal(self, goal_map): - goal_name = goal_map['name'] - goal_display_name = goal_map['display_name'] + goal_name = goal_map.name + goal_display_name = goal_map.display_name goal_mapping = dict() + # Goals that are matching by name with the given discovered goal name matching_goals = [g for g in self.available_goals if g.name == goal_name] - stale_goals = self.soft_delete_stale_goals(goal_map, matching_goals) + stale_goals = self._soft_delete_stale_goals(goal_map, matching_goals) if stale_goals or not matching_goals: goal = objects.Goal(self.ctx) @@ -120,14 +133,16 @@ class Syncer(object): return goal_mapping def _sync_strategy(self, strategy_map): - strategy_name = strategy_map['name'] - strategy_display_name = strategy_map['display_name'] - goal_name = strategy_map['goal_name'] + strategy_name = strategy_map.name + strategy_display_name = strategy_map.display_name + goal_name = strategy_map.goal_name strategy_mapping = dict() + # Strategies that are matching by name with the given + # discovered strategy name matching_strategies = [s for s in self.available_strategies if s.name == strategy_name] - stale_strategies = self.soft_delete_stale_strategies( + stale_strategies = self._soft_delete_stale_strategies( strategy_map, matching_strategies) if stale_strategies or not matching_strategies: @@ -146,7 +161,95 @@ class Syncer(object): return strategy_mapping - def discover(self): + def _sync_audit_templates(self): + # First we find audit templates that are stale because their associated + # goal or strategy has been modified and we update them in-memory + self._find_stale_audit_templates_due_to_goal() + self._find_stale_audit_templates_due_to_strategy() + + # Then we handle the case where an audit template became + # stale because its related goal does not exist anymore. + self._soft_delete_removed_goals() + # Then we handle the case where an audit template became + # stale because its related strategy does not exist anymore. + self._soft_delete_removed_strategies() + + # Finally, we save into the DB the updated stale audit templates + for stale_audit_template in self.stale_audit_templates_map.values(): + stale_audit_template.save() + LOG.info(_LI("Audit Template '%s' synced"), + stale_audit_template.name) + + def _find_stale_audit_templates_due_to_goal(self): + for goal_id, synced_goal in self.goal_mapping.items(): + filters = {"goal_id": goal_id} + stale_audit_templates = objects.AuditTemplate.list( + self.ctx, filters=filters) + + # Update the goal ID for the stale audit templates (w/o saving) + for audit_template in stale_audit_templates: + if audit_template.id not in self.stale_audit_templates_map: + audit_template.goal_id = synced_goal.id + self.stale_audit_templates_map[audit_template.id] = ( + audit_template) + else: + self.stale_audit_templates_map[ + audit_template.id].goal_id = synced_goal.id + + def _find_stale_audit_templates_due_to_strategy(self): + for strategy_id, synced_strategy in self.strategy_mapping.items(): + filters = {"strategy_id": strategy_id} + stale_audit_templates = objects.AuditTemplate.list( + self.ctx, filters=filters) + + # Update strategy IDs for all stale audit templates (w/o saving) + for audit_template in stale_audit_templates: + if audit_template.id not in self.stale_audit_templates_map: + audit_template.strategy_id = synced_strategy.id + self.stale_audit_templates_map[audit_template.id] = ( + audit_template) + else: + self.stale_audit_templates_map[ + audit_template.id].strategy_id = synced_strategy.id + + def _soft_delete_removed_goals(self): + removed_goals = [ + g for g in self.available_goals + if g.name not in self.discovered_map['goals']] + for removed_goal in removed_goals: + removed_goal.soft_delete() + filters = {"goal_id": removed_goal.id} + invalid_ats = objects.AuditTemplate.list(self.ctx, filters=filters) + for at in invalid_ats: + LOG.warning( + _LE("Audit Template '%(audit_template)s' references a " + "goal that does not exist"), + audit_template=at.uuid) + + def _soft_delete_removed_strategies(self): + removed_strategies = [ + s for s in self.available_strategies + if s.name not in self.discovered_map['strategies']] + + for removed_strategy in removed_strategies: + removed_strategy.soft_delete() + filters = {"strategy_id": removed_strategy.id} + invalid_ats = objects.AuditTemplate.list(self.ctx, filters=filters) + for at in invalid_ats: + LOG.info( + _LI("Audit Template '%(audit_template)s' references a " + "strategy that does not exist"), + audit_template=at.uuid) + # In this case we can reset the strategy ID to None + # so the audit template can still achieve the same goal + # but with a different strategy + if at.id not in self.stale_audit_templates_map: + at.strategy_id = None + self.stale_audit_templates_map[at.id] = at + else: + self.stale_audit_templates_map[at.id].strategy_id = None + + def _discover(self): strategies_map = {} goals_map = {} discovered_map = {"goals": goals_map, "strategies": strategies_map} @@ -154,25 +257,25 @@ class Syncer(object): implemented_strategies = strategy_loader.list_available() for _, strategy_cls in implemented_strategies.items(): - goals_map[strategy_cls.get_goal_name()] = { - "name": strategy_cls.get_goal_name(), - "display_name": - strategy_cls.get_translatable_goal_display_name()} + goals_map[strategy_cls.get_goal_name()] = GoalMapping( + name=strategy_cls.get_goal_name(), + display_name=strategy_cls.get_translatable_goal_display_name()) - strategies_map[strategy_cls.get_name()] = { - "name": strategy_cls.get_name(), - "goal_name": strategy_cls.get_goal_name(), - "display_name": strategy_cls.get_translatable_display_name()} + strategies_map[strategy_cls.get_name()] = StrategyMapping( + name=strategy_cls.get_name(), + goal_name=strategy_cls.get_goal_name(), + display_name=strategy_cls.get_translatable_display_name()) return discovered_map - def soft_delete_stale_goals(self, goal_map, matching_goals): - goal_name = goal_map['name'] - goal_display_name = goal_map['display_name'] + def _soft_delete_stale_goals(self, goal_map, matching_goals): + goal_name = goal_map.name + goal_display_name = goal_map.display_name stale_goals = [] for matching_goal in matching_goals: - if matching_goal.display_name == goal_display_name: + if (matching_goal.display_name == goal_display_name and + matching_goal.strategy_id not in self.strategy_mapping): LOG.info(_LI("Goal %s unchanged"), goal_name) else: LOG.info(_LI("Goal %s modified"), goal_name) @@ -181,13 +284,14 @@ class Syncer(object): return stale_goals - def soft_delete_stale_strategies(self, strategy_map, matching_strategies): - strategy_name = strategy_map['name'] - strategy_display_name = strategy_map['display_name'] + def _soft_delete_stale_strategies(self, strategy_map, matching_strategies): + strategy_name = strategy_map.name + strategy_display_name = strategy_map.display_name stale_strategies = [] for matching_strategy in matching_strategies: - if matching_strategy.display_name == strategy_display_name: + if (matching_strategy.display_name == strategy_display_name and + matching_strategy.goal_id not in self.goal_mapping): LOG.info(_LI("Strategy %s unchanged"), strategy_name) else: LOG.info(_LI("Strategy %s modified"), strategy_name) diff --git a/watcher/tests/decision_engine/test_sync.py b/watcher/tests/decision_engine/test_sync.py index efa2a286e..5aae86a98 100644 --- a/watcher/tests/decision_engine/test_sync.py +++ b/watcher/tests/decision_engine/test_sync.py @@ -31,6 +31,7 @@ class TestSyncer(base.DbTestCase): super(TestSyncer, self).setUp() self.ctx = context.make_context() + # This mock simulates the strategies discovery done in discover() self.m_available_strategies = mock.Mock(return_value={ fake_strategies.FakeDummy1Strategy1.get_name(): fake_strategies.FakeDummy1Strategy1, @@ -201,54 +202,254 @@ class TestSyncer(base.DbTestCase): self.assertEqual(1, m_s_soft_delete.call_count) def test_end2end_sync_goals_with_modified_goal_and_strategy(self): - goal = objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(), - name="DUMMY_1", display_name="Original") - goal.create() - strategy = objects.Strategy( - self.ctx, id=1, name="STRATEGY_1", - display_name="Original", goal_id=goal.id) - strategy.create() - # audit_template = objects.AuditTemplate( - # self.ctx, id=1, name="Synced AT", goal_id=goal.id, - # strategy_id=strategy.id) - # audit_template.create() + # ### Setup ### # - # before_audit_templates = objects.AuditTemplate.list(self.ctx) + # Here, we simulate goals and strategies already discovered in the past + # that were saved in DB + + # Should stay unmodified after sync() + goal1 = objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(), + name="DUMMY_1", display_name="Dummy 1") + # Should be modified by the sync() + goal2 = objects.Goal(self.ctx, id=2, uuid=utils.generate_uuid(), + name="DUMMY_2", display_name="Original") + goal1.create() + goal2.create() + + # Should stay unmodified after sync() + strategy1 = objects.Strategy( + self.ctx, id=1, name="STRATEGY_1", uuid=utils.generate_uuid(), + display_name="Strategy 1", goal_id=goal1.id) + # Should stay unmodified after sync() + strategy2 = objects.Strategy( + self.ctx, id=2, name="STRATEGY_2", uuid=utils.generate_uuid(), + display_name="Strategy 2", goal_id=goal2.id) + # Should be modified by the sync() + strategy3 = objects.Strategy( + self.ctx, id=3, name="STRATEGY_3", uuid=utils.generate_uuid(), + display_name="Original", goal_id=goal2.id) + # Should be modified by the sync() + strategy4 = objects.Strategy( + self.ctx, id=4, name="STRATEGY_4", uuid=utils.generate_uuid(), + display_name="Original", goal_id=goal2.id) + strategy1.create() + strategy2.create() + strategy3.create() + strategy4.create() + + # Here we simulate audit_templates that were already created in the + # past and hence saved within the Watcher DB + + # Should stay unmodified after sync() + audit_template1 = objects.AuditTemplate( + self.ctx, id=1, uuid=utils.generate_uuid(), + name="Synced AT1", goal_id=goal1.id, strategy_id=strategy1.id) + # Should be modified by the sync() because its associated goal + # should be modified + audit_template2 = objects.AuditTemplate( + self.ctx, id=2, name="Synced AT2", uuid=utils.generate_uuid(), + goal_id=goal2.id, strategy_id=strategy2.id) + # Should be modified by the sync() because its associated strategy + # should be modified + audit_template3 = objects.AuditTemplate( + self.ctx, id=3, name="Synced AT3", uuid=utils.generate_uuid(), + goal_id=goal2.id, strategy_id=strategy3.id) + # Modified because of both because its associated goal and associated + # strategy should be modified + audit_template4 = objects.AuditTemplate( + self.ctx, id=4, name="Synced AT4", uuid=utils.generate_uuid(), + goal_id=goal2.id, strategy_id=strategy4.id) + audit_template1.create() + audit_template2.create() + audit_template3.create() + audit_template4.create() + + before_audit_templates = objects.AuditTemplate.list(self.ctx) before_goals = objects.Goal.list(self.ctx) before_strategies = objects.Strategy.list(self.ctx) + # ### Action under test ### # + try: self.syncer.sync() except Exception as exc: self.fail(exc) - # after_audit_templates = objects.AuditTemplate.list(self.ctx) + # ### Assertions ### # + + after_audit_templates = objects.AuditTemplate.list(self.ctx) after_goals = objects.Goal.list(self.ctx) after_strategies = objects.Strategy.list(self.ctx) - self.assertEqual(1, len(before_goals)) - self.assertEqual(1, len(before_strategies)) - # self.assertEqual(1, len(before_audit_templates)) + self.assertEqual(2, len(before_goals)) + self.assertEqual(4, len(before_strategies)) + self.assertEqual(4, len(before_audit_templates)) self.assertEqual(2, len(after_goals)) self.assertEqual(4, len(after_strategies)) - # self.assertEqual(1, len(after_audit_templates)) + self.assertEqual(4, len(after_audit_templates)) self.assertEqual( {"DUMMY_1", "DUMMY_2"}, set([g.name for g in after_goals])) self.assertEqual( {"STRATEGY_1", "STRATEGY_2", "STRATEGY_3", "STRATEGY_4"}, set([s.name for s in after_strategies])) + created_goals = { + ag.name: ag for ag in after_goals + if ag.uuid not in [bg.uuid for bg in before_goals] + } + created_strategies = { + a_s.name: a_s for a_s in after_strategies + if a_s.uuid not in [b_s.uuid for b_s in before_strategies] + } + + self.assertEqual(1, len(created_goals)) + self.assertEqual(3, len(created_strategies)) + + modified_audit_templates = { + a_at.id for a_at in after_audit_templates + if a_at.goal_id not in ( + # initial goal IDs + b_at.goal_id for b_at in before_audit_templates) or + a_at.strategy_id not in ( + # initial strategy IDs + b_at.strategy_id for b_at in before_audit_templates + if b_at.strategy_id is not None) + } + + unmodified_audit_templates = { + a_at.id for a_at in after_audit_templates + if a_at.goal_id in ( + # initial goal IDs + b_at.goal_id for b_at in before_audit_templates) and + a_at.strategy_id in ( + # initial strategy IDs + b_at.strategy_id for b_at in before_audit_templates + if b_at.strategy_id is not None) + } + + self.assertEqual(2, strategy2.goal_id) + self.assertIn(strategy2.name, created_strategies) + self.assertTrue(strategy2.id != created_strategies[strategy2.name].id) + + self.assertEqual(set([audit_template2.id, + audit_template3.id, + audit_template4.id]), + modified_audit_templates) + self.assertEqual(set([audit_template1.id]), + unmodified_audit_templates) + + def test_end2end_sync_goals_with_removed_goal_and_strategy(self): + # ### Setup ### # + + # We Simulate the fact that we removed 2 strategies + # as well as the DUMMY_2 goal + self.m_available_strategies.return_value = { + fake_strategies.FakeDummy1Strategy1.get_name(): + fake_strategies.FakeDummy1Strategy1 + } + + # Should stay unmodified after sync() + goal1 = objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(), + name="DUMMY_1", display_name="Dummy 1") + # To be removed by the sync() + goal2 = objects.Goal(self.ctx, id=2, uuid=utils.generate_uuid(), + name="DUMMY_2", display_name="Dummy 2") + goal1.create() + goal2.create() + + # Should stay unmodified after sync() + strategy1 = objects.Strategy( + self.ctx, id=1, name="STRATEGY_1", uuid=utils.generate_uuid(), + display_name="Strategy 1", goal_id=goal1.id) + # To be removed by the sync() + strategy2 = objects.Strategy( + self.ctx, id=2, name="STRATEGY_2", uuid=utils.generate_uuid(), + display_name="Strategy 2", goal_id=goal1.id) + # To be removed by the sync() + strategy3 = objects.Strategy( + self.ctx, id=3, name="STRATEGY_3", uuid=utils.generate_uuid(), + display_name="Original", goal_id=goal2.id) + strategy1.create() + strategy2.create() + strategy3.create() + + # Here we simulate audit_templates that were already created in the + # past and hence saved within the Watcher DB + + # The strategy of this audit template will be dereferenced + # as it does not exist anymore + audit_template1 = objects.AuditTemplate( + self.ctx, id=1, uuid=utils.generate_uuid(), + name="Synced AT1", goal_id=goal1.id, strategy_id=strategy1.id) + # Stale even after syncing because the goal has been soft deleted + audit_template2 = objects.AuditTemplate( + self.ctx, id=2, name="Synced AT2", uuid=utils.generate_uuid(), + goal_id=goal2.id, strategy_id=strategy2.id) + + audit_template1.create() + audit_template2.create() + + before_audit_templates = objects.AuditTemplate.list(self.ctx) + before_goals = objects.Goal.list(self.ctx) + before_strategies = objects.Strategy.list(self.ctx) + + # ### Action under test ### # + + try: + self.syncer.sync() + except Exception as exc: + self.fail(exc) + + # ### Assertions ### # + + after_audit_templates = objects.AuditTemplate.list(self.ctx) + after_goals = objects.Goal.list(self.ctx) + after_strategies = objects.Strategy.list(self.ctx) + + self.assertEqual(2, len(before_goals)) + self.assertEqual(3, len(before_strategies)) + self.assertEqual(2, len(before_audit_templates)) + self.assertEqual(1, len(after_goals)) + self.assertEqual(1, len(after_strategies)) + self.assertEqual(2, len(after_audit_templates)) + self.assertEqual( + {"DUMMY_1"}, + set([g.name for g in after_goals])) + self.assertEqual( + {"STRATEGY_1"}, + set([s.name for s in after_strategies])) created_goals = [ag for ag in after_goals if ag.uuid not in [bg.uuid for bg in before_goals]] created_strategies = [ a_s for a_s in after_strategies if a_s.uuid not in [b_s.uuid for b_s in before_strategies]] - self.assertEqual(2, len(created_goals)) - self.assertEqual(4, len(created_strategies)) + self.assertEqual(0, len(created_goals)) + self.assertEqual(0, len(created_strategies)) - # synced_audit_template = after_audit_templates[0] - # self.assertTrue( - # audit_template.goal_id != synced_audit_template.goal_id) - # self.assertIn(synced_audit_template.goal_id, - # (g.id for g in after_goals)) + modified_audit_templates = { + a_at.id for a_at in after_audit_templates + if a_at.goal_id not in ( + # initial goal IDs + b_at.goal_id for b_at in before_audit_templates) or + a_at.strategy_id not in ( + # initial strategy IDs + b_at.strategy_id for b_at in before_audit_templates + if b_at.strategy_id is not None) + } + + unmodified_audit_templates = { + a_at.id for a_at in after_audit_templates + if a_at.goal_id in ( + # initial goal IDs + b_at.goal_id for b_at in before_audit_templates) and + a_at.strategy_id in ( + # initial strategy IDs + b_at.strategy_id for b_at in before_audit_templates + if b_at.strategy_id is not None) + } + + self.assertEqual(set([audit_template2.id]), + modified_audit_templates) + self.assertEqual(set([audit_template1.id]), + unmodified_audit_templates)