Syncer now syncs stale audit templates

In this changeset, I introduce the syncing of audit templates.

Partially Implements: blueprint get-goal-from-strategy

Change-Id: Ie394c12fe51f73eff95465fd5140d82ebd212599

watcher/decision_engine/sync.py

@@ -14,22 +14,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
+
 from oslo_log import log
 
-from watcher._i18n import _LI
+from watcher._i18n import _LE, _LI
 from watcher.common import context
 from watcher.decision_engine.strategy.loading import default
 from watcher import objects
 
 LOG = log.getLogger(__name__)
 
+GoalMapping = collections.namedtuple('GoalMapping', ['name', 'display_name'])
+StrategyMapping = collections.namedtuple(
+    'StrategyMapping', ['name', 'goal_name', 'display_name'])
+
 
 class Syncer(object):
     """Syncs all available goals and strategies with the Watcher DB"""
 
     def __init__(self):
         self.ctx = context.make_context()
-        self._discovered_map = None
+        self.discovered_map = None
 
         self._available_goals = None
         self._available_goals_map = None
@@ -42,6 +48,8 @@ class Syncer(object):
         # This strategy mapping maps stale strategy IDs to the synced goal
         self.strategy_mapping = dict()
 
+        self.stale_audit_templates_map = {}
+
     @property
     def available_goals(self):
         if self._available_goals is None:
@@ -58,7 +66,8 @@ class Syncer(object):
     def available_goals_map(self):
         if self._available_goals_map is None:
             self._available_goals_map = {
-                g: {"name": g.name, "display_name": g.display_name}
+                GoalMapping(
+                    name=g.name, display_name=g.display_name): g
                 for g in self.available_goals
             }
         return self._available_goals_map
@@ -68,41 +77,45 @@ class Syncer(object):
         if self._available_strategies_map is None:
             goals_map = {g.id: g.name for g in self.available_goals}
             self._available_strategies_map = {
-                s: {"name": s.name, "goal_name": goals_map[s.goal_id],
+                StrategyMapping(
-                    "display_name": s.display_name}
+                    name=s.name, goal_name=goals_map[s.goal_id],
+                    display_name=s.display_name): s
                 for s in self.available_strategies
             }
         return self._available_strategies_map
 
     def sync(self):
-        discovered_map = self.discover()
+        self.discovered_map = self._discover()
-        goals_map = discovered_map["goals"]
+        goals_map = self.discovered_map["goals"]
-        strategies_map = discovered_map["strategies"]
+        strategies_map = self.discovered_map["strategies"]
 
         for goal_name, goal_map in goals_map.items():
-            if goal_map in self.available_goals_map.values():
+            if goal_map in self.available_goals_map:
                 LOG.info(_LI("Goal %s already exists"), goal_name)
                 continue
 
             self.goal_mapping.update(self._sync_goal(goal_map))
 
         for strategy_name, strategy_map in strategies_map.items():
-            if strategy_map in self.available_strategies_map.values():
+            if (strategy_map in self.available_strategies_map and
+                    strategy_map.goal_name not in
+                    [g.name for g in self.goal_mapping.values()]):
                 LOG.info(_LI("Strategy %s already exists"), strategy_name)
                 continue
 
             self.strategy_mapping.update(self._sync_strategy(strategy_map))
 
-        # TODO(v-francoise): Sync the audit templates
+        self._sync_audit_templates()
 
     def _sync_goal(self, goal_map):
-        goal_name = goal_map['name']
+        goal_name = goal_map.name
-        goal_display_name = goal_map['display_name']
+        goal_display_name = goal_map.display_name
         goal_mapping = dict()
 
+        # Goals that are matching by name with the given discovered goal name
         matching_goals = [g for g in self.available_goals
                           if g.name == goal_name]
-        stale_goals = self.soft_delete_stale_goals(goal_map, matching_goals)
+        stale_goals = self._soft_delete_stale_goals(goal_map, matching_goals)
 
         if stale_goals or not matching_goals:
             goal = objects.Goal(self.ctx)
@@ -120,14 +133,16 @@ class Syncer(object):
         return goal_mapping
 
     def _sync_strategy(self, strategy_map):
-        strategy_name = strategy_map['name']
+        strategy_name = strategy_map.name
-        strategy_display_name = strategy_map['display_name']
+        strategy_display_name = strategy_map.display_name
-        goal_name = strategy_map['goal_name']
+        goal_name = strategy_map.goal_name
         strategy_mapping = dict()
 
+        # Strategies that are matching by name with the given
+        # discovered strategy name
         matching_strategies = [s for s in self.available_strategies
                                if s.name == strategy_name]
-        stale_strategies = self.soft_delete_stale_strategies(
+        stale_strategies = self._soft_delete_stale_strategies(
             strategy_map, matching_strategies)
 
         if stale_strategies or not matching_strategies:
@@ -146,7 +161,95 @@ class Syncer(object):
 
         return strategy_mapping
 
-    def discover(self):
+    def _sync_audit_templates(self):
+        # First we find audit templates that are stale because their associated
+        # goal or strategy has been modified and we update them in-memory
+        self._find_stale_audit_templates_due_to_goal()
+        self._find_stale_audit_templates_due_to_strategy()
+
+        # Then we handle the case where an audit template became
+        # stale because its related goal does not exist anymore.
+        self._soft_delete_removed_goals()
+        # Then we handle the case where an audit template became
+        # stale because its related strategy does not exist anymore.
+        self._soft_delete_removed_strategies()
+
+        # Finally, we save into the DB the updated stale audit templates
+        for stale_audit_template in self.stale_audit_templates_map.values():
+            stale_audit_template.save()
+            LOG.info(_LI("Audit Template '%s' synced"),
+                     stale_audit_template.name)
+
+    def _find_stale_audit_templates_due_to_goal(self):
+        for goal_id, synced_goal in self.goal_mapping.items():
+            filters = {"goal_id": goal_id}
+            stale_audit_templates = objects.AuditTemplate.list(
+                self.ctx, filters=filters)
+
+            # Update the goal ID for the stale audit templates (w/o saving)
+            for audit_template in stale_audit_templates:
+                if audit_template.id not in self.stale_audit_templates_map:
+                    audit_template.goal_id = synced_goal.id
+                    self.stale_audit_templates_map[audit_template.id] = (
+                        audit_template)
+                else:
+                    self.stale_audit_templates_map[
+                        audit_template.id].goal_id = synced_goal.id
+
+    def _find_stale_audit_templates_due_to_strategy(self):
+        for strategy_id, synced_strategy in self.strategy_mapping.items():
+            filters = {"strategy_id": strategy_id}
+            stale_audit_templates = objects.AuditTemplate.list(
+                self.ctx, filters=filters)
+
+            # Update strategy IDs for all stale audit templates (w/o saving)
+            for audit_template in stale_audit_templates:
+                if audit_template.id not in self.stale_audit_templates_map:
+                    audit_template.strategy_id = synced_strategy.id
+                    self.stale_audit_templates_map[audit_template.id] = (
+                        audit_template)
+                else:
+                    self.stale_audit_templates_map[
+                        audit_template.id].strategy_id = synced_strategy.id
+
+    def _soft_delete_removed_goals(self):
+        removed_goals = [
+            g for g in self.available_goals
+            if g.name not in self.discovered_map['goals']]
+        for removed_goal in removed_goals:
+            removed_goal.soft_delete()
+            filters = {"goal_id": removed_goal.id}
+            invalid_ats = objects.AuditTemplate.list(self.ctx, filters=filters)
+            for at in invalid_ats:
+                LOG.warning(
+                    _LE("Audit Template '%(audit_template)s' references a "
+                        "goal that does not exist"),
+                    audit_template=at.uuid)
+
+    def _soft_delete_removed_strategies(self):
+        removed_strategies = [
+            s for s in self.available_strategies
+            if s.name not in self.discovered_map['strategies']]
+
+        for removed_strategy in removed_strategies:
+            removed_strategy.soft_delete()
+            filters = {"strategy_id": removed_strategy.id}
+            invalid_ats = objects.AuditTemplate.list(self.ctx, filters=filters)
+            for at in invalid_ats:
+                LOG.info(
+                    _LI("Audit Template '%(audit_template)s' references a "
+                        "strategy that does not exist"),
+                    audit_template=at.uuid)
+                # In this case we can reset the strategy ID to None
+                # so the audit template can still achieve the same goal
+                # but with a different strategy
+                if at.id not in self.stale_audit_templates_map:
+                    at.strategy_id = None
+                    self.stale_audit_templates_map[at.id] = at
+                else:
+                    self.stale_audit_templates_map[at.id].strategy_id = None
+
+    def _discover(self):
         strategies_map = {}
         goals_map = {}
         discovered_map = {"goals": goals_map, "strategies": strategies_map}
@@ -154,25 +257,25 @@ class Syncer(object):
         implemented_strategies = strategy_loader.list_available()
 
         for _, strategy_cls in implemented_strategies.items():
-            goals_map[strategy_cls.get_goal_name()] = {
+            goals_map[strategy_cls.get_goal_name()] = GoalMapping(
-                "name": strategy_cls.get_goal_name(),
+                name=strategy_cls.get_goal_name(),
-                "display_name":
+                display_name=strategy_cls.get_translatable_goal_display_name())
-                    strategy_cls.get_translatable_goal_display_name()}
 
-            strategies_map[strategy_cls.get_name()] = {
+            strategies_map[strategy_cls.get_name()] = StrategyMapping(
-                "name": strategy_cls.get_name(),
+                name=strategy_cls.get_name(),
-                "goal_name": strategy_cls.get_goal_name(),
+                goal_name=strategy_cls.get_goal_name(),
-                "display_name": strategy_cls.get_translatable_display_name()}
+                display_name=strategy_cls.get_translatable_display_name())
 
         return discovered_map
 
-    def soft_delete_stale_goals(self, goal_map, matching_goals):
+    def _soft_delete_stale_goals(self, goal_map, matching_goals):
-        goal_name = goal_map['name']
+        goal_name = goal_map.name
-        goal_display_name = goal_map['display_name']
+        goal_display_name = goal_map.display_name
 
         stale_goals = []
         for matching_goal in matching_goals:
-            if matching_goal.display_name == goal_display_name:
+            if (matching_goal.display_name == goal_display_name and
+                    matching_goal.strategy_id not in self.strategy_mapping):
                 LOG.info(_LI("Goal %s unchanged"), goal_name)
             else:
                 LOG.info(_LI("Goal %s modified"), goal_name)
@@ -181,13 +284,14 @@ class Syncer(object):
 
         return stale_goals
 
-    def soft_delete_stale_strategies(self, strategy_map, matching_strategies):
+    def _soft_delete_stale_strategies(self, strategy_map, matching_strategies):
-        strategy_name = strategy_map['name']
+        strategy_name = strategy_map.name
-        strategy_display_name = strategy_map['display_name']
+        strategy_display_name = strategy_map.display_name
 
         stale_strategies = []
         for matching_strategy in matching_strategies:
-            if matching_strategy.display_name == strategy_display_name:
+            if (matching_strategy.display_name == strategy_display_name and
+                    matching_strategy.goal_id not in self.goal_mapping):
                 LOG.info(_LI("Strategy %s unchanged"), strategy_name)
             else:
                 LOG.info(_LI("Strategy %s modified"), strategy_name)

watcher/tests/decision_engine/test_sync.py

@@ -31,6 +31,7 @@ class TestSyncer(base.DbTestCase):
         super(TestSyncer, self).setUp()
         self.ctx = context.make_context()
 
+        # This mock simulates the strategies discovery done in discover()
         self.m_available_strategies = mock.Mock(return_value={
             fake_strategies.FakeDummy1Strategy1.get_name():
                 fake_strategies.FakeDummy1Strategy1,
@@ -201,54 +202,254 @@ class TestSyncer(base.DbTestCase):
         self.assertEqual(1, m_s_soft_delete.call_count)
 
     def test_end2end_sync_goals_with_modified_goal_and_strategy(self):
-        goal = objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
+        # ### Setup ### #
-                            name="DUMMY_1", display_name="Original")
-        goal.create()
-        strategy = objects.Strategy(
-            self.ctx, id=1, name="STRATEGY_1",
-            display_name="Original", goal_id=goal.id)
-        strategy.create()
-        # audit_template = objects.AuditTemplate(
-        #     self.ctx, id=1, name="Synced AT", goal_id=goal.id,
-        #     strategy_id=strategy.id)
-        # audit_template.create()
 
-        # before_audit_templates = objects.AuditTemplate.list(self.ctx)
+        # Here, we simulate goals and strategies already discovered in the past
+        # that were saved in DB
+
+        # Should stay unmodified after sync()
+        goal1 = objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
+                             name="DUMMY_1", display_name="Dummy 1")
+        # Should be modified by the sync()
+        goal2 = objects.Goal(self.ctx, id=2, uuid=utils.generate_uuid(),
+                             name="DUMMY_2", display_name="Original")
+        goal1.create()
+        goal2.create()
+
+        # Should stay unmodified after sync()
+        strategy1 = objects.Strategy(
+            self.ctx, id=1, name="STRATEGY_1", uuid=utils.generate_uuid(),
+            display_name="Strategy 1", goal_id=goal1.id)
+        # Should stay unmodified after sync()
+        strategy2 = objects.Strategy(
+            self.ctx, id=2, name="STRATEGY_2", uuid=utils.generate_uuid(),
+            display_name="Strategy 2", goal_id=goal2.id)
+        # Should be modified by the sync()
+        strategy3 = objects.Strategy(
+            self.ctx, id=3, name="STRATEGY_3", uuid=utils.generate_uuid(),
+            display_name="Original", goal_id=goal2.id)
+        # Should be modified by the sync()
+        strategy4 = objects.Strategy(
+            self.ctx, id=4, name="STRATEGY_4", uuid=utils.generate_uuid(),
+            display_name="Original", goal_id=goal2.id)
+        strategy1.create()
+        strategy2.create()
+        strategy3.create()
+        strategy4.create()
+
+        # Here we simulate audit_templates that were already created in the
+        # past and hence saved within the Watcher DB
+
+        # Should stay unmodified after sync()
+        audit_template1 = objects.AuditTemplate(
+            self.ctx, id=1, uuid=utils.generate_uuid(),
+            name="Synced AT1", goal_id=goal1.id, strategy_id=strategy1.id)
+        # Should be modified by the sync() because its associated goal
+        # should be modified
+        audit_template2 = objects.AuditTemplate(
+            self.ctx, id=2, name="Synced AT2", uuid=utils.generate_uuid(),
+            goal_id=goal2.id, strategy_id=strategy2.id)
+        # Should be modified by the sync() because its associated strategy
+        # should be modified
+        audit_template3 = objects.AuditTemplate(
+            self.ctx, id=3, name="Synced AT3", uuid=utils.generate_uuid(),
+            goal_id=goal2.id, strategy_id=strategy3.id)
+        # Modified because of both because its associated goal and associated
+        # strategy should be modified
+        audit_template4 = objects.AuditTemplate(
+            self.ctx, id=4, name="Synced AT4", uuid=utils.generate_uuid(),
+            goal_id=goal2.id, strategy_id=strategy4.id)
+        audit_template1.create()
+        audit_template2.create()
+        audit_template3.create()
+        audit_template4.create()
+
+        before_audit_templates = objects.AuditTemplate.list(self.ctx)
         before_goals = objects.Goal.list(self.ctx)
         before_strategies = objects.Strategy.list(self.ctx)
 
+        # ### Action under test ### #
+
         try:
             self.syncer.sync()
         except Exception as exc:
             self.fail(exc)
 
-        # after_audit_templates = objects.AuditTemplate.list(self.ctx)
+        # ### Assertions ### #
+
+        after_audit_templates = objects.AuditTemplate.list(self.ctx)
         after_goals = objects.Goal.list(self.ctx)
         after_strategies = objects.Strategy.list(self.ctx)
 
-        self.assertEqual(1, len(before_goals))
+        self.assertEqual(2, len(before_goals))
-        self.assertEqual(1, len(before_strategies))
+        self.assertEqual(4, len(before_strategies))
-        # self.assertEqual(1, len(before_audit_templates))
+        self.assertEqual(4, len(before_audit_templates))
         self.assertEqual(2, len(after_goals))
         self.assertEqual(4, len(after_strategies))
-        # self.assertEqual(1, len(after_audit_templates))
+        self.assertEqual(4, len(after_audit_templates))
         self.assertEqual(
             {"DUMMY_1", "DUMMY_2"},
             set([g.name for g in after_goals]))
         self.assertEqual(
             {"STRATEGY_1", "STRATEGY_2", "STRATEGY_3", "STRATEGY_4"},
             set([s.name for s in after_strategies]))
+        created_goals = {
+            ag.name: ag for ag in after_goals
+            if ag.uuid not in [bg.uuid for bg in before_goals]
+        }
+        created_strategies = {
+            a_s.name: a_s for a_s in after_strategies
+            if a_s.uuid not in [b_s.uuid for b_s in before_strategies]
+        }
+
+        self.assertEqual(1, len(created_goals))
+        self.assertEqual(3, len(created_strategies))
+
+        modified_audit_templates = {
+            a_at.id for a_at in after_audit_templates
+            if a_at.goal_id not in (
+                # initial goal IDs
+                b_at.goal_id for b_at in before_audit_templates) or
+            a_at.strategy_id not in (
+                # initial strategy IDs
+                b_at.strategy_id for b_at in before_audit_templates
+                if b_at.strategy_id is not None)
+        }
+
+        unmodified_audit_templates = {
+            a_at.id for a_at in after_audit_templates
+            if a_at.goal_id in (
+                # initial goal IDs
+                b_at.goal_id for b_at in before_audit_templates) and
+            a_at.strategy_id in (
+                # initial strategy IDs
+                b_at.strategy_id for b_at in before_audit_templates
+                if b_at.strategy_id is not None)
+        }
+
+        self.assertEqual(2, strategy2.goal_id)
+        self.assertIn(strategy2.name, created_strategies)
+        self.assertTrue(strategy2.id != created_strategies[strategy2.name].id)
+
+        self.assertEqual(set([audit_template2.id,
+                              audit_template3.id,
+                              audit_template4.id]),
+                         modified_audit_templates)
+        self.assertEqual(set([audit_template1.id]),
+                         unmodified_audit_templates)
+
+    def test_end2end_sync_goals_with_removed_goal_and_strategy(self):
+        # ### Setup ### #
+
+        # We Simulate the fact that we removed 2 strategies
+        # as well as the DUMMY_2 goal
+        self.m_available_strategies.return_value = {
+            fake_strategies.FakeDummy1Strategy1.get_name():
+                fake_strategies.FakeDummy1Strategy1
+        }
+
+        # Should stay unmodified after sync()
+        goal1 = objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
+                             name="DUMMY_1", display_name="Dummy 1")
+        # To be removed by the sync()
+        goal2 = objects.Goal(self.ctx, id=2, uuid=utils.generate_uuid(),
+                             name="DUMMY_2", display_name="Dummy 2")
+        goal1.create()
+        goal2.create()
+
+        # Should stay unmodified after sync()
+        strategy1 = objects.Strategy(
+            self.ctx, id=1, name="STRATEGY_1", uuid=utils.generate_uuid(),
+            display_name="Strategy 1", goal_id=goal1.id)
+        # To be removed by the sync()
+        strategy2 = objects.Strategy(
+            self.ctx, id=2, name="STRATEGY_2", uuid=utils.generate_uuid(),
+            display_name="Strategy 2", goal_id=goal1.id)
+        # To be removed by the sync()
+        strategy3 = objects.Strategy(
+            self.ctx, id=3, name="STRATEGY_3", uuid=utils.generate_uuid(),
+            display_name="Original", goal_id=goal2.id)
+        strategy1.create()
+        strategy2.create()
+        strategy3.create()
+
+        # Here we simulate audit_templates that were already created in the
+        # past and hence saved within the Watcher DB
+
+        # The strategy of this audit template will be dereferenced
+        # as it does not exist anymore
+        audit_template1 = objects.AuditTemplate(
+            self.ctx, id=1, uuid=utils.generate_uuid(),
+            name="Synced AT1", goal_id=goal1.id, strategy_id=strategy1.id)
+        # Stale even after syncing because the goal has been soft deleted
+        audit_template2 = objects.AuditTemplate(
+            self.ctx, id=2, name="Synced AT2", uuid=utils.generate_uuid(),
+            goal_id=goal2.id, strategy_id=strategy2.id)
+
+        audit_template1.create()
+        audit_template2.create()
+
+        before_audit_templates = objects.AuditTemplate.list(self.ctx)
+        before_goals = objects.Goal.list(self.ctx)
+        before_strategies = objects.Strategy.list(self.ctx)
+
+        # ### Action under test ### #
+
+        try:
+            self.syncer.sync()
+        except Exception as exc:
+            self.fail(exc)
+
+        # ### Assertions ### #
+
+        after_audit_templates = objects.AuditTemplate.list(self.ctx)
+        after_goals = objects.Goal.list(self.ctx)
+        after_strategies = objects.Strategy.list(self.ctx)
+
+        self.assertEqual(2, len(before_goals))
+        self.assertEqual(3, len(before_strategies))
+        self.assertEqual(2, len(before_audit_templates))
+        self.assertEqual(1, len(after_goals))
+        self.assertEqual(1, len(after_strategies))
+        self.assertEqual(2, len(after_audit_templates))
+        self.assertEqual(
+            {"DUMMY_1"},
+            set([g.name for g in after_goals]))
+        self.assertEqual(
+            {"STRATEGY_1"},
+            set([s.name for s in after_strategies]))
         created_goals = [ag for ag in after_goals
                          if ag.uuid not in [bg.uuid for bg in before_goals]]
         created_strategies = [
             a_s for a_s in after_strategies
             if a_s.uuid not in [b_s.uuid for b_s in before_strategies]]
 
-        self.assertEqual(2, len(created_goals))
+        self.assertEqual(0, len(created_goals))
-        self.assertEqual(4, len(created_strategies))
+        self.assertEqual(0, len(created_strategies))
 
-        # synced_audit_template = after_audit_templates[0]
+        modified_audit_templates = {
-        # self.assertTrue(
+            a_at.id for a_at in after_audit_templates
-        #     audit_template.goal_id != synced_audit_template.goal_id)
+            if a_at.goal_id not in (
-        # self.assertIn(synced_audit_template.goal_id,
+                # initial goal IDs
-        #               (g.id for g in after_goals))
+                b_at.goal_id for b_at in before_audit_templates) or
+            a_at.strategy_id not in (
+                # initial strategy IDs
+                b_at.strategy_id for b_at in before_audit_templates
+                if b_at.strategy_id is not None)
+        }
+
+        unmodified_audit_templates = {
+            a_at.id for a_at in after_audit_templates
+            if a_at.goal_id in (
+                # initial goal IDs
+                b_at.goal_id for b_at in before_audit_templates) and
+            a_at.strategy_id in (
+                # initial strategy IDs
+                b_at.strategy_id for b_at in before_audit_templates
+                if b_at.strategy_id is not None)
+        }
+
+        self.assertEqual(set([audit_template2.id]),
+                         modified_audit_templates)
+        self.assertEqual(set([audit_template1.id]),
+                         unmodified_audit_templates)