From a3ac26870a60ea69f90017592823677de5965ff7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Fran=C3=A7oise?= Date: Fri, 29 Apr 2016 17:46:49 +0200 Subject: [PATCH] DB sync for Strategies In this changeset, I added the ability to synchronize the strategies into the Wather DB so that it can later be served through the Watcher API. Partially Implements: blueprint get-goal-from-strategy Change-Id: Ifeaa1f6e1f4ff7d7efc1b221cf57797a49dc5bc5 --- watcher/decision_engine/sync.py | 106 ++++++++-- watcher/tests/decision_engine/test_sync.py | 232 ++++++++++++++++----- 2 files changed, 268 insertions(+), 70 deletions(-) diff --git a/watcher/decision_engine/sync.py b/watcher/decision_engine/sync.py index 45435c453..f4a6163d9 100644 --- a/watcher/decision_engine/sync.py +++ b/watcher/decision_engine/sync.py @@ -32,9 +32,16 @@ class Syncer(object): self._discovered_map = None self._available_goals = None - self._available_goal_names = None self._available_goals_map = None + self._available_strategies = None + self._available_strategies_map = None + + # This goal mapping maps stale goal IDs to the synced goal + self.goal_mapping = dict() + # This strategy mapping maps stale strategy IDs to the synced goal + self.strategy_mapping = dict() + @property def available_goals(self): if self._available_goals is None: @@ -42,10 +49,10 @@ class Syncer(object): return self._available_goals @property - def available_goal_names(self): - if self._available_goal_names is None: - self._available_goal_names = [g.name for g in self.available_goals] - return self._available_goal_names + def available_strategies(self): + if self._available_strategies is None: + self._available_strategies = objects.Strategy.list(self.ctx) + return self._available_strategies @property def available_goals_map(self): @@ -56,20 +63,42 @@ class Syncer(object): } return self._available_goals_map + @property + def available_strategies_map(self): + if self._available_strategies_map is None: + goals_map = {g.id: g.name for g in self.available_goals} + self._available_strategies_map = { + s: {"name": s.name, "goal_name": goals_map[s.goal_id], + "display_name": s.display_name} + for s in self.available_strategies + } + return self._available_strategies_map + def sync(self): discovered_map = self.discover() goals_map = discovered_map["goals"] + strategies_map = discovered_map["strategies"] for goal_name, goal_map in goals_map.items(): if goal_map in self.available_goals_map.values(): LOG.info(_LI("Goal %s already exists"), goal_name) continue - self._sync_goal(goal_map) + self.goal_mapping.update(self._sync_goal(goal_map)) + + for strategy_name, strategy_map in strategies_map.items(): + if strategy_map in self.available_strategies_map.values(): + LOG.info(_LI("Strategy %s already exists"), strategy_name) + continue + + self.strategy_mapping.update(self._sync_strategy(strategy_map)) + + # TODO(v-francoise): Sync the audit templates def _sync_goal(self, goal_map): goal_name = goal_map['name'] goal_display_name = goal_map['display_name'] + goal_mapping = dict() matching_goals = [g for g in self.available_goals if g.name == goal_name] @@ -81,22 +110,41 @@ class Syncer(object): goal.display_name = goal_display_name goal.create() LOG.info(_LI("Goal %s created"), goal_name) - self.available_goal_names.append(goal_name) + + # Updating the internal states self.available_goals_map[goal] = goal_map - # We have to update the audit templates that were pointing - # self._sync_audit_templates(stale_goals, goal) + # Map the old goal IDs to the new (equivalent) goal + for matching_goal in matching_goals: + goal_mapping[matching_goal.id] = goal - # def _sync_audit_templates(self, stale_goals, goal): - # related_audit_templates = [] - # for stale_goal in stale_goals: - # filters = {"goal_id": stale_goal.id} - # related_audit_templates.extend( - # objects.AuditTemplate.list(self.ctx, filters=filters)) + return goal_mapping - # for audit_template in related_audit_templates: - # LOG.info(_LI("Audit Template '%s' updated with synced goal")) - # audit_template.goal_id = goal.id - # audit_template.save() + def _sync_strategy(self, strategy_map): + strategy_name = strategy_map['name'] + strategy_display_name = strategy_map['display_name'] + goal_name = strategy_map['goal_name'] + strategy_mapping = dict() + + matching_strategies = [s for s in self.available_strategies + if s.name == strategy_name] + stale_strategies = self.soft_delete_stale_strategies( + strategy_map, matching_strategies) + + if stale_strategies or not matching_strategies: + strategy = objects.Strategy(self.ctx) + strategy.name = strategy_name + strategy.display_name = strategy_display_name + strategy.goal_id = objects.Goal.get_by_name(self.ctx, goal_name).id + strategy.create() + LOG.info(_LI("Strategy %s created"), strategy_name) + + # Updating the internal states + self.available_strategies_map[strategy] = strategy_map + # Map the old strategy IDs to the new (equivalent) strategy + for matching_strategy in matching_strategies: + strategy_mapping[matching_strategy.id] = strategy + + return strategy_mapping def discover(self): strategies_map = {} @@ -116,6 +164,11 @@ class Syncer(object): "name": strategy_cls.DEFAULT_NAME, "display_name": strategy_cls.DEFAULT_DESCRIPTION} + strategies_map[strategy_cls.__name__] = { + "name": strategy_cls.__name__, + "goal_name": strategy_cls.DEFAULT_NAME, + "display_name": strategy_cls.DEFAULT_DESCRIPTION} + return discovered_map def soft_delete_stale_goals(self, goal_map, matching_goals): @@ -132,3 +185,18 @@ class Syncer(object): stale_goals.append(matching_goal) return stale_goals + + def soft_delete_stale_strategies(self, strategy_map, matching_strategies): + strategy_name = strategy_map['name'] + strategy_display_name = strategy_map['display_name'] + + stale_strategies = [] + for matching_strategy in matching_strategies: + if matching_strategy.display_name == strategy_display_name: + LOG.info(_LI("Strategy %s unchanged"), strategy_name) + else: + LOG.info(_LI("Strategy %s modified"), strategy_name) + matching_strategy.soft_delete() + stale_strategies.append(matching_strategy) + + return stale_strategies diff --git a/watcher/tests/decision_engine/test_sync.py b/watcher/tests/decision_engine/test_sync.py index 3f3799635..167141d9e 100644 --- a/watcher/tests/decision_engine/test_sync.py +++ b/watcher/tests/decision_engine/test_sync.py @@ -21,7 +21,7 @@ from watcher.common import utils from watcher.decision_engine.strategy.loading import default from watcher.decision_engine.strategy.strategies import base as base_strategy from watcher.decision_engine import sync -from watcher.objects import goal +from watcher import objects from watcher.tests.db import base @@ -36,25 +36,21 @@ class FakeStrategy(base_strategy.BaseStrategy): class FakeDummy1Strategy1(FakeStrategy): DEFAULT_NAME = "DUMMY_1" DEFAULT_DESCRIPTION = "Dummy 1" - ID = "STRATEGY_1" class FakeDummy1Strategy2(FakeStrategy): DEFAULT_NAME = "DUMMY_1" DEFAULT_DESCRIPTION = "Dummy 1" - ID = "STRATEGY_2" class FakeDummy2Strategy3(FakeStrategy): DEFAULT_NAME = "DUMMY_2" DEFAULT_DESCRIPTION = "Dummy 2" - ID = "STRATEGY_3" class FakeDummy2Strategy4(FakeStrategy): DEFAULT_NAME = "DUMMY_2" DEFAULT_DESCRIPTION = "Other Dummy 2" - ID = "STRATEGY_4" class TestSyncer(base.DbTestCase): @@ -64,10 +60,10 @@ class TestSyncer(base.DbTestCase): self.ctx = context.make_context() self.m_available_strategies = mock.Mock(return_value={ - FakeDummy1Strategy1.DEFAULT_NAME: FakeDummy1Strategy1, - FakeDummy1Strategy2.DEFAULT_NAME: FakeDummy1Strategy2, - FakeDummy2Strategy3.DEFAULT_NAME: FakeDummy2Strategy3, - FakeDummy2Strategy4.DEFAULT_NAME: FakeDummy2Strategy4, + FakeDummy1Strategy1.__name__: FakeDummy1Strategy1, + FakeDummy1Strategy2.__name__: FakeDummy1Strategy2, + FakeDummy2Strategy3.__name__: FakeDummy2Strategy3, + FakeDummy2Strategy4.__name__: FakeDummy2Strategy4, }) p_strategies = mock.patch.object( @@ -78,72 +74,206 @@ class TestSyncer(base.DbTestCase): self.syncer = sync.Syncer() self.addCleanup(p_strategies.stop) - @mock.patch.object(goal.Goal, "soft_delete") - @mock.patch.object(goal.Goal, "save") - @mock.patch.object(goal.Goal, "create") - @mock.patch.object(goal.Goal, "list") - def test_sync_goals_empty_db(self, m_list, m_create, - m_save, m_soft_delete): - m_list.return_value = [] + @mock.patch.object(objects.Strategy, "soft_delete") + @mock.patch.object(objects.Strategy, "save") + @mock.patch.object(objects.Strategy, "create") + @mock.patch.object(objects.Strategy, "list") + @mock.patch.object(objects.Goal, "get_by_name") + @mock.patch.object(objects.Goal, "soft_delete") + @mock.patch.object(objects.Goal, "save") + @mock.patch.object(objects.Goal, "create") + @mock.patch.object(objects.Goal, "list") + def test_sync_empty_db( + self, m_g_list, m_g_create, m_g_save, m_g_soft_delete, + m_g_get_by_name, m_s_list, m_s_create, m_s_save, m_s_soft_delete): + m_g_get_by_name.side_effect = [ + objects.Goal(self.ctx, id=i) for i in range(1, 10)] + m_g_list.return_value = [] + m_s_list.return_value = [] self.syncer.sync() - self.assertEqual(2, m_create.call_count) - self.assertEqual(0, m_save.call_count) - self.assertEqual(0, m_soft_delete.call_count) + self.assertEqual(2, m_g_create.call_count) + self.assertEqual(0, m_g_save.call_count) + self.assertEqual(0, m_g_soft_delete.call_count) - @mock.patch.object(goal.Goal, "soft_delete") - @mock.patch.object(goal.Goal, "save") - @mock.patch.object(goal.Goal, "create") - @mock.patch.object(goal.Goal, "list") - def test_sync_goals_with_existing_goal(self, m_list, m_create, - m_save, m_soft_delete): - m_list.return_value = [ - goal.Goal(self.ctx, id=1, uuid=utils.generate_uuid(), - name="DUMMY_1", display_name="Dummy 1") + self.assertEqual(4, m_s_create.call_count) + self.assertEqual(0, m_s_save.call_count) + self.assertEqual(0, m_s_soft_delete.call_count) + + @mock.patch.object(objects.Strategy, "soft_delete") + @mock.patch.object(objects.Strategy, "save") + @mock.patch.object(objects.Strategy, "create") + @mock.patch.object(objects.Strategy, "list") + @mock.patch.object(objects.Goal, "get_by_name") + @mock.patch.object(objects.Goal, "soft_delete") + @mock.patch.object(objects.Goal, "save") + @mock.patch.object(objects.Goal, "create") + @mock.patch.object(objects.Goal, "list") + def test_sync_with_existing_goal( + self, m_g_list, m_g_create, m_g_save, m_g_soft_delete, + m_g_get_by_name, m_s_list, m_s_create, m_s_save, m_s_soft_delete): + m_g_get_by_name.side_effect = [ + objects.Goal(self.ctx, id=i) for i in range(1, 10)] + m_g_list.return_value = [ + objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(), + name="DUMMY_1", display_name="Dummy 1") + ] + m_s_list.return_value = [] + + self.syncer.sync() + + self.assertEqual(1, m_g_create.call_count) + self.assertEqual(0, m_g_save.call_count) + self.assertEqual(0, m_g_soft_delete.call_count) + + self.assertEqual(4, m_s_create.call_count) + self.assertEqual(0, m_s_save.call_count) + self.assertEqual(0, m_s_soft_delete.call_count) + + @mock.patch.object(objects.Strategy, "soft_delete") + @mock.patch.object(objects.Strategy, "save") + @mock.patch.object(objects.Strategy, "create") + @mock.patch.object(objects.Strategy, "list") + @mock.patch.object(objects.Goal, "get_by_name") + @mock.patch.object(objects.Goal, "soft_delete") + @mock.patch.object(objects.Goal, "save") + @mock.patch.object(objects.Goal, "create") + @mock.patch.object(objects.Goal, "list") + def test_sync_with_existing_strategy( + self, m_g_list, m_g_create, m_g_save, m_g_soft_delete, + m_g_get_by_name, m_s_list, m_s_create, m_s_save, m_s_soft_delete): + m_g_get_by_name.side_effect = [ + objects.Goal(self.ctx, id=i) for i in range(1, 10)] + m_g_list.return_value = [ + objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(), + name="DUMMY_1", display_name="Dummy 1") + ] + m_s_list.return_value = [ + objects.Strategy(self.ctx, id=1, name="FakeDummy1Strategy1", + goal_id=1, display_name="Dummy 1") ] self.syncer.sync() - self.assertEqual(1, m_create.call_count) - self.assertEqual(0, m_save.call_count) - self.assertEqual(0, m_soft_delete.call_count) + self.assertEqual(1, m_g_create.call_count) + self.assertEqual(0, m_g_save.call_count) + self.assertEqual(0, m_g_soft_delete.call_count) - @mock.patch.object(goal.Goal, "soft_delete") - @mock.patch.object(goal.Goal, "save") - @mock.patch.object(goal.Goal, "create") - @mock.patch.object(goal.Goal, "list") - def test_sync_goals_with_modified_goal(self, m_list, m_create, - m_save, m_soft_delete): - m_list.return_value = [ - goal.Goal(self.ctx, id=1, uuid=utils.generate_uuid(), - name="DUMMY_2", display_name="original") + self.assertEqual(3, m_s_create.call_count) + self.assertEqual(0, m_s_save.call_count) + self.assertEqual(0, m_s_soft_delete.call_count) + + @mock.patch.object(objects.Strategy, "soft_delete") + @mock.patch.object(objects.Strategy, "save") + @mock.patch.object(objects.Strategy, "create") + @mock.patch.object(objects.Strategy, "list") + @mock.patch.object(objects.Goal, "get_by_name") + @mock.patch.object(objects.Goal, "soft_delete") + @mock.patch.object(objects.Goal, "save") + @mock.patch.object(objects.Goal, "create") + @mock.patch.object(objects.Goal, "list") + def test_sync_with_modified_goal( + self, m_g_list, m_g_create, m_g_save, m_g_soft_delete, + m_g_get_by_name, m_s_list, m_s_create, m_s_save, m_s_soft_delete): + m_g_get_by_name.side_effect = [ + objects.Goal(self.ctx, id=i) for i in range(1, 10)] + m_g_list.return_value = [ + objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(), + name="DUMMY_2", display_name="original") + ] + m_s_list.return_value = [] + self.syncer.sync() + + self.assertEqual(2, m_g_create.call_count) + self.assertEqual(0, m_g_save.call_count) + self.assertEqual(1, m_g_soft_delete.call_count) + + self.assertEqual(4, m_s_create.call_count) + self.assertEqual(0, m_s_save.call_count) + self.assertEqual(0, m_s_soft_delete.call_count) + + @mock.patch.object(objects.Strategy, "soft_delete") + @mock.patch.object(objects.Strategy, "save") + @mock.patch.object(objects.Strategy, "create") + @mock.patch.object(objects.Strategy, "list") + @mock.patch.object(objects.Goal, "get_by_name") + @mock.patch.object(objects.Goal, "soft_delete") + @mock.patch.object(objects.Goal, "save") + @mock.patch.object(objects.Goal, "create") + @mock.patch.object(objects.Goal, "list") + def test_sync_with_modified_strategy( + self, m_g_list, m_g_create, m_g_save, m_g_soft_delete, + m_g_get_by_name, m_s_list, m_s_create, m_s_save, m_s_soft_delete): + m_g_get_by_name.side_effect = [ + objects.Goal(self.ctx, id=i) for i in range(1, 10)] + m_g_list.return_value = [ + objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(), + name="DUMMY_1", display_name="Dummy 1") + ] + m_s_list.return_value = [ + objects.Strategy(self.ctx, id=1, name="FakeDummy1Strategy1", + goal_id=1, display_name="original") ] self.syncer.sync() - self.assertEqual(2, m_create.call_count) - self.assertEqual(0, m_save.call_count) - self.assertEqual(1, m_soft_delete.call_count) + self.assertEqual(1, m_g_create.call_count) + self.assertEqual(0, m_g_save.call_count) + self.assertEqual(0, m_g_soft_delete.call_count) - def test_end2end_sync_goals_with_modified_goal(self): - goal1 = goal.Goal(self.ctx, id=1, uuid=utils.generate_uuid(), - name="DUMMY_2", display_name="original") - goal1.create() + self.assertEqual(4, m_s_create.call_count) + self.assertEqual(0, m_s_save.call_count) + self.assertEqual(1, m_s_soft_delete.call_count) - before_goals = goal.Goal.list(self.ctx) + def test_end2end_sync_goals_with_modified_goal_and_strategy(self): + goal = objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(), + name="DUMMY_1", display_name="Original") + goal.create() + strategy = objects.Strategy( + self.ctx, id=1, name="FakeDummy1Strategy1", + display_name="Original", goal_id=goal.id) + strategy.create() + # audit_template = objects.AuditTemplate( + # self.ctx, id=1, name="Synced AT", goal_id=goal.id, + # strategy_id=strategy.id) + # audit_template.create() + + # before_audit_templates = objects.AuditTemplate.list(self.ctx) + before_goals = objects.Goal.list(self.ctx) + before_strategies = objects.Strategy.list(self.ctx) try: self.syncer.sync() except Exception as exc: self.fail(exc) - after_goals = goal.Goal.list(self.ctx) + # after_audit_templates = objects.AuditTemplate.list(self.ctx) + after_goals = objects.Goal.list(self.ctx) + after_strategies = objects.Strategy.list(self.ctx) + self.assertEqual(1, len(before_goals)) + self.assertEqual(1, len(before_strategies)) + # self.assertEqual(1, len(before_audit_templates)) self.assertEqual(2, len(after_goals)) + self.assertEqual(4, len(after_strategies)) + # self.assertEqual(1, len(after_audit_templates)) self.assertEqual( {"DUMMY_1", "DUMMY_2"}, set([g.name for g in after_goals])) + self.assertEqual( + {'FakeDummy1Strategy1', 'FakeDummy1Strategy2', + 'FakeDummy2Strategy3', 'FakeDummy2Strategy4'}, + set([s.name for s in after_strategies])) created_goals = [ag for ag in after_goals if ag.uuid not in [bg.uuid for bg in before_goals]] + created_strategies = [ + a_s for a_s in after_strategies + if a_s.uuid not in [b_s.uuid for b_s in before_strategies]] + self.assertEqual(2, len(created_goals)) - # TODO(v-francoise): check that the audit templates are re-synced with - # the new goal version + self.assertEqual(4, len(created_strategies)) + + # synced_audit_template = after_audit_templates[0] + # self.assertTrue( + # audit_template.goal_id != synced_audit_template.goal_id) + # self.assertIn(synced_audit_template.goal_id, + # (g.id for g in after_goals))