Arnike/watcher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

DB sync for Strategies

Browse Source 
In this changeset, I added the ability to synchronize the strategies
into the Wather DB so that it can later be served through the Watcher
API.

Partially Implements: blueprint get-goal-from-strategy

Change-Id: Ifeaa1f6e1f4ff7d7efc1b221cf57797a49dc5bc5
This commit is contained in:
Vincent Françoise
2016-04-29 17:46:49 +02:00
parent 192d8e262c
commit a3ac26870a
2 changed files with 268 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
watcher/decision_engine/sync.py
View File

@@ -32,9 +32,16 @@ class Syncer(object):
self._discovered_map = None
self._available_goals = None
self._available_goal_names = None
self._available_goals_map = None
self._available_strategies = None
self._available_strategies_map = None
# This goal mapping maps stale goal IDs to the synced goal
self.goal_mapping = dict()
# This strategy mapping maps stale strategy IDs to the synced goal
self.strategy_mapping = dict()
@property
def available_goals(self):
if self._available_goals is None:
@@ -42,10 +49,10 @@ class Syncer(object):
return self._available_goals
@property
def available_goal_names(self):
if self._available_goal_names is None:
self._available_goal_names = [g.name for g in self.available_goals]
return self._available_goal_names
def available_strategies(self):
if self._available_strategies is None:
self._available_strategies = objects.Strategy.list(self.ctx)
return self._available_strategies
@property
def available_goals_map(self):
@@ -56,20 +63,42 @@ class Syncer(object):
}
return self._available_goals_map
@property
def available_strategies_map(self):
if self._available_strategies_map is None:
goals_map = {g.id: g.name for g in self.available_goals}
self._available_strategies_map = {
s: {"name": s.name, "goal_name": goals_map[s.goal_id],
"display_name": s.display_name}
for s in self.available_strategies
}
return self._available_strategies_map
def sync(self):
discovered_map = self.discover()
goals_map = discovered_map["goals"]
strategies_map = discovered_map["strategies"]
for goal_name, goal_map in goals_map.items():
if goal_map in self.available_goals_map.values():
LOG.info(_LI("Goal %s already exists"), goal_name)
continue
self._sync_goal(goal_map)
self.goal_mapping.update(self._sync_goal(goal_map))
for strategy_name, strategy_map in strategies_map.items():
if strategy_map in self.available_strategies_map.values():
LOG.info(_LI("Strategy %s already exists"), strategy_name)
continue
self.strategy_mapping.update(self._sync_strategy(strategy_map))
# TODO(v-francoise): Sync the audit templates
def _sync_goal(self, goal_map):
goal_name = goal_map['name']
goal_display_name = goal_map['display_name']
goal_mapping = dict()
matching_goals = [g for g in self.available_goals
if g.name == goal_name]
@@ -81,22 +110,41 @@ class Syncer(object):
goal.display_name = goal_display_name
goal.create()
LOG.info(_LI("Goal %s created"), goal_name)
self.available_goal_names.append(goal_name)
# Updating the internal states
self.available_goals_map[goal] = goal_map
# We have to update the audit templates that were pointing
# self._sync_audit_templates(stale_goals, goal)
# Map the old goal IDs to the new (equivalent) goal
for matching_goal in matching_goals:
goal_mapping[matching_goal.id] = goal
# def _sync_audit_templates(self, stale_goals, goal):
# related_audit_templates = []
# for stale_goal in stale_goals:
# filters = {"goal_id": stale_goal.id}
# related_audit_templates.extend(
# objects.AuditTemplate.list(self.ctx, filters=filters))
return goal_mapping
# for audit_template in related_audit_templates:
# LOG.info(_LI("Audit Template '%s' updated with synced goal"))
# audit_template.goal_id = goal.id
# audit_template.save()
def _sync_strategy(self, strategy_map):
strategy_name = strategy_map['name']
strategy_display_name = strategy_map['display_name']
goal_name = strategy_map['goal_name']
strategy_mapping = dict()
matching_strategies = [s for s in self.available_strategies
if s.name == strategy_name]
stale_strategies = self.soft_delete_stale_strategies(
strategy_map, matching_strategies)
if stale_strategies or not matching_strategies:
strategy = objects.Strategy(self.ctx)
strategy.name = strategy_name
strategy.display_name = strategy_display_name
strategy.goal_id = objects.Goal.get_by_name(self.ctx, goal_name).id
strategy.create()
LOG.info(_LI("Strategy %s created"), strategy_name)
# Updating the internal states
self.available_strategies_map[strategy] = strategy_map
# Map the old strategy IDs to the new (equivalent) strategy
for matching_strategy in matching_strategies:
strategy_mapping[matching_strategy.id] = strategy
return strategy_mapping
def discover(self):
strategies_map = {}
@@ -116,6 +164,11 @@ class Syncer(object):
"name": strategy_cls.DEFAULT_NAME,
"display_name": strategy_cls.DEFAULT_DESCRIPTION}
strategies_map[strategy_cls.__name__] = {
"name": strategy_cls.__name__,
"goal_name": strategy_cls.DEFAULT_NAME,
"display_name": strategy_cls.DEFAULT_DESCRIPTION}
return discovered_map
def soft_delete_stale_goals(self, goal_map, matching_goals):
@@ -132,3 +185,18 @@ class Syncer(object):
stale_goals.append(matching_goal)
return stale_goals
def soft_delete_stale_strategies(self, strategy_map, matching_strategies):
strategy_name = strategy_map['name']
strategy_display_name = strategy_map['display_name']
stale_strategies = []
for matching_strategy in matching_strategies:
if matching_strategy.display_name == strategy_display_name:
LOG.info(_LI("Strategy %s unchanged"), strategy_name)
else:
LOG.info(_LI("Strategy %s modified"), strategy_name)
matching_strategy.soft_delete()
stale_strategies.append(matching_strategy)
return stale_strategies

232
watcher/tests/decision_engine/test_sync.py
View File

@@ -21,7 +21,7 @@ from watcher.common import utils
from watcher.decision_engine.strategy.loading import default
from watcher.decision_engine.strategy.strategies import base as base_strategy
from watcher.decision_engine import sync
from watcher.objects import goal
from watcher import objects
from watcher.tests.db import base
@@ -36,25 +36,21 @@ class FakeStrategy(base_strategy.BaseStrategy):
class FakeDummy1Strategy1(FakeStrategy):
DEFAULT_NAME = "DUMMY_1"
DEFAULT_DESCRIPTION = "Dummy 1"
ID = "STRATEGY_1"
class FakeDummy1Strategy2(FakeStrategy):
DEFAULT_NAME = "DUMMY_1"
DEFAULT_DESCRIPTION = "Dummy 1"
ID = "STRATEGY_2"
class FakeDummy2Strategy3(FakeStrategy):
DEFAULT_NAME = "DUMMY_2"
DEFAULT_DESCRIPTION = "Dummy 2"
ID = "STRATEGY_3"
class FakeDummy2Strategy4(FakeStrategy):
DEFAULT_NAME = "DUMMY_2"
DEFAULT_DESCRIPTION = "Other Dummy 2"
ID = "STRATEGY_4"
class TestSyncer(base.DbTestCase):
@@ -64,10 +60,10 @@ class TestSyncer(base.DbTestCase):
self.ctx = context.make_context()
self.m_available_strategies = mock.Mock(return_value={
FakeDummy1Strategy1.DEFAULT_NAME: FakeDummy1Strategy1,
FakeDummy1Strategy2.DEFAULT_NAME: FakeDummy1Strategy2,
FakeDummy2Strategy3.DEFAULT_NAME: FakeDummy2Strategy3,
FakeDummy2Strategy4.DEFAULT_NAME: FakeDummy2Strategy4,
FakeDummy1Strategy1.__name__: FakeDummy1Strategy1,
FakeDummy1Strategy2.__name__: FakeDummy1Strategy2,
FakeDummy2Strategy3.__name__: FakeDummy2Strategy3,
FakeDummy2Strategy4.__name__: FakeDummy2Strategy4,
})
p_strategies = mock.patch.object(
@@ -78,72 +74,206 @@ class TestSyncer(base.DbTestCase):
self.syncer = sync.Syncer()
self.addCleanup(p_strategies.stop)
@mock.patch.object(goal.Goal, "soft_delete")
@mock.patch.object(goal.Goal, "save")
@mock.patch.object(goal.Goal, "create")
@mock.patch.object(goal.Goal, "list")
def test_sync_goals_empty_db(self, m_list, m_create,
m_save, m_soft_delete):
m_list.return_value = []
@mock.patch.object(objects.Strategy, "soft_delete")
@mock.patch.object(objects.Strategy, "save")
@mock.patch.object(objects.Strategy, "create")
@mock.patch.object(objects.Strategy, "list")
@mock.patch.object(objects.Goal, "get_by_name")
@mock.patch.object(objects.Goal, "soft_delete")
@mock.patch.object(objects.Goal, "save")
@mock.patch.object(objects.Goal, "create")
@mock.patch.object(objects.Goal, "list")
def test_sync_empty_db(
self, m_g_list, m_g_create, m_g_save, m_g_soft_delete,
m_g_get_by_name, m_s_list, m_s_create, m_s_save, m_s_soft_delete):
m_g_get_by_name.side_effect = [
objects.Goal(self.ctx, id=i) for i in range(1, 10)]
m_g_list.return_value = []
m_s_list.return_value = []
self.syncer.sync()
self.assertEqual(2, m_create.call_count)
self.assertEqual(0, m_save.call_count)
self.assertEqual(0, m_soft_delete.call_count)
self.assertEqual(2, m_g_create.call_count)
self.assertEqual(0, m_g_save.call_count)
self.assertEqual(0, m_g_soft_delete.call_count)
@mock.patch.object(goal.Goal, "soft_delete")
@mock.patch.object(goal.Goal, "save")
@mock.patch.object(goal.Goal, "create")
@mock.patch.object(goal.Goal, "list")
def test_sync_goals_with_existing_goal(self, m_list, m_create,
m_save, m_soft_delete):
m_list.return_value = [
goal.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
name="DUMMY_1", display_name="Dummy 1")
self.assertEqual(4, m_s_create.call_count)
self.assertEqual(0, m_s_save.call_count)
self.assertEqual(0, m_s_soft_delete.call_count)
@mock.patch.object(objects.Strategy, "soft_delete")
@mock.patch.object(objects.Strategy, "save")
@mock.patch.object(objects.Strategy, "create")
@mock.patch.object(objects.Strategy, "list")
@mock.patch.object(objects.Goal, "get_by_name")
@mock.patch.object(objects.Goal, "soft_delete")
@mock.patch.object(objects.Goal, "save")
@mock.patch.object(objects.Goal, "create")
@mock.patch.object(objects.Goal, "list")
def test_sync_with_existing_goal(
self, m_g_list, m_g_create, m_g_save, m_g_soft_delete,
m_g_get_by_name, m_s_list, m_s_create, m_s_save, m_s_soft_delete):
m_g_get_by_name.side_effect = [
objects.Goal(self.ctx, id=i) for i in range(1, 10)]
m_g_list.return_value = [
objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
name="DUMMY_1", display_name="Dummy 1")
]
m_s_list.return_value = []
self.syncer.sync()
self.assertEqual(1, m_g_create.call_count)
self.assertEqual(0, m_g_save.call_count)
self.assertEqual(0, m_g_soft_delete.call_count)
self.assertEqual(4, m_s_create.call_count)
self.assertEqual(0, m_s_save.call_count)
self.assertEqual(0, m_s_soft_delete.call_count)
@mock.patch.object(objects.Strategy, "soft_delete")
@mock.patch.object(objects.Strategy, "save")
@mock.patch.object(objects.Strategy, "create")
@mock.patch.object(objects.Strategy, "list")
@mock.patch.object(objects.Goal, "get_by_name")
@mock.patch.object(objects.Goal, "soft_delete")
@mock.patch.object(objects.Goal, "save")
@mock.patch.object(objects.Goal, "create")
@mock.patch.object(objects.Goal, "list")
def test_sync_with_existing_strategy(
self, m_g_list, m_g_create, m_g_save, m_g_soft_delete,
m_g_get_by_name, m_s_list, m_s_create, m_s_save, m_s_soft_delete):
m_g_get_by_name.side_effect = [
objects.Goal(self.ctx, id=i) for i in range(1, 10)]
m_g_list.return_value = [
objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
name="DUMMY_1", display_name="Dummy 1")
]
m_s_list.return_value = [
objects.Strategy(self.ctx, id=1, name="FakeDummy1Strategy1",
goal_id=1, display_name="Dummy 1")
]
self.syncer.sync()
self.assertEqual(1, m_create.call_count)
self.assertEqual(0, m_save.call_count)
self.assertEqual(0, m_soft_delete.call_count)
self.assertEqual(1, m_g_create.call_count)
self.assertEqual(0, m_g_save.call_count)
self.assertEqual(0, m_g_soft_delete.call_count)
@mock.patch.object(goal.Goal, "soft_delete")
@mock.patch.object(goal.Goal, "save")
@mock.patch.object(goal.Goal, "create")
@mock.patch.object(goal.Goal, "list")
def test_sync_goals_with_modified_goal(self, m_list, m_create,
m_save, m_soft_delete):
m_list.return_value = [
goal.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
name="DUMMY_2", display_name="original")
self.assertEqual(3, m_s_create.call_count)
self.assertEqual(0, m_s_save.call_count)
self.assertEqual(0, m_s_soft_delete.call_count)
@mock.patch.object(objects.Strategy, "soft_delete")
@mock.patch.object(objects.Strategy, "save")
@mock.patch.object(objects.Strategy, "create")
@mock.patch.object(objects.Strategy, "list")
@mock.patch.object(objects.Goal, "get_by_name")
@mock.patch.object(objects.Goal, "soft_delete")
@mock.patch.object(objects.Goal, "save")
@mock.patch.object(objects.Goal, "create")
@mock.patch.object(objects.Goal, "list")
def test_sync_with_modified_goal(
self, m_g_list, m_g_create, m_g_save, m_g_soft_delete,
m_g_get_by_name, m_s_list, m_s_create, m_s_save, m_s_soft_delete):
m_g_get_by_name.side_effect = [
objects.Goal(self.ctx, id=i) for i in range(1, 10)]
m_g_list.return_value = [
objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
name="DUMMY_2", display_name="original")
]
m_s_list.return_value = []
self.syncer.sync()
self.assertEqual(2, m_g_create.call_count)
self.assertEqual(0, m_g_save.call_count)
self.assertEqual(1, m_g_soft_delete.call_count)
self.assertEqual(4, m_s_create.call_count)
self.assertEqual(0, m_s_save.call_count)
self.assertEqual(0, m_s_soft_delete.call_count)
@mock.patch.object(objects.Strategy, "soft_delete")
@mock.patch.object(objects.Strategy, "save")
@mock.patch.object(objects.Strategy, "create")
@mock.patch.object(objects.Strategy, "list")
@mock.patch.object(objects.Goal, "get_by_name")
@mock.patch.object(objects.Goal, "soft_delete")
@mock.patch.object(objects.Goal, "save")
@mock.patch.object(objects.Goal, "create")
@mock.patch.object(objects.Goal, "list")
def test_sync_with_modified_strategy(
self, m_g_list, m_g_create, m_g_save, m_g_soft_delete,
m_g_get_by_name, m_s_list, m_s_create, m_s_save, m_s_soft_delete):
m_g_get_by_name.side_effect = [
objects.Goal(self.ctx, id=i) for i in range(1, 10)]
m_g_list.return_value = [
objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
name="DUMMY_1", display_name="Dummy 1")
]
m_s_list.return_value = [
objects.Strategy(self.ctx, id=1, name="FakeDummy1Strategy1",
goal_id=1, display_name="original")
]
self.syncer.sync()
self.assertEqual(2, m_create.call_count)
self.assertEqual(0, m_save.call_count)
self.assertEqual(1, m_soft_delete.call_count)
self.assertEqual(1, m_g_create.call_count)
self.assertEqual(0, m_g_save.call_count)
self.assertEqual(0, m_g_soft_delete.call_count)
def test_end2end_sync_goals_with_modified_goal(self):
goal1 = goal.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
name="DUMMY_2", display_name="original")
goal1.create()
self.assertEqual(4, m_s_create.call_count)
self.assertEqual(0, m_s_save.call_count)
self.assertEqual(1, m_s_soft_delete.call_count)
before_goals = goal.Goal.list(self.ctx)
def test_end2end_sync_goals_with_modified_goal_and_strategy(self):
goal = objects.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
name="DUMMY_1", display_name="Original")
goal.create()
strategy = objects.Strategy(
self.ctx, id=1, name="FakeDummy1Strategy1",
display_name="Original", goal_id=goal.id)
strategy.create()
# audit_template = objects.AuditTemplate(
# self.ctx, id=1, name="Synced AT", goal_id=goal.id,
# strategy_id=strategy.id)
# audit_template.create()
# before_audit_templates = objects.AuditTemplate.list(self.ctx)
before_goals = objects.Goal.list(self.ctx)
before_strategies = objects.Strategy.list(self.ctx)
try:
self.syncer.sync()
except Exception as exc:
self.fail(exc)
after_goals = goal.Goal.list(self.ctx)
# after_audit_templates = objects.AuditTemplate.list(self.ctx)
after_goals = objects.Goal.list(self.ctx)
after_strategies = objects.Strategy.list(self.ctx)
self.assertEqual(1, len(before_goals))
self.assertEqual(1, len(before_strategies))
# self.assertEqual(1, len(before_audit_templates))
self.assertEqual(2, len(after_goals))
self.assertEqual(4, len(after_strategies))
# self.assertEqual(1, len(after_audit_templates))
self.assertEqual(
{"DUMMY_1", "DUMMY_2"},
set([g.name for g in after_goals]))
self.assertEqual(
{'FakeDummy1Strategy1', 'FakeDummy1Strategy2',
'FakeDummy2Strategy3', 'FakeDummy2Strategy4'},
set([s.name for s in after_strategies]))
created_goals = [ag for ag in after_goals
if ag.uuid not in [bg.uuid for bg in before_goals]]
created_strategies = [
a_s for a_s in after_strategies
if a_s.uuid not in [b_s.uuid for b_s in before_strategies]]
self.assertEqual(2, len(created_goals))
# TODO(v-francoise): check that the audit templates are re-synced with
# the new goal version
self.assertEqual(4, len(created_strategies))
# synced_audit_template = after_audit_templates[0]
# self.assertTrue(
# audit_template.goal_id != synced_audit_template.goal_id)
# self.assertIn(synced_audit_template.goal_id,
# (g.id for g in after_goals))