diff --git a/watcher/decision_engine/strategy/strategies/__init__.py b/watcher/decision_engine/strategy/strategies/__init__.py index 84a9e5fcd..87f5f092c 100644 --- a/watcher/decision_engine/strategy/strategies/__init__.py +++ b/watcher/decision_engine/strategy/strategies/__init__.py @@ -18,14 +18,13 @@ from watcher.decision_engine.strategy.strategies import basic_consolidation from watcher.decision_engine.strategy.strategies import dummy_strategy from watcher.decision_engine.strategy.strategies import outlet_temp_control -from watcher.decision_engine.strategy.strategies \ - import vm_workload_consolidation +from watcher.decision_engine.strategy.strategies import \ + vm_workload_consolidation BasicConsolidation = basic_consolidation.BasicConsolidation OutletTempControl = outlet_temp_control.OutletTempControl DummyStrategy = dummy_strategy.DummyStrategy VMWorkloadConsolidation = vm_workload_consolidation.VMWorkloadConsolidation - -__all__ = (BasicConsolidation, OutletTempControl, DummyStrategy, - VMWorkloadConsolidation) +__all__ = ("BasicConsolidation", "OutletTempControl", + "DummyStrategy", "VMWorkloadConsolidation") diff --git a/watcher/decision_engine/sync.py b/watcher/decision_engine/sync.py new file mode 100644 index 000000000..45435c453 --- /dev/null +++ b/watcher/decision_engine/sync.py @@ -0,0 +1,134 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2016 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_log import log + +from watcher._i18n import _LI +from watcher.common import context +from watcher.decision_engine.strategy.loading import default +from watcher import objects + +LOG = log.getLogger(__name__) + + +class Syncer(object): + """Syncs all available goals and strategies with the Watcher DB""" + + def __init__(self): + self.ctx = context.make_context() + self._discovered_map = None + + self._available_goals = None + self._available_goal_names = None + self._available_goals_map = None + + @property + def available_goals(self): + if self._available_goals is None: + self._available_goals = objects.Goal.list(self.ctx) + return self._available_goals + + @property + def available_goal_names(self): + if self._available_goal_names is None: + self._available_goal_names = [g.name for g in self.available_goals] + return self._available_goal_names + + @property + def available_goals_map(self): + if self._available_goals_map is None: + self._available_goals_map = { + g: {"name": g.name, "display_name": g.display_name} + for g in self.available_goals + } + return self._available_goals_map + + def sync(self): + discovered_map = self.discover() + goals_map = discovered_map["goals"] + + for goal_name, goal_map in goals_map.items(): + if goal_map in self.available_goals_map.values(): + LOG.info(_LI("Goal %s already exists"), goal_name) + continue + + self._sync_goal(goal_map) + + def _sync_goal(self, goal_map): + goal_name = goal_map['name'] + goal_display_name = goal_map['display_name'] + + matching_goals = [g for g in self.available_goals + if g.name == goal_name] + stale_goals = self.soft_delete_stale_goals(goal_map, matching_goals) + + if stale_goals or not matching_goals: + goal = objects.Goal(self.ctx) + goal.name = goal_name + goal.display_name = goal_display_name + goal.create() + LOG.info(_LI("Goal %s created"), goal_name) + self.available_goal_names.append(goal_name) + self.available_goals_map[goal] = goal_map + # We have to update the audit templates that were pointing + # self._sync_audit_templates(stale_goals, goal) + + # def _sync_audit_templates(self, stale_goals, goal): + # related_audit_templates = [] + # for stale_goal in stale_goals: + # filters = {"goal_id": stale_goal.id} + # related_audit_templates.extend( + # objects.AuditTemplate.list(self.ctx, filters=filters)) + + # for audit_template in related_audit_templates: + # LOG.info(_LI("Audit Template '%s' updated with synced goal")) + # audit_template.goal_id = goal.id + # audit_template.save() + + def discover(self): + strategies_map = {} + goals_map = {} + discovered_map = {"goals": goals_map, "strategies": strategies_map} + strategy_loader = default.DefaultStrategyLoader() + implemented_strategies = strategy_loader.list_available() + + # TODO(v-francoise): At this point I only register the goals, but later + # on this will be extended to also populate the strategies map. + for _, strategy_cls in implemented_strategies.items(): + # This mapping is a temporary trick where I use the strategy + # DEFAULT_NAME as the goal name because we used to have a 1-to-1 + # mapping between the goal and the strategy. + # TODO(v-francoise): Dissociate the goal name and the strategy name + goals_map[strategy_cls.DEFAULT_NAME] = { + "name": strategy_cls.DEFAULT_NAME, + "display_name": strategy_cls.DEFAULT_DESCRIPTION} + + return discovered_map + + def soft_delete_stale_goals(self, goal_map, matching_goals): + goal_name = goal_map['name'] + goal_display_name = goal_map['display_name'] + + stale_goals = [] + for matching_goal in matching_goals: + if matching_goal.display_name == goal_display_name: + LOG.info(_LI("Goal %s unchanged"), goal_name) + else: + LOG.info(_LI("Goal %s modified"), goal_name) + matching_goal.soft_delete() + stale_goals.append(matching_goal) + + return stale_goals diff --git a/watcher/objects/__init__.py b/watcher/objects/__init__.py index dee1a19f2..93fa7855c 100644 --- a/watcher/objects/__init__.py +++ b/watcher/objects/__init__.py @@ -17,10 +17,12 @@ from watcher.objects import action from watcher.objects import action_plan from watcher.objects import audit from watcher.objects import audit_template +from watcher.objects import goal Audit = audit.Audit AuditTemplate = audit_template.AuditTemplate Action = action.Action ActionPlan = action_plan.ActionPlan +Goal = goal.Goal -__all__ = (Audit, AuditTemplate, Action, ActionPlan) +__all__ = ("Audit", "AuditTemplate", "Action", "ActionPlan", "Goal") diff --git a/watcher/objects/goal.py b/watcher/objects/goal.py new file mode 100644 index 000000000..238632484 --- /dev/null +++ b/watcher/objects/goal.py @@ -0,0 +1,184 @@ +# -*- encoding: utf-8 -*- +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from watcher.common import exception +from watcher.common import utils +from watcher.db import api as dbapi +from watcher.objects import base +from watcher.objects import utils as obj_utils + + +class Goal(base.WatcherObject): + # Version 1.0: Initial version + VERSION = '1.0' + + dbapi = dbapi.get_instance() + + fields = { + 'id': int, + 'uuid': obj_utils.str_or_none, + 'name': obj_utils.str_or_none, + 'display_name': obj_utils.str_or_none, + } + + @staticmethod + def _from_db_object(goal, db_goal): + """Converts a database entity to a formal object.""" + for field in goal.fields: + goal[field] = db_goal[field] + + goal.obj_reset_changes() + return goal + + @staticmethod + def _from_db_object_list(db_objects, cls, context): + """Converts a list of database entities to a list of formal objects.""" + return [cls._from_db_object(cls(context), obj) for obj in db_objects] + + @classmethod + def get(cls, context, goal_id): + """Find a goal based on its id or uuid + + :param context: Security context. NOTE: This should only + be used internally by the indirection_api. + Unfortunately, RPC requires context as the first + argument, even though we don't use it. + A context should be set when instantiating the + object, e.g.: Goal(context) + :param goal_id: the id *or* uuid of a goal. + :returns: a :class:`Goal` object. + """ + if utils.is_int_like(goal_id): + return cls.get_by_id(context, goal_id) + elif utils.is_uuid_like(goal_id): + return cls.get_by_uuid(context, goal_id) + else: + raise exception.InvalidIdentity(identity=goal_id) + + @classmethod + def get_by_id(cls, context, goal_id): + """Find a goal based on its integer id + + :param context: Security context. NOTE: This should only + be used internally by the indirection_api. + Unfortunately, RPC requires context as the first + argument, even though we don't use it. + A context should be set when instantiating the + object, e.g.: Goal(context) + :param goal_id: the id *or* uuid of a goal. + :returns: a :class:`Goal` object. + """ + db_goal = cls.dbapi.get_goal_by_id(context, goal_id) + goal = cls._from_db_object(cls(context), db_goal) + return goal + + @classmethod + def get_by_uuid(cls, context, uuid): + """Find a goal based on uuid + + :param context: Security context. NOTE: This should only + be used internally by the indirection_api. + Unfortunately, RPC requires context as the first + argument, even though we don't use it. + A context should be set when instantiating the + object, e.g.: Goal(context) + :param uuid: the uuid of a goal. + :returns: a :class:`Goal` object. + """ + + db_goal = cls.dbapi.get_goal_by_uuid(context, uuid) + goal = cls._from_db_object(cls(context), db_goal) + return goal + + @classmethod + def get_by_name(cls, context, name): + """Find a goal based on name + + :param name: the name of a goal. + :param context: Security context + :returns: a :class:`Goal` object. + """ + + db_goal = cls.dbapi.get_goal_by_name(context, name) + goal = cls._from_db_object(cls(context), db_goal) + return goal + + @classmethod + def list(cls, context, limit=None, marker=None, filters=None, + sort_key=None, sort_dir=None): + """Return a list of :class:`Goal` objects. + + :param context: Security context. NOTE: This should only + be used internally by the indirection_api. + Unfortunately, RPC requires context as the first + argument, even though we don't use it. + A context should be set when instantiating the + object, e.g.: Goal(context) + :param filters: dict mapping the filter key to a value. + :param limit: maximum number of resources to return in a single result. + :param marker: pagination marker for large data sets. + :param sort_key: column to sort results by. + :param sort_dir: direction to sort. "asc" or "desc". + :returns: a list of :class:`Goal` object. + """ + db_goals = cls.dbapi.get_goal_list( + context, + filters=filters, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(db_goals, cls, context) + + def create(self): + """Create a :class:`Goal` record in the DB.""" + + values = self.obj_get_changes() + db_goal = self.dbapi.create_goal(values) + self._from_db_object(self, db_goal) + + def destroy(self): + """Delete the :class:`Goal` from the DB.""" + self.dbapi.destroy_goal(self.id) + self.obj_reset_changes() + + def save(self): + """Save updates to this :class:`Goal`. + + Updates will be made column by column based on the result + of self.what_changed(). + """ + updates = self.obj_get_changes() + self.dbapi.update_goal(self.id, updates) + + self.obj_reset_changes() + + def refresh(self): + """Loads updates for this :class:`Goal`. + + Loads a goal with the same uuid from the database and + checks for updated attributes. Updates are applied from + the loaded goal column by column, if there are any updates. + """ + current = self.get_by_uuid(self._context, uuid=self.uuid) + for field in self.fields: + if (hasattr(self, base.get_attrname(field)) and + self[field] != current[field]): + self[field] = current[field] + + def soft_delete(self): + """Soft Delete the :class:`Goal` from the DB.""" + self.dbapi.soft_delete_goal(self.uuid) diff --git a/watcher/tests/db/test_goal.py b/watcher/tests/db/test_goal.py index 44451a747..3e8d5c03e 100644 --- a/watcher/tests/db/test_goal.py +++ b/watcher/tests/db/test_goal.py @@ -51,11 +51,11 @@ class TestDbGoalFilters(base.DbTestCase): def _soft_delete_goals(self): with freezegun.freeze_time(self.FAKE_TODAY): - self.dbapi.soft_delete_goal(self.goal1.uuid) + self.dbapi.soft_delete_goal(self.goal1.id) with freezegun.freeze_time(self.FAKE_OLD_DATE): - self.dbapi.soft_delete_goal(self.goal2.uuid) + self.dbapi.soft_delete_goal(self.goal2.id) with freezegun.freeze_time(self.FAKE_OLDER_DATE): - self.dbapi.soft_delete_goal(self.goal3.uuid) + self.dbapi.soft_delete_goal(self.goal3.id) def _update_goals(self): with freezegun.freeze_time(self.FAKE_TODAY): @@ -70,7 +70,7 @@ class TestDbGoalFilters(base.DbTestCase): def test_get_goal_list_filter_deleted_true(self): with freezegun.freeze_time(self.FAKE_TODAY): - self.dbapi.soft_delete_goal(self.goal1.uuid) + self.dbapi.soft_delete_goal(self.goal1.id) res = self.dbapi.get_goal_list( self.context, filters={'deleted': True}) @@ -79,7 +79,7 @@ class TestDbGoalFilters(base.DbTestCase): def test_get_goal_list_filter_deleted_false(self): with freezegun.freeze_time(self.FAKE_TODAY): - self.dbapi.soft_delete_goal(self.goal1.uuid) + self.dbapi.soft_delete_goal(self.goal1.id) res = self.dbapi.get_goal_list( self.context, filters={'deleted': False}) diff --git a/watcher/tests/decision_engine/test_sync.py b/watcher/tests/decision_engine/test_sync.py new file mode 100644 index 000000000..3f3799635 --- /dev/null +++ b/watcher/tests/decision_engine/test_sync.py @@ -0,0 +1,149 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2016 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +from watcher.common import context +from watcher.common import utils +from watcher.decision_engine.strategy.loading import default +from watcher.decision_engine.strategy.strategies import base as base_strategy +from watcher.decision_engine import sync +from watcher.objects import goal +from watcher.tests.db import base + + +class FakeStrategy(base_strategy.BaseStrategy): + DEFAULT_NAME = "" + DEFAULT_DESCRIPTION = "" + + def execute(self, original_model): + pass + + +class FakeDummy1Strategy1(FakeStrategy): + DEFAULT_NAME = "DUMMY_1" + DEFAULT_DESCRIPTION = "Dummy 1" + ID = "STRATEGY_1" + + +class FakeDummy1Strategy2(FakeStrategy): + DEFAULT_NAME = "DUMMY_1" + DEFAULT_DESCRIPTION = "Dummy 1" + ID = "STRATEGY_2" + + +class FakeDummy2Strategy3(FakeStrategy): + DEFAULT_NAME = "DUMMY_2" + DEFAULT_DESCRIPTION = "Dummy 2" + ID = "STRATEGY_3" + + +class FakeDummy2Strategy4(FakeStrategy): + DEFAULT_NAME = "DUMMY_2" + DEFAULT_DESCRIPTION = "Other Dummy 2" + ID = "STRATEGY_4" + + +class TestSyncer(base.DbTestCase): + + def setUp(self): + super(TestSyncer, self).setUp() + self.ctx = context.make_context() + + self.m_available_strategies = mock.Mock(return_value={ + FakeDummy1Strategy1.DEFAULT_NAME: FakeDummy1Strategy1, + FakeDummy1Strategy2.DEFAULT_NAME: FakeDummy1Strategy2, + FakeDummy2Strategy3.DEFAULT_NAME: FakeDummy2Strategy3, + FakeDummy2Strategy4.DEFAULT_NAME: FakeDummy2Strategy4, + }) + + p_strategies = mock.patch.object( + default.DefaultStrategyLoader, 'list_available', + self.m_available_strategies) + p_strategies.start() + + self.syncer = sync.Syncer() + self.addCleanup(p_strategies.stop) + + @mock.patch.object(goal.Goal, "soft_delete") + @mock.patch.object(goal.Goal, "save") + @mock.patch.object(goal.Goal, "create") + @mock.patch.object(goal.Goal, "list") + def test_sync_goals_empty_db(self, m_list, m_create, + m_save, m_soft_delete): + m_list.return_value = [] + + self.syncer.sync() + + self.assertEqual(2, m_create.call_count) + self.assertEqual(0, m_save.call_count) + self.assertEqual(0, m_soft_delete.call_count) + + @mock.patch.object(goal.Goal, "soft_delete") + @mock.patch.object(goal.Goal, "save") + @mock.patch.object(goal.Goal, "create") + @mock.patch.object(goal.Goal, "list") + def test_sync_goals_with_existing_goal(self, m_list, m_create, + m_save, m_soft_delete): + m_list.return_value = [ + goal.Goal(self.ctx, id=1, uuid=utils.generate_uuid(), + name="DUMMY_1", display_name="Dummy 1") + ] + self.syncer.sync() + + self.assertEqual(1, m_create.call_count) + self.assertEqual(0, m_save.call_count) + self.assertEqual(0, m_soft_delete.call_count) + + @mock.patch.object(goal.Goal, "soft_delete") + @mock.patch.object(goal.Goal, "save") + @mock.patch.object(goal.Goal, "create") + @mock.patch.object(goal.Goal, "list") + def test_sync_goals_with_modified_goal(self, m_list, m_create, + m_save, m_soft_delete): + m_list.return_value = [ + goal.Goal(self.ctx, id=1, uuid=utils.generate_uuid(), + name="DUMMY_2", display_name="original") + ] + self.syncer.sync() + + self.assertEqual(2, m_create.call_count) + self.assertEqual(0, m_save.call_count) + self.assertEqual(1, m_soft_delete.call_count) + + def test_end2end_sync_goals_with_modified_goal(self): + goal1 = goal.Goal(self.ctx, id=1, uuid=utils.generate_uuid(), + name="DUMMY_2", display_name="original") + goal1.create() + + before_goals = goal.Goal.list(self.ctx) + + try: + self.syncer.sync() + except Exception as exc: + self.fail(exc) + + after_goals = goal.Goal.list(self.ctx) + self.assertEqual(1, len(before_goals)) + self.assertEqual(2, len(after_goals)) + self.assertEqual( + {"DUMMY_1", "DUMMY_2"}, + set([g.name for g in after_goals])) + created_goals = [ag for ag in after_goals + if ag.uuid not in [bg.uuid for bg in before_goals]] + self.assertEqual(2, len(created_goals)) + # TODO(v-francoise): check that the audit templates are re-synced with + # the new goal version diff --git a/watcher/tests/objects/test_goal.py b/watcher/tests/objects/test_goal.py new file mode 100644 index 000000000..01334871c --- /dev/null +++ b/watcher/tests/objects/test_goal.py @@ -0,0 +1,138 @@ +# Copyright 2015 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from watcher import objects +from watcher.tests.db import base +from watcher.tests.db import utils + + +class TestGoalObject(base.DbTestCase): + + def setUp(self): + super(TestGoalObject, self).setUp() + self.fake_goal = utils.get_test_goal() + + def test_get_by_id(self): + goal_id = self.fake_goal['id'] + with mock.patch.object(self.dbapi, 'get_goal_by_id', + autospec=True) as mock_get_goal: + mock_get_goal.return_value = self.fake_goal + goal = objects.Goal.get(self.context, goal_id) + mock_get_goal.assert_called_once_with(self.context, goal_id) + self.assertEqual(self.context, goal._context) + + def test_get_by_uuid(self): + uuid = self.fake_goal['uuid'] + with mock.patch.object(self.dbapi, 'get_goal_by_uuid', + autospec=True) as mock_get_goal: + mock_get_goal.return_value = self.fake_goal + goal = objects.Goal.get(self.context, uuid) + mock_get_goal.assert_called_once_with(self.context, uuid) + self.assertEqual(self.context, goal._context) + + def test_get_by_name(self): + name = self.fake_goal['name'] + with mock.patch.object(self.dbapi, 'get_goal_by_name', + autospec=True) as mock_get_goal: + mock_get_goal.return_value = self.fake_goal + goal = objects.Goal.get_by_name( + self.context, + name) + mock_get_goal.assert_called_once_with(self.context, name) + self.assertEqual(self.context, goal._context) + + def test_list(self): + with mock.patch.object(self.dbapi, 'get_goal_list', + autospec=True) as mock_get_list: + mock_get_list.return_value = [self.fake_goal] + goals = objects.Goal.list(self.context) + self.assertEqual(1, mock_get_list.call_count) + self.assertEqual(1, len(goals)) + self.assertIsInstance(goals[0], objects.Goal) + self.assertEqual(self.context, goals[0]._context) + + def test_create(self): + with mock.patch.object(self.dbapi, 'create_goal', + autospec=True) as mock_create_goal: + mock_create_goal.return_value = self.fake_goal + goal = objects.Goal(self.context, **self.fake_goal) + goal.create() + mock_create_goal.assert_called_once_with(self.fake_goal) + self.assertEqual(self.context, goal._context) + + def test_destroy(self): + goal_id = self.fake_goal['id'] + with mock.patch.object(self.dbapi, 'get_goal_by_id', + autospec=True) as mock_get_goal: + mock_get_goal.return_value = self.fake_goal + with mock.patch.object(self.dbapi, 'destroy_goal', + autospec=True) \ + as mock_destroy_goal: + goal = objects.Goal.get_by_id(self.context, goal_id) + goal.destroy() + mock_get_goal.assert_called_once_with( + self.context, goal_id) + mock_destroy_goal.assert_called_once_with(goal_id) + self.assertEqual(self.context, goal._context) + + def test_save(self): + goal_id = self.fake_goal['id'] + with mock.patch.object(self.dbapi, 'get_goal_by_id', + autospec=True) as mock_get_goal: + mock_get_goal.return_value = self.fake_goal + with mock.patch.object(self.dbapi, 'update_goal', + autospec=True) as mock_update_goal: + goal = objects.Goal.get_by_id(self.context, goal_id) + goal.display_name = 'DUMMY' + goal.save() + + mock_get_goal.assert_called_once_with(self.context, goal_id) + mock_update_goal.assert_called_once_with( + goal_id, {'display_name': 'DUMMY'}) + self.assertEqual(self.context, goal._context) + + def test_refresh(self): + uuid = self.fake_goal['uuid'] + fake_goal2 = utils.get_test_goal(name="BALANCE_LOAD") + returns = [self.fake_goal, fake_goal2] + expected = [mock.call(self.context, uuid), + mock.call(self.context, uuid)] + with mock.patch.object(self.dbapi, 'get_goal_by_uuid', + side_effect=returns, + autospec=True) as mock_get_goal: + goal = objects.Goal.get(self.context, uuid) + self.assertEqual("TEST", goal.name) + goal.refresh() + self.assertEqual("BALANCE_LOAD", goal.name) + self.assertEqual(expected, mock_get_goal.call_args_list) + self.assertEqual(self.context, goal._context) + + def test_soft_delete(self): + uuid = self.fake_goal['uuid'] + with mock.patch.object(self.dbapi, 'get_goal_by_uuid', + autospec=True) as mock_get_goal: + mock_get_goal.return_value = self.fake_goal + with mock.patch.object(self.dbapi, 'soft_delete_goal', + autospec=True) \ + as mock_soft_delete_goal: + goal = objects.Goal.get_by_uuid( + self.context, uuid) + goal.soft_delete() + mock_get_goal.assert_called_once_with( + self.context, uuid) + mock_soft_delete_goal.assert_called_once_with(uuid) + self.assertEqual(self.context, goal._context)