Added Goal object + goal syncing

In this changeset, I added the Goal object into Watcher along with
a sync module that is responsible for syncing the goals with the
Watcher DB.

Partially Implements: blueprint get-goal-from-strategy

Change-Id: Ia3a2032dd9023d668c6f32ebbce44f8c1d77b0a3
This commit is contained in:
Vincent Françoise
2016-03-23 17:04:34 +01:00
parent be9058f3e3
commit 3b5ef15db6
7 changed files with 617 additions and 11 deletions

View File

@@ -18,14 +18,13 @@
from watcher.decision_engine.strategy.strategies import basic_consolidation
from watcher.decision_engine.strategy.strategies import dummy_strategy
from watcher.decision_engine.strategy.strategies import outlet_temp_control
from watcher.decision_engine.strategy.strategies \
import vm_workload_consolidation
from watcher.decision_engine.strategy.strategies import \
vm_workload_consolidation
BasicConsolidation = basic_consolidation.BasicConsolidation
OutletTempControl = outlet_temp_control.OutletTempControl
DummyStrategy = dummy_strategy.DummyStrategy
VMWorkloadConsolidation = vm_workload_consolidation.VMWorkloadConsolidation
__all__ = (BasicConsolidation, OutletTempControl, DummyStrategy,
VMWorkloadConsolidation)
__all__ = ("BasicConsolidation", "OutletTempControl",
"DummyStrategy", "VMWorkloadConsolidation")

View File

@@ -0,0 +1,134 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2016 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_log import log
from watcher._i18n import _LI
from watcher.common import context
from watcher.decision_engine.strategy.loading import default
from watcher import objects
LOG = log.getLogger(__name__)
class Syncer(object):
"""Syncs all available goals and strategies with the Watcher DB"""
def __init__(self):
self.ctx = context.make_context()
self._discovered_map = None
self._available_goals = None
self._available_goal_names = None
self._available_goals_map = None
@property
def available_goals(self):
if self._available_goals is None:
self._available_goals = objects.Goal.list(self.ctx)
return self._available_goals
@property
def available_goal_names(self):
if self._available_goal_names is None:
self._available_goal_names = [g.name for g in self.available_goals]
return self._available_goal_names
@property
def available_goals_map(self):
if self._available_goals_map is None:
self._available_goals_map = {
g: {"name": g.name, "display_name": g.display_name}
for g in self.available_goals
}
return self._available_goals_map
def sync(self):
discovered_map = self.discover()
goals_map = discovered_map["goals"]
for goal_name, goal_map in goals_map.items():
if goal_map in self.available_goals_map.values():
LOG.info(_LI("Goal %s already exists"), goal_name)
continue
self._sync_goal(goal_map)
def _sync_goal(self, goal_map):
goal_name = goal_map['name']
goal_display_name = goal_map['display_name']
matching_goals = [g for g in self.available_goals
if g.name == goal_name]
stale_goals = self.soft_delete_stale_goals(goal_map, matching_goals)
if stale_goals or not matching_goals:
goal = objects.Goal(self.ctx)
goal.name = goal_name
goal.display_name = goal_display_name
goal.create()
LOG.info(_LI("Goal %s created"), goal_name)
self.available_goal_names.append(goal_name)
self.available_goals_map[goal] = goal_map
# We have to update the audit templates that were pointing
# self._sync_audit_templates(stale_goals, goal)
# def _sync_audit_templates(self, stale_goals, goal):
# related_audit_templates = []
# for stale_goal in stale_goals:
# filters = {"goal_id": stale_goal.id}
# related_audit_templates.extend(
# objects.AuditTemplate.list(self.ctx, filters=filters))
# for audit_template in related_audit_templates:
# LOG.info(_LI("Audit Template '%s' updated with synced goal"))
# audit_template.goal_id = goal.id
# audit_template.save()
def discover(self):
strategies_map = {}
goals_map = {}
discovered_map = {"goals": goals_map, "strategies": strategies_map}
strategy_loader = default.DefaultStrategyLoader()
implemented_strategies = strategy_loader.list_available()
# TODO(v-francoise): At this point I only register the goals, but later
# on this will be extended to also populate the strategies map.
for _, strategy_cls in implemented_strategies.items():
# This mapping is a temporary trick where I use the strategy
# DEFAULT_NAME as the goal name because we used to have a 1-to-1
# mapping between the goal and the strategy.
# TODO(v-francoise): Dissociate the goal name and the strategy name
goals_map[strategy_cls.DEFAULT_NAME] = {
"name": strategy_cls.DEFAULT_NAME,
"display_name": strategy_cls.DEFAULT_DESCRIPTION}
return discovered_map
def soft_delete_stale_goals(self, goal_map, matching_goals):
goal_name = goal_map['name']
goal_display_name = goal_map['display_name']
stale_goals = []
for matching_goal in matching_goals:
if matching_goal.display_name == goal_display_name:
LOG.info(_LI("Goal %s unchanged"), goal_name)
else:
LOG.info(_LI("Goal %s modified"), goal_name)
matching_goal.soft_delete()
stale_goals.append(matching_goal)
return stale_goals

View File

@@ -17,10 +17,12 @@ from watcher.objects import action
from watcher.objects import action_plan
from watcher.objects import audit
from watcher.objects import audit_template
from watcher.objects import goal
Audit = audit.Audit
AuditTemplate = audit_template.AuditTemplate
Action = action.Action
ActionPlan = action_plan.ActionPlan
Goal = goal.Goal
__all__ = (Audit, AuditTemplate, Action, ActionPlan)
__all__ = ("Audit", "AuditTemplate", "Action", "ActionPlan", "Goal")

184
watcher/objects/goal.py Normal file
View File

@@ -0,0 +1,184 @@
# -*- encoding: utf-8 -*-
# Copyright 2013 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from watcher.common import exception
from watcher.common import utils
from watcher.db import api as dbapi
from watcher.objects import base
from watcher.objects import utils as obj_utils
class Goal(base.WatcherObject):
# Version 1.0: Initial version
VERSION = '1.0'
dbapi = dbapi.get_instance()
fields = {
'id': int,
'uuid': obj_utils.str_or_none,
'name': obj_utils.str_or_none,
'display_name': obj_utils.str_or_none,
}
@staticmethod
def _from_db_object(goal, db_goal):
"""Converts a database entity to a formal object."""
for field in goal.fields:
goal[field] = db_goal[field]
goal.obj_reset_changes()
return goal
@staticmethod
def _from_db_object_list(db_objects, cls, context):
"""Converts a list of database entities to a list of formal objects."""
return [cls._from_db_object(cls(context), obj) for obj in db_objects]
@classmethod
def get(cls, context, goal_id):
"""Find a goal based on its id or uuid
:param context: Security context. NOTE: This should only
be used internally by the indirection_api.
Unfortunately, RPC requires context as the first
argument, even though we don't use it.
A context should be set when instantiating the
object, e.g.: Goal(context)
:param goal_id: the id *or* uuid of a goal.
:returns: a :class:`Goal` object.
"""
if utils.is_int_like(goal_id):
return cls.get_by_id(context, goal_id)
elif utils.is_uuid_like(goal_id):
return cls.get_by_uuid(context, goal_id)
else:
raise exception.InvalidIdentity(identity=goal_id)
@classmethod
def get_by_id(cls, context, goal_id):
"""Find a goal based on its integer id
:param context: Security context. NOTE: This should only
be used internally by the indirection_api.
Unfortunately, RPC requires context as the first
argument, even though we don't use it.
A context should be set when instantiating the
object, e.g.: Goal(context)
:param goal_id: the id *or* uuid of a goal.
:returns: a :class:`Goal` object.
"""
db_goal = cls.dbapi.get_goal_by_id(context, goal_id)
goal = cls._from_db_object(cls(context), db_goal)
return goal
@classmethod
def get_by_uuid(cls, context, uuid):
"""Find a goal based on uuid
:param context: Security context. NOTE: This should only
be used internally by the indirection_api.
Unfortunately, RPC requires context as the first
argument, even though we don't use it.
A context should be set when instantiating the
object, e.g.: Goal(context)
:param uuid: the uuid of a goal.
:returns: a :class:`Goal` object.
"""
db_goal = cls.dbapi.get_goal_by_uuid(context, uuid)
goal = cls._from_db_object(cls(context), db_goal)
return goal
@classmethod
def get_by_name(cls, context, name):
"""Find a goal based on name
:param name: the name of a goal.
:param context: Security context
:returns: a :class:`Goal` object.
"""
db_goal = cls.dbapi.get_goal_by_name(context, name)
goal = cls._from_db_object(cls(context), db_goal)
return goal
@classmethod
def list(cls, context, limit=None, marker=None, filters=None,
sort_key=None, sort_dir=None):
"""Return a list of :class:`Goal` objects.
:param context: Security context. NOTE: This should only
be used internally by the indirection_api.
Unfortunately, RPC requires context as the first
argument, even though we don't use it.
A context should be set when instantiating the
object, e.g.: Goal(context)
:param filters: dict mapping the filter key to a value.
:param limit: maximum number of resources to return in a single result.
:param marker: pagination marker for large data sets.
:param sort_key: column to sort results by.
:param sort_dir: direction to sort. "asc" or "desc".
:returns: a list of :class:`Goal` object.
"""
db_goals = cls.dbapi.get_goal_list(
context,
filters=filters,
limit=limit,
marker=marker,
sort_key=sort_key,
sort_dir=sort_dir)
return cls._from_db_object_list(db_goals, cls, context)
def create(self):
"""Create a :class:`Goal` record in the DB."""
values = self.obj_get_changes()
db_goal = self.dbapi.create_goal(values)
self._from_db_object(self, db_goal)
def destroy(self):
"""Delete the :class:`Goal` from the DB."""
self.dbapi.destroy_goal(self.id)
self.obj_reset_changes()
def save(self):
"""Save updates to this :class:`Goal`.
Updates will be made column by column based on the result
of self.what_changed().
"""
updates = self.obj_get_changes()
self.dbapi.update_goal(self.id, updates)
self.obj_reset_changes()
def refresh(self):
"""Loads updates for this :class:`Goal`.
Loads a goal with the same uuid from the database and
checks for updated attributes. Updates are applied from
the loaded goal column by column, if there are any updates.
"""
current = self.get_by_uuid(self._context, uuid=self.uuid)
for field in self.fields:
if (hasattr(self, base.get_attrname(field)) and
self[field] != current[field]):
self[field] = current[field]
def soft_delete(self):
"""Soft Delete the :class:`Goal` from the DB."""
self.dbapi.soft_delete_goal(self.uuid)

View File

@@ -51,11 +51,11 @@ class TestDbGoalFilters(base.DbTestCase):
def _soft_delete_goals(self):
with freezegun.freeze_time(self.FAKE_TODAY):
self.dbapi.soft_delete_goal(self.goal1.uuid)
self.dbapi.soft_delete_goal(self.goal1.id)
with freezegun.freeze_time(self.FAKE_OLD_DATE):
self.dbapi.soft_delete_goal(self.goal2.uuid)
self.dbapi.soft_delete_goal(self.goal2.id)
with freezegun.freeze_time(self.FAKE_OLDER_DATE):
self.dbapi.soft_delete_goal(self.goal3.uuid)
self.dbapi.soft_delete_goal(self.goal3.id)
def _update_goals(self):
with freezegun.freeze_time(self.FAKE_TODAY):
@@ -70,7 +70,7 @@ class TestDbGoalFilters(base.DbTestCase):
def test_get_goal_list_filter_deleted_true(self):
with freezegun.freeze_time(self.FAKE_TODAY):
self.dbapi.soft_delete_goal(self.goal1.uuid)
self.dbapi.soft_delete_goal(self.goal1.id)
res = self.dbapi.get_goal_list(
self.context, filters={'deleted': True})
@@ -79,7 +79,7 @@ class TestDbGoalFilters(base.DbTestCase):
def test_get_goal_list_filter_deleted_false(self):
with freezegun.freeze_time(self.FAKE_TODAY):
self.dbapi.soft_delete_goal(self.goal1.uuid)
self.dbapi.soft_delete_goal(self.goal1.id)
res = self.dbapi.get_goal_list(
self.context, filters={'deleted': False})

View File

@@ -0,0 +1,149 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2016 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from watcher.common import context
from watcher.common import utils
from watcher.decision_engine.strategy.loading import default
from watcher.decision_engine.strategy.strategies import base as base_strategy
from watcher.decision_engine import sync
from watcher.objects import goal
from watcher.tests.db import base
class FakeStrategy(base_strategy.BaseStrategy):
DEFAULT_NAME = ""
DEFAULT_DESCRIPTION = ""
def execute(self, original_model):
pass
class FakeDummy1Strategy1(FakeStrategy):
DEFAULT_NAME = "DUMMY_1"
DEFAULT_DESCRIPTION = "Dummy 1"
ID = "STRATEGY_1"
class FakeDummy1Strategy2(FakeStrategy):
DEFAULT_NAME = "DUMMY_1"
DEFAULT_DESCRIPTION = "Dummy 1"
ID = "STRATEGY_2"
class FakeDummy2Strategy3(FakeStrategy):
DEFAULT_NAME = "DUMMY_2"
DEFAULT_DESCRIPTION = "Dummy 2"
ID = "STRATEGY_3"
class FakeDummy2Strategy4(FakeStrategy):
DEFAULT_NAME = "DUMMY_2"
DEFAULT_DESCRIPTION = "Other Dummy 2"
ID = "STRATEGY_4"
class TestSyncer(base.DbTestCase):
def setUp(self):
super(TestSyncer, self).setUp()
self.ctx = context.make_context()
self.m_available_strategies = mock.Mock(return_value={
FakeDummy1Strategy1.DEFAULT_NAME: FakeDummy1Strategy1,
FakeDummy1Strategy2.DEFAULT_NAME: FakeDummy1Strategy2,
FakeDummy2Strategy3.DEFAULT_NAME: FakeDummy2Strategy3,
FakeDummy2Strategy4.DEFAULT_NAME: FakeDummy2Strategy4,
})
p_strategies = mock.patch.object(
default.DefaultStrategyLoader, 'list_available',
self.m_available_strategies)
p_strategies.start()
self.syncer = sync.Syncer()
self.addCleanup(p_strategies.stop)
@mock.patch.object(goal.Goal, "soft_delete")
@mock.patch.object(goal.Goal, "save")
@mock.patch.object(goal.Goal, "create")
@mock.patch.object(goal.Goal, "list")
def test_sync_goals_empty_db(self, m_list, m_create,
m_save, m_soft_delete):
m_list.return_value = []
self.syncer.sync()
self.assertEqual(2, m_create.call_count)
self.assertEqual(0, m_save.call_count)
self.assertEqual(0, m_soft_delete.call_count)
@mock.patch.object(goal.Goal, "soft_delete")
@mock.patch.object(goal.Goal, "save")
@mock.patch.object(goal.Goal, "create")
@mock.patch.object(goal.Goal, "list")
def test_sync_goals_with_existing_goal(self, m_list, m_create,
m_save, m_soft_delete):
m_list.return_value = [
goal.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
name="DUMMY_1", display_name="Dummy 1")
]
self.syncer.sync()
self.assertEqual(1, m_create.call_count)
self.assertEqual(0, m_save.call_count)
self.assertEqual(0, m_soft_delete.call_count)
@mock.patch.object(goal.Goal, "soft_delete")
@mock.patch.object(goal.Goal, "save")
@mock.patch.object(goal.Goal, "create")
@mock.patch.object(goal.Goal, "list")
def test_sync_goals_with_modified_goal(self, m_list, m_create,
m_save, m_soft_delete):
m_list.return_value = [
goal.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
name="DUMMY_2", display_name="original")
]
self.syncer.sync()
self.assertEqual(2, m_create.call_count)
self.assertEqual(0, m_save.call_count)
self.assertEqual(1, m_soft_delete.call_count)
def test_end2end_sync_goals_with_modified_goal(self):
goal1 = goal.Goal(self.ctx, id=1, uuid=utils.generate_uuid(),
name="DUMMY_2", display_name="original")
goal1.create()
before_goals = goal.Goal.list(self.ctx)
try:
self.syncer.sync()
except Exception as exc:
self.fail(exc)
after_goals = goal.Goal.list(self.ctx)
self.assertEqual(1, len(before_goals))
self.assertEqual(2, len(after_goals))
self.assertEqual(
{"DUMMY_1", "DUMMY_2"},
set([g.name for g in after_goals]))
created_goals = [ag for ag in after_goals
if ag.uuid not in [bg.uuid for bg in before_goals]]
self.assertEqual(2, len(created_goals))
# TODO(v-francoise): check that the audit templates are re-synced with
# the new goal version

View File

@@ -0,0 +1,138 @@
# Copyright 2015 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from watcher import objects
from watcher.tests.db import base
from watcher.tests.db import utils
class TestGoalObject(base.DbTestCase):
def setUp(self):
super(TestGoalObject, self).setUp()
self.fake_goal = utils.get_test_goal()
def test_get_by_id(self):
goal_id = self.fake_goal['id']
with mock.patch.object(self.dbapi, 'get_goal_by_id',
autospec=True) as mock_get_goal:
mock_get_goal.return_value = self.fake_goal
goal = objects.Goal.get(self.context, goal_id)
mock_get_goal.assert_called_once_with(self.context, goal_id)
self.assertEqual(self.context, goal._context)
def test_get_by_uuid(self):
uuid = self.fake_goal['uuid']
with mock.patch.object(self.dbapi, 'get_goal_by_uuid',
autospec=True) as mock_get_goal:
mock_get_goal.return_value = self.fake_goal
goal = objects.Goal.get(self.context, uuid)
mock_get_goal.assert_called_once_with(self.context, uuid)
self.assertEqual(self.context, goal._context)
def test_get_by_name(self):
name = self.fake_goal['name']
with mock.patch.object(self.dbapi, 'get_goal_by_name',
autospec=True) as mock_get_goal:
mock_get_goal.return_value = self.fake_goal
goal = objects.Goal.get_by_name(
self.context,
name)
mock_get_goal.assert_called_once_with(self.context, name)
self.assertEqual(self.context, goal._context)
def test_list(self):
with mock.patch.object(self.dbapi, 'get_goal_list',
autospec=True) as mock_get_list:
mock_get_list.return_value = [self.fake_goal]
goals = objects.Goal.list(self.context)
self.assertEqual(1, mock_get_list.call_count)
self.assertEqual(1, len(goals))
self.assertIsInstance(goals[0], objects.Goal)
self.assertEqual(self.context, goals[0]._context)
def test_create(self):
with mock.patch.object(self.dbapi, 'create_goal',
autospec=True) as mock_create_goal:
mock_create_goal.return_value = self.fake_goal
goal = objects.Goal(self.context, **self.fake_goal)
goal.create()
mock_create_goal.assert_called_once_with(self.fake_goal)
self.assertEqual(self.context, goal._context)
def test_destroy(self):
goal_id = self.fake_goal['id']
with mock.patch.object(self.dbapi, 'get_goal_by_id',
autospec=True) as mock_get_goal:
mock_get_goal.return_value = self.fake_goal
with mock.patch.object(self.dbapi, 'destroy_goal',
autospec=True) \
as mock_destroy_goal:
goal = objects.Goal.get_by_id(self.context, goal_id)
goal.destroy()
mock_get_goal.assert_called_once_with(
self.context, goal_id)
mock_destroy_goal.assert_called_once_with(goal_id)
self.assertEqual(self.context, goal._context)
def test_save(self):
goal_id = self.fake_goal['id']
with mock.patch.object(self.dbapi, 'get_goal_by_id',
autospec=True) as mock_get_goal:
mock_get_goal.return_value = self.fake_goal
with mock.patch.object(self.dbapi, 'update_goal',
autospec=True) as mock_update_goal:
goal = objects.Goal.get_by_id(self.context, goal_id)
goal.display_name = 'DUMMY'
goal.save()
mock_get_goal.assert_called_once_with(self.context, goal_id)
mock_update_goal.assert_called_once_with(
goal_id, {'display_name': 'DUMMY'})
self.assertEqual(self.context, goal._context)
def test_refresh(self):
uuid = self.fake_goal['uuid']
fake_goal2 = utils.get_test_goal(name="BALANCE_LOAD")
returns = [self.fake_goal, fake_goal2]
expected = [mock.call(self.context, uuid),
mock.call(self.context, uuid)]
with mock.patch.object(self.dbapi, 'get_goal_by_uuid',
side_effect=returns,
autospec=True) as mock_get_goal:
goal = objects.Goal.get(self.context, uuid)
self.assertEqual("TEST", goal.name)
goal.refresh()
self.assertEqual("BALANCE_LOAD", goal.name)
self.assertEqual(expected, mock_get_goal.call_args_list)
self.assertEqual(self.context, goal._context)
def test_soft_delete(self):
uuid = self.fake_goal['uuid']
with mock.patch.object(self.dbapi, 'get_goal_by_uuid',
autospec=True) as mock_get_goal:
mock_get_goal.return_value = self.fake_goal
with mock.patch.object(self.dbapi, 'soft_delete_goal',
autospec=True) \
as mock_soft_delete_goal:
goal = objects.Goal.get_by_uuid(
self.context, uuid)
goal.soft_delete()
mock_get_goal.assert_called_once_with(
self.context, uuid)
mock_soft_delete_goal.assert_called_once_with(uuid)
self.assertEqual(self.context, goal._context)