From bd8636f3f0622c18cfc6dd0f1698474a992da083 Mon Sep 17 00:00:00 2001 From: Dantali0n Date: Thu, 21 Mar 2019 15:17:44 +0100 Subject: [PATCH] Allow for global datasources preference from config Allows to define a global preference for metric datasources with the ability for strategy specific overrides. In addition, strategies which do not require datasources have the config options removed this is done to prevent confusion. Some documentation that details the inner workings of selecting datasources is updated. Imports for some files in watcher/common have been changed to resolve circular dependencies and now match the overall method to import configuration. Addtional datasources will be retrieved by the manager if the datasource throws an error. Implements: blueprint global-datasource-preference Change-Id: I6fc455b288e338c20d2c4cfec5a0c95350bebc36 --- .../contributor/plugin/strategy-plugin.rst | 9 ++- ...atasource-preference-3ab47b4be09ff3a5.yaml | 11 ++++ watcher/common/clients.py | 5 +- watcher/common/exception.py | 5 +- watcher/common/utils.py | 5 +- watcher/conf/__init__.py | 2 + watcher/conf/datasources.py | 47 ++++++++++++++ .../{datasource => datasources}/__init__.py | 0 watcher/{datasource => datasources}/base.py | 0 .../{datasource => datasources}/ceilometer.py | 2 +- .../{datasource => datasources}/gnocchi.py | 2 +- .../{datasource => datasources}/manager.py | 29 ++++++--- .../{datasource => datasources}/monasca.py | 2 +- .../strategy/strategies/actuation.py | 6 ++ .../strategy/strategies/base.py | 50 ++++++++++++++- .../strategies/basic_consolidation.py | 16 ++--- .../strategy/strategies/noisy_neighbor.py | 13 ---- .../strategies/outlet_temp_control.py | 14 ----- .../strategies/storage_capacity_balance.py | 2 +- .../strategy/strategies/uniform_airflow.py | 14 ----- .../strategies/vm_workload_consolidation.py | 14 ----- .../strategy/strategies/workload_balance.py | 14 ----- .../strategies/workload_stabilization.py | 13 ---- watcher/tests/conf/test_list_opts.py | 4 +- .../{datasource => datasources}/__init__.py | 0 .../test_ceilometer_helper.py | 2 +- .../test_gnocchi_helper.py | 2 +- .../test_manager.py | 13 +++- .../test_monasca_helper.py | 2 +- .../strategy/strategies/test_base.py | 62 +++++++++++++++++++ 30 files changed, 234 insertions(+), 126 deletions(-) create mode 100644 releasenotes/notes/global-datasource-preference-3ab47b4be09ff3a5.yaml create mode 100644 watcher/conf/datasources.py rename watcher/{datasource => datasources}/__init__.py (100%) rename watcher/{datasource => datasources}/base.py (100%) rename watcher/{datasource => datasources}/ceilometer.py (99%) rename watcher/{datasource => datasources}/gnocchi.py (99%) rename watcher/{datasource => datasources}/manager.py (71%) rename watcher/{datasource => datasources}/monasca.py (99%) rename watcher/tests/{datasource => datasources}/__init__.py (100%) rename watcher/tests/{datasource => datasources}/test_ceilometer_helper.py (99%) rename watcher/tests/{datasource => datasources}/test_gnocchi_helper.py (99%) rename watcher/tests/{datasource => datasources}/test_manager.py (75%) rename watcher/tests/{datasource => datasources}/test_monasca_helper.py (98%) diff --git a/doc/source/contributor/plugin/strategy-plugin.rst b/doc/source/contributor/plugin/strategy-plugin.rst index 5f38af147..1cde35773 100644 --- a/doc/source/contributor/plugin/strategy-plugin.rst +++ b/doc/source/contributor/plugin/strategy-plugin.rst @@ -285,8 +285,15 @@ The following code snippet shows how datasource_backend is defined: @property def datasource_backend(self): if not self._datasource_backend: + + # Load the global preferred datasources order but override it + # if the strategy has a specific datasources config + datasources = CONF.watcher_datasources + if self.config.datasources: + datasources = self.config + self._datasource_backend = ds_manager.DataSourceManager( - config=self.config, + config=datasources, osc=self.osc ).get_backend(self.DATASOURCE_METRICS) return self._datasource_backend diff --git a/releasenotes/notes/global-datasource-preference-3ab47b4be09ff3a5.yaml b/releasenotes/notes/global-datasource-preference-3ab47b4be09ff3a5.yaml new file mode 100644 index 000000000..884faf338 --- /dev/null +++ b/releasenotes/notes/global-datasource-preference-3ab47b4be09ff3a5.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + Watcher now supports configuring which datasource to use and in which + order. This configuration is done by specifying datasources in the + watcher_datasources section: + + - ``[watcher_datasources] datasources = gnocchi,monasca,ceilometer`` + + Specific strategies can override this order and use datasources which + are not listed in the global preference. \ No newline at end of file diff --git a/watcher/common/clients.py b/watcher/common/clients.py index fc504a6c5..bfe351ba8 100755 --- a/watcher/common/clients.py +++ b/watcher/common/clients.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo_config import cfg from cinderclient import client as ciclient from glanceclient import client as glclient @@ -23,15 +24,13 @@ from novaclient import client as nvclient from watcher.common import exception -from watcher import conf - try: from ceilometerclient import client as ceclient HAS_CEILCLIENT = True except ImportError: HAS_CEILCLIENT = False -CONF = conf.CONF +CONF = cfg.CONF _CLIENTS_AUTH_GROUP = 'watcher_clients_auth' diff --git a/watcher/common/exception.py b/watcher/common/exception.py index 912729f89..2fc2ff4f8 100644 --- a/watcher/common/exception.py +++ b/watcher/common/exception.py @@ -26,16 +26,15 @@ import functools import sys from keystoneclient import exceptions as keystone_exceptions +from oslo_config import cfg from oslo_log import log import six from watcher._i18n import _ -from watcher import conf - LOG = log.getLogger(__name__) -CONF = conf.CONF +CONF = cfg.CONF def wrap_keystone_exception(func): diff --git a/watcher/common/utils.py b/watcher/common/utils.py index 3a3408de3..ad2f7cf7e 100644 --- a/watcher/common/utils.py +++ b/watcher/common/utils.py @@ -24,6 +24,7 @@ import string from croniter import croniter from jsonschema import validators +from oslo_config import cfg from oslo_log import log from oslo_utils import strutils from oslo_utils import uuidutils @@ -31,9 +32,7 @@ import six from watcher.common import exception -from watcher import conf - -CONF = conf.CONF +CONF = cfg.CONF LOG = log.getLogger(__name__) diff --git a/watcher/conf/__init__.py b/watcher/conf/__init__.py index ca7f77aff..9ca08e798 100755 --- a/watcher/conf/__init__.py +++ b/watcher/conf/__init__.py @@ -25,6 +25,7 @@ from watcher.conf import ceilometer_client from watcher.conf import cinder_client from watcher.conf import clients_auth from watcher.conf import collector +from watcher.conf import datasources from watcher.conf import db from watcher.conf import decision_engine from watcher.conf import exception @@ -44,6 +45,7 @@ service.register_opts(CONF) api.register_opts(CONF) paths.register_opts(CONF) exception.register_opts(CONF) +datasources.register_opts(CONF) db.register_opts(CONF) planner.register_opts(CONF) applier.register_opts(CONF) diff --git a/watcher/conf/datasources.py b/watcher/conf/datasources.py new file mode 100644 index 000000000..6fcc5449e --- /dev/null +++ b/watcher/conf/datasources.py @@ -0,0 +1,47 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2019 European Organization for Nuclear Research (CERN) +# +# Authors: Corne Lukken +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +from watcher.datasources import manager + +datasources = cfg.OptGroup(name='watcher_datasources', + title='Configuration Options for watcher' + ' datasources') + +possible_datasources = list(manager.DataSourceManager.metric_map.keys()) + +DATASOURCES_OPTS = [ + cfg.ListOpt("datasources", + help="Datasources to use in order to query the needed metrics." + " If one of strategy metric is not available in the first" + " datasource, the next datasource will be chosen. This is" + " the default for all strategies unless a strategy has a" + " specific override.", + item_type=cfg.types.String(choices=possible_datasources), + default=possible_datasources) + ] + + +def register_opts(conf): + conf.register_group(datasources) + conf.register_opts(DATASOURCES_OPTS, group=datasources) + + +def list_opts(): + return [('watcher_datasources', DATASOURCES_OPTS)] diff --git a/watcher/datasource/__init__.py b/watcher/datasources/__init__.py similarity index 100% rename from watcher/datasource/__init__.py rename to watcher/datasources/__init__.py diff --git a/watcher/datasource/base.py b/watcher/datasources/base.py similarity index 100% rename from watcher/datasource/base.py rename to watcher/datasources/base.py diff --git a/watcher/datasource/ceilometer.py b/watcher/datasources/ceilometer.py similarity index 99% rename from watcher/datasource/ceilometer.py rename to watcher/datasources/ceilometer.py index f774ef8c2..290c1d7ed 100644 --- a/watcher/datasource/ceilometer.py +++ b/watcher/datasources/ceilometer.py @@ -25,7 +25,7 @@ from oslo_utils import timeutils from watcher._i18n import _ from watcher.common import clients from watcher.common import exception -from watcher.datasource import base +from watcher.datasources import base LOG = log.getLogger(__name__) diff --git a/watcher/datasource/gnocchi.py b/watcher/datasources/gnocchi.py similarity index 99% rename from watcher/datasource/gnocchi.py rename to watcher/datasources/gnocchi.py index b60607009..a1ec97161 100644 --- a/watcher/datasource/gnocchi.py +++ b/watcher/datasources/gnocchi.py @@ -26,7 +26,7 @@ from oslo_log import log from watcher.common import clients from watcher.common import exception from watcher.common import utils as common_utils -from watcher.datasource import base +from watcher.datasources import base CONF = cfg.CONF LOG = log.getLogger(__name__) diff --git a/watcher/datasource/manager.py b/watcher/datasources/manager.py similarity index 71% rename from watcher/datasource/manager.py rename to watcher/datasources/manager.py index 31303b213..1162e7bed 100644 --- a/watcher/datasource/manager.py +++ b/watcher/datasources/manager.py @@ -13,25 +13,31 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections import OrderedDict + from watcher.common import exception -from watcher.datasource import ceilometer as ceil -from watcher.datasource import gnocchi as gnoc -from watcher.datasource import monasca as mon +from watcher.datasources import ceilometer as ceil +from watcher.datasources import gnocchi as gnoc +from watcher.datasources import monasca as mon class DataSourceManager(object): + metric_map = OrderedDict([ + (gnoc.GnocchiHelper.NAME, gnoc.GnocchiHelper.METRIC_MAP), + (ceil.CeilometerHelper.NAME, ceil.CeilometerHelper.METRIC_MAP), + (mon.MonascaHelper.NAME, mon.MonascaHelper.METRIC_MAP), + ]) + """Dictionary with all possible datasources, dictionary order is the default + order for attempting to use datasources + """ + def __init__(self, config=None, osc=None): self.osc = osc self.config = config self._ceilometer = None self._monasca = None self._gnocchi = None - self.metric_map = { - mon.MonascaHelper.NAME: mon.MonascaHelper.METRIC_MAP, - gnoc.GnocchiHelper.NAME: gnoc.GnocchiHelper.METRIC_MAP, - ceil.CeilometerHelper.NAME: ceil.CeilometerHelper.METRIC_MAP - } self.datasources = self.config.datasources @property @@ -73,5 +79,10 @@ class DataSourceManager(object): no_metric = True break if not no_metric: - return getattr(self, datasource) + # Try to use a specific datasource but attempt additional + # datasources upon exceptions (if config has more datasources) + try: + return getattr(self, datasource) + except Exception: + pass raise exception.NoSuchMetric() diff --git a/watcher/datasource/monasca.py b/watcher/datasources/monasca.py similarity index 99% rename from watcher/datasource/monasca.py rename to watcher/datasources/monasca.py index 6b02419ef..c64c4fac0 100644 --- a/watcher/datasource/monasca.py +++ b/watcher/datasources/monasca.py @@ -22,7 +22,7 @@ from monascaclient import exc from watcher.common import clients from watcher.common import exception -from watcher.datasource import base +from watcher.datasources import base class MonascaHelper(base.DataSourceBase): diff --git a/watcher/decision_engine/strategy/strategies/actuation.py b/watcher/decision_engine/strategy/strategies/actuation.py index c54fbc4b3..3d88e34c9 100644 --- a/watcher/decision_engine/strategy/strategies/actuation.py +++ b/watcher/decision_engine/strategy/strategies/actuation.py @@ -80,6 +80,12 @@ class Actuator(base.UnclassifiedStrategy): ] } + @classmethod + def get_config_opts(cls): + """Override base class config options as do not use datasource """ + + return [] + @property def actions(self): return self.input_parameters.get('actions', []) diff --git a/watcher/decision_engine/strategy/strategies/base.py b/watcher/decision_engine/strategy/strategies/base.py index 7eeaefb2b..dca394eed 100755 --- a/watcher/decision_engine/strategy/strategies/base.py +++ b/watcher/decision_engine/strategy/strategies/base.py @@ -48,7 +48,7 @@ from watcher.common import context from watcher.common import exception from watcher.common.loader import loadable from watcher.common import utils -from watcher.datasource import manager as ds_manager +from watcher.datasources import manager as ds_manager from watcher.decision_engine.loading import default as loading from watcher.decision_engine.model.collector import manager from watcher.decision_engine.solution import default @@ -130,6 +130,8 @@ class BaseStrategy(loadable.Loadable): """ DATASOURCE_METRICS = [] + """Contains all metrics the strategy requires from a datasource to properly + execute""" def __init__(self, config, osc=None): """Constructor: the signature should be identical within the subclasses @@ -197,7 +199,18 @@ class BaseStrategy(loadable.Loadable): :return: A list of configuration options relative to this Loadable :rtype: list of :class:`oslo_config.cfg.Opt` instances """ - return [] + + datasources_ops = list(ds_manager.DataSourceManager.metric_map.keys()) + + return [ + cfg.ListOpt( + "datasources", + help="Datasources to use in order to query the needed metrics." + " This option overrides the global preference." + " options: {0}".format(datasources_ops), + item_type=cfg.types.String(choices=datasources_ops), + default=None) + ] @abc.abstractmethod def pre_execute(self): @@ -341,8 +354,15 @@ class BaseStrategy(loadable.Loadable): @property def datasource_backend(self): if not self._datasource_backend: + + # Load the global preferred datasources order but override it + # if the strategy has a specific datasources config + datasources = CONF.watcher_datasources + if self.config.datasources: + datasources = self.config + self._datasource_backend = ds_manager.DataSourceManager( - config=self.config, + config=datasources, osc=self.osc ).get_backend(self.DATASOURCE_METRICS) return self._datasource_backend @@ -429,6 +449,12 @@ class DummyBaseStrategy(BaseStrategy): def get_goal_name(cls): return "dummy" + @classmethod + def get_config_opts(cls): + """Override base class config options as do not use datasource """ + + return [] + @six.add_metaclass(abc.ABCMeta) class UnclassifiedStrategy(BaseStrategy): @@ -486,6 +512,12 @@ class SavingEnergyBaseStrategy(BaseStrategy): def get_goal_name(cls): return "saving_energy" + @classmethod + def get_config_opts(cls): + """Override base class config options as do not use datasource """ + + return [] + @six.add_metaclass(abc.ABCMeta) class ZoneMigrationBaseStrategy(BaseStrategy): @@ -494,6 +526,12 @@ class ZoneMigrationBaseStrategy(BaseStrategy): def get_goal_name(cls): return "hardware_maintenance" + @classmethod + def get_config_opts(cls): + """Override base class config options as do not use datasource """ + + return [] + @six.add_metaclass(abc.ABCMeta) class HostMaintenanceBaseStrategy(BaseStrategy): @@ -503,3 +541,9 @@ class HostMaintenanceBaseStrategy(BaseStrategy): @classmethod def get_goal_name(cls): return "cluster_maintaining" + + @classmethod + def get_config_opts(cls): + """Override base class config options as do not use datasource """ + + return [] diff --git a/watcher/decision_engine/strategy/strategies/basic_consolidation.py b/watcher/decision_engine/strategy/strategies/basic_consolidation.py index 0b4669658..edb313076 100644 --- a/watcher/decision_engine/strategy/strategies/basic_consolidation.py +++ b/watcher/decision_engine/strategy/strategies/basic_consolidation.py @@ -172,19 +172,11 @@ class BasicConsolidation(base.ServerConsolidationBaseStrategy): @classmethod def get_config_opts(cls): - return [ - cfg.ListOpt( - "datasources", - help="Datasources to use in order to query the needed metrics." - " If one of strategy metric isn't available in the first" - " datasource, the next datasource will be chosen.", - item_type=cfg.types.String(choices=['gnocchi', 'ceilometer', - 'monasca']), - default=['gnocchi', 'ceilometer', 'monasca']), + return super(BasicConsolidation, cls).get_config_opts() + [ cfg.BoolOpt( - "check_optimize_metadata", - help="Check optimize metadata field in instance before " - "migration", + 'check_optimize_metadata', + help='Check optimize metadata field in instance before' + ' migration', default=False), ] diff --git a/watcher/decision_engine/strategy/strategies/noisy_neighbor.py b/watcher/decision_engine/strategy/strategies/noisy_neighbor.py index 4893b162c..49568cd78 100644 --- a/watcher/decision_engine/strategy/strategies/noisy_neighbor.py +++ b/watcher/decision_engine/strategy/strategies/noisy_neighbor.py @@ -94,19 +94,6 @@ class NoisyNeighbor(base.NoisyNeighborBaseStrategy): }, } - @classmethod - def get_config_opts(cls): - return [ - cfg.ListOpt( - "datasources", - help="Datasources to use in order to query the needed metrics." - " If one of strategy metric isn't available in the first" - " datasource, the next datasource will be chosen.", - item_type=cfg.types.String(choices=['gnocchi', 'ceilometer', - 'monasca']), - default=['gnocchi', 'ceilometer', 'monasca']) - ] - def get_current_and_previous_cache(self, instance): try: curr_cache = self.datasource_backend.get_instance_l3_cache_usage( diff --git a/watcher/decision_engine/strategy/strategies/outlet_temp_control.py b/watcher/decision_engine/strategy/strategies/outlet_temp_control.py index 601a8a9ab..b7dada6b1 100644 --- a/watcher/decision_engine/strategy/strategies/outlet_temp_control.py +++ b/watcher/decision_engine/strategy/strategies/outlet_temp_control.py @@ -32,7 +32,6 @@ thermal condition (lowest outlet temperature) when the outlet temperature of source hosts reach a configurable threshold. """ -from oslo_config import cfg from oslo_log import log from watcher._i18n import _ @@ -141,19 +140,6 @@ class OutletTempControl(base.ThermalOptimizationBaseStrategy): def granularity(self): return self.input_parameters.get('granularity', 300) - @classmethod - def get_config_opts(cls): - return [ - cfg.ListOpt( - "datasources", - help="Datasources to use in order to query the needed metrics." - " If one of strategy metric isn't available in the first" - " datasource, the next datasource will be chosen.", - item_type=cfg.types.String(choices=['gnocchi', 'ceilometer', - 'monasca']), - default=['gnocchi', 'ceilometer', 'monasca']), - ] - def get_available_compute_nodes(self): default_node_scope = [element.ServiceState.ENABLED.value] return {uuid: cn for uuid, cn in diff --git a/watcher/decision_engine/strategy/strategies/storage_capacity_balance.py b/watcher/decision_engine/strategy/strategies/storage_capacity_balance.py index 1acf91c16..648839b62 100644 --- a/watcher/decision_engine/strategy/strategies/storage_capacity_balance.py +++ b/watcher/decision_engine/strategy/strategies/storage_capacity_balance.py @@ -99,7 +99,7 @@ class StorageCapacityBalance(base.WorkloadStabilizationBaseStrategy): @classmethod def get_config_opts(cls): - return [ + return super(StorageCapacityBalance, cls).get_config_opts() + [ cfg.ListOpt( "ex_pools", help="exclude pools", diff --git a/watcher/decision_engine/strategy/strategies/uniform_airflow.py b/watcher/decision_engine/strategy/strategies/uniform_airflow.py index 82c7c45ce..5091b38f9 100644 --- a/watcher/decision_engine/strategy/strategies/uniform_airflow.py +++ b/watcher/decision_engine/strategy/strategies/uniform_airflow.py @@ -17,7 +17,6 @@ # limitations under the License. # -from oslo_config import cfg from oslo_log import log from watcher._i18n import _ @@ -148,19 +147,6 @@ class UniformAirflow(base.BaseStrategy): }, } - @classmethod - def get_config_opts(cls): - return [ - cfg.ListOpt( - "datasources", - help="Datasources to use in order to query the needed metrics." - " If one of strategy metric isn't available in the first" - " datasource, the next datasource will be chosen.", - item_type=cfg.types.String(choices=['gnocchi', 'ceilometer', - 'monasca']), - default=['gnocchi', 'ceilometer', 'monasca']), - ] - def get_available_compute_nodes(self): default_node_scope = [element.ServiceState.ENABLED.value] return {uuid: cn for uuid, cn in diff --git a/watcher/decision_engine/strategy/strategies/vm_workload_consolidation.py b/watcher/decision_engine/strategy/strategies/vm_workload_consolidation.py index 06928d558..a983e7a1a 100644 --- a/watcher/decision_engine/strategy/strategies/vm_workload_consolidation.py +++ b/watcher/decision_engine/strategy/strategies/vm_workload_consolidation.py @@ -18,7 +18,6 @@ # limitations under the License. # -from oslo_config import cfg from oslo_log import log import six @@ -138,19 +137,6 @@ class VMWorkloadConsolidation(base.ServerConsolidationBaseStrategy): } } - @classmethod - def get_config_opts(cls): - return [ - cfg.ListOpt( - "datasources", - help="Datasources to use in order to query the needed metrics." - " If one of strategy metric isn't available in the first" - " datasource, the next datasource will be chosen.", - item_type=cfg.types.String(choices=['gnocchi', 'ceilometer', - 'monasca']), - default=['gnocchi', 'ceilometer', 'monasca']) - ] - def get_available_compute_nodes(self): default_node_scope = [element.ServiceState.ENABLED.value, element.ServiceState.DISABLED.value] diff --git a/watcher/decision_engine/strategy/strategies/workload_balance.py b/watcher/decision_engine/strategy/strategies/workload_balance.py index 10e8d0eba..764a4c00c 100644 --- a/watcher/decision_engine/strategy/strategies/workload_balance.py +++ b/watcher/decision_engine/strategy/strategies/workload_balance.py @@ -19,7 +19,6 @@ from __future__ import division -from oslo_config import cfg from oslo_log import log from watcher._i18n import _ @@ -130,19 +129,6 @@ class WorkloadBalance(base.WorkloadStabilizationBaseStrategy): }, } - @classmethod - def get_config_opts(cls): - return [ - cfg.ListOpt( - "datasources", - help="Datasources to use in order to query the needed metrics." - " If one of strategy metric isn't available in the first" - " datasource, the next datasource will be chosen.", - item_type=cfg.types.String(choices=['gnocchi', 'ceilometer', - 'monasca']), - default=['gnocchi', 'ceilometer', 'monasca']) - ] - def get_available_compute_nodes(self): default_node_scope = [element.ServiceState.ENABLED.value] return {uuid: cn for uuid, cn in diff --git a/watcher/decision_engine/strategy/strategies/workload_stabilization.py b/watcher/decision_engine/strategy/strategies/workload_stabilization.py index b0f8e3456..827ee1988 100644 --- a/watcher/decision_engine/strategy/strategies/workload_stabilization.py +++ b/watcher/decision_engine/strategy/strategies/workload_stabilization.py @@ -227,19 +227,6 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy): } } - @classmethod - def get_config_opts(cls): - return [ - cfg.ListOpt( - "datasources", - help="Datasources to use in order to query the needed metrics." - " If one of strategy metric isn't available in the first" - " datasource, the next datasource will be chosen.", - item_type=cfg.types.String(choices=['gnocchi', 'ceilometer', - 'monasca']), - default=['gnocchi', 'ceilometer', 'monasca']) - ] - def transform_instance_cpu(self, instance_load, host_vcpus): """Transform instance cpu utilization to overall host cpu utilization. diff --git a/watcher/tests/conf/test_list_opts.py b/watcher/tests/conf/test_list_opts.py index 8ecf0208a..026364745 100755 --- a/watcher/tests/conf/test_list_opts.py +++ b/watcher/tests/conf/test_list_opts.py @@ -29,8 +29,8 @@ class TestListOpts(base.TestCase): super(TestListOpts, self).setUp() self.base_sections = [ 'DEFAULT', 'api', 'database', 'watcher_decision_engine', - 'watcher_applier', 'watcher_planner', 'nova_client', - 'glance_client', 'gnocchi_client', 'cinder_client', + 'watcher_applier', 'watcher_datasources', 'watcher_planner', + 'nova_client', 'glance_client', 'gnocchi_client', 'cinder_client', 'ceilometer_client', 'monasca_client', 'ironic_client', 'neutron_client', 'watcher_clients_auth', 'collector'] self.opt_sections = list(dict(opts.list_opts()).keys()) diff --git a/watcher/tests/datasource/__init__.py b/watcher/tests/datasources/__init__.py similarity index 100% rename from watcher/tests/datasource/__init__.py rename to watcher/tests/datasources/__init__.py diff --git a/watcher/tests/datasource/test_ceilometer_helper.py b/watcher/tests/datasources/test_ceilometer_helper.py similarity index 99% rename from watcher/tests/datasource/test_ceilometer_helper.py rename to watcher/tests/datasources/test_ceilometer_helper.py index cc79f53bd..f07321804 100644 --- a/watcher/tests/datasource/test_ceilometer_helper.py +++ b/watcher/tests/datasources/test_ceilometer_helper.py @@ -20,7 +20,7 @@ from __future__ import unicode_literals import mock from watcher.common import clients -from watcher.datasource import ceilometer as ceilometer_helper +from watcher.datasources import ceilometer as ceilometer_helper from watcher.tests import base diff --git a/watcher/tests/datasource/test_gnocchi_helper.py b/watcher/tests/datasources/test_gnocchi_helper.py similarity index 99% rename from watcher/tests/datasource/test_gnocchi_helper.py rename to watcher/tests/datasources/test_gnocchi_helper.py index f3f1227a7..5d5030675 100644 --- a/watcher/tests/datasource/test_gnocchi_helper.py +++ b/watcher/tests/datasources/test_gnocchi_helper.py @@ -18,7 +18,7 @@ import mock from oslo_config import cfg from watcher.common import clients -from watcher.datasource import gnocchi as gnocchi_helper +from watcher.datasources import gnocchi as gnocchi_helper from watcher.tests import base CONF = cfg.CONF diff --git a/watcher/tests/datasource/test_manager.py b/watcher/tests/datasources/test_manager.py similarity index 75% rename from watcher/tests/datasource/test_manager.py rename to watcher/tests/datasources/test_manager.py index c0bd5df9d..19bb40088 100644 --- a/watcher/tests/datasource/test_manager.py +++ b/watcher/tests/datasources/test_manager.py @@ -17,7 +17,8 @@ import mock from watcher.common import exception -from watcher.datasource import manager as ds_manager +from watcher.datasources import gnocchi +from watcher.datasources import manager as ds_manager from watcher.tests import base @@ -46,3 +47,13 @@ class TestDataSourceManager(base.BaseTestCase): osc=mock.MagicMock()) self.assertRaises(exception.NoSuchMetric, manager.get_backend, ['host_cpu', 'instance_cpu_usage']) + + @mock.patch.object(gnocchi, 'GnocchiHelper') + def test_get_backend_error_datasource(self, m_gnocchi): + m_gnocchi.side_effect = exception.DataSourceNotAvailable + manager = ds_manager.DataSourceManager( + config=mock.MagicMock( + datasources=['gnocchi', 'ceilometer', 'monasca']), + osc=mock.MagicMock()) + backend = manager.get_backend(['host_cpu_usage', 'instance_cpu_usage']) + self.assertEqual(backend, manager.ceilometer) diff --git a/watcher/tests/datasource/test_monasca_helper.py b/watcher/tests/datasources/test_monasca_helper.py similarity index 98% rename from watcher/tests/datasource/test_monasca_helper.py rename to watcher/tests/datasources/test_monasca_helper.py index 93e63d6a6..840318d0e 100644 --- a/watcher/tests/datasource/test_monasca_helper.py +++ b/watcher/tests/datasources/test_monasca_helper.py @@ -18,7 +18,7 @@ import mock from oslo_config import cfg from watcher.common import clients -from watcher.datasource import monasca as monasca_helper +from watcher.datasources import monasca as monasca_helper from watcher.tests import base CONF = cfg.CONF diff --git a/watcher/tests/decision_engine/strategy/strategies/test_base.py b/watcher/tests/decision_engine/strategy/strategies/test_base.py index 0c71d931d..b6e0bfe2d 100644 --- a/watcher/tests/decision_engine/strategy/strategies/test_base.py +++ b/watcher/tests/decision_engine/strategy/strategies/test_base.py @@ -17,6 +17,7 @@ import mock from watcher.common import exception +from watcher.datasources import manager from watcher.decision_engine.model import model_root from watcher.decision_engine.strategy import strategies from watcher.tests import base @@ -49,6 +50,67 @@ class TestBaseStrategy(base.TestCase): self.strategy = strategies.DummyStrategy(config=mock.Mock()) +class TestBaseStrategyDatasource(TestBaseStrategy): + + def setUp(self): + super(TestBaseStrategyDatasource, self).setUp() + self.strategy = strategies.DummyStrategy( + config=mock.Mock(datasources=None)) + + @mock.patch.object(strategies.BaseStrategy, 'osc', None) + @mock.patch.object(manager, 'DataSourceManager') + @mock.patch.object(strategies.base, 'CONF') + def test_global_preference(self, m_conf, m_manager): + """Test if the global preference is used""" + + m_conf.watcher_datasources.datasources = \ + ['gnocchi', 'monasca', 'ceilometer'] + + # Access the property so that the configuration is read in order to + # get the correct datasource + self.strategy.datasource_backend() + + m_manager.assert_called_once_with( + config=m_conf.watcher_datasources, osc=None) + + @mock.patch.object(strategies.BaseStrategy, 'osc', None) + @mock.patch.object(manager, 'DataSourceManager') + @mock.patch.object(strategies.base, 'CONF') + def test_global_preference_reverse(self, m_conf, m_manager): + """Test if the global preference is used with another order""" + + m_conf.watcher_datasources.datasources = \ + ['ceilometer', 'monasca', 'gnocchi'] + + # Access the property so that the configuration is read in order to + # get the correct datasource + self.strategy.datasource_backend() + + m_manager.assert_called_once_with( + config=m_conf.watcher_datasources, osc=None) + + @mock.patch.object(strategies.BaseStrategy, 'osc', None) + @mock.patch.object(manager, 'DataSourceManager') + @mock.patch.object(strategies.base, 'CONF') + def test_strategy_preference_override(self, m_conf, m_manager): + """Test if the global preference can be overridden""" + + datasources = mock.Mock(datasources=['ceilometer']) + + self.strategy = strategies.DummyStrategy( + config=datasources) + + m_conf.watcher_datasources.datasources = \ + ['ceilometer', 'monasca', 'gnocchi'] + + # Access the property so that the configuration is read in order to + # get the correct datasource + self.strategy.datasource_backend() + + m_manager.assert_called_once_with( + config=datasources, osc=None) + + class TestBaseStrategyException(TestBaseStrategy): def setUp(self):