Allow for global datasources preference from config

Allows to define a global preference for metric datasources with the
ability for strategy specific overrides. In addition, strategies which
do not require datasources have the config options removed this is
done to prevent confusion.

Some documentation that details the inner workings of selecting
datasources is updated.

Imports for some files in watcher/common have been changed to resolve
circular dependencies and now match the overall method to import
configuration.

Addtional datasources will be retrieved by the manager if the
datasource throws an error.

Implements: blueprint global-datasource-preference
Change-Id: I6fc455b288e338c20d2c4cfec5a0c95350bebc36
This commit is contained in:
Dantali0n
2019-03-21 15:17:44 +01:00
parent 92c94f61ca
commit bd8636f3f0
30 changed files with 234 additions and 126 deletions

View File

View File

@@ -0,0 +1,217 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
from __future__ import unicode_literals
import mock
from watcher.common import clients
from watcher.datasources import ceilometer as ceilometer_helper
from watcher.tests import base
@mock.patch.object(clients.OpenStackClients, 'ceilometer')
class TestCeilometerHelper(base.BaseTestCase):
def test_build_query(self, mock_ceilometer):
mock_ceilometer.return_value = mock.MagicMock()
cm = ceilometer_helper.CeilometerHelper()
expected = [{'field': 'user_id', 'op': 'eq', 'value': u'user_id'},
{'field': 'project_id', 'op': 'eq', 'value': u'tenant_id'},
{'field': 'resource_id', 'op': 'eq',
'value': u'resource_id'}]
query = cm.build_query(user_id="user_id",
tenant_id="tenant_id",
resource_id="resource_id",
user_ids=["user_ids"],
tenant_ids=["tenant_ids"],
resource_ids=["resource_ids"])
self.assertEqual(expected, query)
def test_statistic_aggregation(self, mock_ceilometer):
cm = ceilometer_helper.CeilometerHelper()
ceilometer = mock.MagicMock()
statistic = mock.MagicMock()
expected_result = 100
statistic[-1]._info = {'aggregate': {'avg': expected_result}}
ceilometer.statistics.list.return_value = statistic
mock_ceilometer.return_value = ceilometer
cm = ceilometer_helper.CeilometerHelper()
val = cm.statistic_aggregation(
resource_id="INSTANCE_ID",
meter_name="cpu_util",
period="7300",
granularity=None
)
self.assertEqual(expected_result, val)
def test_get_last_sample(self, mock_ceilometer):
ceilometer = mock.MagicMock()
statistic = mock.MagicMock()
expected_result = 100
statistic[-1]._info = {'counter_volume': expected_result}
ceilometer.samples.list.return_value = statistic
mock_ceilometer.return_value = ceilometer
cm = ceilometer_helper.CeilometerHelper()
val = cm.get_last_sample_value(
resource_id="id",
meter_name="compute.node.percent"
)
self.assertEqual(expected_result, val)
def test_get_last_sample_none(self, mock_ceilometer):
ceilometer = mock.MagicMock()
expected = []
ceilometer.samples.list.return_value = expected
mock_ceilometer.return_value = ceilometer
cm = ceilometer_helper.CeilometerHelper()
val = cm.get_last_sample_values(
resource_id="id",
meter_name="compute.node.percent"
)
self.assertEqual(expected, val)
def test_statistic_list(self, mock_ceilometer):
ceilometer = mock.MagicMock()
expected_value = []
ceilometer.statistics.list.return_value = expected_value
mock_ceilometer.return_value = ceilometer
cm = ceilometer_helper.CeilometerHelper()
val = cm.statistic_list(meter_name="cpu_util")
self.assertEqual(expected_value, val)
@mock.patch.object(ceilometer_helper.CeilometerHelper,
'statistic_aggregation')
def test_get_host_cpu_usage(self, mock_aggregation, mock_ceilometer):
helper = ceilometer_helper.CeilometerHelper()
helper.get_host_cpu_usage('compute1', 600, 'mean')
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['host_cpu_usage'], 600, None,
aggregate='mean')
@mock.patch.object(ceilometer_helper.CeilometerHelper,
'statistic_aggregation')
def test_get_instance_cpu_usage(self, mock_aggregation, mock_ceilometer):
helper = ceilometer_helper.CeilometerHelper()
helper.get_instance_cpu_usage('compute1', 600, 'mean')
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['instance_cpu_usage'], 600, None,
aggregate='mean')
@mock.patch.object(ceilometer_helper.CeilometerHelper,
'statistic_aggregation')
def test_get_host_memory_usage(self, mock_aggregation, mock_ceilometer):
helper = ceilometer_helper.CeilometerHelper()
helper.get_host_memory_usage('compute1', 600, 'mean')
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['host_memory_usage'], 600, None,
aggregate='mean')
@mock.patch.object(ceilometer_helper.CeilometerHelper,
'statistic_aggregation')
def test_get_instance_memory_usage(self, mock_aggregation,
mock_ceilometer):
helper = ceilometer_helper.CeilometerHelper()
helper.get_instance_ram_usage('compute1', 600, 'mean')
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['instance_ram_usage'], 600, None,
aggregate='mean')
@mock.patch.object(ceilometer_helper.CeilometerHelper,
'statistic_aggregation')
def test_get_instance_l3_cache_usage(self, mock_aggregation,
mock_ceilometer):
helper = ceilometer_helper.CeilometerHelper()
helper.get_instance_l3_cache_usage('compute1', 600, 'mean')
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['instance_l3_cache_usage'], 600,
None, aggregate='mean')
@mock.patch.object(ceilometer_helper.CeilometerHelper,
'statistic_aggregation')
def test_get_instance_ram_allocated(self, mock_aggregation,
mock_ceilometer):
helper = ceilometer_helper.CeilometerHelper()
helper.get_instance_ram_allocated('compute1', 600, 'mean')
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['instance_ram_allocated'], 600, None,
aggregate='mean')
@mock.patch.object(ceilometer_helper.CeilometerHelper,
'statistic_aggregation')
def test_get_instance_root_disk_allocated(self, mock_aggregation,
mock_ceilometer):
helper = ceilometer_helper.CeilometerHelper()
helper.get_instance_root_disk_size('compute1', 600, 'mean')
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['instance_root_disk_size'], 600,
None, aggregate='mean')
@mock.patch.object(ceilometer_helper.CeilometerHelper,
'statistic_aggregation')
def test_get_host_outlet_temperature(self, mock_aggregation,
mock_ceilometer):
helper = ceilometer_helper.CeilometerHelper()
helper.get_host_outlet_temp('compute1', 600, 'mean')
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['host_outlet_temp'], 600, None,
aggregate='mean')
@mock.patch.object(ceilometer_helper.CeilometerHelper,
'statistic_aggregation')
def test_get_host_inlet_temperature(self, mock_aggregation,
mock_ceilometer):
helper = ceilometer_helper.CeilometerHelper()
helper.get_host_inlet_temp('compute1', 600, 'mean')
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['host_inlet_temp'], 600, None,
aggregate='mean')
@mock.patch.object(ceilometer_helper.CeilometerHelper,
'statistic_aggregation')
def test_get_host_airflow(self, mock_aggregation, mock_ceilometer):
helper = ceilometer_helper.CeilometerHelper()
helper.get_host_airflow('compute1', 600, 'mean')
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['host_airflow'], 600, None,
aggregate='mean')
@mock.patch.object(ceilometer_helper.CeilometerHelper,
'statistic_aggregation')
def test_get_host_power(self, mock_aggregation, mock_ceilometer):
helper = ceilometer_helper.CeilometerHelper()
helper.get_host_power('compute1', 600, 'mean')
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['host_power'], 600, None,
aggregate='mean')
def test_check_availability(self, mock_ceilometer):
ceilometer = mock.MagicMock()
ceilometer.resources.list.return_value = True
mock_ceilometer.return_value = ceilometer
helper = ceilometer_helper.CeilometerHelper()
result = helper.check_availability()
self.assertEqual('available', result)
def test_check_availability_with_failure(self, mock_ceilometer):
ceilometer = mock.MagicMock()
ceilometer.resources.list.side_effect = Exception()
mock_ceilometer.return_value = ceilometer
helper = ceilometer_helper.CeilometerHelper()
self.assertEqual('not available', helper.check_availability())

View File

@@ -0,0 +1,172 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2017 Servionica
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from oslo_config import cfg
from watcher.common import clients
from watcher.datasources import gnocchi as gnocchi_helper
from watcher.tests import base
CONF = cfg.CONF
@mock.patch.object(clients.OpenStackClients, 'gnocchi')
class TestGnocchiHelper(base.BaseTestCase):
def test_gnocchi_statistic_aggregation(self, mock_gnocchi):
gnocchi = mock.MagicMock()
expected_result = 5.5
expected_measures = [["2017-02-02T09:00:00.000000", 360, 5.5]]
gnocchi.metric.get_measures.return_value = expected_measures
mock_gnocchi.return_value = gnocchi
helper = gnocchi_helper.GnocchiHelper()
result = helper.statistic_aggregation(
resource_id='16a86790-327a-45f9-bc82-45839f062fdc',
meter_name='cpu_util',
period=300,
granularity=360,
dimensions=None,
aggregation='mean',
group_by='*'
)
self.assertEqual(expected_result, result)
@mock.patch.object(gnocchi_helper.GnocchiHelper, 'statistic_aggregation')
def test_get_host_cpu_usage(self, mock_aggregation, mock_gnocchi):
helper = gnocchi_helper.GnocchiHelper()
helper.get_host_cpu_usage('compute1', 600, 'mean', granularity=300)
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['host_cpu_usage'], 600, 300,
aggregation='mean')
@mock.patch.object(gnocchi_helper.GnocchiHelper, 'statistic_aggregation')
def test_get_instance_cpu_usage(self, mock_aggregation, mock_gnocchi):
helper = gnocchi_helper.GnocchiHelper()
helper.get_instance_cpu_usage('compute1', 600, 'mean', granularity=300)
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['instance_cpu_usage'], 600, 300,
aggregation='mean')
@mock.patch.object(gnocchi_helper.GnocchiHelper, 'statistic_aggregation')
def test_get_host_memory_usage(self, mock_aggregation, mock_gnocchi):
helper = gnocchi_helper.GnocchiHelper()
helper.get_host_memory_usage('compute1', 600, 'mean', granularity=300)
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['host_memory_usage'], 600, 300,
aggregation='mean')
@mock.patch.object(gnocchi_helper.GnocchiHelper, 'statistic_aggregation')
def test_get_instance_memory_usage(self, mock_aggregation, mock_gnocchi):
helper = gnocchi_helper.GnocchiHelper()
helper.get_instance_ram_usage('compute1', 600, 'mean',
granularity=300)
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['instance_ram_usage'], 600, 300,
aggregation='mean')
@mock.patch.object(gnocchi_helper.GnocchiHelper, 'statistic_aggregation')
def test_get_instance_ram_allocated(self, mock_aggregation, mock_gnocchi):
helper = gnocchi_helper.GnocchiHelper()
helper.get_instance_ram_allocated('compute1', 600, 'mean',
granularity=300)
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['instance_ram_allocated'], 600, 300,
aggregation='mean')
@mock.patch.object(gnocchi_helper.GnocchiHelper, 'statistic_aggregation')
def test_get_instance_root_disk_allocated(self, mock_aggregation,
mock_gnocchi):
helper = gnocchi_helper.GnocchiHelper()
helper.get_instance_root_disk_size('compute1', 600, 'mean',
granularity=300)
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['instance_root_disk_size'], 600,
300, aggregation='mean')
@mock.patch.object(gnocchi_helper.GnocchiHelper, 'statistic_aggregation')
def test_get_host_outlet_temperature(self, mock_aggregation, mock_gnocchi):
helper = gnocchi_helper.GnocchiHelper()
helper.get_host_outlet_temp('compute1', 600, 'mean',
granularity=300)
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['host_outlet_temp'], 600, 300,
aggregation='mean')
@mock.patch.object(gnocchi_helper.GnocchiHelper, 'statistic_aggregation')
def test_get_host_inlet_temperature(self, mock_aggregation, mock_gnocchi):
helper = gnocchi_helper.GnocchiHelper()
helper.get_host_inlet_temp('compute1', 600, 'mean',
granularity=300)
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['host_inlet_temp'], 600, 300,
aggregation='mean')
@mock.patch.object(gnocchi_helper.GnocchiHelper, 'statistic_aggregation')
def test_get_host_airflow(self, mock_aggregation, mock_gnocchi):
helper = gnocchi_helper.GnocchiHelper()
helper.get_host_airflow('compute1', 600, 'mean', granularity=300)
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['host_airflow'], 600, 300,
aggregation='mean')
@mock.patch.object(gnocchi_helper.GnocchiHelper, 'statistic_aggregation')
def test_get_host_power(self, mock_aggregation, mock_gnocchi):
helper = gnocchi_helper.GnocchiHelper()
helper.get_host_power('compute1', 600, 'mean', granularity=300)
mock_aggregation.assert_called_once_with(
'compute1', helper.METRIC_MAP['host_power'], 600, 300,
aggregation='mean')
def test_gnocchi_check_availability(self, mock_gnocchi):
gnocchi = mock.MagicMock()
gnocchi.status.get.return_value = True
mock_gnocchi.return_value = gnocchi
helper = gnocchi_helper.GnocchiHelper()
result = helper.check_availability()
self.assertEqual('available', result)
def test_gnocchi_check_availability_with_failure(self, mock_gnocchi):
cfg.CONF.set_override("query_max_retries", 1,
group='gnocchi_client')
gnocchi = mock.MagicMock()
gnocchi.status.get.side_effect = Exception()
mock_gnocchi.return_value = gnocchi
helper = gnocchi_helper.GnocchiHelper()
self.assertEqual('not available', helper.check_availability())
def test_gnocchi_list_metrics(self, mock_gnocchi):
gnocchi = mock.MagicMock()
metrics = [{"name": "metric1"}, {"name": "metric2"}]
expected_metrics = set(["metric1", "metric2"])
gnocchi.metric.list.return_value = metrics
mock_gnocchi.return_value = gnocchi
helper = gnocchi_helper.GnocchiHelper()
result = helper.list_metrics()
self.assertEqual(expected_metrics, result)
def test_gnocchi_list_metrics_with_failure(self, mock_gnocchi):
cfg.CONF.set_override("query_max_retries", 1,
group='gnocchi_client')
gnocchi = mock.MagicMock()
gnocchi.metric.list.side_effect = Exception()
mock_gnocchi.return_value = gnocchi
helper = gnocchi_helper.GnocchiHelper()
self.assertFalse(helper.list_metrics())

View File

@@ -0,0 +1,59 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2017 Servionica
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from watcher.common import exception
from watcher.datasources import gnocchi
from watcher.datasources import manager as ds_manager
from watcher.tests import base
class TestDataSourceManager(base.BaseTestCase):
def test_get_backend(self):
manager = ds_manager.DataSourceManager(
config=mock.MagicMock(
datasources=['gnocchi', 'ceilometer', 'monasca']),
osc=mock.MagicMock())
backend = manager.get_backend(['host_cpu_usage', 'instance_cpu_usage'])
self.assertEqual(backend, manager.gnocchi)
def test_get_backend_order(self):
manager = ds_manager.DataSourceManager(
config=mock.MagicMock(
datasources=['monasca', 'ceilometer', 'gnocchi']),
osc=mock.MagicMock())
backend = manager.get_backend(['host_cpu_usage', 'instance_cpu_usage'])
self.assertEqual(backend, manager.monasca)
def test_get_backend_wrong_metric(self):
manager = ds_manager.DataSourceManager(
config=mock.MagicMock(
datasources=['gnocchi', 'ceilometer', 'monasca']),
osc=mock.MagicMock())
self.assertRaises(exception.NoSuchMetric, manager.get_backend,
['host_cpu', 'instance_cpu_usage'])
@mock.patch.object(gnocchi, 'GnocchiHelper')
def test_get_backend_error_datasource(self, m_gnocchi):
m_gnocchi.side_effect = exception.DataSourceNotAvailable
manager = ds_manager.DataSourceManager(
config=mock.MagicMock(
datasources=['gnocchi', 'ceilometer', 'monasca']),
osc=mock.MagicMock())
backend = manager.get_backend(['host_cpu_usage', 'instance_cpu_usage'])
self.assertEqual(backend, manager.ceilometer)

View File

@@ -0,0 +1,129 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from oslo_config import cfg
from watcher.common import clients
from watcher.datasources import monasca as monasca_helper
from watcher.tests import base
CONF = cfg.CONF
@mock.patch.object(clients.OpenStackClients, 'monasca')
class TestMonascaHelper(base.BaseTestCase):
def test_monasca_statistic_aggregation(self, mock_monasca):
monasca = mock.MagicMock()
expected_stat = [{
'columns': ['timestamp', 'avg'],
'dimensions': {
'hostname': 'rdev-indeedsrv001',
'service': 'monasca'},
'id': '0',
'name': 'cpu.percent',
'statistics': [
['2016-07-29T12:45:00Z', 0.0],
['2016-07-29T12:50:00Z', 0.9],
['2016-07-29T12:55:00Z', 0.9]]}]
monasca.metrics.list_statistics.return_value = expected_stat
mock_monasca.return_value = monasca
helper = monasca_helper.MonascaHelper()
result = helper.statistic_aggregation(
resource_id=None,
meter_name='cpu.percent',
period=7200,
granularity=300,
dimensions={'hostname': 'NODE_UUID'},
aggregation='avg',
group_by='*',
)
self.assertEqual(0.6, result)
def test_check_availability(self, mock_monasca):
monasca = mock.MagicMock()
monasca.metrics.list.return_value = True
mock_monasca.return_value = monasca
helper = monasca_helper.MonascaHelper()
result = helper.check_availability()
self.assertEqual('available', result)
def test_check_availability_with_failure(self, mock_monasca):
monasca = mock.MagicMock()
monasca.metrics.list.side_effect = Exception()
mock_monasca.return_value = monasca
helper = monasca_helper.MonascaHelper()
self.assertEqual('not available', helper.check_availability())
def test_monasca_statistic_list(self, mock_monasca):
monasca = mock.MagicMock()
expected_result = [{
'columns': ['timestamp', 'value', 'value_meta'],
'dimensions': {
'hostname': 'rdev-indeedsrv001',
'service': 'monasca'},
'id': '0',
'measurements': [
['2016-07-29T12:54:06.000Z', 0.9, {}],
['2016-07-29T12:54:36.000Z', 0.9, {}],
['2016-07-29T12:55:06.000Z', 0.9, {}],
['2016-07-29T12:55:36.000Z', 0.8, {}]],
'name': 'cpu.percent'}]
monasca.metrics.list_measurements.return_value = expected_result
mock_monasca.return_value = monasca
helper = monasca_helper.MonascaHelper()
val = helper.statistics_list(meter_name="cpu.percent", dimensions={})
self.assertEqual(expected_result, val)
def test_monasca_statistic_list_query_retry(self, mock_monasca):
monasca = mock.MagicMock()
expected_result = [{
'columns': ['timestamp', 'value', 'value_meta'],
'dimensions': {
'hostname': 'rdev-indeedsrv001',
'service': 'monasca'},
'id': '0',
'measurements': [
['2016-07-29T12:54:06.000Z', 0.9, {}],
['2016-07-29T12:54:36.000Z', 0.9, {}],
['2016-07-29T12:55:06.000Z', 0.9, {}],
['2016-07-29T12:55:36.000Z', 0.8, {}]],
'name': 'cpu.percent'}]
monasca.metrics.list_measurements.side_effect = [expected_result]
mock_monasca.return_value = monasca
helper = monasca_helper.MonascaHelper()
val = helper.statistics_list(meter_name="cpu.percent", dimensions={})
self.assertEqual(expected_result, val)
@mock.patch.object(monasca_helper.MonascaHelper, 'statistic_aggregation')
def test_get_host_cpu_usage(self, mock_aggregation, mock_monasca):
node = "compute1_compute1"
mock_aggregation.return_value = 0.6
helper = monasca_helper.MonascaHelper()
cpu_usage = helper.get_host_cpu_usage(node, 600, 'mean')
self.assertEqual(0.6, cpu_usage)
@mock.patch.object(monasca_helper.MonascaHelper, 'statistic_aggregation')
def test_get_instance_cpu_usage(self, mock_aggregation, mock_monasca):
mock_aggregation.return_value = 0.6
helper = monasca_helper.MonascaHelper()
cpu_usage = helper.get_instance_cpu_usage('vm1', 600, 'mean')
self.assertEqual(0.6, cpu_usage)