From 03c09825f7421eac429034fb4e25ba089671077d Mon Sep 17 00:00:00 2001 From: Douglas Viroel Date: Thu, 17 Jul 2025 17:03:15 -0300 Subject: [PATCH] Extend compute model attributes This patch extends compute model attributes by adding new fields to Instance element. Values are populated by nova the collector, using the same nova list call, but requires a more recent compute API microversion. A new config option was added to allow users to enable or disable the extended attributes and it is disable by default. Configure prometheus-based jobs to run on newer version of nova api (2.96) and enables the extended attributes collection. Implements: bp/extend-compute-model-attributes Assisted-By: Cursor (claude-4-sonnet) Change-Id: Ibf31105d780dce510a59fc74241fa04e28529ade Signed-off-by: Douglas Viroel --- .zuul.yaml | 10 ++- api-ref/source/parameters.yaml | 14 +++ .../samples/datamodel-list-response.json | 6 ++ api-ref/source/watcher-api-v1-datamodel.inc | 2 + devstack/lib/watcher | 2 +- ...ute-model-attributes-b56bc093e8637bb4.yaml | 14 +++ test-requirements.txt | 1 + watcher/api/controllers/v1/data_model.py | 17 ++++ watcher/api/controllers/v1/utils.py | 10 +++ watcher/api/controllers/v1/versions.py | 3 +- watcher/common/nova_helper.py | 13 +++ watcher/conf/__init__.py | 2 + watcher/conf/models.py | 40 +++++++++ .../decision_engine/model/collector/nova.py | 21 +++-- .../decision_engine/model/element/instance.py | 3 + watcher/decision_engine/model/model_root.py | 15 ++++ .../model/notification/nova.py | 16 +++- watcher/tests/api/v1/test_data_model.py | 44 ++++++--- .../decision_engine/cluster/test_nova_cdmc.py | 75 +++++++++++++++- .../data/instance-update-2-1.json | 90 +++++++++++++++++++ .../notification/test_nova_notifications.py | 75 +++++++++++++++- .../decision_engine/model/test_element.py | 10 +++ 22 files changed, 454 insertions(+), 29 deletions(-) create mode 100644 releasenotes/notes/bp-extend-compute-model-attributes-b56bc093e8637bb4.yaml create mode 100644 watcher/conf/models.py create mode 100644 watcher/tests/decision_engine/model/notification/data/instance-update-2-1.json diff --git a/.zuul.yaml b/.zuul.yaml index 2681790bf..09a704b74 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -198,6 +198,10 @@ period: 120 watcher_cluster_data_model_collectors.storage: period: 120 + compute_model: + enable_extended_attributes: true + nova_client: + api_version: "2.96" test-config: $TEMPEST_CONFIG: compute: @@ -217,12 +221,14 @@ ceilometer_polling_interval: 15 optimize: datasource: prometheus + extended_attributes_nova_microversion: "2.96" + data_model_collectors_period: 120 tempest_plugins: - watcher-tempest-plugin # All tests inside watcher_tempest_plugin.tests.scenario with tag "strategy" - # or test_execute_strategies file + # and test_execute_strategies, test_data_model files # excluding tests with tag "real_load" - tempest_test_regex: (watcher_tempest_plugin.tests.scenario)(.*\[.*\bstrategy\b.*\].*)|(watcher_tempest_plugin.tests.scenario.test_execute_strategies) + tempest_test_regex: (watcher_tempest_plugin.tests.scenario)(.*\[.*\bstrategy\b.*\].*)|(watcher_tempest_plugin.tests.scenario.(test_execute_strategies|test_data_model)) tempest_exclude_regex: .*\[.*\breal_load\b.*\].* tempest_concurrency: 1 tox_envlist: all diff --git a/api-ref/source/parameters.yaml b/api-ref/source/parameters.yaml index c8cd2a3b7..eff542ccd 100644 --- a/api-ref/source/parameters.yaml +++ b/api-ref/source/parameters.yaml @@ -552,6 +552,13 @@ server_disk: in: body required: true type: integer +server_flavor_extra_specs: + description: | + The flavor extra specs of the server. + in: body + required: true + type: JSON + min_version: 1.6 server_locked: description: | Whether the server is locked. @@ -576,6 +583,13 @@ server_name: in: body required: true type: string +server_pinned_az: + description: | + The pinned availability zone of the server. + in: body + required: true + type: string + min_version: 1.6 server_project_id: description: | The project ID of the server. diff --git a/api-ref/source/samples/datamodel-list-response.json b/api-ref/source/samples/datamodel-list-response.json index 919569c7b..9e412dffc 100644 --- a/api-ref/source/samples/datamodel-list-response.json +++ b/api-ref/source/samples/datamodel-list-response.json @@ -11,6 +11,10 @@ "server_project_id": "baea342fc74b4a1785b4a40c69a8d958", "server_locked":false, "server_uuid": "1bf91464-9b41-428d-a11e-af691e5563bb", + "server_pinned_az": "nova", + "server_flavor_extra_specs": { + "hw_rng:allowed": true + }, "node_hostname": "localhost.localdomain", "node_status": "enabled", "node_disabled_reason": null, @@ -37,6 +41,8 @@ "server_project_id": "baea342fc74b4a1785b4a40c69a8d958", "server_locked": false, "server_uuid": "e2cb5f6f-fa1d-4ba2-be1e-0bf02fa86ba4", + "server_pinned_az": "nova", + "server_flavor_extra_specs": {}, "node_hostname": "localhost.localdomain", "node_status": "enabled", "node_disabled_reason": null, diff --git a/api-ref/source/watcher-api-v1-datamodel.inc b/api-ref/source/watcher-api-v1-datamodel.inc index 57c5093b5..be944fb34 100644 --- a/api-ref/source/watcher-api-v1-datamodel.inc +++ b/api-ref/source/watcher-api-v1-datamodel.inc @@ -45,6 +45,8 @@ Response - server_project_id: server_project_id - server_locked: server_locked - server_uuid: server_uuid + - server_pinned_az: server_pinned_az + - server_flavor_extra_specs: server_flavor_extra_specs - node_hostname: node_hostname - node_status: node_status - node_disabled_reason: node_disabled_reason diff --git a/devstack/lib/watcher b/devstack/lib/watcher index fe04d4c11..962070999 100644 --- a/devstack/lib/watcher +++ b/devstack/lib/watcher @@ -268,7 +268,7 @@ function configure_tempest_for_watcher { # Please make sure to update this when the microversion is updated, otherwise # new tests may be skipped. TEMPEST_WATCHER_MIN_MICROVERSION=${TEMPEST_WATCHER_MIN_MICROVERSION:-"1.0"} - TEMPEST_WATCHER_MAX_MICROVERSION=${TEMPEST_WATCHER_MAX_MICROVERSION:-"1.5"} + TEMPEST_WATCHER_MAX_MICROVERSION=${TEMPEST_WATCHER_MAX_MICROVERSION:-"1.6"} # Set microversion options in tempest.conf iniset $TEMPEST_CONFIG optimize min_microversion $TEMPEST_WATCHER_MIN_MICROVERSION diff --git a/releasenotes/notes/bp-extend-compute-model-attributes-b56bc093e8637bb4.yaml b/releasenotes/notes/bp-extend-compute-model-attributes-b56bc093e8637bb4.yaml new file mode 100644 index 000000000..952fce49b --- /dev/null +++ b/releasenotes/notes/bp-extend-compute-model-attributes-b56bc093e8637bb4.yaml @@ -0,0 +1,14 @@ +--- +features: + - | + The compute model was extended with additional server attributes to provide + more detailed information about compute instances. These additions will + enable strategies to make more precise decisions by considering more server + placement constraints. The new attributes are ``flavor extra specs`` and + ``pinned availability zone``. Each new attribute depends on a minimal + microversion to be supported in nova and configured in the watcher + configuration, at ``nova_client`` section. Please refer to the nova api-ref + documentation for more details on which microversion is required: + https://docs.openstack.org/api-ref/compute/ + A new configuration option was added to allow the user to enable or disable + the extended attributes collection, which is disabled by default. diff --git a/test-requirements.txt b/test-requirements.txt index 005f0704e..f668bcc01 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,5 @@ coverage>=4.5.1 # Apache-2.0 +ddt>=1.2.1 # MIT freezegun>=0.3.10 # Apache-2.0 oslotest>=3.3.0 # Apache-2.0 testscenarios>=0.5.0 # Apache-2.0/BSD diff --git a/watcher/api/controllers/v1/data_model.py b/watcher/api/controllers/v1/data_model.py index a3c794ade..9e6d164f5 100644 --- a/watcher/api/controllers/v1/data_model.py +++ b/watcher/api/controllers/v1/data_model.py @@ -30,6 +30,22 @@ from watcher.common import policy from watcher.decision_engine import rpcapi +def hide_fields_in_newer_versions(obj): + """This method hides fields that were added in newer API versions. + + Certain node fields were introduced at certain API versions. + These fields are only made available when the request's API version + matches or exceeds the versions when these fields were introduced. + """ + if not utils.allow_list_extend_compute_model(): + # NOTE(dviroel): the content returned by the rpc is + # a list of dicts, so we need to remove the elements based + # on the api version. + for elem in obj.get('context', []): + elem.pop('server_pinned_az', None) + elem.pop('server_flavor_extra_specs', None) + + class DataModelController(rest.RestController): """REST controller for data model""" @@ -63,4 +79,5 @@ class DataModelController(rest.RestController): context, data_model_type, audit_uuid) + hide_fields_in_newer_versions(rpc_all_data_model) return rpc_all_data_model diff --git a/watcher/api/controllers/v1/utils.py b/watcher/api/controllers/v1/utils.py index 23cce2920..790160f74 100644 --- a/watcher/api/controllers/v1/utils.py +++ b/watcher/api/controllers/v1/utils.py @@ -203,3 +203,13 @@ def allow_skipped_action(): """ return pecan.request.version.minor >= ( versions.VERSIONS.MINOR_5_SKIPPED_ACTION.value) + + +def allow_list_extend_compute_model(): + """Check if it should list extended compute model attributes. + + Version 1.6 of the API added support to additional attributes + to the compute data model. + """ + return pecan.request.version.minor >= ( + versions.VERSIONS.MINOR_6_EXT_COMPUTE_MODEL.value) diff --git a/watcher/api/controllers/v1/versions.py b/watcher/api/controllers/v1/versions.py index f7dfbae77..ddb526959 100644 --- a/watcher/api/controllers/v1/versions.py +++ b/watcher/api/controllers/v1/versions.py @@ -24,7 +24,8 @@ class VERSIONS(enum.Enum): MINOR_3_DATAMODEL = 3 # v1.3: Add list datamodel API MINOR_4_WEBHOOK_API = 4 # v1.4: Add webhook trigger API MINOR_5_SKIPPED_ACTION = 5 # v1.5: Add skipped action support - MINOR_MAX_VERSION = 5 + MINOR_6_EXT_COMPUTE_MODEL = 6 # v1.6: Extend compute data model API + MINOR_MAX_VERSION = 6 # This is the version 1 API diff --git a/watcher/common/nova_helper.py b/watcher/common/nova_helper.py index 997414daa..e8719d8ca 100644 --- a/watcher/common/nova_helper.py +++ b/watcher/common/nova_helper.py @@ -43,6 +43,19 @@ class NovaHelper(object): self.cinder = self.osc.cinder() self.nova = self.osc.nova() self.glance = self.osc.glance() + self._is_pinned_az_available = None + + def is_pinned_az_available(self): + """Check if pinned AZ is available in GET /servers/detail response. + + :returns: True if is available, False otherwise. + """ + if self._is_pinned_az_available is None: + self._is_pinned_az_available = ( + api_versions.APIVersion( + version_str=CONF.nova_client.api_version) >= + api_versions.APIVersion(version_str='2.96')) + return self._is_pinned_az_available def get_compute_node_list(self): hypervisors = self.nova.hypervisors.list() diff --git a/watcher/conf/__init__.py b/watcher/conf/__init__.py index 12dcb35ae..337339e6d 100644 --- a/watcher/conf/__init__.py +++ b/watcher/conf/__init__.py @@ -36,6 +36,7 @@ from watcher.conf import grafana_translators from watcher.conf import ironic_client from watcher.conf import keystone_client from watcher.conf import maas_client +from watcher.conf import models from watcher.conf import monasca_client from watcher.conf import neutron_client from watcher.conf import nova_client @@ -59,6 +60,7 @@ applier.register_opts(CONF) decision_engine.register_opts(CONF) maas_client.register_opts(CONF) monasca_client.register_opts(CONF) +models.register_opts(CONF) nova_client.register_opts(CONF) glance_client.register_opts(CONF) gnocchi_client.register_opts(CONF) diff --git a/watcher/conf/models.py b/watcher/conf/models.py new file mode 100644 index 000000000..03671cea3 --- /dev/null +++ b/watcher/conf/models.py @@ -0,0 +1,40 @@ +# Copyright 2025 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +from oslo_config import cfg + +compute_model = cfg.OptGroup(name='compute_model', + title='Configuration Options for Compute Model', + help="Additional configuration options for the " + "compute model.") + +COMPUTE_MODEL_OPTS = [ + cfg.BoolOpt( + 'enable_extended_attributes', + default=False, + help="Enable the collection of compute model extended attributes. " + "Note that some attributes require a more recent api " + "microversion to be configured in nova_client section." + ), +] + + +def register_opts(conf): + conf.register_group(compute_model) + conf.register_opts(COMPUTE_MODEL_OPTS, group=compute_model) + + +def list_opts(): + return [(compute_model, COMPUTE_MODEL_OPTS)] diff --git a/watcher/decision_engine/model/collector/nova.py b/watcher/decision_engine/model/collector/nova.py index 154b8b94d..b65631031 100644 --- a/watcher/decision_engine/model/collector/nova.py +++ b/watcher/decision_engine/model/collector/nova.py @@ -471,13 +471,22 @@ class NovaModelBuilder(base.BaseModelBuilder): "state": getattr(instance, "OS-EXT-STS:vm_state"), "metadata": instance.metadata, "project_id": instance.tenant_id, - "locked": instance.locked} + "locked": instance.locked, + # NOTE(dviroel): new attributes are updated if + # extended feature is enabled + "flavor_extra_specs": {}, + "pinned_az": "", + } + + if self.model.extended_attributes_enabled: + instance_attributes.update({ + "flavor_extra_specs": flavor["extra_specs"], + }) + if self.nova_helper.is_pinned_az_available(): + instance_attributes["pinned_az"] = ( + instance.pinned_availability_zone + ) - # node_attributes = dict() - # node_attributes["layer"] = "virtual" - # node_attributes["category"] = "compute" - # node_attributes["type"] = "compute" - # node_attributes["attributes"] = instance_attributes return element.Instance(**instance_attributes) def _merge_compute_scope(self, compute_scope): diff --git a/watcher/decision_engine/model/element/instance.py b/watcher/decision_engine/model/element/instance.py index f2e9aeadf..bb892e874 100644 --- a/watcher/decision_engine/model/element/instance.py +++ b/watcher/decision_engine/model/element/instance.py @@ -54,6 +54,9 @@ class Instance(compute_resource.ComputeResource): "metadata": wfields.JsonField(), "project_id": wfields.UUIDField(), "locked": wfields.BooleanField(default=False), + # New fields for extended compute model + "pinned_az": wfields.StringField(default=""), + "flavor_extra_specs": wfields.JsonField(default={}), } def accept(self, visitor): diff --git a/watcher/decision_engine/model/model_root.py b/watcher/decision_engine/model/model_root.py index c05101ec8..4a7b740e5 100644 --- a/watcher/decision_engine/model/model_root.py +++ b/watcher/decision_engine/model/model_root.py @@ -17,9 +17,11 @@ Openstack implementation of the cluster graph. """ import ast + from lxml import etree # nosec: B410 import networkx as nx from oslo_concurrency import lockutils +from oslo_config import cfg from oslo_log import log from watcher._i18n import _ @@ -28,6 +30,7 @@ from watcher.decision_engine.model import base from watcher.decision_engine.model import element LOG = log.getLogger(__name__) +CONF = cfg.CONF class ModelRoot(nx.DiGraph, base.Model): @@ -36,12 +39,24 @@ class ModelRoot(nx.DiGraph, base.Model): def __init__(self, stale=False): super(ModelRoot, self).__init__() self.stale = stale + self._extended_attributes_enabled = None def __nonzero__(self): return not self.stale __bool__ = __nonzero__ + @property + def extended_attributes_enabled(self): + if self._extended_attributes_enabled is None: + self._extended_attributes_enabled = ( + CONF.compute_model.enable_extended_attributes) + return self._extended_attributes_enabled + + @extended_attributes_enabled.setter + def extended_attributes_enabled(self, value): + self._extended_attributes_enabled = value + @staticmethod def assert_node(obj): if not isinstance(obj, element.ComputeNode): diff --git a/watcher/decision_engine/model/notification/nova.py b/watcher/decision_engine/model/notification/nova.py index eb776f2f1..7a11645bd 100644 --- a/watcher/decision_engine/model/notification/nova.py +++ b/watcher/decision_engine/model/notification/nova.py @@ -16,8 +16,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from novaclient import api_versions import os_resource_classes as orc from oslo_log import log + from watcher.common import exception from watcher.common import nova_helper from watcher.common import placement_helper @@ -72,7 +74,7 @@ class NovaNotification(base.NotificationEndpoint): return instance def update_instance(self, instance, data): - n_version = float(data['nova_object.version']) + n_version = api_versions.APIVersion(data['nova_object.version']) instance_data = data['nova_object.data'] instance_flavor_data = instance_data['flavor']['nova_object.data'] @@ -91,12 +93,20 @@ class NovaNotification(base.NotificationEndpoint): 'vcpus': num_cores, 'disk': disk_gb, 'metadata': instance_metadata, - 'project_id': instance_data['tenant_id'] + 'project_id': instance_data['tenant_id'], }) + # locked was added in nova notification payload version 1.1 - if n_version > 1.0: + if n_version > api_versions.APIVersion(version_str='1.0'): instance.update({'locked': instance_data['locked']}) + # NOTE(dviroel): extra_specs can change due to a resize operation. + # 'extra_specs' was added in nova notification payload version 1.2 + if (n_version > api_versions.APIVersion(version_str='1.1') and + self.cluster_data_model.extended_attributes_enabled): + extra_specs = instance_flavor_data.get("extra_specs", {}) + instance.update({'flavor_extra_specs': extra_specs}) + try: node = self.get_or_create_node(instance_data['host']) except exception.ComputeNodeNotFound as exc: diff --git a/watcher/tests/api/v1/test_data_model.py b/watcher/tests/api/v1/test_data_model.py index d6e59e490..d2f4e9bee 100644 --- a/watcher/tests/api/v1/test_data_model.py +++ b/watcher/tests/api/v1/test_data_model.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import ddt from unittest import mock from http import HTTPStatus @@ -30,15 +31,16 @@ class TestListDataModel(api_base.FunctionalTest): super(TestListDataModel, self).setUp() p_dcapi = mock.patch.object(deapi, 'DecisionEngineAPI') self.mock_dcapi = p_dcapi.start() - self.mock_dcapi().get_data_model_info.return_value = \ - 'fake_response_value' + self.fake_response = {'context': [{'server_uuid': 'fake_uuid'}]} + self.mock_dcapi().get_data_model_info.return_value = ( + self.fake_response) self.addCleanup(p_dcapi.stop) def test_get_all(self): response = self.get_json( '/data_model/?data_model_type=compute', headers={'OpenStack-API-Version': 'infra-optim 1.3'}) - self.assertEqual('fake_response_value', response) + self.assertEqual(self.fake_response, response) def test_get_all_not_acceptable(self): response = self.get_json( @@ -48,6 +50,7 @@ class TestListDataModel(api_base.FunctionalTest): self.assertEqual(HTTPStatus.NOT_ACCEPTABLE, response.status_int) +@ddt.ddt class TestListDataModelResponse(api_base.FunctionalTest): NODE_FIELDS_1_3 = [ @@ -67,6 +70,13 @@ class TestListDataModelResponse(api_base.FunctionalTest): 'node_uuid' ] + # Map of API version to expected node fields + NODE_FIELDS_MAP = { + '1.3': NODE_FIELDS_1_3, + '1.6': NODE_FIELDS_1_3, + 'latest': NODE_FIELDS_1_3, + } + SERVER_FIELDS_1_3 = [ 'server_watcher_exclude', 'server_name', @@ -80,8 +90,17 @@ class TestListDataModelResponse(api_base.FunctionalTest): 'server_uuid' ] - NODE_FIELDS_LATEST = NODE_FIELDS_1_3 - SERVER_FIELDS_LATEST = SERVER_FIELDS_1_3 + SERVER_FIELDS_1_6 = [ + 'server_pinned_az', + 'server_flavor_extra_specs' + ] + SERVER_FIELDS_1_3 + + # Map of API version to expected server fields + SERVER_FIELDS_MAP = { + '1.3': SERVER_FIELDS_1_3, + '1.6': SERVER_FIELDS_1_6, + 'latest': SERVER_FIELDS_1_6, + } def setUp(self): super(TestListDataModelResponse, self).setUp() @@ -89,36 +108,39 @@ class TestListDataModelResponse(api_base.FunctionalTest): self.mock_dcapi = p_dcapi.start() self.addCleanup(p_dcapi.stop) - def test_model_list_compute_no_instance(self): + @ddt.data("1.3", "1.6", versions.max_version_string()) + def test_model_list_compute_no_instance(self, version): fake_cluster = faker_cluster_state.FakerModelCollector() model = fake_cluster.generate_scenario_11_with_1_node_no_instance() get_model_resp = {'context': model.to_list()} self.mock_dcapi().get_data_model_info.return_value = get_model_resp - infra_max_version = 'infra-optim ' + versions.max_version_string() + infra_max_version = 'infra-optim ' + version response = self.get_json( '/data_model/?data_model_type=compute', headers={'OpenStack-API-Version': infra_max_version}) server_info = response.get("context")[0] - expected_keys = self.NODE_FIELDS_LATEST + expected_keys = self.NODE_FIELDS_MAP[version] self.assertEqual(len(response.get("context")), 1) self.assertEqual(set(expected_keys), set(server_info.keys())) - def test_model_list_compute_with_instances(self): + @ddt.data("1.3", "1.6", versions.max_version_string()) + def test_model_list_compute_with_instances(self, version): fake_cluster = faker_cluster_state.FakerModelCollector() model = fake_cluster.generate_scenario_11_with_2_nodes_2_instances() get_model_resp = {'context': model.to_list()} self.mock_dcapi().get_data_model_info.return_value = get_model_resp - infra_max_version = 'infra-optim ' + versions.max_version_string() + infra_max_version = 'infra-optim ' + version response = self.get_json( '/data_model/?data_model_type=compute', headers={'OpenStack-API-Version': infra_max_version}) server_info = response.get("context")[0] - expected_keys = self.NODE_FIELDS_LATEST + self.SERVER_FIELDS_LATEST + expected_keys = (self.NODE_FIELDS_MAP[version] + + self.SERVER_FIELDS_MAP[version]) self.assertEqual(len(response.get("context")), 2) self.assertEqual(set(expected_keys), set(server_info.keys())) diff --git a/watcher/tests/decision_engine/cluster/test_nova_cdmc.py b/watcher/tests/decision_engine/cluster/test_nova_cdmc.py index 749e1c068..cb76d0197 100644 --- a/watcher/tests/decision_engine/cluster/test_nova_cdmc.py +++ b/watcher/tests/decision_engine/cluster/test_nova_cdmc.py @@ -16,16 +16,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import ddt import os_resource_classes as orc from unittest import mock from watcher.common import nova_helper from watcher.common import placement_helper from watcher.decision_engine.model.collector import nova +from watcher.decision_engine.model import model_root from watcher.tests import base from watcher.tests import conf_fixture +@ddt.ddt class TestNovaClusterDataModelCollector(base.TestCase): def setUp(self): @@ -35,8 +38,17 @@ class TestNovaClusterDataModelCollector(base.TestCase): @mock.patch('keystoneclient.v3.client.Client', mock.Mock()) @mock.patch.object(placement_helper, 'PlacementHelper') @mock.patch.object(nova_helper, 'NovaHelper') - def test_nova_cdmc_execute(self, m_nova_helper_cls, + @mock.patch.object(model_root.ModelRoot, 'extended_attributes_enabled', + new_callable=mock.PropertyMock) + @ddt.data(False, True) + def test_nova_cdmc_execute(self, + extended_attr_enabled, + m_extended_attr_enabled, + m_nova_helper_cls, m_placement_helper_cls): + # Set the extended_attributes_enabled configuration value + m_extended_attr_enabled.return_value = extended_attr_enabled + m_placement_helper = mock.Mock(name="placement_helper") m_placement_helper.get_inventories.return_value = { orc.VCPU: { @@ -117,12 +129,15 @@ class TestNovaClusterDataModelCollector(base.TestCase): fake_instance = mock.Mock( id='ef500f7e-dac8-470f-960c-169486fce71b', name='fake_instance', - flavor={'ram': 333, 'disk': 222, 'vcpus': 4, 'id': 1}, + flavor={'ram': 333, 'disk': 222, 'vcpus': 4, 'id': 1, + 'extra_specs': {'hw_rng:allowed': 'True'}}, metadata={'hi': 'hello'}, tenant_id='ff560f7e-dbc8-771f-960c-164482fce21b', + pinned_availability_zone='nova', ) setattr(fake_instance, 'OS-EXT-STS:vm_state', 'VM_STATE') setattr(fake_instance, 'name', 'fake_instance') + # Returns the hypervisors with details (service) but no servers. m_nova_helper.get_compute_node_list.return_value = [fake_compute_node] # Returns the hypervisor with servers and details (service). @@ -161,12 +176,21 @@ class TestNovaClusterDataModelCollector(base.TestCase): vcpus_total = (node.vcpus-node.vcpu_reserved)*node.vcpu_ratio self.assertEqual(node.vcpu_capacity, vcpus_total) + if extended_attr_enabled: + self.assertEqual('nova', instance.pinned_az) + self.assertEqual({'hw_rng:allowed': 'True'}, + instance.flavor_extra_specs) + else: + self.assertEqual('', instance.pinned_az) + self.assertEqual({}, instance.flavor_extra_specs) + m_nova_helper.get_compute_node_by_name.assert_called_once_with( minimal_node['hypervisor_hostname'], servers=True, detailed=True) m_nova_helper.get_instance_list.assert_called_once_with( filters={'host': fake_compute_node.service['host']}, limit=1) +@ddt.ddt class TestNovaModelBuilder(base.TestCase): @mock.patch.object(nova_helper, 'NovaHelper', mock.MagicMock()) @@ -180,11 +204,15 @@ class TestNovaModelBuilder(base.TestCase): tenant_id='ff560f7e-dbc8-771f-960c-164482fce21b') setattr(inst1, 'OS-EXT-STS:vm_state', 'deleted') setattr(inst1, 'name', 'instance1') + setattr(inst1, 'pinned_availability_zone', 'nova') + inst2 = mock.MagicMock( id='ef500f7e-dac8-470f-960c-169486fce722', tenant_id='ff560f7e-dbc8-771f-960c-164482fce21b') setattr(inst2, 'OS-EXT-STS:vm_state', 'active') setattr(inst2, 'name', 'instance2') + setattr(inst2, 'pinned_availability_zone', 'nova') + mock_instances = [inst1, inst2] model_builder.nova_helper.get_instance_list.return_value = ( mock_instances) @@ -203,6 +231,49 @@ class TestNovaModelBuilder(base.TestCase): model_builder.nova_helper.get_instance_list.assert_called_with( filters={'host': mock_host}, limit=-1) + @ddt.unpack + # unpacks (extended_attr_enabled, pinned_az_available) + @ddt.data((True, True), (True, False), (False, True)) + @mock.patch.object(nova_helper.NovaHelper, 'is_pinned_az_available') + def test__build_instance_node_extended_fields(self, + extended_attr_enabled, + pinned_az_available, + m_is_pinned_az_available): + model_builder = nova.NovaModelBuilder(osc=mock.MagicMock()) + model_builder.model = mock.MagicMock() + model_builder.model.extended_attributes_enabled = extended_attr_enabled + m_is_pinned_az_available.return_value = pinned_az_available + + inst1 = mock.MagicMock( + id='ef500f7e-dac8-470f-960c-169486fce711', + tenant_id='ff560f7e-dbc8-771f-960c-164482fce21b', + flavor={'ram': 333, 'disk': 222, 'vcpus': 4, 'id': 1, + 'extra_specs': {'hw_rng:allowed': 'True'}},) + setattr(inst1, 'OS-EXT-STS:vm_state', 'active') + setattr(inst1, 'name', 'instance1') + setattr(inst1, 'pinned_availability_zone', 'nova') + + fake_instance = model_builder._build_instance_node(inst1) + + self.assertEqual(fake_instance.uuid, + 'ef500f7e-dac8-470f-960c-169486fce711') + self.assertEqual(fake_instance.name, 'instance1') + self.assertEqual(fake_instance.state, 'active') + self.assertEqual(fake_instance.memory, 333) + self.assertEqual(fake_instance.disk, 222) + self.assertEqual(fake_instance.vcpus, 4) + self.assertEqual(fake_instance.project_id, + 'ff560f7e-dbc8-771f-960c-164482fce21b') + + if extended_attr_enabled: + expected_pinned_az = 'nova' if pinned_az_available else '' + self.assertEqual(expected_pinned_az, fake_instance.pinned_az) + self.assertEqual({'hw_rng:allowed': 'True'}, + fake_instance.flavor_extra_specs) + else: + self.assertEqual('', fake_instance.pinned_az) + self.assertEqual({}, fake_instance.flavor_extra_specs) + def test_check_model(self): """Initialize collector ModelBuilder and test check model""" diff --git a/watcher/tests/decision_engine/model/notification/data/instance-update-2-1.json b/watcher/tests/decision_engine/model/notification/data/instance-update-2-1.json new file mode 100644 index 000000000..3b5f92b0b --- /dev/null +++ b/watcher/tests/decision_engine/model/notification/data/instance-update-2-1.json @@ -0,0 +1,90 @@ +{ + "event_type": "instance.update", + "payload": { + "nova_object.data": { + "action_initiator_project": "6f70656e737461636b20342065766572", + "action_initiator_user": "fake", + "architecture": "x86_64", + "audit_period": { + "nova_object.data": { + "audit_period_beginning": "2012-10-01T00:00:00Z", + "audit_period_ending": "2012-10-29T13:42:11Z" + }, + "nova_object.name": "AuditPeriodPayload", + "nova_object.namespace": "nova", + "nova_object.version": "1.0" + }, + "auto_disk_config": "MANUAL", + "availability_zone": "nova", + "block_devices": [], + "created_at": "2012-10-29T13:42:11Z", + "deleted_at": null, + "display_description": "some-server", + "display_name": "some-server", + "host": "compute", + "host_name": "some-server", + "image_uuid": "155d900f-4e14-4e4c-a73d-069cbf4541e6", + "ip_addresses": [], + "kernel_id": "", + "key_name": "my-key", + "launched_at": null, + "locked": false, + "locked_reason": null, + "metadata": {}, + "node": "fake-mini", + "old_display_name": null, + "os_type": null, + "power_state": "pending", + "progress": 0, + "ramdisk_id": "", + "request_id": "req-5b6c791d-5709-4f36-8fbe-c3e02869e35d", + "reservation_id": "r-npxv0e40", + "shares": [], + "state": "paused", + "state_update": { + "nova_object.data": { + "new_task_state": null, + "old_state": null, + "old_task_state": null, + "state": "active"}, + "nova_object.name": "InstanceStateUpdatePayload", + "nova_object.namespace": "nova", + "nova_object.version": "1.0"}, + "tags": [], + "task_state": "scheduling", + "tenant_id": "6f70656e737461636b20342065766572", + "flavor": { + "nova_object.data": { + "description": null, + "disabled": false, + "ephemeral_gb": 0, + "extra_specs": { + "hw:watchdog_action": "disabled" + }, + "flavorid": "a22d5517-147c-4147-a0d1-e698df5cd4e3", + "is_public": true, + "memory_mb": 512, + "name": "test_flavor", + "projects": null, + "root_gb": 1, + "rxtx_factor": 1.0, + "swap": 0, + "vcpu_weight": 0, + "vcpus": 1 + }, + "nova_object.name": "FlavorPayload", + "nova_object.namespace": "nova", + "nova_object.version": "1.4" + }, + "terminated_at": null, + "updated_at": null, + "user_id": "fake", + "uuid": "73b09e16-35b7-4922-804e-e8f5d9b740fc" + }, + "nova_object.name": "InstanceUpdatePayload", + "nova_object.namespace": "nova", + "nova_object.version": "2.1" + }, + "priority": "INFO", + "publisher_id": "nova-compute:compute" +} diff --git a/watcher/tests/decision_engine/model/notification/test_nova_notifications.py b/watcher/tests/decision_engine/model/notification/test_nova_notifications.py index d5b9f6e38..1d94f8fb2 100644 --- a/watcher/tests/decision_engine/model/notification/test_nova_notifications.py +++ b/watcher/tests/decision_engine/model/notification/test_nova_notifications.py @@ -16,6 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import ddt import os import os_resource_classes as orc from unittest import mock @@ -110,6 +111,7 @@ class TestReceiveNovaNotifications(NotificationTestCase): expected_message, self.FAKE_METADATA) +@ddt.ddt class TestNovaNotifications(NotificationTestCase): FAKE_METADATA = {'message_id': None, 'timestamp': None} @@ -285,8 +287,10 @@ class TestNovaNotifications(NotificationTestCase): @mock.patch.object(placement_helper, 'PlacementHelper') @mock.patch.object(nova_helper, "NovaHelper") + @ddt.data(False, True) def test_nova_instance_update_notfound_still_creates( - self, m_nova_helper_cls, m_placement_helper): + self, extended_attr_enabled, m_nova_helper_cls, + m_placement_helper): mock_placement = mock.Mock(name="placement_helper") mock_placement.get_inventories.return_value = dict() mock_placement.get_usages_for_resource_provider.return_value = { @@ -313,6 +317,10 @@ class TestNovaNotifications(NotificationTestCase): name='m_nova_helper') compute_model = self.fake_cdmc.generate_scenario_3_with_2_nodes() self.fake_cdmc.cluster_data_model = compute_model + # Set the extended_attributes_enabled configuration value + self.fake_cdmc.cluster_data_model.extended_attributes_enabled = ( + extended_attr_enabled) + handler = novanotification.VersionedNotification(self.fake_cdmc) instance0_uuid = '9966d6bd-a45c-4e1c-9d57-3054899a3ec7' @@ -333,6 +341,10 @@ class TestNovaNotifications(NotificationTestCase): self.assertEqual(1, instance0.vcpus) self.assertEqual(1, instance0.disk) self.assertEqual(512, instance0.memory) + # NOTE(dviroel): pinned_az is not yet available in nova notifications + # and flavor_extra_specs is not available in nova notifications 1.0 + self.assertEqual({}, instance0.flavor_extra_specs) + self.assertEqual('', instance0.pinned_az) m_get_compute_node_by_hostname.assert_called_once_with('Node_2') node_2 = compute_model.get_node_by_name('Node_2') @@ -380,7 +392,10 @@ class TestNovaNotifications(NotificationTestCase): @mock.patch.object(placement_helper, 'PlacementHelper') @mock.patch.object(nova_helper, 'NovaHelper') - def test_nova_instance_create(self, m_nova_helper_cls, + @ddt.data(False, True) + def test_nova_instance_create(self, + extended_attr_enabled, + m_nova_helper_cls, m_placement_helper): mock_placement = mock.Mock(name="placement_helper") mock_placement.get_inventories.return_value = dict() @@ -410,6 +425,10 @@ class TestNovaNotifications(NotificationTestCase): compute_model = self.fake_cdmc.generate_scenario_3_with_2_nodes() self.fake_cdmc.cluster_data_model = compute_model + # Set the extended_attributes_enabled configuration value + self.fake_cdmc.cluster_data_model.extended_attributes_enabled = ( + extended_attr_enabled) + handler = novanotification.VersionedNotification(self.fake_cdmc) instance0_uuid = 'c03c0bf9-f46e-4e4f-93f1-817568567ee2' @@ -440,6 +459,14 @@ class TestNovaNotifications(NotificationTestCase): self.assertEqual(1, instance0.vcpus) self.assertEqual(1, instance0.disk) self.assertEqual(512, instance0.memory) + if extended_attr_enabled: + self.assertEqual({'hw:watchdog_action': 'disabled'}, + instance0.flavor_extra_specs) + # NOTE(dviroel): pinned_az is not available in nova notifications + self.assertEqual('', instance0.pinned_az) + else: + self.assertEqual({}, instance0.flavor_extra_specs) + self.assertEqual('', instance0.pinned_az) def test_nova_instance_delete_end(self): compute_model = self.fake_cdmc.generate_scenario_3_with_2_nodes() @@ -564,6 +591,34 @@ class TestNovaNotifications(NotificationTestCase): self.assertFalse(instance0.locked) + @ddt.data(False, True) + def test_nova_instance_update_extra_specs(self, extended_attr_enabled): + compute_model = self.fake_cdmc.generate_scenario_3_with_2_nodes() + self.fake_cdmc.cluster_data_model = compute_model + self.fake_cdmc.cluster_data_model.extended_attributes_enabled = ( + extended_attr_enabled) + handler = novanotification.VersionedNotification(self.fake_cdmc) + + instance0_uuid = '73b09e16-35b7-4922-804e-e8f5d9b740fc' + instance0 = compute_model.get_instance_by_uuid(instance0_uuid) + + message = self.load_message('instance-update-2-1.json') + + self.assertEqual({}, instance0.flavor_extra_specs) + + handler.info( + ctxt=self.context, + publisher_id=message['publisher_id'], + event_type=message['event_type'], + payload=message['payload'], + metadata=self.FAKE_METADATA, + ) + if extended_attr_enabled: + self.assertEqual({'hw:watchdog_action': 'disabled'}, + instance0.flavor_extra_specs) + else: + self.assertEqual({}, instance0.flavor_extra_specs) + def test_nova_instance_pause(self): compute_model = self.fake_cdmc.generate_scenario_3_with_2_nodes() self.fake_cdmc.cluster_data_model = compute_model @@ -685,14 +740,22 @@ class TestNovaNotifications(NotificationTestCase): self.assertEqual(element.InstanceState.ACTIVE.value, instance0.state) - def test_instance_resize_confirm_end(self): + @ddt.data(False, True) + def test_instance_resize_confirm_end(self, extended_attr_enabled): compute_model = self.fake_cdmc.generate_scenario_3_with_2_nodes() self.fake_cdmc.cluster_data_model = compute_model + self.fake_cdmc.cluster_data_model.extended_attributes_enabled = ( + extended_attr_enabled) + handler = novanotification.VersionedNotification(self.fake_cdmc) instance0_uuid = '73b09e16-35b7-4922-804e-e8f5d9b740fc' instance0 = compute_model.get_instance_by_uuid(instance0_uuid) node = compute_model.get_node_by_instance_uuid(instance0_uuid) + self.assertEqual('fa69c544-906b-4a6a-a9c6-c1f7a8078c73', node.uuid) + # NOTE(dviroel): extra_specs are empty generated scenario + self.assertEqual({}, instance0.flavor_extra_specs) + message = self.load_message( 'instance-resize_confirm-end.json') handler.info( @@ -706,6 +769,11 @@ class TestNovaNotifications(NotificationTestCase): self.assertEqual('fa69c544-906b-4a6a-a9c6-c1f7a8078c73', node.uuid) self.assertEqual(element.InstanceState.ACTIVE.value, instance0.state) + expected_extra_specs = { + 'hw:watchdog_action': 'disabled' + } if extended_attr_enabled else {} + self.assertEqual(expected_extra_specs, instance0.flavor_extra_specs) + def test_nova_instance_restore_end(self): compute_model = self.fake_cdmc.generate_scenario_3_with_2_nodes() self.fake_cdmc.cluster_data_model = compute_model @@ -846,6 +914,7 @@ class TestNovaNotifications(NotificationTestCase): def test_fake_instance_create(self): self.fake_cdmc.cluster_data_model = mock.Mock() + self.fake_cdmc.cluster_data_model.extended_attributes_enabled = False handler = novanotification.VersionedNotification(self.fake_cdmc) message = self.load_message('instance-create-end.json') diff --git a/watcher/tests/decision_engine/model/test_element.py b/watcher/tests/decision_engine/model/test_element.py index 91dbb737f..d7a060902 100644 --- a/watcher/tests/decision_engine/model/test_element.py +++ b/watcher/tests/decision_engine/model/test_element.py @@ -60,6 +60,16 @@ class TestElement(base.TestCase): 'vcpus': 222, 'disk': 333, })), + ("Instance_with_extended_fields", dict( + cls=element.Instance, + data={ + 'uuid': 'FAKE_UUID', + 'state': 'state', + 'vcpus': 222, + 'disk': 333, + 'pinned_az': 'nova', + 'flavor_extra_specs': {"spec1": "value1", "spec2": "value2"}, + })), ] def test_as_xml_element(self):