From d84f8c50f5f52207ebaf7824df0c585ec10f5a9c Mon Sep 17 00:00:00 2001 From: Dantali0n Date: Sat, 20 Apr 2019 12:42:25 +0200 Subject: [PATCH] Resolve problems with audit scope and add tests This resolves problems with the audit scope such as the scope being ignored, the scope not merging due to a type in .append, change update into .add method when adding single elements to a set and making the access of dict keys and values as lists work in python 3.7. All these methods from the model builder now have tests to prevent regressions. Co-Authored-By: Canwei Li Change-Id: I287763d5e426ff860aefabc4a1f3fe3f51accd76 --- .../decision_engine/model/collector/nova.py | 176 +----------------- .../cluster/test_model_builder.py | 175 +++++++++++++++++ 2 files changed, 185 insertions(+), 166 deletions(-) create mode 100644 watcher/tests/decision_engine/cluster/test_model_builder.py diff --git a/watcher/decision_engine/model/collector/nova.py b/watcher/decision_engine/model/collector/nova.py index 60a4f7fc0..95c10a989 100644 --- a/watcher/decision_engine/model/collector/nova.py +++ b/watcher/decision_engine/model/collector/nova.py @@ -208,10 +208,9 @@ class ModelBuilder(object): self.osc = osc self.model = None self.model_scope = dict() + self.no_model_scope_flag = False self.nova = osc.nova() self.nova_helper = nova_helper.NovaHelper(osc=self.osc) - # self.neutron = osc.neutron() - # self.cinder = osc.cinder() def _collect_aggregates(self, host_aggregates, _nodes): aggregate_list = self.nova_helper.get_aggregate_list() @@ -237,7 +236,7 @@ class ModelBuilder(object): include_all_nodes = True for service in service_list: if service.zone in zone_names or include_all_nodes: - _nodes.update(service.host) + _nodes.add(service.host) def _add_physical_layer(self): """Add the physical layer of the graph. @@ -254,6 +253,7 @@ class ModelBuilder(object): self._collect_zones(availability_zones, compute_nodes) if not compute_nodes: + self.no_model_scope_flag = True all_nodes = self.nova_helper.get_compute_node_list() compute_nodes = set( [node.hypervisor_hostname for node in all_nodes]) @@ -315,47 +315,6 @@ class ModelBuilder(object): # node_attributes) return compute_node - # def _build_network_compute_node(self, base_node): - # attributes = {} - # net_node = self._build_node("physical", "network", "NIC", attributes) - # net_id = "{}_network".format(base_node) - # return net_id, net_node - - # def build_disk_compute_node(self, base_node, compute): - # # Build disk node attributes. - # disk_attributes = { - # "size_gb": compute.local_gb, - # "used_gb": compute.local_gb_used, - # "available_gb": compute.free_disk_gb} - # disk_node = self._build_node("physical", "storage", "disk", - # disk_attributes) - # disk_id = "{}_disk".format(base_node) - # return disk_id, disk_node - - # def build_memory_compute_node(self, base_node, compute): - # # Build memory node attributes. - # memory_attrs = {"size_mb": compute.memory_mb, - # "used_mb": compute.memory_mb_used, - # "available_mb": compute.free_ram_mb} - # memory_node = self._build_node("physical", "memory", "memory", - # memory_attrs) - # memory_id = "{}_memory".format(base_node) - # return memory_id, memory_node - - # def build_cpu_compute_node(self, base_node, compute): - # # Build memory node attributes. - # cpu_attributes = {"vcpus": compute.vcpus, - # "vcpus_used": compute.vcpus_used, - # "info": jsonutils.loads(compute.cpu_info)} - # cpu_node = self._build_node("physical", "cpu", "cpu", cpu_attributes) - # cpu_id = "{}_cpu".format(base_node) - # return cpu_id, cpu_node - - # @staticmethod - # def _build_node(layer, category, node_type, attributes): - # return {"layer": layer, "category": category, "type": node_type, - # "attributes": attributes} - def _add_virtual_layer(self): """Add the virtual layer to the graph. @@ -439,137 +398,23 @@ class ModelBuilder(object): # node_attributes["attributes"] = instance_attributes return element.Instance(**instance_attributes) - # def _add_virtual_storage(self): - # try: - # volumes = self.cinder.volumes.list() - # except Exception: - # return - # for volume in volumes: - # volume_id, volume_node = self._build_storage_node(volume) - # self.add_node(volume_id, volume_node) - # host = self._get_volume_host_id(volume_node) - # self.add_edge(volume_id, host) - # # Add connections to an instance. - # if volume_node['attributes']['attachments']: - # for attachment in volume_node['attributes']['attachments']: - # self.add_edge(volume_id, attachment['server_id'], - # label='ATTACHED_TO') - # volume_node['attributes'].pop('attachments') - - # def _add_virtual_network(self): - # try: - # routers = self.neutron.list_routers() - # except Exception: - # return - - # for network in self.neutron.list_networks()['networks']: - # self.add_node(*self._build_network(network)) - - # for router in routers['routers']: - # self.add_node(*self._build_router(router)) - - # router_interfaces, _, compute_ports = self._group_ports() - # for router_interface in router_interfaces: - # interface = self._build_router_interface(router_interface) - # router_interface_id = interface[0] - # router_interface_node = interface[1] - # router_id = interface[2] - # self.add_node(router_interface_id, router_interface_node) - # self.add_edge(router_id, router_interface_id) - # network_id = router_interface_node['attributes']['network_id'] - # self.add_edge(router_interface_id, network_id) - - # for compute_port in compute_ports: - # cp_id, cp_node, instance_id = self._build_compute_port_node( - # compute_port) - # self.add_node(cp_id, cp_node) - # self.add_edge(cp_id, vm_id) - # net_id = cp_node['attributes']['network_id'] - # self.add_edge(net_id, cp_id) - # # Connect port to physical node - # phys_net_node = "{}_network".format(cp_node['attributes'] - # ['binding:host_id']) - # self.add_edge(cp_id, phys_net_node) - - # def _get_volume_host_id(self, volume_node): - # host = volume_node['attributes']['os-vol-host-attr:host'] - # if host.find('@') != -1: - # host = host.split('@')[0] - # elif host.find('#') != -1: - # host = host.split('#')[0] - # return "{}_disk".format(host) - - # def _build_storage_node(self, volume_obj): - # volume = volume_obj.__dict__ - # volume["name"] = volume["id"] - # volume.pop("id") - # volume.pop("manager") - # node = self._build_node("virtual", "storage", 'volume', volume) - # return volume["name"], node - - # def _build_compute_port_node(self, compute_port): - # compute_port["name"] = compute_port["id"] - # compute_port.pop("id") - # nde_type = "{}_port".format( - # compute_port["device_owner"].split(":")[0]) - # compute_port.pop("device_owner") - # device_id = compute_port["device_id"] - # compute_port.pop("device_id") - # node = self._build_node("virtual", "network", nde_type, compute_port) - # return compute_port["name"], node, device_id - - # def _group_ports(self): - # router_interfaces = [] - # floating_ips = [] - # compute_ports = [] - # interface_types = ["network:router_interface", - # 'network:router_gateway'] - - # for port in self.neutron.list_ports()['ports']: - # if port['device_owner'] in interface_types: - # router_interfaces.append(port) - # elif port['device_owner'].startswith('compute:'): - # compute_ports.append(port) - # elif port['device_owner'] == 'network:floatingip': - # floating_ips.append(port) - - # return router_interfaces, floating_ips, compute_ports - - # def _build_router_interface(self, interface): - # interface["name"] = interface["id"] - # interface.pop("id") - # node_type = interface["device_owner"].split(":")[1] - # node = self._build_node("virtual", "network", node_type, interface) - # return interface["name"], node, interface["device_id"] - - # def _build_router(self, router): - # router_attrs = {"uuid": router['id'], - # "name": router['name'], - # "state": router['status']} - # node = self._build_node('virtual', 'network', 'router', router_attrs) - # return str(router['id']), node - - # def _build_network(self, network): - # node = self._build_node('virtual', 'network', 'network', network) - # return network['id'], node - def _merge_compute_scope(self, compute_scope): model_keys = self.model_scope.keys() update_flag = False role_keys = ("host_aggregates", "availability_zones") for role in compute_scope: - role_key = role.keys()[0] + role_key = list(role.keys())[0] if role_key not in role_keys: continue - role_values = role.values()[0] + role_values = list(role.values())[0] if role_key in model_keys: for value in role_values: if value not in self.model_scope[role_key]: - self.model_scope[role_key].sppend(value) + self.model_scope[role_key].append(value) update_flag = True else: - self.self.model_scope[role_key] = role_values + self.model_scope[role_key] = role_values update_flag = True return update_flag @@ -581,10 +426,9 @@ class ModelBuilder(object): compute_scope = _scope['compute'] break - if self.model_scope: - if model_scope: - if compute_scope: - update_flag = self._merge_compute_scope(compute_scope) + if self.no_model_scope_flag is False: + if compute_scope: + update_flag = self._merge_compute_scope(compute_scope) else: self.model_scope = dict() update_flag = True diff --git a/watcher/tests/decision_engine/cluster/test_model_builder.py b/watcher/tests/decision_engine/cluster/test_model_builder.py new file mode 100644 index 000000000..147edc83a --- /dev/null +++ b/watcher/tests/decision_engine/cluster/test_model_builder.py @@ -0,0 +1,175 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2019 European Organization for Nuclear Research (CERN) +# +# Authors: Corne Lukken +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +from oslo_config import cfg +from oslo_log import log + +from watcher.common import nova_helper +from watcher.decision_engine.model.collector import nova +from watcher.tests import base + +CONF = cfg.CONF +LOG = log.getLogger(__name__) + + +class TestModelBuilder(base.BaseTestCase): + """Test the collector ModelBuilder + + Objects under test are preceded with t_ and mocked objects are preceded + with m_ , additionally, patched objects are preceded with p_ no object + under test should be created in setUp this can influence the results. + """ + + def setUp(self): + super(TestModelBuilder, self).setUp() + + def test_check_model(self): + """Initialize collector ModelBuilder and test check model""" + + m_scope = [{"compute": [ + {"host_aggregates": [{"id": 5}]}, + {"availability_zones": [{"name": "av_a"}]} + ]}] + + t_nova_cluster = nova.ModelBuilder(mock.Mock()) + self.assertTrue(t_nova_cluster._check_model_scope(m_scope)) + + def test_check_model_update_false(self): + """Initialize check model with multiple identical scopes + + The seconds check_model should return false as the models are the same + """ + + m_scope = [{"compute": [ + {"host_aggregates": [{"id": 5}]}, + {"availability_zones": [{"name": "av_a"}]} + ]}] + + t_nova_cluster = nova.ModelBuilder(mock.Mock()) + self.assertTrue(t_nova_cluster._check_model_scope(m_scope)) + self.assertFalse(t_nova_cluster._check_model_scope(m_scope)) + + def test_check_model_update_true(self): + """Initialize check model with multiple different scopes + + Since the models differ both should return True for the update flag + """ + + m_scope_one = [{"compute": [ + {"host_aggregates": [{"id": 5}]}, + {"availability_zones": [{"name": "av_a"}]} + ]}] + + m_scope_two = [{"compute": [ + {"host_aggregates": [{"id": 2}]}, + {"availability_zones": [{"name": "av_b"}]} + ]}] + + t_nova_cluster = nova.ModelBuilder(mock.Mock()) + self.assertTrue(t_nova_cluster._check_model_scope(m_scope_one)) + self.assertTrue(t_nova_cluster._check_model_scope(m_scope_two)) + + def test_merge_compute_scope(self): + """""" + + m_scope_one = [ + {"host_aggregates": [{"id": 5}]}, + {"availability_zones": [{"name": "av_a"}]} + ] + + m_scope_two = [ + {"host_aggregates": [{"id": 4}]}, + {"availability_zones": [{"name": "av_b"}]} + ] + + reference = {'availability_zones': + [{'name': 'av_a'}, {'name': 'av_b'}], + 'host_aggregates': + [{'id': 5}, {'id': 4}]} + + t_nova_cluster = nova.ModelBuilder(mock.Mock()) + t_nova_cluster._merge_compute_scope(m_scope_one) + t_nova_cluster._merge_compute_scope(m_scope_two) + + self.assertEqual(reference, t_nova_cluster.model_scope) + + @mock.patch.object(nova_helper, 'NovaHelper') + def test_collect_aggregates(self, m_nova): + """""" + + m_nova.return_value.get_aggregate_list.return_value = \ + [mock.Mock(id=1, name='example'), + mock.Mock(id=5, name='example', hosts=['hostone', 'hosttwo'])] + + m_nova.return_value.get_compute_node_by_name.return_value = False + + m_scope = [{'id': 5}] + + t_nova_cluster = nova.ModelBuilder(mock.Mock()) + result = set() + t_nova_cluster._collect_aggregates(m_scope, result) + + self.assertEqual(set(['hostone', 'hosttwo']), result) + + @mock.patch.object(nova_helper, 'NovaHelper') + def test_collect_zones(self, m_nova): + """""" + + m_nova.return_value.get_service_list.return_value = \ + [mock.Mock(zone='av_b'), + mock.Mock(zone='av_a', host='hostone')] + + m_nova.return_value.get_compute_node_by_name.return_value = False + + m_scope = [{'name': 'av_a'}] + + t_nova_cluster = nova.ModelBuilder(mock.Mock()) + result = set() + t_nova_cluster._collect_zones(m_scope, result) + + self.assertEqual(set(['hostone']), result) + + @mock.patch.object(nova_helper, 'NovaHelper') + def test_add_physical_layer(self, m_nova): + """""" + + m_nova.return_value.get_aggregate_list.return_value = \ + [mock.Mock(id=1, name='example'), + mock.Mock(id=5, name='example', hosts=['hostone', 'hosttwo'])] + + m_nova.return_value.get_service_list.return_value = \ + [mock.Mock(zone='av_b', host='hostthree'), + mock.Mock(zone='av_a', host='hostone')] + + m_nova.return_value.get_compute_node_by_name.return_value = False + + m_scope = [{"compute": [ + {"host_aggregates": [{"id": 5}]}, + {"availability_zones": [{"name": "av_a"}]} + ]}] + + t_nova_cluster = nova.ModelBuilder(mock.Mock()) + t_nova_cluster.execute(m_scope) + m_nova.return_value.get_compute_node_by_name.assert_any_call( + 'hostone', servers=True) + m_nova.return_value.get_compute_node_by_name.assert_any_call( + 'hosttwo', servers=True) + self.assertEqual( + m_nova.return_value.get_compute_node_by_name.call_count, 2)