From f337c67bfe2108fa326f556db711b544fcce3690 Mon Sep 17 00:00:00 2001 From: licanwei Date: Sat, 2 Mar 2019 14:53:21 +0800 Subject: [PATCH] scope for datamodel This patch adds a scope to the datamodel, which only gets the VMs of the specified nodes, and no longer gets all VMs from nova. Implements: blueprint scope-for-watcher-datamodel Change-Id: Ic4659d1f18af181203439a8bf1b38805ff34c309 --- watcher/common/nova_helper.py | 3 + .../decision_engine/model/collector/base.py | 1 + .../decision_engine/model/collector/nova.py | 141 ++++++++++++++++-- .../decision_engine/cluster/test_nova_cdmc.py | 14 +- 4 files changed, 145 insertions(+), 14 deletions(-) diff --git a/watcher/common/nova_helper.py b/watcher/common/nova_helper.py index 2517bd8c3..eed5204af 100644 --- a/watcher/common/nova_helper.py +++ b/watcher/common/nova_helper.py @@ -53,6 +53,9 @@ class NovaHelper(object): # We need to pass an object with an 'id' attribute to make it work return self.nova.hypervisors.get(utils.Struct(id=node_id)) + def get_compute_node_by_name(self, node_name, servers=False): + return self.nova.hypervisors.search(node_name, servers) + def get_compute_node_by_hostname(self, node_hostname): """Get compute node by hostname""" try: diff --git a/watcher/decision_engine/model/collector/base.py b/watcher/decision_engine/model/collector/base.py index a0ff3e50c..055d618db 100644 --- a/watcher/decision_engine/model/collector/base.py +++ b/watcher/decision_engine/model/collector/base.py @@ -129,6 +129,7 @@ class BaseClusterDataModelCollector(loadable.LoadableSingleton): self._cluster_data_model = None self.lock = threading.RLock() self._audit_scope_handler = None + self._data_model_scope = None @property def cluster_data_model(self): diff --git a/watcher/decision_engine/model/collector/nova.py b/watcher/decision_engine/model/collector/nova.py index 62abe4716..60a4f7fc0 100644 --- a/watcher/decision_engine/model/collector/nova.py +++ b/watcher/decision_engine/model/collector/nova.py @@ -165,14 +165,24 @@ class NovaClusterDataModelCollector(base.BaseClusterDataModelCollector): def get_audit_scope_handler(self, audit_scope): self._audit_scope_handler = compute_scope.ComputeScope( audit_scope, self.config) + if self._data_model_scope is None or ( + len(self._data_model_scope) > 0 and ( + self._data_model_scope != audit_scope)): + self._data_model_scope = audit_scope + self._cluster_data_model = None + LOG.debug("audit scope %s", audit_scope) return self._audit_scope_handler def execute(self): """Build the compute cluster data model""" LOG.debug("Building latest Nova cluster data model") + if self._audit_scope_handler is None: + LOG.debug("No audit, Don't Build compute data model") + return + builder = ModelBuilder(self.osc) - return builder.execute() + return builder.execute(self._data_model_scope) class ModelBuilder(object): @@ -196,24 +206,70 @@ class ModelBuilder(object): def __init__(self, osc): self.osc = osc - self.model = model_root.ModelRoot() + self.model = None + self.model_scope = dict() self.nova = osc.nova() self.nova_helper = nova_helper.NovaHelper(osc=self.osc) # self.neutron = osc.neutron() # self.cinder = osc.cinder() + def _collect_aggregates(self, host_aggregates, _nodes): + aggregate_list = self.nova_helper.get_aggregate_list() + aggregate_ids = [aggregate['id'] for aggregate + in host_aggregates if 'id' in aggregate] + aggregate_names = [aggregate['name'] for aggregate + in host_aggregates if 'name' in aggregate] + include_all_nodes = any('*' in field + for field in (aggregate_ids, aggregate_names)) + + for aggregate in aggregate_list: + if (aggregate.id in aggregate_ids or + aggregate.name in aggregate_names or + include_all_nodes): + _nodes.update(aggregate.hosts) + + def _collect_zones(self, availability_zones, _nodes): + service_list = self.nova_helper.get_service_list() + zone_names = [zone['name'] for zone + in availability_zones] + include_all_nodes = False + if '*' in zone_names: + include_all_nodes = True + for service in service_list: + if service.zone in zone_names or include_all_nodes: + _nodes.update(service.host) + def _add_physical_layer(self): """Add the physical layer of the graph. This includes components which represent actual infrastructure hardware. """ - for cnode in self.nova_helper.get_compute_node_list(): - self.add_compute_node(cnode) + compute_nodes = set() + host_aggregates = self.model_scope.get("host_aggregates") + availability_zones = self.model_scope.get("availability_zones") + if host_aggregates: + self._collect_aggregates(host_aggregates, compute_nodes) + if availability_zones: + self._collect_zones(availability_zones, compute_nodes) + + if not compute_nodes: + all_nodes = self.nova_helper.get_compute_node_list() + compute_nodes = set( + [node.hypervisor_hostname for node in all_nodes]) + LOG.debug("compute nodes: %s", compute_nodes) + for node_name in compute_nodes: + cnode = self.nova_helper.get_compute_node_by_name(node_name, + servers=True) + if cnode: + self.add_compute_node(cnode[0]) + self.add_instance_node(cnode[0]) def add_compute_node(self, node): # Build and add base node. - compute_node = self.build_compute_node(node) + node_info = self.nova_helper.get_compute_node_by_id(node.id) + LOG.debug("node info: %s", node_info) + compute_node = self.build_compute_node(node_info) self.model.add_node(compute_node) # NOTE(v-francoise): we can encapsulate capabilities of the node @@ -242,10 +298,9 @@ class ModelBuilder(object): :type node: :py:class:`~novaclient.v2.hypervisors.Hypervisor` """ # build up the compute node. - compute_service = self.nova_helper.get_service(node.service["id"]) node_attributes = { "id": node.id, - "uuid": compute_service.host, + "uuid": node.service["host"], "hostname": node.hypervisor_hostname, "memory": node.memory_mb, "disk": node.free_disk_gb, @@ -253,7 +308,7 @@ class ModelBuilder(object): "vcpus": node.vcpus, "state": node.state, "status": node.status, - "disabled_reason": compute_service.disabled_reason} + "disabled_reason": node.service["disabled_reason"]} compute_node = element.ComputeNode(**node_attributes) # compute_node = self._build_node("physical", "compute", "hypervisor", @@ -333,6 +388,29 @@ class ModelBuilder(object): except exception.ComputeNodeNotFound: continue + def add_instance_node(self, node): + # node.servers is a list of server objects + # New in nova version 2.53 + instances = getattr(node, "servers", None) + if instances is None: + # no instances on this node + return + instance_uuids = [s['uuid'] for s in instances] + for uuid in instance_uuids: + try: + inst = self.nova_helper.find_instance(uuid) + except Exception as exc: + LOG.exception(exc) + continue + # Add Node + instance = self._build_instance_node(inst) + self.model.add_instance(instance) + cnode_uuid = getattr(inst, "OS-EXT-SRV-ATTR:host") + compute_node = self.model.get_node_by_uuid( + cnode_uuid) + # Connect the instance to its compute node + self.model.map_instance(instance, compute_node) + def _build_instance_node(self, instance): """Build an instance node @@ -475,12 +553,53 @@ class ModelBuilder(object): # node = self._build_node('virtual', 'network', 'network', network) # return network['id'], node - def execute(self): + def _merge_compute_scope(self, compute_scope): + model_keys = self.model_scope.keys() + update_flag = False + + role_keys = ("host_aggregates", "availability_zones") + for role in compute_scope: + role_key = role.keys()[0] + if role_key not in role_keys: + continue + role_values = role.values()[0] + if role_key in model_keys: + for value in role_values: + if value not in self.model_scope[role_key]: + self.model_scope[role_key].sppend(value) + update_flag = True + else: + self.self.model_scope[role_key] = role_values + update_flag = True + return update_flag + + def _check_model_scope(self, model_scope): + compute_scope = [] + update_flag = False + for _scope in model_scope: + if 'compute' in _scope: + compute_scope = _scope['compute'] + break + + if self.model_scope: + if model_scope: + if compute_scope: + update_flag = self._merge_compute_scope(compute_scope) + else: + self.model_scope = dict() + update_flag = True + + return update_flag + + def execute(self, model_scope): """Instantiates the graph with the openstack cluster data. The graph is populated along 2 layers: virtual and physical. As each new layer is built connections are made back to previous layers. """ - self._add_physical_layer() - self._add_virtual_layer() + updata_model_flag = self._check_model_scope(model_scope) + if self.model is None or updata_model_flag: + self.model = self.model or model_root.ModelRoot() + self._add_physical_layer() + return self.model diff --git a/watcher/tests/decision_engine/cluster/test_nova_cdmc.py b/watcher/tests/decision_engine/cluster/test_nova_cdmc.py index 424b4d93b..a2ffc2764 100644 --- a/watcher/tests/decision_engine/cluster/test_nova_cdmc.py +++ b/watcher/tests/decision_engine/cluster/test_nova_cdmc.py @@ -46,7 +46,8 @@ class TestNovaClusterDataModelCollector(base.TestCase): fake_compute_node = mock.Mock( id=1337, - service={'id': 123}, + service={'id': 123, 'host': 'test_hostname', + 'disabled_reason': ''}, hypervisor_hostname='test_hostname', memory_mb=333, free_disk_gb=222, @@ -54,6 +55,10 @@ class TestNovaClusterDataModelCollector(base.TestCase): vcpus=4, state='TEST_STATE', status='TEST_STATUS', + servers=[ + {'name': 'fake_instance', + 'uuid': 'ef500f7e-dac8-470f-960c-169486fce71b'} + ] ) fake_instance = mock.Mock( id='ef500f7e-dac8-470f-960c-169486fce71b', @@ -65,8 +70,10 @@ class TestNovaClusterDataModelCollector(base.TestCase): setattr(fake_instance, 'OS-EXT-STS:vm_state', 'VM_STATE') setattr(fake_instance, 'OS-EXT-SRV-ATTR:host', 'test_hostname') m_nova_helper.get_compute_node_list.return_value = [fake_compute_node] - # m_nova_helper.get_instances_by_node.return_value = [fake_instance] - m_nova_helper.get_instance_list.return_value = [fake_instance] + m_nova_helper.get_compute_node_by_name.return_value = [ + fake_compute_node] + m_nova_helper.get_compute_node_by_id.return_value = fake_compute_node + m_nova_helper.find_instance.return_value = fake_instance m_config = mock.Mock() m_osc = mock.Mock() @@ -74,6 +81,7 @@ class TestNovaClusterDataModelCollector(base.TestCase): nova_cdmc = nova.NovaClusterDataModelCollector( config=m_config, osc=m_osc) + nova_cdmc.get_audit_scope_handler([]) model = nova_cdmc.execute() compute_nodes = model.get_all_compute_nodes()