3f76f9cfdb
The nova CDM builder code and notification handling code had some inefficiencies when it came to looking up a hypevisor to get details. The general pattern used before was: 1. get the minimal hypervisor information by hypervisor_hostname 2. make another query to get the hypervisor details by id In the notifications case, it was actually three calls because the first is listing hyprvisors to filter client-side by service host. This change collapses 1 and 2 above into a single API call to get the hypervisor by hypervisor_hostname with details which will include the service (compute) host information which is what get_compute_node_by_id() was being used for. Now that nothing is using get_compute_node_by_id it is removed. There is more work we could do in get_compute_node_by_hostname if the compute API allowed filtering hypervisors by service host so a TODO is left for that. One final thing: the TODO in get_compute_node_by_hostname about there being more than one hypervisor per compute service host for vmware vcenter is not accurate - nova's vcenter driver hasn't supported a host:node 1:M topology like that since the Liberty release [1]. The only in-tree driver in nova that supports 1:M is the ironic baremetal driver, so the comment is updated. [1] Ifc17c5049e3ed29c8dd130339207907b00433960 Depends-On: https://review.opendev.org/661785/ Change-Id: I5e0e88d7b2dd1a69117ab03e0e66851c687606da
413 lines
16 KiB
Python
413 lines
16 KiB
Python
# -*- encoding: utf-8 -*-
|
|
# Copyright (c) 2017 Intel Innovation and Research Ireland Ltd.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from oslo_log import log
|
|
|
|
from watcher.common import nova_helper
|
|
from watcher.decision_engine.model.collector import base
|
|
from watcher.decision_engine.model import element
|
|
from watcher.decision_engine.model import model_root
|
|
from watcher.decision_engine.model.notification import nova
|
|
from watcher.decision_engine.scope import compute as compute_scope
|
|
|
|
LOG = log.getLogger(__name__)
|
|
|
|
|
|
class NovaClusterDataModelCollector(base.BaseClusterDataModelCollector):
|
|
"""Nova cluster data model collector
|
|
|
|
The Nova cluster data model collector creates an in-memory
|
|
representation of the resources exposed by the compute service.
|
|
"""
|
|
|
|
HOST_AGGREGATES = "#/items/properties/compute/host_aggregates/"
|
|
SCHEMA = {
|
|
"$schema": "http://json-schema.org/draft-04/schema#",
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"host_aggregates": {
|
|
"type": "array",
|
|
"items": {
|
|
"anyOf": [
|
|
{"$ref": HOST_AGGREGATES + "id"},
|
|
{"$ref": HOST_AGGREGATES + "name"},
|
|
]
|
|
}
|
|
},
|
|
"availability_zones": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"name": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"additionalProperties": False
|
|
}
|
|
},
|
|
"exclude": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"instances": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"uuid": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"additionalProperties": False
|
|
}
|
|
},
|
|
"compute_nodes": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"name": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"additionalProperties": False
|
|
}
|
|
},
|
|
"host_aggregates": {
|
|
"type": "array",
|
|
"items": {
|
|
"anyOf": [
|
|
{"$ref": HOST_AGGREGATES + "id"},
|
|
{"$ref": HOST_AGGREGATES + "name"},
|
|
]
|
|
}
|
|
},
|
|
"instance_metadata": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object"
|
|
}
|
|
},
|
|
"projects": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"uuid": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"additionalProperties": False
|
|
}
|
|
}
|
|
},
|
|
"additionalProperties": False
|
|
}
|
|
}
|
|
},
|
|
"additionalProperties": False
|
|
},
|
|
"host_aggregates": {
|
|
"id": {
|
|
"properties": {
|
|
"id": {
|
|
"oneOf": [
|
|
{"type": "integer"},
|
|
{"enum": ["*"]}
|
|
]
|
|
}
|
|
},
|
|
"additionalProperties": False
|
|
},
|
|
"name": {
|
|
"properties": {
|
|
"name": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"additionalProperties": False
|
|
}
|
|
},
|
|
"additionalProperties": False
|
|
}
|
|
|
|
def __init__(self, config, osc=None):
|
|
super(NovaClusterDataModelCollector, self).__init__(config, osc)
|
|
|
|
@property
|
|
def notification_endpoints(self):
|
|
"""Associated notification endpoints
|
|
|
|
:return: Associated notification endpoints
|
|
:rtype: List of :py:class:`~.EventsNotificationEndpoint` instances
|
|
"""
|
|
return [
|
|
nova.VersionedNotification(self),
|
|
]
|
|
|
|
def get_audit_scope_handler(self, audit_scope):
|
|
self._audit_scope_handler = compute_scope.ComputeScope(
|
|
audit_scope, self.config)
|
|
if self._data_model_scope is None or (
|
|
len(self._data_model_scope) > 0 and (
|
|
self._data_model_scope != audit_scope)):
|
|
self._data_model_scope = audit_scope
|
|
self._cluster_data_model = None
|
|
LOG.debug("audit scope %s", audit_scope)
|
|
return self._audit_scope_handler
|
|
|
|
def execute(self):
|
|
"""Build the compute cluster data model"""
|
|
LOG.debug("Building latest Nova cluster data model")
|
|
|
|
if self._audit_scope_handler is None:
|
|
LOG.debug("No audit, Don't Build compute data model")
|
|
return
|
|
|
|
builder = ModelBuilder(self.osc)
|
|
return builder.execute(self._data_model_scope)
|
|
|
|
|
|
class ModelBuilder(object):
|
|
"""Build the graph-based model
|
|
|
|
This model builder adds the following data"
|
|
|
|
- Compute-related knowledge (Nova)
|
|
- TODO(v-francoise): Storage-related knowledge (Cinder)
|
|
- TODO(v-francoise): Network-related knowledge (Neutron)
|
|
|
|
NOTE(v-francoise): This model builder is meant to be extended in the future
|
|
to also include both storage and network information respectively coming
|
|
from Cinder and Neutron. Some prelimary work has been done in this
|
|
direction in https://review.opendev.org/#/c/362730 but since we cannot
|
|
guarantee a sufficient level of consistency for neither the storage nor the
|
|
network part before the end of the Ocata cycle, this work has been
|
|
re-scheduled for Pike. In the meantime, all the associated code has been
|
|
commented out.
|
|
"""
|
|
|
|
def __init__(self, osc):
|
|
self.osc = osc
|
|
self.model = None
|
|
self.model_scope = dict()
|
|
self.no_model_scope_flag = False
|
|
self.nova = osc.nova()
|
|
self.nova_helper = nova_helper.NovaHelper(osc=self.osc)
|
|
|
|
def _collect_aggregates(self, host_aggregates, _nodes):
|
|
aggregate_list = self.nova_helper.get_aggregate_list()
|
|
aggregate_ids = [aggregate['id'] for aggregate
|
|
in host_aggregates if 'id' in aggregate]
|
|
aggregate_names = [aggregate['name'] for aggregate
|
|
in host_aggregates if 'name' in aggregate]
|
|
include_all_nodes = any('*' in field
|
|
for field in (aggregate_ids, aggregate_names))
|
|
|
|
for aggregate in aggregate_list:
|
|
if (aggregate.id in aggregate_ids or
|
|
aggregate.name in aggregate_names or
|
|
include_all_nodes):
|
|
_nodes.update(aggregate.hosts)
|
|
|
|
def _collect_zones(self, availability_zones, _nodes):
|
|
service_list = self.nova_helper.get_service_list()
|
|
zone_names = [zone['name'] for zone
|
|
in availability_zones]
|
|
include_all_nodes = False
|
|
if '*' in zone_names:
|
|
include_all_nodes = True
|
|
for service in service_list:
|
|
if service.zone in zone_names or include_all_nodes:
|
|
_nodes.add(service.host)
|
|
|
|
def _add_physical_layer(self):
|
|
"""Add the physical layer of the graph.
|
|
|
|
This includes components which represent actual infrastructure
|
|
hardware.
|
|
"""
|
|
compute_nodes = set()
|
|
host_aggregates = self.model_scope.get("host_aggregates")
|
|
availability_zones = self.model_scope.get("availability_zones")
|
|
if host_aggregates:
|
|
self._collect_aggregates(host_aggregates, compute_nodes)
|
|
if availability_zones:
|
|
self._collect_zones(availability_zones, compute_nodes)
|
|
|
|
if not compute_nodes:
|
|
self.no_model_scope_flag = True
|
|
all_nodes = self.nova_helper.get_compute_node_list()
|
|
compute_nodes = set(
|
|
[node.hypervisor_hostname for node in all_nodes])
|
|
LOG.debug("compute nodes: %s", compute_nodes)
|
|
for node_name in compute_nodes:
|
|
cnode = self.nova_helper.get_compute_node_by_name(node_name,
|
|
servers=True,
|
|
detailed=True)
|
|
if cnode:
|
|
node_info = cnode[0]
|
|
self.add_compute_node(node_info)
|
|
# node.servers is a list of server objects
|
|
# New in nova version 2.53
|
|
instances = getattr(node_info, "servers", None)
|
|
self.add_instance_node(node_info, instances)
|
|
|
|
def add_compute_node(self, node):
|
|
# Build and add base node.
|
|
LOG.debug("node info: %s", node)
|
|
compute_node = self.build_compute_node(node)
|
|
self.model.add_node(compute_node)
|
|
|
|
# NOTE(v-francoise): we can encapsulate capabilities of the node
|
|
# (special instruction sets of CPUs) in the attributes; as well as
|
|
# sub-nodes can be added re-presenting e.g. GPUs/Accelerators etc.
|
|
|
|
# # Build & add disk, memory, network and cpu nodes.
|
|
# disk_id, disk_node = self.build_disk_compute_node(base_id, node)
|
|
# self.add_node(disk_id, disk_node)
|
|
# mem_id, mem_node = self.build_memory_compute_node(base_id, node)
|
|
# self.add_node(mem_id, mem_node)
|
|
# net_id, net_node = self._build_network_compute_node(base_id)
|
|
# self.add_node(net_id, net_node)
|
|
# cpu_id, cpu_node = self.build_cpu_compute_node(base_id, node)
|
|
# self.add_node(cpu_id, cpu_node)
|
|
|
|
# # Connect the base compute node to the dependent nodes.
|
|
# self.add_edges_from([(base_id, disk_id), (base_id, mem_id),
|
|
# (base_id, cpu_id), (base_id, net_id)],
|
|
# label="contains")
|
|
|
|
def build_compute_node(self, node):
|
|
"""Build a compute node from a Nova compute node
|
|
|
|
:param node: A node hypervisor instance
|
|
:type node: :py:class:`~novaclient.v2.hypervisors.Hypervisor`
|
|
"""
|
|
# build up the compute node.
|
|
node_attributes = {
|
|
"id": node.id,
|
|
"uuid": node.service["host"],
|
|
"hostname": node.hypervisor_hostname,
|
|
"memory": node.memory_mb,
|
|
"disk": node.free_disk_gb,
|
|
"disk_capacity": node.local_gb,
|
|
"vcpus": node.vcpus,
|
|
"state": node.state,
|
|
"status": node.status,
|
|
"disabled_reason": node.service["disabled_reason"]}
|
|
|
|
compute_node = element.ComputeNode(**node_attributes)
|
|
# compute_node = self._build_node("physical", "compute", "hypervisor",
|
|
# node_attributes)
|
|
return compute_node
|
|
|
|
def add_instance_node(self, node, instances):
|
|
if instances is None:
|
|
# no instances on this node
|
|
return
|
|
host = node.service["host"]
|
|
compute_node = self.model.get_node_by_uuid(host)
|
|
# Get all servers on this compute host.
|
|
instances = self.nova_helper.get_instance_list({'host': host})
|
|
for inst in instances:
|
|
# Add Node
|
|
instance = self._build_instance_node(inst)
|
|
self.model.add_instance(instance)
|
|
# Connect the instance to its compute node
|
|
self.model.map_instance(instance, compute_node)
|
|
|
|
def _build_instance_node(self, instance):
|
|
"""Build an instance node
|
|
|
|
Create an instance node for the graph using nova and the
|
|
`server` nova object.
|
|
:param instance: Nova VM object.
|
|
:return: An instance node for the graph.
|
|
"""
|
|
flavor = instance.flavor
|
|
instance_attributes = {
|
|
"uuid": instance.id,
|
|
"human_id": instance.human_id,
|
|
"memory": flavor["ram"],
|
|
"disk": flavor["disk"],
|
|
"disk_capacity": flavor["disk"],
|
|
"vcpus": flavor["vcpus"],
|
|
"state": getattr(instance, "OS-EXT-STS:vm_state"),
|
|
"metadata": instance.metadata,
|
|
"project_id": instance.tenant_id,
|
|
"locked": instance.locked}
|
|
|
|
# node_attributes = dict()
|
|
# node_attributes["layer"] = "virtual"
|
|
# node_attributes["category"] = "compute"
|
|
# node_attributes["type"] = "compute"
|
|
# node_attributes["attributes"] = instance_attributes
|
|
return element.Instance(**instance_attributes)
|
|
|
|
def _merge_compute_scope(self, compute_scope):
|
|
model_keys = self.model_scope.keys()
|
|
update_flag = False
|
|
|
|
role_keys = ("host_aggregates", "availability_zones")
|
|
for role in compute_scope:
|
|
role_key = list(role.keys())[0]
|
|
if role_key not in role_keys:
|
|
continue
|
|
role_values = list(role.values())[0]
|
|
if role_key in model_keys:
|
|
for value in role_values:
|
|
if value not in self.model_scope[role_key]:
|
|
self.model_scope[role_key].append(value)
|
|
update_flag = True
|
|
else:
|
|
self.model_scope[role_key] = role_values
|
|
update_flag = True
|
|
return update_flag
|
|
|
|
def _check_model_scope(self, model_scope):
|
|
compute_scope = []
|
|
update_flag = False
|
|
for _scope in model_scope:
|
|
if 'compute' in _scope:
|
|
compute_scope = _scope['compute']
|
|
break
|
|
|
|
if self.no_model_scope_flag is False:
|
|
if compute_scope:
|
|
update_flag = self._merge_compute_scope(compute_scope)
|
|
else:
|
|
self.model_scope = dict()
|
|
update_flag = True
|
|
|
|
return update_flag
|
|
|
|
def execute(self, model_scope):
|
|
"""Instantiates the graph with the openstack cluster data.
|
|
|
|
The graph is populated along 2 layers: virtual and physical. As each
|
|
new layer is built connections are made back to previous layers.
|
|
"""
|
|
updata_model_flag = self._check_model_scope(model_scope)
|
|
if self.model is None or updata_model_flag:
|
|
self.model = self.model or model_root.ModelRoot()
|
|
self._add_physical_layer()
|
|
|
|
return self.model
|