Arnike/watcher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge "scope for datamodel"

Browse Source
This commit is contained in:
Zuul
2019-03-11 12:40:24 +00:00
committed by Gerrit Code Review
parent 8d2c249f4b f337c67bfe
commit 23cac6813b
4 changed files with 145 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
watcher/common/nova_helper.py
View File

@@ -53,6 +53,9 @@ class NovaHelper(object):
# We need to pass an object with an 'id' attribute to make it work
return self.nova.hypervisors.get(utils.Struct(id=node_id))
def get_compute_node_by_name(self, node_name, servers=False):
return self.nova.hypervisors.search(node_name, servers)
def get_compute_node_by_hostname(self, node_hostname):
"""Get compute node by hostname"""
try:

1
watcher/decision_engine/model/collector/base.py
View File

@@ -129,6 +129,7 @@ class BaseClusterDataModelCollector(loadable.LoadableSingleton):
self._cluster_data_model = None
self.lock = threading.RLock()
self._audit_scope_handler = None
self._data_model_scope = None
@property
def cluster_data_model(self):

141
watcher/decision_engine/model/collector/nova.py
View File

@@ -165,14 +165,24 @@ class NovaClusterDataModelCollector(base.BaseClusterDataModelCollector):
def get_audit_scope_handler(self, audit_scope):
self._audit_scope_handler = compute_scope.ComputeScope(
audit_scope, self.config)
if self._data_model_scope is None or (
len(self._data_model_scope) > 0 and (
self._data_model_scope != audit_scope)):
self._data_model_scope = audit_scope
self._cluster_data_model = None
LOG.debug("audit scope %s", audit_scope)
return self._audit_scope_handler
def execute(self):
"""Build the compute cluster data model"""
LOG.debug("Building latest Nova cluster data model")
if self._audit_scope_handler is None:
LOG.debug("No audit, Don't Build compute data model")
return
builder = ModelBuilder(self.osc)
return builder.execute()
return builder.execute(self._data_model_scope)
class ModelBuilder(object):
@@ -196,24 +206,70 @@ class ModelBuilder(object):
def __init__(self, osc):
self.osc = osc
self.model = model_root.ModelRoot()
self.model = None
self.model_scope = dict()
self.nova = osc.nova()
self.nova_helper = nova_helper.NovaHelper(osc=self.osc)
# self.neutron = osc.neutron()
# self.cinder = osc.cinder()
def _collect_aggregates(self, host_aggregates, _nodes):
aggregate_list = self.nova_helper.get_aggregate_list()
aggregate_ids = [aggregate['id'] for aggregate
in host_aggregates if 'id' in aggregate]
aggregate_names = [aggregate['name'] for aggregate
in host_aggregates if 'name' in aggregate]
include_all_nodes = any('*' in field
for field in (aggregate_ids, aggregate_names))
for aggregate in aggregate_list:
if (aggregate.id in aggregate_ids or
aggregate.name in aggregate_names or
include_all_nodes):
_nodes.update(aggregate.hosts)
def _collect_zones(self, availability_zones, _nodes):
service_list = self.nova_helper.get_service_list()
zone_names = [zone['name'] for zone
in availability_zones]
include_all_nodes = False
if '*' in zone_names:
include_all_nodes = True
for service in service_list:
if service.zone in zone_names or include_all_nodes:
_nodes.update(service.host)
def _add_physical_layer(self):
"""Add the physical layer of the graph.
This includes components which represent actual infrastructure
hardware.
"""
for cnode in self.nova_helper.get_compute_node_list():
self.add_compute_node(cnode)
compute_nodes = set()
host_aggregates = self.model_scope.get("host_aggregates")
availability_zones = self.model_scope.get("availability_zones")
if host_aggregates:
self._collect_aggregates(host_aggregates, compute_nodes)
if availability_zones:
self._collect_zones(availability_zones, compute_nodes)
if not compute_nodes:
all_nodes = self.nova_helper.get_compute_node_list()
compute_nodes = set(
[node.hypervisor_hostname for node in all_nodes])
LOG.debug("compute nodes: %s", compute_nodes)
for node_name in compute_nodes:
cnode = self.nova_helper.get_compute_node_by_name(node_name,
servers=True)
if cnode:
self.add_compute_node(cnode[0])
self.add_instance_node(cnode[0])
def add_compute_node(self, node):
# Build and add base node.
compute_node = self.build_compute_node(node)
node_info = self.nova_helper.get_compute_node_by_id(node.id)
LOG.debug("node info: %s", node_info)
compute_node = self.build_compute_node(node_info)
self.model.add_node(compute_node)
# NOTE(v-francoise): we can encapsulate capabilities of the node
@@ -242,10 +298,9 @@ class ModelBuilder(object):
:type node: :py:class:`~novaclient.v2.hypervisors.Hypervisor`
"""
# build up the compute node.
compute_service = self.nova_helper.get_service(node.service["id"])
node_attributes = {
"id": node.id,
"uuid": compute_service.host,
"uuid": node.service["host"],
"hostname": node.hypervisor_hostname,
"memory": node.memory_mb,
"disk": node.free_disk_gb,
@@ -253,7 +308,7 @@ class ModelBuilder(object):
"vcpus": node.vcpus,
"state": node.state,
"status": node.status,
"disabled_reason": compute_service.disabled_reason}
"disabled_reason": node.service["disabled_reason"]}
compute_node = element.ComputeNode(**node_attributes)
# compute_node = self._build_node("physical", "compute", "hypervisor",
@@ -333,6 +388,29 @@ class ModelBuilder(object):
except exception.ComputeNodeNotFound:
continue
def add_instance_node(self, node):
# node.servers is a list of server objects
# New in nova version 2.53
instances = getattr(node, "servers", None)
if instances is None:
# no instances on this node
return
instance_uuids = [s['uuid'] for s in instances]
for uuid in instance_uuids:
try:
inst = self.nova_helper.find_instance(uuid)
except Exception as exc:
LOG.exception(exc)
continue
# Add Node
instance = self._build_instance_node(inst)
self.model.add_instance(instance)
cnode_uuid = getattr(inst, "OS-EXT-SRV-ATTR:host")
compute_node = self.model.get_node_by_uuid(
cnode_uuid)
# Connect the instance to its compute node
self.model.map_instance(instance, compute_node)
def _build_instance_node(self, instance):
"""Build an instance node
@@ -475,12 +553,53 @@ class ModelBuilder(object):
# node = self._build_node('virtual', 'network', 'network', network)
# return network['id'], node
def execute(self):
def _merge_compute_scope(self, compute_scope):
model_keys = self.model_scope.keys()
update_flag = False
role_keys = ("host_aggregates", "availability_zones")
for role in compute_scope:
role_key = role.keys()[0]
if role_key not in role_keys:
continue
role_values = role.values()[0]
if role_key in model_keys:
for value in role_values:
if value not in self.model_scope[role_key]:
self.model_scope[role_key].sppend(value)
update_flag = True
else:
self.self.model_scope[role_key] = role_values
update_flag = True
return update_flag
def _check_model_scope(self, model_scope):
compute_scope = []
update_flag = False
for _scope in model_scope:
if 'compute' in _scope:
compute_scope = _scope['compute']
break
if self.model_scope:
if model_scope:
if compute_scope:
update_flag = self._merge_compute_scope(compute_scope)
else:
self.model_scope = dict()
update_flag = True
return update_flag
def execute(self, model_scope):
"""Instantiates the graph with the openstack cluster data.
The graph is populated along 2 layers: virtual and physical. As each
new layer is built connections are made back to previous layers.
"""
self._add_physical_layer()
self._add_virtual_layer()
updata_model_flag = self._check_model_scope(model_scope)
if self.model is None or updata_model_flag:
self.model = self.model or model_root.ModelRoot()
self._add_physical_layer()
return self.model

14
watcher/tests/decision_engine/cluster/test_nova_cdmc.py
View File

@@ -46,7 +46,8 @@ class TestNovaClusterDataModelCollector(base.TestCase):
fake_compute_node = mock.Mock(
id=1337,
service={'id': 123},
service={'id': 123, 'host': 'test_hostname',
'disabled_reason': ''},
hypervisor_hostname='test_hostname',
memory_mb=333,
free_disk_gb=222,
@@ -54,6 +55,10 @@ class TestNovaClusterDataModelCollector(base.TestCase):
vcpus=4,
state='TEST_STATE',
status='TEST_STATUS',
servers=[
{'name': 'fake_instance',
'uuid': 'ef500f7e-dac8-470f-960c-169486fce71b'}
]
)
fake_instance = mock.Mock(
id='ef500f7e-dac8-470f-960c-169486fce71b',
@@ -65,8 +70,10 @@ class TestNovaClusterDataModelCollector(base.TestCase):
setattr(fake_instance, 'OS-EXT-STS:vm_state', 'VM_STATE')
setattr(fake_instance, 'OS-EXT-SRV-ATTR:host', 'test_hostname')
m_nova_helper.get_compute_node_list.return_value = [fake_compute_node]
# m_nova_helper.get_instances_by_node.return_value = [fake_instance]
m_nova_helper.get_instance_list.return_value = [fake_instance]
m_nova_helper.get_compute_node_by_name.return_value = [
fake_compute_node]
m_nova_helper.get_compute_node_by_id.return_value = fake_compute_node
m_nova_helper.find_instance.return_value = fake_instance
m_config = mock.Mock()
m_osc = mock.Mock()
@@ -74,6 +81,7 @@ class TestNovaClusterDataModelCollector(base.TestCase):
nova_cdmc = nova.NovaClusterDataModelCollector(
config=m_config, osc=m_osc)
nova_cdmc.get_audit_scope_handler([])
model = nova_cdmc.execute()
compute_nodes = model.get_all_compute_nodes()