Arnike/watcher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

consolidation of watcher

Browse Source 
Change-Id: I9c82ef4d8a81af98afdfc34f5ad496bcade4af6a
This commit is contained in:
Jean-Emile DARTOIS
2015-10-22 17:02:45 +02:00
parent 8c76c7fbef
commit 74160c5e78
140 changed files with 2991 additions and 1271 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
watcher/decision_engine/api/collector/__init__.py
View File

22
watcher/decision_engine/api/collector/cluster_state_collector.py
View File

@@ -1,22 +0,0 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
class ClusterStateCollector(object):
def get_latest_state_cluster(self):
raise NotImplementedError("Should have implemented this")
# todo(jed) think abouts needed interfaces
# todo(jed) stream incremental diff

35
watcher/decision_engine/api/collector/metrics_resource_collector.py
View File

@@ -1,35 +0,0 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
class MetricsResourceCollector(object):
def __init__(self):
pass
def get_average_usage_vm_cpu(self, uuid):
raise NotImplementedError("Should have implemented this")
def get_average_usage_vm_memory(self, uuid):
raise NotImplementedError("Should have implemented this")
def get_virtual_machine_capacity(self, uuid):
raise NotImplementedError("Should have implemented this")
def get_average_network_incomming(self, uuid):
raise NotImplementedError("Should have implemented this")
def get_average_network_outcomming(self, uuid):
raise NotImplementedError("Should have implemented this")

2
watcher/decision_engine/api/messaging/decision_engine_command.py
View File

@@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

3
watcher/decision_engine/api/messaging/event_consumer.py
View File

@@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class EventConsumer(object):

5
watcher/decision_engine/api/planner/planner.py
View File

@@ -1,11 +1,13 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class Planner(object):

0
watcher/decision_engine/api/selector/__init__.py
View File

2
watcher/decision_engine/api/solution/solution.py
View File

@@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

2
watcher/decision_engine/api/solution/solution_comparator.py
View File

@@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

2
watcher/decision_engine/api/solution/solution_evaluator.py
View File

@@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

5
watcher/decision_engine/api/strategy/meta_action.py
View File

@@ -1,11 +1,13 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.decision_engine.api.strategy.strategy import StrategyLevel

6
watcher/decision_engine/api/selector/selector.py → watcher/decision_engine/api/strategy/selector.py
View File

@@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -13,7 +15,9 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class Selector(object):
pass
def define_from_goal(self, goal_name):
raise NotImplementedError("Should have implemented this")

2
watcher/decision_engine/api/strategy/strategy.py
View File

@@ -24,8 +24,6 @@ from watcher.decision_engine.framework.default_solution import DefaultSolution
LOG = log.getLogger(__name__)
# todo(jed) add interface
@six.add_metaclass(abc.ABCMeta)
class Strategy(object):

5
watcher/decision_engine/api/strategy/strategy_context.py
View File

@@ -1,11 +1,13 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class StrategyContext(object):

5
watcher/decision_engine/api/strategy/strategy_level.py
View File

@@ -1,11 +1,13 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from enum import Enum

5
watcher/decision_engine/api/strategy/strategy_state.py
View File

@@ -1,11 +1,13 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from enum import Enum

31
watcher/decision_engine/framework/client_selector_strategy.py
View File

@@ -1,31 +0,0 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from watcher.common.messaging.messaging_core import MessagingCore
from watcher.decision_engine.api.selector.selector import Selector
class ClientSelectorStrategy(Selector, MessagingCore):
"""Trigger an audit (a request for optimizing a cluster)
:param goal: the strategy selected by the strategy selector
:param hosts: the list of hypervisors where a nova-compute service
is running (host aggregate)
:return: None
"""
def launch_audit(self, goal):
# TODO(jed):
# client = ClientScheduler()
pass

59
watcher/decision_engine/framework/command/trigger_audit_command.py
View File

@@ -45,41 +45,42 @@ class TriggerAuditCommand(DecisionEngineCommand):
self.messaging.topic_status.publish_event(event.get_type().name,
payload)
# todo(jed) remove params
def update_audit(self, request_context, audit_uuid, state):
LOG.debug("update audit " + str(state))
audit = Audit.get_by_uuid(request_context, audit_uuid)
audit.state = state
audit.save()
self.notify(audit_uuid, Events.TRIGGER_AUDIT, state)
return audit
def execute(self, audit_uuid, request_context):
LOG.debug("Execute TriggerAuditCommand ")
try:
LOG.debug("Execute TriggerAuditCommand ")
# 1 - change status to ONGOING
audit = Audit.get_by_uuid(request_context, audit_uuid)
audit.state = AuditStatus.ONGOING
audit.save()
# 1 - change status to ONGOING
audit = self.update_audit(request_context, audit_uuid,
AuditStatus.ONGOING)
# 2 - notify the others components of the system
self.notify(audit_uuid, Events.TRIGGER_AUDIT, AuditStatus.ONGOING)
# 3 - Retrieve metrics
cluster = self.statedb.get_latest_state_cluster()
# 3 - Retrieve metrics
cluster = self.statedb.get_latest_state_cluster()
# 4 - Select appropriate strategy
audit_template = AuditTemplate.get_by_id(request_context,
audit.audit_template_id)
# 4 - Select appropriate strategy
audit_template = AuditTemplate.get_by_id(request_context,
audit.audit_template_id)
self.strategy_context.set_goal(audit_template.goal)
self.strategy_context.set_metrics_resource_collector(
self.ressourcedb)
self.strategy_context.set_goal(audit_template.goal)
self.strategy_context.set_metrics_resource_collector(self.ressourcedb)
# 5 - compute change requests
solution = self.strategy_context.execute_strategy(cluster)
# 5 - compute change requests
solution = self.strategy_context.execute_strategy(cluster)
# 6 - create an action plan
planner = DefaultPlanner()
planner.schedule(request_context, audit.id, solution)
# 6 - create an action plan
planner = DefaultPlanner()
planner.schedule(request_context, audit.id, solution)
# 7 - change status to SUCCESS
audit = Audit.get_by_uuid(request_context, audit_uuid)
audit.state = AuditStatus.SUCCESS
audit.save()
# 8 - notify the others components of the system
self.notify(audit_uuid, Events.TRIGGER_AUDIT,
AuditStatus.SUCCESS)
# 7 - change status to SUCCESS and notify
self.update_audit(request_context, audit_uuid, AuditStatus.SUCCESS)
except Exception as e:
self.update_audit(request_context, audit_uuid, AuditStatus.FAILED)
LOG.error(" " + unicode(e))

18
watcher/decision_engine/framework/default_planner.py
View File

@@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -88,11 +90,11 @@ class DefaultPlanner(Planner):
# TODO(jed) type
primitive = self.create_action(action_plan.id,
Primitives.LIVE_MIGRATE.value,
action.get_vm().get_uuid(),
action.get_vm().uuid,
action.get_source_hypervisor().
get_uuid(),
uuid,
action.get_dest_hypervisor().
get_uuid(),
uuid,
description=str(action)
)
@@ -100,18 +102,16 @@ class DefaultPlanner(Planner):
primitive = self.create_action(action_plan_id=action_plan.id,
action_type=Primitives.
POWER_STATE.value,
applies_to=action.target.
get_uuid(),
applies_to=action.target.uuid,
parameter=action.
get_power_state().
powerstate.
value, description=str(action))
elif isinstance(action, ChangeHypervisorState):
primitive = self.create_action(action_plan_id=action_plan.id,
action_type=Primitives.
HYPERVISOR_STATE.value,
applies_to=action.target.
get_uuid(),
parameter=action.get_state().
applies_to=action.target.uuid,
parameter=action.state.
value,
description=str(action))

4
watcher/decision_engine/framework/default_solution.py
View File

@@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -13,7 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# jed <jean-emile.dartois@b-com.com>
#
from watcher.decision_engine.api.solution.solution import Solution
from watcher.openstack.common import log

3
watcher/decision_engine/framework/manager_decision_engine.py
View File

@@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from concurrent.futures import ThreadPoolExecutor
from oslo_config import cfg

14
watcher/decision_engine/framework/messaging/audit_endpoint.py
View File

@@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -13,24 +15,24 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.decision_engine.framework.command.trigger_audit_command import \
TriggerAuditCommand
from watcher.decision_engine.framework.ressourcedb_collector import RessourceDB
from watcher.decision_engine.framework.statedb_collector import NovaCollector
from watcher.metrics_engine.framework.collector_manager import CollectorManager
from watcher.openstack.common import log
LOG = log.getLogger(__name__)
class AuditEndpoint(object):
def __init__(self, de):
self.de = de
self.manager = CollectorManager()
def do_trigger_audit(self, context, audit_uuid):
statedb = NovaCollector()
ressourcedb = RessourceDB()
statedb = self.manager.get_statedb_collector()
ressourcedb = self.manager.get_metric_collector()
audit = TriggerAuditCommand(self.de, statedb,
ressourcedb)
audit.execute(audit_uuid, context)

5
watcher/decision_engine/framework/messaging/events.py
View File

@@ -1,11 +1,13 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from enum import Enum

25
watcher/decision_engine/framework/meta_actions/hypervisor_state.py
View File

@@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.decision_engine.api.strategy.meta_action import MetaAction
from watcher.decision_engine.framework.model.hypervisor_state import \
HypervisorState
@@ -26,14 +29,24 @@ class ChangeHypervisorState(MetaAction):
:param target:
:return:
'''
self.target = target
self.state = HypervisorState.ONLINE
self._target = target
self._state = HypervisorState.ONLINE
def set_state(self, state):
self.state = state
@property
def state(self):
return self._state
def get_state(self):
return self.state
@state.setter
def state(self, state):
self._state = state
@property
def target(self):
return self._target
@target.setter
def target(self, p):
self._target = p
def __str__(self):
return MetaAction.__str__(self) + " ChangeHypervisorState" + str(

3
watcher/decision_engine/framework/meta_actions/migrate.py
View File

@@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from enum import Enum

27
watcher/decision_engine/framework/meta_actions/power_state.py
View File

@@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.decision_engine.api.strategy.meta_action import MetaAction
from watcher.decision_engine.framework.model.power_state import PowerState
@@ -25,15 +28,25 @@ class ChangePowerState(MetaAction):
:param target:
:return:
"""
self.target = target
self.power_state = PowerState.g0
self._target = target
self._power_state = PowerState.g0
def set_power_state(self, state):
self.power_state = state
@property
def powerstate(self):
return self._power_state
def get_power_state(self):
return self.power_state
@powerstate.setter
def powerstate(self, p):
self._power_state = p
@property
def target(self):
return self._target
@target.setter
def target(self, p):
self._target = p
def __str__(self):
return MetaAction.__str__(self) + "ChangePowerState " + str(
self.target) + " => " + str(self.power_state)
self.target) + " => " + str(self.powerstate)

31
watcher/decision_engine/framework/model/hypervisor.py
View File

@@ -21,11 +21,30 @@ from watcher.decision_engine.framework.model.power_state import PowerState
class Hypervisor(NamedElement):
def __init__(self):
self.state = HypervisorState.ONLINE
self.power_state = PowerState.g0
self._state = HypervisorState.ONLINE
self._status = HypervisorState.ENABLED
self._power_state = PowerState.g0
def set_state(self, state):
self.state = state
@property
def state(self):
return self._state
def get_state(self):
return self.state
@state.setter
def state(self, state):
self._state = state
@property
def status(self):
return self._status
@status.setter
def status(self, s):
self._status = s
@property
def powerstate(self):
return self._power_state
@powerstate.setter
def powerstate(self, p):
self._power_state = p

6
watcher/decision_engine/framework/model/hypervisor_state.py
View File

@@ -18,5 +18,7 @@ from enum import Enum
class HypervisorState(Enum):
ONLINE = 'ONLINE'
OFFLINE = 'OFFLINE'
ONLINE = 'up'
OFFLINE = 'down'
ENABLED = 'enabled'
DISABLED = 'disabled'

25
watcher/decision_engine/framework/model/mapping.py
View File

@@ -37,15 +37,15 @@ class Mapping(object):
self.lock.acquire()
# init first
if hypervisor.get_uuid() not in self._mapping_hypervisors.keys():
self._mapping_hypervisors[hypervisor.get_uuid()] = []
if hypervisor.uuid not in self._mapping_hypervisors.keys():
self._mapping_hypervisors[hypervisor.uuid] = []
# map node => vms
self._mapping_hypervisors[hypervisor.get_uuid()].append(
vm.get_uuid())
self._mapping_hypervisors[hypervisor.uuid].append(
vm.uuid)
# map vm => node
self.mapping_vm[vm.get_uuid()] = hypervisor.get_uuid()
self.mapping_vm[vm.uuid] = hypervisor.uuid
finally:
self.lock.release()
@@ -56,16 +56,23 @@ class Mapping(object):
:param hypervisor: the hypervisor
:param vm: the virtual machine or instance
"""
self.unmap_from_id(hypervisor.get_uuid(), vm.get_uuid())
self.unmap_from_id(hypervisor.uuid, vm.uuid)
def unmap_from_id(self, node_uuid, vm_uuid):
"""
:rtype : object
"""
try:
self.lock.acquire()
if str(node_uuid) in self._mapping_hypervisors:
self._mapping_hypervisors[str(node_uuid)].remove(str(vm_uuid))
# remove vm
self.mapping_vm.pop(vm_uuid)
else:
LOG.warn("trying to delete the virtual machine " + str(
vm_uuid) + " but it was not found")
vm_uuid) + " but it was not found on hypervisor" + str(
node_uuid))
finally:
self.lock.release()
@@ -76,7 +83,7 @@ class Mapping(object):
return self.mapping_vm
def get_node_from_vm(self, vm):
return self.get_node_from_vm_id(vm.get_uuid())
return self.get_node_from_vm_id(vm.uuid)
def get_node_from_vm_id(self, vm_uuid):
"""Getting host information from the guest VM
@@ -93,7 +100,7 @@ class Mapping(object):
:param hypervisor:
:return:
"""
return self.get_node_vms_from_id(hypervisor.get_uuid())
return self.get_node_vms_from_id(hypervisor.uuid)
def get_node_vms_from_id(self, node_uuid):
if str(node_uuid) in self._mapping_hypervisors.keys():

17
watcher/decision_engine/framework/model/model_root.py
View File

@@ -15,6 +15,7 @@
# limitations under the License.
from watcher.common.exception import HypervisorNotFound
from watcher.common.exception import IllegalArgumentException
from watcher.common.exception import VMNotFound
from watcher.decision_engine.framework.model.hypervisor import Hypervisor
from watcher.decision_engine.framework.model.mapping import Mapping
@@ -33,26 +34,28 @@ class ModelRoot(object):
def assert_hypervisor(self, hypervisor):
if not isinstance(hypervisor, Hypervisor):
raise Exception("assert_vm")
raise IllegalArgumentException(
"Hypervisor must be an instance of hypervisor")
def assert_vm(self, vm):
if not isinstance(vm, VM):
raise Exception("assert_vm")
raise IllegalArgumentException(
"VM must be an instance of VM")
def add_hypervisor(self, hypervisor):
self.assert_hypervisor(hypervisor)
self._hypervisors[hypervisor.get_uuid()] = hypervisor
self._hypervisors[hypervisor.uuid] = hypervisor
def remove_hypervisor(self, hypervisor):
self.assert_hypervisor(hypervisor)
if str(hypervisor.get_uuid()) not in self._hypervisors.keys():
raise HypervisorNotFound(hypervisor.get_uuid())
if str(hypervisor.uuid) not in self._hypervisors.keys():
raise HypervisorNotFound(hypervisor.uuid)
else:
del self._hypervisors[hypervisor.get_uuid()]
del self._hypervisors[hypervisor.uuid]
def add_vm(self, vm):
self.assert_vm(vm)
self._vms[vm.get_uuid()] = vm
self._vms[vm.uuid] = vm
def get_all_hypervisors(self):
return self._hypervisors

21
watcher/decision_engine/framework/model/named_element.py
View File

@@ -18,13 +18,24 @@
class NamedElement(object):
def __init__(self):
self.uuid = ""
self._uuid = ""
self._human_id = ""
def set_uuid(self, uuid):
self.uuid = uuid
@property
def uuid(self):
return self._uuid
def get_uuid(self):
return self.uuid
@uuid.setter
def uuid(self, u):
self._uuid = u
@property
def human_id(self):
return self._human_id
@human_id.setter
def human_id(self, h):
self._human_id = h
def __str__(self):
return "[" + str(self.uuid) + "]"

4
watcher/decision_engine/framework/model/resource.py
View File

@@ -39,7 +39,7 @@ class Resource(object):
return self.name
def set_capacity(self, element, value):
self.mapping[element.get_uuid()] = value
self.mapping[element.uuid] = value
def get_capacity_from_id(self, uuid):
if str(uuid) in self.mapping.keys():
@@ -49,4 +49,4 @@ class Resource(object):
return None
def get_capacity(self, element):
return self.get_capacity_from_id(element.get_uuid())
return self.get_capacity_from_id(element.uuid)

12
watcher/decision_engine/framework/model/vm.py
View File

@@ -19,10 +19,12 @@ from watcher.decision_engine.framework.model.vm_state import VMState
class VM(NamedElement):
def __init__(self):
self.state = VMState.INIT
self._state = VMState.ACTIVE.value
def set_state(self, state):
self.state = state
@property
def state(self):
return self._state
def get_state(self):
return self.state
@state.setter
def state(self, state):
self._state = state

20
watcher/decision_engine/framework/model/vm_state.py
View File

@@ -18,9 +18,17 @@ from enum import Enum
class VMState(Enum):
INIT = 1,
READY = 2,
RUNNING = 3,
SLEEPING = 4,
KILLED = 5,
LIVE_MIGRATION = 6
ACTIVE = 'active' # VM is running
BUILDING = 'building' # VM only exists in DB
PAUSED = 'paused'
SUSPENDED = 'suspended' # VM is suspended to disk.
STOPPED = 'stopped' # VM is powered off, the disk image is still there.
RESCUED = 'rescued' # A rescue image is running with the original VM image
# attached.
RESIZED = 'resized' # a VM with the new size is active.
SOFT_DELETED = 'soft-delete'
# still available to restore.
DELETED = 'deleted' # VM is permanently deleted.
ERROR = 'error'

117
watcher/decision_engine/framework/ressourcedb_collector.py
View File

@@ -1,117 +0,0 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import ceilometerclient.v2 as c_client
import keystoneclient.v3.client as ksclient
from oslo_config import cfg
CONF = cfg.CONF
from watcher.decision_engine.api.collector.metrics_resource_collector import \
MetricsResourceCollector
CONF.import_opt('admin_user', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
CONF.import_opt('admin_tenant_name', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
CONF.import_opt('admin_password', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
CONF.import_opt('auth_uri', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
class RessourceDB(MetricsResourceCollector):
def __init__(self):
creds = \
{'auth_url': CONF.keystone_authtoken.auth_uri,
'username': CONF.keystone_authtoken.admin_user,
'password': CONF.keystone_authtoken.admin_password,
'project_name': CONF.keystone_authtoken.admin_tenant_name,
'user_domain_name': "default",
'project_domain_name': "default"}
self.keystone = ksclient.Client(**creds)
self.ceilometer = c_client.Client(
endpoint=self.get_ceilometer_uri(),
token=self.keystone.auth_token)
def make_query(user_id=None, tenant_id=None, resource_id=None,
user_ids=None, tenant_ids=None, resource_ids=None):
"""Returns query built form given parameters.
This query can be then used for querying resources, meters and
statistics.
:Parameters:
- `user_id`: user_id, has a priority over list of ids
- `tenant_id`: tenant_id, has a priority over list of ids
- `resource_id`: resource_id, has a priority over list of ids
- `user_ids`: list of user_ids
- `tenant_ids`: list of tenant_ids
- `resource_ids`: list of resource_ids
"""
user_ids = user_ids or []
tenant_ids = tenant_ids or []
resource_ids = resource_ids or []
query = []
if user_id:
user_ids = [user_id]
for u_id in user_ids:
query.append({"field": "user_id", "op": "eq", "value": u_id})
if tenant_id:
tenant_ids = [tenant_id]
for t_id in tenant_ids:
query.append({"field": "project_id", "op": "eq", "value": t_id})
if resource_id:
resource_ids = [resource_id]
for r_id in resource_ids:
query.append({"field": "resource_id", "op": "eq", "value": r_id})
return query
def get_ceilometer_uri(self):
a = self.keystone.services.list(**{'type': 'metering'})
e = self.keystone.endpoints.list()
for s in e:
if s.service_id == a[0].id and s.interface == 'internal':
return s.url
raise Exception("Ceilometer Metering Service internal not defined")
def get_average_usage_vm_cpu(self, instance_uuid):
"""The last VM CPU usage values to average
:param uuid:00
:return:
"""
# query influxdb stream
query = self.make_query(resource_id=instance_uuid)
cpu_util_sample = self.ceilometer.samples.list('cpu_util',
q=query)
cpu_usage = 0
count = len(cpu_util_sample)
for each in cpu_util_sample:
# print each.timestamp, each.counter_name, each.counter_volume
cpu_usage = cpu_usage + each.counter_volume
if count == 0:
return 0
else:
return cpu_usage / len(cpu_util_sample)
def get_average_usage_vm_memory(self, uuid):
# Obtaining Memory Usage is not implemented for LibvirtInspector
# waiting for kilo memory.resident
return 1
def get_average_usage_vm_disk(self, uuid):
# waiting for kilo disk.usage
return 1

3
watcher/decision_engine/framework/rpcapi.py
View File

@@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from oslo_config import cfg

104
watcher/decision_engine/framework/statedb_collector.py
View File

@@ -1,104 +0,0 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from keystoneclient.auth.identity import v3
from keystoneclient import session
from watcher.applier.framework.command.wrapper.nova_wrapper import NovaWrapper
from watcher.decision_engine.api.collector.cluster_state_collector import \
ClusterStateCollector
from watcher.decision_engine.framework.model.hypervisor import Hypervisor
from watcher.decision_engine.framework.model.model_root import ModelRoot
from watcher.decision_engine.framework.model.resource import Resource
from watcher.decision_engine.framework.model.resource import ResourceType
from watcher.decision_engine.framework.model.vm import VM
from watcher.openstack.common import log
from oslo_config import cfg
CONF = cfg.CONF
LOG = log.getLogger(__name__)
CONF.import_opt('admin_user', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
CONF.import_opt('admin_tenant_name', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
CONF.import_opt('admin_password', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
CONF.import_opt('auth_uri', 'keystonemiddleware.auth_token',
group='keystone_authtoken')
class NovaCollector(ClusterStateCollector):
def get_latest_state_cluster(self):
try:
creds = \
{'auth_url': CONF.keystone_authtoken.auth_uri,
'username': CONF.keystone_authtoken.admin_user,
'password': CONF.keystone_authtoken.admin_password,
'project_name': CONF.keystone_authtoken.admin_tenant_name,
'user_domain_name': "default",
'project_domain_name': "default"}
auth = v3.Password(auth_url=creds['auth_url'],
username=creds['username'],
password=creds['password'],
project_name=creds['project_name'],
user_domain_name=creds[
'user_domain_name'],
project_domain_name=creds[
'project_domain_name'])
sess = session.Session(auth=auth)
wrapper = NovaWrapper(creds, session=sess)
cluster = ModelRoot()
mem = Resource(ResourceType.memory)
num_cores = Resource(ResourceType.cpu_cores)
disk = Resource(ResourceType.disk)
cluster.create_resource(mem)
cluster.create_resource(num_cores)
cluster.create_resource(disk)
flavor_cache = {}
hypervisors = wrapper.get_hypervisors_list()
for h in hypervisors:
i = h.hypervisor_hostname.index('.')
name = h.hypervisor_hostname[0:i]
# create hypervisor in stateDB
hypervisor = Hypervisor()
hypervisor.set_uuid(name)
# set capacity
mem.set_capacity(hypervisor, h.memory_mb)
disk.set_capacity(hypervisor, h.disk_available_least)
num_cores.set_capacity(hypervisor, h.vcpus)
cluster.add_hypervisor(hypervisor)
vms = wrapper.get_vms_by_hypervisor(str(name))
for v in vms:
# create VM in stateDB
vm = VM()
vm.set_uuid(v.id)
# set capacity
wrapper.get_flavor_instance(v, flavor_cache)
mem.set_capacity(vm, v.flavor['ram'])
disk.set_capacity(vm, v.flavor['disk'])
num_cores.set_capacity(vm, v.flavor['vcpus'])
# print(dir(v))
cluster.get_mapping().map(hypervisor, vm)
cluster.add_vm(vm)
return cluster
except Exception as e:
LOG.error("nova collector " + unicode(e))
return None

12
watcher/decision_engine/framework/strategy/strategy_loader.py
View File

@@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -13,7 +15,9 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from oslo_config import cfg
from stevedore import driver
from watcher.decision_engine.strategies.basic_consolidation import \
BasicConsolidation
from watcher.openstack.common import log
@@ -52,5 +56,13 @@ class StrategyLoader(object):
"Basic offline consolidation")
}
def load_driver(self, algo):
_algo = driver.DriverManager(
namespace='watcher_strategies',
name=algo,
invoke_on_load=True,
)
return _algo
def load(self, model):
return self.strategies[model]

3
watcher/decision_engine/framework/strategy/strategy_selector.py
View File

@@ -15,6 +15,7 @@
# limitations under the License.
from oslo_config import cfg
from watcher.decision_engine.api.strategy.selector import Selector
from watcher.decision_engine.framework.strategy.strategy_loader import \
StrategyLoader
from watcher.objects.audit_template import Goal
@@ -40,7 +41,7 @@ CONF.register_group(goals_opt_group)
CONF.register_opts(WATCHER_GOALS_OPTS, goals_opt_group)
class StrategySelector(object):
class StrategySelector(Selector):
def __init__(self):
self.strategy_loader = StrategyLoader()

117
watcher/decision_engine/strategies/basic_consolidation.py
View File

@@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -13,10 +15,15 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.decision_engine.framework.model.vm_state import VMState
from watcher.metrics_engine.api.metrics_resource_collector import \
AggregationFunction
from watcher.common.exception import ClusterEmpty
from watcher.common.exception import ClusteStateNotDefined
from watcher.common.exception import MetricCollectorNotDefined
from watcher.common.exception import NoDataFound
from watcher.decision_engine.api.strategy.strategy import Strategy
from watcher.decision_engine.api.strategy.strategy import StrategyLevel
from watcher.decision_engine.framework.meta_actions.hypervisor_state import \
@@ -78,8 +85,8 @@ class BasicConsolidation(Strategy):
self.efficiency = 100
# TODO(jed) improve threshold overbooking ?,...
self.threshold_mem = 0.90
self.threshold_disk = 0.80
self.threshold_mem = 1
self.threshold_disk = 1
self.threshold_cores = 1
# TODO(jed) target efficiency
@@ -115,6 +122,11 @@ class BasicConsolidation(Strategy):
if src_hypervisor == dest_hypervisor:
return False
LOG.debug('Migrate VM %s from %s to %s ',
str(src_hypervisor),
str(dest_hypervisor),
str(vm_to_mig))
total_cores = 0
total_disk = 0
total_mem = 0
@@ -162,6 +174,13 @@ class BasicConsolidation(Strategy):
cores_available = cap_cores.get_capacity(dest_hypervisor)
disk_available = cap_disk.get_capacity(dest_hypervisor)
mem_available = cap_mem.get_capacity(dest_hypervisor)
LOG.debug("VCPU %s/%s ", str(total_cores * self.threshold_cores),
str(cores_available), )
LOG.debug("DISK %s/%s ", str(total_disk * self.threshold_disk),
str(disk_available), )
LOG.debug("MEM %s/%s ", str(total_mem * self.threshold_mem),
str(mem_available))
if cores_available >= total_cores * self.threshold_cores \
and disk_available >= total_disk * self.threshold_disk \
and mem_available >= total_mem * self.threshold_mem:
@@ -232,21 +251,25 @@ class BasicConsolidation(Strategy):
metrics_collector = self.get_metrics_resource_collector()
if metrics_collector is None:
raise MetricCollectorNotDefined()
total_cores_used = 0
total_memory_used = 0
total_disk_used = 0
for vm_id in model.get_mapping().get_node_vms(hypervisor):
total_cores_used += metrics_collector.get_average_usage_vm_cpu(
vm_id)
total_memory_used += metrics_collector.get_average_usage_vm_memory(
vm_id)
total_disk_used += metrics_collector.get_average_usage_vm_disk(
vm_id)
cpu_compute_mean_16h = metrics_collector.get_measurement(
metric='compute_cpu_user_percent_gauge',
aggregation_function=AggregationFunction.MEAN,
start_time="16 hours before now",
filters=["resource_id=" + hypervisor.uuid + ""])
if len(cpu_compute_mean_16h) > 0:
cpu_capacity = model.get_resource_from_id(
ResourceType.cpu_cores).get_capacity(hypervisor)
cpu_utilization = float(cpu_compute_mean_16h[0].value)
total_cores_used = cpu_capacity * (cpu_utilization / 100)
else:
raise NoDataFound(
"No values returned for " + str(hypervisor.uuid) +
" compute_cpu_percent_gauge")
return self.calculate_weight(model, hypervisor, total_cores_used,
total_disk_used,
total_memory_used)
0,
0)
def calculate_migration_efficiency(self):
"""Calculate migration efficiency
@@ -275,15 +298,25 @@ class BasicConsolidation(Strategy):
if model is None:
raise ClusteStateNotDefined()
vm = model.get_vm_from_id(vm.get_uuid())
cores_used = metric_collector.get_average_usage_vm_cpu(vm.get_uuid())
memory_used = metric_collector.get_average_usage_vm_memory(
vm.get_uuid())
disk_used = metric_collector.get_average_usage_vm_disk(vm.get_uuid())
vm = model.get_vm_from_id(vm.uuid)
instance_cpu_mean_16 = metric_collector.get_measurement(
metric='instance_cpu_percent_gauge',
aggregation_function=AggregationFunction.MEAN,
start_time="16 hours before now",
filters=["resource_id=" + vm.uuid + ""])
return self.calculate_weight(model, vm, cores_used,
disk_used,
memory_used)
if len(instance_cpu_mean_16) > 0:
cpu_capacity = model.get_resource_from_id(
ResourceType.cpu_cores).get_capacity(vm)
vm_cpu_utilization = instance_cpu_mean_16[0].value
total_cores_used = cpu_capacity * (vm_cpu_utilization / 100)
else:
raise NoDataFound("No values returned for " + str(vm.uuid) +
" instance_cpu_percent_gauge")
return self.calculate_weight(model, vm, total_cores_used,
0,
0)
def print_utilization(self, model):
if model is None:
@@ -308,13 +341,27 @@ class BasicConsolidation(Strategy):
unsuccessful_migration = 0
first = True
self.print_utilization(current_model)
size_cluster = len(current_model.get_all_hypervisors())
if size_cluster == 0:
raise ClusterEmpty()
self.compute_attempts(size_cluster)
for hypevisor_id in current_model.get_all_hypervisors():
hypervisor = current_model.get_hypervisor_from_id(hypevisor_id)
count = current_model.get_mapping(). \
get_node_vms_from_id(hypevisor_id)
if len(count) == 0:
change_power = ChangePowerState(hypervisor)
change_power.powerstate = PowerState.g1_S1
change_power.set_level(StrategyLevel.conservative)
self.solution.add_change_request(change_power)
if hypervisor.state == HypervisorState.ONLINE:
h = ChangeHypervisorState(hypervisor)
h.set_level(StrategyLevel.aggressive)
h.state = HypervisorState.OFFLINE
self.solution.add_change_request(h)
while self.get_allowed_migration_attempts() >= unsuccessful_migration:
if first is not True:
self.efficiency = self.calculate_migration_efficiency()
@@ -325,9 +372,16 @@ class BasicConsolidation(Strategy):
''' calculate score of nodes based on load by VMs '''
for hypevisor_id in current_model.get_all_hypervisors():
hypevisor = current_model.get_hypervisor_from_id(hypevisor_id)
result = self.calculate_score_node(hypevisor, current_model)
if result != 0:
hypervisor = current_model.get_hypervisor_from_id(hypevisor_id)
count = current_model.get_mapping(). \
get_node_vms_from_id(hypevisor_id)
if len(count) > 0:
result = self.calculate_score_node(hypervisor,
current_model)
else:
''' the hypervisor has not VMs '''
result = 0
if len(count) > 0:
score.append((hypevisor_id, result))
''' sort compute nodes by Score decreasing '''''
@@ -350,8 +404,9 @@ class BasicConsolidation(Strategy):
vm_score = []
for vm_id in vms_to_mig:
vm = current_model.get_vm_from_id(vm_id)
vm_score.append(
(vm_id, self.calculate_score_vm(vm, current_model)))
if vm.state == VMState.ACTIVE.value:
vm_score.append(
(vm_id, self.calculate_score_vm(vm, current_model)))
''' sort VM's by Score '''
v = sorted(vm_score, reverse=True, key=lambda x: (x[1]))
@@ -392,7 +447,7 @@ class BasicConsolidation(Strategy):
# TODO(jed) how to manage strategy level
# from conservative to aggressive
change_power = ChangePowerState(mig_src_hypervisor)
change_power.set_power_state(PowerState.g1_S1)
change_power.powerstate = PowerState.g1_S1
change_power.set_level(
StrategyLevel.conservative)
tmp_vm_migration_schedule.append(change_power)
@@ -400,7 +455,7 @@ class BasicConsolidation(Strategy):
h = ChangeHypervisorState(mig_src_hypervisor)
h.set_level(StrategyLevel.aggressive)
h.set_state(HypervisorState.OFFLINE)
h.state = HypervisorState.OFFLINE
tmp_vm_migration_schedule.append(h)
self.number_of_released_nodes += 1
@@ -414,7 +469,7 @@ class BasicConsolidation(Strategy):
self.solution.add_change_request(a)
else:
unsuccessful_migration += 1
self.print_utilization(current_model)
# self.print_utilization(current_model)
infos = {
"number_of_migrations": self.number_of_migrations,
"number_of_nodes_released": self.number_of_released_nodes,

3
watcher/decision_engine/strategies/dummy_strategy.py
View File

@@ -1,6 +1,8 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Authors: Jean-Emile DARTOIS <jean-emile.dartois@b-com.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -13,6 +15,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from watcher.decision_engine.api.strategy.strategy import Strategy
from watcher.openstack.common import log