diff --git a/watcher/applier/framework/command/hypervisor_state_command.py b/watcher/applier/framework/command/hypervisor_state_command.py index a0fdbba29..95d8b9f42 100644 --- a/watcher/applier/framework/command/hypervisor_state_command.py +++ b/watcher/applier/framework/command/hypervisor_state_command.py @@ -23,9 +23,9 @@ from oslo_config import cfg from watcher.applier.api.primitive_command import PrimitiveCommand from watcher.applier.api.promise import Promise from watcher.applier.framework.command.wrapper.nova_wrapper import NovaWrapper -from watcher.common.keystone import Client -from watcher.decision_engine.model.hypervisor_state import \ - HypervisorState + +from watcher.common.keystone import KeystoneClient +from watcher.decision_engine.model.hypervisor_state import HypervisorState CONF = cfg.CONF @@ -36,7 +36,7 @@ class HypervisorStateCommand(PrimitiveCommand): self.status = status def nova_manage_service(self, status): - keystone = Client() + keystone = KeystoneClient() wrapper = NovaWrapper(keystone.get_credentials(), session=keystone.get_session()) if status is True: diff --git a/watcher/applier/framework/command/migrate_command.py b/watcher/applier/framework/command/migrate_command.py index 13cad4adc..8eb05ae2e 100644 --- a/watcher/applier/framework/command/migrate_command.py +++ b/watcher/applier/framework/command/migrate_command.py @@ -24,7 +24,8 @@ from oslo_config import cfg from watcher.applier.api.primitive_command import PrimitiveCommand from watcher.applier.api.promise import Promise from watcher.applier.framework.command.wrapper.nova_wrapper import NovaWrapper -from watcher.common.keystone import Client + +from watcher.common.keystone import KeystoneClient from watcher.decision_engine.planner.default import Primitives CONF = cfg.CONF @@ -41,7 +42,7 @@ class MigrateCommand(PrimitiveCommand): self.destination_hypervisor = destination_hypervisor def migrate(self, destination): - keystone = Client() + keystone = KeystoneClient() wrapper = NovaWrapper(keystone.get_credentials(), session=keystone.get_session()) instance = wrapper.find_instance(self.instance_uuid) diff --git a/watcher/common/ceilometer.py b/watcher/common/ceilometer.py index bdb15dd06..1047e0afb 100644 --- a/watcher/common/ceilometer.py +++ b/watcher/common/ceilometer.py @@ -16,34 +16,32 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from urlparse import urlparse from ceilometerclient import client from ceilometerclient.exc import HTTPUnauthorized from watcher.common import keystone -class Client(object): - # todo olso conf: this must be sync with ceilometer - CEILOMETER_API_VERSION = '2' +class CeilometerClient(object): + def __init__(self, api_version='2'): + self._cmclient = None + self._api_version = api_version - def __init__(self): - ksclient = keystone.Client() - self.creds = ksclient.get_credentials() - self.creds['os_auth_token'] = ksclient.get_token() - self.creds['token'] = ksclient.get_token() - self.creds['ceilometer_url'] = "http://" + urlparse( - ksclient.get_endpoint( - service_type='metering', - endpoint_type='publicURL')).netloc - self.connect() - - def connect(self): + @property + def cmclient(self): """Initialization of Ceilometer client.""" - self.cmclient = client.get_client(self.CEILOMETER_API_VERSION, - **self.creds) + if not self._cmclient: + ksclient = keystone.KeystoneClient() + creds = ksclient.get_credentials() + endpoint = ksclient.get_endpoint( + service_type='metering', + endpoint_type='publicURL') + self._cmclient = client.get_client(self._api_version, + ceilometer_url=endpoint, + **creds) + return self._cmclient - def build_query(user_id=None, tenant_id=None, resource_id=None, + def build_query(self, user_id=None, tenant_id=None, resource_id=None, user_ids=None, tenant_ids=None, resource_ids=None): """Returns query built from given parameters. This query can be then used for querying resources, meters and @@ -78,37 +76,32 @@ class Client(object): return query - def query_sample(self, meter_name, query, limit=1): + def query_retry(self, f, *args, **kargs): try: - samples = self.ceilometerclient().samples.list( - meter_name=meter_name, - limit=limit, - q=query) + return f(*args, **kargs) except HTTPUnauthorized: - self.connect() - samples = self.ceilometerclient().samples.list( - meter_name=meter_name, - limit=limit, - q=query) + self.reset_client() + return f(*args, **kargs) except Exception: raise - return samples - def get_endpoint(self, service_type, endpoint_type=None): - ksclient = keystone.Client() - endpoint = ksclient.get_endpoint(service_type=service_type, - endpoint_type=endpoint_type) - return endpoint + def query_sample(self, meter_name, query, limit=1): + return self.query_retry(f=self.cmclient.samples.list, + meter_name=meter_name, + limit=limit, + q=query) def statistic_list(self, meter_name, query=None, period=None): """List of statistics.""" - statistics = self.ceilometerclient().statistics.list( - meter_name=meter_name, q=query, period=period) + statistics = self.cmclient.statistics.list( + meter_name=meter_name, + q=query, + period=period) return statistics def meter_list(self, query=None): """List the user's meters.""" - meters = self.ceilometerclient().meters.list(query) + meters = self.query_retry(f=self.cmclient.meters.list, query=query) return meters def statistic_aggregation(self, @@ -129,25 +122,14 @@ class Client(object): """Representing a statistic aggregate by operators""" query = self.build_query(resource_id=resource_id) - try: - statistic = self.cmclient.statistics.list( - meter_name=meter_name, - q=query, - period=period, - aggregates=[ - {'func': aggregate}], - groupby=['resource_id']) - except HTTPUnauthorized: - self.connect() - statistic = self.cmclient.statistics.list( - meter_name=meter_name, - q=query, - period=period, - aggregates=[ - {'func': aggregate}], - groupby=['resource_id']) - except Exception: - raise + statistic = self.query_retry(f=self.cmclient.statistics.list, + meter_name=meter_name, + q=query, + period=period, + aggregates=[ + {'func': aggregate}], + groupby=['resource_id']) + item_value = None if statistic: item_value = statistic[-1]._info.get('aggregate').get('avg') @@ -171,3 +153,6 @@ class Client(object): return samples[-1]._info['counter_volume'] else: return False + + def reset_client(self): + self._cmclient = None diff --git a/watcher/common/exception.py b/watcher/common/exception.py index 31f2bd600..47c9e8bdc 100644 --- a/watcher/common/exception.py +++ b/watcher/common/exception.py @@ -282,6 +282,10 @@ class NoDataFound(BaseException): return self._desc +class KeystoneFailure(WatcherException): + message = _("'Keystone API endpoint is missing''") + + class ClusterEmpty(WatcherException): message = _("The list of hypervisor(s) in the cluster is empty.'") diff --git a/watcher/common/keystone.py b/watcher/common/keystone.py index 69b7ad84a..e79cf9de9 100644 --- a/watcher/common/keystone.py +++ b/watcher/common/keystone.py @@ -16,14 +16,19 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import datetime -from keystoneclient.auth.identity import v3 -from keystoneclient import session -import keystoneclient.v3.client as ksclient from oslo_config import cfg from oslo_log import log +from urlparse import urljoin +from urlparse import urlparse + +from keystoneclient.auth.identity import generic +from keystoneclient import session as keystone_session + +from watcher.common.exception import KeystoneFailure + + LOG = log.getLogger(__name__) CONF = cfg.CONF @@ -35,30 +40,66 @@ CONF.import_opt('admin_password', 'keystonemiddleware.auth_token', group='keystone_authtoken') CONF.import_opt('auth_uri', 'keystonemiddleware.auth_token', group='keystone_authtoken') +CONF.import_opt('auth_version', 'keystonemiddleware.auth_token', + group='keystone_authtoken') +CONF.import_opt('insecure', 'keystonemiddleware.auth_token', + group='keystone_authtoken') -class Client(object): +class KeystoneClient(object): def __init__(self): - ks_args = self.get_credentials() - self.ks_client = ksclient.Client(**ks_args) + self._ks_client = None + self._session = None + self._auth = None + self._token = None def get_endpoint(self, **kwargs): + kc = self._get_ksclient() + if not kc.has_service_catalog(): + raise KeystoneFailure('No Keystone service catalog ' + 'loaded') attr = None filter_value = None if kwargs.get('region_name'): attr = 'region' filter_value = kwargs.get('region_name') - return self.ks_client.service_catalog.url_for( + return self._get_ksclient().service_catalog.url_for( service_type=kwargs.get('service_type') or 'metering', attr=attr, filter_value=filter_value, endpoint_type=kwargs.get('endpoint_type') or 'publicURL') - def get_token(self): - return self.ks_client.auth_token + def _is_apiv3(self, auth_url, auth_version): + return auth_version == 'v3.0' or '/v3' in urlparse(auth_url).path - @staticmethod - def get_credentials(): + def get_keystone_url(self, auth_url, auth_version): + """Gives an http/https url to contact keystone. + """ + api_v3 = self._is_apiv3(auth_url, auth_version) + api_version = 'v3' if api_v3 else 'v2.0' + # NOTE(lucasagomes): Get rid of the trailing '/' otherwise urljoin() + # fails to override the version in the URL + return urljoin(auth_url.rstrip('/'), api_version) + + def _get_ksclient(self): + """Get an endpoint and auth token from Keystone. + """ + ks_args = self.get_credentials() + auth_version = CONF.keystone_authtoken.auth_version + auth_url = CONF.keystone_authtoken.auth_uri + api_version = self._is_apiv3(auth_url, auth_version) + + if api_version: + from keystoneclient.v3 import client + else: + from keystoneclient.v2_0 import client + # generic + # ksclient = client.Client(version=api_version, + # session=session,**ks_args) + + return client.Client(**ks_args) + + def get_credentials(self): creds = \ {'auth_url': CONF.keystone_authtoken.auth_uri, 'username': CONF.keystone_authtoken.admin_user, @@ -71,10 +112,6 @@ class Client(object): def get_session(self): creds = self.get_credentials() - auth = v3.Password(**creds) - return session.Session(auth=auth) - - def is_token_expired(self, token): - expires = datetime.datetime.strptime(token['expires'], - '%Y-%m-%dT%H:%M:%SZ') - return datetime.datetime.now() >= expires + self._auth = generic.Password(**creds) + session = keystone_session.Session(auth=self._auth) + return session diff --git a/watcher/metrics_engine/cluster_history/ceilometer.py b/watcher/metrics_engine/cluster_history/ceilometer.py index 601b57cee..a1faee275 100644 --- a/watcher/metrics_engine/cluster_history/ceilometer.py +++ b/watcher/metrics_engine/cluster_history/ceilometer.py @@ -20,7 +20,7 @@ from oslo_config import cfg from oslo_log import log -from watcher.common.ceilometer import Client +from watcher.common.ceilometer import CeilometerClient from watcher.metrics_engine.cluster_history.api import BaseClusterHistory @@ -30,7 +30,7 @@ LOG = log.getLogger(__name__) class CeilometerClusterHistory(BaseClusterHistory): def __init__(self): - self.ceilometer = Client() + self.ceilometer = CeilometerClient() def statistic_list(self, meter_name, query=None, period=None): return self.ceilometer.statistic_list(meter_name, query, period) diff --git a/watcher/metrics_engine/cluster_model_collector/manager.py b/watcher/metrics_engine/cluster_model_collector/manager.py index f024bcd4d..b4e215d99 100644 --- a/watcher/metrics_engine/cluster_model_collector/manager.py +++ b/watcher/metrics_engine/cluster_model_collector/manager.py @@ -21,7 +21,7 @@ from oslo_config import cfg from oslo_log import log from watcher.applier.framework.command.wrapper.nova_wrapper import NovaWrapper -from watcher.common.keystone import Client +from watcher.common.keystone import KeystoneClient from watcher.metrics_engine.cluster_model_collector.nova import \ NovaClusterModelCollector @@ -31,7 +31,7 @@ CONF = cfg.CONF class CollectorManager(object): def get_cluster_model_collector(self): - keystone = Client() + keystone = KeystoneClient() wrapper = NovaWrapper(keystone.get_credentials(), session=keystone.get_session()) return NovaClusterModelCollector(wrapper=wrapper) diff --git a/watcher/tests/common/ceilometer.py b/watcher/tests/common/ceilometer.py new file mode 100644 index 000000000..90b9c0659 --- /dev/null +++ b/watcher/tests/common/ceilometer.py @@ -0,0 +1,93 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import unicode_literals + +from mock import MagicMock +from mock import mock + +from watcher.common.ceilometer import CeilometerClient + +from watcher.tests.base import BaseTestCase + + +class TestCeilometer(BaseTestCase): + def setUp(self): + super(TestCeilometer, self).setUp() + self.cm = CeilometerClient() + + def test_build_query(self): + expected = [{'field': 'user_id', 'op': 'eq', 'value': u'user_id'}, + {'field': 'project_id', 'op': 'eq', 'value': u'tenant_id'}, + {'field': 'resource_id', 'op': 'eq', + 'value': u'resource_id'}] + + query = self.cm.build_query(user_id="user_id", + tenant_id="tenant_id", + resource_id="resource_id", + user_ids=["user_ids"], + tenant_ids=["tenant_ids"], + resource_ids=["resource_ids"]) + self.assertEqual(query, expected) + + @mock.patch("watcher.common.keystone.Keystoneclient") + def test_get_ceilometer_v2(self, mock_keystone): + c = CeilometerClient(api_version='2') + from ceilometerclient.v2 import Client + self.assertIsInstance(c.cmclient, Client) + + @mock.patch.object(CeilometerClient, "cmclient") + def test_statistic_aggregation(self, mock_keystone): + statistic = MagicMock() + expected_result = 100 + statistic[-1]._info = {'aggregate': {'avg': expected_result}} + mock_keystone.statistics.list.return_value = statistic + val = self.cm.statistic_aggregation( + resource_id="VM_ID", + meter_name="cpu_util", + period="7300" + ) + self.assertEqual(val, expected_result) + + @mock.patch.object(CeilometerClient, "cmclient") + def test_get_last_sample(self, mock_keystone): + statistic = MagicMock() + expected_result = 100 + statistic[-1]._info = {'counter_volume': expected_result} + mock_keystone.samples.list.return_value = statistic + val = self.cm.get_last_sample_value( + resource_id="id", + meter_name="compute.node.percent" + ) + self.assertEqual(val, expected_result) + + @mock.patch.object(CeilometerClient, "cmclient") + def test_get_last_sample_none(self, mock_keystone): + expected = False + mock_keystone.samples.list.return_value = None + val = self.cm.get_last_sample_values( + resource_id="id", + meter_name="compute.node.percent" + ) + self.assertEqual(val, expected) + + @mock.patch.object(CeilometerClient, "cmclient") + def test_statistic_list(self, mock_keystone): + expected_value = [] + mock_keystone.statistics.list.return_value = expected_value + val = self.cm.statistic_list(meter_name="cpu_util") + self.assertEqual(val, expected_value) diff --git a/watcher/tests/common/keystone.py b/watcher/tests/common/keystone.py new file mode 100644 index 000000000..6b304cab9 --- /dev/null +++ b/watcher/tests/common/keystone.py @@ -0,0 +1,60 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2015 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import unicode_literals +from keystoneclient.auth.identity import Password + +from keystoneclient.session import Session +from mock import mock + +from watcher.common.keystone import KeystoneClient + +from watcher.tests.base import BaseTestCase + + +class TestKeyStone(BaseTestCase): + def setUp(self): + super(TestKeyStone, self).setUp() + self.ckeystone = KeystoneClient() + + @mock.patch('keystoneclient.client.Client') + def test_get_endpoint(self, keystone): + expected_endpoint = "http://IP:PORT" + ks = mock.Mock() + ks.service_catalog.url_for.return_value = expected_endpoint + keystone.return_value = ks + ep = self.ckeystone.get_endpoint(service_type='metering', + endpoint_type='publicURL', + region_name='RegionOne') + + self.assertEqual(ep, expected_endpoint) + + def test_get_session(self): + k = KeystoneClient() + session = k.get_session() + self.assertIsInstance(session.auth, Password) + self.assertIsInstance(session, Session) + + def test_get_credentials(self): + expected_creds = {'auth_url': None, + 'password': None, + 'project_domain_name': 'default', + 'project_name': 'admin', + 'user_domain_name': 'default', + 'username': None} + creds = self.ckeystone.get_credentials() + self.assertEqual(creds, expected_creds)