Merge "Improve OpenStack clients API"

watcher/applier/framework/command/hypervisor_state_command.py

@@ -23,9 +23,9 @@ from oslo_config import cfg
 from watcher.applier.api.primitive_command import PrimitiveCommand
 from watcher.applier.api.promise import Promise
 from watcher.applier.framework.command.wrapper.nova_wrapper import NovaWrapper
-from watcher.common.keystone import Client
+
-from watcher.decision_engine.model.hypervisor_state import \
+from watcher.common.keystone import KeystoneClient
-    HypervisorState
+from watcher.decision_engine.model.hypervisor_state import HypervisorState
 
 CONF = cfg.CONF
 
@@ -36,7 +36,7 @@ class HypervisorStateCommand(PrimitiveCommand):
         self.status = status
 
     def nova_manage_service(self, status):
-        keystone = Client()
+        keystone = KeystoneClient()
         wrapper = NovaWrapper(keystone.get_credentials(),
                               session=keystone.get_session())
         if status is True:

watcher/applier/framework/command/migrate_command.py

@@ -24,7 +24,8 @@ from oslo_config import cfg
 from watcher.applier.api.primitive_command import PrimitiveCommand
 from watcher.applier.api.promise import Promise
 from watcher.applier.framework.command.wrapper.nova_wrapper import NovaWrapper
-from watcher.common.keystone import Client
+
+from watcher.common.keystone import KeystoneClient
 from watcher.decision_engine.planner.default import Primitives
 
 CONF = cfg.CONF
@@ -41,7 +42,7 @@ class MigrateCommand(PrimitiveCommand):
         self.destination_hypervisor = destination_hypervisor
 
     def migrate(self, destination):
-        keystone = Client()
+        keystone = KeystoneClient()
         wrapper = NovaWrapper(keystone.get_credentials(),
                               session=keystone.get_session())
         instance = wrapper.find_instance(self.instance_uuid)

watcher/common/ceilometer.py

@@ -16,34 +16,32 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from urlparse import urlparse
 
 from ceilometerclient import client
 from ceilometerclient.exc import HTTPUnauthorized
 from watcher.common import keystone
 
 
-class Client(object):
+class CeilometerClient(object):
-    # todo olso conf: this must be sync with ceilometer
+    def __init__(self, api_version='2'):
-    CEILOMETER_API_VERSION = '2'
+        self._cmclient = None
+        self._api_version = api_version
 
-    def __init__(self):
+    @property
-        ksclient = keystone.Client()
+    def cmclient(self):
-        self.creds = ksclient.get_credentials()
-        self.creds['os_auth_token'] = ksclient.get_token()
-        self.creds['token'] = ksclient.get_token()
-        self.creds['ceilometer_url'] = "http://" + urlparse(
-            ksclient.get_endpoint(
-                service_type='metering',
-                endpoint_type='publicURL')).netloc
-        self.connect()
-
-    def connect(self):
         """Initialization of Ceilometer client."""
-        self.cmclient = client.get_client(self.CEILOMETER_API_VERSION,
+        if not self._cmclient:
-                                          **self.creds)
+            ksclient = keystone.KeystoneClient()
+            creds = ksclient.get_credentials()
+            endpoint = ksclient.get_endpoint(
+                service_type='metering',
+                endpoint_type='publicURL')
+            self._cmclient = client.get_client(self._api_version,
+                                               ceilometer_url=endpoint,
+                                               **creds)
+        return self._cmclient
 
-    def build_query(user_id=None, tenant_id=None, resource_id=None,
+    def build_query(self, user_id=None, tenant_id=None, resource_id=None,
                     user_ids=None, tenant_ids=None, resource_ids=None):
         """Returns query built from given parameters.
         This query can be then used for querying resources, meters and
@@ -78,37 +76,32 @@ class Client(object):
 
         return query
 
-    def query_sample(self, meter_name, query, limit=1):
+    def query_retry(self, f, *args, **kargs):
         try:
-            samples = self.ceilometerclient().samples.list(
+            return f(*args, **kargs)
-                meter_name=meter_name,
-                limit=limit,
-                q=query)
         except HTTPUnauthorized:
-            self.connect()
+            self.reset_client()
-            samples = self.ceilometerclient().samples.list(
+            return f(*args, **kargs)
-                meter_name=meter_name,
-                limit=limit,
-                q=query)
         except Exception:
             raise
-        return samples
 
-    def get_endpoint(self, service_type, endpoint_type=None):
+    def query_sample(self, meter_name, query, limit=1):
-        ksclient = keystone.Client()
+        return self.query_retry(f=self.cmclient.samples.list,
-        endpoint = ksclient.get_endpoint(service_type=service_type,
+                                meter_name=meter_name,
-                                         endpoint_type=endpoint_type)
+                                limit=limit,
-        return endpoint
+                                q=query)
 
     def statistic_list(self, meter_name, query=None, period=None):
         """List of statistics."""
-        statistics = self.ceilometerclient().statistics.list(
+        statistics = self.cmclient.statistics.list(
-            meter_name=meter_name, q=query, period=period)
+            meter_name=meter_name,
+            q=query,
+            period=period)
         return statistics
 
     def meter_list(self, query=None):
         """List the user's meters."""
-        meters = self.ceilometerclient().meters.list(query)
+        meters = self.query_retry(f=self.cmclient.meters.list, query=query)
         return meters
 
     def statistic_aggregation(self,
@@ -129,25 +122,14 @@ class Client(object):
         """Representing a statistic aggregate by operators"""
 
         query = self.build_query(resource_id=resource_id)
-        try:
+        statistic = self.query_retry(f=self.cmclient.statistics.list,
-            statistic = self.cmclient.statistics.list(
+                                     meter_name=meter_name,
-                meter_name=meter_name,
+                                     q=query,
-                q=query,
+                                     period=period,
-                period=period,
+                                     aggregates=[
-                aggregates=[
+                                         {'func': aggregate}],
-                    {'func': aggregate}],
+                                     groupby=['resource_id'])
-                groupby=['resource_id'])
+
-        except HTTPUnauthorized:
-            self.connect()
-            statistic = self.cmclient.statistics.list(
-                meter_name=meter_name,
-                q=query,
-                period=period,
-                aggregates=[
-                    {'func': aggregate}],
-                groupby=['resource_id'])
-        except Exception:
-            raise
         item_value = None
         if statistic:
             item_value = statistic[-1]._info.get('aggregate').get('avg')
@@ -171,3 +153,6 @@ class Client(object):
             return samples[-1]._info['counter_volume']
         else:
             return False
+
+    def reset_client(self):
+        self._cmclient = None

watcher/common/exception.py

@@ -282,6 +282,10 @@ class NoDataFound(BaseException):
         return self._desc
 
 
+class KeystoneFailure(WatcherException):
+    message = _("'Keystone API endpoint is missing''")
+
+
 class ClusterEmpty(WatcherException):
     message = _("The list of hypervisor(s) in the cluster is empty.'")
 

watcher/common/keystone.py

@@ -16,14 +16,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import datetime
 
-from keystoneclient.auth.identity import v3
-from keystoneclient import session
-import keystoneclient.v3.client as ksclient
 from oslo_config import cfg
 from oslo_log import log
 
+from urlparse import urljoin
+from urlparse import urlparse
+
+from keystoneclient.auth.identity import generic
+from keystoneclient import session as keystone_session
+
+from watcher.common.exception import KeystoneFailure
+
+
 LOG = log.getLogger(__name__)
 CONF = cfg.CONF
 
@@ -35,30 +40,66 @@ CONF.import_opt('admin_password', 'keystonemiddleware.auth_token',
                 group='keystone_authtoken')
 CONF.import_opt('auth_uri', 'keystonemiddleware.auth_token',
                 group='keystone_authtoken')
+CONF.import_opt('auth_version', 'keystonemiddleware.auth_token',
+                group='keystone_authtoken')
+CONF.import_opt('insecure', 'keystonemiddleware.auth_token',
+                group='keystone_authtoken')
 
 
-class Client(object):
+class KeystoneClient(object):
     def __init__(self):
-        ks_args = self.get_credentials()
+        self._ks_client = None
-        self.ks_client = ksclient.Client(**ks_args)
+        self._session = None
+        self._auth = None
+        self._token = None
 
     def get_endpoint(self, **kwargs):
+        kc = self._get_ksclient()
+        if not kc.has_service_catalog():
+            raise KeystoneFailure('No Keystone service catalog '
+                                  'loaded')
         attr = None
         filter_value = None
         if kwargs.get('region_name'):
             attr = 'region'
             filter_value = kwargs.get('region_name')
-        return self.ks_client.service_catalog.url_for(
+        return self._get_ksclient().service_catalog.url_for(
             service_type=kwargs.get('service_type') or 'metering',
             attr=attr,
             filter_value=filter_value,
             endpoint_type=kwargs.get('endpoint_type') or 'publicURL')
 
-    def get_token(self):
+    def _is_apiv3(self, auth_url, auth_version):
-        return self.ks_client.auth_token
+        return auth_version == 'v3.0' or '/v3' in urlparse(auth_url).path
 
-    @staticmethod
+    def get_keystone_url(self, auth_url, auth_version):
-    def get_credentials():
+        """Gives an http/https url to contact keystone.
+        """
+        api_v3 = self._is_apiv3(auth_url, auth_version)
+        api_version = 'v3' if api_v3 else 'v2.0'
+        # NOTE(lucasagomes): Get rid of the trailing '/' otherwise urljoin()
+        #   fails to override the version in the URL
+        return urljoin(auth_url.rstrip('/'), api_version)
+
+    def _get_ksclient(self):
+        """Get an endpoint and auth token from Keystone.
+        """
+        ks_args = self.get_credentials()
+        auth_version = CONF.keystone_authtoken.auth_version
+        auth_url = CONF.keystone_authtoken.auth_uri
+        api_version = self._is_apiv3(auth_url, auth_version)
+
+        if api_version:
+            from keystoneclient.v3 import client
+        else:
+            from keystoneclient.v2_0 import client
+        # generic
+        # ksclient = client.Client(version=api_version,
+        # session=session,**ks_args)
+
+        return client.Client(**ks_args)
+
+    def get_credentials(self):
         creds = \
             {'auth_url': CONF.keystone_authtoken.auth_uri,
              'username': CONF.keystone_authtoken.admin_user,
@@ -71,10 +112,6 @@ class Client(object):
 
     def get_session(self):
         creds = self.get_credentials()
-        auth = v3.Password(**creds)
+        self._auth = generic.Password(**creds)
-        return session.Session(auth=auth)
+        session = keystone_session.Session(auth=self._auth)
-
+        return session
-    def is_token_expired(self, token):
-        expires = datetime.datetime.strptime(token['expires'],
-                                             '%Y-%m-%dT%H:%M:%SZ')
-        return datetime.datetime.now() >= expires

watcher/metrics_engine/cluster_history/ceilometer.py

@@ -20,7 +20,7 @@
 
 from oslo_config import cfg
 from oslo_log import log
-from watcher.common.ceilometer import Client
+from watcher.common.ceilometer import CeilometerClient
 
 from watcher.metrics_engine.cluster_history.api import BaseClusterHistory
 
@@ -30,7 +30,7 @@ LOG = log.getLogger(__name__)
 
 class CeilometerClusterHistory(BaseClusterHistory):
     def __init__(self):
-        self.ceilometer = Client()
+        self.ceilometer = CeilometerClient()
 
     def statistic_list(self, meter_name, query=None, period=None):
         return self.ceilometer.statistic_list(meter_name, query, period)

watcher/metrics_engine/cluster_model_collector/manager.py

@@ -21,7 +21,7 @@ from oslo_config import cfg
 from oslo_log import log
 
 from watcher.applier.framework.command.wrapper.nova_wrapper import NovaWrapper
-from watcher.common.keystone import Client
+from watcher.common.keystone import KeystoneClient
 from watcher.metrics_engine.cluster_model_collector.nova import \
     NovaClusterModelCollector
 
@@ -31,7 +31,7 @@ CONF = cfg.CONF
 
 class CollectorManager(object):
     def get_cluster_model_collector(self):
-        keystone = Client()
+        keystone = KeystoneClient()
         wrapper = NovaWrapper(keystone.get_credentials(),
                               session=keystone.get_session())
         return NovaClusterModelCollector(wrapper=wrapper)

watcher/tests/common/ceilometer.py

@@ -0,0 +1,93 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2015 b<>com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+from mock import MagicMock
+from mock import mock
+
+from watcher.common.ceilometer import CeilometerClient
+
+from watcher.tests.base import BaseTestCase
+
+
+class TestCeilometer(BaseTestCase):
+    def setUp(self):
+        super(TestCeilometer, self).setUp()
+        self.cm = CeilometerClient()
+
+    def test_build_query(self):
+        expected = [{'field': 'user_id', 'op': 'eq', 'value': u'user_id'},
+                    {'field': 'project_id', 'op': 'eq', 'value': u'tenant_id'},
+                    {'field': 'resource_id', 'op': 'eq',
+                     'value': u'resource_id'}]
+
+        query = self.cm.build_query(user_id="user_id",
+                                    tenant_id="tenant_id",
+                                    resource_id="resource_id",
+                                    user_ids=["user_ids"],
+                                    tenant_ids=["tenant_ids"],
+                                    resource_ids=["resource_ids"])
+        self.assertEqual(query, expected)
+
+    @mock.patch("watcher.common.keystone.Keystoneclient")
+    def test_get_ceilometer_v2(self, mock_keystone):
+        c = CeilometerClient(api_version='2')
+        from ceilometerclient.v2 import Client
+        self.assertIsInstance(c.cmclient, Client)
+
+    @mock.patch.object(CeilometerClient, "cmclient")
+    def test_statistic_aggregation(self, mock_keystone):
+        statistic = MagicMock()
+        expected_result = 100
+        statistic[-1]._info = {'aggregate': {'avg': expected_result}}
+        mock_keystone.statistics.list.return_value = statistic
+        val = self.cm.statistic_aggregation(
+            resource_id="VM_ID",
+            meter_name="cpu_util",
+            period="7300"
+        )
+        self.assertEqual(val, expected_result)
+
+    @mock.patch.object(CeilometerClient, "cmclient")
+    def test_get_last_sample(self, mock_keystone):
+        statistic = MagicMock()
+        expected_result = 100
+        statistic[-1]._info = {'counter_volume': expected_result}
+        mock_keystone.samples.list.return_value = statistic
+        val = self.cm.get_last_sample_value(
+            resource_id="id",
+            meter_name="compute.node.percent"
+        )
+        self.assertEqual(val, expected_result)
+
+    @mock.patch.object(CeilometerClient, "cmclient")
+    def test_get_last_sample_none(self, mock_keystone):
+        expected = False
+        mock_keystone.samples.list.return_value = None
+        val = self.cm.get_last_sample_values(
+            resource_id="id",
+            meter_name="compute.node.percent"
+        )
+        self.assertEqual(val, expected)
+
+    @mock.patch.object(CeilometerClient, "cmclient")
+    def test_statistic_list(self, mock_keystone):
+        expected_value = []
+        mock_keystone.statistics.list.return_value = expected_value
+        val = self.cm.statistic_list(meter_name="cpu_util")
+        self.assertEqual(val, expected_value)

watcher/tests/common/keystone.py

@@ -0,0 +1,60 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2015 b<>com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from keystoneclient.auth.identity import Password
+
+from keystoneclient.session import Session
+from mock import mock
+
+from watcher.common.keystone import KeystoneClient
+
+from watcher.tests.base import BaseTestCase
+
+
+class TestKeyStone(BaseTestCase):
+    def setUp(self):
+        super(TestKeyStone, self).setUp()
+        self.ckeystone = KeystoneClient()
+
+    @mock.patch('keystoneclient.client.Client')
+    def test_get_endpoint(self, keystone):
+        expected_endpoint = "http://IP:PORT"
+        ks = mock.Mock()
+        ks.service_catalog.url_for.return_value = expected_endpoint
+        keystone.return_value = ks
+        ep = self.ckeystone.get_endpoint(service_type='metering',
+                                         endpoint_type='publicURL',
+                                         region_name='RegionOne')
+
+        self.assertEqual(ep, expected_endpoint)
+
+    def test_get_session(self):
+        k = KeystoneClient()
+        session = k.get_session()
+        self.assertIsInstance(session.auth, Password)
+        self.assertIsInstance(session, Session)
+
+    def test_get_credentials(self):
+        expected_creds = {'auth_url': None,
+                          'password': None,
+                          'project_domain_name': 'default',
+                          'project_name': 'admin',
+                          'user_domain_name': 'default',
+                          'username': None}
+        creds = self.ckeystone.get_credentials()
+        self.assertEqual(creds, expected_creds)