Don't throw exception when missing metrics

When querying data from datasource, it's possible to miss some data.
In this case if we throw an exception, Audit will failed because of
the exception. We should remove the exception and give the decision
to the strategy.

Change-Id: I1b0e6b78b3bba4df9ba16e093b3910aab1de922e
Closes-Bug: #1847434
can not cherry picke from master because of code refactoring
(cherry picked from commit 306224f70c)

.gitreview

@@ -1,4 +1,5 @@
 [gerrit]
-host=review.openstack.org
+host=review.opendev.org
 port=29418
 project=openstack/watcher.git
+defaultbranch=stable/rocky

.zuul.yaml

@@ -1,4 +1,10 @@
 - project:
+    templates:
+      - openstack-python-jobs
+      - openstack-python35-jobs
+      - publish-openstack-sphinx-docs
+      - check-requirements
+      - release-notes-jobs
     check:
       jobs:
         - watcher-tempest-functional
@@ -10,6 +16,7 @@
         - watcher-tempest-zone_migration
         - openstack-tox-lower-constraints
     gate:
+      queue: watcher
       jobs:
         - watcher-tempest-functional
         - openstack-tox-lower-constraints
@@ -19,35 +26,35 @@
     parent: watcher-tempest-multinode
     voting: false
     vars:
-      tempest_test_regex: 'watcher_tempest_plugin.tests.scenario.test_execute_dummy_optim'
+      tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_dummy_optim
 
 - job:
     name: watcher-tempest-actuator
     parent: watcher-tempest-multinode
     voting: false
     vars:
-      tempest_test_regex: 'watcher_tempest_plugin.tests.scenario.test_execute_actuator'
+      tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_actuator
 
 - job:
     name: watcher-tempest-basic_optim
     parent: watcher-tempest-multinode
     voting: false
     vars:
-      tempest_test_regex: 'watcher_tempest_plugin.tests.scenario.test_execute_basic_optim'
+      tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_basic_optim
 
 - job:
     name: watcher-tempest-workload_balancing
     parent: watcher-tempest-multinode
     voting: false
     vars:
-      tempest_test_regex: 'watcher_tempest_plugin.tests.scenario.test_execute_workload_balancing'
+      tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_workload_balancing
 
 - job:
     name: watcher-tempest-zone_migration
     parent: watcher-tempest-multinode
     voting: false
     vars:
-      tempest_test_regex: 'watcher_tempest_plugin.tests.scenario.test_execute_zone_migration'
+      tempest_test_regex: watcher_tempest_plugin.tests.scenario.test_execute_zone_migration
 
 - job:
     name: watcher-tempest-multinode
@@ -63,7 +70,7 @@
           post-config:
             $NOVA_CONF:
               libvirt:
-                live_migration_uri: 'qemu+ssh://root@%s/system'
+                live_migration_uri: qemu+ssh://root@%s/system
             $WATCHER_CONF:
               watcher_cluster_data_model_collectors.compute:
                 period: 120
@@ -91,7 +98,7 @@
         post-config:
           $NOVA_CONF:
             libvirt:
-              live_migration_uri: 'qemu+ssh://root@%s/system'
+              live_migration_uri: qemu+ssh://root@%s/system
           $WATCHER_CONF:
             watcher_cluster_data_model_collectors.compute:
               period: 120
@@ -107,7 +114,7 @@
               live_migration: true
               block_migration_for_live_migration: true
       devstack_plugins:
-        ceilometer: https://git.openstack.org/openstack/ceilometer
+        ceilometer: https://opendev.org/openstack/ceilometer
 
 - job:
     name: watcher-tempest-functional
@@ -115,7 +122,7 @@
     timeout: 7200
     required-projects:
       - openstack/ceilometer
-      - openstack-infra/devstack-gate
+      - openstack/devstack-gate
       - openstack/python-openstackclient
       - openstack/python-watcherclient
       - openstack/watcher
@@ -123,7 +130,7 @@
       - openstack/tempest
     vars:
       devstack_plugins:
-        watcher: https://git.openstack.org/openstack/watcher
+        watcher: https://opendev.org/openstack/watcher
       devstack_services:
         tls-proxy: false
         watcher-api: true
@@ -135,8 +142,8 @@
         s-object: false
         s-proxy: false
       devstack_localrc:
-        TEMPEST_PLUGINS: '/opt/stack/watcher-tempest-plugin'
-      tempest_test_regex: 'watcher_tempest_plugin.tests.api'
+        TEMPEST_PLUGINS: /opt/stack/watcher-tempest-plugin
+      tempest_test_regex: watcher_tempest_plugin.tests.api
       tox_envlist: all
       tox_environment:
         # Do we really need to set this? It's cargo culted
@@ -148,9 +155,10 @@
     # This job is used in python-watcherclient repo
     name: watcherclient-tempest-functional
     parent: watcher-tempest-functional
+    voting: false
     timeout: 4200
     vars:
       tempest_concurrency: 1
       devstack_localrc:
-        TEMPEST_PLUGINS: '/opt/stack/python-watcherclient'
-      tempest_test_regex: 'watcherclient.tests.functional'
+        TEMPEST_PLUGINS: /opt/stack/python-watcherclient
+      tempest_test_regex: watcherclient.tests.functional

devstack/lib/watcher

@@ -317,6 +317,7 @@ function start_watcher {
 function stop_watcher {
     if [[ "$WATCHER_USE_MOD_WSGI" == "True" ]]; then
         disable_apache_site watcher-api
+	restart_apache_server
     else
         stop_process watcher-api
     fi

devstack/local.conf.compute

@@ -35,7 +35,7 @@ VNCSERVER_PROXYCLIENT_ADDRESS=$HOST_IP
 NOVA_INSTANCES_PATH=/opt/stack/data/instances
 
 # Enable the Ceilometer plugin for the compute agent
-enable_plugin ceilometer git://git.openstack.org/openstack/ceilometer
+enable_plugin ceilometer https://git.openstack.org/openstack/ceilometer
 disable_service ceilometer-acentral,ceilometer-collector,ceilometer-api
 
 LOGFILE=$DEST/logs/stack.sh.log

devstack/local.conf.controller

@@ -25,13 +25,13 @@ MULTI_HOST=1
 disable_service n-cpu
 
 # Enable the Watcher Dashboard plugin
-enable_plugin watcher-dashboard git://git.openstack.org/openstack/watcher-dashboard
+enable_plugin watcher-dashboard https://git.openstack.org/openstack/watcher-dashboard
 
 # Enable the Watcher plugin
-enable_plugin watcher git://git.openstack.org/openstack/watcher
+enable_plugin watcher https://git.openstack.org/openstack/watcher
 
 # Enable the Ceilometer plugin
-enable_plugin ceilometer git://git.openstack.org/openstack/ceilometer
+enable_plugin ceilometer https://git.openstack.org/openstack/ceilometer
 
 # This is the controller node, so disable the ceilometer compute agent
 disable_service ceilometer-acompute

doc/source/configuration/configuring.rst

@@ -178,7 +178,7 @@ You can easily generate and update a sample configuration file
 named :ref:`watcher.conf.sample <watcher_sample_configuration_files>` by using
 these following commands::
 
-    $ git clone git://git.openstack.org/openstack/watcher
+    $ git clone https://git.openstack.org/openstack/watcher
     $ cd watcher/
     $ tox -e genconfig
     $ vi etc/watcher/watcher.conf.sample
@@ -239,10 +239,6 @@ so that the watcher service is configured for your needs.
 
     [DEFAULT]
 
-    # The messaging driver to use, defaults to rabbit. Other drivers
-    # include qpid and zmq. (string value)
-    #rpc_backend = rabbit
-
     # The default exchange under which topics are scoped. May be
     # overridden by an exchange name specified in the transport_url
     # option. (string value)

doc/source/contributor/devstack.rst

@@ -19,7 +19,7 @@ model. To enable the Watcher plugin with DevStack, add the following to the
 `[[local|localrc]]` section of your controller's `local.conf` to enable the
 Watcher plugin::
 
-    enable_plugin watcher git://git.openstack.org/openstack/watcher
+    enable_plugin watcher https://git.openstack.org/openstack/watcher
 
 For more detailed instructions, see `Detailed DevStack Instructions`_. Check
 out the `DevStack documentation`_ for more information regarding DevStack.

tox.ini

@@ -7,7 +7,7 @@ skipsdist = True
 usedevelop = True
 whitelist_externals = find
                       rm
-install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
+install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=stable/rocky} {opts} {packages}
 setenv =
    VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/test-requirements.txt

watcher/api/hooks.py

@@ -16,7 +16,6 @@
 
 
 from oslo_config import cfg
-from oslo_utils import importutils
 from pecan import hooks
 from six.moves import http_client
 
@@ -60,14 +59,8 @@ class ContextHook(hooks.PecanHook):
         roles = (headers.get('X-Roles', None) and
                  headers.get('X-Roles').split(','))
 
-        auth_url = headers.get('X-Auth-Url')
-        if auth_url is None:
-            importutils.import_module('keystonemiddleware.auth_token')
-            auth_url = cfg.CONF.keystone_authtoken.www_authenticate_uri
-
         state.request.context = context.make_context(
             auth_token=auth_token,
-            auth_url=auth_url,
             auth_token_info=auth_token_info,
             user=user,
             user_id=user_id,

watcher/common/clients.py

@@ -10,7 +10,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from ceilometerclient import client as ceclient
+
 from cinderclient import client as ciclient
 from glanceclient import client as glclient
 from gnocchiclient import client as gnclient
@@ -25,6 +25,12 @@ from watcher.common import exception
 
 from watcher import conf
 
+try:
+    from ceilometerclient import client as ceclient
+    HAS_CEILCLIENT = True
+except ImportError:
+    HAS_CEILCLIENT = False
+
 CONF = conf.CONF
 
 _CLIENTS_AUTH_GROUP = 'watcher_clients_auth'
@@ -83,8 +89,10 @@ class OpenStackClients(object):
 
         novaclient_version = self._get_client_option('nova', 'api_version')
         nova_endpoint_type = self._get_client_option('nova', 'endpoint_type')
+        nova_region_name = self._get_client_option('nova', 'region_name')
         self._nova = nvclient.Client(novaclient_version,
                                      endpoint_type=nova_endpoint_type,
+                                     region_name=nova_region_name,
                                      session=self.session)
         return self._nova
 
@@ -96,8 +104,10 @@ class OpenStackClients(object):
         glanceclient_version = self._get_client_option('glance', 'api_version')
         glance_endpoint_type = self._get_client_option('glance',
                                                        'endpoint_type')
+        glance_region_name = self._get_client_option('glance', 'region_name')
         self._glance = glclient.Client(glanceclient_version,
                                        interface=glance_endpoint_type,
+                                       region_name=glance_region_name,
                                        session=self.session)
         return self._glance
 
@@ -110,8 +120,11 @@ class OpenStackClients(object):
                                                         'api_version')
         gnocchiclient_interface = self._get_client_option('gnocchi',
                                                           'endpoint_type')
+        gnocchiclient_region_name = self._get_client_option('gnocchi',
+                                                            'region_name')
         adapter_options = {
-            "interface": gnocchiclient_interface
+            "interface": gnocchiclient_interface,
+            "region_name": gnocchiclient_region_name
         }
 
         self._gnocchi = gnclient.Client(gnocchiclient_version,
@@ -127,8 +140,10 @@ class OpenStackClients(object):
         cinderclient_version = self._get_client_option('cinder', 'api_version')
         cinder_endpoint_type = self._get_client_option('cinder',
                                                        'endpoint_type')
+        cinder_region_name = self._get_client_option('cinder', 'region_name')
         self._cinder = ciclient.Client(cinderclient_version,
                                        endpoint_type=cinder_endpoint_type,
+                                       region_name=cinder_region_name,
                                        session=self.session)
         return self._cinder
 
@@ -141,9 +156,12 @@ class OpenStackClients(object):
                                                            'api_version')
         ceilometer_endpoint_type = self._get_client_option('ceilometer',
                                                            'endpoint_type')
+        ceilometer_region_name = self._get_client_option('ceilometer',
+                                                         'region_name')
         self._ceilometer = ceclient.get_client(
             ceilometerclient_version,
             endpoint_type=ceilometer_endpoint_type,
+            region_name=ceilometer_region_name,
             session=self.session)
         return self._ceilometer
 
@@ -156,6 +174,8 @@ class OpenStackClients(object):
             'monasca', 'api_version')
         monascaclient_interface = self._get_client_option(
             'monasca', 'interface')
+        monascaclient_region = self._get_client_option(
+            'monasca', 'region_name')
         token = self.session.get_token()
         watcher_clients_auth_config = CONF.get(_CLIENTS_AUTH_GROUP)
         service_type = 'monitoring'
@@ -172,7 +192,8 @@ class OpenStackClients(object):
             'password': watcher_clients_auth_config.password,
         }
         endpoint = self.session.get_endpoint(service_type=service_type,
-                                             interface=monascaclient_interface)
+                                             interface=monascaclient_interface,
+                                             region_name=monascaclient_region)
 
         self._monasca = monclient.Client(
             monascaclient_version, endpoint, **monasca_kwargs)
@@ -188,9 +209,11 @@ class OpenStackClients(object):
                                                         'api_version')
         neutron_endpoint_type = self._get_client_option('neutron',
                                                         'endpoint_type')
+        neutron_region_name = self._get_client_option('neutron', 'region_name')
 
         self._neutron = netclient.Client(neutronclient_version,
                                          endpoint_type=neutron_endpoint_type,
+                                         region_name=neutron_region_name,
                                          session=self.session)
         self._neutron.format = 'json'
         return self._neutron
@@ -202,7 +225,9 @@ class OpenStackClients(object):
 
         ironicclient_version = self._get_client_option('ironic', 'api_version')
         endpoint_type = self._get_client_option('ironic', 'endpoint_type')
+        ironic_region_name = self._get_client_option('ironic', 'region_name')
         self._ironic = irclient.get_client(ironicclient_version,
                                            os_endpoint_type=endpoint_type,
+                                           region_name=ironic_region_name,
                                            session=self.session)
         return self._ironic

watcher/common/config.py

@@ -21,12 +21,15 @@ from watcher.common import rpc
 from watcher import version
 
 
-def parse_args(argv, default_config_files=None):
+def parse_args(argv, default_config_files=None, default_config_dirs=None):
     default_config_files = (default_config_files or
                             cfg.find_config_files(project='watcher'))
+    default_config_dirs = (default_config_dirs or
+                           cfg.find_config_dirs(project='watcher'))
     rpc.set_defaults(control_exchange='watcher')
     cfg.CONF(argv[1:],
              project='python-watcher',
              version=version.version_info.release_string(),
+             default_config_dirs=default_config_dirs,
              default_config_files=default_config_files)
     rpc.init(cfg.CONF)

watcher/common/context.py

@@ -23,9 +23,9 @@ class RequestContext(context.RequestContext):
 
     def __init__(self, user_id=None, project_id=None, is_admin=None,
                  roles=None, timestamp=None, request_id=None, auth_token=None,
-                 auth_url=None, overwrite=True, user_name=None,
-                 project_name=None, domain_name=None, domain_id=None,
-                 auth_token_info=None, **kwargs):
+                 overwrite=True, user_name=None, project_name=None,
+                 domain_name=None, domain_id=None, auth_token_info=None,
+                 **kwargs):
         """Stores several additional request parameters:
 
         :param domain_id: The ID of the domain.
@@ -70,7 +70,6 @@ class RequestContext(context.RequestContext):
         # FIXME(dims): user_id and project_id duplicate information that is
         # already present in the oslo_context's RequestContext. We need to
         # get rid of them.
-        self.auth_url = auth_url
         self.domain_name = domain_name
         self.domain_id = domain_id
         self.auth_token_info = auth_token_info

watcher/common/nova_helper.py

@@ -74,7 +74,8 @@ class NovaHelper(object):
             raise exception.ComputeNodeNotFound(name=node_hostname)
 
     def get_instance_list(self):
-        return self.nova.servers.list(search_opts={'all_tenants': True})
+        return self.nova.servers.list(search_opts={'all_tenants': True},
+                                      limit=-1)
 
     def get_flavor_list(self):
         return self.nova.flavors.list(**{'is_public': None})
@@ -704,8 +705,9 @@ class NovaHelper(object):
 
     def get_instances_by_node(self, host):
         return [instance for instance in
-                self.nova.servers.list(search_opts={"all_tenants": True})
-                if self.get_hostname(instance) == host]
+                self.nova.servers.list(search_opts={"all_tenants": True,
+                                                    "host": host},
+                                       limit=-1)]
 
     def get_hostname(self, instance):
         return str(getattr(instance, 'OS-EXT-SRV-ATTR:host'))

watcher/common/service.py

@@ -265,8 +265,7 @@ class Service(service.ServiceBase):
             allow_requeue=False, pool=CONF.host)
 
     def start(self):
-        LOG.debug("Connecting to '%s' (%s)",
-                  CONF.transport_url, CONF.rpc_backend)
+        LOG.debug("Connecting to '%s'", CONF.transport_url)
         if self.conductor_topic_handler:
             self.conductor_topic_handler.start()
         if self.notification_handler:
@@ -275,8 +274,7 @@ class Service(service.ServiceBase):
             self.heartbeat.start()
 
     def stop(self):
-        LOG.debug("Disconnecting from '%s' (%s)",
-                  CONF.transport_url, CONF.rpc_backend)
+        LOG.debug("Disconnecting from '%s' (%s)'", CONF.transport_url)
         if self.conductor_topic_handler:
             self.conductor_topic_handler.stop()
         if self.notification_handler:

watcher/conf/ceilometer_client.py

@@ -30,7 +30,10 @@ CEILOMETER_CLIENT_OPTS = [
                default='internalURL',
                help='Type of endpoint to use in ceilometerclient.'
                     'Supported values: internalURL, publicURL, adminURL'
-                    'The default is internalURL.')]
+                    'The default is internalURL.'),
+    cfg.StrOpt('region_name',
+               help='Region in Identity service catalog to use for '
+                    'communication with the OpenStack service.')]
 
 
 def register_opts(conf):

watcher/conf/cinder_client.py

@@ -29,7 +29,10 @@ CINDER_CLIENT_OPTS = [
                default='publicURL',
                help='Type of endpoint to use in cinderclient.'
                     'Supported values: internalURL, publicURL, adminURL'
-                    'The default is publicURL.')]
+                    'The default is publicURL.'),
+    cfg.StrOpt('region_name',
+               help='Region in Identity service catalog to use for '
+                    'communication with the OpenStack service.')]
 
 
 def register_opts(conf):

watcher/conf/glance_client.py

@@ -29,7 +29,10 @@ GLANCE_CLIENT_OPTS = [
                default='publicURL',
                help='Type of endpoint to use in glanceclient.'
                     'Supported values: internalURL, publicURL, adminURL'
-                    'The default is publicURL.')]
+                    'The default is publicURL.'),
+    cfg.StrOpt('region_name',
+               help='Region in Identity service catalog to use for '
+                    'communication with the OpenStack service.')]
 
 
 def register_opts(conf):

watcher/conf/gnocchi_client.py

@@ -30,6 +30,9 @@ GNOCCHI_CLIENT_OPTS = [
                help='Type of endpoint to use in gnocchi client.'
                     'Supported values: internal, public, admin'
                     'The default is public.'),
+    cfg.StrOpt('region_name',
+               help='Region in Identity service catalog to use for '
+                    'communication with the OpenStack service.'),
     cfg.IntOpt('query_max_retries',
                default=10,
                mutable=True,

watcher/conf/ironic_client.py

@@ -29,7 +29,10 @@ IRONIC_CLIENT_OPTS = [
                default='publicURL',
                help='Type of endpoint to use in ironicclient.'
                     'Supported values: internalURL, publicURL, adminURL'
-                    'The default is publicURL.')]
+                    'The default is publicURL.'),
+    cfg.StrOpt('region_name',
+               help='Region in Identity service catalog to use for '
+                    'communication with the OpenStack service.')]
 
 
 def register_opts(conf):

watcher/conf/monasca_client.py

@@ -29,7 +29,10 @@ MONASCA_CLIENT_OPTS = [
                default='internal',
                help='Type of interface used for monasca endpoint.'
                     'Supported values: internal, public, admin'
-                    'The default is internal.')]
+                    'The default is internal.'),
+    cfg.StrOpt('region_name',
+               help='Region in Identity service catalog to use for '
+                    'communication with the OpenStack service.')]
 
 
 def register_opts(conf):

watcher/conf/neutron_client.py

@@ -29,7 +29,10 @@ NEUTRON_CLIENT_OPTS = [
                default='publicURL',
                help='Type of endpoint to use in neutronclient.'
                     'Supported values: internalURL, publicURL, adminURL'
-                    'The default is publicURL.')]
+                    'The default is publicURL.'),
+    cfg.StrOpt('region_name',
+               help='Region in Identity service catalog to use for '
+                    'communication with the OpenStack service.')]
 
 
 def register_opts(conf):

watcher/conf/nova_client.py

@@ -29,7 +29,10 @@ NOVA_CLIENT_OPTS = [
                default='publicURL',
                help='Type of endpoint to use in novaclient.'
                     'Supported values: internalURL, publicURL, adminURL'
-                    'The default is publicURL.')]
+                    'The default is publicURL.'),
+    cfg.StrOpt('region_name',
+               help='Region in Identity service catalog to use for '
+                    'communication with the OpenStack service.')]
 
 
 def register_opts(conf):

watcher/datasource/ceilometer.py

@@ -19,6 +19,7 @@
 import datetime
 
 from ceilometerclient import exc
+from oslo_log import log
 from oslo_utils import timeutils
 
 from watcher._i18n import _
@@ -26,6 +27,8 @@ from watcher.common import clients
 from watcher.common import exception
 from watcher.datasource import base
 
+LOG = log.getLogger(__name__)
+
 
 class CeilometerHelper(base.DataSourceBase):
 
@@ -112,15 +115,15 @@ class CeilometerHelper(base.DataSourceBase):
             self.osc.reset_clients()
             self.ceilometer = self.osc.ceilometer()
             return f(*args, **kargs)
-        except Exception:
-            raise
+        except Exception as e:
+            LOG.exception(e)
 
     def check_availability(self):
-        try:
-            self.query_retry(self.ceilometer.resources.list)
-        except Exception:
-            return 'not available'
+        status = self.query_retry(self.ceilometer.resources.list)
+        if status:
             return 'available'
+        else:
+            return 'not available'
 
     def query_sample(self, meter_name, query, limit=1):
         return self.query_retry(f=self.ceilometer.samples.list,
@@ -138,9 +141,8 @@ class CeilometerHelper(base.DataSourceBase):
 
     def list_metrics(self):
         """List the user's meters."""
-        try:
         meters = self.query_retry(f=self.ceilometer.meters.list)
-        except Exception:
+        if not meters:
             return set()
         else:
             return meters

watcher/datasource/gnocchi.py

@@ -24,7 +24,6 @@ from oslo_config import cfg
 from oslo_log import log
 
 from watcher.common import clients
-from watcher.common import exception
 from watcher.common import utils as common_utils
 from watcher.datasource import base
 
@@ -49,27 +48,25 @@ class GnocchiHelper(base.DataSourceBase):
             except Exception as e:
                 LOG.exception(e)
                 time.sleep(CONF.gnocchi_client.query_timeout)
-        raise exception.DataSourceNotAvailable(datasource='gnocchi')
 
     def check_availability(self):
-        try:
-            self.query_retry(self.gnocchi.status.get)
-        except Exception:
-            return 'not available'
+        status = self.query_retry(self.gnocchi.status.get)
+        if status:
             return 'available'
+        else:
+            return 'not available'
 
     def list_metrics(self):
         """List the user's meters."""
-        try:
         response = self.query_retry(f=self.gnocchi.metric.list)
-        except Exception:
+        if not response:
             return set()
         else:
             return set([metric['name'] for metric in response])
 
     def statistic_aggregation(self, resource_id=None, meter_name=None,
                               period=300, granularity=300, dimensions=None,
-                              aggregation='avg', group_by='*'):
+                              aggregation='mean', group_by='*'):
         """Representing a statistic aggregate by operators
 
         :param resource_id: id of resource to list statistics for.
@@ -95,7 +92,9 @@ class GnocchiHelper(base.DataSourceBase):
                 f=self.gnocchi.resource.search, **kwargs)
 
             if not resources:
-                raise exception.ResourceNotFound(name=resource_id)
+                LOG.warning("The {0} resource {1} could not be "
+                            "found".format(self.NAME, resource_id))
+                return
 
             resource_id = resources[0]['id']
 

watcher/datasource/manager.py

@@ -29,7 +29,12 @@ class DataSourceManager(object):
         self._monasca = None
         self._gnocchi = None
         self.metric_map = base.DataSourceBase.METRIC_MAP
+        if hasattr(self.config, 'datasources'):
             self.datasources = self.config.datasources
+        elif hasattr(self.config, 'datasource'):
+            self.datasources = [self.config.datasource]
+        else:
+            self.datasources = []
 
     @property
     def ceilometer(self):

watcher/datasource/monasca.py

@@ -19,11 +19,14 @@
 import datetime
 
 from monascaclient import exc
+from oslo_log import log
 
 from watcher.common import clients
 from watcher.common import exception
 from watcher.datasource import base
 
+LOG = log.getLogger(__name__)
+
 
 class MonascaHelper(base.DataSourceBase):
 
@@ -42,8 +45,8 @@ class MonascaHelper(base.DataSourceBase):
             self.osc.reset_clients()
             self.monasca = self.osc.monasca()
             return f(*args, **kwargs)
-        except Exception:
-            raise
+        except Exception as e:
+            LOG.exception(e)
 
     def _format_time_params(self, start_time, end_time, period):
         """Format time-related params to the correct Monasca format
@@ -67,11 +70,11 @@ class MonascaHelper(base.DataSourceBase):
         return start_timestamp, end_timestamp, period
 
     def check_availability(self):
-        try:
-            self.query_retry(self.monasca.metrics.list)
-        except Exception:
-            return 'not available'
+        status = self.query_retry(self.monasca.metrics.list)
+        if status:
             return 'available'
+        else:
+            return 'not available'
 
     def list_metrics(self):
         # TODO(alexchadin): this method should be implemented in accordance to

watcher/decision_engine/model/collector/cinder.py

@@ -230,21 +230,17 @@ class ModelBuilder(object):
                  "free_capacity_gb", "provisioned_capacity_gb",
                  "allocated_capacity_gb"]
 
+        node_attributes = {"name": pool.name}
         for attr in attrs:
             try:
-                int(getattr(pool, attr))
+                node_attributes[attr] = int(getattr(pool, attr))
+            except AttributeError:
+                LOG.debug("Attribute %s for pool %s is not provided",
+                          attr, pool.name)
             except ValueError:
                 raise exception.InvalidPoolAttributeValue(
                     name=pool.name, attribute=attr)
 
-        node_attributes = {
-            "name": pool.name,
-            "total_volumes": pool.total_volumes,
-            "total_capacity_gb": pool.total_capacity_gb,
-            "free_capacity_gb": pool.free_capacity_gb,
-            "provisioned_capacity_gb": pool.provisioned_capacity_gb,
-            "allocated_capacity_gb": pool.allocated_capacity_gb}
-
         storage_pool = element.Pool(**node_attributes)
         return storage_pool
 

watcher/decision_engine/planner/workload_stabilization.py

@@ -279,7 +279,7 @@ class ChangeNovaServiceStateActionValidator(BaseActionValidator):
 
     def validate_parents(self, resource_action_map, action):
         host_name = action['input_parameters']['resource_id']
-        self._mapping(resource_action_map, host_name, action.uuid,
+        self._mapping(resource_action_map, host_name, action['uuid'],
                       'change_nova_service_state')
         return []
 

watcher/tests/api/test_hooks.py

@@ -222,7 +222,6 @@ class TestContextHook(base.FunctionalTest):
             user_id=headers['X-User-Id'],
             domain_id=headers['X-User-Domain-Id'],
             domain_name=headers['X-User-Domain-Name'],
-            auth_url=cfg.CONF.keystone_authtoken.www_authenticate_uri,
             project=headers['X-Project-Name'],
             project_id=headers['X-Project-Id'],
             show_deleted=None,
@@ -243,7 +242,6 @@ class TestContextHook(base.FunctionalTest):
             user_id=headers['X-User-Id'],
             domain_id=headers['X-User-Domain-Id'],
             domain_name=headers['X-User-Domain-Name'],
-            auth_url=cfg.CONF.keystone_authtoken.www_authenticate_uri,
             project=headers['X-Project-Name'],
             project_id=headers['X-Project-Id'],
             show_deleted=None,
@@ -265,7 +263,6 @@ class TestContextHook(base.FunctionalTest):
             user_id=headers['X-User-Id'],
             domain_id=headers['X-User-Domain-Id'],
             domain_name=headers['X-User-Domain-Name'],
-            auth_url=cfg.CONF.keystone_authtoken.www_authenticate_uri,
             project=headers['X-Project-Name'],
             project_id=headers['X-Project-Id'],
             show_deleted=None,

watcher/tests/base.py

@@ -60,9 +60,6 @@ class TestCase(BaseTestCase):
 
         cfg.CONF.set_override("auth_type", "admin_token",
                               group='keystone_authtoken')
-        cfg.CONF.set_override("www_authenticate_uri",
-                              "http://127.0.0.1/identity",
-                              group='keystone_authtoken')
 
         app_config_path = os.path.join(os.path.dirname(__file__), 'config.py')
         self.app = testing.load_test_app(app_config_path)

watcher/tests/common/test_clients.py

@@ -120,6 +120,7 @@ class TestClients(base.TestCase):
         mock_call.assert_called_once_with(
             CONF.nova_client.api_version,
             endpoint_type=CONF.nova_client.endpoint_type,
+            region_name=CONF.nova_client.region_name,
             session=mock_session)
 
     @mock.patch.object(clients.OpenStackClients, 'session')
@@ -155,6 +156,7 @@ class TestClients(base.TestCase):
         mock_call.assert_called_once_with(
             CONF.glance_client.api_version,
             interface=CONF.glance_client.endpoint_type,
+            region_name=CONF.glance_client.region_name,
             session=mock_session)
 
     @mock.patch.object(clients.OpenStackClients, 'session')
@@ -191,7 +193,8 @@ class TestClients(base.TestCase):
         mock_call.assert_called_once_with(
             CONF.gnocchi_client.api_version,
             adapter_options={
-                "interface": CONF.gnocchi_client.endpoint_type},
+                "interface": CONF.gnocchi_client.endpoint_type,
+                "region_name": CONF.gnocchi_client.region_name},
             session=mock_session)
 
     @mock.patch.object(clients.OpenStackClients, 'session')
@@ -229,6 +232,7 @@ class TestClients(base.TestCase):
         mock_call.assert_called_once_with(
             CONF.cinder_client.api_version,
             endpoint_type=CONF.cinder_client.endpoint_type,
+            region_name=CONF.cinder_client.region_name,
             session=mock_session)
 
     @mock.patch.object(clients.OpenStackClients, 'session')
@@ -266,6 +270,7 @@ class TestClients(base.TestCase):
             CONF.ceilometer_client.api_version,
             None,
             endpoint_type=CONF.ceilometer_client.endpoint_type,
+            region_name=CONF.ceilometer_client.region_name,
             session=mock_session)
 
     @mock.patch.object(clients.OpenStackClients, 'session')
@@ -314,6 +319,7 @@ class TestClients(base.TestCase):
         mock_call.assert_called_once_with(
             CONF.neutron_client.api_version,
             endpoint_type=CONF.neutron_client.endpoint_type,
+            region_name=CONF.neutron_client.region_name,
             session=mock_session)
 
     @mock.patch.object(clients.OpenStackClients, 'session')
@@ -404,6 +410,7 @@ class TestClients(base.TestCase):
         mock_call.assert_called_once_with(
             CONF.ironic_client.api_version,
             endpoint_override=ironic_url,
+            interface='publicURL',
             max_retries=None,
             os_ironic_api_version=None,
             retry_interval=None,

watcher/tests/fakes.py

@@ -20,7 +20,6 @@ fakeAuthTokenHeaders = {'X-User-Id': u'773a902f022949619b5c2f32cd89d419',
                         'X-Auth-Token': u'5588aebbcdc24e17a061595f80574376',
                         'X-Forwarded-For': u'10.10.10.10, 11.11.11.11',
                         'X-Service-Catalog': u'{test: 12345}',
-                        'X-Auth-Url': 'fake_auth_url',
                         'X-Identity-Status': 'Confirmed',
                         'X-User-Domain-Name': 'domain',
                         'X-Project-Domain-Id': 'project_domain_id',