diff --git a/releasenotes/notes/support-placement-api-58ce6bef1bbbe98a.yaml b/releasenotes/notes/support-placement-api-58ce6bef1bbbe98a.yaml new file mode 100644 index 000000000..6ca92338f --- /dev/null +++ b/releasenotes/notes/support-placement-api-58ce6bef1bbbe98a.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Added Placement API helper to Watcher. Now Watcher can get information + about resource providers, it can be used for the data model and strategies. + Config group placement_client with options 'api_version', 'interface' and + 'region_name' is also added. The default values for 'api_version' and + 'interface' are 1.29 and 'public', respectively. diff --git a/watcher/common/clients.py b/watcher/common/clients.py index 1aca5500d..a71eabc64 100755 --- a/watcher/common/clients.py +++ b/watcher/common/clients.py @@ -16,6 +16,7 @@ from cinderclient import client as ciclient from glanceclient import client as glclient from gnocchiclient import client as gnclient from ironicclient import client as irclient +from keystoneauth1 import adapter as ka_adapter from keystoneauth1 import loading as ka_loading from keystoneclient import client as keyclient from monascaclient import client as monclient @@ -73,6 +74,7 @@ class OpenStackClients(object): self._monasca = None self._neutron = None self._ironic = None + self._placement = None def _get_keystone_session(self): auth = ka_loading.load_auth_from_conf_options(CONF, @@ -262,3 +264,27 @@ class OpenStackClients(object): region_name=ironic_region_name, session=self.session) return self._ironic + + @exception.wrap_keystone_exception + def placement(self): + if self._placement: + return self._placement + + placement_version = self._get_client_option('placement', + 'api_version') + placement_interface = self._get_client_option('placement', + 'interface') + placement_region_name = self._get_client_option('placement', + 'region_name') + # Set accept header on every request to ensure we notify placement + # service of our response body media type preferences. + headers = {'accept': 'application/json'} + self._placement = ka_adapter.Adapter( + session=self.session, + service_type='placement', + default_microversion=placement_version, + interface=placement_interface, + region_name=placement_region_name, + additional_headers=headers) + + return self._placement diff --git a/watcher/common/placement_helper.py b/watcher/common/placement_helper.py new file mode 100644 index 000000000..580ef9e96 --- /dev/null +++ b/watcher/common/placement_helper.py @@ -0,0 +1,179 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg +from oslo_log import log as logging + +from watcher.common import clients + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + + +class PlacementHelper(object): + + def __init__(self, osc=None): + """:param osc: an OpenStackClients instance""" + self.osc = osc if osc else clients.OpenStackClients() + self._placement = self.osc.placement() + + def get(self, url): + return self._placement.get(url, raise_exc=False) + + @staticmethod + def get_error_msg(resp): + json_resp = resp.json() + # https://developer.openstack.org/api-ref/placement/#errors + if 'errors' in json_resp: + error_msg = json_resp['errors'][0].get('detail') + else: + error_msg = resp.text + + return error_msg + + def get_resource_providers(self, rp_name=None): + """Calls the placement API for a resource provider record. + + :param rp_name: Name of the resource provider, if None, + list all resource providers. + :return: A list of resource providers information + or None if the resource provider doesn't exist. + """ + url = '/resource_providers' + if rp_name: + url += '?name=%s' % rp_name + resp = self.get(url) + if resp.status_code == 200: + json_resp = resp.json() + return json_resp['resource_providers'] + + if rp_name: + msg = "Failed to get resource provider %(name)s. " + else: + msg = "Failed to get all resource providers. " + msg += "Got %(status_code)d: %(err_text)s." + args = { + 'name': rp_name, + 'status_code': resp.status_code, + 'err_text': self.get_error_msg(resp), + } + LOG.error(msg, args) + + def get_inventories(self, rp_uuid): + """Calls the placement API to get resource inventory information. + + :param rp_uuid: UUID of the resource provider to get. + :return: A dictionary of inventories keyed by resource classes. + """ + url = '/resource_providers/%s/inventories' % rp_uuid + resp = self.get(url) + if resp.status_code == 200: + json = resp.json() + return json['inventories'] + msg = ("Failed to get resource provider %(rp_uuid) inventories. " + "Got %(status_code)d: %(err_text)s.") + args = { + 'rp_uuid': rp_uuid, + 'status_code': resp.status_code, + 'err_text': self.get_error_msg(resp), + } + LOG.error(msg, args) + + def get_provider_traits(self, rp_uuid): + """Queries the placement API for a resource provider's traits. + + :param rp_uuid: UUID of the resource provider to grab traits for. + :return: A list of traits. + """ + resp = self.get("/resource_providers/%s/traits" % rp_uuid) + + if resp.status_code == 200: + json = resp.json() + return json['traits'] + msg = ("Failed to get resource provider %(rp_uuid) traits. " + "Got %(status_code)d: %(err_text)s.") + args = { + 'rp_uuid': rp_uuid, + 'status_code': resp.status_code, + 'err_text': self.get_error_msg(resp), + } + LOG.error(msg, args) + + def get_allocations_for_consumer(self, consumer_uuid): + """Retrieves the allocations for a specific consumer. + + :param consumer_uuid: the UUID of the consumer resource. + :return: A dictionary of allocation records keyed by resource + provider uuid. + """ + url = '/allocations/%s' % consumer_uuid + resp = self.get(url) + if resp.status_code == 200: + json = resp.json() + return json['allocations'] + msg = ("Failed to get allocations for consumer %(c_uuid). " + "Got %(status_code)d: %(err_text)s.") + args = { + 'c_uuid': consumer_uuid, + 'status_code': resp.status_code, + 'err_text': self.get_error_msg(resp), + } + LOG.error(msg, args) + + def get_usages_for_resource_provider(self, rp_uuid): + """Retrieves the usages for a specific provider. + + :param rp_uuid: The UUID of the provider. + :return: A dictionary that describes how much each class of + resource is being consumed on this resource provider. + """ + url = '/resource_providers/%s/usages' % rp_uuid + resp = self.get(url) + if resp.status_code == 200: + json = resp.json() + return json['usages'] + msg = ("Failed to get resource provider %(rp_uuid) usages. " + "Got %(status_code)d: %(err_text)s.") + args = { + 'rp_uuid': rp_uuid, + 'status_code': resp.status_code, + 'err_text': self.get_error_msg(resp), + } + LOG.error(msg, args) + + def get_candidate_providers(self, resources): + """Returns a dictionary of resource provider summaries. + + :param resources: A comma-separated list of strings indicating + an amount of resource of a specified class that + providers in each allocation request must collectively + have the capacity and availability to serve: + resources=VCPU:4,DISK_GB:64,MEMORY_MB:2048 + :returns: A dict, keyed by resource provider UUID, which can + provide the required resources. + """ + url = "/allocation_candidates?%s" % resources + resp = self.get(url) + if resp.status_code == 200: + data = resp.json() + return data['provider_summaries'] + + args = { + 'resource_request': resources, + 'status_code': resp.status_code, + 'err_text': self.get_error_msg(resp), + } + msg = ("Failed to get allocation candidates from placement " + "API for resources: %(resource_request)s\n" + "Got %(status_code)d: %(err_text)s.") + LOG.error(msg, args) diff --git a/watcher/conf/__init__.py b/watcher/conf/__init__.py index dddee25c9..17b7293b6 100755 --- a/watcher/conf/__init__.py +++ b/watcher/conf/__init__.py @@ -37,6 +37,7 @@ from watcher.conf import monasca_client from watcher.conf import neutron_client from watcher.conf import nova_client from watcher.conf import paths +from watcher.conf import placement_client from watcher.conf import planner from watcher.conf import service @@ -62,3 +63,4 @@ neutron_client.register_opts(CONF) clients_auth.register_opts(CONF) ironic_client.register_opts(CONF) collector.register_opts(CONF) +placement_client.register_opts(CONF) diff --git a/watcher/conf/placement_client.py b/watcher/conf/placement_client.py new file mode 100644 index 000000000..2afb887fc --- /dev/null +++ b/watcher/conf/placement_client.py @@ -0,0 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + + +placement_group = cfg.OptGroup( + 'placement_client', + title='Placement Service Options', + help="Configuration options for connecting to the placement API service") + +placement_opts = [ + cfg.StrOpt('api_version', + default='1.29', + help='microversion of placement API when using ' + 'placement service.'), + cfg.StrOpt('interface', + default='public', + choices=['internal', 'public', 'admin'], + help='Type of endpoint when using placement service.'), + cfg.StrOpt('region_name', + help='Region in Identity service catalog to use for ' + 'communication with the OpenStack service.')] + + +def register_opts(conf): + conf.register_group(placement_group) + conf.register_opts(placement_opts, group=placement_group) + + +def list_opts(): + return [(placement_group.name, placement_opts)] diff --git a/watcher/tests/common/test_clients.py b/watcher/tests/common/test_clients.py index 091b325d4..847d226f9 100755 --- a/watcher/tests/common/test_clients.py +++ b/watcher/tests/common/test_clients.py @@ -19,6 +19,7 @@ from gnocchiclient import client as gnclient from gnocchiclient.v1 import client as gnclient_v1 from ironicclient import client as irclient from ironicclient.v1 import client as irclient_v1 +from keystoneauth1 import adapter as ka_adapter from keystoneauth1 import loading as ka_loading import mock from monascaclient import client as monclient @@ -459,3 +460,17 @@ class TestClients(base.TestCase): ironic = osc.ironic() ironic_cached = osc.ironic() self.assertEqual(ironic, ironic_cached) + + @mock.patch.object(ka_adapter, 'Adapter') + @mock.patch.object(clients.OpenStackClients, 'session') + def test_clients_placement(self, mock_session, mock_call): + osc = clients.OpenStackClients() + osc.placement() + headers = {'accept': 'application/json'} + mock_call.assert_called_once_with( + session=mock_session, + service_type='placement', + default_microversion=CONF.placement_client.api_version, + interface=CONF.placement_client.interface, + region_name=CONF.placement_client.region_name, + additional_headers=headers) diff --git a/watcher/tests/common/test_placement_helper.py b/watcher/tests/common/test_placement_helper.py new file mode 100644 index 000000000..fc2de290b --- /dev/null +++ b/watcher/tests/common/test_placement_helper.py @@ -0,0 +1,312 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +from watcher.common import placement_helper +from watcher.tests import base +from watcher.tests import fakes as fake_requests + +from keystoneauth1 import loading as ka_loading +from oslo_config import cfg +from oslo_serialization import jsonutils +from oslo_utils import uuidutils + +CONF = cfg.CONF + + +@mock.patch('keystoneauth1.session.Session.request') +class TestPlacementHelper(base.TestCase): + def setUp(self): + super(TestPlacementHelper, self).setUp() + _AUTH_CONF_GROUP = 'watcher_clients_auth' + ka_loading.register_auth_conf_options(CONF, _AUTH_CONF_GROUP) + ka_loading.register_session_conf_options(CONF, _AUTH_CONF_GROUP) + self.client = placement_helper.PlacementHelper() + self.fake_err_msg = { + 'errors': [{ + 'detail': 'The resource could not be found.', + }] + } + + def _add_default_kwargs(self, kwargs): + kwargs['endpoint_filter'] = { + 'service_type': 'placement', + 'interface': CONF.placement_client.interface} + kwargs['headers'] = {'accept': 'application/json'} + kwargs['microversion'] = CONF.placement_client.api_version + kwargs['raise_exc'] = False + + def _assert_keystone_called_once(self, kss_req, url, method, **kwargs): + self._add_default_kwargs(kwargs) + # request method has added param rate_semaphore since Stein cycle + if 'rate_semaphore' in kss_req.call_args[1]: + kwargs['rate_semaphore'] = mock.ANY + kss_req.assert_called_once_with(url, method, **kwargs) + + def test_get(self, kss_req): + kss_req.return_value = fake_requests.FakeResponse(200) + url = '/resource_providers' + resp = self.client.get(url) + self.assertEqual(200, resp.status_code) + self._assert_keystone_called_once(kss_req, url, 'GET') + + def test_get_resource_providers_OK(self, kss_req): + rp_name = 'compute' + rp_uuid = uuidutils.generate_uuid() + parent_uuid = uuidutils.generate_uuid() + + fake_rp = [{'uuid': rp_uuid, + 'name': rp_name, + 'generation': 0, + 'parent_provider_uuid': parent_uuid}] + + mock_json_data = { + 'resource_providers': fake_rp + } + + kss_req.return_value = fake_requests.FakeResponse( + 200, content=jsonutils.dump_as_bytes(mock_json_data)) + + result = self.client.get_resource_providers(rp_name) + + expected_url = '/resource_providers?name=compute' + self._assert_keystone_called_once(kss_req, expected_url, 'GET') + self.assertEqual(fake_rp, result) + + def test_get_resource_providers_no_rp_OK(self, kss_req): + rp_name = None + rp_uuid = uuidutils.generate_uuid() + parent_uuid = uuidutils.generate_uuid() + + fake_rp = [{'uuid': rp_uuid, + 'name': 'compute', + 'generation': 0, + 'parent_provider_uuid': parent_uuid}] + + mock_json_data = { + 'resource_providers': fake_rp + } + + kss_req.return_value = fake_requests.FakeResponse( + 200, content=jsonutils.dump_as_bytes(mock_json_data)) + + result = self.client.get_resource_providers(rp_name) + + expected_url = '/resource_providers' + self._assert_keystone_called_once(kss_req, expected_url, 'GET') + self.assertEqual(fake_rp, result) + + def test_get_resource_providers_fail(self, kss_req): + rp_name = 'compute' + kss_req.return_value = fake_requests.FakeResponse( + 400, content=jsonutils.dump_as_bytes(self.fake_err_msg)) + result = self.client.get_resource_providers(rp_name) + self.assertIsNone(result) + + def test_get_inventories_OK(self, kss_req): + rp_uuid = uuidutils.generate_uuid() + + fake_inventories = { + "DISK_GB": { + "allocation_ratio": 1.0, + "max_unit": 35, + "min_unit": 1, + "reserved": 0, + "step_size": 1, + "total": 35 + }, + "MEMORY_MB": { + "allocation_ratio": 1.5, + "max_unit": 5825, + "min_unit": 1, + "reserved": 512, + "step_size": 1, + "total": 5825 + }, + "VCPU": { + "allocation_ratio": 16.0, + "max_unit": 4, + "min_unit": 1, + "reserved": 0, + "step_size": 1, + "total": 4 + }, + } + mock_json_data = { + 'inventories': fake_inventories, + "resource_provider_generation": 7 + } + + kss_req.return_value = fake_requests.FakeResponse( + 200, content=jsonutils.dump_as_bytes(mock_json_data)) + + result = self.client.get_inventories(rp_uuid) + + expected_url = '/resource_providers/%s/inventories' % rp_uuid + self._assert_keystone_called_once(kss_req, expected_url, 'GET') + self.assertEqual(fake_inventories, result) + + def test_get_inventories_fail(self, kss_req): + rp_uuid = uuidutils.generate_uuid() + kss_req.return_value = fake_requests.FakeResponse( + 404, content=jsonutils.dump_as_bytes(self.fake_err_msg)) + result = self.client.get_inventories(rp_uuid) + self.assertIsNone(result) + + def test_get_provider_traits_OK(self, kss_req): + rp_uuid = uuidutils.generate_uuid() + + fake_traits = ["CUSTOM_HW_FPGA_CLASS1", + "CUSTOM_HW_FPGA_CLASS3"] + mock_json_data = { + 'traits': fake_traits, + "resource_provider_generation": 7 + } + + kss_req.return_value = fake_requests.FakeResponse( + 200, content=jsonutils.dump_as_bytes(mock_json_data)) + + result = self.client.get_provider_traits(rp_uuid) + + expected_url = '/resource_providers/%s/traits' % rp_uuid + self._assert_keystone_called_once(kss_req, expected_url, 'GET') + self.assertEqual(fake_traits, result) + + def test_get_provider_traits_fail(self, kss_req): + rp_uuid = uuidutils.generate_uuid() + kss_req.return_value = fake_requests.FakeResponse( + 404, content=jsonutils.dump_as_bytes(self.fake_err_msg)) + result = self.client.get_provider_traits(rp_uuid) + self.assertIsNone(result) + + def test_get_allocations_for_consumer_OK(self, kss_req): + c_uuid = uuidutils.generate_uuid() + + fake_allocations = { + "92637880-2d79-43c6-afab-d860886c6391": { + "generation": 2, + "resources": { + "DISK_GB": 5 + } + }, + "ba8e1ef8-7fa3-41a4-9bb4-d7cb2019899b": { + "generation": 8, + "resources": { + "MEMORY_MB": 512, + "VCPU": 2 + } + } + } + mock_json_data = { + 'allocations': fake_allocations, + "consumer_generation": 1, + "project_id": "7e67cbf7-7c38-4a32-b85b-0739c690991a", + "user_id": "067f691e-725a-451a-83e2-5c3d13e1dffc" + } + + kss_req.return_value = fake_requests.FakeResponse( + 200, content=jsonutils.dump_as_bytes(mock_json_data)) + + result = self.client.get_allocations_for_consumer(c_uuid) + + expected_url = '/allocations/%s' % c_uuid + self._assert_keystone_called_once(kss_req, expected_url, 'GET') + self.assertEqual(fake_allocations, result) + + def test_get_allocations_for_consumer_fail(self, kss_req): + c_uuid = uuidutils.generate_uuid() + kss_req.return_value = fake_requests.FakeResponse( + 404, content=jsonutils.dump_as_bytes(self.fake_err_msg)) + result = self.client.get_allocations_for_consumer(c_uuid) + self.assertIsNone(result) + + def test_get_usages_for_resource_provider_OK(self, kss_req): + rp_uuid = uuidutils.generate_uuid() + + fake_usages = { + "DISK_GB": 1, + "MEMORY_MB": 512, + "VCPU": 1 + } + mock_json_data = { + 'usages': fake_usages, + "resource_provider_generation": 7 + } + + kss_req.return_value = fake_requests.FakeResponse( + 200, content=jsonutils.dump_as_bytes(mock_json_data)) + + result = self.client.get_usages_for_resource_provider(rp_uuid) + + expected_url = '/resource_providers/%s/usages' % rp_uuid + self._assert_keystone_called_once(kss_req, expected_url, 'GET') + self.assertEqual(fake_usages, result) + + def test_get_usages_for_resource_provider_fail(self, kss_req): + rp_uuid = uuidutils.generate_uuid() + kss_req.return_value = fake_requests.FakeResponse( + 404, content=jsonutils.dump_as_bytes(self.fake_err_msg)) + result = self.client.get_usages_for_resource_provider(rp_uuid) + self.assertIsNone(result) + + def test_get_candidate_providers_OK(self, kss_req): + resources = 'VCPU:4,DISK_GB:64,MEMORY_MB:2048' + + fake_provider_summaries = { + "a99bad54-a275-4c4f-a8a3-ac00d57e5c64": { + "resources": { + "DISK_GB": { + "used": 0, + "capacity": 1900 + }, + }, + "traits": ["MISC_SHARES_VIA_AGGREGATE"], + "parent_provider_uuid": None, + "root_provider_uuid": "a99bad54-a275-4c4f-a8a3-ac00d57e5c64" + }, + "35791f28-fb45-4717-9ea9-435b3ef7c3b3": { + "resources": { + "VCPU": { + "used": 0, + "capacity": 384 + }, + "MEMORY_MB": { + "used": 0, + "capacity": 196608 + }, + }, + "traits": ["HW_CPU_X86_SSE2", "HW_CPU_X86_AVX2"], + "parent_provider_uuid": None, + "root_provider_uuid": "35791f28-fb45-4717-9ea9-435b3ef7c3b3" + }, + } + mock_json_data = { + 'provider_summaries': fake_provider_summaries, + } + + kss_req.return_value = fake_requests.FakeResponse( + 200, content=jsonutils.dump_as_bytes(mock_json_data)) + + result = self.client.get_candidate_providers(resources) + + expected_url = "/allocation_candidates?%s" % resources + self._assert_keystone_called_once(kss_req, expected_url, 'GET') + self.assertEqual(fake_provider_summaries, result) + + def test_get_candidate_providers_fail(self, kss_req): + rp_uuid = uuidutils.generate_uuid() + kss_req.return_value = fake_requests.FakeResponse( + 404, content=jsonutils.dump_as_bytes(self.fake_err_msg)) + result = self.client.get_candidate_providers(rp_uuid) + self.assertIsNone(result) diff --git a/watcher/tests/conf/test_list_opts.py b/watcher/tests/conf/test_list_opts.py index 4b36ea9d2..a54670ac1 100755 --- a/watcher/tests/conf/test_list_opts.py +++ b/watcher/tests/conf/test_list_opts.py @@ -33,7 +33,7 @@ class TestListOpts(base.TestCase): 'nova_client', 'glance_client', 'gnocchi_client', 'cinder_client', 'ceilometer_client', 'monasca_client', 'ironic_client', 'keystone_client', 'neutron_client', 'watcher_clients_auth', - 'collector'] + 'collector', 'placement_client'] self.opt_sections = list(dict(opts.list_opts()).keys()) def test_run_list_opts(self): diff --git a/watcher/tests/fakes.py b/watcher/tests/fakes.py index 24ed570f7..012b8a198 100644 --- a/watcher/tests/fakes.py +++ b/watcher/tests/fakes.py @@ -11,6 +11,7 @@ # under the License. import mock +import requests fakeAuthTokenHeaders = {'X-User-Id': u'773a902f022949619b5c2f32cd89d419', 'X-Roles': u'admin, ResellerAdmin, _member_', @@ -88,3 +89,25 @@ class FakeAuthProtocol(mock.Mock): super(FakeAuthProtocol, self).__init__(**kwargs) self.app = FakeApp() self.config = '' + + +class FakeResponse(requests.Response): + def __init__(self, status_code, content=None, headers=None): + """A requests.Response that can be used as a mock return_value. + + A key feature is that the instance will evaluate to True or False like + a real Response, based on the status_code. + Properties like ok, status_code, text, and content, and methods like + json(), work as expected based on the inputs. + :param status_code: Integer HTTP response code (200, 404, etc.) + :param content: String supplying the payload content of the response. + Using a json-encoded string will make the json() method + behave as expected. + :param headers: Dict of HTTP header values to set. + """ + super(FakeResponse, self).__init__() + self.status_code = status_code + if content: + self._content = content + if headers: + self.headers = headers