Update TOX_CONSTRAINTS_FILE for stable/2023.1

Update the URL to the upper-constraints file to point to the redirect
rule on releases.openstack.org so that anyone working on this branch
will switch to the correct upper-constraints list automatically when
the requirements repository branches.

Until the requirements repository has as stable/2023.1 branch, tests will
continue to use the upper-constraints list on master.

Change-Id: I733663a069ea2887ee8f63c56673e3960f8d1a0f

.gitreview

@@ -2,4 +2,4 @@
 host=review.opendev.org
 port=29418
 project=openstack/watcher.git
-defaultbranch=stable/2023.2
+defaultbranch=stable/2023.1

.zuul.yaml

@@ -89,7 +89,7 @@
 - job:
     name: watcher-tempest-multinode
     parent: watcher-tempest-functional
-    nodeset: openstack-two-node-jammy
+    nodeset: openstack-two-node-focal
     roles:
       - zuul: openstack/tempest
     group-vars:
@@ -107,7 +107,6 @@
           watcher-api: false
           watcher-decision-engine: true
           watcher-applier: false
-          c-bak: false
           ceilometer: false
           ceilometer-acompute: false
           ceilometer-acentral: false

doc/source/configuration/configuring.rst

@@ -372,7 +372,7 @@ You can configure and install Ceilometer by following the documentation below :
 #. https://docs.openstack.org/ceilometer/latest
 
 The built-in strategy 'basic_consolidation' provided by watcher requires
-"**compute.node.cpu.percent**" and "**cpu**" measurements to be collected
+"**compute.node.cpu.percent**" and "**cpu_util**" measurements to be collected
 by Ceilometer.
 The measurements available depend on the hypervisors that OpenStack manages on
 the specific implementation.

doc/source/contributor/plugin/strategy-plugin.rst

@@ -300,6 +300,6 @@ Using that you can now query the values for that specific metric:
 .. code-block:: py
 
     avg_meter = self.datasource_backend.statistic_aggregation(
-        instance.uuid, 'instance_cpu_usage', self.periods['instance'],
+        instance.uuid, 'cpu_util', self.periods['instance'],
         self.granularity,
         aggregation=self.aggregation_method['instance'])

doc/source/strategies/basic-server-consolidation.rst

@@ -26,7 +26,8 @@ metric                       service name plugins comment
                                                   ``compute_monitors`` option
                                                   to ``cpu.virt_driver`` in
                                                   the nova.conf.
-``cpu``                      ceilometer_  none
+``cpu_util``                 ceilometer_  none    cpu_util has been removed
+                                                  since Stein.
 ============================ ============ ======= ===========================
 
 .. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute

doc/source/strategies/vm_workload_consolidation.rst

@@ -22,7 +22,8 @@ The *vm_workload_consolidation* strategy requires the following metrics:
 ============================ ============ ======= =========================
 metric                       service name plugins comment
 ============================ ============ ======= =========================
-``cpu``                      ceilometer_  none
+``cpu_util``                 ceilometer_  none    cpu_util has been removed
+                                                  since Stein.
 ``memory.resident``          ceilometer_  none
 ``memory``                   ceilometer_  none
 ``disk.root.size``           ceilometer_  none

doc/source/strategies/workload-stabilization.rst

@@ -27,8 +27,9 @@ metric                       service name plugins comment
                                                   to ``cpu.virt_driver`` in the
                                                   nova.conf.
 ``hardware.memory.used``     ceilometer_  SNMP_
-``cpu``                      ceilometer_  none
-``instance_ram_usage``       ceilometer_  none
+``cpu_util``                 ceilometer_  none    cpu_util has been removed
+                                                  since Stein.
+``memory.resident``          ceilometer_  none
 ============================ ============ ======= =============================
 
 .. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute
@@ -106,10 +107,10 @@ parameter            type   default Value         description
                                                   period of all received ones.
 ==================== ====== ===================== =============================
 
-.. |metrics| replace:: ["instance_cpu_usage", "instance_ram_usage"]
-.. |thresholds| replace:: {"instance_cpu_usage": 0.2, "instance_ram_usage": 0.2}
-.. |weights| replace:: {"instance_cpu_usage_weight": 1.0, "instance_ram_usage_weight": 1.0}
-.. |instance_metrics| replace:: {"instance_cpu_usage": "compute.node.cpu.percent", "instance_ram_usage": "hardware.memory.used"}
+.. |metrics| replace:: ["cpu_util", "memory.resident"]
+.. |thresholds| replace:: {"cpu_util": 0.2, "memory.resident": 0.2}
+.. |weights| replace:: {"cpu_util_weight": 1.0, "memory.resident_weight": 1.0}
+.. |instance_metrics| replace:: {"cpu_util": "compute.node.cpu.percent", "memory.resident": "hardware.memory.used"}
 .. |periods| replace:: {"instance": 720, "node": 600}
 
 Efficacy Indicator
@@ -135,8 +136,8 @@ How to use it ?
       at1 workload_balancing --strategy workload_stabilization
 
     $ openstack optimize audit create -a at1 \
-      -p thresholds='{"instance_ram_usage": 0.05}' \
-      -p metrics='["instance_ram_usage"]'
+      -p thresholds='{"memory.resident": 0.05}' \
+      -p metrics='["memory.resident"]'
 
 External Links
 --------------

doc/source/strategies/workload_balance.rst

@@ -24,7 +24,8 @@ The *workload_balance* strategy requires the following metrics:
 ======================= ============ ======= =========================
 metric                  service name plugins comment
 ======================= ============ ======= =========================
-``cpu``                 ceilometer_  none
+``cpu_util``            ceilometer_  none    cpu_util has been removed
+                                             since Stein.
 ``memory.resident``     ceilometer_  none
 ======================= ============ ======= =========================
 
@@ -64,16 +65,15 @@ Configuration
 
 Strategy parameters are:
 
-============== ====== ==================== ====================================
-parameter      type   default Value        description
-============== ====== ==================== ====================================
-``metrics``    String 'instance_cpu_usage' Workload balance base on cpu or ram
-                                           utilization. Choices:
-                                           ['instance_cpu_usage',
-                                           'instance_ram_usage']
-``threshold``  Number 25.0                 Workload threshold for migration
-``period``     Number 300                  Aggregate time period of ceilometer
-============== ====== ==================== ====================================
+============== ====== ============= ====================================
+parameter      type   default Value description
+============== ====== ============= ====================================
+``metrics``    String 'cpu_util'    Workload balance base on cpu or ram
+                                    utilization. choice: ['cpu_util',
+                                    'memory.resident']
+``threshold``  Number 25.0          Workload threshold for migration
+``period``     Number 300           Aggregate time period of ceilometer
+============== ====== ============= ====================================
 
 Efficacy Indicator
 ------------------
@@ -95,7 +95,7 @@ How to use it ?
       at1 workload_balancing --strategy workload_balance
 
     $ openstack optimize audit create -a at1 -p threshold=26.0 \
-            -p period=310 -p metrics=instance_cpu_usage
+            -p period=310 -p metrics=cpu_util
 
 External Links
 --------------

releasenotes/source/2023.1.rst

@@ -1,6 +0,0 @@
-===========================
-2023.1 Series Release Notes
-===========================
-
-.. release-notes::
-   :branch: stable/2023.1

releasenotes/source/index.rst

@@ -21,7 +21,6 @@ Contents:
    :maxdepth: 1
 
    unreleased
-   2023.1
    zed
    yoga
    xena

releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po

@@ -2,16 +2,15 @@
 # Andi Chandler <andi@gowling.com>, 2018. #zanata
 # Andi Chandler <andi@gowling.com>, 2020. #zanata
 # Andi Chandler <andi@gowling.com>, 2022. #zanata
-# Andi Chandler <andi@gowling.com>, 2023. #zanata
 msgid ""
 msgstr ""
 "Project-Id-Version: python-watcher\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2023-08-14 03:05+0000\n"
+"POT-Creation-Date: 2022-08-29 03:02+0000\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"PO-Revision-Date: 2023-06-21 07:54+0000\n"
+"PO-Revision-Date: 2022-05-31 08:39+0000\n"
 "Last-Translator: Andi Chandler <andi@gowling.com>\n"
 "Language-Team: English (United Kingdom)\n"
 "Language: en_GB\n"
@@ -60,9 +59,6 @@ msgstr "1.9.0"
 msgid "2.0.0"
 msgstr "2.0.0"
 
-msgid "2023.1 Series Release Notes"
-msgstr "2023.1 Series Release Notes"
-
 msgid "3.0.0"
 msgstr "3.0.0"
 
@@ -973,9 +969,6 @@ msgstr "Xena Series Release Notes"
 msgid "Yoga Series Release Notes"
 msgstr "Yoga Series Release Notes"
 
-msgid "Zed Series Release Notes"
-msgstr "Zed Series Release Notes"
-
 msgid "``[watcher_datasources] datasources = gnocchi,monasca,ceilometer``"
 msgstr "``[watcher_datasources] datasources = gnocchi,monasca,ceilometer``"
 

requirements.txt

@@ -17,7 +17,7 @@ oslo.context>=2.21.0 # Apache-2.0
 oslo.db>=4.44.0 # Apache-2.0
 oslo.i18n>=3.20.0 # Apache-2.0
 oslo.log>=3.37.0 # Apache-2.0
-oslo.messaging>=14.1.0 # Apache-2.0
+oslo.messaging>=8.1.2 # Apache-2.0
 oslo.policy>=3.6.0 # Apache-2.0
 oslo.reports>=1.27.0 # Apache-2.0
 oslo.serialization>=2.25.0 # Apache-2.0

tox.ini

@@ -8,7 +8,7 @@ basepython = python3
 usedevelop = True
 allowlist_externals = find
                       rm
-install_command = pip install -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2023.2} {opts} {packages}
+install_command = pip install -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2023.1} {opts} {packages}
 setenv =
    VIRTUAL_ENV={envdir}
 deps =
@@ -30,7 +30,7 @@ passenv =
 commands =
     doc8 doc/source/ CONTRIBUTING.rst HACKING.rst README.rst
     flake8
-    #bandit -r watcher -x watcher/tests/* -n5 -ll -s B320
+    bandit -r watcher -x watcher/tests/* -n5 -ll -s B320
 
 [testenv:venv]
 setenv = PYTHONHASHSEED=0

watcher/common/rpc.py

@@ -121,7 +121,7 @@ class RequestContextSerializer(messaging.Serializer):
 def get_client(target, version_cap=None, serializer=None):
     assert TRANSPORT is not None
     serializer = RequestContextSerializer(serializer)
-    return messaging.get_rpc_client(
+    return messaging.RPCClient(
         TRANSPORT,
         target,
         version_cap=version_cap,

watcher/conf/grafana_client.py

@@ -134,13 +134,7 @@ GRAFANA_CLIENT_OPTS = [
                      "InfluxDB this will be the retention period. "
                      "These queries will need to be constructed using tools "
                      "such as Postman. Example: SELECT cpu FROM {4}."
-                     "cpu_percent WHERE host == '{1}' AND time > now()-{2}s"),
-    cfg.IntOpt('http_timeout',
-               min=0,
-               default=60,
-               mutable=True,
-               help='Timeout for Grafana request')
-]
+                     "cpu_percent WHERE host == '{1}' AND time > now()-{2}s")]
 
 
 def register_opts(conf):

watcher/decision_engine/datasources/gnocchi.py

@@ -38,7 +38,7 @@ class GnocchiHelper(base.DataSourceBase):
                       host_inlet_temp='hardware.ipmi.node.temperature',
                       host_airflow='hardware.ipmi.node.airflow',
                       host_power='hardware.ipmi.node.power',
-                      instance_cpu_usage='cpu',
+                      instance_cpu_usage='cpu_util',
                       instance_ram_usage='memory.resident',
                       instance_ram_allocated='memory',
                       instance_l3_cache_usage='cpu_l3_cache',
@@ -93,25 +93,6 @@ class GnocchiHelper(base.DataSourceBase):
 
             resource_id = resources[0]['id']
 
-        if meter_name == "instance_cpu_usage":
-            if resource_type != "instance":
-                LOG.warning("Unsupported resource type for metric "
-                            "'instance_cpu_usage': ", resource_type)
-                return
-
-            # The "cpu_util" gauge (percentage) metric has been removed.
-            # We're going to obtain the same result by using the rate of change
-            # aggregate operation.
-            if aggregate not in ("mean", "rate:mean"):
-                LOG.warning("Unsupported aggregate for instance_cpu_usage "
-                            "metric: %s. "
-                            "Supported aggregates: mean, rate:mean ",
-                            aggregate)
-                return
-
-            # TODO(lpetrut): consider supporting other aggregates.
-            aggregate = "rate:mean"
-
         raw_kwargs = dict(
             metric=meter,
             start=start_time,
@@ -136,17 +117,6 @@ class GnocchiHelper(base.DataSourceBase):
                 # Airflow from hardware.ipmi.node.airflow is reported as
                 # 1/10 th of actual CFM
                 return_value *= 10
-            if meter_name == "instance_cpu_usage":
-                # "rate:mean" can return negative values for migrated vms.
-                return_value = max(0, return_value)
-
-                # We're converting the cumulative cpu time (ns) to cpu usage
-                # percentage.
-                vcpus = resource.vcpus
-                if not vcpus:
-                    LOG.warning("instance vcpu count not set, assuming 1")
-                    vcpus = 1
-                return_value *= 100 / (granularity * 10e+8) / vcpus
 
         return return_value
 

watcher/decision_engine/datasources/grafana.py

@@ -138,8 +138,7 @@ class GrafanaHelper(base.DataSourceBase):
             raise exception.DataSourceNotAvailable(self.NAME)
 
         resp = requests.get(self._base_url + str(project_id) + '/query',
-                            params=params, headers=self._headers,
-                            timeout=CONF.grafana_client.http_timeout)
+                            params=params, headers=self._headers)
         if resp.status_code == HTTPStatus.OK:
             return resp
         elif resp.status_code == HTTPStatus.BAD_REQUEST:

watcher/decision_engine/model/collector/nova.py

@@ -48,7 +48,7 @@ class NovaClusterDataModelCollector(base.BaseClusterDataModelCollector):
                     "type": "array",
                     "items": {
                         "anyOf": [
-                            {"$ref": HOST_AGGREGATES + "host_aggr_id"},
+                            {"$ref": HOST_AGGREGATES + "id"},
                             {"$ref": HOST_AGGREGATES + "name"},
                         ]
                     }
@@ -98,8 +98,7 @@ class NovaClusterDataModelCollector(base.BaseClusterDataModelCollector):
                                 "type": "array",
                                 "items": {
                                     "anyOf": [
-                                        {"$ref":
-                                            HOST_AGGREGATES + "host_aggr_id"},
+                                        {"$ref": HOST_AGGREGATES + "id"},
                                         {"$ref": HOST_AGGREGATES + "name"},
                                     ]
                                 }
@@ -130,7 +129,7 @@ class NovaClusterDataModelCollector(base.BaseClusterDataModelCollector):
             "additionalProperties": False
         },
         "host_aggregates": {
-            "host_aggr_id": {
+            "id": {
                 "properties": {
                     "id": {
                         "oneOf": [

watcher/decision_engine/strategy/strategies/base.py

@@ -252,6 +252,9 @@ class BaseStrategy(loadable.Loadable, metaclass=abc.ABCMeta):
         if not self.compute_model:
             raise exception.ClusterStateNotDefined()
 
+        if self.compute_model.stale:
+            raise exception.ClusterStateStale()
+
         LOG.debug(self.compute_model.to_string())
 
     def execute(self, audit=None):

watcher/decision_engine/strategy/strategies/workload_balance.py

@@ -295,7 +295,7 @@ class WorkloadBalance(base.WorkloadStabilizationBaseStrategy):
                         self.threshold)
             return self.solution
 
-        # choose the server with largest cpu usage
+        # choose the server with largest cpu_util
         source_nodes = sorted(source_nodes,
                               reverse=True,
                               key=lambda x: (x[self._meter]))

watcher/tests/decision_engine/datasources/test_gnocchi_helper.py

@@ -40,25 +40,17 @@ class TestGnocchiHelper(base.BaseTestCase):
         self.addCleanup(stat_agg_patcher.stop)
 
     def test_gnocchi_statistic_aggregation(self, mock_gnocchi):
-        vcpus = 2
-        mock_instance = mock.Mock(
-            id='16a86790-327a-45f9-bc82-45839f062fdc',
-            vcpus=vcpus)
-
         gnocchi = mock.MagicMock()
-        # cpu time rate of change (ns)
-        mock_rate_measure = 360 * 10e+8 * vcpus * 5.5 / 100
         expected_result = 5.5
 
-        expected_measures = [
-            ["2017-02-02T09:00:00.000000", 360, mock_rate_measure]]
+        expected_measures = [["2017-02-02T09:00:00.000000", 360, 5.5]]
 
         gnocchi.metric.get_measures.return_value = expected_measures
         mock_gnocchi.return_value = gnocchi
 
         helper = gnocchi_helper.GnocchiHelper()
         result = helper.statistic_aggregation(
-            resource=mock_instance,
+            resource=mock.Mock(id='16a86790-327a-45f9-bc82-45839f062fdc'),
             resource_type='instance',
             meter_name='instance_cpu_usage',
             period=300,
@@ -67,14 +59,6 @@ class TestGnocchiHelper(base.BaseTestCase):
         )
         self.assertEqual(expected_result, result)
 
-        gnocchi.metric.get_measures.assert_called_once_with(
-            metric="cpu",
-            start=mock.ANY,
-            stop=mock.ANY,
-            resource_id=mock_instance.uuid,
-            granularity=360,
-            aggregation="rate:mean")
-
     def test_gnocchi_statistic_series(self, mock_gnocchi):
         gnocchi = mock.MagicMock()
         expected_result = {