OpenDev Migration Patch

This commit was bulk generated and pushed by the OpenDev sysadmins
as a part of the Git hosting and code review systems migration
detailed in these mailing list posts:

http://lists.openstack.org/pipermail/openstack-discuss/2019-March/003603.html
http://lists.openstack.org/pipermail/openstack-discuss/2019-April/004920.html

Attempts have been made to correct repository namespaces and
hostnames based on simple pattern matching, but it's possible some
were updated incorrectly or missed entirely. Please reach out to us
via the contact information listed at https://opendev.org/ with any
questions you may have.

.gitreview

@@ -1,5 +1,5 @@
 [gerrit]
-host=review.openstack.org
+host=review.opendev.org
 port=29418
 project=openstack/watcher.git
 defaultbranch=stable/pike

.zuul.yaml

@@ -0,0 +1,9 @@
+- project:
+    templates:
+      - openstack-python-jobs
+      - openstack-python35-jobs
+      - publish-openstack-sphinx-docs
+      - check-requirements
+      - release-notes-jobs
+    gate:
+      queue: watcher

devstack/local.conf.compute

@@ -35,7 +35,7 @@ VNCSERVER_PROXYCLIENT_ADDRESS=$HOST_IP
 NOVA_INSTANCES_PATH=/opt/stack/data/instances
 
 # Enable the Ceilometer plugin for the compute agent
-enable_plugin ceilometer git://git.openstack.org/openstack/ceilometer
+enable_plugin ceilometer https://git.openstack.org/openstack/ceilometer
 disable_service ceilometer-acentral,ceilometer-collector,ceilometer-api
 
 LOGFILE=$DEST/logs/stack.sh.log

devstack/local.conf.controller

@@ -32,13 +32,13 @@ ENABLED_SERVICES+=,q-svc,q-dhcp,q-meta,q-agt,q-l3,neutron
 enable_service n-cauth
 
 # Enable the Watcher Dashboard plugin
-enable_plugin watcher-dashboard git://git.openstack.org/openstack/watcher-dashboard
+enable_plugin watcher-dashboard https://git.openstack.org/openstack/watcher-dashboard
 
 # Enable the Watcher plugin
-enable_plugin watcher git://git.openstack.org/openstack/watcher
+enable_plugin watcher https://git.openstack.org/openstack/watcher
 
 # Enable the Ceilometer plugin
-enable_plugin ceilometer git://git.openstack.org/openstack/ceilometer
+enable_plugin ceilometer https://git.openstack.org/openstack/ceilometer
 
 # This is the controller node, so disable the ceilometer compute agent
 disable_service ceilometer-acompute

doc/source/configuration/configuring.rst

@@ -165,7 +165,7 @@ You can easily generate and update a sample configuration file
 named :ref:`watcher.conf.sample <watcher_sample_configuration_files>` by using
 these following commands::
 
-    $ git clone git://git.openstack.org/openstack/watcher
+    $ git clone https://git.openstack.org/openstack/watcher
     $ cd watcher/
     $ tox -e genconfig
     $ vi etc/watcher/watcher.conf.sample

doc/source/contributor/devstack.rst

@@ -19,7 +19,7 @@ model. To enable the Watcher plugin with DevStack, add the following to the
 `[[local|localrc]]` section of your controller's `local.conf` to enable the
 Watcher plugin::
 
-    enable_plugin watcher git://git.openstack.org/openstack/watcher
+    enable_plugin watcher https://git.openstack.org/openstack/watcher
 
 For more detailed instructions, see `Detailed DevStack Instructions`_. Check
 out the `DevStack documentation`_ for more information regarding DevStack.

requirements.txt

@@ -37,7 +37,7 @@ python-keystoneclient>=3.8.0 # Apache-2.0
 python-monascaclient>=1.7.0 # Apache-2.0
 python-neutronclient>=6.3.0 # Apache-2.0
 python-novaclient>=9.0.0 # Apache-2.0
-python-openstackclient!=3.10.0,>=3.3.0 # Apache-2.0
+python-openstackclient>=3.11.0 # Apache-2.0
 python-ironicclient>=1.14.0 # Apache-2.0
 six>=1.9.0 # MIT
 SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT
@@ -45,5 +45,5 @@ stevedore>=1.20.0 # Apache-2.0
 taskflow>=2.7.0 # Apache-2.0
 WebOb>=1.7.1 # MIT
 WSME>=0.8 # MIT
-networkx>=1.10 # BSD
+networkx<2.0,>=1.10 # BSD
 

watcher/common/clients.py

@@ -110,8 +110,12 @@ class OpenStackClients(object):
                                                         'api_version')
         gnocchiclient_interface = self._get_client_option('gnocchi',
                                                           'endpoint_type')
+        adapter_options = {
+            "interface": gnocchiclient_interface
+        }
+
         self._gnocchi = gnclient.Client(gnocchiclient_version,
-                                        interface=gnocchiclient_interface,
+                                        adapter_options=adapter_options,
                                         session=self.session)
         return self._gnocchi
 

watcher/decision_engine/audit/continuous.py

@@ -62,11 +62,12 @@ class ContinuousAuditHandler(base.AuditHandler):
         if objects.audit.AuditStateTransitionManager().is_inactive(audit):
             # if audit isn't in active states, audit's job must be removed to
             # prevent using of inactive audit in future.
-            if self.scheduler.get_jobs():
+            jobs = [job for job in self.scheduler.get_jobs()
-                [job for job in self.scheduler.get_jobs()
+                    if job.name == 'execute_audit' and
-                 if job.name == 'execute_audit' and
+                    job.args[0].uuid == audit.uuid]
-                 job.args[0].uuid == audit.uuid][0].remove()
+            if jobs:
-                return True
+                jobs[0].remove()
+            return True
 
         return False
 

watcher/decision_engine/planner/weight.py

@@ -87,6 +87,7 @@ class WeightPlanner(base.BasePlanner):
     @staticmethod
     def chunkify(lst, n):
         """Yield successive n-sized chunks from lst."""
+        n = int(n)
         if n < 1:
             # Just to make sure the number is valid
             n = 1

watcher/decision_engine/strategy/strategies/workload_stabilization.py

@@ -252,7 +252,7 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
                     "No values returned by %(resource_id)s "
                     "for %(metric_name)s" % dict(
                         resource_id=instance.uuid, metric_name=meter))
-                avg_meter = 0
+                return
             if meter == 'cpu_util':
                 avg_meter /= float(100)
             instance_load[meter] = avg_meter
@@ -308,12 +308,10 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
                     )
 
                 if avg_meter is None:
-                    if meter_name == 'hardware.memory.used':
-                        avg_meter = node.memory
-                    if meter_name == 'compute.node.cpu.percent':
-                        avg_meter = 1
                     LOG.warning('No values returned by node %s for %s',
                                 node_id, meter_name)
+                    del hosts_load[node_id]
+                    break
                 else:
                     if meter_name == 'hardware.memory.used':
                         avg_meter /= oslo_utils.units.Ki
@@ -362,10 +360,12 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
         migration_case = []
         new_hosts = copy.deepcopy(hosts)
         instance_load = self.get_instance_load(instance)
+        if not instance_load:
+            return
         s_host_vcpus = new_hosts[src_node.uuid]['vcpus']
         d_host_vcpus = new_hosts[dst_node.uuid]['vcpus']
         for metric in self.metrics:
-            if metric is 'cpu_util':
+            if metric == 'cpu_util':
                 new_hosts[src_node.uuid][metric] -= (
                     self.transform_instance_cpu(instance_load, s_host_vcpus))
                 new_hosts[dst_node.uuid][metric] += (
@@ -408,6 +408,8 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
                     dst_node = self.compute_model.get_node_by_uuid(dst_host)
                     sd_case = self.calculate_migration_case(
                         hosts, instance, src_node, dst_node)
+                    if sd_case is None:
+                        break
 
                     weighted_sd = self.calculate_weighted_sd(sd_case[:-1])
 
@@ -416,6 +418,8 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
                             'host': dst_node.uuid, 'value': weighted_sd,
                             's_host': src_node.uuid, 'instance': instance.uuid}
                         instance_host_map.append(min_sd_case)
+                if sd_case is None:
+                    continue
         return sorted(instance_host_map, key=lambda x: x['value'])
 
     def check_threshold(self):
@@ -424,7 +428,12 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
         normalized_load = self.normalize_hosts_load(hosts_load)
         for metric in self.metrics:
             metric_sd = self.get_sd(normalized_load, metric)
+            LOG.info("Standard deviation for %s is %s."
+                     % (metric, metric_sd))
             if metric_sd > float(self.thresholds[metric]):
+                LOG.info("Standard deviation of %s exceeds"
+                         " appropriate threshold %s."
+                         % (metric, metric_sd))
                 return self.simulate_migrations(hosts_load)
 
     def add_migration(self,

watcher/tests/common/test_clients.py

@@ -190,7 +190,8 @@ class TestClients(base.TestCase):
         osc.gnocchi()
         mock_call.assert_called_once_with(
             CONF.gnocchi_client.api_version,
-            interface=CONF.gnocchi_client.endpoint_type,
+            adapter_options={
+                "interface": CONF.gnocchi_client.endpoint_type},
             session=mock_session)
 
     @mock.patch.object(clients.OpenStackClients, 'session')

watcher/tests/decision_engine/audit/test_audit_handlers.py

@@ -379,8 +379,27 @@ class TestContinuousAuditHandler(base.DbTestCase):
         self.audits[0].next_run_time = (datetime.datetime.now() -
                                         datetime.timedelta(seconds=1800))
         m_is_inactive.return_value = True
-        m_get_jobs.return_value = None
+        m_get_jobs.return_value = []
 
         audit_handler.execute_audit(self.audits[0], self.context)
-        m_execute.assert_called_once_with(self.audits[0], self.context)
         self.assertIsNotNone(self.audits[0].next_run_time)
+
+    @mock.patch.object(scheduling.BackgroundSchedulerService, 'get_jobs')
+    def test_is_audit_inactive(self, mock_jobs):
+        audit_handler = continuous.ContinuousAuditHandler()
+        mock_jobs.return_value = mock.MagicMock()
+        audit_handler._scheduler = mock.MagicMock()
+
+        ap_jobs = [job.Job(mock.MagicMock(), name='execute_audit',
+                           func=audit_handler.execute_audit,
+                           args=(self.audits[0], mock.MagicMock()),
+                           kwargs={}),
+                   ]
+
+        audit_handler.update_audit_state(self.audits[1],
+                                         objects.audit.State.CANCELLED)
+        mock_jobs.return_value = ap_jobs
+        is_inactive = audit_handler._is_audit_inactive(self.audits[1])
+        self.assertTrue(is_inactive)
+        is_inactive = audit_handler._is_audit_inactive(self.audits[0])
+        self.assertFalse(is_inactive)

watcher/tests/decision_engine/model/ceilometer_metrics.py

@@ -176,6 +176,8 @@ class FakeCeilometerMetrics(object):
 
         # node 3
         mock['Node_6_hostname_6'] = 8
+        # This node doesn't send metrics
+        mock['LOST_NODE_hostname_7'] = None
         mock['Node_19_hostname_19'] = 10
         # node 4
         mock['INSTANCE_7_hostname_7'] = 4
@@ -190,7 +192,10 @@ class FakeCeilometerMetrics(object):
             # mock[uuid] = random.randint(1, 4)
             mock[uuid] = 8
 
-        return float(mock[str(uuid)])
+        if mock[str(uuid)] is not None:
+            return float(mock[str(uuid)])
+        else:
+            return mock[str(uuid)]
 
     @staticmethod
     def get_average_usage_instance_cpu_wb(uuid):
@@ -255,6 +260,8 @@ class FakeCeilometerMetrics(object):
 
         # node 4
         mock['INSTANCE_7'] = 4
+
+        mock['LOST_INSTANCE'] = None
         if uuid not in mock.keys():
             # mock[uuid] = random.randint(1, 4)
             mock[uuid] = 8

watcher/tests/decision_engine/model/data/scenario_1_with_1_node_unavailable.xml

@@ -0,0 +1,50 @@
+<ModelRoot>
+  <ComputeNode human_id="" uuid="Node_0" status="enabled" state="up" id="0" hostname="hostname_0" vcpus="40" disk="250" disk_capacity="250" memory="132">
+    <Instance state="active" human_id="" uuid="INSTANCE_0" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+    <Instance state="active" human_id="" uuid="INSTANCE_1" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  </ComputeNode>
+  <ComputeNode human_id="" uuid="Node_1" status="enabled" state="up" id="1" hostname="hostname_1" vcpus="40" disk="250" disk_capacity="250" memory="132">
+    <Instance state="active" human_id="" uuid="INSTANCE_2" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  </ComputeNode>
+  <ComputeNode human_id="" uuid="Node_2" status="enabled" state="up" id="2" hostname="hostname_2" vcpus="40" disk="250" disk_capacity="250" memory="132">
+    <Instance state="active" human_id="" uuid="INSTANCE_3" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+    <Instance state="active" human_id="" uuid="INSTANCE_4" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+    <Instance state="active" human_id="" uuid="INSTANCE_5" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  </ComputeNode>
+  <ComputeNode human_id="" uuid="Node_3" status="enabled" state="up" id="3" hostname="hostname_3" vcpus="40" disk="250" disk_capacity="250" memory="132">
+    <Instance state="active" human_id="" uuid="INSTANCE_6" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  </ComputeNode>
+  <ComputeNode human_id="" uuid="Node_4" status="enabled" state="up" id="4" hostname="hostname_4" vcpus="40" disk="250" disk_capacity="250" memory="132">
+    <Instance state="active" human_id="" uuid="INSTANCE_7" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  </ComputeNode>
+  <ComputeNode human_id="" uuid="LOST_NODE" status="enabled" state="up" id="1" hostname="hostname_7" vcpus="40" disk="250" disk_capacity="250" memory="132">
+    <Instance state="active" human_id="" uuid="LOST_INSTANCE" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  </ComputeNode>
+  <Instance state="active" human_id="" uuid="INSTANCE_10" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_11" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_12" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_13" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_14" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_15" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_16" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_17" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_18" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_19" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_20" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_21" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_22" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_23" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_24" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_25" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_26" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_27" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_28" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_29" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_30" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_31" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_32" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_33" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_34" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_8" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_9" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+</ModelRoot>

watcher/tests/decision_engine/model/faker_cluster_state.py

@@ -114,6 +114,9 @@ class FakerModelCollector(base.BaseClusterDataModelCollector):
     def generate_scenario_1(self):
         return self.load_model('scenario_1.xml')
 
+    def generate_scenario_1_with_1_node_unavailable(self):
+        return self.load_model('scenario_1_with_1_node_unavailable.xml')
+
     def generate_scenario_3_with_2_nodes(self):
         return self.load_model('scenario_3_with_2_nodes.xml')
 

watcher/tests/decision_engine/model/gnocchi_metrics.py

@@ -132,6 +132,8 @@ class FakeGnocchiMetrics(object):
 
         # node 3
         mock['Node_6_hostname_6'] = 8
+        # This node doesn't send metrics
+        mock['LOST_NODE_hostname_7'] = None
         mock['Node_19_hostname_19'] = 10
         # node 4
         mock['INSTANCE_7_hostname_7'] = 4
@@ -145,7 +147,10 @@ class FakeGnocchiMetrics(object):
         if uuid not in mock.keys():
             mock[uuid] = 8
 
-        return float(mock[str(uuid)])
+        if mock[str(uuid)] is not None:
+            return float(mock[str(uuid)])
+        else:
+            return mock[str(uuid)]
 
     @staticmethod
     def get_average_usage_instance_cpu(uuid):
@@ -172,6 +177,8 @@ class FakeGnocchiMetrics(object):
 
         # node 4
         mock['INSTANCE_7'] = 4
+
+        mock['LOST_INSTANCE'] = None
         if uuid not in mock.keys():
             mock[uuid] = 8
 

watcher/tests/decision_engine/strategy/strategies/test_workload_stabilization.py

@@ -172,6 +172,12 @@ class TestWorkloadStabilization(base.TestCase):
                 granularity=300, start_time=start_time, stop_time=stop_time,
                 aggregation='mean')
 
+    def test_get_instance_load_with_no_metrics(self):
+        model = self.fake_cluster.generate_scenario_1_with_1_node_unavailable()
+        self.m_model.return_value = model
+        lost_instance = model.get_instance_by_uuid("LOST_INSTANCE")
+        self.assertIsNone(self.strategy.get_instance_load(lost_instance))
+
     def test_normalize_hosts_load(self):
         self.m_model.return_value = self.fake_cluster.generate_scenario_1()
         fake_hosts = {'Node_0': {'cpu_util': 0.07, 'memory.resident': 7},
@@ -196,6 +202,12 @@ class TestWorkloadStabilization(base.TestCase):
         self.assertEqual(self.strategy.get_hosts_load(),
                          self.hosts_load_assert)
 
+    def test_get_hosts_load_with_node_missing(self):
+        self.m_model.return_value = \
+            self.fake_cluster.generate_scenario_1_with_1_node_unavailable()
+        self.assertEqual(self.hosts_load_assert,
+                         self.strategy.get_hosts_load())
+
     def test_get_sd(self):
         test_cpu_sd = 0.296
         test_ram_sd = 9.3