Replace openstack.org git:// URLs with https://

This is a mechanically generated change to replace openstack.org
git:// URLs with https:// equivalents.

This is in aid of a planned future move of the git hosting
infrastructure to a self-hosted instance of gitea (https://gitea.io),
which does not support the git wire protocol at this stage.

This update should result in no functional change.

For more information see the thread at

 http://lists.openstack.org/pipermail/openstack-discuss/2019-March/003825.html

Change-Id: I5ffcaf509ec6901f7e221a4312cc6b0577090440

.zuul.yaml

@@ -0,0 +1,9 @@
+- project:
+    templates:
+      - openstack-python-jobs
+      - openstack-python35-jobs
+      - publish-openstack-sphinx-docs
+      - check-requirements
+      - release-notes-jobs
+    gate:
+      queue: watcher

devstack/local.conf.compute

@@ -35,7 +35,7 @@ VNCSERVER_PROXYCLIENT_ADDRESS=$HOST_IP
 NOVA_INSTANCES_PATH=/opt/stack/data/instances
 
 # Enable the Ceilometer plugin for the compute agent
-enable_plugin ceilometer git://git.openstack.org/openstack/ceilometer
+enable_plugin ceilometer https://git.openstack.org/openstack/ceilometer
 disable_service ceilometer-acentral,ceilometer-collector,ceilometer-api
 
 LOGFILE=$DEST/logs/stack.sh.log

devstack/local.conf.controller

@@ -32,13 +32,13 @@ ENABLED_SERVICES+=,q-svc,q-dhcp,q-meta,q-agt,q-l3,neutron
 enable_service n-cauth
 
 # Enable the Watcher Dashboard plugin
-enable_plugin watcher-dashboard git://git.openstack.org/openstack/watcher-dashboard
+enable_plugin watcher-dashboard https://git.openstack.org/openstack/watcher-dashboard
 
 # Enable the Watcher plugin
-enable_plugin watcher git://git.openstack.org/openstack/watcher
+enable_plugin watcher https://git.openstack.org/openstack/watcher
 
 # Enable the Ceilometer plugin
-enable_plugin ceilometer git://git.openstack.org/openstack/ceilometer
+enable_plugin ceilometer https://git.openstack.org/openstack/ceilometer
 
 # This is the controller node, so disable the ceilometer compute agent
 disable_service ceilometer-acompute

doc/source/configuration/configuring.rst

@@ -165,7 +165,7 @@ You can easily generate and update a sample configuration file
 named :ref:`watcher.conf.sample <watcher_sample_configuration_files>` by using
 these following commands::
 
-    $ git clone git://git.openstack.org/openstack/watcher
+    $ git clone https://git.openstack.org/openstack/watcher
     $ cd watcher/
     $ tox -e genconfig
     $ vi etc/watcher/watcher.conf.sample

doc/source/contributor/devstack.rst

@@ -19,7 +19,7 @@ model. To enable the Watcher plugin with DevStack, add the following to the
 `[[local|localrc]]` section of your controller's `local.conf` to enable the
 Watcher plugin::
 
-    enable_plugin watcher git://git.openstack.org/openstack/watcher
+    enable_plugin watcher https://git.openstack.org/openstack/watcher
 
 For more detailed instructions, see `Detailed DevStack Instructions`_. Check
 out the `DevStack documentation`_ for more information regarding DevStack.

requirements.txt

@@ -37,7 +37,7 @@ python-keystoneclient>=3.8.0 # Apache-2.0
 python-monascaclient>=1.7.0 # Apache-2.0
 python-neutronclient>=6.3.0 # Apache-2.0
 python-novaclient>=9.0.0 # Apache-2.0
-python-openstackclient!=3.10.0,>=3.3.0 # Apache-2.0
+python-openstackclient>=3.11.0 # Apache-2.0
 python-ironicclient>=1.14.0 # Apache-2.0
 six>=1.9.0 # MIT
 SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT
@@ -45,5 +45,5 @@ stevedore>=1.20.0 # Apache-2.0
 taskflow>=2.7.0 # Apache-2.0
 WebOb>=1.7.1 # MIT
 WSME>=0.8 # MIT
-networkx>=1.10 # BSD
+networkx<2.0,>=1.10 # BSD
 

watcher/common/clients.py

@@ -110,8 +110,12 @@ class OpenStackClients(object):
                                                         'api_version')
         gnocchiclient_interface = self._get_client_option('gnocchi',
                                                           'endpoint_type')
+        adapter_options = {
+            "interface": gnocchiclient_interface
+        }
+
         self._gnocchi = gnclient.Client(gnocchiclient_version,
-                                        interface=gnocchiclient_interface,
+                                        adapter_options=adapter_options,
                                         session=self.session)
         return self._gnocchi
 

watcher/decision_engine/audit/continuous.py

@@ -62,11 +62,12 @@ class ContinuousAuditHandler(base.AuditHandler):
         if objects.audit.AuditStateTransitionManager().is_inactive(audit):
             # if audit isn't in active states, audit's job must be removed to
             # prevent using of inactive audit in future.
-            if self.scheduler.get_jobs():
+            jobs = [job for job in self.scheduler.get_jobs()
-                [job for job in self.scheduler.get_jobs()
+                    if job.name == 'execute_audit' and
-                 if job.name == 'execute_audit' and
+                    job.args[0].uuid == audit.uuid]
-                 job.args[0].uuid == audit.uuid][0].remove()
+            if jobs:
-                return True
+                jobs[0].remove()
+            return True
 
         return False
 

watcher/decision_engine/planner/weight.py

@@ -87,6 +87,7 @@ class WeightPlanner(base.BasePlanner):
     @staticmethod
     def chunkify(lst, n):
         """Yield successive n-sized chunks from lst."""
+        n = int(n)
         if n < 1:
             # Just to make sure the number is valid
             n = 1

watcher/decision_engine/strategy/strategies/workload_stabilization.py

@@ -252,7 +252,7 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
                     "No values returned by %(resource_id)s "
                     "for %(metric_name)s" % dict(
                         resource_id=instance.uuid, metric_name=meter))
-                avg_meter = 0
+                return
             if meter == 'cpu_util':
                 avg_meter /= float(100)
             instance_load[meter] = avg_meter
@@ -308,12 +308,10 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
                     )
 
                 if avg_meter is None:
-                    if meter_name == 'hardware.memory.used':
-                        avg_meter = node.memory
-                    if meter_name == 'compute.node.cpu.percent':
-                        avg_meter = 1
                     LOG.warning('No values returned by node %s for %s',
                                 node_id, meter_name)
+                    del hosts_load[node_id]
+                    break
                 else:
                     if meter_name == 'hardware.memory.used':
                         avg_meter /= oslo_utils.units.Ki
@@ -362,10 +360,12 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
         migration_case = []
         new_hosts = copy.deepcopy(hosts)
         instance_load = self.get_instance_load(instance)
+        if not instance_load:
+            return
         s_host_vcpus = new_hosts[src_node.uuid]['vcpus']
         d_host_vcpus = new_hosts[dst_node.uuid]['vcpus']
         for metric in self.metrics:
-            if metric is 'cpu_util':
+            if metric == 'cpu_util':
                 new_hosts[src_node.uuid][metric] -= (
                     self.transform_instance_cpu(instance_load, s_host_vcpus))
                 new_hosts[dst_node.uuid][metric] += (
@@ -408,6 +408,8 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
                     dst_node = self.compute_model.get_node_by_uuid(dst_host)
                     sd_case = self.calculate_migration_case(
                         hosts, instance, src_node, dst_node)
+                    if sd_case is None:
+                        break
 
                     weighted_sd = self.calculate_weighted_sd(sd_case[:-1])
 
@@ -416,6 +418,8 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
                             'host': dst_node.uuid, 'value': weighted_sd,
                             's_host': src_node.uuid, 'instance': instance.uuid}
                         instance_host_map.append(min_sd_case)
+                if sd_case is None:
+                    continue
         return sorted(instance_host_map, key=lambda x: x['value'])
 
     def check_threshold(self):
@@ -424,7 +428,12 @@ class WorkloadStabilization(base.WorkloadStabilizationBaseStrategy):
         normalized_load = self.normalize_hosts_load(hosts_load)
         for metric in self.metrics:
             metric_sd = self.get_sd(normalized_load, metric)
+            LOG.info("Standard deviation for %s is %s."
+                     % (metric, metric_sd))
             if metric_sd > float(self.thresholds[metric]):
+                LOG.info("Standard deviation of %s exceeds"
+                         " appropriate threshold %s."
+                         % (metric, metric_sd))
                 return self.simulate_migrations(hosts_load)
 
     def add_migration(self,

watcher/tests/common/test_clients.py

@@ -190,7 +190,8 @@ class TestClients(base.TestCase):
         osc.gnocchi()
         mock_call.assert_called_once_with(
             CONF.gnocchi_client.api_version,
-            interface=CONF.gnocchi_client.endpoint_type,
+            adapter_options={
+                "interface": CONF.gnocchi_client.endpoint_type},
             session=mock_session)
 
     @mock.patch.object(clients.OpenStackClients, 'session')

watcher/tests/decision_engine/audit/test_audit_handlers.py

@@ -379,8 +379,27 @@ class TestContinuousAuditHandler(base.DbTestCase):
         self.audits[0].next_run_time = (datetime.datetime.now() -
                                         datetime.timedelta(seconds=1800))
         m_is_inactive.return_value = True
-        m_get_jobs.return_value = None
+        m_get_jobs.return_value = []
 
         audit_handler.execute_audit(self.audits[0], self.context)
-        m_execute.assert_called_once_with(self.audits[0], self.context)
         self.assertIsNotNone(self.audits[0].next_run_time)
+
+    @mock.patch.object(scheduling.BackgroundSchedulerService, 'get_jobs')
+    def test_is_audit_inactive(self, mock_jobs):
+        audit_handler = continuous.ContinuousAuditHandler()
+        mock_jobs.return_value = mock.MagicMock()
+        audit_handler._scheduler = mock.MagicMock()
+
+        ap_jobs = [job.Job(mock.MagicMock(), name='execute_audit',
+                           func=audit_handler.execute_audit,
+                           args=(self.audits[0], mock.MagicMock()),
+                           kwargs={}),
+                   ]
+
+        audit_handler.update_audit_state(self.audits[1],
+                                         objects.audit.State.CANCELLED)
+        mock_jobs.return_value = ap_jobs
+        is_inactive = audit_handler._is_audit_inactive(self.audits[1])
+        self.assertTrue(is_inactive)
+        is_inactive = audit_handler._is_audit_inactive(self.audits[0])
+        self.assertFalse(is_inactive)

watcher/tests/decision_engine/model/ceilometer_metrics.py

@@ -176,6 +176,8 @@ class FakeCeilometerMetrics(object):
 
         # node 3
         mock['Node_6_hostname_6'] = 8
+        # This node doesn't send metrics
+        mock['LOST_NODE_hostname_7'] = None
         mock['Node_19_hostname_19'] = 10
         # node 4
         mock['INSTANCE_7_hostname_7'] = 4
@@ -190,7 +192,10 @@ class FakeCeilometerMetrics(object):
             # mock[uuid] = random.randint(1, 4)
             mock[uuid] = 8
 
-        return float(mock[str(uuid)])
+        if mock[str(uuid)] is not None:
+            return float(mock[str(uuid)])
+        else:
+            return mock[str(uuid)]
 
     @staticmethod
     def get_average_usage_instance_cpu_wb(uuid):
@@ -255,6 +260,8 @@ class FakeCeilometerMetrics(object):
 
         # node 4
         mock['INSTANCE_7'] = 4
+
+        mock['LOST_INSTANCE'] = None
         if uuid not in mock.keys():
             # mock[uuid] = random.randint(1, 4)
             mock[uuid] = 8

watcher/tests/decision_engine/model/data/scenario_1_with_1_node_unavailable.xml

@@ -0,0 +1,50 @@
+<ModelRoot>
+  <ComputeNode human_id="" uuid="Node_0" status="enabled" state="up" id="0" hostname="hostname_0" vcpus="40" disk="250" disk_capacity="250" memory="132">
+    <Instance state="active" human_id="" uuid="INSTANCE_0" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+    <Instance state="active" human_id="" uuid="INSTANCE_1" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  </ComputeNode>
+  <ComputeNode human_id="" uuid="Node_1" status="enabled" state="up" id="1" hostname="hostname_1" vcpus="40" disk="250" disk_capacity="250" memory="132">
+    <Instance state="active" human_id="" uuid="INSTANCE_2" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  </ComputeNode>
+  <ComputeNode human_id="" uuid="Node_2" status="enabled" state="up" id="2" hostname="hostname_2" vcpus="40" disk="250" disk_capacity="250" memory="132">
+    <Instance state="active" human_id="" uuid="INSTANCE_3" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+    <Instance state="active" human_id="" uuid="INSTANCE_4" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+    <Instance state="active" human_id="" uuid="INSTANCE_5" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  </ComputeNode>
+  <ComputeNode human_id="" uuid="Node_3" status="enabled" state="up" id="3" hostname="hostname_3" vcpus="40" disk="250" disk_capacity="250" memory="132">
+    <Instance state="active" human_id="" uuid="INSTANCE_6" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  </ComputeNode>
+  <ComputeNode human_id="" uuid="Node_4" status="enabled" state="up" id="4" hostname="hostname_4" vcpus="40" disk="250" disk_capacity="250" memory="132">
+    <Instance state="active" human_id="" uuid="INSTANCE_7" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  </ComputeNode>
+  <ComputeNode human_id="" uuid="LOST_NODE" status="enabled" state="up" id="1" hostname="hostname_7" vcpus="40" disk="250" disk_capacity="250" memory="132">
+    <Instance state="active" human_id="" uuid="LOST_INSTANCE" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  </ComputeNode>
+  <Instance state="active" human_id="" uuid="INSTANCE_10" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_11" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_12" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_13" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_14" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_15" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_16" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_17" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_18" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_19" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_20" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_21" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_22" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_23" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_24" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_25" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_26" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_27" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_28" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_29" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_30" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_31" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_32" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_33" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_34" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_8" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+  <Instance state="active" human_id="" uuid="INSTANCE_9" vcpus="10" disk="20" disk_capacity="20" memory="2" metadata='{"optimize": true,"top": "floor", "nested": {"x": "y"}}'/>
+</ModelRoot>

watcher/tests/decision_engine/model/faker_cluster_state.py

@@ -114,6 +114,9 @@ class FakerModelCollector(base.BaseClusterDataModelCollector):
     def generate_scenario_1(self):
         return self.load_model('scenario_1.xml')
 
+    def generate_scenario_1_with_1_node_unavailable(self):
+        return self.load_model('scenario_1_with_1_node_unavailable.xml')
+
     def generate_scenario_3_with_2_nodes(self):
         return self.load_model('scenario_3_with_2_nodes.xml')
 

watcher/tests/decision_engine/model/gnocchi_metrics.py

@@ -132,6 +132,8 @@ class FakeGnocchiMetrics(object):
 
         # node 3
         mock['Node_6_hostname_6'] = 8
+        # This node doesn't send metrics
+        mock['LOST_NODE_hostname_7'] = None
         mock['Node_19_hostname_19'] = 10
         # node 4
         mock['INSTANCE_7_hostname_7'] = 4
@@ -145,7 +147,10 @@ class FakeGnocchiMetrics(object):
         if uuid not in mock.keys():
             mock[uuid] = 8
 
-        return float(mock[str(uuid)])
+        if mock[str(uuid)] is not None:
+            return float(mock[str(uuid)])
+        else:
+            return mock[str(uuid)]
 
     @staticmethod
     def get_average_usage_instance_cpu(uuid):
@@ -172,6 +177,8 @@ class FakeGnocchiMetrics(object):
 
         # node 4
         mock['INSTANCE_7'] = 4
+
+        mock['LOST_INSTANCE'] = None
         if uuid not in mock.keys():
             mock[uuid] = 8
 

watcher/tests/decision_engine/strategy/strategies/test_workload_stabilization.py

@@ -172,6 +172,12 @@ class TestWorkloadStabilization(base.TestCase):
                 granularity=300, start_time=start_time, stop_time=stop_time,
                 aggregation='mean')
 
+    def test_get_instance_load_with_no_metrics(self):
+        model = self.fake_cluster.generate_scenario_1_with_1_node_unavailable()
+        self.m_model.return_value = model
+        lost_instance = model.get_instance_by_uuid("LOST_INSTANCE")
+        self.assertIsNone(self.strategy.get_instance_load(lost_instance))
+
     def test_normalize_hosts_load(self):
         self.m_model.return_value = self.fake_cluster.generate_scenario_1()
         fake_hosts = {'Node_0': {'cpu_util': 0.07, 'memory.resident': 7},
@@ -196,6 +202,12 @@ class TestWorkloadStabilization(base.TestCase):
         self.assertEqual(self.strategy.get_hosts_load(),
                          self.hosts_load_assert)
 
+    def test_get_hosts_load_with_node_missing(self):
+        self.m_model.return_value = \
+            self.fake_cluster.generate_scenario_1_with_1_node_unavailable()
+        self.assertEqual(self.hosts_load_assert,
+                         self.strategy.get_hosts_load())
+
     def test_get_sd(self):
         test_cpu_sd = 0.296
         test_ram_sd = 9.3