Merge "Fix TypeError in LOG.debug"

Change-Id: I4a4050081d0a22cc66fdb311ef676d0ba802bb72
Closes-Bug: #1785962

.zuul.yaml

@@ -1,18 +1,17 @@
 - project:
     check:
       jobs:
-        - watcher-tempest-functional:
-            voting: false
+        - watcher-tempest-functional
         - watcher-tempest-dummy_optim
         - watcher-tempest-actuator
         - watcher-tempest-basic_optim
         - watcher-tempest-workload_balancing
-        - watcherclient-tempest-functional:
-            voting: false
+        - watcherclient-tempest-functional
+        - watcher-tempest-zone_migration
         - openstack-tox-lower-constraints
     gate:
       jobs:
-        # - watcher-tempest-functional
+        - watcher-tempest-functional
         - openstack-tox-lower-constraints
 
 - job:
@@ -43,6 +42,13 @@
     vars:
       tempest_test_regex: 'watcher_tempest_plugin.tests.scenario.test_execute_workload_balancing'
 
+- job:
+    name: watcher-tempest-zone_migration
+    parent: watcher-tempest-multinode
+    voting: false
+    vars:
+      tempest_test_regex: 'watcher_tempest_plugin.tests.scenario.test_execute_zone_migration'
+
 - job:
     name: watcher-tempest-multinode
     parent: watcher-tempest-functional
@@ -58,6 +64,13 @@
             $NOVA_CONF:
               libvirt:
                 live_migration_uri: 'qemu+ssh://root@%s/system'
+            $WATCHER_CONF:
+              watcher_cluster_data_model_collectors.compute:
+                period: 120
+              watcher_cluster_data_model_collectors.baremetal:
+                period: 120
+              watcher_cluster_data_model_collectors.storage:
+                period: 120
         devstack_services:
           watcher-api: false
           watcher-decision-engine: true
@@ -79,6 +92,13 @@
           $NOVA_CONF:
             libvirt:
               live_migration_uri: 'qemu+ssh://root@%s/system'
+          $WATCHER_CONF:
+              watcher_cluster_data_model_collectors.compute:
+                period: 120
+              watcher_cluster_data_model_collectors.baremetal:
+                period: 120
+              watcher_cluster_data_model_collectors.storage:
+                period: 120
         test-config:
           $TEMPEST_CONFIG:
             compute:

doc/source/man/watcher-db-manage.rst

@@ -241,10 +241,9 @@ purge
   The maximum number of database objects we expect to be deleted. If exceeded,
   this will prevent any deletion.
 
-.. option:: -t, --audit-template
+.. option:: -t, --goal
 
-  Either the UUID or name of the soft deleted audit template to purge. This
-  will also include any related objects with it.
+  Either the UUID or name of the goal to purge.
 
 .. option:: -e, --exclude-orphans
 

doc/source/strategies/noisy_neighbor.rst

@@ -0,0 +1,97 @@
+==============
+Noisy neighbor
+==============
+
+Synopsis
+--------
+
+**display name**: ``Noisy Neighbor``
+
+**goal**: ``noisy_neighbor``
+
+    .. watcher-term:: watcher.decision_engine.strategy.strategies.noisy_neighbor.NoisyNeighbor
+
+Requirements
+------------
+
+Metrics
+*******
+
+The *noisy_neighbor* strategy requires the following metrics:
+
+============================ ============ ======= =======================
+metric                       service name plugins comment
+============================ ============ ======= =======================
+``cpu_l3_cache``             ceilometer_  none     Intel CMT_ is required
+============================ ============ ======= =======================
+
+.. _CMT: http://www.intel.com/content/www/us/en/architecture-and-technology/resource-director-technology.html
+.. _ceilometer: https://docs.openstack.org/ceilometer/latest/admin/telemetry-measurements.html#openstack-compute
+
+Cluster data model
+******************
+
+Default Watcher's Compute cluster data model:
+
+    .. watcher-term:: watcher.decision_engine.model.collector.nova.NovaClusterDataModelCollector
+
+Actions
+*******
+
+Default Watcher's actions:
+
+    .. list-table::
+       :widths: 30 30
+       :header-rows: 1
+
+       * - action
+         - description
+       * - ``migration``
+         - .. watcher-term:: watcher.applier.actions.migration.Migrate
+
+Planner
+*******
+
+Default Watcher's planner:
+
+    .. watcher-term:: watcher.decision_engine.planner.weight.WeightPlanner
+
+Configuration
+-------------
+
+Strategy parameter is:
+
+==================== ====== ============= ============================
+parameter            type   default       Value description
+==================== ====== ============= ============================
+``cache_threshold``  Number 35.0          Performance drop in L3_cache
+                                          threshold for migration
+==================== ====== ============= ============================
+
+
+Efficacy Indicator
+------------------
+
+None
+
+Algorithm
+---------
+
+For more information on the noisy neighbor strategy please refer to:
+http://specs.openstack.org/openstack/watcher-specs/specs/pike/implemented/noisy_neighbor_strategy.html
+
+How to use it ?
+---------------
+
+.. code-block:: shell
+
+    $ openstack optimize audittemplate create \
+      at1 noisy_neighbor --strategy noisy_neighbor
+
+    $ openstack optimize audit create -a at1 \
+      -p cache_threshold=45.0
+
+External Links
+--------------
+
+None

lower-constraints.txt

@@ -155,7 +155,6 @@ ujson==1.35
 unittest2==1.1.0
 urllib3==1.22
 vine==1.1.4
-voluptuous==0.11.1
 waitress==1.1.0
 warlock==1.3.0
 WebOb==1.7.4

requirements.txt

@@ -28,7 +28,6 @@ PasteDeploy>=1.5.2 # MIT
 pbr>=3.1.1 # Apache-2.0
 pecan>=1.2.1 # BSD
 PrettyTable<0.8,>=0.7.2 # BSD
-voluptuous>=0.11.1 # BSD License
 gnocchiclient>=7.0.1 # Apache-2.0
 python-ceilometerclient>=2.9.0 # Apache-2.0
 python-cinderclient>=3.5.0 # Apache-2.0

watcher/api/controllers/v1/action_plan.py

@@ -581,7 +581,7 @@ class ActionPlansController(rest.RestController):
 
         if action_plan_to_start['state'] != \
                 objects.action_plan.State.RECOMMENDED:
-            raise Exception.StartError(
+            raise exception.StartError(
                 state=action_plan_to_start.state)
 
         action_plan_to_start['state'] = objects.action_plan.State.PENDING

watcher/api/scheduling.py

@@ -52,7 +52,8 @@ class APISchedulingService(scheduling.BackgroundSchedulerService):
                 self.services_status[service.id] = result
                 notifications.service.send_service_update(context, service,
                                                           state=result)
-                if result == failed_s:
+                if (result == failed_s) and (
+                        service.name == 'watcher-decision-engine'):
                     audit_filters = {
                         'audit_type': objects.audit.AuditType.CONTINUOUS.value,
                         'state': objects.audit.State.ONGOING,

watcher/common/exception.py

@@ -150,7 +150,7 @@ class ResourceNotFound(ObjectNotFound):
 
 
 class InvalidParameter(Invalid):
-        msg_fmt = _("%(parameter)s has to be of type %(parameter_type)s")
+    msg_fmt = _("%(parameter)s has to be of type %(parameter_type)s")
 
 
 class InvalidIdentity(Invalid):
@@ -514,9 +514,9 @@ class NegativeLimitError(WatcherException):
 
 
 class NotificationPayloadError(WatcherException):
-    _msg_fmt = _("Payload not populated when trying to send notification "
-                 "\"%(class_name)s\"")
+    msg_fmt = _("Payload not populated when trying to send notification "
+                "\"%(class_name)s\"")
 
 
 class InvalidPoolAttributeValue(Invalid):
-        msg_fmt = _("The %(name)s pool %(attribute)s is not integer")
+    msg_fmt = _("The %(name)s pool %(attribute)s is not integer")

watcher/common/nova_helper.py

@@ -22,7 +22,6 @@ import time
 from novaclient import api_versions
 from oslo_log import log
 
-import cinderclient.exceptions as ciexceptions
 import glanceclient.exc as glexceptions
 import novaclient.exceptions as nvexceptions
 
@@ -711,25 +710,6 @@ class NovaHelper(object):
     def get_hostname(self, instance):
         return str(getattr(instance, 'OS-EXT-SRV-ATTR:host'))
 
-    def get_flavor_instance(self, instance, cache):
-        fid = instance.flavor['id']
-        if fid in cache:
-            flavor = cache.get(fid)
-        else:
-            try:
-                flavor = self.nova.flavors.get(fid)
-            except ciexceptions.NotFound:
-                flavor = None
-            cache[fid] = flavor
-        attr_defaults = [('name', 'unknown-id-%s' % fid),
-                         ('vcpus', 0), ('ram', 0), ('disk', 0),
-                         ('ephemeral', 0), ('extra_specs', {})]
-        for attr, default in attr_defaults:
-            if not flavor:
-                instance.flavor[attr] = default
-                continue
-            instance.flavor[attr] = getattr(flavor, attr, default)
-
     def get_running_migration(self, instance_id):
         return self.nova.server_migrations.list(server=instance_id)
 

watcher/db/sqlalchemy/models.py

@@ -23,8 +23,10 @@ from sqlalchemy import Boolean
 from sqlalchemy import Column
 from sqlalchemy import DateTime
 from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import Float
 from sqlalchemy import ForeignKey
 from sqlalchemy import Integer
+from sqlalchemy import LargeBinary
 from sqlalchemy import Numeric
 from sqlalchemy import orm
 from sqlalchemy import String
@@ -296,3 +298,23 @@ class ActionDescription(Base):
     id = Column(Integer, primary_key=True)
     action_type = Column(String(255), nullable=False)
     description = Column(String(255), nullable=False)
+
+
+class APScheulerJob(Base):
+    """Represents apscheduler jobs"""
+
+    __tablename__ = 'apscheduler_jobs'
+    __table_args__ = (
+        UniqueConstraint('id',
+                         name="uniq_apscheduler_jobs0id"),
+        table_args()
+    )
+    id = Column(String(191), nullable=False, primary_key=True)
+    next_run_time = Column(Float(25), index=True)
+    job_state = Column(LargeBinary, nullable=False)
+    tag = Column(JSONEncodedDict(), nullable=True)
+    service_id = Column(Integer, ForeignKey('services.id'),
+                        nullable=False)
+
+    service = orm.relationship(
+        Service, foreign_keys=service_id, lazy=None)

watcher/decision_engine/strategy/strategies/saving_energy.py

@@ -31,6 +31,8 @@ LOG = log.getLogger(__name__)
 class SavingEnergy(base.SavingEnergyBaseStrategy):
     """Saving Energy Strategy
 
+    *Description*
+
     Saving Energy Strategy together with VM Workload Consolidation Strategy
     can perform the Dynamic Power Management (DPM) functionality, which tries
     to save power by dynamically consolidating workloads even further during
@@ -51,19 +53,29 @@ class SavingEnergy(base.SavingEnergyBaseStrategy):
     the given number and there are spare unused nodes(in poweroff state),
     randomly select some nodes(unused,poweroff) and power on them.
 
+    *Requirements*
+
     In this policy, in order to calculate the min_free_hosts_num,
     users must provide two parameters:
 
-      * One parameter("min_free_hosts_num") is a constant int number.
-        This number should be int type and larger than zero.
+    * One parameter("min_free_hosts_num") is a constant int number.
+      This number should be int type and larger than zero.
 
-      * The other parameter("free_used_percent") is a percentage number, which
-        describes the quotient of min_free_hosts_num/nodes_with_VMs_num,
-        where nodes_with_VMs_num is the number of nodes with VMs running on it.
-        This parameter is used to calculate a dynamic min_free_hosts_num.
-        The nodes with VMs refer to those nodes with VMs running on it.
+    * The other parameter("free_used_percent") is a percentage number, which
+      describes the quotient of min_free_hosts_num/nodes_with_VMs_num,
+      where nodes_with_VMs_num is the number of nodes with VMs running on it.
+      This parameter is used to calculate a dynamic min_free_hosts_num.
+      The nodes with VMs refer to those nodes with VMs running on it.
 
     Then choose the larger one as the final min_free_hosts_num.
+
+    *Limitations*
+
+    * at least 2 physical compute hosts
+
+    *Spec URL*
+
+    http://specs.openstack.org/openstack/watcher-specs/specs/pike/implemented/energy-saving-strategy.html
     """
 
     def __init__(self, config, osc=None):
@@ -113,16 +125,16 @@ class SavingEnergy(base.SavingEnergyBaseStrategy):
             "properties": {
                 "free_used_percent": {
                     "description": ("a rational number, which describes the"
-                                    "quotient of"
+                                    " quotient of"
                                     " min_free_hosts_num/nodes_with_VMs_num"
-                                    "where nodes_with_VMs_num is the number"
-                                    "of nodes with VMs"),
+                                    " where nodes_with_VMs_num is the number"
+                                    " of nodes with VMs"),
                     "type": "number",
                     "default": 10.0
                 },
                 "min_free_hosts_num": {
                     "description": ("minimum number of hosts without VMs"
-                                    "but still powered on"),
+                                    " but still powered on"),
                     "type": "number",
                     "default": 1
                 },

watcher/decision_engine/strategy/strategies/storage_capacity_balance.py

@@ -41,6 +41,15 @@ class StorageCapacityBalance(base.WorkloadStabilizationBaseStrategy):
 
     * You must have at least 2 cinder volume pools to run
       this strategy.
+
+    *Limitations*
+
+    * Volume migration depends on the storage device.
+      It may take a long time.
+
+    *Spec URL*
+
+    http://specs.openstack.org/openstack/watcher-specs/specs/queens/implemented/storage-capacity-balance.html
     """
 
     def __init__(self, config, osc=None):

watcher/decision_engine/strategy/strategies/zone_migration.py

@@ -309,10 +309,10 @@ class ZoneMigration(base.ZoneMigrationBaseStrategy):
                     else:
                         self.instances_migration(targets, action_counter)
 
-        LOG.debug("action total: %s, pools: %s, nodes %s ", (
+        LOG.debug("action total: %s, pools: %s, nodes %s ",
                   action_counter.total_count,
                   action_counter.per_pool_count,
-                  action_counter.per_node_count))
+                  action_counter.per_node_count)
 
     def post_execute(self):
         """Post-execution phase
@@ -416,7 +416,7 @@ class ZoneMigration(base.ZoneMigrationBaseStrategy):
             src_type = volume.volume_type
             dst_pool, dst_type = self.get_dst_pool_and_type(pool, src_type)
             LOG.debug(src_type)
-            LOG.debug("%s %s", (dst_pool, dst_type))
+            LOG.debug("%s %s", dst_pool, dst_type)
 
             if self.is_available(volume):
                 if src_type == dst_type:
@@ -640,8 +640,8 @@ class ActionCounter(object):
         if not self.is_total_max() and not self.is_pool_max(pool):
             self.per_pool_count[pool] += 1
             self.total_count += 1
-            LOG.debug("total: %s, per_pool: %s", (
-                      self.total_count, self.per_pool_count))
+            LOG.debug("total: %s, per_pool: %s",
+                      self.total_count, self.per_pool_count)
             return True
         return False
 
@@ -657,8 +657,8 @@ class ActionCounter(object):
         if not self.is_total_max() and not self.is_node_max(node):
             self.per_node_count[node] += 1
             self.total_count += 1
-            LOG.debug("total: %s, per_node: %s", (
-                      self.total_count, self.per_node_count))
+            LOG.debug("total: %s, per_node: %s",
+                      self.total_count, self.per_node_count)
             return True
         return False
 
@@ -677,7 +677,7 @@ class ActionCounter(object):
         if pool not in self.per_pool_count:
             self.per_pool_count[pool] = 0
         LOG.debug("the number of parallel per pool %s is %s ",
-                  (pool, self.per_pool_count[pool]))
+                  pool, self.per_pool_count[pool])
         LOG.debug("per pool limit is %s", self.per_pool_limit)
         return self.per_pool_count[pool] >= self.per_pool_limit
 
@@ -721,7 +721,7 @@ class BaseFilter(object):
             for k, v in six.iteritems(targets):
                 if not self.is_allowed(k):
                     continue
-                LOG.debug("filter:%s with the key: %s", (cond, k))
+                LOG.debug("filter:%s with the key: %s", cond, k)
                 targets[k] = self.exec_filter(v, cond)
 
         LOG.debug(targets)
@@ -775,7 +775,7 @@ class ProjectSortFilter(SortMovingToFrontFilter):
         """
 
         project_id = self.get_project_id(item)
-        LOG.debug("project_id: %s, sort_key: %s", (project_id, sort_key))
+        LOG.debug("project_id: %s, sort_key: %s", project_id, sort_key)
         return project_id == sort_key
 
     def get_project_id(self, item):
@@ -809,7 +809,7 @@ class ComputeHostSortFilter(SortMovingToFrontFilter):
         """
 
         host = self.get_host(item)
-        LOG.debug("host: %s, sort_key: %s", (host, sort_key))
+        LOG.debug("host: %s, sort_key: %s", host, sort_key)
         return host == sort_key
 
     def get_host(self, item):
@@ -837,7 +837,7 @@ class StorageHostSortFilter(SortMovingToFrontFilter):
         """
 
         host = self.get_host(item)
-        LOG.debug("host: %s, sort_key: %s", (host, sort_key))
+        LOG.debug("host: %s, sort_key: %s", host, sort_key)
         return host == sort_key
 
     def get_host(self, item):
@@ -909,9 +909,9 @@ class ComputeSpecSortFilter(BaseFilter):
         :returns: memory size of item
         """
 
-        LOG.debug("item: %s, flavors: %s", (item, flavors))
+        LOG.debug("item: %s, flavors: %s", item, flavors)
         for flavor in flavors:
-            LOG.debug("item.flavor: %s, flavor: %s", (item.flavor, flavor))
+            LOG.debug("item.flavor: %s, flavor: %s", item.flavor, flavor)
             if item.flavor.get('id') == flavor.id:
                 LOG.debug("flavor.ram: %s", flavor.ram)
                 return flavor.ram
@@ -924,9 +924,9 @@ class ComputeSpecSortFilter(BaseFilter):
         :returns: vcpu number of item
         """
 
-        LOG.debug("item: %s, flavors: %s", (item, flavors))
+        LOG.debug("item: %s, flavors: %s", item, flavors)
         for flavor in flavors:
-            LOG.debug("item.flavor: %s, flavor: %s", (item.flavor, flavor))
+            LOG.debug("item.flavor: %s, flavor: %s", item.flavor, flavor)
             if item.flavor.get('id') == flavor.id:
                 LOG.debug("flavor.vcpus: %s", flavor.vcpus)
                 return flavor.vcpus
@@ -939,9 +939,9 @@ class ComputeSpecSortFilter(BaseFilter):
         :returns: disk size of item
         """
 
-        LOG.debug("item: %s, flavors: %s", (item, flavors))
+        LOG.debug("item: %s, flavors: %s", item, flavors)
         for flavor in flavors:
-            LOG.debug("item.flavor: %s, flavor: %s", (item.flavor, flavor))
+            LOG.debug("item.flavor: %s, flavor: %s", item.flavor, flavor)
             if item.flavor.get('id') == flavor.id:
                 LOG.debug("flavor.disk: %s", flavor.disk)
                 return flavor.disk

watcher/tests/common/test_nova_helper.py

@@ -321,19 +321,6 @@ class TestNovaHelper(base.TestCase):
         instance = nova_util.create_instance(self.source_node)
         self.assertIsNotNone(instance)
 
-    def test_get_flavor_instance(self, mock_glance, mock_cinder,
-                                 mock_neutron, mock_nova):
-        nova_util = nova_helper.NovaHelper()
-        instance = self.fake_server(self.instance_uuid)
-        flavor = {'id': 1, 'name': 'm1.tiny', 'ram': 512, 'vcpus': 1,
-                  'disk': 0, 'ephemeral': 0}
-        instance.flavor = flavor
-        nova_util.nova.flavors.get.return_value = flavor
-        cache = flavor
-
-        nova_util.get_flavor_instance(instance, cache)
-        self.assertEqual(instance.flavor['name'], cache['name'])
-
     @staticmethod
     def fake_volume(**kwargs):
         volume = mock.MagicMock()

watcher/tests/decision_engine/audit/test_audit_handlers.py

@@ -385,11 +385,10 @@ class TestContinuousAuditHandler(base.DbTestCase):
         audit_handler = continuous.ContinuousAuditHandler()
         self.audits[0].next_run_time = (datetime.datetime.now() -
                                         datetime.timedelta(seconds=1800))
-        m_is_inactive.return_value = False
-        m_get_jobs.return_value = None
+        m_is_inactive.return_value = True
+        m_get_jobs.return_value = []
 
         audit_handler.execute_audit(self.audits[0], self.context)
-        m_execute.assert_called_once_with(self.audits[0], self.context)
         self.assertIsNotNone(self.audits[0].next_run_time)
 
     @mock.patch.object(objects.service.Service, 'list')

watcher/tests/decision_engine/test_sync.py

@@ -451,11 +451,10 @@ class TestSyncer(base.DbTestCase):
              self._find_created_modified_unmodified_ids(
                  before_action_plans, after_action_plans))
 
-        dummy_1_spec = [
-            {'description': 'Dummy indicator', 'name': 'dummy',
-             'schema': jsonutils.dumps({'minimum': 0, 'type': 'integer'}),
-             'unit': '%'}]
-        dummy_2_spec = []
+        dummy_1_spec = jsonutils.loads(
+            self.goal1_spec.serialize_indicators_specs())
+        dummy_2_spec = jsonutils.loads(
+            self.goal2_spec.serialize_indicators_specs())
         self.assertEqual(
             [dummy_1_spec, dummy_2_spec],
             [g.efficacy_specification for g in after_goals])