Added purge script for soft deleted objects

This patchset implements the purge script as specified in its
related blueprint:

- The '--age-in-days' option allows to specify the number of
  days before expiry
- The '--max-number' option allows us to specify a limit on the number
  of objects to delete
- The '--audit-template' option allows you to only delete objects
  related to the specified audit template UUID or name
- The '--dry-run' option to go through the purge procedure without
  actually deleting anything
- The '--exclude-orphans' option which allows you to exclude from the
  purge any object that does not have a parent (i.e. and audit without
  a related audit template)

A prompt has been added to also propose to narrow down the number of
deletions to be below the specified limit.

Change-Id: I3ce83ab95277c109df67a6b5b920a878f6e59d3f
Implements: blueprint db-purge-engine

requirements.txt

@@ -17,6 +17,7 @@ oslo.utils>=3.5.0 # Apache-2.0
 PasteDeploy>=1.5.0 # MIT
 pbr>=1.6 # Apache-2.0
 pecan>=1.0.0 # BSD
+PrettyTable>=0.7,<0.8 # BSD
 voluptuous>=0.8.6 # BSD License
 python-ceilometerclient>=2.2.1 # Apache-2.0
 python-cinderclient>=1.3.1 # Apache-2.0

watcher/cmd/dbmanage.py

@@ -25,7 +25,7 @@ from oslo_config import cfg
 
 from watcher.common import service
 from watcher.db import migration
-
+from watcher.db import purge
 
 CONF = cfg.CONF
 
@@ -56,6 +56,12 @@ class DBCommand(object):
     def create_schema():
         migration.create_schema()
 
+    @staticmethod
+    def purge():
+        purge.purge(CONF.command.age_in_days, CONF.command.max_number,
+                    CONF.command.audit_template, CONF.command.exclude_orphans,
+                    CONF.command.dry_run)
+
 
 def add_command_parsers(subparsers):
     parser = subparsers.add_parser(
@@ -96,6 +102,33 @@ def add_command_parsers(subparsers):
         help="Create the database schema.")
     parser.set_defaults(func=DBCommand.create_schema)
 
+    parser = subparsers.add_parser(
+        'purge',
+        help="Purge the database.")
+    parser.add_argument('-d', '--age-in-days',
+                        help="Number of days since deletion (from today) "
+                             "to exclude from the purge. If None, everything "
+                             "will be purged.",
+                        type=int, default=None, nargs='?')
+    parser.add_argument('-n', '--max-number',
+                        help="Max number of objects expected to be deleted. "
+                             "Prevents the deletion if exceeded. No limit if "
+                             "set to None.",
+                        type=int, default=None, nargs='?')
+    parser.add_argument('-t', '--audit-template',
+                        help="UUID or name of the audit template to purge.",
+                        type=str, default=None, nargs='?')
+    parser.add_argument('-e', '--exclude-orphans', action='store_true',
+                        help="Flag to indicate whether or not you want to "
+                             "exclude orphans from deletion (default: False).",
+                        default=False)
+    parser.add_argument('--dry-run', action='store_true',
+                        help="Flag to indicate whether or not you want to "
+                             "perform a dry run (no deletion).",
+                        default=False)
+
+    parser.set_defaults(func=DBCommand.purge)
+
 
 command_opt = cfg.SubCommandOpt('command',
                                 title='Command',
@@ -114,6 +147,7 @@ def main():
     valid_commands = set([
         'upgrade', 'downgrade', 'revision',
         'version', 'stamp', 'create_schema',
+        'purge',
     ])
     if not set(sys.argv).intersection(valid_commands):
         sys.argv.append('upgrade')

watcher/common/config.py

@@ -22,6 +22,8 @@ from watcher import version
 
 
 def parse_args(argv, default_config_files=None):
+    default_config_files = (default_config_files or
+                            cfg.find_config_files(project='watcher'))
     rpc.set_defaults(control_exchange='watcher')
     cfg.CONF(argv[1:],
              project='python-watcher',

watcher/common/exception.py

@@ -283,3 +283,11 @@ class LoadingError(WatcherException):
 
 class ReservedWord(WatcherException):
     msg_fmt = _("The identifier '%(name)s' is a reserved word")
+
+
+class NotSoftDeletedStateError(WatcherException):
+    msg_fmt = _("The %(name)s resource %(id)s is not soft deleted")
+
+
+class NegativeLimitError(WatcherException):
+    msg_fmt = _("Limit should be positive")

watcher/db/purge.py

@@ -0,0 +1,410 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2016 b<>com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import print_function
+
+import collections
+import datetime
+import itertools
+import sys
+
+from oslo_log import log
+from oslo_utils import strutils
+import prettytable as ptable
+from six.moves import input
+
+from watcher._i18n import _, _LI
+from watcher.common import context
+from watcher.common import exception
+from watcher.common import utils
+from watcher import objects
+
+LOG = log.getLogger(__name__)
+
+
+class WatcherObjectsMap(object):
+    """Wrapper to deal with watcher objects per type
+
+    This wrapper object contains a list of watcher objects per type.
+    Its main use is to simplify the merge of watcher objects by avoiding
+    duplicates, but also for representing the relationships between these
+    objects.
+    """
+
+    # This is for generating the .pot translations
+    keymap = collections.OrderedDict([
+        ("audit_templates", _("Audit Templates")),
+        ("audits", _("Audits")),
+        ("action_plans", _("Action Plans")),
+        ("actions", _("Actions")),
+    ])
+
+    def __init__(self):
+        for attr_name in self.__class__.keys():
+            setattr(self, attr_name, [])
+
+    def values(self):
+        return (getattr(self, key) for key in self.__class__.keys())
+
+    @classmethod
+    def keys(cls):
+        return cls.keymap.keys()
+
+    def __iter__(self):
+        return itertools.chain(*self.values())
+
+    def __add__(self, other):
+        new_map = self.__class__()
+
+        # Merge the 2 items dicts into a new object (and avoid dupes)
+        for attr_name, initials, others in zip(self.keys(), self.values(),
+                                               other.values()):
+            # Creates a copy
+            merged = initials[:]
+            initials_ids = [item.id for item in initials]
+            non_dupes = [item for item in others
+                         if item.id not in initials_ids]
+            merged += non_dupes
+
+            setattr(new_map, attr_name, merged)
+
+        return new_map
+
+    def __str__(self):
+        out = ""
+        for key, vals in zip(self.keys(), self.values()):
+            ids = [val.id for val in vals]
+            out += "%(key)s: %(val)s" % (dict(key=key, val=ids))
+            out += "\n"
+        return out
+
+    def __len__(self):
+        return sum(len(getattr(self, key)) for key in self.keys())
+
+    def get_count_table(self):
+        headers = list(self.keymap.values())
+        headers.append(_("Total"))  # We also add a total count
+        counters = [len(cat_vals) for cat_vals in self.values()] + [len(self)]
+        table = ptable.PrettyTable(field_names=headers)
+        table.add_row(counters)
+        return table.get_string()
+
+
+class PurgeCommand(object):
+    """Purges the DB by removing soft deleted entries
+
+    The workflow for this purge is the following:
+
+    # Find soft deleted objects which are expired
+    # Find orphan objects
+    # Find their related objects whether they are expired or not
+    # Merge them together
+    # If it does not exceed the limit, destroy them all
+    """
+
+    ctx = context.make_context(show_deleted=True)
+
+    def __init__(self, age_in_days=None, max_number=None,
+                 uuid=None, exclude_orphans=False, dry_run=None):
+        self.age_in_days = age_in_days
+        self.max_number = max_number
+        self.uuid = uuid
+        self.exclude_orphans = exclude_orphans
+        self.dry_run = dry_run
+
+        self._delete_up_to_max = None
+        self._objects_map = WatcherObjectsMap()
+
+    def get_expiry_date(self):
+        if not self.age_in_days:
+            return None
+        today = datetime.datetime.today()
+        expiry_date = today - datetime.timedelta(days=self.age_in_days)
+        return expiry_date
+
+    @classmethod
+    def get_audit_template_uuid(cls, uuid_or_name):
+        if uuid_or_name is None:
+            return
+
+        query_func = None
+        if not utils.is_uuid_like(uuid_or_name):
+            query_func = objects.audit_template.AuditTemplate.get_by_name
+        else:
+            query_func = objects.audit_template.AuditTemplate.get_by_uuid
+
+        try:
+            audit_template = query_func(cls.ctx, uuid_or_name)
+        except Exception as exc:
+            LOG.exception(exc)
+            raise exception.AuditTemplateNotFound(audit_template=uuid_or_name)
+
+        if not audit_template.deleted_at:
+            raise exception.NotSoftDeletedStateError(
+                name=_('Audit Template'), id=uuid_or_name)
+
+        return audit_template.uuid
+
+    def _find_audit_templates(self, filters=None):
+        return objects.audit_template.AuditTemplate.list(
+            self.ctx, filters=filters)
+
+    def _find_audits(self, filters=None):
+        return objects.audit.Audit.list(self.ctx, filters=filters)
+
+    def _find_action_plans(self, filters=None):
+        return objects.action_plan.ActionPlan.list(self.ctx, filters=filters)
+
+    def _find_actions(self, filters=None):
+        return objects.action.Action.list(self.ctx, filters=filters)
+
+    def _find_orphans(self):
+        orphans = WatcherObjectsMap()
+
+        filters = dict(deleted=False)
+        audit_templates = objects.audit_template.AuditTemplate.list(
+            self.ctx, filters=filters)
+        audits = objects.audit.Audit.list(self.ctx, filters=filters)
+        action_plans = objects.action_plan.ActionPlan.list(
+            self.ctx, filters=filters)
+        actions = objects.action.Action.list(self.ctx, filters=filters)
+
+        audit_template_ids = set(at.id for at in audit_templates)
+        orphans.audits = [
+            audit for audit in audits
+            if audit.audit_template_id not in audit_template_ids]
+
+        # Objects with orphan parents are themselves orphans
+        audit_ids = [audit.id for audit in (a for a in audits
+                                            if a not in orphans.audits)]
+        orphans.action_plans = [
+            ap for ap in action_plans
+            if ap.audit_id not in audit_ids]
+
+        # Objects with orphan parents are themselves orphans
+        action_plan_ids = [ap.id for ap in (a for a in action_plans
+                                            if a not in orphans.action_plans)]
+        orphans.actions = [
+            action for action in actions
+            if action.action_plan_id not in action_plan_ids]
+
+        LOG.debug("Orphans found:\n%s", orphans)
+        LOG.info(_LI("Orphans found:\n%s"), orphans.get_count_table())
+
+        return orphans
+
+    def _find_soft_deleted_objects(self):
+        to_be_deleted = WatcherObjectsMap()
+
+        expiry_date = self.get_expiry_date()
+        filters = dict(deleted=True)
+        if self.uuid:
+            filters["uuid"] = self.uuid
+        if expiry_date:
+            filters.update(dict(deleted_at__lt=expiry_date))
+
+        to_be_deleted.audit_templates.extend(
+            self._find_audit_templates(filters))
+        to_be_deleted.audits.extend(self._find_audits(filters))
+        to_be_deleted.action_plans.extend(self._find_action_plans(filters))
+        to_be_deleted.actions.extend(self._find_actions(filters))
+
+        soft_deleted_objs = self._find_related_objects(
+            to_be_deleted, base_filters=dict(deleted=True))
+
+        LOG.debug("Soft deleted objects:\n%s", soft_deleted_objs)
+
+        return soft_deleted_objs
+
+    def _find_related_objects(self, objects_map, base_filters=None):
+        base_filters = base_filters or {}
+
+        for audit_template in objects_map.audit_templates:
+            filters = {}
+            filters.update(base_filters)
+            filters.update(dict(audit_template_id=audit_template.id))
+            related_objs = WatcherObjectsMap()
+            related_objs.audits = self._find_audits(filters)
+            objects_map += related_objs
+
+        for audit in objects_map.audits:
+            filters = {}
+            filters.update(base_filters)
+            filters.update(dict(audit_id=audit.id))
+            related_objs = WatcherObjectsMap()
+            related_objs.action_plans = self._find_action_plans(filters)
+            objects_map += related_objs
+
+        for action_plan in objects_map.action_plans:
+            filters = {}
+            filters.update(base_filters)
+            filters.update(dict(action_plan_id=action_plan.id))
+            related_objs = WatcherObjectsMap()
+            related_objs.actions = self._find_actions(filters)
+            objects_map += related_objs
+
+        return objects_map
+
+    def confirmation_prompt(self):
+        print(self._objects_map.get_count_table())
+        raw_val = input(
+            _("There are %(count)d objects set for deletion. "
+              "Continue? [y/N]") % dict(count=len(self._objects_map)))
+
+        return strutils.bool_from_string(raw_val)
+
+    def delete_up_to_max_prompt(self, objects_map):
+        print(objects_map.get_count_table())
+        print(_("The number of objects (%(num)s) to delete from the database "
+                "exceeds the maximum number of objects (%(max_number)s) "
+                "specified.") % dict(max_number=self.max_number,
+                                     num=len(objects_map)))
+        raw_val = input(
+            _("Do you want to delete objects up to the specified maximum "
+              "number? [y/N]"))
+
+        self._delete_up_to_max = strutils.bool_from_string(raw_val)
+
+        return self._delete_up_to_max
+
+    def _aggregate_objects(self):
+        """Objects aggregated on a 'per audit template' basis"""
+        # todo: aggregate orphans as well
+        aggregate = []
+        for audit_template in self._objects_map.audit_templates:
+            related_objs = WatcherObjectsMap()
+            related_objs.audit_templates = [audit_template]
+            related_objs.audits = [
+                audit for audit in self._objects_map.audits
+                if audit.audit_template_id == audit_template.id
+            ]
+            audit_ids = [audit.id for audit in related_objs.audits]
+            related_objs.action_plans = [
+                action_plan for action_plan in self._objects_map.action_plans
+                if action_plan.audit_id in audit_ids
+            ]
+            action_plan_ids = [
+                action_plan.id for action_plan in related_objs.action_plans
+            ]
+            related_objs.actions = [
+                action for action in self._objects_map.actions
+                if action.action_plan_id in action_plan_ids
+            ]
+            aggregate.append(related_objs)
+
+        return aggregate
+
+    def _get_objects_up_to_limit(self):
+        aggregated_objects = self._aggregate_objects()
+        to_be_deleted_subset = WatcherObjectsMap()
+
+        for aggregate in aggregated_objects:
+            if len(aggregate) + len(to_be_deleted_subset) <= self.max_number:
+                to_be_deleted_subset += aggregate
+            else:
+                break
+
+        LOG.debug(to_be_deleted_subset)
+        return to_be_deleted_subset
+
+    def find_objects_to_delete(self):
+        """Finds all the objects to be purged
+
+        :returns: A mapping with all the Watcher objects to purged
+        :rtype: :py:class:`~.WatcherObjectsMap` instance
+        """
+        to_be_deleted = self._find_soft_deleted_objects()
+
+        if not self.exclude_orphans:
+            to_be_deleted += self._find_orphans()
+
+        LOG.debug("Objects to be deleted:\n%s", to_be_deleted)
+
+        return to_be_deleted
+
+    def do_delete(self):
+        LOG.info(_LI("Deleting..."))
+        # Reversed to avoid errors with foreign keys
+        for entry in reversed(list(self._objects_map)):
+            entry.destroy()
+
+    def execute(self):
+        LOG.info(_LI("Starting purge command"))
+        self._objects_map = self.find_objects_to_delete()
+
+        if (self.max_number is not None and
+                len(self._objects_map) > self.max_number):
+            if self.delete_up_to_max_prompt(self._objects_map):
+                self._objects_map = self._get_objects_up_to_limit()
+            else:
+                return
+
+        _orphans_note = (_(" (orphans excluded)") if self.exclude_orphans
+                         else _(" (may include orphans)"))
+        if not self.dry_run and self.confirmation_prompt():
+            self.do_delete()
+            print(_("Purge results summary%s:") % _orphans_note)
+            LOG.info(_LI("Purge results summary%s:"), _orphans_note)
+        else:
+            LOG.debug(self._objects_map)
+            print(_("Here below is a table containing the objects "
+                    "that can be purged%s:") % _orphans_note)
+
+        LOG.info("\n%s", self._objects_map.get_count_table())
+        print(self._objects_map.get_count_table())
+        LOG.info(_LI("Purge process completed"))
+
+
+def purge(age_in_days, max_number, audit_template, exclude_orphans, dry_run):
+    """Removes soft deleted objects from the database
+
+    :param age_in_days: Number of days since deletion (from today)
+        to exclude from the purge. If None, everything will be purged.
+    :type age_in_days: int
+    :param max_number: Max number of objects expected to be deleted.
+                  Prevents the deletion if exceeded. No limit if set to None.
+    :type max_number: int
+    :param audit_template: UUID or name of the audit template to purge.
+    :type audit_template: str
+    :param exclude_orphans: Flag to indicate whether or not you want to
+                            exclude orphans from deletion (default: False).
+    :type exclude_orphans: bool
+    :param dry_run: Flag to indicate whether or not you want to perform
+                    a dry run (no deletion).
+    :type dry_run: bool
+    """
+    try:
+        if max_number and max_number < 0:
+            raise exception.NegativeLimitError
+
+        LOG.info("[options] age_in_days = %s", age_in_days)
+        LOG.info("[options] max_number = %s", max_number)
+        LOG.info("[options] audit_template = %s", audit_template)
+        LOG.info("[options] exclude_orphans = %s", exclude_orphans)
+        LOG.info("[options] dry_run = %s", dry_run)
+
+        uuid = PurgeCommand.get_audit_template_uuid(audit_template)
+
+        cmd = PurgeCommand(age_in_days, max_number, uuid,
+                           exclude_orphans, dry_run)
+
+        cmd.execute()
+
+    except Exception as exc:
+        LOG.exception(exc)
+        print(exc)
+        sys.exit(1)

watcher/locale/watcher.pot

@@ -7,9 +7,9 @@
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: python-watcher 0.24.1.dev1\n"
+"Project-Id-Version: python-watcher 0.24.1.dev4\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2016-03-09 11:18+0100\n"
+"POT-Creation-Date: 2016-03-14 15:29+0100\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -344,6 +344,15 @@ msgstr ""
 msgid "The identifier '%(name)s' is a reserved word"
 msgstr ""
 
+#: watcher/common/exception.py:289
+#, python-format
+msgid "The %(name)s resource %(id)s is not soft deleted"
+msgstr ""
+
+#: watcher/common/exception.py:293
+msgid "Limit should be positive"
+msgstr ""
+
 #: watcher/common/service.py:83
 #, python-format
 msgid "Created RPC server for service %(service)s on host %(host)s."
@@ -386,25 +395,102 @@ msgstr ""
 msgid "Messaging configuration error"
 msgstr ""
 
-#: watcher/db/sqlalchemy/api.py:256
+#: watcher/db/purge.py:50
+msgid "Audit Templates"
+msgstr ""
+
+#: watcher/db/purge.py:51
+msgid "Audits"
+msgstr ""
+
+#: watcher/db/purge.py:52
+msgid "Action Plans"
+msgstr ""
+
+#: watcher/db/purge.py:53
+msgid "Actions"
+msgstr ""
+
+#: watcher/db/purge.py:100
+msgid "Total"
+msgstr ""
+
+#: watcher/db/purge.py:158
+msgid "Audit Template"
+msgstr ""
+
+#: watcher/db/purge.py:206
+#, python-format
+msgid ""
+"Orphans found:\n"
+"%s"
+msgstr ""
+
+#: watcher/db/purge.py:265
+#, python-format
+msgid "There are %(count)d objects set for deletion. Continue? [y/N]"
+msgstr ""
+
+#: watcher/db/purge.py:272
+#, python-format
+msgid ""
+"The number of objects (%(num)s) to delete from the database exceeds the "
+"maximum number of objects (%(max_number)s) specified."
+msgstr ""
+
+#: watcher/db/purge.py:277
+msgid "Do you want to delete objects up to the specified maximum number? [y/N]"
+msgstr ""
+
+#: watcher/db/purge.py:340
+msgid "Deleting..."
+msgstr ""
+
+#: watcher/db/purge.py:346
+msgid "Starting purge command"
+msgstr ""
+
+#: watcher/db/purge.py:356
+msgid " (orphans excluded)"
+msgstr ""
+
+#: watcher/db/purge.py:357
+msgid " (may include orphans)"
+msgstr ""
+
+#: watcher/db/purge.py:360 watcher/db/purge.py:361
+#, python-format
+msgid "Purge results summary%s:"
+msgstr ""
+
+#: watcher/db/purge.py:364
+#, python-format
+msgid "Here below is a table containing the objects that can be purged%s:"
+msgstr ""
+
+#: watcher/db/purge.py:369
+msgid "Purge process completed"
+msgstr ""
+
+#: watcher/db/sqlalchemy/api.py:362
 msgid ""
 "Multiple audit templates exist with the same name. Please use the audit "
 "template uuid instead"
 msgstr ""
 
-#: watcher/db/sqlalchemy/api.py:278
+#: watcher/db/sqlalchemy/api.py:384
 msgid "Cannot overwrite UUID for an existing Audit Template."
 msgstr ""
 
-#: watcher/db/sqlalchemy/api.py:388
+#: watcher/db/sqlalchemy/api.py:495
 msgid "Cannot overwrite UUID for an existing Audit."
 msgstr ""
 
-#: watcher/db/sqlalchemy/api.py:480
+#: watcher/db/sqlalchemy/api.py:588
 msgid "Cannot overwrite UUID for an existing Action."
 msgstr ""
 
-#: watcher/db/sqlalchemy/api.py:590
+#: watcher/db/sqlalchemy/api.py:699
 msgid "Cannot overwrite UUID for an existing Action Plan."
 msgstr ""
 

watcher/tests/cmd/test_db_manage.py

@@ -1,5 +1,5 @@
 # -*- encoding: utf-8 -*-
-# Copyright (c) 2015 b<>com
+# Copyright (c) 2016 b<>com
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,15 +14,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from mock import Mock
+import sys
-from mock import patch
+
+import mock
 from oslo_config import cfg
+
 from watcher.cmd import dbmanage
 from watcher.db import migration
-from watcher.tests.base import TestCase
+from watcher.db import purge
+from watcher.tests import base
 
 
-class TestDBManageRunApp(TestCase):
+class TestDBManageRunApp(base.TestCase):
 
     scenarios = (
         ("upgrade", {"command": "upgrade", "expected": "upgrade"}),
@@ -32,15 +35,16 @@ class TestDBManageRunApp(TestCase):
         ("version", {"command": "version", "expected": "version"}),
         ("create_schema", {"command": "create_schema",
                            "expected": "create_schema"}),
+        ("purge", {"command": "purge", "expected": "purge"}),
         ("no_param", {"command": None, "expected": "upgrade"}),
     )
 
-    @patch.object(dbmanage, "register_sub_command_opts", Mock())
+    @mock.patch.object(dbmanage, "register_sub_command_opts", mock.Mock())
-    @patch("watcher.cmd.dbmanage.service.prepare_service")
+    @mock.patch("watcher.cmd.dbmanage.service.prepare_service")
-    @patch("watcher.cmd.dbmanage.sys")
+    @mock.patch("watcher.cmd.dbmanage.sys")
     def test_run_db_manage_app(self, m_sys, m_prepare_service):
         # Patch command function
-        m_func = Mock()
+        m_func = mock.Mock()
         cfg.CONF.register_opt(cfg.SubCommandOpt("command"))
         cfg.CONF.command.func = m_func
 
@@ -53,9 +57,9 @@ class TestDBManageRunApp(TestCase):
             ["watcher-db-manage", self.expected])
 
 
-class TestDBManageRunCommand(TestCase):
+class TestDBManageRunCommand(base.TestCase):
 
-    @patch.object(migration, "upgrade")
+    @mock.patch.object(migration, "upgrade")
     def test_run_db_upgrade(self, m_upgrade):
         cfg.CONF.register_opt(cfg.StrOpt("revision"), group="command")
         cfg.CONF.set_default("revision", "dummy", group="command")
@@ -63,7 +67,7 @@ class TestDBManageRunCommand(TestCase):
 
         m_upgrade.assert_called_once_with("dummy")
 
-    @patch.object(migration, "downgrade")
+    @mock.patch.object(migration, "downgrade")
     def test_run_db_downgrade(self, m_downgrade):
         cfg.CONF.register_opt(cfg.StrOpt("revision"), group="command")
         cfg.CONF.set_default("revision", "dummy", group="command")
@@ -71,7 +75,7 @@ class TestDBManageRunCommand(TestCase):
 
         m_downgrade.assert_called_once_with("dummy")
 
-    @patch.object(migration, "revision")
+    @mock.patch.object(migration, "revision")
     def test_run_db_revision(self, m_revision):
         cfg.CONF.register_opt(cfg.StrOpt("message"), group="command")
         cfg.CONF.register_opt(cfg.StrOpt("autogenerate"), group="command")
@@ -87,14 +91,85 @@ class TestDBManageRunCommand(TestCase):
             "dummy_message", "dummy_autogenerate"
         )
 
-    @patch.object(migration, "stamp")
+    @mock.patch.object(migration, "stamp")
     def test_run_db_stamp(self, m_stamp):
         cfg.CONF.register_opt(cfg.StrOpt("revision"), group="command")
         cfg.CONF.set_default("revision", "dummy", group="command")
         dbmanage.DBCommand.stamp()
 
-    @patch.object(migration, "version")
+    @mock.patch.object(migration, "version")
     def test_run_db_version(self, m_version):
         dbmanage.DBCommand.version()
 
         self.assertEqual(1, m_version.call_count)
+
+    @mock.patch.object(purge, "PurgeCommand")
+    def test_run_db_purge(self, m_purge_cls):
+        m_purge = mock.Mock()
+        m_purge_cls.return_value = m_purge
+        m_purge_cls.get_audit_template_uuid.return_value = 'Some UUID'
+        cfg.CONF.register_opt(cfg.IntOpt("age_in_days"), group="command")
+        cfg.CONF.register_opt(cfg.IntOpt("max_number"), group="command")
+        cfg.CONF.register_opt(cfg.StrOpt("audit_template"), group="command")
+        cfg.CONF.register_opt(cfg.BoolOpt("exclude_orphans"), group="command")
+        cfg.CONF.register_opt(cfg.BoolOpt("dry_run"), group="command")
+        cfg.CONF.set_default("age_in_days", None, group="command")
+        cfg.CONF.set_default("max_number", None, group="command")
+        cfg.CONF.set_default("audit_template", None, group="command")
+        cfg.CONF.set_default("exclude_orphans", True, group="command")
+        cfg.CONF.set_default("dry_run", False, group="command")
+
+        dbmanage.DBCommand.purge()
+
+        m_purge_cls.assert_called_once_with(
+            None, None, 'Some UUID', True, False)
+        m_purge.execute.assert_called_once_with()
+
+    @mock.patch.object(sys, "exit")
+    @mock.patch.object(purge, "PurgeCommand")
+    def test_run_db_purge_negative_max_number(self, m_purge_cls, m_exit):
+        m_purge = mock.Mock()
+        m_purge_cls.return_value = m_purge
+        m_purge_cls.get_audit_template_uuid.return_value = 'Some UUID'
+        cfg.CONF.register_opt(cfg.IntOpt("age_in_days"), group="command")
+        cfg.CONF.register_opt(cfg.IntOpt("max_number"), group="command")
+        cfg.CONF.register_opt(cfg.StrOpt("audit_template"), group="command")
+        cfg.CONF.register_opt(cfg.BoolOpt("exclude_orphans"), group="command")
+        cfg.CONF.register_opt(cfg.BoolOpt("dry_run"), group="command")
+        cfg.CONF.set_default("age_in_days", None, group="command")
+        cfg.CONF.set_default("max_number", -1, group="command")
+        cfg.CONF.set_default("audit_template", None, group="command")
+        cfg.CONF.set_default("exclude_orphans", True, group="command")
+        cfg.CONF.set_default("dry_run", False, group="command")
+
+        dbmanage.DBCommand.purge()
+
+        self.assertEqual(0, m_purge_cls.call_count)
+        self.assertEqual(0, m_purge.execute.call_count)
+        self.assertEqual(0, m_purge.do_delete.call_count)
+        self.assertEqual(1, m_exit.call_count)
+
+    @mock.patch.object(sys, "exit")
+    @mock.patch.object(purge, "PurgeCommand")
+    def test_run_db_purge_dry_run(self, m_purge_cls, m_exit):
+        m_purge = mock.Mock()
+        m_purge_cls.return_value = m_purge
+        m_purge_cls.get_audit_template_uuid.return_value = 'Some UUID'
+        cfg.CONF.register_opt(cfg.IntOpt("age_in_days"), group="command")
+        cfg.CONF.register_opt(cfg.IntOpt("max_number"), group="command")
+        cfg.CONF.register_opt(cfg.StrOpt("audit_template"), group="command")
+        cfg.CONF.register_opt(cfg.BoolOpt("exclude_orphans"), group="command")
+        cfg.CONF.register_opt(cfg.BoolOpt("dry_run"), group="command")
+        cfg.CONF.set_default("age_in_days", None, group="command")
+        cfg.CONF.set_default("max_number", None, group="command")
+        cfg.CONF.set_default("audit_template", None, group="command")
+        cfg.CONF.set_default("exclude_orphans", True, group="command")
+        cfg.CONF.set_default("dry_run", True, group="command")
+
+        dbmanage.DBCommand.purge()
+
+        m_purge_cls.assert_called_once_with(
+            None, None, 'Some UUID', True, True)
+        self.assertEqual(1, m_purge.execute.call_count)
+        self.assertEqual(0, m_purge.do_delete.call_count)
+        self.assertEqual(0, m_exit.call_count)

watcher/tests/db/test_purge.py

@@ -0,0 +1,372 @@
+# -*- encoding: utf-8 -*-
+# Copyright (c) 2016 b<>com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import uuid
+
+import freezegun
+import mock
+
+from watcher.common import context as watcher_context
+from watcher.db import purge
+from watcher.db.sqlalchemy import api as dbapi
+from watcher.tests.db import base
+from watcher.tests.objects import utils as obj_utils
+
+
+class TestPurgeCommand(base.DbTestCase):
+
+    def setUp(self):
+        super(TestPurgeCommand, self).setUp()
+        self.cmd = purge.PurgeCommand()
+        token_info = {
+            'token': {
+                'project': {
+                    'id': 'fake_project'
+                },
+                'user': {
+                    'id': 'fake_user'
+                }
+            }
+        }
+        self.context = watcher_context.RequestContext(
+            auth_token_info=token_info,
+            project_id='fake_project',
+            user_id='fake_user',
+            show_deleted=True,
+        )
+
+        self.fake_today = '2016-02-24T09:52:05.219414+00:00'
+        self.expired_date = '2016-01-24T09:52:05.219414+00:00'
+
+        self.m_input = mock.Mock()
+        p = mock.patch("watcher.db.purge.input", self.m_input)
+        self.m_input.return_value = 'y'
+        p.start()
+        self.addCleanup(p.stop)
+
+        self._id_generator = None
+        self._data_setup()
+
+    def _generate_id(self):
+        if self._id_generator is None:
+            self._id_generator = self._get_id_generator()
+        return next(self._id_generator)
+
+    def _get_id_generator(self):
+        seed = 1
+        while True:
+            yield seed
+            seed += 1
+
+    def _data_setup(self):
+        # All the 1's are soft_deleted and are expired
+        # All the 2's are soft_deleted but are not expired
+        # All the 3's are *not* soft_deleted
+
+        # Number of days we want to keep in DB (no purge for them)
+        self.cmd.age_in_days = 10
+        self.cmd.max_number = None
+        self.cmd.orphans = True
+        gen_name = lambda: "Audit Template %s" % uuid.uuid4()
+        self.audit_template1_name = gen_name()
+        self.audit_template2_name = gen_name()
+        self.audit_template3_name = gen_name()
+
+        with freezegun.freeze_time(self.expired_date):
+            self.audit_template1 = obj_utils.create_test_audit_template(
+                self.context, name=self.audit_template1_name,
+                id=self._generate_id(), uuid=None)
+            self.audit_template2 = obj_utils.create_test_audit_template(
+                self.context, name=self.audit_template2_name,
+                id=self._generate_id(), uuid=None)
+            self.audit_template3 = obj_utils.create_test_audit_template(
+                self.context, name=self.audit_template3_name,
+                id=self._generate_id(), uuid=None)
+            self.audit_template1.soft_delete()
+
+        with freezegun.freeze_time(self.expired_date):
+            self.audit1 = obj_utils.create_test_audit(
+                self.context, audit_template_id=self.audit_template1.id,
+                id=self._generate_id(), uuid=None)
+            self.audit2 = obj_utils.create_test_audit(
+                self.context, audit_template_id=self.audit_template2.id,
+                id=self._generate_id(), uuid=None)
+            self.audit3 = obj_utils.create_test_audit(
+                self.context, audit_template_id=self.audit_template3.id,
+                id=self._generate_id(), uuid=None)
+            self.audit1.soft_delete()
+
+        with freezegun.freeze_time(self.expired_date):
+            self.action_plan1 = obj_utils.create_test_action_plan(
+                self.context, audit_id=self.audit1.id,
+                id=self._generate_id(), uuid=None)
+            self.action_plan2 = obj_utils.create_test_action_plan(
+                self.context, audit_id=self.audit2.id,
+                id=self._generate_id(), uuid=None)
+            self.action_plan3 = obj_utils.create_test_action_plan(
+                self.context, audit_id=self.audit3.id,
+                id=self._generate_id(), uuid=None)
+
+            self.action1 = obj_utils.create_test_action(
+                self.context, action_plan_id=self.action_plan1.id,
+                id=self._generate_id(), uuid=None)
+            self.action2 = obj_utils.create_test_action(
+                self.context, action_plan_id=self.action_plan2.id,
+                id=self._generate_id(), uuid=None)
+            self.action3 = obj_utils.create_test_action(
+                self.context, action_plan_id=self.action_plan3.id,
+                id=self._generate_id(), uuid=None)
+            self.action_plan1.soft_delete()
+
+    @mock.patch.object(dbapi.Connection, "destroy_action")
+    @mock.patch.object(dbapi.Connection, "destroy_action_plan")
+    @mock.patch.object(dbapi.Connection, "destroy_audit")
+    @mock.patch.object(dbapi.Connection, "destroy_audit_template")
+    def test_execute_max_number_exceeded(self, m_destroy_audit_template,
+                                         m_destroy_audit,
+                                         m_destroy_action_plan,
+                                         m_destroy_action):
+        self.cmd.age_in_days = None
+        self.cmd.max_number = 5
+
+        with freezegun.freeze_time(self.fake_today):
+            self.audit_template2.soft_delete()
+            self.audit2.soft_delete()
+            self.action_plan2.soft_delete()
+
+        with freezegun.freeze_time(self.fake_today):
+            self.cmd.execute()
+
+        # The 1's and the 2's are purgeable (due to age of day set to 0),
+        # but max_number = 5, and because of no Db integrity violation, we
+        # should be able to purge only 4 objects.
+        self.assertEqual(m_destroy_audit_template.call_count, 1)
+        self.assertEqual(m_destroy_audit.call_count, 1)
+        self.assertEqual(m_destroy_action_plan.call_count, 1)
+        self.assertEqual(m_destroy_action.call_count, 1)
+
+    def test_find_deleted_entries(self):
+        self.cmd.age_in_days = None
+
+        with freezegun.freeze_time(self.fake_today):
+            objects_map = self.cmd.find_objects_to_delete()
+
+        self.assertEqual(len(objects_map.audit_templates), 1)
+        self.assertEqual(len(objects_map.audits), 1)
+        self.assertEqual(len(objects_map.action_plans), 1)
+        self.assertEqual(len(objects_map.actions), 1)
+
+    def test_find_deleted_and_expired_entries(self):
+        with freezegun.freeze_time(self.fake_today):
+            self.audit_template2.soft_delete()
+            self.audit2.soft_delete()
+            self.action_plan2.soft_delete()
+
+        with freezegun.freeze_time(self.fake_today):
+            objects_map = self.cmd.find_objects_to_delete()
+
+        # The 1's are purgeable (due to age of day set to 10)
+        self.assertEqual(len(objects_map.audit_templates), 1)
+        self.assertEqual(len(objects_map.audits), 1)
+        self.assertEqual(len(objects_map.action_plans), 1)
+        self.assertEqual(len(objects_map.actions), 1)
+
+    def test_find_deleted_and_nonexpired_related_entries(self):
+        with freezegun.freeze_time(self.fake_today):
+            # orphan audit
+            audit4 = obj_utils.create_test_audit(
+                self.context, audit_template_id=404,  # Does not exist
+                id=self._generate_id(), uuid=None)
+            action_plan4 = obj_utils.create_test_action_plan(
+                self.context, audit_id=audit4.id,
+                id=self._generate_id(), uuid=None)
+            action4 = obj_utils.create_test_action(
+                self.context, action_plan_id=action_plan4.id,
+                id=self._generate_id(), uuid=None)
+
+            audit5 = obj_utils.create_test_audit(
+                self.context, audit_template_id=self.audit_template1.id,
+                id=self._generate_id(), uuid=None)
+            action_plan5 = obj_utils.create_test_action_plan(
+                self.context, audit_id=audit5.id,
+                id=self._generate_id(), uuid=None)
+            action5 = obj_utils.create_test_action(
+                self.context, action_plan_id=action_plan5.id,
+                id=self._generate_id(), uuid=None)
+
+            self.audit_template2.soft_delete()
+            self.audit2.soft_delete()
+            self.action_plan2.soft_delete()
+
+            # All the 4's should be purged as well because they are orphans
+            # even though they were not deleted
+
+            # All the 5's should be purged as well even though they are not
+            # expired because their related audit template is itself expired
+            audit5.soft_delete()
+            action_plan5.soft_delete()
+
+        with freezegun.freeze_time(self.fake_today):
+            objects_map = self.cmd.find_objects_to_delete()
+
+        self.assertEqual(len(objects_map.audit_templates), 1)
+        self.assertEqual(len(objects_map.audits), 3)
+        self.assertEqual(len(objects_map.action_plans), 3)
+        self.assertEqual(len(objects_map.actions), 3)
+        self.assertEqual(
+            set([self.action1.id, action4.id, action5.id]),
+            set([entry.id for entry in objects_map.actions]))
+
+    @mock.patch.object(dbapi.Connection, "destroy_action")
+    @mock.patch.object(dbapi.Connection, "destroy_action_plan")
+    @mock.patch.object(dbapi.Connection, "destroy_audit")
+    @mock.patch.object(dbapi.Connection, "destroy_audit_template")
+    def test_purge_command(self, m_destroy_audit_template,
+                           m_destroy_audit, m_destroy_action_plan,
+                           m_destroy_action):
+        with freezegun.freeze_time(self.fake_today):
+            self.cmd.execute()
+
+        m_destroy_audit_template.assert_called_once_with(
+            self.audit_template1.uuid)
+        m_destroy_audit.assert_called_once_with(
+            self.audit1.uuid)
+        m_destroy_action_plan.assert_called_once_with(
+            self.action_plan1.uuid)
+        m_destroy_action.assert_called_once_with(
+            self.action1.uuid)
+
+    @mock.patch.object(dbapi.Connection, "destroy_action")
+    @mock.patch.object(dbapi.Connection, "destroy_action_plan")
+    @mock.patch.object(dbapi.Connection, "destroy_audit")
+    @mock.patch.object(dbapi.Connection, "destroy_audit_template")
+    def test_purge_command_with_nonexpired_related_entries(
+            self, m_destroy_audit_template, m_destroy_audit,
+            m_destroy_action_plan, m_destroy_action):
+        with freezegun.freeze_time(self.fake_today):
+            # orphan audit
+            audit4 = obj_utils.create_test_audit(
+                self.context, audit_template_id=404,  # Does not exist
+                id=self._generate_id(), uuid=None)
+            action_plan4 = obj_utils.create_test_action_plan(
+                self.context, audit_id=audit4.id,
+                id=self._generate_id(), uuid=None)
+            action4 = obj_utils.create_test_action(
+                self.context, action_plan_id=action_plan4.id,
+                id=self._generate_id(), uuid=None)
+
+            audit5 = obj_utils.create_test_audit(
+                self.context, audit_template_id=self.audit_template1.id,
+                id=self._generate_id(), uuid=None)
+            action_plan5 = obj_utils.create_test_action_plan(
+                self.context, audit_id=audit5.id,
+                id=self._generate_id(), uuid=None)
+            action5 = obj_utils.create_test_action(
+                self.context, action_plan_id=action_plan5.id,
+                id=self._generate_id(), uuid=None)
+
+            self.audit_template2.soft_delete()
+            self.audit2.soft_delete()
+            self.action_plan2.soft_delete()
+
+            # All the 4's should be purged as well because they are orphans
+            # even though they were not deleted
+
+            # All the 5's should be purged as well even though they are not
+            # expired because their related audit template is itself expired
+            audit5.soft_delete()
+            action_plan5.soft_delete()
+
+        with freezegun.freeze_time(self.fake_today):
+            self.cmd.execute()
+
+        self.assertEqual(m_destroy_audit_template.call_count, 1)
+        self.assertEqual(m_destroy_audit.call_count, 3)
+        self.assertEqual(m_destroy_action_plan.call_count, 3)
+        self.assertEqual(m_destroy_action.call_count, 3)
+
+        m_destroy_audit_template.assert_any_call(self.audit_template1.uuid)
+        m_destroy_audit.assert_any_call(self.audit1.uuid)
+        m_destroy_audit.assert_any_call(audit4.uuid)
+        m_destroy_action_plan.assert_any_call(self.action_plan1.uuid)
+        m_destroy_action_plan.assert_any_call(action_plan4.uuid)
+        m_destroy_action_plan.assert_any_call(action_plan5.uuid)
+        m_destroy_action.assert_any_call(self.action1.uuid)
+        m_destroy_action.assert_any_call(action4.uuid)
+        m_destroy_action.assert_any_call(action5.uuid)
+
+    @mock.patch.object(dbapi.Connection, "destroy_action")
+    @mock.patch.object(dbapi.Connection, "destroy_action_plan")
+    @mock.patch.object(dbapi.Connection, "destroy_audit")
+    @mock.patch.object(dbapi.Connection, "destroy_audit_template")
+    def test_purge_command_with_audit_template_ok(
+            self, m_destroy_audit_template, m_destroy_audit,
+            m_destroy_action_plan, m_destroy_action):
+        self.cmd.orphans = False
+        self.cmd.uuid = self.audit_template1.uuid
+
+        with freezegun.freeze_time(self.fake_today):
+            self.cmd.execute()
+
+        self.assertEqual(m_destroy_audit_template.call_count, 1)
+        self.assertEqual(m_destroy_audit.call_count, 1)
+        self.assertEqual(m_destroy_action_plan.call_count, 1)
+        self.assertEqual(m_destroy_action.call_count, 1)
+
+        m_destroy_audit_template.assert_called_once_with(
+            self.audit_template1.uuid)
+        m_destroy_audit.assert_called_once_with(
+            self.audit1.uuid)
+        m_destroy_action_plan.assert_called_once_with(
+            self.action_plan1.uuid)
+        m_destroy_action.assert_called_once_with(
+            self.action1.uuid)
+
+    @mock.patch.object(dbapi.Connection, "destroy_action")
+    @mock.patch.object(dbapi.Connection, "destroy_action_plan")
+    @mock.patch.object(dbapi.Connection, "destroy_audit")
+    @mock.patch.object(dbapi.Connection, "destroy_audit_template")
+    def test_purge_command_with_audit_template_not_expired(
+            self, m_destroy_audit_template, m_destroy_audit,
+            m_destroy_action_plan, m_destroy_action):
+        self.cmd.orphans = False
+        self.cmd.uuid = self.audit_template2.uuid
+
+        with freezegun.freeze_time(self.fake_today):
+            self.cmd.execute()
+
+        self.assertEqual(m_destroy_audit_template.call_count, 0)
+        self.assertEqual(m_destroy_audit.call_count, 0)
+        self.assertEqual(m_destroy_action_plan.call_count, 0)
+        self.assertEqual(m_destroy_action.call_count, 0)
+
+    @mock.patch.object(dbapi.Connection, "destroy_action")
+    @mock.patch.object(dbapi.Connection, "destroy_action_plan")
+    @mock.patch.object(dbapi.Connection, "destroy_audit")
+    @mock.patch.object(dbapi.Connection, "destroy_audit_template")
+    def test_purge_command_with_audit_template_not_soft_deleted(
+            self, m_destroy_audit_template, m_destroy_audit,
+            m_destroy_action_plan, m_destroy_action):
+        self.cmd.orphans = False
+        self.cmd.uuid = self.audit_template3.uuid
+
+        with freezegun.freeze_time(self.fake_today):
+            self.cmd.execute()
+
+        self.assertEqual(m_destroy_audit_template.call_count, 0)
+        self.assertEqual(m_destroy_audit.call_count, 0)
+        self.assertEqual(m_destroy_action_plan.call_count, 0)
+        self.assertEqual(m_destroy_action.call_count, 0)