From a6508a00132d586af4db1d417359671abfef92de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vincent=20Fran=C3=A7oise?= Date: Mon, 29 Feb 2016 09:22:48 +0100 Subject: [PATCH] Added purge script for soft deleted objects This patchset implements the purge script as specified in its related blueprint: - The '--age-in-days' option allows to specify the number of days before expiry - The '--max-number' option allows us to specify a limit on the number of objects to delete - The '--audit-template' option allows you to only delete objects related to the specified audit template UUID or name - The '--dry-run' option to go through the purge procedure without actually deleting anything - The '--exclude-orphans' option which allows you to exclude from the purge any object that does not have a parent (i.e. and audit without a related audit template) A prompt has been added to also propose to narrow down the number of deletions to be below the specified limit. Change-Id: I3ce83ab95277c109df67a6b5b920a878f6e59d3f Implements: blueprint db-purge-engine --- requirements.txt | 1 + watcher/cmd/dbmanage.py | 36 ++- watcher/common/config.py | 2 + watcher/common/exception.py | 8 + watcher/db/purge.py | 410 ++++++++++++++++++++++++++++ watcher/locale/watcher.pot | 100 ++++++- watcher/tests/cmd/test_db_manage.py | 105 ++++++- watcher/tests/db/test_purge.py | 372 +++++++++++++++++++++++++ 8 files changed, 1011 insertions(+), 23 deletions(-) create mode 100644 watcher/db/purge.py create mode 100644 watcher/tests/db/test_purge.py diff --git a/requirements.txt b/requirements.txt index 2af3fb078..c220708ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ oslo.utils>=3.5.0 # Apache-2.0 PasteDeploy>=1.5.0 # MIT pbr>=1.6 # Apache-2.0 pecan>=1.0.0 # BSD +PrettyTable>=0.7,<0.8 # BSD voluptuous>=0.8.6 # BSD License python-ceilometerclient>=2.2.1 # Apache-2.0 python-cinderclient>=1.3.1 # Apache-2.0 diff --git a/watcher/cmd/dbmanage.py b/watcher/cmd/dbmanage.py index 90667d521..37ea15f53 100644 --- a/watcher/cmd/dbmanage.py +++ b/watcher/cmd/dbmanage.py @@ -25,7 +25,7 @@ from oslo_config import cfg from watcher.common import service from watcher.db import migration - +from watcher.db import purge CONF = cfg.CONF @@ -56,6 +56,12 @@ class DBCommand(object): def create_schema(): migration.create_schema() + @staticmethod + def purge(): + purge.purge(CONF.command.age_in_days, CONF.command.max_number, + CONF.command.audit_template, CONF.command.exclude_orphans, + CONF.command.dry_run) + def add_command_parsers(subparsers): parser = subparsers.add_parser( @@ -96,6 +102,33 @@ def add_command_parsers(subparsers): help="Create the database schema.") parser.set_defaults(func=DBCommand.create_schema) + parser = subparsers.add_parser( + 'purge', + help="Purge the database.") + parser.add_argument('-d', '--age-in-days', + help="Number of days since deletion (from today) " + "to exclude from the purge. If None, everything " + "will be purged.", + type=int, default=None, nargs='?') + parser.add_argument('-n', '--max-number', + help="Max number of objects expected to be deleted. " + "Prevents the deletion if exceeded. No limit if " + "set to None.", + type=int, default=None, nargs='?') + parser.add_argument('-t', '--audit-template', + help="UUID or name of the audit template to purge.", + type=str, default=None, nargs='?') + parser.add_argument('-e', '--exclude-orphans', action='store_true', + help="Flag to indicate whether or not you want to " + "exclude orphans from deletion (default: False).", + default=False) + parser.add_argument('--dry-run', action='store_true', + help="Flag to indicate whether or not you want to " + "perform a dry run (no deletion).", + default=False) + + parser.set_defaults(func=DBCommand.purge) + command_opt = cfg.SubCommandOpt('command', title='Command', @@ -114,6 +147,7 @@ def main(): valid_commands = set([ 'upgrade', 'downgrade', 'revision', 'version', 'stamp', 'create_schema', + 'purge', ]) if not set(sys.argv).intersection(valid_commands): sys.argv.append('upgrade') diff --git a/watcher/common/config.py b/watcher/common/config.py index c88a4ab87..5ca04e3ff 100644 --- a/watcher/common/config.py +++ b/watcher/common/config.py @@ -22,6 +22,8 @@ from watcher import version def parse_args(argv, default_config_files=None): + default_config_files = (default_config_files or + cfg.find_config_files(project='watcher')) rpc.set_defaults(control_exchange='watcher') cfg.CONF(argv[1:], project='python-watcher', diff --git a/watcher/common/exception.py b/watcher/common/exception.py index 45a657481..7f65906ef 100644 --- a/watcher/common/exception.py +++ b/watcher/common/exception.py @@ -283,3 +283,11 @@ class LoadingError(WatcherException): class ReservedWord(WatcherException): msg_fmt = _("The identifier '%(name)s' is a reserved word") + + +class NotSoftDeletedStateError(WatcherException): + msg_fmt = _("The %(name)s resource %(id)s is not soft deleted") + + +class NegativeLimitError(WatcherException): + msg_fmt = _("Limit should be positive") diff --git a/watcher/db/purge.py b/watcher/db/purge.py new file mode 100644 index 000000000..7241878b8 --- /dev/null +++ b/watcher/db/purge.py @@ -0,0 +1,410 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2016 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import print_function + +import collections +import datetime +import itertools +import sys + +from oslo_log import log +from oslo_utils import strutils +import prettytable as ptable +from six.moves import input + +from watcher._i18n import _, _LI +from watcher.common import context +from watcher.common import exception +from watcher.common import utils +from watcher import objects + +LOG = log.getLogger(__name__) + + +class WatcherObjectsMap(object): + """Wrapper to deal with watcher objects per type + + This wrapper object contains a list of watcher objects per type. + Its main use is to simplify the merge of watcher objects by avoiding + duplicates, but also for representing the relationships between these + objects. + """ + + # This is for generating the .pot translations + keymap = collections.OrderedDict([ + ("audit_templates", _("Audit Templates")), + ("audits", _("Audits")), + ("action_plans", _("Action Plans")), + ("actions", _("Actions")), + ]) + + def __init__(self): + for attr_name in self.__class__.keys(): + setattr(self, attr_name, []) + + def values(self): + return (getattr(self, key) for key in self.__class__.keys()) + + @classmethod + def keys(cls): + return cls.keymap.keys() + + def __iter__(self): + return itertools.chain(*self.values()) + + def __add__(self, other): + new_map = self.__class__() + + # Merge the 2 items dicts into a new object (and avoid dupes) + for attr_name, initials, others in zip(self.keys(), self.values(), + other.values()): + # Creates a copy + merged = initials[:] + initials_ids = [item.id for item in initials] + non_dupes = [item for item in others + if item.id not in initials_ids] + merged += non_dupes + + setattr(new_map, attr_name, merged) + + return new_map + + def __str__(self): + out = "" + for key, vals in zip(self.keys(), self.values()): + ids = [val.id for val in vals] + out += "%(key)s: %(val)s" % (dict(key=key, val=ids)) + out += "\n" + return out + + def __len__(self): + return sum(len(getattr(self, key)) for key in self.keys()) + + def get_count_table(self): + headers = list(self.keymap.values()) + headers.append(_("Total")) # We also add a total count + counters = [len(cat_vals) for cat_vals in self.values()] + [len(self)] + table = ptable.PrettyTable(field_names=headers) + table.add_row(counters) + return table.get_string() + + +class PurgeCommand(object): + """Purges the DB by removing soft deleted entries + + The workflow for this purge is the following: + + # Find soft deleted objects which are expired + # Find orphan objects + # Find their related objects whether they are expired or not + # Merge them together + # If it does not exceed the limit, destroy them all + """ + + ctx = context.make_context(show_deleted=True) + + def __init__(self, age_in_days=None, max_number=None, + uuid=None, exclude_orphans=False, dry_run=None): + self.age_in_days = age_in_days + self.max_number = max_number + self.uuid = uuid + self.exclude_orphans = exclude_orphans + self.dry_run = dry_run + + self._delete_up_to_max = None + self._objects_map = WatcherObjectsMap() + + def get_expiry_date(self): + if not self.age_in_days: + return None + today = datetime.datetime.today() + expiry_date = today - datetime.timedelta(days=self.age_in_days) + return expiry_date + + @classmethod + def get_audit_template_uuid(cls, uuid_or_name): + if uuid_or_name is None: + return + + query_func = None + if not utils.is_uuid_like(uuid_or_name): + query_func = objects.audit_template.AuditTemplate.get_by_name + else: + query_func = objects.audit_template.AuditTemplate.get_by_uuid + + try: + audit_template = query_func(cls.ctx, uuid_or_name) + except Exception as exc: + LOG.exception(exc) + raise exception.AuditTemplateNotFound(audit_template=uuid_or_name) + + if not audit_template.deleted_at: + raise exception.NotSoftDeletedStateError( + name=_('Audit Template'), id=uuid_or_name) + + return audit_template.uuid + + def _find_audit_templates(self, filters=None): + return objects.audit_template.AuditTemplate.list( + self.ctx, filters=filters) + + def _find_audits(self, filters=None): + return objects.audit.Audit.list(self.ctx, filters=filters) + + def _find_action_plans(self, filters=None): + return objects.action_plan.ActionPlan.list(self.ctx, filters=filters) + + def _find_actions(self, filters=None): + return objects.action.Action.list(self.ctx, filters=filters) + + def _find_orphans(self): + orphans = WatcherObjectsMap() + + filters = dict(deleted=False) + audit_templates = objects.audit_template.AuditTemplate.list( + self.ctx, filters=filters) + audits = objects.audit.Audit.list(self.ctx, filters=filters) + action_plans = objects.action_plan.ActionPlan.list( + self.ctx, filters=filters) + actions = objects.action.Action.list(self.ctx, filters=filters) + + audit_template_ids = set(at.id for at in audit_templates) + orphans.audits = [ + audit for audit in audits + if audit.audit_template_id not in audit_template_ids] + + # Objects with orphan parents are themselves orphans + audit_ids = [audit.id for audit in (a for a in audits + if a not in orphans.audits)] + orphans.action_plans = [ + ap for ap in action_plans + if ap.audit_id not in audit_ids] + + # Objects with orphan parents are themselves orphans + action_plan_ids = [ap.id for ap in (a for a in action_plans + if a not in orphans.action_plans)] + orphans.actions = [ + action for action in actions + if action.action_plan_id not in action_plan_ids] + + LOG.debug("Orphans found:\n%s", orphans) + LOG.info(_LI("Orphans found:\n%s"), orphans.get_count_table()) + + return orphans + + def _find_soft_deleted_objects(self): + to_be_deleted = WatcherObjectsMap() + + expiry_date = self.get_expiry_date() + filters = dict(deleted=True) + if self.uuid: + filters["uuid"] = self.uuid + if expiry_date: + filters.update(dict(deleted_at__lt=expiry_date)) + + to_be_deleted.audit_templates.extend( + self._find_audit_templates(filters)) + to_be_deleted.audits.extend(self._find_audits(filters)) + to_be_deleted.action_plans.extend(self._find_action_plans(filters)) + to_be_deleted.actions.extend(self._find_actions(filters)) + + soft_deleted_objs = self._find_related_objects( + to_be_deleted, base_filters=dict(deleted=True)) + + LOG.debug("Soft deleted objects:\n%s", soft_deleted_objs) + + return soft_deleted_objs + + def _find_related_objects(self, objects_map, base_filters=None): + base_filters = base_filters or {} + + for audit_template in objects_map.audit_templates: + filters = {} + filters.update(base_filters) + filters.update(dict(audit_template_id=audit_template.id)) + related_objs = WatcherObjectsMap() + related_objs.audits = self._find_audits(filters) + objects_map += related_objs + + for audit in objects_map.audits: + filters = {} + filters.update(base_filters) + filters.update(dict(audit_id=audit.id)) + related_objs = WatcherObjectsMap() + related_objs.action_plans = self._find_action_plans(filters) + objects_map += related_objs + + for action_plan in objects_map.action_plans: + filters = {} + filters.update(base_filters) + filters.update(dict(action_plan_id=action_plan.id)) + related_objs = WatcherObjectsMap() + related_objs.actions = self._find_actions(filters) + objects_map += related_objs + + return objects_map + + def confirmation_prompt(self): + print(self._objects_map.get_count_table()) + raw_val = input( + _("There are %(count)d objects set for deletion. " + "Continue? [y/N]") % dict(count=len(self._objects_map))) + + return strutils.bool_from_string(raw_val) + + def delete_up_to_max_prompt(self, objects_map): + print(objects_map.get_count_table()) + print(_("The number of objects (%(num)s) to delete from the database " + "exceeds the maximum number of objects (%(max_number)s) " + "specified.") % dict(max_number=self.max_number, + num=len(objects_map))) + raw_val = input( + _("Do you want to delete objects up to the specified maximum " + "number? [y/N]")) + + self._delete_up_to_max = strutils.bool_from_string(raw_val) + + return self._delete_up_to_max + + def _aggregate_objects(self): + """Objects aggregated on a 'per audit template' basis""" + # todo: aggregate orphans as well + aggregate = [] + for audit_template in self._objects_map.audit_templates: + related_objs = WatcherObjectsMap() + related_objs.audit_templates = [audit_template] + related_objs.audits = [ + audit for audit in self._objects_map.audits + if audit.audit_template_id == audit_template.id + ] + audit_ids = [audit.id for audit in related_objs.audits] + related_objs.action_plans = [ + action_plan for action_plan in self._objects_map.action_plans + if action_plan.audit_id in audit_ids + ] + action_plan_ids = [ + action_plan.id for action_plan in related_objs.action_plans + ] + related_objs.actions = [ + action for action in self._objects_map.actions + if action.action_plan_id in action_plan_ids + ] + aggregate.append(related_objs) + + return aggregate + + def _get_objects_up_to_limit(self): + aggregated_objects = self._aggregate_objects() + to_be_deleted_subset = WatcherObjectsMap() + + for aggregate in aggregated_objects: + if len(aggregate) + len(to_be_deleted_subset) <= self.max_number: + to_be_deleted_subset += aggregate + else: + break + + LOG.debug(to_be_deleted_subset) + return to_be_deleted_subset + + def find_objects_to_delete(self): + """Finds all the objects to be purged + + :returns: A mapping with all the Watcher objects to purged + :rtype: :py:class:`~.WatcherObjectsMap` instance + """ + to_be_deleted = self._find_soft_deleted_objects() + + if not self.exclude_orphans: + to_be_deleted += self._find_orphans() + + LOG.debug("Objects to be deleted:\n%s", to_be_deleted) + + return to_be_deleted + + def do_delete(self): + LOG.info(_LI("Deleting...")) + # Reversed to avoid errors with foreign keys + for entry in reversed(list(self._objects_map)): + entry.destroy() + + def execute(self): + LOG.info(_LI("Starting purge command")) + self._objects_map = self.find_objects_to_delete() + + if (self.max_number is not None and + len(self._objects_map) > self.max_number): + if self.delete_up_to_max_prompt(self._objects_map): + self._objects_map = self._get_objects_up_to_limit() + else: + return + + _orphans_note = (_(" (orphans excluded)") if self.exclude_orphans + else _(" (may include orphans)")) + if not self.dry_run and self.confirmation_prompt(): + self.do_delete() + print(_("Purge results summary%s:") % _orphans_note) + LOG.info(_LI("Purge results summary%s:"), _orphans_note) + else: + LOG.debug(self._objects_map) + print(_("Here below is a table containing the objects " + "that can be purged%s:") % _orphans_note) + + LOG.info("\n%s", self._objects_map.get_count_table()) + print(self._objects_map.get_count_table()) + LOG.info(_LI("Purge process completed")) + + +def purge(age_in_days, max_number, audit_template, exclude_orphans, dry_run): + """Removes soft deleted objects from the database + + :param age_in_days: Number of days since deletion (from today) + to exclude from the purge. If None, everything will be purged. + :type age_in_days: int + :param max_number: Max number of objects expected to be deleted. + Prevents the deletion if exceeded. No limit if set to None. + :type max_number: int + :param audit_template: UUID or name of the audit template to purge. + :type audit_template: str + :param exclude_orphans: Flag to indicate whether or not you want to + exclude orphans from deletion (default: False). + :type exclude_orphans: bool + :param dry_run: Flag to indicate whether or not you want to perform + a dry run (no deletion). + :type dry_run: bool + """ + try: + if max_number and max_number < 0: + raise exception.NegativeLimitError + + LOG.info("[options] age_in_days = %s", age_in_days) + LOG.info("[options] max_number = %s", max_number) + LOG.info("[options] audit_template = %s", audit_template) + LOG.info("[options] exclude_orphans = %s", exclude_orphans) + LOG.info("[options] dry_run = %s", dry_run) + + uuid = PurgeCommand.get_audit_template_uuid(audit_template) + + cmd = PurgeCommand(age_in_days, max_number, uuid, + exclude_orphans, dry_run) + + cmd.execute() + + except Exception as exc: + LOG.exception(exc) + print(exc) + sys.exit(1) diff --git a/watcher/locale/watcher.pot b/watcher/locale/watcher.pot index a02690321..b1b9553b9 100644 --- a/watcher/locale/watcher.pot +++ b/watcher/locale/watcher.pot @@ -7,9 +7,9 @@ #, fuzzy msgid "" msgstr "" -"Project-Id-Version: python-watcher 0.24.1.dev1\n" +"Project-Id-Version: python-watcher 0.24.1.dev4\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2016-03-09 11:18+0100\n" +"POT-Creation-Date: 2016-03-14 15:29+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -344,6 +344,15 @@ msgstr "" msgid "The identifier '%(name)s' is a reserved word" msgstr "" +#: watcher/common/exception.py:289 +#, python-format +msgid "The %(name)s resource %(id)s is not soft deleted" +msgstr "" + +#: watcher/common/exception.py:293 +msgid "Limit should be positive" +msgstr "" + #: watcher/common/service.py:83 #, python-format msgid "Created RPC server for service %(service)s on host %(host)s." @@ -386,25 +395,102 @@ msgstr "" msgid "Messaging configuration error" msgstr "" -#: watcher/db/sqlalchemy/api.py:256 +#: watcher/db/purge.py:50 +msgid "Audit Templates" +msgstr "" + +#: watcher/db/purge.py:51 +msgid "Audits" +msgstr "" + +#: watcher/db/purge.py:52 +msgid "Action Plans" +msgstr "" + +#: watcher/db/purge.py:53 +msgid "Actions" +msgstr "" + +#: watcher/db/purge.py:100 +msgid "Total" +msgstr "" + +#: watcher/db/purge.py:158 +msgid "Audit Template" +msgstr "" + +#: watcher/db/purge.py:206 +#, python-format +msgid "" +"Orphans found:\n" +"%s" +msgstr "" + +#: watcher/db/purge.py:265 +#, python-format +msgid "There are %(count)d objects set for deletion. Continue? [y/N]" +msgstr "" + +#: watcher/db/purge.py:272 +#, python-format +msgid "" +"The number of objects (%(num)s) to delete from the database exceeds the " +"maximum number of objects (%(max_number)s) specified." +msgstr "" + +#: watcher/db/purge.py:277 +msgid "Do you want to delete objects up to the specified maximum number? [y/N]" +msgstr "" + +#: watcher/db/purge.py:340 +msgid "Deleting..." +msgstr "" + +#: watcher/db/purge.py:346 +msgid "Starting purge command" +msgstr "" + +#: watcher/db/purge.py:356 +msgid " (orphans excluded)" +msgstr "" + +#: watcher/db/purge.py:357 +msgid " (may include orphans)" +msgstr "" + +#: watcher/db/purge.py:360 watcher/db/purge.py:361 +#, python-format +msgid "Purge results summary%s:" +msgstr "" + +#: watcher/db/purge.py:364 +#, python-format +msgid "Here below is a table containing the objects that can be purged%s:" +msgstr "" + +#: watcher/db/purge.py:369 +msgid "Purge process completed" +msgstr "" + +#: watcher/db/sqlalchemy/api.py:362 msgid "" "Multiple audit templates exist with the same name. Please use the audit " "template uuid instead" msgstr "" -#: watcher/db/sqlalchemy/api.py:278 +#: watcher/db/sqlalchemy/api.py:384 msgid "Cannot overwrite UUID for an existing Audit Template." msgstr "" -#: watcher/db/sqlalchemy/api.py:388 +#: watcher/db/sqlalchemy/api.py:495 msgid "Cannot overwrite UUID for an existing Audit." msgstr "" -#: watcher/db/sqlalchemy/api.py:480 +#: watcher/db/sqlalchemy/api.py:588 msgid "Cannot overwrite UUID for an existing Action." msgstr "" -#: watcher/db/sqlalchemy/api.py:590 +#: watcher/db/sqlalchemy/api.py:699 msgid "Cannot overwrite UUID for an existing Action Plan." msgstr "" diff --git a/watcher/tests/cmd/test_db_manage.py b/watcher/tests/cmd/test_db_manage.py index 95b54e92e..e08424d59 100644 --- a/watcher/tests/cmd/test_db_manage.py +++ b/watcher/tests/cmd/test_db_manage.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -# Copyright (c) 2015 b<>com +# Copyright (c) 2016 b<>com # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,15 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from mock import Mock -from mock import patch +import sys + +import mock from oslo_config import cfg + from watcher.cmd import dbmanage from watcher.db import migration -from watcher.tests.base import TestCase +from watcher.db import purge +from watcher.tests import base -class TestDBManageRunApp(TestCase): +class TestDBManageRunApp(base.TestCase): scenarios = ( ("upgrade", {"command": "upgrade", "expected": "upgrade"}), @@ -32,15 +35,16 @@ class TestDBManageRunApp(TestCase): ("version", {"command": "version", "expected": "version"}), ("create_schema", {"command": "create_schema", "expected": "create_schema"}), + ("purge", {"command": "purge", "expected": "purge"}), ("no_param", {"command": None, "expected": "upgrade"}), ) - @patch.object(dbmanage, "register_sub_command_opts", Mock()) - @patch("watcher.cmd.dbmanage.service.prepare_service") - @patch("watcher.cmd.dbmanage.sys") + @mock.patch.object(dbmanage, "register_sub_command_opts", mock.Mock()) + @mock.patch("watcher.cmd.dbmanage.service.prepare_service") + @mock.patch("watcher.cmd.dbmanage.sys") def test_run_db_manage_app(self, m_sys, m_prepare_service): # Patch command function - m_func = Mock() + m_func = mock.Mock() cfg.CONF.register_opt(cfg.SubCommandOpt("command")) cfg.CONF.command.func = m_func @@ -53,9 +57,9 @@ class TestDBManageRunApp(TestCase): ["watcher-db-manage", self.expected]) -class TestDBManageRunCommand(TestCase): +class TestDBManageRunCommand(base.TestCase): - @patch.object(migration, "upgrade") + @mock.patch.object(migration, "upgrade") def test_run_db_upgrade(self, m_upgrade): cfg.CONF.register_opt(cfg.StrOpt("revision"), group="command") cfg.CONF.set_default("revision", "dummy", group="command") @@ -63,7 +67,7 @@ class TestDBManageRunCommand(TestCase): m_upgrade.assert_called_once_with("dummy") - @patch.object(migration, "downgrade") + @mock.patch.object(migration, "downgrade") def test_run_db_downgrade(self, m_downgrade): cfg.CONF.register_opt(cfg.StrOpt("revision"), group="command") cfg.CONF.set_default("revision", "dummy", group="command") @@ -71,7 +75,7 @@ class TestDBManageRunCommand(TestCase): m_downgrade.assert_called_once_with("dummy") - @patch.object(migration, "revision") + @mock.patch.object(migration, "revision") def test_run_db_revision(self, m_revision): cfg.CONF.register_opt(cfg.StrOpt("message"), group="command") cfg.CONF.register_opt(cfg.StrOpt("autogenerate"), group="command") @@ -87,14 +91,85 @@ class TestDBManageRunCommand(TestCase): "dummy_message", "dummy_autogenerate" ) - @patch.object(migration, "stamp") + @mock.patch.object(migration, "stamp") def test_run_db_stamp(self, m_stamp): cfg.CONF.register_opt(cfg.StrOpt("revision"), group="command") cfg.CONF.set_default("revision", "dummy", group="command") dbmanage.DBCommand.stamp() - @patch.object(migration, "version") + @mock.patch.object(migration, "version") def test_run_db_version(self, m_version): dbmanage.DBCommand.version() self.assertEqual(1, m_version.call_count) + + @mock.patch.object(purge, "PurgeCommand") + def test_run_db_purge(self, m_purge_cls): + m_purge = mock.Mock() + m_purge_cls.return_value = m_purge + m_purge_cls.get_audit_template_uuid.return_value = 'Some UUID' + cfg.CONF.register_opt(cfg.IntOpt("age_in_days"), group="command") + cfg.CONF.register_opt(cfg.IntOpt("max_number"), group="command") + cfg.CONF.register_opt(cfg.StrOpt("audit_template"), group="command") + cfg.CONF.register_opt(cfg.BoolOpt("exclude_orphans"), group="command") + cfg.CONF.register_opt(cfg.BoolOpt("dry_run"), group="command") + cfg.CONF.set_default("age_in_days", None, group="command") + cfg.CONF.set_default("max_number", None, group="command") + cfg.CONF.set_default("audit_template", None, group="command") + cfg.CONF.set_default("exclude_orphans", True, group="command") + cfg.CONF.set_default("dry_run", False, group="command") + + dbmanage.DBCommand.purge() + + m_purge_cls.assert_called_once_with( + None, None, 'Some UUID', True, False) + m_purge.execute.assert_called_once_with() + + @mock.patch.object(sys, "exit") + @mock.patch.object(purge, "PurgeCommand") + def test_run_db_purge_negative_max_number(self, m_purge_cls, m_exit): + m_purge = mock.Mock() + m_purge_cls.return_value = m_purge + m_purge_cls.get_audit_template_uuid.return_value = 'Some UUID' + cfg.CONF.register_opt(cfg.IntOpt("age_in_days"), group="command") + cfg.CONF.register_opt(cfg.IntOpt("max_number"), group="command") + cfg.CONF.register_opt(cfg.StrOpt("audit_template"), group="command") + cfg.CONF.register_opt(cfg.BoolOpt("exclude_orphans"), group="command") + cfg.CONF.register_opt(cfg.BoolOpt("dry_run"), group="command") + cfg.CONF.set_default("age_in_days", None, group="command") + cfg.CONF.set_default("max_number", -1, group="command") + cfg.CONF.set_default("audit_template", None, group="command") + cfg.CONF.set_default("exclude_orphans", True, group="command") + cfg.CONF.set_default("dry_run", False, group="command") + + dbmanage.DBCommand.purge() + + self.assertEqual(0, m_purge_cls.call_count) + self.assertEqual(0, m_purge.execute.call_count) + self.assertEqual(0, m_purge.do_delete.call_count) + self.assertEqual(1, m_exit.call_count) + + @mock.patch.object(sys, "exit") + @mock.patch.object(purge, "PurgeCommand") + def test_run_db_purge_dry_run(self, m_purge_cls, m_exit): + m_purge = mock.Mock() + m_purge_cls.return_value = m_purge + m_purge_cls.get_audit_template_uuid.return_value = 'Some UUID' + cfg.CONF.register_opt(cfg.IntOpt("age_in_days"), group="command") + cfg.CONF.register_opt(cfg.IntOpt("max_number"), group="command") + cfg.CONF.register_opt(cfg.StrOpt("audit_template"), group="command") + cfg.CONF.register_opt(cfg.BoolOpt("exclude_orphans"), group="command") + cfg.CONF.register_opt(cfg.BoolOpt("dry_run"), group="command") + cfg.CONF.set_default("age_in_days", None, group="command") + cfg.CONF.set_default("max_number", None, group="command") + cfg.CONF.set_default("audit_template", None, group="command") + cfg.CONF.set_default("exclude_orphans", True, group="command") + cfg.CONF.set_default("dry_run", True, group="command") + + dbmanage.DBCommand.purge() + + m_purge_cls.assert_called_once_with( + None, None, 'Some UUID', True, True) + self.assertEqual(1, m_purge.execute.call_count) + self.assertEqual(0, m_purge.do_delete.call_count) + self.assertEqual(0, m_exit.call_count) diff --git a/watcher/tests/db/test_purge.py b/watcher/tests/db/test_purge.py new file mode 100644 index 000000000..44f7b0d00 --- /dev/null +++ b/watcher/tests/db/test_purge.py @@ -0,0 +1,372 @@ +# -*- encoding: utf-8 -*- +# Copyright (c) 2016 b<>com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid + +import freezegun +import mock + +from watcher.common import context as watcher_context +from watcher.db import purge +from watcher.db.sqlalchemy import api as dbapi +from watcher.tests.db import base +from watcher.tests.objects import utils as obj_utils + + +class TestPurgeCommand(base.DbTestCase): + + def setUp(self): + super(TestPurgeCommand, self).setUp() + self.cmd = purge.PurgeCommand() + token_info = { + 'token': { + 'project': { + 'id': 'fake_project' + }, + 'user': { + 'id': 'fake_user' + } + } + } + self.context = watcher_context.RequestContext( + auth_token_info=token_info, + project_id='fake_project', + user_id='fake_user', + show_deleted=True, + ) + + self.fake_today = '2016-02-24T09:52:05.219414+00:00' + self.expired_date = '2016-01-24T09:52:05.219414+00:00' + + self.m_input = mock.Mock() + p = mock.patch("watcher.db.purge.input", self.m_input) + self.m_input.return_value = 'y' + p.start() + self.addCleanup(p.stop) + + self._id_generator = None + self._data_setup() + + def _generate_id(self): + if self._id_generator is None: + self._id_generator = self._get_id_generator() + return next(self._id_generator) + + def _get_id_generator(self): + seed = 1 + while True: + yield seed + seed += 1 + + def _data_setup(self): + # All the 1's are soft_deleted and are expired + # All the 2's are soft_deleted but are not expired + # All the 3's are *not* soft_deleted + + # Number of days we want to keep in DB (no purge for them) + self.cmd.age_in_days = 10 + self.cmd.max_number = None + self.cmd.orphans = True + gen_name = lambda: "Audit Template %s" % uuid.uuid4() + self.audit_template1_name = gen_name() + self.audit_template2_name = gen_name() + self.audit_template3_name = gen_name() + + with freezegun.freeze_time(self.expired_date): + self.audit_template1 = obj_utils.create_test_audit_template( + self.context, name=self.audit_template1_name, + id=self._generate_id(), uuid=None) + self.audit_template2 = obj_utils.create_test_audit_template( + self.context, name=self.audit_template2_name, + id=self._generate_id(), uuid=None) + self.audit_template3 = obj_utils.create_test_audit_template( + self.context, name=self.audit_template3_name, + id=self._generate_id(), uuid=None) + self.audit_template1.soft_delete() + + with freezegun.freeze_time(self.expired_date): + self.audit1 = obj_utils.create_test_audit( + self.context, audit_template_id=self.audit_template1.id, + id=self._generate_id(), uuid=None) + self.audit2 = obj_utils.create_test_audit( + self.context, audit_template_id=self.audit_template2.id, + id=self._generate_id(), uuid=None) + self.audit3 = obj_utils.create_test_audit( + self.context, audit_template_id=self.audit_template3.id, + id=self._generate_id(), uuid=None) + self.audit1.soft_delete() + + with freezegun.freeze_time(self.expired_date): + self.action_plan1 = obj_utils.create_test_action_plan( + self.context, audit_id=self.audit1.id, + id=self._generate_id(), uuid=None) + self.action_plan2 = obj_utils.create_test_action_plan( + self.context, audit_id=self.audit2.id, + id=self._generate_id(), uuid=None) + self.action_plan3 = obj_utils.create_test_action_plan( + self.context, audit_id=self.audit3.id, + id=self._generate_id(), uuid=None) + + self.action1 = obj_utils.create_test_action( + self.context, action_plan_id=self.action_plan1.id, + id=self._generate_id(), uuid=None) + self.action2 = obj_utils.create_test_action( + self.context, action_plan_id=self.action_plan2.id, + id=self._generate_id(), uuid=None) + self.action3 = obj_utils.create_test_action( + self.context, action_plan_id=self.action_plan3.id, + id=self._generate_id(), uuid=None) + self.action_plan1.soft_delete() + + @mock.patch.object(dbapi.Connection, "destroy_action") + @mock.patch.object(dbapi.Connection, "destroy_action_plan") + @mock.patch.object(dbapi.Connection, "destroy_audit") + @mock.patch.object(dbapi.Connection, "destroy_audit_template") + def test_execute_max_number_exceeded(self, m_destroy_audit_template, + m_destroy_audit, + m_destroy_action_plan, + m_destroy_action): + self.cmd.age_in_days = None + self.cmd.max_number = 5 + + with freezegun.freeze_time(self.fake_today): + self.audit_template2.soft_delete() + self.audit2.soft_delete() + self.action_plan2.soft_delete() + + with freezegun.freeze_time(self.fake_today): + self.cmd.execute() + + # The 1's and the 2's are purgeable (due to age of day set to 0), + # but max_number = 5, and because of no Db integrity violation, we + # should be able to purge only 4 objects. + self.assertEqual(m_destroy_audit_template.call_count, 1) + self.assertEqual(m_destroy_audit.call_count, 1) + self.assertEqual(m_destroy_action_plan.call_count, 1) + self.assertEqual(m_destroy_action.call_count, 1) + + def test_find_deleted_entries(self): + self.cmd.age_in_days = None + + with freezegun.freeze_time(self.fake_today): + objects_map = self.cmd.find_objects_to_delete() + + self.assertEqual(len(objects_map.audit_templates), 1) + self.assertEqual(len(objects_map.audits), 1) + self.assertEqual(len(objects_map.action_plans), 1) + self.assertEqual(len(objects_map.actions), 1) + + def test_find_deleted_and_expired_entries(self): + with freezegun.freeze_time(self.fake_today): + self.audit_template2.soft_delete() + self.audit2.soft_delete() + self.action_plan2.soft_delete() + + with freezegun.freeze_time(self.fake_today): + objects_map = self.cmd.find_objects_to_delete() + + # The 1's are purgeable (due to age of day set to 10) + self.assertEqual(len(objects_map.audit_templates), 1) + self.assertEqual(len(objects_map.audits), 1) + self.assertEqual(len(objects_map.action_plans), 1) + self.assertEqual(len(objects_map.actions), 1) + + def test_find_deleted_and_nonexpired_related_entries(self): + with freezegun.freeze_time(self.fake_today): + # orphan audit + audit4 = obj_utils.create_test_audit( + self.context, audit_template_id=404, # Does not exist + id=self._generate_id(), uuid=None) + action_plan4 = obj_utils.create_test_action_plan( + self.context, audit_id=audit4.id, + id=self._generate_id(), uuid=None) + action4 = obj_utils.create_test_action( + self.context, action_plan_id=action_plan4.id, + id=self._generate_id(), uuid=None) + + audit5 = obj_utils.create_test_audit( + self.context, audit_template_id=self.audit_template1.id, + id=self._generate_id(), uuid=None) + action_plan5 = obj_utils.create_test_action_plan( + self.context, audit_id=audit5.id, + id=self._generate_id(), uuid=None) + action5 = obj_utils.create_test_action( + self.context, action_plan_id=action_plan5.id, + id=self._generate_id(), uuid=None) + + self.audit_template2.soft_delete() + self.audit2.soft_delete() + self.action_plan2.soft_delete() + + # All the 4's should be purged as well because they are orphans + # even though they were not deleted + + # All the 5's should be purged as well even though they are not + # expired because their related audit template is itself expired + audit5.soft_delete() + action_plan5.soft_delete() + + with freezegun.freeze_time(self.fake_today): + objects_map = self.cmd.find_objects_to_delete() + + self.assertEqual(len(objects_map.audit_templates), 1) + self.assertEqual(len(objects_map.audits), 3) + self.assertEqual(len(objects_map.action_plans), 3) + self.assertEqual(len(objects_map.actions), 3) + self.assertEqual( + set([self.action1.id, action4.id, action5.id]), + set([entry.id for entry in objects_map.actions])) + + @mock.patch.object(dbapi.Connection, "destroy_action") + @mock.patch.object(dbapi.Connection, "destroy_action_plan") + @mock.patch.object(dbapi.Connection, "destroy_audit") + @mock.patch.object(dbapi.Connection, "destroy_audit_template") + def test_purge_command(self, m_destroy_audit_template, + m_destroy_audit, m_destroy_action_plan, + m_destroy_action): + with freezegun.freeze_time(self.fake_today): + self.cmd.execute() + + m_destroy_audit_template.assert_called_once_with( + self.audit_template1.uuid) + m_destroy_audit.assert_called_once_with( + self.audit1.uuid) + m_destroy_action_plan.assert_called_once_with( + self.action_plan1.uuid) + m_destroy_action.assert_called_once_with( + self.action1.uuid) + + @mock.patch.object(dbapi.Connection, "destroy_action") + @mock.patch.object(dbapi.Connection, "destroy_action_plan") + @mock.patch.object(dbapi.Connection, "destroy_audit") + @mock.patch.object(dbapi.Connection, "destroy_audit_template") + def test_purge_command_with_nonexpired_related_entries( + self, m_destroy_audit_template, m_destroy_audit, + m_destroy_action_plan, m_destroy_action): + with freezegun.freeze_time(self.fake_today): + # orphan audit + audit4 = obj_utils.create_test_audit( + self.context, audit_template_id=404, # Does not exist + id=self._generate_id(), uuid=None) + action_plan4 = obj_utils.create_test_action_plan( + self.context, audit_id=audit4.id, + id=self._generate_id(), uuid=None) + action4 = obj_utils.create_test_action( + self.context, action_plan_id=action_plan4.id, + id=self._generate_id(), uuid=None) + + audit5 = obj_utils.create_test_audit( + self.context, audit_template_id=self.audit_template1.id, + id=self._generate_id(), uuid=None) + action_plan5 = obj_utils.create_test_action_plan( + self.context, audit_id=audit5.id, + id=self._generate_id(), uuid=None) + action5 = obj_utils.create_test_action( + self.context, action_plan_id=action_plan5.id, + id=self._generate_id(), uuid=None) + + self.audit_template2.soft_delete() + self.audit2.soft_delete() + self.action_plan2.soft_delete() + + # All the 4's should be purged as well because they are orphans + # even though they were not deleted + + # All the 5's should be purged as well even though they are not + # expired because their related audit template is itself expired + audit5.soft_delete() + action_plan5.soft_delete() + + with freezegun.freeze_time(self.fake_today): + self.cmd.execute() + + self.assertEqual(m_destroy_audit_template.call_count, 1) + self.assertEqual(m_destroy_audit.call_count, 3) + self.assertEqual(m_destroy_action_plan.call_count, 3) + self.assertEqual(m_destroy_action.call_count, 3) + + m_destroy_audit_template.assert_any_call(self.audit_template1.uuid) + m_destroy_audit.assert_any_call(self.audit1.uuid) + m_destroy_audit.assert_any_call(audit4.uuid) + m_destroy_action_plan.assert_any_call(self.action_plan1.uuid) + m_destroy_action_plan.assert_any_call(action_plan4.uuid) + m_destroy_action_plan.assert_any_call(action_plan5.uuid) + m_destroy_action.assert_any_call(self.action1.uuid) + m_destroy_action.assert_any_call(action4.uuid) + m_destroy_action.assert_any_call(action5.uuid) + + @mock.patch.object(dbapi.Connection, "destroy_action") + @mock.patch.object(dbapi.Connection, "destroy_action_plan") + @mock.patch.object(dbapi.Connection, "destroy_audit") + @mock.patch.object(dbapi.Connection, "destroy_audit_template") + def test_purge_command_with_audit_template_ok( + self, m_destroy_audit_template, m_destroy_audit, + m_destroy_action_plan, m_destroy_action): + self.cmd.orphans = False + self.cmd.uuid = self.audit_template1.uuid + + with freezegun.freeze_time(self.fake_today): + self.cmd.execute() + + self.assertEqual(m_destroy_audit_template.call_count, 1) + self.assertEqual(m_destroy_audit.call_count, 1) + self.assertEqual(m_destroy_action_plan.call_count, 1) + self.assertEqual(m_destroy_action.call_count, 1) + + m_destroy_audit_template.assert_called_once_with( + self.audit_template1.uuid) + m_destroy_audit.assert_called_once_with( + self.audit1.uuid) + m_destroy_action_plan.assert_called_once_with( + self.action_plan1.uuid) + m_destroy_action.assert_called_once_with( + self.action1.uuid) + + @mock.patch.object(dbapi.Connection, "destroy_action") + @mock.patch.object(dbapi.Connection, "destroy_action_plan") + @mock.patch.object(dbapi.Connection, "destroy_audit") + @mock.patch.object(dbapi.Connection, "destroy_audit_template") + def test_purge_command_with_audit_template_not_expired( + self, m_destroy_audit_template, m_destroy_audit, + m_destroy_action_plan, m_destroy_action): + self.cmd.orphans = False + self.cmd.uuid = self.audit_template2.uuid + + with freezegun.freeze_time(self.fake_today): + self.cmd.execute() + + self.assertEqual(m_destroy_audit_template.call_count, 0) + self.assertEqual(m_destroy_audit.call_count, 0) + self.assertEqual(m_destroy_action_plan.call_count, 0) + self.assertEqual(m_destroy_action.call_count, 0) + + @mock.patch.object(dbapi.Connection, "destroy_action") + @mock.patch.object(dbapi.Connection, "destroy_action_plan") + @mock.patch.object(dbapi.Connection, "destroy_audit") + @mock.patch.object(dbapi.Connection, "destroy_audit_template") + def test_purge_command_with_audit_template_not_soft_deleted( + self, m_destroy_audit_template, m_destroy_audit, + m_destroy_action_plan, m_destroy_action): + self.cmd.orphans = False + self.cmd.uuid = self.audit_template3.uuid + + with freezegun.freeze_time(self.fake_today): + self.cmd.execute() + + self.assertEqual(m_destroy_audit_template.call_count, 0) + self.assertEqual(m_destroy_audit.call_count, 0) + self.assertEqual(m_destroy_action_plan.call_count, 0) + self.assertEqual(m_destroy_action.call_count, 0)