Arnike/watcher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added purge script for soft deleted objects

Browse Source 
This patchset implements the purge script as specified in its
related blueprint:

- The '--age-in-days' option allows to specify the number of
  days before expiry
- The '--max-number' option allows us to specify a limit on the number
  of objects to delete
- The '--audit-template' option allows you to only delete objects
  related to the specified audit template UUID or name
- The '--dry-run' option to go through the purge procedure without
  actually deleting anything
- The '--exclude-orphans' option which allows you to exclude from the
  purge any object that does not have a parent (i.e. and audit without
  a related audit template)

A prompt has been added to also propose to narrow down the number of
deletions to be below the specified limit.

Change-Id: I3ce83ab95277c109df67a6b5b920a878f6e59d3f
Implements: blueprint db-purge-engine
This commit is contained in:
Vincent Françoise
2016-02-29 09:22:48 +01:00
parent c3db66ca09
commit a6508a0013
8 changed files with 1011 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
requirements.txt
View File

@@ -17,6 +17,7 @@ oslo.utils>=3.5.0 # Apache-2.0
PasteDeploy>=1.5.0 # MIT
pbr>=1.6 # Apache-2.0
pecan>=1.0.0 # BSD
PrettyTable>=0.7,<0.8 # BSD
voluptuous>=0.8.6 # BSD License
python-ceilometerclient>=2.2.1 # Apache-2.0
python-cinderclient>=1.3.1 # Apache-2.0

36
watcher/cmd/dbmanage.py
View File

@@ -25,7 +25,7 @@ from oslo_config import cfg
from watcher.common import service
from watcher.db import migration
from watcher.db import purge
CONF = cfg.CONF
@@ -56,6 +56,12 @@ class DBCommand(object):
def create_schema():
migration.create_schema()
@staticmethod
def purge():
purge.purge(CONF.command.age_in_days, CONF.command.max_number,
CONF.command.audit_template, CONF.command.exclude_orphans,
CONF.command.dry_run)
def add_command_parsers(subparsers):
parser = subparsers.add_parser(
@@ -96,6 +102,33 @@ def add_command_parsers(subparsers):
help="Create the database schema.")
parser.set_defaults(func=DBCommand.create_schema)
parser = subparsers.add_parser(
'purge',
help="Purge the database.")
parser.add_argument('-d', '--age-in-days',
help="Number of days since deletion (from today) "
"to exclude from the purge. If None, everything "
"will be purged.",
type=int, default=None, nargs='?')
parser.add_argument('-n', '--max-number',
help="Max number of objects expected to be deleted. "
"Prevents the deletion if exceeded. No limit if "
"set to None.",
type=int, default=None, nargs='?')
parser.add_argument('-t', '--audit-template',
help="UUID or name of the audit template to purge.",
type=str, default=None, nargs='?')
parser.add_argument('-e', '--exclude-orphans', action='store_true',
help="Flag to indicate whether or not you want to "
"exclude orphans from deletion (default: False).",
default=False)
parser.add_argument('--dry-run', action='store_true',
help="Flag to indicate whether or not you want to "
"perform a dry run (no deletion).",
default=False)
parser.set_defaults(func=DBCommand.purge)
command_opt = cfg.SubCommandOpt('command',
title='Command',
@@ -114,6 +147,7 @@ def main():
valid_commands = set([
'upgrade', 'downgrade', 'revision',
'version', 'stamp', 'create_schema',
'purge',
])
if not set(sys.argv).intersection(valid_commands):
sys.argv.append('upgrade')

2
watcher/common/config.py
View File

@@ -22,6 +22,8 @@ from watcher import version
def parse_args(argv, default_config_files=None):
default_config_files = (default_config_files or
cfg.find_config_files(project='watcher'))
rpc.set_defaults(control_exchange='watcher')
cfg.CONF(argv[1:],
project='python-watcher',

8
watcher/common/exception.py
View File

@@ -283,3 +283,11 @@ class LoadingError(WatcherException):
class ReservedWord(WatcherException):
msg_fmt = _("The identifier '%(name)s' is a reserved word")
class NotSoftDeletedStateError(WatcherException):
msg_fmt = _("The %(name)s resource %(id)s is not soft deleted")
class NegativeLimitError(WatcherException):
msg_fmt = _("Limit should be positive")

410
watcher/db/purge.py Normal file
View File

@@ -0,0 +1,410 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2016 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import print_function
import collections
import datetime
import itertools
import sys
from oslo_log import log
from oslo_utils import strutils
import prettytable as ptable
from six.moves import input
from watcher._i18n import _, _LI
from watcher.common import context
from watcher.common import exception
from watcher.common import utils
from watcher import objects
LOG = log.getLogger(__name__)
class WatcherObjectsMap(object):
"""Wrapper to deal with watcher objects per type
This wrapper object contains a list of watcher objects per type.
Its main use is to simplify the merge of watcher objects by avoiding
duplicates, but also for representing the relationships between these
objects.
"""
# This is for generating the .pot translations
keymap = collections.OrderedDict([
("audit_templates", _("Audit Templates")),
("audits", _("Audits")),
("action_plans", _("Action Plans")),
("actions", _("Actions")),
])
def __init__(self):
for attr_name in self.__class__.keys():
setattr(self, attr_name, [])
def values(self):
return (getattr(self, key) for key in self.__class__.keys())
@classmethod
def keys(cls):
return cls.keymap.keys()
def __iter__(self):
return itertools.chain(*self.values())
def __add__(self, other):
new_map = self.__class__()
# Merge the 2 items dicts into a new object (and avoid dupes)
for attr_name, initials, others in zip(self.keys(), self.values(),
other.values()):
# Creates a copy
merged = initials[:]
initials_ids = [item.id for item in initials]
non_dupes = [item for item in others
if item.id not in initials_ids]
merged += non_dupes
setattr(new_map, attr_name, merged)
return new_map
def __str__(self):
out = ""
for key, vals in zip(self.keys(), self.values()):
ids = [val.id for val in vals]
out += "%(key)s: %(val)s" % (dict(key=key, val=ids))
out += "\n"
return out
def __len__(self):
return sum(len(getattr(self, key)) for key in self.keys())
def get_count_table(self):
headers = list(self.keymap.values())
headers.append(_("Total")) # We also add a total count
counters = [len(cat_vals) for cat_vals in self.values()] + [len(self)]
table = ptable.PrettyTable(field_names=headers)
table.add_row(counters)
return table.get_string()
class PurgeCommand(object):
"""Purges the DB by removing soft deleted entries
The workflow for this purge is the following:
# Find soft deleted objects which are expired
# Find orphan objects
# Find their related objects whether they are expired or not
# Merge them together
# If it does not exceed the limit, destroy them all
"""
ctx = context.make_context(show_deleted=True)
def __init__(self, age_in_days=None, max_number=None,
uuid=None, exclude_orphans=False, dry_run=None):
self.age_in_days = age_in_days
self.max_number = max_number
self.uuid = uuid
self.exclude_orphans = exclude_orphans
self.dry_run = dry_run
self._delete_up_to_max = None
self._objects_map = WatcherObjectsMap()
def get_expiry_date(self):
if not self.age_in_days:
return None
today = datetime.datetime.today()
expiry_date = today - datetime.timedelta(days=self.age_in_days)
return expiry_date
@classmethod
def get_audit_template_uuid(cls, uuid_or_name):
if uuid_or_name is None:
return
query_func = None
if not utils.is_uuid_like(uuid_or_name):
query_func = objects.audit_template.AuditTemplate.get_by_name
else:
query_func = objects.audit_template.AuditTemplate.get_by_uuid
try:
audit_template = query_func(cls.ctx, uuid_or_name)
except Exception as exc:
LOG.exception(exc)
raise exception.AuditTemplateNotFound(audit_template=uuid_or_name)
if not audit_template.deleted_at:
raise exception.NotSoftDeletedStateError(
name=_('Audit Template'), id=uuid_or_name)
return audit_template.uuid
def _find_audit_templates(self, filters=None):
return objects.audit_template.AuditTemplate.list(
self.ctx, filters=filters)
def _find_audits(self, filters=None):
return objects.audit.Audit.list(self.ctx, filters=filters)
def _find_action_plans(self, filters=None):
return objects.action_plan.ActionPlan.list(self.ctx, filters=filters)
def _find_actions(self, filters=None):
return objects.action.Action.list(self.ctx, filters=filters)
def _find_orphans(self):
orphans = WatcherObjectsMap()
filters = dict(deleted=False)
audit_templates = objects.audit_template.AuditTemplate.list(
self.ctx, filters=filters)
audits = objects.audit.Audit.list(self.ctx, filters=filters)
action_plans = objects.action_plan.ActionPlan.list(
self.ctx, filters=filters)
actions = objects.action.Action.list(self.ctx, filters=filters)
audit_template_ids = set(at.id for at in audit_templates)
orphans.audits = [
audit for audit in audits
if audit.audit_template_id not in audit_template_ids]
# Objects with orphan parents are themselves orphans
audit_ids = [audit.id for audit in (a for a in audits
if a not in orphans.audits)]
orphans.action_plans = [
ap for ap in action_plans
if ap.audit_id not in audit_ids]
# Objects with orphan parents are themselves orphans
action_plan_ids = [ap.id for ap in (a for a in action_plans
if a not in orphans.action_plans)]
orphans.actions = [
action for action in actions
if action.action_plan_id not in action_plan_ids]
LOG.debug("Orphans found:\n%s", orphans)
LOG.info(_LI("Orphans found:\n%s"), orphans.get_count_table())
return orphans
def _find_soft_deleted_objects(self):
to_be_deleted = WatcherObjectsMap()
expiry_date = self.get_expiry_date()
filters = dict(deleted=True)
if self.uuid:
filters["uuid"] = self.uuid
if expiry_date:
filters.update(dict(deleted_at__lt=expiry_date))
to_be_deleted.audit_templates.extend(
self._find_audit_templates(filters))
to_be_deleted.audits.extend(self._find_audits(filters))
to_be_deleted.action_plans.extend(self._find_action_plans(filters))
to_be_deleted.actions.extend(self._find_actions(filters))
soft_deleted_objs = self._find_related_objects(
to_be_deleted, base_filters=dict(deleted=True))
LOG.debug("Soft deleted objects:\n%s", soft_deleted_objs)
return soft_deleted_objs
def _find_related_objects(self, objects_map, base_filters=None):
base_filters = base_filters or {}
for audit_template in objects_map.audit_templates:
filters = {}
filters.update(base_filters)
filters.update(dict(audit_template_id=audit_template.id))
related_objs = WatcherObjectsMap()
related_objs.audits = self._find_audits(filters)
objects_map += related_objs
for audit in objects_map.audits:
filters = {}
filters.update(base_filters)
filters.update(dict(audit_id=audit.id))
related_objs = WatcherObjectsMap()
related_objs.action_plans = self._find_action_plans(filters)
objects_map += related_objs
for action_plan in objects_map.action_plans:
filters = {}
filters.update(base_filters)
filters.update(dict(action_plan_id=action_plan.id))
related_objs = WatcherObjectsMap()
related_objs.actions = self._find_actions(filters)
objects_map += related_objs
return objects_map
def confirmation_prompt(self):
print(self._objects_map.get_count_table())
raw_val = input(
_("There are %(count)d objects set for deletion. "
"Continue? [y/N]") % dict(count=len(self._objects_map)))
return strutils.bool_from_string(raw_val)
def delete_up_to_max_prompt(self, objects_map):
print(objects_map.get_count_table())
print(_("The number of objects (%(num)s) to delete from the database "
"exceeds the maximum number of objects (%(max_number)s) "
"specified.") % dict(max_number=self.max_number,
num=len(objects_map)))
raw_val = input(
_("Do you want to delete objects up to the specified maximum "
"number? [y/N]"))
self._delete_up_to_max = strutils.bool_from_string(raw_val)
return self._delete_up_to_max
def _aggregate_objects(self):
"""Objects aggregated on a 'per audit template' basis"""
# todo: aggregate orphans as well
aggregate = []
for audit_template in self._objects_map.audit_templates:
related_objs = WatcherObjectsMap()
related_objs.audit_templates = [audit_template]
related_objs.audits = [
audit for audit in self._objects_map.audits
if audit.audit_template_id == audit_template.id
]
audit_ids = [audit.id for audit in related_objs.audits]
related_objs.action_plans = [
action_plan for action_plan in self._objects_map.action_plans
if action_plan.audit_id in audit_ids
]
action_plan_ids = [
action_plan.id for action_plan in related_objs.action_plans
]
related_objs.actions = [
action for action in self._objects_map.actions
if action.action_plan_id in action_plan_ids
]
aggregate.append(related_objs)
return aggregate
def _get_objects_up_to_limit(self):
aggregated_objects = self._aggregate_objects()
to_be_deleted_subset = WatcherObjectsMap()
for aggregate in aggregated_objects:
if len(aggregate) + len(to_be_deleted_subset) <= self.max_number:
to_be_deleted_subset += aggregate
else:
break
LOG.debug(to_be_deleted_subset)
return to_be_deleted_subset
def find_objects_to_delete(self):
"""Finds all the objects to be purged
:returns: A mapping with all the Watcher objects to purged
:rtype: :py:class:`~.WatcherObjectsMap` instance
"""
to_be_deleted = self._find_soft_deleted_objects()
if not self.exclude_orphans:
to_be_deleted += self._find_orphans()
LOG.debug("Objects to be deleted:\n%s", to_be_deleted)
return to_be_deleted
def do_delete(self):
LOG.info(_LI("Deleting..."))
# Reversed to avoid errors with foreign keys
for entry in reversed(list(self._objects_map)):
entry.destroy()
def execute(self):
LOG.info(_LI("Starting purge command"))
self._objects_map = self.find_objects_to_delete()
if (self.max_number is not None and
len(self._objects_map) > self.max_number):
if self.delete_up_to_max_prompt(self._objects_map):
self._objects_map = self._get_objects_up_to_limit()
else:
return
_orphans_note = (_(" (orphans excluded)") if self.exclude_orphans
else _(" (may include orphans)"))
if not self.dry_run and self.confirmation_prompt():
self.do_delete()
print(_("Purge results summary%s:") % _orphans_note)
LOG.info(_LI("Purge results summary%s:"), _orphans_note)
else:
LOG.debug(self._objects_map)
print(_("Here below is a table containing the objects "
"that can be purged%s:") % _orphans_note)
LOG.info("\n%s", self._objects_map.get_count_table())
print(self._objects_map.get_count_table())
LOG.info(_LI("Purge process completed"))
def purge(age_in_days, max_number, audit_template, exclude_orphans, dry_run):
"""Removes soft deleted objects from the database
:param age_in_days: Number of days since deletion (from today)
to exclude from the purge. If None, everything will be purged.
:type age_in_days: int
:param max_number: Max number of objects expected to be deleted.
Prevents the deletion if exceeded. No limit if set to None.
:type max_number: int
:param audit_template: UUID or name of the audit template to purge.
:type audit_template: str
:param exclude_orphans: Flag to indicate whether or not you want to
exclude orphans from deletion (default: False).
:type exclude_orphans: bool
:param dry_run: Flag to indicate whether or not you want to perform
a dry run (no deletion).
:type dry_run: bool
"""
try:
if max_number and max_number < 0:
raise exception.NegativeLimitError
LOG.info("[options] age_in_days = %s", age_in_days)
LOG.info("[options] max_number = %s", max_number)
LOG.info("[options] audit_template = %s", audit_template)
LOG.info("[options] exclude_orphans = %s", exclude_orphans)
LOG.info("[options] dry_run = %s", dry_run)
uuid = PurgeCommand.get_audit_template_uuid(audit_template)
cmd = PurgeCommand(age_in_days, max_number, uuid,
exclude_orphans, dry_run)
cmd.execute()
except Exception as exc:
LOG.exception(exc)
print(exc)
sys.exit(1)

100
watcher/locale/watcher.pot
View File

@@ -7,9 +7,9 @@
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: python-watcher 0.24.1.dev1\n"
"Project-Id-Version: python-watcher 0.24.1.dev4\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2016-03-09 11:18+0100\n"
"POT-Creation-Date: 2016-03-14 15:29+0100\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
@@ -344,6 +344,15 @@ msgstr ""
msgid "The identifier '%(name)s' is a reserved word"
msgstr ""
#: watcher/common/exception.py:289
#, python-format
msgid "The %(name)s resource %(id)s is not soft deleted"
msgstr ""
#: watcher/common/exception.py:293
msgid "Limit should be positive"
msgstr ""
#: watcher/common/service.py:83
#, python-format
msgid "Created RPC server for service %(service)s on host %(host)s."
@@ -386,25 +395,102 @@ msgstr ""
msgid "Messaging configuration error"
msgstr ""
#: watcher/db/sqlalchemy/api.py:256
#: watcher/db/purge.py:50
msgid "Audit Templates"
msgstr ""
#: watcher/db/purge.py:51
msgid "Audits"
msgstr ""
#: watcher/db/purge.py:52
msgid "Action Plans"
msgstr ""
#: watcher/db/purge.py:53
msgid "Actions"
msgstr ""
#: watcher/db/purge.py:100
msgid "Total"
msgstr ""
#: watcher/db/purge.py:158
msgid "Audit Template"
msgstr ""
#: watcher/db/purge.py:206
#, python-format
msgid ""
"Orphans found:\n"
"%s"
msgstr ""
#: watcher/db/purge.py:265
#, python-format
msgid "There are %(count)d objects set for deletion. Continue? [y/N]"
msgstr ""
#: watcher/db/purge.py:272
#, python-format
msgid ""
"The number of objects (%(num)s) to delete from the database exceeds the "
"maximum number of objects (%(max_number)s) specified."
msgstr ""
#: watcher/db/purge.py:277
msgid "Do you want to delete objects up to the specified maximum number? [y/N]"
msgstr ""
#: watcher/db/purge.py:340
msgid "Deleting..."
msgstr ""
#: watcher/db/purge.py:346
msgid "Starting purge command"
msgstr ""
#: watcher/db/purge.py:356
msgid " (orphans excluded)"
msgstr ""
#: watcher/db/purge.py:357
msgid " (may include orphans)"
msgstr ""
#: watcher/db/purge.py:360 watcher/db/purge.py:361
#, python-format
msgid "Purge results summary%s:"
msgstr ""
#: watcher/db/purge.py:364
#, python-format
msgid "Here below is a table containing the objects that can be purged%s:"
msgstr ""
#: watcher/db/purge.py:369
msgid "Purge process completed"
msgstr ""
#: watcher/db/sqlalchemy/api.py:362
msgid ""
"Multiple audit templates exist with the same name. Please use the audit "
"template uuid instead"
msgstr ""
#: watcher/db/sqlalchemy/api.py:278
#: watcher/db/sqlalchemy/api.py:384
msgid "Cannot overwrite UUID for an existing Audit Template."
msgstr ""
#: watcher/db/sqlalchemy/api.py:388
#: watcher/db/sqlalchemy/api.py:495
msgid "Cannot overwrite UUID for an existing Audit."
msgstr ""
#: watcher/db/sqlalchemy/api.py:480
#: watcher/db/sqlalchemy/api.py:588
msgid "Cannot overwrite UUID for an existing Action."
msgstr ""
#: watcher/db/sqlalchemy/api.py:590
#: watcher/db/sqlalchemy/api.py:699
msgid "Cannot overwrite UUID for an existing Action Plan."
msgstr ""

105
watcher/tests/cmd/test_db_manage.py
View File

@@ -1,5 +1,5 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
# Copyright (c) 2016 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -14,15 +14,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from mock import Mock
from mock import patch
import sys
import mock
from oslo_config import cfg
from watcher.cmd import dbmanage
from watcher.db import migration
from watcher.tests.base import TestCase
from watcher.db import purge
from watcher.tests import base
class TestDBManageRunApp(TestCase):
class TestDBManageRunApp(base.TestCase):
scenarios = (
("upgrade", {"command": "upgrade", "expected": "upgrade"}),
@@ -32,15 +35,16 @@ class TestDBManageRunApp(TestCase):
("version", {"command": "version", "expected": "version"}),
("create_schema", {"command": "create_schema",
"expected": "create_schema"}),
("purge", {"command": "purge", "expected": "purge"}),
("no_param", {"command": None, "expected": "upgrade"}),
)
@patch.object(dbmanage, "register_sub_command_opts", Mock())
@patch("watcher.cmd.dbmanage.service.prepare_service")
@patch("watcher.cmd.dbmanage.sys")
@mock.patch.object(dbmanage, "register_sub_command_opts", mock.Mock())
@mock.patch("watcher.cmd.dbmanage.service.prepare_service")
@mock.patch("watcher.cmd.dbmanage.sys")
def test_run_db_manage_app(self, m_sys, m_prepare_service):
# Patch command function
m_func = Mock()
m_func = mock.Mock()
cfg.CONF.register_opt(cfg.SubCommandOpt("command"))
cfg.CONF.command.func = m_func
@@ -53,9 +57,9 @@ class TestDBManageRunApp(TestCase):
["watcher-db-manage", self.expected])
class TestDBManageRunCommand(TestCase):
class TestDBManageRunCommand(base.TestCase):
@patch.object(migration, "upgrade")
@mock.patch.object(migration, "upgrade")
def test_run_db_upgrade(self, m_upgrade):
cfg.CONF.register_opt(cfg.StrOpt("revision"), group="command")
cfg.CONF.set_default("revision", "dummy", group="command")
@@ -63,7 +67,7 @@ class TestDBManageRunCommand(TestCase):
m_upgrade.assert_called_once_with("dummy")
@patch.object(migration, "downgrade")
@mock.patch.object(migration, "downgrade")
def test_run_db_downgrade(self, m_downgrade):
cfg.CONF.register_opt(cfg.StrOpt("revision"), group="command")
cfg.CONF.set_default("revision", "dummy", group="command")
@@ -71,7 +75,7 @@ class TestDBManageRunCommand(TestCase):
m_downgrade.assert_called_once_with("dummy")
@patch.object(migration, "revision")
@mock.patch.object(migration, "revision")
def test_run_db_revision(self, m_revision):
cfg.CONF.register_opt(cfg.StrOpt("message"), group="command")
cfg.CONF.register_opt(cfg.StrOpt("autogenerate"), group="command")
@@ -87,14 +91,85 @@ class TestDBManageRunCommand(TestCase):
"dummy_message", "dummy_autogenerate"
)
@patch.object(migration, "stamp")
@mock.patch.object(migration, "stamp")
def test_run_db_stamp(self, m_stamp):
cfg.CONF.register_opt(cfg.StrOpt("revision"), group="command")
cfg.CONF.set_default("revision", "dummy", group="command")
dbmanage.DBCommand.stamp()
@patch.object(migration, "version")
@mock.patch.object(migration, "version")
def test_run_db_version(self, m_version):
dbmanage.DBCommand.version()
self.assertEqual(1, m_version.call_count)
@mock.patch.object(purge, "PurgeCommand")
def test_run_db_purge(self, m_purge_cls):
m_purge = mock.Mock()
m_purge_cls.return_value = m_purge
m_purge_cls.get_audit_template_uuid.return_value = 'Some UUID'
cfg.CONF.register_opt(cfg.IntOpt("age_in_days"), group="command")
cfg.CONF.register_opt(cfg.IntOpt("max_number"), group="command")
cfg.CONF.register_opt(cfg.StrOpt("audit_template"), group="command")
cfg.CONF.register_opt(cfg.BoolOpt("exclude_orphans"), group="command")
cfg.CONF.register_opt(cfg.BoolOpt("dry_run"), group="command")
cfg.CONF.set_default("age_in_days", None, group="command")
cfg.CONF.set_default("max_number", None, group="command")
cfg.CONF.set_default("audit_template", None, group="command")
cfg.CONF.set_default("exclude_orphans", True, group="command")
cfg.CONF.set_default("dry_run", False, group="command")
dbmanage.DBCommand.purge()
m_purge_cls.assert_called_once_with(
None, None, 'Some UUID', True, False)
m_purge.execute.assert_called_once_with()
@mock.patch.object(sys, "exit")
@mock.patch.object(purge, "PurgeCommand")
def test_run_db_purge_negative_max_number(self, m_purge_cls, m_exit):
m_purge = mock.Mock()
m_purge_cls.return_value = m_purge
m_purge_cls.get_audit_template_uuid.return_value = 'Some UUID'
cfg.CONF.register_opt(cfg.IntOpt("age_in_days"), group="command")
cfg.CONF.register_opt(cfg.IntOpt("max_number"), group="command")
cfg.CONF.register_opt(cfg.StrOpt("audit_template"), group="command")
cfg.CONF.register_opt(cfg.BoolOpt("exclude_orphans"), group="command")
cfg.CONF.register_opt(cfg.BoolOpt("dry_run"), group="command")
cfg.CONF.set_default("age_in_days", None, group="command")
cfg.CONF.set_default("max_number", -1, group="command")
cfg.CONF.set_default("audit_template", None, group="command")
cfg.CONF.set_default("exclude_orphans", True, group="command")
cfg.CONF.set_default("dry_run", False, group="command")
dbmanage.DBCommand.purge()
self.assertEqual(0, m_purge_cls.call_count)
self.assertEqual(0, m_purge.execute.call_count)
self.assertEqual(0, m_purge.do_delete.call_count)
self.assertEqual(1, m_exit.call_count)
@mock.patch.object(sys, "exit")
@mock.patch.object(purge, "PurgeCommand")
def test_run_db_purge_dry_run(self, m_purge_cls, m_exit):
m_purge = mock.Mock()
m_purge_cls.return_value = m_purge
m_purge_cls.get_audit_template_uuid.return_value = 'Some UUID'
cfg.CONF.register_opt(cfg.IntOpt("age_in_days"), group="command")
cfg.CONF.register_opt(cfg.IntOpt("max_number"), group="command")
cfg.CONF.register_opt(cfg.StrOpt("audit_template"), group="command")
cfg.CONF.register_opt(cfg.BoolOpt("exclude_orphans"), group="command")
cfg.CONF.register_opt(cfg.BoolOpt("dry_run"), group="command")
cfg.CONF.set_default("age_in_days", None, group="command")
cfg.CONF.set_default("max_number", None, group="command")
cfg.CONF.set_default("audit_template", None, group="command")
cfg.CONF.set_default("exclude_orphans", True, group="command")
cfg.CONF.set_default("dry_run", True, group="command")
dbmanage.DBCommand.purge()
m_purge_cls.assert_called_once_with(
None, None, 'Some UUID', True, True)
self.assertEqual(1, m_purge.execute.call_count)
self.assertEqual(0, m_purge.do_delete.call_count)
self.assertEqual(0, m_exit.call_count)

372
watcher/tests/db/test_purge.py Normal file
View File

@@ -0,0 +1,372 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2016 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import uuid
import freezegun
import mock
from watcher.common import context as watcher_context
from watcher.db import purge
from watcher.db.sqlalchemy import api as dbapi
from watcher.tests.db import base
from watcher.tests.objects import utils as obj_utils
class TestPurgeCommand(base.DbTestCase):
def setUp(self):
super(TestPurgeCommand, self).setUp()
self.cmd = purge.PurgeCommand()
token_info = {
'token': {
'project': {
'id': 'fake_project'
},
'user': {
'id': 'fake_user'
}
}
}
self.context = watcher_context.RequestContext(
auth_token_info=token_info,
project_id='fake_project',
user_id='fake_user',
show_deleted=True,
)
self.fake_today = '2016-02-24T09:52:05.219414+00:00'
self.expired_date = '2016-01-24T09:52:05.219414+00:00'
self.m_input = mock.Mock()
p = mock.patch("watcher.db.purge.input", self.m_input)
self.m_input.return_value = 'y'
p.start()
self.addCleanup(p.stop)
self._id_generator = None
self._data_setup()
def _generate_id(self):
if self._id_generator is None:
self._id_generator = self._get_id_generator()
return next(self._id_generator)
def _get_id_generator(self):
seed = 1
while True:
yield seed
seed += 1
def _data_setup(self):
# All the 1's are soft_deleted and are expired
# All the 2's are soft_deleted but are not expired
# All the 3's are *not* soft_deleted
# Number of days we want to keep in DB (no purge for them)
self.cmd.age_in_days = 10
self.cmd.max_number = None
self.cmd.orphans = True
gen_name = lambda: "Audit Template %s" % uuid.uuid4()
self.audit_template1_name = gen_name()
self.audit_template2_name = gen_name()
self.audit_template3_name = gen_name()
with freezegun.freeze_time(self.expired_date):
self.audit_template1 = obj_utils.create_test_audit_template(
self.context, name=self.audit_template1_name,
id=self._generate_id(), uuid=None)
self.audit_template2 = obj_utils.create_test_audit_template(
self.context, name=self.audit_template2_name,
id=self._generate_id(), uuid=None)
self.audit_template3 = obj_utils.create_test_audit_template(
self.context, name=self.audit_template3_name,
id=self._generate_id(), uuid=None)
self.audit_template1.soft_delete()
with freezegun.freeze_time(self.expired_date):
self.audit1 = obj_utils.create_test_audit(
self.context, audit_template_id=self.audit_template1.id,
id=self._generate_id(), uuid=None)
self.audit2 = obj_utils.create_test_audit(
self.context, audit_template_id=self.audit_template2.id,
id=self._generate_id(), uuid=None)
self.audit3 = obj_utils.create_test_audit(
self.context, audit_template_id=self.audit_template3.id,
id=self._generate_id(), uuid=None)
self.audit1.soft_delete()
with freezegun.freeze_time(self.expired_date):
self.action_plan1 = obj_utils.create_test_action_plan(
self.context, audit_id=self.audit1.id,
id=self._generate_id(), uuid=None)
self.action_plan2 = obj_utils.create_test_action_plan(
self.context, audit_id=self.audit2.id,
id=self._generate_id(), uuid=None)
self.action_plan3 = obj_utils.create_test_action_plan(
self.context, audit_id=self.audit3.id,
id=self._generate_id(), uuid=None)
self.action1 = obj_utils.create_test_action(
self.context, action_plan_id=self.action_plan1.id,
id=self._generate_id(), uuid=None)
self.action2 = obj_utils.create_test_action(
self.context, action_plan_id=self.action_plan2.id,
id=self._generate_id(), uuid=None)
self.action3 = obj_utils.create_test_action(
self.context, action_plan_id=self.action_plan3.id,
id=self._generate_id(), uuid=None)
self.action_plan1.soft_delete()
@mock.patch.object(dbapi.Connection, "destroy_action")
@mock.patch.object(dbapi.Connection, "destroy_action_plan")
@mock.patch.object(dbapi.Connection, "destroy_audit")
@mock.patch.object(dbapi.Connection, "destroy_audit_template")
def test_execute_max_number_exceeded(self, m_destroy_audit_template,
m_destroy_audit,
m_destroy_action_plan,
m_destroy_action):
self.cmd.age_in_days = None
self.cmd.max_number = 5
with freezegun.freeze_time(self.fake_today):
self.audit_template2.soft_delete()
self.audit2.soft_delete()
self.action_plan2.soft_delete()
with freezegun.freeze_time(self.fake_today):
self.cmd.execute()
# The 1's and the 2's are purgeable (due to age of day set to 0),
# but max_number = 5, and because of no Db integrity violation, we
# should be able to purge only 4 objects.
self.assertEqual(m_destroy_audit_template.call_count, 1)
self.assertEqual(m_destroy_audit.call_count, 1)
self.assertEqual(m_destroy_action_plan.call_count, 1)
self.assertEqual(m_destroy_action.call_count, 1)
def test_find_deleted_entries(self):
self.cmd.age_in_days = None
with freezegun.freeze_time(self.fake_today):
objects_map = self.cmd.find_objects_to_delete()
self.assertEqual(len(objects_map.audit_templates), 1)
self.assertEqual(len(objects_map.audits), 1)
self.assertEqual(len(objects_map.action_plans), 1)
self.assertEqual(len(objects_map.actions), 1)
def test_find_deleted_and_expired_entries(self):
with freezegun.freeze_time(self.fake_today):
self.audit_template2.soft_delete()
self.audit2.soft_delete()
self.action_plan2.soft_delete()
with freezegun.freeze_time(self.fake_today):
objects_map = self.cmd.find_objects_to_delete()
# The 1's are purgeable (due to age of day set to 10)
self.assertEqual(len(objects_map.audit_templates), 1)
self.assertEqual(len(objects_map.audits), 1)
self.assertEqual(len(objects_map.action_plans), 1)
self.assertEqual(len(objects_map.actions), 1)
def test_find_deleted_and_nonexpired_related_entries(self):
with freezegun.freeze_time(self.fake_today):
# orphan audit
audit4 = obj_utils.create_test_audit(
self.context, audit_template_id=404, # Does not exist
id=self._generate_id(), uuid=None)
action_plan4 = obj_utils.create_test_action_plan(
self.context, audit_id=audit4.id,
id=self._generate_id(), uuid=None)
action4 = obj_utils.create_test_action(
self.context, action_plan_id=action_plan4.id,
id=self._generate_id(), uuid=None)
audit5 = obj_utils.create_test_audit(
self.context, audit_template_id=self.audit_template1.id,
id=self._generate_id(), uuid=None)
action_plan5 = obj_utils.create_test_action_plan(
self.context, audit_id=audit5.id,
id=self._generate_id(), uuid=None)
action5 = obj_utils.create_test_action(
self.context, action_plan_id=action_plan5.id,
id=self._generate_id(), uuid=None)
self.audit_template2.soft_delete()
self.audit2.soft_delete()
self.action_plan2.soft_delete()
# All the 4's should be purged as well because they are orphans
# even though they were not deleted
# All the 5's should be purged as well even though they are not
# expired because their related audit template is itself expired
audit5.soft_delete()
action_plan5.soft_delete()
with freezegun.freeze_time(self.fake_today):
objects_map = self.cmd.find_objects_to_delete()
self.assertEqual(len(objects_map.audit_templates), 1)
self.assertEqual(len(objects_map.audits), 3)
self.assertEqual(len(objects_map.action_plans), 3)
self.assertEqual(len(objects_map.actions), 3)
self.assertEqual(
set([self.action1.id, action4.id, action5.id]),
set([entry.id for entry in objects_map.actions]))
@mock.patch.object(dbapi.Connection, "destroy_action")
@mock.patch.object(dbapi.Connection, "destroy_action_plan")
@mock.patch.object(dbapi.Connection, "destroy_audit")
@mock.patch.object(dbapi.Connection, "destroy_audit_template")
def test_purge_command(self, m_destroy_audit_template,
m_destroy_audit, m_destroy_action_plan,
m_destroy_action):
with freezegun.freeze_time(self.fake_today):
self.cmd.execute()
m_destroy_audit_template.assert_called_once_with(
self.audit_template1.uuid)
m_destroy_audit.assert_called_once_with(
self.audit1.uuid)
m_destroy_action_plan.assert_called_once_with(
self.action_plan1.uuid)
m_destroy_action.assert_called_once_with(
self.action1.uuid)
@mock.patch.object(dbapi.Connection, "destroy_action")
@mock.patch.object(dbapi.Connection, "destroy_action_plan")
@mock.patch.object(dbapi.Connection, "destroy_audit")
@mock.patch.object(dbapi.Connection, "destroy_audit_template")
def test_purge_command_with_nonexpired_related_entries(
self, m_destroy_audit_template, m_destroy_audit,
m_destroy_action_plan, m_destroy_action):
with freezegun.freeze_time(self.fake_today):
# orphan audit
audit4 = obj_utils.create_test_audit(
self.context, audit_template_id=404, # Does not exist
id=self._generate_id(), uuid=None)
action_plan4 = obj_utils.create_test_action_plan(
self.context, audit_id=audit4.id,
id=self._generate_id(), uuid=None)
action4 = obj_utils.create_test_action(
self.context, action_plan_id=action_plan4.id,
id=self._generate_id(), uuid=None)
audit5 = obj_utils.create_test_audit(
self.context, audit_template_id=self.audit_template1.id,
id=self._generate_id(), uuid=None)
action_plan5 = obj_utils.create_test_action_plan(
self.context, audit_id=audit5.id,
id=self._generate_id(), uuid=None)
action5 = obj_utils.create_test_action(
self.context, action_plan_id=action_plan5.id,
id=self._generate_id(), uuid=None)
self.audit_template2.soft_delete()
self.audit2.soft_delete()
self.action_plan2.soft_delete()
# All the 4's should be purged as well because they are orphans
# even though they were not deleted
# All the 5's should be purged as well even though they are not
# expired because their related audit template is itself expired
audit5.soft_delete()
action_plan5.soft_delete()
with freezegun.freeze_time(self.fake_today):
self.cmd.execute()
self.assertEqual(m_destroy_audit_template.call_count, 1)
self.assertEqual(m_destroy_audit.call_count, 3)
self.assertEqual(m_destroy_action_plan.call_count, 3)
self.assertEqual(m_destroy_action.call_count, 3)
m_destroy_audit_template.assert_any_call(self.audit_template1.uuid)
m_destroy_audit.assert_any_call(self.audit1.uuid)
m_destroy_audit.assert_any_call(audit4.uuid)
m_destroy_action_plan.assert_any_call(self.action_plan1.uuid)
m_destroy_action_plan.assert_any_call(action_plan4.uuid)
m_destroy_action_plan.assert_any_call(action_plan5.uuid)
m_destroy_action.assert_any_call(self.action1.uuid)
m_destroy_action.assert_any_call(action4.uuid)
m_destroy_action.assert_any_call(action5.uuid)
@mock.patch.object(dbapi.Connection, "destroy_action")
@mock.patch.object(dbapi.Connection, "destroy_action_plan")
@mock.patch.object(dbapi.Connection, "destroy_audit")
@mock.patch.object(dbapi.Connection, "destroy_audit_template")
def test_purge_command_with_audit_template_ok(
self, m_destroy_audit_template, m_destroy_audit,
m_destroy_action_plan, m_destroy_action):
self.cmd.orphans = False
self.cmd.uuid = self.audit_template1.uuid
with freezegun.freeze_time(self.fake_today):
self.cmd.execute()
self.assertEqual(m_destroy_audit_template.call_count, 1)
self.assertEqual(m_destroy_audit.call_count, 1)
self.assertEqual(m_destroy_action_plan.call_count, 1)
self.assertEqual(m_destroy_action.call_count, 1)
m_destroy_audit_template.assert_called_once_with(
self.audit_template1.uuid)
m_destroy_audit.assert_called_once_with(
self.audit1.uuid)
m_destroy_action_plan.assert_called_once_with(
self.action_plan1.uuid)
m_destroy_action.assert_called_once_with(
self.action1.uuid)
@mock.patch.object(dbapi.Connection, "destroy_action")
@mock.patch.object(dbapi.Connection, "destroy_action_plan")
@mock.patch.object(dbapi.Connection, "destroy_audit")
@mock.patch.object(dbapi.Connection, "destroy_audit_template")
def test_purge_command_with_audit_template_not_expired(
self, m_destroy_audit_template, m_destroy_audit,
m_destroy_action_plan, m_destroy_action):
self.cmd.orphans = False
self.cmd.uuid = self.audit_template2.uuid
with freezegun.freeze_time(self.fake_today):
self.cmd.execute()
self.assertEqual(m_destroy_audit_template.call_count, 0)
self.assertEqual(m_destroy_audit.call_count, 0)
self.assertEqual(m_destroy_action_plan.call_count, 0)
self.assertEqual(m_destroy_action.call_count, 0)
@mock.patch.object(dbapi.Connection, "destroy_action")
@mock.patch.object(dbapi.Connection, "destroy_action_plan")
@mock.patch.object(dbapi.Connection, "destroy_audit")
@mock.patch.object(dbapi.Connection, "destroy_audit_template")
def test_purge_command_with_audit_template_not_soft_deleted(
self, m_destroy_audit_template, m_destroy_audit,
m_destroy_action_plan, m_destroy_action):
self.cmd.orphans = False
self.cmd.uuid = self.audit_template3.uuid
with freezegun.freeze_time(self.fake_today):
self.cmd.execute()
self.assertEqual(m_destroy_audit_template.call_count, 0)
self.assertEqual(m_destroy_audit.call_count, 0)
self.assertEqual(m_destroy_action_plan.call_count, 0)
self.assertEqual(m_destroy_action.call_count, 0)