Arnike/watcher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add continuously optimization

Browse Source 
This patch set adds implementation for CONTINUOUS type
of audit.

Change-Id: I5f4ec97b2082c8a6b3ccebe36b2a343fa4a67d19
Implements: blueprint continuously-optimization
This commit is contained in:
Alexander Chadin
2016-05-16 20:16:07 +03:00
parent 518b4c82f1
commit 1de00086f5
18 changed files with 516 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
watcher/tests/api/v1/test_audits.py
View File

@@ -477,6 +477,7 @@ class TestPost(api_base.FunctionalTest):
audit_dict = post_get_test_audit(state=objects.audit.State.PENDING)
del audit_dict['uuid']
del audit_dict['state']
del audit_dict['interval']
response = self.post_json('/audits', audit_dict)
self.assertEqual('application/json', response.content_type)
@@ -517,6 +518,7 @@ class TestPost(api_base.FunctionalTest):
audit_dict = post_get_test_audit()
del audit_dict['uuid']
del audit_dict['state']
del audit_dict['interval']
# Make the audit template UUID some garbage value
audit_dict['audit_template_uuid'] = (
'01234567-8910-1112-1314-151617181920')
@@ -537,6 +539,7 @@ class TestPost(api_base.FunctionalTest):
state = audit_dict['state']
del audit_dict['uuid']
del audit_dict['state']
del audit_dict['interval']
with mock.patch.object(self.dbapi, 'create_audit',
wraps=self.dbapi.create_audit) as cn_mock:
response = self.post_json('/audits', audit_dict)
@@ -552,6 +555,7 @@ class TestPost(api_base.FunctionalTest):
audit_dict = post_get_test_audit()
del audit_dict['uuid']
del audit_dict['state']
del audit_dict['interval']
response = self.post_json('/audits', audit_dict)
self.assertEqual('application/json', response.content_type)
@@ -560,12 +564,66 @@ class TestPost(api_base.FunctionalTest):
response.json['state'])
self.assertTrue(utils.is_uuid_like(response.json['uuid']))
@mock.patch.object(deapi.DecisionEngineAPI, 'trigger_audit')
def test_create_continuous_audit_with_period(self, mock_trigger_audit):
mock_trigger_audit.return_value = mock.ANY
audit_dict = post_get_test_audit()
del audit_dict['uuid']
del audit_dict['state']
audit_dict['audit_type'] = objects.audit.AuditType.CONTINUOUS.value
audit_dict['interval'] = 1200
response = self.post_json('/audits', audit_dict)
self.assertEqual('application/json', response.content_type)
self.assertEqual(201, response.status_int)
self.assertEqual(objects.audit.State.PENDING,
response.json['state'])
self.assertEqual(audit_dict['interval'], response.json['interval'])
self.assertTrue(utils.is_uuid_like(response.json['uuid']))
@mock.patch.object(deapi.DecisionEngineAPI, 'trigger_audit')
def test_create_continuous_audit_without_period(self, mock_trigger_audit):
mock_trigger_audit.return_value = mock.ANY
audit_dict = post_get_test_audit()
del audit_dict['uuid']
del audit_dict['state']
audit_dict['audit_type'] = objects.audit.AuditType.CONTINUOUS.value
del audit_dict['interval']
response = self.post_json('/audits', audit_dict, expect_errors=True)
self.assertEqual(400, response.status_int)
self.assertEqual('application/json', response.content_type)
expected_error_msg = ('Interval of audit must be specified '
'for CONTINUOUS.')
self.assertTrue(response.json['error_message'])
self.assertTrue(expected_error_msg in response.json['error_message'])
@mock.patch.object(deapi.DecisionEngineAPI, 'trigger_audit')
def test_create_oneshot_audit_with_period(self, mock_trigger_audit):
mock_trigger_audit.return_value = mock.ANY
audit_dict = post_get_test_audit()
del audit_dict['uuid']
del audit_dict['state']
audit_dict['audit_type'] = objects.audit.AuditType.ONESHOT.value
audit_dict['interval'] = 1200
response = self.post_json('/audits', audit_dict, expect_errors=True)
self.assertEqual(400, response.status_int)
self.assertEqual('application/json', response.content_type)
expected_error_msg = 'Interval of audit must not be set for ONESHOT.'
self.assertTrue(response.json['error_message'])
self.assertTrue(expected_error_msg in response.json['error_message'])
def test_create_audit_trigger_decision_engine(self):
with mock.patch.object(deapi.DecisionEngineAPI,
'trigger_audit') as de_mock:
audit_dict = post_get_test_audit(state=objects.audit.State.PENDING)
del audit_dict['uuid']
del audit_dict['state']
del audit_dict['interval']
response = self.post_json('/audits', audit_dict)
de_mock.assert_called_once_with(mock.ANY, response.json['uuid'])
@@ -586,6 +644,7 @@ class TestPost(api_base.FunctionalTest):
audit_dict = post_get_test_audit(parameters={'name': 'Tom'})
del audit_dict['uuid']
del audit_dict['state']
del audit_dict['interval']
response = self.post_json('/audits', audit_dict, expect_errors=True)
self.assertEqual('application/json', response.content_type)
@@ -605,6 +664,7 @@ class TestPost(api_base.FunctionalTest):
parameters={'name': 'Tom'})
del audit_dict['uuid']
del audit_dict['state']
del audit_dict['interval']
response = self.post_json('/audits', audit_dict, expect_errors=True)
self.assertEqual('application/json', response.content_type)

1
watcher/tests/db/utils.py
View File

@@ -62,6 +62,7 @@ def get_test_audit(**kwargs):
'updated_at': kwargs.get('updated_at'),
'deleted_at': kwargs.get('deleted_at'),
'parameters': kwargs.get('parameters', {}),
'interval': kwargs.get('period', 3600),
}

139
watcher/tests/decision_engine/audit/test_audit_handlers.py Normal file
View File

@@ -0,0 +1,139 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import uuid
from apscheduler.schedulers import background
from watcher.decision_engine.audit import continuous
from watcher.decision_engine.audit import oneshot
from watcher.decision_engine.messaging import events
from watcher.metrics_engine.cluster_model_collector import manager
from watcher.objects import audit as audit_objects
from watcher.tests.db import base
from watcher.tests.decision_engine.strategy.strategies import \
faker_cluster_state as faker
from watcher.tests.objects import utils as obj_utils
class TestOneShotAuditHandler(base.DbTestCase):
def setUp(self):
super(TestOneShotAuditHandler, self).setUp()
obj_utils.create_test_goal(self.context, id=1, name="dummy")
audit_template = obj_utils.create_test_audit_template(
self.context)
self.audit = obj_utils.create_test_audit(
self.context,
audit_template_id=audit_template.id)
@mock.patch.object(manager.CollectorManager, "get_cluster_model_collector")
def test_trigger_audit_without_errors(self, mock_collector):
mock_collector.return_value = faker.FakerModelCollector()
audit_handler = oneshot.OneShotAuditHandler(mock.MagicMock())
audit_handler.execute(self.audit, self.context)
@mock.patch.object(manager.CollectorManager, "get_cluster_model_collector")
def test_trigger_audit_state_succeeded(self, mock_collector):
mock_collector.return_value = faker.FakerModelCollector()
audit_handler = oneshot.OneShotAuditHandler(mock.MagicMock())
audit_handler.execute(self.audit, self.context)
audit = audit_objects.Audit.get_by_uuid(self.context, self.audit.uuid)
self.assertEqual(audit_objects.State.SUCCEEDED, audit.state)
@mock.patch.object(manager.CollectorManager, "get_cluster_model_collector")
def test_trigger_audit_send_notification(self, mock_collector):
messaging = mock.MagicMock()
mock_collector.return_value = faker.FakerModelCollector()
audit_handler = oneshot.OneShotAuditHandler(messaging)
audit_handler.execute(self.audit, self.context)
call_on_going = mock.call(events.Events.TRIGGER_AUDIT.name, {
'audit_status': audit_objects.State.ONGOING,
'audit_uuid': self.audit.uuid})
call_succeeded = mock.call(events.Events.TRIGGER_AUDIT.name, {
'audit_status': audit_objects.State.SUCCEEDED,
'audit_uuid': self.audit.uuid})
calls = [call_on_going, call_succeeded]
messaging.status_topic_handler.publish_event.assert_has_calls(calls)
self.assertEqual(
2, messaging.status_topic_handler.publish_event.call_count)
class TestContinuousAuditHandler(base.DbTestCase):
def setUp(self):
super(TestContinuousAuditHandler, self).setUp()
obj_utils.create_test_goal(self.context, id=1, name="DUMMY")
audit_template = obj_utils.create_test_audit_template(
self.context)
self.audits = [obj_utils.create_test_audit(
self.context,
uuid=uuid.uuid4(),
audit_template_id=audit_template.id,
audit_type=audit_objects.AuditType.CONTINUOUS.value)
for i in range(2)]
@mock.patch.object(background.BackgroundScheduler, 'add_job')
@mock.patch.object(background.BackgroundScheduler, 'get_jobs')
@mock.patch.object(audit_objects.Audit, 'list')
def test_launch_audits_periodically(self, mock_list,
mock_jobs, mock_add_job):
audit_handler = continuous.ContinuousAuditHandler(mock.MagicMock())
audits = [audit_objects.Audit.get_by_uuid(self.context,
self.audits[0].uuid)]
mock_list.return_value = audits
mock_jobs.return_value = mock.MagicMock()
audit_handler.launch_audits_periodically()
mock_add_job.assert_called()
@mock.patch.object(background.BackgroundScheduler, 'add_job')
@mock.patch.object(background.BackgroundScheduler, 'get_jobs')
@mock.patch.object(audit_objects.Audit, 'list')
def test_launch_multiply_audits_periodically(self, mock_list,
mock_jobs, mock_add_job):
audit_handler = continuous.ContinuousAuditHandler(mock.MagicMock())
audits = [audit_objects.Audit.get_by_uuid(
self.context,
audit.uuid) for audit in self.audits]
mock_list.return_value = audits
mock_jobs.return_value = mock.MagicMock()
calls = [mock.call(audit_handler.execute_audit, 'interval',
args=[mock.ANY, mock.ANY],
seconds=3600,
name='execute_audit',
next_run_time=mock.ANY) for audit in self.audits]
audit_handler.launch_audits_periodically()
mock_add_job.assert_has_calls(calls)
@mock.patch.object(background.BackgroundScheduler, 'add_job')
@mock.patch.object(background.BackgroundScheduler, 'get_jobs')
@mock.patch.object(audit_objects.Audit, 'list')
def test_period_audit_not_called_when_deleted(self, mock_list,
mock_jobs, mock_add_job):
audit_handler = continuous.ContinuousAuditHandler(mock.MagicMock())
audits = [audit_objects.Audit.get_by_uuid(
self.context,
audit.uuid) for audit in self.audits]
mock_list.return_value = audits
mock_jobs.return_value = mock.MagicMock()
audits[1].state = audit_objects.State.CANCELLED
calls = [mock.call(audit_handler.execute_audit, 'interval',
args=[mock.ANY, mock.ANY],
seconds=3600,
name='execute_audit',
next_run_time=mock.ANY)]
audit_handler.launch_audits_periodically()
mock_add_job.assert_has_calls(calls)

69
watcher/tests/decision_engine/audit/test_default_audit_handler.py
View File

@@ -1,69 +0,0 @@
# -*- encoding: utf-8 -*-
# Copyright (c) 2015 b<>com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from watcher.decision_engine.audit import default as default
from watcher.decision_engine.messaging import events
from watcher.metrics_engine.cluster_model_collector import manager
from watcher.objects import audit as audit_objects
from watcher.tests.db import base
from watcher.tests.decision_engine.strategy.strategies import \
faker_cluster_state as faker
from watcher.tests.objects import utils as obj_utils
class TestDefaultAuditHandler(base.DbTestCase):
def setUp(self):
super(TestDefaultAuditHandler, self).setUp()
obj_utils.create_test_goal(self.context, id=1, name="dummy")
audit_template = obj_utils.create_test_audit_template(
self.context)
self.audit = obj_utils.create_test_audit(
self.context,
audit_template_id=audit_template.id)
@mock.patch.object(manager.CollectorManager, "get_cluster_model_collector")
def test_trigger_audit_without_errors(self, mock_collector):
mock_collector.return_value = faker.FakerModelCollector()
audit_handler = default.DefaultAuditHandler(mock.MagicMock())
audit_handler.execute(self.audit.uuid, self.context)
@mock.patch.object(manager.CollectorManager, "get_cluster_model_collector")
def test_trigger_audit_state_succeeded(self, mock_collector):
mock_collector.return_value = faker.FakerModelCollector()
audit_handler = default.DefaultAuditHandler(mock.MagicMock())
audit_handler.execute(self.audit.uuid, self.context)
audit = audit_objects.Audit.get_by_uuid(self.context, self.audit.uuid)
self.assertEqual(audit_objects.State.SUCCEEDED, audit.state)
@mock.patch.object(manager.CollectorManager, "get_cluster_model_collector")
def test_trigger_audit_send_notification(self, mock_collector):
messaging = mock.MagicMock()
mock_collector.return_value = faker.FakerModelCollector()
audit_handler = default.DefaultAuditHandler(messaging)
audit_handler.execute(self.audit.uuid, self.context)
call_on_going = mock.call(events.Events.TRIGGER_AUDIT.name, {
'audit_status': audit_objects.State.ONGOING,
'audit_uuid': self.audit.uuid})
call_succeeded = mock.call(events.Events.TRIGGER_AUDIT.name, {
'audit_status': audit_objects.State.SUCCEEDED,
'audit_uuid': self.audit.uuid})
calls = [call_on_going, call_succeeded]
messaging.status_topic_handler.publish_event.assert_has_calls(calls)
self.assertEqual(
2, messaging.status_topic_handler.publish_event.call_count)

28
watcher/tests/decision_engine/messaging/test_audit_endpoint.py
View File

@@ -16,8 +16,7 @@
import mock
from watcher.common import utils
from watcher.decision_engine.audit import default
from watcher.decision_engine.audit import oneshot as oneshot_handler
from watcher.decision_engine.messaging import audit_endpoint
from watcher.metrics_engine.cluster_model_collector import manager
from watcher.tests.db import base
@@ -38,28 +37,29 @@ class TestAuditEndpoint(base.DbTestCase):
@mock.patch.object(manager.CollectorManager, "get_cluster_model_collector")
def test_do_trigger_audit(self, mock_collector):
mock_collector.return_value = faker_cluster_state.FakerModelCollector()
audit_uuid = utils.generate_uuid()
audit_handler = default.DefaultAuditHandler(mock.MagicMock())
audit_handler = oneshot_handler.OneShotAuditHandler(mock.MagicMock())
endpoint = audit_endpoint.AuditEndpoint(audit_handler)
with mock.patch.object(default.DefaultAuditHandler,
with mock.patch.object(oneshot_handler.OneShotAuditHandler,
'execute') as mock_call:
mock_call.return_value = 0
endpoint.do_trigger_audit(audit_handler, audit_uuid)
endpoint.do_trigger_audit(self.context, self.audit.uuid)
mock_call.assert_called_once_with(audit_uuid, audit_handler)
self.assertEqual(mock_call.call_count, 1)
@mock.patch.object(manager.CollectorManager, "get_cluster_model_collector")
def test_trigger_audit(self, mock_collector):
mock_collector.return_value = faker_cluster_state.FakerModelCollector()
audit_uuid = utils.generate_uuid()
audit_handler = default.DefaultAuditHandler(mock.MagicMock())
audit_handler = oneshot_handler.OneShotAuditHandler(mock.MagicMock())
endpoint = audit_endpoint.AuditEndpoint(audit_handler)
with mock.patch.object(default.DefaultAuditHandler, 'execute') \
as mock_call:
mock_call.return_value = 0
endpoint.trigger_audit(audit_handler, audit_uuid)
with mock.patch.object(endpoint.executor, 'submit') as mock_call:
mock_execute = mock.call(endpoint.do_trigger_audit,
self.context,
self.audit.uuid)
endpoint.trigger_audit(self.context, self.audit.uuid)
mock_call.assert_called_once_with(audit_uuid, audit_handler)
mock_call.assert_has_calls([mock_execute])
self.assertEqual(mock_call.call_count, 1)