From 7ea30df9cdd416bdc41762f1bc31c3971d55ae34 Mon Sep 17 00:00:00 2001 From: paul luse Date: Thu, 10 Jul 2014 06:21:56 -0700 Subject: [PATCH] Update docs to highlight that the auditor chunk size can be set May not be obvious, but existing code will let you change the disk_chunk_size just for the auditor so this just points that out in the docs. In one short test I ran with a 4 node cluster with 18GB of 4MB objects on it, changint he auditor chunk size from the default of 64K to 1MB creased the auditor CPU time from 10% to 4%. Also added test code to make sure this overridden value is actually used and checked other auditWorker conf values as well. Change-Id: Ia12e1c6127877dc2124b60cd963cd0b6d5f3d6ef --- doc/source/deployment_guide.rst | 7 ++++--- etc/object-server.conf-sample | 3 +++ test/unit/obj/test_auditor.py | 27 +++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index 07874cd286..ed6843dc2e 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -146,7 +146,7 @@ General Service Configuration ----------------------------- Most Swift services fall into two categories. Swift's wsgi servers and -background daemons. +background daemons. For more information specific to the configuration of Swift's wsgi servers with paste deploy see :ref:`general-server-configuration` @@ -177,7 +177,7 @@ exist in a single file ``/etc/swift/object-server.conf``:: Swift services expect a configuration path as the first argument:: - $ swift-object-auditor + $ swift-object-auditor Usage: swift-object-auditor CONFIG [options] Error: missing config path argument @@ -185,7 +185,7 @@ Swift services expect a configuration path as the first argument:: If you omit the object-auditor section this file could not be used as the configuration path when starting the ``swift-object-auditor`` daemon:: - $ swift-object-auditor /etc/swift/object-server.conf + $ swift-object-auditor /etc/swift/object-server.conf Unable to find object-auditor config section in /etc/swift/object-server.conf If the configuration path is a directory instead of a file all of the files in @@ -520,6 +520,7 @@ log_name object-auditor Label used when logging log_facility LOG_LOCAL0 Syslog log facility log_level INFO Logging level log_time 3600 Frequency of status logs in seconds. +disk_chunk_size 65536 Size of chunks read during auditing files_per_second 20 Maximum files audited per second per auditor process. Should be tuned according to individual system specs. 0 is unlimited. diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index 6ffc4f33cb..5079e36ba2 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -226,6 +226,9 @@ use = egg:swift#recon # log_level = INFO # log_address = /dev/log # +# You can set the disk chunk size that the auditor uses making it larger if +# you like for more efficient local auditing of larger objects +# disk_chunk_size = 65536 # files_per_second = 20 # concurrency = 1 # bytes_per_second = 10000000 diff --git a/test/unit/obj/test_auditor.py b/test/unit/obj/test_auditor.py index 7d47a0346d..49f9eed6e2 100644 --- a/test/unit/obj/test_auditor.py +++ b/test/unit/obj/test_auditor.py @@ -78,6 +78,33 @@ class TestAuditor(unittest.TestCase): rmtree(os.path.dirname(self.testdir), ignore_errors=1) unit.xattr_data = {} + def test_worker_conf_parms(self): + def check_common_defaults(): + self.assertEquals(auditor_worker.max_bytes_per_second, 10000000) + self.assertEquals(auditor_worker.log_time, 3600) + + # test default values + conf = dict( + devices=self.devices, + mount_check='false', + object_size_stats='10,100,1024,10240') + auditor_worker = auditor.AuditorWorker(conf, self.logger, + self.rcache, self.devices) + check_common_defaults() + self.assertEquals(auditor_worker.diskfile_mgr.disk_chunk_size, 65536) + self.assertEquals(auditor_worker.max_files_per_second, 20) + self.assertEquals(auditor_worker.zero_byte_only_at_fps, 0) + + # test specified audit value overrides + conf.update({'disk_chunk_size': 4096}) + auditor_worker = auditor.AuditorWorker(conf, self.logger, + self.rcache, self.devices, + zero_byte_only_at_fps=50) + check_common_defaults() + self.assertEquals(auditor_worker.diskfile_mgr.disk_chunk_size, 4096) + self.assertEquals(auditor_worker.max_files_per_second, 50) + self.assertEquals(auditor_worker.zero_byte_only_at_fps, 50) + def test_object_audit_extra_data(self): def run_tests(disk_file): auditor_worker = auditor.AuditorWorker(self.conf, self.logger,