diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index 8f360f8f1b..e5cec2c283 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -367,6 +367,9 @@ max_clients 1024 Maximum number of clients one worker can concurrently. disable_fallocate false Disable "fast fail" fallocate checks if the underlying filesystem does not support it. +log_max_line_length 0 Caps the length of log lines to the + value given; no limit if set to 0, the + default. log_custom_handlers None Comma-separated list of functions to call to setup custom log handlers. eventlet_debug false If true, turn on debug logging for eventlet @@ -566,6 +569,9 @@ max_clients 1024 Maximum number of clients one worker can user swift User to run as disable_fallocate false Disable "fast fail" fallocate checks if the underlying filesystem does not support it. +log_max_line_length 0 Caps the length of log lines to the + value given; no limit if set to 0, the + default. log_custom_handlers None Comma-separated list of functions to call to setup custom log handlers. eventlet_debug false If true, turn on debug logging for eventlet @@ -695,6 +701,9 @@ db_preallocation off If you don't mind the extra disk space usage in fragmentation. disable_fallocate false Disable "fast fail" fallocate checks if the underlying filesystem does not support it. +log_max_line_length 0 Caps the length of log lines to the + value given; no limit if set to 0, the + default. log_custom_handlers None Comma-separated list of functions to call to setup custom log handlers. eventlet_debug false If true, turn on debug logging for eventlet @@ -825,6 +834,10 @@ cors_allow_origin This is a list of hosts that Access-Control-Allow-Origin header in addition to what the container has set. +log_max_line_length 0 Caps the length of log + lines to the value given; + no limit if set to 0, the + default. log_custom_handlers None Comma separated list of functions to call to setup custom log handlers. diff --git a/doc/source/logs.rst b/doc/source/logs.rst index c2981aac34..6239b93d25 100644 --- a/doc/source/logs.rst +++ b/doc/source/logs.rst @@ -8,6 +8,18 @@ overview, Swift's logs are sent to syslog and organized by log level and syslog facility. All log lines related to the same request have the same transaction id. This page documents the log formats used in the system. +.. note:: + + By default, Swift will log full log lines. However, with the + ``log_max_line_length`` setting and depending on your logging server + software, lines may be truncated or shortened. With ``log_max_line_length < + 7``, the log line will be truncated. With ``log_max_line_length >= 7``, the + log line will be "shortened": about half the max length followed by " ... " + followed by the other half the max length. Unless you use exceptionally + short values, you are unlikely to run across this with the following + documented log lines, but you may see it with debugging and error log + lines. + ---------- Proxy Logs ---------- diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample index 7bbb283786..85f330f3df 100644 --- a/etc/account-server.conf-sample +++ b/etc/account-server.conf-sample @@ -21,6 +21,9 @@ # log_facility = LOG_LOCAL0 # log_level = INFO # log_address = /dev/log +# The following caps the length of log lines to the value given; no limit if +# set to 0, the default. +# log_max_line_length = 0 # # comma separated list of functions to call to setup custom log handlers. # functions get passed: conf, name, log_to_console, log_route, fmt, logger, diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index 6a209038a4..be94ea48cc 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -27,6 +27,9 @@ # log_facility = LOG_LOCAL0 # log_level = INFO # log_address = /dev/log +# The following caps the length of log lines to the value given; no limit if +# set to 0, the default. +# log_max_line_length = 0 # # comma separated list of functions to call to setup custom log handlers. # functions get passed: conf, name, log_to_console, log_route, fmt, logger, diff --git a/etc/drive-audit.conf-sample b/etc/drive-audit.conf-sample index f57b4fc5b2..a07f0a8acd 100644 --- a/etc/drive-audit.conf-sample +++ b/etc/drive-audit.conf-sample @@ -3,6 +3,9 @@ # log_facility = LOG_LOCAL0 # log_level = INFO # log_address = /dev/log +# The following caps the length of log lines to the value given; no limit if +# set to 0, the default. +# log_max_line_length = 0 # minutes = 60 # error_limit = 1 # diff --git a/etc/object-expirer.conf-sample b/etc/object-expirer.conf-sample index 552f57b50a..7d9e515ff1 100644 --- a/etc/object-expirer.conf-sample +++ b/etc/object-expirer.conf-sample @@ -6,6 +6,9 @@ # log_facility = LOG_LOCAL0 # log_level = INFO # log_address = /dev/log +# The following caps the length of log lines to the value given; no limit if +# set to 0, the default. +# log_max_line_length = 0 # # comma separated list of functions to call to setup custom log handlers. # functions get passed: conf, name, log_to_console, log_route, fmt, logger, diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index bc33eff92a..a71bcc1c01 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -23,6 +23,9 @@ # log_facility = LOG_LOCAL0 # log_level = INFO # log_address = /dev/log +# The following caps the length of log lines to the value given; no limit if +# set to 0, the default. +# log_max_line_length = 0 # # comma separated list of functions to call to setup custom log handlers. # functions get passed: conf, name, log_to_console, log_route, fmt, logger, diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 7cb87ac68e..d8b926e714 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -41,6 +41,9 @@ # log_level = INFO # log_headers = false # log_address = /dev/log +# The following caps the length of log lines to the value given; no limit if +# set to 0, the default. +# log_max_line_length = 0 # # This optional suffix (default is empty) that would be appended to the swift transaction # id allows one to easily figure out from which cluster that X-Trans-Id belongs to. diff --git a/swift/common/utils.py b/swift/common/utils.py index 434c701673..be5cca3f63 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -1023,10 +1023,15 @@ class LogAdapter(logging.LoggerAdapter, object): class SwiftLogFormatter(logging.Formatter): """ - Custom logging.Formatter will append txn_id to a log message if the record - has one and the message does not. + Custom logging.Formatter will append txn_id to a log message if the + record has one and the message does not. Optionally it can shorten + overly long log lines. """ + def __init__(self, fmt=None, datefmt=None, max_line_length=0): + logging.Formatter.__init__(self, fmt=fmt, datefmt=datefmt) + self.max_line_length = max_line_length + def format(self, record): if not hasattr(record, 'server'): # Catch log messages that were not initiated by swift @@ -1058,6 +1063,12 @@ class SwiftLogFormatter(logging.Formatter): record.levelno != logging.INFO and record.client_ip not in msg): msg = "%s (client_ip: %s)" % (msg, record.client_ip) + if self.max_line_length > 0 and len(msg) > self.max_line_length: + if self.max_line_length < 7: + msg = msg[:self.max_line_length] + else: + approxhalf = (self.max_line_length - 5) / 2 + msg = msg[:approxhalf] + " ... " + msg[-approxhalf:] return msg @@ -1071,6 +1082,7 @@ def get_logger(conf, name=None, log_to_console=False, log_route=None, log_facility = LOG_LOCAL0 log_level = INFO log_name = swift + log_max_line_length = 0 log_udp_host = (disabled) log_udp_port = logging.handlers.SYSLOG_UDP_PORT log_address = /dev/log @@ -1096,7 +1108,8 @@ def get_logger(conf, name=None, log_to_console=False, log_route=None, logger = logging.getLogger(log_route) logger.propagate = False # all new handlers will get the same formatter - formatter = SwiftLogFormatter(fmt) + formatter = SwiftLogFormatter( + fmt=fmt, max_line_length=int(conf.get('log_max_line_length', 0))) # get_logger will only ever add one SysLog Handler to a logger if not hasattr(get_logger, 'handler4logger'): diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index caaaa5ed62..dd63851540 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -702,6 +702,61 @@ class TestUtils(unittest.TestCase): logger.logger.removeHandler(handler) reset_loggers() + def test_swift_log_formatter_max_line_length(self): + # setup stream logging + sio = StringIO() + logger = utils.get_logger(None) + handler = logging.StreamHandler(sio) + formatter = utils.SwiftLogFormatter(max_line_length=10) + handler.setFormatter(formatter) + logger.logger.addHandler(handler) + + def strip_value(sio): + v = sio.getvalue() + sio.truncate(0) + return v + + try: + logger.info('12345') + self.assertEqual(strip_value(sio), '12345\n') + logger.info('1234567890') + self.assertEqual(strip_value(sio), '1234567890\n') + logger.info('1234567890abcde') + self.assertEqual(strip_value(sio), '12 ... de\n') + formatter.max_line_length = 11 + logger.info('1234567890abcde') + self.assertEqual(strip_value(sio), '123 ... cde\n') + formatter.max_line_length = 0 + logger.info('1234567890abcde') + self.assertEqual(strip_value(sio), '1234567890abcde\n') + formatter.max_line_length = 1 + logger.info('1234567890abcde') + self.assertEqual(strip_value(sio), '1\n') + formatter.max_line_length = 2 + logger.info('1234567890abcde') + self.assertEqual(strip_value(sio), '12\n') + formatter.max_line_length = 3 + logger.info('1234567890abcde') + self.assertEqual(strip_value(sio), '123\n') + formatter.max_line_length = 4 + logger.info('1234567890abcde') + self.assertEqual(strip_value(sio), '1234\n') + formatter.max_line_length = 5 + logger.info('1234567890abcde') + self.assertEqual(strip_value(sio), '12345\n') + formatter.max_line_length = 6 + logger.info('1234567890abcde') + self.assertEqual(strip_value(sio), '123456\n') + formatter.max_line_length = 7 + logger.info('1234567890abcde') + self.assertEqual(strip_value(sio), '1 ... e\n') + formatter.max_line_length = -10 + logger.info('1234567890abcde') + self.assertEqual(strip_value(sio), '1234567890abcde\n') + finally: + logger.logger.removeHandler(handler) + reset_loggers() + def test_swift_log_formatter(self): # setup stream logging sio = StringIO()