From ffb06b20d3c06bdecaaa9caf1b15e8664438245e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 2 Jul 2015 15:05:12 +0200 Subject: [PATCH] Fix Python 3 issues in nova.utils and nova.tests * Fix sort(): dictionaries are no more comparable on Python 3, use a key function building a sorted list of the dictionary items * Replace __builtin__ with six.moves.builtins * get_hash_str() now accepts Unicode: Unicode is encoded to UTF-8 before hashing the string. Mention also the hash method (MD5) in the docstring. * LastBytesTestCase: use binary files instead of text files. Use also a context manager on the TemporaryFile to ensure that the temporary file is closed (even on error). * Functions: use the __code__ attribute instead of func_code, and __closure__ instead of func_closure. These attributes are also available on Python 2.6. * Replace unichr() with six.unichr() * SafeTruncateTestCase(): write directly the chinese Unicode character instead of using safe_decode() (which is not reliable, it depends on the locale encoding!) * sanitize_hostname(): On Python 3, decode the hostname from Latin1 to get back Unicode. Add also a comment explaining the purpose of the conversion to Latin1. * last_bytes(): replace the hardcoded constant 22 with errno.EINVAL and add a comment explaining how the function works. * tox.ini: add nova.tests.unit.test_utils to Python 3.4 Blueprint nova-python3 Change-Id: I96d9b0581ceaeccf9c35e0c9bada369e9d19fd15 --- nova/test.py | 22 ++++++++++------------ nova/utils.py | 36 +++++++++++++++++++++++++++--------- tox.ini | 1 + 3 files changed, 38 insertions(+), 21 deletions(-) diff --git a/nova/test.py b/nova/test.py index fd4ec0d28..44dede65d 100644 --- a/nova/test.py +++ b/nova/test.py @@ -294,16 +294,14 @@ class TestCase(testtools.TestCase): if isinstance(observed, six.string_types): observed = jsonutils.loads(observed) - def sort(what): - def get_key(item): - if isinstance(item, (datetime.datetime, set)): - return str(item) - if six.PY3 and isinstance(item, dict): - return str(sort(list(six.iterkeys(item)) + - list(six.itervalues(item)))) - return str(item) if six.PY3 else item - - return sorted(what, key=get_key) + def sort_key(x): + if isinstance(x, set) or isinstance(x, datetime.datetime): + return str(x) + if isinstance(x, dict): + items = ((sort_key(key), sort_key(value)) + for key, value in x.items()) + return sorted(items) + return x def inner(expected, observed): if isinstance(expected, dict) and isinstance(observed, dict): @@ -318,8 +316,8 @@ class TestCase(testtools.TestCase): isinstance(observed, (list, tuple, set))): self.assertEqual(len(expected), len(observed)) - expected_values_iter = iter(sort(expected)) - observed_values_iter = iter(sort(observed)) + expected_values_iter = iter(sorted(expected, key=sort_key)) + observed_values_iter = iter(sorted(observed, key=sort_key)) for i in range(len(expected)): inner(next(expected_values_iter), diff --git a/nova/utils.py b/nova/utils.py index 496f49f18..7728c2460 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -19,6 +19,7 @@ import contextlib import datetime +import errno import functools import hashlib import hmac @@ -565,6 +566,12 @@ def monkey_patch(): # If CONF.monkey_patch is not True, this function do nothing. if not CONF.monkey_patch: return + if six.PY3: + def is_method(obj): + # Unbound methods became regular functions on Python 3 + return inspect.ismethod(obj) or inspect.isfunction(obj) + else: + is_method = inspect.ismethod # Get list of modules and decorators for module_and_decorator in CONF.monkey_patch_modules: module, decorator_name = module_and_decorator.split(':') @@ -573,15 +580,15 @@ def monkey_patch(): __import__(module) # Retrieve module information using pyclbr module_data = pyclbr.readmodule_ex(module) - for key in module_data.keys(): + for key, value in module_data.items(): # set the decorator for the class methods - if isinstance(module_data[key], pyclbr.Class): + if isinstance(value, pyclbr.Class): clz = importutils.import_class("%s.%s" % (module, key)) - for method, func in inspect.getmembers(clz, inspect.ismethod): + for method, func in inspect.getmembers(clz, is_method): setattr(clz, method, decorator("%s.%s.%s" % (module, key, method), func)) # set the decorator for the function - if isinstance(module_data[key], pyclbr.Function): + if isinstance(value, pyclbr.Function): func = importutils.import_class("%s.%s" % (module, key)) setattr(sys.modules[module], key, decorator("%s.%s" % (module, key), func)) @@ -614,7 +621,10 @@ def make_dev_path(dev, partition=None, base='/dev'): def sanitize_hostname(hostname): """Return a hostname which conforms to RFC-952 and RFC-1123 specs.""" if isinstance(hostname, six.text_type): + # Remove characters outside the Unicode range U+0000-U+00FF hostname = hostname.encode('latin-1', 'ignore') + if six.PY3: + hostname = hostname.decode('latin-1') hostname = re.sub('[ _]', '-', hostname) hostname = re.sub('[^\w.-]+', '', hostname) @@ -830,7 +840,10 @@ def last_bytes(file_like_object, num): try: file_like_object.seek(-num, os.SEEK_END) except IOError as e: - if e.errno == 22: + # seek() fails with EINVAL when trying to go before the start of the + # file. It means that num is larger than the file size, so just + # go to the start. + if e.errno == errno.EINVAL: file_like_object.seek(0, os.SEEK_SET) else: raise @@ -874,14 +887,14 @@ def instance_sys_meta(instance): def get_wrapped_function(function): """Get the method at the bottom of a stack of decorators.""" - if not hasattr(function, 'func_closure') or not function.func_closure: + if not hasattr(function, '__closure__') or not function.__closure__: return function def _get_wrapped_function(function): - if not hasattr(function, 'func_closure') or not function.func_closure: + if not hasattr(function, '__closure__') or not function.__closure__: return None - for closure in function.func_closure: + for closure in function.__closure__: func = closure.cell_contents deeper_func = _get_wrapped_function(func) @@ -1190,7 +1203,12 @@ def get_image_metadata_from_volume(volume): def get_hash_str(base_str): - """returns string that represents hash of base_str (in hex format).""" + """Returns string that represents MD5 hash of base_str (in hex format). + + If base_str is a Unicode string, encode it to UTF-8. + """ + if isinstance(base_str, six.text_type): + base_str = base_str.encode('utf-8') return hashlib.md5(base_str).hexdigest() if hasattr(hmac, 'compare_digest'): diff --git a/tox.ini b/tox.ini index fc7fda72b..96f33eb82 100644 --- a/tox.ini +++ b/tox.ini @@ -78,6 +78,7 @@ commands = nova.tests.unit.objects.test_virtual_interface \ nova.tests.unit.test_crypto \ nova.tests.unit.test_exception \ + nova.tests.unit.test_utils \ nova.tests.unit.test_versions [testenv:functional]