From 7b706926a8ed5bbcec3a678e868e301c9a6ed8f1 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Mon, 23 May 2016 15:20:06 +0100 Subject: [PATCH] Fix setup of manifest responses in SLO tests The swift_bytes param is removed from the content-type in the proxy object controller, so the SLO unit tests should not be registering GET responses with FakeSwift that have swift_bytes appended to the content-type. Nor should submanifest segment dicts have swift_bytes appended to their content-type values. Also adds a test for the object controller and container server handling of SLO swift_bytes. Change-Id: Icf9bd87eee25002c8d9728b16e60c8347060f320 --- test/unit/common/middleware/test_slo.py | 79 +++++++++++++++++++------ test/unit/container/test_server.py | 42 +++++++++++++ test/unit/proxy/controllers/test_obj.py | 14 +++++ 3 files changed, 116 insertions(+), 19 deletions(-) diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index 2a27d1c315..0435ae5544 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -349,7 +349,7 @@ class TestSloPutManifest(SloTestCase): 'GET', '/v1/AUTH_test/checktest/slob', swob.HTTPOk, {'X-Static-Large-Object': 'true', 'Etag': 'slob-etag', - 'Content-Type': 'cat/picture;swift_bytes=12345', + 'Content-Type': 'cat/picture', 'Content-Length': len(_manifest_json)}, _manifest_json) @@ -1106,7 +1106,7 @@ class TestSloGetRawManifest(SloTestCase): 'last_modified': '1970-01-01T00:00:00.000000'}, {'name': '/gettest/d_10', 'hash': md5hex(md5hex("e" * 5) + md5hex("f" * 5)), 'bytes': '10', - 'content_type': 'application/json;swift_bytes=10', + 'content_type': 'application/json', 'sub_slo': True, 'last_modified': '1970-01-01T00:00:00.000000'}]) self.bc_etag = md5hex(_bc_manifest_json) @@ -1262,7 +1262,7 @@ class TestSloGetManifest(SloTestCase): 'content_type': 'text/plain'}]) self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest-bc', - swob.HTTPOk, {'Content-Type': 'application/json;swift_bytes=25', + swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true', 'X-Object-Meta-Plant': 'Ficus', 'Etag': md5hex(_bc_manifest_json)}, @@ -1272,9 +1272,9 @@ class TestSloGetManifest(SloTestCase): [{'name': '/gettest/a_5', 'hash': md5hex("a" * 5), 'content_type': 'text/plain', 'bytes': '5'}, {'name': '/gettest/manifest-bc', 'sub_slo': True, - 'content_type': 'application/json;swift_bytes=25', + 'content_type': 'application/json', 'hash': md5hex(md5hex("b" * 10) + md5hex("c" * 15)), - 'bytes': len(_bc_manifest_json)}, + 'bytes': 25}, {'name': '/gettest/d_20', 'hash': md5hex("d" * 20), 'content_type': 'text/plain', 'bytes': '20'}]) self.app.register( @@ -1284,6 +1284,34 @@ class TestSloGetManifest(SloTestCase): 'Etag': md5(_abcd_manifest_json).hexdigest()}, _abcd_manifest_json) + # A submanifest segment is created using the response headers from a + # HEAD on the submanifest. That HEAD is passed through SLO which will + # modify the response content-length to be equal to the size of the + # submanifest's large object. The swift_bytes value appended to the + # submanifest's content-type will have been removed. So the sub-slo + # segment dict that is written to the parent manifest should have the + # correct bytes and content-type values. However, if somehow the + # submanifest HEAD response wasn't modified by SLO (maybe + # historically?) and we ended up with the parent manifest sub-slo entry + # having swift_bytes appended to it's content-type and the actual + # submanifest size in its bytes field, then SLO can cope, so we create + # a deviant manifest to verify that SLO can deal with it. + _abcd_manifest_json_alt = json.dumps( + [{'name': '/gettest/a_5', 'hash': md5hex("a" * 5), + 'content_type': 'text/plain', 'bytes': '5'}, + {'name': '/gettest/manifest-bc', 'sub_slo': True, + 'content_type': 'application/json; swift_bytes=25', + 'hash': md5hex(md5hex("b" * 10) + md5hex("c" * 15)), + 'bytes': len(_bc_manifest_json)}, + {'name': '/gettest/d_20', 'hash': md5hex("d" * 20), + 'content_type': 'text/plain', 'bytes': '20'}]) + self.app.register( + 'GET', '/v1/AUTH_test/gettest/manifest-abcd-alt', + swob.HTTPOk, {'Content-Type': 'application/json', + 'X-Static-Large-Object': 'true', + 'Etag': md5(_abcd_manifest_json_alt).hexdigest()}, + _abcd_manifest_json_alt) + _abcdefghijkl_manifest_json = json.dumps( [{'name': '/gettest/a_5', 'hash': md5hex("a" * 5), 'content_type': 'text/plain', 'bytes': '5'}, @@ -1337,7 +1365,7 @@ class TestSloGetManifest(SloTestCase): self.bc_ranges_etag = md5hex(_bc_ranges_manifest_json) self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest-bc-ranges', - swob.HTTPOk, {'Content-Type': 'application/json;swift_bytes=16', + swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true', 'X-Object-Meta-Plant': 'Ficus', 'Etag': self.bc_ranges_etag}, @@ -1351,12 +1379,12 @@ class TestSloGetManifest(SloTestCase): 'content_type': 'text/plain', 'bytes': '5', 'range': '1-4'}, {'name': '/gettest/manifest-bc-ranges', 'sub_slo': True, - 'content_type': 'application/json;swift_bytes=16', + 'content_type': 'application/json', 'hash': self.bc_ranges_etag, - 'bytes': len(_bc_ranges_manifest_json), + 'bytes': 16, 'range': '8-15'}, {'name': '/gettest/manifest-bc-ranges', 'sub_slo': True, - 'content_type': 'application/json;swift_bytes=16', + 'content_type': 'application/json', 'hash': self.bc_ranges_etag, 'bytes': len(_bc_ranges_manifest_json), 'range': '0-7'}, @@ -1655,6 +1683,22 @@ class TestSloGetManifest(SloTestCase): self.assertEqual( body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd') + def test_get_manifest_with_submanifest_bytes_in_content_type(self): + # verify correct content-length when the sub-slo segment in the + # manifest has its actual object content-length appended as swift_bytes + # to the content-type, and the submanifest length in the bytes field. + req = Request.blank( + '/v1/AUTH_test/gettest/manifest-abcd-alt', + environ={'REQUEST_METHOD': 'GET'}) + status, headers, body = self.call_slo(req) + headers = HeaderKeyDict(headers) + + self.assertEqual(status, '200 OK') + self.assertEqual(headers['Content-Length'], '50') + self.assertEqual(headers['Etag'], '"%s"' % self.manifest_abcd_etag) + self.assertEqual( + body, 'aaaaabbbbbbbbbbcccccccccccccccdddddddddddddddddddd') + def test_range_get_manifest(self): req = Request.blank( '/v1/AUTH_test/gettest/manifest-abcd', @@ -2274,8 +2318,7 @@ class TestSloGetManifest(SloTestCase): 'hash': 'man%d' % (i + 1), 'sub_slo': True, 'bytes': len(manifest_json), - 'content_type': - 'application/json;swift_bytes=%d' % ((21 - i) * 6)}] + 'content_type': 'application/json'}] manifest_json = json.dumps(manifest_data) self.app.register( @@ -2330,9 +2373,8 @@ class TestSloGetManifest(SloTestCase): {'name': '/gettest/man%d' % (i + 1), 'hash': 'man%d' % (i + 1), 'sub_slo': True, - 'bytes': len(manifest_json), - 'content_type': - 'application/json;swift_bytes=%d' % ((10 - i) * 6)}, + 'bytes': (10 - i) * 6, + 'content_type': 'application/json'}, {'name': '/gettest/obj%d' % i, 'hash': md5hex('body%02d' % i), 'bytes': '6', @@ -2387,9 +2429,8 @@ class TestSloGetManifest(SloTestCase): {'name': '/gettest/man%d' % (i + 1), 'hash': 'man%d' % (i + 1), 'sub_slo': True, - 'bytes': len(manifest_json), - 'content_type': - 'application/json;swift_bytes=%d' % ((12 - i) * 6)}, + 'bytes': (12 - i) * 6, + 'content_type': 'application/json'}, {'name': '/gettest/obj%d' % i, 'hash': md5hex('body%02d' % i), 'bytes': '6', @@ -2479,7 +2520,7 @@ class TestSloGetManifest(SloTestCase): swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true'}, json.dumps([{'name': '/gettest/manifest-a', 'sub_slo': True, - 'content_type': 'application/json;swift_bytes=5', + 'content_type': 'application/json', 'hash': 'manifest-a', 'bytes': '12345'}])) @@ -2497,7 +2538,7 @@ class TestSloGetManifest(SloTestCase): def test_invalid_json_submanifest(self): self.app.register( 'GET', '/v1/AUTH_test/gettest/manifest-bc', - swob.HTTPOk, {'Content-Type': 'application/json;swift_bytes=25', + swob.HTTPOk, {'Content-Type': 'application/json', 'X-Static-Large-Object': 'true', 'X-Object-Meta-Plant': 'Ficus'}, "[this {isn't (JSON") diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index 642c12e7b6..706f3c3366 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -2299,6 +2299,48 @@ class TestContainerController(unittest.TestCase): result = [x['content_type'] for x in json.loads(resp.body)] self.assertEqual(result, [u'\u2603', 'text/plain;charset="utf-8"']) + def test_swift_bytes_in_content_type(self): + # create container + req = Request.blank( + '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + req.get_response(self.controller) + + # regular object update + ctype = 'text/plain; charset="utf-8"' + req = Request.blank( + '/sda1/p/a/c/o1', environ={ + 'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': ctype, + 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 99}) + self._update_object_put_headers(req) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 201) + + # slo object update + ctype = 'text/plain; charset="utf-8"; swift_bytes=12345678' + req = Request.blank( + '/sda1/p/a/c/o2', environ={ + 'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1', 'HTTP_X_CONTENT_TYPE': ctype, + 'HTTP_X_ETAG': 'x', 'HTTP_X_SIZE': 99}) + self._update_object_put_headers(req) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 201) + + # verify listing + req = Request.blank('/sda1/p/a/c?format=json', + environ={'REQUEST_METHOD': 'GET'}) + resp = req.get_response(self.controller) + listing = json.loads(resp.body) + self.assertEqual(2, len(listing)) + self.assertEqual('text/plain;charset="utf-8"', + listing[0]['content_type']) + self.assertEqual(99, listing[0]['bytes']) + self.assertEqual('text/plain;charset="utf-8"', + listing[1]['content_type']) + self.assertEqual(12345678, listing[1]['bytes']) + def test_GET_accept_not_valid(self): req = Request.blank('/sda1/p/a/c', method='PUT', headers={ 'X-Timestamp': Timestamp(0).internal}) diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py index 35a2178c1c..be0893dbb2 100755 --- a/test/unit/proxy/controllers/test_obj.py +++ b/test/unit/proxy/controllers/test_obj.py @@ -747,6 +747,20 @@ class TestReplicatedObjController(BaseObjectControllerMixin, self.assertEqual(resp.status_int, 200) self.assertEqual(resp.headers['Transfer-Encoding'], 'chunked') + def _test_removes_swift_bytes(self, method): + req = swift.common.swob.Request.blank('/v1/a/c/o', method=method) + with set_http_connect( + 200, headers={'content-type': 'image/jpeg; swift_bytes=99'}): + resp = req.get_response(self.app) + self.assertEqual(resp.status_int, 200) + self.assertEqual(resp.headers['Content-Type'], 'image/jpeg') + + def test_GET_removes_swift_bytes(self): + self._test_removes_swift_bytes('GET') + + def test_HEAD_removes_swift_bytes(self): + self._test_removes_swift_bytes('HEAD') + def test_GET_error(self): req = swift.common.swob.Request.blank('/v1/a/c/o') self.app.logger.txn_id = req.environ['swift.trans_id'] = 'my-txn-id'