openstack/swift
Code Issues Proposed changes

break up that giant chunked put test

Browse Source
This commit is contained in:
Michael Barton
2011-01-19 15:16:44 +00:00
committed by Tarmac
parent 72999be598 2fc5c401fa
commit 56791413b8
1 changed files with 619 additions and 562 deletions
Download Patch File Download Diff File Expand all files Collapse all files

257
test/unit/proxy/test_server.py
View File

@@ -47,11 +47,111 @@ from swift.common import ring
from swift.common.constraints import MAX_META_NAME_LENGTH, \
MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, MAX_FILE_SIZE
from swift.common.utils import mkdirs, normalize_timestamp, NullLogger
from swift.common.wsgi import monkey_patch_mimetools
# mocks
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
def setup():
global _testdir, _test_servers, _test_sockets, \
_orig_container_listing_limit, _test_coros
monkey_patch_mimetools()
# Since we're starting up a lot here, we're going to test more than
# just chunked puts; we're also going to test parts of
# proxy_server.Application we couldn't get to easily otherwise.
path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
if not path_to_test_xfs or not os.path.exists(path_to_test_xfs):
print >> sys.stderr, 'WARNING: PATH_TO_TEST_XFS not set or not ' \
'pointing to a valid directory.\n' \
'Please set PATH_TO_TEST_XFS to a directory on an XFS file ' \
'system for testing.'
raise SkipTest
_testdir = \
os.path.join(path_to_test_xfs, 'tmp_test_proxy_server_chunked')
mkdirs(_testdir)
rmtree(_testdir)
mkdirs(os.path.join(_testdir, 'sda1'))
mkdirs(os.path.join(_testdir, 'sda1', 'tmp'))
mkdirs(os.path.join(_testdir, 'sdb1'))
mkdirs(os.path.join(_testdir, 'sdb1', 'tmp'))
_orig_container_listing_limit = proxy_server.CONTAINER_LISTING_LIMIT
conf = {'devices': _testdir, 'swift_dir': _testdir,
'mount_check': 'false'}
prolis = listen(('localhost', 0))
acc1lis = listen(('localhost', 0))
acc2lis = listen(('localhost', 0))
con1lis = listen(('localhost', 0))
con2lis = listen(('localhost', 0))
obj1lis = listen(('localhost', 0))
obj2lis = listen(('localhost', 0))
_test_sockets = \
(prolis, acc1lis, acc2lis, con2lis, con2lis, obj1lis, obj2lis)
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': acc1lis.getsockname()[1]},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': acc2lis.getsockname()[1]}], 30),
GzipFile(os.path.join(_testdir, 'account.ring.gz'), 'wb'))
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': con1lis.getsockname()[1]},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': con2lis.getsockname()[1]}], 30),
GzipFile(os.path.join(_testdir, 'container.ring.gz'), 'wb'))
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': obj1lis.getsockname()[1]},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': obj2lis.getsockname()[1]}], 30),
GzipFile(os.path.join(_testdir, 'object.ring.gz'), 'wb'))
prosrv = proxy_server.Application(conf, FakeMemcacheReturnsNone())
acc1srv = account_server.AccountController(conf)
acc2srv = account_server.AccountController(conf)
con1srv = container_server.ContainerController(conf)
con2srv = container_server.ContainerController(conf)
obj1srv = object_server.ObjectController(conf)
obj2srv = object_server.ObjectController(conf)
_test_servers = \
(prosrv, acc1srv, acc2srv, con2srv, con2srv, obj1srv, obj2srv)
nl = NullLogger()
prospa = spawn(wsgi.server, prolis, prosrv, nl)
acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl)
acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl)
con1spa = spawn(wsgi.server, con1lis, con1srv, nl)
con2spa = spawn(wsgi.server, con2lis, con2srv, nl)
obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl)
obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl)
_test_coros = \
(prospa, acc1spa, acc2spa, con2spa, con2spa, obj1spa, obj2spa)
# Create account
ts = normalize_timestamp(time())
partition, nodes = prosrv.account_ring.get_nodes('a')
for node in nodes:
conn = proxy_server.http_connect(node['ip'], node['port'],
node['device'], partition, 'PUT', '/a',
{'X-Timestamp': ts, 'X-CF-Trans-Id': 'test'})
resp = conn.getresponse()
assert(resp.status == 201)
# Create container
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/c HTTP/1.1\r\nHost: localhost\r\n'
'Connection: close\r\nX-Auth-Token: t\r\n'
'Content-Length: 0\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
assert(headers[:len(exp)] == exp)
def teardown():
for server in _test_coros:
server.kill()
proxy_server.CONTAINER_LISTING_LIMIT = _orig_container_listing_limit
rmtree(_testdir)
def fake_http_connect(*code_iter, **kwargs):
class FakeConn(object):
@@ -427,6 +527,10 @@ class TestObjectController(unittest.TestCase):
account_ring=FakeRing(), container_ring=FakeRing(),
object_ring=FakeRing())
def tearDown(self):
proxy_server.CONTAINER_LISTING_LIMIT = _orig_container_listing_limit
def assert_status_map(self, method, statuses, expected, raise_exc=False):
with save_globals():
kwargs = {}
@@ -1586,71 +1690,10 @@ class TestObjectController(unittest.TestCase):
finally:
server.MAX_FILE_SIZE = MAX_FILE_SIZE
def test_chunked_put_and_a_bit_more(self):
# Since we're starting up a lot here, we're going to test more than
# just chunked puts; we're also going to test parts of
# proxy_server.Application we couldn't get to easily otherwise.
path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
if not path_to_test_xfs or not os.path.exists(path_to_test_xfs):
print >> sys.stderr, 'WARNING: PATH_TO_TEST_XFS not set or not ' \
'pointing to a valid directory.\n' \
'Please set PATH_TO_TEST_XFS to a directory on an XFS file ' \
'system for testing.'
raise SkipTest
testdir = \
os.path.join(path_to_test_xfs, 'tmp_test_proxy_server_chunked')
mkdirs(testdir)
rmtree(testdir)
mkdirs(os.path.join(testdir, 'sda1'))
mkdirs(os.path.join(testdir, 'sda1', 'tmp'))
mkdirs(os.path.join(testdir, 'sdb1'))
mkdirs(os.path.join(testdir, 'sdb1', 'tmp'))
try:
orig_container_listing_limit = proxy_server.CONTAINER_LISTING_LIMIT
conf = {'devices': testdir, 'swift_dir': testdir,
'mount_check': 'false'}
prolis = listen(('localhost', 0))
acc1lis = listen(('localhost', 0))
acc2lis = listen(('localhost', 0))
con1lis = listen(('localhost', 0))
con2lis = listen(('localhost', 0))
obj1lis = listen(('localhost', 0))
obj2lis = listen(('localhost', 0))
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': acc1lis.getsockname()[1]},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': acc2lis.getsockname()[1]}], 30),
GzipFile(os.path.join(testdir, 'account.ring.gz'), 'wb'))
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': con1lis.getsockname()[1]},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': con2lis.getsockname()[1]}], 30),
GzipFile(os.path.join(testdir, 'container.ring.gz'), 'wb'))
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': obj1lis.getsockname()[1]},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': obj2lis.getsockname()[1]}], 30),
GzipFile(os.path.join(testdir, 'object.ring.gz'), 'wb'))
prosrv = proxy_server.Application(conf, FakeMemcacheReturnsNone())
acc1srv = account_server.AccountController(conf)
acc2srv = account_server.AccountController(conf)
con1srv = container_server.ContainerController(conf)
con2srv = container_server.ContainerController(conf)
obj1srv = object_server.ObjectController(conf)
obj2srv = object_server.ObjectController(conf)
nl = NullLogger()
prospa = spawn(wsgi.server, prolis, prosrv, nl)
acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl)
acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl)
con1spa = spawn(wsgi.server, con1lis, con1srv, nl)
con2spa = spawn(wsgi.server, con2lis, con2srv, nl)
obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl)
obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl)
try:
def test_chunked_put_bad_version(self):
# Check bad version
(prolis, acc1lis, acc2lis, con2lis, con2lis, obj1lis, obj2lis) = \
_test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v0 HTTP/1.1\r\nHost: localhost\r\n'
@@ -1659,7 +1702,11 @@ class TestObjectController(unittest.TestCase):
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 412'
self.assertEquals(headers[:len(exp)], exp)
def test_chunked_put_bad_path(self):
# Check bad path
(prolis, acc1lis, acc2lis, con2lis, con2lis, obj1lis, obj2lis) = \
_test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET invalid HTTP/1.1\r\nHost: localhost\r\n'
@@ -1668,7 +1715,11 @@ class TestObjectController(unittest.TestCase):
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 404'
self.assertEquals(headers[:len(exp)], exp)
def test_chunked_put_bad_utf8(self):
# Check invalid utf-8
(prolis, acc1lis, acc2lis, con2lis, con2lis, obj1lis, obj2lis) = \
_test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a%80 HTTP/1.1\r\nHost: localhost\r\n'
@@ -1678,7 +1729,11 @@ class TestObjectController(unittest.TestCase):
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 412'
self.assertEquals(headers[:len(exp)], exp)
def test_chunked_put_bad_path_no_controller(self):
# Check bad path, no controller
(prolis, acc1lis, acc2lis, con2lis, con2lis, obj1lis, obj2lis) = \
_test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1 HTTP/1.1\r\nHost: localhost\r\n'
@@ -1688,7 +1743,11 @@ class TestObjectController(unittest.TestCase):
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 412'
self.assertEquals(headers[:len(exp)], exp)
def test_chunked_put_bad_method(self):
# Check bad method
(prolis, acc1lis, acc2lis, con2lis, con2lis, obj1lis, obj2lis) = \
_test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('LICK /v1/a HTTP/1.1\r\nHost: localhost\r\n'
@@ -1698,7 +1757,13 @@ class TestObjectController(unittest.TestCase):
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 405'
self.assertEquals(headers[:len(exp)], exp)
def test_chunked_put_unhandled_exception(self):
# Check unhandled exception
(prosrv, acc1srv, acc2srv, con2srv, con2srv, obj1srv, obj2srv) = \
_test_servers
(prolis, acc1lis, acc2lis, con2lis, con2lis, obj1lis, obj2lis) = \
_test_sockets
orig_update_request = prosrv.update_request
def broken_update_request(env, req):
@@ -1715,18 +1780,13 @@ class TestObjectController(unittest.TestCase):
exp = 'HTTP/1.1 500'
self.assertEquals(headers[:len(exp)], exp)
prosrv.update_request = orig_update_request
# Okay, back to chunked put testing; Create account
ts = normalize_timestamp(time())
partition, nodes = prosrv.account_ring.get_nodes('a')
for node in nodes:
conn = proxy_server.http_connect(node['ip'], node['port'],
node['device'], partition, 'PUT', '/a',
{'X-Timestamp': ts, 'X-CF-Trans-Id': 'test'})
resp = conn.getresponse()
self.assertEquals(resp.status, 201)
def test_chunked_put_head_account(self):
# Head account, just a double check and really is here to test
# the part Application.log_request that 'enforces' a
# content_length on the response.
(prolis, acc1lis, acc2lis, con2lis, con2lis, obj1lis, obj2lis) = \
_test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('HEAD /v1/a HTTP/1.1\r\nHost: localhost\r\n'
@@ -1737,19 +1797,15 @@ class TestObjectController(unittest.TestCase):
exp = 'HTTP/1.1 204'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('\r\nContent-Length: 0\r\n' in headers)
# Create container
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/c HTTP/1.1\r\nHost: localhost\r\n'
'Connection: close\r\nX-Auth-Token: t\r\n'
'Content-Length: 0\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
def test_chunked_put_logging(self):
# GET account with a query string to test that
# Application.log_request logs the query string. Also, throws
# in a test for logging x-forwarded-for (first entry only).
(prosrv, acc1srv, acc2srv, con2srv, con2srv, obj1srv, obj2srv) = \
_test_servers
(prolis, acc1lis, acc2lis, con2lis, con2lis, obj1lis, obj2lis) = \
_test_sockets
class Logger(object):
@@ -1776,11 +1832,6 @@ class TestObjectController(unittest.TestCase):
prosrv.logger = orig_logger
# Turn on header logging.
class Logger(object):
def info(self, msg):
self.msg = msg
orig_logger = prosrv.logger
prosrv.logger = Logger()
prosrv.log_headers = True
@@ -1797,7 +1848,11 @@ class TestObjectController(unittest.TestCase):
prosrv.logger.msg)
prosrv.log_headers = False
prosrv.logger = orig_logger
def test_chunked_put_utf8_all_the_way_down(self):
# Test UTF-8 Unicode all the way through the system
(prolis, acc1lis, acc2lis, con2lis, con2lis, obj1lis, obj2lis) = \
_test_sockets
ustr = '\xe1\xbc\xb8\xce\xbf\xe1\xbd\xba \xe1\xbc\xb0\xce' \
'\xbf\xe1\xbd\xbb\xce\x87 \xcf\x84\xe1\xbd\xb0 \xcf' \
'\x80\xe1\xbd\xb1\xce\xbd\xcf\x84\xca\xbc \xe1\xbc' \
@@ -1806,6 +1861,8 @@ class TestObjectController(unittest.TestCase):
'\xbf\x86.Test'
ustr_short = '\xe1\xbc\xb8\xce\xbf\xe1\xbd\xbatest'
# Create ustr container
(prolis, acc1lis, acc2lis, con2lis, con2lis, obj1lis, obj2lis) = \
_test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/%s HTTP/1.1\r\nHost: localhost\r\n'
@@ -1838,7 +1895,7 @@ class TestObjectController(unittest.TestCase):
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
listing = simplejson.loads(fd.read())
self.assertEquals(listing[1]['name'], ustr.decode('utf8'))
self.assert_(ustr.decode('utf8') in [l['name'] for l in listing])
# List account with ustr container (test xml)
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
@@ -1912,7 +1969,11 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(headers[:len(exp)], exp)
self.assert_('\r\nX-Object-Meta-%s: %s\r\n' %
(quote(ustr_short).lower(), quote(ustr)) in headers)
def test_chunked_put_chunked_put(self):
# Do chunked object put
(prolis, acc1lis, acc2lis, con2lis, con2lis, obj1lis, obj2lis) = \
_test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
# Also happens to assert that x-storage-token is taken as a
@@ -1937,7 +1998,11 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(headers[:len(exp)], exp)
body = fd.read()
self.assertEquals(body, 'oh hai123456789abcdef')
def test_chunked_put_lobjects(self):
# Create a container for our segmented/manifest object testing
(prolis, acc1lis, acc2lis, con2lis, con2lis, obj1lis, obj2lis) = \
_test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented HTTP/1.1\r\nHost: localhost\r\n'
@@ -2098,6 +2163,7 @@ class TestObjectController(unittest.TestCase):
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
print headers
self.assert_('Content-Type: text/jibberish' in headers)
# Check set content type
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
@@ -2145,17 +2211,6 @@ class TestObjectController(unittest.TestCase):
self.assertEquals(headers[:len(exp)], exp)
self.assert_('Content-Type: foo/bar; charset=UTF-8' in
headers.split('\r\n'), repr(headers.split('\r\n')))
finally:
prospa.kill()
acc1spa.kill()
acc2spa.kill()
con1spa.kill()
con2spa.kill()
obj1spa.kill()
obj2spa.kill()
finally:
proxy_server.CONTAINER_LISTING_LIMIT = orig_container_listing_limit
rmtree(testdir)
def test_mismatched_etags(self):
with save_globals():
@@ -3377,4 +3432,6 @@ class TestSegmentedIterable(unittest.TestCase):
if __name__ == '__main__':
setup()
unittest.main()
teardown()