diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index 25bb8c810d..838cfc21df 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -20,6 +20,8 @@ import random import math import time import shutil +import uuid +import errno from eventlet import GreenPool, sleep, Timeout, TimeoutError from eventlet.green import subprocess @@ -49,7 +51,13 @@ def quarantine_db(object_file, server_type): quarantine_dir = os.path.abspath(os.path.join(object_dir, '..', '..', '..', '..', 'quarantined', server_type + 's', os.path.basename(object_dir))) - renamer(object_dir, quarantine_dir) + try: + renamer(object_dir, quarantine_dir) + except OSError, e: + if e.errno not in (errno.EEXIST, errno.ENOTEMPTY): + raise + quarantine_dir = "%s-%s" % (quarantine_dir, uuid.uuid4().hex) + renamer(object_dir, quarantine_dir) class ReplConnection(BufferedHTTPConnection): diff --git a/test/unit/common/test_db_replicator.py b/test/unit/common/test_db_replicator.py index 9e77f2c92f..30db570868 100644 --- a/test/unit/common/test_db_replicator.py +++ b/test/unit/common/test_db_replicator.py @@ -17,8 +17,10 @@ import unittest from contextlib import contextmanager import os import logging +import errno from swift.common import db_replicator +from swift.common import utils from swift.common.utils import normalize_timestamp from swift.container import server as container_server @@ -86,6 +88,8 @@ class ChangingMtimesOs: class FakeBroker: db_file = __file__ + get_repl_missing_table = False + db_type = 'container' def __init__(self, *args, **kwargs): return None @contextmanager @@ -104,6 +108,8 @@ class FakeBroker: def merge_items(self, *args): self.args = args def get_replication_info(self): + if self.get_repl_missing_table: + raise Exception('no such table') return {'delete_timestamp': 0, 'put_timestamp': 1, 'count': 0} def reclaim(self, item_timestamp, sync_timestamp): pass @@ -202,6 +208,35 @@ class TestDBReplicator(unittest.TestCase): replicator = TestReplicator({}) replicator._replicate_object('0', 'file', 'node_id') + def test_replicate_object_quarantine(self): + replicator = TestReplicator({}) + was_db_file = replicator.brokerclass.db_file + try: + + def mock_renamer(was, new, cause_colision=False): + if cause_colision and '-' not in new: + raise OSError(errno.EEXIST, "File already exists") + self.assertEquals('/a/b/c/d/e', was) + if '-' in new: + self.assert_( + new.startswith('/a/quarantined/containers/e-')) + else: + self.assertEquals('/a/quarantined/containers/e', new) + + def mock_renamer_error(was, new): + return mock_renamer(was, new, cause_colision=True) + was_renamer = db_replicator.renamer + db_replicator.renamer = mock_renamer + db_replicator.lock_parent_directory = lock_parent_directory + replicator.brokerclass.get_repl_missing_table = True + replicator.brokerclass.db_file = '/a/b/c/d/e/hey' + replicator._replicate_object('0', 'file', 'node_id') + # try the double quarantine + db_replicator.renamer = mock_renamer_error + replicator._replicate_object('0', 'file', 'node_id') + finally: + replicator.brokerclass.db_file = was_db_file + db_replicator.renamer = was_renamer # def test_dispatch(self): # rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False)