)
args, remaining = conf_parser.parse_known_args()
defaults = dict(
- sync_scope='partial',
+ sync_scope='incremental',
log_lock_time=10,
)
if args.conf is not None:
)
parser.add_argument(
'--sync-scope',
- choices=['full', 'partial'],
- default='partial',
+ choices=['full', 'incremental'],
+ default='incremental',
help='synchronize everything (for a new region) or only things that '
'have changed since the last run'
)
'continuous sync',
)
parser.add_argument(
- '--partial-sync-delay',
+ '--incremental-sync-delay',
type=check_positive_int,
default=20,
help='seconds to wait between syncs',
log.exception('error doing full sync')
status = 500
resp = str(e)
- elif self.path.startswith('/metadata/partial'):
+ elif self.path.startswith('/metadata/incremental'):
try:
- TestHandler.syncer.sync_partial(TestHandler.num_workers,
+ TestHandler.syncer.sync_incremental(TestHandler.num_workers,
TestHandler.lock_timeout,
TestHandler.max_entries)
except Exception as e:
- log.exception('error doing partial sync')
+ log.exception('error doing incremental sync')
status = 500
resp = str(e)
else:
else:
while True:
try:
- syncer.sync_partial(args.num_workers, args.lock_timeout,
+ syncer.sync_incremental(args.num_workers, args.lock_timeout,
args.max_entries)
except:
- log.exception('error doing partial sync, trying again later')
+ log.exception('error doing incremental sync, trying again later')
log.debug('waiting %d seconds until next sync',
- args.partial_sync_delay)
- time.sleep(args.partial_sync_delay)
+ args.incremental_sync_delay)
+ time.sleep(args.incremental_sync_delay)
self.dest_conn = client.connection(dest)
self.daemon_id = daemon_id
- def sync_partial(self, num_workers, log_lock_time, max_entries):
+ def sync_incremental(self, num_workers, log_lock_time, max_entries):
try:
num_shards = client.num_log_shards(self.src_conn, self._type)
except:
# create the worker processes
if self._type == 'data':
- worker_cls = worker.DataWorkerPartial
+ worker_cls = worker.DataWorkerIncremental
else:
- worker_cls = worker.MetadataWorkerPartial
+ worker_cls = worker.MetadataWorkerIncremental
processes = [worker_cls(workQueue,
resultQueue,
log_lock_time,
process.daemon = True
process.start()
- log.info('Starting partial sync')
+ log.info('Starting incremental sync')
# enqueue the shards to be synced
for i in xrange(num_shards):
workQueue.put(i)
else:
client.update_metadata(self.dest_conn, section, name, metadata)
-class MetadataWorkerPartial(MetadataWorker):
+class MetadataWorkerIncremental(MetadataWorker):
def __init__(self, *args, **kwargs):
self.daemon_id = kwargs['daemon_id']
self.max_entries = kwargs['max_entries']
- super(MetadataWorkerPartial, self).__init__(*args, **kwargs)
+ super(MetadataWorkerIncremental, self).__init__(*args, **kwargs)
def get_and_process_entries(self, marker, shard_num):
num_entries = self.max_entries