def main():
    base_directory = "/mnt/u001/sysops-api/"

    start_time = int(time.time())
    print "Starting Cycle at " + time.ctime()
    sites = {}
    global total_redis_corelist
    total_redis_corelist = execute_range_query()

    for core in total_redis_corelist:
        dc = core.split('.')[2].split('-')[0]
        sites[dc.upper()] = 1

    files_written = {}
    for datacenter in sites.iterkeys():
        start_dc_time = time.time()
        if not os.path.exists(base_directory + datacenter):
            os.makedirs(base_directory + datacenter)

        # Find the unique files for the datacenter.  Spawn a sysopsapi.cache_extractor
        # object for every key.  Dumping the entire datacenter in a single
        # object requires too much RAM.
        uniqueFiles = sysopsapi.cache_extractor.CacheExtractor(
            scope='site', site=datacenter.lower(), list_files=True)

        number_of_keys = len(uniqueFiles._gold.keys())
        keys_processed = 0
        for key in sorted(uniqueFiles._gold.iterkeys()):
            keys_processed += 1
            start_file_time = time.time()
            filename = key.split("#")[1]
            # Spawn a sysopsapi.cache_extractor object for every unique key, in every
            # datacenter.
            try:
                dataDump = sysopsapi.cache_extractor.CacheExtractor(
                    scope='site', site=datacenter.lower(), search_string=filename, contents=True)
            except Exception, e:
                print "Lost filename " + filename + " The object has dropped out of the cache"
                continue

            for key in sorted(dataDump._gold.iterkeys()):
                host, file = key.split('#')
                hostdir = base_directory + datacenter + "/" + host
                if not os.path.exists(hostdir + os.path.dirname(file)):
                    os.makedirs(hostdir + os.path.dirname(file))
                if dataDump._gold[key]:
                    tmp_target_file_name = hostdir + \
                        os.path.dirname(file) + "/." + os.path.basename(file)
                    target_file_name = hostdir + file
                    tmp_target_file = open(tmp_target_file_name, 'w')
                    tmp_target_file.write(dataDump._gold[key])
                    tmp_target_file.close()
                    shutil.move(tmp_target_file_name, target_file_name)
                    files_written[target_file_name] = 1
            del dataDump
            gc.collect()
            print "Completed filename " + filename + " in datacenter " + datacenter + " in " + str(int(time.time() - start_file_time)) + " seconds. " + str(number_of_keys - keys_processed) + " keys remain for the datacenter."
            print "Total time elapsed is " + str(datetime.timedelta(seconds=int(start_time - time.time())))

        del uniqueFiles
        gc.collect()
        print "Completed datacenter " + datacenter + " in " + str(int(time.time() - start_dc_time)) + " seconds"

    purge_start = time.time()
    print "Starting purge of stale data at " + time.ctime()
    files_on_disk = {}
    for (path, dirs, files) in os.walk(base_directory):
        for file in files:
            files_on_disk[path + "/" + file] = 1

    for file in files_on_disk.iterkeys():
        if not files_written.get(file):
            print "removing stale file " + file
            os.remove(file)
    print "Purge of stale data completed in " + str(int(time.time() - purge_start)) + " seconds"

    print "Completed execution in " + str(int(time.time() - start_time)) + " seconds"
    # 20 minutes * 60 seconds = 1200
    seconds_left = 1200 - int(time.time() - start_time)
    print "Sleeping for " + str(seconds_left) + " seconds"
    if seconds_left > 0:
        time.sleep(seconds_left)
dataset/ETHPy150Open linkedin/sysops-api/scripts/extract_sysops_api_to_disk.py/main/swapped operands of `-`