def runParallel(opts):
    # FIXME Currently assumes you run this from ./${DATASET}-utils

    # First distribute all the jobs
    unsub_jobs = list()
    for i in xrange(opts.start_file, opts.start_file + opts.numFiles):
        unsub_jobs.append(i)

    # Sort files by size so that biggest files are running the whole time
    # Note that we pop from the end
    unsub_jobs = sorted(unsub_jobs, key=lambda x: os.path.getsize(pjoin(LIKELIHOODS_DIR, 'loglikelihoods_%d.pk' % x)))

    while len(unsub_jobs):
        # Get free machines and split the files between them
        free_nodes = get_free_nodes('deep')  # PARAM
        for free_node in free_nodes:
            if len(unsub_jobs):
                job = unsub_jobs.pop()
                runNode(free_node, job, opts)
        if len(unsub_jobs):
            print '-' * 80
            print '%d jobs to be submitted...' % len(unsub_jobs)
            print '-' * 80
            time.sleep(SLEEP_SEC)

    # Now wait until all the jobs are finished
    # FIXME Hacky, just checks whether files exist and aren't empty
    def jobs_left():
        wait_count = 0
        for i in xrange(opts.start_file, opts.start_file + opts.numFiles):
            fi = '%s.%d' % (opts.out_file, i)
            if not os.path.exists(fi) or os.path.getsize(fi) == 0:
                wait_count += 1
        return wait_count
    while jobs_left() != 0:
        print '-' * 80
        print 'Waiting for %d jobs to finish...' % jobs_left()
        print '-' * 80
        time.sleep(SLEEP_SEC)

    # Now merge the results together into single file like
    # we would get by running sequentially
    concat_list = list()
    hyps = list()
    refs = list()
    hypscores = list()
    refscores = list()
    numphones = list()
    subsets = list()
    alignments = list()
    for i in xrange(opts.start_file, opts.start_file + opts.numFiles):
        fi = '%s.%d' % (opts.out_file, i)
        fi_pk = fi.replace('.txt', '.pk')
        assert os.path.exists(fi), '%s does not exist' % fi
        assert os.path.exists(fi_pk), '%s does not exist' % fi_pk
        with open(fi, 'r') as f:
            concat_list.append(f.read().strip())
        with open(fi_pk, 'rb') as f:
            h = pickle.load(f)
            r = pickle.load(f)
            hs = pickle.load(f)
            rs = pickle.load(f)
            np = pickle.load(f)
            ss = pickle.load(f)
            al = pickle.load(f)
            hyps += h
            refs += r
            hypscores += hs
            refscores += rs
            numphones += np
            subsets += ss
            alignments += al
        # Cleanup
        os.remove(fi)
        os.remove(fi_pk)
    with open(opts.out_file, 'w') as fout:
        fout.write('\n'.join(concat_list))
    with open(opts.out_file.replace('.txt', '.pk'), 'wb') as fout:
        pickle.dump(hyps, fout)
        pickle.dump(refs, fout)
        pickle.dump(hypscores, fout)
        pickle.dump(refscores, fout)
        pickle.dump(numphones, fout)
        pickle.dump(subsets, fout)
        pickle.dump(alignments, fout)
dataset/ETHPy150Open amaas/stanford-ctc/ctc_fast/runDecode.py/runParallel/original