def get_free_gpus(node_name):
    '''
    Based off Awni's runAll.py
    '''
    output = subprocess.Popen(SSH_CMD.split(' ') + [node_name, 'nvidia-smi', '-q', '-x'],
            stdout=subprocess.PIPE).communicate()[0]
    if not output:
        print 'No output for %s' % node_name
        return []
    try:
        tree = et.fromstring(output.strip())
    except xml.parsers.expat.ExpatError:
        print 'Invalid XML: ', output.strip()
        return []

    gpus = tree.findall('gpu')
    print 'Detected %d gpus on %s' % (len(gpus), node_name)
    free_gpus = []
    for i, gpu in enumerate(gpus):
        mem = gpu.findall('memory_usage')
        if len(mem) == 0:
            mem = gpu.findall('fb_memory_usage')
        if len(mem) == 0:
            print node_name % 'Couldn\'t get memory usage on %s'
            return []
        mem = mem[0]
        tot = int(mem.findall('total')[0].text.split()[0])
        used = int(mem.findall('used')[0].text.split()[0])
        print used, '/', tot
        if float(used) / tot < GPU_FREE_TOL:
            free_gpus.append(i)

    return free_gpus
dataset/ETHPy150Open amaas/stanford-ctc/ctc_fast/cluster/utils.py/get_free_gpus/swapped operands of `%`