def main():

    parser = optparse.OptionParser( version = "%prog version: $Id: benchmark_cluster.py 2794 2009-09-16 15:27:49Z andreas $", usage = globals()["__doc__"])

    parser.add_option( "-s", "--suite", dest="suite", type="choice",
                       choices=("CPU", "hosts"),
                       help="suite to use [default=%default]." )

    parser.set_defaults(
        iterations_cpu = 36,
        iterations_sample = 5,
        saturation = 2,
        host_block = 10,
        suite = "CPU",
        cluster_prefix = """qrsh -q %(queue)s@%(host)s -cwd -now n 'python %(script)s | grep -e "job started" -e "job finished"'""",
        )

    (options, args) = E.Start( parser, add_cluster_options = True )

    hosts = [ x for x in getSGEHosts() if x.ncpu != "-" ]
    E.info( "found %i active hosts" % len(hosts) )

    curdir = os.path.abspath( os.getcwd() )

    outfile = options.stdout

    if options.suite == "hosts":
        outfile.write( "%s\n" % "\t".join(HOST._fields))
        for host in hosts:
            outfile.write( "%s\n" % "\t".join(map(str, host )))
    elif suite == "CPU":

        pythonscript = os.path.join( curdir, "benchmarkscript.py" ) 

        if os.path.exists( pythonscript ):
            raise OSError( "file %s already exists - not overwritten" % pythonscript )

        with open(pythonscript,"w") as outfile:
            if options.suite == "CPU":
                iterations = options.iterations_cpu
                outfile.write( CODE_CPU % locals() )

        results = collections.defaultdict( list )

        def run( script, host, options ):
            """run python script on node in the cluster.
            """

            queue = options.cluster_queue

            command = options.cluster_prefix % locals()
            E.debug( "started: %s" % command )

            try:
                child = subprocess.Popen( command, 
                                          shell=True,
                                          stdout=subprocess.PIPE,
                                          stdin=None,
                                          close_fds = True )
                (stdout, stderr) = child.communicate()
            except OSError, e:
                E.warn( "execution of %s failed" % cmd)
                raise

            E.debug( "finished: %s" % command )

            data = Logfile.LogFileData()
            for line in stdout.split("\n"): 
                data.add( line )
            results[host].append( data )

        # do on host-by-host basis, as otherwise 
        # we run out of ports. Several hosts are run
        # as one block. The order of hosts is randomized
        # at each step.
        idx = 0
        for n in range(0,options.iterations_sample):
            myhosts = list( hosts[:] )
            random.shuffle( myhosts )

            for h in range(0, len(myhosts), options.host_block) :
                for host in (myhosts[x] for x in range(h, min(len(myhosts), h+options.host_block))):
                    processes = [] 
                    for cpu in range(0, int( host.ncpu * options.saturation) ):
                        p = Process( target = run, args = ( pythonscript, host.host, options ) )
                        processes.append( p )
                        p.start()
                    E.info( "iteration %i: host %s: started %i threads" % (n, host.host, len(processes)))
                for p in processes: p.join()

            filename = "results_%i.table" % idx
            E.info( (n, filename ) % "iteration %i: output goes to %s")
            idx += 1
            outfile = open( filename, "w")
            writeResults( outfile, results )
            outfile.close()

        writeResults( options.stdout, results )

    E.Stop()
dataset/ETHPy150Open CGATOxford/cgat/obsolete/benchmark_cluster.py/main/swapped operands of `%`