def read_n_merge(fileLst):
    '''Read entries from the individual files and them counts on the fly'''

    total_rules = 0
    total_1nt_rules = 0
    candLst = []
    ruleDict = {}
    rulesDict1NT = {}
    fileTrackLst = [ 1 for file in fileLst ]
    stop_iteration = False

    print "Reading rules and finding the total model size ..."
    fHLst = [ open(file, 'r') for file in fileLst ]
    while True:
        stop_iteration = True
        for f_track in fileTrackLst:
            if f_track != 9:
                stop_iteration = False
                break

        if stop_iteration:
            break

        for indx, f_track in enumerate( fileTrackLst ):
            if f_track == 0 or f_track == 9:
                continue

            fileTrackLst[indx] = 0
            line = fHLst[indx].readline()
            line = line.strip()
            if line == '':
                fileTrackLst[indx] = 9          # Set 9 if 'EOF' is reached
                continue

            (src, tgt, _) = line.split(' ||| ', 2)
            rule = src + ' ||| ' + tgt

            if ruleDict.has_key(rule):
                ruleDict[rule] += [indx]
            else:
                ruleDict[rule] = [indx]
                heapq.heappush(candLst, rule)
                if src.find('X__2') == -1:
                    rulesDict1NT[rule] = 1

        if len(candLst) == 0: continue

        popped_rule = heapq.heappop(candLst)
        total_rules += 1
        total_1nt_rules += rulesDict1NT.pop(popped_rule, 0)

        for indx1 in ruleDict.pop(popped_rule):
            fileTrackLst[indx1] = 1

    for fH in fHLst:
        fH.close()
    sys.stdout.write( "Size of the original 2NT model   : %d\n" % (total_rules) )
    sys.stdout.write( "Size of the slimmer 1NT model    : %d\n" % (total_1nt_rules) )
dataset/ETHPy150Open sfu-natlang/Kriya/src/Kriya-Grammar-Xtrctr/fullModelSize.py/read_n_merge/original