def parse_names_files(zfile, N_names, end_year):
    def parse_state(string):
        data = defaultdict(lambda: defaultdict(int))
        for line in string.strip().split('\r\n'):
            state, gender, year, name, count = line.split(',')
            if gender.lower() == 'm' and int(year) <= end_year:
                data[name.lower()][int(year)] += int(count)
        return data

    def get_years(dct):
        sum = lambda lst: reduce(operator.or_, lst)
        return sorted(sum(set(dd.keys()) for d in dct.values() for dd in d.values()))

    def get_top_names(dct, N_names):
        counts = defaultdict(int)
        for statedict in dct.values():
            for name, yeardict in statedict.iteritems():
                counts[name] += sum(yeardict.values())  # sum over years
        return sorted(counts.keys(), key=counts.__getitem__, reverse=True)[:N_names]

    def get_data_array(dct, years, states, names):
        counts = np.zeros((len(years), len(states), len(names)))
        for year_idx, year in enumerate(years):
            for state_idx, state in enumerate(states):
                for name_idx, name in enumerate(names):
                    counts[year_idx, state_idx, name_idx] = \
                        dct[state][name][year]
        return counts

    # dct[state][name][year] = count
    keys = [key for key in zfile.namelist() if key.endswith('.TXT')]
    dct = {key[:2]: parse_state(zfile.read(key)) for key in keys}

    years = get_years(dct)
    states = sorted(dct.keys())
    names = get_top_names(dct, N_names)
    data = get_data_array(dct, years, states, names)

    return data, years, states, names
dataset/ETHPy150Open HIPS/pgmult/experiments/census_gp.py/parse_names_files/original