def data(path, D):
    ''' GENERATOR: Robotly hash-trick to the original csv row
                   and for simplicity, we one-hot-encode everything

        INPUT:
            path: path to training or testing file
            D: the max index that we can hash to

        YIELDS:
            ID: id of the instance, mainly useless
            x: a list of hashed and one-hot-encoded 'indices'
               we only need the index since all values are either 0 or 1
            y: y = 1 if we have a click, else we have y = 0
    '''

    for t, row in enumerate(DictReader(open(path))):
        # process id
        ID = row['id']
        del row['id']

        # process clicks
        y = 0.
        if 'click' in row:
            if row['click'] == '1':
                y = 1.
            del row['click']

        # extract date
        date = int(row['hour'][4:6])
        row["C28"] = convt(row["C28"],"C28")
        del row["hour"]
        row["ips"] = row["device_ip"]
        if row["device_ip"][-3:] == "ips":
            row["device_ip"] = "ips"
            row["ips"] = row["ips"][:-3]
        row["ipcate"] = "ipcate_null"
        if row["ips"][2:] in ipcate:
            row["ipcate"] = "ipcate_" + str(ipcate[row["ips"][2:]])
        # turn hour really into hour, it was originally YYMMDDHH
        isApp = row["site_id"] == "c_85f751fd" 
        if isApp:
            del row["site_id"]
            del row["site_domain"]
            del row["site_category"]
        else:
            del row["app_id"]
            del row["app_domain"]
            del row["app_category"]
        # build x
        x = []
        if interaction:
            if not isApp:
                for pair in inter:
                    x.append(row[pair[0]] + "_" + row[pair[1]])
            else:
                for pair in inter2:
                    x.append(row[pair[0]] + "_" + row[pair[1]])
        del row["ips"]
        for key in row:
            value = row[key]
            # one-hot encode everything with hash trick
            x.append(value)

        yield t, date, ID, x, y, isApp


##############################################################################
# start training #############################################################
##############################################################################
dataset/ETHPy150Open infinitezxc/kaggle-avazu/script/ftrl_2.py/data/original