def check_for_differences():    
    base_path = os.path.abspath("./data") 
    onlyfiles = [ f for f in listdir(base_path) if isfile(join(base_path,f)) ]
    headers = {}
    # for each of the file 
    for filepath in onlyfiles:        
        #form the full file path
        refFile_path = os.path.join(base_path, filepath)

        #get the file name without extension
        filename, _ = os.path.splitext(filepath)

        #open the csv file   
        with open(refFile_path,'rb') as f:
            #decode file first
            sr = Recoder(f, 'utf-16le', 'utf-8')
            #only get the first row
            headerRow = csv.reader(sr).next()
            #force to replace 
        headers[filename] = headerRow   
    
    
    #get all the unique column names    
    columnList = []
    for header in headers:          
        columnName = headers[header][0].split("\t")       
        
        #remove empty string from list 
        columnName = filter(len, columnName)
        numOfColumn = len(columnName)
        columnList.extend(columnName)


    uniqueColumns = set(columnList)
    print "Total number of unique columns: ", len(uniqueColumns)
    # print them out
    for i in uniqueColumns:
        if 'ENGLISH CATEGORY' in i:
            i = '"ENGLISH CATEGORY"'
        print unicode(i.decode('UTF-8'))

    #same column names among 79 tables 
    sameColumnList = []
    firstTime = True
    for header in headers:
        if firstTime:
            sameColumnList = headers[header][0].split("\t")
            sameColumnList = filter(len, sameColumnList) #remove empty string from list    
            firstTime = False
            continue
        
        columnName = headers[header][0].split("\t")      
        #remove empty string from list 
        columnName = filter(len, columnName)        
        #check if it existed in previous csv file
        sameColumnList = set(columnName) & set(sameColumnList)

    sameColumnList = set(sameColumnList)
    print "Total number of same columns among all csv: ", len(sameColumnList)
    # print them out
    for i in sameColumnList:
        #Because the csv file in utf-16-le, the first column header
        # with extra string \ufeff e.g. \ufeff"ENGLISH CATEGORY"
        # it needs to remove \ufeff for the first column header
        if 'ENGLISH CATEGORY' in i:
            i = '"ENGLISH CATEGORY"'
        print unicode(i.decode('UTF-8'))
dataset/ETHPy150Open gazetteerhk/census_explorer/scripts/checkPubFacilityCSV.py/check_for_differences/original