def WordsTableCreate(maxDictionaryLength=1000000, maxMemoryStorage=20000000):
    """
    This function reads already-tokenized words from sys.stdin,
    and uses them to count all the words in the document.

    The only real challenge here is that there may be more unique words than can fit
    in memory, so we need some kind of buffering on disk.

    It does this by maintaining a dictionary in memory, and periodically 
    writing the least used portions of that to disk; once it's read everything,
    it writes everything to disk, sorts the list of all words so that 
    """
    database = open('.bookworm/texts/wordlist/raw.txt','w')
    start_time = timeit.default_timer()
    n = 1
    #When flushing the dictionary to disk, they are kept in memory if there count is
    #greater than keepThreshold to avoid overhead reinitializing and rewriting.
    keepThreshold=2

    wordcounts = dict()
    for row in sys.stdin:
        for item in row.split(" "):
            n+=1
            if n % 100000000==0:
                elapsed = timeit.default_timer() - start_time
                logging.info(str(float(len(wordcounts))/1000000) + " million distinct entries at " + str(float(n)/1000000000) + " billion words---" + str(elapsed) + " seconds since last print")
                start_time = timeit.default_timer()
            item = item.rstrip("\n")
            try: 
                wordcounts[item] += 1
            except KeyError:
                wordcounts[item] = 1

            while len(wordcounts) > maxMemoryStorage:
                logging.info("exporting to disk at " + str(float(len(wordcounts))/1000000) + " million words")
                wordcounts = exportToDisk(wordcounts,diskFile=database,keepThreshold=keepThreshold)
                logging.info("after export, it's " + str(1000000/float(len(wordcounts))) + " million words")
                if len(wordcounts) > .8*float(maxMemoryStorage):
                    #If that's not enough to get down to a small dictionary,
                    #try again with a new higher limit.
                    keepThreshold = keepThreshold*2
                    logging.info("upping the keep threshold to " + str(keepThreshold))

    #Write all remaining items to disk
    nothing = exportToDisk(wordcounts,diskFile=database,keepThreshold=float("inf"))
    database.close()
    sortWordlist(maxDictionaryLength=maxDictionaryLength)
dataset/ETHPy150Open Bookworm-project/BookwormDB/bookwormDB/wordcounter.py/WordsTableCreate/swapped operands of `/`