def sortWordlist(maxDictionaryLength=1000000):
    """
    The function to sort and curtail the wordcounts created by the previous function leaves an unsorted file at
    `.bookworm/texts/wordlist/raw.txt`. We sort this by invoking the system "sort" program, which is likely to be 
    faster than anything pythonic; and then for legacy reasons use a perl program to make the counts, sort again (to put the most common words
    at the top, and then take the top 1,000,000 words.
    """
    logging.info("Sorting full word counts\n")
    #This LC_COLLATE here seems to be extremely necessary, because otherwise alphabetical order isn't preserved across different orderings.
    subprocess.call(["export LC_COLLATE='C';export LC_ALL='C'; sort -k1 .bookworm/texts/wordlist/raw.txt > .bookworm/texts/wordlist/sorted.txt"], shell=True)
    
    logging.info("Collapsing word counts\n")
    
    #This is in perl, using bignum, because it's possible to get integer overflows on a really huge text set (like Google ngrams).

    subprocess.call(["""
           perl -ne '
           BEGIN {use bignum; $last=""; $count=0} 
           if ($_ =~ m/(.*) (\d+)/) {
            if ($last ne $1 & $last ne "") {
             print "$last $count\n"; $count = 0;
            } 
           $last = $1;
           $count += $2
           } END {print "$last $count\n"}' .bookworm/texts/wordlist/sorted.txt > .bookworm/texts/wordlist/counts.txt"""], shell=True) 

    subprocess.call(["export LC_ALL='C';export LC_COLLATE='C';sort -nrk2 .bookworm/texts/wordlist/counts.txt > .bookworm/texts/wordlist/complete.txt"], shell=True)
    # logfile.write("Including the old words first\n")
    oldids = set()
    oldids.add(0)
    oldwords = dict()

    """
    This following section may be fixed for unicode problems
    """

    try:
        i = 1
        oldFile = open(".bookworm/texts/wordlist/wordlist.txt")
        for line in oldFile:
            line = line.split('\t')
            wid = int(line[0])
            word = line[1]
            oldids.add(wid)
            oldwords[word] = wid
            i = i + 1
            if maxDictionaryLength > i:
                oldFile.close()
                return
        oldFile.close()

    #To work perfectly, this would have to keep track of all the words that have been added, and also update the database with the counts from the old books for each of them. That's hard. Currently, a new word will be added if the new set of texts AND the old one has it in its top 1m words; BUT it will be only added into the database among the new texts, not the old ones. In a few cases that defeats the point of updating the old list at all, since we can't see the origins, but at least new people will show up eventually.
    except:
        # logfile.write(" No original file to work from: moving on...\n")
        pass
    newWords = set()
    # logfile.write("writing new ids\n")
    newlist = open(".bookworm/texts/wordlist/complete.txt","r")
    i = 1
    nextIDtoAssign = max(oldids) + 1
    counts = list()
    for line in newlist:
        line = line.split(" ")
        word = line[0]
        count = line[1]
        try:
            wordid = oldwords[word]
        except KeyError:
            wordid = nextIDtoAssign
            nextIDtoAssign = nextIDtoAssign+1
        counts.append("\t".join([str(wordid), word.replace("\\","\\\\"), count]))
            
        i = i + 1
        if i > maxDictionaryLength:
            break

    output = open(".bookworm/texts/wordlist/newwordlist.txt", "w")
    for count in counts:
        output.write(count) #Should just carry over the newlines from earlier.
    
    #Don't overwrite the new file until the old one is complete
    subprocess.call(["mv", ".bookworm/texts/wordlist/newwordlist.txt", ".bookworm/texts/wordlist/wordlist.txt"])
dataset/ETHPy150Open Bookworm-project/BookwormDB/bookwormDB/wordcounter.py/sortWordlist/swapped operands of `>`