def tokenize(document, stopwords=None):
    """
    Tokenize and return (docid, word) pairs with a counter.
    """

    def inner_tokenizer(lines):
        """
        Inner generator for word tokenization.
        """
        for line in lines:
            for word in re.split(tokenizer, line):
                if word and stopwords.value not in word and word.isalpha():
                    yield word

    docid, lines = document
    return [
        ((docid, word), 1) for word in inner_tokenizer(lines)
    ]
dataset/ETHPy150Open bbengfort/hadoop-fundamentals/tfidf/spark-app.py/tokenize/swapped operands of `not in`