@staticmethod
    def get_vocab(tokens, vocab=None):
        """
        Construct vocabulary from the given tokens.

        Args:
            tokens (list) : List of tokens.

        Returns:
            python.set : A set of unique tokens
        """
        # (if vocab is not None, we check that it contains all tokens)
        if vocab is None:
            return set(tokens)
        else:
            vocab = set(vocab)
            assert vocab >= set(tokens), "the predefined vocab must contain all the tokens"
            return vocab
dataset/ETHPy150Open NervanaSystems/neon/neon/data/text.py/Text.get_vocab/original