def tokens_to_ids(tokenizer, tokens):
    """
    Returns a dict of 'token: id' for tokens in a tokenizer.
    """
    if hasattr(tokenizer, "token_to_id"):
        return {t: tokenizer.token_to_id(t) for t in tokens}
    else:
        return {t: tokenizer.convert_tokens_to_ids(t) for t in tokens}