import re


def parse_tweet_text(tweet_text):
    """
    Input:  tweet_text: a string with the text of a single tweet
                        or a concatenation of tweets
                
    Output: lists of tokens in the text:
              words (many emoticons are recognized as words)
              hashtags
              users mentioned
              urls
              
    Usage: words, hashes, users, urls = parse_tweet_text(tweet_text)
    """
    
    content = tweet_text
           
    # collect and remove URLs
    urls    = re.findall(r"\b((?:https?|ftp|file)://[-A-Z0-9+&@#/%?=~_|$!:,.;]*[A-Z0-9+&@#/%=~_|$])", content, re.IGNORECASE)
    content = re.sub(r"\b((?:https?|ftp|file)://[-A-Z0-9+&@#/%?=~_|$!:,.;]*[A-Z0-9+&@#/%=~_|$])", "", content, 0, re.IGNORECASE)
    
    content = content.lower()
    
    # collect and remove users mentioned
    users   = re.findall(r"@(\w+)", content)
    content = re.sub(r"@(\w+)", "", content, 0)
   
    # collect and remove hashtags
    hashes  = re.findall(r"#(\w+)", content)
    content = re.sub(r"#(\w+)", "", content, 0)
    
    # strip out extra whitespace in the remaining text
    content = re.sub(r"\s{2,}", " ", content)
    
    # strip out singleton punctuation
    raw_words   = content.split()
    words = []
    for word in raw_words:
        if word in ['.',':','!',',',';',"-","-","?",'\xe2\x80\xa6',"!","|",'"','~','..','/']: continue
        re_pattern = re.compile(u'[^\u0000-\uD7FF\uE000-\uFFFF]', re.UNICODE)
        word       = re_pattern.sub(u'\uFFFD', word) 
        #if word.encode('utf-8') in ['\xe2\x80\xa6']: continue
        
        # remove trailing commas, periods, question marks, colons, exclamation marks
        word = re.sub(r"(.*)[,\.,\?,:,!]$", r"\1", word, 0, re.MULTILINE)
        words.append(word)
        
    return (words, hashes, users, urls)