title: Scraping Reddit in R with RedditExtractoR
body: <p>I'm trying to scrape Reddit data (I'm pretty new to web scraping and half decent at R). The <a href="https://cran.r-project.org/web/packages/RedditExtractoR/RedditExtractoR.pdf" rel="nofollow">RedditExtractor</a> package has a nice function that does 90% of what I need, but it doesn't grab the "flair" associated with users who make comments. I'm trying to play around with the package's function but I'm a bit over my head.</p>
  
  <p>There are examples of Reddit threads with flairs <a href="https://www.reddit.com/r/Christianity/comments/4m8qgv/i_love_orthodoxy_but_sometimes_their_political/" rel="nofollow">here</a>. I think I'm looking for the text in these bits of XML:</p>
  
  <pre><code>&lt;span class="flair flair-orthodox" title="Eastern Orthodox"&gt;Eastern Orthodox&lt;/span&gt;
  </code></pre>
  
  <p>I've pasted the code from the <code>reddit_content()</code> function along with comments where I think the extra code should go, but I'm not quite sure where to go from here. At the moment the function returns a data frame with columns for the comment, time stamp, user, etc. I need it to also produce a comment with user flairs if they exist. Thanks in advance!</p>
  
  <pre><code>redd_content_flair &lt;- function (URL, wait_time = 2) 
  {
    if (is.null(URL) | length(URL) == 0 | !is.character(URL)) {
      stop("invalid URL parameter")
    }
    GetAttribute = function(node, feature) {
      Attribute = node$data[[feature]]
      replies = node$data$replies
      reply.nodes = if (is.list(replies)) 
        replies$data$children
      else NULL
      return(list(Attribute, lapply(reply.nodes, function(x) {
        GetAttribute(x, feature)
      })))
    }
    get.structure = function(node, depth = 0) {
      if (is.null(node)) {
        return(list())
      }
      filter = is.null(node$data$author)
      replies = node$data$replies
      reply.nodes = if (is.list(replies)) 
        replies$data$children
      else NULL
      return(list(paste0(filter, " ", depth), lapply(1:length(reply.nodes), 
                                                     function(x) get.structure(reply.nodes[[x]], paste0(depth, 
                                                                                                        "_", x)))))
    }
    data_extract = data.frame(id = numeric(), structure = character(), 
                              post_date = as.Date(character()), comm_date = as.Date(character()), 
                              num_comments = numeric(), subreddit = character(), upvote_prop = numeric(), 
                              post_score = numeric(), author = character(), user = character(), 
                              comment_score = numeric(), controversiality = numeric(), 
                              comment = character(), title = character(), post_text = character(), 
                              link = character(), domain = character(), 
                              #flair = character(),
                              URL = character())
    pb = utils::txtProgressBar(min = 0, max = length(URL), style = 3)
    for (i in seq(URL)) {
      if (!grepl("^https?://(.*)", URL[i])) 
        URL[i] = paste0("https://www.", gsub("^.*(reddit\\..*$)", 
                                             "\\1", URL[i]))
      if (!grepl("\\?ref=search_posts$", URL[i])) 
        URL[i] = paste0(gsub("/$", "", URL[i]), "/?ref=search_posts")
      X = paste0(gsub("\\?ref=search_posts$", "", URL[i]), 
                 ".json?limit=500")
      raw_data = tryCatch(RJSONIO::fromJSON(readLines(X, warn = FALSE)), 
                          error = function(e) NULL)
      if (is.null(raw_data)) {
        Sys.sleep(min(1, wait_time))
        raw_data = tryCatch(RJSONIO::fromJSON(readLines(X, 
                                                        warn = FALSE)), error = function(e) NULL)
      }
      if (is.null(raw_data) == FALSE) {
        meta.node = raw_data[[1]]$data$children[[1]]$data
        main.node = raw_data[[2]]$data$children
        if (min(length(meta.node), length(main.node)) &gt; 0) {
          structure = unlist(lapply(1:length(main.node), 
                                    function(x) get.structure(main.node[[x]], x)))
          TEMP = data.frame(id = NA, structure = gsub("FALSE ", 
                                                      "", structure[!grepl("TRUE", structure)]), 
                            post_date = format(as.Date(as.POSIXct(meta.node$created_utc, 
                                                                  origin = "1970-01-01")), "%d-%m-%y"),
                            comm_date = format(as.Date(as.POSIXct(unlist(lapply(main.node, 
                                                                                                                                                            function(x) {
                                                                                                                                                              GetAttribute(x, "created_utc")
                                                                                                                                                            })), origin = "1970-01-01")), "%d-%m-%y"), 
                            num_comments = meta.node$num_comments,
                            subreddit = ifelse(is.null(meta.node$subreddit), 
                                                                                      "UNKNOWN", meta.node$subreddit), upvote_prop = meta.node$upvote_ratio, 
                            post_score = meta.node$score, author = meta.node$author, 
                            user = unlist(lapply(main.node, function(x) {
                              GetAttribute(x, "author")
                            })),
                            comment_score = unlist(lapply(main.node, 
                                                               function(x) {
                                                                 GetAttribute(x, "score")
                                                               })),
                            controversiality = unlist(lapply(main.node, 
                                                                                                     function(x) {
                                                                                                       GetAttribute(x, "controversiality")
                                                                                                     })),
                            comment = unlist(lapply(main.node, function(x) {
                                                                                                       GetAttribute(x, "body")
                                                                                                     })),
                            title = meta.node$title, post_text = meta.node$selftext, 
                            link = meta.node$url, domain = meta.node$domain, 
                            #flair = unlist(lapply(main.node, function(x) {GetAttribute(x, "flair")})),
                            URL = URL[i], stringsAsFactors = FALSE)
          TEMP$id = 1:nrow(TEMP)
          if (dim(TEMP)[1] &gt; 0 &amp; dim(TEMP)[2] &gt; 0) 
            data_extract = rbind(TEMP, data_extract)
          else print(paste("missed", i, ":", URL[i]))
        }
      }
      utils::setTxtProgressBar(pb, i)
      Sys.sleep(min(2, wait_time))
    }
    close(pb)
    return(data_extract)
  }
  </code></pre>
  
  <p>Edit: I'd also like to grab the URL for the "parent" comment, which looks like its in tags like</p>
  
  <pre><code>&lt;p class="parent"&gt;&lt;a name="d3t1p1r"&gt;&lt;/a&gt;&lt;/p&gt;
  </code></pre>