title: How to process many txt files with my code in R
body: <p>I'm quite a novice, but I've successfully managed to make some code do what I want.
  Right now my code does what I want for one file at a time.
  I want to make my code automate this process for 600 files.
  I kind of have an idea, that I need to put the list of files in a vector, then maybe use lapply and a function, but I'm not sure how to do this. The syntax and code are beyond me at the moment.
  Here's my code...</p>
  <p>#Packages are callled</p>
  <pre><code>library(tm) #text mining
  library(SnowballC) #stemming - reducing words to their root
  library(stringr) #for str_trim
  library(plyr)
  library(dplyr)
  library(readtext)
  </code></pre>
  <p>#this is my code to run the code on a bunch of text files. Obviously it's unfinished, and I'm not sure if this is the right approach. Where do I put this? Will it even work?</p>
  <pre><code>data_files &lt;- list.files(path = &quot;data/&quot;, pattern = '*.txt', full.names = T, recursive = T)
  
  lapply(
  
  # 
  # where do I put this chunk of code?
  # do I need to make all the code below a function?
  </code></pre>
  <p>##this bit cleans the document</p>
  <pre><code>company &lt;- &quot;CompanyXReport2015&quot;
  txt_raw = readLines(&quot;data/CompanyXReport2015.txt&quot;)
  
  # remove all extra white space, also splits on lines
  txt_format1 &lt;- gsub(&quot; *\\b[[:alpha:]]{1,2}\\b *&quot;, &quot; &quot;, txt_raw)
  txt_format1.5 &lt;- gsub(&quot;^ +| +$|( ) +&quot;, &quot;\\1&quot;, txt_format1) 
  
  # recombine now that all white space is stripped
  txt_format2 &lt;- str_c(txt_format1.5, collapse=&quot; &quot;)
  
  #split strings on space now to get a list of all words
  txt_format3 &lt;- str_split(txt_format2,&quot; &quot;)
  txt_format3
  
  # convert to vector
  txt_format4 &lt;- unlist(txt_format3)
  
  # remove empty strings and those with words shorter than 3 length
  txt_format5 &lt;- txt_format4[str_length(txt_format4) &gt; 3]
  
  # combine document back to single string
  cleaned &lt;- str_c(txt_format5, collapse=&quot; &quot;)
  
  head(cleaned, 2)
  </code></pre>
  <p>##import key words and run analysis on frequency for the document</p>
  <pre><code>s1_raw = readLines(&quot;data/stage1r.txt&quot;)
  str(s1_raw)
  s2_raw = readLines(&quot;data/stage2r.txt&quot;)
  str(s2_raw)
  s3_raw = readLines(&quot;data/stage3r.txt&quot;)
  str(s3_raw)
  s4_raw = readLines(&quot;data/stage4r.txt&quot;)
  str(s4_raw)
  s5_raw = readLines(&quot;data/stage5r.txt&quot;)
  str(s5_raw)
  
  # str_count(cleaned, &quot;legal&quot;)
  
  # apply str_count function using each stage vector
  level1 &lt;- sapply(s1_raw, str_count, string=cleaned)
  level2 &lt;- sapply(s2_raw, str_count, string=cleaned)
  level3 &lt;- sapply(s3_raw, str_count, string=cleaned)
  level4 &lt;- sapply(s4_raw, str_count, string=cleaned)
  level5 &lt;- sapply(s5_raw, str_count, string=cleaned)
  
  #make a vector from this for the report later
  wordcountresult &lt;- c(level1,level2,level3,level4,level5)
  
  # convert to dataframes
  s1 &lt;- as.data.frame(level1)
  s2 &lt;- as.data.frame(level2)
  s3 &lt;- as.data.frame(level3)
  s4 &lt;- as.data.frame(level4)
  s5 &lt;- as.data.frame(level5)
  
  # add a count column that each df shares
  s1$count &lt;- s1$level1
  s2$count &lt;- s2$level2
  s3$count &lt;- s3$level3
  s4$count &lt;- s4$level4
  s5$count &lt;- s5$level5
  
  # add a stage column to identify what stage the word is in
  s1$stage &lt;- &quot;Stage 1&quot;
  s2$stage &lt;- &quot;Stage 2&quot;
  s3$stage &lt;- &quot;Stage 3&quot;
  s4$stage &lt;- &quot;Stage 4&quot;
  s5$stage &lt;- &quot;Stage 5&quot;
  
  # drop the unique column
  s1 &lt;- s1[c(&quot;count&quot;,&quot;stage&quot;)]
  s2 &lt;- s2[c(&quot;count&quot;,&quot;stage&quot;)]
  s3 &lt;- s3[c(&quot;count&quot;,&quot;stage&quot;)]
  s4 &lt;- s4[c(&quot;count&quot;,&quot;stage&quot;)]
  s5 &lt;- s5[c(&quot;count&quot;,&quot;stage&quot;)]
  
  # s1
  
  df &lt;- rbind(s1, s2,s3, s4, s5)
  
  
  df
  
  </code></pre>
  <p>#write the summary for each company to a csv</p>
  <pre><code>#Making the report
  
  #Make a vector to put in the report
  
  #get stage counts and make a vector
  s1c &lt;- sum(s1$count)
  s2c &lt;- sum(s2$count)
  s3c &lt;- sum(s3$count)
  s4c &lt;- sum(s4$count)
  s5c &lt;- sum(s5$count)
  stagesvec &lt;- c(s1c,s2c,s3c,s4c,s5c)
  names(stagesvec) &lt;- c(&quot;Stage1&quot;,&quot;Stage2&quot;,&quot;Stage3&quot;,&quot;Stage4&quot;,&quot;Stage5&quot;)
  
  #get the company report name for a vector
  companyvec &lt;- c(company)
  names(companyvec) &lt;- c(&quot;company&quot;)
  
  # combine the vectors for the vector row to be inserted into the report
  reportresult &lt;- c(companyvec, wordcountresult, stagesvec)
  rrdf &lt;- data.frame(t(reportresult))
  newdf &lt;- data.frame(t(reportresult))
  
  #if working file exists-use it
  if (file.exists(&quot;data/WordCount12.csv&quot;)){
      write.csv(
      rrdf,
      &quot;data/WordCountTemp12.csv&quot;, row.names=FALSE
    )
    rrdf2 &lt;-
      read.csv(&quot;data/WordCountTemp12.csv&quot;)
    df2 &lt;-
      read.csv(&quot;data/WordCount12.csv&quot;)
    df2 &lt;- rbind(df2, rrdf2)
    write.csv(df2,
              &quot;data/WordCount12.csv&quot;, row.names=FALSE)
  }else{ #if NO working file exists-make it
    write.csv(newdf,
              &quot;data/WordCount12.csv&quot;, row.names=FALSE)
  }
  </code></pre>