title: How can I use Spark to tokenize remote documents where I store only the URL?
body: <p>I'm a Spark newbie just reading a few books. Is it possible to tokenize remote documents with only their url? For example would there be a may to modify this script (taken from <a href="http://blog.knoldus.com/2014/06/12/tutorial-how-to-build-a-tokenizer-in-spark-and-scala/" rel="nofollow">this blog post</a>):</p>
  
  <pre><code>import org.apache.spark.SparkContext
  import org.apache.spark.SparkContext._
  
  object TokenizerApp {
   def main(args: Array[String]) {
   val logFile = "src/data/sample.txt" // Should be some file on your system
   val sc = new SparkContext("local", "Tokenizer App", "/path/to/spark-0.9.1-incubating",
   List("target/scala-2.10/simple-project_2.10-1.0.jar"))
   val logData = sc.textFile(logFile, 2).cache()
   val tokens = sc.textFile(logFile, 2).flatMap(line =&gt; line.split(" "))
   val termFrequency = tokens.map(word =&gt; (word, 1)).reduceByKey((a, b) =&gt; a + b)
   termFrequency.collect.map(tf =&gt; println("Term, Frequency: " + tf))
   tokens.saveAsTextFile("src/data/tokens")
   termFrequency.saveAsTextFile("src/data/term_frequency")
   }
  }
  </code></pre>