title: How to test Java-Spark using JUNit?
body: <p>I cannot figure out how to unit-test java spark app...</p>
  
  <p>I have a simple test here, but it fails with Jackson exception.  I've also tried setting the master to "local"</p>
  
  <pre><code>    private JavaSparkContext sparkCtx;
  
      @Before
      public void init() throws IllegalArgumentException, IOException {
          //ctxtBuilder = new ContextBuilder(tempFolder);
          SparkConf conf = new SparkConf();
          conf.setMaster("local[2]");
          conf.setAppName("junit");
          sparkCtx = new JavaSparkContext(conf);     
      }
  
      @Test
      public void test() {
          final List&lt;Integer&gt; nums = new ArrayList&lt;Integer&gt;();
          nums.add(3);
          nums.add(4);
          nums.add(2);
          JavaRDD&lt;Integer&gt; rdd = sparkCtx.parallelize(nums,1);
          assertEquals(3, rdd.count());
  
      }
  </code></pre>
  
  <hr>
  
  <p>Exception</p>
  
  <pre><code>java.lang.VerifyError: class com.fasterxml.jackson.module.scala.ser.ScalaIteratorSerializer overrides final method withResolved.(Lcom/fasterxml/jackson/databind/BeanProperty;Lcom/fasterxml/jackson/databind/jsontype/TypeSerializer;Lcom/fasterxml/jackson/databind/JsonSerializer;)Lcom/fasterxml/jackson/databind/ser/std/AsArraySerializerBase;
      at java.lang.ClassLoader.defineClass1(Native Method)
      at java.lang.ClassLoader.defineClass(ClassLoader.java:760)
      at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
      at java.net.URLClassLoader.defineClass(URLClassLoader.java:467)
      at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
      at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
      at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
      at java.security.AccessController.doPrivileged(Native Method)
      at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
      at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
      at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
      at com.fasterxml.jackson.module.scala.ser.IteratorSerializerModule$class.$init$(IteratorSerializerModule.scala:70)
      at com.fasterxml.jackson.module.scala.DefaultScalaModule.&lt;init&gt;(DefaultScalaModule.scala:19)
      at com.fasterxml.jackson.module.scala.DefaultScalaModule$.&lt;init&gt;(DefaultScalaModule.scala:35)
      at com.fasterxml.jackson.module.scala.DefaultScalaModule$.&lt;clinit&gt;(DefaultScalaModule.scala)
      at org.apache.spark.rdd.RDDOperationScope$.&lt;init&gt;(RDDOperationScope.scala:78)
      at org.apache.spark.rdd.RDDOperationScope$.&lt;clinit&gt;(RDDOperationScope.scala)
      at org.apache.spark.SparkContext.withScope(SparkContext.scala:700)
      at org.apache.spark.SparkContext.parallelize(SparkContext.scala:714)
      at org.apache.spark.api.java.JavaSparkContext.parallelize(JavaSparkContext.scala:145)
      at sal_mr.ingestion.storage.IngestionOutputFormatTest.test(IngestionOutputFormatTest.java:38)
      at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:497)
      at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47)
      at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
      at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44)
      at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
      at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
      at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:271)
      at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70)
      at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50)
      at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238)
      at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63)
      at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236)
      at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53)
      at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229)
      at org.junit.runners.ParentRunner.run(ParentRunner.java:309)
      at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:86)
      at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
      at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:459)
      at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:675)
      at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:382)
      at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:192)
  </code></pre>
  
  <hr>
  
  <p>Console output:</p>
  
  <pre><code>2016-03-23 09:56:58 INFO  SparkContext:59 - Running Spark version 1.5.0
  2016-03-23 09:56:59 INFO  Utils:59 - Successfully started service 'sparkDriver' on port 38028.
  2016-03-23 09:57:00 INFO  Utils:59 - Successfully started service 'SparkUI' on port 4040.
  2016-03-23 09:57:00 INFO  Executor:59 - Starting executor ID driver on host localhost
  2016-03-23 09:57:00 INFO  Utils:59 - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 35071.
  2016-03-23 09:57:00 INFO  BlockManagerMasterEndpoint:59 - Registering block manager localhost:35071 with 1918.1 MB RAM, BlockManagerId(driver, localhost, 35071)
  2016-03-23 09:57:00 INFO  BlockManagerMaster:59 - Registered BlockManager
  2016-03-23 09:57:00 INFO  SparkContext:59 - Invoking stop() from shutdown hook
  </code></pre>