title: An error occurred while calling o7344.save
body: <p>I am running spark jobs using datafactory in azure databricks.
  My cluster vesion is <strong>9.1 LTS ML (includes Apache Spark 3.1.2, Scala 2.12)</strong>.
  I am writing data on azure blob storage.</p>
  <p>While writing job gives me error.
  It shows error on line
  <strong>AllDataDF.coalesce(1).write.format(&quot;delta&quot;).mode(&quot;append&quot;).option(&quot;mergeSchema&quot;, &quot;true&quot;).save(path)</strong></p>
  <p>And traceback in as below.</p>
  <pre><code>/databricks/spark/python/pyspark/sql/readwriter.py in save(self, path, format, mode, partitionBy, **options)
     1134             self._jwrite.save()
     1135         else:
  -&gt; 1136             self._jwrite.save(path)
     1137 
     1138     @since(1.4)
  
  /databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py in __call__(self, *args)
     1302 
     1303         answer = self.gateway_client.send_command(command)
  -&gt; 1304         return_value = get_return_value(
     1305             answer, self.gateway_client, self.target_id, self.name)
     1306 
  
  /databricks/spark/python/pyspark/sql/utils.py in deco(*a, **kw)
      115     def deco(*a, **kw):
      116         try:
  --&gt; 117             return f(*a, **kw)
      118         except py4j.protocol.Py4JJavaError as e:
      119             converted = convert_exception(e.java_exception)
  
  /databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
      324             value = OUTPUT_CONVERTER[type](answer[2:], gateway_client)
      325             if answer[1] == REFERENCE_TYPE:
  --&gt; 326                 raise Py4JJavaError(
      327                     &quot;An error occurred while calling {0}{1}{2}.\n&quot;.
      328                     format(target_id, &quot;.&quot;, name), value)
  
  Py4JJavaError: An error occurred while calling o7344.save.
  : org.apache.spark.SparkException: Job aborted.
      at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:307)
      at com.databricks.sql.transaction.tahoe.files.TransactionalWriteEdge.$anonfun$writeFiles$5(TransactionalWriteEdge.scala:349)
      at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
      at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withCustomExecutionEnv$5(SQLExecution.scala:130)
      at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:273)
      at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withCustomExecutionEnv$1(SQLExecution.scala:104)
      at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:854)
      at org.apache.spark.sql.execution.SQLExecution$.withCustomExecutionEnv(SQLExecution.scala:77)
      at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:223)
      at com.databricks.sql.transaction.tahoe.files.TransactionalWriteEdge.$anonfun$writeFiles$1(TransactionalWriteEdge.scala:296)
      at com.databricks.spark.util.FrameProfiler$.record(FrameProfiler.scala:80)
      at com.databricks.sql.transaction.tahoe.metering.DeltaLogging.$anonfun$recordDeltaOperation$5(DeltaLogging.scala:122)
      at com.databricks.logging.UsageLogging.$anonfun$recordOperation$1(UsageLogging.scala:395)
      at com.databricks.logging.UsageLogging.executeThunkAndCaptureResultTags$1(UsageLogging.scala:484)
      at com.databricks.logging.UsageLogging.$anonfun$recordOperationWithResultTags$4(UsageLogging.scala:504)
      at com.databricks.logging.UsageLogging.$anonfun$withAttributionContext$1(UsageLogging.scala:266)
      at scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)
      at com.databricks.logging.UsageLogging.withAttributionContext(UsageLogging.scala:261)
      at com.databricks.logging.UsageLogging.withAttributionContext$(UsageLogging.scala:258)
      at com.databricks.spark.util.PublicDBLogging.withAttributionContext(DatabricksSparkUsageLogger.scala:20)
      at com.databricks.logging.UsageLogging.withAttributionTags(UsageLogging.scala:305)
      at com.databricks.logging.UsageLogging.withAttributionTags$(UsageLogging.scala:297)
      at com.databricks.spark.util.PublicDBLogging.withAttributionTags(DatabricksSparkUsageLogger.scala:20)
      at com.databricks.logging.UsageLogging.recordOperationWithResultTags(UsageLogging.scala:479)
      at com.databricks.logging.UsageLogging.recordOperationWithResultTags$(UsageLogging.scala:404)
      at com.databricks.spark.util.PublicDBLogging.recordOperationWithResultTags(DatabricksSparkUsageLogger.scala:20)
      at com.databricks.logging.UsageLogging.recordOperation(UsageLogging.scala:395)
      at com.databricks.logging.UsageLogging.recordOperation$(UsageLogging.scala:367)
      at com.databricks.spark.util.PublicDBLogging.recordOperation(DatabricksSparkUsageLogger.scala:20)
      at com.databricks.spark.util.PublicDBLogging.recordOperation0(DatabricksSparkUsageLogger.scala:57)
      at com.databricks.spark.util.DatabricksSparkUsageLogger.recordOperation(DatabricksSparkUsageLogger.scala:137)
      at com.databricks.spark.util.UsageLogger.recordOperation(UsageLogger.scala:71)
      at com.databricks.spark.util.UsageLogger.recordOperation$(UsageLogger.scala:58)
      at com.databricks.spark.util.DatabricksSparkUsageLogger.recordOperation(DatabricksSparkUsageLogger.scala:98)
      at com.databricks.spark.util.UsageLogging.recordOperation(UsageLogger.scala:429)
      at com.databricks.spark.util.UsageLogging.recordOperation$(UsageLogger.scala:408)
      at com.databricks.sql.transaction.tahoe.OptimisticTransaction.recordOperation(OptimisticTransaction.scala:98)
      at com.databricks.sql.transaction.tahoe.metering.DeltaLogging.recordDeltaOperation(DeltaLogging.scala:120)
      at com.databricks.sql.transaction.tahoe.metering.DeltaLogging.recordDeltaOperation$(DeltaLogging.scala:104)
      at com.databricks.sql.transaction.tahoe.OptimisticTransaction.recordDeltaOperation(OptimisticTransaction.scala:98)
      at com.databricks.sql.transaction.tahoe.files.TransactionalWriteEdge.writeFiles(TransactionalWriteEdge.scala:213)
      at com.databricks.sql.transaction.tahoe.files.TransactionalWriteEdge.writeFiles$(TransactionalWriteEdge.scala:207)
      at com.databricks.sql.transaction.tahoe.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:98)
      at com.databricks.sql.transaction.tahoe.files.TransactionalWriteEdge.writeFiles(TransactionalWriteEdge.scala:389)
      at com.databricks.sql.transaction.tahoe.files.TransactionalWriteEdge.writeFiles$(TransactionalWriteEdge.scala:382)
      at com.databricks.sql.transaction.tahoe.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:98)
      at com.databricks.sql.transaction.tahoe.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:158)
      at com.databricks.sql.transaction.tahoe.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:155)
      at com.databricks.sql.transaction.tahoe.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:98)
      at com.databricks.sql.transaction.tahoe.commands.WriteIntoDelta.write(WriteIntoDelta.scala:175)
      at com.databricks.sql.transaction.tahoe.commands.WriteIntoDelta.$anonfun$run$2(WriteIntoDelta.scala:91)
      at com.databricks.sql.transaction.tahoe.commands.WriteIntoDelta.$anonfun$run$2$adapted(WriteIntoDelta.scala:83)
      at com.databricks.sql.transaction.tahoe.DeltaLog.withNewTransaction(DeltaLog.scala:206)
      at com.databricks.sql.transaction.tahoe.commands.WriteIntoDelta.$anonfun$run$1(WriteIntoDelta.scala:83)
      at com.databricks.sql.acl.CheckPermissions$.trusted(CheckPermissions.scala:1413)
      at com.databricks.sql.transaction.tahoe.commands.WriteIntoDelta.run(WriteIntoDelta.scala:82)
      at com.databricks.sql.transaction.tahoe.sources.DeltaDataSource.createRelation(DeltaDataSource.scala:164)
      at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:48)
      at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:75)
      at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:73)
      at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:96)
      at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:213)
      at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:257)
      at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:165)
      at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:253)
      at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:209)
      at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:167)
      at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:166)
      at org.apache.spark.sql.DataFrameWriter.$anonfun$runCommand$1(DataFrameWriter.scala:1080)
      at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withCustomExecutionEnv$5(SQLExecution.scala:130)
      at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:273)
      at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withCustomExecutionEnv$1(SQLExecution.scala:104)
      at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:854)
      at org.apache.spark.sql.execution.SQLExecution$.withCustomExecutionEnv(SQLExecution.scala:77)
      at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:223)
      at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:1080)
      at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:469)
      at org.apache.spark.sql.DataFrameWriter.saveInternal(DataFrameWriter.scala:386)
      at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:304)
      at sun.reflect.GeneratedMethodAccessor531.invoke(Unknown Source)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:498)
      at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
      at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)
      at py4j.Gateway.invoke(Gateway.java:295)
      at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
      at py4j.commands.CallCommand.execute(CallCommand.java:79)
      at py4j.GatewayConnection.run(GatewayConnection.java:251)
      at java.lang.Thread.run(Thread.java:748)
  Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 8352.0 failed 4 times, most recent failure: Lost task 0.3 in stage 8352.0 (TID 10576) (10.139.64.18 executor 17): org.apache.spark.SparkException: Task failed while writing rows.
      at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:396)
      at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:284)
      at org.apache.spark.scheduler.ResultTask.$anonfun$runTask$3(ResultTask.scala:75)
      at com.databricks.spark.util.ExecutorFrameProfiler$.record(ExecutorFrameProfiler.scala:110)
      at org.apache.spark.scheduler.ResultTask.$anonfun$runTask$1(ResultTask.scala:75)
      at com.databricks.spark.util.ExecutorFrameProfiler$.record(ExecutorFrameProfiler.scala:110)
      at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:55)
      at org.apache.spark.scheduler.Task.doRunTask(Task.scala:150)
      at org.apache.spark.scheduler.Task.$anonfun$run$1(Task.scala:119)
      at com.databricks.spark.util.ExecutorFrameProfiler$.record(ExecutorFrameProfiler.scala:110)
      at org.apache.spark.scheduler.Task.run(Task.scala:91)
      at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$13(Executor.scala:813)
      at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1620)
      at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:816)
      at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
      at com.databricks.spark.util.ExecutorFrameProfiler$.record(ExecutorFrameProfiler.scala:110)
      at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:672)
      at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
      at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
      at java.lang.Thread.run(Thread.java:748)
  Caused by: java.util.NoSuchElementException
      at org.apache.spark.sql.vectorized.ColumnarBatch$1.next(ColumnarBatch.java:69)
      at org.apache.spark.sql.vectorized.ColumnarBatch$1.next(ColumnarBatch.java:58)
      at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:44)
      at org.apache.spark.sql.execution.arrow.ArrowConverters$$anon$4.next(ArrowConverters.scala:401)
      at org.apache.spark.sql.execution.arrow.ArrowConverters$$anon$4.next(ArrowConverters.scala:382)
      at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage138.processNext(Unknown Source)
      at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
      at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:757)
      at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage183.processNext(Unknown Source)
      at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
      at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:757)
      at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:488)
      at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeTask$2(FileFormatWriter.scala:374)
      at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1654)
      at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:383)
      ... 19 more
      Suppressed: java.io.IOException: can not write PageHeader(type:DICTIONARY_PAGE, uncompressed_page_size:176, compressed_page_size:101, crc:-1384879305, dictionary_page_header:DictionaryPageHeader(num_values:22, encoding:PLAIN_DICTIONARY))
          at org.apache.parquet.format.Util.write(Util.java:224)
          at org.apache.parquet.format.Util.writePageHeader(Util.java:61)
          at org.apache.parquet.format.converter.ParquetMetadataConverter.writeDictionaryPageHeader(ParquetMetadataConverter.java:1190)
          at org.apache.parquet.hadoop.ParquetFileWriter.writeDictionaryPage(ParquetFileWriter.java:374)
          at org.apache.parquet.hadoop.ColumnChunkPageWriteStore$ColumnChunkPageWriter.writeToFileWriter(ColumnChunkPageWriteStore.java:238)
          at org.apache.parquet.hadoop.ColumnChunkPageWriteStore.flushToFileWriter(ColumnChunkPageWriteStore.java:316)
          at org.apache.parquet.hadoop.InternalParquetRecordWriter.flushRowGroupToStore(InternalParquetRecordWriter.java:202)
          at org.apache.parquet.hadoop.InternalParquetRecordWriter.close(InternalParquetRecordWriter.java:127)
          at org.apache.parquet.hadoop.ParquetRecordWriter.close(ParquetRecordWriter.java:165)
          at org.apache.spark.sql.execution.datasources.parquet.ParquetOutputWriter.close(ParquetOutputWriter.scala:41)
          at org.apache.spark.sql.execution.datasources.FileFormatDataWriter.releaseResources(FileFormatDataWriter.scala:58)
          at org.apache.spark.sql.execution.datasources.FileFormatDataWriter.abort(FileFormatDataWriter.scala:84)
          at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeTask$3(FileFormatWriter.scala:380)
          at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1665)
          ... 20 more
      Caused by: shaded.parquet.org.apache.thrift.transport.TTransportException: java.io.IOException: Stream is closed!
          at shaded.parquet.org.apache.thrift.transport.TIOStreamTransport.write(TIOStreamTransport.java:147)
          at shaded.parquet.org.apache.thrift.transport.TTransport.write(TTransport.java:107)
          at shaded.parquet.org.apache.thrift.protocol.TCompactProtocol.writeByteDirect(TCompactProtocol.java:482)
          at shaded.parquet.org.apache.thrift.protocol.TCompactProtocol.writeByteDirect(TCompactProtocol.java:489)
          at shaded.parquet.org.apache.thrift.protocol.TCompactProtocol.writeFieldBeginInternal(TCompactProtocol.java:252)
          at shaded.parquet.org.apache.thrift.protocol.TCompactProtocol.writeFieldBegin(TCompactProtocol.java:234)
          at org.apache.parquet.format.InterningProtocol.writeFieldBegin(InterningProtocol.java:74)
          at org.apache.parquet.format.PageHeader$PageHeaderStandardScheme.write(PageHeader.java:1068)
          at org.apache.parquet.format.PageHeader$PageHeaderStandardScheme.write(PageHeader.java:966)
          at org.apache.parquet.format.PageHeader.write(PageHeader.java:847)
          at org.apache.parquet.format.Util.write(Util.java:222)
          ... 33 more
      Caused by: java.io.IOException: Stream is closed!
          at shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream.close(AbfsOutputStream.java:344)
          at shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.services.PageCRCVerifyingAbfsOutputStream.close(PageCRCVerifyingAbfsOutputStream.java:48)
          at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72)
          at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106)
          at com.databricks.backend.daemon.data.common.CancellableOutputStream.cancel(CancellableOutputStream.scala:24)
          at com.databricks.backend.daemon.data.client.DatabricksFileSystemV2.$anonfun$create$4(DatabricksFileSystemV2.scala:630)
          at org.apache.spark.SparkUtils$.$anonfun$onTaskFailure$2(SparkUtils.scala:58)
          at org.apache.spark.SparkUtils$.$anonfun$onTaskFailure$2$adapted(SparkUtils.scala:57)
          at org.apache.spark.TaskContext$$anon$2.onTaskFailure(TaskContext.scala:177)
          at org.apache.spark.TaskContextImpl.$anonfun$invokeTaskFailureListeners$1(TaskContextImpl.scala:150)
          at org.apache.spark.TaskContextImpl.$anonfun$invokeTaskFailureListeners$1$adapted(TaskContextImpl.scala:150)
          at org.apache.spark.TaskContextImpl.invokeListeners(TaskContextImpl.scala:197)
          at org.apache.spark.TaskContextImpl.invokeTaskFailureListeners(TaskContextImpl.scala:150)
          at org.apache.spark.TaskContextImpl.markTaskFailed(TaskContextImpl.scala:127)
          at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1663)
          ... 20 more
  
  Driver stacktrace:
      at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2828)
      at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2775)
      at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2769)
      at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
      at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
      at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
      at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2769)
      at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1305)
      at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1305)
      at scala.Option.foreach(Option.scala:407)
      at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1305)
      at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:3036)
      at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2977)
      at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2965)
      at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
      at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:1067)
      at org.apache.spark.SparkContext.runJobInternal(SparkContext.scala:2477)
      at org.apache.spark.SparkContext.runJob(SparkContext.scala:2460)
      at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:274)
      ... 88 more
  Caused by: org.apache.spark.SparkException: Task failed while writing rows.
      at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:396)
      at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$write$15(FileFormatWriter.scala:284)
      at org.apache.spark.scheduler.ResultTask.$anonfun$runTask$3(ResultTask.scala:75)
      at com.databricks.spark.util.ExecutorFrameProfiler$.record(ExecutorFrameProfiler.scala:110)
      at org.apache.spark.scheduler.ResultTask.$anonfun$runTask$1(ResultTask.scala:75)
      at com.databricks.spark.util.ExecutorFrameProfiler$.record(ExecutorFrameProfiler.scala:110)
      at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:55)
      at org.apache.spark.scheduler.Task.doRunTask(Task.scala:150)
      at org.apache.spark.scheduler.Task.$anonfun$run$1(Task.scala:119)
      at com.databricks.spark.util.ExecutorFrameProfiler$.record(ExecutorFrameProfiler.scala:110)
      at org.apache.spark.scheduler.Task.run(Task.scala:91)
      at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$13(Executor.scala:813)
      at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1620)
      at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:816)
      at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
      at com.databricks.spark.util.ExecutorFrameProfiler$.record(ExecutorFrameProfiler.scala:110)
      at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:672)
      at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
      at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
      ... 1 more
  Caused by: java.util.NoSuchElementException
      at org.apache.spark.sql.vectorized.ColumnarBatch$1.next(ColumnarBatch.java:69)
      at org.apache.spark.sql.vectorized.ColumnarBatch$1.next(ColumnarBatch.java:58)
      at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:44)
      at org.apache.spark.sql.execution.arrow.ArrowConverters$$anon$4.next(ArrowConverters.scala:401)
      at org.apache.spark.sql.execution.arrow.ArrowConverters$$anon$4.next(ArrowConverters.scala:382)
      at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage138.processNext(Unknown Source)
      at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
      at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:757)
      at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage183.processNext(Unknown Source)
      at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
      at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:757)
      at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:488)
      at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeTask$2(FileFormatWriter.scala:374)
      at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1654)
      at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:383)
      ... 19 more
      Suppressed: java.io.IOException: can not write PageHeader(type:DICTIONARY_PAGE, uncompressed_page_size:176, compressed_page_size:101, crc:-1384879305, dictionary_page_header:DictionaryPageHeader(num_values:22, encoding:PLAIN_DICTIONARY))
          at org.apache.parquet.format.Util.write(Util.java:224)
          at org.apache.parquet.format.Util.writePageHeader(Util.java:61)
          at org.apache.parquet.format.converter.ParquetMetadataConverter.writeDictionaryPageHeader(ParquetMetadataConverter.java:1190)
          at org.apache.parquet.hadoop.ParquetFileWriter.writeDictionaryPage(ParquetFileWriter.java:374)
          at org.apache.parquet.hadoop.ColumnChunkPageWriteStore$ColumnChunkPageWriter.writeToFileWriter(ColumnChunkPageWriteStore.java:238)
          at org.apache.parquet.hadoop.ColumnChunkPageWriteStore.flushToFileWriter(ColumnChunkPageWriteStore.java:316)
          at org.apache.parquet.hadoop.InternalParquetRecordWriter.flushRowGroupToStore(InternalParquetRecordWriter.java:202)
          at org.apache.parquet.hadoop.InternalParquetRecordWriter.close(InternalParquetRecordWriter.java:127)
          at org.apache.parquet.hadoop.ParquetRecordWriter.close(ParquetRecordWriter.java:165)
          at org.apache.spark.sql.execution.datasources.parquet.ParquetOutputWriter.close(ParquetOutputWriter.scala:41)
          at org.apache.spark.sql.execution.datasources.FileFormatDataWriter.releaseResources(FileFormatDataWriter.scala:58)
          at org.apache.spark.sql.execution.datasources.FileFormatDataWriter.abort(FileFormatDataWriter.scala:84)
          at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeTask$3(FileFormatWriter.scala:380)
          at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1665)
          ... 20 more
      Caused by: shaded.parquet.org.apache.thrift.transport.TTransportException: java.io.IOException: Stream is closed!
          at shaded.parquet.org.apache.thrift.transport.TIOStreamTransport.write(TIOStreamTransport.java:147)
          at shaded.parquet.org.apache.thrift.transport.TTransport.write(TTransport.java:107)
          at shaded.parquet.org.apache.thrift.protocol.TCompactProtocol.writeByteDirect(TCompactProtocol.java:482)
          at shaded.parquet.org.apache.thrift.protocol.TCompactProtocol.writeByteDirect(TCompactProtocol.java:489)
          at shaded.parquet.org.apache.thrift.protocol.TCompactProtocol.writeFieldBeginInternal(TCompactProtocol.java:252)
          at shaded.parquet.org.apache.thrift.protocol.TCompactProtocol.writeFieldBegin(TCompactProtocol.java:234)
          at org.apache.parquet.format.InterningProtocol.writeFieldBegin(InterningProtocol.java:74)
          at org.apache.parquet.format.PageHeader$PageHeaderStandardScheme.write(PageHeader.java:1068)
          at org.apache.parquet.format.PageHeader$PageHeaderStandardScheme.write(PageHeader.java:966)
          at org.apache.parquet.format.PageHeader.write(PageHeader.java:847)
          at org.apache.parquet.format.Util.write(Util.java:222)
          ... 33 more
      Caused by: java.io.IOException: Stream is closed!
          at shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream.close(AbfsOutputStream.java:344)
          at shaded.databricks.azurebfs.org.apache.hadoop.fs.azurebfs.services.PageCRCVerifyingAbfsOutputStream.close(PageCRCVerifyingAbfsOutputStream.java:48)
          at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72)
          at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106)
          at com.databricks.backend.daemon.data.common.CancellableOutputStream.cancel(CancellableOutputStream.scala:24)
          at com.databricks.backend.daemon.data.client.DatabricksFileSystemV2.$anonfun$create$4(DatabricksFileSystemV2.scala:630)
          at org.apache.spark.SparkUtils$.$anonfun$onTaskFailure$2(SparkUtils.scala:58)
          at org.apache.spark.SparkUtils$.$anonfun$onTaskFailure$2$adapted(SparkUtils.scala:57)
          at org.apache.spark.TaskContext$$anon$2.onTaskFailure(TaskContext.scala:177)
          at org.apache.spark.TaskContextImpl.$anonfun$invokeTaskFailureListeners$1(TaskContextImpl.scala:150)
          at org.apache.spark.TaskContextImpl.$anonfun$invokeTaskFailureListeners$1$adapted(TaskContextImpl.scala:150)
          at org.apache.spark.TaskContextImpl.invokeListeners(TaskContextImpl.scala:197)
          at org.apache.spark.TaskContextImpl.invokeTaskFailureListeners(TaskContextImpl.scala:150)
          at org.apache.spark.TaskContextImpl.markTaskFailed(TaskContextImpl.scala:127)
          at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1663)
          ... 20 more
  </code></pre>
  <p>Job runs every month so it should not be a code error.</p>