-
Notifications
You must be signed in to change notification settings - Fork 19
Description
Possible concurrency problem when using the BufferedOutputStream. Process hangs here:
"Executor task launch worker for task 143" #152 daemon prio=5 os_prio=0 tid=0x00007f8cd0e78000 nid=0x6ae3 runnable [0x00007f8cd4253000]
java.lang.Thread.State: RUNNABLE
at com.ibm.crail.storage.rdma.client.RdmaStoragePassiveEndpoint.write(RdmaStoragePassiveEndpoint.java:194)
at com.ibm.crail.core.CoreOutputStream.trigger(CoreOutputStream.java:110)
at com.ibm.crail.core.CoreStream.prepareAndTrigger(CoreStream.java:238)
at com.ibm.crail.core.CoreStream.dataOperation(CoreStream.java:104)
at com.ibm.crail.core.CoreOutputStream.write(CoreOutputStream.java:67)
at com.ibm.crail.core.DirectoryOutputStream.writeRecord(DirectoryOutputStream.java:53)
at com.ibm.crail.core.CoreFileSystem._createNode(CoreFileSystem.java:211)
at com.ibm.crail.core.CreateNodeFuture.process(CoreMetaDataOperation.java:164)
at com.ibm.crail.core.CreateNodeFuture.process(CoreMetaDataOperation.java:150)
at com.ibm.crail.core.CoreMetaDataOperation.get(CoreMetaDataOperation.java:87)
at com.ibm.crail.core.CoreEarlyFile.file(CoreFile.java:167)
- eliminated <0x00007f8d7bb59890> (a com.ibm.crail.core.CoreEarlyFile)
at com.ibm.crail.core.CoreEarlyFile.getDirectOutputStream(CoreFile.java:104)
- locked <0x00007f8d7bb59890> (a com.ibm.crail.core.CoreEarlyFile)
at com.ibm.crail.CrailBufferedOutputStream.outputStream(CrailBufferedOutputStream.java:329)
at com.ibm.crail.CrailBufferedOutputStream.syncSlice(CrailBufferedOutputStream.java:320)
at com.ibm.crail.CrailBufferedOutputStream.write(CrailBufferedOutputStream.java:124)
at com.ibm.crail.CrailBufferedOutputStream.write(CrailBufferedOutputStream.java:102)
at com.ibm.crail.terasort.serializer.F22SerializerStream.writeObject(F22Serializer.scala:92)
at com.ibm.crail.terasort.serializer.F22SerializerStream.writeValue(F22Serializer.scala:102)
at org.apache.spark.storage.CrailObjectWriter.write(CrailStore.scala:717)
at org.apache.spark.shuffle.crail.CrailShuffleWriter$$anonfun$write$1.apply(CrailShuffleWriter.scala:67)
at org.apache.spark.shuffle.crail.CrailShuffleWriter$$anonfun$write$1.apply(CrailShuffleWriter.scala:65)
at scala.collection.Iterator$class.foreach(Iterator.scala:893)
at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28)
at org.apache.spark.shuffle.crail.CrailShuffleWriter.write(CrailShuffleWriter.scala:65)
at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)
at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)
at org.apache.spark.scheduler.Task.run(Task.scala:108)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:335)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)