TaskVals

Spark Measure package: proof-of-concept tool for measuring Spark performance metrics This is based on using Spark Listeners as data source and collecting metrics in a ListBuffer The list buffer is then transformed into a DataFrame for analysis

Stage Metrics: collects and aggregates metrics at the end of each stage Task Metrics: collects data at task granularity

Use modes: Interactive mode from the REPL Flight recorder mode: records data and saves it for later processing

Supported languages: The tool is written in Scala, but it can be used both from Scala and Python

Example usage for stage metrics: val stageMetrics = ch.cern.sparkmeasure.StageMetrics(spark) stageMetrics.runAndMeasure(spark.sql("select count(*) from range(1000) cross join range(1000) cross join range(1000)").show)

for task metrics: val taskMetrics = ch.cern.sparkmeasure.TaskMetrics(spark) spark.sql("select count(*) from range(1000) cross join range(1000) cross join range(1000)").show() val df = taskMetrics.createTaskMetricsDF()

To use in flight recorder mode add: --conf spark.extraListeners=ch.cern.sparkmeasure.FlightRecorderStageMetrics

Created by [email protected], March 2017

Linear Supertypes

Serializable, Serializable, Product, Equals, AnyRef, Any

Instance Constructors

new TaskVals(jobId: Int, jobGroup: String, stageId: Int, index: Long, launchTime: Long, finishTime: Long, duration: Long, schedulerDelay: Long, executorId: String, host: String, taskLocality: Int, speculative: Boolean, gettingResultTime: Long, successful: Boolean, executorRunTime: Long, executorCpuTime: Long, executorDeserializeTime: Long, executorDeserializeCpuTime: Long, resultSerializationTime: Long, jvmGCTime: Long, resultSize: Long, numUpdatedBlockStatuses: Int, diskBytesSpilled: Long, memoryBytesSpilled: Long, peakExecutionMemory: Long, recordsRead: Long, bytesRead: Long, recordsWritten: Long, bytesWritten: Long, shuffleFetchWaitTime: Long, shuffleTotalBytesRead: Long, shuffleTotalBlocksFetched: Long, shuffleLocalBlocksFetched: Long, shuffleRemoteBlocksFetched: Long, shuffleWriteTime: Long, shuffleBytesWritten: Long, shuffleRecordsWritten: Long)

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
val bytesRead: Long
val bytesWritten: Long
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
val diskBytesSpilled: Long
val duration: Long
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
val executorCpuTime: Long
val executorDeserializeCpuTime: Long
val executorDeserializeTime: Long
val executorId: String
val executorRunTime: Long
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
val finishTime: Long
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
val gettingResultTime: Long
val host: String
val index: Long
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
val jobGroup: String
val jobId: Int
val jvmGCTime: Long
val launchTime: Long
val memoryBytesSpilled: Long
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val numUpdatedBlockStatuses: Int
val peakExecutionMemory: Long
val recordsRead: Long
val recordsWritten: Long
val resultSerializationTime: Long
val resultSize: Long
val schedulerDelay: Long
val shuffleBytesWritten: Long
val shuffleFetchWaitTime: Long
val shuffleLocalBlocksFetched: Long
val shuffleRecordsWritten: Long
val shuffleRemoteBlocksFetched: Long
val shuffleTotalBlocksFetched: Long
val shuffleTotalBytesRead: Long
val shuffleWriteTime: Long
val speculative: Boolean
val stageId: Int
val successful: Boolean
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
val taskLocality: Int
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package sparkmeasure

Instance Constructors

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

val bytesRead: Long

val bytesWritten: Long

def clone(): AnyRef

val diskBytesSpilled: Long

val duration: Long

final def eq(arg0: AnyRef): Boolean

val executorCpuTime: Long

val executorDeserializeCpuTime: Long

val executorDeserializeTime: Long

val executorId: String

val executorRunTime: Long

def finalize(): Unit

val finishTime: Long

final def getClass(): Class[_]

val gettingResultTime: Long

val host: String

val index: Long

final def isInstanceOf[T0]: Boolean

val jobGroup: String

val jobId: Int

val jvmGCTime: Long

val launchTime: Long

val memoryBytesSpilled: Long

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val numUpdatedBlockStatuses: Int

val peakExecutionMemory: Long

val recordsRead: Long

val recordsWritten: Long

val resultSerializationTime: Long

val resultSize: Long

val schedulerDelay: Long

val shuffleBytesWritten: Long

val shuffleFetchWaitTime: Long

val shuffleLocalBlocksFetched: Long

val shuffleRecordsWritten: Long

val shuffleRemoteBlocksFetched: Long

val shuffleTotalBlocksFetched: Long

val shuffleTotalBytesRead: Long

val shuffleWriteTime: Long

val speculative: Boolean

val stageId: Int

val successful: Boolean

final def synchronized[T0](arg0: ⇒ T0): T0

val taskLocality: Int

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Serializable

Inherited from Serializable

Inherited from Product

Inherited from Equals

Inherited from AnyRef

Inherited from Any

Ungrouped