scheduler

Type Members

1. class AccumulableInfo extends AnyRef

   

   :: DeveloperApi :: Information about an org.apache.spark.Accumulable modified during a task or stage.

   :: DeveloperApi :: Information about an org.apache.spark.Accumulable modified during a task or stage.

   Annotations

   @DeveloperApi()

2. class InputFormatInfo extends Logging

   

   :: DeveloperApi :: Parses and holds information about inputFormat (and files) specified as a parameter.

   :: DeveloperApi :: Parses and holds information about inputFormat (and files) specified as a parameter.

   Annotations

   @DeveloperApi()

3. sealed trait JobResult extends AnyRef

   

   :: DeveloperApi :: A result of a job in the DAGScheduler.

   :: DeveloperApi :: A result of a job in the DAGScheduler.

   Annotations

   @DeveloperApi()

4. trait SparkListener extends AnyRef

   

   :: DeveloperApi :: Interface for listening to events from the Spark scheduler.

   :: DeveloperApi :: Interface for listening to events from the Spark scheduler. Note that this is an internal interface which might change in different Spark releases. Java clients should extend JavaSparkListener

   Annotations

   @DeveloperApi()

5. case class SparkListenerApplicationEnd(time: Long) extends SparkListenerEvent with Product with Serializable

   

   Annotations

   @DeveloperApi()

6. case class SparkListenerApplicationStart(appName: String, appId: Option[String], time: Long, sparkUser: String, appAttemptId: Option[String]) extends SparkListenerEvent with Product with Serializable

   

   Annotations

   @DeveloperApi()

7. case class SparkListenerBlockManagerAdded(time: Long, blockManagerId: BlockManagerId, maxMem: Long) extends SparkListenerEvent with Product with Serializable

   

   Annotations

   @DeveloperApi()

8. case class SparkListenerBlockManagerRemoved(time: Long, blockManagerId: BlockManagerId) extends SparkListenerEvent with Product with Serializable

   

   Annotations

   @DeveloperApi()

9. case class SparkListenerEnvironmentUpdate(environmentDetails: Map[String, Seq[(String, String)]]) extends SparkListenerEvent with Product with Serializable

   

   Annotations

   @DeveloperApi()

10. sealed trait SparkListenerEvent extends AnyRef

    

    Annotations

    @DeveloperApi()

11. case class SparkListenerExecutorAdded(time: Long, executorId: String, executorInfo: ExecutorInfo) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

12. case class SparkListenerExecutorMetricsUpdate(execId: String, taskMetrics: Seq[(Long, Int, Int, TaskMetrics)]) extends SparkListenerEvent with Product with Serializable

    

    Periodic updates from executors.

    Periodic updates from executors.

    execId

    executor id

    taskMetrics

    sequence of (task id, stage id, stage attempt, metrics)

    Annotations

    @DeveloperApi()

13. case class SparkListenerExecutorRemoved(time: Long, executorId: String, reason: String) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

14. case class SparkListenerJobEnd(jobId: Int, time: Long, jobResult: JobResult) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

15. case class SparkListenerJobStart(jobId: Int, time: Long, stageInfos: Seq[StageInfo], properties: Properties = null) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

16. case class SparkListenerStageCompleted(stageInfo: StageInfo) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

17. case class SparkListenerStageSubmitted(stageInfo: StageInfo, properties: Properties = null) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

18. case class SparkListenerTaskEnd(stageId: Int, stageAttemptId: Int, taskType: String, reason: TaskEndReason, taskInfo: TaskInfo, taskMetrics: TaskMetrics) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

19. case class SparkListenerTaskGettingResult(taskInfo: TaskInfo) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

20. case class SparkListenerTaskStart(stageId: Int, stageAttemptId: Int, taskInfo: TaskInfo) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

21. case class SparkListenerUnpersistRDD(rddId: Int) extends SparkListenerEvent with Product with Serializable

    

    Annotations

    @DeveloperApi()

22. class SplitInfo extends AnyRef

    

    Annotations

    @DeveloperApi()

23. class StageInfo extends AnyRef

    

    :: DeveloperApi :: Stores information about a stage to pass from the scheduler to SparkListeners.

    :: DeveloperApi :: Stores information about a stage to pass from the scheduler to SparkListeners.

    Annotations

    @DeveloperApi()

24. class StatsReportListener extends SparkListener with Logging

    

    :: DeveloperApi :: Simple SparkListener that logs a few summary statistics when each stage completes

    :: DeveloperApi :: Simple SparkListener that logs a few summary statistics when each stage completes

    Annotations

    @DeveloperApi()

25. class TaskInfo extends AnyRef

    

    :: DeveloperApi :: Information about a running task attempt inside a TaskSet.

    :: DeveloperApi :: Information about a running task attempt inside a TaskSet.

    Annotations

    @DeveloperApi()

26. class JobLogger extends SparkListener with Logging

    

    :: DeveloperApi :: A logger class to record runtime information for jobs in Spark.

    :: DeveloperApi :: A logger class to record runtime information for jobs in Spark. This class outputs one log file for each Spark job, containing tasks start/stop and shuffle information. JobLogger is a subclass of SparkListener, use addSparkListener to add JobLogger to a SparkContext after the SparkContext is created. Note that each JobLogger only works for one SparkContext

    NOTE: The functionality of this class is heavily stripped down to accommodate for a general refactor of the SparkListener interface. In its place, the EventLoggingListener is introduced to log application information as SparkListenerEvents. To enable this functionality, set spark.eventLog.enabled to true.

    Annotations

    @DeveloperApi() @deprecated

    Deprecated

    (Since version 1.0.0) Log application information by setting spark.eventLog.enabled.