JobProgressListener

Instance Constructors

new JobProgressListener(conf: SparkConf)

Type Members

type ExecutorId = String
type JobGroupId = String
type JobId = Int
type PoolName = String
type StageAttemptId = Int
type StageId = Int

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
val activeJobs: HashMap[JobId, JobUIData]
val activeStages: HashMap[StageId, StageInfo]
final def asInstanceOf[T0]: T0

Definition Classes
Any
def blockManagerIds: Seq[BlockManagerId]
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
val completedJobs: ListBuffer[JobUIData]
val completedStages: ListBuffer[StageInfo]
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
val executorIdToBlockManagerId: HashMap[String, BlockManagerId]
val failedJobs: ListBuffer[JobUIData]
val failedStages: ListBuffer[StageInfo]
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
val jobGroupToJobIds: HashMap[JobGroupId, HashSet[JobId]]
val jobIdToData: HashMap[JobId, JobUIData]
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
var numCompletedJobs: Int
var numCompletedStages: Int
var numFailedJobs: Int
var numFailedStages: Int
def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit

Called when the application ends
Called when the application ends

Definition Classes
SparkListener
def onApplicationStart(appStarted: SparkListenerApplicationStart): Unit

Called when the application starts
Called when the application starts

Definition Classes
JobProgressListener → SparkListener
def onBlockManagerAdded(blockManagerAdded: SparkListenerBlockManagerAdded): Unit

Called when a new block manager has joined
Called when a new block manager has joined

Definition Classes
JobProgressListener → SparkListener
def onBlockManagerRemoved(blockManagerRemoved: SparkListenerBlockManagerRemoved): Unit

Called when an existing block manager has been removed
Called when an existing block manager has been removed

Definition Classes
JobProgressListener → SparkListener
def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit

Called when the driver receives a block update info.
Called when the driver receives a block update info.

Definition Classes
SparkListener
def onEnvironmentUpdate(environmentUpdate: SparkListenerEnvironmentUpdate): Unit

Called when environment properties have been updated
Called when environment properties have been updated

Definition Classes
JobProgressListener → SparkListener
def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit

Called when the driver registers a new executor.
Called when the driver registers a new executor.

Definition Classes
SparkListener
def onExecutorMetricsUpdate(executorMetricsUpdate: SparkListenerExecutorMetricsUpdate): Unit

Called when the driver receives task metrics from an executor in a heartbeat.
Called when the driver receives task metrics from an executor in a heartbeat.

Definition Classes
JobProgressListener → SparkListener
def onExecutorRemoved(executorRemoved: SparkListenerExecutorRemoved): Unit

Called when the driver removes an executor.
Called when the driver removes an executor.

Definition Classes
SparkListener
def onJobEnd(jobEnd: SparkListenerJobEnd): Unit

Called when a job ends
Called when a job ends

Definition Classes
JobProgressListener → SparkListener
def onJobStart(jobStart: SparkListenerJobStart): Unit

Called when a job starts
Called when a job starts

Definition Classes
JobProgressListener → SparkListener
def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit

Called when a stage completes successfully or fails, with information on the completed stage.
Called when a stage completes successfully or fails, with information on the completed stage.

Definition Classes
JobProgressListener → SparkListener
def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit

For FIFO, all stages are contained by "default" pool but "default" pool here is meaningless
For FIFO, all stages are contained by "default" pool but "default" pool here is meaningless

Definition Classes
JobProgressListener → SparkListener
def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit

Called when a task ends
Called when a task ends

Definition Classes
JobProgressListener → SparkListener
def onTaskGettingResult(taskGettingResult: SparkListenerTaskGettingResult): Unit

Called when a task begins remotely fetching its result (will not be called for tasks that do not need to fetch the result remotely).
Called when a task begins remotely fetching its result (will not be called for tasks that do not need to fetch the result remotely).

Definition Classes
JobProgressListener → SparkListener
def onTaskStart(taskStart: SparkListenerTaskStart): Unit

Called when a task starts
Called when a task starts

Definition Classes
JobProgressListener → SparkListener
def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit

Called when an RDD is manually unpersisted by the application
Called when an RDD is manually unpersisted by the application

Definition Classes
SparkListener
val pendingStages: HashMap[StageId, StageInfo]
val poolToActiveStages: HashMap[String, HashMap[Int, StageInfo]]
val retainedJobs: Int
val retainedStages: Int
var schedulingMode: Option[SchedulingMode]
val skippedStages: ListBuffer[StageInfo]
val stageIdToActiveJobIds: HashMap[StageId, HashSet[JobId]]
val stageIdToData: HashMap[(StageId, StageAttemptId), StageUIData]
val stageIdToInfo: HashMap[StageId, StageInfo]
var startTime: Long
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
def updateAggregateMetrics(stageData: StageUIData, execId: String, taskMetrics: TaskMetrics, oldMetrics: Option[TaskMetrics]): Unit

Upon receiving new metrics for a task, updates the per-stage and per-executor-per-stage aggregate metrics by calculating deltas between the currently recorded metrics and the new metrics.
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package jobs

class JobProgressListener extends SparkListener with Logging

Instance Constructors

new JobProgressListener(conf: SparkConf)

Type Members

type ExecutorId = String

type JobGroupId = String

type JobId = Int

type PoolName = String

type StageAttemptId = Int

type StageId = Int

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

val activeJobs: HashMap[JobId, JobUIData]

val activeStages: HashMap[StageId, StageInfo]

final def asInstanceOf[T0]: T0

def blockManagerIds: Seq[BlockManagerId]

def clone(): AnyRef

val completedJobs: ListBuffer[JobUIData]

val completedStages: ListBuffer[StageInfo]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

val executorIdToBlockManagerId: HashMap[String, BlockManagerId]

val failedJobs: ListBuffer[JobUIData]

val failedStages: ListBuffer[StageInfo]

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

def isTraceEnabled(): Boolean

val jobGroupToJobIds: HashMap[JobGroupId, HashSet[JobId]]

val jobIdToData: HashMap[JobId, JobUIData]

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

var numCompletedJobs: Int

var numCompletedStages: Int

var numFailedJobs: Int

var numFailedStages: Int

def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit

def onApplicationStart(appStarted: SparkListenerApplicationStart): Unit

def onBlockManagerAdded(blockManagerAdded: SparkListenerBlockManagerAdded): Unit

def onBlockManagerRemoved(blockManagerRemoved: SparkListenerBlockManagerRemoved): Unit

def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit

def onEnvironmentUpdate(environmentUpdate: SparkListenerEnvironmentUpdate): Unit

def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit

def onExecutorMetricsUpdate(executorMetricsUpdate: SparkListenerExecutorMetricsUpdate): Unit

def onExecutorRemoved(executorRemoved: SparkListenerExecutorRemoved): Unit

def onJobEnd(jobEnd: SparkListenerJobEnd): Unit

def onJobStart(jobStart: SparkListenerJobStart): Unit

def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit

def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit

def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit

def onTaskGettingResult(taskGettingResult: SparkListenerTaskGettingResult): Unit

def onTaskStart(taskStart: SparkListenerTaskStart): Unit

def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit

val pendingStages: HashMap[StageId, StageInfo]

val poolToActiveStages: HashMap[String, HashMap[Int, StageInfo]]

val retainedJobs: Int

val retainedStages: Int

var schedulingMode: Option[SchedulingMode]

val skippedStages: ListBuffer[StageInfo]

val stageIdToActiveJobIds: HashMap[StageId, HashSet[JobId]]

val stageIdToData: HashMap[(StageId, StageAttemptId), StageUIData]

val stageIdToInfo: HashMap[StageId, StageInfo]