OpSparkListener

Instance Constructors

new OpSparkListener(appName: String, appId: String, runType: String, customTagName: Option[String], customTagValue: Option[String], logStageMetrics: Boolean, collectStageMetrics: Boolean)

appName
application name
appId
application id
runType
OpWorkflowRunType
customTagName
tag name printed on log lines
customTagValue
the value for the tag printed on log lines
logStageMetrics
should log metrics for every stage Note: can increase logging significantly if app has too many stages.
collectStageMetrics
should collect metrics for every stage Note: can increase memory usage on the driver if app has too many stages.

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
val appId: String

application id
val appName: String

application name
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
val collectStageMetrics: Boolean

should collect metrics for every stage Note: can increase memory usage on the driver if app has too many stages.
val customTagName: Option[String]

tag name printed on log lines
val customTagValue: Option[String]

the value for the tag printed on log lines
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
val logPrefix: String
val logStageMetrics: Boolean

should log metrics for every stage Note: can increase logging significantly if app has too many stages.
def metrics: AppMetrics

All the metrics computed by the spark listener
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit

Definition Classes
OpSparkListener → SparkListener → SparkListenerInterface
def onApplicationStart(applicationStart: SparkListenerApplicationStart): Unit

Definition Classes
OpSparkListener → SparkListener → SparkListenerInterface
def onBlockManagerAdded(blockManagerAdded: SparkListenerBlockManagerAdded): Unit

Definition Classes
SparkListener → SparkListenerInterface
def onBlockManagerRemoved(blockManagerRemoved: SparkListenerBlockManagerRemoved): Unit

Definition Classes
SparkListener → SparkListenerInterface
def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit

Definition Classes
SparkListener → SparkListenerInterface
def onEnvironmentUpdate(environmentUpdate: SparkListenerEnvironmentUpdate): Unit

Definition Classes
SparkListener → SparkListenerInterface
def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit

Definition Classes
SparkListener → SparkListenerInterface
def onExecutorBlacklisted(executorBlacklisted: SparkListenerExecutorBlacklisted): Unit

Definition Classes
SparkListener → SparkListenerInterface
def onExecutorMetricsUpdate(executorMetricsUpdate: SparkListenerExecutorMetricsUpdate): Unit

Definition Classes
SparkListener → SparkListenerInterface
def onExecutorRemoved(executorRemoved: SparkListenerExecutorRemoved): Unit

Definition Classes
SparkListener → SparkListenerInterface
def onExecutorUnblacklisted(executorUnblacklisted: SparkListenerExecutorUnblacklisted): Unit

Definition Classes
SparkListener → SparkListenerInterface
def onJobEnd(jobEnd: SparkListenerJobEnd): Unit

Definition Classes
OpSparkListener → SparkListener → SparkListenerInterface
def onJobStart(jobStart: SparkListenerJobStart): Unit

Definition Classes
OpSparkListener → SparkListener → SparkListenerInterface
def onNodeBlacklisted(nodeBlacklisted: SparkListenerNodeBlacklisted): Unit

Definition Classes
SparkListener → SparkListenerInterface
def onNodeUnblacklisted(nodeUnblacklisted: SparkListenerNodeUnblacklisted): Unit

Definition Classes
SparkListener → SparkListenerInterface
def onOtherEvent(event: SparkListenerEvent): Unit

Definition Classes
SparkListener → SparkListenerInterface
def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit

Definition Classes
OpSparkListener → SparkListener → SparkListenerInterface
def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit

Definition Classes
SparkListener → SparkListenerInterface
def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit

Definition Classes
SparkListener → SparkListenerInterface
def onTaskGettingResult(taskGettingResult: SparkListenerTaskGettingResult): Unit

Definition Classes
SparkListener → SparkListenerInterface
def onTaskStart(taskStart: SparkListenerTaskStart): Unit

Definition Classes
SparkListener → SparkListenerInterface
def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit

Definition Classes
SparkListener → SparkListenerInterface
val runType: String

OpWorkflowRunType
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package spark

class OpSparkListener extends SparkListener

Instance Constructors

new OpSparkListener(appName: String, appId: String, runType: String, customTagName: Option[String], customTagValue: Option[String], logStageMetrics: Boolean, collectStageMetrics: Boolean)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

val appId: String

val appName: String

final def asInstanceOf[T0]: T0

def clone(): AnyRef

val collectStageMetrics: Boolean

val customTagName: Option[String]

val customTagValue: Option[String]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

val logPrefix: String

val logStageMetrics: Boolean

def metrics: AppMetrics

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit

def onApplicationStart(applicationStart: SparkListenerApplicationStart): Unit

def onBlockManagerAdded(blockManagerAdded: SparkListenerBlockManagerAdded): Unit

def onBlockManagerRemoved(blockManagerRemoved: SparkListenerBlockManagerRemoved): Unit

def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit

def onEnvironmentUpdate(environmentUpdate: SparkListenerEnvironmentUpdate): Unit

def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit

def onExecutorBlacklisted(executorBlacklisted: SparkListenerExecutorBlacklisted): Unit

def onExecutorMetricsUpdate(executorMetricsUpdate: SparkListenerExecutorMetricsUpdate): Unit

def onExecutorRemoved(executorRemoved: SparkListenerExecutorRemoved): Unit

def onExecutorUnblacklisted(executorUnblacklisted: SparkListenerExecutorUnblacklisted): Unit

def onJobEnd(jobEnd: SparkListenerJobEnd): Unit

def onJobStart(jobStart: SparkListenerJobStart): Unit

def onNodeBlacklisted(nodeBlacklisted: SparkListenerNodeBlacklisted): Unit

def onNodeUnblacklisted(nodeUnblacklisted: SparkListenerNodeUnblacklisted): Unit

def onOtherEvent(event: SparkListenerEvent): Unit

def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit

def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit

def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit

def onTaskGettingResult(taskGettingResult: SparkListenerTaskGettingResult): Unit

def onTaskStart(taskStart: SparkListenerTaskStart): Unit

def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit

val runType: String

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from SparkListener

Inherited from SparkListenerInterface

Inherited from AnyRef

Inherited from Any

Ungrouped