SparkTaskActivity

Type Members

type Self = SparkTaskActivity

Definition Classes
SparkTaskActivity → EmrTaskActivity → EmrActivity → PipelineActivity → NamedPipelineObject → PipelineObject

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
def <(that: PipelineObject): Boolean

Definition Classes
Ordered
def <=(that: PipelineObject): Boolean

Definition Classes
Ordered
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def >(that: PipelineObject): Boolean

Definition Classes
Ordered
def >=(that: PipelineObject): Boolean

Definition Classes
Ordered
val activityFields: ActivityFields[SparkCluster]

Definition Classes
SparkTaskActivity → PipelineActivity
val arguments: Seq[HString]
final def asInstanceOf[T0]: T0

Definition Classes
Any
def attemptTimeout: Option[HDuration]

Definition Classes
PipelineActivity
val baseFields: BaseFields

Definition Classes
SparkTaskActivity → NamedPipelineObject
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def compare(that: PipelineObject): Int

Definition Classes
PipelineObject → Ordered
def compareTo(that: PipelineObject): Int

Definition Classes
Ordered → Comparable
def dependsOn: Seq[PipelineActivity[_]]

Definition Classes
PipelineActivity
val emrTaskActivityFields: EmrTaskActivityFields

Definition Classes
SparkTaskActivity → EmrTaskActivity
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def failureAndRerunMode: Option[FailureAndRerunMode]

Definition Classes
PipelineActivity
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def groupedBy(group: String): Self

Postfix the name field
Postfix the name field

Definition Classes
NamedPipelineObject
val hadoopQueue: Option[HString]
def id: PipelineObjectId

Definition Classes
NamedPipelineObject → PipelineObject
def idGroupedBy(group: String): Self

Have a grouping postfix in the id field
Have a grouping postfix in the id field

Definition Classes
NamedPipelineObject
Note
Id naming is more restrictive, it is recommended to not changing the id unleass you have a good reason
def idNamed(namePrefix: String): Self

Id field will be prefixed with name
Id field will be prefixed with name

Definition Classes
NamedPipelineObject
Note
Id naming is more restrictive, it is recommended to not changing the id unless you have a good reason
val inputs: Seq[S3DataNode]
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
val jarUri: HString
val jobRunner: HString
def lateAfterTimeout: Option[HDuration]

Definition Classes
PipelineActivity
val mainClass: MainClass
def maximumRetries: Option[HInt]

Definition Classes
PipelineActivity
def name: Option[String]

Name of the pipeline object, if not set, it will defaults to
Name of the pipeline object, if not set, it will defaults to
```
Option(id)
```
Definition Classes
NamedPipelineObject
def named(namePrefix: String): Self

Give the object a name prefix
Give the object a name prefix

Definition Classes
NamedPipelineObject
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def objects: Iterable[PipelineObject]

Definition Classes
SparkTaskActivity → EmrTaskActivity → PipelineActivity → PipelineObject
def onFail(alarms: SnsAlarm*): Self

Definition Classes
PipelineActivity
def onFailAlarms: Seq[SnsAlarm]

Definition Classes
PipelineActivity
def onLateAction(alarms: SnsAlarm*): Self

Definition Classes
PipelineActivity
def onLateActionAlarms: Seq[SnsAlarm]

Definition Classes
PipelineActivity
def onSuccess(alarms: SnsAlarm*): Self

Definition Classes
PipelineActivity
def onSuccessAlarms: Seq[SnsAlarm]

Definition Classes
PipelineActivity
val outputs: Seq[S3DataNode]
def postActivityTaskConfig: Option[ShellScriptConfig]

Definition Classes
EmrTaskActivity
def preActivityTaskConfig: Option[ShellScriptConfig]

Definition Classes
EmrTaskActivity
def preconditions: Seq[Precondition]

Definition Classes
PipelineActivity
def ref: AdpRef[AdpActivity]

Definition Classes
PipelineActivity → PipelineObject
def retryDelay: Option[HDuration]

Definition Classes
PipelineActivity
def runsOn: Resource[SparkCluster]

Definition Classes
PipelineActivity
val scriptRunner: HString
implicit def seq2Option[A](anySeq: Seq[A]): Option[Seq[A]]

Definition Classes
PipelineObject
def seqToOption[A, B](anySeq: Seq[A])(transform: (A) ⇒ B): Option[Seq[B]]

Definition Classes
PipelineObject
lazy val serialize: AdpHadoopActivity

Definition Classes
SparkTaskActivity → PipelineActivity → PipelineObject
val sparkConfig: Map[HString, HString]
val sparkOptions: Seq[HString]
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
implicit def uniquePipelineId2String(id: PipelineObjectId): String

Definition Classes
PipelineObject
def updateActivityFields(fields: ActivityFields[SparkCluster]): SparkTaskActivity

Definition Classes
SparkTaskActivity → PipelineActivity
def updateBaseFields(fields: BaseFields): SparkTaskActivity

Definition Classes
SparkTaskActivity → NamedPipelineObject
def updateEmrTaskActivityFields(fields: EmrTaskActivityFields): SparkTaskActivity

Definition Classes
SparkTaskActivity → EmrTaskActivity
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def whenMet(conditions: Precondition*): Self

Definition Classes
PipelineActivity
def withArguments(args: HString*): SparkTaskActivity
def withAttemptTimeout(duration: HDuration): Self

Definition Classes
PipelineActivity
def withDriverCores(n: HInt): SparkTaskActivity
def withDriverMemory(memory: Memory): SparkTaskActivity
def withExecutorCores(n: HInt): SparkTaskActivity
def withExecutorMemory(memory: Memory): SparkTaskActivity
def withFailureAndRerunMode(mode: FailureAndRerunMode): Self

Definition Classes
PipelineActivity
def withFiles(files: HString*): SparkTaskActivity
def withHadoopQueue(queue: HString): SparkTaskActivity
def withInput(input: S3DataNode*): SparkTaskActivity
def withLateAfterTimeout(duration: HDuration): Self

Definition Classes
PipelineActivity
def withMaster(master: HString): SparkTaskActivity
def withMaximumRetries(retries: HInt): Self

Definition Classes
PipelineActivity
def withNumExecutors(n: HInt): SparkTaskActivity
def withOutput(output: S3DataNode*): SparkTaskActivity
def withPostActivityTaskConfig(config: ShellScriptConfig): Self

Definition Classes
EmrTaskActivity
def withPreActivityTaskConfig(config: ShellScriptConfig): Self

Definition Classes
EmrTaskActivity
def withRetryDelay(duration: HDuration): Self

Definition Classes
PipelineActivity
def withSparkConfig(key: HString, value: HString): SparkTaskActivity
def withSparkOption(option: HString*): SparkTaskActivity
def withTotalExecutorCores(n: HInt): SparkTaskActivity

Related Docs: object SparkTaskActivity | package activity

case class SparkTaskActivity extends EmrTaskActivity[SparkCluster] with Product with Serializable

Type Members

type Self = SparkTaskActivity

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

def <(that: PipelineObject): Boolean

def <=(that: PipelineObject): Boolean

final def ==(arg0: Any): Boolean

def >(that: PipelineObject): Boolean

def >=(that: PipelineObject): Boolean

val activityFields: ActivityFields[SparkCluster]

val arguments: Seq[HString]

final def asInstanceOf[T0]: T0

def attemptTimeout: Option[HDuration]

val baseFields: BaseFields

def clone(): AnyRef

def compare(that: PipelineObject): Int

def compareTo(that: PipelineObject): Int

def dependsOn: Seq[PipelineActivity[_]]

val emrTaskActivityFields: EmrTaskActivityFields

final def eq(arg0: AnyRef): Boolean

def failureAndRerunMode: Option[FailureAndRerunMode]

def finalize(): Unit

final def getClass(): Class[_]

def groupedBy(group: String): Self

val hadoopQueue: Option[HString]

def id: PipelineObjectId

def idGroupedBy(group: String): Self

def idNamed(namePrefix: String): Self

val inputs: Seq[S3DataNode]

final def isInstanceOf[T0]: Boolean

val jarUri: HString

val jobRunner: HString

def lateAfterTimeout: Option[HDuration]

val mainClass: MainClass

def maximumRetries: Option[HInt]

def name: Option[String]

def named(namePrefix: String): Self

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def objects: Iterable[PipelineObject]

def onFail(alarms: SnsAlarm*): Self

def onFailAlarms: Seq[SnsAlarm]

def onLateAction(alarms: SnsAlarm*): Self

def onLateActionAlarms: Seq[SnsAlarm]

def onSuccess(alarms: SnsAlarm*): Self

def onSuccessAlarms: Seq[SnsAlarm]

val outputs: Seq[S3DataNode]

def postActivityTaskConfig: Option[ShellScriptConfig]

def preActivityTaskConfig: Option[ShellScriptConfig]

def preconditions: Seq[Precondition]

def ref: AdpRef[AdpActivity]

def retryDelay: Option[HDuration]

def runsOn: Resource[SparkCluster]

val scriptRunner: HString

implicit def seq2Option[A](anySeq: Seq[A]): Option[Seq[A]]

def seqToOption[A, B](anySeq: Seq[A])(transform: (A) ⇒ B): Option[Seq[B]]

lazy val serialize: AdpHadoopActivity

val sparkConfig: Map[HString, HString]

val sparkOptions: Seq[HString]

final def synchronized[T0](arg0: ⇒ T0): T0

implicit def uniquePipelineId2String(id: PipelineObjectId): String

def updateActivityFields(fields: ActivityFields[SparkCluster]): SparkTaskActivity

def updateBaseFields(fields: BaseFields): SparkTaskActivity

def updateEmrTaskActivityFields(fields: EmrTaskActivityFields): SparkTaskActivity

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

def whenMet(conditions: Precondition*): Self

def withArguments(args: HString*): SparkTaskActivity

def withAttemptTimeout(duration: HDuration): Self

def withDriverCores(n: HInt): SparkTaskActivity

def withDriverMemory(memory: Memory): SparkTaskActivity

def withExecutorCores(n: HInt): SparkTaskActivity

def withExecutorMemory(memory: Memory): SparkTaskActivity

def withFailureAndRerunMode(mode: FailureAndRerunMode): Self