class
SparkJobActivity extends EmrActivity
Value Members
-
final
def
!=(arg0: AnyRef): Boolean
-
final
def
!=(arg0: Any): Boolean
-
final
def
##(): Int
-
-
-
final
def
==(arg0: AnyRef): Boolean
-
final
def
==(arg0: Any): Boolean
-
-
-
val
args: Seq[String]
-
final
def
asInstanceOf[T0]: T0
-
-
def
clone(): AnyRef
-
-
-
def
copy(id: PipelineObjectId = id, scriptRunner: String = scriptRunner, jobRunner: String = jobRunner, jarUri: String = jarUri, mainClass: MainClass = mainClass, args: Seq[String] = args, hadoopQueue: Option[String] = hadoopQueue, preActivityTaskConfig: Option[ShellScriptConfig] = preActivityTaskConfig, postActivityTaskConfig: Option[ShellScriptConfig] = postActivityTaskConfig, inputs: Seq[S3DataNode] = inputs, outputs: Seq[S3DataNode] = outputs, runsOn: Resource[SparkCluster] = runsOn, dependsOn: Seq[PipelineActivity] = dependsOn, preconditions: Seq[Precondition] = preconditions, onFailAlarms: Seq[SnsAlarm] = onFailAlarms, onSuccessAlarms: Seq[SnsAlarm] = onSuccessAlarms, onLateActionAlarms: Seq[SnsAlarm] = onLateActionAlarms, attemptTimeout: Option[Parameter[Duration]] = attemptTimeout, lateAfterTimeout: Option[Parameter[Duration]] = lateAfterTimeout, maximumRetries: Option[Parameter[Int]] = maximumRetries, retryDelay: Option[Parameter[Duration]] = retryDelay, failureAndRerunMode: Option[FailureAndRerunMode] = failureAndRerunMode, sparkOptions: Seq[String] = sparkOptions, sparkConfig: Map[String, String] = sparkConfig): SparkJobActivity
-
-
final
def
eq(arg0: AnyRef): Boolean
-
def
equals(arg0: Any): Boolean
-
-
def
finalize(): Unit
-
final
def
getClass(): Class[_]
-
-
val
hadoopQueue: Option[String]
-
def
hashCode(): Int
-
-
-
final
def
isInstanceOf[T0]: Boolean
-
val
jarUri: String
-
val
jobRunner: String
-
-
-
val
maximumRetries: Option[Parameter[Int]]
-
-
final
def
ne(arg0: AnyRef): Boolean
-
final
def
notify(): Unit
-
final
def
notifyAll(): Unit
-
-
-
val
onFailAlarms: Seq[SnsAlarm]
-
-
val
onLateActionAlarms: Seq[SnsAlarm]
-
-
val
onSuccessAlarms: Seq[SnsAlarm]
-
-
val
postActivityTaskConfig: Option[ShellScriptConfig]
-
val
preActivityTaskConfig: Option[ShellScriptConfig]
-
-
-
-
-
val
scriptRunner: String
-
implicit
def
seq2Option[A](anySeq: Seq[A]): Option[Seq[A]]
-
def
seqToOption[A, B](anySeq: Seq[A])(transform: (A) ⇒ B): Option[Seq[B]]
-
-
val
sparkConfig: Map[String, String]
-
val
sparkOptions: Seq[String]
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
-
def
toString(): String
-
implicit
def
uniquePipelineId2String(id: PipelineObjectId): String
-
final
def
wait(): Unit
-
final
def
wait(arg0: Long, arg1: Int): Unit
-
final
def
wait(arg0: Long): Unit
-
-
def
withArguments(argument: String*): SparkJobActivity
-
-
-
def
withHadoopQueue(queue: String): SparkJobActivity
-
-
-
-
-
-
-
-
def
withSparkConfig(key: String, value: String): SparkJobActivity
-
def
withSparkOption(option: String*): SparkJobActivity
Inherited from AnyRef
Inherited from Any
Runs a Spark job on a cluster. The cluster can be an EMR cluster managed by AWS Data Pipeline or another resource if you use TaskRunner. Use SparkJobActivity when you want to run work in parallel. This allows you to use the scheduling resources of the YARN framework or the MapReduce resource negotiator in Hadoop 1. If you would like to run work sequentially using the Amazon EMR Step action, you can still use SparkActivity.