com.krux.hyperion.activity

SparkJobActivity

class SparkJobActivity extends EmrActivity

Runs a Spark job on a cluster. The cluster can be an EMR cluster managed by AWS Data Pipeline or another resource if you use TaskRunner. Use SparkJobActivity when you want to run work in parallel. This allows you to use the scheduling resources of the YARN framework or the MapReduce resource negotiator in Hadoop 1. If you would like to run work sequentially using the Amazon EMR Step action, you can still use SparkActivity.

Source
SparkJobActivity.scala
Linear Supertypes
EmrActivity, PipelineActivity, PipelineObject, Ordered[PipelineObject], Comparable[PipelineObject], AnyRef, Any
Ordering
  1. Alphabetic
  2. By inheritance
Inherited
  1. SparkJobActivity
  2. EmrActivity
  3. PipelineActivity
  4. PipelineObject
  5. Ordered
  6. Comparable
  7. AnyRef
  8. Any
  1. Hide All
  2. Show all
Learn more about member selection
Visibility
  1. Public
  2. All

Value Members

  1. final def !=(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  2. final def !=(arg0: Any): Boolean

    Definition Classes
    Any

  3. final def ##(): Int

    Definition Classes
    AnyRef → Any

  4. def <(that: PipelineObject): Boolean

    Definition Classes
    Ordered

  5. def <=(that: PipelineObject): Boolean

    Definition Classes
    Ordered

  6. final def ==(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  7. final def ==(arg0: Any): Boolean

    Definition Classes
    Any

  8. def >(that: PipelineObject): Boolean

    Definition Classes
    Ordered

  9. def >=(that: PipelineObject): Boolean

    Definition Classes
    Ordered

  10. val args: Seq[String]

  11. final def asInstanceOf[T0]: T0

    Definition Classes
    Any

  12. val attemptTimeout: Option[Parameter[Duration]]

  13. def clone(): AnyRef

    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  14. def compare(that: PipelineObject): Int

    Definition Classes
    PipelineObject → Ordered

  15. def compareTo(that: PipelineObject): Int

    Definition Classes
    Ordered → Comparable

  16. def copy(id: PipelineObjectId = id, scriptRunner: String = scriptRunner, jobRunner: String = jobRunner, jarUri: String = jarUri, mainClass: MainClass = mainClass, args: Seq[String] = args, hadoopQueue: Option[String] = hadoopQueue, preActivityTaskConfig: Option[ShellScriptConfig] = preActivityTaskConfig, postActivityTaskConfig: Option[ShellScriptConfig] = postActivityTaskConfig, inputs: Seq[S3DataNode] = inputs, outputs: Seq[S3DataNode] = outputs, runsOn: Resource[SparkCluster] = runsOn, dependsOn: Seq[PipelineActivity] = dependsOn, preconditions: Seq[Precondition] = preconditions, onFailAlarms: Seq[SnsAlarm] = onFailAlarms, onSuccessAlarms: Seq[SnsAlarm] = onSuccessAlarms, onLateActionAlarms: Seq[SnsAlarm] = onLateActionAlarms, attemptTimeout: Option[Parameter[Duration]] = attemptTimeout, lateAfterTimeout: Option[Parameter[Duration]] = lateAfterTimeout, maximumRetries: Option[Parameter[Int]] = maximumRetries, retryDelay: Option[Parameter[Duration]] = retryDelay, failureAndRerunMode: Option[FailureAndRerunMode] = failureAndRerunMode, sparkOptions: Seq[String] = sparkOptions, sparkConfig: Map[String, String] = sparkConfig): SparkJobActivity

  17. val dependsOn: Seq[PipelineActivity]

    Definition Classes
    SparkJobActivity → PipelineActivity

  18. final def eq(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  19. def equals(arg0: Any): Boolean

    Definition Classes
    AnyRef → Any

  20. val failureAndRerunMode: Option[FailureAndRerunMode]

  21. def finalize(): Unit

    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )

  22. final def getClass(): Class[_]

    Definition Classes
    AnyRef → Any

  23. def groupedBy(group: String): SparkJobActivity

    Definition Classes
    SparkJobActivity → PipelineActivity

  24. val hadoopQueue: Option[String]

  25. def hashCode(): Int

    Definition Classes
    AnyRef → Any

  26. val id: PipelineObjectId

    Definition Classes
    SparkJobActivity → PipelineObject

  27. val inputs: Seq[S3DataNode]

  28. final def isInstanceOf[T0]: Boolean

    Definition Classes
    Any

  29. val jarUri: String

  30. val jobRunner: String

  31. val lateAfterTimeout: Option[Parameter[Duration]]

  32. val mainClass: MainClass

  33. val maximumRetries: Option[Parameter[Int]]

  34. def named(name: String): SparkJobActivity

    Definition Classes
    SparkJobActivity → PipelineActivity

  35. final def ne(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  36. final def notify(): Unit

    Definition Classes
    AnyRef

  37. final def notifyAll(): Unit

    Definition Classes
    AnyRef

  38. def objects: Iterable[PipelineObject]

    Definition Classes
    SparkJobActivity → PipelineObject

  39. def onFail(alarms: SnsAlarm*): SparkJobActivity

    Definition Classes
    SparkJobActivity → PipelineActivity

  40. val onFailAlarms: Seq[SnsAlarm]

  41. def onLateAction(alarms: SnsAlarm*): SparkJobActivity

    Definition Classes
    SparkJobActivity → PipelineActivity

  42. val onLateActionAlarms: Seq[SnsAlarm]

  43. def onSuccess(alarms: SnsAlarm*): SparkJobActivity

    Definition Classes
    SparkJobActivity → PipelineActivity

  44. val onSuccessAlarms: Seq[SnsAlarm]

  45. val outputs: Seq[S3DataNode]

  46. val postActivityTaskConfig: Option[ShellScriptConfig]

  47. val preActivityTaskConfig: Option[ShellScriptConfig]

  48. val preconditions: Seq[Precondition]

  49. def ref: AdpRef[AdpActivity]

    Definition Classes
    PipelineActivity → PipelineObject

  50. val retryDelay: Option[Parameter[Duration]]

  51. val runsOn: Resource[SparkCluster]

  52. val scriptRunner: String

  53. implicit def seq2Option[A](anySeq: Seq[A]): Option[Seq[A]]

    Definition Classes
    PipelineObject

  54. def seqToOption[A, B](anySeq: Seq[A])(transform: (A) ⇒ B): Option[Seq[B]]

    Definition Classes
    PipelineObject

  55. lazy val serialize: AdpHadoopActivity

    Definition Classes
    SparkJobActivity → PipelineActivity → PipelineObject

  56. val sparkConfig: Map[String, String]

  57. val sparkOptions: Seq[String]

  58. final def synchronized[T0](arg0: ⇒ T0): T0

    Definition Classes
    AnyRef

  59. def toString(): String

    Definition Classes
    AnyRef → Any

  60. implicit def uniquePipelineId2String(id: PipelineObjectId): String

    Definition Classes
    PipelineObject

  61. final def wait(): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  62. final def wait(arg0: Long, arg1: Int): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  63. final def wait(arg0: Long): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  64. def whenMet(conditions: Precondition*): SparkJobActivity

    Definition Classes
    SparkJobActivity → PipelineActivity

  65. def withArguments(argument: String*): SparkJobActivity

  66. def withAttemptTimeout(timeout: Parameter[Duration]): SparkJobActivity

  67. def withFailureAndRerunMode(mode: FailureAndRerunMode): SparkJobActivity

  68. def withHadoopQueue(queue: String): SparkJobActivity

  69. def withInput(input: S3DataNode*): SparkJobActivity

  70. def withLateAfterTimeout(timeout: Parameter[Duration]): SparkJobActivity

  71. def withMaximumRetries(retries: Parameter[Int]): SparkJobActivity

  72. def withOutput(output: S3DataNode*): SparkJobActivity

  73. def withPostActivityTaskConfig(script: ShellScriptConfig): SparkJobActivity

  74. def withPreActivityTaskConfig(script: ShellScriptConfig): SparkJobActivity

  75. def withRetryDelay(delay: Parameter[Duration]): SparkJobActivity

  76. def withSparkConfig(key: String, value: String): SparkJobActivity

  77. def withSparkOption(option: String*): SparkJobActivity

Inherited from EmrActivity

Inherited from PipelineActivity

Inherited from PipelineObject

Inherited from Ordered[PipelineObject]

Inherited from Comparable[PipelineObject]

Inherited from AnyRef

Inherited from Any

Ungrouped