Class/Object

com.krux.hyperion.activity

SparkJobActivity

Related Docs: object SparkJobActivity | package activity

Permalink

case class SparkJobActivity extends EmrActivity with Product with Serializable

Runs a Spark job on a cluster. The cluster can be an EMR cluster managed by AWS Data Pipeline or another resource if you use TaskRunner. Use SparkJobActivity when you want to run work in parallel. This allows you to use the scheduling resources of the YARN framework or the MapReduce resource negotiator in Hadoop 1. If you would like to run work sequentially using the Amazon EMR Step action, you can still use SparkActivity.

Source
SparkJobActivity.scala
Linear Supertypes
Serializable, Serializable, Product, Equals, EmrActivity, PipelineActivity, PipelineObject, Ordered[PipelineObject], Comparable[PipelineObject], AnyRef, Any
Ordering
  1. Alphabetic
  2. By inheritance
Inherited
  1. SparkJobActivity
  2. Serializable
  3. Serializable
  4. Product
  5. Equals
  6. EmrActivity
  7. PipelineActivity
  8. PipelineObject
  9. Ordered
  10. Comparable
  11. AnyRef
  12. Any
  1. Hide All
  2. Show all
Visibility
  1. Public
  2. All

Value Members

  1. final def !=(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any

  2. final def ##(): Int

    Permalink
    Definition Classes
    AnyRef → Any

  3. def <(that: PipelineObject): Boolean

    Permalink
    Definition Classes
    Ordered

  4. def <=(that: PipelineObject): Boolean

    Permalink
    Definition Classes
    Ordered

  5. final def ==(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any

  6. def >(that: PipelineObject): Boolean

    Permalink
    Definition Classes
    Ordered

  7. def >=(that: PipelineObject): Boolean

    Permalink
    Definition Classes
    Ordered

  8. val args: Seq[String]

    Permalink

  9. final def asInstanceOf[T0]: T0

    Permalink
    Definition Classes
    Any

  10. val attemptTimeout: Option[Parameter[Duration]]

    Permalink

  11. def clone(): AnyRef

    Permalink
    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  12. def compare(that: PipelineObject): Int

    Permalink
    Definition Classes
    PipelineObject → Ordered

  13. def compareTo(that: PipelineObject): Int

    Permalink
    Definition Classes
    Ordered → Comparable

  14. val dependsOn: Seq[PipelineActivity]

    Permalink
    Definition Classes
    SparkJobActivity → PipelineActivity

  15. final def eq(arg0: AnyRef): Boolean

    Permalink
    Definition Classes
    AnyRef

  16. val failureAndRerunMode: Option[FailureAndRerunMode]

    Permalink

  17. def finalize(): Unit

    Permalink
    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )

  18. final def getClass(): Class[_]

    Permalink
    Definition Classes
    AnyRef → Any

  19. def groupedBy(group: String): SparkJobActivity

    Permalink
    Definition Classes
    SparkJobActivity → PipelineActivity

  20. val hadoopQueue: Option[String]

    Permalink

  21. val id: PipelineObjectId

    Permalink
    Definition Classes
    SparkJobActivity → PipelineObject

  22. val inputs: Seq[S3DataNode]

    Permalink

  23. final def isInstanceOf[T0]: Boolean

    Permalink
    Definition Classes
    Any

  24. val jarUri: String

    Permalink

  25. val jobRunner: String

    Permalink

  26. val lateAfterTimeout: Option[Parameter[Duration]]

    Permalink

  27. val mainClass: MainClass

    Permalink

  28. val maximumRetries: Option[Parameter[Int]]

    Permalink

  29. def named(name: String): SparkJobActivity

    Permalink
    Definition Classes
    SparkJobActivity → PipelineActivity

  30. final def ne(arg0: AnyRef): Boolean

    Permalink
    Definition Classes
    AnyRef

  31. final def notify(): Unit

    Permalink
    Definition Classes
    AnyRef

  32. final def notifyAll(): Unit

    Permalink
    Definition Classes
    AnyRef

  33. def objects: Iterable[PipelineObject]

    Permalink
    Definition Classes
    SparkJobActivity → PipelineObject

  34. def onFail(alarms: SnsAlarm*): SparkJobActivity

    Permalink
    Definition Classes
    SparkJobActivity → PipelineActivity

  35. val onFailAlarms: Seq[SnsAlarm]

    Permalink

  36. def onLateAction(alarms: SnsAlarm*): SparkJobActivity

    Permalink
    Definition Classes
    SparkJobActivity → PipelineActivity

  37. val onLateActionAlarms: Seq[SnsAlarm]

    Permalink

  38. def onSuccess(alarms: SnsAlarm*): SparkJobActivity

    Permalink
    Definition Classes
    SparkJobActivity → PipelineActivity

  39. val onSuccessAlarms: Seq[SnsAlarm]

    Permalink

  40. val outputs: Seq[S3DataNode]

    Permalink

  41. val postActivityTaskConfig: Option[ShellScriptConfig]

    Permalink

  42. val preActivityTaskConfig: Option[ShellScriptConfig]

    Permalink

  43. val preconditions: Seq[Precondition]

    Permalink

  44. def ref: AdpRef[AdpActivity]

    Permalink
    Definition Classes
    PipelineActivity → PipelineObject

  45. val retryDelay: Option[Parameter[Duration]]

    Permalink

  46. val runsOn: Resource[SparkCluster]

    Permalink

  47. val scriptRunner: String

    Permalink

  48. implicit def seq2Option[A](anySeq: Seq[A]): Option[Seq[A]]

    Permalink
    Definition Classes
    PipelineObject

  49. def seqToOption[A, B](anySeq: Seq[A])(transform: (A) ⇒ B): Option[Seq[B]]

    Permalink
    Definition Classes
    PipelineObject

  50. lazy val serialize: AdpHadoopActivity

    Permalink
    Definition Classes
    SparkJobActivity → PipelineActivity → PipelineObject

  51. final def synchronized[T0](arg0: ⇒ T0): T0

    Permalink
    Definition Classes
    AnyRef

  52. implicit def uniquePipelineId2String(id: PipelineObjectId): String

    Permalink
    Definition Classes
    PipelineObject

  53. final def wait(): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  54. final def wait(arg0: Long, arg1: Int): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  55. final def wait(arg0: Long): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  56. def whenMet(conditions: Precondition*): SparkJobActivity

    Permalink
    Definition Classes
    SparkJobActivity → PipelineActivity

  57. def withArguments(argument: String*): SparkJobActivity

    Permalink

  58. def withAttemptTimeout(timeout: Parameter[Duration]): SparkJobActivity

    Permalink

  59. def withFailureAndRerunMode(mode: FailureAndRerunMode): SparkJobActivity

    Permalink

  60. def withHadoopQueue(queue: String): SparkJobActivity

    Permalink

  61. def withInput(input: S3DataNode*): SparkJobActivity

    Permalink

  62. def withLateAfterTimeout(timeout: Parameter[Duration]): SparkJobActivity

    Permalink

  63. def withMaximumRetries(retries: Parameter[Int]): SparkJobActivity

    Permalink

  64. def withOutput(output: S3DataNode*): SparkJobActivity

    Permalink

  65. def withPostActivityTaskConfig(script: ShellScriptConfig): SparkJobActivity

    Permalink

  66. def withPreActivityTaskConfig(script: ShellScriptConfig): SparkJobActivity

    Permalink

  67. def withRetryDelay(delay: Parameter[Duration]): SparkJobActivity

    Permalink

Inherited from Serializable

Inherited from Serializable

Inherited from Product

Inherited from Equals

Inherited from EmrActivity

Inherited from PipelineActivity

Inherited from PipelineObject

Inherited from Ordered[PipelineObject]

Inherited from Comparable[PipelineObject]

Inherited from AnyRef

Inherited from Any

Ungrouped