com.krux.hyperion.activity

HadoopActivity

case class HadoopActivity extends EmrActivity with Product with Serializable

Runs a MapReduce job on a cluster. The cluster can be an EMR cluster managed by AWS Data Pipeline or another resource if you use TaskRunner. Use HadoopActivity when you want to run work in parallel. This allows you to use the scheduling resources of the YARN framework or the MapReduce resource negotiator in Hadoop 1. If you would like to run work sequentially using the Amazon EMR Step action, you can still use EmrActivity.

Source
HadoopActivity.scala
Linear Supertypes
Serializable, Serializable, Product, Equals, EmrActivity, PipelineActivity, PipelineObject, Ordered[PipelineObject], Comparable[PipelineObject], AnyRef, Any
Ordering
  1. Alphabetic
  2. By inheritance
Inherited
  1. HadoopActivity
  2. Serializable
  3. Serializable
  4. Product
  5. Equals
  6. EmrActivity
  7. PipelineActivity
  8. PipelineObject
  9. Ordered
  10. Comparable
  11. AnyRef
  12. Any
  1. Hide All
  2. Show all
Learn more about member selection
Visibility
  1. Public
  2. All

Value Members

  1. final def !=(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  2. final def !=(arg0: Any): Boolean

    Definition Classes
    Any

  3. final def ##(): Int

    Definition Classes
    AnyRef → Any

  4. def <(that: PipelineObject): Boolean

    Definition Classes
    Ordered

  5. def <=(that: PipelineObject): Boolean

    Definition Classes
    Ordered

  6. final def ==(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  7. final def ==(arg0: Any): Boolean

    Definition Classes
    Any

  8. def >(that: PipelineObject): Boolean

    Definition Classes
    Ordered

  9. def >=(that: PipelineObject): Boolean

    Definition Classes
    Ordered

  10. val argument: Seq[String]

  11. final def asInstanceOf[T0]: T0

    Definition Classes
    Any

  12. val attemptTimeout: Option[Parameter[Duration]]

  13. def clone(): AnyRef

    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  14. def compare(that: PipelineObject): Int

    Definition Classes
    PipelineObject → Ordered

  15. def compareTo(that: PipelineObject): Int

    Definition Classes
    Ordered → Comparable

  16. val dependsOn: Seq[PipelineActivity]

    Definition Classes
    HadoopActivity → PipelineActivity

  17. final def eq(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  18. val failureAndRerunMode: Option[FailureAndRerunMode]

  19. def finalize(): Unit

    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )

  20. final def getClass(): Class[_]

    Definition Classes
    AnyRef → Any

  21. def groupedBy(group: String): HadoopActivity

    Definition Classes
    HadoopActivity → PipelineActivity

  22. val hadoopQueue: Option[String]

  23. val id: PipelineObjectId

    Definition Classes
    HadoopActivity → PipelineObject

  24. val inputs: Seq[S3DataNode]

  25. final def isInstanceOf[T0]: Boolean

    Definition Classes
    Any

  26. val jarUri: String

  27. val lateAfterTimeout: Option[Parameter[Duration]]

  28. val mainClass: Option[MainClass]

  29. val maximumRetries: Option[Parameter[Int]]

  30. def named(name: String): HadoopActivity

    Definition Classes
    HadoopActivity → PipelineActivity

  31. final def ne(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  32. final def notify(): Unit

    Definition Classes
    AnyRef

  33. final def notifyAll(): Unit

    Definition Classes
    AnyRef

  34. def objects: Iterable[PipelineObject]

    Definition Classes
    HadoopActivity → PipelineObject

  35. def onFail(alarms: SnsAlarm*): HadoopActivity

    Definition Classes
    HadoopActivity → PipelineActivity

  36. val onFailAlarms: Seq[SnsAlarm]

  37. def onLateAction(alarms: SnsAlarm*): HadoopActivity

    Definition Classes
    HadoopActivity → PipelineActivity

  38. val onLateActionAlarms: Seq[SnsAlarm]

  39. def onSuccess(alarms: SnsAlarm*): HadoopActivity

    Definition Classes
    HadoopActivity → PipelineActivity

  40. val onSuccessAlarms: Seq[SnsAlarm]

  41. val outputs: Seq[S3DataNode]

  42. val postActivityTaskConfig: Option[ShellScriptConfig]

  43. val preActivityTaskConfig: Option[ShellScriptConfig]

  44. val preconditions: Seq[Precondition]

  45. def ref: AdpRef[AdpActivity]

    Definition Classes
    PipelineActivity → PipelineObject

  46. val retryDelay: Option[Parameter[Duration]]

  47. val runsOn: Resource[EmrCluster]

  48. implicit def seq2Option[A](anySeq: Seq[A]): Option[Seq[A]]

    Definition Classes
    PipelineObject

  49. def seqToOption[A, B](anySeq: Seq[A])(transform: (A) ⇒ B): Option[Seq[B]]

    Definition Classes
    PipelineObject

  50. lazy val serialize: AdpHadoopActivity

    Definition Classes
    HadoopActivity → PipelineActivity → PipelineObject

  51. final def synchronized[T0](arg0: ⇒ T0): T0

    Definition Classes
    AnyRef

  52. implicit def uniquePipelineId2String(id: PipelineObjectId): String

    Definition Classes
    PipelineObject

  53. final def wait(): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  54. final def wait(arg0: Long, arg1: Int): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  55. final def wait(arg0: Long): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  56. def whenMet(conditions: Precondition*): HadoopActivity

    Definition Classes
    HadoopActivity → PipelineActivity

  57. def withArguments(arguments: String*): HadoopActivity

  58. def withAttemptTimeout(timeout: Parameter[Duration]): HadoopActivity

  59. def withFailureAndRerunMode(mode: FailureAndRerunMode): HadoopActivity

  60. def withHadoopQueue(queue: String): HadoopActivity

  61. def withInput(input: S3DataNode*): HadoopActivity

  62. def withLateAfterTimeout(timeout: Parameter[Duration]): HadoopActivity

  63. def withMaximumRetries(retries: Parameter[Int]): HadoopActivity

  64. def withOutput(output: S3DataNode*): HadoopActivity

  65. def withPostActivityTaskConfig(script: ShellScriptConfig): HadoopActivity

  66. def withPreActivityTaskConfig(script: ShellScriptConfig): HadoopActivity

  67. def withRetryDelay(delay: Parameter[Duration]): HadoopActivity

Inherited from Serializable

Inherited from Serializable

Inherited from Product

Inherited from Equals

Inherited from EmrActivity

Inherited from PipelineActivity

Inherited from PipelineObject

Inherited from Ordered[PipelineObject]

Inherited from Comparable[PipelineObject]

Inherited from AnyRef

Inherited from Any

Ungrouped