Class/Object

com.krux.hyperion.activity

SparkJobActivity

Related Docs: object SparkJobActivity | package activity

Permalink

class SparkJobActivity extends EmrActivity

Runs a Spark job on a cluster. The cluster can be an EMR cluster managed by AWS Data Pipeline or another resource if you use TaskRunner. Use SparkJobActivity when you want to run work in parallel. This allows you to use the scheduling resources of the YARN framework or the MapReduce resource negotiator in Hadoop 1. If you would like to run work sequentially using the Amazon EMR Step action, you can still use SparkActivity.

Source
SparkJobActivity.scala
Linear Supertypes
EmrActivity, PipelineActivity, PipelineObject, Ordered[PipelineObject], Comparable[PipelineObject], AnyRef, Any
Ordering
  1. Alphabetic
  2. By inheritance
Inherited
  1. SparkJobActivity
  2. EmrActivity
  3. PipelineActivity
  4. PipelineObject
  5. Ordered
  6. Comparable
  7. AnyRef
  8. Any
  1. Hide All
  2. Show all
Visibility
  1. Public
  2. All

Value Members

  1. final def !=(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any

  2. final def ##(): Int

    Permalink
    Definition Classes
    AnyRef → Any

  3. def <(that: PipelineObject): Boolean

    Permalink
    Definition Classes
    Ordered

  4. def <=(that: PipelineObject): Boolean

    Permalink
    Definition Classes
    Ordered

  5. final def ==(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any

  6. def >(that: PipelineObject): Boolean

    Permalink
    Definition Classes
    Ordered

  7. def >=(that: PipelineObject): Boolean

    Permalink
    Definition Classes
    Ordered

  8. val args: Seq[String]

    Permalink

  9. final def asInstanceOf[T0]: T0

    Permalink
    Definition Classes
    Any

  10. val attemptTimeout: Option[Parameter[Duration]]

    Permalink

  11. def clone(): AnyRef

    Permalink
    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  12. def compare(that: PipelineObject): Int

    Permalink
    Definition Classes
    PipelineObject → Ordered

  13. def compareTo(that: PipelineObject): Int

    Permalink
    Definition Classes
    Ordered → Comparable

  14. def copy(id: PipelineObjectId = id, scriptRunner: String = scriptRunner, jobRunner: String = jobRunner, jarUri: String = jarUri, mainClass: MainClass = mainClass, args: Seq[String] = args, hadoopQueue: Option[String] = hadoopQueue, preActivityTaskConfig: Option[ShellScriptConfig] = preActivityTaskConfig, postActivityTaskConfig: Option[ShellScriptConfig] = postActivityTaskConfig, inputs: Seq[S3DataNode] = inputs, outputs: Seq[S3DataNode] = outputs, runsOn: Resource[SparkCluster] = runsOn, dependsOn: Seq[PipelineActivity] = dependsOn, preconditions: Seq[Precondition] = preconditions, onFailAlarms: Seq[SnsAlarm] = onFailAlarms, onSuccessAlarms: Seq[SnsAlarm] = onSuccessAlarms, onLateActionAlarms: Seq[SnsAlarm] = onLateActionAlarms, attemptTimeout: Option[Parameter[Duration]] = attemptTimeout, lateAfterTimeout: Option[Parameter[Duration]] = lateAfterTimeout, maximumRetries: Option[Parameter[Int]] = maximumRetries, retryDelay: Option[Parameter[Duration]] = retryDelay, failureAndRerunMode: Option[FailureAndRerunMode] = failureAndRerunMode, sparkOptions: Seq[String] = sparkOptions, sparkConfig: Map[String, String] = sparkConfig): SparkJobActivity

    Permalink

  15. val dependsOn: Seq[PipelineActivity]

    Permalink
    Definition Classes
    SparkJobActivity → PipelineActivity

  16. final def eq(arg0: AnyRef): Boolean

    Permalink
    Definition Classes
    AnyRef

  17. def equals(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any

  18. val failureAndRerunMode: Option[FailureAndRerunMode]

    Permalink

  19. def finalize(): Unit

    Permalink
    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )

  20. final def getClass(): Class[_]

    Permalink
    Definition Classes
    AnyRef → Any

  21. def groupedBy(group: String): SparkJobActivity

    Permalink
    Definition Classes
    SparkJobActivity → PipelineActivity

  22. val hadoopQueue: Option[String]

    Permalink

  23. def hashCode(): Int

    Permalink
    Definition Classes
    AnyRef → Any

  24. val id: PipelineObjectId

    Permalink
    Definition Classes
    SparkJobActivity → PipelineObject

  25. val inputs: Seq[S3DataNode]

    Permalink

  26. final def isInstanceOf[T0]: Boolean

    Permalink
    Definition Classes
    Any

  27. val jarUri: String

    Permalink

  28. val jobRunner: String

    Permalink

  29. val lateAfterTimeout: Option[Parameter[Duration]]

    Permalink

  30. val mainClass: MainClass

    Permalink

  31. val maximumRetries: Option[Parameter[Int]]

    Permalink

  32. def named(name: String): SparkJobActivity

    Permalink
    Definition Classes
    SparkJobActivity → PipelineActivity

  33. final def ne(arg0: AnyRef): Boolean

    Permalink
    Definition Classes
    AnyRef

  34. final def notify(): Unit

    Permalink
    Definition Classes
    AnyRef

  35. final def notifyAll(): Unit

    Permalink
    Definition Classes
    AnyRef

  36. def objects: Iterable[PipelineObject]

    Permalink
    Definition Classes
    SparkJobActivity → PipelineObject

  37. def onFail(alarms: SnsAlarm*): SparkJobActivity

    Permalink
    Definition Classes
    SparkJobActivity → PipelineActivity

  38. val onFailAlarms: Seq[SnsAlarm]

    Permalink

  39. def onLateAction(alarms: SnsAlarm*): SparkJobActivity

    Permalink
    Definition Classes
    SparkJobActivity → PipelineActivity

  40. val onLateActionAlarms: Seq[SnsAlarm]

    Permalink

  41. def onSuccess(alarms: SnsAlarm*): SparkJobActivity

    Permalink
    Definition Classes
    SparkJobActivity → PipelineActivity

  42. val onSuccessAlarms: Seq[SnsAlarm]

    Permalink

  43. val outputs: Seq[S3DataNode]

    Permalink

  44. val postActivityTaskConfig: Option[ShellScriptConfig]

    Permalink

  45. val preActivityTaskConfig: Option[ShellScriptConfig]

    Permalink

  46. val preconditions: Seq[Precondition]

    Permalink

  47. def ref: AdpRef[AdpActivity]

    Permalink
    Definition Classes
    PipelineActivity → PipelineObject

  48. val retryDelay: Option[Parameter[Duration]]

    Permalink

  49. val runsOn: Resource[SparkCluster]

    Permalink

  50. val scriptRunner: String

    Permalink

  51. implicit def seq2Option[A](anySeq: Seq[A]): Option[Seq[A]]

    Permalink
    Definition Classes
    PipelineObject

  52. def seqToOption[A, B](anySeq: Seq[A])(transform: (A) ⇒ B): Option[Seq[B]]

    Permalink
    Definition Classes
    PipelineObject

  53. lazy val serialize: AdpHadoopActivity

    Permalink
    Definition Classes
    SparkJobActivity → PipelineActivity → PipelineObject

  54. val sparkConfig: Map[String, String]

    Permalink

  55. val sparkOptions: Seq[String]

    Permalink

  56. final def synchronized[T0](arg0: ⇒ T0): T0

    Permalink
    Definition Classes
    AnyRef

  57. def toString(): String

    Permalink
    Definition Classes
    AnyRef → Any

  58. implicit def uniquePipelineId2String(id: PipelineObjectId): String

    Permalink
    Definition Classes
    PipelineObject

  59. final def wait(): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  60. final def wait(arg0: Long, arg1: Int): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  61. final def wait(arg0: Long): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  62. def whenMet(conditions: Precondition*): SparkJobActivity

    Permalink
    Definition Classes
    SparkJobActivity → PipelineActivity

  63. def withArguments(argument: String*): SparkJobActivity

    Permalink

  64. def withAttemptTimeout(timeout: Parameter[Duration]): SparkJobActivity

    Permalink

  65. def withDriverCores(n: Int): SparkJobActivity

    Permalink

  66. def withDriverMemory(memory: Memory): SparkJobActivity

    Permalink

  67. def withExecutorCores(n: Int): SparkJobActivity

    Permalink

  68. def withExecutorMemory(memory: Memory): SparkJobActivity

    Permalink

  69. def withFailureAndRerunMode(mode: FailureAndRerunMode): SparkJobActivity

    Permalink

  70. def withFiles(files: String*): SparkJobActivity

    Permalink

  71. def withHadoopQueue(queue: String): SparkJobActivity

    Permalink

  72. def withInput(input: S3DataNode*): SparkJobActivity

    Permalink

  73. def withLateAfterTimeout(timeout: Parameter[Duration]): SparkJobActivity

    Permalink

  74. def withMaster(master: String): SparkJobActivity

    Permalink

  75. def withMaximumRetries(retries: Parameter[Int]): SparkJobActivity

    Permalink

  76. def withNumExecutors(n: Int): SparkJobActivity

    Permalink

  77. def withOutput(output: S3DataNode*): SparkJobActivity

    Permalink

  78. def withPostActivityTaskConfig(script: ShellScriptConfig): SparkJobActivity

    Permalink

  79. def withPreActivityTaskConfig(script: ShellScriptConfig): SparkJobActivity

    Permalink

  80. def withRetryDelay(delay: Parameter[Duration]): SparkJobActivity

    Permalink

  81. def withSparkConfig(key: String, value: String): SparkJobActivity

    Permalink

  82. def withSparkOption(option: String*): SparkJobActivity

    Permalink

  83. def withTotalExecutorCores(n: Int): SparkJobActivity

    Permalink

Inherited from EmrActivity

Inherited from PipelineActivity

Inherited from PipelineObject

Inherited from Ordered[PipelineObject]

Inherited from Comparable[PipelineObject]

Inherited from AnyRef

Inherited from Any

Ungrouped