objects

Type Members

1. case class CopyActivity extends PipelineActivity with Product with Serializable

   The activity that copys data from one data node to the other.

2. trait Copyable extends DataNode

3. case class CsvDataFormat(id: String, column: Option[Seq[String]] = scala.None, escapeChar: Option[String] = scala.None) extends DataFormat with Product with Serializable

   CSV data format

4. trait DataFormat extends PipelineObject

5. trait DataNode extends PipelineObject

6. trait Database extends PipelineObject

7. case class DefaultObject(schedule: Schedule)(implicit hc: HyperionContext) extends AdpDataPipelineAbstractObject with AdpDataPipelineDefaultObject with PipelineObject with Product with Serializable

8. case class DeleteS3PathActivity(id: String, s3Path: String, runsOn: Ec2Resource, stdout: Option[String] = scala.None, stderr: Option[String] = scala.None, dependsOn: Seq[PipelineActivity] = ..., preconditions: Seq[Precondition] = ..., onFailAlarms: Seq[SnsAlarm] = ..., onSuccessAlarms: Seq[SnsAlarm] = ..., onLateActionAlarms: Seq[SnsAlarm] = ...)(implicit hc: HyperionContext) extends PipelineActivity with Product with Serializable

   Activity to recursively delete files in an S3 path.

9. case class DoubleParameter(id: String, value: Double, description: Option[String] = scala.None, allowedValues: Seq[Double] = ..., encrypted: Boolean = false) extends Parameter with Product with Serializable

10. case class DynamoDBDataExistsPrecondition(id: String, tableName: String, preconditionTimeout: Option[String] = scala.None, role: Option[String] = scala.None)(implicit hc: HyperionContext) extends Precondition with Product with Serializable

    A precondition to check that data exists in a DynamoDB table.

11. case class DynamoDBTableExistsPrecondition(id: String, tableName: String, preconditionTimeout: Option[String] = scala.None, role: Option[String] = scala.None)(implicit hc: HyperionContext) extends Precondition with Product with Serializable

    A precondition to check that the DynamoDB table exists.

12. case class Ec2Resource(id: String, terminateAfter: String, role: Option[String], resourceRole: Option[String], instanceType: String, region: Option[String], imageId: Option[String], securityGroups: Seq[String], securityGroupIds: Seq[String], associatePublicIpAddress: Boolean)(implicit hc: HyperionContext) extends ResourceObject with Product with Serializable

    EC2 resource

13. trait EmrActivity extends PipelineActivity

14. trait EmrCluster extends ResourceObject

15. case class ExistsPrecondition(id: String, preconditionTimeout: Option[String] = scala.None, role: Option[String] = scala.None)(implicit hc: HyperionContext) extends Precondition with Product with Serializable

    Checks whether a data node object exists.

16. trait GoogleStorageActivity extends PipelineActivity

17. case class GoogleStorageDownloadActivity(id: String, runsOn: Ec2Resource, input: String = "", output: Option[S3DataNode] = scala.None, botoConfigUrl: String = "", dependsOn: Seq[PipelineActivity] = ..., preconditions: Seq[Precondition] = ..., onFailAlarms: Seq[SnsAlarm] = ..., onSuccessAlarms: Seq[SnsAlarm] = ..., onLateActionAlarms: Seq[SnsAlarm] = ...)(implicit hc: HyperionContext) extends GoogleStorageActivity with Product with Serializable

    Google Storage Download activity

18. case class GoogleStorageUploadActivity(id: String, runsOn: Ec2Resource, input: Option[S3DataNode] = scala.None, output: String = "", botoConfigUrl: String = "", dependsOn: Seq[PipelineActivity] = ..., preconditions: Seq[Precondition] = ..., onFailAlarms: Seq[SnsAlarm] = ..., onSuccessAlarms: Seq[SnsAlarm] = ..., onLateActionAlarms: Seq[SnsAlarm] = ...)(implicit hc: HyperionContext) extends GoogleStorageActivity with Product with Serializable

    Google Storage Upload activity

19. case class HiveActivity(id: String, runsOn: EmrCluster, hiveScript: Option[String] = scala.None, scriptUri: Option[String] = scala.None, scriptVariable: Option[String] = scala.None, input: Option[DataNode] = scala.None, output: Option[DataNode] = scala.None, stage: Option[Boolean] = scala.None, dependsOn: Seq[PipelineActivity] = ..., preconditions: Seq[Precondition] = ..., onFailAlarms: Seq[SnsAlarm] = ..., onSuccessAlarms: Seq[SnsAlarm] = ..., onLateActionAlarms: Seq[SnsAlarm] = ...)(implicit hc: HyperionContext) extends PipelineActivity with Product with Serializable

20. case class HiveCopyActivity(id: String, runsOn: EmrCluster, filterSql: Option[String] = scala.None, generatedScriptsPath: Option[String] = scala.None, input: Option[DataNode] = scala.None, output: Option[DataNode] = scala.None, dependsOn: Seq[PipelineActivity] = ..., preconditions: Seq[Precondition] = ..., onFailAlarms: Seq[SnsAlarm] = ..., onSuccessAlarms: Seq[SnsAlarm] = ..., onLateActionAlarms: Seq[SnsAlarm] = ...)(implicit hc: HyperionContext) extends PipelineActivity with Product with Serializable

21. case class IntegerParameter(id: String, value: Int, description: Option[String] = scala.None, allowedValues: Seq[Int] = ..., encrypted: Boolean = false) extends Parameter with Product with Serializable

22. case class JarActivity(id: String, runsOn: Ec2Resource, jar: Option[String] = scala.None, mainClass: Option[String] = scala.None, arguments: Seq[String] = ..., input: Seq[S3DataNode] = ..., output: Seq[S3DataNode] = ..., stdout: Option[String] = scala.None, stderr: Option[String] = scala.None, dependsOn: Seq[PipelineActivity] = ..., preconditions: Seq[Precondition] = ..., onFailAlarms: Seq[SnsAlarm] = ..., onSuccessAlarms: Seq[SnsAlarm] = ..., onLateActionAlarms: Seq[SnsAlarm] = ...)(implicit hc: HyperionContext) extends PipelineActivity with Product with Serializable

    Shell command activity

23. trait JdbcDatabase extends Database

24. case class MapReduceActivity(id: String, runsOn: EmrCluster, steps: Seq[MapReduceStep] = ..., dependsOn: Seq[PipelineActivity] = ..., preconditions: Seq[Precondition] = ..., onFailAlarms: Seq[SnsAlarm] = ..., onSuccessAlarms: Seq[SnsAlarm] = ..., onLateActionAlarms: Seq[SnsAlarm] = ...) extends EmrActivity with Product with Serializable

    Defines a MapReduce activity

25. case class MapReduceCluster(id: String = "MapReduceCluster", taskInstanceCount: Int = 0)(implicit hc: HyperionContext) extends EmrCluster with Product with Serializable

    Launch a MapReduce cluster

26. case class MapReduceStep(jar: String = "", mainClass: String = "", args: Seq[String] = immutable.this.Nil) extends Product with Serializable

    A MapReduce step that runs on MapReduce Cluster

27. trait Parameter extends AnyRef

28. case class PigActivity(id: String, runsOn: EmrCluster, generatedScriptsPath: Option[String] = scala.None, script: Option[String] = scala.None, scriptUri: Option[String] = scala.None, scriptVariable: Option[String] = scala.None, input: Option[DataNode] = scala.None, output: Option[DataNode] = scala.None, stage: Option[Boolean] = scala.None, dependsOn: Seq[PipelineActivity] = ..., preconditions: Seq[Precondition] = ..., onFailAlarms: Seq[SnsAlarm] = ..., onSuccessAlarms: Seq[SnsAlarm] = ..., onLateActionAlarms: Seq[SnsAlarm] = ...)(implicit hc: HyperionContext) extends PipelineActivity with Product with Serializable

29. trait PipelineActivity extends PipelineObject

30. trait PipelineObject extends AnyRef

    The base trait of krux data pipeline objects.

31. trait Precondition extends PipelineObject

    A condition that must be met before the object can run.

32. case class RedshiftCopyActivity(id: String, input: S3DataNode, output: RedshiftDataNode, insertMode: InsertMode, runsOn: Ec2Resource, transformSql: Option[String] = scala.None, commandOptions: Seq[RedshiftCopyOption] = ..., dependsOn: Seq[PipelineActivity] = ..., preconditions: Seq[Precondition] = ..., onFailAlarms: Seq[SnsAlarm] = ..., onSuccessAlarms: Seq[SnsAlarm] = ..., onLateActionAlarms: Seq[SnsAlarm] = ...) extends PipelineActivity with Product with Serializable

    Redshift copy activity

33. trait RedshiftCopyOption extends AnyRef

34. case class RedshiftDataNode(id: String, database: RedshiftDatabase, tableName: String, createTableSql: Option[String] = scala.None, schemaName: Option[String] = scala.None, primaryKeys: Option[Seq[String]] = scala.None, preconditions: Seq[Precondition] = ..., onSuccessAlarms: Seq[SnsAlarm] = ..., onFailAlarms: Seq[SnsAlarm] = ...) extends DataNode with Product with Serializable

    The abstracted RedshiftDataNode

35. trait RedshiftDatabase extends Database

    Redshift Database Trait, to use this please extend with an object.

36. case class RedshiftUnloadActivity(id: String, database: RedshiftDatabase, script: String, s3Path: String, runsOn: Ec2Resource, unloadOptions: Seq[RedshiftUnloadOption] = ..., dependsOn: Seq[PipelineActivity] = ..., preconditions: Seq[Precondition] = ..., onFailAlarms: Seq[SnsAlarm] = ..., onSuccessAlarms: Seq[SnsAlarm] = ..., onLateActionAlarms: Seq[SnsAlarm] = ...)(implicit hc: HyperionContext) extends PipelineActivity with Product with Serializable

    Redshift unload activity

37. trait RedshiftUnloadOption extends AnyRef

38. trait ResourceObject extends PipelineObject

39. trait RunnableObject extends AnyRef

    Run time references of runnable objects

40. trait S3DataNode extends Copyable

41. case class S3File(id: String, filePath: String = "", dataFormat: Option[DataFormat] = scala.None, preconditions: Seq[Precondition] = ..., onSuccessAlarms: Seq[SnsAlarm] = ..., onFailAlarms: Seq[SnsAlarm] = ...) extends S3DataNode with Product with Serializable

    Defines data from s3

42. case class S3Folder(id: String, directoryPath: String = "", dataFormat: Option[DataFormat] = scala.None, preconditions: Seq[Precondition] = ..., onSuccessAlarms: Seq[SnsAlarm] = ..., onFailAlarms: Seq[SnsAlarm] = ...) extends S3DataNode with Product with Serializable

    Defines data from s3 directory

43. case class S3KeyExistsPrecondition(id: String, s3Key: String, name: Option[String] = scala.None, preconditionTimeout: Option[String] = scala.None, role: Option[String] = scala.None)(implicit hc: HyperionContext) extends Precondition with Product with Serializable

    Checks whether a key exists in an Amazon S3 data node.

44. case class S3KeyParameter(id: String, value: String, description: Option[String] = scala.None, encrypted: Boolean = false) extends Parameter with Product with Serializable

45. case class S3PrefixNotEmptyPrecondition(id: String, s3Prefix: String, preconditionTimeout: Option[String] = scala.None, role: Option[String] = scala.None)(implicit hc: HyperionContext) extends Precondition with Product with Serializable

    A precondition to check that the Amazon S3 objects with the given prefix (represented as a URI) are present.

46. case class Schedule(id: String = "PipelineSchedule", start: Option[github.nscala_time.time.Imports.DateTime] = scala.None, period: DpPeriod = ..., occurrences: Option[Int] = scala.None, scheduleType: ScheduleType = ...) extends PipelineObject with Product with Serializable

    Cron liked schedule that runs at defined period.

47. case class ShellCommandActivity(id: String, runsOn: Ec2Resource, command: Option[String] = scala.None, scriptUri: Option[String] = scala.None, scriptArguments: Seq[String] = ..., stage: Boolean = true, input: Seq[S3DataNode] = ..., output: Seq[S3DataNode] = ..., stdout: Option[String] = scala.None, stderr: Option[String] = scala.None, dependsOn: Seq[PipelineActivity] = ..., preconditions: Seq[Precondition] = ..., onFailAlarms: Seq[SnsAlarm] = ..., onSuccessAlarms: Seq[SnsAlarm] = ..., onLateActionAlarms: Seq[SnsAlarm] = ...) extends PipelineActivity with Product with Serializable

    Shell command activity

48. case class ShellCommandPrecondition(id: String, command: String, scriptArgument: Seq[String] = ..., scriptUri: Option[String] = scala.None, stdout: Option[String] = scala.None, stderr: Option[String] = scala.None, preconditionTimeout: Option[String] = scala.None, role: Option[String] = scala.None)(implicit hc: HyperionContext) extends Precondition with Product with Serializable

    A Unix/Linux shell command that can be run as a precondition.

49. case class SnsAlarm(id: String, subject: String = "", message: String = "", topicArn: Option[String] = scala.None, role: Option[String] = scala.None)(implicit hc: HyperionContext) extends PipelineObject with Product with Serializable

50. case class SparkActivity(id: String, runsOn: SparkCluster, steps: Seq[SparkStep] = ..., dependsOn: Seq[PipelineActivity] = ..., preconditions: Seq[Precondition] = ..., onFailAlarms: Seq[SnsAlarm] = ..., onSuccessAlarms: Seq[SnsAlarm] = ..., onLateActionAlarms: Seq[SnsAlarm] = ...) extends EmrActivity with Product with Serializable

    Defines a spark activity

51. case class SparkCluster(id: String, taskInstanceCount: Int, coreInstanceCount: Int, instanceType: String, amiVersion: String, sparkVersion: String, terminateAfter: String)(implicit hc: HyperionContext) extends EmrCluster with Product with Serializable

    Launch a Spark cluster

52. case class SparkStep(jar: String = "", mainClass: String = "", args: Seq[String] = immutable.this.Nil)(implicit hc: HyperionContext) extends Product with Serializable

    A spark step that runs on Spark Cluster

53. case class SqlActivity(id: String, runsOn: Ec2Resource, database: Database, script: String, scriptArgument: Seq[String] = ..., queue: Option[String] = scala.None, dependsOn: Seq[PipelineActivity] = ..., preconditions: Seq[Precondition] = ..., onFailAlarms: Seq[SnsAlarm] = ..., onSuccessAlarms: Seq[SnsAlarm] = ..., onLateActionAlarms: Seq[SnsAlarm] = ...) extends PipelineActivity with Product with Serializable

54. case class SqlDataNode(id: String, tableQuery: TableQuery, database: JdbcDatabase, preconditions: Seq[Precondition] = ..., onSuccessAlarms: Seq[SnsAlarm] = ..., onFailAlarms: Seq[SnsAlarm] = ...) extends Copyable with Product with Serializable

55. case class StringParameter(id: String, value: String, description: Option[String] = scala.None, allowedValues: Seq[String] = ..., encrypted: Boolean = false) extends Parameter with Product with Serializable

56. class Terminate extends PipelineObject

57. case class TsvDataFormat(id: String, column: Option[Seq[String]] = scala.None, escapeChar: Option[String] = scala.None) extends DataFormat with Product with Serializable

    TSV data format