API

Type Members

1. case class ARCContext(jobId: Option[String], jobName: Option[String], environment: Option[String], environmentId: Option[String], configUri: Option[String], isStreaming: Boolean, ignoreEnvironments: Boolean, storageLevel: StorageLevel, immutableViews: Boolean, ipynb: Boolean = true, inlineSQL: Boolean = true, commandLineArguments: Map[String, String], dynamicConfigurationPlugins: List[DynamicConfigurationPlugin], lifecyclePlugins: List[LifecyclePlugin], activeLifecyclePlugins: List[LifecyclePluginInstance], pipelineStagePlugins: List[PipelineStagePlugin], udfPlugins: List[UDFPlugin], userData: Map[String, AnyRef], serializableConfiguration: SerializableConfiguration) extends Product with Serializable

   

   ARCContext is used to define immutable global run parameters.

2. sealed trait AmazonS3EncryptionType extends AnyRef

   

3. case class ArrayColumn(id: Option[String], name: String, description: Option[String], nullable: Boolean, nullReplacementValue: Option[String], trim: Boolean, nullableValues: List[String], elementType: ExtractColumn, metadata: Option[String]) extends ExtractColumn with Product with Serializable

   

4. sealed trait Authentication extends AnyRef

   

5. case class BinaryColumn(id: Option[String], name: String, description: Option[String], nullable: Boolean, nullReplacementValue: Option[String], trim: Boolean, nullableValues: List[String], encoding: EncodingType, metadata: Option[String]) extends ExtractColumn with Product with Serializable

   

6. case class BooleanColumn(id: Option[String], name: String, description: Option[String], nullable: Boolean, nullReplacementValue: Option[String], trim: Boolean, nullableValues: List[String], trueValues: List[String], falseValues: List[String], metadata: Option[String]) extends ExtractColumn with Product with Serializable

   

   true / false values are lists of strings that are considered equivalent to true or false e.g.

   true / false values are lists of strings that are considered equivalent to true or false e.g. "Y", "yes", "N", "no".

7. trait ConfigPlugin[T] extends VersionedPlugin

   

8. case class CsvReaderOptions(hasHeader: Boolean, ignoreHeader: Boolean, delimiter: Char) extends ExtractReaderOptions with Product with Serializable

   

9. case class DateColumn(id: Option[String], name: String, description: Option[String], nullable: Boolean, nullReplacementValue: Option[String], trim: Boolean, nullableValues: List[String], formatters: List[String], metadata: Option[String], strict: Boolean, caseSensitive: Boolean) extends ExtractColumn with Product with Serializable

   

   Formatters is a list of valid Java Time formats.

   Formatters is a list of valid Java Time formats. Will attemp to parse in order so most likely match should be first.

10. case class DecimalColumn(id: Option[String], name: String, description: Option[String], nullable: Boolean, nullReplacementValue: Option[String], trim: Boolean, nullableValues: List[String], precision: Int, scale: Int, metadata: Option[String], formatters: Option[List[String]]) extends ExtractColumn with Product with Serializable

    

11. case class DoubleColumn(id: Option[String], name: String, description: Option[String], nullable: Boolean, nullReplacementValue: Option[String], trim: Boolean, nullableValues: List[String], metadata: Option[String], formatters: Option[List[String]]) extends ExtractColumn with Product with Serializable

    

12. case class ETLPipeline(stages: List[PipelineStage]) extends Product with Serializable

    

13. case class EmptySchemaExtractError(path: Option[String]) extends ExtractError with Product with Serializable

    

14. sealed trait EncodingType extends AnyRef

    

15. case class ErrorRow(row: String, rowIndex: Long, err: String) extends Product with Serializable

    

16. sealed trait ExtractColumn extends AnyRef

    

    ExtractColumns are used to define schemas for typing transforms as well as when extracting from sources that lack a schema such as CSV.

17. sealed trait ExtractError extends AnyRef

    

18. sealed trait ExtractReaderOptions extends AnyRef

    

19. sealed trait FailMode extends AnyRef

    

20. case class FileNotFoundExtractError(path: Option[String]) extends ExtractError with Product with Serializable

    

21. case class IntegerColumn(id: Option[String], name: String, description: Option[String], nullable: Boolean, nullReplacementValue: Option[String], trim: Boolean = true, nullableValues: List[String], metadata: Option[String], formatters: Option[List[String]]) extends ExtractColumn with Product with Serializable

    

22. sealed trait IsolationLevel extends AnyRef

    

23. trait LifecyclePluginInstance extends AnyRef

    

24. case class LongColumn(id: Option[String], name: String, description: Option[String], nullable: Boolean, nullReplacementValue: Option[String], trim: Boolean, nullableValues: List[String], metadata: Option[String], formatters: Option[List[String]]) extends ExtractColumn with Product with Serializable

    

25. sealed trait MetadataFormat extends AnyRef

    

26. case class MetadataSchema(name: String, format: MetadataFormat) extends Product with Serializable

    

27. sealed trait OutputModeType extends AnyRef

    

28. case class PathNotExistsExtractError(path: Option[String]) extends ExtractError with Product with Serializable

    

29. trait PipelineStage extends AnyRef

    

30. sealed trait ResponseType extends AnyRef

    

31. case class StringColumn(id: Option[String], name: String, description: Option[String], nullable: Boolean, nullReplacementValue: Option[String], trim: Boolean, nullableValues: List[String], metadata: Option[String], minLength: Option[Int], maxLength: Option[Int], regex: Option[Regex]) extends ExtractColumn with Product with Serializable

    

32. case class StructColumn(id: Option[String], name: String, description: Option[String], nullable: Boolean, nullReplacementValue: Option[String], trim: Boolean, nullableValues: List[String], fields: List[ExtractColumn], metadata: Option[String]) extends ExtractColumn with Product with Serializable

    

33. case class TimeColumn(id: Option[String], name: String, description: Option[String], nullable: Boolean, nullReplacementValue: Option[String], trim: Boolean, nullableValues: List[String], formatters: List[String], metadata: Option[String]) extends ExtractColumn with Product with Serializable

    

    Formatters is a list of valid Java Time formats.

    Formatters is a list of valid Java Time formats. Will attemp to parse in order so most likely match should be first.

34. case class TimestampColumn(id: Option[String], name: String, description: Option[String], nullable: Boolean, nullReplacementValue: Option[String], trim: Boolean, nullableValues: List[String], timezoneId: String, formatters: List[String], time: Option[LocalTime], metadata: Option[String], strict: Boolean, caseSensitive: Boolean) extends ExtractColumn with Product with Serializable

    

    Formatters is a list of valid Java Time formats.

    Formatters is a list of valid Java Time formats. Will attemp to parse in order so most likely match should be first.

35. case class TypingError(field: String, message: String) extends Product with Serializable

    

36. trait VersionedPlugin extends Serializable

    

37. case class Watermark(eventTime: String, delayThreshold: String) extends Product with Serializable