etlflow.task
package etlflow.task
Members list
Concise view
Type members
Classlikes
case class BQExportTask(name: String, sourceProject: Option[String], sourceDataset: String, sourceTable: String, destinationPath: String, destinationFormat: FileType, destinationFileName: Option[String], destinationCompressionType: String) extends EtlTask[BQ, Unit]
Attributes
- Graph
- Supertypes
- trait Serializabletrait Producttrait Equalstrait EtlTask[BQ, Unit]trait ApplicationLoggerclass Objecttrait Matchableclass Any
case class BQLoadTask(name: String, inputLocation: Either[String, Seq[(String, String)]], inputType: FileType, outputProject: Option[String], outputDataset: String, outputTable: String, outputWriteDisposition: WriteDisposition, outputCreateDisposition: CreateDisposition, schema: Option[Schema]) extends EtlTask[BQ, Unit]
Attributes
- Graph
- Supertypes
- trait Serializabletrait Producttrait Equalstrait EtlTask[BQ, Unit]trait ApplicationLoggerclass Objecttrait Matchableclass Any
Attributes
- Graph
- Supertypes
- trait Serializabletrait Producttrait Equalstrait EtlTask[BQ, Unit]trait ApplicationLoggerclass Objecttrait Matchableclass Any
case class BQReadTask[T](name: String, query: String)(fn: FieldValueList => T) extends EtlTask[BQ, Iterable[T]]
Attributes
- Graph
- Supertypes
- trait Serializabletrait Producttrait Equalstrait EtlTask[BQ, Iterable[T]]trait ApplicationLoggerclass Objecttrait Matchableclass Any
case class BQSensorTask(name: String, query: String, sensor: Iterable[FieldValueList] => Boolean, spaced: Duration, retry: Option[Int]) extends EtlTask[BQ, Unit]
BQ Sensor on SQL query based on output rows satisfying condition defined by sensor function
BQ Sensor on SQL query based on output rows satisfying condition defined by sensor function
Attributes
- name
Task name
- query
BigQuery SQL query
- retry
Number of times we need to check query the BQ table before this effect terminates
- sensor
Function which takes Iterable[FieldValueList] and return bool, which on returning true exits the poll
- spaced
Specifies duration each repetition should be spaced from the last run
- Graph
- Supertypes
- trait Serializabletrait Producttrait Equalstrait EtlTask[BQ, Unit]trait ApplicationLoggerclass Objecttrait Matchableclass Any
case class CMPSSensorTask(name: String, project: String, subscription: String, interval: TimeInterval, spaced: Duration) extends EtlTask[Monitoring, Unit]
Cloud Monitoring sensor on Pubsub Subscription to check for num_undelivered_messages
Cloud Monitoring sensor on Pubsub Subscription to check for num_undelivered_messages
Attributes
- interval
Cloud Monitoring Interval with start time and end time.
- name
Name for the task
- project
GCP Project ID
- spaced
Check continuously with every repetition spaced by the specified duration from the last run.
- subscription
PubSub Subscription ID
- Graph
- Supertypes
- trait Serializabletrait Producttrait Equalstrait EtlTask[Monitoring, Unit]trait ApplicationLoggerclass Objecttrait Matchableclass Any
case class DPCreateTask(name: String, cluster: String, project: String, region: String, props: ClusterProps) extends EtlTask[DPCluster, Cluster]
Attributes
- Graph
- Supertypes
- trait Serializabletrait Producttrait Equalstrait EtlTask[DPCluster, Cluster]trait ApplicationLoggerclass Objecttrait Matchableclass Any
case class DPDeleteTask(name: String, cluster: String, project: String, region: String) extends EtlTask[DPCluster, Unit]
Attributes
- Graph
- Supertypes
- trait Serializabletrait Producttrait Equalstrait EtlTask[DPCluster, Unit]trait ApplicationLoggerclass Objecttrait Matchableclass Any
case class DPHiveJobTask(name: String, query: String, cluster: String, project: String, region: String) extends EtlTask[DPJob, Job]
Attributes
- Graph
- Supertypes
- trait Serializabletrait Producttrait Equalstrait EtlTask[DPJob, Job]trait ApplicationLoggerclass Objecttrait Matchableclass Any
case class DPSparkJobTask(name: String, args: List[String], mainClass: String, libs: List[String], conf: Map[String, String], cluster: String, project: String, region: String) extends EtlTask[DPJob, Job]
Attributes
- Graph
- Supertypes
- trait Serializabletrait Producttrait Equalstrait EtlTask[DPJob, Job]trait ApplicationLoggerclass Objecttrait Matchableclass Any
case class GCSCopyTask(name: String, input: Location, inputRecursive: Boolean, output: Location, parallelism: Int, overwrite: Boolean) extends EtlTask[GCS, Long]
Attributes
- Graph
- Supertypes
- trait Serializabletrait Producttrait Equalstrait EtlTask[GCS, Long]trait ApplicationLoggerclass Objecttrait Matchableclass Any
Attributes
- Graph
- Supertypes
- trait Serializabletrait Producttrait Equalstrait EtlTask[GCS, Unit]trait ApplicationLoggerclass Objecttrait Matchableclass Any
case class GCSListTask(name: String, bucket: String, prefix: Option[String], recursive: Boolean) extends EtlTask[GCS, Chunk[Blob]]
Attributes
- Graph
- Supertypes
- trait Serializabletrait Producttrait Equalstrait EtlTask[GCS, Chunk[Blob]]trait ApplicationLoggerclass Objecttrait Matchableclass Any
case class GCSPutTask(name: String, bucket: String, prefix: String, file: String) extends EtlTask[GCS, Unit]
Attributes
- Graph
- Supertypes
- trait Serializabletrait Producttrait Equalstrait EtlTask[GCS, Unit]trait ApplicationLoggerclass Objecttrait Matchableclass Any
case class GCSSensorTask(name: String, bucket: String, prefix: String, retry: Int, spaced: Duration) extends EtlTask[GCS, Unit]
Attributes
- Graph
- Supertypes
- trait Serializabletrait Producttrait Equalstrait EtlTask[GCS, Unit]trait ApplicationLoggerclass Objecttrait Matchableclass Any