etlflow.gcp

package etlflow.gcp

Type members

Classlikes

case
class BQ(client: BigQuery) extends Service[Task]
Companion
object
object BQ
Companion
class
object BQApi
object BQClient
Companion
class
sealed
trait BQInputType extends Serializable
Companion
object
case
class BQLoadException(msg: String) extends RuntimeException
case
class DP(client: ClusterControllerClient) extends Service[Task]
Companion
object
object DP
Companion
class
object DPApi
object DPClient
case
class DPJob(client: JobControllerClient) extends Service[Task]
Companion
object
object DPJob
Companion
class
object DPJobApi
case
class DataprocProperties(bucket_name: String, subnet_uri: Option[String], network_tags: List[String], service_account: Option[String], idle_deletion_duration_sec: Option[Long], master_machine_type_uri: String, worker_machine_type_uri: String, image_version: String, boot_disk_type: String, master_boot_disk_size_gb: Int, worker_boot_disk_size_gb: Int, master_num_instance: Int, worker_num_instance: Int)
object FSType
Companion
class
sealed
trait FSType
Companion
object
case
class GCS(client: Storage) extends Service[Task]
Companion
object
object GCS
Companion
class
object GCSApi
object GCSClient
object Location
Companion
class
sealed
trait Location
Companion
object

Types

type BQEnv = Has[Service[Task]]
type DPEnv = Has[Service[Task]]
type DPJobEnv = Has[Service[Task]]
type GCSEnv = Has[Service[Task]]

Value members

Concrete methods

def getBQType(sp_type: String): LegacySQLTypeName
def getBqSchema[T : Tag]: Option[Schema]
def getFields[T : Tag]: Array[(String, String)]

Inherited fields

lazy
val logger: Logger
Inherited from
ApplicationLogger