AutoTaskJob

Instance Constructors

new AutoTaskJob(name: String, defaultArea: Option[StorageArea], format: Option[String], coalesce: Boolean, udf: Option[String], views: Views, engine: Engine, task: AutoTaskDesc, storageHandler: StorageHandler, sqlParameters: Map[String, String], schemaHandler: SchemaHandler)(implicit settings: Settings)

name
: Job Name as defined in the YML job description file
defaultArea
: Where the resulting dataset is stored by default if not specified in the task
task
: Task to run
sqlParameters
: Sql Parameters to pass to SQL statements

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def analyze(fullTableName: String): Any

Attributes
protected
Definition Classes
SparkJob
def appendToFile(storageHandler: StorageHandler, dataToSave: DataFrame, path: Path): Unit

Saves a dataset.
Saves a dataset. If the path is empty (the first time we call metrics on the schema) then we can write.
If there's already parquet files stored in it, then create a temporary directory to compute on, and flush the path to move updated metrics in it
dataToSave
: dataset to be saved
path
: Path to save the file at

Attributes
protected
Definition Classes
SparkJob
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
val createDisposition: String
def createViews(views: Views, sqlParameters: Map[String, String]): Unit

Attributes
protected
Definition Classes
SparkJob
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
val logger: Logger

Attributes
protected
Definition Classes
StrictLogging
val name: String

: Job Name as defined in the YML job description file
: Job Name as defined in the YML job description file

Definition Classes
AutoTaskJob → JobBase
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def partitionDataset(dataset: DataFrame, partition: List[String]): DataFrame

Attributes
protected
Definition Classes
SparkJob
def partitionedDatasetWriter(dataset: DataFrame, partition: List[String]): DataFrameWriter[Row]

Partition a dataset using dataset columns.
Partition a dataset using dataset columns. To partition the dataset using the ingestion time, use the reserved column names :
- comet_date
- comet_year
- comet_month
- comet_day
- comet_hour
- comet_minute These columns are renamed to "date", "year", "month", "day", "hour", "minute" in the dataset and their values is set to the current date/time.
dataset
: Input dataset
partition
: list of columns to use for partitioning.
returns
The Spark session used to run this job

Attributes
protected
Definition Classes
SparkJob
def registerUdf(udf: String): Unit

Attributes
protected
Definition Classes
SparkJob
def run(): Try[JobResult]

Just to force any job to implement its entry point using within the "run" method
Just to force any job to implement its entry point using within the "run" method
returns
: Spark Dataframe for Spark Jobs None otherwise

Definition Classes
AutoTaskJob → JobBase
def runBQ(): Try[JobResult]
def runSpark(): Try[SparkJobResult]
def runView(viewName: String, viewDir: Option[String], viewCount: Int): Try[JobResult]
lazy val session: SparkSession

Definition Classes
SparkJob
implicit val settings: Settings

Definition Classes
AutoTaskJob → JobBase
lazy val sparkEnv: SparkEnv

Definition Classes
SparkJob
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
val writeDisposition: String

Related Doc: package transform

class AutoTaskJob extends SparkJob

Instance Constructors

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def analyze(fullTableName: String): Any

def appendToFile(storageHandler: StorageHandler, dataToSave: DataFrame, path: Path): Unit

final def asInstanceOf[T0]: T0

def clone(): AnyRef

val createDisposition: String

def createViews(views: Views, sqlParameters: Map[String, String]): Unit

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

val logger: Logger

val name: String

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def partitionDataset(dataset: DataFrame, partition: List[String]): DataFrame

def partitionedDatasetWriter(dataset: DataFrame, partition: List[String]): DataFrameWriter[Row]

def registerUdf(udf: String): Unit

def run(): Try[JobResult]

def runBQ(): Try[JobResult]

def runSpark(): Try[SparkJobResult]

def runView(viewName: String, viewDir: Option[String], viewCount: Int): Try[JobResult]

lazy val session: SparkSession

implicit val settings: Settings

lazy val sparkEnv: SparkEnv

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

val writeDisposition: String

Inherited from SparkJob

Inherited from JobBase

Inherited from StrictLogging

Inherited from AnyRef

Inherited from Any

Ungrouped