IngestionJob

Abstract Value Members

abstract def domain: Domain
abstract def ingest(dataset: DataFrame): (RDD[_], RDD[_])

ingestion algorithm
abstract def loadDataSet(): Try[DataFrame]

Dataset loading strategy (JSON / CSV / ...)
Dataset loading strategy (JSON / CSV / ...)
returns
Spark Dataframe loaded using metadata options
abstract def name: String

Definition Classes
SparkJob
abstract def path: List[Path]
abstract def schema: Schema
abstract def schemaHandler: SchemaHandler
implicit abstract def settings: Settings

Definition Classes
SparkJob
abstract def storageHandler: StorageHandler
abstract def types: List[Type]

Concrete Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getWriteMode(): WriteMode
def hashCode(): Int

Definition Classes
AnyRef → Any
def index(mergedDF: DataFrame): Unit
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
val logger: Logger

Attributes
protected
Definition Classes
StrictLogging
def merge(inputDF: DataFrame, existingDF: DataFrame, merge: MergeOptions): DataFrame

Merge incoming and existing dataframes using merge options
Merge incoming and existing dataframes using merge options
returns
merged dataframe
lazy val metadata: Metadata

Merged metadata
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val now: Timestamp
def partitionDataset(dataset: DataFrame, partition: List[String]): DataFrame

Definition Classes
SparkJob
def partitionedDatasetWriter(dataset: DataFrame, partition: List[String]): DataFrameWriter[Row]

Partition a dataset using dataset columns.
Partition a dataset using dataset columns. To partition the dataset using the igestion time, use the reserved column names :
- comet_year
- comet_month
- comet_day
- comet_hour
- comet_minute These columsn are renamed to "year", "month", "day", "hour", "minute" in the dataset and their values is set to the current date/time.
dataset
: Input dataset
partition
: list of columns to use for partitioning.
returns
The Spark session used to run this job

Definition Classes
SparkJob
def run(): Try[SparkSession]

Main entry point as required by the Spark Job interface
Main entry point as required by the Spark Job interface
returns
: Spark Session used for the job

Definition Classes
IngestionJob → SparkJob
def saveAccepted(acceptedDF: DataFrame): (DataFrame, Path)

Merge new and existing dataset if required Save using overwrite / Append mode
def saveRejected(rejectedRDD: RDD[String]): Try[Path]
def saveRows(dataset: DataFrame, targetPath: Path, writeMode: WriteMode, area: StorageArea, merge: Boolean): (DataFrameWriter[Row], String)

Save typed dataset in parquet.
Save typed dataset in parquet. If hive support is active, also register it as a Hive Table and if analyze is active, also compute basic statistics
dataset
: dataset to save
targetPath
: absolute path
writeMode
: Append or overwrite
area
: accepted or rejected area
lazy val session: SparkSession

Definition Classes
SparkJob
lazy val sparkEnv: SparkEnv

Definition Classes
SparkJob
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package ingest

trait IngestionJob extends SparkJob

Abstract Value Members

abstract def domain: Domain

abstract def ingest(dataset: DataFrame): (RDD[_], RDD[_])

abstract def loadDataSet(): Try[DataFrame]

abstract def name: String

abstract def path: List[Path]

abstract def schema: Schema

abstract def schemaHandler: SchemaHandler

implicit abstract def settings: Settings

abstract def storageHandler: StorageHandler

abstract def types: List[Type]

Concrete Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def getWriteMode(): WriteMode

def hashCode(): Int

def index(mergedDF: DataFrame): Unit

final def isInstanceOf[T0]: Boolean

val logger: Logger

def merge(inputDF: DataFrame, existingDF: DataFrame, merge: MergeOptions): DataFrame

lazy val metadata: Metadata

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val now: Timestamp

def partitionDataset(dataset: DataFrame, partition: List[String]): DataFrame

def partitionedDatasetWriter(dataset: DataFrame, partition: List[String]): DataFrameWriter[Row]

def run(): Try[SparkSession]

def saveAccepted(acceptedDF: DataFrame): (DataFrame, Path)

def saveRejected(rejectedRDD: RDD[String]): Try[Path]

def saveRows(dataset: DataFrame, targetPath: Path, writeMode: WriteMode, area: StorageArea, merge: Boolean): (DataFrameWriter[Row], String)

lazy val session: SparkSession

lazy val sparkEnv: SparkEnv

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from SparkJob

Inherited from StrictLogging

Inherited from AnyRef

Inherited from Any

Ungrouped