DsvIngestionJob

Instance Constructors

new DsvIngestionJob(domain: Domain, schema: Schema, types: List[Type], path: List[Path], storageHandler: StorageHandler, schemaHandler: SchemaHandler)(implicit settings: Settings)

domain
: Input Dataset Domain
schema
: Input Dataset Schema
types
: List of globally defined types
path
: Input dataset path
storageHandler
: Storage Handler

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def cleanHeaderCol(header: String): String

remove any extra quote / BOM in the header
remove any extra quote / BOM in the header
header
: Header column name
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
val domain: Domain

: Input Dataset Domain
: Input Dataset Domain

Definition Classes
DsvIngestionJob → IngestionJob
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getWriteMode(): WriteMode

Definition Classes
IngestionJob
def hashCode(): Int

Definition Classes
AnyRef → Any
def index(mergedDF: DataFrame): Unit

Definition Classes
IngestionJob
def ingest(dataset: DataFrame): (RDD[_], RDD[_])

Apply the schema to the dataset.
Apply the schema to the dataset. This is where all the magic happen Valid records are stored in the accepted path / table and invalid records in the rejected path / table
dataset
: Spark Dataset

Definition Classes
DsvIngestionJob → IngestionJob
def intersectHeaders(datasetHeaders: List[String], schemaHeaders: List[String]): (List[String], List[String])

datasetHeaders
: Headers found in the dataset
schemaHeaders
: Headers defined in the schema
returns
two lists : One with thecolumns present in the schema and the dataset and onther with the headers present in the dataset only
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def loadDataSet(): Try[DataFrame]

Load dataset using spark csv reader and all metadata.
Load dataset using spark csv reader and all metadata. Does not infer schema. columns not defined in the schema are dropped fro the dataset (require datsets with a header)
returns
Spark Dataset

Definition Classes
DsvIngestionJob → IngestionJob
val logger: Logger

Attributes
protected
Definition Classes
StrictLogging
def merge(inputDF: DataFrame, existingDF: DataFrame, merge: MergeOptions): DataFrame

Merge incoming and existing dataframes using merge options
Merge incoming and existing dataframes using merge options
returns
merged dataframe

Definition Classes
IngestionJob
lazy val metadata: Metadata

Merged metadata
Merged metadata

Definition Classes
IngestionJob
def name: String

returns
Spark Job name

Definition Classes
DsvIngestionJob → SparkJob
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val now: Timestamp

Definition Classes
IngestionJob
def partitionDataset(dataset: DataFrame, partition: List[String]): DataFrame

Definition Classes
SparkJob
def partitionedDatasetWriter(dataset: DataFrame, partition: List[String]): DataFrameWriter[Row]

Partition a dataset using dataset columns.
Partition a dataset using dataset columns. To partition the dataset using the igestion time, use the reserved column names :
- comet_year
- comet_month
- comet_day
- comet_hour
- comet_minute These columsn are renamed to "year", "month", "day", "hour", "minute" in the dataset and their values is set to the current date/time.
dataset
: Input dataset
partition
: list of columns to use for partitioning.
returns
The Spark session used to run this job

Definition Classes
SparkJob
val path: List[Path]

: Input dataset path
: Input dataset path

Definition Classes
DsvIngestionJob → IngestionJob
def run(): Try[SparkSession]

Main entry point as required by the Spark Job interface
Main entry point as required by the Spark Job interface
returns
: Spark Session used for the job

Definition Classes
IngestionJob → SparkJob
def saveAccepted(acceptedRDD: RDD[Row], orderedSparkTypes: StructType): (DataFrame, Path)
def saveAccepted(acceptedDF: DataFrame): (DataFrame, Path)

Merge new and existing dataset if required Save using overwrite / Append mode
Merge new and existing dataset if required Save using overwrite / Append mode

Definition Classes
IngestionJob
def saveRejected(rejectedRDD: RDD[String]): Try[Path]

Definition Classes
IngestionJob
def saveRows(dataset: DataFrame, targetPath: Path, writeMode: WriteMode, area: StorageArea, merge: Boolean): (DataFrameWriter[Row], String)

Save typed dataset in parquet.
Save typed dataset in parquet. If hive support is active, also register it as a Hive Table and if analyze is active, also compute basic statistics
dataset
: dataset to save
targetPath
: absolute path
writeMode
: Append or overwrite
area
: accepted or rejected area

Definition Classes
IngestionJob
val schema: Schema

: Input Dataset Schema
: Input Dataset Schema

Definition Classes
DsvIngestionJob → IngestionJob
val schemaHandler: SchemaHandler

Definition Classes
DsvIngestionJob → IngestionJob
val schemaHeaders: List[String]

dataset Header names as defined by the schema
lazy val session: SparkSession

Definition Classes
SparkJob
implicit val settings: Settings

Definition Classes
DsvIngestionJob → SparkJob
lazy val sparkEnv: SparkEnv

Definition Classes
SparkJob
val storageHandler: StorageHandler

: Storage Handler
: Storage Handler

Definition Classes
DsvIngestionJob → IngestionJob
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
val types: List[Type]

: List of globally defined types
: List of globally defined types

Definition Classes
DsvIngestionJob → IngestionJob
def validateHeader(datasetHeaders: List[String], schemaHeaders: List[String]): Boolean

datasetHeaders
: Headers found in the dataset
schemaHeaders
: Headers defined in the schema
returns
success if all headers in the schema exist in the dataset
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package ingest

class DsvIngestionJob extends IngestionJob

Instance Constructors

new DsvIngestionJob(domain: Domain, schema: Schema, types: List[Type], path: List[Path], storageHandler: StorageHandler, schemaHandler: SchemaHandler)(implicit settings: Settings)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def cleanHeaderCol(header: String): String

def clone(): AnyRef

val domain: Domain

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def getWriteMode(): WriteMode

def hashCode(): Int

def index(mergedDF: DataFrame): Unit

def ingest(dataset: DataFrame): (RDD[_], RDD[_])

def intersectHeaders(datasetHeaders: List[String], schemaHeaders: List[String]): (List[String], List[String])

final def isInstanceOf[T0]: Boolean

def loadDataSet(): Try[DataFrame]

val logger: Logger

def merge(inputDF: DataFrame, existingDF: DataFrame, merge: MergeOptions): DataFrame

lazy val metadata: Metadata

def name: String

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val now: Timestamp

def partitionDataset(dataset: DataFrame, partition: List[String]): DataFrame

def partitionedDatasetWriter(dataset: DataFrame, partition: List[String]): DataFrameWriter[Row]

val path: List[Path]

def run(): Try[SparkSession]

def saveAccepted(acceptedRDD: RDD[Row], orderedSparkTypes: StructType): (DataFrame, Path)

def saveAccepted(acceptedDF: DataFrame): (DataFrame, Path)

def saveRejected(rejectedRDD: RDD[String]): Try[Path]

def saveRows(dataset: DataFrame, targetPath: Path, writeMode: WriteMode, area: StorageArea, merge: Boolean): (DataFrameWriter[Row], String)

val schema: Schema

val schemaHandler: SchemaHandler

val schemaHeaders: List[String]

lazy val session: SparkSession

implicit val settings: Settings

lazy val sparkEnv: SparkEnv

val storageHandler: StorageHandler

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

val types: List[Type]

def validateHeader(datasetHeaders: List[String], schemaHeaders: List[String]): Boolean

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from IngestionJob

Inherited from SparkJob

Inherited from StrictLogging

Inherited from AnyRef

Inherited from Any

Ungrouped