JsonIngestionJob

Instance Constructors

new JsonIngestionJob(domain: Domain, schema: Schema, types: List[Type], path: List[Path], storageHandler: StorageHandler, schemaHandler: SchemaHandler)(implicit settings: Settings)

domain
: Input Dataset Domain
schema
: Input Dataset Schema
types
: List of globally defined types
path
: Input dataset path
storageHandler
: Storage Handler

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
val domain: Domain

: Input Dataset Domain
: Input Dataset Domain

Definition Classes
JsonIngestionJob → IngestionJob
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getWriteMode(): WriteMode

Definition Classes
IngestionJob
def hashCode(): Int

Definition Classes
AnyRef → Any
def index(mergedDF: DataFrame): Unit

Definition Classes
IngestionJob
def ingest(dataset: DataFrame): (RDD[_], RDD[_])

Where the magic happen
Where the magic happen
dataset
input dataset as a RDD of string

Definition Classes
JsonIngestionJob → IngestionJob
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def loadDataSet(): Try[DataFrame]

load the json as an RDD of String
load the json as an RDD of String
returns
Spark Dataframe loaded using metadata options

Definition Classes
JsonIngestionJob → IngestionJob
val logger: Logger

Attributes
protected
Definition Classes
StrictLogging
def merge(inputDF: DataFrame, existingDF: DataFrame, merge: MergeOptions): DataFrame

Merge incoming and existing dataframes using merge options
Merge incoming and existing dataframes using merge options
returns
merged dataframe

Definition Classes
IngestionJob
lazy val metadata: Metadata

Merged metadata
Merged metadata

Definition Classes
IngestionJob
def name: String

Definition Classes
JsonIngestionJob → SparkJob
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val now: Timestamp

Definition Classes
IngestionJob
def partitionDataset(dataset: DataFrame, partition: List[String]): DataFrame

Definition Classes
SparkJob
def partitionedDatasetWriter(dataset: DataFrame, partition: List[String]): DataFrameWriter[Row]

Partition a dataset using dataset columns.
Partition a dataset using dataset columns. To partition the dataset using the igestion time, use the reserved column names :
- comet_year
- comet_month
- comet_day
- comet_hour
- comet_minute These columsn are renamed to "year", "month", "day", "hour", "minute" in the dataset and their values is set to the current date/time.
dataset
: Input dataset
partition
: list of columns to use for partitioning.
returns
The Spark session used to run this job

Definition Classes
SparkJob
val path: List[Path]

: Input dataset path
: Input dataset path

Definition Classes
JsonIngestionJob → IngestionJob
def run(): Try[SparkSession]

Main entry point as required by the Spark Job interface
Main entry point as required by the Spark Job interface
returns
: Spark Session used for the job

Definition Classes
IngestionJob → SparkJob
def saveAccepted(acceptedDF: DataFrame): (DataFrame, Path)

Merge new and existing dataset if required Save using overwrite / Append mode
Merge new and existing dataset if required Save using overwrite / Append mode

Definition Classes
IngestionJob
def saveRejected(rejectedRDD: RDD[String]): Try[Path]

Definition Classes
IngestionJob
def saveRows(dataset: DataFrame, targetPath: Path, writeMode: WriteMode, area: StorageArea, merge: Boolean): (DataFrameWriter[Row], String)

Save typed dataset in parquet.
Save typed dataset in parquet. If hive support is active, also register it as a Hive Table and if analyze is active, also compute basic statistics
dataset
: dataset to save
targetPath
: absolute path
writeMode
: Append or overwrite
area
: accepted or rejected area

Definition Classes
IngestionJob
val schema: Schema

: Input Dataset Schema
: Input Dataset Schema

Definition Classes
JsonIngestionJob → IngestionJob
val schemaHandler: SchemaHandler

Definition Classes
JsonIngestionJob → IngestionJob
lazy val schemaSparkType: StructType
lazy val session: SparkSession

Definition Classes
SparkJob
implicit val settings: Settings

Definition Classes
JsonIngestionJob → SparkJob
lazy val sparkEnv: SparkEnv

Definition Classes
SparkJob
val storageHandler: StorageHandler

: Storage Handler
: Storage Handler

Definition Classes
JsonIngestionJob → IngestionJob
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
val types: List[Type]

: List of globally defined types
: List of globally defined types

Definition Classes
JsonIngestionJob → IngestionJob
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Deprecated Value Members

def saveAccepted(acceptedRDD: RDD[Row]): Path

Use the schema we used for validation when saving
Use the schema we used for validation when saving

Annotations
@deprecated
Deprecated
(Since version ) We let Spark compute the final schema

Related Doc: package ingest

class JsonIngestionJob extends IngestionJob

Instance Constructors

new JsonIngestionJob(domain: Domain, schema: Schema, types: List[Type], path: List[Path], storageHandler: StorageHandler, schemaHandler: SchemaHandler)(implicit settings: Settings)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

val domain: Domain

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def getWriteMode(): WriteMode

def hashCode(): Int

def index(mergedDF: DataFrame): Unit

def ingest(dataset: DataFrame): (RDD[_], RDD[_])

final def isInstanceOf[T0]: Boolean

def loadDataSet(): Try[DataFrame]

val logger: Logger

def merge(inputDF: DataFrame, existingDF: DataFrame, merge: MergeOptions): DataFrame

lazy val metadata: Metadata

def name: String

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val now: Timestamp

def partitionDataset(dataset: DataFrame, partition: List[String]): DataFrame

def partitionedDatasetWriter(dataset: DataFrame, partition: List[String]): DataFrameWriter[Row]

val path: List[Path]

def run(): Try[SparkSession]

def saveAccepted(acceptedDF: DataFrame): (DataFrame, Path)

def saveRejected(rejectedRDD: RDD[String]): Try[Path]

def saveRows(dataset: DataFrame, targetPath: Path, writeMode: WriteMode, area: StorageArea, merge: Boolean): (DataFrameWriter[Row], String)

val schema: Schema

val schemaHandler: SchemaHandler

lazy val schemaSparkType: StructType

lazy val session: SparkSession

implicit val settings: Settings

lazy val sparkEnv: SparkEnv

val storageHandler: StorageHandler

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

val types: List[Type]

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Deprecated Value Members

def saveAccepted(acceptedRDD: RDD[Row]): Path

Inherited from IngestionJob

Inherited from SparkJob

Inherited from StrictLogging

Inherited from AnyRef

Inherited from Any

Ungrouped