SparkApplication

Instance Constructors

new SparkApplication()

Abstract Value Members

abstract val configDefinition: ConfigDefinition

The configuration files definition
abstract def executionPlans(implicit spark: SparkSession): Map[String, ExecutionConfig]

The defined execution plans
The defined execution plans
spark
an implicit spark session
returns
A JobName/ExecutionPlan Map
abstract def load(dataFrame: DataFrame, database: String, table: String, optionalJobParameters: OptionalJobParameters[U, V]): Unit

Saves the resulting dataFrame to disk
Saves the resulting dataFrame to disk
dataFrame
The resulting DataFrame
database
The output database name
table
The output table name (job name)
optionalJobParameters
An OptionalJobParameters object containing any custom argument/application files we defined through our application.

Definition Classes
ETL

Concrete Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def extract(jobName: String, jobSourcesConf: List[JobSource], sourcesConf: SourcesConf, env: String)(implicit spark: SparkSession): Sources

Extracts data from a provided sources configuration
Extracts data from a provided sources configuration
jobName
The Job name
jobSourcesConf
The Job input sources configuration
sourcesConf
The different input sources configuration
env
The environment in which we want to extract the input sources from
spark
An implicit spark session
returns
A Map of sourceName/SourcePipeline containing the extracted sources.

Definition Classes
SparkApplication → ETL
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
final def getSparkSession(sparkParamsConf: SparkParamsConf): SparkSession

Definition Classes
SparkSessionProvider
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def loadExecutionData(args: Array[String]): ExecutionData

Loads a set of parameters needed for the ETL Operation
Loads a set of parameters needed for the ETL Operation
through : config files loading, argument parsing, execution parameters creation, etc...
args
The list of arguments to parse
returns
An ExecutionData object containing all the required parameters

Attributes
protected
Definition Classes
SparkApplication → ETL
val logger: Logger

Definition Classes
LoggerProvider
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def runETL[Config, Argument, ConfigInput, ArgumentInput](executionData: ExecutionData): Unit

The main method containing the logic for running our ETL job
The main method containing the logic for running our ETL job
executionData
The ExecutionData object that will be used

Definition Classes
ETL
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
def transform(jobName: String, sources: Sources, executionPlan: ExecutionPlan, outputColumns: Option[Seq[String]], exportDateColumn: Option[String])(implicit spark: SparkSession): DataFrame

Apply transformations to a given set of sources
Apply transformations to a given set of sources
jobName
The Job name
sources
The extracted input sources
executionPlan
The execution plan to apply
exportDateColumn
An optional date column name to tie the result computation date
spark
An implicit spark session
returns
The resulting DataFrame

Definition Classes
SparkApplication → ETL
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Docs: object SparkApplication | package bigdata

abstract class SparkApplication[U, V] extends SparkSessionProvider with ETL[U, V] with LoggerProvider

Instance Constructors

new SparkApplication()

Abstract Value Members

abstract val configDefinition: ConfigDefinition

abstract def executionPlans(implicit spark: SparkSession): Map[String, ExecutionConfig]

abstract def load(dataFrame: DataFrame, database: String, table: String, optionalJobParameters: OptionalJobParameters[U, V]): Unit

Concrete Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def extract(jobName: String, jobSourcesConf: List[JobSource], sourcesConf: SourcesConf, env: String)(implicit spark: SparkSession): Sources

def finalize(): Unit

final def getClass(): Class[_]

final def getSparkSession(sparkParamsConf: SparkParamsConf): SparkSession

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

def loadExecutionData(args: Array[String]): ExecutionData

val logger: Logger

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def runETL[Config, Argument, ConfigInput, ArgumentInput](executionData: ExecutionData): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

def transform(jobName: String, sources: Sources, executionPlan: ExecutionPlan, outputColumns: Option[Seq[String]], exportDateColumn: Option[String])(implicit spark: SparkSession): DataFrame

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from LoggerProvider

Inherited from ETL[U, V]

Inherited from SparkSessionProvider

Inherited from AnyRef

Inherited from Any

Ungrouped