HiveTableDataObject

Instance Constructors

new HiveTableDataObject(id: DataObjectId, path: String, partitions: Seq[String] = Seq(), analyzeTableAfterWrite: Boolean = false, dateColumnType: DateColumnType = DateColumnType.Date, schemaMin: Option[StructType] = None, table: Table, numInitialHdfsPartitions: Int = 16, saveMode: SaveMode = SaveMode.Overwrite, acl: Option[AclDef] = None, connectionId: Option[ConnectionId] = None, metadata: Option[DataObjectMetadata] = None)(implicit instanceRegistry: InstanceRegistry)

id
unique name of this data object
path
hadoop directory for this table. If it doesn't contain scheme and authority, the connections pathPrefix is applied. If pathPrefix is not defined or doesn't define scheme and authority, default schema and authority is applied.
partitions
partition columns for this data object
analyzeTableAfterWrite
enable compute statistics after writing data (default=false)
dateColumnType
type of date column
schemaMin
An optional, minimal schema that this DataObject must have to pass schema validation on reading and writing.
table
hive table to be written by this output
numInitialHdfsPartitions
number of files created when writing into an empty table (otherwise the number will be derived from the existing data)
saveMode
spark SaveMode to use when writing files, default is "overwrite"
acl
override connections permissions for files created tables hadoop directory with this connection
connectionId
optional id of io.smartdatalake.workflow.connection.HiveTableConnection
metadata
meta data

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
val acl: Option[AclDef]

override connections permissions for files created tables hadoop directory with this connection
val analyzeTableAfterWrite: Boolean

enable compute statistics after writing data (default=false)
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
val connectionId: Option[ConnectionId]

optional id of io.smartdatalake.workflow.connection.HiveTableConnection
def createEmptyPartition(partitionValues: PartitionValues)(implicit session: SparkSession): Unit

create empty partition
create empty partition

Definition Classes
HiveTableDataObject → CanHandlePartitions
final def createMissingPartitions(partitionValues: Seq[PartitionValues])(implicit session: SparkSession): Unit

Create empty partitions for partition values not yet existing
Create empty partitions for partition values not yet existing

Definition Classes
CanHandlePartitions
val dateColumnType: DateColumnType

type of date column
def deletePartitions(partitionValues: Seq[PartitionValues])(implicit session: SparkSession): Unit

Delete given partitions.
Delete given partitions. This is used to cleanup partitions after they are processed.

Definition Classes
CanHandlePartitions
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def factory: FromConfigFactory[DataObject]

Returns the factory that can parse this type (that is, type CO).
Returns the factory that can parse this type (that is, type CO).
Typically, implementations of this method should return the companion object of the implementing class. The companion object in turn should implement FromConfigFactory.
returns
the factory (object) for this class.

Definition Classes
HiveTableDataObject → ParsableFromConfig
def filesystem(implicit session: SparkSession): FileSystem
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getConnection[T <: Connection](connectionId: ConnectionId)(implicit registry: InstanceRegistry, ct: ClassTag[T], tt: scala.reflect.api.JavaUniverse.TypeTag[T]): T

Handle class cast exception when getting objects from instance registry
Handle class cast exception when getting objects from instance registry

Attributes
protected
Definition Classes
DataObject
def getConnectionReg[T <: Connection](connectionId: ConnectionId, registry: InstanceRegistry)(implicit ct: ClassTag[T], tt: scala.reflect.api.JavaUniverse.TypeTag[T]): T

Attributes
protected
Definition Classes
DataObject
def getDataFrame(partitionValues: Seq[PartitionValues] = Seq())(implicit session: SparkSession): DataFrame

Definition Classes
HiveTableDataObject → CanCreateDataFrame
def getPKduplicates(implicit session: SparkSession): DataFrame

Definition Classes
TableDataObject
def getPKnulls(implicit session: SparkSession): DataFrame

Definition Classes
TableDataObject
def getPKviolators(implicit session: SparkSession): DataFrame

Definition Classes
TableDataObject
val id: DataObjectId

unique name of this data object
unique name of this data object

Definition Classes
HiveTableDataObject → DataObject → SdlConfigObject
def init(df: DataFrame, partitionValues: Seq[PartitionValues])(implicit session: SparkSession): Unit

Initialize callback before writing data out to disk/sinks.
Initialize callback before writing data out to disk/sinks.

Definition Classes
HiveTableDataObject → CanWriteDataFrame
implicit val instanceRegistry: InstanceRegistry
def isDbExisting(implicit session: SparkSession): Boolean

Definition Classes
HiveTableDataObject → TableDataObject
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isPKcandidateKey(implicit session: SparkSession): Boolean

Definition Classes
TableDataObject
def isTableExisting(implicit session: SparkSession): Boolean

Definition Classes
HiveTableDataObject → TableDataObject
def listPartitions(implicit session: SparkSession): Seq[PartitionValues]

list hive table partitions
list hive table partitions

Definition Classes
HiveTableDataObject → CanHandlePartitions
lazy val logger: Logger

Attributes
protected
Definition Classes
SmartDataLakeLogger
val metadata: Option[DataObjectMetadata]

meta data
meta data

Definition Classes
HiveTableDataObject → DataObject
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val numInitialHdfsPartitions: Int

number of files created when writing into an empty table (otherwise the number will be derived from the existing data)
val partitions: Seq[String]

partition columns for this data object
partition columns for this data object

Definition Classes
HiveTableDataObject → CanHandlePartitions
val path: String

hadoop directory for this table.
hadoop directory for this table. If it doesn't contain scheme and authority, the connections pathPrefix is applied. If pathPrefix is not defined or doesn't define scheme and authority, default schema and authority is applied.
def postRead(implicit session: SparkSession): Unit

Runs operations after reading from DataObject
Runs operations after reading from DataObject

Definition Classes
DataObject
def postWrite(implicit session: SparkSession): Unit

Runs operations after writing to DataObject
Runs operations after writing to DataObject

Definition Classes
DataObject
def preRead(implicit session: SparkSession): Unit

Runs operations before reading from DataObject
Runs operations before reading from DataObject

Definition Classes
DataObject
def preWrite(implicit session: SparkSession): Unit

Runs operations before writing to DataObject
Runs operations before writing to DataObject

Definition Classes
DataObject
def prepare(implicit session: SparkSession): Unit

Prepare & test DataObject's prerequisits
Prepare & test DataObject's prerequisits
This runs during the "prepare" operation of the DAG.

Definition Classes
HiveTableDataObject → DataObject
val saveMode: SaveMode

spark SaveMode to use when writing files, default is "overwrite"
val schemaMin: Option[StructType]

An optional, minimal schema that this DataObject must have to pass schema validation on reading and writing.
An optional, minimal schema that this DataObject must have to pass schema validation on reading and writing.

Definition Classes
HiveTableDataObject → SchemaValidation
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
var table: Table

hive table to be written by this output
hive table to be written by this output

Definition Classes
HiveTableDataObject → TableDataObject
var tableSchema: StructType

Definition Classes
TableDataObject
def toStringShort: String

Definition Classes
DataObject
def validateSchemaMin(df: DataFrame): Unit

Validate the schema of a given Spark Data Frame df against schemaMin.
Validate the schema of a given Spark Data Frame df against schemaMin.
df
The data frame to validate.

Definition Classes
SchemaValidation
Exceptions thrown
SchemaViolationException is the schemaMin does not validate.
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def writeDataFrame(df: DataFrame, partitionValues: Seq[PartitionValues])(implicit session: SparkSession): Unit

Definition Classes
HiveTableDataObject → CanWriteDataFrame

Related Docs: object HiveTableDataObject | package dataobject

Instance Constructors

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

val acl: Option[AclDef]

val analyzeTableAfterWrite: Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

val connectionId: Option[ConnectionId]

def createEmptyPartition(partitionValues: PartitionValues)(implicit session: SparkSession): Unit

final def createMissingPartitions(partitionValues: Seq[PartitionValues])(implicit session: SparkSession): Unit

val dateColumnType: DateColumnType

def deletePartitions(partitionValues: Seq[PartitionValues])(implicit session: SparkSession): Unit

final def eq(arg0: AnyRef): Boolean

def factory: FromConfigFactory[DataObject]

def filesystem(implicit session: SparkSession): FileSystem

def finalize(): Unit

final def getClass(): Class[_]

def getConnection[T <: Connection](connectionId: ConnectionId)(implicit registry: InstanceRegistry, ct: ClassTag[T], tt: scala.reflect.api.JavaUniverse.TypeTag[T]): T

def getConnectionReg[T <: Connection](connectionId: ConnectionId, registry: InstanceRegistry)(implicit ct: ClassTag[T], tt: scala.reflect.api.JavaUniverse.TypeTag[T]): T

def getDataFrame(partitionValues: Seq[PartitionValues] = Seq())(implicit session: SparkSession): DataFrame

def getPKduplicates(implicit session: SparkSession): DataFrame

def getPKnulls(implicit session: SparkSession): DataFrame

def getPKviolators(implicit session: SparkSession): DataFrame

val id: DataObjectId

def init(df: DataFrame, partitionValues: Seq[PartitionValues])(implicit session: SparkSession): Unit

implicit val instanceRegistry: InstanceRegistry

def isDbExisting(implicit session: SparkSession): Boolean

final def isInstanceOf[T0]: Boolean

def isPKcandidateKey(implicit session: SparkSession): Boolean

def isTableExisting(implicit session: SparkSession): Boolean

def listPartitions(implicit session: SparkSession): Seq[PartitionValues]

lazy val logger: Logger

val metadata: Option[DataObjectMetadata]

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val numInitialHdfsPartitions: Int

val partitions: Seq[String]

val path: String

def postRead(implicit session: SparkSession): Unit

def postWrite(implicit session: SparkSession): Unit

def preRead(implicit session: SparkSession): Unit

def preWrite(implicit session: SparkSession): Unit

def prepare(implicit session: SparkSession): Unit

val saveMode: SaveMode

val schemaMin: Option[StructType]

final def synchronized[T0](arg0: ⇒ T0): T0

var table: Table

var tableSchema: StructType

def toStringShort: String

def validateSchemaMin(df: DataFrame): Unit

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

def writeDataFrame(df: DataFrame, partitionValues: Seq[PartitionValues])(implicit session: SparkSession): Unit

Inherited from Serializable

Inherited from Serializable

Inherited from Product

Inherited from Equals

Inherited from CanHandlePartitions

Inherited from CanWriteDataFrame

Inherited from TableDataObject

Inherited from SchemaValidation

Inherited from CanCreateDataFrame

Inherited from DataObject

Inherited from SmartDataLakeLogger

Inherited from ParsableFromConfig[DataObject]

Inherited from SdlConfigObject

Inherited from AnyRef

Inherited from Any

Ungrouped