SnowflakeTableDataObject

Instance Constructors

new SnowflakeTableDataObject(id: DataObjectId, schemaMin: Option[StructType] = None, table: Table, saveMode: SaveMode = SaveMode.Overwrite, connectionId: ConnectionId, comment: Option[String], metadata: Option[DataObjectMetadata] = None)(implicit instanceRegistry: InstanceRegistry)

id
unique name of this data object
schemaMin
An optional, minimal schema that this DataObject must have to pass schema validation on reading and writing.
table
Snowflake table to be written by this output
saveMode
spark SaveMode to use when writing files, default is "overwrite"
connectionId
The SnowflakeTableConnection to use for the table
comment
An optional comment to add to the table after writing a DataFrame to it
metadata
meta data

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def addFieldIfNotExisting(writeSchema: StructType, colName: String, dataType: DataType): StructType

Attributes
protected
Definition Classes
CanCreateDataFrame
final def asInstanceOf[T0]: T0

Definition Classes
Any
def atlasName: String

Definition Classes
TableDataObject → DataObject → AtlasExportable
def atlasQualifiedName(prefix: String): String

Definition Classes
TableDataObject → AtlasExportable
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
val comment: Option[String]

An optional comment to add to the table after writing a DataFrame to it
val connectionId: ConnectionId

The SnowflakeTableConnection to use for the table
def createReadSchema(writeSchema: StructType)(implicit session: SparkSession): StructType

Definition Classes
CanCreateDataFrame
def dropTable(implicit session: SparkSession): Unit

Definition Classes
SnowflakeTableDataObject → TableDataObject
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def factory: FromConfigFactory[DataObject]

<invalid inheritdoc annotation>
<invalid inheritdoc annotation>

Definition Classes
SnowflakeTableDataObject → ParsableFromConfig
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getConnection[T <: Connection](connectionId: ConnectionId)(implicit registry: InstanceRegistry, ct: ClassTag[T], tt: scala.reflect.api.JavaUniverse.TypeTag[T]): T

Attributes
protected
Definition Classes
DataObject
def getConnectionReg[T <: Connection](connectionId: ConnectionId, registry: InstanceRegistry)(implicit ct: ClassTag[T], tt: scala.reflect.api.JavaUniverse.TypeTag[T]): T

Attributes
protected
Definition Classes
DataObject
def getDataFrame(partitionValues: Seq[PartitionValues] = Seq())(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

Definition Classes
SnowflakeTableDataObject → CanCreateDataFrame
def getPKduplicates(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

Definition Classes
TableDataObject
def getPKnulls(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

Definition Classes
TableDataObject
def getPKviolators(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

Definition Classes
TableDataObject
def housekeepingMode: Option[HousekeepingMode]

Definition Classes
DataObject
val id: DataObjectId

unique name of this data object
unique name of this data object

Definition Classes
SnowflakeTableDataObject → DataObject → SdlConfigObject
def init(df: DataFrame, partitionValues: Seq[PartitionValues])(implicit session: SparkSession, context: ActionPipelineContext): Unit

Definition Classes
CanWriteDataFrame
implicit val instanceRegistry: InstanceRegistry
def isDbExisting(implicit session: SparkSession): Boolean

Definition Classes
SnowflakeTableDataObject → TableDataObject
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isPKcandidateKey(implicit session: SparkSession, context: ActionPipelineContext): Boolean

Definition Classes
TableDataObject
def isTableExisting(implicit session: SparkSession): Boolean

Definition Classes
SnowflakeTableDataObject → TableDataObject
lazy val logger: Logger

Attributes
protected
Definition Classes
SmartDataLakeLogger
val metadata: Option[DataObjectMetadata]

meta data
meta data

Definition Classes
SnowflakeTableDataObject → DataObject
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val saveMode: SaveMode

spark SaveMode to use when writing files, default is "overwrite"
val schemaMin: Option[StructType]

An optional, minimal schema that this DataObject must have to pass schema validation on reading and writing.
An optional, minimal schema that this DataObject must have to pass schema validation on reading and writing.

Definition Classes
SnowflakeTableDataObject → SchemaValidation
def streamingOptions: Map[String, String]

Definition Classes
CanWriteDataFrame
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
var table: Table

Snowflake table to be written by this output
Snowflake table to be written by this output

Definition Classes
SnowflakeTableDataObject → TableDataObject
var tableSchema: StructType

Definition Classes
TableDataObject
def toStringShort: String

Definition Classes
DataObject
def validateSchema(df: DataFrame, schemaExpected: StructType, role: String): Unit

Definition Classes
SchemaValidation
def validateSchemaMin(df: DataFrame, role: String): Unit

Definition Classes
SchemaValidation
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def writeDataFrame(df: DataFrame, createTableOnly: Boolean, partitionValues: Seq[PartitionValues])(implicit session: SparkSession): Unit

Writes DataFrame to Snowflake Snowflake does not support explicit partitions, so any passed partition values are ignored
def writeDataFrame(df: DataFrame, partitionValues: Seq[PartitionValues] = Seq(), isRecursiveInput: Boolean = false)(implicit session: SparkSession, context: ActionPipelineContext): Unit

Definition Classes
SnowflakeTableDataObject → CanWriteDataFrame
def writeStreamingDataFrame(df: DataFrame, trigger: Trigger, options: Map[String, String], checkpointLocation: String, queryName: String, outputMode: OutputMode)(implicit session: SparkSession, context: ActionPipelineContext): StreamingQuery

Definition Classes
CanWriteDataFrame

Related Docs: object SnowflakeTableDataObject | package dataobject

Instance Constructors

new SnowflakeTableDataObject(id: DataObjectId, schemaMin: Option[StructType] = None, table: Table, saveMode: SaveMode = SaveMode.Overwrite, connectionId: ConnectionId, comment: Option[String], metadata: Option[DataObjectMetadata] = None)(implicit instanceRegistry: InstanceRegistry)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def addFieldIfNotExisting(writeSchema: StructType, colName: String, dataType: DataType): StructType

final def asInstanceOf[T0]: T0

def atlasName: String

def atlasQualifiedName(prefix: String): String

def clone(): AnyRef

val comment: Option[String]

val connectionId: ConnectionId

def createReadSchema(writeSchema: StructType)(implicit session: SparkSession): StructType

def dropTable(implicit session: SparkSession): Unit

final def eq(arg0: AnyRef): Boolean

def factory: FromConfigFactory[DataObject]

def finalize(): Unit

final def getClass(): Class[_]

def getConnection[T <: Connection](connectionId: ConnectionId)(implicit registry: InstanceRegistry, ct: ClassTag[T], tt: scala.reflect.api.JavaUniverse.TypeTag[T]): T

def getConnectionReg[T <: Connection](connectionId: ConnectionId, registry: InstanceRegistry)(implicit ct: ClassTag[T], tt: scala.reflect.api.JavaUniverse.TypeTag[T]): T

def getDataFrame(partitionValues: Seq[PartitionValues] = Seq())(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

def getPKduplicates(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

def getPKnulls(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

def getPKviolators(implicit session: SparkSession, context: ActionPipelineContext): DataFrame

def housekeepingMode: Option[HousekeepingMode]

val id: DataObjectId

def init(df: DataFrame, partitionValues: Seq[PartitionValues])(implicit session: SparkSession, context: ActionPipelineContext): Unit

implicit val instanceRegistry: InstanceRegistry

def isDbExisting(implicit session: SparkSession): Boolean

final def isInstanceOf[T0]: Boolean

def isPKcandidateKey(implicit session: SparkSession, context: ActionPipelineContext): Boolean

def isTableExisting(implicit session: SparkSession): Boolean

lazy val logger: Logger

val metadata: Option[DataObjectMetadata]

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val saveMode: SaveMode

val schemaMin: Option[StructType]

def streamingOptions: Map[String, String]

final def synchronized[T0](arg0: ⇒ T0): T0

var table: Table

var tableSchema: StructType

def toStringShort: String

def validateSchema(df: DataFrame, schemaExpected: StructType, role: String): Unit

def validateSchemaMin(df: DataFrame, role: String): Unit

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

def writeDataFrame(df: DataFrame, createTableOnly: Boolean, partitionValues: Seq[PartitionValues])(implicit session: SparkSession): Unit

def writeDataFrame(df: DataFrame, partitionValues: Seq[PartitionValues] = Seq(), isRecursiveInput: Boolean = false)(implicit session: SparkSession, context: ActionPipelineContext): Unit

def writeStreamingDataFrame(df: DataFrame, trigger: Trigger, options: Map[String, String], checkpointLocation: String, queryName: String, outputMode: OutputMode)(implicit session: SparkSession, context: ActionPipelineContext): StreamingQuery

Inherited from Serializable

Inherited from Serializable

Inherited from Product

Inherited from Equals

Inherited from TransactionalSparkTableDataObject

Inherited from CanWriteDataFrame

Inherited from TableDataObject

Inherited from SchemaValidation

Inherited from CanCreateDataFrame

Inherited from DataObject

Inherited from AtlasExportable

Inherited from SmartDataLakeLogger

Inherited from ParsableFromConfig[DataObject]

Inherited from SdlConfigObject

Inherited from AnyRef

Inherited from Any

Ungrouped