Atum

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def cachingStorageLevel: StorageLevel

Returns the current caching storage level.
Returns the current caching storage level.

Definition Classes
Atum
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def disableCaching(): Unit

Turns off caching that happens every time a checkpoint is generated.
Turns off caching that happens every time a checkpoint is generated.

Definition Classes
Atum
def enableCaching(cacheStorageLevel: StorageLevel = StorageLevel.MEMORY_AND_DISK): Unit

Turns on caching that happens every time a checkpoint is generated (default behavior).
Turns on caching that happens every time a checkpoint is generated (default behavior).
Atum uses the default storage level for Spark 2.4 and later. But you can specify a different Spark storage level to use for caching. (See the definition of org.apache.spark.storage.StorageLevel for the details).
It can be one of the following: NONE, DISK_ONLY, DISK_ONLY_2, MEMORY_ONLY, MEMORY_ONLY_2, MEMORY_ONLY_SER, MEMORY_ONLY_SER_2, MEMORY_AND_DISK, MEMORY_AND_DISK_2, MEMORY_AND_DISK_SER, MEMORY_AND_DISK_SER_2, MEMORY_AND_DISK_SER_2, OFF_HEAP.
cacheStorageLevel
A caching storage level.

Definition Classes
Atum
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getControlMeasure: ControlMeasure

Returns the current control measures object containing all the checkpoints up to the current point.
Returns the current control measures object containing all the checkpoints up to the current point.

Definition Classes
Atum
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def preventDoubleInitialization(sparkSession: SparkSession): Unit

Attributes
protected
Definition Classes
Atum
var queryExecutionListener: QueryExecutionListener

Attributes
protected
Definition Classes
Atum
def setAdditionalInfo(kv: (String, String), replaceIfExists: Boolean = false): Unit

Sets an additional info in metadata of the _INFO file
Sets an additional info in metadata of the _INFO file

Definition Classes
Atum
def setAllowUnpersistOldDatasets(allowUnpersist: Boolean): Unit

Allows Atum to automatically unpersist all cached datasets it creates before calculating a checkpoint.
Allows Atum to automatically unpersist all cached datasets it creates before calculating a checkpoint. This may lead to performance improvement,
This feature is experimental and so it is disabled by default.

Definition Classes
Atum
def setCachingStorageLevel(cacheStorageLevelStr: String): Unit

Same as enableCaching() with cacheStorageLevel being a mandatory parameter.
Same as enableCaching() with cacheStorageLevel being a mandatory parameter. The storage level is passed as a String here for forward compatibility.

Definition Classes
Atum
def setCachingStorageLevel(cacheStorageLevel: StorageLevel): Unit

Same as enableCaching() with cacheStorageLevel being a mandatory parameter.
Same as enableCaching() with cacheStorageLevel being a mandatory parameter.

Definition Classes
Atum
def setRunUniqueId(uniqueId: String): Unit

Set's an unique id of a run stored in the _INFO file
Set's an unique id of a run stored in the _INFO file

Definition Classes
Atum
var sparkListener: SparkListener

Attributes
protected
Definition Classes
Atum
var state: ControlFrameworkState

Attributes
protected
Definition Classes
Atum
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Deprecated Value Members

def getControMeasure: ControlMeasure

Returns the current control measures object containing all the checkpoints up to the current point.
Returns the current control measures object containing all the checkpoints up to the current point.

Definition Classes
Atum
Annotations
@Deprecated
Deprecated
This method is deprecated. Please use getControlMeasure

Related Docs: trait Atum | package core

object Atum extends Atum

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def cachingStorageLevel: StorageLevel

def clone(): AnyRef

def disableCaching(): Unit

def enableCaching(cacheStorageLevel: StorageLevel = StorageLevel.MEMORY_AND_DISK): Unit

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def getControlMeasure: ControlMeasure

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def preventDoubleInitialization(sparkSession: SparkSession): Unit

var queryExecutionListener: QueryExecutionListener

def setAdditionalInfo(kv: (String, String), replaceIfExists: Boolean = false): Unit

def setAllowUnpersistOldDatasets(allowUnpersist: Boolean): Unit

def setCachingStorageLevel(cacheStorageLevelStr: String): Unit

def setCachingStorageLevel(cacheStorageLevel: StorageLevel): Unit

def setRunUniqueId(uniqueId: String): Unit

var sparkListener: SparkListener

var state: ControlFrameworkState

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Deprecated Value Members

def getControMeasure: ControlMeasure

Inherited from Atum

Inherited from AnyRef

Inherited from Any

Ungrouped