Analyzer

Instance Constructors

new Analyzer(tableUtils: TableUtils, conf: Any, startDate: String, endDate: String, count: Int = 64, sample: Double = 0.1, enableHitter: Boolean = false, silenceMode: Boolean = false, skipTimestampCheck: Boolean = false)

Type Members

case class AggregationMetadata(name: String, columnType: DataType, operation: String = null, window: String = null, inputColumn: String = null, groupByName: String = null) extends Product with Serializable

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def analyze(df: DataFrame, keys: Array[String], sourceTable: String): String
def analyzeGroupBy(groupByConf: api.GroupBy, prefix: String = "", includeOutputTableName: Boolean = false, enableHitter: Boolean = false, skipTimestampCheck: Boolean = skipTimestampCheck, validateTablePermission: Boolean = false): (Array[AggregationMetadata], Map[String, DataType], Set[String])
def analyzeJoin(joinConf: api.Join, enableHitter: Boolean = false, validateTablePermission: Boolean = true, validationAssert: Boolean = false, skipTimestampCheck: Boolean = skipTimestampCheck): (Map[String, DataType], ListBuffer[AggregationMetadata])
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def dataFrameToMap(inputDf: DataFrame): Map[String, String]
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
def heavyHitters(df: DataFrame, frequentItemKeys: Array[String], frequentItemMapSize: Int = 1024, sampleFraction: Double = 0.1): Array[(String, Array[(String, Long)])]
def heavyHittersWithTsAndCount(df: DataFrame, keys: Array[String], frequentItemMapSize: Int = 1024, sampleFraction: Double = 0.1): Array[(String, Array[(String, Long)])]
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
lazy val logger: Logger
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def run(): Unit
def runSchemaValidation(left: Map[String, DataType], right: Map[String, DataType], keyMapping: Map[String, String]): Map[String, String]
def runTablePermissionValidation(sources: Set[String], partitionColOpt: Option[String] = None): Set[String]
def runTimestampChecks(df: DataFrame, sampleNumber: Int = 100): Map[String, String]
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
def validateTimestampChecks(timestampCheckMap: Map[String, String], configType: String, configName: String): Unit

This method can be used to trigger the assertion checks or print the summary stats once the timestamp checks have been run
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package spark

class Analyzer extends AnyRef

Instance Constructors

new Analyzer(tableUtils: TableUtils, conf: Any, startDate: String, endDate: String, count: Int = 64, sample: Double = 0.1, enableHitter: Boolean = false, silenceMode: Boolean = false, skipTimestampCheck: Boolean = false)

Type Members

case class AggregationMetadata(name: String, columnType: DataType, operation: String = null, window: String = null, inputColumn: String = null, groupByName: String = null) extends Product with Serializable

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def analyze(df: DataFrame, keys: Array[String], sourceTable: String): String

def analyzeGroupBy(groupByConf: api.GroupBy, prefix: String = "", includeOutputTableName: Boolean = false, enableHitter: Boolean = false, skipTimestampCheck: Boolean = skipTimestampCheck, validateTablePermission: Boolean = false): (Array[AggregationMetadata], Map[String, DataType], Set[String])

def analyzeJoin(joinConf: api.Join, enableHitter: Boolean = false, validateTablePermission: Boolean = true, validationAssert: Boolean = false, skipTimestampCheck: Boolean = skipTimestampCheck): (Map[String, DataType], ListBuffer[AggregationMetadata])

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def dataFrameToMap(inputDf: DataFrame): Map[String, String]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

def heavyHitters(df: DataFrame, frequentItemKeys: Array[String], frequentItemMapSize: Int = 1024, sampleFraction: Double = 0.1): Array[(String, Array[(String, Long)])]

def heavyHittersWithTsAndCount(df: DataFrame, keys: Array[String], frequentItemMapSize: Int = 1024, sampleFraction: Double = 0.1): Array[(String, Array[(String, Long)])]

final def isInstanceOf[T0]: Boolean

lazy val logger: Logger

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def run(): Unit

def runSchemaValidation(left: Map[String, DataType], right: Map[String, DataType], keyMapping: Map[String, String]): Map[String, String]

def runTablePermissionValidation(sources: Set[String], partitionColOpt: Option[String] = None): Set[String]

def runTimestampChecks(df: DataFrame, sampleNumber: Int = 100): Map[String, String]

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

def validateTimestampChecks(timestampCheckMap: Map[String, String], configType: String, configName: String): Unit

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from AnyRef

Inherited from Any

Ungrouped