Utils

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def ERROR_NO_QCS(module: String): String
final val SKIP_ANALYSIS_PREFIX: String("SAMPLE_")
final val WEIGHTAGE_COLUMN_NAME: String("SNAPPY_SAMPLER_WEIGHTAGE")
final val Z95Percent: Double
final val Z95Squared: Double
def analysisException(msg: String, cause: Option[Throwable] = None): AnalysisException
final def asInstanceOf[T0]: T0

Definition Classes
Any
def charMetadata(size: Int, md: Metadata): Metadata

Utility function to return a metadata for a StructField of StringType, to ensure that the field is stored (and rendered) as CHAR by SnappyStore.
Utility function to return a metadata for a StructField of StringType, to ensure that the field is stored (and rendered) as CHAR by SnappyStore.
size
the size parameter of the CHAR() column type
md
optional Metadata object to be merged into the result
returns
the result Metadata object to use for StructField
def charMetadata(size: Int): Metadata

Utility function to return a metadata for a StructField of StringType, to ensure that the field is stored (and rendered) as CHAR by SnappyStore.
Utility function to return a metadata for a StructField of StringType, to ensure that the field is stored (and rendered) as CHAR by SnappyStore.
size
the size parameter of the CHAR() column type
returns
the result Metadata object to use for StructField
def charMetadata(): Metadata

Utility function to return a metadata for a StructField of StringType, to ensure that the field is stored (and rendered) as CHAR by SnappyStore.
Utility function to return a metadata for a StructField of StringType, to ensure that the field is stored (and rendered) as CHAR by SnappyStore.
returns
the result Metadata object to use for StructField
def classForName(className: String): Class[_]
def clearDefaultSerializerAndCodec(): Unit
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def closeJsonGenerator(gen: AnyRef): Unit
def columnIndex(col: String, cols: Array[String], module: String): Int
def createCatalystConverter(dataType: DataType): (Any) ⇒ Any
def createScalaConverter(dataType: DataType): (Any) ⇒ Any
def createStatsBuffer(statsData: Array[Byte], allocator: BufferAllocator): ByteBuffer
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def executorsListener(sc: SparkContext): Option[ExecutorsListener]
def fieldIndex(relationOutput: Seq[Attribute], columnName: String, caseSensitive: Boolean): Int
def fieldName(f: StructField): String
def fillArray[T](a: Array[_ >: T], v: T, start: Int, endP1: Int): Unit
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def genTaskContextFunction(ctx: CodegenContext): String
def generateJson(gen: AnyRef, row: InternalRow, columnIndex: Int, columnType: DataType): Unit
def getActiveSession: Option[SparkSession]
def getAllExecutorsMemoryStatus(sc: SparkContext): Map[BlockManagerId, (Long, Long)]
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getClientHostPort(netServer: String): String
def getDriverClassName(url: String): String
def getFields(o: Any): Map[String, Any]
def getFixedPartitionRDD[T](sc: SparkContext, f: (TaskContext, Partition) ⇒ Iterator[T], partitioner: Partitioner, numPartitions: Int)(implicit arg0: ClassTag[T]): RDD[T]
def getGenericRowValues(row: GenericRow): Array[Any]
def getHostExecutorId(blockId: BlockManagerId): String
def getInternalSparkConf(sc: SparkContext): SparkConf
def getInternalType(dataType: DataType): Class[_]
def getJsonGenerator(dataType: DataType, columnName: String, writer: Writer): AnyRef
def getNumColumns(partitioning: Partitioning): Int
def getPartitionData(blockId: BlockId, bm: BlockManager): ByteBuffer
def getSQLDataType(dataType: DataType): DataType
def getSchemaAndPlanFromBase(schemaOpt: Option[StructType], baseTableOpt: Option[String], catalog: SnappyStoreHiveCatalog, asSelect: Boolean, table: String, tableType: String): (StructType, Option[LogicalPlan])

Get the result schema given an optional explicit schema and base table.
Get the result schema given an optional explicit schema and base table. In case both are specified, then check compatibility between the two.
def hasLowerCase(k: String): Boolean
def hashCode(): Int

Definition Classes
AnyRef → Any
def immutableMap[A, B](m: Map[A, B]): Map[A, B]
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def isLoner(sc: SparkContext): Boolean
def mapExecutors[T](sc: SparkContext, f: (TaskContext, ExecutorLocalPartition) ⇒ Iterator[T], maxTries: Int)(implicit arg0: ClassTag[T]): Array[T]
def mapExecutors[T](sc: SparkContext, f: () ⇒ Iterator[T], maxTries: Int = 30, blockManagerIds: Seq[BlockManagerId] = Seq.empty)(implicit arg0: ClassTag[T]): Array[T]
def matchOption(optName: String, options: Map[String, Any]): Option[(String, Any)]
def metricMethods: ((String) ⇒ String, (String) ⇒ String)
def millisToDays(millisUtc: Long, tz: TimeZone): Int
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def newChunkedByteBuffer(chunks: Array[ByteBuffer]): ChunkedByteBuffer
def newClusterSparkConf(props: Map[AnyRef, AnyRef]): SparkConf
def newClusterSparkConf(): SparkConf
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def parseColumn(cv: Any, cols: Array[String], module: String, option: String): Int
def parseColumnsAsClob(s: String): (Boolean, Set[String])
def parseDouble(v: Any, module: String, option: String, min: Double, max: Double, exclusive: Boolean = true): Double
def parseInteger(v: Any, module: String, option: String, min: Int = 1, max: Int = Int.MaxValue): Int
def parseTimeInterval(optV: Any, module: String): Long

Parse the given time interval value as long milliseconds.
Parse the given time interval value as long milliseconds.

See also
timeIntervalSpec for the allowed string specification
def parseTimestamp(ts: String, module: String, col: String): Long
def qcsOf(qa: Array[String], cols: Array[String], module: String): (Array[Int], Array[String])
def registerDriver(driver: String): Unit

Register given driver class with Spark's loader.
def registerDriverUrl(url: String): String

Register driver for given JDBC URL and return the driver class name.
def resolveQCS(options: Map[String, Any], fieldNames: Array[String], module: String): (Array[Int], Array[String])
def resolveQCS(qcsV: Option[Any], fieldNames: Array[String], module: String): (Array[Int], Array[String])
def schemaFields(schema: StructType): Map[String, StructField]
def setDefaultConfProperty(conf: SparkConf, name: String, default: String): Unit
def setDefaultSerializerAndCodec(conf: SparkConf): Unit
def stringMetadata(md: Metadata = Metadata.empty): Metadata

Utility function to return a metadata for a StructField of StringType, to ensure that the field is rendered as CLOB by SnappyStore.
Utility function to return a metadata for a StructField of StringType, to ensure that the field is rendered as CLOB by SnappyStore.
md
optional Metadata object to be merged into the result
returns
the result Metadata object to use for StructField
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def taskMemoryManager(context: TaskContext): TaskMemoryManager
def toLowerCase(k: String): String
def toOpenHashMap[K, V](map: Map[K, V]): ObjectObjectHashMap[K, V]
def toString(): String

Definition Classes
AnyRef → Any
def toUnsafeRow(buffer: ByteBuffer, numColumns: Int): UnsafeRow
def toUpperCase(k: String): String
def unAlias(e: Expression, childClass: Class[_] = null): Expression

Annotations
@tailrec()
lazy val usingEnhancedSpark: Boolean
def varcharMetadata(size: Int, md: Metadata): Metadata

Utility function to return a metadata for a StructField of StringType, to ensure that the field is stored (and rendered) as VARCHAR by SnappyStore.
Utility function to return a metadata for a StructField of StringType, to ensure that the field is stored (and rendered) as VARCHAR by SnappyStore.
size
the size parameter of the VARCHAR() column type
md
optional Metadata object to be merged into the result
returns
the result Metadata object to use for StructField
def varcharMetadata(size: Int): Metadata

Utility function to return a metadata for a StructField of StringType, to ensure that the field is stored (and rendered) as VARCHAR by SnappyStore.
Utility function to return a metadata for a StructField of StringType, to ensure that the field is stored (and rendered) as VARCHAR by SnappyStore.
size
the size parameter of the VARCHAR() column type
returns
the result Metadata object to use for StructField
def varcharMetadata(): Metadata

Utility function to return a metadata for a StructField of StringType, to ensure that the field is stored (and rendered) as VARCHAR by SnappyStore.
Utility function to return a metadata for a StructField of StringType, to ensure that the field is stored (and rendered) as VARCHAR by SnappyStore.
returns
the result Metadata object to use for StructField
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def withNewExecutionId[T](df: DataFrame, body: ⇒ T): T

Wrap a DataFrame action to track all Spark jobs in the body so that we can connect them with an execution.

Related Doc: package collection

object Utils

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def ERROR_NO_QCS(module: String): String

final val SKIP_ANALYSIS_PREFIX: String("SAMPLE_")

final val WEIGHTAGE_COLUMN_NAME: String("SNAPPY_SAMPLER_WEIGHTAGE")

final val Z95Percent: Double

final val Z95Squared: Double

def analysisException(msg: String, cause: Option[Throwable] = None): AnalysisException

final def asInstanceOf[T0]: T0

def charMetadata(size: Int, md: Metadata): Metadata

def charMetadata(size: Int): Metadata

def charMetadata(): Metadata

def classForName(className: String): Class[_]

def clearDefaultSerializerAndCodec(): Unit

def clone(): AnyRef

def closeJsonGenerator(gen: AnyRef): Unit

def columnIndex(col: String, cols: Array[String], module: String): Int

def createCatalystConverter(dataType: DataType): (Any) ⇒ Any

def createScalaConverter(dataType: DataType): (Any) ⇒ Any

def createStatsBuffer(statsData: Array[Byte], allocator: BufferAllocator): ByteBuffer

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def executorsListener(sc: SparkContext): Option[ExecutorsListener]

def fieldIndex(relationOutput: Seq[Attribute], columnName: String, caseSensitive: Boolean): Int

def fieldName(f: StructField): String

def fillArray[T](a: Array[_ >: T], v: T, start: Int, endP1: Int): Unit

def finalize(): Unit

def genTaskContextFunction(ctx: CodegenContext): String

def generateJson(gen: AnyRef, row: InternalRow, columnIndex: Int, columnType: DataType): Unit

def getActiveSession: Option[SparkSession]

def getAllExecutorsMemoryStatus(sc: SparkContext): Map[BlockManagerId, (Long, Long)]

final def getClass(): Class[_]

def getClientHostPort(netServer: String): String

def getDriverClassName(url: String): String

def getFields(o: Any): Map[String, Any]

def getFixedPartitionRDD[T](sc: SparkContext, f: (TaskContext, Partition) ⇒ Iterator[T], partitioner: Partitioner, numPartitions: Int)(implicit arg0: ClassTag[T]): RDD[T]

def getGenericRowValues(row: GenericRow): Array[Any]

def getHostExecutorId(blockId: BlockManagerId): String

def getInternalSparkConf(sc: SparkContext): SparkConf

def getInternalType(dataType: DataType): Class[_]

def getJsonGenerator(dataType: DataType, columnName: String, writer: Writer): AnyRef

def getNumColumns(partitioning: Partitioning): Int

def getPartitionData(blockId: BlockId, bm: BlockManager): ByteBuffer

def getSQLDataType(dataType: DataType): DataType

def getSchemaAndPlanFromBase(schemaOpt: Option[StructType], baseTableOpt: Option[String], catalog: SnappyStoreHiveCatalog, asSelect: Boolean, table: String, tableType: String): (StructType, Option[LogicalPlan])

def hasLowerCase(k: String): Boolean

def hashCode(): Int

def immutableMap[A, B](m: Map[A, B]): Map[A, B]

final def isInstanceOf[T0]: Boolean

final def isLoner(sc: SparkContext): Boolean

def mapExecutors[T](sc: SparkContext, f: (TaskContext, ExecutorLocalPartition) ⇒ Iterator[T], maxTries: Int)(implicit arg0: ClassTag[T]): Array[T]

def mapExecutors[T](sc: SparkContext, f: () ⇒ Iterator[T], maxTries: Int = 30, blockManagerIds: Seq[BlockManagerId] = Seq.empty)(implicit arg0: ClassTag[T]): Array[T]

def matchOption(optName: String, options: Map[String, Any]): Option[(String, Any)]

def metricMethods: ((String) ⇒ String, (String) ⇒ String)

def millisToDays(millisUtc: Long, tz: TimeZone): Int

final def ne(arg0: AnyRef): Boolean

def newChunkedByteBuffer(chunks: Array[ByteBuffer]): ChunkedByteBuffer

def newClusterSparkConf(props: Map[AnyRef, AnyRef]): SparkConf

def newClusterSparkConf(): SparkConf

final def notify(): Unit

final def notifyAll(): Unit

def parseColumn(cv: Any, cols: Array[String], module: String, option: String): Int

def parseColumnsAsClob(s: String): (Boolean, Set[String])

def parseDouble(v: Any, module: String, option: String, min: Double, max: Double, exclusive: Boolean = true): Double

def parseInteger(v: Any, module: String, option: String, min: Int = 1, max: Int = Int.MaxValue): Int

def parseTimeInterval(optV: Any, module: String): Long

def parseTimestamp(ts: String, module: String, col: String): Long

def qcsOf(qa: Array[String], cols: Array[String], module: String): (Array[Int], Array[String])

def registerDriver(driver: String): Unit

def registerDriverUrl(url: String): String

def resolveQCS(options: Map[String, Any], fieldNames: Array[String], module: String): (Array[Int], Array[String])

def resolveQCS(qcsV: Option[Any], fieldNames: Array[String], module: String): (Array[Int], Array[String])

def schemaFields(schema: StructType): Map[String, StructField]

def setDefaultConfProperty(conf: SparkConf, name: String, default: String): Unit

def setDefaultSerializerAndCodec(conf: SparkConf): Unit