HiveSource

Instance Constructors

new HiveSource(dbName: String, tableName: String, projection: List[String] = Nil, predicate: Option[Predicate] = None, partitionConstraints: Seq[PartitionConstraint] = Nil, principal: Option[String] = None, keytabPath: Option[Path] = None)(implicit fs: FileSystem, client: IMetaStoreClient)

projection
sets which fields are required by the caller.
predicate
optional predicate which will filter rows at the read level
partitionConstraints
optional constraits on the partition data to narrow which partitions are read

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def addPartitionConstraint(constraint: PartitionConstraint): HiveSource
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
val dbName: String
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isPartitionOnlyProjection: Boolean

Returns true if the currently set projection is for partition only fields.
val keytabPath: Option[Path]
def load[T](implicit arg0: Manifest[T]): Seq[T]

Definition Classes
Source
val logger: Logger

Attributes
protected
Definition Classes
Logging
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val partitionConstraints: Seq[PartitionConstraint]

optional constraits on the partition data to narrow which partitions are read
def parts(): Seq[Publisher[Seq[Row]]]

Definition Classes
HiveSource → Source
val predicate: Option[Predicate]

optional predicate which will filter rows at the read level
val principal: Option[String]
val projection: List[String]

sets which fields are required by the caller.
def schema: StructType

The returned schema should take into account:
The returned schema should take into account:
1) Any projection. If a projection is set, then it should return the schema in the same order as the projection. If no projection is set then the schema should be driven from the hive metastore.
If the projection requests a field that does not exist, then this method will throw an exception.
2) Any partitions set. These should be included in the schema columns.

Definition Classes
HiveSource → Source
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def table: HiveTable
val tableName: String
def toDataStream(listener: Listener): DataStream

Definition Classes
Source
def toDataStream(): DataStream

Definition Classes
Source
def using[T, U <: AnyRef { def close(): Unit }](closeable: U)(f: (U) ⇒ T): T

Definition Classes
Using
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def withKeytabFile(principal: String, keytabPath: Path): HiveSource
def withPartitionConstraints(constraints: Seq[PartitionConstraint]): HiveSource
def withPredicate(predicate: Predicate): HiveSource
def withProjection(columns: Seq[String]): HiveSource
def withProjection(first: String, rest: String*): HiveSource

Related Doc: package hive

Instance Constructors

new HiveSource(dbName: String, tableName: String, projection: List[String] = Nil, predicate: Option[Predicate] = None, partitionConstraints: Seq[PartitionConstraint] = Nil, principal: Option[String] = None, keytabPath: Option[Path] = None)(implicit fs: FileSystem, client: IMetaStoreClient)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def addPartitionConstraint(constraint: PartitionConstraint): HiveSource

final def asInstanceOf[T0]: T0

def clone(): AnyRef

val dbName: String

final def eq(arg0: AnyRef): Boolean

def finalize(): Unit

final def getClass(): Class[_]

final def isInstanceOf[T0]: Boolean

def isPartitionOnlyProjection: Boolean

val keytabPath: Option[Path]

def load[T](implicit arg0: Manifest[T]): Seq[T]

val logger: Logger

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val partitionConstraints: Seq[PartitionConstraint]

def parts(): Seq[Publisher[Seq[Row]]]

val predicate: Option[Predicate]

val principal: Option[String]

val projection: List[String]

def schema: StructType

final def synchronized[T0](arg0: ⇒ T0): T0

def table: HiveTable

val tableName: String

def toDataStream(listener: Listener): DataStream

def toDataStream(): DataStream

def using[T, U <: AnyRef { def close(): Unit }](closeable: U)(f: (U) ⇒ T): T

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

def withKeytabFile(principal: String, keytabPath: Path): HiveSource

def withPartitionConstraints(constraints: Seq[PartitionConstraint]): HiveSource

def withPredicate(predicate: Predicate): HiveSource

def withProjection(columns: Seq[String]): HiveSource

def withProjection(first: String, rest: String*): HiveSource

Inherited from Serializable

Inherited from Serializable

Inherited from Product

Inherited from scala.Equals

Inherited from Using

Inherited from Source

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Ungrouped