ExternalCatalog

Instance Constructors

new ExternalCatalog()

Abstract Value Members

abstract def alterDatabase(dbDefinition: CatalogDatabase): Unit

Alter a database whose name matches the one specified in dbDefinition, assuming the database exists.
Alter a database whose name matches the one specified in dbDefinition, assuming the database exists.
Note: If the underlying implementation does not support altering a certain field, this becomes a no-op.
abstract def alterPartitions(db: String, table: String, parts: Seq[CatalogTablePartition]): Unit

Alter one or many table partitions whose specs that match those specified in parts, assuming the partitions exist.
Alter one or many table partitions whose specs that match those specified in parts, assuming the partitions exist.
Note: If the underlying implementation does not support altering a certain field, this becomes a no-op.
abstract def alterTable(tableDefinition: CatalogTable): Unit

Alter a table whose database and name match the ones specified in tableDefinition, assuming the table exists.
Alter a table whose database and name match the ones specified in tableDefinition, assuming the table exists. Note that, even though we can specify database in tableDefinition, it's used to identify the table, not to alter the table's database, which is not allowed.
Note: If the underlying implementation does not support altering a certain field, this becomes a no-op.
abstract def alterTableSchema(db: String, table: String, schema: StructType): Unit

Alter the schema of a table identified by the provided database and table name.
Alter the schema of a table identified by the provided database and table name. The new schema should still contain the existing bucket columns and partition columns used by the table. This method will also update any Spark SQL-related parameters stored as Hive table properties (such as the schema itself).
db
Database that table to alter schema for exists in
table
Name of table to alter schema for
schema
Updated schema to be used for the table (must contain existing partition and bucket columns)
abstract def createPartitions(db: String, table: String, parts: Seq[CatalogTablePartition], ignoreIfExists: Boolean): Unit
abstract def databaseExists(db: String): Boolean
abstract def doCreateDatabase(dbDefinition: CatalogDatabase, ignoreIfExists: Boolean): Unit

Attributes
protected
abstract def doCreateFunction(db: String, funcDefinition: CatalogFunction): Unit

Attributes
protected
abstract def doCreateTable(tableDefinition: CatalogTable, ignoreIfExists: Boolean): Unit

Attributes
protected
abstract def doDropDatabase(db: String, ignoreIfNotExists: Boolean, cascade: Boolean): Unit

Attributes
protected
abstract def doDropFunction(db: String, funcName: String): Unit

Attributes
protected
abstract def doDropTable(db: String, table: String, ignoreIfNotExists: Boolean, purge: Boolean): Unit

Attributes
protected
abstract def doRenameFunction(db: String, oldName: String, newName: String): Unit

Attributes
protected
abstract def doRenameTable(db: String, oldName: String, newName: String): Unit

Attributes
protected
abstract def dropPartitions(db: String, table: String, parts: Seq[TablePartitionSpec], ignoreIfNotExists: Boolean, purge: Boolean, retainData: Boolean): Unit
abstract def functionExists(db: String, funcName: String): Boolean
abstract def getDatabase(db: String): CatalogDatabase
abstract def getFunction(db: String, funcName: String): CatalogFunction
abstract def getPartition(db: String, table: String, spec: TablePartitionSpec): CatalogTablePartition
abstract def getPartitionOption(db: String, table: String, spec: TablePartitionSpec): Option[CatalogTablePartition]

Returns the specified partition or None if it does not exist.
abstract def getTable(db: String, table: String): CatalogTable
abstract def getTableOption(db: String, table: String): Option[CatalogTable]
abstract def listDatabases(pattern: String): Seq[String]
abstract def listDatabases(): Seq[String]
abstract def listFunctions(db: String, pattern: String): Seq[String]
abstract def listPartitionNames(db: String, table: String, partialSpec: Option[TablePartitionSpec] = None): Seq[String]

List the names of all partitions that belong to the specified table, assuming it exists.
List the names of all partitions that belong to the specified table, assuming it exists.
For a table with partition columns p1, p2, p3, each partition name is formatted as p1=v1/p2=v2/p3=v3. Each partition column name and value is an escaped path name, and can be decoded with the ExternalCatalogUtils.unescapePathName method.
The returned sequence is sorted as strings.
A partial partition spec may optionally be provided to filter the partitions returned, as described in the listPartitions method.
db
database name
table
table name
partialSpec
partition spec
abstract def listPartitions(db: String, table: String, partialSpec: Option[TablePartitionSpec] = None): Seq[CatalogTablePartition]

List the metadata of all partitions that belong to the specified table, assuming it exists.
List the metadata of all partitions that belong to the specified table, assuming it exists.
A partial partition spec may optionally be provided to filter the partitions returned. For instance, if there exist partitions (a='1', b='2'), (a='1', b='3') and (a='2', b='4'), then a partial spec of (a='1') will return the first two only.
db
database name
table
table name
partialSpec
partition spec
abstract def listPartitionsByFilter(db: String, table: String, predicates: Seq[Expression], defaultTimeZoneId: String): Seq[CatalogTablePartition]

List the metadata of partitions that belong to the specified table, assuming it exists, that satisfy the given partition-pruning predicate expressions.
List the metadata of partitions that belong to the specified table, assuming it exists, that satisfy the given partition-pruning predicate expressions.
db
database name
table
table name
predicates
partition-pruning predicates
defaultTimeZoneId
default timezone id to parse partition values of TimestampType
abstract def listTables(db: String, pattern: String): Seq[String]
abstract def listTables(db: String): Seq[String]
abstract def loadDynamicPartitions(db: String, table: String, loadPath: String, partition: TablePartitionSpec, replace: Boolean, numDP: Int): Unit
abstract def loadPartition(db: String, table: String, loadPath: String, partition: TablePartitionSpec, isOverwrite: Boolean, inheritTableSpecs: Boolean, isSrcLocal: Boolean): Unit

Loads data into a partition.
Loads data into a partition.
isSrcLocal
Whether the source data is local, as defined by the "LOAD DATA LOCAL" HiveQL command.
abstract def loadTable(db: String, table: String, loadPath: String, isOverwrite: Boolean, isSrcLocal: Boolean): Unit

Loads data into a table.
Loads data into a table.
isSrcLocal
Whether the source data is local, as defined by the "LOAD DATA LOCAL" HiveQL command.
abstract def renamePartitions(db: String, table: String, specs: Seq[TablePartitionSpec], newSpecs: Seq[TablePartitionSpec]): Unit

Override the specs of one or many existing table partitions, assuming they exist.
Override the specs of one or many existing table partitions, assuming they exist. This assumes index i of specs corresponds to index i of newSpecs.
abstract def setCurrentDatabase(db: String): Unit
abstract def tableExists(db: String, table: String): Boolean

Concrete Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def addListener(listener: ExternalCatalogEventListener): Unit

Definition Classes
ListenerBus
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def createDatabase(dbDefinition: CatalogDatabase, ignoreIfExists: Boolean): Unit
final def createFunction(db: String, funcDefinition: CatalogFunction): Unit
final def createTable(tableDefinition: CatalogTable, ignoreIfExists: Boolean): Unit
def doPostEvent(listener: ExternalCatalogEventListener, event: ExternalCatalogEvent): Unit

Attributes
protected
Definition Classes
ExternalCatalog → ListenerBus
final def dropDatabase(db: String, ignoreIfNotExists: Boolean, cascade: Boolean): Unit
final def dropFunction(db: String, funcName: String): Unit
final def dropTable(db: String, table: String, ignoreIfNotExists: Boolean, purge: Boolean): Unit
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes
protected
Definition Classes
Logging
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def postToAll(event: ExternalCatalogEvent): Unit

Definition Classes
ListenerBus
final def removeListener(listener: ExternalCatalogEventListener): Unit

Definition Classes
ListenerBus
final def renameFunction(db: String, oldName: String, newName: String): Unit
final def renameTable(db: String, oldName: String, newName: String): Unit
def requireDbExists(db: String): Unit

Attributes
protected
def requireFunctionExists(db: String, funcName: String): Unit

Attributes
protected
def requireFunctionNotExists(db: String, funcName: String): Unit

Attributes
protected
def requireTableExists(db: String, table: String): Unit

Attributes
protected
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package catalog

abstract class ExternalCatalog extends ListenerBus[ExternalCatalogEventListener, ExternalCatalogEvent]

Instance Constructors

new ExternalCatalog()

Abstract Value Members

abstract def alterDatabase(dbDefinition: CatalogDatabase): Unit

abstract def alterPartitions(db: String, table: String, parts: Seq[CatalogTablePartition]): Unit

abstract def alterTable(tableDefinition: CatalogTable): Unit

abstract def alterTableSchema(db: String, table: String, schema: StructType): Unit

abstract def createPartitions(db: String, table: String, parts: Seq[CatalogTablePartition], ignoreIfExists: Boolean): Unit

abstract def databaseExists(db: String): Boolean

abstract def doCreateDatabase(dbDefinition: CatalogDatabase, ignoreIfExists: Boolean): Unit

abstract def doCreateFunction(db: String, funcDefinition: CatalogFunction): Unit

abstract def doCreateTable(tableDefinition: CatalogTable, ignoreIfExists: Boolean): Unit

abstract def doDropDatabase(db: String, ignoreIfNotExists: Boolean, cascade: Boolean): Unit

abstract def doDropFunction(db: String, funcName: String): Unit

abstract def doDropTable(db: String, table: String, ignoreIfNotExists: Boolean, purge: Boolean): Unit

abstract def doRenameFunction(db: String, oldName: String, newName: String): Unit

abstract def doRenameTable(db: String, oldName: String, newName: String): Unit

abstract def dropPartitions(db: String, table: String, parts: Seq[TablePartitionSpec], ignoreIfNotExists: Boolean, purge: Boolean, retainData: Boolean): Unit

abstract def functionExists(db: String, funcName: String): Boolean

abstract def getDatabase(db: String): CatalogDatabase

abstract def getFunction(db: String, funcName: String): CatalogFunction

abstract def getPartition(db: String, table: String, spec: TablePartitionSpec): CatalogTablePartition

abstract def getPartitionOption(db: String, table: String, spec: TablePartitionSpec): Option[CatalogTablePartition]

abstract def getTable(db: String, table: String): CatalogTable

abstract def getTableOption(db: String, table: String): Option[CatalogTable]

abstract def listDatabases(pattern: String): Seq[String]

abstract def listDatabases(): Seq[String]

abstract def listFunctions(db: String, pattern: String): Seq[String]

abstract def listPartitionNames(db: String, table: String, partialSpec: Option[TablePartitionSpec] = None): Seq[String]

abstract def listPartitions(db: String, table: String, partialSpec: Option[TablePartitionSpec] = None): Seq[CatalogTablePartition]

abstract def listPartitionsByFilter(db: String, table: String, predicates: Seq[Expression], defaultTimeZoneId: String): Seq[CatalogTablePartition]

abstract def listTables(db: String, pattern: String): Seq[String]

abstract def listTables(db: String): Seq[String]

abstract def loadDynamicPartitions(db: String, table: String, loadPath: String, partition: TablePartitionSpec, replace: Boolean, numDP: Int): Unit

abstract def loadPartition(db: String, table: String, loadPath: String, partition: TablePartitionSpec, isOverwrite: Boolean, inheritTableSpecs: Boolean, isSrcLocal: Boolean): Unit

abstract def loadTable(db: String, table: String, loadPath: String, isOverwrite: Boolean, isSrcLocal: Boolean): Unit

abstract def renamePartitions(db: String, table: String, specs: Seq[TablePartitionSpec], newSpecs: Seq[TablePartitionSpec]): Unit

abstract def setCurrentDatabase(db: String): Unit

abstract def tableExists(db: String, table: String): Boolean

Concrete Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def addListener(listener: ExternalCatalogEventListener): Unit

final def asInstanceOf[T0]: T0

def clone(): AnyRef

final def createDatabase(dbDefinition: CatalogDatabase, ignoreIfExists: Boolean): Unit

final def createFunction(db: String, funcDefinition: CatalogFunction): Unit

final def createTable(tableDefinition: CatalogTable, ignoreIfExists: Boolean): Unit

def doPostEvent(listener: ExternalCatalogEventListener, event: ExternalCatalogEvent): Unit

final def dropDatabase(db: String, ignoreIfNotExists: Boolean, cascade: Boolean): Unit

final def dropFunction(db: String, funcName: String): Unit

final def dropTable(db: String, table: String, ignoreIfNotExists: Boolean, purge: Boolean): Unit

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

final def isInstanceOf[T0]: Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def postToAll(event: ExternalCatalogEvent): Unit