ParquetFileFormat

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
val apacheParquetLogger: Logger
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes
protected
Definition Classes
Logging
final def isDebugEnabled: Boolean

Attributes
protected
Definition Classes
Logging
final def isInfoEnabled: Boolean

Attributes
protected
Definition Classes
Logging
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def isTraceEnabled: Boolean

Attributes
protected
Definition Classes
Logging
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def mergeMetastoreParquetSchema(metastoreSchema: StructType, parquetSchema: StructType): StructType

Reconciles Hive Metastore case insensitivity issue and data type conflicts between Metastore schema and Parquet schema.
Reconciles Hive Metastore case insensitivity issue and data type conflicts between Metastore schema and Parquet schema.
Hive doesn't retain case information, while Parquet is case sensitive. On the other hand, the schema read from Parquet files may be incomplete (e.g. older versions of Parquet doesn't distinguish binary and string). This method generates a correct schema by merging Metastore schema data types and Parquet schema field names.
def mergeSchemasInParallel(filesToTouch: Seq[FileStatus], sparkSession: SparkSession): Option[StructType]

Figures out a merged Parquet schema with a distributed Spark job.
Figures out a merged Parquet schema with a distributed Spark job.
Note that locality is not taken into consideration here because:
1. For a single Parquet part-file, in most cases the footer only resides in the last block of that file. Thus we only need to retrieve the location of the last block. However, Hadoop FileSystem only provides API to retrieve locations of all blocks, which can be potentially expensive.
2. This optimization is mainly useful for S3, where file metadata operations can be pretty slow. And basically locality is not available when using S3 (you can't run computation on S3 nodes).
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val parquetLogger: Logger
def readSchemaFromFooter(footer: Footer, converter: ParquetSchemaConverter): StructType

Reads Spark SQL schema from a Parquet footer.
Reads Spark SQL schema from a Parquet footer. If a valid serialized Spark SQL schema string can be found in the file metadata, returns the deserialized StructType, otherwise, returns a StructType converted from the MessageType stored in this footer.
val redirectParquetLogsViaSLF4J: Unit
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Docs: class ParquetFileFormat | package parquet

object ParquetFileFormat extends Logging with Serializable

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

val apacheParquetLogger: Logger

final def asInstanceOf[T0]: T0

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

final def isDebugEnabled: Boolean

final def isInfoEnabled: Boolean

final def isInstanceOf[T0]: Boolean

final def isTraceEnabled: Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

def mergeMetastoreParquetSchema(metastoreSchema: StructType, parquetSchema: StructType): StructType

def mergeSchemasInParallel(filesToTouch: Seq[FileStatus], sparkSession: SparkSession): Option[StructType]

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val parquetLogger: Logger

def readSchemaFromFooter(footer: Footer, converter: ParquetSchemaConverter): StructType

val redirectParquetLogsViaSLF4J: Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Serializable

Inherited from Serializable

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Ungrouped