PucketCompanion

Type Members

abstract type DescriptorType[T <: HigherType] <: PucketDescriptor[T]
abstract type HigherType
abstract type V

Abstract Value Members

abstract def apply[T <: HigherType](path: Path, fs: FileSystem, other: V, blockSize: Int): \/[Throwable, Pucket[T]]

Find an existing pucket on the filesystem
Find an existing pucket on the filesystem
T
the expected type of the pucket data
path
the path to the pucket
fs
hadoop filesystem instance
other
the implementing object's parameter for verifying the found pucket matches expect format
blockSize
the size of a parquet block before memory is flushed
returns
an error if any of the validation fails or the pucket
abstract def create[T <: HigherType](path: Path, fs: FileSystem, descriptor: DescriptorType[T], blockSize: Int = defaultBlockSize): \/[Throwable, Pucket[T]]

Create a new pucket
Create a new pucket
T
the type of the pucket data
path
the path to the pucket
fs
hadoop filesystem instance
descriptor
a descriptor to create a new pucket with
blockSize
the size of a parquet block before memory is flushed
returns
an error if any of the validation fails or the new pucket
abstract def getDescriptor[T <: HigherType](schemaSpec: V, compression: CompressionCodecName, partitioner: Option[PucketPartitioner[T]]): DescriptorType[T]

Returns a descriptor of the correct type
abstract def getDescriptorSchemaSpec[T <: HigherType](descriptor: DescriptorType[T]): V

Concrete Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def create[T <: HigherType](path: Path, fs: FileSystem, schemaSpec: V, compression: CompressionCodecName, partitioner: Option[PucketPartitioner[T]]): \/[Throwable, Pucket[T]]

Create a new pucket
Create a new pucket
T
the expected type of the pucket data
path
the path to the pucket
fs
hadoop filesystem instance
schemaSpec
schema for the data type
compression
parquet compression codec to use
partitioner
optional partitioning scheme
returns
an error if any of the validation fails or the new pucket
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def findOrCreate[T <: HigherType](path: Path, fs: FileSystem, schemaSpec: V, compression: CompressionCodecName, partitioner: Option[PucketPartitioner[T]]): \/[Throwable, Pucket[T]]
def findOrCreate[T <: HigherType](path: Path, fs: FileSystem, descriptor: DescriptorType[T], blockSize: Int = defaultBlockSize): \/[Throwable, Pucket[T]]
def findOrCreateRetry[T <: HigherType](path: Path, fs: FileSystem, schemaSpec: V, compression: CompressionCodecName, partitioner: Option[PucketPartitioner[T]], attempts: Int, retryIntervalMs: Int): \/[Throwable, Pucket[T]]

Find an existing pucket or create one if it does not exist
Find an existing pucket or create one if it does not exist
T
the expected type of the pucket data
path
the path to the pucket
fs
hadoop filesystem instance
schemaSpec
schema for the data type
compression
parquet compression codec to use
partitioner
optional partitioning scheme
returns
an error if any of the validation fails or the pucket
def findOrCreateRetry[T <: HigherType](path: Path, fs: FileSystem, descriptor: DescriptorType[T], blockSize: Int = defaultBlockSize, attempts: Int = Pucket.defaultCreationAttempts, retryIntervalMs: Int = Pucket.defaultRetryIntervalMs): \/[Throwable, Pucket[T]]

Find an existing pucket or create one if it does not exist
Find an existing pucket or create one if it does not exist
T
the expected type of the pucket data
path
the path to the pucket
fs
hadoop filesystem instance
descriptor
a descriptor to create a new pucket with or validate an existing pucket against
blockSize
the size of a parquet block before memory is flushed
returns
an error if any of the validation fails or the pucket
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package pucket

trait PucketCompanion extends AnyRef

Type Members

abstract type DescriptorType[T <: HigherType] <: PucketDescriptor[T]

abstract type HigherType

abstract type V

Abstract Value Members

abstract def apply[T <: HigherType](path: Path, fs: FileSystem, other: V, blockSize: Int): \/[Throwable, Pucket[T]]

abstract def create[T <: HigherType](path: Path, fs: FileSystem, descriptor: DescriptorType[T], blockSize: Int = defaultBlockSize): \/[Throwable, Pucket[T]]

abstract def getDescriptor[T <: HigherType](schemaSpec: V, compression: CompressionCodecName, partitioner: Option[PucketPartitioner[T]]): DescriptorType[T]

abstract def getDescriptorSchemaSpec[T <: HigherType](descriptor: DescriptorType[T]): V

Concrete Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def create[T <: HigherType](path: Path, fs: FileSystem, schemaSpec: V, compression: CompressionCodecName, partitioner: Option[PucketPartitioner[T]]): \/[Throwable, Pucket[T]]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

def findOrCreate[T <: HigherType](path: Path, fs: FileSystem, schemaSpec: V, compression: CompressionCodecName, partitioner: Option[PucketPartitioner[T]]): \/[Throwable, Pucket[T]]

def findOrCreate[T <: HigherType](path: Path, fs: FileSystem, descriptor: DescriptorType[T], blockSize: Int = defaultBlockSize): \/[Throwable, Pucket[T]]

def findOrCreateRetry[T <: HigherType](path: Path, fs: FileSystem, schemaSpec: V, compression: CompressionCodecName, partitioner: Option[PucketPartitioner[T]], attempts: Int, retryIntervalMs: Int): \/[Throwable, Pucket[T]]

def findOrCreateRetry[T <: HigherType](path: Path, fs: FileSystem, descriptor: DescriptorType[T], blockSize: Int = defaultBlockSize, attempts: Int = Pucket.defaultCreationAttempts, retryIntervalMs: Int = Pucket.defaultRetryIntervalMs): \/[Throwable, Pucket[T]]

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from AnyRef

Inherited from Any

Ungrouped