RDDOperations

Type Members

implicit class ImplicitArrayDecorator[A] extends AnyRef

Definition Classes
CollectionOperations
implicit class ImplicitBufferedIteratorDecorator[A] extends AnyRef

Adds operations to BufferedIterator
Adds operations to BufferedIterator

Definition Classes
CollectionOperations
implicit class ImplicitKVTraversableOnceDecorator[K, V] extends AnyRef

Adds operations to TraversableOnce[(K, V)]
Adds operations to TraversableOnce[(K, V)]

Definition Classes
CollectionOperations
implicit class ImplicitRDDDecorator[T] extends Serializable

Definition Classes
RDDOperations
implicit class ImplicitTraversableOnceDecorator[A] extends AnyRef

Adds operations to TraversableOnce
Adds operations to TraversableOnce

Definition Classes
CollectionOperations
implicit class PairRDDDecorator[K, A] extends AnyRef

Definition Classes
RDDOperations

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def distributeToPartitions[K, A, B, D](data: RDD[(K, A)], mapping: RDD[(K, D)], partitioner: Partitioner)(f: (A) ⇒ B)(implicit arg0: ClassTag[K], arg1: ClassTag[A], arg2: ClassTag[D]): RDD[(D, (K, B))]

Creates an RDD having each of required objects distributed to all required destinations
Creates an RDD having each of required objects distributed to all required destinations
If the data and/or mapping are partitioned with the same partitioner, no shuffle is required.
data
Source data to process
mapping
Mapping of keys to destinations. Duplicate destinations from different partitions are coalesced.
partitioner
Partitioner to use. Operation is more efficient if data is partitioned with the same partitioner
f
Function to apply to values. It's called exactly once for each value in data
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def groupWithinPartitions[K, V](rdd: RDD[(K, V)])(implicit arg0: ClassTag[K], arg1: ClassTag[V]): RDD[(K, Iterator[V])]

Utility used to group data by key within RDD partitions by a key, assuming that RDD is already partitioned and sorted by key.
Utility used to group data by key within RDD partitions by a key, assuming that RDD is already partitioned and sorted by key. The RDD data are processed sequentialy without shuffling and materializing them in memory.
K
Type of the key
V
Type of the value
rdd
RDD to group
returns
RDD with key -> values.
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def joinUnique[K, A, B](rdd: RDD[(K, A)], other: RDD[(K, B)], partitioner: Partitioner)(implicit arg0: ClassTag[K], arg1: ClassTag[A], arg2: ClassTag[B]): RDD[(K, (A, B))]
def joinUnique[K, A, B](rdd: RDD[(K, A)], other: RDD[(K, B)], numPartitions: Int)(implicit arg0: ClassTag[K], arg1: ClassTag[A], arg2: ClassTag[B]): RDD[(K, (A, B))]
def joinUnique[K, A, B](rdd: RDD[(K, A)], other: RDD[(K, B)])(implicit arg0: ClassTag[K], arg1: ClassTag[A], arg2: ClassTag[B]): RDD[(K, (A, B))]
def leftJoinUnique[K, A, B](rdd: RDD[(K, A)], other: RDD[(K, B)], partitioner: Partitioner)(implicit arg0: ClassTag[K], arg1: ClassTag[A], arg2: ClassTag[B]): RDD[(K, (A, Option[B]))]
def leftJoinUnique[K, A, B](rdd: RDD[(K, A)], other: RDD[(K, B)], numPartitions: Int)(implicit arg0: ClassTag[K], arg1: ClassTag[A], arg2: ClassTag[B]): RDD[(K, (A, Option[B]))]
def leftJoinUnique[K, A, B](rdd: RDD[(K, A)], other: RDD[(K, B)])(implicit arg0: ClassTag[K], arg1: ClassTag[A], arg2: ClassTag[B]): RDD[(K, (A, Option[B]))]
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Docs: trait RDDOperations | package util

object RDDOperations extends RDDOperations with CollectionOperations with Serializable

Type Members

implicit class ImplicitArrayDecorator[A] extends AnyRef

implicit class ImplicitBufferedIteratorDecorator[A] extends AnyRef

implicit class ImplicitKVTraversableOnceDecorator[K, V] extends AnyRef

implicit class ImplicitRDDDecorator[T] extends Serializable

implicit class ImplicitTraversableOnceDecorator[A] extends AnyRef

implicit class PairRDDDecorator[K, A] extends AnyRef

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def distributeToPartitions[K, A, B, D](data: RDD[(K, A)], mapping: RDD[(K, D)], partitioner: Partitioner)(f: (A) ⇒ B)(implicit arg0: ClassTag[K], arg1: ClassTag[A], arg2: ClassTag[D]): RDD[(D, (K, B))]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def groupWithinPartitions[K, V](rdd: RDD[(K, V)])(implicit arg0: ClassTag[K], arg1: ClassTag[V]): RDD[(K, Iterator[V])]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

def joinUnique[K, A, B](rdd: RDD[(K, A)], other: RDD[(K, B)], partitioner: Partitioner)(implicit arg0: ClassTag[K], arg1: ClassTag[A], arg2: ClassTag[B]): RDD[(K, (A, B))]

def joinUnique[K, A, B](rdd: RDD[(K, A)], other: RDD[(K, B)], numPartitions: Int)(implicit arg0: ClassTag[K], arg1: ClassTag[A], arg2: ClassTag[B]): RDD[(K, (A, B))]

def joinUnique[K, A, B](rdd: RDD[(K, A)], other: RDD[(K, B)])(implicit arg0: ClassTag[K], arg1: ClassTag[A], arg2: ClassTag[B]): RDD[(K, (A, B))]

def leftJoinUnique[K, A, B](rdd: RDD[(K, A)], other: RDD[(K, B)], partitioner: Partitioner)(implicit arg0: ClassTag[K], arg1: ClassTag[A], arg2: ClassTag[B]): RDD[(K, (A, Option[B]))]

def leftJoinUnique[K, A, B](rdd: RDD[(K, A)], other: RDD[(K, B)], numPartitions: Int)(implicit arg0: ClassTag[K], arg1: ClassTag[A], arg2: ClassTag[B]): RDD[(K, (A, Option[B]))]

def leftJoinUnique[K, A, B](rdd: RDD[(K, A)], other: RDD[(K, B)])(implicit arg0: ClassTag[K], arg1: ClassTag[A], arg2: ClassTag[B]): RDD[(K, (A, Option[B]))]

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Serializable

Inherited from Serializable

Inherited from CollectionOperations

Inherited from RDDOperations

Inherited from AnyRef

Inherited from Any

Ungrouped