SparkJob

Instance Constructors

new SparkJob(args: Args)

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def anyToFieldArg(f: Any): Comparable[_]

Attributes
protected
Definition Classes
LowPriorityFieldConversions
val args: Args

Definition Classes
Job
final def asInstanceOf[T0]: T0

Definition Classes
Any
def asList(f: Fields): List[Comparable[_]]

Definition Classes
FieldConversions
def asSet(f: Fields): Set[Comparable[_]]

Definition Classes
FieldConversions
def buildFlow: Flow[_]

Definition Classes
Job
def classIdentifier: String

Definition Classes
Job
def clear: Unit

Definition Classes
Job
def clone(nextargs: Args): Job

Definition Classes
Job
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def config: Map[AnyRef, AnyRef]

Overrides the Scalding Job config method in order not to use the default serializations, but to pass different ones instead.
Overrides the Scalding Job config method in order not to use the default serializations, but to pass different ones instead.
returns

Definition Classes
SparkJob → Job
implicit def dateParser: DateParser

Definition Classes
Job
def defaultComparator: Option[Class[_ <: Comparator[_]]]

Definition Classes
Job
def defaultMode(fromFields: Fields, toFields: Fields): Fields

Definition Classes
FieldConversions
def defaultSpillThreshold: Int

Definition Classes
Job
final def ensureUniqueFields(left: Fields, right: Fields, rightPipe: Pipe): (Fields, Pipe)

Definition Classes
FieldConversions
implicit def enumValueToFields(x: Value): Fields

Definition Classes
FieldConversions
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
implicit def fieldFields[T <: TraversableOnce[Field[_]]](f: T): RichFields

Definition Classes
FieldConversions
implicit def fieldToFields(f: Field[_]): RichFields

Definition Classes
FieldConversions
implicit def fields[T <: TraversableOnce[Symbol]](f: T): Fields

Definition Classes
FieldConversions
implicit def fieldsToRichFields(fields: Fields): RichFields

Definition Classes
FieldConversions
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
implicit val flowDef: FlowDef

Attributes
protected
Definition Classes
Job
implicit def fromEnum[T <: Enumeration](enumeration: T): Fields

Definition Classes
FieldConversions
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getField(f: Fields, idx: Int): Fields

Definition Classes
FieldConversions
implicit def hadoopConf: Configuration
def handleStats(statsData: CascadingStats): Unit

Attributes
protected
Definition Classes
Job
def hasInts(f: Fields): Boolean

Definition Classes
FieldConversions
def hashCode(): Int

Definition Classes
AnyRef → Any
implicit def intFields[T <: TraversableOnce[Int]](f: T): Fields

Definition Classes
FieldConversions
implicit def intToFields(x: Int): Fields

Definition Classes
FieldConversions
implicit def integerToFields(x: Integer): Fields

Definition Classes
FieldConversions
def ioSerializations: List[Class[_ <: Serialization[_]]]

Definition Classes
Job
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
implicit def iterableToRichPipe[T](iter: Iterable[T])(implicit set: TupleSetter[T], conv: TupleConverter[T]): RichPipe

Definition Classes
Job
def keepAlive: Unit

Definition Classes
Job
def listeners: List[FlowListener]

Definition Classes
Job
implicit def mode: Mode

Definition Classes
Job
def name: String

Definition Classes
Job
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def newSymbol(avoid: Set[Symbol], guess: Symbol, trial: Int): Symbol

Definition Classes
FieldConversions
Annotations
@tailrec()
def next: Option[Job]

Definition Classes
Job
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
implicit def parseAnySeqToFields[T <: TraversableOnce[Any]](anyf: T): Fields

Definition Classes
FieldConversions
implicit def pipeToRichPipe(pipe: Pipe): RichPipe

Definition Classes
Job
implicit def productToFields(f: Product): Fields

Definition Classes
LowPriorityFieldConversions
implicit def read(src: Source): Pipe

Definition Classes
Job
def run: Boolean

Definition Classes
Job
implicit lazy val sc: SparkContext
implicit def scaldingConfig: Config

Attributes
protected
Definition Classes
Job
def setSerialization(userHadoop: Seq[Class[_ <: Serialization[_]]] = Nil): Config

Hadoop and Cascading serialization needs to be first, and the Kryo serialization needs to be last and this method handles this for you: hadoop, cascading, [userHadoop,] kyro is the order.
Hadoop and Cascading serialization needs to be first, and the Kryo serialization needs to be last and this method handles this for you: hadoop, cascading, [userHadoop,] kyro is the order.
Kryo uses a specific class here.
userHadoop
returns

Attributes
protected
def skipStrategy: Option[FlowSkipStrategy]

Definition Classes
Job
implicit def sourceToRichPipe(src: Source): RichPipe

Definition Classes
Job
implicit def sourceToSparkRichSource(source: Source): RichSource
def stepListeners: List[FlowStepListener]

Definition Classes
Job
def stepStrategy: Option[FlowStepStrategy[_]]

Definition Classes
Job
implicit def strFields[T <: TraversableOnce[String]](f: T): Fields

Definition Classes
FieldConversions
implicit def stringToFields(x: String): Fields

Definition Classes
FieldConversions
implicit def symbolToFields(x: Symbol): Fields

Definition Classes
FieldConversions
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def timeout[T](timeout: AbsoluteDuration)(t: ⇒ T): Option[T]

Definition Classes
Job
implicit def toPipe[T](iter: Iterable[T])(implicit set: TupleSetter[T], conv: TupleConverter[T]): Pipe

Definition Classes
Job
def toString(): String

Definition Classes
AnyRef → Any
implicit def tuple2ToFieldsPair[T, U](pair: (T, U))(implicit tf: (T) ⇒ Fields, uf: (U) ⇒ Fields): (Fields, Fields)

Definition Classes
FieldConversions
implicit def unitToFields(u: Unit): Fields

Definition Classes
FieldConversions
def validate: Unit

Definition Classes
Job
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def write(pipe: Pipe, src: Source): Unit

Definition Classes
Job

Related Doc: package scalding

abstract class SparkJob extends Job

Instance Constructors

new SparkJob(args: Args)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def anyToFieldArg(f: Any): Comparable[_]

val args: Args

final def asInstanceOf[T0]: T0

def asList(f: Fields): List[Comparable[_]]

def asSet(f: Fields): Set[Comparable[_]]

def buildFlow: Flow[_]

def classIdentifier: String

def clear: Unit

def clone(nextargs: Args): Job

def clone(): AnyRef

def config: Map[AnyRef, AnyRef]

implicit def dateParser: DateParser

def defaultComparator: Option[Class[_ <: Comparator[_]]]

def defaultMode(fromFields: Fields, toFields: Fields): Fields

def defaultSpillThreshold: Int

final def ensureUniqueFields(left: Fields, right: Fields, rightPipe: Pipe): (Fields, Pipe)

implicit def enumValueToFields(x: Value): Fields

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

implicit def fieldFields[T <: TraversableOnce[Field[_]]](f: T): RichFields

implicit def fieldToFields(f: Field[_]): RichFields

implicit def fields[T <: TraversableOnce[Symbol]](f: T): Fields

implicit def fieldsToRichFields(fields: Fields): RichFields

def finalize(): Unit

implicit val flowDef: FlowDef

implicit def fromEnum[T <: Enumeration](enumeration: T): Fields

final def getClass(): Class[_]

def getField(f: Fields, idx: Int): Fields

implicit def hadoopConf: Configuration

def handleStats(statsData: CascadingStats): Unit

def hasInts(f: Fields): Boolean

def hashCode(): Int

implicit def intFields[T <: TraversableOnce[Int]](f: T): Fields

implicit def intToFields(x: Int): Fields

implicit def integerToFields(x: Integer): Fields

def ioSerializations: List[Class[_ <: Serialization[_]]]

final def isInstanceOf[T0]: Boolean

implicit def iterableToRichPipe[T](iter: Iterable[T])(implicit set: TupleSetter[T], conv: TupleConverter[T]): RichPipe

def keepAlive: Unit

def listeners: List[FlowListener]

implicit def mode: Mode

def name: String

final def ne(arg0: AnyRef): Boolean

final def newSymbol(avoid: Set[Symbol], guess: Symbol, trial: Int): Symbol

def next: Option[Job]

final def notify(): Unit

final def notifyAll(): Unit

implicit def parseAnySeqToFields[T <: TraversableOnce[Any]](anyf: T): Fields

implicit def pipeToRichPipe(pipe: Pipe): RichPipe

implicit def productToFields(f: Product): Fields

implicit def read(src: Source): Pipe

def run: Boolean

implicit lazy val sc: SparkContext

implicit def scaldingConfig: Config

def setSerialization(userHadoop: Seq[Class[_ <: Serialization[_]]] = Nil): Config

def skipStrategy: Option[FlowSkipStrategy]

implicit def sourceToRichPipe(src: Source): RichPipe

implicit def sourceToSparkRichSource(source: Source): RichSource

def stepListeners: List[FlowStepListener]

def stepStrategy: Option[FlowStepStrategy[_]]

implicit def strFields[T <: TraversableOnce[String]](f: T): Fields

implicit def stringToFields(x: String): Fields

implicit def symbolToFields(x: Symbol): Fields

final def synchronized[T0](arg0: ⇒ T0): T0

def timeout[T](timeout: AbsoluteDuration)(t: ⇒ T): Option[T]

implicit def toPipe[T](iter: Iterable[T])(implicit set: TupleSetter[T], conv: TupleConverter[T]): Pipe

def toString(): String

implicit def tuple2ToFieldsPair[T, U](pair: (T, U))(implicit tf: (T) ⇒ Fields, uf: (U) ⇒ Fields): (Fields, Fields)

implicit def unitToFields(u: Unit): Fields

def validate: Unit

final def wait(): Unit