FrameSubProducer

Instance Constructors

new FrameSubProducer(body: ConnectionSource[DataFrame, SparkRuntime])

Value Members

final def !=(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def !=(arg0: Any): Boolean

Definition Classes
Any
final def ##(): Int

Definition Classes
AnyRef → Any
def -->(targets: ConnectionTarget[DataFrame, SparkRuntime]*): Unit

Definition Classes
ConnectionSource
def -->(tgt: MultiInputStep[DataFrame, SparkRuntime]): tgt.type

Definition Classes
ConnectionSource
def -->(tgt: ConnectionTarget[DataFrame, SparkRuntime]): tgt.type

Definition Classes
ConnectionSource
final def ==(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def ==(arg0: Any): Boolean

Definition Classes
Any
def addStepListener(listener: StepListener[DataFrame, SparkRuntime]): Unit

Registers a new listener.
Registers a new listener.

Definition Classes
Step
def allInputsRequired: Boolean

Specifies if the step should throw an error if one of the inputs is not connected.
Specifies if the step should throw an error if one of the inputs is not connected.

Definition Classes
Step
final def asInstanceOf[T0]: T0

Definition Classes
Any
val body: ConnectionSource[DataFrame, SparkRuntime]
def buildSchema(index: Int)(implicit runtime: SparkRuntime): StructType

Returns the output schema of the step at a given index by calling output() in preview mode and retrieving the schema from the DataFrame.
Returns the output schema of the step at a given index by calling output() in preview mode and retrieving the schema from the DataFrame. Since the default implementation of this method calls output(), invoking it from output() directly or indirectly will result in infinite cycle.

Attributes
protected
Definition Classes
FrameStep
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def compute(implicit runtime: SparkRuntime): DataFrame

Attributes
protected
Definition Classes
SubProducer → Producer
def compute(index: Int)(implicit runtime: SparkRuntime): DataFrame

Computes a step output value at the specified index.
Computes a step output value at the specified index. This method is invoked from output() and can safely throw any exception, which will be wrapped into ExecutionException.
index
the output value index.

Attributes
protected
Definition Classes
Producer → Step
lazy val connections: Set[Connection[DataFrame, SparkRuntime]]

Collects all connections given a set of steps.
Collects all connections given a set of steps.

Definition Classes
SubFlow
def ctx(implicit runtime: SparkRuntime): SQLContext

Returns the implicit SQLContext.
Returns the implicit SQLContext.

Attributes
protected
Definition Classes
FrameStep
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def evaluate(implicit runtime: SparkRuntime): IndexedSeq[DataFrame]

Evaluates all step's outputs and returns a list of results.
Evaluates all step's outputs and returns a list of results.

Definition Classes
Step
Annotations
@throws( classOf[ExecutionException] )
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
lazy val inPoints: Nil.type

Definition Classes
SubProducer → SubFlow
val index: Int

Definition Classes
SingleOutputStep → ConnectionSource
val inputCount: Int

The maximum number of input ports.
The maximum number of input ports.

Definition Classes
NoInputStep → Step
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def optLimit(df: DataFrame, preview: Boolean): DataFrame

Takes one row from the data frame is preview is true.
Takes one row from the data frame is preview is true.

Attributes
protected
Definition Classes
FrameStep
lazy val outPoints: List[ConnectionSource[DataFrame, SparkRuntime]]

Definition Classes
SubProducer → SubFlow
final def outSchema(implicit runtime: SparkRuntime): StructType

Returns the output schema of the step at index 0.
Returns the output schema of the step at index 0.

Definition Classes
FrameStep
final def outSchema(index: Int)(implicit runtime: SparkRuntime): StructType

Returns the output schema of the step at a given index.
Returns the output schema of the step at a given index.

Definition Classes
FrameStep
def outbounds(index: Int): SingleOutputStep[DataFrame, SparkRuntime]

Definition Classes
SingleOutputStep
final def output(implicit runtime: SparkRuntime): DataFrame

Shortcut for output(0).
Shortcut for output(0). Computes a step output at index 0.

Definition Classes
Step
Annotations
@throws( classOf[ExecutionException] )
Exceptions thrown
ExecutionException
in case of an error, or if the step is not connected.
final def output(index: Int)(implicit runtime: SparkRuntime): DataFrame

Computes a step output value at the specified index.
Computes a step output value at the specified index.
index
the output value index.

Definition Classes
Step
Annotations
@throws( classOf[ExecutionException] )
Exceptions thrown
ExecutionException
in case of an error, or if the step is not connected.
val outputCount: Int

The number of output ports.
The number of output ports.

Definition Classes
SingleOutputStep → Step
def outputToJson(implicit idGen: (Step[DataFrame, SparkRuntime]) ⇒ String): JValue

Outputs subflow into JSON.
Outputs subflow into JSON.

Definition Classes
SubFlow
def outputToXml(implicit idGen: (Step[DataFrame, SparkRuntime]) ⇒ String): Elem

Outputs the subflow into XML.
Outputs the subflow into XML.

Attributes
protected
Definition Classes
SubFlow
def removeStepListener(listener: StepListener[DataFrame, SparkRuntime]): Unit

Unregisters a listener.
Unregisters a listener.

Definition Classes
Step
def resetCache(predecessors: Boolean, descendants: Boolean): Unit

Reset the cache of the subflow and its constituents.
Reset the cache of the subflow and its constituents.

Definition Classes
SubFlow → Step
def sc(implicit runtime: SparkRuntime): SparkContext

Returns the implicit SparkContext.
Returns the implicit SparkContext.

Attributes
protected
Definition Classes
FrameStep
val step: SingleOutputStep[DataFrame, SparkRuntime]

Definition Classes
SingleOutputStep → ConnectionSource
lazy val steps: Set[Step[DataFrame, SparkRuntime]]

Collects all steps starting with the targets and going back through the predecessors.
Collects all steps starting with the targets and going back through the predecessors.

Definition Classes
SubFlow
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
val tag: String

Definition Classes
FrameSubFlow
def to(targets: ConnectionTarget[DataFrame, SparkRuntime]*): Unit

Definition Classes
ConnectionSource
def to(tgt: MultiInputStep[DataFrame, SparkRuntime]): tgt.type

Definition Classes
ConnectionSource
def to(tgt: ConnectionTarget[DataFrame, SparkRuntime]): tgt.type

Definition Classes
ConnectionSource
def toJson: JValue

The default implementation uses default id generator.
The default implementation uses default id generator.

Definition Classes
SubFlow → JsonExport
def toXml: Elem

The default implementation uses default id generator.
The default implementation uses default id generator.

Definition Classes
SubFlow → XmlExport
def value(implicit runtime: SparkRuntime): DataFrame

Definition Classes
SingleOutputStep → ConnectionSource
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wrap[U](body: ⇒ U): U

Wraps exceptions into ExecutionException instances.
Wraps exceptions into ExecutionException instances.

Attributes
protected
Definition Classes
AbstractStep

case class FrameSubProducer(body: ConnectionSource[DataFrame, SparkRuntime]) extends SubProducer[DataFrame, SparkRuntime] with FrameSubFlow with Product with Serializable

Instance Constructors

new FrameSubProducer(body: ConnectionSource[DataFrame, SparkRuntime])

Value Members

final def !=(arg0: AnyRef): Boolean

final def !=(arg0: Any): Boolean

final def ##(): Int

def -->(targets: ConnectionTarget[DataFrame, SparkRuntime]*): Unit

def -->(tgt: MultiInputStep[DataFrame, SparkRuntime]): tgt.type

def -->(tgt: ConnectionTarget[DataFrame, SparkRuntime]): tgt.type

final def ==(arg0: AnyRef): Boolean

final def ==(arg0: Any): Boolean

def addStepListener(listener: StepListener[DataFrame, SparkRuntime]): Unit

def allInputsRequired: Boolean

final def asInstanceOf[T0]: T0

val body: ConnectionSource[DataFrame, SparkRuntime]

def buildSchema(index: Int)(implicit runtime: SparkRuntime): StructType

def clone(): AnyRef

def compute(implicit runtime: SparkRuntime): DataFrame

def compute(index: Int)(implicit runtime: SparkRuntime): DataFrame

lazy val connections: Set[Connection[DataFrame, SparkRuntime]]

def ctx(implicit runtime: SparkRuntime): SQLContext

final def eq(arg0: AnyRef): Boolean

final def evaluate(implicit runtime: SparkRuntime): IndexedSeq[DataFrame]

def finalize(): Unit

final def getClass(): Class[_]

lazy val inPoints: Nil.type

val index: Int

val inputCount: Int

final def isInstanceOf[T0]: Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def optLimit(df: DataFrame, preview: Boolean): DataFrame

lazy val outPoints: List[ConnectionSource[DataFrame, SparkRuntime]]

final def outSchema(implicit runtime: SparkRuntime): StructType

final def outSchema(index: Int)(implicit runtime: SparkRuntime): StructType

def outbounds(index: Int): SingleOutputStep[DataFrame, SparkRuntime]

final def output(implicit runtime: SparkRuntime): DataFrame

final def output(index: Int)(implicit runtime: SparkRuntime): DataFrame

val outputCount: Int

def outputToJson(implicit idGen: (Step[DataFrame, SparkRuntime]) ⇒ String): JValue

def outputToXml(implicit idGen: (Step[DataFrame, SparkRuntime]) ⇒ String): Elem

def removeStepListener(listener: StepListener[DataFrame, SparkRuntime]): Unit

def resetCache(predecessors: Boolean, descendants: Boolean): Unit

def sc(implicit runtime: SparkRuntime): SparkContext

val step: SingleOutputStep[DataFrame, SparkRuntime]

lazy val steps: Set[Step[DataFrame, SparkRuntime]]

final def synchronized[T0](arg0: ⇒ T0): T0

val tag: String

def to(targets: ConnectionTarget[DataFrame, SparkRuntime]*): Unit

def to(tgt: MultiInputStep[DataFrame, SparkRuntime]): tgt.type

def to(tgt: ConnectionTarget[DataFrame, SparkRuntime]): tgt.type

def toJson: JValue

def toXml: Elem

def value(implicit runtime: SparkRuntime): DataFrame

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

final def wrap[U](body: ⇒ U): U

Inherited from Product

Inherited from Equals

Inherited from FrameSubFlow

Inherited from FrameStep

Inherited from SubProducer[DataFrame, SparkRuntime]

Inherited from SubFlow[DataFrame, SparkRuntime]

Inherited from Producer[DataFrame, SparkRuntime]

Inherited from NoInputStep[DataFrame, SparkRuntime]

Inherited from SingleOutputStep[DataFrame, SparkRuntime]

Inherited from ConnectionSource[DataFrame, SparkRuntime]

Inherited from Step[DataFrame, SparkRuntime]

Inherited from JsonExport

Inherited from XmlExport

Inherited from AbstractStep

Inherited from Serializable

Inherited from Serializable

Inherited from AnyRef

Inherited from Any

Ungrouped