stream

Type Members

1. type AfterStreamStepComputed = AfterStepComputed[DataStream, SparkStreamingRuntime]

2. type BeforeStreamStepComputed = BeforeStepComputed[DataStream, SparkStreamingRuntime]

3. type DataStream = DStream[Row]

4. class DefaultSparkStreamingRuntime extends DefaultSparkRuntime with SparkStreamingRuntime

   The default implementation of SparkStreamingRuntime.

5. case class Filter(condition: String) extends StreamSplitter with Product with Serializable

   Filters the stream based on a combination of boolean conditions against fields.

6. case class Foreach(flow: Step[DataFrame, SparkRuntime]) extends StreamSplitter with Product with Serializable

   Invokes the embedded flow on each batch.

7. case class Join(condition: Option[String], joinType: JoinType) extends StreamMerger with Product with Serializable

   Performs join of the two data streams.

8. type JsonStreamStepFactory = JsonStepFactory[StreamStep, DataStream, SparkStreamingRuntime]

9. case class KafkaInput(brokers: Iterable[String], topics: Iterable[String], kafkaProperties: Map[String, String] = ..., field: String = "payload") extends StreamProducer with Product with Serializable

   Creates a text stream from Apache Kafka.

10. type MapListState = Iterable[Map[String, Any]]

11. type MapState = Map[String, Any]

12. case class MvelMapListStateUpdate(schema: StructType, expr: String, keyFields: Iterable[String]) extends MvelStateUpdate[MapListState] with Product with Serializable

    Models state as a java Iterable[Map[String, Any]].

13. case class MvelMapStateUpdate(schema: StructType, expr: String, keyFields: Iterable[String]) extends MvelStateUpdate[MapState] with Product with Serializable

    Models state as a java Map[String, Any].

14. abstract class MvelStateUpdate[S <: AnyRef] extends StateUpdate[S]

    Starting point for MVEL-based implementations of the update state function.

15. trait PairFunctions extends AnyRef

    Supplies helper functions for pair streams.

16. case class QueueInput(schema: StructType, data: Seq[Seq[Row]] = immutable.this.Nil) extends StreamProducer with Product with Serializable

    Creates a stream from a static data set, passing one RDD at a time.

17. case class SetVariables(vars: Map[String, Any]) extends StreamTransformer with Product with Serializable

    Sets or drops the ignition runtime variables.

18. trait SparkStreamingRuntime extends SparkRuntime

    An extension of com.ignition.frame.SparkRuntime which adds a streaming context to the mix.

19. abstract class StateUpdate[S] extends StreamTransformer with PairFunctions

    Applies Spark updateStateByKey function to produce a stream of states.

20. case class StreamFlow(targets: Iterable[ConnectionSource[DataStream, SparkStreamingRuntime]]) extends SubModule[DataStream, SparkStreamingRuntime] with StreamStep with Product with Serializable

    Stream Flow represents a DStream workflow.

21. sealed trait StreamFlowEvent extends AnyRef

    Base trait for all stream flow events.

22. trait StreamFlowListener extends AnyRef

    Listener which will be notified on stream flow events.

23. case class StreamFlowStarted(flow: StreamFlow) extends StreamFlowEvent with Product with Serializable

24. case class StreamFlowTerminated(flow: StreamFlow) extends StreamFlowEvent with Product with Serializable

25. abstract class StreamMerger extends Merger[DataStream, SparkStreamingRuntime] with StreamStep

26. abstract class StreamModule extends Module[DataStream, SparkStreamingRuntime] with StreamStep

27. abstract class StreamProducer extends Producer[DataStream, SparkStreamingRuntime] with StreamStep

28. abstract class StreamSplitter extends Splitter[DataStream, SparkStreamingRuntime] with StreamStep

29. trait StreamStep extends AbstractStep with Step[DataStream, SparkStreamingRuntime]

    Workflow step that emits DataStream as the output.

30. case class StreamStepBatchProcessed(step: StreamStep, index: Int, time: DateTime, preview: Boolean, rows: RDD[Row]) extends Product with Serializable

    Encapsulates the details about the processed batch.

31. trait StreamStepDataListener extends AnyRef

    Listener which will be notified on each processed stream batch.

32. type StreamStepListener = StepListener[DataStream, SparkStreamingRuntime]

33. trait StreamSubFlow extends AbstractStep with StreamStep

34. case class StreamSubMerger(body: (Seq[ConnectionTarget[DataStream, SparkStreamingRuntime]], ConnectionSource[DataStream, SparkStreamingRuntime])) extends SubMerger[DataStream, SparkStreamingRuntime] with StreamSubFlow with Product with Serializable

35. case class StreamSubModule(body: (Seq[ConnectionTarget[DataStream, SparkStreamingRuntime]], Seq[ConnectionSource[DataStream, SparkStreamingRuntime]])) extends SubModule[DataStream, SparkStreamingRuntime] with StreamSubFlow with Product with Serializable

36. case class StreamSubProducer(body: ConnectionSource[DataStream, SparkStreamingRuntime]) extends SubProducer[DataStream, SparkStreamingRuntime] with StreamSubFlow with Product with Serializable

37. case class StreamSubSplitter(body: (ConnectionTarget[DataStream, SparkStreamingRuntime], Seq[ConnectionSource[DataStream, SparkStreamingRuntime]])) extends SubSplitter[DataStream, SparkStreamingRuntime] with StreamSubFlow with Product with Serializable

38. case class StreamSubTransformer(body: (ConnectionTarget[DataStream, SparkStreamingRuntime], ConnectionSource[DataStream, SparkStreamingRuntime])) extends SubTransformer[DataStream, SparkStreamingRuntime] with StreamSubFlow with Product with Serializable

39. abstract class StreamTransformer extends Transformer[DataStream, SparkStreamingRuntime] with StreamStep

40. case class Window(windowDuration: Duration, slideDuration: Option[Duration] = scala.None) extends StreamTransformer with Product with Serializable

    Sliding time window.

41. type XmlStreamStepFactory = XmlStepFactory[StreamStep, DataStream, SparkStreamingRuntime]