com.twitter.summingbird.spark

SparkPlatform

class SparkPlatform extends Platform[SparkPlatform] with PlatformPlanner[SparkPlatform]

This is a first pass at an offline Platform that executes on apache spark.

TODO: add the logic for seeing what time spans each source / store / service can cover and finding the max they all cover together etc.

Linear Supertypes
PlatformPlanner[SparkPlatform], Platform[SparkPlatform], AnyRef, Any
Ordering
  1. Alphabetic
  2. By inheritance
Inherited
  1. SparkPlatform
  2. PlatformPlanner
  3. Platform
  4. AnyRef
  5. Any
  1. Hide All
  2. Show all
Learn more about member selection
Visibility
  1. Public
  2. All

Instance Constructors

  1. new SparkPlatform(sc: SparkContext, timeSpan: Interval[Timestamp], options: Map[String, Options] = ...)

Type Members

  1. type Buffer[k, v] = Service[k, v] with Sink[(k, v)]

    Definition Classes
    Platform

  2. type Plan[T] = RDD[(Timestamp, T)]

    Definition Classes
    SparkPlatform → Platform

  3. case class PlanState[T](plan: P.Plan[T], visited: Visited) extends Product with Serializable

    Definition Classes
    PlatformPlanner

  4. type Prod[T] = Producer[SparkPlatform, T]

    Definition Classes
    PlatformPlanner

  5. type Service[K, LV] = SparkService[K, LV]

    Definition Classes
    SparkPlatform → Platform

  6. type Sink[T] = SparkSink[T]

    Definition Classes
    SparkPlatform → Platform

  7. type Source[T] = SparkSource[T]

    Definition Classes
    SparkPlatform → Platform

  8. type Store[K, V] = SparkStore[K, V]

    Definition Classes
    SparkPlatform → Platform

  9. type TailProd[T] = TailProducer[SparkPlatform, T]

    Definition Classes
    PlatformPlanner

  10. type Visited = Map[Prod[_], Plan[_]]

    Definition Classes
    PlatformPlanner

Value Members

  1. final def !=(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  2. final def !=(arg0: Any): Boolean

    Definition Classes
    Any

  3. final def ##(): Int

    Definition Classes
    AnyRef → Any

  4. final def ==(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  5. final def ==(arg0: Any): Boolean

    Definition Classes
    Any

  6. final def asInstanceOf[T0]: T0

    Definition Classes
    Any

  7. def clone(): AnyRef

    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  8. final def eq(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  9. def equals(arg0: Any): Boolean

    Definition Classes
    AnyRef → Any

  10. def finalize(): Unit

    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )

  11. final def getClass(): Class[_]

    Definition Classes
    AnyRef → Any

  12. def getCommutativity(summer: Summer[SparkPlatform, _, _]): Commutativity

  13. def hashCode(): Int

    Definition Classes
    AnyRef → Any

  14. final def isInstanceOf[T0]: Boolean

    Definition Classes
    Any

  15. final def ne(arg0: AnyRef): Boolean

    Definition Classes
    AnyRef

  16. final def notify(): Unit

    Definition Classes
    AnyRef

  17. final def notifyAll(): Unit

    Definition Classes
    AnyRef

  18. def plan[T](completed: TailProducer[SparkPlatform, T]): RDD[(Timestamp, T)]

    Definition Classes
    SparkPlatform → Platform

  19. def planAlsoProducer[E, R](ensure: TailProd[E], result: Prod[R], visited: Visited)(implicit arg0: ClassTag[R]): PlanState[R]

    Definition Classes
    SparkPlatform → PlatformPlanner

  20. def planFlatMappedProducer[T, U](prod: Prod[T], visited: Visited, fn: (T) ⇒ TraversableOnce[U])(implicit arg0: ClassTag[U]): PlanState[U]

    Definition Classes
    SparkPlatform → PlatformPlanner

  21. def planIdentityKeyedProducer[K, V](prod: Prod[(K, V)], visited: Visited): PlanState[(K, V)]

    Definition Classes
    SparkPlatform → PlatformPlanner

  22. def planKeyFlatMappedProducer[K, V, K2](prod: Prod[(K, V)], visited: Visited, fn: (K) ⇒ TraversableOnce[K2]): PlanState[(K2, V)]

    Definition Classes
    SparkPlatform → PlatformPlanner

  23. def planLeftJoinedProducer[K, V, JoinedV](prod: Prod[(K, V)], visited: Visited, service: Service[K, JoinedV])(implicit arg0: ClassTag[K], arg1: ClassTag[V]): PlanState[(K, (V, Option[JoinedV]))]

    Definition Classes
    SparkPlatform → PlatformPlanner

  24. def planMergedProducer[T](left: Prod[T], right: Prod[T], visited: Visited): PlanState[T]

    Definition Classes
    SparkPlatform → PlatformPlanner

  25. def planNamedProducer[T](prod: Prod[T], name: String, visited: Visited): PlanState[T]

    Definition Classes
    SparkPlatform → PlatformPlanner

  26. def planOptionMappedProducer[T, U](prod: Prod[T], visited: Visited, fn: (T) ⇒ Option[U])(implicit arg0: ClassTag[U]): PlanState[U]

    Definition Classes
    SparkPlatform → PlatformPlanner

  27. def planSource[T](source: Source[T], visited: Visited): PlanState[T]

    Definition Classes
    SparkPlatform → PlatformPlanner

  28. def planSummer[K, V](summer: Summer[SparkPlatform, K, V], prod: Prod[(K, V)], visited: Visited, store: Store[K, V], semigroup: Semigroup[V])(implicit arg0: ClassTag[K], arg1: ClassTag[V]): PlanState[(K, (Option[V], V))]

    Definition Classes
    SparkPlatform → PlatformPlanner

  29. def planWrittenProducer[T](prod: Prod[T], visited: Visited, sink: Sink[T])(implicit arg0: ClassTag[T]): PlanState[T]

    Definition Classes
    SparkPlatform → PlatformPlanner

  30. def run[T](completed: TailProducer[SparkPlatform, T]): Unit

  31. def run(): Unit

  32. final def synchronized[T0](arg0: ⇒ T0): T0

    Definition Classes
    AnyRef

  33. def toPlan[T](producer: Prod[T], visited: Visited): PlanState[T]

    Attributes
    protected
    Definition Classes
    PlatformPlanner

  34. def toString(): String

    Definition Classes
    AnyRef → Any

  35. def visit[T](producer: Prod[T]): PlanState[T]

    Definition Classes
    PlatformPlanner

  36. final def wait(): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  37. final def wait(arg0: Long, arg1: Int): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

  38. final def wait(arg0: Long): Unit

    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

Inherited from PlatformPlanner[SparkPlatform]

Inherited from Platform[SparkPlatform]

Inherited from AnyRef

Inherited from Any

Ungrouped