Scalding

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. def also[L, R](ensure: FlowToPipe[L], result: FlowToPipe[R]): FlowToPipe[R]

   

   This does the AlsoProducer logic of making ensure a part of the flow, but not this output.

5. def apply(jobName: String, options: Map[String, Options] = Map.empty): Scalding

   

6. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

7. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

8. implicit val dateRangeInjection: Injection[DateRange, Interval[Timestamp]]

   

9. def emptyFlowProducer[T]: FlowProducer[TypedPipe[T]]

   

10. final def eq(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

11. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

12. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

13. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

14. def getCommutativity(names: List[String], options: Map[String, Options], s: Summer[Scalding, _, _]): Commutativity

    

15. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

16. def intersect(dr1: DateRange, dr2: DateRange): Option[DateRange]

    

17. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

18. def limitTimes[T](range: Interval[Timestamp], in: FlowToPipe[T]): FlowToPipe[T]

    

    This makes sure that the output FlowToPipe[T] produces a TypedPipe[T] with only times in the given time interval.

19. def mappedPipeFactory[T, U](factory: (DateRange) ⇒ Mappable[T])(fn: (T) ⇒ U)(implicit timeOf: TimeExtractor[U]): PipeFactory[U]

    

    Like pipeFactory, but allows the output of the factory to be mapped.

    Like pipeFactory, but allows the output of the factory to be mapped.

    Useful when using TextLine, for example, where the lines need to be parsed before you can extract the timestamps.

20. def memoize[T](pf: PipeFactory[T]): PipeFactory[T]

    

    Memoize the inner reader This is not a performance optimization, but a correctness one applicable to some cases (namely any function that mutates the FlowDef or does IO).

    Memoize the inner reader This is not a performance optimization, but a correctness one applicable to some cases (namely any function that mutates the FlowDef or does IO). Though we are working in a referentially transparent manner, the application of the function inside the PipeFactory (the Reader) mutates the FlowDef. For a fixed PipeFactory, we only want to mutate a given FlowDef once. If we memoize with this function, it guarantees that the PipeFactory is idempotent.

21. def merge[T](left: FlowToPipe[T], right: FlowToPipe[T]): FlowToPipe[T]

    

22. def minify(mode: Mode, desired: DateRange)(factory: (DateRange) ⇒ scalding.Source): Either[List[FailureReason], DateRange]

    

    Given a constructor function, computes the maximum available range of time or gives an error.

    Given a constructor function, computes the maximum available range of time or gives an error.

    Works by calling validateTaps on the Mappable, so if that does not work correctly this will be incorrect.

23. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

24. final def notify(): Unit

    

    Definition Classes

    AnyRef

25. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

26. def optionMappedPipeFactory[T, U](factory: (DateRange) ⇒ Mappable[T])(fn: (T) ⇒ Option[U])(implicit timeOf: TimeExtractor[U]): PipeFactory[U]

    

    Like pipeFactory, but allows the output of the factory to be mapped to an optional value.

    Like pipeFactory, but allows the output of the factory to be mapped to an optional value.

    Useful when using TextLine, for example, where the lines need to be parsed before you can extract the timestamps.

27. def pipeFactory[T](factory: (DateRange) ⇒ Mappable[T])(implicit timeOf: TimeExtractor[T]): PipeFactory[T]

    

    This uses minify to find the smallest subset we can run.

    This uses minify to find the smallest subset we can run. If you don't want this behavior, then use pipeFactoryExact which either produces all the DateRange or the whole job fails.

28. def pipeFactoryExact[T](factory: (DateRange) ⇒ Mappable[T])(implicit timeOf: TimeExtractor[T]): PipeFactory[T]

    

29. def plan[T](options: Map[String, Options], prod: TailProducer[Scalding, T]): PipeFactory[T]

    

30. def sourceFromMappable[T](factory: (DateRange) ⇒ Mappable[T])(implicit arg0: TimeExtractor[T], arg1: Manifest[T]): Producer[Scalding, T]

    

31. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

32. def toDateRange(timeSpan: Interval[Timestamp]): Try[DateRange]

    

33. def toPipe[T](timeSpan: Interval[Timestamp], flowDef: FlowDef, mode: Mode, pf: PipeFactory[T]): Try[(Interval[Timestamp], TimedPipe[T])]

    

34. def toPipe[T](dr: DateRange, prod: Producer[Scalding, T], opts: Map[String, Options] = Map.empty)(implicit fd: FlowDef, mode: Mode): Try[(DateRange, TypedPipe[(Timestamp, T)])]

    

    Use this method to interop with existing scalding code Note this may return a smaller DateRange than you ask for If you need an exact DateRange see toPipeExact.

35. def toPipeExact[T](timeSpan: Interval[Timestamp], flowDef: FlowDef, mode: Mode, pf: PipeFactory[T]): Try[TimedPipe[T]]

    

36. def toPipeExact[T](dr: DateRange, prod: Producer[Scalding, T], opts: Map[String, Options] = Map.empty)(implicit fd: FlowDef, mode: Mode): Try[TypedPipe[(Timestamp, T)]]

    

    Use this method to interop with existing scalding code that expects to schedule an exact DateRange or fail.

37. def toString(): String

    

    Definition Classes

    AnyRef → Any

38. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

39. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

40. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )