IntervalRDD

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. def ++(other: RDD[(K, V)]): RDD[(K, V)]

   

   Definition Classes

   RDD

4. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

5. def aggregate[U](zeroValue: U)(seqOp: (U, (K, V)) ⇒ U, combOp: (U, U) ⇒ U)(implicit arg0: ClassTag[U]): U

   

   Definition Classes

   RDD

6. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

7. def barrier(): RDDBarrier[(K, V)]

   

   Definition Classes

   RDD

   Annotations

   @Experimental() @Since( "2.4.0" )

8. def cache(): IntervalRDD.this.type

   

   Definition Classes

   RDD

9. def cartesian[U](other: RDD[U])(implicit arg0: ClassTag[U]): RDD[((K, V), U)]

   

   Definition Classes

   RDD

10. def checkpoint(): Unit

    

    Definition Classes

    RDD

11. def clearDependencies(): Unit

    

    Attributes

    protected

    Definition Classes

    RDD

12. def clone(): AnyRef

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

13. def coalesce(numPartitions: Int, shuffle: Boolean, partitionCoalescer: Option[PartitionCoalescer])(implicit ord: Ordering[(K, V)]): RDD[(K, V)]

    

    Definition Classes

    RDD

14. def collect(interval: K): List[V]

    

    Gets the values corresponding to the specified key, if any Assume that we're only getting data that exists (if it doesn't exist, would have been handled by upper LazyMaterialization layer

    Gets the values corresponding to the specified key, if any Assume that we're only getting data that exists (if it doesn't exist, would have been handled by upper LazyMaterialization layer

    interval

    Interval to collect overlapping data from

    returns

    List of V records overlapping interval

15. def collect(): Array[(K, V)]

    

    Collects all elements in RDD into an array of (K, V) pairs.

    Collects all elements in RDD into an array of (K, V) pairs.

    returns

    Array of (K, V) pairs

    Definition Classes

    IntervalRDD → RDD

16. def collect[U](f: PartialFunction[(K, V), U])(implicit arg0: ClassTag[U]): RDD[U]

    

    Definition Classes

    RDD

17. def compute(part: Partition, context: TaskContext): Iterator[(K, V)]

    

    Provides the RDD[(K, V)] equivalent output.

    Provides the RDD[(K, V)] equivalent output.

    part

    Partition

    context

    TaskContext

    returns

    Iterator of (K, V) pairs

    Definition Classes

    IntervalRDD → RDD

18. def context: SparkContext

    

    Definition Classes

    RDD

19. def count(): Long

    

    The number of elements in the RDD.

    The number of elements in the RDD.

    Definition Classes

    IntervalRDD → RDD

20. def countApprox(timeout: Long, confidence: Double): PartialResult[BoundedDouble]

    

    Definition Classes

    RDD

21. def countApproxDistinct(relativeSD: Double): Long

    

    Definition Classes

    RDD

22. def countApproxDistinct(p: Int, sp: Int): Long

    

    Definition Classes

    RDD

23. def countByValue()(implicit ord: Ordering[(K, V)]): Map[(K, V), Long]

    

    Definition Classes

    RDD

24. def countByValueApprox(timeout: Long, confidence: Double)(implicit ord: Ordering[(K, V)]): PartialResult[Map[(K, V), BoundedDouble]]

    

    Definition Classes

    RDD

25. final def dependencies: Seq[Dependency[_]]

    

    Definition Classes

    RDD

26. def distinct(): RDD[(K, V)]

    

    Definition Classes

    RDD

27. def distinct(numPartitions: Int)(implicit ord: Ordering[(K, V)]): RDD[(K, V)]

    

    Definition Classes

    RDD

28. final def eq(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

29. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

30. def filter(pred: ((K, V)) ⇒ Boolean): IntervalRDD[K, V]

    

    Performs filtering given a predicate.

    Performs filtering given a predicate.

    pred

    Predicate to filter (K,V) pairs by

    returns

    new IntervalRDD with filtered predicate

    Definition Classes

    IntervalRDD → RDD

31. def filterByInterval(r: K): IntervalRDD[K, V]

    

    Filters IntervalRDD by interval and returns new IntervalRDD.

    Filters IntervalRDD by interval and returns new IntervalRDD.

    r

    Interval to filter by

    returns

    filtered IntervalRDD

32. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

33. def first(): (K, V)

    

    Gets first element in the RDD.

    Gets first element in the RDD.

    returns

    first (K,V) pair

    Definition Classes

    IntervalRDD → RDD

34. def firstParent[U](implicit arg0: ClassTag[U]): RDD[U]

    

    Attributes

    protected[org.apache.spark]

    Definition Classes

    RDD

35. def flatMap[U](f: ((K, V)) ⇒ TraversableOnce[U])(implicit arg0: ClassTag[U]): RDD[U]

    

    Definition Classes

    RDD

36. def fold(zeroValue: (K, V))(op: ((K, V), (K, V)) ⇒ (K, V)): (K, V)

    

    Definition Classes

    RDD

37. def foreach(f: ((K, V)) ⇒ Unit): Unit

    

    Definition Classes

    RDD

38. def foreachPartition(f: (Iterator[(K, V)]) ⇒ Unit): Unit

    

    Definition Classes

    RDD

39. def getCheckpointFile: Option[String]

    

    Definition Classes

    RDD

40. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

41. def getDependencies: Seq[Dependency[_]]

    

    Attributes

    protected

    Definition Classes

    RDD

42. final def getNumPartitions: Int

    

    Definition Classes

    RDD

    Annotations

    @Since( "1.6.0" )

43. def getOutputDeterministicLevel: apache.spark.rdd.DeterministicLevel.Value

    

    Attributes

    protected

    Definition Classes

    RDD

    Annotations

    @DeveloperApi()

44. def getPartitions: Array[Partition]

    

    Attributes

    protected

    Definition Classes

    IntervalRDD → RDD

45. def getPreferredLocations(split: Partition): Seq[String]

    

    Attributes

    protected

    Definition Classes

    RDD

46. def getStorageLevel: StorageLevel

    

    Definition Classes

    RDD

47. def glom(): RDD[Array[(K, V)]]

    

    Definition Classes

    RDD

48. def groupBy[K](f: ((K, V)) ⇒ K, p: Partitioner)(implicit kt: ClassTag[K], ord: Ordering[K]): RDD[(K, Iterable[(K, V)])]

    

    Definition Classes

    RDD

49. def groupBy[K](f: ((K, V)) ⇒ K, numPartitions: Int)(implicit kt: ClassTag[K]): RDD[(K, Iterable[(K, V)])]

    

    Definition Classes

    RDD

50. def groupBy[K](f: ((K, V)) ⇒ K)(implicit kt: ClassTag[K]): RDD[(K, Iterable[(K, V)])]

    

    Definition Classes

    RDD

51. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

52. val id: Int

    

    Definition Classes

    RDD

53. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

54. def initializeLogIfNecessary(isInterpreter: Boolean): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

55. def intersection(other: RDD[(K, V)], numPartitions: Int): RDD[(K, V)]

    

    Definition Classes

    RDD

56. def intersection(other: RDD[(K, V)], partitioner: Partitioner)(implicit ord: Ordering[(K, V)]): RDD[(K, V)]

    

    Definition Classes

    RDD

57. def intersection(other: RDD[(K, V)]): RDD[(K, V)]

    

    Definition Classes

    RDD

58. lazy val isBarrier_: Boolean

    

    Attributes

    protected

    Definition Classes

    RDD

59. def isCheckpointed: Boolean

    

    Definition Classes

    RDD

60. def isEmpty(): Boolean

    

    Definition Classes

    RDD

61. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

62. def isTraceEnabled(): Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

63. final def iterator(split: Partition, context: TaskContext): Iterator[(K, V)]

    

    Definition Classes

    RDD

64. def keyBy[K](f: ((K, V)) ⇒ K): RDD[(K, (K, V))]

    

    Definition Classes

    RDD

65. def localCheckpoint(): IntervalRDD.this.type

    

    Definition Classes

    RDD

66. def log: Logger

    

    Attributes

    protected

    Definition Classes

    Logging

67. def logDebug(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

68. def logDebug(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

69. def logError(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

70. def logError(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

71. def logInfo(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

72. def logInfo(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

73. def logName: String

    

    Attributes

    protected

    Definition Classes

    Logging

74. def logTrace(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

75. def logTrace(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

76. def logWarning(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

77. def logWarning(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

78. def map[U](f: ((K, V)) ⇒ U)(implicit arg0: ClassTag[U]): RDD[U]

    

    Definition Classes

    RDD

79. def mapIntervalPartitions[K2 <: Interval[K2], V2](f: (IntervalPartition[K, V]) ⇒ IntervalPartition[K2, V2])(implicit arg0: ClassTag[K2], arg1: ClassTag[V2]): IntervalRDD[K2, V2]

    

    Maps interval partitions in partitionsRDD.

    Maps interval partitions in partitionsRDD.

    K2

    key type extending Interval

    V2

    data type

    f

    mapping function for interval partitions

    returns

    new IntervalRDD of type [K2, V2]

80. def mapPartitions[U](f: (Iterator[(K, V)]) ⇒ Iterator[U], preservesPartitioning: Boolean)(implicit arg0: ClassTag[U]): RDD[U]

    

    Definition Classes

    RDD

81. def mapPartitionsWithIndex[U](f: (Int, Iterator[(K, V)]) ⇒ Iterator[U], preservesPartitioning: Boolean)(implicit arg0: ClassTag[U]): RDD[U]

    

    Definition Classes

    RDD

82. def mapValues[V2](f: (V) ⇒ V2)(implicit arg0: ClassTag[V2]): IntervalRDD[K, V2]

    

    Maps each value, preserving the index.

    Maps each value, preserving the index.

    returns

    new IntervalRDD with values V2

83. def max()(implicit ord: Ordering[(K, V)]): (K, V)

    

    Definition Classes

    RDD

84. def min()(implicit ord: Ordering[(K, V)]): (K, V)

    

    Definition Classes

    RDD

85. def multiput(elems: Array[(K, V)], isSorted: Boolean = false): IntervalRDD[K, V]

    

    Unconditionally updates the specified keys to have the specified value.

    Unconditionally updates the specified keys to have the specified value. Returns a new IntervalRDD.

    elems

    Array of (K,V) pairs

    returns

    IntervalRDD with inserted elems

86. def multiputRDD(elems: RDD[(K, V)], sorted: Boolean = false): IntervalRDD[K, V]

    

    Unconditionally updates the specified keys to have the specified value.

    Unconditionally updates the specified keys to have the specified value. Returns a new IntervalRDD.

    elems

    RDD of (K,V) pairs

    sorted

    Boolean that determines whether elems is sorted

    returns

    IntervalRDD with inserted elems

87. var name: String

    

    Definition Classes

    RDD

88. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

89. final def notify(): Unit

    

    Definition Classes

    AnyRef

90. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

91. def parent[U](j: Int)(implicit arg0: ClassTag[U]): RDD[U]

    

    Attributes

    protected[org.apache.spark]

    Definition Classes

    RDD

92. def partitionBy(partitioner: Partitioner): IntervalRDD[K, V]

    

    Repartitions underlying data in IntervalRDD.

    Repartitions underlying data in IntervalRDD.

    partitioner

    new Partitioner to partition data by

    returns

    repartitioned IntervalRDD

93. val partitioner: Option[Partitioner]

    

    Definition Classes

    IntervalRDD → RDD

94. final def partitions: Array[Partition]

    

    Definition Classes

    RDD

95. def persist(newLevel: StorageLevel): IntervalRDD.this.type

    

    Persists the interval partitions using targetStorageLevel, which defaults to MEMORY_ONLY.

    Persists the interval partitions using targetStorageLevel, which defaults to MEMORY_ONLY.

    Definition Classes

    IntervalRDD → RDD

96. def persist(): IntervalRDD.this.type

    

    Definition Classes

    RDD

97. def pipe(command: Seq[String], env: Map[String, String], printPipeContext: ((String) ⇒ Unit) ⇒ Unit, printRDDElement: ((K, V), (String) ⇒ Unit) ⇒ Unit, separateWorkingDir: Boolean, bufferSize: Int, encoding: String): RDD[String]

    

    Definition Classes

    RDD

98. def pipe(command: String, env: Map[String, String]): RDD[String]

    

    Definition Classes

    RDD

99. def pipe(command: String): RDD[String]

    

    Definition Classes

    RDD

100. final def preferredLocations(split: Partition): Seq[String]

     

     Definition Classes

     RDD

101. def put(elem: (K, V)): IntervalRDD[K, V]

     

     Unconditionally updates the specified keys to have the specified value.

     Unconditionally updates the specified keys to have the specified value. Returns a new IntervalRDD.

     elem

     (K,V) pair

     returns

     IntervalRDD with inserted elem

102. def randomSplit(weights: Array[Double], seed: Long): Array[RDD[(K, V)]]

     

     Definition Classes

     RDD

103. def reduce(f: ((K, V), (K, V)) ⇒ (K, V)): (K, V)

     

     Definition Classes

     RDD

104. def repartition(numPartitions: Int)(implicit ord: Ordering[(K, V)]): RDD[(K, V)]

     

     Definition Classes

     RDD

105. def sample(withReplacement: Boolean, fraction: Double, seed: Long): RDD[(K, V)]

     

     Definition Classes

     RDD

106. def saveAsObjectFile(path: String): Unit

     

     Definition Classes

     RDD

107. def saveAsTextFile(path: String, codec: Class[_ <: CompressionCodec]): Unit

     

     Definition Classes

     RDD

108. def saveAsTextFile(path: String): Unit

     

     Definition Classes

     RDD

109. def setName(_name: String): IntervalRDD.this.type

     

     Resets RDD name

     Resets RDD name

     _name

     RDD name

     Definition Classes

     IntervalRDD → RDD

110. def sortBy[K](f: ((K, V)) ⇒ K, ascending: Boolean, numPartitions: Int)(implicit ord: Ordering[K], ctag: ClassTag[K]): RDD[(K, V)]

     

     Definition Classes

     RDD

111. def sparkContext: SparkContext

     

     Definition Classes

     RDD

112. def subtract(other: RDD[(K, V)], p: Partitioner)(implicit ord: Ordering[(K, V)]): RDD[(K, V)]

     

     Definition Classes

     RDD

113. def subtract(other: RDD[(K, V)], numPartitions: Int): RDD[(K, V)]

     

     Definition Classes

     RDD

114. def subtract(other: RDD[(K, V)]): RDD[(K, V)]

     

     Definition Classes

     RDD

115. final def synchronized[T0](arg0: ⇒ T0): T0

     

     Definition Classes

     AnyRef

116. def take(num: Int): Array[(K, V)]

     

     Definition Classes

     RDD

117. def takeOrdered(num: Int)(implicit ord: Ordering[(K, V)]): Array[(K, V)]

     

     Definition Classes

     RDD

118. def takeSample(withReplacement: Boolean, num: Int, seed: Long): Array[(K, V)]

     

     Definition Classes

     RDD

119. def toDebugString: String

     

     Definition Classes

     RDD

120. def toJavaRDD(): JavaRDD[(K, V)]

     

     Definition Classes

     RDD

121. def toLocalIterator: Iterator[(K, V)]

     

     Definition Classes

     RDD

122. def toRDD: RDD[(K, V)]

     

     Converts to normal RDD of (K, V) pairs

     Converts to normal RDD of (K, V) pairs

     returns

     RDD of (K, V) pairs

123. def toString(): String

     

     Definition Classes

     RDD → AnyRef → Any

124. def top(num: Int)(implicit ord: Ordering[(K, V)]): Array[(K, V)]

     

     Definition Classes

     RDD

125. def treeAggregate[U](zeroValue: U)(seqOp: (U, (K, V)) ⇒ U, combOp: (U, U) ⇒ U, depth: Int)(implicit arg0: ClassTag[U]): U

     

     Definition Classes

     RDD

126. def treeReduce(f: ((K, V), (K, V)) ⇒ (K, V), depth: Int): (K, V)

     

     Definition Classes

     RDD

127. def union(other: RDD[(K, V)]): RDD[(K, V)]

     

     Definition Classes

     RDD

128. def unpersist(blocking: Boolean = true): IntervalRDD.this.type

     

     Unpersists the interval partitions using

     Unpersists the interval partitions using

     Definition Classes

     IntervalRDD → RDD

129. final def wait(): Unit

     

     Definition Classes

     AnyRef

     Annotations

     @throws( ... )

130. final def wait(arg0: Long, arg1: Int): Unit

     

     Definition Classes

     AnyRef

     Annotations

     @throws( ... )

131. final def wait(arg0: Long): Unit

     

     Definition Classes

     AnyRef

     Annotations

     @throws( ... )

132. def zip[U](other: RDD[U])(implicit arg0: ClassTag[U]): RDD[((K, V), U)]

     

     Definition Classes

     RDD

133. def zipPartitions[B, C, D, V](rdd2: RDD[B], rdd3: RDD[C], rdd4: RDD[D])(f: (Iterator[(K, V)], Iterator[B], Iterator[C], Iterator[D]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[D], arg3: ClassTag[V]): RDD[V]

     

     Definition Classes

     RDD

134. def zipPartitions[B, C, D, V](rdd2: RDD[B], rdd3: RDD[C], rdd4: RDD[D], preservesPartitioning: Boolean)(f: (Iterator[(K, V)], Iterator[B], Iterator[C], Iterator[D]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[D], arg3: ClassTag[V]): RDD[V]

     

     Definition Classes

     RDD

135. def zipPartitions[B, C, V](rdd2: RDD[B], rdd3: RDD[C])(f: (Iterator[(K, V)], Iterator[B], Iterator[C]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[V]): RDD[V]

     

     Definition Classes

     RDD

136. def zipPartitions[B, C, V](rdd2: RDD[B], rdd3: RDD[C], preservesPartitioning: Boolean)(f: (Iterator[(K, V)], Iterator[B], Iterator[C]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[V]): RDD[V]

     

     Definition Classes

     RDD

137. def zipPartitions[B, V](rdd2: RDD[B])(f: (Iterator[(K, V)], Iterator[B]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[V]): RDD[V]

     

     Definition Classes

     RDD

138. def zipPartitions[B, V](rdd2: RDD[B], preservesPartitioning: Boolean)(f: (Iterator[(K, V)], Iterator[B]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[V]): RDD[V]

     

     Definition Classes

     RDD

139. def zipWithIndex(): RDD[((K, V), Long)]

     

     Definition Classes

     RDD

140. def zipWithUniqueId(): RDD[((K, V), Long)]

     

     Definition Classes

     RDD