FacetedLuceneRDD

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. def ++(other: RDD[T]): RDD[T]

   

   Definition Classes

   RDD

4. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

5. lazy val Config: Config

   

   Definition Classes

   Configurable

6. val DefaultFacetNum: Int

   

   Attributes

   protected

   Definition Classes

   LuceneRDDConfigurable

7. val DefaultTopK: Int

   

   Default value for topK queries

   Default value for topK queries

   Attributes

   protected

   Definition Classes

   LuceneRDDConfigurable

8. val MaxDefaultTopKValue: Int

   

   Attributes

   protected

   Definition Classes

   LuceneRDDConfigurable

9. val StringFieldsAnalyzed: Boolean

   

   Attributes

   protected

   Definition Classes

   LuceneRDDConfigurable

10. val StringFieldsIndexOptions: IndexOptions

    

    Attributes

    protected

    Definition Classes

    LuceneRDDConfigurable

11. val StringFieldsOmitNorms: Boolean

    

    Attributes

    protected

    Definition Classes

    LuceneRDDConfigurable

12. val StringFieldsStoreTermPositions: Boolean

    

    Attributes

    protected

    Definition Classes

    LuceneRDDConfigurable

13. val StringFieldsStoreTermVector: Boolean

    

    Attributes

    protected

    Definition Classes

    LuceneRDDConfigurable

14. def aggregate[U](zeroValue: U)(seqOp: (U, T) ⇒ U, combOp: (U, U) ⇒ U)(implicit arg0: ClassTag[U]): U

    

    Definition Classes

    RDD

15. final def asInstanceOf[T0]: T0

    

    Definition Classes

    Any

16. def cache(): FacetedLuceneRDD.this.type

    

    Definition Classes

    FacetedLuceneRDD → LuceneRDD → RDD

17. def cartesian[U](other: RDD[U])(implicit arg0: ClassTag[U]): RDD[(T, U)]

    

    Definition Classes

    RDD

18. def checkpoint(): Unit

    

    Definition Classes

    RDD

19. def clearDependencies(): Unit

    

    Attributes

    protected

    Definition Classes

    RDD

20. def clone(): AnyRef

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

21. def close(): Unit

    

    Definition Classes

    LuceneRDD

22. def coalesce(numPartitions: Int, shuffle: Boolean, partitionCoalescer: Option[PartitionCoalescer])(implicit ord: Ordering[T]): RDD[T]

    

    Definition Classes

    RDD

23. def collect[U](f: PartialFunction[T, U])(implicit arg0: ClassTag[U]): RDD[U]

    

    Definition Classes

    RDD

24. def collect(): Array[T]

    

    Definition Classes

    RDD

25. def compute(part: Partition, context: TaskContext): Iterator[T]

    

    RDD compute method.

    RDD compute method.

    Definition Classes

    LuceneRDD → RDD

26. def context: SparkContext

    

    Definition Classes

    RDD

27. def count(): Long

    

    Definition Classes

    LuceneRDD → RDD

28. def countApprox(timeout: Long, confidence: Double): PartialResult[BoundedDouble]

    

    Definition Classes

    RDD

29. def countApproxDistinct(relativeSD: Double): Long

    

    Definition Classes

    RDD

30. def countApproxDistinct(p: Int, sp: Int): Long

    

    Definition Classes

    RDD

31. def countByValue()(implicit ord: Ordering[T]): Map[T, Long]

    

    Definition Classes

    RDD

32. def countByValueApprox(timeout: Long, confidence: Double)(implicit ord: Ordering[T]): PartialResult[Map[T, BoundedDouble]]

    

    Definition Classes

    RDD

33. def dedup[T1](searchQueryGen: (T1) ⇒ String, topK: Int = DefaultTopK, linkerMethod: String = getLinkerMethod)(implicit arg0: ClassTag[T1]): RDD[(T1, Array[SparkScoreDoc])]

    

    Deduplication of self

    Deduplication of self

    searchQueryGen

    Search query mapper function

    topK

    Number of results to deduplication

    linkerMethod

    Method to perform linkage

    Definition Classes

    LuceneRDD

34. final def dependencies: Seq[Dependency[_]]

    

    Definition Classes

    RDD

35. def distinct(): RDD[T]

    

    Definition Classes

    RDD

36. def distinct(numPartitions: Int)(implicit ord: Ordering[T]): RDD[T]

    

    Definition Classes

    RDD

37. final def eq(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

38. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

39. def exists(elem: T): Boolean

    

    Definition Classes

    LuceneRDD

40. def exists(doc: Map[String, String]): Boolean

    

    Lucene generic query

    Lucene generic query

    Definition Classes

    LuceneRDD

41. def facetQueries(searchString: String, facetFields: Seq[String], topK: Int = DefaultTopK, facetNum: Int = DefaultFacetNum): (LuceneRDDResponse, Map[String, SparkFacetResult])

    

    Faceted query with multiple facets

    Faceted query with multiple facets

    searchString

    Lucene query string

    facetFields

    Fields on which to compute facets

    topK

    Number of results

    facetNum

    Number of faceted results

42. def facetQuery(searchString: String, facetField: String, topK: Int = DefaultTopK, facetNum: Int = DefaultFacetNum): (LuceneRDDResponse, SparkFacetResult)

    

    Faceted query

    Faceted query

    searchString

    Lucene query string

    facetField

    Field on which to compute facet

    topK

    Number of results

    facetNum

    Number of faceted results

43. def fields(): Set[String]

    

    Return all document fields

    Return all document fields

    Definition Classes

    LuceneRDD

44. def filter(pred: (T) ⇒ Boolean): LuceneRDD[T]

    

    Definition Classes

    LuceneRDD → RDD

45. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

46. def first(): T

    

    Definition Classes

    RDD

47. def firstParent[U](implicit arg0: ClassTag[U]): RDD[U]

    

    Attributes

    protected[org.apache.spark]

    Definition Classes

    RDD

48. def flatMap[U](f: (T) ⇒ TraversableOnce[U])(implicit arg0: ClassTag[U]): RDD[U]

    

    Definition Classes

    RDD

49. def fold(zeroValue: T)(op: (T, T) ⇒ T): T

    

    Definition Classes

    RDD

50. def foreach(f: (T) ⇒ Unit): Unit

    

    Definition Classes

    RDD

51. def foreachPartition(f: (Iterator[T]) ⇒ Unit): Unit

    

    Definition Classes

    RDD

52. def fuzzyQuery(fieldName: String, query: String, maxEdits: Int, topK: Int = DefaultTopK): LuceneRDDResponse

    

    Lucene fuzzy query

    Lucene fuzzy query

    fieldName

    Name of field

    query

    Query text

    maxEdits

    Fuzziness, edit distance

    topK

    Number of documents to return

    Definition Classes

    LuceneRDD

53. def getCheckpointFile: Option[String]

    

    Definition Classes

    RDD

54. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

55. def getDependencies: Seq[Dependency[_]]

    

    Attributes

    protected

    Definition Classes

    RDD

56. val getLinkerMethod: String

    

    Attributes

    protected

    Definition Classes

    LuceneRDDConfigurable

57. final def getNumPartitions: Int

    

    Definition Classes

    RDD

    Annotations

    @Since( "1.6.0" )

58. def getPartitions: Array[Partition]

    

    Attributes

    protected

    Definition Classes

    LuceneRDD → RDD

59. def getPreferredLocations(s: Partition): Seq[String]

    

    Attributes

    protected

    Definition Classes

    LuceneRDD → RDD

60. def getStorageLevel: StorageLevel

    

    Definition Classes

    RDD

61. def glom(): RDD[Array[T]]

    

    Definition Classes

    RDD

62. def groupBy[K](f: (T) ⇒ K, p: Partitioner)(implicit kt: ClassTag[K], ord: Ordering[K]): RDD[(K, Iterable[T])]

    

    Definition Classes

    RDD

63. def groupBy[K](f: (T) ⇒ K, numPartitions: Int)(implicit kt: ClassTag[K]): RDD[(K, Iterable[T])]

    

    Definition Classes

    RDD

64. def groupBy[K](f: (T) ⇒ K)(implicit kt: ClassTag[K]): RDD[(K, Iterable[T])]

    

    Definition Classes

    RDD

65. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

66. val id: Int

    

    Definition Classes

    RDD

67. val indexAnalyzer: String

    

    Definition Classes

    FacetedLuceneRDD → LuceneRDD

68. def indexStats(): RDD[IndexStatistics]

    

    Definition Classes

    LuceneRDD

69. def initializeLogIfNecessary(isInterpreter: Boolean): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

70. def intersection(other: RDD[T], numPartitions: Int): RDD[T]

    

    Definition Classes

    RDD

71. def intersection(other: RDD[T], partitioner: Partitioner)(implicit ord: Ordering[T]): RDD[T]

    

    Definition Classes

    RDD

72. def intersection(other: RDD[T]): RDD[T]

    

    Definition Classes

    RDD

73. def isCheckpointed: Boolean

    

    Definition Classes

    RDD

74. def isEmpty(): Boolean

    

    Definition Classes

    RDD

75. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

76. def isTraceEnabled(): Boolean

    

    Attributes

    protected

    Definition Classes

    Logging

77. final def iterator(split: Partition, context: TaskContext): Iterator[T]

    

    Definition Classes

    RDD

78. def keyBy[K](f: (T) ⇒ K): RDD[(K, T)]

    

    Definition Classes

    RDD

79. def link[T1](other: RDD[T1], searchQueryGen: (T1) ⇒ String, topK: Int = DefaultTopK, linkerMethod: String = getLinkerMethod)(implicit arg0: ClassTag[T1]): RDD[(T1, Array[SparkScoreDoc])]

    

    Entity linkage via Lucene query over all elements of an RDD.

    Entity linkage via Lucene query over all elements of an RDD.

    T1

    A type

    other

    RDD to be linked

    searchQueryGen

    Function that generates a search query for each element of other

    linkerMethod

    Method to perform linkage, default value from configuration

    returns

    an RDD of Tuple2 that contains the linked search Lucene documents in the second Note: Currently the query strings of the other RDD are collected to the driver and broadcast to the workers.

    Definition Classes

    LuceneRDD

80. def linkByQuery[T1](other: RDD[T1], searchQueryGen: (T1) ⇒ Query, topK: Int = DefaultTopK, linkerMethod: String = getLinkerMethod)(implicit arg0: ClassTag[T1]): RDD[(T1, Array[SparkScoreDoc])]

    

    Entity linkage via Lucene query over all elements of an RDD.

    Entity linkage via Lucene query over all elements of an RDD.

    T1

    A type

    other

    RDD to be linked

    searchQueryGen

    Function that generates a Lucene Query object for each element of other

    linkerMethod

    Method to perform linkage

    returns

    an RDD of Tuple2 that contains the linked search Lucene Document in the second position

    Definition Classes

    LuceneRDD

81. def linkDataFrame(other: DataFrame, searchQueryGen: (Row) ⇒ String, topK: Int = DefaultTopK, linkerMethod: String = getLinkerMethod): RDD[(Row, Array[SparkScoreDoc])]

    

    Entity linkage via Lucene query over all elements of an RDD.

    Entity linkage via Lucene query over all elements of an RDD.

    other

    DataFrame to be linked

    searchQueryGen

    Function that generates a search query for each element of other

    linkerMethod

    Method to perform linkage

    returns

    an RDD of Tuple2 that contains the linked search Lucene documents in the second

    Definition Classes

    LuceneRDD

82. def localCheckpoint(): FacetedLuceneRDD.this.type

    

    Definition Classes

    RDD

83. def log: Logger

    

    Attributes

    protected

    Definition Classes

    Logging

84. def logDebug(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

85. def logDebug(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

86. def logError(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

87. def logError(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

88. def logInfo(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

89. def logInfo(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

90. def logName: String

    

    Attributes

    protected

    Definition Classes

    Logging

91. def logTrace(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

92. def logTrace(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

93. def logWarning(msg: ⇒ String, throwable: Throwable): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

94. def logWarning(msg: ⇒ String): Unit

    

    Attributes

    protected

    Definition Classes

    Logging

95. def map[U](f: (T) ⇒ U)(implicit arg0: ClassTag[U]): RDD[U]

    

    Definition Classes

    RDD

96. def mapPartitions[U](f: (Iterator[T]) ⇒ Iterator[U], preservesPartitioning: Boolean)(implicit arg0: ClassTag[U]): RDD[U]

    

    Definition Classes

    RDD

97. def mapPartitionsWithIndex[U](f: (Int, Iterator[T]) ⇒ Iterator[U], preservesPartitioning: Boolean)(implicit arg0: ClassTag[U]): RDD[U]

    

    Definition Classes

    RDD

98. def max()(implicit ord: Ordering[T]): T

    

    Definition Classes

    RDD

99. def min()(implicit ord: Ordering[T]): T

    

    Definition Classes

    RDD

100. def moreLikeThis(fieldName: String, query: String, minTermFreq: Int, minDocFreq: Int, topK: Int = DefaultTopK): LuceneRDDResponse

     

     Lucene's More Like This (MLT) functionality

     Lucene's More Like This (MLT) functionality

     fieldName

     Field name

     query

     Query text

     minTermFreq

     Minimum term frequency

     minDocFreq

     Minimum document frequency

     topK

     Number of returned documents

     Definition Classes

     LuceneRDD

101. var name: String

     

     Definition Classes

     RDD

102. final def ne(arg0: AnyRef): Boolean

     

     Definition Classes

     AnyRef

103. final def notify(): Unit

     

     Definition Classes

     AnyRef

104. final def notifyAll(): Unit

     

     Definition Classes

     AnyRef

105. def parent[U](j: Int)(implicit arg0: ClassTag[U]): RDD[U]

     

     Attributes

     protected[org.apache.spark]

     Definition Classes

     RDD

106. def partitionMapper(f: (AbstractLuceneRDDPartition[T]) ⇒ LuceneRDDResponsePartition, k: Int): LuceneRDDResponse

     

     Maps partition results

     Maps partition results

     f

     Function to apply on each partition / distributed index

     k

     number of documents to return

     Attributes

     protected

     Definition Classes

     LuceneRDD

107. val partitioner: Option[Partitioner]

     

     Definition Classes

     RDD

108. final def partitions: Array[Partition]

     

     Definition Classes

     RDD

109. val partitionsRDD: RDD[AbstractLuceneRDDPartition[T]]

     

     Attributes

     protected

     Definition Classes

     FacetedLuceneRDD → LuceneRDD

110. def persist(newLevel: StorageLevel): FacetedLuceneRDD.this.type

     

     Definition Classes

     FacetedLuceneRDD → LuceneRDD → RDD

111. def persist(): FacetedLuceneRDD.this.type

     

     Definition Classes

     RDD

112. def phraseQuery(fieldName: String, query: String, topK: Int = DefaultTopK): LuceneRDDResponse

     

     Lucene phrase Query

     Lucene phrase Query

     fieldName

     Name of field

     query

     Query text

     topK

     Number of documents to return

     Definition Classes

     LuceneRDD

113. def pipe(command: Seq[String], env: Map[String, String], printPipeContext: ((String) ⇒ Unit) ⇒ Unit, printRDDElement: (T, (String) ⇒ Unit) ⇒ Unit, separateWorkingDir: Boolean, bufferSize: Int, encoding: String): RDD[String]

     

     Definition Classes

     RDD

114. def pipe(command: String, env: Map[String, String]): RDD[String]

     

     Definition Classes

     RDD

115. def pipe(command: String): RDD[String]

     

     Definition Classes

     RDD

116. final def preferredLocations(split: Partition): Seq[String]

     

     Definition Classes

     RDD

117. def prefixQuery(fieldName: String, query: String, topK: Int = DefaultTopK): LuceneRDDResponse

     

     Lucene prefix query

     Lucene prefix query

     fieldName

     Name of field

     query

     Prefix query text

     topK

     Number of documents to return

     Definition Classes

     LuceneRDD

118. def query(searchString: String, topK: Int = DefaultTopK): LuceneRDDResponse

     

     Generic query using Lucene's query parser

     Generic query using Lucene's query parser

     searchString

     Query String

     Definition Classes

     LuceneRDD

119. val queryAnalyzer: String

     

     Definition Classes

     FacetedLuceneRDD → LuceneRDD

120. def randomSplit(weights: Array[Double], seed: Long): Array[RDD[T]]

     

     Definition Classes

     RDD

121. def reduce(f: (T, T) ⇒ T): T

     

     Definition Classes

     RDD

122. def repartition(numPartitions: Int)(implicit ord: Ordering[T]): RDD[T]

     

     Definition Classes

     RDD

123. def sample(withReplacement: Boolean, fraction: Double, seed: Long): RDD[T]

     

     Definition Classes

     RDD

124. def saveAsObjectFile(path: String): Unit

     

     Definition Classes

     RDD

125. def saveAsTextFile(path: String, codec: Class[_ <: CompressionCodec]): Unit

     

     Definition Classes

     RDD

126. def saveAsTextFile(path: String): Unit

     

     Definition Classes

     RDD

127. def setName(_name: String): FacetedLuceneRDD.this.type

     

     Set the name for the RDD; By default set to "LuceneRDD"

     Set the name for the RDD; By default set to "LuceneRDD"

     Definition Classes

     LuceneRDD → RDD

128. def sortBy[K](f: (T) ⇒ K, ascending: Boolean, numPartitions: Int)(implicit ord: Ordering[K], ctag: ClassTag[K]): RDD[T]

     

     Definition Classes

     RDD

129. def sparkContext: SparkContext

     

     Definition Classes

     RDD

130. def subtract(other: RDD[T], p: Partitioner)(implicit ord: Ordering[T]): RDD[T]

     

     Definition Classes

     RDD

131. def subtract(other: RDD[T], numPartitions: Int): RDD[T]

     

     Definition Classes

     RDD

132. def subtract(other: RDD[T]): RDD[T]

     

     Definition Classes

     RDD

133. final def synchronized[T0](arg0: ⇒ T0): T0

     

     Definition Classes

     AnyRef

134. def take(num: Int): Array[T]

     

     Definition Classes

     RDD

135. def takeOrdered(num: Int)(implicit ord: Ordering[T]): Array[T]

     

     Definition Classes

     RDD

136. def takeSample(withReplacement: Boolean, num: Int, seed: Long): Array[T]

     

     Definition Classes

     RDD

137. def termQuery(fieldName: String, query: String, topK: Int = DefaultTopK): LuceneRDDResponse

     

     Lucene term query

     Lucene term query

     fieldName

     Name of field

     query

     Term to search on

     topK

     Number of documents to return

     Definition Classes

     LuceneRDD

138. def termVectors(fieldName: String, idFieldName: Option[String] = None): RDD[TermVectorEntry]

     

     Return Term vector for a Lucene field

     Return Term vector for a Lucene field

     fieldName

     Field name for term vectors

     idFieldName

     Lucene field that contains unique id: default set to None, in which case id equals (docId, partitionId)

     returns

     RDD of term vector entries, i.e., (document id, term as String, term frequency in document)

     Definition Classes

     LuceneRDD

139. def toDebugString: String

     

     Definition Classes

     RDD

140. def toJavaRDD(): JavaRDD[T]

     

     Definition Classes

     RDD

141. def toLocalIterator: Iterator[T]

     

     Definition Classes

     RDD

142. def toString(): String

     

     Definition Classes

     RDD → AnyRef → Any

143. def top(num: Int)(implicit ord: Ordering[T]): Array[T]

     

     Definition Classes

     RDD

144. def treeAggregate[U](zeroValue: U)(seqOp: (U, T) ⇒ U, combOp: (U, U) ⇒ U, depth: Int)(implicit arg0: ClassTag[U]): U

     

     Definition Classes

     RDD

145. def treeReduce(f: (T, T) ⇒ T, depth: Int): T

     

     Definition Classes

     RDD

146. def union(other: RDD[T]): RDD[T]

     

     Definition Classes

     RDD

147. def unpersist(blocking: Boolean = true): FacetedLuceneRDD.this.type

     

     Definition Classes

     FacetedLuceneRDD → LuceneRDD → RDD

148. final def wait(): Unit

     

     Definition Classes

     AnyRef

     Annotations

     @throws( ... )

149. final def wait(arg0: Long, arg1: Int): Unit

     

     Definition Classes

     AnyRef

     Annotations

     @throws( ... )

150. final def wait(arg0: Long): Unit

     

     Definition Classes

     AnyRef

     Annotations

     @throws( ... )

151. def zip[U](other: RDD[U])(implicit arg0: ClassTag[U]): RDD[(T, U)]

     

     Definition Classes

     RDD

152. def zipPartitions[B, C, D, V](rdd2: RDD[B], rdd3: RDD[C], rdd4: RDD[D])(f: (Iterator[T], Iterator[B], Iterator[C], Iterator[D]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[D], arg3: ClassTag[V]): RDD[V]

     

     Definition Classes

     RDD

153. def zipPartitions[B, C, D, V](rdd2: RDD[B], rdd3: RDD[C], rdd4: RDD[D], preservesPartitioning: Boolean)(f: (Iterator[T], Iterator[B], Iterator[C], Iterator[D]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[D], arg3: ClassTag[V]): RDD[V]

     

     Definition Classes

     RDD

154. def zipPartitions[B, C, V](rdd2: RDD[B], rdd3: RDD[C])(f: (Iterator[T], Iterator[B], Iterator[C]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[V]): RDD[V]

     

     Definition Classes

     RDD

155. def zipPartitions[B, C, V](rdd2: RDD[B], rdd3: RDD[C], preservesPartitioning: Boolean)(f: (Iterator[T], Iterator[B], Iterator[C]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[V]): RDD[V]

     

     Definition Classes

     RDD

156. def zipPartitions[B, V](rdd2: RDD[B])(f: (Iterator[T], Iterator[B]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[V]): RDD[V]

     

     Definition Classes

     RDD

157. def zipPartitions[B, V](rdd2: RDD[B], preservesPartitioning: Boolean)(f: (Iterator[T], Iterator[B]) ⇒ Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[V]): RDD[V]

     

     Definition Classes

     RDD

158. def zipWithIndex(): RDD[(T, Long)]

     

     Definition Classes

     RDD

159. def zipWithUniqueId(): RDD[(T, Long)]

     

     Definition Classes

     RDD