Packages

class FileScanRDD extends RDD[InternalRow]

An RDD that scans a list of file partitions.

Linear Supertypes
RDD[InternalRow], Logging, Serializable, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. FileScanRDD
  2. RDD
  3. Logging
  4. Serializable
  5. AnyRef
  6. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. Protected

Instance Constructors

  1. new FileScanRDD(sparkSession: SparkSession, readFunction: (PartitionedFile) => Iterator[InternalRow], filePartitions: Seq[FilePartition], readSchema: StructType, metadataColumns: Seq[AttributeReference] = Seq.empty, metadataExtractors: Map[String, (PartitionedFile) => Any] = Map.empty, options: FileSourceOptions = new FileSourceOptions(CaseInsensitiveMap(Map.empty)))

    metadataColumns

    File-constant metadata columns to append to end of schema.

Value Members

  1. def ++(other: RDD[InternalRow]): RDD[InternalRow]
    Definition Classes
    RDD
  2. def aggregate[U](zeroValue: U)(seqOp: (U, InternalRow) => U, combOp: (U, U) => U)(implicit arg0: ClassTag[U]): U
    Definition Classes
    RDD
  3. def barrier(): RDDBarrier[InternalRow]
    Definition Classes
    RDD
    Annotations
    @Experimental() @Since("2.4.0")
  4. def cache(): FileScanRDD.this.type
    Definition Classes
    RDD
  5. def cartesian[U](other: RDD[U])(implicit arg0: ClassTag[U]): RDD[(InternalRow, U)]
    Definition Classes
    RDD
  6. def checkpoint(): Unit
    Definition Classes
    RDD
  7. def cleanShuffleDependencies(blocking: Boolean): Unit
    Definition Classes
    RDD
    Annotations
    @DeveloperApi() @Since("3.1.0")
  8. def coalesce(numPartitions: Int, shuffle: Boolean, partitionCoalescer: Option[PartitionCoalescer])(implicit ord: Ordering[InternalRow]): RDD[InternalRow]
    Definition Classes
    RDD
  9. def collect[U](f: PartialFunction[InternalRow, U])(implicit arg0: ClassTag[U]): RDD[U]
    Definition Classes
    RDD
  10. def collect(): Array[InternalRow]
    Definition Classes
    RDD
  11. def compute(split: Partition, context: TaskContext): Iterator[InternalRow]
    Definition Classes
    FileScanRDD → RDD
  12. def context: SparkContext
    Definition Classes
    RDD
  13. def count(): Long
    Definition Classes
    RDD
  14. def countApprox(timeout: Long, confidence: Double): PartialResult[BoundedDouble]
    Definition Classes
    RDD
  15. def countApproxDistinct(relativeSD: Double): Long
    Definition Classes
    RDD
  16. def countApproxDistinct(p: Int, sp: Int): Long
    Definition Classes
    RDD
  17. def countByValue()(implicit ord: Ordering[InternalRow]): Map[InternalRow, Long]
    Definition Classes
    RDD
  18. def countByValueApprox(timeout: Long, confidence: Double)(implicit ord: Ordering[InternalRow]): PartialResult[Map[InternalRow, BoundedDouble]]
    Definition Classes
    RDD
  19. final def dependencies: Seq[Dependency[_]]
    Definition Classes
    RDD
  20. def distinct(): RDD[InternalRow]
    Definition Classes
    RDD
  21. def distinct(numPartitions: Int)(implicit ord: Ordering[InternalRow]): RDD[InternalRow]
    Definition Classes
    RDD
  22. val filePartitions: Seq[FilePartition]
  23. def filter(f: (InternalRow) => Boolean): RDD[InternalRow]
    Definition Classes
    RDD
  24. def first(): InternalRow
    Definition Classes
    RDD
  25. def flatMap[U](f: (InternalRow) => TraversableOnce[U])(implicit arg0: ClassTag[U]): RDD[U]
    Definition Classes
    RDD
  26. def fold(zeroValue: InternalRow)(op: (InternalRow, InternalRow) => InternalRow): InternalRow
    Definition Classes
    RDD
  27. def foreach(f: (InternalRow) => Unit): Unit
    Definition Classes
    RDD
  28. def foreachPartition(f: (Iterator[InternalRow]) => Unit): Unit
    Definition Classes
    RDD
  29. def getCheckpointFile: Option[String]
    Definition Classes
    RDD
  30. final def getNumPartitions: Int
    Definition Classes
    RDD
    Annotations
    @Since("1.6.0")
  31. def getResourceProfile(): ResourceProfile
    Definition Classes
    RDD
    Annotations
    @Experimental() @Since("3.1.0")
  32. def getStorageLevel: StorageLevel
    Definition Classes
    RDD
  33. def glom(): RDD[Array[InternalRow]]
    Definition Classes
    RDD
  34. def groupBy[K](f: (InternalRow) => K, p: Partitioner)(implicit kt: ClassTag[K], ord: Ordering[K]): RDD[(K, Iterable[InternalRow])]
    Definition Classes
    RDD
  35. def groupBy[K](f: (InternalRow) => K, numPartitions: Int)(implicit kt: ClassTag[K]): RDD[(K, Iterable[InternalRow])]
    Definition Classes
    RDD
  36. def groupBy[K](f: (InternalRow) => K)(implicit kt: ClassTag[K]): RDD[(K, Iterable[InternalRow])]
    Definition Classes
    RDD
  37. val id: Int
    Definition Classes
    RDD
  38. def intersection(other: RDD[InternalRow], numPartitions: Int): RDD[InternalRow]
    Definition Classes
    RDD
  39. def intersection(other: RDD[InternalRow], partitioner: Partitioner)(implicit ord: Ordering[InternalRow]): RDD[InternalRow]
    Definition Classes
    RDD
  40. def intersection(other: RDD[InternalRow]): RDD[InternalRow]
    Definition Classes
    RDD
  41. def isCheckpointed: Boolean
    Definition Classes
    RDD
  42. def isEmpty(): Boolean
    Definition Classes
    RDD
  43. final def iterator(split: Partition, context: TaskContext): Iterator[InternalRow]
    Definition Classes
    RDD
  44. def keyBy[K](f: (InternalRow) => K): RDD[(K, InternalRow)]
    Definition Classes
    RDD
  45. def localCheckpoint(): FileScanRDD.this.type
    Definition Classes
    RDD
  46. def map[U](f: (InternalRow) => U)(implicit arg0: ClassTag[U]): RDD[U]
    Definition Classes
    RDD
  47. def mapPartitions[U](f: (Iterator[InternalRow]) => Iterator[U], preservesPartitioning: Boolean)(implicit arg0: ClassTag[U]): RDD[U]
    Definition Classes
    RDD
  48. def mapPartitionsWithEvaluator[U](evaluatorFactory: PartitionEvaluatorFactory[InternalRow, U])(implicit arg0: ClassTag[U]): RDD[U]
    Definition Classes
    RDD
    Annotations
    @DeveloperApi() @Since("3.5.0")
  49. def mapPartitionsWithIndex[U](f: (Int, Iterator[InternalRow]) => Iterator[U], preservesPartitioning: Boolean)(implicit arg0: ClassTag[U]): RDD[U]
    Definition Classes
    RDD
  50. def max()(implicit ord: Ordering[InternalRow]): InternalRow
    Definition Classes
    RDD
  51. val metadataColumns: Seq[AttributeReference]
  52. def min()(implicit ord: Ordering[InternalRow]): InternalRow
    Definition Classes
    RDD
  53. var name: String
    Definition Classes
    RDD
  54. val partitioner: Option[Partitioner]
    Definition Classes
    RDD
  55. final def partitions: Array[Partition]
    Definition Classes
    RDD
  56. def persist(): FileScanRDD.this.type
    Definition Classes
    RDD
  57. def persist(newLevel: StorageLevel): FileScanRDD.this.type
    Definition Classes
    RDD
  58. def pipe(command: Seq[String], env: Map[String, String], printPipeContext: ((String) => Unit) => Unit, printRDDElement: (InternalRow, (String) => Unit) => Unit, separateWorkingDir: Boolean, bufferSize: Int, encoding: String): RDD[String]
    Definition Classes
    RDD
  59. def pipe(command: String, env: Map[String, String]): RDD[String]
    Definition Classes
    RDD
  60. def pipe(command: String): RDD[String]
    Definition Classes
    RDD
  61. final def preferredLocations(split: Partition): Seq[String]
    Definition Classes
    RDD
  62. def randomSplit(weights: Array[Double], seed: Long): Array[RDD[InternalRow]]
    Definition Classes
    RDD
  63. val readSchema: StructType
  64. def reduce(f: (InternalRow, InternalRow) => InternalRow): InternalRow
    Definition Classes
    RDD
  65. def repartition(numPartitions: Int)(implicit ord: Ordering[InternalRow]): RDD[InternalRow]
    Definition Classes
    RDD
  66. def sample(withReplacement: Boolean, fraction: Double, seed: Long): RDD[InternalRow]
    Definition Classes
    RDD
  67. def saveAsObjectFile(path: String): Unit
    Definition Classes
    RDD
  68. def saveAsTextFile(path: String, codec: Class[_ <: CompressionCodec]): Unit
    Definition Classes
    RDD
  69. def saveAsTextFile(path: String): Unit
    Definition Classes
    RDD
  70. def setName(_name: String): FileScanRDD.this.type
    Definition Classes
    RDD
  71. def sortBy[K](f: (InternalRow) => K, ascending: Boolean, numPartitions: Int)(implicit ord: Ordering[K], ctag: ClassTag[K]): RDD[InternalRow]
    Definition Classes
    RDD
  72. def sparkContext: SparkContext
    Definition Classes
    RDD
  73. def subtract(other: RDD[InternalRow], p: Partitioner)(implicit ord: Ordering[InternalRow]): RDD[InternalRow]
    Definition Classes
    RDD
  74. def subtract(other: RDD[InternalRow], numPartitions: Int): RDD[InternalRow]
    Definition Classes
    RDD
  75. def subtract(other: RDD[InternalRow]): RDD[InternalRow]
    Definition Classes
    RDD
  76. def take(num: Int): Array[InternalRow]
    Definition Classes
    RDD
  77. def takeOrdered(num: Int)(implicit ord: Ordering[InternalRow]): Array[InternalRow]
    Definition Classes
    RDD
  78. def takeSample(withReplacement: Boolean, num: Int, seed: Long): Array[InternalRow]
    Definition Classes
    RDD
  79. def toDebugString: String
    Definition Classes
    RDD
  80. def toJavaRDD(): JavaRDD[InternalRow]
    Definition Classes
    RDD
  81. def toLocalIterator: Iterator[InternalRow]
    Definition Classes
    RDD
  82. def toString(): String
    Definition Classes
    RDD → AnyRef → Any
  83. def top(num: Int)(implicit ord: Ordering[InternalRow]): Array[InternalRow]
    Definition Classes
    RDD
  84. def treeAggregate[U](zeroValue: U, seqOp: (U, InternalRow) => U, combOp: (U, U) => U, depth: Int, finalAggregateOnExecutor: Boolean)(implicit arg0: ClassTag[U]): U
    Definition Classes
    RDD
  85. def treeAggregate[U](zeroValue: U)(seqOp: (U, InternalRow) => U, combOp: (U, U) => U, depth: Int)(implicit arg0: ClassTag[U]): U
    Definition Classes
    RDD
  86. def treeReduce(f: (InternalRow, InternalRow) => InternalRow, depth: Int): InternalRow
    Definition Classes
    RDD
  87. def union(other: RDD[InternalRow]): RDD[InternalRow]
    Definition Classes
    RDD
  88. def unpersist(blocking: Boolean): FileScanRDD.this.type
    Definition Classes
    RDD
  89. def withResources(rp: ResourceProfile): FileScanRDD.this.type
    Definition Classes
    RDD
    Annotations
    @Experimental() @Since("3.1.0")
  90. def zip[U](other: RDD[U])(implicit arg0: ClassTag[U]): RDD[(InternalRow, U)]
    Definition Classes
    RDD
  91. def zipPartitions[B, C, D, V](rdd2: RDD[B], rdd3: RDD[C], rdd4: RDD[D])(f: (Iterator[InternalRow], Iterator[B], Iterator[C], Iterator[D]) => Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[D], arg3: ClassTag[V]): RDD[V]
    Definition Classes
    RDD
  92. def zipPartitions[B, C, D, V](rdd2: RDD[B], rdd3: RDD[C], rdd4: RDD[D], preservesPartitioning: Boolean)(f: (Iterator[InternalRow], Iterator[B], Iterator[C], Iterator[D]) => Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[D], arg3: ClassTag[V]): RDD[V]
    Definition Classes
    RDD
  93. def zipPartitions[B, C, V](rdd2: RDD[B], rdd3: RDD[C])(f: (Iterator[InternalRow], Iterator[B], Iterator[C]) => Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[V]): RDD[V]
    Definition Classes
    RDD
  94. def zipPartitions[B, C, V](rdd2: RDD[B], rdd3: RDD[C], preservesPartitioning: Boolean)(f: (Iterator[InternalRow], Iterator[B], Iterator[C]) => Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[V]): RDD[V]
    Definition Classes
    RDD
  95. def zipPartitions[B, V](rdd2: RDD[B])(f: (Iterator[InternalRow], Iterator[B]) => Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[V]): RDD[V]
    Definition Classes
    RDD
  96. def zipPartitions[B, V](rdd2: RDD[B], preservesPartitioning: Boolean)(f: (Iterator[InternalRow], Iterator[B]) => Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[V]): RDD[V]
    Definition Classes
    RDD
  97. def zipPartitionsWithEvaluator[U](rdd2: RDD[InternalRow], evaluatorFactory: PartitionEvaluatorFactory[InternalRow, U])(implicit arg0: ClassTag[U]): RDD[U]
    Definition Classes
    RDD
    Annotations
    @DeveloperApi() @Since("3.5.0")
  98. def zipWithIndex(): RDD[(InternalRow, Long)]
    Definition Classes
    RDD
  99. def zipWithUniqueId(): RDD[(InternalRow, Long)]
    Definition Classes
    RDD