Packages

case class LogicalRDD(output: Seq[Attribute], rdd: RDD[InternalRow], outputPartitioning: Partitioning = UnknownPartitioning(0), outputOrdering: Seq[SortOrder] = Nil, isStreaming: Boolean = false)(session: SparkSession, originStats: Option[Statistics] = None, originConstraints: Option[ExpressionSet] = None) extends LogicalPlan with LeafNode with MultiInstanceRelation with Product with Serializable

Logical plan node for scanning data from an RDD of InternalRow.

It is advised to set the field originStats and originConstraints if the RDD is directly built from DataFrame, so that Spark can make better optimizations.

Linear Supertypes
Serializable, MultiInstanceRelation, LeafNode, LeafLike[LogicalPlan], LogicalPlan, Logging, QueryPlanConstraints, ConstraintHelper, LogicalPlanDistinctKeys, LogicalPlanStats, AnalysisHelper, QueryPlan[LogicalPlan], SQLConfHelper, TreeNode[LogicalPlan], WithOrigin, TreePatternBits, Product, Equals, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. LogicalRDD
  2. Serializable
  3. MultiInstanceRelation
  4. LeafNode
  5. LeafLike
  6. LogicalPlan
  7. Logging
  8. QueryPlanConstraints
  9. ConstraintHelper
  10. LogicalPlanDistinctKeys
  11. LogicalPlanStats
  12. AnalysisHelper
  13. QueryPlan
  14. SQLConfHelper
  15. TreeNode
  16. WithOrigin
  17. TreePatternBits
  18. Product
  19. Equals
  20. AnyRef
  21. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. Protected

Instance Constructors

  1. new LogicalRDD(output: Seq[Attribute], rdd: RDD[InternalRow], outputPartitioning: Partitioning = UnknownPartitioning(0), outputOrdering: Seq[SortOrder] = Nil, isStreaming: Boolean = false)(session: SparkSession, originStats: Option[Statistics] = None, originConstraints: Option[ExpressionSet] = None)

Value Members

  1. lazy val allAttributes: AttributeSeq
    Definition Classes
    QueryPlan
  2. def analyzed: Boolean
    Definition Classes
    AnalysisHelper
  3. def apply(number: Int): TreeNode[_]
    Definition Classes
    TreeNode
  4. def argString(maxFields: Int): String
    Definition Classes
    TreeNode
  5. def asCode: String
    Definition Classes
    TreeNode
  6. final lazy val canonicalized: LogicalPlan
    Definition Classes
    QueryPlan
    Annotations
    @transient()
  7. final def children: Seq[LogicalPlan]
    Definition Classes
    LeafLike
  8. def childrenResolved: Boolean
    Definition Classes
    LogicalPlan
  9. def clone(): LogicalPlan
    Definition Classes
    AnalysisHelper → TreeNode → AnyRef
  10. def collect[B](pf: PartialFunction[LogicalPlan, B]): Seq[B]
    Definition Classes
    TreeNode
  11. def collectFirst[B](pf: PartialFunction[LogicalPlan, B]): Option[B]
    Definition Classes
    TreeNode
  12. def collectLeaves(): Seq[LogicalPlan]
    Definition Classes
    TreeNode
  13. def collectWithSubqueries[B](f: PartialFunction[LogicalPlan, B]): Seq[B]
    Definition Classes
    QueryPlan
  14. def computeStats(): Statistics
    Definition Classes
    LogicalRDD → LeafNode
  15. def conf: SQLConf
    Definition Classes
    SQLConfHelper
  16. lazy val constraints: ExpressionSet
    Definition Classes
    LogicalRDD → QueryPlanConstraints
  17. def constructIsNotNullConstraints(constraints: ExpressionSet, output: Seq[Attribute]): ExpressionSet
    Definition Classes
    ConstraintHelper
  18. final def containsAllPatterns(patterns: TreePattern*): Boolean
    Definition Classes
    TreePatternBits
  19. final def containsAnyPattern(patterns: TreePattern*): Boolean
    Definition Classes
    TreePatternBits
  20. lazy val containsChild: Set[TreeNode[_]]
    Definition Classes
    TreeNode
  21. final def containsPattern(t: TreePattern): Boolean
    Definition Classes
    TreePatternBits
    Annotations
    @inline()
  22. def copyTagsFrom(other: LogicalPlan): Unit
    Definition Classes
    TreeNode
  23. lazy val deterministic: Boolean
    Definition Classes
    QueryPlan
  24. lazy val distinctKeys: Set[ExpressionSet]
    Definition Classes
    LogicalPlanDistinctKeys
  25. def exists(f: (LogicalPlan) => Boolean): Boolean
    Definition Classes
    TreeNode
  26. final def expressions: Seq[Expression]
    Definition Classes
    QueryPlan
  27. def fastEquals(other: TreeNode[_]): Boolean
    Definition Classes
    TreeNode
  28. def find(f: (LogicalPlan) => Boolean): Option[LogicalPlan]
    Definition Classes
    TreeNode
  29. def flatMap[A](f: (LogicalPlan) => TraversableOnce[A]): Seq[A]
    Definition Classes
    TreeNode
  30. def foreach(f: (LogicalPlan) => Unit): Unit
    Definition Classes
    TreeNode
  31. def foreachUp(f: (LogicalPlan) => Unit): Unit
    Definition Classes
    TreeNode
  32. def generateTreeString(depth: Int, lastChildren: ArrayList[Boolean], append: (String) => Unit, verbose: Boolean, prefix: String, addSuffix: Boolean, maxFields: Int, printNodeId: Boolean, indent: Int): Unit
    Definition Classes
    TreeNode
  33. def getMetadataAttributeByName(name: String): AttributeReference
    Definition Classes
    LogicalPlan
  34. def getMetadataAttributeByNameOpt(name: String): Option[AttributeReference]
    Definition Classes
    LogicalPlan
  35. def getTagValue[T](tag: TreeNodeTag[T]): Option[T]
    Definition Classes
    TreeNode
  36. def hashCode(): Int
    Definition Classes
    TreeNode → AnyRef → Any
  37. def inferAdditionalConstraints(constraints: ExpressionSet): ExpressionSet
    Definition Classes
    ConstraintHelper
  38. def innerChildren: Seq[QueryPlan[_]]
    Definition Classes
    QueryPlan → TreeNode
  39. def inputSet: AttributeSet
    Definition Classes
    QueryPlan
  40. final def invalidateStatsCache(): Unit
    Definition Classes
    LogicalPlanStats
  41. val isStreaming: Boolean
    Definition Classes
    LogicalRDD → LogicalPlan
  42. def makeCopy(newArgs: Array[AnyRef]): LogicalPlan
    Definition Classes
    TreeNode
  43. def map[A](f: (LogicalPlan) => A): Seq[A]
    Definition Classes
    TreeNode
  44. final def mapChildren(f: (LogicalPlan) => LogicalPlan): LogicalPlan
    Definition Classes
    LeafLike
  45. def mapExpressions(f: (Expression) => Expression): LogicalRDD.this.type
    Definition Classes
    QueryPlan
  46. def maxRows: Option[Long]
    Definition Classes
    LogicalPlan
  47. def maxRowsPerPartition: Option[Long]
    Definition Classes
    LogicalPlan
  48. def metadataOutput: Seq[Attribute]
    Definition Classes
    LogicalPlan
  49. final def missingInput: AttributeSet
    Definition Classes
    QueryPlan
  50. def multiTransformDown(rule: PartialFunction[LogicalPlan, Seq[LogicalPlan]]): Stream[LogicalPlan]
    Definition Classes
    TreeNode
  51. def multiTransformDownWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[LogicalPlan, Seq[LogicalPlan]]): Stream[LogicalPlan]
    Definition Classes
    TreeNode
  52. def newInstance(): LogicalRDD.this.type
    Definition Classes
    LogicalRDD → MultiInstanceRelation
  53. def nodeName: String
    Definition Classes
    TreeNode
  54. def numberedTreeString: String
    Definition Classes
    TreeNode
  55. val origin: Origin
    Definition Classes
    TreeNode → WithOrigin
  56. val output: Seq[Attribute]
    Definition Classes
    LogicalRDD → QueryPlan
  57. val outputOrdering: Seq[SortOrder]
    Definition Classes
    LogicalRDD → QueryPlan
  58. val outputPartitioning: Partitioning
  59. lazy val outputSet: AttributeSet
    Definition Classes
    QueryPlan
    Annotations
    @transient()
  60. def p(number: Int): LogicalPlan
    Definition Classes
    TreeNode
  61. def prettyJson: String
    Definition Classes
    TreeNode
  62. def printSchema(): Unit
    Definition Classes
    QueryPlan
  63. def producedAttributes: AttributeSet
    Definition Classes
    LeafNode → QueryPlan
  64. def productElementNames: Iterator[String]
    Definition Classes
    Product
  65. val rdd: RDD[InternalRow]
  66. lazy val references: AttributeSet
    Definition Classes
    QueryPlan
    Annotations
    @transient()
  67. def refresh(): Unit
    Definition Classes
    LogicalPlan
  68. def resolve(nameParts: Seq[String], resolver: Resolver): Option[NamedExpression]
    Definition Classes
    LogicalPlan
  69. def resolve(schema: StructType, resolver: Resolver): Seq[Attribute]
    Definition Classes
    LogicalPlan
  70. def resolveChildren(nameParts: Seq[String], resolver: Resolver): Option[NamedExpression]
    Definition Classes
    LogicalPlan
  71. def resolveExpressions(r: PartialFunction[Expression, Expression]): LogicalPlan
    Definition Classes
    AnalysisHelper
  72. def resolveExpressionsWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[Expression, Expression]): LogicalPlan
    Definition Classes
    AnalysisHelper
  73. def resolveOperators(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
    Definition Classes
    AnalysisHelper
  74. def resolveOperatorsDown(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
    Definition Classes
    AnalysisHelper
  75. def resolveOperatorsDownWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
    Definition Classes
    AnalysisHelper
  76. def resolveOperatorsUp(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
    Definition Classes
    AnalysisHelper
  77. def resolveOperatorsUpWithNewOutput(rule: PartialFunction[LogicalPlan, (LogicalPlan, Seq[(Attribute, Attribute)])]): LogicalPlan
    Definition Classes
    AnalysisHelper
  78. def resolveOperatorsUpWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
    Definition Classes
    AnalysisHelper
  79. def resolveOperatorsWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
    Definition Classes
    AnalysisHelper
  80. def resolveQuoted(name: String, resolver: Resolver): Option[NamedExpression]
    Definition Classes
    LogicalPlan
  81. lazy val resolved: Boolean
    Definition Classes
    LogicalPlan
  82. def rewriteAttrs(attrMap: AttributeMap[Attribute]): LogicalPlan
    Definition Classes
    QueryPlan
  83. def sameOutput(other: LogicalPlan): Boolean
    Definition Classes
    LogicalPlan
  84. final def sameResult(other: LogicalPlan): Boolean
    Definition Classes
    QueryPlan
  85. lazy val schema: StructType
    Definition Classes
    QueryPlan
  86. def schemaString: String
    Definition Classes
    QueryPlan
  87. final def semanticHash(): Int
    Definition Classes
    QueryPlan
  88. def setTagValue[T](tag: TreeNodeTag[T], value: T): Unit
    Definition Classes
    TreeNode
  89. def simpleString(maxFields: Int): String
    Definition Classes
    QueryPlan → TreeNode
  90. def simpleStringWithNodeId(): String
    Definition Classes
    QueryPlan → TreeNode
  91. def stats: Statistics
    Definition Classes
    LogicalPlanStats
  92. lazy val subqueries: Seq[LogicalPlan]
    Definition Classes
    QueryPlan
    Annotations
    @transient()
  93. def subqueriesAll: Seq[LogicalPlan]
    Definition Classes
    QueryPlan
  94. def toJSON: String
    Definition Classes
    TreeNode
  95. def toString(): String
    Definition Classes
    TreeNode → AnyRef → Any
  96. def transform(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
    Definition Classes
    TreeNode
  97. def transformAllExpressions(rule: PartialFunction[Expression, Expression]): LogicalRDD.this.type
    Definition Classes
    QueryPlan
  98. def transformAllExpressionsWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[Expression, Expression]): LogicalRDD.this.type
    Definition Classes
    AnalysisHelper → QueryPlan
  99. def transformAllExpressionsWithSubqueries(rule: PartialFunction[Expression, Expression]): LogicalRDD.this.type
    Definition Classes
    QueryPlan
  100. def transformDown(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
    Definition Classes
    TreeNode
  101. def transformDownWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
    Definition Classes
    AnalysisHelper → TreeNode
  102. def transformDownWithSubqueries(f: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
    Definition Classes
    QueryPlan
  103. def transformDownWithSubqueriesAndPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(f: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
    Definition Classes
    QueryPlan
  104. def transformExpressions(rule: PartialFunction[Expression, Expression]): LogicalRDD.this.type
    Definition Classes
    QueryPlan
  105. def transformExpressionsDown(rule: PartialFunction[Expression, Expression]): LogicalRDD.this.type
    Definition Classes
    QueryPlan
  106. def transformExpressionsDownWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[Expression, Expression]): LogicalRDD.this.type
    Definition Classes
    QueryPlan
  107. def transformExpressionsUp(rule: PartialFunction[Expression, Expression]): LogicalRDD.this.type
    Definition Classes
    QueryPlan
  108. def transformExpressionsUpWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[Expression, Expression]): LogicalRDD.this.type
    Definition Classes
    QueryPlan
  109. def transformExpressionsWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[Expression, Expression]): LogicalRDD.this.type
    Definition Classes
    QueryPlan
  110. def transformUp(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
    Definition Classes
    TreeNode
  111. def transformUpWithBeforeAndAfterRuleOnChildren(cond: (LogicalPlan) => Boolean, ruleId: RuleId)(rule: PartialFunction[(LogicalPlan, LogicalPlan), LogicalPlan]): LogicalPlan
    Definition Classes
    TreeNode
  112. def transformUpWithNewOutput(rule: PartialFunction[LogicalPlan, (LogicalPlan, Seq[(Attribute, Attribute)])], skipCond: (LogicalPlan) => Boolean, canGetOutput: (LogicalPlan) => Boolean): LogicalPlan
    Definition Classes
    AnalysisHelper → QueryPlan
  113. def transformUpWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
    Definition Classes
    AnalysisHelper → TreeNode
  114. def transformUpWithSubqueries(f: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
    Definition Classes
    QueryPlan
  115. def transformWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
    Definition Classes
    TreeNode
  116. def transformWithSubqueries(f: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
    Definition Classes
    QueryPlan
  117. lazy val treePatternBits: BitSet
    Definition Classes
    QueryPlan → TreeNode → TreePatternBits
  118. def treeString(append: (String) => Unit, verbose: Boolean, addSuffix: Boolean, maxFields: Int, printOperatorId: Boolean): Unit
    Definition Classes
    TreeNode
  119. final def treeString(verbose: Boolean, addSuffix: Boolean, maxFields: Int, printOperatorId: Boolean): String
    Definition Classes
    TreeNode
  120. final def treeString: String
    Definition Classes
    TreeNode
  121. def unsetTagValue[T](tag: TreeNodeTag[T]): Unit
    Definition Classes
    TreeNode
  122. def updateOuterReferencesInSubquery(plan: LogicalPlan, attrMap: AttributeMap[Attribute]): LogicalPlan
    Definition Classes
    AnalysisHelper → QueryPlan
  123. def verboseString(maxFields: Int): String
    Definition Classes
    QueryPlan → TreeNode
  124. def verboseStringWithOperatorId(): String
    Definition Classes
    QueryPlan
  125. def verboseStringWithSuffix(maxFields: Int): String
    Definition Classes
    LogicalPlan → TreeNode
  126. final def withNewChildren(newChildren: Seq[LogicalPlan]): LogicalPlan
    Definition Classes
    TreeNode
  127. def withNewChildrenInternal(newChildren: IndexedSeq[LogicalPlan]): LogicalPlan
    Definition Classes
    LeafLike