case class InsertIntoHadoopFsRelationCommand(outputPath: Path, staticPartitions: TablePartitionSpec, ifPartitionNotExists: Boolean, partitionColumns: Seq[Attribute], bucketSpec: Option[BucketSpec], fileFormat: FileFormat, options: Map[String, String], query: LogicalPlan, mode: SaveMode, catalogTable: Option[CatalogTable], fileIndex: Option[FileIndex], outputColumnNames: Seq[String]) extends LogicalPlan with DataWritingCommand with Product with Serializable
A command for writing data to a HadoopFsRelation. Supports both overwriting and appending. Writing to dynamic partitions is also supported.
- staticPartitions
partial partitioning spec for write. This defines the scope of partition overwrites: when the spec is empty, all partitions are overwritten. When it covers a prefix of the partition keys, only partitions matching the prefix are overwritten.
- ifPartitionNotExists
If true, only write if the partition does not exist. Only valid for static partitions.
- Alphabetic
- By Inheritance
- InsertIntoHadoopFsRelationCommand
- Serializable
- DataWritingCommand
- UnaryCommand
- UnaryLike
- Command
- LogicalPlan
- Logging
- QueryPlanConstraints
- ConstraintHelper
- LogicalPlanDistinctKeys
- LogicalPlanStats
- AnalysisHelper
- QueryPlan
- SQLConfHelper
- TreeNode
- TreePatternBits
- Product
- Equals
- AnyRef
- Any
- Hide All
- Show All
- Public
- Protected
Instance Constructors
- new InsertIntoHadoopFsRelationCommand(outputPath: Path, staticPartitions: TablePartitionSpec, ifPartitionNotExists: Boolean, partitionColumns: Seq[Attribute], bucketSpec: Option[BucketSpec], fileFormat: FileFormat, options: Map[String, String], query: LogicalPlan, mode: SaveMode, catalogTable: Option[CatalogTable], fileIndex: Option[FileIndex], outputColumnNames: Seq[String])
- staticPartitions
partial partitioning spec for write. This defines the scope of partition overwrites: when the spec is empty, all partitions are overwritten. When it covers a prefix of the partition keys, only partitions matching the prefix are overwritten.
- ifPartitionNotExists
If true, only write if the partition does not exist. Only valid for static partitions.
Value Members
- final def !=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- final def ##: Int
- Definition Classes
- AnyRef → Any
- final def ==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- lazy val allAttributes: AttributeSeq
- Definition Classes
- QueryPlan
- def analyzed: Boolean
- Definition Classes
- AnalysisHelper
- def apply(number: Int): TreeNode[_]
- Definition Classes
- TreeNode
- def argString(maxFields: Int): String
- Definition Classes
- TreeNode
- def asCode: String
- Definition Classes
- TreeNode
- final def asInstanceOf[T0]: T0
- Definition Classes
- Any
- def assertNotAnalysisRule(): Unit
- Attributes
- protected
- Definition Classes
- AnalysisHelper
- def basicWriteJobStatsTracker(hadoopConf: Configuration): BasicWriteJobStatsTracker
- Definition Classes
- DataWritingCommand
- val bucketSpec: Option[BucketSpec]
- final lazy val canonicalized: LogicalPlan
- Definition Classes
- QueryPlan
- Annotations
- @transient()
- val catalogTable: Option[CatalogTable]
- final def child: LogicalPlan
- Definition Classes
- DataWritingCommand → UnaryLike
- final lazy val children: Seq[LogicalPlan]
- Definition Classes
- UnaryLike
- Annotations
- @transient()
- def childrenResolved: Boolean
- Definition Classes
- LogicalPlan
- def clone(): LogicalPlan
- Definition Classes
- AnalysisHelper → TreeNode → AnyRef
- def collect[B](pf: PartialFunction[LogicalPlan, B]): Seq[B]
- Definition Classes
- TreeNode
- def collectFirst[B](pf: PartialFunction[LogicalPlan, B]): Option[B]
- Definition Classes
- TreeNode
- def collectLeaves(): Seq[LogicalPlan]
- Definition Classes
- TreeNode
- def collectWithSubqueries[B](f: PartialFunction[LogicalPlan, B]): Seq[B]
- Definition Classes
- QueryPlan
- def conf: SQLConf
- Definition Classes
- SQLConfHelper
- lazy val constraints: ExpressionSet
- Definition Classes
- QueryPlanConstraints
- def constructIsNotNullConstraints(constraints: ExpressionSet, output: Seq[Attribute]): ExpressionSet
- Definition Classes
- ConstraintHelper
- final def containsAllPatterns(patterns: TreePattern*): Boolean
- Definition Classes
- TreePatternBits
- final def containsAnyPattern(patterns: TreePattern*): Boolean
- Definition Classes
- TreePatternBits
- lazy val containsChild: Set[TreeNode[_]]
- Definition Classes
- TreeNode
- final def containsPattern(t: TreePattern): Boolean
- Definition Classes
- TreePatternBits
- Annotations
- @inline()
- def copyTagsFrom(other: LogicalPlan): Unit
- Definition Classes
- TreeNode
- lazy val deterministic: Boolean
- Definition Classes
- QueryPlan
- lazy val distinctKeys: Set[ExpressionSet]
- Definition Classes
- LogicalPlanDistinctKeys
- def doCanonicalize(): LogicalPlan
- Attributes
- protected
- Definition Classes
- QueryPlan
- final def eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- def exists(f: (LogicalPlan) => Boolean): Boolean
- Definition Classes
- TreeNode
- final def expressions: Seq[Expression]
- Definition Classes
- QueryPlan
- def fastEquals(other: TreeNode[_]): Boolean
- Definition Classes
- TreeNode
- val fileFormat: FileFormat
- val fileIndex: Option[FileIndex]
- def finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.Throwable])
- def find(f: (LogicalPlan) => Boolean): Option[LogicalPlan]
- Definition Classes
- TreeNode
- def flatMap[A](f: (LogicalPlan) => TraversableOnce[A]): Seq[A]
- Definition Classes
- TreeNode
- def foreach(f: (LogicalPlan) => Unit): Unit
- Definition Classes
- TreeNode
- def foreachUp(f: (LogicalPlan) => Unit): Unit
- Definition Classes
- TreeNode
- def formattedNodeName: String
- Attributes
- protected
- Definition Classes
- QueryPlan
- def generateTreeString(depth: Int, lastChildren: Seq[Boolean], append: (String) => Unit, verbose: Boolean, prefix: String, addSuffix: Boolean, maxFields: Int, printNodeId: Boolean, indent: Int): Unit
- Definition Classes
- TreeNode
- final def getClass(): Class[_ <: AnyRef]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
- def getDefaultTreePatternBits: BitSet
- Attributes
- protected
- Definition Classes
- TreeNode
- def getTagValue[T](tag: TreeNodeTag[T]): Option[T]
- Definition Classes
- TreeNode
- def hashCode(): Int
- Definition Classes
- TreeNode → AnyRef → Any
- val ifPartitionNotExists: Boolean
- def inferAdditionalConstraints(constraints: ExpressionSet): ExpressionSet
- Definition Classes
- ConstraintHelper
- def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
- Attributes
- protected
- Definition Classes
- Logging
- def initializeLogIfNecessary(isInterpreter: Boolean): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def innerChildren: Seq[QueryPlan[_]]
- Definition Classes
- QueryPlan → TreeNode
- def inputSet: AttributeSet
- Definition Classes
- QueryPlan
- final def invalidateStatsCache(): Unit
- Definition Classes
- LogicalPlanStats
- def isCanonicalizedPlan: Boolean
- Attributes
- protected
- Definition Classes
- QueryPlan
- final def isInstanceOf[T0]: Boolean
- Definition Classes
- Any
- def isRuleIneffective(ruleId: RuleId): Boolean
- Attributes
- protected
- Definition Classes
- TreeNode
- def isStreaming: Boolean
- Definition Classes
- LogicalPlan
- def isTraceEnabled(): Boolean
- Attributes
- protected
- Definition Classes
- Logging
- def jsonFields: List[JField]
- Attributes
- protected
- Definition Classes
- TreeNode
- final def legacyWithNewChildren(newChildren: Seq[LogicalPlan]): LogicalPlan
- Attributes
- protected
- Definition Classes
- TreeNode
- def log: Logger
- Attributes
- protected
- Definition Classes
- Logging
- def logDebug(msg: => String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logDebug(msg: => String): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logError(msg: => String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logError(msg: => String): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logInfo(msg: => String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logInfo(msg: => String): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logName: String
- Attributes
- protected
- Definition Classes
- Logging
- def logTrace(msg: => String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logTrace(msg: => String): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logWarning(msg: => String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def logWarning(msg: => String): Unit
- Attributes
- protected
- Definition Classes
- Logging
- def makeCopy(newArgs: Array[AnyRef]): LogicalPlan
- Definition Classes
- TreeNode
- def map[A](f: (LogicalPlan) => A): Seq[A]
- Definition Classes
- TreeNode
- final def mapChildren(f: (LogicalPlan) => LogicalPlan): LogicalPlan
- Definition Classes
- UnaryLike
- def mapExpressions(f: (Expression) => Expression): InsertIntoHadoopFsRelationCommand.this.type
- Definition Classes
- QueryPlan
- def mapProductIterator[B](f: (Any) => B)(implicit arg0: ClassTag[B]): Array[B]
- Attributes
- protected
- Definition Classes
- TreeNode
- def markRuleAsIneffective(ruleId: RuleId): Unit
- Attributes
- protected
- Definition Classes
- TreeNode
- def maxRows: Option[Long]
- Definition Classes
- LogicalPlan
- def maxRowsPerPartition: Option[Long]
- Definition Classes
- LogicalPlan
- def metadataOutput: Seq[Attribute]
- Definition Classes
- LogicalPlan
- lazy val metrics: Map[String, SQLMetric]
- Definition Classes
- DataWritingCommand
- final def missingInput: AttributeSet
- Definition Classes
- QueryPlan
- val mode: SaveMode
- final def ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- def nodeName: String
- Definition Classes
- TreeNode
- final val nodePatterns: Seq[TreePattern]
- Definition Classes
- Command → TreeNode
- final def notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
- final def notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
- def numberedTreeString: String
- Definition Classes
- TreeNode
- val options: Map[String, String]
- val origin: Origin
- Definition Classes
- TreeNode
- def otherCopyArgs: Seq[AnyRef]
- Attributes
- protected
- Definition Classes
- TreeNode
- def output: Seq[Attribute]
- Definition Classes
- Command → QueryPlan
- val outputColumnNames: Seq[String]
- Definition Classes
- InsertIntoHadoopFsRelationCommand → DataWritingCommand
- def outputColumns: Seq[Attribute]
- Definition Classes
- DataWritingCommand
- def outputOrdering: Seq[SortOrder]
- Definition Classes
- LogicalPlan
- val outputPath: Path
- lazy val outputSet: AttributeSet
- Definition Classes
- QueryPlan
- Annotations
- @transient()
- def p(number: Int): LogicalPlan
- Definition Classes
- TreeNode
- val partitionColumns: Seq[Attribute]
- def prettyJson: String
- Definition Classes
- TreeNode
- def printSchema(): Unit
- Definition Classes
- QueryPlan
- def producedAttributes: AttributeSet
- Definition Classes
- Command → QueryPlan
- def productElementNames: Iterator[String]
- Definition Classes
- Product
- val query: LogicalPlan
The input query plan that produces the data to be written.
The input query plan that produces the data to be written. IMPORTANT: the input query plan MUST be analyzed, so that we can carry its output columns to org.apache.spark.sql.execution.datasources.FileFormatWriter.
- Definition Classes
- InsertIntoHadoopFsRelationCommand → DataWritingCommand
- lazy val references: AttributeSet
- Definition Classes
- QueryPlan
- Annotations
- @transient()
- def refresh(): Unit
- Definition Classes
- LogicalPlan
- def resolve(nameParts: Seq[String], resolver: Resolver): Option[NamedExpression]
- Definition Classes
- LogicalPlan
- def resolve(schema: StructType, resolver: Resolver): Seq[Attribute]
- Definition Classes
- LogicalPlan
- def resolveChildren(nameParts: Seq[String], resolver: Resolver): Option[NamedExpression]
- Definition Classes
- LogicalPlan
- def resolveExpressions(r: PartialFunction[Expression, Expression]): LogicalPlan
- Definition Classes
- AnalysisHelper
- def resolveExpressionsWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[Expression, Expression]): LogicalPlan
- Definition Classes
- AnalysisHelper
- def resolveOperators(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
- Definition Classes
- AnalysisHelper
- def resolveOperatorsDown(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
- Definition Classes
- AnalysisHelper
- def resolveOperatorsDownWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
- Definition Classes
- AnalysisHelper
- def resolveOperatorsUp(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
- Definition Classes
- AnalysisHelper
- def resolveOperatorsUpWithNewOutput(rule: PartialFunction[LogicalPlan, (LogicalPlan, Seq[(Attribute, Attribute)])]): LogicalPlan
- Definition Classes
- AnalysisHelper
- def resolveOperatorsUpWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
- Definition Classes
- AnalysisHelper
- def resolveOperatorsWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
- Definition Classes
- AnalysisHelper
- def resolveQuoted(name: String, resolver: Resolver): Option[NamedExpression]
- Definition Classes
- LogicalPlan
- lazy val resolved: Boolean
- Definition Classes
- LogicalPlan
- def rewriteAttrs(attrMap: AttributeMap[Attribute]): LogicalPlan
- Definition Classes
- QueryPlan
- def run(sparkSession: SparkSession, child: SparkPlan): Seq[Row]
- Definition Classes
- InsertIntoHadoopFsRelationCommand → DataWritingCommand
- def sameOutput(other: LogicalPlan): Boolean
- Definition Classes
- LogicalPlan
- final def sameResult(other: LogicalPlan): Boolean
- Definition Classes
- QueryPlan
- lazy val schema: StructType
- Definition Classes
- QueryPlan
- def schemaString: String
- Definition Classes
- QueryPlan
- final def semanticHash(): Int
- Definition Classes
- QueryPlan
- def setTagValue[T](tag: TreeNodeTag[T], value: T): Unit
- Definition Classes
- TreeNode
- def simpleString(maxFields: Int): String
- Definition Classes
- QueryPlan → TreeNode
- def simpleStringWithNodeId(): String
- Definition Classes
- QueryPlan → TreeNode
- def statePrefix: String
- Attributes
- protected
- Definition Classes
- LogicalPlan → QueryPlan
- val staticPartitions: TablePartitionSpec
- def stats: Statistics
- Definition Classes
- Command → LogicalPlanStats
- val statsCache: Option[Statistics]
- Attributes
- protected
- Definition Classes
- LogicalPlanStats
- def stringArgs: Iterator[Any]
- Attributes
- protected
- Definition Classes
- TreeNode
- lazy val subqueries: Seq[LogicalPlan]
- Definition Classes
- QueryPlan
- Annotations
- @transient()
- def subqueriesAll: Seq[LogicalPlan]
- Definition Classes
- QueryPlan
- final def synchronized[T0](arg0: => T0): T0
- Definition Classes
- AnyRef
- def toJSON: String
- Definition Classes
- TreeNode
- def toString(): String
- Definition Classes
- TreeNode → AnyRef → Any
- def transform(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
- Definition Classes
- TreeNode
- def transformAllExpressions(rule: PartialFunction[Expression, Expression]): InsertIntoHadoopFsRelationCommand.this.type
- Definition Classes
- QueryPlan
- def transformAllExpressionsWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[Expression, Expression]): InsertIntoHadoopFsRelationCommand.this.type
- Definition Classes
- AnalysisHelper → QueryPlan
- def transformDown(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
- Definition Classes
- TreeNode
- def transformDownWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
- Definition Classes
- AnalysisHelper → TreeNode
- def transformDownWithSubqueries(f: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
- Definition Classes
- QueryPlan
- def transformDownWithSubqueriesAndPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(f: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
- Definition Classes
- QueryPlan
- def transformExpressions(rule: PartialFunction[Expression, Expression]): InsertIntoHadoopFsRelationCommand.this.type
- Definition Classes
- QueryPlan
- def transformExpressionsDown(rule: PartialFunction[Expression, Expression]): InsertIntoHadoopFsRelationCommand.this.type
- Definition Classes
- QueryPlan
- def transformExpressionsDownWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[Expression, Expression]): InsertIntoHadoopFsRelationCommand.this.type
- Definition Classes
- QueryPlan
- def transformExpressionsUp(rule: PartialFunction[Expression, Expression]): InsertIntoHadoopFsRelationCommand.this.type
- Definition Classes
- QueryPlan
- def transformExpressionsUpWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[Expression, Expression]): InsertIntoHadoopFsRelationCommand.this.type
- Definition Classes
- QueryPlan
- def transformExpressionsWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[Expression, Expression]): InsertIntoHadoopFsRelationCommand.this.type
- Definition Classes
- QueryPlan
- def transformUp(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
- Definition Classes
- TreeNode
- def transformUpWithBeforeAndAfterRuleOnChildren(cond: (LogicalPlan) => Boolean, ruleId: RuleId)(rule: PartialFunction[(LogicalPlan, LogicalPlan), LogicalPlan]): LogicalPlan
- Definition Classes
- TreeNode
- def transformUpWithNewOutput(rule: PartialFunction[LogicalPlan, (LogicalPlan, Seq[(Attribute, Attribute)])], skipCond: (LogicalPlan) => Boolean, canGetOutput: (LogicalPlan) => Boolean): LogicalPlan
- Definition Classes
- AnalysisHelper → QueryPlan
- def transformUpWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
- Definition Classes
- AnalysisHelper → TreeNode
- def transformUpWithSubqueries(f: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
- Definition Classes
- QueryPlan
- def transformWithPruning(cond: (TreePatternBits) => Boolean, ruleId: RuleId)(rule: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
- Definition Classes
- TreeNode
- def transformWithSubqueries(f: PartialFunction[LogicalPlan, LogicalPlan]): LogicalPlan
- Definition Classes
- QueryPlan
- lazy val treePatternBits: BitSet
- Definition Classes
- QueryPlan → TreeNode → TreePatternBits
- def treeString(append: (String) => Unit, verbose: Boolean, addSuffix: Boolean, maxFields: Int, printOperatorId: Boolean): Unit
- Definition Classes
- TreeNode
- final def treeString(verbose: Boolean, addSuffix: Boolean, maxFields: Int, printOperatorId: Boolean): String
- Definition Classes
- TreeNode
- final def treeString: String
- Definition Classes
- TreeNode
- def unsetTagValue[T](tag: TreeNodeTag[T]): Unit
- Definition Classes
- TreeNode
- def updateOuterReferencesInSubquery(plan: LogicalPlan, attrMap: AttributeMap[Attribute]): LogicalPlan
- Definition Classes
- AnalysisHelper → QueryPlan
- lazy val validConstraints: ExpressionSet
- Attributes
- protected
- Definition Classes
- QueryPlanConstraints
- def verboseString(maxFields: Int): String
- Definition Classes
- QueryPlan → TreeNode
- def verboseStringWithOperatorId(): String
- Definition Classes
- QueryPlan
- def verboseStringWithSuffix(maxFields: Int): String
- Definition Classes
- LogicalPlan → TreeNode
- final def wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException])
- final def wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException])
- final def wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException]) @native()
- def withNewChildInternal(newChild: LogicalPlan): InsertIntoHadoopFsRelationCommand
- Attributes
- protected
- Definition Classes
- InsertIntoHadoopFsRelationCommand → UnaryLike
- final def withNewChildren(newChildren: Seq[LogicalPlan]): LogicalPlan
- Definition Classes
- TreeNode
- final def withNewChildrenInternal(newChildren: IndexedSeq[LogicalPlan]): LogicalPlan
- Definition Classes
- UnaryLike