DeltaTable

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. def alias(alias: String): DeltaTable

   

   :: Evolving ::

   :: Evolving ::

   Apply an alias to the DeltaTable. This is similar to Dataset.as(alias) or SQL tableName AS alias.

   Annotations

   @Evolving()

   Since

   0.3.0

5. def as(alias: String): DeltaTable

   

   :: Evolving ::

   :: Evolving ::

   Apply an alias to the DeltaTable. This is similar to Dataset.as(alias) or SQL tableName AS alias.

   Annotations

   @Evolving()

   Since

   0.3.0

6. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

7. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

8. def delete(): Unit

   

   :: Evolving ::

   :: Evolving ::

   Delete data from the table.

   Annotations

   @Evolving()

   Since

   0.3.0

9. def delete(condition: Column): Unit

   

   :: Evolving ::

   :: Evolving ::

   Delete data from the table that match the given condition.

   condition

   Boolean SQL expression

   Annotations

   @Evolving()

   Since

   0.3.0

10. def delete(condition: String): Unit

    

    :: Evolving ::

    :: Evolving ::

    Delete data from the table that match the given condition.

    condition

    Boolean SQL expression

    Annotations

    @Evolving()

    Since

    0.3.0

11. final def eq(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

12. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

13. def executeDelete(condition: Option[Expression]): Unit

    

    Attributes

    protected

    Definition Classes

    DeltaTableOperations

14. def executeGenerate(tblIdentifier: String, mode: String): Unit

    

    Attributes

    protected

    Definition Classes

    DeltaTableOperations

15. def executeHistory(deltaLog: DeltaLog, limit: Option[Int]): DataFrame

    

    Attributes

    protected

    Definition Classes

    DeltaTableOperations

16. def executeUpdate(set: Map[String, Column], condition: Option[Column]): Unit

    

    Attributes

    protected

    Definition Classes

    DeltaTableOperations

17. def executeVacuum(deltaLog: DeltaLog, retentionHours: Option[Double]): DataFrame

    

    Attributes

    protected

    Definition Classes

    DeltaTableOperations

18. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

19. def generate(mode: String): Unit

    

    :: Evolving ::

    :: Evolving ::

    Generate a manifest for the given Delta Table

    mode

    Specifies the mode for the generation of the manifest. The valid modes are as follows (not case sensitive):

    • "symlink_format_manifest" : This will generate manifests in symlink format for Presto and Athena read support. See the online documentation for more information.

    Annotations

    @Evolving()

    Since

    0.5.0

20. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

21. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

22. def history(): DataFrame

    

    :: Evolving ::

    :: Evolving ::

    Get the information available commits on this table as a Spark DataFrame. The information is in reverse chronological order.

    Annotations

    @Evolving()

    Since

    0.3.0

23. def history(limit: Int): DataFrame

    

    :: Evolving ::

    :: Evolving ::

    Get the information of the latest limit commits on this table as a Spark DataFrame. The information is in reverse chronological order.

    limit

    The number of previous commands to get history for

    Annotations

    @Evolving()

    Since

    0.3.0

24. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

25. def makeUpdateTable(target: DeltaTable, onCondition: Option[Column], setColumns: Seq[(String, Column)]): DeltaUpdateTable

    

    Attributes

    protected

    Definition Classes

    DeltaTableOperations

26. def merge(source: DataFrame, condition: Column): DeltaMergeBuilder

    

    :: Evolving ::

    :: Evolving ::

    Merge data from the source DataFrame based on the given merge condition. This returns a DeltaMergeBuilder object that can be used to specify the update, delete, or insert actions to be performed on rows based on whether the rows matched the condition or not.

    See the DeltaMergeBuilder for a full description of this operation and what combinations of update, delete and insert operations are allowed.

    Scala example to update a key-value Delta table with new key-values from a source DataFrame:

    deltaTable
     .as("target")
     .merge(
       source.as("source"),
       "target.key = source.key")
     .whenMatched
     .updateExpr(Map(
       "value" -> "source.value"))
     .whenNotMatched
     .insertExpr(Map(
       "key" -> "source.key",
       "value" -> "source.value"))
     .execute()

    Java example to update a key-value Delta table with new key-values from a source DataFrame:

    deltaTable
     .as("target")
     .merge(
       source.as("source"),
       "target.key = source.key")
     .whenMatched
     .updateExpr(
        new HashMap<String, String>() {{
          put("value" -> "source.value")
        }})
     .whenNotMatched
     .insertExpr(
        new HashMap<String, String>() {{
         put("key", "source.key");
         put("value", "source.value");
       }})
     .execute()

    source

    source Dataframe to be merged.

    condition

    boolean expression as a Column object

    Annotations

    @Evolving()

    Since

    0.3.0

27. def merge(source: DataFrame, condition: String): DeltaMergeBuilder

    

    :: Evolving ::

    :: Evolving ::

    Merge data from the source DataFrame based on the given merge condition. This returns a DeltaMergeBuilder object that can be used to specify the update, delete, or insert actions to be performed on rows based on whether the rows matched the condition or not.

    See the DeltaMergeBuilder for a full description of this operation and what combinations of update, delete and insert operations are allowed.

    Scala example to update a key-value Delta table with new key-values from a source DataFrame:

    deltaTable
     .as("target")
     .merge(
       source.as("source"),
       "target.key = source.key")
     .whenMatched
     .updateExpr(Map(
       "value" -> "source.value"))
     .whenNotMatched
     .insertExpr(Map(
       "key" -> "source.key",
       "value" -> "source.value"))
     .execute()

    Java example to update a key-value Delta table with new key-values from a source DataFrame:

    deltaTable
     .as("target")
     .merge(
       source.as("source"),
       "target.key = source.key")
     .whenMatched
     .updateExpr(
        new HashMap<String, String>() {{
          put("value" -> "source.value");
        }})
     .whenNotMatched
     .insertExpr(
        new HashMap<String, String>() {{
         put("key", "source.key");
         put("value", "source.value");
       }})
     .execute();

    source

    source Dataframe to be merged.

    condition

    boolean expression as SQL formatted string

    Annotations

    @Evolving()

    Since

    0.3.0

28. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

29. final def notify(): Unit

    

    Definition Classes

    AnyRef

30. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

31. def sparkSession: SparkSession

    

    Attributes

    protected

    Definition Classes

    DeltaTableOperations

32. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

33. def toDF: Dataset[Row]

    

    :: Evolving ::

    :: Evolving ::

    Get a DataFrame (that is, Dataset[Row]) representation of this Delta table.

    Annotations

    @Evolving()

    Since

    0.3.0

34. def toStrColumnMap(map: Map[String, String]): Map[String, Column]

    

    Attributes

    protected

    Definition Classes

    DeltaTableOperations

35. def toString(): String

    

    Definition Classes

    AnyRef → Any

36. def tryResolveReferences(sparkSession: SparkSession)(expr: Expression, planContainingExpr: LogicalPlan): Expression

    

    Attributes

    protected

    Definition Classes

    AnalysisHelper

37. def update(condition: Column, set: Map[String, Column]): Unit

    

    :: Evolving ::

    :: Evolving ::

    Update data from the table on the rows that match the given condition based on the rules defined by set.

    Java example to increment the column data.

    import org.apache.spark.sql.Column;
    import org.apache.spark.sql.functions;
    
    deltaTable.update(
      functions.col("date").gt("2018-01-01"),
      new HashMap<String, Column>() {{
        put("data", functions.col("data").plus(1));
      }}
    );

    condition

    boolean expression as Column object specifying which rows to update.

    set

    rules to update a row as a Java map between target column names and corresponding update expressions as Column objects.

    Annotations

    @Evolving()

    Since

    0.3.0

38. def update(condition: Column, set: Map[String, Column]): Unit

    

    :: Evolving ::

    :: Evolving ::

    Update data from the table on the rows that match the given condition based on the rules defined by set.

    Scala example to increment the column data.

    import org.apache.spark.sql.functions._
    
    deltaTable.update(
      col("date") > "2018-01-01",
      Map("data" -> col("data") + 1))

    condition

    boolean expression as Column object specifying which rows to update.

    set

    rules to update a row as a Scala map between target column names and corresponding update expressions as Column objects.

    Annotations

    @Evolving()

    Since

    0.3.0

39. def update(set: Map[String, Column]): Unit

    

    :: Evolving ::

    :: Evolving ::

    Update rows in the table based on the rules defined by set.

    Java example to increment the column data.

    import org.apache.spark.sql.Column;
    import org.apache.spark.sql.functions;
    
    deltaTable.update(
      new HashMap<String, Column>() {{
        put("data", functions.col("data").plus(1));
      }}
    );

    set

    rules to update a row as a Java map between target column names and corresponding update expressions as Column objects.

    Annotations

    @Evolving()

    Since

    0.3.0

40. def update(set: Map[String, Column]): Unit

    

    :: Evolving ::

    :: Evolving ::

    Update rows in the table based on the rules defined by set.

    Scala example to increment the column data.

    import org.apache.spark.sql.functions._
    
    deltaTable.update(Map("data" -> col("data") + 1))

    set

    rules to update a row as a Scala map between target column names and corresponding update expressions as Column objects.

    Annotations

    @Evolving()

    Since

    0.3.0

41. def updateExpr(condition: String, set: Map[String, String]): Unit

    

    :: Evolving ::

    :: Evolving ::

    Update data from the table on the rows that match the given condition, which performs the rules defined by set.

    Java example to increment the column data.

    deltaTable.update(
      "date > '2018-01-01'",
      new HashMap<String, String>() {{
        put("data", "data + 1");
      }}
    );

    condition

    boolean expression as SQL formatted string object specifying which rows to update.

    set

    rules to update a row as a Java map between target column names and corresponding update expressions as SQL formatted strings.

    Annotations

    @Evolving()

    Since

    0.3.0

42. def updateExpr(condition: String, set: Map[String, String]): Unit

    

    :: Evolving ::

    :: Evolving ::

    Update data from the table on the rows that match the given condition, which performs the rules defined by set.

    Scala example to increment the column data.

    deltaTable.update(
      "date > '2018-01-01'",
      Map("data" -> "data + 1"))

    condition

    boolean expression as SQL formatted string object specifying which rows to update.

    set

    rules to update a row as a Scala map between target column names and corresponding update expressions as SQL formatted strings.

    Annotations

    @Evolving()

    Since

    0.3.0

43. def updateExpr(set: Map[String, String]): Unit

    

    :: Evolving ::

    :: Evolving ::

    Update rows in the table based on the rules defined by set.

    Java example to increment the column data.

    deltaTable.updateExpr(
      new HashMap<String, String>() {{
        put("data", "data + 1");
      }}
    );

    set

    rules to update a row as a Java map between target column names and corresponding update expressions as SQL formatted strings.

    Annotations

    @Evolving()

    Since

    0.3.0

44. def updateExpr(set: Map[String, String]): Unit

    

    :: Evolving ::

    :: Evolving ::

    Update rows in the table based on the rules defined by set.

    Scala example to increment the column data.

    deltaTable.updateExpr(Map("data" -> "data + 1")))

    set

    rules to update a row as a Scala map between target column names and corresponding update expressions as SQL formatted strings.

    Annotations

    @Evolving()

    Since

    0.3.0

45. def vacuum(): DataFrame

    

    :: Evolving ::

    :: Evolving ::

    Recursively delete files and directories in the table that are not needed by the table for maintaining older versions up to the given retention threshold. This method will return an empty DataFrame on successful completion.

    note: This will use the default retention period of 7 days.

    Annotations

    @Evolving()

    Since

    0.3.0

46. def vacuum(retentionHours: Double): DataFrame

    

    :: Evolving ::

    :: Evolving ::

    Recursively delete files and directories in the table that are not needed by the table for maintaining older versions up to the given retention threshold. This method will return an empty DataFrame on successful completion.

    retentionHours

    The retention threshold in hours. Files required by the table for reading versions earlier than this will be preserved and the rest of them will be deleted.

    Annotations

    @Evolving()

    Since

    0.3.0

47. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

48. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

49. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )