JoinUtils

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

5. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

6. def coalescedJoin(leftDf: DataFrame, rightDf: DataFrame, keys: Seq[String], joinType: String = "left"): DataFrame

   

   * join left and right dataframes, merging any shared columns if exists by the coalesce rule.

   * join left and right dataframes, merging any shared columns if exists by the coalesce rule. fails if there is any data type mismatch between shared columns.

   The order of output joined dataframe is:

   • all keys
   • all columns on left (incl. both shared and non-shared) in the original order of left
   • all columns on right that are NOT shared by left, in the original order of right

7. val contains_any: UserDefinedFunction

   

8. def createLatestLabelView(viewName: String, baseView: String, tableUtils: TableUtils, propertiesOverride: Map[String, String] = null): Unit

   

   * Method to create a view with latest available label_ds for a given ds.

   * Method to create a view with latest available label_ds for a given ds. This view is built on top of final label view which has all label versions available. This view will inherit the final label view properties as well.

9. def createOrReplaceView(viewName: String, leftTable: String, rightTable: String, joinKeys: Array[String], tableUtils: TableUtils, viewProperties: Map[String, String] = null, labelColumnPrefix: String = Constants.LabelColumnPrefix): Unit

   

   * Method to create or replace a view for feature table joining with labels.

   * Method to create or replace a view for feature table joining with labels. Label columns will be prefixed with "label" or custom prefix for easy identification

10. final def eq(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

11. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

12. def filterColumns(df: DataFrame, filter: Seq[String]): DataFrame

    

13. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

14. def genBloomFilterIfNeeded(joinPart: JoinPart, joinConf: api.Join, leftRowCount: Long, unfilledRange: PartitionRange, joinLevelBloomMapOpt: Option[Map[String, BloomFilter]]): Option[Map[String, BloomFilter]]

    

    Generate a Bloom filter for 'joinPart' when the row count to be backfilled falls below a specified threshold.

    Generate a Bloom filter for 'joinPart' when the row count to be backfilled falls below a specified threshold. This method anticipates that there will likely be a substantial number of rows on the right side that need to be filtered out.

    returns

    bloomfilter map option for right part

15. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

16. def getLatestLabelMapping(tableName: String, tableUtils: TableUtils): Map[String, Seq[PartitionRange]]

    

    compute the mapping label_ds -> PartitionRange of ds which has this label_ds as latest version

    compute the mapping label_ds -> PartitionRange of ds which has this label_ds as latest version

    • Get all partitions from table
    • For each ds, find the latest available label_ds
    • Reverse the mapping and get the ds partition range for each label version(label_ds)

    returns

    Mapping of the label ds -> partition ranges of ds which has this label available as latest

17. def getRangesToFill(leftSource: Source, tableUtils: TableUtils, endPartition: String, overrideStartPartition: Option[String] = None, historicalBackfill: Boolean = true): PartitionRange

    

    * Compute partition range to be filled for given join conf

18. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

19. def injectKeyFilter(leftDf: DataFrame, joinPart: JoinPart): Unit

    

20. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

21. def leftDf(joinConf: api.Join, range: PartitionRange, tableUtils: TableUtils, allowEmpty: Boolean = false, limit: Option[Int] = None): Option[DataFrame]

    

    * Util methods for join computation

22. lazy val logger: Logger

    

23. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

24. final def notify(): Unit

    

    Definition Classes

    AnyRef

25. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

26. val set_add: UserDefinedFunction

    

27. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

28. def toString(): String

    

    Definition Classes

    AnyRef → Any

29. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

30. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

31. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )