SparkActionHelpers

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. def applyFileReduce(numFiles: Option[Int]): (Dataset[_]) ⇒ Dataset[_]

   

5. def applyMode(mode: SaveMode): (DataFrameWriter[_]) ⇒ DataFrameWriter[_]

   

6. def applyOpenCSV(path: String): (DataFrameReader) ⇒ Dataset[_]

   

7. def applyOpenDataFrameReader: (SparkFlowContext) ⇒ DataFrameReader

   

8. def applyOpenParquet(path: String): (DataFrameReader) ⇒ Dataset[_]

   

9. def applyOverwrite(overwrite: Boolean): (DataFrameWriter[_]) ⇒ DataFrameWriter[_]

   

10. def applyPartitionBy(partitionCols: Seq[String]): (DataFrameWriter[_]) ⇒ DataFrameWriter[_]

    

11. def applyReaderOptions(options: Map[String, String]): (DataFrameReader) ⇒ DataFrameReader

    

12. def applyRepartition(repartition: Int): (Dataset[_]) ⇒ Dataset[_]

    

13. def applyRepartition(partitionCols: Seq[String], repartition: Boolean): (Dataset[_]) ⇒ Dataset[_]

    

14. def applyRepartitionAndPartitionBy(partitions: Option[Either[Seq[String], Int]], repartition: Boolean): ((Dataset[_]) ⇒ Dataset[_], (DataFrameWriter[_]) ⇒ DataFrameWriter[_])

    

15. def applySaveAsTable(database: String, table: String): (DataFrameWriter[_]) ⇒ Unit

    

16. def applyWriteCSV(path: String): (DataFrameWriter[_]) ⇒ Unit

    

17. def applyWriteParquet(path: String): (DataFrameWriter[_]) ⇒ Unit

    

18. def applyWriterOptions(options: Map[String, String]): (DataFrameWriter[_]) ⇒ DataFrameWriter[_]

    

19. final def asInstanceOf[T0]: T0

    

    Definition Classes

    Any

20. def checkValidSqlLabels(sparkSession: SparkSession, labels: Seq[String], actionName: String): Unit

    

21. def clone(): AnyRef

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

22. final def eq(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

23. def equals(arg0: Any): Boolean

    

    Definition Classes

    AnyRef → Any

24. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

25. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

26. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

27. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

28. def isValidViewName(sparkSession: SparkSession)(label: String): Boolean

    

29. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

30. final def notify(): Unit

    

    Definition Classes

    AnyRef

31. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

32. def openBase(dataFlow: SparkDataFlow, label: String)(open: (SparkFlowContext) ⇒ Dataset[_]): SparkDataFlow

    

    Base function for all read operation, in all cases users should use more specialised one.

    Base function for all read operation, in all cases users should use more specialised one. This one is used by other builders.

    dataFlow

    - flow to which to add the write action

    label

    - Label of the output Dataset

    open

    - dataset opening function

33. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

34. def toString(): String

    

    Definition Classes

    AnyRef → Any

35. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

36. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

37. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

38. def writeBase(dataFlow: SparkDataFlow, label: String)(pre: (Dataset[_]) ⇒ Dataset[_])(dfr: (DataFrameWriter[_]) ⇒ Unit): SparkDataFlow

    

    Base function for all write operation, in most of the cases users should use more specialised one.

    Base function for all write operation, in most of the cases users should use more specialised one. This one is used by other builders.

    dataFlow

    - flow to which to add the write action

    label

    - label whose data set will be written out

    pre

    - dataset transformation function

    dfr

    - dataframe writer function