DataFrameReader

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

5. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

6. def csv(csvDataset: Dataset[String])(implicit trace: Trace): SIO[DataFrame]

   

   Loads a dataframe from Dataset[String].

   Loads a dataframe from Dataset[String].

   See UnderlyingDataFrameReader.csv for more information.

7. def csv(paths: String*)(implicit trace: Trace): SIO[DataFrame]

   

   Loads a dataframe from CSV files.

   Loads a dataframe from CSV files.

   See UnderlyingDataFrameReader.csv for more information.

8. def csv(path: String)(implicit trace: Trace): SIO[DataFrame]

   

   Loads a dataframe from a CSV file.

   Loads a dataframe from a CSV file.

   See UnderlyingDataFrameReader.csv for more information.

9. final def eq(arg0: AnyRef): Boolean

   

   Definition Classes

   AnyRef

10. def finalize(): Unit

    

    Attributes

    protected[java.lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

11. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

12. def inferSchema: DataFrameReader[State]

    

    Adds an option to say that spark should infer the schema.

13. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

14. def jdbc(url: String, table: String, columnName: String, lowerBound: Long, upperBound: Long, numPartitions: Int, connectionProperties: Properties)(implicit trace: Trace, ev: =:=[State, WithoutSchema]): SIO[DataFrame]

    

    Construct a DataFrame representing the database table accessible via JDBC.

    Construct a DataFrame representing the database table accessible via JDBC.

    See UnderlyingDataFrameReader.jdbc for more information.

15. def jdbc(url: String, table: String, predicates: Array[String], connectionProperties: Properties)(implicit trace: Trace, ev: =:=[State, WithoutSchema]): SIO[DataFrame]

    

    Construct a DataFrame representing the database table accessible via JDBC.

    Construct a DataFrame representing the database table accessible via JDBC.

    See UnderlyingDataFrameReader.jdbc for more information.

16. def jdbc(url: String, table: String, properties: Properties)(implicit trace: Trace, ev: =:=[State, WithoutSchema]): SIO[DataFrame]

    

    Construct a DataFrame representing the database table accessible via JDBC.

    Construct a DataFrame representing the database table accessible via JDBC.

    See UnderlyingDataFrameReader.jdbc for more information.

17. def json(jsonDataset: Dataset[String])(implicit trace: Trace): SIO[DataFrame]

    

    Loads a dataframe from a Dataset[String].

    Loads a dataframe from a Dataset[String].

    See UnderlyingDataFrameReader.json for more information.

18. def json(paths: String*)(implicit trace: Trace): SIO[DataFrame]

    

    Loads a dataframe from JSON files.

    Loads a dataframe from JSON files.

    See UnderlyingDataFrameReader.json for more information.

19. def json(path: String)(implicit trace: Trace): SIO[DataFrame]

    

    Loads a dataframe from a JSON file.

    Loads a dataframe from a JSON file.

    See UnderlyingDataFrameReader.json for more information.

20. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

21. final def notify(): Unit

    

    Definition Classes

    AnyRef

22. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

23. def option(key: String, value: Double): DataFrameReader[State]

    

    Adds an option to the DataFrameReader.

24. def option(key: String, value: Float): DataFrameReader[State]

    

    Adds an option to the DataFrameReader.

25. def option(key: String, value: Int): DataFrameReader[State]

    

    Adds an option to the DataFrameReader.

26. def option(key: String, value: Boolean): DataFrameReader[State]

    

    Adds an option to the DataFrameReader.

27. def option(key: String, value: String): DataFrameReader[State]

    

    Adds an option to the DataFrameReader.

28. def options(options: Map[String, String]): DataFrameReader[State]

    

    Adds multiple options to the DataFrameReader.

29. val options: Map[String, String]

    

30. def orc(paths: String*)(implicit trace: Trace): SIO[DataFrame]

    

    Loads a dataframe from ORC files.

    Loads a dataframe from ORC files.

    See UnderlyingDataFrameReader.orc for more information.

31. def orc(path: String)(implicit trace: Trace): SIO[DataFrame]

    

    Loads a dataframe from a ORC file.

    Loads a dataframe from a ORC file.

    See UnderlyingDataFrameReader.orc for more information.

32. def parquet(paths: String*)(implicit trace: Trace): SIO[DataFrame]

    

    Loads a dataframe from PARQUET files.

    Loads a dataframe from PARQUET files.

    See UnderlyingDataFrameReader.parquet for more information.

33. def parquet(path: String)(implicit trace: Trace): SIO[DataFrame]

    

    Loads a dataframe from a PARQUET file.

    Loads a dataframe from a PARQUET file.

    See UnderlyingDataFrameReader.parquet for more information.

34. def schema[T](implicit T: ToStructSchema[T]): DataFrameReader[WithSchema]

    

    ZIO-Spark specifics function to generate the schema from a case class.

    ZIO-Spark specifics function to generate the schema from a case class.

    E.g.:

    import zio.spark._
    
    case class Person(name: String, age: Int)
    val ds: Dataset[Person] = SparkSession.read.schema[Person].csv("./path.csv").as[Person].getOrThrow

35. def schema(schemaString: String): Either[ParseException, DataFrameReader[WithSchema]]

    

    Replace the data schema of the DataFrameReader.

    Replace the data schema of the DataFrameReader.

    We advice you to always use a schema even if some data sources can infer the schema. It allows you to increase your job speed, it ensures that the schema is the expected one and it is useful as documentation.

    E.g.:

    schema("a INT, b STRING, c DOUBLE").csv("test.csv")

    See UnderlyingDataFrameReader.schema for more information.

36. def schema(schema: StructType): DataFrameReader[WithSchema]

    

    Replace the data schema of the DataFrameReader.

    Replace the data schema of the DataFrameReader.

    We advice you to always use a schema even if some data sources can infer the schema. It allows you to increase your job speed, it ensures that the schema is the expected one and it is useful as documentation.

    See UnderlyingDataFrameReader.schema for more information.

37. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

38. def table(tableName: String)(implicit trace: Trace, ev: =:=[State, WithoutSchema]): SIO[DataFrame]

    

    Returns the specified table/view as a DataFrame.

    Returns the specified table/view as a DataFrame.

    See UnderlyingDataFrameReader.table for more information.

39. def text(path: String*)(implicit trace: Trace, ev: =:=[State, WithoutSchema]): SIO[DataFrame]

    

    Loads a dataframe from text files.

    Loads a dataframe from text files.

    The underlying schema of the Dataset contains a single string column named "value". The text files must be encoded as UTF-8.

    See UnderlyingDataFrameReader.textFile for more information.

40. def text(path: String)(implicit trace: Trace, ev: =:=[State, WithoutSchema]): SIO[DataFrame]

    

    Loads a dataframe from a text file.

    Loads a dataframe from a text file.

    The underlying schema of the Dataset contains a single string column named "value". The text files must be encoded as UTF-8.

    See UnderlyingDataFrameReader.textFile for more information.

41. def textFile(path: String*)(implicit trace: Trace, ev: =:=[State, WithoutSchema]): SIO[Dataset[String]]

    

    Loads a dataset[String] from text files.

    Loads a dataset[String] from text files.

    See UnderlyingDataFrameReader.textFile for more information.

42. def textFile(path: String)(implicit trace: Trace, ev: =:=[State, WithoutSchema]): SIO[Dataset[String]]

    

    Loads a dataset[String] from a text file.

    Loads a dataset[String] from a text file.

    See UnderlyingDataFrameReader.textFile for more information.

43. val userSpecifiedSchema: Option[StructType]

    

44. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

45. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

46. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

47. def withDelimiter(delimiter: String): DataFrameReader[State]

    

    Adds an option to delimit the column from a csv file.

48. def withHeader: DataFrameReader[State]

    

    Adds an option to say that the file has a header.