Packages

class SparkSessionBuilder extends Builder[SparkSession]

Configure and build new sparkSession according to given usages


Usage:

// Auto-configure
val spark: SparkSession = new SparkSessionBuilder("cassandra", "postgres").build().get()

// Build with your own SparkConf
val spark: SparkSession = new SparkSessionBuilder().configure(yourSparkConf).build().get()
Annotations
@Evolving()
Linear Supertypes
Builder[SparkSession], Logging, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. SparkSessionBuilder
  2. Builder
  3. Logging
  4. AnyRef
  5. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Instance Constructors

  1. new SparkSessionBuilder(usages: String*)

    usages

    usages of the sparkSession, could be a list of the following elements:

    • cassandra

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. def appName: String

    Get Spark application name

  5. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  6. def build(): SparkSessionBuilder.this.type

    Automatically build a SparkSession

    Automatically build a SparkSession

    Definition Classes
    SparkSessionBuilder → Builder
  7. def cassandraHost: String

    Get cassandar host value

  8. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  9. def configure(conf: SparkConf): SparkSessionBuilder.this.type

    Wrapper of withSparkConf

    Wrapper of withSparkConf

    conf

    spark configuration

  10. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  11. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  12. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  13. def get(): SparkSession

    Build a spark session with the current configuration

    Build a spark session with the current configuration

    returns

    spark session

    Definition Classes
    SparkSessionBuilder → Builder
  14. def get(key: String): String

    Get a SparkConf value

    Get a SparkConf value

    key

    key of spark conf

    returns

    string if the key exists, null otherwise

  15. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  16. def getOrCreate(): SparkSession
    Definition Classes
    Builder
  17. def getShufflePartitions: String

    Get spark.sql.shuffle.partitions

  18. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  19. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  20. def log: Logger
    Attributes
    protected
    Definition Classes
    Logging
  21. def logDebug(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  22. def logError(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  23. def logInfo(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  24. def logName: String
    Attributes
    protected
    Definition Classes
    Logging
  25. def logTrace(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  26. def logWarning(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  27. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  28. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  29. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  30. def registerClass(cls: Class[_]): SparkSessionBuilder.this.type
  31. def registerClasses(cls: Array[Class[_]]): SparkSessionBuilder.this.type
  32. def set(options: Map[String, String]): SparkSessionBuilder.this.type
  33. def set(key: String, value: String): SparkSessionBuilder.this.type

    Set a SparkConf property

    Set a SparkConf property

    key

    key of spark conf

    value

    value of spark conf

  34. def setAppName(name: String): SparkSessionBuilder.this.type

    Set the name of spark application

    Set the name of spark application

    name

    name of app

  35. def setCassandraHost(host: String): SparkSessionBuilder.this.type

    Set the application envir

    Set the application envir

    host

    cassandra host

  36. def setEnv(env: String): SparkSessionBuilder.this.type

    Set application environment

    Set application environment

    env

    LOCAL, DEV, PREPROD, PROD, EMR

  37. def setKryoRegistrationRequired(boolean: Boolean): SparkSessionBuilder.this.type
  38. def setShufflePartitions(par: Int): SparkSessionBuilder.this.type

    Set spark.sql.shuffle.partitions

    Set spark.sql.shuffle.partitions

    par

    default number of partition

  39. def setSparkMaster(url: String): SparkSessionBuilder.this.type

    Set Master URL for Spark

    Set Master URL for Spark

    url

    url of master

  40. def sparkMasterUrl: String

    Get Spark Master URL

  41. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  42. def toString(): String
    Definition Classes
    AnyRef → Any
  43. def useKryo: Boolean
  44. def useKryo(boo: Boolean): SparkSessionBuilder.this.type
  45. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  46. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  47. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  48. def withSparkConf(conf: SparkConf): SparkSessionBuilder.this.type

    Override the existing configuration with an user defined configuration

    Override the existing configuration with an user defined configuration

    conf

    spark configuration

Inherited from Builder[SparkSession]

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Ungrouped