class SparkSession extends sql.SparkSession with Logging

The entry point to programming Spark with the Dataset and DataFrame API.

In environments that this has been created upfront (e.g. REPL, notebooks), use the builder to get an existing session:

SparkSession.builder().getOrCreate()

The builder can also be used to create a new session:

SparkSession.builder
  .remote("sc://localhost:15001/myapp")
  .getOrCreate()
Linear Supertypes
Logging, sql.SparkSession, Closeable, AutoCloseable, Serializable, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. SparkSession
  2. Logging
  3. SparkSession
  4. Closeable
  5. AutoCloseable
  6. Serializable
  7. AnyRef
  8. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. Protected

Type Members

  1. implicit class LogStringContext extends AnyRef
    Definition Classes
    Logging

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##: Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. def addArtifact(source: String, target: String): Unit

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
    Annotations
    @Experimental()
  5. def addArtifact(bytes: Array[Byte], target: String): Unit

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
    Annotations
    @Experimental()
  6. def addArtifact(uri: URI): Unit

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
    Annotations
    @Experimental()
  7. def addArtifact(path: String): Unit

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
    Annotations
    @Experimental()
  8. def addArtifacts(uri: URI*): Unit

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
    Annotations
    @Experimental() @varargs()
  9. def addTag(tag: String): Unit

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  10. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  11. def baseRelationToDataFrame(baseRelation: BaseRelation): DataFrame

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  12. lazy val catalog: Catalog

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  13. def clearTags(): Unit

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  14. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.CloneNotSupportedException]) @IntrinsicCandidate() @native()
  15. def close(): Unit

    Close the SparkSession.

    Close the SparkSession.

    Release the current session and close the GRPC connection to the server. The API will not error if any of these operations fail. Closing a closed session is a no-op.

    Close the allocator. Fail if there are still open SparkResults.

    Definition Classes
    SparkSession → Closeable → AutoCloseable
    Since

    3.4.0

  16. val conf: RuntimeConfig

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  17. def createDataFrame(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  18. def createDataFrame(rdd: RDD[_], beanClass: Class[_]): DataFrame

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  19. def createDataFrame(rowRDD: JavaRDD[Row], schema: StructType): DataFrame

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  20. def createDataFrame(rowRDD: RDD[Row], schema: StructType): DataFrame

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  21. def createDataFrame[A <: Product](rdd: RDD[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  22. def createDataFrame(data: List[_], beanClass: Class[_]): DataFrame

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  23. def createDataFrame(rows: List[Row], schema: StructType): DataFrame

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  24. def createDataFrame[A <: Product](data: Seq[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  25. def createDataset[T](data: RDD[T])(implicit arg0: Encoder[T]): Dataset[T]

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  26. def createDataset[T](data: List[T])(implicit arg0: Encoder[T]): Dataset[T]

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  27. def createDataset[T](data: Seq[T])(implicit arg0: Encoder[T]): Dataset[T]

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  28. val emptyDataFrame: DataFrame

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  29. def emptyDataset[T](implicit arg0: Encoder[T]): Dataset[T]

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  30. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  31. def equals(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef → Any
  32. def execute(command: Command): Seq[ExecutePlanResponse]
    Annotations
    @Since("4.0.0") @DeveloperApi()
  33. def executeCommand(runner: String, command: String, options: Map[String, String]): DataFrame

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  34. def experimental: ExperimentalMethods

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  35. final def getClass(): Class[_ <: AnyRef]
    Definition Classes
    AnyRef → Any
    Annotations
    @IntrinsicCandidate() @native()
  36. def getTags(): Set[String]

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  37. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @IntrinsicCandidate() @native()
  38. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  39. def initializeLogIfNecessary(isInterpreter: Boolean): Unit
    Attributes
    protected
    Definition Classes
    Logging
  40. def interruptAll(): Seq[String]

    Interrupt all operations of this session currently running on the connected server.

    Interrupt all operations of this session currently running on the connected server.

    returns

    sequence of operationIds of interrupted operations. Note: there is still a possibility of operation finishing just as it is interrupted.

    Definition Classes
    SparkSession → SparkSession
    Since

    3.5.0

  41. def interruptOperation(operationId: String): Seq[String]

    Interrupt an operation of this session with the given operationId.

    Interrupt an operation of this session with the given operationId.

    returns

    sequence of operationIds of interrupted operations. Note: there is still a possibility of operation finishing just as it is interrupted.

    Definition Classes
    SparkSession → SparkSession
    Since

    3.5.0

  42. def interruptTag(tag: String): Seq[String]

    Interrupt all operations of this session with the given operation tag.

    Interrupt all operations of this session with the given operation tag.

    returns

    sequence of operationIds of interrupted operations. Note: there is still a possibility of operation finishing just as it is interrupted.

    Definition Classes
    SparkSession → SparkSession
    Since

    3.5.0

  43. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  44. def isTraceEnabled(): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  45. def listenerManager: ExecutionListenerManager

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  46. def log: Logger
    Attributes
    protected
    Definition Classes
    Logging
  47. def logDebug(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  48. def logDebug(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  49. def logDebug(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  50. def logDebug(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  51. def logError(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  52. def logError(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  53. def logError(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  54. def logError(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  55. def logInfo(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  56. def logInfo(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  57. def logInfo(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  58. def logInfo(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  59. def logName: String
    Attributes
    protected
    Definition Classes
    Logging
  60. def logTrace(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  61. def logTrace(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  62. def logTrace(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  63. def logTrace(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  64. def logWarning(msg: => String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  65. def logWarning(entry: LogEntry, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  66. def logWarning(entry: LogEntry): Unit
    Attributes
    protected
    Definition Classes
    Logging
  67. def logWarning(msg: => String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  68. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  69. def newDataFrame(cols: Seq[Column])(f: (Builder) => Unit): DataFrame

    Create a DataFrame including the proto plan built by the given function.

    Create a DataFrame including the proto plan built by the given function.

    Use this method when columns are used to create a new DataFrame. When there are columns referring to other Dataset or DataFrame, the plan will be wrapped with a WithRelation.

    with_relations [id 10]
  root: plan  [id 9]  using columns referring to other Dataset or DataFrame, holding plan ids
  reference:
       refs#1: [id 8]  plan for the reference 1
       refs#2: [id 5]  plan for the reference 2
    cols

    The columns to be used in the DataFrame.

    f

    The function to build the proto plan.

    returns

    The DataFrame created from the proto plan.

    Annotations
    @Since("4.0.0") @DeveloperApi()
  70. def newDataFrame(f: (Builder) => Unit): DataFrame

    Create a DataFrame including the proto plan built by the given function.

    Create a DataFrame including the proto plan built by the given function.

    f

    The function to build the proto plan.

    returns

    The DataFrame created from the proto plan.

    Annotations
    @Since("4.0.0") @DeveloperApi()
  71. def newDataset[T](encoder: AgnosticEncoder[T], cols: Seq[Column])(f: (Builder) => Unit): Dataset[T]

    Create a Dataset including the proto plan built by the given function.

    Create a Dataset including the proto plan built by the given function.

    Use this method when columns are used to create a new Dataset. When there are columns referring to other Dataset or DataFrame, the plan will be wrapped with a WithRelation.

    with_relations [id 10]
  root: plan  [id 9]  using columns referring to other Dataset or DataFrame, holding plan ids
  reference:
       refs#1: [id 8]  plan for the reference 1
       refs#2: [id 5]  plan for the reference 2
    encoder

    The encoder for the Dataset.

    cols

    The columns to be used in the DataFrame.

    f

    The function to build the proto plan.

    returns

    The Dataset created from the proto plan.

    Annotations
    @Since("4.0.0") @DeveloperApi()
  72. def newDataset[T](encoder: AgnosticEncoder[T])(f: (Builder) => Unit): Dataset[T]

    Create a Dataset including the proto plan built by the given function.

    Create a Dataset including the proto plan built by the given function.

    encoder

    The encoder for the Dataset.

    f

    The function to build the proto plan.

    returns

    The Dataset created from the proto plan.

    Annotations
    @Since("4.0.0") @DeveloperApi()
  73. def newSession(): SparkSession

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  74. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @IntrinsicCandidate() @native()
  75. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @IntrinsicCandidate() @native()
  76. def range(start: Long, end: Long, step: Long, numPartitions: Int): Dataset[Long]

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  77. def range(start: Long, end: Long, step: Long): Dataset[Long]

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  78. def range(start: Long, end: Long): Dataset[Long]

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  79. def range(end: Long): Dataset[Long]

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  80. def read: DataFrameReader

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  81. def readStream: DataStreamReader

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  82. def registerClassFinder(finder: ClassFinder): Unit

    Register a ClassFinder for dynamically generated classes.

    Register a ClassFinder for dynamically generated classes.

    Annotations
    @Experimental()
    Since

    3.5.0

  83. def removeTag(tag: String): Unit

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  84. def sessionState: SessionState

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  85. def sharedState: SharedState

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  86. def sparkContext: SparkContext

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  87. def sql(query: String): DataFrame

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  88. def sql(sqlText: String, args: Map[String, Any]): DataFrame

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  89. def sql(sqlText: String, args: Map[String, Any]): DataFrame

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  90. def sql(sqlText: String, args: Array[_]): DataFrame

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  91. val sqlContext: SQLContext

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  92. def stop(): Unit
    Definition Classes
    SparkSession
  93. lazy val streams: StreamingQueryManager

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  94. final def synchronized[T0](arg0: => T0): T0
    Definition Classes
    AnyRef
  95. def table(tableName: String): DataFrame

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  96. def time[T](f: => T): T
    Definition Classes
    SparkSession
  97. def toString(): String
    Definition Classes
    AnyRef → Any
  98. def tvf: TableValuedFunction

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  99. lazy val udf: UDFRegistration

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession
  100. lazy val version: String
    Definition Classes
    SparkSession → SparkSession
  101. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  102. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException]) @native()
  103. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  104. def withActive[T](block: => T): T
    Definition Classes
    SparkSession
    Annotations
    @DeveloperApi()
  105. def withLogContext(context: Map[String, String])(body: => Unit): Unit
    Attributes
    protected
    Definition Classes
    Logging
  106. object implicits extends SQLImplicits

    <invalid inheritdoc annotation>

    <invalid inheritdoc annotation>

    Definition Classes
    SparkSession → SparkSession

Deprecated Value Members

  1. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.Throwable]) @Deprecated
    Deprecated

    (Since version 9)

Inherited from Logging

Inherited from sql.SparkSession

Inherited from Closeable

Inherited from AutoCloseable

Inherited from Serializable

Inherited from AnyRef

Inherited from Any

Ungrouped