SQLContext

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

lazy val analyzer: Analyzer

def applySchemaToPythonRDD(rdd: RDD[Array[Any]], schema: StructType): DataFrame

Apply a schema defined by the schema to an RDD.

def applySchemaToPythonRDD(rdd: RDD[Array[Any]], schemaString: String): DataFrame

Apply a schema defined by the schemaString to an RDD.

final def asInstanceOf[T0]: T0

def baseRelationToDataFrame(baseRelation: BaseRelation): DataFrame

Convert a BaseRelation created for external data sources into a DataFrame.

val cacheManager: execution.CacheManager

def cacheTable(tableName: String): Unit

Caches the specified table in-memory.

lazy val catalog: Catalog

def clearCache(): Unit

Removes all cached tables from the in-memory cache.

def clone(): AnyRef

def conf: SQLConf

def createDataFrame(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame

Applies a schema to an RDD of Java Beans.

def createDataFrame(rdd: RDD[_], beanClass: Class[_]): DataFrame

Applies a schema to an RDD of Java Beans.

def createDataFrame(rowRDD: JavaRDD[Row], schema: StructType): DataFrame

:: DeveloperApi :: Creates a DataFrame from an JavaRDD containing Rows using the given schema.

def createDataFrame(rowRDD: RDD[Row], schema: StructType): DataFrame

:: DeveloperApi :: Creates a DataFrame from an RDD containing Rows using the given schema.

import org.apache.spark.sql._
import org.apache.spark.sql.types._
val sqlContext = new org.apache.spark.sql.SQLContext(sc)

val schema =
  StructType(
    StructField("name", StringType, false) ::
    StructField("age", IntegerType, true) :: Nil)

val people =
  sc.textFile("examples/src/main/resources/people.txt").map(
    _.split(",")).map(p => Row(p(0), p(1).trim.toInt))
val dataFrame = sqlContext.createDataFrame(people, schema)
dataFrame.printSchema
// root
// |-- name: string (nullable = false)
// |-- age: integer (nullable = true)

dataFrame.registerTempTable("people")
sqlContext.sql("select name from people").collect.foreach(println)

def createDataFrame[A <: Product](data: Seq[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame

:: Experimental :: Creates a DataFrame from a local Seq of Product.

def createDataFrame[A <: Product](rdd: RDD[A])(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[A]): DataFrame

:: Experimental :: Creates a DataFrame from an RDD of Product (e.g.

def createExternalTable(tableName: String, source: String, schema: StructType, options: Map[String, String]): DataFrame

:: Experimental :: (Scala-specific) Create an external table from the given path based on a data source, a schema and a set of options.

def createExternalTable(tableName: String, source: String, schema: StructType, options: Map[String, String]): DataFrame

:: Experimental :: Create an external table from the given path based on a data source, a schema and a set of options.

def createExternalTable(tableName: String, source: String, options: Map[String, String]): DataFrame

:: Experimental :: (Scala-specific) Creates an external table from the given path based on a data source and a set of options.

def createExternalTable(tableName: String, source: String, options: Map[String, String]): DataFrame

:: Experimental :: Creates an external table from the given path based on a data source and a set of options.

def createExternalTable(tableName: String, path: String, source: String): DataFrame

:: Experimental :: Creates an external table from the given path based on a data source and returns the corresponding DataFrame.

def createExternalTable(tableName: String, path: String): DataFrame

:: Experimental :: Creates an external table from the given path and returns the corresponding DataFrame.

def createSession(): SQLSession

def currentSession(): SQLSession

val ddlParser: DDLParser

val defaultSession: SQLSession

def detachSession(): Unit

def dialectClassName: String

def dropTempTable(tableName: String): Unit

Drops the temporary table with the given table name in the catalog.

lazy val emptyDataFrame: DataFrame

:: Experimental :: Returns a DataFrame with no rows or columns.

lazy val emptyResult: RDD[InternalRow]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def executePlan(plan: LogicalPlan): QueryExecution

def executeSql(sql: String): QueryExecution

val experimental: ExperimentalMethods

:: Experimental :: A collection of methods that are considered experimental, but can be used to hook into the query planner for advanced functionality.

def finalize(): Unit

lazy val functionRegistry: FunctionRegistry

def getAllConfs: Map[String, String]

Return all the configuration properties that have been set (i.e.

final def getClass(): Class[_]

def getConf(key: String, defaultValue: String): String

Return the value of Spark SQL configuration property for the given key.

def getConf(key: String): String

Return the value of Spark SQL configuration property for the given key.

def getSQLDialect(): ParserDialect

def getSchema(beanClass: Class[_]): Seq[AttributeReference]

Returns a Catalyst Schema for the given java bean class.

def hashCode(): Int

object implicits extends SQLImplicits with Serializable

:: Experimental :: (Scala-specific) Implicit methods available in Scala for converting common Scala objects into DataFrames.

val sqlContext = new SQLContext(sc)
import sqlContext.implicits._

def isCached(tableName: String): Boolean

Returns true if the table is currently cached in-memory.

final def isInstanceOf[T0]: Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def openSession(): SQLSession

lazy val optimizer: Optimizer

def parseDataType(dataTypeString: String): DataType

Parses the data type in our internal string representation.

def parseSql(sql: String): LogicalPlan

val planner: SparkPlanner

val prepareForExecution: RuleExecutor[SparkPlan]

Prepares a planned SparkPlan for execution by inserting shuffle operations and internal row format conversions as needed.

def range(start: Long, end: Long, step: Long, numPartitions: Int): DataFrame

:: Experimental :: Creates a DataFrame with a single LongType column named id, containing elements in an range from start to end (exclusive) with an step value, with partition number specified.

def range(start: Long, end: Long): DataFrame

:: Experimental :: Creates a DataFrame with a single LongType column named id, containing elements in an range from start to end (exclusive) with step value 1.

def range(end: Long): DataFrame

:: Experimental :: Creates a DataFrame with a single LongType column named id, containing elements in an range from 0 to end (exclusive) with step value 1.

def read: DataFrameReader

:: Experimental :: Returns a DataFrameReader that can be used to read data in as a DataFrame.

sqlContext.read.parquet("/path/to/file.parquet")
sqlContext.read.schema(schema).json("/path/to/file.json")

def setConf(key: String, value: String): Unit

Set the given Spark SQL configuration property.

def setConf(props: Properties): Unit

Set Spark SQL configuration properties.

def setSession(session: SQLSession): Unit

val sparkContext: SparkContext

def sql(sqlText: String): DataFrame

Executes a SQL query using Spark, returning the result as a DataFrame.

val sqlParser: SparkSQLParser

final def synchronized[T0](arg0: ⇒ T0): T0

def table(tableName: String): DataFrame

Returns the specified table as a DataFrame.

def tableNames(databaseName: String): Array[String]

Returns the names of tables in the given database as an array.

def tableNames(): Array[String]

Returns the names of tables in the current database as an array.

def tables(databaseName: String): DataFrame

Returns a DataFrame containing names of existing tables in the given database.

def tables(): DataFrame

Returns a DataFrame containing names of existing tables in the current database.

val tlSession: ThreadLocal[SQLSession]

def toString(): String

val udf: UDFRegistration

A collection of methods for registering user-defined functions (UDF).

sqlContext.udf.register("myUDF", (arg1: Int, arg2: String) => arg2 + arg1)

sqlContext.udf().register("myUDF",
    new UDF2<Integer, String, String>() {
        @Override
        public String call(Integer arg1, String arg2) {
            return arg2 + arg1;
        }
   }, DataTypes.StringType);

sqlContext.udf().register("myUDF",
    (Integer arg1, String arg2) -> arg2 + arg1,
    DataTypes.StringType);

def uncacheTable(tableName: String): Unit

Removes the specified table from the in-memory cache.

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

def applySchema(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame

def applySchema(rdd: RDD[_], beanClass: Class[_]): DataFrame

def applySchema(rowRDD: JavaRDD[Row], schema: StructType): DataFrame

def applySchema(rowRDD: RDD[Row], schema: StructType): DataFrame

def jdbc(url: String, table: String, theParts: Array[String]): DataFrame

Construct a DataFrame representing the database table accessible via JDBC URL url named table.

def jdbc(url: String, table: String, columnName: String, lowerBound: Long, upperBound: Long, numPartitions: Int): DataFrame

Construct a DataFrame representing the database table accessible via JDBC URL url named table.

def jdbc(url: String, table: String): DataFrame

Construct a DataFrame representing the database table accessible via JDBC URL url named table.

def jsonFile(path: String, samplingRatio: Double): DataFrame

def jsonFile(path: String, schema: StructType): DataFrame

Loads a JSON file (one object per line) and applies the given schema, returning the result as a DataFrame.

def jsonFile(path: String): DataFrame

Loads a JSON file (one object per line), returning the result as a DataFrame.

def jsonRDD(json: JavaRDD[String], samplingRatio: Double): DataFrame

Loads a JavaRDD[String] storing JSON objects (one object per record) inferring the schema, returning the result as a DataFrame.

def jsonRDD(json: RDD[String], samplingRatio: Double): DataFrame

Loads an RDD[String] storing JSON objects (one object per record) inferring the schema, returning the result as a DataFrame.

def jsonRDD(json: JavaRDD[String], schema: StructType): DataFrame

Loads an JavaRDD<String> storing JSON objects (one object per record) and applies the given schema, returning the result as a DataFrame.

def jsonRDD(json: RDD[String], schema: StructType): DataFrame

Loads an RDD[String] storing JSON objects (one object per record) and applies the given schema, returning the result as a DataFrame.

def jsonRDD(json: JavaRDD[String]): DataFrame

Loads an RDD[String] storing JSON objects (one object per record), returning the result as a DataFrame.

def jsonRDD(json: RDD[String]): DataFrame

Loads an RDD[String] storing JSON objects (one object per record), returning the result as a DataFrame.

def load(source: String, schema: StructType, options: Map[String, String]): DataFrame

(Scala-specific) Returns the dataset specified by the given data source and a set of options as a DataFrame, using the given schema as the schema of the DataFrame.

def load(source: String, schema: StructType, options: Map[String, String]): DataFrame

(Java-specific) Returns the dataset specified by the given data source and a set of options as a DataFrame, using the given schema as the schema of the DataFrame.

def load(source: String, options: Map[String, String]): DataFrame

(Scala-specific) Returns the dataset specified by the given data source and a set of options as a DataFrame.

def load(source: String, options: Map[String, String]): DataFrame

(Java-specific) Returns the dataset specified by the given data source and a set of options as a DataFrame.

def load(path: String, source: String): DataFrame

Returns the dataset stored at path as a DataFrame, using the given data source.

def load(path: String): DataFrame

Returns the dataset stored at path as a DataFrame, using the default data source configured by spark.sql.sources.default.

def parquetFile(paths: String*): DataFrame

Loads a Parquet file, returning the result as a DataFrame.