ForkedSparkEstimator

This utility is used to support evaluation of the part of pipeline in a separate Spark app. There are at least three identified use cases: 1. Spark App with different settings for ETL and ML 2. Support for larger fork factor in segmented hyperopt (scale driver if it became a bootleneck) 3. Support for parallel XGBoost training (resolves internal conflict on the Rabbit part)

Simple example with linear SGD and Zeppelin in yarn-client mode:

// This estimator will start new Spark app from an app running in yarn-cluster mode
val secondLevel = new ForkedSparkEstimator[LinearRegressionModel, LinearRegressionSGD](new LinearRegressionSGD().setCacheTrainData(true))
            .setTempPath("tmp/forkedModels")
            // Match only files transfered with the app, re-point to the hdfs for faster start
            .withClassPathPropagation(".*__spark_libs__.*", ".+/" -> "hdfs://my-hadoop-nn/spark/lib/")
            // These files are localy available on all nodes
            .withClassPathPropagation("/opt/.*", "^/" -> "local://")
            // For convinience propagate configuration when working in non-interactive mode
            .setPropagateConfig(true)
            .setConfOverrides(
                // Enable log aggregation and disable dynamic allocation
                "spark.hadoop.yarn.log-aggregation-enable" -> "true",
                "spark.dynamicAllocation.enabled" -> "false",
                // These files might sneeak in when submited from Zeppelin, suppress them
                "spark.yarn.dist.jars" -> "",
                "spark.yarn.dist.files" -> "",
                "spark.yarn.dist.archives" -> ""
                )
            .setMaster("yarn")
            .setDeployMode("cluster")
            .setSubmitArgs(
                "--num-executors", "1")
            .setName("secondLevel")

// This estimator is will start neq Spark app from an interactive Zeppelin session
val firstLevel = new ForkedSparkEstimator[LinearRegressionModel, ForkedSparkEstimator[LinearRegressionModel,LinearRegressionSGD]](secondLevel)
        .setTempPath("tmp/forkedModels")
        // Propagate only odkl-analysiss jars, repoint to HDFS for faster start
        .withClassPathPropagation("/home/.*", ".+/" -> "hdfs://my-hadoop-nn/user/myuser/spark/lib/")
        // Do not propagate hell a lot of Zeppelin configs, rely on spark-defaults
        .setPropagateConfig(false)
        .setConfOverrides(
            // Enable log aggregation and disable dynamic execution
            "spark.hadoop.yarn.log-aggregation-enable" -> "true",
            "spark.dynamicAllocation.enabled" -> "false",
            // This is required to be able to start new spark apps from our app
            "spark.yarn.appMasterEnv.HADOOP_CONF_DIR" -> "/opt/hadoop/etc/hadoop/",
            // This is required to make sure Zeppelin does not full us the we are a Python app
            "spark.yarn.isPython" -> "false"
             )
        .setMaster("yarn")
        .setDeployMode("cluster")
        .setSubmitArgs(
            "--num-executors", "1")
        .setName("firstLevel")


val doubleForkedPipeiline = new Pipeline().setStages(Array(
    new VectorAssembler()
        .setInputCols(Array("first", "second"))
        .setOutputCol("features"),
    firstLevel
    ))

Linear Supertypes

MLWritable, SummarizableEstimator[M], Estimator[M], PipelineStage, Logging, Params, Serializable, Serializable, Identifiable, AnyRef, Any

Instance Constructors

new ForkedSparkEstimator(nested: E)
new ForkedSparkEstimator(uid: String)

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def $[T](param: Param[T]): T

Attributes
protected
Definition Classes
Params
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
final val classPathPropagations: JacksonParam[Array[ClassPathExpression]]
final def clear(param: Param[_]): ForkedSparkEstimator.this.type

Definition Classes
Params
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final val confOverrides: JacksonParam[Map[String, String]]
def copy(extra: ParamMap): ForkedSparkEstimator[M, E]

Definition Classes
ForkedSparkEstimator → SummarizableEstimator → Estimator → PipelineStage → Params
def copyValues[T <: Params](to: T, extra: ParamMap): T

Attributes
protected
Definition Classes
Params
final def defaultCopy[T <: Params](extra: ParamMap): T

Attributes
protected
Definition Classes
Params
final val deployMode: Param[String]
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def explainParam(param: Param[_]): String

Definition Classes
Params
def explainParams(): String

Definition Classes
Params
final def extractParamMap(): ParamMap

Definition Classes
Params
final def extractParamMap(extra: ParamMap): ParamMap

Definition Classes
Params
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def fit(dataset: Dataset[_]): M

Definition Classes
ForkedSparkEstimator → Estimator
def fit(dataset: Dataset[_], paramMaps: Array[ParamMap]): Seq[M]

Definition Classes
Estimator
Annotations
@Since( "2.0.0" )
def fit(dataset: Dataset[_], paramMap: ParamMap): M

Definition Classes
Estimator
Annotations
@Since( "2.0.0" )
def fit(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): M

Definition Classes
Estimator
Annotations
@Since( "2.0.0" ) @varargs()
final def get[T](param: Param[T]): Option[T]

Definition Classes
Params
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
final def getDefault[T](param: Param[T]): Option[T]

Definition Classes
Params
final def getOrDefault[T](param: Param[T]): T

Definition Classes
Params
def getParam(paramName: String): Param[Any]

Definition Classes
Params
final def hasDefault[T](param: Param[T]): Boolean

Definition Classes
Params
def hasParam(paramName: String): Boolean

Definition Classes
Params
def hashCode(): Int

Definition Classes
AnyRef → Any
def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

Attributes
protected
Definition Classes
Logging
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes
protected
Definition Classes
Logging
final def isDefined(param: Param[_]): Boolean

Definition Classes
Params
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def isSet(param: Param[_]): Boolean

Definition Classes
Params
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
final val mainJar: Param[String]
final val master: Param[String]
final val name: Param[String]
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def nested: E
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
lazy val params: Array[Param[_]]

Definition Classes
Params
final val propagateConfig: BooleanParam
def save(path: String): Unit

Definition Classes
MLWritable
Annotations
@Since( "1.6.0" ) @throws( ... )
final def set(paramPair: ParamPair[_]): ForkedSparkEstimator.this.type

Attributes
protected
Definition Classes
Params
final def set(param: String, value: Any): ForkedSparkEstimator.this.type

Attributes
protected
Definition Classes
Params
final def set[T](param: Param[T], value: T): ForkedSparkEstimator.this.type

Definition Classes
Params
def setConfOverrides(conf: (String, String)*): ForkedSparkEstimator.this.type
final def setDefault(paramPairs: ParamPair[_]*): ForkedSparkEstimator.this.type

Attributes
protected
Definition Classes
Params
final def setDefault[T](param: Param[T], value: T): ForkedSparkEstimator.this.type

Attributes
protected
Definition Classes
Params
def setDeployMode(value: String): ForkedSparkEstimator.this.type
def setMainJar(value: String): ForkedSparkEstimator.this.type
def setMaster(value: String): ForkedSparkEstimator.this.type
def setName(value: String): ForkedSparkEstimator.this.type
def setPropagateConfig(value: Boolean): ForkedSparkEstimator.this.type
def setSubmitArgs(args: String*): ForkedSparkEstimator.this.type
def setSuppressConfigs(name: String*): ForkedSparkEstimator.this.type
def setTempPath(path: String): ForkedSparkEstimator.this.type
final val submitArgs: StringArrayParam
final val suppressConfig: StringArrayParam
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
final val tempPath: Param[String]
def toString(): String

Definition Classes
Identifiable → AnyRef → Any
def transformSchema(schema: StructType): StructType

Definition Classes
ForkedSparkEstimator → PipelineStage
def transformSchema(schema: StructType, logging: Boolean): StructType

Attributes
protected
Definition Classes
PipelineStage
Annotations
@DeveloperApi()
val uid: String

Definition Classes
ForkedSparkEstimator → Identifiable
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def withClassPathPropagation(filter: String, transformations: (String, String)*): ForkedSparkEstimator.this.type
def write: MLWriter

Definition Classes
ForkedSparkEstimator → MLWritable

Related Docs: object ForkedSparkEstimator | package odkl

class ForkedSparkEstimator[M <: ModelWithSummary[M] with MLWritable, E <: SummarizableEstimator[M] with MLWritable] extends Estimator[M] with SummarizableEstimator[M] with MLWritable

Instance Constructors

new ForkedSparkEstimator(nested: E)

new ForkedSparkEstimator(uid: String)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def $[T](param: Param[T]): T

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

final val classPathPropagations: JacksonParam[Array[ClassPathExpression]]

final def clear(param: Param[_]): ForkedSparkEstimator.this.type

def clone(): AnyRef

final val confOverrides: JacksonParam[Map[String, String]]

def copy(extra: ParamMap): ForkedSparkEstimator[M, E]

def copyValues[T <: Params](to: T, extra: ParamMap): T

final def defaultCopy[T <: Params](extra: ParamMap): T

final val deployMode: Param[String]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def explainParam(param: Param[_]): String

def explainParams(): String

final def extractParamMap(): ParamMap

final def extractParamMap(extra: ParamMap): ParamMap

def finalize(): Unit

def fit(dataset: Dataset[_]): M

def fit(dataset: Dataset[_], paramMaps: Array[ParamMap]): Seq[M]

def fit(dataset: Dataset[_], paramMap: ParamMap): M

def fit(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): M

final def get[T](param: Param[T]): Option[T]

final def getClass(): Class[_]

final def getDefault[T](param: Param[T]): Option[T]

final def getOrDefault[T](param: Param[T]): T

def getParam(paramName: String): Param[Any]

final def hasDefault[T](param: Param[T]): Boolean

def hasParam(paramName: String): Boolean

def hashCode(): Int

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

final def isDefined(param: Param[_]): Boolean

final def isInstanceOf[T0]: Boolean

final def isSet(param: Param[_]): Boolean

def isTraceEnabled(): Boolean

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final val mainJar: Param[String]

final val master: Param[String]

final val name: Param[String]

final def ne(arg0: AnyRef): Boolean

def nested: E

final def notify(): Unit

final def notifyAll(): Unit

lazy val params: Array[Param[_]]

final val propagateConfig: BooleanParam

def save(path: String): Unit

final def set(paramPair: ParamPair[_]): ForkedSparkEstimator.this.type

final def set(param: String, value: Any): ForkedSparkEstimator.this.type

final def set[T](param: Param[T], value: T): ForkedSparkEstimator.this.type

def setConfOverrides(conf: (String, String)*): ForkedSparkEstimator.this.type

final def setDefault(paramPairs: ParamPair[_]*): ForkedSparkEstimator.this.type

final def setDefault[T](param: Param[T], value: T): ForkedSparkEstimator.this.type

def setDeployMode(value: String): ForkedSparkEstimator.this.type

def setMainJar(value: String): ForkedSparkEstimator.this.type

def setMaster(value: String): ForkedSparkEstimator.this.type

def setName(value: String): ForkedSparkEstimator.this.type

def setPropagateConfig(value: Boolean): ForkedSparkEstimator.this.type

def setSubmitArgs(args: String*): ForkedSparkEstimator.this.type

def setSuppressConfigs(name: String*): ForkedSparkEstimator.this.type