DeVectorizedDSVRGD

abstract def addGradient(weights: Matrix, features: DenseMatrix, labels: DenseMatrix, updateTerm: DenseMatrix, marginCache: DenseMatrix, lossCache: DenseVector): Unit

For single instance and weights calculates gradient and loss.

abstract def extractModel(labelAttributeGroup: AttributeGroup, numLabels: Int, weights: Matrix, dataset: DataFrame): M

Given labels info and weights matrice create appropriate ML models.

abstract def weightsDistanceForLabel(oldWeights: Matrix, newWeights: DenseMatrix, label: Int): Double

Evaluates weight distance based on old and new weights images.

final def !=(arg0: Any): Boolean

final def ##(): Int

final def $[T](param: Param[T]): T

final def ==(arg0: Any): Boolean

def addL1Reg(l1regParam: Vector, weights: DenseMatrix, updateTerm: DenseMatrix, lossCache: DenseVector, skipRegFeature: Int): DenseMatrix

def addL2Reg(l2regParam: Vector, weights: DenseMatrix, updateTerm: DenseMatrix, lossCache: DenseVector, skipRegFeature: Int): DenseMatrix

Adds L2 regularization part to the gradient and loss.

def adjust(direction: Int, learningRates: DenseMatrix, updateTerm: DenseMatrix, weights: DenseMatrix): DenseMatrix

def applyL1Shrinkage(regParam: Vector, weights: DenseMatrix, skipRegFeature: Int, notDegraded: Set[Int]): DenseMatrix

Apply L1 shrinkage to the updated weights.

final def asInstanceOf[T0]: T0

def axpy(a: Double, x: Vector, y: Array[Double]): Unit

def axpy(a: Double, x: Array[Double], y: Array[Double]): Unit

def axpyCompensated(updateTerm: Array[Double], sum: Array[Double], compensator: Array[Double], y: Array[Double], t: Array[Double]): Unit

def blas: BLAS

final val cacheTrainData: BooleanParam

final def clear(param: Param[_]): DeVectorizedDSVRGD.this.type

def clone(): AnyRef

val convergenceMode: Param[String]

def copy(extra: ParamMap): DSVRGD[M]

def copy(x: Array[Double], y: Array[Double]): Unit

def copyValues[T <: Params](to: T, extra: ParamMap): T

final def defaultCopy[T <: Params](extra: ParamMap): T

def dscal(a: Double, data: Array[Double]): Unit

final val elasticNetParam: DoubleParam

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def evaluateL1Regularization(data: DataFrame, l1Scalar: Double, numLabels: Int): Vector

Given L1 regularization config create a vector with per-label reg param (by default - constant).

def evaluateL2Regularization(data: DataFrame, l2Scalar: Double, numLabels: Int): Vector

Given L2 regularization config create a vector with per-label reg param (by default - constant).

def explainParam(param: Param[_]): String

def explainParams(): String

def extractBlock(lossHistory: Array[CompactBuffer[Double]], dataset: DataFrame, names: Map[Int, String], sc: SparkContext): DataFrame

def extractLabelVectors(labelAttributeGroup: AttributeGroup, numLabels: Int, weights: Matrix): Map[String, Vector]

Utility used to split weights matrice into label -> vector map

final def extractParamMap(): ParamMap

final def extractParamMap(extra: ParamMap): ParamMap

def extractRow(label: Int, weights: Matrix): Vector

Extracts a single row from a matrice.

def extractSummaryBlocks(lossHistory: Array[CompactBuffer[Double]], weightDiffHistory: Array[CompactBuffer[Double]], weightNormHistory: Array[CompactBuffer[Double]], dataset: DataFrame, labelAttributeGroup: AttributeGroup): Map[Block, DataFrame]

Extracts summary blocks from iterations loss history.

def f2jBLAS: BLAS

final val featuresCol: Param[String]

def finalize(): Unit

def fit(dataset: Dataset[_]): M

def fit(dataset: Dataset[_], paramMaps: Array[ParamMap]): Seq[M]

def fit(dataset: Dataset[_], paramMap: ParamMap): M

def fit(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): M

def fullGradientAndLoss(l1regParam: Vector, l2regParam: Vector, localWeights: DenseMatrix, marginCache: DenseMatrix, lossCache: DenseVector, updateTerm: DenseMatrix, skipRegFeature: Int, features: DenseMatrix, labels: DenseMatrix): Any

final def get[T](param: Param[T]): Option[T]

final def getCacheTrainData: Boolean

final def getClass(): Class[_]

final def getDefault[T](param: Param[T]): Option[T]

final def getElasticNetParam: Double

final def getFeaturesCol: String

final def getLabelCol: String

final def getMaxIter: Int

def getNotConverged(activeLabels: Map[Int, Int], lossHistory: Array[CompactBuffer[Double]], weightDiffHistory: Array[CompactBuffer[Double]], weightNormHistory: Array[CompactBuffer[Double]], tolerance: Double): Array[Int]

Extracts not converged labels based on actual and previous weights and on the loss history.

final def getOrDefault[T](param: Param[T]): T

def getParam(paramName: String): Param[Any]

final def getPredictionCol: String

final def getRegParam: Double

final def getTol: Double

final def hasDefault[T](param: Param[T]): Boolean

def hasParam(paramName: String): Boolean

def hashCode(): Int

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

def initializeWeights(data: DataFrame, numLabels: Int, numFeatures: Int): Matrix

final def isDefined(param: Param[_]): Boolean

final def isInstanceOf[T0]: Boolean

final def isSet(param: Param[_]): Boolean

def isTraceEnabled(): Boolean

final val labelCol: Param[String]

val lastIsIntercept: BooleanParam

val learningRate: DoubleParam

val localMinibatchSize: IntParam

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

def lossDifferenceForLabel(lossHistory: Array[CompactBuffer[Double]], label: Int): Double

Evaluates loss difference simply as relative change

val lossIncreaseTolerance: DoubleParam

final val maxIter: IntParam

def merge(labelsMap: Map[Int, Int], weights: Matrix, newWeights: DenseMatrix): DenseMatrix

Merges weights from the new epoch with overal weights.

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

lazy val params: Array[Param[_]]

final val predictionCol: Param[String]

final val regParam: DoubleParam

def relabel(activeLabels: Array[Int], labels: Vector): DenseVector

Used to preserve only active (not yet converged) labels into a vector

def relabelMatrix(activeLabels: Array[Int], matrix: Matrix): Matrix

Used to preserve only active (not yet converged) labels into a matrix

final def set(paramPair: ParamPair[_]): DeVectorizedDSVRGD.this.type

final def set(param: String, value: Any): DeVectorizedDSVRGD.this.type

final def set[T](param: Param[T], value: T): DeVectorizedDSVRGD.this.type

def setCacheTrainData(value: Boolean): DeVectorizedDSVRGD.this.type

def setConvergenceMode(value: String): DeVectorizedDSVRGD.this.type

final def setDefault(paramPairs: ParamPair[_]*): DeVectorizedDSVRGD.this.type

final def setDefault[T](param: Param[T], value: T): DeVectorizedDSVRGD.this.type

def setElasticNetParam(value: Double): DeVectorizedDSVRGD.this.type

def setLastIsIntercept(value: Boolean): DeVectorizedDSVRGD.this.type

def setLearningRate(value: Double): DeVectorizedDSVRGD.this.type

def setLocalMinibatchSize(value: Int): DeVectorizedDSVRGD.this.type

def setMaxIter(value: Int): DeVectorizedDSVRGD.this.type

def setRegParam(value: Double): DeVectorizedDSVRGD.this.type

def setSlowDownFactor(value: Double): DeVectorizedDSVRGD.this.type

def setSpeedUpFactor(value: Double): DeVectorizedDSVRGD.this.type

def setTol(value: Double): DeVectorizedDSVRGD.this.type

def singleStep(data: RDD[(Vector, DenseVector)], weights: Broadcast[Matrix], avgWeights: Broadcast[Matrix], avgGradient: Broadcast[Matrix], l1regParam: Vector, l2regParam: Vector, stepNum: Int, labelLearningRates: DenseVector): DistributedSgdState

Single epoch of the descend

val slowDownFactor: DoubleParam

val speedUpFactor: DoubleParam

final def synchronized[T0](arg0: ⇒ T0): T0

def toDense(weights: Broadcast[Matrix]): DenseMatrix

def toString(): String

final val tol: DoubleParam

def transformSchema(schema: StructType): StructType

def transformSchema(schema: StructType, logging: Boolean): StructType

val uid: String

def updateWeights(stepSize: Double, updateTerm: DenseMatrix, weights: DenseMatrix): Unit

Updates the weights given update term and current value.

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

def weightNorm(newWeights: Matrix, label: Int, skipRegFeature: Int): Double

Evaluates weight norm for a given label.