lamp-core
API
Generated with
lamp-core/lamp/lamp.nn

lamp.nn

package lamp.nn

Provides building blocks for neural networks

Notable types:

Optimizers:

Modules facilitating composing other modules:

  • nn.Sequential composes a homogenous list of modules (analogous to List)
  • nn.sequence composes a heterogeneous list of modules (analogous to tuples)
  • nn.EitherModule composes two modules in a scala.Either

Examples of neural network building blocks, layers etc:

Attributes

Members list

Packages

package lamp.nn.bert
package lamp.nn.graph
package lamp.nn.languagemodel

Type members

Classlikes

object AdamW

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
AdamW.type
case class AdamW(parameters: Seq[(STen, PTag)], weightDecay: OptimizerHyperparameter, learningRate: OptimizerHyperparameter, beta1: OptimizerHyperparameter, beta2: OptimizerHyperparameter, eps: Double, clip0: Option[Double], debias: Boolean, mixedPrecision: Boolean) extends Optimizer

Attributes

See also
Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait Optimizer
class Object
trait Matchable
class Any
Show all
case class AdversarialTraining(eps: Double) extends LossCalculation[Variable]

Attributes

Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
case class BatchNorm(weight: Constant, bias: Constant, runningMean: Constant, runningVar: Constant, training: Boolean, momentum: Double, eps: Double, forceTrain: Boolean, forceEval: Boolean, evalIfBatchSizeIsOne: Boolean) extends Module

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
object BatchNorm

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
BatchNorm.type
case class BatchNorm2D(weight: Constant, bias: Constant, runningMean: Constant, runningVar: Constant, training: Boolean, momentum: Double, eps: Double) extends Module

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
object BatchNorm2D

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case class Conv1D(weights: Constant, bias: Constant, stride: Long, padding: Long, dilation: Long, groups: Long) extends Module

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
object Conv1D

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
Conv1D.type
case class Conv2D(weights: Constant, bias: Constant, stride: Long, padding: Long, dilation: Long, groups: Long) extends Module

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
object Conv2D

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
Conv2D.type
case class Conv2DTransposed(weights: Constant, bias: Constant, stride: Long, padding: Long, dilation: Long) extends Module

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
object Conv2DTransposed

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case class Debug(fun: (STen, Boolean, Boolean) => Unit) extends Module

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
object Debug

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
Debug.type
case class DependentHyperparameter(default: Double)(pf: PartialFunction[PTag, Double]) extends OptimizerHyperparameter

Attributes

Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
case class Dropout(prob: Double, training: Boolean) extends Module

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
object Dropout

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
Dropout.type
case class EitherModule[A, B, M1 <: GenericModule[A, B], M2 <: GenericModule[A, B]](members: Either[M1 & GenericModule[A, B], M2 & GenericModule[A, B]]) extends GenericModule[A, B]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[A, B]
class Object
trait Matchable
class Any
Show all
object EitherModule

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case class Embedding(weights: Constant) extends Module

Learnable mapping from classes to dense vectors. Equivalent to L * W where L is the n x C one-hot encoded matrix of the classes * is matrix multiplication W is the C x dim dense matrix. W is learnable. L is never computed directly. C is the number of classes. n is the size of the batch.

Learnable mapping from classes to dense vectors. Equivalent to L * W where L is the n x C one-hot encoded matrix of the classes * is matrix multiplication W is the C x dim dense matrix. W is learnable. L is never computed directly. C is the number of classes. n is the size of the batch.

Input is a long tensor with values in [0,C-1]. Input shape is arbitrary, (). Output shape is ( x D) where D is the embedding dimension.

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
object Embedding

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
Embedding.type
case class FreeRunningRNN[T, M <: StatefulModule[Variable, Variable, T]](module: M & StatefulModule[Variable, Variable, T], timeSteps: Int) extends StatefulModule[Variable, Variable, T]

Wraps a (sequence x batch) long -> (sequence x batch x dim) double stateful module and runs in it greedy (argmax) generation mode over timeSteps steps.

Wraps a (sequence x batch) long -> (sequence x batch x dim) double stateful module and runs in it greedy (argmax) generation mode over timeSteps steps.

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[(Variable, T), (Variable, T)]
class Object
trait Matchable
class Any
Show all
object FreeRunningRNN

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case class Fun(fun: Scope => Variable => Variable) extends Module

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
object Fun

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
Fun.type
case class GRU(weightXh: Constant, weightHh: Constant, weightXr: Constant, weightXz: Constant, weightHr: Constant, weightHz: Constant, biasR: Constant, biasZ: Constant, biasH: Constant) extends StatefulModule[Variable, Variable, Option[Variable]]

Inputs of size (sequence length * batch * in dim) Outputs of size (sequence length * batch * hidden dim)

Inputs of size (sequence length * batch * in dim) Outputs of size (sequence length * batch * hidden dim)

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[(Variable, Option[Variable]), (Variable, Option[Variable])]
class Object
trait Matchable
class Any
Show all
object GRU

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
GRU.type
case class GenericFun[A, B](fun: Scope => A => B) extends GenericModule[A, B]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[A, B]
class Object
trait Matchable
class Any
Show all
object GenericFun

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
GenericFun.type
object GenericModule

Attributes

Companion
trait
Supertypes
class Object
trait Matchable
class Any
Self type
trait GenericModule[A, B]

Base type of modules

Base type of modules

Modules are functions of type (Seq[lamp.autograd.Constant],A) => B, where the Seq[lamp.autograd.Constant] arguments are optimizable parameters and A is a non-optimizable input.

Modules provide a way to build composite functions while also keep track of the parameter list of the composite function.

===Example===

case object Weights extends LeafTag
case object Bias extends LeafTag
case class Linear(weights: Constant, bias: Option[Constant]) extends Module {

 override val state = List(
   weights -> Weights
 ) ++ bias.toList.map(b => (b, Bias))

 def forward[S: Sc](x: Variable): Variable = {
   val v = x.mm(weights)
   bias.map(_ + v).getOrElse(v)

 }
}

Some other attributes of modules are attached by type classes e.g. with the nn.TrainingMode, nn.Load type classes.

Type parameters

A

the argument type of the module

B

the value type of the module

Attributes

See also

nn.Module is an alias for simple Variable => Variable modules

Companion
object
Supertypes
class Object
trait Matchable
class Any
Known subtypes
class BertEncoder
class BertLoss
class EitherModule[A, B, M1, M2]
class GenericFun[A, B]
class Recursive[A, M]
class Seq2[T1, T2, T3, M1, M2]
class Seq3[T1, T2, T3, T4, M1, M2, M3]
class Seq4[T1, T2, T3, T4, T5, M1, M2, M3, M4]
class Seq5[T1, T2, T3, T4, T5, T6, M1, M2, M3, M4, M5]
class Seq6[T1, T2, T3, T4, T5, T6, T7, M1, M2, M3, M4, M5, M6]
class Sequential[A, M]
class Transformer
class UnliftedModule[A, B, C, D, M]
class WrapFun[A, B, M, O]
Show all
trait InitState[M, C]

Type class about how to initialize recurrent neural networks

Type class about how to initialize recurrent neural networks

Attributes

Companion
object
Supertypes
class Object
trait Matchable
class Any
object InitState

Attributes

Companion
trait
Supertypes
class Object
trait Matchable
class Any
Self type
InitState.type
implicit class InitStateSyntax[M, C](m: M)(implicit is: InitState[M, C])

Attributes

Supertypes
class Object
trait Matchable
class Any
case class LSTM(weightXi: Constant, weightXf: Constant, weightXo: Constant, weightHi: Constant, weightHf: Constant, weightHo: Constant, weightXc: Constant, weightHc: Constant, biasI: Constant, biasF: Constant, biasO: Constant, biasC: Constant) extends StatefulModule[Variable, Variable, Option[(Variable, Variable)]]

Inputs of size (sequence length * batch * vocab) Outputs of size (sequence length * batch * output dim)

Inputs of size (sequence length * batch * vocab) Outputs of size (sequence length * batch * output dim)

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[(Variable, Option[(Variable, Variable)]), (Variable, Option[(Variable, Variable)])]
class Object
trait Matchable
class Any
Show all
object LSTM

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
LSTM.type
case class LayerNorm(scale: Constant, bias: Constant, eps: Double, normalizedShape: List[Long]) extends Module

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
object LayerNorm

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
LayerNorm.type
trait LeafTag extends PTag

Attributes

Supertypes
trait PTag
class Object
trait Matchable
class Any
Known subtypes
object Weights.type
object Bias.type
object RunningMean.type
object RunningVar.type
object Weights.type
object Bias.type
object Weights.type
object Bias.type
object Weights.type
object Bias.type
object Weights.type
object Bias.type
object Weights.type
object Weights.type
object BiasH.type
object BiasR.type
object BiasZ.type
object WeightHh.type
object WeightHr.type
object WeightHz.type
object WeightXh.type
object WeightXr.type
object WeightXz.type
object BiasC.type
object BiasF.type
object BiasI.type
object BiasO.type
object WeightHc.type
object WeightHf.type
object WeightHi.type
object WeightHo.type
object WeightXc.type
object WeightXf.type
object WeightXi.type
object WeightXo.type
object Bias.type
object Scale.type
object Bias.type
object Weights.type
object WeightsK.type
object WeightsO.type
object WeightsQ.type
object WeightsV.type
object NoTag.type
object BiasH.type
object WeightHh.type
object WeightXh.type
object Bias.type
object Weight.type
object Bias1.type
object Bias2.type
object Weights1.type
object Weights2.type
object Embedding.type
object Bias1.type
object Bias2.type
object Scale1.type
object Scale2.type
object Weights1.type
object Weights2.type
object Bias.type
object WeightsG.type
object WeightsV.type
Show all
trait LearningRateSchedule[State]

Attributes

Companion
object
Supertypes
class Object
trait Matchable
class Any
object LearningRateSchedule

Attributes

Companion
trait
Supertypes
class Object
trait Matchable
class Any
Self type
case class LiftedModule[M <: Module](mod: M & Module) extends StatefulModule[Variable, Variable, Unit]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[(Variable, Unit), (Variable, Unit)]
class Object
trait Matchable
class Any
Show all
object LiftedModule

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case class Linear(weights: Constant, bias: Option[Constant]) extends Module

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
object Linear

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
Linear.type
trait Load[M]

Type class about how to load the contents of the state of modules from external tensors

Type class about how to load the contents of the state of modules from external tensors

Attributes

Companion
object
Supertypes
class Object
trait Matchable
class Any
object Load

Attributes

Companion
trait
Supertypes
class Object
trait Matchable
class Any
Self type
Load.type
implicit class LoadSyntax[M](m: M)(implicit evidence$2: Load[M])

Attributes

Supertypes
class Object
trait Matchable
class Any
trait LossCalculation[I]

Loss and Gradient calculation

Loss and Gradient calculation

Takes samples, target, module, loss function and computes the loss and the gradients

Attributes

Supertypes
class Object
trait Matchable
class Any
Known subtypes
trait LossFunction

Attributes

Supertypes
class Object
trait Matchable
class Any
Known subtypes
object Identity.type
object MSE.type
class NLL
class SequenceNLL
class SmoothL1Loss
Show all
object LossFunctions

Attributes

Supertypes
class Object
trait Matchable
class Any
Self type
object MLP

Factory for multilayer fully connected feed forward networks

Factory for multilayer fully connected feed forward networks

Returned network has the following repeated structure: [linear -> batchnorm -> nonlinearity -> dropout]*

The last block does not include the nonlinearity and the dropout.

Value parameters

dropout

dropout applied to each block

hidden

list of hidden dimensions

in

input dimensions

out

output dimensions

Attributes

Supertypes
class Object
trait Matchable
class Any
Self type
MLP.type
case class MappedState[A, B, C, D, M <: StatefulModule[A, B, C]](statefulModule: M & StatefulModule[A, B, C], map: C => D) extends StatefulModule2[A, B, C, D]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[(A, C), (B, D)]
class Object
trait Matchable
class Any
Show all
object MappedState

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case class ModelWithOptimizer[I, M <: GenericModule[I, Variable]](model: SupervisedModel[I, M], optimizer: Optimizer)

Attributes

Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
case class MultiheadAttention(wQ: Constant, wK: Constant, wV: Constant, wO: Constant, dropout: Double, train: Boolean, numHeads: Int, linearized: Boolean, causalMask: Boolean) extends GenericModule[(Variable, Variable, Variable, Option[STen]), Variable]

Multi-head scaled dot product attention module

Multi-head scaled dot product attention module

Input: (query,key,value,maxLength) where

  • query: batch x num queries x query dim
  • key: batch x num k-v x key dim
  • value: batch x num k-v x key value
  • maxLength: 1D or 2D long tensor for attention masking

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[(Variable, Variable, Variable, Option[STen]), Variable]
class Object
trait Matchable
class Any
Show all
object MultiheadAttention

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case object NoTag extends LeafTag

Attributes

Supertypes
trait Singleton
trait Product
trait Mirror
trait Serializable
trait Product
trait Equals
trait LeafTag
trait PTag
class Object
trait Matchable
class Any
Show all
Self type
NoTag.type
trait Optimizer

Attributes

Supertypes
class Object
trait Matchable
class Any
Known subtypes
class AdamW
class RAdam
class SGDW
class Shampoo
class Yogi
trait OptimizerHyperparameter

Attributes

Supertypes
class Object
trait Matchable
class Any
Known subtypes
class simple
trait PTag

A small trait to mark paramters for unique identification

A small trait to mark paramters for unique identification

Attributes

Companion
object
Supertypes
class Object
trait Matchable
class Any
Known subtypes
class Tag[T]
trait LeafTag
object Weights.type
object Bias.type
object RunningMean.type
object RunningVar.type
object Weights.type
object Bias.type
object Weights.type
object Bias.type
object Weights.type
object Bias.type
object Weights.type
object Bias.type
object Weights.type
object Weights.type
object BiasH.type
object BiasR.type
object BiasZ.type
object WeightHh.type
object WeightHr.type
object WeightHz.type
object WeightXh.type
object WeightXr.type
object WeightXz.type
object BiasC.type
object BiasF.type
object BiasI.type
object BiasO.type
object WeightHc.type
object WeightHf.type
object WeightHi.type
object WeightHo.type
object WeightXc.type
object WeightXf.type
object WeightXi.type
object WeightXo.type
object Bias.type
object Scale.type
object Bias.type
object Weights.type
object WeightsK.type
object WeightsO.type
object WeightsQ.type
object WeightsV.type
object NoTag.type
object BiasH.type
object WeightHh.type
object WeightXh.type
object Bias.type
object Weight.type
object Bias1.type
object Bias2.type
object Weights1.type
object Weights2.type
object Embedding.type
object Bias1.type
object Bias2.type
object Scale1.type
object Scale2.type
object Weights1.type
object Weights2.type
object Bias.type
object WeightsG.type
object WeightsV.type
class Tag[T]
Show all
object PTag

Attributes

Companion
trait
Supertypes
class Object
trait Matchable
class Any
Self type
PTag.type
class PerturbedLossCalculation[I](noiseLevel: Double) extends LossCalculation[I]

Evaluates the gradient at current point + eps where eps is I * N(0,noiseLevel)

Evaluates the gradient at current point + eps where eps is I * N(0,noiseLevel)

Attributes

Supertypes
trait LossCalculation[I]
class Object
trait Matchable
class Any
object PositionalEmbedding

Attributes

Supertypes
class Object
trait Matchable
class Any
Self type
object RAdam

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
RAdam.type
case class RAdam(parameters: Seq[(STen, PTag)], weightDecay: OptimizerHyperparameter, learningRate: OptimizerHyperparameter, beta1: OptimizerHyperparameter, beta2: OptimizerHyperparameter, eps: Double, clip0: Option[Double]) extends Optimizer

Rectified Adam optimizer algorithm

Rectified Adam optimizer algorithm

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait Optimizer
class Object
trait Matchable
class Any
Show all
case class RNN(weightXh: Constant, weightHh: Constant, biasH: Constant) extends StatefulModule[Variable, Variable, Option[Variable]]

Inputs of size (sequence length * batch * in dim) Outputs of size (sequence length * batch * hidden dim)

Inputs of size (sequence length * batch * in dim) Outputs of size (sequence length * batch * hidden dim)

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[(Variable, Option[Variable]), (Variable, Option[Variable])]
class Object
trait Matchable
class Any
Show all
object RNN

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
RNN.type
case class Recursive[A, M <: GenericModule[A, A]](member: M & GenericModule[A, A], n: Int) extends GenericModule[A, A]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[A, A]
class Object
trait Matchable
class Any
Show all
object Recursive

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
Recursive.type
case class ResidualModule[M <: Module](transform: M & Module) extends Module

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
object ResidualModule

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
object SGDW

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
SGDW.type
case class SGDW(parameters: Seq[(STen, PTag)], learningRate: OptimizerHyperparameter, weightDecay: OptimizerHyperparameter, momentum: Option[OptimizerHyperparameter], clip0: Option[Double]) extends Optimizer

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait Optimizer
class Object
trait Matchable
class Any
Show all
case class Seq2[T1, T2, T3, M1 <: GenericModule[T1, T2], M2 <: GenericModule[T2, T3]](m1: M1 & GenericModule[T1, T2], m2: M2 & GenericModule[T2, T3]) extends GenericModule[T1, T3]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[T1, T3]
class Object
trait Matchable
class Any
Show all
object Seq2

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
Seq2.type
case class Seq2Seq[S0, S1, M1 <: StatefulModule2[Variable, Variable, S0, S1], M2 <: StatefulModule[Variable, Variable, S1]](encoder: M1 & StatefulModule2[Variable, Variable, S0, S1], decoder: M2 & StatefulModule[Variable, Variable, S1]) extends StatefulModule2[(Variable, Variable), Variable, S0, S1]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[((Variable, Variable), S0), (Variable, S1)]
class Object
trait Matchable
class Any
Show all
object Seq2Seq

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
Seq2Seq.type
case class Seq3[T1, T2, T3, T4, M1 <: GenericModule[T1, T2], M2 <: GenericModule[T2, T3], M3 <: GenericModule[T3, T4]](m1: M1 & GenericModule[T1, T2], m2: M2 & GenericModule[T2, T3], m3: M3 & GenericModule[T3, T4]) extends GenericModule[T1, T4]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[T1, T4]
class Object
trait Matchable
class Any
Show all
object Seq3

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
Seq3.type
case class Seq4[T1, T2, T3, T4, T5, M1 <: GenericModule[T1, T2], M2 <: GenericModule[T2, T3], M3 <: GenericModule[T3, T4], M4 <: GenericModule[T4, T5]](m1: M1 & GenericModule[T1, T2], m2: M2 & GenericModule[T2, T3], m3: M3 & GenericModule[T3, T4], m4: M4 & GenericModule[T4, T5]) extends GenericModule[T1, T5]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[T1, T5]
class Object
trait Matchable
class Any
Show all
object Seq4

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
Seq4.type
case class Seq5[T1, T2, T3, T4, T5, T6, M1 <: GenericModule[T1, T2], M2 <: GenericModule[T2, T3], M3 <: GenericModule[T3, T4], M4 <: GenericModule[T4, T5], M5 <: GenericModule[T5, T6]](m1: M1 & GenericModule[T1, T2], m2: M2 & GenericModule[T2, T3], m3: M3 & GenericModule[T3, T4], m4: M4 & GenericModule[T4, T5], m5: M5 & GenericModule[T5, T6]) extends GenericModule[T1, T6]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[T1, T6]
class Object
trait Matchable
class Any
Show all
object Seq5

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
Seq5.type
case class Seq6[T1, T2, T3, T4, T5, T6, T7, M1 <: GenericModule[T1, T2], M2 <: GenericModule[T2, T3], M3 <: GenericModule[T3, T4], M4 <: GenericModule[T4, T5], M5 <: GenericModule[T5, T6], M6 <: GenericModule[T6, T7]](m1: M1 & GenericModule[T1, T2], m2: M2 & GenericModule[T2, T3], m3: M3 & GenericModule[T3, T4], m4: M4 & GenericModule[T4, T5], m5: M5 & GenericModule[T5, T6], m6: M6 & GenericModule[T6, T7]) extends GenericModule[T1, T7]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[T1, T7]
class Object
trait Matchable
class Any
Show all
object Seq6

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
Seq6.type
case class SeqLinear(weight: Constant, bias: Constant) extends Module

Inputs of size (sequence length * batch * in dim) Outputs of size (sequence length * batch * output dim) Applies a linear function to each time step

Inputs of size (sequence length * batch * in dim) Outputs of size (sequence length * batch * output dim) Applies a linear function to each time step

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
object SeqLinear

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
SeqLinear.type
case class Sequential[A, M <: GenericModule[A, A]](members: M & GenericModule[A, A]*) extends GenericModule[A, A]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[A, A]
class Object
trait Matchable
class Any
Show all
object Sequential

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
Sequential.type
object Shampoo

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
Shampoo.type
case class Shampoo(parameters: Seq[(STen, PTag)], learningRate: OptimizerHyperparameter, clip0: Option[Double], eps: Double, diagonalThreshold: Int, updatePreconditionerEveryNIterations: Int, momentum: OptimizerHyperparameter) extends Optimizer

Attributes

See also
Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait Optimizer
class Object
trait Matchable
class Any
Show all
class SimpleLossCalculation[I] extends LossCalculation[I]

Attributes

Supertypes
trait LossCalculation[I]
class Object
trait Matchable
class Any
case class StatefulSeq2[T1, T2, T3, S1, S2, M1 <: StatefulModule[T1, T2, S1], M2 <: StatefulModule[T2, T3, S2]](m1: M1 & StatefulModule[T1, T2, S1], m2: M2 & StatefulModule[T2, T3, S2]) extends StatefulModule[T1, T3, (S1, S2)]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[(T1, (S1, S2)), (T3, (S1, S2))]
class Object
trait Matchable
class Any
Show all
object StatefulSeq2

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case class StatefulSeq3[T1, T2, T3, T4, S1, S2, S3, M1 <: StatefulModule[T1, T2, S1], M2 <: StatefulModule[T2, T3, S2], M3 <: StatefulModule[T3, T4, S3]](m1: M1 & StatefulModule[T1, T2, S1], m2: M2 & StatefulModule[T2, T3, S2], m3: M3 & StatefulModule[T3, T4, S3]) extends StatefulModule[T1, T4, (S1, S2, S3)]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[(T1, (S1, S2, S3)), (T4, (S1, S2, S3))]
class Object
trait Matchable
class Any
Show all
object StatefulSeq3

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case class StatefulSeq4[T1, T2, T3, T4, T5, S1, S2, S3, S4, M1 <: StatefulModule[T1, T2, S1], M2 <: StatefulModule[T2, T3, S2], M3 <: StatefulModule[T3, T4, S3], M4 <: StatefulModule[T4, T5, S4]](m1: M1 & StatefulModule[T1, T2, S1], m2: M2 & StatefulModule[T2, T3, S2], m3: M3 & StatefulModule[T3, T4, S3], m4: M4 & StatefulModule[T4, T5, S4]) extends StatefulModule[T1, T5, (S1, S2, S3, S4)]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[(T1, (S1, S2, S3, S4)), (T5, (S1, S2, S3, S4))]
class Object
trait Matchable
class Any
Show all
object StatefulSeq4

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case class StatefulSeq5[T1, T2, T3, T4, T5, T6, S1, S2, S3, S4, S5, M1 <: StatefulModule[T1, T2, S1], M2 <: StatefulModule[T2, T3, S2], M3 <: StatefulModule[T3, T4, S3], M4 <: StatefulModule[T4, T5, S4], M5 <: StatefulModule[T5, T6, S5]](m1: M1 & StatefulModule[T1, T2, S1], m2: M2 & StatefulModule[T2, T3, S2], m3: M3 & StatefulModule[T3, T4, S3], m4: M4 & StatefulModule[T4, T5, S4], m5: M5 & StatefulModule[T5, T6, S5]) extends StatefulModule[T1, T6, (S1, S2, S3, S4, S5)]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[(T1, (S1, S2, S3, S4, S5)), (T6, (S1, S2, S3, S4, S5))]
class Object
trait Matchable
class Any
Show all
object StatefulSeq5

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case class SupervisedModel[I, M <: GenericModule[I, Variable]](module: M & GenericModule[I, Variable], lossFunction: LossFunction, lossCalculation: LossCalculation[I], printMemoryAllocations: Boolean)(implicit tm: TrainingMode[M])

Attributes

Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
implicit class ToLift[M <: Module](mod: M & Module)

Attributes

Supertypes
class Object
trait Matchable
class Any
implicit class ToMappedState[A, B, C, M <: StatefulModule[A, B, C]](mod: M & StatefulModule[A, B, C])

Attributes

Supertypes
class Object
trait Matchable
class Any
implicit class ToUnlift[A, B, C, D, M <: StatefulModule2[A, B, C, D]](mod: M & StatefulModule2[A, B, C, D])(implicit is: InitState[M, C])

Attributes

Supertypes
class Object
trait Matchable
class Any
implicit class ToWithInit[A, B, C, M <: StatefulModule[A, B, C]](mod: M & StatefulModule[A, B, C])

Attributes

Supertypes
class Object
trait Matchable
class Any
trait TrainingMode[M]

Type class about how to switch a module into training or evaluation mode

Type class about how to switch a module into training or evaluation mode

Attributes

Companion
object
Supertypes
class Object
trait Matchable
class Any
object TrainingMode

Attributes

Companion
trait
Supertypes
class Object
trait Matchable
class Any
Self type
implicit class TrainingModeSyntax[M](m: M)(implicit evidence$1: TrainingMode[M])

Attributes

Supertypes
class Object
trait Matchable
class Any
case class Transformer(encoder: TransformerEncoder, decoder: TransformerDecoder) extends GenericModule[(Variable, Variable, Option[STen], Option[STen]), Variable]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[(Variable, Variable, Option[STen], Option[STen]), Variable]
class Object
trait Matchable
class Any
Show all
object Transformer

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case class TransformerDecoder(blocks: Seq[TransformerDecoderBlock]) extends GenericModule[(Variable, Variable, Option[STen]), Variable]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[(Variable, Variable, Option[STen]), Variable]
class Object
trait Matchable
class Any
Show all
object TransformerDecoder

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case class TransformerDecoderBlock(attentionDecoderDecoder: MultiheadAttention, attentionEncoderDecoder: MultiheadAttention, layerNorm1: LayerNorm, layerNorm2: LayerNorm, layerNorm3: LayerNorm, layerNorm4: LayerNorm, w1: Constant, b1: Constant, w2: Constant, b2: Constant, dropout: Double, train: Boolean) extends GenericModule[(Variable, Variable, Option[STen]), Variable]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[(Variable, Variable, Option[STen]), Variable]
class Object
trait Matchable
class Any
Show all
object TransformerDecoderBlock

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case class TransformerEmbedding(embedding: Embedding, addPositionalEmbedding: Boolean, positionalEmbedding: Constant) extends GenericModule[Variable, Variable]

A module with positional and token embeddings

A module with positional and token embeddings

Token embeddings are lookup embeddings. Positional embeddings are supplied as a constant. They are supposed to come from a fixed unlearned derivation of the positions.

Token and positional embeddings are summed.

Gradients are not computed for positionalEmbedding

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
object TransformerEmbedding

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case class TransformerEncoder(blocks: Seq[TransformerEncoderBlock]) extends GenericModule[(Variable, Option[STen]), Variable]

TransformerEncoder module

TransformerEncoder module

Does not include initial embedding or position encoding.

Input is (data, maxLength) where data is (batch, sequence, input dimension), double tensor maxLength is a 1D or 2D long tensor used for attention masking.

Attention masking is implemented similarly to chapter 11.3.2.1 in d2l.ai v1.0.0-beta0. It supports unmasked attention, attention on variable length input, and left-to-right attention.

Output is (bach, sequence, output dimension)

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[(Variable, Option[STen]), Variable]
class Object
trait Matchable
class Any
Show all
object TransformerEncoder

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case class TransformerEncoderBlock(attention: MultiheadAttention, layerNorm1: LayerNorm, layerNorm2: LayerNorm, w1: Constant, b1: Constant, w2: Constant, b2: Constant, scale1: Constant, scale2: Constant, dropout: Double, train: Boolean, gptOrder: Boolean) extends GenericModule[(Variable, Option[STen]), Variable]

A single block of the transformer self attention encoder using GELU

A single block of the transformer self attention encoder using GELU

Input is (data, maxLength) where data is (batch, sequence, input dimension), double tensor maxLength is a 1D or 2D long tensor used for attention masking.

The order of operations depends on gptOrder param. If gptOrder is true then:

  • y = attention(norm(input))+input
  • result = mlp(norm(y))+y
  • Note that in this case there is no normalization at the end of the transformer. One may wants to add one separately. This is how GPT2 is defined in hugging face or nanoGPT.
  • Note that the residual connection has a path which does not flow through the normalization.
    • dimension wise learnable scale parameter in each residual path

If gptOrder is false then:

  • y = norm(attention(input)+input )
  • result = norm(mlp(y)+y)
  • This follows chapter 11.7 in d2l.ai v1.0.0-beta0. (Same as in https://arxiv.org/pdf/1706.03762.pdf)
  • Note that the residual connection has a path which flows through the normalization.

Output is (bach, sequence, output dimension)

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[(Variable, Option[STen]), Variable]
class Object
trait Matchable
class Any
Show all
object TransformerEncoderBlock

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case class UnliftedModule[A, B, C, D, M <: StatefulModule2[A, B, C, D]](statefulModule: M & StatefulModule2[A, B, C, D])(implicit init: InitState[M, C]) extends GenericModule[A, B]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[A, B]
class Object
trait Matchable
class Any
Show all
object UnliftedModule

Attributes

Companion
class
Supertypes
class Object
trait Matchable
class Any
Self type
case class WeightNormLinear(weightsV: Constant, weightsG: Constant, bias: Option[Constant]) extends Module

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
object WeightNormLinear

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
case class WithInit[A, B, C, M <: StatefulModule[A, B, C]](module: M & StatefulModule[A, B, C], init: C) extends StatefulModule[A, B, C]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[(A, C), (B, C)]
class Object
trait Matchable
class Any
Show all
object WithInit

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
WithInit.type
case class WrapFun[A, B, M <: GenericModule[A, B], O](module: M, fun: (A, B) => O) extends GenericModule[A, (B, O)]

Attributes

Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait GenericModule[A, (B, O)]
class Object
trait Matchable
class Any
Show all
object WrapFun

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
WrapFun.type
object Yogi

Attributes

Companion
class
Supertypes
trait Product
trait Mirror
class Object
trait Matchable
class Any
Self type
Yogi.type
case class Yogi(parameters: Seq[(STen, PTag)], weightDecay: OptimizerHyperparameter, learningRate: OptimizerHyperparameter, beta1: OptimizerHyperparameter, beta2: OptimizerHyperparameter, eps: Double, clip0: Option[Double], debias: Boolean) extends Optimizer

The Yogi optimizer algorithm I added the decoupled weight decay term following https://arxiv.org/pdf/1711.05101.pdf

The Yogi optimizer algorithm I added the decoupled weight decay term following https://arxiv.org/pdf/1711.05101.pdf

Attributes

See also
Companion
object
Supertypes
trait Serializable
trait Product
trait Equals
trait Optimizer
class Object
trait Matchable
class Any
Show all
object sequence

Attributes

Supertypes
class Object
trait Matchable
class Any
Self type
sequence.type
case class simple(v: Double) extends OptimizerHyperparameter

Attributes

Supertypes
trait Serializable
trait Product
trait Equals
class Object
trait Matchable
class Any
Show all
object statefulSequence

Attributes

Supertypes
class Object
trait Matchable
class Any
Self type

Types

type StatefulModule[A, B, C] = GenericModule[(A, C), (B, C)]
type StatefulModule2[A, B, C, D] = GenericModule[(A, C), (B, D)]

Value members

Concrete methods

def gradientClippingInPlace(gradients: Seq[Option[STen]], theta: STen): Unit
def initLinear[S : Sc](in: Int, out: Int, tOpt: STenOptions): Constant
def loadMultiple[T1 <: GenericModule[_, _] : Load, T2 <: GenericModule[_, _] : Load](t1: T1, t2: T2, tensors: Seq[STen]): Unit
def loadMultiple[T1 <: GenericModule[_, _] : Load, T2 <: GenericModule[_, _] : Load, T3 <: GenericModule[_, _] : Load](t1: T1, t2: T2, t3: T3, tensors: Seq[STen]): Unit
def loadMultiple[T1 <: GenericModule[_, _] : Load, T2 <: GenericModule[_, _] : Load, T3 <: GenericModule[_, _] : Load, T4 <: GenericModule[_, _] : Load](t1: T1, t2: T2, t3: T3, t4: T4, tensors: Seq[STen]): Unit
def loadMultiple[T1 <: GenericModule[_, _] : Load, T2 <: GenericModule[_, _] : Load, T3 <: GenericModule[_, _] : Load, T4 <: GenericModule[_, _] : Load, T5 <: GenericModule[_, _] : Load](t1: T1, t2: T2, t3: T3, t4: T4, t5: T5, tensors: Seq[STen]): Unit
def loadMultiple[T1 <: GenericModule[_, _] : Load, T2 <: GenericModule[_, _] : Load, T3 <: GenericModule[_, _] : Load, T4 <: GenericModule[_, _] : Load, T5 <: GenericModule[_, _] : Load, T6 <: GenericModule[_, _] : Load](t1: T1, t2: T2, t3: T3, t4: T4, t5: T5, t6: T6, tensors: Seq[STen]): Unit
def loadMultiple[T1 <: GenericModule[_, _] : Load, T2 <: GenericModule[_, _] : Load, T3 <: GenericModule[_, _] : Load, T4 <: GenericModule[_, _] : Load, T5 <: GenericModule[_, _] : Load, T6 <: GenericModule[_, _] : Load, T7 <: GenericModule[_, _] : Load](t1: T1, t2: T2, t3: T3, t4: T4, t5: T5, t6: T6, t7: T7, tensors: Seq[STen]): Unit
def loadMultiple[T1 <: GenericModule[_, _] : Load, T2 <: GenericModule[_, _] : Load, T3 <: GenericModule[_, _] : Load, T4 <: GenericModule[_, _] : Load, T5 <: GenericModule[_, _] : Load, T6 <: GenericModule[_, _] : Load, T7 <: GenericModule[_, _] : Load, T8 <: GenericModule[_, _] : Load](t1: T1, t2: T2, t3: T3, t4: T4, t5: T5, t6: T6, t7: T7, t8: T8, tensors: Seq[STen]): Unit
def loadMultiple[T1 <: GenericModule[_, _] : Load, T2 <: GenericModule[_, _] : Load, T3 <: GenericModule[_, _] : Load, T4 <: GenericModule[_, _] : Load, T5 <: GenericModule[_, _] : Load, T6 <: GenericModule[_, _] : Load, T7 <: GenericModule[_, _] : Load, T8 <: GenericModule[_, _] : Load, T9 <: GenericModule[_, _] : Load](t1: T1, t2: T2, t3: T3, t4: T4, t5: T5, t6: T6, t7: T7, t8: T8, t9: T9, tensors: Seq[STen]): Unit
def loadMultiple[T1 <: GenericModule[_, _] : Load, T2 <: GenericModule[_, _] : Load, T3 <: GenericModule[_, _] : Load, T4 <: GenericModule[_, _] : Load, T5 <: GenericModule[_, _] : Load, T6 <: GenericModule[_, _] : Load, T7 <: GenericModule[_, _] : Load, T8 <: GenericModule[_, _] : Load, T9 <: GenericModule[_, _] : Load, T10 <: GenericModule[_, _] : Load](t1: T1, t2: T2, t3: T3, t4: T4, t5: T5, t6: T6, t7: T7, t8: T8, t9: T9, t10: T10, tensors: Seq[STen]): Unit
def loadMultiple[T1 <: GenericModule[_, _] : Load, T2 <: GenericModule[_, _] : Load, T3 <: GenericModule[_, _] : Load, T4 <: GenericModule[_, _] : Load, T5 <: GenericModule[_, _] : Load, T6 <: GenericModule[_, _] : Load, T7 <: GenericModule[_, _] : Load, T8 <: GenericModule[_, _] : Load, T9 <: GenericModule[_, _] : Load, T10 <: GenericModule[_, _] : Load, T11 <: GenericModule[_, _] : Load](t1: T1, t2: T2, t3: T3, t4: T4, t5: T5, t6: T6, t7: T7, t8: T8, t9: T9, t10: T10, t11: T11, tensors: Seq[STen]): Unit
def loadMultiple[T1 <: GenericModule[_, _] : Load, T2 <: GenericModule[_, _] : Load, T3 <: GenericModule[_, _] : Load, T4 <: GenericModule[_, _] : Load, T5 <: GenericModule[_, _] : Load, T6 <: GenericModule[_, _] : Load, T7 <: GenericModule[_, _] : Load, T8 <: GenericModule[_, _] : Load, T9 <: GenericModule[_, _] : Load, T10 <: GenericModule[_, _] : Load, T11 <: GenericModule[_, _] : Load, T12 <: GenericModule[_, _] : Load](t1: T1, t2: T2, t3: T3, t4: T4, t5: T5, t6: T6, t7: T7, t8: T8, t9: T9, t10: T10, t11: T11, t12: T12, tensors: Seq[STen]): Unit

Implicits

Implicits

final implicit def InitStateSyntax[M, C](m: M)(implicit is: InitState[M, C]): InitStateSyntax[M, C]
final implicit def LoadSyntax[M : Load](m: M): LoadSyntax[M]
final implicit def ToLift[M <: Module](mod: M & Module): ToLift[M]
final implicit def ToMappedState[A, B, C, M <: StatefulModule[A, B, C]](mod: M & StatefulModule[A, B, C]): ToMappedState[A, B, C, M]
final implicit def ToUnlift[A, B, C, D, M <: StatefulModule2[A, B, C, D]](mod: M & StatefulModule2[A, B, C, D])(implicit is: InitState[M, C]): ToUnlift[A, B, C, D, M]
final implicit def ToWithInit[A, B, C, M <: StatefulModule[A, B, C]](mod: M & StatefulModule[A, B, C]): ToWithInit[A, B, C, M]
final implicit def TrainingModeSyntax[M : TrainingMode](m: M): TrainingModeSyntax[M]
Generated with