OpParams

Instance Constructors

new OpParams()
new OpParams(stageParams: Map[String, Map[String, Any]], readerParams: Map[String, ReaderParams], modelLocation: Option[String], writeLocation: Option[String], metricsLocation: Option[String], metricsCompress: Option[Boolean], metricsCodec: Option[String], batchDurationSecs: Option[Int], awaitTerminationTimeoutSecs: Option[Int], customTagName: Option[String], customTagValue: Option[String], logStageMetrics: Option[Boolean], collectStageMetrics: Option[Boolean], customParams: Map[String, Any], alternateReaderParams: Map[String, ReaderParams])

stageParams
a map of parameters to inject into stages. Format is Map(StageSimpleName -> Map(ParameterName -> Value)). Allows changing parameters away from defaults (and only defaults) at the level of stage type (all stages of the same type will get the same setting. Note: Will NOT override parameter values that have been set previously in code OR with a previous set of parameters.
readerParams
a map of parameters to inject into readers. Format is Map(ReaderType -> Map(ParameterName -> Value)). In order to use these parameters the read method in the reader must be overwritten to specifically take and use these parameters.
modelLocation
location to save model to or read model from
writeLocation
location to write out any data generated by flow
metricsLocation
location to write out any metrics generated by flow
metricsCompress
should compress metrics file
metricsCodec
compress with the supplied codec
batchDurationSecs
the time interval at which streaming data will be divided into batches
awaitTerminationTimeoutSecs
the time to await until streaming context termination
customTagName
tag name printed on log lines (e.g., by com.salesforce.op.utils.spark.OpSparkListener)
customTagValue
the value for the tag printed on log lines (e.g., by com.salesforce.op.utils.spark.OpSparkListener)
logStageMetrics
if com.salesforce.op.utils.spark.OpSparkListener should log metrics for every stage. Note: can increase logging significantly if app has too many stages.
collectStageMetrics
if com.salesforce.op.utils.spark.OpSparkListener should collect metrics for every stage. Note: can increase memory usage on the driver if app has too many stages.
customParams
any custom parameters
alternateReaderParams
a map of parameters to inject into readers - other than the main reader being called in the workflow run Format is Map(ReaderType -> Map(ParameterName -> Value)). In order to use these parameters the read method in the reader must be overwritten to specifically take and use these parameters.

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
val alternateReaderParams: Map[String, ReaderParams]

a map of parameters to inject into readers - other than the main reader being called in the workflow run Format is Map(ReaderType -> Map(ParameterName -> Value)).
a map of parameters to inject into readers - other than the main reader being called in the workflow run Format is Map(ReaderType -> Map(ParameterName -> Value)). In order to use these parameters the read method in the reader must be overwritten to specifically take and use these parameters.
final def asInstanceOf[T0]: T0

Definition Classes
Any
val awaitTerminationTimeoutSecs: Option[Int]

the time to await until streaming context termination
val batchDurationSecs: Option[Int]

the time interval at which streaming data will be divided into batches
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
val collectStageMetrics: Option[Boolean]

if com.salesforce.op.utils.spark.OpSparkListener should collect metrics for every stage.
if com.salesforce.op.utils.spark.OpSparkListener should collect metrics for every stage. Note: can increase memory usage on the driver if app has too many stages.
def copy(stageParams: Map[String, Map[String, Any]] = this.stageParams, readerParams: Map[String, ReaderParams] = this.readerParams, modelLocation: Option[String] = this.modelLocation, writeLocation: Option[String] = this.writeLocation, metricsLocation: Option[String] = this.metricsLocation, metricsCompress: Option[Boolean] = this.metricsCompress, metricsCodec: Option[String] = this.metricsCodec, batchDurationSecs: Option[Int] = this.batchDurationSecs, awaitTerminationTimeoutSecs: Option[Int] = this.awaitTerminationTimeoutSecs, customTagName: Option[String] = this.customTagName, customTagValue: Option[String] = this.customTagValue, logStageMetrics: Option[Boolean] = this.logStageMetrics, collectStageMetrics: Option[Boolean] = this.collectStageMetrics, customParams: Map[String, Any] = this.customParams, alternateReaderParams: Map[String, ReaderParams] = this.alternateReaderParams): OpParams
val customParams: Map[String, Any]

any custom parameters
val customTagName: Option[String]

tag name printed on log lines (e.g., by com.salesforce.op.utils.spark.OpSparkListener)
val customTagValue: Option[String]

the value for the tag printed on log lines (e.g., by com.salesforce.op.utils.spark.OpSparkListener)
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
val logStageMetrics: Option[Boolean]

if com.salesforce.op.utils.spark.OpSparkListener should log metrics for every stage.
if com.salesforce.op.utils.spark.OpSparkListener should log metrics for every stage. Note: can increase logging significantly if app has too many stages.
val metricsCodec: Option[String]

compress with the supplied codec
val metricsCompress: Option[Boolean]

should compress metrics file
val metricsLocation: Option[String]

location to write out any metrics generated by flow
val modelLocation: Option[String]

location to save model to or read model from
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val readerParams: Map[String, ReaderParams]

a map of parameters to inject into readers.
a map of parameters to inject into readers. Format is Map(ReaderType -> Map(ParameterName -> Value)). In order to use these parameters the read method in the reader must be overwritten to specifically take and use these parameters.
val stageParams: Map[String, Map[String, Any]]

a map of parameters to inject into stages.
a map of parameters to inject into stages. Format is Map(StageSimpleName -> Map(ParameterName -> Value)). Allows changing parameters away from defaults (and only defaults) at the level of stage type (all stages of the same type will get the same setting. Note: Will NOT override parameter values that have been set previously in code OR with a previous set of parameters.
def switchReaderParams(): OpParams

Switch the reader params with the alternate reader params and return a new params instance this is necessary because the readers will always try to use readerParams so if you want to use the alternate reader params they need to be moved to readerParams
Switch the reader params with the alternate reader params and return a new params instance this is necessary because the readers will always try to use readerParams so if you want to use the alternate reader params they need to be moved to readerParams
returns
a new params instance
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toJson(pretty: Boolean): String

Definition Classes
JsonLike
def toString(): String

Definition Classes
JsonLike → AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def withValues(readLocations: Map[String, String] = Map.empty, writeLocation: Option[String] = None, modelLocation: Option[String] = None, metricsLocation: Option[String] = None, batchDurationSecs: Option[Int] = None, awaitTerminationTimeoutSecs: Option[Int] = None, alternateReadLocations: Map[String, String] = Map.empty): OpParams

Copy op params with new specified values
Copy op params with new specified values
readLocations
read locations
writeLocation
write locations
modelLocation
model location
metricsLocation
metrics location
batchDurationSecs
the time interval at which streaming data will be divided into batches
awaitTerminationTimeoutSecs
the time to await until streaming context termination
alternateReadLocations
read locations for readers other than the main reader in the workflow
returns
a copy of params with new values
val writeLocation: Option[String]

location to write out any data generated by flow

Related Docs: object OpParams | package op

final class OpParams extends JsonLike

Instance Constructors

new OpParams()

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

val alternateReaderParams: Map[String, ReaderParams]

final def asInstanceOf[T0]: T0

val awaitTerminationTimeoutSecs: Option[Int]

val batchDurationSecs: Option[Int]

def clone(): AnyRef

val collectStageMetrics: Option[Boolean]

val customParams: Map[String, Any]

val customTagName: Option[String]

val customTagValue: Option[String]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

val logStageMetrics: Option[Boolean]

val metricsCodec: Option[String]

val metricsCompress: Option[Boolean]

val metricsLocation: Option[String]

val modelLocation: Option[String]

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val readerParams: Map[String, ReaderParams]

val stageParams: Map[String, Map[String, Any]]

def switchReaderParams(): OpParams

final def synchronized[T0](arg0: ⇒ T0): T0

def toJson(pretty: Boolean): String

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

val writeLocation: Option[String]

Inherited from JsonLike

Inherited from Serializable

Inherited from Serializable

Inherited from AnyRef

Inherited from Any

Ungrouped