models

Type Members

1. sealed trait BatchETL extends AnyRef

   

2. case class BatchETLModel(name: String, inputs: List[ReaderModel], output: WriterModel, mlModels: List[MlModelOnlyInfo], strategy: Option[StrategyModel], kafkaAccessType: String, group: String = "default", isActive: Boolean = false) extends BatchETL with Product with Serializable

   

3. case class BatchGdprETLModel(name: String, dataStores: List[DataStoreConf], strategyConfig: String, inputs: List[ReaderModel], output: WriterModel, group: String = "default", isActive: Boolean = false) extends BatchETL with Product with Serializable

   

4. case class BatchJobExclusionConfig(isFullyExclusive: Boolean, restConfigExclusiveParams: Seq[String]) extends Product with Serializable

   

5. case class BatchJobInstanceModel(name: String, instanceOf: String, startTimestamp: Long, currentStatusTimestamp: Long, status: JobStatus, restConfig: Config = ConfigFactory.empty, error: Option[String] = None) extends Model with Product with Serializable

   

6. trait BatchJobJsonSupport extends DefaultJsonProtocol

   

7. case class BatchJobModel(name: String, description: String, owner: String, system: Boolean, creationTime: Long, etl: BatchETL, exclusivityConfig: BatchJobExclusionConfig = ...) extends Model with Product with Serializable

   

8. case class BatchSchedulerModel(name: String, cronExpression: String, batchJob: Option[String], options: Option[BsonDocument] = None, isActive: Boolean = true) extends Model with Product with Serializable

   

9. case class CompletionModel(toComplete: String, info: String) extends Product with Serializable

   

10. case class ContainsRawMatchingStrategy(dataframeKeyMatchingExpression: String) extends RawMatchingStrategy with Product with Serializable

    

11. case class CountEntry(timestamp: Instant, count: Map[String, Int]) extends Product with Serializable

    

12. case class Counts(logs: Seq[CountEntry], telemetry: Seq[CountEntry], events: Seq[CountEntry]) extends Product with Serializable

    

13. case class DashboardModel(url: String, needsFilterBox: Boolean) extends Product with Serializable

    

14. sealed trait DataStoreConf extends AnyRef

    

15. trait DataStoreConfJsonSupport extends DefaultJsonProtocol

    

16. abstract class DatastoreModel[DSC <: DatastoreCategory] extends Model

    

    Base datastore model.

17. case class DocumentModel(name: String, connectionString: String, schema: String) extends DatastoreModel[DocumentCategory] with Product with Serializable

    

18. case class ErrorModel(fileName: String, where: String, errorType: String, msg: String, content: String, indicator: String) extends Product with Serializable

    

19. case class EventEntry(eventType: String, eventId: String, severity: String, payload: String, timestamp: Instant, source: String, sourceId: String, eventRuleName: String) extends Product with Serializable

    

20. case class Events(found: Long, entries: Seq[EventEntry]) extends Product with Serializable

    

21. final case class ExactKeyValueMatchingStrategy() extends KeyValueMatchingStrategy with Product with Serializable

    

22. case class ExactRawMatchingStrategy(dataframeKeyMatchingExpression: String) extends RawMatchingStrategy with Product with Serializable

    

23. case class FreeCode(code: String) extends Product with Serializable

    

24. case class FreeCodeModel(name: String, code: String) extends Model with Product with Serializable

    

25. case class GdprStrategyModel(className: String, dataStoresConf: List[DataStoreConf], configuration: Option[String] = None) extends Product with Serializable

    

26. case class IndexModel(name: String, creationTime: Long, schema: Option[String], query: Option[String] = None, numShards: Option[Int] = Some(1), replicationFactor: Option[Int] = Some(1), rollingIndex: Boolean = true, idField: Option[String] = None, options: Map[String, String] = Map.empty) extends DatastoreModel[IndexCategory] with Product with Serializable

    

27. class IndexModelBuilder[Stage <: Stage, Kind <: DataStoreKind] extends AnyRef

    

    A builder able to create instances of IndexModel.

    A builder able to create instances of IndexModel.

    Stage

    The current Stage of the builder.

    Kind

    The kind of DataStore whose index is being built.

28. final case class KeyValueDataStoreConf(inputKeyColumn: String, correlationIdColumn: String, keyValueModel: KeyValueModel, keyValueMatchingStrategy: KeyValueMatchingStrategy) extends DataStoreConf with Product with Serializable

    

29. sealed trait KeyValueMatchingStrategy extends AnyRef

    

30. case class KeyValueModel(name: String, tableCatalog: String, dataFrameSchema: Option[String], options: Option[Seq[KeyValueOption]], useAvroSchemaManager: Boolean, avroSchemas: Option[Map[String, String]]) extends DatastoreModel[KeyValueCategory] with Product with Serializable

    

31. case class KeyValueOption(key: String, value: String) extends Product with Serializable

    

32. case class LogEntry(log_source: String, log_level: String, message: String, timestamp: Instant, thread: String, cause: Option[String] = None, stacktrace: Option[String] = None) extends Product with Serializable

    

33. case class Logs(found: Long, entries: Seq[LogEntry]) extends Product with Serializable

    

34. trait Metadata extends AnyRef

    

35. case class MetadataModel(id: String, sourceId: String, arrivalTimestamp: Long, lastSeenTimestamp: Long, path: Array[PathModel]) extends Product with Serializable

    

36. case class MetricEntry(source: SourceEntry, name: String) extends Product with Serializable

    

37. case class Metrics(found: Long, entries: Seq[MetricEntry]) extends Product with Serializable

    

38. case class MlModelOnlyInfo(name: String, version: String, className: Option[String] = None, timestamp: Option[Long] = None, modelFileId: Option[BsonObjectId] = None, favorite: Boolean = false, description: String = "") extends Model with Product with Serializable

    

39. trait Model extends AnyRef

    

40. case class MultiTopicModel extends DatastoreModel[TopicCategory] with Product with Serializable

    

    A model for grouping of topics.

    A model for grouping of topics.

    The name field specifies the name of the model, which is used as the unique identifier for the model in the models database.

    The topicNameField field specifies the field whose contents will be used as the name of the topic to which the message will be sent when writing to Kafka. The field must be of type string. The original field will be left as-is, so your schema must handle it (or you can use valueFieldsNames).

    The topicModelNames contains the names of the topic model that constitute this grouping of topics.

    The topic models that constitute this grouping of topics must: - consist of at least one topic model - be all different models - refer to different topics - use the same settings for everything but partitions and replicas

41. case class NoPartitionPruningStrategy() extends PartitionPruningStrategy with Product with Serializable

    

42. sealed trait PartitionPruningStrategy extends AnyRef

    

43. case class PathModel(name: String, ts: Long) extends Product with Serializable

    

44. final case class PipegraphInstanceModel(name: String, instanceOf: String, startTimestamp: Long, currentStatusTimestamp: Long, status: PipegraphStatus, executedByNode: Option[String], peerActor: Option[String], error: Option[String] = None) extends Model with Product with Serializable

    

45. case class PipegraphModel(name: String, description: String, owner: String, isSystem: Boolean, creationTime: Long, legacyStreamingComponents: List[LegacyStreamingETLModel], structuredStreamingComponents: List[StructuredStreamingETLModel], rtComponents: List[RTModel], dashboard: Option[DashboardModel] = None, labels: Set[String] = Set.empty) extends Model with Product with Serializable

    

    A model for a pipegraph, a processing pipeline abstraction.

    A model for a pipegraph, a processing pipeline abstraction.

    name

    name of the pipegraph

    description

    description of the pipegraph

    owner

    owner of the pipegraph

    isSystem

    whether the pipegraph is from the WASP system

    creationTime

    time of creation of the pipegraph

    legacyStreamingComponents

    components describing processing built on Spark Legacy Streaming

    structuredStreamingComponents

    components describing processing built on Spark Structured Streaming

    rtComponents

    components describing processing built on Akka actors

    dashboard

    dashboard of the pipegraph

46. final case class PrefixAndTimeBoundKeyValueMatchingStrategy(separator: String, pattern: String, locale: String = "UTC") extends KeyValueMatchingStrategy with Product with Serializable

    

47. final case class PrefixKeyValueMatchingStrategy() extends KeyValueMatchingStrategy with Product with Serializable

    

48. case class PrefixRawMatchingStrategy(dataframeKeyMatchingExpression: String) extends RawMatchingStrategy with Product with Serializable

    

49. case class ProcessGroupModel(name: String, content: BsonDocument, errorPort: String) extends Model with Product with Serializable

    

50. trait ProcessingComponentModel extends AnyRef

    

51. case class ProducerModel(name: String, className: String, topicName: Option[String], isActive: Boolean = false, configuration: Option[String] = None, isRemote: Boolean, isSystem: Boolean) extends Model with Product with Serializable

    

    DataSource class.

    DataSource class. The fields must be the same as the ones inside the MongoDB document associated with this model *

52. case class RTModel(name: String, inputs: List[ReaderModel], isActive: Boolean = false, strategy: Option[StrategyModel] = None, endpoint: Option[WriterModel] = None) extends ProcessingComponentModel with Product with Serializable

    

53. final case class RawDataStoreConf(inputKeyColumn: String, correlationIdColumn: String, rawModel: RawModel, rawMatchingStrategy: RawMatchingStrategy, partitionPruningStrategy: PartitionPruningStrategy, missingPathFailure: Boolean = false) extends DataStoreConf with Product with Serializable

    

54. sealed trait RawMatchingStrategy extends AnyRef

    

55. case class RawModel(name: String, uri: String, timed: Boolean = true, schema: String, options: RawOptions = RawOptions.default) extends DatastoreModel[RawCategory] with Product with Serializable

    

    A named model for data stored as files on a raw datastore (eg HDFS).

    A named model for data stored as files on a raw datastore (eg HDFS).

    The uri is augmented with time information if timed is true. For writers this means whether to use uri as-is or create timed namespaces (eg for HDFS, a subdirectory) inside; for readers whether to read from uri as-is or from the most recent timed namespace inside.

    schema is a json-encoded DataFrame schema, that is, a StructType. See DataType.fromJson and DataType.json.

    options control the underlying spark DataFrameWriter/Reader in the writers/readers using an instance of this model.

    name

    the name of the datastore

    uri

    the uri where the data files reside

    timed

    whether the uri must be augmented with time information

    schema

    the schema of the data

    options

    the options for the datastore

56. case class RawOptions(saveMode: String, format: String, extraOptions: Option[Map[String, String]] = None, partitionBy: Option[List[String]] = None) extends Product with Serializable

    

    Options for a raw datastore.

    Options for a raw datastore.

    saveMode specifies the behaviour when saving and the output uri already exists; valid values are:

    • "error", throw an error and do not save anything
    • "overwrite", overwrite existing data
    • "append", append to existing data
    • "ignore", do not save anything and don't throw any errors
    • "default", like "error" for it.agilelab.bigdata.wasp.consumers.SparkWriter, like "append" for it.agilelab.bigdata.wasp.consumers.SparkStreamingWriter

    format specifies the data format to use; valid values are:

    • "parquet" (this is the default)
    • "orc"
    • "json"
    • any format accepted by the available Spark DataFrameWriters

    extraOptions allows specifying any writer-specific options accepted by DataFrameReader/Writer.option

    partitionBy allows specifying columns to be used to partition the data by using different directories for different values

    saveMode

    specifies the behaviour when the output uri exists

    format

    specifies the format to use

    extraOptions

    extra options for the underlying writer

57. case class ReaderModel extends Product with Serializable

    

    A model for a reader, composed by a name, a datastoreModelName defining the datastore, a datastoreProduct defining the datastore software product to use, and any additional options needed to configure the reader.

58. case class SourceEntry(name: String) extends Product with Serializable

    

59. case class Sources(found: Long, entries: Seq[SourceEntry]) extends Product with Serializable

    

60. case class SqlSourceModel(name: String, connectionName: String, dbtable: String, partitioningInfo: Option[JdbcPartitioningInfo], numPartitions: Option[Int], fetchSize: Option[Int]) extends DatastoreModel[DatabaseCategory] with Product with Serializable

    

    Class representing a SqlSource model

    Class representing a SqlSource model

    name

    The name of the SqlSource model

    connectionName

    The name of the connection to use. N.B. have to be present in jdbc-subConfig

    dbtable

    The name of the table

    partitioningInfo

    optional - Partition info (column, lowerBound, upperBound)

    numPartitions

    optional - Number of partitions

    fetchSize

    optional - Fetch size

61. case class StrategyModel(className: String, configuration: Option[String] = None) extends Product with Serializable

    

62. case class StreamingReaderModel extends Product with Serializable

    

63. case class StructuredStreamingETLModel(name: String, group: String = "default", streamingInput: StreamingReaderModel, staticInputs: List[ReaderModel], streamingOutput: WriterModel, mlModels: List[MlModelOnlyInfo], strategy: Option[StrategyModel], triggerIntervalMs: Option[Long], options: Map[String, String] = Map.empty) extends ProcessingComponentModel with Product with Serializable

    

    A streaming processing component that leverages Spark's Structured Streaming API.

    A streaming processing component that leverages Spark's Structured Streaming API.

    name

    unique name of the processing component

    group

    group of which the processing component is part

    staticInputs

    list of inputs for static datasets

    streamingOutput

    streaming output

    mlModels

    machine learning models to be used in the processing

    strategy

    strategy model that defines the processing

    triggerIntervalMs

    trigger interval to use, in milliseconds

    options

    has no effect at all

64. case class TelemetryEntry(source: SourceEntry, metric: MetricEntry, messageId: String, value: Long, timestamp: Instant) extends Product with Serializable

    

65. case class TelemetryPoint(timestamp: Instant, value: Double) extends Product with Serializable

    

66. case class TelemetrySeries(source: SourceEntry, metric: MetricEntry, series: Seq[TelemetryPoint]) extends Product with Serializable

    

67. final case class TimeBasedBetweenPartitionPruningStrategy(columnName: String, isDateNumeric: Boolean, pattern: String, granularity: String) extends PartitionPruningStrategy with Product with Serializable

    

68. sealed abstract class TopicCompression extends AnyRef

    

69. case class TopicModel(name: String, creationTime: Long, partitions: Int, replicas: Int, topicDataType: String, keyFieldName: Option[String], headersFieldName: Option[String], valueFieldsNames: Option[Seq[String]], useAvroSchemaManager: Boolean, schema: BsonDocument, topicCompression: TopicCompression = TopicCompression.Disabled) extends DatastoreModel[TopicCategory] with Product with Serializable

    

    A model for a topic, that is, a message queue of some sort.

    A model for a topic, that is, a message queue of some sort. Right now this means just Kafka topics.

    The name field specifies the name of the topic, and doubles as the unique identifier for the model in the models database.

    The creationTime marks the time at which the model was generated.

    The partitions and replicas fields are used the configure the topic when the framework creates it.

    The topicDataType field specifies the format to use when encoding/decoding data to/from messages.

    The keyFieldName field allows you to optionally specify a field whose contents will be used as a message key when writing to Kafka. The field must be of type string or binary. The original field will be left as-is, so you schema must handle it (or you can use valueFieldsNames).

    The headersFieldName field allows you to optionally specify a field whose contents will be used as message headers when writing to Kafka. The field must contain an array of non-null objects which must have a non-null field headerKey of type string and a field headerValue of type binary. The original field will be left as-is, so your schema must handle it (or you can use valueFieldsNames).

    The valueFieldsNames field allows you to specify a list of field names to be used to filter the fields that get passed to the value encoding; with this you can filter out fields that you don't need in the value, obviating the need to handle them in the schema. This is especially useful when specifying the keyFieldName or headersFieldName. For the avro and json topic data type this is optional; for the plaintext and binary topic data types this field is mandatory and the list must contain a single value field name that has the proper type (string for plaintext and binary for binary).

    The schema field contains the schema to use when encoding the value.

70. case class WebMailModel(name: String) extends DatastoreModel[WebMailCategory] with Product with Serializable

    

71. case class WebsocketModel(name: String, host: String, port: String, resourceName: String, options: Option[BsonDocument] = None) extends DatastoreModel[WebSocketCategory] with Product with Serializable

    

72. case class WriterModel extends Product with Serializable

    

    A model for a writer, composed by a name, a datastoreModelName defining the datastore, a datastoreProduct defining the datastore software product to use, and any additional options needed to configure the writer.

73. case class LegacyStreamingETLModel(name: String, inputs: List[ReaderModel], output: WriterModel, mlModels: List[MlModelOnlyInfo], strategy: Option[StrategyModel], kafkaAccessType: String, group: String = "default", isActive: Boolean = false) extends ProcessingComponentModel with Product with Serializable

    

    Annotations

    @deprecated

    Deprecated

    (Since version 2.8.0)