AggregatorTransformer

Instance Constructors

new AggregatorTransformer(statsReceiver: StatsReceiver, stateStoreName: String, timerStoreName: String, windowSize: Duration, allowedLateness: Duration, initializer: () ⇒ Aggregate, aggregator: ((K, V), Aggregate) ⇒ Aggregate, customWindowStart: (Time, K, V) ⇒ Long, emitOnClose: Boolean = false, queryableAfterClose: Duration, emitUpdatedEntriesOnCommit: Boolean, commitInterval: Duration)

statsReceiver
The StatsReceiver for collecting stats
stateStoreName
the name of the StateStore used to maintain the counts.
timerStoreName
the name of the StateStore used to maintain the timers.
windowSize
splits the stream of data into buckets of data of windowSize, based on the timestamp of each message.
allowedLateness
allow messages that are upto this amount late to be added to the store, otherwise they are emitted as restatements.
initializer
Initializer function that computes an initial intermediate aggregation result
aggregator
Aggregator function that computes a new aggregate result
emitOnClose
Emit messages for each entry in the window when the window close. Emitted entries will have a WindowResultType set to WindowClosed.
queryableAfterClose
allow state to be queried upto this amount after the window is closed.
emitUpdatedEntriesOnCommit
Emit messages for each updated entry in the window on the Kafka Streams commit interval. Emitted entries will have a WindowResultType set to WindowOpen.

Type Members

implicit class RichLong extends AnyRef

Definition Classes
ProcessorContextLogging

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def close(): Unit

Definition Classes
FinatraTransformerV2 → Transformer
val commitInterval: Duration

Definition Classes
AggregatorTransformer → Flushing
final def debug(message: ⇒ Any): Unit

Attributes
protected[this]
Definition Classes
ProcessorContextLogging
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def error(message: ⇒ Any): Unit

Attributes
protected[this]
Definition Classes
ProcessorContextLogging
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
val finatraKeyValueStoresMap: Map[String, FinatraKeyValueStore[_, _]]

Attributes
protected[com.twitter.finatra.streams]
Definition Classes
FinatraTransformerV2
final def forward(key: TimeWindowed[K], value: WindowedValue[Aggregate], timestamp: Long): Unit

Attributes
protected
Definition Classes
FinatraTransformerV2
final def forward(key: TimeWindowed[K], value: WindowedValue[Aggregate]): Unit

Attributes
protected
Definition Classes
FinatraTransformerV2
def getCachingKeyValueStore[KK, VV](name: String)(implicit arg0: ClassTag[KK]): CachingFinatraKeyValueStore[KK, VV]

Lookup a caching key value store by name
Lookup a caching key value store by name
KK
Type of keys in the store
VV
Type of values in the store
name
The name of the store
returns
A caching key value store

Attributes
protected
Definition Classes
CachingKeyValueStores
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
final def getKeyValueStore[KK, VV](name: String)(implicit arg0: ClassTag[KK]): FinatraKeyValueStore[KK, VV]

Attributes
protected
Definition Classes
FinatraTransformerV2
def getPersistentTimerStore[TimerKey](timerStoreName: String, onTimer: (Time, TimerMetadata, TimerKey) ⇒ Unit, punctuationType: PunctuationType, maxTimerFiresPerWatermark: Int = 10000): PersistentTimerStore[TimerKey]

Attributes
protected
Definition Classes
PersistentTimers
def hashCode(): Int

Definition Classes
AnyRef → Any
final def info(message: ⇒ Any): Unit

Attributes
protected[this]
Definition Classes
ProcessorContextLogging
final def init(processorContext: ProcessorContext): Unit

Definition Classes
FinatraTransformerV2 → Transformer
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def onClose(): Unit

Definition Classes
Flushing → OnClose
def onFlush(): Unit

Definition Classes
CachingKeyValueStores → Flushing
def onInit(): Unit

Definition Classes
AggregatorTransformer → PersistentTimers → Flushing → OnInit
def onMessage(time: Time, key: K, value: V): Unit

Callback method which is called for every message in the stream this Transformer is attached to.
Callback method which is called for every message in the stream this Transformer is attached to. Implementers of this method may emit 0 or more records by using the processorContext.
key
the key of the message
value
the value of the message

Definition Classes
AggregatorTransformer → FinatraTransformerV2
final def onWatermark(watermark: Watermark): Unit

Definition Classes
PersistentTimers → OnWatermark
def processorContext: ProcessorContext

Attributes
protected
Definition Classes
FinatraTransformerV2 → ProcessorContextLogging
val statsReceiver: StatsReceiver

The StatsReceiver for collecting stats
The StatsReceiver for collecting stats

Definition Classes
AggregatorTransformer → CachingKeyValueStores
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
final def taskIdStr: String

Attributes
protected
Definition Classes
ProcessorContextLogging
final def timeStr: String

Attributes
protected
Definition Classes
ProcessorContextLogging
def toString(): String

Definition Classes
AnyRef → Any
final def trace(message: ⇒ Any): Unit

Attributes
protected[this]
Definition Classes
ProcessorContextLogging
final def transform(k: K, v: V): (TimeWindowed[K], WindowedValue[Aggregate])

Definition Classes
FinatraTransformerV2 → Transformer
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def warn(message: ⇒ Any): Unit

Attributes
protected[this]
Definition Classes
ProcessorContextLogging
final def watermark: Watermark

Attributes
protected
Definition Classes
FinatraTransformerV2

Deprecated Value Members

def logger: Logger

Attributes
protected
Definition Classes
ProcessorContextLogging
Annotations
@deprecated
Deprecated
Use error, warn, info, debug, or trace methods directly

Related Doc: package transformer

class AggregatorTransformer[K, V, Aggregate] extends FinatraTransformerV2[K, V, TimeWindowed[K], WindowedValue[Aggregate]] with CachingKeyValueStores[K, V, TimeWindowed[K], WindowedValue[Aggregate]] with PersistentTimers

Instance Constructors

Type Members

implicit class RichLong extends AnyRef

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

final def close(): Unit

val commitInterval: Duration

final def debug(message: ⇒ Any): Unit

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

final def error(message: ⇒ Any): Unit

def finalize(): Unit

val finatraKeyValueStoresMap: Map[String, FinatraKeyValueStore[_, _]]

final def forward(key: TimeWindowed[K], value: WindowedValue[Aggregate], timestamp: Long): Unit

final def forward(key: TimeWindowed[K], value: WindowedValue[Aggregate]): Unit

def getCachingKeyValueStore[KK, VV](name: String)(implicit arg0: ClassTag[KK]): CachingFinatraKeyValueStore[KK, VV]

final def getClass(): Class[_]

final def getKeyValueStore[KK, VV](name: String)(implicit arg0: ClassTag[KK]): FinatraKeyValueStore[KK, VV]

def getPersistentTimerStore[TimerKey](timerStoreName: String, onTimer: (Time, TimerMetadata, TimerKey) ⇒ Unit, punctuationType: PunctuationType, maxTimerFiresPerWatermark: Int = 10000): PersistentTimerStore[TimerKey]

def hashCode(): Int

final def info(message: ⇒ Any): Unit

final def init(processorContext: ProcessorContext): Unit

final def isInstanceOf[T0]: Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def onClose(): Unit

def onFlush(): Unit

def onInit(): Unit

def onMessage(time: Time, key: K, value: V): Unit

final def onWatermark(watermark: Watermark): Unit

def processorContext: ProcessorContext

val statsReceiver: StatsReceiver

final def synchronized[T0](arg0: ⇒ T0): T0

final def taskIdStr: String

final def timeStr: String

def toString(): String

final def trace(message: ⇒ Any): Unit

final def transform(k: K, v: V): (TimeWindowed[K], WindowedValue[Aggregate])

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

final def warn(message: ⇒ Any): Unit

final def watermark: Watermark

Deprecated Value Members

def logger: Logger

Inherited from PersistentTimers

Inherited from CachingKeyValueStores[K, V, TimeWindowed[K], WindowedValue[Aggregate]]

Inherited from FlushingTransformer[K, V, TimeWindowed[K], WindowedValue[Aggregate]]

Inherited from Flushing

Inherited from FinatraTransformerV2[K, V, TimeWindowed[K], WindowedValue[Aggregate]]

Inherited from ProcessorContextLogging

Inherited from OnClose

Inherited from OnWatermark

Inherited from OnInit

Inherited from Transformer[K, V, (TimeWindowed[K], WindowedValue[Aggregate])]

Inherited from AnyRef

Inherited from Any

Ungrouped