case class MemoryStream[A](id: Int, sqlContext: SQLContext, numPartitions: Option[Int] = None)(implicit evidence$4: Encoder[A]) extends MemoryStreamBase[A] with MicroBatchStream with Logging with Product with Serializable
A Source that produces value stored in memory as they are added by the user. This Source is intended for use in unit tests as it can only replay data when the object is still available.
If numPartitions is provided, the rows will be redistributed to the given number of partitions.
- Alphabetic
- By Inheritance
- MemoryStream
- Serializable
- Serializable
- Product
- Equals
- Logging
- MicroBatchStream
- MemoryStreamBase
- SparkDataStream
- AnyRef
- Any
- Hide All
- Show All
- Public
- All
Instance Constructors
- new MemoryStream(id: Int, sqlContext: SQLContext, numPartitions: Option[Int] = None)(implicit arg0: Encoder[A])
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
addData(data: TraversableOnce[A]): Offset
- Definition Classes
- MemoryStream → MemoryStreamBase
-
def
addData(data: A*): connector.read.streaming.Offset
- Definition Classes
- MemoryStreamBase
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
-
val
attributes: Seq[AttributeReference]
- Attributes
- protected
- Definition Classes
- MemoryStreamBase
-
val
batches: ListBuffer[Array[UnsafeRow]]
All batches from
lastCommittedOffset + 1
tocurrentOffset
, inclusive.All batches from
lastCommittedOffset + 1
tocurrentOffset
, inclusive. Stored in a ListBuffer to facilitate removing committed batches.- Attributes
- protected
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
def
commit(end: connector.read.streaming.Offset): Unit
- Definition Classes
- MemoryStream → MemoryStreamBase → SparkDataStream
-
def
createReaderFactory(): PartitionReaderFactory
- Definition Classes
- MemoryStream → MicroBatchStream
-
var
currentOffset: LongOffset
- Attributes
- protected
-
def
deserializeOffset(json: String): connector.read.streaming.Offset
- Definition Classes
- MemoryStream → MemoryStreamBase → SparkDataStream
-
val
encoder: ExpressionEncoder[A]
- Definition Classes
- MemoryStreamBase
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
def
fullSchema(): StructType
- Definition Classes
- MemoryStreamBase
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
- val id: Int
-
def
initialOffset(): connector.read.streaming.Offset
- Definition Classes
- MemoryStream → MemoryStreamBase → SparkDataStream
-
def
initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
def
initializeLogIfNecessary(isInterpreter: Boolean): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
def
isTraceEnabled(): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
var
lastOffsetCommitted: LongOffset
Last offset that was discarded, or -1 if no commits have occurred.
Last offset that was discarded, or -1 if no commits have occurred. Note that the value -1 is used in calculations below and isn't just an arbitrary constant.
- Attributes
- protected
-
def
latestOffset(): connector.read.streaming.Offset
- Definition Classes
- MemoryStream → MicroBatchStream
-
def
log: Logger
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logName: String
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
val
logicalPlan: LogicalPlan
- Attributes
- protected
- Definition Classes
- MemoryStreamBase
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
- val numPartitions: Option[Int]
-
val
output: Seq[Attribute]
- Attributes
- protected
-
def
planInputPartitions(start: connector.read.streaming.Offset, end: connector.read.streaming.Offset): Array[InputPartition]
- Definition Classes
- MemoryStream → MicroBatchStream
- def reset(): Unit
- val sqlContext: SQLContext
-
var
startOffset: LongOffset
- Attributes
- protected
-
def
stop(): Unit
- Definition Classes
- MemoryStream → SparkDataStream
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
def
toDF(): DataFrame
- Definition Classes
- MemoryStreamBase
-
def
toDS(): Dataset[A]
- Definition Classes
- MemoryStreamBase
-
lazy val
toRow: Serializer[A]
- Attributes
- protected
- Definition Classes
- MemoryStreamBase
-
def
toString(): String
- Definition Classes
- MemoryStream → AnyRef → Any
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()