class FileStreamSource extends SupportsAdmissionControl with Source with Logging
A very simple source that reads files from the given directory as they appear.
- Alphabetic
- By Inheritance
- FileStreamSource
- Logging
- Source
- SupportsAdmissionControl
- SparkDataStream
- AnyRef
- Any
- Hide All
- Show All
- Public
- All
Instance Constructors
- new FileStreamSource(sparkSession: SparkSession, path: String, fileFormatClassName: String, schema: StructType, partitionColumns: Seq[String], metadataPath: String, options: Map[String, String])
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
def
commit(end: Offset): Unit
Informs the source that Spark has completed processing all data for offsets less than or equal to
end
and will only request offsets greater thanend
in the future.Informs the source that Spark has completed processing all data for offsets less than or equal to
end
and will only request offsets greater thanend
in the future.- Definition Classes
- FileStreamSource → Source
-
def
commit(end: connector.read.streaming.Offset): Unit
- Definition Classes
- Source → SparkDataStream
-
def
currentLogOffset: Long
Return the latest offset in the FileStreamSourceLog
-
def
deserializeOffset(json: String): connector.read.streaming.Offset
- Definition Classes
- Source → SparkDataStream
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
equals(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
def
getBatch(start: Option[Offset], end: Offset): DataFrame
Returns the data that is between the offsets (
start
,end
].Returns the data that is between the offsets (
start
,end
].- Definition Classes
- FileStreamSource → Source
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
def
getDefaultReadLimit(): ReadLimit
- Definition Classes
- FileStreamSource → SupportsAdmissionControl
-
def
getOffset: Option[Offset]
Returns the maximum available offset for this source.
Returns the maximum available offset for this source. Returns
None
if this source has never received any data.- Definition Classes
- FileStreamSource → Source
-
def
hashCode(): Int
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
def
initialOffset(): connector.read.streaming.Offset
- Definition Classes
- Source → SparkDataStream
-
def
initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
def
initializeLogIfNecessary(isInterpreter: Boolean): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
def
isTraceEnabled(): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
def
latestOffset(startOffset: connector.read.streaming.Offset, limit: ReadLimit): connector.read.streaming.Offset
- Definition Classes
- FileStreamSource → SupportsAdmissionControl
-
def
log: Logger
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logName: String
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
val
schema: StructType
Returns the schema of the data from this source
Returns the schema of the data from this source
- Definition Classes
- FileStreamSource → Source
-
val
seenFiles: SeenFilesMap
A mapping from a file that we have processed to some timestamp it was last modified.
-
def
stop(): Unit
- Definition Classes
- FileStreamSource → SparkDataStream
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
def
toString(): String
- Definition Classes
- FileStreamSource → AnyRef → Any
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
def
withBatchingLocked[T](func: ⇒ T): T
For test only.
For test only. Run
func
with the internal lock to make sure whenfunc
is running, the current offset won't be changed and no new batch will be emitted.