KafkaSparkStructuredStreamingReader

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

2. final def ##(): Int

   

   Definition Classes

   AnyRef → Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

4. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

5. def clone(): AnyRef

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( ... )

6. def createStructuredStream(etl: StructuredStreamingETLModel, streamingReaderModel: StreamingReaderModel)(implicit ss: SparkSession): DataFrame

   

   Creates a streaming DataFrame from a Kafka streaming source.

   Creates a streaming DataFrame from a Kafka streaming source.

   The returned DataFrame will contain a column named "kafkaMetadata" column with message metadata and the message contents either as a single column named "value" or as multiple columns named after the value contents depending on the topic data type.

   The "kafkaMetadata" column contains the following: - key: bytes - headers: array of {headerKey: string, headerValue: bytes} - topic: string - partition: int - offset: long - timestamp: timestamp - timestampType: int

   The behaviour for message contents column(s) is the following: - the "avro" and "json" topic data types will output the columns specified by their schemas - the "plaintext" and "bytes" topic data types output a "value" column with the contents as string or bytes respectively

   Definition Classes

   KafkaSparkStructuredStreamingReader → SparkStructuredStreamingReader

7. final def eq(arg0: AnyRef): Boolean

   

   Definition Classes

   AnyRef

8. def equals(arg0: Any): Boolean

   

   Definition Classes

   AnyRef → Any

9. def finalize(): Unit

   

   Attributes

   protected[java.lang]

   Definition Classes

   AnyRef

   Annotations

   @throws( classOf[java.lang.Throwable] )

10. final def getClass(): Class[_]

    

    Definition Classes

    AnyRef → Any

11. def hashCode(): Int

    

    Definition Classes

    AnyRef → Any

12. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

13. val logger: WaspLogger

    

    Attributes

    protected

    Definition Classes

    Logging

14. final def ne(arg0: AnyRef): Boolean

    

    Definition Classes

    AnyRef

15. final def notify(): Unit

    

    Definition Classes

    AnyRef

16. final def notifyAll(): Unit

    

    Definition Classes

    AnyRef

17. final def synchronized[T0](arg0: ⇒ T0): T0

    

    Definition Classes

    AnyRef

18. def toString(): String

    

    Definition Classes

    AnyRef → Any

19. final def wait(): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

20. final def wait(arg0: Long, arg1: Int): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

21. final def wait(arg0: Long): Unit

    

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )