org.elasticsearch.spark.sql.streaming

Class EsStreamQueryWriter

java.lang.Object

org.elasticsearch.spark.rdd.EsRDDWriter<org.apache.spark.sql.catalyst.InternalRow>

org.elasticsearch.spark.sql.streaming.EsStreamQueryWriter

All Implemented Interfaces:

Serializable

public class EsStreamQueryWriter
extends EsRDDWriter<org.apache.spark.sql.catalyst.InternalRow>

Takes in iterator of InternalRow objects from a partition of data, writes it to Elasticsearch, and manages the streaming commit protocol.

See Also:

Serialized Form

Constructor Summary

Constructors

Constructor and Description
EsStreamQueryWriter(String serializedSettings, StructType schema, EsCommitProtocol commitProtocol)

Method Summary

Modifier and Type	Method and Description
protected Class<? extends org.elasticsearch.hadoop.serialization.BytesConverter>	bytesConverter()
protected Class<? extends org.elasticsearch.hadoop.serialization.field.FieldExtractor>	fieldExtractor()
protected Object	processData(scala.collection.Iterator<org.apache.spark.sql.catalyst.InternalRow> data)
TaskCommit	run(TaskContext taskContext, scala.collection.Iterator<org.apache.spark.sql.catalyst.InternalRow> data)
protected Class<? extends org.elasticsearch.hadoop.serialization.builder.ValueWriter<?>>	valueWriter()
void	write(TaskContext taskContext, scala.collection.Iterator<org.apache.spark.sql.catalyst.InternalRow> data)

Methods inherited from class org.elasticsearch.spark.rdd.EsRDDWriter

log, metadataExtractor, metaExtractor, runtimeMetadata, serializedSettings, settings, userProvider

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

EsStreamQueryWriter

public EsStreamQueryWriter(String serializedSettings,
                           StructType schema,
                           EsCommitProtocol commitProtocol)

Method Detail

bytesConverter

protected Class<? extends org.elasticsearch.hadoop.serialization.BytesConverter> bytesConverter()

Overrides:

bytesConverter in class EsRDDWriter<org.apache.spark.sql.catalyst.InternalRow>

fieldExtractor

protected Class<? extends org.elasticsearch.hadoop.serialization.field.FieldExtractor> fieldExtractor()

Overrides:

fieldExtractor in class EsRDDWriter<org.apache.spark.sql.catalyst.InternalRow>

processData

protected Object processData(scala.collection.Iterator<org.apache.spark.sql.catalyst.InternalRow> data)

Overrides:

processData in class EsRDDWriter<org.apache.spark.sql.catalyst.InternalRow>

run

public TaskCommit run(TaskContext taskContext,
                      scala.collection.Iterator<org.apache.spark.sql.catalyst.InternalRow> data)

valueWriter

protected Class<? extends org.elasticsearch.hadoop.serialization.builder.ValueWriter<?>> valueWriter()

Overrides:

valueWriter in class EsRDDWriter<org.apache.spark.sql.catalyst.InternalRow>

write

public void write(TaskContext taskContext,
                  scala.collection.Iterator<org.apache.spark.sql.catalyst.InternalRow> data)

Overrides:

write in class EsRDDWriter<org.apache.spark.sql.catalyst.InternalRow>