org.elasticsearch.hadoop.pig

Class EsStorage

java.lang.Object

org.apache.pig.LoadFunc

org.elasticsearch.hadoop.pig.EsStorage

All Implemented Interfaces:

org.apache.pig.LoadMetadata, org.apache.pig.LoadPushDown, org.apache.pig.StoreFuncInterface, org.apache.pig.StoreMetadata

public class EsStorage
extends org.apache.pig.LoadFunc
implements org.apache.pig.LoadMetadata, org.apache.pig.LoadPushDown, org.apache.pig.StoreFuncInterface, org.apache.pig.StoreMetadata

Pig storage for reading and writing data into an ElasticSearch index. Uses the tuple implied schema to create the resulting JSON string sent to ElasticSearch.

Typical usage is:

 A = LOAD 'twitter/_search?q=kimchy' USING org.elasticsearch.hadoop.pig.ESStorage();
 

 STORE A INTO '' USING org.elasticsearch.hadoop.pig.ESStorage();
 

The ElasticSearch host/port can be specified through Hadoop properties (see package description) or passed to the EsStorage(String...) constructor.

Nested Class Summary

Nested classes/interfaces inherited from interface org.apache.pig.LoadPushDown

org.apache.pig.LoadPushDown.OperatorSet, org.apache.pig.LoadPushDown.RequiredField, org.apache.pig.LoadPushDown.RequiredFieldList, org.apache.pig.LoadPushDown.RequiredFieldResponse

Constructor Summary

Constructors

Constructor and Description
EsStorage()
EsStorage(java.lang.String... configuration)

Method Summary

Modifier and Type	Method and Description
void	checkSchema(org.apache.pig.ResourceSchema s)
void	cleanupOnFailure(java.lang.String location, org.apache.hadoop.mapreduce.Job job)
void	cleanupOnSuccess(java.lang.String location, org.apache.hadoop.mapreduce.Job job)
java.util.List<org.apache.pig.LoadPushDown.OperatorSet>	getFeatures()
org.apache.hadoop.mapreduce.InputFormat	getInputFormat()
org.apache.pig.data.Tuple	getNext()
org.apache.hadoop.mapreduce.OutputFormat<java.lang.Object,java.util.Map<org.apache.hadoop.io.Writable,org.apache.hadoop.io.Writable>>	getOutputFormat()
java.lang.String[]	getPartitionKeys(java.lang.String location, org.apache.hadoop.mapreduce.Job job)
org.apache.pig.ResourceSchema	getSchema(java.lang.String location, org.apache.hadoop.mapreduce.Job job)
org.apache.pig.ResourceStatistics	getStatistics(java.lang.String location, org.apache.hadoop.mapreduce.Job job)
void	prepareToRead(org.apache.hadoop.mapreduce.RecordReader reader, org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit split)
void	prepareToWrite(org.apache.hadoop.mapreduce.RecordWriter writer)
org.apache.pig.LoadPushDown.RequiredFieldResponse	pushProjection(org.apache.pig.LoadPushDown.RequiredFieldList requiredFieldList)
void	putNext(org.apache.pig.data.Tuple t)
java.lang.String	relativeToAbsolutePath(java.lang.String location, org.apache.hadoop.fs.Path curDir)
java.lang.String	relToAbsPathForStoreLocation(java.lang.String location, org.apache.hadoop.fs.Path curDir)
void	setLocation(java.lang.String location, org.apache.hadoop.mapreduce.Job job)
void	setPartitionFilter(org.apache.pig.Expression partitionFilter)
void	setStoreFuncUDFContextSignature(java.lang.String signature)
void	setStoreLocation(java.lang.String location, org.apache.hadoop.mapreduce.Job job)
void	setUDFContextSignature(java.lang.String signature)
void	storeSchema(org.apache.pig.ResourceSchema schema, java.lang.String location, org.apache.hadoop.mapreduce.Job job)
void	storeStatistics(org.apache.pig.ResourceStatistics stats, java.lang.String location, org.apache.hadoop.mapreduce.Job job)

Methods inherited from class org.apache.pig.LoadFunc

getAbsolutePath, getCacheFiles, getLoadCaster, getPathStrings, getShipFiles, join, warn

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

EsStorage

public EsStorage()

EsStorage

public EsStorage(java.lang.String... configuration)

Method Detail

relToAbsPathForStoreLocation

public java.lang.String relToAbsPathForStoreLocation(java.lang.String location,
                                                     org.apache.hadoop.fs.Path curDir)
                                              throws java.io.IOException

Specified by:

relToAbsPathForStoreLocation in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

setStoreFuncUDFContextSignature

public void setStoreFuncUDFContextSignature(java.lang.String signature)

Specified by:

setStoreFuncUDFContextSignature in interface org.apache.pig.StoreFuncInterface

checkSchema

public void checkSchema(org.apache.pig.ResourceSchema s)
                 throws java.io.IOException

Specified by:

checkSchema in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

setStoreLocation

public void setStoreLocation(java.lang.String location,
                             org.apache.hadoop.mapreduce.Job job)
                      throws java.io.IOException

Specified by:

setStoreLocation in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

getOutputFormat

public org.apache.hadoop.mapreduce.OutputFormat<java.lang.Object,java.util.Map<org.apache.hadoop.io.Writable,org.apache.hadoop.io.Writable>> getOutputFormat()
                                                                                                                                                      throws java.io.IOException

Specified by:

getOutputFormat in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

prepareToWrite

public void prepareToWrite(org.apache.hadoop.mapreduce.RecordWriter writer)
                    throws java.io.IOException

Specified by:

prepareToWrite in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

putNext

public void putNext(org.apache.pig.data.Tuple t)
             throws java.io.IOException

Specified by:

putNext in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

cleanupOnFailure

public void cleanupOnFailure(java.lang.String location,
                             org.apache.hadoop.mapreduce.Job job)
                      throws java.io.IOException

Specified by:

cleanupOnFailure in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

cleanupOnSuccess

public void cleanupOnSuccess(java.lang.String location,
                             org.apache.hadoop.mapreduce.Job job)
                      throws java.io.IOException

Specified by:

cleanupOnSuccess in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

storeStatistics

public void storeStatistics(org.apache.pig.ResourceStatistics stats,
                            java.lang.String location,
                            org.apache.hadoop.mapreduce.Job job)
                     throws java.io.IOException

Specified by:

storeStatistics in interface org.apache.pig.StoreMetadata

Throws:

java.io.IOException

storeSchema

public void storeSchema(org.apache.pig.ResourceSchema schema,
                        java.lang.String location,
                        org.apache.hadoop.mapreduce.Job job)
                 throws java.io.IOException

Specified by:

storeSchema in interface org.apache.pig.StoreMetadata

Throws:

java.io.IOException

setLocation

public void setLocation(java.lang.String location,
                        org.apache.hadoop.mapreduce.Job job)
                 throws java.io.IOException

Specified by:

setLocation in class org.apache.pig.LoadFunc

Throws:

java.io.IOException

relativeToAbsolutePath

public java.lang.String relativeToAbsolutePath(java.lang.String location,
                                               org.apache.hadoop.fs.Path curDir)
                                        throws java.io.IOException

Overrides:

relativeToAbsolutePath in class org.apache.pig.LoadFunc

Throws:

java.io.IOException

getInputFormat

public org.apache.hadoop.mapreduce.InputFormat getInputFormat()
                                                       throws java.io.IOException

Specified by:

getInputFormat in class org.apache.pig.LoadFunc

Throws:

java.io.IOException

prepareToRead

public void prepareToRead(org.apache.hadoop.mapreduce.RecordReader reader,
                          org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit split)
                   throws java.io.IOException

Specified by:

prepareToRead in class org.apache.pig.LoadFunc

Throws:

java.io.IOException

getNext

public org.apache.pig.data.Tuple getNext()
                                  throws java.io.IOException

Specified by:

getNext in class org.apache.pig.LoadFunc

Throws:

java.io.IOException

getFeatures

public java.util.List<org.apache.pig.LoadPushDown.OperatorSet> getFeatures()

Specified by:

getFeatures in interface org.apache.pig.LoadPushDown

pushProjection

public org.apache.pig.LoadPushDown.RequiredFieldResponse pushProjection(org.apache.pig.LoadPushDown.RequiredFieldList requiredFieldList)
                                                                 throws org.apache.pig.impl.logicalLayer.FrontendException

Specified by:

pushProjection in interface org.apache.pig.LoadPushDown

Throws:

org.apache.pig.impl.logicalLayer.FrontendException

getSchema

public org.apache.pig.ResourceSchema getSchema(java.lang.String location,
                                               org.apache.hadoop.mapreduce.Job job)
                                        throws java.io.IOException

Specified by:

getSchema in interface org.apache.pig.LoadMetadata

Throws:

java.io.IOException

getStatistics

public org.apache.pig.ResourceStatistics getStatistics(java.lang.String location,
                                                       org.apache.hadoop.mapreduce.Job job)
                                                throws java.io.IOException

Specified by:

getStatistics in interface org.apache.pig.LoadMetadata

Throws:

java.io.IOException

getPartitionKeys

public java.lang.String[] getPartitionKeys(java.lang.String location,
                                           org.apache.hadoop.mapreduce.Job job)
                                    throws java.io.IOException

Specified by:

getPartitionKeys in interface org.apache.pig.LoadMetadata

Throws:

java.io.IOException

setPartitionFilter

public void setPartitionFilter(org.apache.pig.Expression partitionFilter)
                        throws java.io.IOException

Specified by:

setPartitionFilter in interface org.apache.pig.LoadMetadata

Throws:

java.io.IOException

setUDFContextSignature

public void setUDFContextSignature(java.lang.String signature)

Overrides:

setUDFContextSignature in class org.apache.pig.LoadFunc