org.apache.cassandra.hadoop.pig

Class CqlNativeStorage

java.lang.Object

org.apache.pig.LoadFunc

org.apache.cassandra.hadoop.pig.CqlNativeStorage

All Implemented Interfaces:

org.apache.pig.LoadMetadata, org.apache.pig.StoreFuncInterface

public class CqlNativeStorage
extends org.apache.pig.LoadFunc
implements org.apache.pig.StoreFuncInterface, org.apache.pig.LoadMetadata

Field Summary

Fields

Modifier and Type	Field and Description
protected java.lang.String	column_family
protected org.apache.hadoop.conf.Configuration	conf
protected java.lang.String	DEFAULT_INPUT_FORMAT
protected java.lang.String	DEFAULT_OUTPUT_FORMAT
protected java.lang.String	initHostAddress
protected java.lang.String	inputFormatClass
protected java.lang.String	keyspace
protected java.lang.String	loadSignature
protected int	nativeProtocolVersion
protected java.lang.String	outputFormatClass
protected java.lang.String	partitionerClass
protected java.lang.String	password
protected java.lang.String	rpcPort
protected int	splitSize
protected java.lang.String	storeSignature
protected boolean	usePartitionFilter
protected java.lang.String	username

Constructor Summary

Constructors

Constructor and Description
CqlNativeStorage()
CqlNativeStorage(int pageSize)

Method Summary

Modifier and Type	Method and Description
protected static org.apache.cassandra.hadoop.pig.TableInfo	cfdefFromString(java.lang.String st) convert string back to CfDef
protected static java.lang.String	cfdefToString(com.datastax.driver.core.TableMetadata cfDef) convert CfDef to string
void	checkSchema(org.apache.pig.ResourceSchema schema)
void	cleanupOnFailure(java.lang.String failure, org.apache.hadoop.mapreduce.Job job)
void	cleanupOnSuccess(java.lang.String location, org.apache.hadoop.mapreduce.Job job)
protected com.datastax.driver.core.TableMetadata	getCfInfo(com.datastax.driver.core.Session client) return the CfInfo for the column family
protected org.apache.cassandra.hadoop.pig.TableInfo	getCfInfo(java.lang.String signature) get the columnfamily definition for the signature
protected java.lang.String	getFullyQualifiedClassName(java.lang.String classname) get the full class name
org.apache.hadoop.mapreduce.InputFormat	getInputFormat()
org.apache.pig.data.Tuple	getNext() get next row
org.apache.hadoop.mapreduce.OutputFormat	getOutputFormat() output format
java.lang.String[]	getPartitionKeys(java.lang.String location, org.apache.hadoop.mapreduce.Job job) return partition keys
static java.util.Map<java.lang.String,java.lang.String>	getQueryMap(java.lang.String query) decompose the query to store the parameters in a map
org.apache.pig.ResourceSchema	getSchema(java.lang.String location, org.apache.hadoop.mapreduce.Job job) schema: (value, value, value) where keys are in the front.
org.apache.pig.ResourceStatistics	getStatistics(java.lang.String location, org.apache.hadoop.mapreduce.Job job)
protected java.util.Map<java.nio.ByteBuffer,AbstractType>	getValidatorMap(org.apache.cassandra.hadoop.pig.TableInfo cfDef) get the validators
protected void	initSchema(java.lang.String signature) Methods to get the column family schema from Cassandra
java.nio.ByteBuffer	nullToBB()
protected java.nio.ByteBuffer	objToBB(java.lang.Object o) convert object to ByteBuffer
void	prepareToRead(org.apache.hadoop.mapreduce.RecordReader reader, org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit split)
void	prepareToWrite(org.apache.hadoop.mapreduce.RecordWriter writer)
void	putNext(org.apache.pig.data.Tuple t) output: (((name, value), (name, value)), (value ...
java.lang.String	relativeToAbsolutePath(java.lang.String location, org.apache.hadoop.fs.Path curDir)
java.lang.String	relToAbsPathForStoreLocation(java.lang.String location, org.apache.hadoop.fs.Path curDir)
protected void	setConnectionInformation() set hadoop cassandra connection settings
void	setLocation(java.lang.String location, org.apache.hadoop.mapreduce.Job job) set read configuration settings
void	setPartitionFilter(org.apache.pig.Expression partitionFilter)
void	setStoreFuncUDFContextSignature(java.lang.String signature) StoreFunc methods
void	setStoreLocation(java.lang.String location, org.apache.hadoop.mapreduce.Job job) set store configuration settings
void	setUDFContextSignature(java.lang.String signature)

Methods inherited from class org.apache.pig.LoadFunc

getAbsolutePath, getLoadCaster, getPathStrings, join, warn

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

DEFAULT_INPUT_FORMAT

protected java.lang.String DEFAULT_INPUT_FORMAT

DEFAULT_OUTPUT_FORMAT

protected java.lang.String DEFAULT_OUTPUT_FORMAT

username

protected java.lang.String username

password

protected java.lang.String password

keyspace

protected java.lang.String keyspace

column_family

protected java.lang.String column_family

loadSignature

protected java.lang.String loadSignature

storeSignature

protected java.lang.String storeSignature

conf

protected org.apache.hadoop.conf.Configuration conf

inputFormatClass

protected java.lang.String inputFormatClass

outputFormatClass

protected java.lang.String outputFormatClass

splitSize

protected int splitSize

partitionerClass

protected java.lang.String partitionerClass

usePartitionFilter

protected boolean usePartitionFilter

initHostAddress

protected java.lang.String initHostAddress

rpcPort

protected java.lang.String rpcPort

nativeProtocolVersion

protected int nativeProtocolVersion

Constructor Detail

CqlNativeStorage

public CqlNativeStorage()

CqlNativeStorage

public CqlNativeStorage(int pageSize)

Parameters:

pageSize - limit number of CQL rows to fetch in a thrift request

Method Detail

prepareToRead

public void prepareToRead(org.apache.hadoop.mapreduce.RecordReader reader,
                          org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit split)

Specified by:

prepareToRead in class org.apache.pig.LoadFunc

prepareToWrite

public void prepareToWrite(org.apache.hadoop.mapreduce.RecordWriter writer)

Specified by:

prepareToWrite in interface org.apache.pig.StoreFuncInterface

getNext

public org.apache.pig.data.Tuple getNext()
                                  throws java.io.IOException

get next row

Specified by:

getNext in class org.apache.pig.LoadFunc

Throws:

java.io.IOException

getCfInfo

protected org.apache.cassandra.hadoop.pig.TableInfo getCfInfo(java.lang.String signature)
                                                       throws java.io.IOException

get the columnfamily definition for the signature

Throws:

java.io.IOException

getCfInfo

protected com.datastax.driver.core.TableMetadata getCfInfo(com.datastax.driver.core.Session client)
                                                    throws com.datastax.driver.core.exceptions.NoHostAvailableException,
                                                           AuthenticationException,
                                                           java.lang.IllegalStateException

return the CfInfo for the column family

Throws:

com.datastax.driver.core.exceptions.NoHostAvailableException

AuthenticationException

java.lang.IllegalStateException

putNext

public void putNext(org.apache.pig.data.Tuple t)
             throws java.io.IOException

output: (((name, value), (name, value)), (value ... value), (value...value))

Specified by:

putNext in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

objToBB

protected java.nio.ByteBuffer objToBB(java.lang.Object o)

convert object to ByteBuffer

getValidatorMap

protected java.util.Map<java.nio.ByteBuffer,AbstractType> getValidatorMap(org.apache.cassandra.hadoop.pig.TableInfo cfDef)
                                                                   throws java.io.IOException

get the validators

Throws:

java.io.IOException

getSchema

public org.apache.pig.ResourceSchema getSchema(java.lang.String location,
                                               org.apache.hadoop.mapreduce.Job job)
                                        throws java.io.IOException

schema: (value, value, value) where keys are in the front.

Specified by:

getSchema in interface org.apache.pig.LoadMetadata

Throws:

java.io.IOException

setPartitionFilter

public void setPartitionFilter(org.apache.pig.Expression partitionFilter)
                        throws java.io.IOException

Specified by:

setPartitionFilter in interface org.apache.pig.LoadMetadata

Throws:

java.io.IOException

setLocation

public void setLocation(java.lang.String location,
                        org.apache.hadoop.mapreduce.Job job)
                 throws java.io.IOException

set read configuration settings

Specified by:

setLocation in class org.apache.pig.LoadFunc

Throws:

java.io.IOException

setStoreLocation

public void setStoreLocation(java.lang.String location,
                             org.apache.hadoop.mapreduce.Job job)
                      throws java.io.IOException

set store configuration settings

Specified by:

setStoreLocation in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

initSchema

protected void initSchema(java.lang.String signature)
                   throws java.io.IOException

Methods to get the column family schema from Cassandra

Throws:

java.io.IOException

cfdefToString

protected static java.lang.String cfdefToString(com.datastax.driver.core.TableMetadata cfDef)
                                         throws java.io.IOException

convert CfDef to string

Throws:

java.io.IOException

cfdefFromString

protected static org.apache.cassandra.hadoop.pig.TableInfo cfdefFromString(java.lang.String st)
                                                                    throws java.io.IOException,
                                                                           java.lang.ClassNotFoundException

convert string back to CfDef

Throws:

java.io.IOException

java.lang.ClassNotFoundException

getQueryMap

public static java.util.Map<java.lang.String,java.lang.String> getQueryMap(java.lang.String query)
                                                                    throws java.io.UnsupportedEncodingException

decompose the query to store the parameters in a map

Throws:

java.io.UnsupportedEncodingException

nullToBB

public java.nio.ByteBuffer nullToBB()

getOutputFormat

public org.apache.hadoop.mapreduce.OutputFormat getOutputFormat()
                                                         throws java.io.IOException

output format

Specified by:

getOutputFormat in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

cleanupOnFailure

public void cleanupOnFailure(java.lang.String failure,
                             org.apache.hadoop.mapreduce.Job job)

Specified by:

cleanupOnFailure in interface org.apache.pig.StoreFuncInterface

cleanupOnSuccess

public void cleanupOnSuccess(java.lang.String location,
                             org.apache.hadoop.mapreduce.Job job)
                      throws java.io.IOException

Specified by:

cleanupOnSuccess in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

getPartitionKeys

public java.lang.String[] getPartitionKeys(java.lang.String location,
                                           org.apache.hadoop.mapreduce.Job job)
                                    throws java.io.IOException

return partition keys

Specified by:

getPartitionKeys in interface org.apache.pig.LoadMetadata

Throws:

java.io.IOException

checkSchema

public void checkSchema(org.apache.pig.ResourceSchema schema)
                 throws java.io.IOException

Specified by:

checkSchema in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

getStatistics

public org.apache.pig.ResourceStatistics getStatistics(java.lang.String location,
                                                       org.apache.hadoop.mapreduce.Job job)

Specified by:

getStatistics in interface org.apache.pig.LoadMetadata

getInputFormat

public org.apache.hadoop.mapreduce.InputFormat getInputFormat()
                                                       throws java.io.IOException

Specified by:

getInputFormat in class org.apache.pig.LoadFunc

Throws:

java.io.IOException

relToAbsPathForStoreLocation

public java.lang.String relToAbsPathForStoreLocation(java.lang.String location,
                                                     org.apache.hadoop.fs.Path curDir)
                                              throws java.io.IOException

Specified by:

relToAbsPathForStoreLocation in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

relativeToAbsolutePath

public java.lang.String relativeToAbsolutePath(java.lang.String location,
                                               org.apache.hadoop.fs.Path curDir)
                                        throws java.io.IOException

Overrides:

relativeToAbsolutePath in class org.apache.pig.LoadFunc

Throws:

java.io.IOException

setUDFContextSignature

public void setUDFContextSignature(java.lang.String signature)

Overrides:

setUDFContextSignature in class org.apache.pig.LoadFunc

setStoreFuncUDFContextSignature

public void setStoreFuncUDFContextSignature(java.lang.String signature)

StoreFunc methods

Specified by:

setStoreFuncUDFContextSignature in interface org.apache.pig.StoreFuncInterface

setConnectionInformation

protected void setConnectionInformation()
                                 throws java.io.IOException

set hadoop cassandra connection settings

Throws:

java.io.IOException

getFullyQualifiedClassName

protected java.lang.String getFullyQualifiedClassName(java.lang.String classname)

get the full class name