org.apache.cassandra.hadoop.pig

Class AbstractCassandraStorage

java.lang.Object

org.apache.pig.LoadFunc

org.apache.cassandra.hadoop.pig.AbstractCassandraStorage

All Implemented Interfaces:

org.apache.pig.LoadMetadata, org.apache.pig.StoreFuncInterface

Direct Known Subclasses:

CassandraStorage, CqlStorage

public abstract class AbstractCassandraStorage
extends org.apache.pig.LoadFunc
implements org.apache.pig.StoreFuncInterface, org.apache.pig.LoadMetadata

A LoadStoreFunc for retrieving data from and storing data to Cassandra

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
protected class	AbstractCassandraStorage.CfInfo
protected static class	AbstractCassandraStorage.MarshallerType

Field Summary

Fields

Modifier and Type	Field and Description
protected java.lang.String	column_family
protected org.apache.hadoop.conf.Configuration	conf
protected java.lang.String	DEFAULT_INPUT_FORMAT
protected java.lang.String	DEFAULT_OUTPUT_FORMAT
protected java.lang.String	initHostAddress
protected java.lang.String	inputFormatClass
protected java.lang.String	keyspace
protected java.lang.String	loadSignature
protected static org.slf4j.Logger	logger
protected java.lang.String	outputFormatClass
static java.lang.String	PARTITION_FILTER_SIGNATURE
protected java.lang.String	partitionerClass
protected java.lang.String	password
static java.lang.String	PIG_INITIAL_ADDRESS
static java.lang.String	PIG_INPUT_FORMAT
static java.lang.String	PIG_INPUT_INITIAL_ADDRESS
static java.lang.String	PIG_INPUT_PARTITIONER
static java.lang.String	PIG_INPUT_RPC_PORT
static java.lang.String	PIG_INPUT_SPLIT_SIZE
static java.lang.String	PIG_OUTPUT_FORMAT
static java.lang.String	PIG_OUTPUT_INITIAL_ADDRESS
static java.lang.String	PIG_OUTPUT_PARTITIONER
static java.lang.String	PIG_OUTPUT_RPC_PORT
static java.lang.String	PIG_PARTITIONER
static java.lang.String	PIG_RPC_PORT
protected java.lang.String	rpcPort
protected int	splitSize
protected java.lang.String	storeSignature
protected boolean	usePartitionFilter
protected java.lang.String	username

Constructor Summary

Constructors

Constructor and Description
AbstractCassandraStorage()

Method Summary

Methods

Modifier and Type	Method and Description
protected java.lang.Object	cassandraToObj(AbstractType validator, java.nio.ByteBuffer value)
protected static org.apache.cassandra.thrift.CfDef	cfdefFromString(java.lang.String st) convert string back to CfDef
protected static java.lang.String	cfdefToString(org.apache.cassandra.thrift.CfDef cfDef) convert CfDef to string
void	checkSchema(org.apache.pig.ResourceSchema schema)
void	cleanupOnFailure(java.lang.String failure, org.apache.hadoop.mapreduce.Job job)
protected org.apache.pig.data.Tuple	columnToTuple(Column col, AbstractCassandraStorage.CfInfo cfInfo, AbstractType comparator) convert a column to a tuple
protected org.apache.pig.data.Tuple	composeComposite(AbstractCompositeType comparator, java.nio.ByteBuffer name) Deconstructs a composite type to a Tuple.
protected CFDefinition	getCfDefinition(java.lang.String ks, java.lang.String cf, org.apache.cassandra.thrift.Cassandra.Client client) get CFDefinition of a column family
protected AbstractCassandraStorage.CfInfo	getCfInfo(org.apache.cassandra.thrift.Cassandra.Client client) return the CfInfo for the column family
protected AbstractCassandraStorage.CfInfo	getCfInfo(java.lang.String signature) get the columnfamily definition for the signature
protected java.util.List<org.apache.cassandra.thrift.ColumnDef>	getColumnMeta(org.apache.cassandra.thrift.Cassandra.Client client, boolean cassandraStorage, boolean includeCompactValueColumn) get column meta data
protected abstract java.util.List<org.apache.cassandra.thrift.ColumnDef>	getColumnMetadata(org.apache.cassandra.thrift.Cassandra.Client client) get a list of columns
protected java.util.Map<AbstractCassandraStorage.MarshallerType,AbstractType>	getDefaultMarshallers(org.apache.cassandra.thrift.CfDef cfDef) construct a map to store the mashaller type to cassandra data type mapping
protected java.lang.String	getFullyQualifiedClassName(java.lang.String classname) get the full class name
protected java.util.List<org.apache.cassandra.thrift.ColumnDef>	getIndexes() get a list of columns with defined index
protected org.apache.cassandra.thrift.IndexType	getIndexType(java.lang.String type) get index type from string
org.apache.hadoop.mapreduce.InputFormat	getInputFormat()
org.apache.hadoop.mapreduce.OutputFormat	getOutputFormat() output format
java.lang.String[]	getPartitionKeys(java.lang.String location, org.apache.hadoop.mapreduce.Job job) return partition keys
protected byte	getPigType(AbstractType type) get pig type for the cassandra data type
static java.util.Map<java.lang.String,java.lang.String>	getQueryMap(java.lang.String query) decompose the query to store the parameters in a map
org.apache.pig.ResourceStatistics	getStatistics(java.lang.String location, org.apache.hadoop.mapreduce.Job job)
protected java.util.Map<java.nio.ByteBuffer,AbstractType>	getValidatorMap(org.apache.cassandra.thrift.CfDef cfDef) get the validators
protected void	initSchema(java.lang.String signature) Methods to get the column family schema from Cassandra
protected abstract java.nio.ByteBuffer	nullToBB()
protected java.nio.ByteBuffer	objToBB(java.lang.Object o) convert object to ByteBuffer
protected AbstractType	parseType(java.lang.String type) parse the string to a cassandra data type
java.lang.String	relativeToAbsolutePath(java.lang.String location, org.apache.hadoop.fs.Path curDir)
java.lang.String	relToAbsPathForStoreLocation(java.lang.String location, org.apache.hadoop.fs.Path curDir)
protected void	setConnectionInformation() set hadoop cassandra connection settings
void	setStoreFuncUDFContextSignature(java.lang.String signature) StoreFunc methods
protected void	setTupleValue(org.apache.pig.data.Tuple pair, int position, java.lang.Object value) set the value to the position of the tuple
void	setUDFContextSignature(java.lang.String signature)

Methods inherited from class org.apache.pig.LoadFunc

getAbsolutePath, getLoadCaster, getNext, getPathStrings, join, prepareToRead, setLocation, warn

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.pig.StoreFuncInterface

prepareToWrite, putNext, setStoreLocation

Methods inherited from interface org.apache.pig.LoadMetadata

getSchema, setPartitionFilter

Field Detail

PIG_INPUT_RPC_PORT

public static final java.lang.String PIG_INPUT_RPC_PORT

See Also:

Constant Field Values

PIG_INPUT_INITIAL_ADDRESS

public static final java.lang.String PIG_INPUT_INITIAL_ADDRESS

See Also:

Constant Field Values

PIG_INPUT_PARTITIONER

public static final java.lang.String PIG_INPUT_PARTITIONER

See Also:

Constant Field Values

PIG_OUTPUT_RPC_PORT

public static final java.lang.String PIG_OUTPUT_RPC_PORT

See Also:

Constant Field Values

PIG_OUTPUT_INITIAL_ADDRESS

public static final java.lang.String PIG_OUTPUT_INITIAL_ADDRESS

See Also:

Constant Field Values

PIG_OUTPUT_PARTITIONER

public static final java.lang.String PIG_OUTPUT_PARTITIONER

See Also:

Constant Field Values

PIG_RPC_PORT

public static final java.lang.String PIG_RPC_PORT

See Also:

Constant Field Values

PIG_INITIAL_ADDRESS

public static final java.lang.String PIG_INITIAL_ADDRESS

See Also:

Constant Field Values

PIG_PARTITIONER

public static final java.lang.String PIG_PARTITIONER

See Also:

Constant Field Values

PIG_INPUT_FORMAT

public static final java.lang.String PIG_INPUT_FORMAT

See Also:

Constant Field Values

PIG_OUTPUT_FORMAT

public static final java.lang.String PIG_OUTPUT_FORMAT

See Also:

Constant Field Values

PIG_INPUT_SPLIT_SIZE

public static final java.lang.String PIG_INPUT_SPLIT_SIZE

See Also:

Constant Field Values

DEFAULT_INPUT_FORMAT

protected java.lang.String DEFAULT_INPUT_FORMAT

DEFAULT_OUTPUT_FORMAT

protected java.lang.String DEFAULT_OUTPUT_FORMAT

PARTITION_FILTER_SIGNATURE

public static final java.lang.String PARTITION_FILTER_SIGNATURE

See Also:

Constant Field Values

logger

protected static final org.slf4j.Logger logger

username

protected java.lang.String username

password

protected java.lang.String password

keyspace

protected java.lang.String keyspace

column_family

protected java.lang.String column_family

loadSignature

protected java.lang.String loadSignature

storeSignature

protected java.lang.String storeSignature

conf

protected org.apache.hadoop.conf.Configuration conf

inputFormatClass

protected java.lang.String inputFormatClass

outputFormatClass

protected java.lang.String outputFormatClass

splitSize

protected int splitSize

partitionerClass

protected java.lang.String partitionerClass

usePartitionFilter

protected boolean usePartitionFilter

initHostAddress

protected java.lang.String initHostAddress

rpcPort

protected java.lang.String rpcPort

Constructor Detail

AbstractCassandraStorage

public AbstractCassandraStorage()

Method Detail

composeComposite

protected org.apache.pig.data.Tuple composeComposite(AbstractCompositeType comparator,
                                         java.nio.ByteBuffer name)
                                              throws java.io.IOException

Deconstructs a composite type to a Tuple.

Throws:

java.io.IOException

columnToTuple

protected org.apache.pig.data.Tuple columnToTuple(Column col,
                                      AbstractCassandraStorage.CfInfo cfInfo,
                                      AbstractType comparator)
                                           throws java.io.IOException

convert a column to a tuple

Throws:

java.io.IOException

setTupleValue

protected void setTupleValue(org.apache.pig.data.Tuple pair,
                 int position,
                 java.lang.Object value)
                      throws org.apache.pig.backend.executionengine.ExecException

set the value to the position of the tuple

Throws:

org.apache.pig.backend.executionengine.ExecException

getCfInfo

protected AbstractCassandraStorage.CfInfo getCfInfo(java.lang.String signature)
                                             throws java.io.IOException

get the columnfamily definition for the signature

Throws:

java.io.IOException

getDefaultMarshallers

protected java.util.Map<AbstractCassandraStorage.MarshallerType,AbstractType> getDefaultMarshallers(org.apache.cassandra.thrift.CfDef cfDef)
                                                                                             throws java.io.IOException

construct a map to store the mashaller type to cassandra data type mapping

Throws:

java.io.IOException

getValidatorMap

protected java.util.Map<java.nio.ByteBuffer,AbstractType> getValidatorMap(org.apache.cassandra.thrift.CfDef cfDef)
                                                                   throws java.io.IOException

get the validators

Throws:

java.io.IOException

parseType

protected AbstractType parseType(java.lang.String type)
                          throws java.io.IOException

parse the string to a cassandra data type

Throws:

java.io.IOException

getInputFormat

public org.apache.hadoop.mapreduce.InputFormat getInputFormat()
                                                       throws java.io.IOException

Specified by:

getInputFormat in class org.apache.pig.LoadFunc

Throws:

java.io.IOException

getQueryMap

public static java.util.Map<java.lang.String,java.lang.String> getQueryMap(java.lang.String query)
                                                                    throws java.io.UnsupportedEncodingException

decompose the query to store the parameters in a map

Throws:

java.io.UnsupportedEncodingException

setConnectionInformation

protected void setConnectionInformation()
                                 throws java.io.IOException

set hadoop cassandra connection settings

Throws:

java.io.IOException

getFullyQualifiedClassName

protected java.lang.String getFullyQualifiedClassName(java.lang.String classname)

get the full class name

getPigType

protected byte getPigType(AbstractType type)

get pig type for the cassandra data type

getStatistics

public org.apache.pig.ResourceStatistics getStatistics(java.lang.String location,
                                              org.apache.hadoop.mapreduce.Job job)

Specified by:

getStatistics in interface org.apache.pig.LoadMetadata

relativeToAbsolutePath

public java.lang.String relativeToAbsolutePath(java.lang.String location,
                                      org.apache.hadoop.fs.Path curDir)
                                        throws java.io.IOException

Overrides:

relativeToAbsolutePath in class org.apache.pig.LoadFunc

Throws:

java.io.IOException

setUDFContextSignature

public void setUDFContextSignature(java.lang.String signature)

Overrides:

setUDFContextSignature in class org.apache.pig.LoadFunc

setStoreFuncUDFContextSignature

public void setStoreFuncUDFContextSignature(java.lang.String signature)

StoreFunc methods

Specified by:

setStoreFuncUDFContextSignature in interface org.apache.pig.StoreFuncInterface

relToAbsPathForStoreLocation

public java.lang.String relToAbsPathForStoreLocation(java.lang.String location,
                                            org.apache.hadoop.fs.Path curDir)
                                              throws java.io.IOException

Specified by:

relToAbsPathForStoreLocation in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

getOutputFormat

public org.apache.hadoop.mapreduce.OutputFormat getOutputFormat()
                                                         throws java.io.IOException

output format

Specified by:

getOutputFormat in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

checkSchema

public void checkSchema(org.apache.pig.ResourceSchema schema)
                 throws java.io.IOException

Specified by:

checkSchema in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

nullToBB

protected abstract java.nio.ByteBuffer nullToBB()

objToBB

protected java.nio.ByteBuffer objToBB(java.lang.Object o)

convert object to ByteBuffer

cleanupOnFailure

public void cleanupOnFailure(java.lang.String failure,
                    org.apache.hadoop.mapreduce.Job job)

Specified by:

cleanupOnFailure in interface org.apache.pig.StoreFuncInterface

initSchema

protected void initSchema(java.lang.String signature)
                   throws java.io.IOException

Methods to get the column family schema from Cassandra

Throws:

java.io.IOException

cfdefToString

protected static java.lang.String cfdefToString(org.apache.cassandra.thrift.CfDef cfDef)
                                         throws java.io.IOException

convert CfDef to string

Throws:

java.io.IOException

cfdefFromString

protected static org.apache.cassandra.thrift.CfDef cfdefFromString(java.lang.String st)
                                                            throws java.io.IOException

convert string back to CfDef

Throws:

java.io.IOException

getCfInfo

protected AbstractCassandraStorage.CfInfo getCfInfo(org.apache.cassandra.thrift.Cassandra.Client client)
                                             throws org.apache.cassandra.thrift.InvalidRequestException,
                                                    org.apache.cassandra.thrift.UnavailableException,
                                                    org.apache.cassandra.thrift.TimedOutException,
                                                    org.apache.cassandra.thrift.SchemaDisagreementException,
                                                    org.apache.thrift.TException,
                                                    org.apache.cassandra.thrift.NotFoundException,
                                                    InvalidRequestException,
                                                    ConfigurationException,
                                                    java.io.IOException

return the CfInfo for the column family

Throws:

org.apache.cassandra.thrift.InvalidRequestException

org.apache.cassandra.thrift.UnavailableException

org.apache.cassandra.thrift.TimedOutException

org.apache.cassandra.thrift.SchemaDisagreementException

org.apache.thrift.TException

org.apache.cassandra.thrift.NotFoundException

ConfigurationException

java.io.IOException

getColumnMetadata

protected abstract java.util.List<org.apache.cassandra.thrift.ColumnDef> getColumnMetadata(org.apache.cassandra.thrift.Cassandra.Client client)
                                                                                    throws org.apache.cassandra.thrift.InvalidRequestException,
                                                                                           org.apache.cassandra.thrift.UnavailableException,
                                                                                           org.apache.cassandra.thrift.TimedOutException,
                                                                                           org.apache.cassandra.thrift.SchemaDisagreementException,
                                                                                           org.apache.thrift.TException,
                                                                                           java.nio.charset.CharacterCodingException,
                                                                                           InvalidRequestException,
                                                                                           ConfigurationException,
                                                                                           org.apache.cassandra.thrift.NotFoundException

get a list of columns

Throws:

org.apache.cassandra.thrift.InvalidRequestException

org.apache.cassandra.thrift.UnavailableException

org.apache.cassandra.thrift.TimedOutException

org.apache.cassandra.thrift.SchemaDisagreementException

org.apache.thrift.TException

java.nio.charset.CharacterCodingException

ConfigurationException

org.apache.cassandra.thrift.NotFoundException

getColumnMeta

protected java.util.List<org.apache.cassandra.thrift.ColumnDef> getColumnMeta(org.apache.cassandra.thrift.Cassandra.Client client,
                                                                  boolean cassandraStorage,
                                                                  boolean includeCompactValueColumn)
                                                                       throws org.apache.cassandra.thrift.InvalidRequestException,
                                                                              org.apache.cassandra.thrift.UnavailableException,
                                                                              org.apache.cassandra.thrift.TimedOutException,
                                                                              org.apache.cassandra.thrift.SchemaDisagreementException,
                                                                              org.apache.thrift.TException,
                                                                              java.nio.charset.CharacterCodingException,
                                                                              InvalidRequestException,
                                                                              ConfigurationException,
                                                                              org.apache.cassandra.thrift.NotFoundException

get column meta data

Throws:

org.apache.cassandra.thrift.InvalidRequestException

org.apache.cassandra.thrift.UnavailableException

org.apache.cassandra.thrift.TimedOutException

org.apache.cassandra.thrift.SchemaDisagreementException

org.apache.thrift.TException

java.nio.charset.CharacterCodingException

ConfigurationException

org.apache.cassandra.thrift.NotFoundException

getIndexType

protected org.apache.cassandra.thrift.IndexType getIndexType(java.lang.String type)

get index type from string

getPartitionKeys

public java.lang.String[] getPartitionKeys(java.lang.String location,
                                  org.apache.hadoop.mapreduce.Job job)
                                    throws java.io.IOException

return partition keys

Specified by:

getPartitionKeys in interface org.apache.pig.LoadMetadata

Throws:

java.io.IOException

getIndexes

protected java.util.List<org.apache.cassandra.thrift.ColumnDef> getIndexes()
                                                                    throws java.io.IOException

get a list of columns with defined index

Throws:

java.io.IOException

getCfDefinition

protected CFDefinition getCfDefinition(java.lang.String ks,
                           java.lang.String cf,
                           org.apache.cassandra.thrift.Cassandra.Client client)
                                throws org.apache.cassandra.thrift.NotFoundException,
                                       org.apache.cassandra.thrift.InvalidRequestException,
                                       org.apache.thrift.TException,
                                       InvalidRequestException,
                                       ConfigurationException

get CFDefinition of a column family

Throws:

org.apache.cassandra.thrift.NotFoundException

org.apache.cassandra.thrift.InvalidRequestException

org.apache.thrift.TException

ConfigurationException

cassandraToObj

protected java.lang.Object cassandraToObj(AbstractType validator,
                              java.nio.ByteBuffer value)