org.apache.cassandra.hadoop.pig

Class AbstractCassandraStorage

java.lang.Object

org.apache.pig.LoadFunc

org.apache.cassandra.hadoop.pig.AbstractCassandraStorage

All Implemented Interfaces:

org.apache.pig.LoadMetadata, org.apache.pig.StoreFuncInterface

Direct Known Subclasses:

CassandraStorage, CqlStorage

public abstract class AbstractCassandraStorage
extends org.apache.pig.LoadFunc
implements org.apache.pig.StoreFuncInterface, org.apache.pig.LoadMetadata

A LoadStoreFunc for retrieving data from and storing data to Cassandra

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
protected static class	AbstractCassandraStorage.MarshallerType

Field Summary

Fields

Modifier and Type	Field and Description
protected java.lang.String	column_family
protected org.apache.hadoop.conf.Configuration	conf
protected java.lang.String	DEFAULT_INPUT_FORMAT
protected java.lang.String	DEFAULT_OUTPUT_FORMAT
protected java.lang.String	inputFormatClass
protected java.lang.String	keyspace
protected java.lang.String	loadSignature
protected static org.slf4j.Logger	logger
protected java.lang.String	outputFormatClass
static java.lang.String	PARTITION_FILTER_SIGNATURE
protected java.lang.String	partitionerClass
protected java.lang.String	password
static java.lang.String	PIG_INITIAL_ADDRESS
static java.lang.String	PIG_INPUT_FORMAT
static java.lang.String	PIG_INPUT_INITIAL_ADDRESS
static java.lang.String	PIG_INPUT_PARTITIONER
static java.lang.String	PIG_INPUT_RPC_PORT
static java.lang.String	PIG_INPUT_SPLIT_SIZE
static java.lang.String	PIG_OUTPUT_FORMAT
static java.lang.String	PIG_OUTPUT_INITIAL_ADDRESS
static java.lang.String	PIG_OUTPUT_PARTITIONER
static java.lang.String	PIG_OUTPUT_RPC_PORT
static java.lang.String	PIG_PARTITIONER
static java.lang.String	PIG_RPC_PORT
protected int	splitSize
protected java.lang.String	storeSignature
protected boolean	usePartitionFilter
protected java.lang.String	username

Constructor Summary

Constructors

Constructor and Description
AbstractCassandraStorage()

Method Summary

Methods

Modifier and Type	Method and Description
protected static org.apache.cassandra.thrift.CfDef	cfdefFromString(java.lang.String st) convert string back to CfDef
protected static java.lang.String	cfdefToString(org.apache.cassandra.thrift.CfDef cfDef) convert CfDef to string
void	checkSchema(org.apache.pig.ResourceSchema schema)
void	cleanupOnFailure(java.lang.String failure, org.apache.hadoop.mapreduce.Job job)
protected org.apache.pig.data.Tuple	columnToTuple(Column col, org.apache.cassandra.thrift.CfDef cfDef, AbstractType comparator) convert a column to a tuple
protected org.apache.pig.data.Tuple	composeComposite(AbstractCompositeType comparator, java.nio.ByteBuffer name) Deconstructs a composite type to a Tuple.
protected org.apache.cassandra.thrift.CfDef	getCfDef(org.apache.cassandra.thrift.Cassandra.Client client) return the CfDef for the column family
protected org.apache.cassandra.thrift.CfDef	getCfDef(java.lang.String signature) get the columnfamily definition for the signature
protected java.util.List<org.apache.cassandra.thrift.ColumnDef>	getColumnMeta(org.apache.cassandra.thrift.Cassandra.Client client) get column meta data
protected abstract java.util.List<org.apache.cassandra.thrift.ColumnDef>	getColumnMetadata(org.apache.cassandra.thrift.Cassandra.Client client, boolean cql3Table) get a list of columns
protected java.util.Map<AbstractCassandraStorage.MarshallerType,AbstractType>	getDefaultMarshallers(org.apache.cassandra.thrift.CfDef cfDef) construct a map to store the mashaller type to cassandra data type mapping
protected java.lang.String	getFullyQualifiedClassName(java.lang.String classname) get the full class name
protected java.util.List<org.apache.cassandra.thrift.ColumnDef>	getIndexes() get a list of columns with defined index
protected org.apache.cassandra.thrift.IndexType	getIndexType(java.lang.String type) get index type from string
org.apache.hadoop.mapreduce.InputFormat	getInputFormat()
protected java.util.List<org.apache.cassandra.thrift.ColumnDef>	getKeysMeta(org.apache.cassandra.thrift.Cassandra.Client client) get keys meta data
org.apache.hadoop.mapreduce.OutputFormat	getOutputFormat() output format
java.lang.String[]	getPartitionKeys(java.lang.String location, org.apache.hadoop.mapreduce.Job job) return partition keys
protected byte	getPigType(AbstractType type) get pig type for the cassandra data type
static java.util.Map<java.lang.String,java.lang.String>	getQueryMap(java.lang.String query) decompose the query to store the parameters in a map
org.apache.pig.ResourceStatistics	getStatistics(java.lang.String location, org.apache.hadoop.mapreduce.Job job)
protected java.util.Map<java.nio.ByteBuffer,AbstractType>	getValidatorMap(org.apache.cassandra.thrift.CfDef cfDef) get the validators
protected void	initSchema(java.lang.String signature) Methods to get the column family schema from Cassandra
protected java.nio.ByteBuffer	objToBB(java.lang.Object o) convert object to ByteBuffer
protected AbstractType	parseType(java.lang.String type) parse the string to a cassandra data type
java.lang.String	relativeToAbsolutePath(java.lang.String location, org.apache.hadoop.fs.Path curDir)
java.lang.String	relToAbsPathForStoreLocation(java.lang.String location, org.apache.hadoop.fs.Path curDir)
protected void	setConnectionInformation() set hadoop cassandra connection settings
void	setStoreFuncUDFContextSignature(java.lang.String signature) StoreFunc methods
protected void	setTupleValue(org.apache.pig.data.Tuple pair, int position, java.lang.Object value) set the value to the position of the tuple
void	setUDFContextSignature(java.lang.String signature)

Methods inherited from class org.apache.pig.LoadFunc

getAbsolutePath, getLoadCaster, getNext, getPathStrings, join, prepareToRead, setLocation, warn

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.pig.StoreFuncInterface

prepareToWrite, putNext, setStoreLocation

Methods inherited from interface org.apache.pig.LoadMetadata

getSchema, setPartitionFilter

Field Detail

PIG_INPUT_RPC_PORT

public static final java.lang.String PIG_INPUT_RPC_PORT

See Also:

Constant Field Values

PIG_INPUT_INITIAL_ADDRESS

public static final java.lang.String PIG_INPUT_INITIAL_ADDRESS

See Also:

Constant Field Values

PIG_INPUT_PARTITIONER

public static final java.lang.String PIG_INPUT_PARTITIONER

See Also:

Constant Field Values

PIG_OUTPUT_RPC_PORT

public static final java.lang.String PIG_OUTPUT_RPC_PORT

See Also:

Constant Field Values

PIG_OUTPUT_INITIAL_ADDRESS

public static final java.lang.String PIG_OUTPUT_INITIAL_ADDRESS

See Also:

Constant Field Values

PIG_OUTPUT_PARTITIONER

public static final java.lang.String PIG_OUTPUT_PARTITIONER

See Also:

Constant Field Values

PIG_RPC_PORT

public static final java.lang.String PIG_RPC_PORT

See Also:

Constant Field Values

PIG_INITIAL_ADDRESS

public static final java.lang.String PIG_INITIAL_ADDRESS

See Also:

Constant Field Values

PIG_PARTITIONER

public static final java.lang.String PIG_PARTITIONER

See Also:

Constant Field Values

PIG_INPUT_FORMAT

public static final java.lang.String PIG_INPUT_FORMAT

See Also:

Constant Field Values

PIG_OUTPUT_FORMAT

public static final java.lang.String PIG_OUTPUT_FORMAT

See Also:

Constant Field Values

PIG_INPUT_SPLIT_SIZE

public static final java.lang.String PIG_INPUT_SPLIT_SIZE

See Also:

Constant Field Values

DEFAULT_INPUT_FORMAT

protected java.lang.String DEFAULT_INPUT_FORMAT

DEFAULT_OUTPUT_FORMAT

protected java.lang.String DEFAULT_OUTPUT_FORMAT

PARTITION_FILTER_SIGNATURE

public static final java.lang.String PARTITION_FILTER_SIGNATURE

See Also:

Constant Field Values

logger

protected static final org.slf4j.Logger logger

username

protected java.lang.String username

password

protected java.lang.String password

keyspace

protected java.lang.String keyspace

column_family

protected java.lang.String column_family

loadSignature

protected java.lang.String loadSignature

storeSignature

protected java.lang.String storeSignature

conf

protected org.apache.hadoop.conf.Configuration conf

inputFormatClass

protected java.lang.String inputFormatClass

outputFormatClass

protected java.lang.String outputFormatClass

splitSize

protected int splitSize

partitionerClass

protected java.lang.String partitionerClass

usePartitionFilter

protected boolean usePartitionFilter

Constructor Detail

AbstractCassandraStorage

public AbstractCassandraStorage()

Method Detail

composeComposite

protected org.apache.pig.data.Tuple composeComposite(AbstractCompositeType comparator,
                                         java.nio.ByteBuffer name)
                                              throws java.io.IOException

Deconstructs a composite type to a Tuple.

Throws:

java.io.IOException

columnToTuple

protected org.apache.pig.data.Tuple columnToTuple(Column col,
                                      org.apache.cassandra.thrift.CfDef cfDef,
                                      AbstractType comparator)
                                           throws java.io.IOException

convert a column to a tuple

Throws:

java.io.IOException

setTupleValue

protected void setTupleValue(org.apache.pig.data.Tuple pair,
                 int position,
                 java.lang.Object value)
                      throws org.apache.pig.backend.executionengine.ExecException

set the value to the position of the tuple

Throws:

org.apache.pig.backend.executionengine.ExecException

getCfDef

protected org.apache.cassandra.thrift.CfDef getCfDef(java.lang.String signature)

get the columnfamily definition for the signature

getDefaultMarshallers

protected java.util.Map<AbstractCassandraStorage.MarshallerType,AbstractType> getDefaultMarshallers(org.apache.cassandra.thrift.CfDef cfDef)
                                                                                             throws java.io.IOException

construct a map to store the mashaller type to cassandra data type mapping

Throws:

java.io.IOException

getValidatorMap

protected java.util.Map<java.nio.ByteBuffer,AbstractType> getValidatorMap(org.apache.cassandra.thrift.CfDef cfDef)
                                                                   throws java.io.IOException

get the validators

Throws:

java.io.IOException

parseType

protected AbstractType parseType(java.lang.String type)
                          throws java.io.IOException

parse the string to a cassandra data type

Throws:

java.io.IOException

getInputFormat

public org.apache.hadoop.mapreduce.InputFormat getInputFormat()

Specified by:

getInputFormat in class org.apache.pig.LoadFunc

getQueryMap

public static java.util.Map<java.lang.String,java.lang.String> getQueryMap(java.lang.String query)
                                                                    throws java.io.UnsupportedEncodingException

decompose the query to store the parameters in a map

Throws:

java.io.UnsupportedEncodingException

setConnectionInformation

protected void setConnectionInformation()
                                 throws java.io.IOException

set hadoop cassandra connection settings

Throws:

java.io.IOException

getFullyQualifiedClassName

protected java.lang.String getFullyQualifiedClassName(java.lang.String classname)

get the full class name

getPigType

protected byte getPigType(AbstractType type)

get pig type for the cassandra data type

getStatistics

public org.apache.pig.ResourceStatistics getStatistics(java.lang.String location,
                                              org.apache.hadoop.mapreduce.Job job)

Specified by:

getStatistics in interface org.apache.pig.LoadMetadata

relativeToAbsolutePath

public java.lang.String relativeToAbsolutePath(java.lang.String location,
                                      org.apache.hadoop.fs.Path curDir)
                                        throws java.io.IOException

Overrides:

relativeToAbsolutePath in class org.apache.pig.LoadFunc

Throws:

java.io.IOException

setUDFContextSignature

public void setUDFContextSignature(java.lang.String signature)

Overrides:

setUDFContextSignature in class org.apache.pig.LoadFunc

setStoreFuncUDFContextSignature

public void setStoreFuncUDFContextSignature(java.lang.String signature)

StoreFunc methods

Specified by:

setStoreFuncUDFContextSignature in interface org.apache.pig.StoreFuncInterface

relToAbsPathForStoreLocation

public java.lang.String relToAbsPathForStoreLocation(java.lang.String location,
                                            org.apache.hadoop.fs.Path curDir)
                                              throws java.io.IOException

Specified by:

relToAbsPathForStoreLocation in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

getOutputFormat

public org.apache.hadoop.mapreduce.OutputFormat getOutputFormat()

output format

Specified by:

getOutputFormat in interface org.apache.pig.StoreFuncInterface

checkSchema

public void checkSchema(org.apache.pig.ResourceSchema schema)
                 throws java.io.IOException

Specified by:

checkSchema in interface org.apache.pig.StoreFuncInterface

Throws:

java.io.IOException

objToBB

protected java.nio.ByteBuffer objToBB(java.lang.Object o)

convert object to ByteBuffer

cleanupOnFailure

public void cleanupOnFailure(java.lang.String failure,
                    org.apache.hadoop.mapreduce.Job job)

Specified by:

cleanupOnFailure in interface org.apache.pig.StoreFuncInterface

initSchema

protected void initSchema(java.lang.String signature)

Methods to get the column family schema from Cassandra

cfdefToString

protected static java.lang.String cfdefToString(org.apache.cassandra.thrift.CfDef cfDef)

convert CfDef to string

cfdefFromString

protected static org.apache.cassandra.thrift.CfDef cfdefFromString(java.lang.String st)

convert string back to CfDef

getCfDef

protected org.apache.cassandra.thrift.CfDef getCfDef(org.apache.cassandra.thrift.Cassandra.Client client)
                                              throws org.apache.cassandra.thrift.InvalidRequestException,
                                                     org.apache.cassandra.thrift.UnavailableException,
                                                     org.apache.cassandra.thrift.TimedOutException,
                                                     org.apache.cassandra.thrift.SchemaDisagreementException,
                                                     org.apache.thrift.TException,
                                                     java.nio.charset.CharacterCodingException

return the CfDef for the column family

Throws:

org.apache.cassandra.thrift.InvalidRequestException

org.apache.cassandra.thrift.UnavailableException

org.apache.cassandra.thrift.TimedOutException

org.apache.cassandra.thrift.SchemaDisagreementException

org.apache.thrift.TException

java.nio.charset.CharacterCodingException

getColumnMetadata

protected abstract java.util.List<org.apache.cassandra.thrift.ColumnDef> getColumnMetadata(org.apache.cassandra.thrift.Cassandra.Client client,
                                                                      boolean cql3Table)
                                                                                    throws org.apache.cassandra.thrift.InvalidRequestException,
                                                                                           org.apache.cassandra.thrift.UnavailableException,
                                                                                           org.apache.cassandra.thrift.TimedOutException,
                                                                                           org.apache.cassandra.thrift.SchemaDisagreementException,
                                                                                           org.apache.thrift.TException,
                                                                                           java.nio.charset.CharacterCodingException

get a list of columns

Throws:

org.apache.cassandra.thrift.InvalidRequestException

org.apache.cassandra.thrift.UnavailableException

org.apache.cassandra.thrift.TimedOutException

org.apache.cassandra.thrift.SchemaDisagreementException

org.apache.thrift.TException

java.nio.charset.CharacterCodingException

getColumnMeta

protected java.util.List<org.apache.cassandra.thrift.ColumnDef> getColumnMeta(org.apache.cassandra.thrift.Cassandra.Client client)
                                                                       throws org.apache.cassandra.thrift.InvalidRequestException,
                                                                              org.apache.cassandra.thrift.UnavailableException,
                                                                              org.apache.cassandra.thrift.TimedOutException,
                                                                              org.apache.cassandra.thrift.SchemaDisagreementException,
                                                                              org.apache.thrift.TException,
                                                                              java.nio.charset.CharacterCodingException

get column meta data

Throws:

org.apache.cassandra.thrift.InvalidRequestException

org.apache.cassandra.thrift.UnavailableException

org.apache.cassandra.thrift.TimedOutException

org.apache.cassandra.thrift.SchemaDisagreementException

org.apache.thrift.TException

java.nio.charset.CharacterCodingException

getKeysMeta

protected java.util.List<org.apache.cassandra.thrift.ColumnDef> getKeysMeta(org.apache.cassandra.thrift.Cassandra.Client client)
                                                                     throws org.apache.cassandra.thrift.InvalidRequestException,
                                                                            org.apache.cassandra.thrift.UnavailableException,
                                                                            org.apache.cassandra.thrift.TimedOutException,
                                                                            org.apache.cassandra.thrift.SchemaDisagreementException,
                                                                            org.apache.thrift.TException,
                                                                            java.io.IOException

get keys meta data

Throws:

org.apache.cassandra.thrift.InvalidRequestException

org.apache.cassandra.thrift.UnavailableException

org.apache.cassandra.thrift.TimedOutException

org.apache.cassandra.thrift.SchemaDisagreementException

org.apache.thrift.TException

java.io.IOException

getIndexType

protected org.apache.cassandra.thrift.IndexType getIndexType(java.lang.String type)

get index type from string

getPartitionKeys

public java.lang.String[] getPartitionKeys(java.lang.String location,
                                  org.apache.hadoop.mapreduce.Job job)

return partition keys

Specified by:

getPartitionKeys in interface org.apache.pig.LoadMetadata

getIndexes

protected java.util.List<org.apache.cassandra.thrift.ColumnDef> getIndexes()

get a list of columns with defined index