kafka.etl
Class KafkaETLContext

java.lang.Object
  kafka.etl.KafkaETLContext

public class KafkaETLContext
extends java.lang.Object

Field Summary

protected int	_bufferSize
protected kafka.javaapi.consumer.SimpleConsumer	_consumer
protected long	_count
protected int	_index
protected java.lang.String	_input
protected java.util.Iterator<kafka.message.MessageAndOffset>	_messageIt
protected org.apache.hadoop.mapred.lib.MultipleOutputs	_mos
protected long	_offset
protected org.apache.hadoop.mapred.OutputCollector<KafkaETLKey,org.apache.hadoop.io.BytesWritable>	_offsetOut
protected long[]	_offsetRange
protected org.apache.hadoop.mapred.Reporter	_reporter
protected KafkaETLRequest	_request
protected long	_requestTime
protected java.util.Iterator<kafka.javaapi.message.ByteBufferMessageSet>	_respIterator
protected kafka.javaapi.FetchResponse	_response
protected int	_retry
protected long	_startTime
protected int	_timeout
protected kafka.api.FetchRequestBuilder	builder
protected static int	MAX_RETRY_TIME

Constructor Summary

KafkaETLContext(org.apache.hadoop.mapred.JobConf job, Props props, org.apache.hadoop.mapred.Reporter reporter, org.apache.hadoop.mapred.lib.MultipleOutputs mos, int index, java.lang.String input)
construct using input string

Method Summary

void	close()
boolean	fetchMore()
protected boolean	get(KafkaETLKey key, org.apache.hadoop.io.BytesWritable value)
static int	getClientBufferSize(Props props)
static int	getClientTimeout(Props props)
long	getCount()
boolean	getNext(KafkaETLKey key, org.apache.hadoop.io.BytesWritable value)
protected long[]	getOffsetRange() Get offset ranges
long	getReadBytes()
long	getTotalBytes()
boolean	hasMore()
void	output(java.lang.String fileprefix)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

MAX_RETRY_TIME

protected static int MAX_RETRY_TIME

_index

protected int _index

_input

protected java.lang.String _input

_request

protected KafkaETLRequest _request

_consumer

protected kafka.javaapi.consumer.SimpleConsumer _consumer

_offsetRange

protected long[] _offsetRange

_offset

protected long _offset

_count

protected long _count

_response

protected kafka.javaapi.FetchResponse _response

_messageIt

protected java.util.Iterator<kafka.message.MessageAndOffset> _messageIt

_respIterator

protected java.util.Iterator<kafka.javaapi.message.ByteBufferMessageSet> _respIterator

_retry

protected int _retry

_requestTime

protected long _requestTime

_startTime

protected long _startTime

_bufferSize

protected int _bufferSize

_timeout

protected int _timeout

_reporter

protected org.apache.hadoop.mapred.Reporter _reporter

_mos

protected org.apache.hadoop.mapred.lib.MultipleOutputs _mos

_offsetOut

protected org.apache.hadoop.mapred.OutputCollector<KafkaETLKey,org.apache.hadoop.io.BytesWritable> _offsetOut

builder

protected kafka.api.FetchRequestBuilder builder

Constructor Detail

KafkaETLContext

public KafkaETLContext(org.apache.hadoop.mapred.JobConf job,
                       Props props,
                       org.apache.hadoop.mapred.Reporter reporter,
                       org.apache.hadoop.mapred.lib.MultipleOutputs mos,
                       int index,
                       java.lang.String input)
                throws java.lang.Exception

construct using input string

Throws:

java.lang.Exception

Method Detail

getTotalBytes

public long getTotalBytes()

getReadBytes

public long getReadBytes()

getCount

public long getCount()

hasMore

public boolean hasMore()

getNext

public boolean getNext(KafkaETLKey key,
                       org.apache.hadoop.io.BytesWritable value)
                throws java.io.IOException

Throws:

java.io.IOException

fetchMore

public boolean fetchMore()
                  throws java.io.IOException

Throws:

java.io.IOException

output

public void output(java.lang.String fileprefix)
            throws java.io.IOException

Throws:

java.io.IOException

close

public void close()
           throws java.io.IOException

Throws:

java.io.IOException

get

protected boolean get(KafkaETLKey key,
                      org.apache.hadoop.io.BytesWritable value)
               throws java.io.IOException

Throws:

java.io.IOException

getOffsetRange

protected long[] getOffsetRange()
                         throws java.io.IOException

Get offset ranges

Throws:

java.io.IOException

getClientBufferSize

public static int getClientBufferSize(Props props)
                               throws java.lang.Exception

Throws:

java.lang.Exception

getClientTimeout

public static int getClientTimeout(Props props)
                            throws java.lang.Exception

Throws:

java.lang.Exception