org.apache.hadoop.mapred
Class Merger

java.lang.Object
  org.apache.hadoop.mapred.Merger

@InterfaceAudience.Private
@InterfaceStability.Unstable
public class Merger
extends Object

Merger is an utility class used by the Map and Reduce tasks for merging both their memory and disk segments

Nested Class Summary

static class

Merger.Segment<K,V>

Constructor Summary

Merger()

Method Summary

static <K,V> RawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class<K> keyClass, Class<V> valueClass, org.apache.hadoop.io.compress.CompressionCodec codec, List<Merger.Segment<K,V>> segments, int mergeFactor, int inMemSegments, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator<K> comparator, org.apache.hadoop.util.Progressable reporter, boolean sortSegments, Counters.Counter readsCounter, Counters.Counter writesCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> RawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class<K> keyClass, Class<V> valueClass, org.apache.hadoop.io.compress.CompressionCodec codec, List<Merger.Segment<K,V>> segments, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator<K> comparator, org.apache.hadoop.util.Progressable reporter, boolean sortSegments, Counters.Counter readsCounter, Counters.Counter writesCounter, org.apache.hadoop.util.Progress mergePhase, TaskType taskType)
static <K,V> RawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class<K> keyClass, Class<V> valueClass, org.apache.hadoop.io.compress.CompressionCodec codec, org.apache.hadoop.fs.Path[] inputs, boolean deleteInputs, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator<K> comparator, org.apache.hadoop.util.Progressable reporter, Counters.Counter readsCounter, Counters.Counter writesCounter, Counters.Counter mergedMapOutputsCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> RawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class<K> keyClass, Class<V> valueClass, org.apache.hadoop.io.compress.CompressionCodec codec, org.apache.hadoop.fs.Path[] inputs, boolean deleteInputs, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator<K> comparator, org.apache.hadoop.util.Progressable reporter, Counters.Counter readsCounter, Counters.Counter writesCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> RawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class<K> keyClass, Class<V> valueClass, List<Merger.Segment<K,V>> segments, int mergeFactor, int inMemSegments, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator<K> comparator, org.apache.hadoop.util.Progressable reporter, boolean sortSegments, Counters.Counter readsCounter, Counters.Counter writesCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> RawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class<K> keyClass, Class<V> valueClass, List<Merger.Segment<K,V>> segments, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator<K> comparator, org.apache.hadoop.util.Progressable reporter, boolean sortSegments, Counters.Counter readsCounter, Counters.Counter writesCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> RawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class<K> keyClass, Class<V> valueClass, List<Merger.Segment<K,V>> segments, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator<K> comparator, org.apache.hadoop.util.Progressable reporter, Counters.Counter readsCounter, Counters.Counter writesCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> void	writeFile(RawKeyValueIterator records, IFile.Writer<K,V> writer, org.apache.hadoop.util.Progressable progressable, org.apache.hadoop.conf.Configuration conf)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

Merger

public Merger()

Method Detail

merge

public static <K,V> RawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                              org.apache.hadoop.fs.FileSystem fs,
                                              Class<K> keyClass,
                                              Class<V> valueClass,
                                              org.apache.hadoop.io.compress.CompressionCodec codec,
                                              org.apache.hadoop.fs.Path[] inputs,
                                              boolean deleteInputs,
                                              int mergeFactor,
                                              org.apache.hadoop.fs.Path tmpDir,
                                              org.apache.hadoop.io.RawComparator<K> comparator,
                                              org.apache.hadoop.util.Progressable reporter,
                                              Counters.Counter readsCounter,
                                              Counters.Counter writesCounter,
                                              org.apache.hadoop.util.Progress mergePhase)
                                 throws IOException

Throws:

IOException

merge

public static <K,V> RawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                              org.apache.hadoop.fs.FileSystem fs,
                                              Class<K> keyClass,
                                              Class<V> valueClass,
                                              org.apache.hadoop.io.compress.CompressionCodec codec,
                                              org.apache.hadoop.fs.Path[] inputs,
                                              boolean deleteInputs,
                                              int mergeFactor,
                                              org.apache.hadoop.fs.Path tmpDir,
                                              org.apache.hadoop.io.RawComparator<K> comparator,
                                              org.apache.hadoop.util.Progressable reporter,
                                              Counters.Counter readsCounter,
                                              Counters.Counter writesCounter,
                                              Counters.Counter mergedMapOutputsCounter,
                                              org.apache.hadoop.util.Progress mergePhase)
                                 throws IOException

Throws:

IOException

merge

public static <K,V> RawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                              org.apache.hadoop.fs.FileSystem fs,
                                              Class<K> keyClass,
                                              Class<V> valueClass,
                                              List<Merger.Segment<K,V>> segments,
                                              int mergeFactor,
                                              org.apache.hadoop.fs.Path tmpDir,
                                              org.apache.hadoop.io.RawComparator<K> comparator,
                                              org.apache.hadoop.util.Progressable reporter,
                                              Counters.Counter readsCounter,
                                              Counters.Counter writesCounter,
                                              org.apache.hadoop.util.Progress mergePhase)
                                 throws IOException

Throws:

IOException

merge

public static <K,V> RawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                              org.apache.hadoop.fs.FileSystem fs,
                                              Class<K> keyClass,
                                              Class<V> valueClass,
                                              List<Merger.Segment<K,V>> segments,
                                              int mergeFactor,
                                              org.apache.hadoop.fs.Path tmpDir,
                                              org.apache.hadoop.io.RawComparator<K> comparator,
                                              org.apache.hadoop.util.Progressable reporter,
                                              boolean sortSegments,
                                              Counters.Counter readsCounter,
                                              Counters.Counter writesCounter,
                                              org.apache.hadoop.util.Progress mergePhase)
                                 throws IOException

Throws:

IOException

merge

public static <K,V> RawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                              org.apache.hadoop.fs.FileSystem fs,
                                              Class<K> keyClass,
                                              Class<V> valueClass,
                                              org.apache.hadoop.io.compress.CompressionCodec codec,
                                              List<Merger.Segment<K,V>> segments,
                                              int mergeFactor,
                                              org.apache.hadoop.fs.Path tmpDir,
                                              org.apache.hadoop.io.RawComparator<K> comparator,
                                              org.apache.hadoop.util.Progressable reporter,
                                              boolean sortSegments,
                                              Counters.Counter readsCounter,
                                              Counters.Counter writesCounter,
                                              org.apache.hadoop.util.Progress mergePhase,
                                              TaskType taskType)
                                 throws IOException

Throws:

IOException

merge

public static <K,V> RawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                              org.apache.hadoop.fs.FileSystem fs,
                                              Class<K> keyClass,
                                              Class<V> valueClass,
                                              List<Merger.Segment<K,V>> segments,
                                              int mergeFactor,
                                              int inMemSegments,
                                              org.apache.hadoop.fs.Path tmpDir,
                                              org.apache.hadoop.io.RawComparator<K> comparator,
                                              org.apache.hadoop.util.Progressable reporter,
                                              boolean sortSegments,
                                              Counters.Counter readsCounter,
                                              Counters.Counter writesCounter,
                                              org.apache.hadoop.util.Progress mergePhase)
                                 throws IOException

Throws:

IOException

merge

public static <K,V> RawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                                              org.apache.hadoop.fs.FileSystem fs,
                                              Class<K> keyClass,
                                              Class<V> valueClass,
                                              org.apache.hadoop.io.compress.CompressionCodec codec,
                                              List<Merger.Segment<K,V>> segments,
                                              int mergeFactor,
                                              int inMemSegments,
                                              org.apache.hadoop.fs.Path tmpDir,
                                              org.apache.hadoop.io.RawComparator<K> comparator,
                                              org.apache.hadoop.util.Progressable reporter,
                                              boolean sortSegments,
                                              Counters.Counter readsCounter,
                                              Counters.Counter writesCounter,
                                              org.apache.hadoop.util.Progress mergePhase)
                                 throws IOException

Throws:

IOException

writeFile

public static <K,V> void writeFile(RawKeyValueIterator records,
                                   IFile.Writer<K,V> writer,
                                   org.apache.hadoop.util.Progressable progressable,
                                   org.apache.hadoop.conf.Configuration conf)
                      throws IOException

Throws:

IOException