org.apache.hadoop.mapred

Class Merger

java.lang.Object

org.apache.hadoop.mapred.Merger

@InterfaceAudience.Private
@InterfaceStability.Unstable
public class Merger
extends Object

Merger is an utility class used by the Map and Reduce tasks for merging both their memory and disk segments

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	Merger.Segment<K,V>

Constructor Summary

Constructors

Constructor and Description
Merger()

Method Summary

Methods

Modifier and Type	Method and Description
static <K,V> RawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class<K> keyClass, Class<V> valueClass, org.apache.hadoop.io.compress.CompressionCodec codec, List<Merger.Segment<K,V>> segments, int mergeFactor, int inMemSegments, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator<K> comparator, org.apache.hadoop.util.Progressable reporter, boolean sortSegments, Counters.Counter readsCounter, Counters.Counter writesCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> RawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class<K> keyClass, Class<V> valueClass, org.apache.hadoop.io.compress.CompressionCodec codec, List<Merger.Segment<K,V>> segments, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator<K> comparator, org.apache.hadoop.util.Progressable reporter, boolean sortSegments, Counters.Counter readsCounter, Counters.Counter writesCounter, org.apache.hadoop.util.Progress mergePhase, TaskType taskType)
static <K,V> RawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class<K> keyClass, Class<V> valueClass, org.apache.hadoop.io.compress.CompressionCodec codec, org.apache.hadoop.fs.Path[] inputs, boolean deleteInputs, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator<K> comparator, org.apache.hadoop.util.Progressable reporter, Counters.Counter readsCounter, Counters.Counter writesCounter, Counters.Counter mergedMapOutputsCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> RawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class<K> keyClass, Class<V> valueClass, org.apache.hadoop.io.compress.CompressionCodec codec, org.apache.hadoop.fs.Path[] inputs, boolean deleteInputs, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator<K> comparator, org.apache.hadoop.util.Progressable reporter, Counters.Counter readsCounter, Counters.Counter writesCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> RawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class<K> keyClass, Class<V> valueClass, List<Merger.Segment<K,V>> segments, int mergeFactor, int inMemSegments, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator<K> comparator, org.apache.hadoop.util.Progressable reporter, boolean sortSegments, Counters.Counter readsCounter, Counters.Counter writesCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> RawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class<K> keyClass, Class<V> valueClass, List<Merger.Segment<K,V>> segments, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator<K> comparator, org.apache.hadoop.util.Progressable reporter, boolean sortSegments, Counters.Counter readsCounter, Counters.Counter writesCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> RawKeyValueIterator	merge(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem fs, Class<K> keyClass, Class<V> valueClass, List<Merger.Segment<K,V>> segments, int mergeFactor, org.apache.hadoop.fs.Path tmpDir, org.apache.hadoop.io.RawComparator<K> comparator, org.apache.hadoop.util.Progressable reporter, Counters.Counter readsCounter, Counters.Counter writesCounter, org.apache.hadoop.util.Progress mergePhase)
static <K,V> void	writeFile(RawKeyValueIterator records, IFile.Writer<K,V> writer, org.apache.hadoop.util.Progressable progressable, org.apache.hadoop.conf.Configuration conf)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

Merger

public Merger()

Method Detail

merge

public static <K,V> RawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                              org.apache.hadoop.fs.FileSystem fs,
                              Class<K> keyClass,
                              Class<V> valueClass,
                              org.apache.hadoop.io.compress.CompressionCodec codec,
                              org.apache.hadoop.fs.Path[] inputs,
                              boolean deleteInputs,
                              int mergeFactor,
                              org.apache.hadoop.fs.Path tmpDir,
                              org.apache.hadoop.io.RawComparator<K> comparator,
                              org.apache.hadoop.util.Progressable reporter,
                              Counters.Counter readsCounter,
                              Counters.Counter writesCounter,
                              org.apache.hadoop.util.Progress mergePhase)
                                 throws IOException

Throws:

IOException

merge

public static <K,V> RawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                              org.apache.hadoop.fs.FileSystem fs,
                              Class<K> keyClass,
                              Class<V> valueClass,
                              org.apache.hadoop.io.compress.CompressionCodec codec,
                              org.apache.hadoop.fs.Path[] inputs,
                              boolean deleteInputs,
                              int mergeFactor,
                              org.apache.hadoop.fs.Path tmpDir,
                              org.apache.hadoop.io.RawComparator<K> comparator,
                              org.apache.hadoop.util.Progressable reporter,
                              Counters.Counter readsCounter,
                              Counters.Counter writesCounter,
                              Counters.Counter mergedMapOutputsCounter,
                              org.apache.hadoop.util.Progress mergePhase)
                                 throws IOException

Throws:

IOException

merge

public static <K,V> RawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                              org.apache.hadoop.fs.FileSystem fs,
                              Class<K> keyClass,
                              Class<V> valueClass,
                              List<Merger.Segment<K,V>> segments,
                              int mergeFactor,
                              org.apache.hadoop.fs.Path tmpDir,
                              org.apache.hadoop.io.RawComparator<K> comparator,
                              org.apache.hadoop.util.Progressable reporter,
                              Counters.Counter readsCounter,
                              Counters.Counter writesCounter,
                              org.apache.hadoop.util.Progress mergePhase)
                                 throws IOException

Throws:

IOException

merge

public static <K,V> RawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                              org.apache.hadoop.fs.FileSystem fs,
                              Class<K> keyClass,
                              Class<V> valueClass,
                              List<Merger.Segment<K,V>> segments,
                              int mergeFactor,
                              org.apache.hadoop.fs.Path tmpDir,
                              org.apache.hadoop.io.RawComparator<K> comparator,
                              org.apache.hadoop.util.Progressable reporter,
                              boolean sortSegments,
                              Counters.Counter readsCounter,
                              Counters.Counter writesCounter,
                              org.apache.hadoop.util.Progress mergePhase)
                                 throws IOException

Throws:

IOException

merge

public static <K,V> RawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                              org.apache.hadoop.fs.FileSystem fs,
                              Class<K> keyClass,
                              Class<V> valueClass,
                              org.apache.hadoop.io.compress.CompressionCodec codec,
                              List<Merger.Segment<K,V>> segments,
                              int mergeFactor,
                              org.apache.hadoop.fs.Path tmpDir,
                              org.apache.hadoop.io.RawComparator<K> comparator,
                              org.apache.hadoop.util.Progressable reporter,
                              boolean sortSegments,
                              Counters.Counter readsCounter,
                              Counters.Counter writesCounter,
                              org.apache.hadoop.util.Progress mergePhase,
                              TaskType taskType)
                                 throws IOException

Throws:

IOException

merge

public static <K,V> RawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                              org.apache.hadoop.fs.FileSystem fs,
                              Class<K> keyClass,
                              Class<V> valueClass,
                              List<Merger.Segment<K,V>> segments,
                              int mergeFactor,
                              int inMemSegments,
                              org.apache.hadoop.fs.Path tmpDir,
                              org.apache.hadoop.io.RawComparator<K> comparator,
                              org.apache.hadoop.util.Progressable reporter,
                              boolean sortSegments,
                              Counters.Counter readsCounter,
                              Counters.Counter writesCounter,
                              org.apache.hadoop.util.Progress mergePhase)
                                 throws IOException

Throws:

IOException

merge

public static <K,V> RawKeyValueIterator merge(org.apache.hadoop.conf.Configuration conf,
                              org.apache.hadoop.fs.FileSystem fs,
                              Class<K> keyClass,
                              Class<V> valueClass,
                              org.apache.hadoop.io.compress.CompressionCodec codec,
                              List<Merger.Segment<K,V>> segments,
                              int mergeFactor,
                              int inMemSegments,
                              org.apache.hadoop.fs.Path tmpDir,
                              org.apache.hadoop.io.RawComparator<K> comparator,
                              org.apache.hadoop.util.Progressable reporter,
                              boolean sortSegments,
                              Counters.Counter readsCounter,
                              Counters.Counter writesCounter,
                              org.apache.hadoop.util.Progress mergePhase)
                                 throws IOException

Throws:

IOException

writeFile

public static <K,V> void writeFile(RawKeyValueIterator records,
                   IFile.Writer<K,V> writer,
                   org.apache.hadoop.util.Progressable progressable,
                   org.apache.hadoop.conf.Configuration conf)
                      throws IOException

Throws:

IOException