@InterfaceAudience.Private @InterfaceStability.Unstable public class ShuffleSchedulerImpl<K,V> extends Object implements ShuffleScheduler<K,V>
Constructor and Description |
---|
ShuffleSchedulerImpl(JobConf job,
TaskStatus status,
TaskAttemptID reduceId,
ExceptionReporter reporter,
org.apache.hadoop.util.Progress progress,
Counters.Counter shuffledMapsCounter,
Counters.Counter reduceShuffleBytes,
Counters.Counter failedShuffleCounter) |
Modifier and Type | Method and Description |
---|---|
void |
addKnownMapOutput(String hostName,
String hostUrl,
TaskAttemptID mapId) |
void |
close() |
void |
copyFailed(TaskAttemptID mapId,
MapHost host,
boolean readError,
boolean connectExcpt) |
void |
copySucceeded(TaskAttemptID mapId,
MapHost host,
long bytes,
long startMillis,
long endMillis,
MapOutput<K,V> output) |
void |
freeHost(MapHost host) |
MapHost |
getHost() |
List<TaskAttemptID> |
getMapsForHost(MapHost host) |
int |
getMaxHostFailures() |
void |
hostFailed(String hostname) |
void |
obsoleteMapOutput(TaskAttemptID mapId) |
void |
putBackKnownMapOutput(MapHost host,
TaskAttemptID mapId) |
void |
reportLocalError(IOException ioe) |
void |
resetKnownMaps() |
void |
resolve(TaskCompletionEvent event)
Interpret a
TaskCompletionEvent from the event stream. |
void |
tipFailed(TaskID taskId) |
boolean |
waitUntilDone(int millis)
Wait until the shuffle finishes or until the timeout.
|
public ShuffleSchedulerImpl(JobConf job, TaskStatus status, TaskAttemptID reduceId, ExceptionReporter reporter, org.apache.hadoop.util.Progress progress, Counters.Counter shuffledMapsCounter, Counters.Counter reduceShuffleBytes, Counters.Counter failedShuffleCounter)
public void resolve(TaskCompletionEvent event)
ShuffleScheduler
TaskCompletionEvent
from the event stream.resolve
in interface ShuffleScheduler<K,V>
event
- Intermediate output metadatapublic void copySucceeded(TaskAttemptID mapId, MapHost host, long bytes, long startMillis, long endMillis, MapOutput<K,V> output) throws IOException
IOException
public void hostFailed(String hostname)
public void copyFailed(TaskAttemptID mapId, MapHost host, boolean readError, boolean connectExcpt)
public void reportLocalError(IOException ioe)
public void tipFailed(TaskID taskId)
public void addKnownMapOutput(String hostName, String hostUrl, TaskAttemptID mapId)
public void obsoleteMapOutput(TaskAttemptID mapId)
public void putBackKnownMapOutput(MapHost host, TaskAttemptID mapId)
public MapHost getHost() throws InterruptedException
InterruptedException
public List<TaskAttemptID> getMapsForHost(MapHost host)
public void freeHost(MapHost host)
public void resetKnownMaps()
public boolean waitUntilDone(int millis) throws InterruptedException
waitUntilDone
in interface ShuffleScheduler<K,V>
millis
- maximum wait timeInterruptedException
public void close() throws InterruptedException
close
in interface ShuffleScheduler<K,V>
InterruptedException
public int getMaxHostFailures()
Copyright © 2017 Apache Software Foundation. All Rights Reserved.