org.apache.flink.streaming.api.graph

Class StreamGraph

java.lang.Object

org.apache.flink.streaming.api.graph.StreamGraph

All Implemented Interfaces:

org.apache.flink.api.dag.Pipeline

@Internal
public class StreamGraph
extends Object
implements org.apache.flink.api.dag.Pipeline

Class representing the streaming topology. It contains all the information necessary to build the jobgraph for the execution.

Field Summary

Fields

Modifier and Type	Field and Description
static String	ITERATION_SINK_NAME_PREFIX
static String	ITERATION_SOURCE_NAME_PREFIX
protected Map<Integer,String>	vertexIDtoBrokerID
protected Map<Integer,Long>	vertexIDtoLoopTimeout

Constructor Summary

Constructors

Constructor and Description
StreamGraph(org.apache.flink.api.common.ExecutionConfig executionConfig, CheckpointConfig checkpointConfig, org.apache.flink.runtime.jobgraph.SavepointRestoreSettings savepointRestoreSettings)

Method Summary

Modifier and Type	Method and Description
<IN1,IN2,OUT> void	addCoOperator(Integer vertexID, String slotSharingGroup, String coLocationGroup, StreamOperatorFactory<OUT> taskOperatorFactory, org.apache.flink.api.common.typeinfo.TypeInformation<IN1> in1TypeInfo, org.apache.flink.api.common.typeinfo.TypeInformation<IN2> in2TypeInfo, org.apache.flink.api.common.typeinfo.TypeInformation<OUT> outTypeInfo, String operatorName)
void	addEdge(Integer upStreamVertexID, Integer downStreamVertexID, int typeNumber)
<IN,OUT> void	addLegacySource(Integer vertexID, String slotSharingGroup, String coLocationGroup, StreamOperatorFactory<OUT> operatorFactory, org.apache.flink.api.common.typeinfo.TypeInformation<IN> inTypeInfo, org.apache.flink.api.common.typeinfo.TypeInformation<OUT> outTypeInfo, String operatorName)
<OUT> void	addMultipleInputOperator(Integer vertexID, String slotSharingGroup, String coLocationGroup, StreamOperatorFactory<OUT> operatorFactory, List<org.apache.flink.api.common.typeinfo.TypeInformation<?>> inTypeInfos, org.apache.flink.api.common.typeinfo.TypeInformation<OUT> outTypeInfo, String operatorName)
protected StreamNode	addNode(Integer vertexID, String slotSharingGroup, String coLocationGroup, Class<? extends org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable> vertexClass, StreamOperatorFactory<?> operatorFactory, String operatorName)
<IN,OUT> void	addOperator(Integer vertexID, String slotSharingGroup, String coLocationGroup, StreamOperatorFactory<OUT> operatorFactory, org.apache.flink.api.common.typeinfo.TypeInformation<IN> inTypeInfo, org.apache.flink.api.common.typeinfo.TypeInformation<OUT> outTypeInfo, String operatorName)
<IN,OUT> void	addSink(Integer vertexID, String slotSharingGroup, String coLocationGroup, StreamOperatorFactory<OUT> operatorFactory, org.apache.flink.api.common.typeinfo.TypeInformation<IN> inTypeInfo, org.apache.flink.api.common.typeinfo.TypeInformation<OUT> outTypeInfo, String operatorName)
<IN,OUT> void	addSource(Integer vertexID, String slotSharingGroup, String coLocationGroup, SourceOperatorFactory<OUT> operatorFactory, org.apache.flink.api.common.typeinfo.TypeInformation<IN> inTypeInfo, org.apache.flink.api.common.typeinfo.TypeInformation<OUT> outTypeInfo, String operatorName)
void	addVirtualPartitionNode(Integer originalId, Integer virtualId, StreamPartitioner<?> partitioner, ShuffleMode shuffleMode) Adds a new virtual node that is used to connect a downstream vertex to an input with a certain partitioning.
void	addVirtualSideOutputNode(Integer originalId, Integer virtualId, org.apache.flink.util.OutputTag outputTag) Adds a new virtual node that is used to connect a downstream vertex to only the outputs with the selected side-output OutputTag.
void	clear() Remove all registered nodes etc.
org.apache.flink.api.java.tuple.Tuple2<StreamNode,StreamNode>	createIterationSourceAndSink(int loopId, int sourceId, int sinkId, long timeout, int parallelism, int maxParallelism, org.apache.flink.api.common.operators.ResourceSpec minResources, org.apache.flink.api.common.operators.ResourceSpec preferredResources)
Set<org.apache.flink.api.java.tuple.Tuple2<Integer,StreamOperatorFactory<?>>>	getAllOperatorFactory()
String	getBrokerID(Integer vertexID)
CheckpointConfig	getCheckpointConfig()
org.apache.flink.api.common.ExecutionConfig	getExecutionConfig()
GlobalDataExchangeMode	getGlobalDataExchangeMode()
Set<org.apache.flink.api.java.tuple.Tuple2<StreamNode,StreamNode>>	getIterationSourceSinkPairs()
org.apache.flink.runtime.jobgraph.JobGraph	getJobGraph() Gets the assembled JobGraph with a random JobID.
org.apache.flink.runtime.jobgraph.JobGraph	getJobGraph(org.apache.flink.api.common.JobID jobID) Gets the assembled JobGraph with a specified JobID.
String	getJobName()
long	getLoopTimeout(Integer vertexID)
org.apache.flink.runtime.jobgraph.SavepointRestoreSettings	getSavepointRestoreSettings()
org.apache.flink.runtime.jobgraph.ScheduleMode	getScheduleMode()
Collection<Integer>	getSinkIDs()
String	getSlotSharingGroup(Integer id) Determines the slot sharing group of an operation across virtual nodes.
Collection<Integer>	getSourceIDs()
StreamNode	getSourceVertex(StreamEdge edge)
org.apache.flink.runtime.state.StateBackend	getStateBackend()
List<StreamEdge>	getStreamEdges(int sourceId)
List<StreamEdge>	getStreamEdges(int sourceId, int targetId)
List<StreamEdge>	getStreamEdgesOrThrow(int sourceId, int targetId) Deprecated.
String	getStreamingPlanAsJSON()
StreamNode	getStreamNode(Integer vertexID)
Collection<StreamNode>	getStreamNodes()
StreamNode	getTargetVertex(StreamEdge edge)
TimeCharacteristic	getTimeCharacteristic()
InternalTimeServiceManager.Provider	getTimerServiceProvider()
Collection<org.apache.flink.api.java.tuple.Tuple2<String,org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry>>	getUserArtifacts()
protected Collection<? extends Integer>	getVertexIDs()
boolean	isAllVerticesInSameSlotSharingGroupByDefault() Gets whether to put all vertices into the same slot sharing group by default.
boolean	isChainingEnabled()
boolean	isIterative()
void	setAllVerticesInSameSlotSharingGroupByDefault(boolean allVerticesInSameSlotSharingGroupByDefault) Set whether to put all vertices into the same slot sharing group by default.
void	setBufferTimeout(Integer vertexID, long bufferTimeout)
void	setChaining(boolean chaining)
void	setGlobalDataExchangeMode(GlobalDataExchangeMode globalDataExchangeMode)
void	setInputFormat(Integer vertexID, org.apache.flink.api.common.io.InputFormat<?,?> inputFormat)
void	setJobName(String jobName)
void	setManagedMemoryUseCaseWeights(int vertexID, Map<org.apache.flink.core.memory.ManagedMemoryUseCase,Integer> operatorScopeUseCaseWeights, Set<org.apache.flink.core.memory.ManagedMemoryUseCase> slotScopeUseCases)
void	setMaxParallelism(int vertexID, int maxParallelism)
void	setMultipleInputStateKey(Integer vertexID, List<org.apache.flink.api.java.functions.KeySelector<?,?>> keySelectors, org.apache.flink.api.common.typeutils.TypeSerializer<?> keySerializer)
void	setOneInputStateKey(Integer vertexID, org.apache.flink.api.java.functions.KeySelector<?,?> keySelector, org.apache.flink.api.common.typeutils.TypeSerializer<?> keySerializer)
void	setOutputFormat(Integer vertexID, org.apache.flink.api.common.io.OutputFormat<?> outputFormat)
<OUT> void	setOutType(Integer vertexID, org.apache.flink.api.common.typeinfo.TypeInformation<OUT> outType)
void	setParallelism(Integer vertexID, int parallelism)
void	setResources(int vertexID, org.apache.flink.api.common.operators.ResourceSpec minResources, org.apache.flink.api.common.operators.ResourceSpec preferredResources)
void	setSavepointRestoreSettings(org.apache.flink.runtime.jobgraph.SavepointRestoreSettings savepointRestoreSettings)
void	setScheduleMode(org.apache.flink.runtime.jobgraph.ScheduleMode scheduleMode)
void	setSerializers(Integer vertexID, org.apache.flink.api.common.typeutils.TypeSerializer<?> in1, org.apache.flink.api.common.typeutils.TypeSerializer<?> in2, org.apache.flink.api.common.typeutils.TypeSerializer<?> out)
void	setSerializersFrom(Integer from, Integer to)
void	setStateBackend(org.apache.flink.runtime.state.StateBackend backend)
void	setTimeCharacteristic(TimeCharacteristic timeCharacteristic)
void	setTimerServiceProvider(InternalTimeServiceManager.Provider timerServiceProvider)
void	setTransformationUID(Integer nodeId, String transformationId)
void	setTwoInputStateKey(Integer vertexID, org.apache.flink.api.java.functions.KeySelector<?,?> keySelector1, org.apache.flink.api.java.functions.KeySelector<?,?> keySelector2, org.apache.flink.api.common.typeutils.TypeSerializer<?> keySerializer)
void	setUserArtifacts(Collection<org.apache.flink.api.java.tuple.Tuple2<String,org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry>> userArtifacts)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

ITERATION_SOURCE_NAME_PREFIX

public static final String ITERATION_SOURCE_NAME_PREFIX

See Also:

Constant Field Values

ITERATION_SINK_NAME_PREFIX

public static final String ITERATION_SINK_NAME_PREFIX

See Also:

Constant Field Values

vertexIDtoBrokerID

protected Map<Integer,String> vertexIDtoBrokerID

vertexIDtoLoopTimeout

protected Map<Integer,Long> vertexIDtoLoopTimeout

Constructor Detail

StreamGraph

public StreamGraph(org.apache.flink.api.common.ExecutionConfig executionConfig,
                   CheckpointConfig checkpointConfig,
                   org.apache.flink.runtime.jobgraph.SavepointRestoreSettings savepointRestoreSettings)

Method Detail

clear

public void clear()

Remove all registered nodes etc.

getExecutionConfig

public org.apache.flink.api.common.ExecutionConfig getExecutionConfig()

getCheckpointConfig

public CheckpointConfig getCheckpointConfig()

setSavepointRestoreSettings

public void setSavepointRestoreSettings(org.apache.flink.runtime.jobgraph.SavepointRestoreSettings savepointRestoreSettings)

getSavepointRestoreSettings

public org.apache.flink.runtime.jobgraph.SavepointRestoreSettings getSavepointRestoreSettings()

getJobName

public String getJobName()

setJobName

public void setJobName(String jobName)

setChaining

public void setChaining(boolean chaining)

setStateBackend

public void setStateBackend(org.apache.flink.runtime.state.StateBackend backend)

getStateBackend

public org.apache.flink.runtime.state.StateBackend getStateBackend()

getTimerServiceProvider

public InternalTimeServiceManager.Provider getTimerServiceProvider()

setTimerServiceProvider

public void setTimerServiceProvider(InternalTimeServiceManager.Provider timerServiceProvider)

getScheduleMode

public org.apache.flink.runtime.jobgraph.ScheduleMode getScheduleMode()

setScheduleMode

public void setScheduleMode(org.apache.flink.runtime.jobgraph.ScheduleMode scheduleMode)

getUserArtifacts

public Collection<org.apache.flink.api.java.tuple.Tuple2<String,org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry>> getUserArtifacts()

setUserArtifacts

public void setUserArtifacts(Collection<org.apache.flink.api.java.tuple.Tuple2<String,org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry>> userArtifacts)

getTimeCharacteristic

public TimeCharacteristic getTimeCharacteristic()

setTimeCharacteristic

public void setTimeCharacteristic(TimeCharacteristic timeCharacteristic)

getGlobalDataExchangeMode

public GlobalDataExchangeMode getGlobalDataExchangeMode()

setGlobalDataExchangeMode

public void setGlobalDataExchangeMode(GlobalDataExchangeMode globalDataExchangeMode)

setAllVerticesInSameSlotSharingGroupByDefault

public void setAllVerticesInSameSlotSharingGroupByDefault(boolean allVerticesInSameSlotSharingGroupByDefault)

Set whether to put all vertices into the same slot sharing group by default.

Parameters:

allVerticesInSameSlotSharingGroupByDefault - indicates whether to put all vertices into the same slot sharing group by default.

isAllVerticesInSameSlotSharingGroupByDefault

public boolean isAllVerticesInSameSlotSharingGroupByDefault()

Gets whether to put all vertices into the same slot sharing group by default.

Returns:

whether to put all vertices into the same slot sharing group by default.

isChainingEnabled

public boolean isChainingEnabled()

isIterative

public boolean isIterative()

addSource

public <IN,OUT> void addSource(Integer vertexID,
                               @Nullable
                               String slotSharingGroup,
                               @Nullable
                               String coLocationGroup,
                               SourceOperatorFactory<OUT> operatorFactory,
                               org.apache.flink.api.common.typeinfo.TypeInformation<IN> inTypeInfo,
                               org.apache.flink.api.common.typeinfo.TypeInformation<OUT> outTypeInfo,
                               String operatorName)

addLegacySource

public <IN,OUT> void addLegacySource(Integer vertexID,
                                     @Nullable
                                     String slotSharingGroup,
                                     @Nullable
                                     String coLocationGroup,
                                     StreamOperatorFactory<OUT> operatorFactory,
                                     org.apache.flink.api.common.typeinfo.TypeInformation<IN> inTypeInfo,
                                     org.apache.flink.api.common.typeinfo.TypeInformation<OUT> outTypeInfo,
                                     String operatorName)

addSink

public <IN,OUT> void addSink(Integer vertexID,
                             @Nullable
                             String slotSharingGroup,
                             @Nullable
                             String coLocationGroup,
                             StreamOperatorFactory<OUT> operatorFactory,
                             org.apache.flink.api.common.typeinfo.TypeInformation<IN> inTypeInfo,
                             org.apache.flink.api.common.typeinfo.TypeInformation<OUT> outTypeInfo,
                             String operatorName)

addOperator

public <IN,OUT> void addOperator(Integer vertexID,
                                 @Nullable
                                 String slotSharingGroup,
                                 @Nullable
                                 String coLocationGroup,
                                 StreamOperatorFactory<OUT> operatorFactory,
                                 org.apache.flink.api.common.typeinfo.TypeInformation<IN> inTypeInfo,
                                 org.apache.flink.api.common.typeinfo.TypeInformation<OUT> outTypeInfo,
                                 String operatorName)

addCoOperator

public <IN1,IN2,OUT> void addCoOperator(Integer vertexID,
                                        String slotSharingGroup,
                                        @Nullable
                                        String coLocationGroup,
                                        StreamOperatorFactory<OUT> taskOperatorFactory,
                                        org.apache.flink.api.common.typeinfo.TypeInformation<IN1> in1TypeInfo,
                                        org.apache.flink.api.common.typeinfo.TypeInformation<IN2> in2TypeInfo,
                                        org.apache.flink.api.common.typeinfo.TypeInformation<OUT> outTypeInfo,
                                        String operatorName)

addMultipleInputOperator

public <OUT> void addMultipleInputOperator(Integer vertexID,
                                           String slotSharingGroup,
                                           @Nullable
                                           String coLocationGroup,
                                           StreamOperatorFactory<OUT> operatorFactory,
                                           List<org.apache.flink.api.common.typeinfo.TypeInformation<?>> inTypeInfos,
                                           org.apache.flink.api.common.typeinfo.TypeInformation<OUT> outTypeInfo,
                                           String operatorName)

addNode

protected StreamNode addNode(Integer vertexID,
                             @Nullable
                             String slotSharingGroup,
                             @Nullable
                             String coLocationGroup,
                             Class<? extends org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable> vertexClass,
                             StreamOperatorFactory<?> operatorFactory,
                             String operatorName)

addVirtualSideOutputNode

public void addVirtualSideOutputNode(Integer originalId,
                                     Integer virtualId,
                                     org.apache.flink.util.OutputTag outputTag)

Adds a new virtual node that is used to connect a downstream vertex to only the outputs with the selected side-output OutputTag.

Parameters:

originalId - ID of the node that should be connected to.

virtualId - ID of the virtual node.

outputTag - The selected side-output OutputTag.

addVirtualPartitionNode

public void addVirtualPartitionNode(Integer originalId,
                                    Integer virtualId,
                                    StreamPartitioner<?> partitioner,
                                    ShuffleMode shuffleMode)

Adds a new virtual node that is used to connect a downstream vertex to an input with a certain partitioning.

When adding an edge from the virtual node to a downstream node the connection will be made to the original node, but with the partitioning given here.

Parameters:

originalId - ID of the node that should be connected to.

virtualId - ID of the virtual node.

partitioner - The partitioner

getSlotSharingGroup

public String getSlotSharingGroup(Integer id)

Determines the slot sharing group of an operation across virtual nodes.

addEdge

public void addEdge(Integer upStreamVertexID,
                    Integer downStreamVertexID,
                    int typeNumber)

setParallelism

public void setParallelism(Integer vertexID,
                           int parallelism)

setMaxParallelism

public void setMaxParallelism(int vertexID,
                              int maxParallelism)

setResources

public void setResources(int vertexID,
                         org.apache.flink.api.common.operators.ResourceSpec minResources,
                         org.apache.flink.api.common.operators.ResourceSpec preferredResources)

setManagedMemoryUseCaseWeights

public void setManagedMemoryUseCaseWeights(int vertexID,
                                           Map<org.apache.flink.core.memory.ManagedMemoryUseCase,Integer> operatorScopeUseCaseWeights,
                                           Set<org.apache.flink.core.memory.ManagedMemoryUseCase> slotScopeUseCases)

setOneInputStateKey

public void setOneInputStateKey(Integer vertexID,
                                org.apache.flink.api.java.functions.KeySelector<?,?> keySelector,
                                org.apache.flink.api.common.typeutils.TypeSerializer<?> keySerializer)

setTwoInputStateKey

public void setTwoInputStateKey(Integer vertexID,
                                org.apache.flink.api.java.functions.KeySelector<?,?> keySelector1,
                                org.apache.flink.api.java.functions.KeySelector<?,?> keySelector2,
                                org.apache.flink.api.common.typeutils.TypeSerializer<?> keySerializer)

setMultipleInputStateKey

public void setMultipleInputStateKey(Integer vertexID,
                                     List<org.apache.flink.api.java.functions.KeySelector<?,?>> keySelectors,
                                     org.apache.flink.api.common.typeutils.TypeSerializer<?> keySerializer)

setBufferTimeout

public void setBufferTimeout(Integer vertexID,
                             long bufferTimeout)

setSerializers

public void setSerializers(Integer vertexID,
                           org.apache.flink.api.common.typeutils.TypeSerializer<?> in1,
                           org.apache.flink.api.common.typeutils.TypeSerializer<?> in2,
                           org.apache.flink.api.common.typeutils.TypeSerializer<?> out)

setSerializersFrom

public void setSerializersFrom(Integer from,
                               Integer to)

setOutType

public <OUT> void setOutType(Integer vertexID,
                             org.apache.flink.api.common.typeinfo.TypeInformation<OUT> outType)

setInputFormat

public void setInputFormat(Integer vertexID,
                           org.apache.flink.api.common.io.InputFormat<?,?> inputFormat)

setOutputFormat

public void setOutputFormat(Integer vertexID,
                            org.apache.flink.api.common.io.OutputFormat<?> outputFormat)

setTransformationUID

public void setTransformationUID(Integer nodeId,
                                 String transformationId)

getStreamNode

public StreamNode getStreamNode(Integer vertexID)

getVertexIDs

protected Collection<? extends Integer> getVertexIDs()

getStreamEdges

@VisibleForTesting
public List<StreamEdge> getStreamEdges(int sourceId)

getStreamEdges

@VisibleForTesting
public List<StreamEdge> getStreamEdges(int sourceId,
                                                          int targetId)

getStreamEdgesOrThrow

@VisibleForTesting
 @Deprecated
public List<StreamEdge> getStreamEdgesOrThrow(int sourceId,
                                                                              int targetId)

Deprecated.

getSourceIDs

public Collection<Integer> getSourceIDs()

getSinkIDs

public Collection<Integer> getSinkIDs()

getStreamNodes

public Collection<StreamNode> getStreamNodes()

getAllOperatorFactory

public Set<org.apache.flink.api.java.tuple.Tuple2<Integer,StreamOperatorFactory<?>>> getAllOperatorFactory()

getBrokerID

public String getBrokerID(Integer vertexID)

getLoopTimeout

public long getLoopTimeout(Integer vertexID)

createIterationSourceAndSink

public org.apache.flink.api.java.tuple.Tuple2<StreamNode,StreamNode> createIterationSourceAndSink(int loopId,
                                                                                                  int sourceId,
                                                                                                  int sinkId,
                                                                                                  long timeout,
                                                                                                  int parallelism,
                                                                                                  int maxParallelism,
                                                                                                  org.apache.flink.api.common.operators.ResourceSpec minResources,
                                                                                                  org.apache.flink.api.common.operators.ResourceSpec preferredResources)

getIterationSourceSinkPairs

public Set<org.apache.flink.api.java.tuple.Tuple2<StreamNode,StreamNode>> getIterationSourceSinkPairs()

getSourceVertex

public StreamNode getSourceVertex(StreamEdge edge)

getTargetVertex

public StreamNode getTargetVertex(StreamEdge edge)

getJobGraph

public org.apache.flink.runtime.jobgraph.JobGraph getJobGraph()

Gets the assembled JobGraph with a random JobID.

getJobGraph

public org.apache.flink.runtime.jobgraph.JobGraph getJobGraph(@Nullable
                                                              org.apache.flink.api.common.JobID jobID)

Gets the assembled JobGraph with a specified JobID.

getStreamingPlanAsJSON

public String getStreamingPlanAsJSON()