Package org.apache.nifi.processors.standard

Class ControlRate

java.lang.Object
org.apache.nifi.components.AbstractConfigurableComponent
org.apache.nifi.processor.AbstractSessionFactoryProcessor
org.apache.nifi.processor.AbstractProcessor
org.apache.nifi.processors.standard.ControlRate
All Implemented Interfaces:
org.apache.nifi.components.ConfigurableComponent, org.apache.nifi.processor.Processor
@SideEffectFree @TriggerSerially @InputRequirement(INPUT_REQUIRED) @Tags({"rate control","throttle","rate","throughput"}) @CapabilityDescription("Controls the rate at which data is transferred to follow-on processors. If you configure a very small Time Duration, then the accuracy of the throttle gets worse. You can improve this accuracy by decreasing the Yield Duration, at the expense of more Tasks given to the processor.") @UseCase(description="Limit the rate at which data is sent to a downstream system with little to no bursts",keywords={"throttle","limit","slow down","data rate"},configuration="Set the \"Rate Control Criteria\" to `data rate`.\nSet the \"Time Duration\" property to `1 sec`.\nConfigure the \"Maximum Rate\" property to specify how much data should be allowed through each second.\n\nFor example, to allow through 8 MB per second, set \"Maximum Rate\" to `8 MB`.\n") @UseCase(description="Limit the rate at which FlowFiles are sent to a downstream system with little to no bursts",keywords={"throttle","limit","slow down","flowfile rate"},configuration="Set the \"Rate Control Criteria\" to `flowfile count`.\nSet the \"Time Duration\" property to `1 sec`.\nConfigure the \"Maximum Rate\" property to specify how many FlowFiles should be allowed through each second.\n\nFor example, to allow through 100 FlowFiles per second, set \"Maximum Rate\" to `100`.\n") @UseCase(description="Reject requests that exceed a specific rate with little to no bursts",keywords={"throttle","limit","slow down","request rate"},configuration="Set the \"Rate Control Criteria\" to `flowfile count`.\nSet the \"Time Duration\" property to `1 sec`.\nSet the \"Rate Exceeded Strategy\" property to `Route to \'rate exceeded\'`.\nConfigure the \"Maximum Rate\" property to specify how many requests should be allowed through each second.\n\nFor example, to allow through 100 requests per second, set \"Maximum Rate\" to `100`.\nIf more than 100 requests come in during any one second, the additional requests will be routed to `rate exceeded` instead of `success`.\n") @UseCase(description="Reject requests that exceed a specific rate, allowing for bursts",keywords={"throttle","limit","slow down","request rate"},configuration="Set the \"Rate Control Criteria\" to `flowfile count`.\nSet the \"Time Duration\" property to `1 min`.\nSet the \"Rate Exceeded Strategy\" property to `Route to \'rate exceeded\'`.\nConfigure the \"Maximum Rate\" property to specify how many requests should be allowed through each minute.\n\nFor example, to allow through 100 requests per second, set \"Maximum Rate\" to `6000`.\nThis will allow through 6,000 FlowFiles per minute, which averages to 100 FlowFiles per second. However, those 6,000 FlowFiles may come all within the first couple of\nseconds, or they may come in over a period of 60 seconds. As a result, this gives us an average rate of 100 FlowFiles per second but allows for bursts of data.\nIf more than 6,000 requests come in during any one minute, the additional requests will be routed to `rate exceeded` instead of `success`.\n") public class ControlRate extends org.apache.nifi.processor.AbstractProcessor

  • Field Details

    • DATA_RATE

      public static final String DATA_RATE
      See Also:

    • FLOWFILE_RATE

      public static final String FLOWFILE_RATE
      See Also:

    • ATTRIBUTE_RATE

      public static final String ATTRIBUTE_RATE
      See Also:

    • DATA_OR_FLOWFILE_RATE

      public static final String DATA_OR_FLOWFILE_RATE
      See Also:

    • DATA_RATE_VALUE

      public static final org.apache.nifi.components.AllowableValue DATA_RATE_VALUE

    • FLOWFILE_RATE_VALUE

      public static final org.apache.nifi.components.AllowableValue FLOWFILE_RATE_VALUE

    • ATTRIBUTE_RATE_VALUE

      public static final org.apache.nifi.components.AllowableValue ATTRIBUTE_RATE_VALUE

    • DATA_OR_FLOWFILE_RATE_VALUE

      public static final org.apache.nifi.components.AllowableValue DATA_OR_FLOWFILE_RATE_VALUE

    • HOLD_FLOWFILE

      static final org.apache.nifi.components.AllowableValue HOLD_FLOWFILE

    • ROUTE_TO_RATE_EXCEEDED

      static final org.apache.nifi.components.AllowableValue ROUTE_TO_RATE_EXCEEDED

    • MAX_FLOW_FILES_PER_BATCH

      public static final int MAX_FLOW_FILES_PER_BATCH
      See Also:

    • DEFAULT_ACCRUAL_COUNT

      private static final long DEFAULT_ACCRUAL_COUNT
      See Also:

    • RATE_CONTROL_CRITERIA

      public static final org.apache.nifi.components.PropertyDescriptor RATE_CONTROL_CRITERIA

    • MAX_RATE

      public static final org.apache.nifi.components.PropertyDescriptor MAX_RATE

    • MAX_DATA_RATE

      public static final org.apache.nifi.components.PropertyDescriptor MAX_DATA_RATE

    • MAX_COUNT_RATE

      public static final org.apache.nifi.components.PropertyDescriptor MAX_COUNT_RATE

    • RATE_EXCEEDED_STRATEGY

      public static final org.apache.nifi.components.PropertyDescriptor RATE_EXCEEDED_STRATEGY

    • RATE_CONTROL_ATTRIBUTE_NAME

      public static final org.apache.nifi.components.PropertyDescriptor RATE_CONTROL_ATTRIBUTE_NAME

    • TIME_PERIOD

      public static final org.apache.nifi.components.PropertyDescriptor TIME_PERIOD

    • GROUPING_ATTRIBUTE_NAME

      public static final org.apache.nifi.components.PropertyDescriptor GROUPING_ATTRIBUTE_NAME

    • PROPERTIES

      private static final List<org.apache.nifi.components.PropertyDescriptor> PROPERTIES

    • REL_SUCCESS

      static final org.apache.nifi.processor.Relationship REL_SUCCESS

    • REL_FAILURE

      static final org.apache.nifi.processor.Relationship REL_FAILURE

    • REL_RATE_EXCEEDED

      static final org.apache.nifi.processor.Relationship REL_RATE_EXCEEDED

    • DEFAULT_RELATIONSHIPS

      private static final Set<org.apache.nifi.processor.Relationship> DEFAULT_RELATIONSHIPS

    • RATE_EXCEEDED_RELATIONSHIPS

      private static final Set<org.apache.nifi.processor.Relationship> RATE_EXCEEDED_RELATIONSHIPS

    • POSITIVE_LONG_PATTERN

      private static final Pattern POSITIVE_LONG_PATTERN

    • DEFAULT_GROUP_ATTRIBUTE

      private static final String DEFAULT_GROUP_ATTRIBUTE

    • relationships

      private volatile Set<org.apache.nifi.processor.Relationship> relationships

    • dataThrottleMap

      private final ConcurrentMap<String,ControlRate.Throttle> dataThrottleMap

    • countThrottleMap

      private final ConcurrentMap<String,ControlRate.Throttle> countThrottleMap

    • lastThrottleClearTime

      private final AtomicLong lastThrottleClearTime

    • rateControlCriteria

      private volatile String rateControlCriteria

    • rateControlAttribute

      private volatile String rateControlAttribute

    • maximumRateStr

      private volatile String maximumRateStr

    • maximumCountRateStr

      private volatile String maximumCountRateStr

    • groupingAttributeName

      private volatile String groupingAttributeName

    • timePeriodSeconds

      private volatile int timePeriodSeconds

  • Constructor Details

    • ControlRate

      public ControlRate()

  • Method Details

    • getSupportedPropertyDescriptors

      protected List<org.apache.nifi.components.PropertyDescriptor> getSupportedPropertyDescriptors()
      Overrides:
      getSupportedPropertyDescriptors in class org.apache.nifi.components.AbstractConfigurableComponent

    • getRelationships

      public Set<org.apache.nifi.processor.Relationship> getRelationships()
      Specified by:
      getRelationships in interface org.apache.nifi.processor.Processor
      Overrides:
      getRelationships in class org.apache.nifi.processor.AbstractSessionFactoryProcessor

    • customValidate

      protected Collection<org.apache.nifi.components.ValidationResult> customValidate(org.apache.nifi.components.ValidationContext context)
      Overrides:
      customValidate in class org.apache.nifi.components.AbstractConfigurableComponent

    • onPropertyModified

      public void onPropertyModified(org.apache.nifi.components.PropertyDescriptor descriptor, String oldValue, String newValue)
      Specified by:
      onPropertyModified in interface org.apache.nifi.components.ConfigurableComponent
      Overrides:
      onPropertyModified in class org.apache.nifi.components.AbstractConfigurableComponent

    • onScheduled

      @OnScheduled public void onScheduled(org.apache.nifi.processor.ProcessContext context)

    • onTrigger

      public void onTrigger(org.apache.nifi.processor.ProcessContext context, org.apache.nifi.processor.ProcessSession session) throws org.apache.nifi.processor.exception.ProcessException
      Specified by:
      onTrigger in class org.apache.nifi.processor.AbstractProcessor
      Throws:
      org.apache.nifi.processor.exception.ProcessException

    • routeFlowFilesExceedingRate

      private void routeFlowFilesExceedingRate(org.apache.nifi.processor.ProcessContext context, org.apache.nifi.processor.ProcessSession session)

    • holdFlowFilesExceedingRate

      private void holdFlowFilesExceedingRate(org.apache.nifi.processor.ProcessContext context, org.apache.nifi.processor.ProcessSession session)

    • clearExpiredThrottles

      private void clearExpiredThrottles(org.apache.nifi.processor.ProcessContext context)

    • getCurrentTimeMillis

      protected long getCurrentTimeMillis()
      Get current time in milliseconds
      Returns:
      Current time in milliseconds from System

    • isRateAttributeValid

      private boolean isRateAttributeValid(org.apache.nifi.flowfile.FlowFile flowFile)

    • getDataSizeAccrual

      private long getDataSizeAccrual(org.apache.nifi.flowfile.FlowFile flowFile)

    • getCountAccrual

      private long getCountAccrual(org.apache.nifi.flowfile.FlowFile flowFile)

    • dataThrottleRequired

      private boolean dataThrottleRequired()

    • countThrottleRequired

      private boolean countThrottleRequired()