Package org.apache.nifi.processors.standard

Class GenerateTableFetch

java.lang.Object
org.apache.nifi.components.AbstractConfigurableComponent
org.apache.nifi.processor.AbstractSessionFactoryProcessor
org.apache.nifi.processors.standard.AbstractDatabaseFetchProcessor
org.apache.nifi.processors.standard.GenerateTableFetch
All Implemented Interfaces:
org.apache.nifi.components.ConfigurableComponent, org.apache.nifi.processor.Processor
@TriggerSerially @InputRequirement(INPUT_ALLOWED) @Tags({"sql","select","jdbc","query","database","fetch","generate"}) @SeeAlso({QueryDatabaseTable.class,ExecuteSQL.class,ListDatabaseTables.class}) @CapabilityDescription("Generates SQL select queries that fetch \"pages\" of rows from a table. The partition size property, along with the table\'s row count, determine the size and number of pages and generated FlowFiles. In addition, incremental fetching can be achieved by setting Maximum-Value Columns, which causes the processor to track the columns\' maximum values, thus only fetching rows whose columns\' values exceed the observed maximums. This processor is intended to be run on the Primary Node only.\n\nThis processor can accept incoming connections; the behavior of the processor is different whether incoming connections are provided:\n - If no incoming connection(s) are specified, the processor will generate SQL queries on the specified processor schedule. Expression Language is supported for many fields, but no FlowFile attributes are available. However the properties will be evaluated using the Environment/System properties.\n - If incoming connection(s) are specified and no FlowFile is available to a processor task, no work will be performed.\n - If incoming connection(s) are specified and a FlowFile is available to a processor task, the FlowFile\'s attributes may be used in Expression Language for such fields as Table Name and others. However, the Max-Value Columns and Columns to Return fields must be empty or refer to columns that are available in each specified table.") @Stateful(scopes=CLUSTER, description="After performing a query on the specified table, the maximum values for the specified column(s) will be retained for use in future executions of the query. This allows the Processor to fetch only those records that have max values greater than the retained values. This can be used for incremental fetching, fetching of newly added rows, etc. To clear the maximum values, clear the state of the processor per the State Management documentation") @WritesAttribute(attribute="generatetablefetch.sql.error",description="If the processor has incoming connections, and processing an incoming FlowFile causes a SQL Exception, the FlowFile is routed to failure and this attribute is set to the exception message.") @WritesAttribute(attribute="generatetablefetch.tableName",description="The name of the database table to be queried.") @WritesAttribute(attribute="generatetablefetch.columnNames",description="The comma-separated list of column names used in the query.") @WritesAttribute(attribute="generatetablefetch.whereClause",description="Where clause used in the query to get the expected rows.") @WritesAttribute(attribute="generatetablefetch.maxColumnNames",description="The comma-separated list of column names used to keep track of data that has been returned since the processor started running.") @WritesAttribute(attribute="generatetablefetch.limit",description="The number of result rows to be fetched by the SQL statement.") @WritesAttribute(attribute="generatetablefetch.offset",description="Offset to be used to retrieve the corresponding partition.") @WritesAttribute(attribute="fragment.identifier",description="All FlowFiles generated from the same query result set will have the same value for the fragment.identifier attribute. This can then be used to correlate the results.") @WritesAttribute(attribute="fragment.count",description="This is the total number of FlowFiles produced by a single ResultSet. This can be used in conjunction with the fragment.identifier attribute in order to know how many FlowFiles belonged to the same incoming ResultSet.") @WritesAttribute(attribute="fragment.index",description="This is the position of this FlowFile in the list of outgoing FlowFiles that were all generated from the same execution. This can be used in conjunction with the fragment.identifier attribute to know which FlowFiles originated from the same execution and in what order FlowFiles were produced") @DynamicProperty(name="initial.maxvalue.<max_value_column>", value="Initial maximum value for the specified column", expressionLanguageScope=FLOWFILE_ATTRIBUTES, description="Specifies an initial max value for max value columns. Properties should be added in the format `initial.maxvalue.<max_value_column>`. This value is only used the first time the table is accessed (when a Maximum Value Column is specified). In the case of incoming connections, the value is only used the first time for each table specified in the FlowFiles.") public class GenerateTableFetch extends AbstractDatabaseFetchProcessor

  • Field Details

    • PARTITION_SIZE

      public static final org.apache.nifi.components.PropertyDescriptor PARTITION_SIZE

    • COLUMN_FOR_VALUE_PARTITIONING

      static final org.apache.nifi.components.PropertyDescriptor COLUMN_FOR_VALUE_PARTITIONING

    • OUTPUT_EMPTY_FLOWFILE_ON_ZERO_RESULTS

      static final org.apache.nifi.components.PropertyDescriptor OUTPUT_EMPTY_FLOWFILE_ON_ZERO_RESULTS

    • CUSTOM_ORDERBY_COLUMN

      static final org.apache.nifi.components.PropertyDescriptor CUSTOM_ORDERBY_COLUMN

    • PROPERTIES

      private static final List<org.apache.nifi.components.PropertyDescriptor> PROPERTIES

    • REL_FAILURE

      public static final org.apache.nifi.processor.Relationship REL_FAILURE

    • RELATIONSHIPS

      private static final Set<org.apache.nifi.processor.Relationship> RELATIONSHIPS

  • Constructor Details

    • GenerateTableFetch

      public GenerateTableFetch()

  • Method Details

    • getRelationships

      public Set<org.apache.nifi.processor.Relationship> getRelationships()
      Specified by:
      getRelationships in interface org.apache.nifi.processor.Processor
      Overrides:
      getRelationships in class org.apache.nifi.processor.AbstractSessionFactoryProcessor

    • getSupportedPropertyDescriptors

      protected List<org.apache.nifi.components.PropertyDescriptor> getSupportedPropertyDescriptors()
      Overrides:
      getSupportedPropertyDescriptors in class org.apache.nifi.components.AbstractConfigurableComponent

    • getSupportedDynamicPropertyDescriptor

      protected org.apache.nifi.components.PropertyDescriptor getSupportedDynamicPropertyDescriptor(String propertyDescriptorName)
      Overrides:
      getSupportedDynamicPropertyDescriptor in class org.apache.nifi.components.AbstractConfigurableComponent

    • customValidate

      protected Collection<org.apache.nifi.components.ValidationResult> customValidate(org.apache.nifi.components.ValidationContext validationContext)
      Overrides:
      customValidate in class AbstractDatabaseFetchProcessor

    • setup

      @OnScheduled public void setup(org.apache.nifi.processor.ProcessContext context)
      Overrides:
      setup in class AbstractDatabaseFetchProcessor

    • stop

      @OnStopped public void stop()

    • onTrigger

      public void onTrigger(org.apache.nifi.processor.ProcessContext context, org.apache.nifi.processor.ProcessSessionFactory sessionFactory) throws org.apache.nifi.processor.exception.ProcessException
      Throws:
      org.apache.nifi.processor.exception.ProcessException

    • getColumnStateMaxValue

      private String getColumnStateMaxValue(String tableName, Map<String,String> statePropertyMap, String colName, DatabaseAdapter adapter)

    • getColumnType

      private Integer getColumnType(String tableName, String colName, DatabaseAdapter adapter)