org.apache.nifi.grok

Class GrokReader

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.controller.AbstractControllerService

org.apache.nifi.serialization.SchemaRegistryService

org.apache.nifi.grok.GrokReader

All Implemented Interfaces:

ConfigurableComponent, ControllerService, RecordReaderFactory

@Tags(value={"grok","logs","logfiles","parse","unstructured","text","record","reader","regex","pattern","logstash"})
 @CapabilityDescription(value="Provides a mechanism for reading unstructured text data, such as log files, and structuring the data so that it can be processed. The service is configured using Grok patterns. The service reads from a stream of data and splits each message that it finds into a separate Record, each containing the fields that are configured. If a line in the input does not match the expected message pattern, the line of text is either considered to be part of the previous message or is skipped, depending on the configuration, with the exception of stack traces. A stack trace that is found at the end of a log message is considered to be part of the previous message but is added to the \'stackTrace\' field of the Record. If a record has no stack trace, it will have a NULL value for the stackTrace field (assuming that the schema does in fact include a stackTrace field of type String). Assuming that the schema includes a \'_raw\' field of type String, the raw message will be included in the Record.")
public class GrokReader
extends SchemaRegistryService
implements RecordReaderFactory

Field Summary

Fields

Modifier and Type	Field and Description
(package private) static AllowableValue	APPEND_TO_PREVIOUS_MESSAGE
(package private) static String	DEFAULT_PATTERN_NAME
(package private) static PropertyDescriptor	GROK_EXPRESSION
(package private) static PropertyDescriptor	GROK_PATTERNS
private List<io.krakens.grok.api.Grok>	groks
(package private) static PropertyDescriptor	NO_MATCH_BEHAVIOR
private NoMatchStrategy	noMatchStrategy
(package private) static AllowableValue	RAW_LINE
private RecordSchema	recordSchema
private RecordSchema	recordSchemaFromGrok
(package private) static AllowableValue	SKIP_LINE
(package private) static AllowableValue	STRING_FIELDS_FROM_GROK_EXPRESSION

Fields inherited from class org.apache.nifi.serialization.SchemaRegistryService

schemaAccessStrategy

Constructor Summary

Constructors

Constructor and Description
GrokReader()

Method Summary

Modifier and Type	Method and Description
private SchemaAccessStrategy	createAccessStrategy()
RecordReader	createRecordReader(Map<String,String> variables, InputStream in, long inputLength, ComponentLog logger)
(package private) static RecordSchema	createRecordSchema(List<io.krakens.grok.api.Grok> groks)
protected Collection<ValidationResult>	customValidate(ValidationContext validationContext)
private Reader	getDefaultPatterns()
protected AllowableValue	getDefaultSchemaAccessStrategy()
protected SchemaAccessStrategy	getSchemaAccessStrategy(String strategy, SchemaRegistry schemaRegistry, PropertyContext context)
protected List<AllowableValue>	getSchemaAccessStrategyValues()
protected List<PropertyDescriptor>	getSupportedPropertyDescriptors()
private static void	populateSchemaFieldNames(io.krakens.grok.api.Grok grok, String grokExpression, Collection<RecordField> fields)
void	preCompile(ConfigurationContext context)
private List<String>	readGrokExpressions(PropertyContext propertyContext)

Methods inherited from class org.apache.nifi.serialization.SchemaRegistryService

buildStrategyProperty, getConfigurationContext, getSchema, getSchema, getSchemaAccessStrategy, getSchemaAcessStrategyDescriptor, getSuppliedSchemaFields, storeSchemaAccessStrategy

Methods inherited from class org.apache.nifi.controller.AbstractControllerService

abstractClearConfigContext, abstractStoreConfigContext, disabled, enabled, getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, getProperty, getStateManager, init, initialize, isEnabled

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, onPropertyModified, toString, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.serialization.RecordReaderFactory

createRecordReader

Methods inherited from interface org.apache.nifi.controller.ControllerService

initialize, isStateful

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getIdentifier, getPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validate

Field Detail

groks

private volatile List<io.krakens.grok.api.Grok> groks

noMatchStrategy

private volatile NoMatchStrategy noMatchStrategy

recordSchema

private volatile RecordSchema recordSchema

recordSchemaFromGrok

private volatile RecordSchema recordSchemaFromGrok

DEFAULT_PATTERN_NAME

static final String DEFAULT_PATTERN_NAME

See Also:

Constant Field Values

APPEND_TO_PREVIOUS_MESSAGE

static final AllowableValue APPEND_TO_PREVIOUS_MESSAGE

SKIP_LINE

static final AllowableValue SKIP_LINE

RAW_LINE

static final AllowableValue RAW_LINE

STRING_FIELDS_FROM_GROK_EXPRESSION

static final AllowableValue STRING_FIELDS_FROM_GROK_EXPRESSION

GROK_PATTERNS

static final PropertyDescriptor GROK_PATTERNS

GROK_EXPRESSION

static final PropertyDescriptor GROK_EXPRESSION

NO_MATCH_BEHAVIOR

static final PropertyDescriptor NO_MATCH_BEHAVIOR

Constructor Detail

GrokReader

public GrokReader()

Method Detail

getSupportedPropertyDescriptors

protected List<PropertyDescriptor> getSupportedPropertyDescriptors()

Overrides:

getSupportedPropertyDescriptors in class SchemaRegistryService

preCompile

@OnEnabled
public void preCompile(ConfigurationContext context)
                           throws io.krakens.grok.api.exception.GrokException,
                                  IOException

Throws:

io.krakens.grok.api.exception.GrokException

IOException

customValidate

protected Collection<ValidationResult> customValidate(ValidationContext validationContext)

Overrides:

customValidate in class SchemaRegistryService

readGrokExpressions

private List<String> readGrokExpressions(PropertyContext propertyContext)
                                  throws IOException

Throws:

IOException

createRecordSchema

static RecordSchema createRecordSchema(List<io.krakens.grok.api.Grok> groks)

populateSchemaFieldNames

private static void populateSchemaFieldNames(io.krakens.grok.api.Grok grok,
                                             String grokExpression,
                                             Collection<RecordField> fields)

getSchemaAccessStrategyValues

protected List<AllowableValue> getSchemaAccessStrategyValues()

Overrides:

getSchemaAccessStrategyValues in class SchemaRegistryService

getDefaultSchemaAccessStrategy

protected AllowableValue getDefaultSchemaAccessStrategy()

Overrides:

getDefaultSchemaAccessStrategy in class SchemaRegistryService

getSchemaAccessStrategy

protected SchemaAccessStrategy getSchemaAccessStrategy(String strategy,
                                                       SchemaRegistry schemaRegistry,
                                                       PropertyContext context)

Overrides:

getSchemaAccessStrategy in class SchemaRegistryService

createAccessStrategy

private SchemaAccessStrategy createAccessStrategy()

createRecordReader

public RecordReader createRecordReader(Map<String,String> variables,
                                       InputStream in,
                                       long inputLength,
                                       ComponentLog logger)
                                throws IOException,
                                       SchemaNotFoundException

Specified by:

createRecordReader in interface RecordReaderFactory

Throws:

IOException

SchemaNotFoundException

getDefaultPatterns

private Reader getDefaultPatterns()
                           throws IOException

Throws:

IOException