org.apache.nifi.grok

Class GrokReader

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.controller.AbstractControllerService

org.apache.nifi.serialization.SchemaRegistryService

org.apache.nifi.grok.GrokReader

All Implemented Interfaces:

ConfigurableComponent, ControllerService, RecordReaderFactory

@Tags(value={"grok","logs","logfiles","parse","unstructured","text","record","reader","regex","pattern","logstash"})
 @CapabilityDescription(value="Provides a mechanism for reading unstructured text data, such as log files, and structuring the data so that it can be processed. The service is configured using Grok patterns. The service reads from a stream of data and splits each message that it finds into a separate Record, each containing the fields that are configured. If a line in the input does not match the expected message pattern, the line of text is either considered to be part of the previous message or is skipped, depending on the configuration, with the exception of stack traces. A stack trace that is found at the end of a log message is considered to be part of the previous message but is added to the \'stackTrace\' field of the Record. If a record has no stack trace, it will have a NULL value for the stackTrace field (assuming that the schema does in fact include a stackTrace field of type String). Assuming that the schema includes a \'_raw\' field of type String, the raw message will be included in the Record.")
public class GrokReader
extends SchemaRegistryService
implements RecordReaderFactory

Field Summary

Fields

Modifier and Type	Field and Description
(package private) static AllowableValue	APPEND_TO_PREVIOUS_MESSAGE
private boolean	appendUnmatchedLine
(package private) static String	DEFAULT_PATTERN_NAME
private io.krakens.grok.api.Grok	grok
(package private) static PropertyDescriptor	GROK_EXPRESSION
private io.krakens.grok.api.GrokCompiler	grokCompiler
(package private) static PropertyDescriptor	NO_MATCH_BEHAVIOR
(package private) static PropertyDescriptor	PATTERN_FILE
private RecordSchema	recordSchema
private RecordSchema	recordSchemaFromGrok
(package private) static AllowableValue	SKIP_LINE
(package private) static AllowableValue	STRING_FIELDS_FROM_GROK_EXPRESSION

Fields inherited from class org.apache.nifi.serialization.SchemaRegistryService

schemaAccessStrategy

Constructor Summary

Constructors

Constructor and Description
GrokReader()

Method Summary

Modifier and Type	Method and Description
private SchemaAccessStrategy	createAccessStrategy()
RecordReader	createRecordReader(Map<String,String> variables, InputStream in, long inputLength, ComponentLog logger)
(package private) static RecordSchema	createRecordSchema(io.krakens.grok.api.Grok grok)
protected Collection<ValidationResult>	customValidate(ValidationContext validationContext)
protected AllowableValue	getDefaultSchemaAccessStrategy()
protected SchemaAccessStrategy	getSchemaAccessStrategy(String strategy, SchemaRegistry schemaRegistry, PropertyContext context)
protected List<AllowableValue>	getSchemaAccessStrategyValues()
protected List<PropertyDescriptor>	getSupportedPropertyDescriptors()
private static void	populateSchemaFieldNames(io.krakens.grok.api.Grok grok, String grokExpression, List<RecordField> fields)
void	preCompile(ConfigurationContext context)

Methods inherited from class org.apache.nifi.serialization.SchemaRegistryService

buildStrategyProperty, getConfigurationContext, getSchema, getSchema, getSchemaAccessStrategy, getSchemaAcessStrategyDescriptor, getSuppliedSchemaFields, storeSchemaAccessStrategy

Methods inherited from class org.apache.nifi.controller.AbstractControllerService

abstractClearConfigContext, abstractStoreConfigContext, disabled, enabled, getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, getProperty, getStateManager, init, initialize, isEnabled

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, onPropertyModified, toString, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.serialization.RecordReaderFactory

createRecordReader

Methods inherited from interface org.apache.nifi.controller.ControllerService

initialize

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getIdentifier, getPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validate

Field Detail

grokCompiler

private volatile io.krakens.grok.api.GrokCompiler grokCompiler

grok

private volatile io.krakens.grok.api.Grok grok

appendUnmatchedLine

private volatile boolean appendUnmatchedLine

recordSchema

private volatile RecordSchema recordSchema

recordSchemaFromGrok

private volatile RecordSchema recordSchemaFromGrok

DEFAULT_PATTERN_NAME

static final String DEFAULT_PATTERN_NAME

See Also:

Constant Field Values

APPEND_TO_PREVIOUS_MESSAGE

static final AllowableValue APPEND_TO_PREVIOUS_MESSAGE

SKIP_LINE

static final AllowableValue SKIP_LINE

STRING_FIELDS_FROM_GROK_EXPRESSION

static final AllowableValue STRING_FIELDS_FROM_GROK_EXPRESSION

PATTERN_FILE

static final PropertyDescriptor PATTERN_FILE

GROK_EXPRESSION

static final PropertyDescriptor GROK_EXPRESSION

NO_MATCH_BEHAVIOR

static final PropertyDescriptor NO_MATCH_BEHAVIOR

Constructor Detail

GrokReader

public GrokReader()

Method Detail

getSupportedPropertyDescriptors

protected List<PropertyDescriptor> getSupportedPropertyDescriptors()

Overrides:

getSupportedPropertyDescriptors in class SchemaRegistryService

preCompile

@OnEnabled
public void preCompile(ConfigurationContext context)
                           throws io.krakens.grok.api.exception.GrokException,
                                  IOException

Throws:

io.krakens.grok.api.exception.GrokException

IOException

customValidate

protected Collection<ValidationResult> customValidate(ValidationContext validationContext)

Overrides:

customValidate in class SchemaRegistryService

createRecordSchema

static RecordSchema createRecordSchema(io.krakens.grok.api.Grok grok)

populateSchemaFieldNames

private static void populateSchemaFieldNames(io.krakens.grok.api.Grok grok,
                                             String grokExpression,
                                             List<RecordField> fields)

getSchemaAccessStrategyValues

protected List<AllowableValue> getSchemaAccessStrategyValues()

Overrides:

getSchemaAccessStrategyValues in class SchemaRegistryService

getDefaultSchemaAccessStrategy

protected AllowableValue getDefaultSchemaAccessStrategy()

Overrides:

getDefaultSchemaAccessStrategy in class SchemaRegistryService

getSchemaAccessStrategy

protected SchemaAccessStrategy getSchemaAccessStrategy(String strategy,
                                                       SchemaRegistry schemaRegistry,
                                                       PropertyContext context)

Overrides:

getSchemaAccessStrategy in class SchemaRegistryService

createAccessStrategy

private SchemaAccessStrategy createAccessStrategy()

createRecordReader

public RecordReader createRecordReader(Map<String,String> variables,
                                       InputStream in,
                                       long inputLength,
                                       ComponentLog logger)
                                throws IOException,
                                       SchemaNotFoundException

Specified by:

createRecordReader in interface RecordReaderFactory

Throws:

IOException

SchemaNotFoundException