org.jruby.lexer.yacc

Class RubyLexer

java.lang.Object

org.jruby.lexer.yacc.RubyLexer

public class RubyLexer
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	RubyLexer.Keyword
static class	RubyLexer.LexState

Field Summary

Fields

Modifier and Type	Field and Description
boolean	commandStart
protected org.jruby.util.ByteList	delayed
protected boolean	eofp
boolean	inKwarg
protected org.jruby.util.ByteList	lex_lastline
protected int	lex_p
protected int	lex_pbeg
protected int	lex_pend
protected org.jruby.util.ByteList	lexb
ISourcePosition	tokline
protected int	tokp

Constructor Summary

Constructors

Constructor and Description
RubyLexer(ParserSupport support, LexerSource source)

Method Summary

Methods

Modifier and Type	Method and Description
int	column()
void	compile_error(String message)
void	compile_error(SyntaxException.PID pid, String message)
StrNode	createStr(org.jruby.util.ByteList buffer, int flags)
org.jruby.util.ByteList	createTokenByteList()
String	createTokenString()
int	getBraceNest()
StackState	getCmdArgumentState()
StackState	getConditionState()
String	getCurrentLine()
org.jcodings.Encoding	getEncoding()
String	getFile()
static RubyLexer.Keyword	getKeyword(String str)
int	getLeftParenBegin()
ISourcePosition	getPosition()
ISourcePosition	getPosition(ISourcePosition startPosition)
RubyLexer.LexState	getState()
StrTerm	getStrTerm()
protected void	handleFileEncodingComment(org.jruby.util.ByteList encodingLine)
boolean	hasStarted() Has lexing started yet?
void	heredoc_restore(HeredocTerm here)
int	incrementParenNest()
boolean	isASCII(int c)
boolean	isEndSeen()
boolean	isGlobalCharPunct(int c)
boolean	isIdentifierChar(int c) This is a valid character for an identifier?
protected boolean	isMultiByteChar(int c) Is this a multibyte character from a multibyte encoding?
protected void	lex_goto_eol()
void	newtok(boolean unreadOnce)
int	nextc()
int	nextToken()
int	p(int offset)
void	parser_prepare()
boolean	peek(int c)
int	precise_mbclen()
void	pushback(int c)
int	readCodepoint(int first, org.jcodings.Encoding encoding)
int	readEscape()
int	readUTFEscape(org.jruby.util.ByteList buffer, boolean stringLiteral, boolean symbolLiteral)
void	readUTFEscapeRegexpLiteral(org.jruby.util.ByteList buffer)
void	reset()
void	resetStacks()
protected void	set_file_encoding(int str, int send)
void	setBraceNest(int nest)
void	setEncoding(org.jcodings.Encoding encoding)
void	setLeftParenBegin(int value)
void	setParserSupport(ParserSupport parserSupport) Parse must pass its support object for some check at bottom of yylex().
void	setSource(LexerSource source) Allow the parser to set the source for its lexer.
void	setState(RubyLexer.LexState state)
void	setStrTerm(StrTerm strterm)
void	setValue(Object yaccValue)
void	setWarnings(IRubyWarnings warnings)
boolean	tokadd_ident(int c)
boolean	tokadd_mbchar(int first_byte) This differs from MRI in a few ways.
boolean	tokadd_mbchar(int first_byte, org.jruby.util.ByteList buffer)
void	tokAdd(int first_byte, org.jruby.util.ByteList buffer)
void	tokaddmbc(int codepoint, org.jruby.util.ByteList buffer) This looks deceptively like tokadd_mbchar(int, ByteList) but it differs in that it uses the bytelists encoding and the first parameter is a full codepoint and not the first byte of a mbc sequence.
void	tokCopy(int length, org.jruby.util.ByteList buffer)
int	token() Last token read from the lexer at the end of a call to yylex()
void	tokenAddMBCFromSrc(int c, org.jruby.util.ByteList buffer)
int	tokenize_ident(int result)
Object	value() Value of last token (if it is a token which has a value).
boolean	was_bol()
boolean	whole_match_p(org.jruby.util.ByteList eos, boolean indent)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

tokline

public ISourcePosition tokline

commandStart

public boolean commandStart

inKwarg

public boolean inKwarg

tokp

protected int tokp

lexb

protected org.jruby.util.ByteList lexb

lex_p

protected int lex_p

lex_pbeg

protected int lex_pbeg

lex_pend

protected int lex_pend

lex_lastline

protected org.jruby.util.ByteList lex_lastline

eofp

protected boolean eofp

delayed

protected org.jruby.util.ByteList delayed

Constructor Detail

RubyLexer

public RubyLexer(ParserSupport support,
         LexerSource source)

Method Detail

getEncoding

public org.jcodings.Encoding getEncoding()

getKeyword

public static RubyLexer.Keyword getKeyword(String str)

newtok

public void newtok(boolean unreadOnce)

tokadd_ident

public boolean tokadd_ident(int c)

createTokenByteList

public org.jruby.util.ByteList createTokenByteList()

createTokenString

public String createTokenString()

tokenize_ident

public int tokenize_ident(int result)

incrementParenNest

public int incrementParenNest()

getBraceNest

public int getBraceNest()

setBraceNest

public void setBraceNest(int nest)

getLeftParenBegin

public int getLeftParenBegin()

setLeftParenBegin

public void setLeftParenBegin(int value)

reset

public final void reset()

hasStarted

public boolean hasStarted()

Has lexing started yet?

isEndSeen

public boolean isEndSeen()

p

public int p(int offset)

nextc

public int nextc()

peek

public boolean peek(int c)

lex_goto_eol

protected void lex_goto_eol()

column

public int column()

was_bol

public boolean was_bol()

pushback

public void pushback(int c)

compile_error

public void compile_error(String message)

compile_error

public void compile_error(SyntaxException.PID pid,
                 String message)

readCodepoint

public int readCodepoint(int first,
                org.jcodings.Encoding encoding)
                  throws IOException

Throws:

IOException

heredoc_restore

public void heredoc_restore(HeredocTerm here)

parser_prepare

public void parser_prepare()

nextToken

public int nextToken()
              throws IOException

Throws:

IOException

token

public int token()

Last token read from the lexer at the end of a call to yylex()

Returns:

last token read

value

public Object value()

Value of last token (if it is a token which has a value).

Returns:

value of last value-laden token

getPosition

public ISourcePosition getPosition()

getPosition

public ISourcePosition getPosition(ISourcePosition startPosition)

getFile

public String getFile()

getCurrentLine

public String getCurrentLine()

whole_match_p

public boolean whole_match_p(org.jruby.util.ByteList eos,
                    boolean indent)

setParserSupport

public void setParserSupport(ParserSupport parserSupport)

Parse must pass its support object for some check at bottom of yylex(). Ruby does it this way as well (i.e. a little parsing logic in the lexer).

Parameters:

parserSupport -

setEncoding

public void setEncoding(org.jcodings.Encoding encoding)

setSource

public void setSource(LexerSource source)

Allow the parser to set the source for its lexer.

Parameters:

source - where the lexer gets raw data

getStrTerm

public StrTerm getStrTerm()

setStrTerm

public void setStrTerm(StrTerm strterm)

resetStacks

public void resetStacks()

setWarnings

public void setWarnings(IRubyWarnings warnings)

getState

public RubyLexer.LexState getState()

setState

public void setState(RubyLexer.LexState state)

getCmdArgumentState

public StackState getCmdArgumentState()

getConditionState

public StackState getConditionState()

setValue

public void setValue(Object yaccValue)

isIdentifierChar

public boolean isIdentifierChar(int c)

This is a valid character for an identifier?

Parameters:

c - is character to be compared

Returns:

whether c is an identifier or not mri: is_identchar

isASCII

public boolean isASCII(int c)

isMultiByteChar

protected boolean isMultiByteChar(int c)

Is this a multibyte character from a multibyte encoding?

Parameters:

c -

Returns:

whether c is an multibyte char or not

createStr

public StrNode createStr(org.jruby.util.ByteList buffer,
                int flags)

handleFileEncodingComment

protected void handleFileEncodingComment(org.jruby.util.ByteList encodingLine)
                                  throws IOException

Throws:

IOException

set_file_encoding

protected void set_file_encoding(int str,
                     int send)

isGlobalCharPunct

public boolean isGlobalCharPunct(int c)

readUTFEscapeRegexpLiteral

public void readUTFEscapeRegexpLiteral(org.jruby.util.ByteList buffer)
                                throws IOException

Throws:

IOException

tokadd_mbchar

public boolean tokadd_mbchar(int first_byte)

This differs from MRI in a few ways. This version does not apply value to a separate token buffer. It is for use when we know we will not be omitting or including ant non-syntactical characters. Use tokadd_mbchar(int, ByteList) if the string differs from actual source. Secondly, this returns a boolean instead of the first byte passed. MRI only used the return value as a success/failure code to return EOF. Because this version does not use a separate token buffer we only just increment lex_p. When we reach end of the token it will just get the bytes directly from source directly.

tokadd_mbchar

public boolean tokadd_mbchar(int first_byte,
                    org.jruby.util.ByteList buffer)

See Also:

RubyLexer::tokadd_mbchar(int)

tokaddmbc

public void tokaddmbc(int codepoint,
             org.jruby.util.ByteList buffer)

This looks deceptively like tokadd_mbchar(int, ByteList) but it differs in that it uses the bytelists encoding and the first parameter is a full codepoint and not the first byte of a mbc sequence.

tokAdd

public void tokAdd(int first_byte,
          org.jruby.util.ByteList buffer)

tokCopy

public void tokCopy(int length,
           org.jruby.util.ByteList buffer)

precise_mbclen

public int precise_mbclen()

tokenAddMBCFromSrc

public void tokenAddMBCFromSrc(int c,
                      org.jruby.util.ByteList buffer)
                        throws IOException

Throws:

IOException

readUTFEscape

public int readUTFEscape(org.jruby.util.ByteList buffer,
                boolean stringLiteral,
                boolean symbolLiteral)
                  throws IOException

Throws:

IOException

readEscape

public int readEscape()
               throws IOException

Throws:

IOException