Documentation
¶
Index ¶
- Constants
- Variables
- type Error
- type Lexer
- type Location
- type Token
- func (t Token) After(o Token) bool
- func (t Token) Before(o Token) bool
- func (t Token) Bytes() []byte
- func (t Token) GoString() string
- func (t Token) Int() (int64, error)
- func (t Token) IsComment() bool
- func (t Token) IsError() bool
- func (t Token) IsID() bool
- func (t Token) IsKeyword() bool
- func (t Token) IsNumeric() bool
- func (t Token) IsPunct() bool
- func (t Token) IsQuoted() bool
- func (t Token) Near(o Token, byLines, byColumns int) bool
- func (t Token) Rune() (r rune, size int)
- func (t Token) String() string
- func (t Token) Uint() (uint64, error)
- type TokenType
Constants ¶
const ( TEOF = 1 + TokenType(iota) TTrue // true TFalse // false TNil // nil TDot // . TDoubleDot // .. TTripleDot // ... THash // # TAt // @ (word initial) TDollar // $ (word initial) TParenOpen // ( TParenClose // ) TBracketOpen // [ TBracketClose // ] TCurlOpen // { TCurlClose // } TBackslash // \ TComma // , TSemicolon // ; TNewline // \n TQuote // ' TBang // ! TNotEqual // != TSlash // / TSlashEquals // /= TPercent // % TPercentEquals // %= TCaret // ^ TCaretEquals // ^= TTilde // ~ (word initial) TTildeEquals // ~= TGreaterThan // > TShiftRight // >> TGreaterEqual // >= TLessThan // < TShiftLeft // << TLesserEqual // <= TInequality // <> TQuestion // ? TDoubleQuestion // ?? TQuestionEquals // ?= TEquals // = TEquality // == TShovel // => TPipe // | TPipeEquals // |= TOr // || TOrEquals // ||= TAmp // & TAmpEquals // &= TAnd // && TAndEquals // &&= TMul // * TDoubleMul // ** TMulAssign // *= TColon // : TDoubleColon // :: TColonEqual // := TColonArrow // :> TMinus // - TDoubleMinus // -- TMinusEquals // -= TArrow // -> TPlus // + TDoublePlus // ++ TPlusEquals // += TPlusArrow // +> TIdent // any identifier TIntLit // 5 TIntExpLit // 5e+8 TFloatLit // 0.5 TFloatExpLit // 0.5e+8 TOctLit // 0655 THexLit // 0x1234abcd 0X1234ABCD TBinLit // 0b1101 0B1101 TRawStringLit // `raw string` TRuneLit // 'c' TStringLit // "string" TLineComment // // comment TBlockComment // /* comment */ )
const TError = TokenType(-1)
TError is the token type for any failed attempt to parse a token. The token's Err field should be set when this is its type.
Variables ¶
var (
ErrInvalidKind = errors.New("cannot convert token of this kind")
)
Functions ¶
This section is empty.
Types ¶
type Lexer ¶
type Lexer struct {
// contains filtered or unexported fields
}
func New ¶
func New(r io.RuneScanner) *Lexer
New allocates a new Lexer that reads its input from the given io.RuneScanner, r. If r is nil, New panics.
func (*Lexer) ReadToken ¶
ReadToken reads the next token from the Lexer's rune stream. If an error occurs, it is returned. If a parsing error occurs, it will typically be returned as a token of kind TError with its Err field set. Subsequent reads will attempt to resume from wherever the last token stopped. If the last token was an error, the next token may be in the middle of the previous erroneous token.
type Location ¶
type Location struct {
Off int // Offset in bytes from the unit's start.
Line int // Line number (starting at 1).
Col int // Column number (starting at 1).
}
Location is a location in a token's source unit.
type Token ¶
type Token struct {
Start, End Location
Kind TokenType
Text []byte
Err error // Only likely to be set if Kind == TError
}
func (Token) IsComment ¶
IsComment returns whether the token is a comment token (i.e., you might want to discard it).
func (Token) IsError ¶
IsIdent returns whether the token is an error. This is shorthand for t.Kind == TError.
func (Token) IsID ¶
IsID returns whether the token is an identifier. This is shorthand for t.Kind == TIdent.
func (Token) IsNumeric ¶
IsNumeric returns whether the token is numeric. Check its Kind field to determine the specific type of numeric.
func (Token) IsPunct ¶
IsPunct returns whether the token is a punctuation token (newlines, parentheses, brackets, symbols, etc.).
Source Files
Directories