token.go

Documentation: go/token

		 1  // Copyright 2009 The Go Authors. All rights reserved.
		 2  // Use of this source code is governed by a BSD-style
		 3  // license that can be found in the LICENSE file.
		 4  
		 5  // Package token defines constants representing the lexical tokens of the Go
		 6  // programming language and basic operations on tokens (printing, predicates).
		 7  //
		 8  package token
		 9  
		10  import (
		11  	"strconv"
		12  	"unicode"
		13  	"unicode/utf8"
		14  )
		15  
		16  // Token is the set of lexical tokens of the Go programming language.
		17  type Token int
		18  
		19  // The list of tokens.
		20  const (
		21  	// Special tokens
		22  	ILLEGAL Token = iota
		23  	EOF
		24  	COMMENT
		25  
		26  	literal_beg
		27  	// Identifiers and basic type literals
		28  	// (these tokens stand for classes of literals)
		29  	IDENT	// main
		30  	INT		// 12345
		31  	FLOAT	// 123.45
		32  	IMAG	 // 123.45i
		33  	CHAR	 // 'a'
		34  	STRING // "abc"
		35  	literal_end
		36  
		37  	operator_beg
		38  	// Operators and delimiters
		39  	ADD // +
		40  	SUB // -
		41  	MUL // *
		42  	QUO // /
		43  	REM // %
		44  
		45  	AND		 // &
		46  	OR			// |
		47  	XOR		 // ^
		48  	SHL		 // <<
		49  	SHR		 // >>
		50  	AND_NOT // &^
		51  
		52  	ADD_ASSIGN // +=
		53  	SUB_ASSIGN // -=
		54  	MUL_ASSIGN // *=
		55  	QUO_ASSIGN // /=
		56  	REM_ASSIGN // %=
		57  
		58  	AND_ASSIGN		 // &=
		59  	OR_ASSIGN			// |=
		60  	XOR_ASSIGN		 // ^=
		61  	SHL_ASSIGN		 // <<=
		62  	SHR_ASSIGN		 // >>=
		63  	AND_NOT_ASSIGN // &^=
		64  
		65  	LAND	// &&
		66  	LOR	 // ||
		67  	ARROW // <-
		68  	INC	 // ++
		69  	DEC	 // --
		70  
		71  	EQL		// ==
		72  	LSS		// <
		73  	GTR		// >
		74  	ASSIGN // =
		75  	NOT		// !
		76  
		77  	NEQ			// !=
		78  	LEQ			// <=
		79  	GEQ			// >=
		80  	DEFINE	 // :=
		81  	ELLIPSIS // ...
		82  
		83  	LPAREN // (
		84  	LBRACK // [
		85  	LBRACE // {
		86  	COMMA	// ,
		87  	PERIOD // .
		88  
		89  	RPAREN		// )
		90  	RBRACK		// ]
		91  	RBRACE		// }
		92  	SEMICOLON // ;
		93  	COLON		 // :
		94  	operator_end
		95  
		96  	keyword_beg
		97  	// Keywords
		98  	BREAK
		99  	CASE
	 100  	CHAN
	 101  	CONST
	 102  	CONTINUE
	 103  
	 104  	DEFAULT
	 105  	DEFER
	 106  	ELSE
	 107  	FALLTHROUGH
	 108  	FOR
	 109  
	 110  	FUNC
	 111  	GO
	 112  	GOTO
	 113  	IF
	 114  	IMPORT
	 115  
	 116  	INTERFACE
	 117  	MAP
	 118  	PACKAGE
	 119  	RANGE
	 120  	RETURN
	 121  
	 122  	SELECT
	 123  	STRUCT
	 124  	SWITCH
	 125  	TYPE
	 126  	VAR
	 127  	keyword_end
	 128  )
	 129  
	 130  var tokens = [...]string{
	 131  	ILLEGAL: "ILLEGAL",
	 132  
	 133  	EOF:		 "EOF",
	 134  	COMMENT: "COMMENT",
	 135  
	 136  	IDENT:	"IDENT",
	 137  	INT:		"INT",
	 138  	FLOAT:	"FLOAT",
	 139  	IMAG:	 "IMAG",
	 140  	CHAR:	 "CHAR",
	 141  	STRING: "STRING",
	 142  
	 143  	ADD: "+",
	 144  	SUB: "-",
	 145  	MUL: "*",
	 146  	QUO: "/",
	 147  	REM: "%",
	 148  
	 149  	AND:		 "&",
	 150  	OR:			"|",
	 151  	XOR:		 "^",
	 152  	SHL:		 "<<",
	 153  	SHR:		 ">>",
	 154  	AND_NOT: "&^",
	 155  
	 156  	ADD_ASSIGN: "+=",
	 157  	SUB_ASSIGN: "-=",
	 158  	MUL_ASSIGN: "*=",
	 159  	QUO_ASSIGN: "/=",
	 160  	REM_ASSIGN: "%=",
	 161  
	 162  	AND_ASSIGN:		 "&=",
	 163  	OR_ASSIGN:			"|=",
	 164  	XOR_ASSIGN:		 "^=",
	 165  	SHL_ASSIGN:		 "<<=",
	 166  	SHR_ASSIGN:		 ">>=",
	 167  	AND_NOT_ASSIGN: "&^=",
	 168  
	 169  	LAND:	"&&",
	 170  	LOR:	 "||",
	 171  	ARROW: "<-",
	 172  	INC:	 "++",
	 173  	DEC:	 "--",
	 174  
	 175  	EQL:		"==",
	 176  	LSS:		"<",
	 177  	GTR:		">",
	 178  	ASSIGN: "=",
	 179  	NOT:		"!",
	 180  
	 181  	NEQ:			"!=",
	 182  	LEQ:			"<=",
	 183  	GEQ:			">=",
	 184  	DEFINE:	 ":=",
	 185  	ELLIPSIS: "...",
	 186  
	 187  	LPAREN: "(",
	 188  	LBRACK: "[",
	 189  	LBRACE: "{",
	 190  	COMMA:	",",
	 191  	PERIOD: ".",
	 192  
	 193  	RPAREN:		")",
	 194  	RBRACK:		"]",
	 195  	RBRACE:		"}",
	 196  	SEMICOLON: ";",
	 197  	COLON:		 ":",
	 198  
	 199  	BREAK:		"break",
	 200  	CASE:		 "case",
	 201  	CHAN:		 "chan",
	 202  	CONST:		"const",
	 203  	CONTINUE: "continue",
	 204  
	 205  	DEFAULT:		 "default",
	 206  	DEFER:			 "defer",
	 207  	ELSE:				"else",
	 208  	FALLTHROUGH: "fallthrough",
	 209  	FOR:				 "for",
	 210  
	 211  	FUNC:	 "func",
	 212  	GO:		 "go",
	 213  	GOTO:	 "goto",
	 214  	IF:		 "if",
	 215  	IMPORT: "import",
	 216  
	 217  	INTERFACE: "interface",
	 218  	MAP:			 "map",
	 219  	PACKAGE:	 "package",
	 220  	RANGE:		 "range",
	 221  	RETURN:		"return",
	 222  
	 223  	SELECT: "select",
	 224  	STRUCT: "struct",
	 225  	SWITCH: "switch",
	 226  	TYPE:	 "type",
	 227  	VAR:		"var",
	 228  }
	 229  
	 230  // String returns the string corresponding to the token tok.
	 231  // For operators, delimiters, and keywords the string is the actual
	 232  // token character sequence (e.g., for the token ADD, the string is
	 233  // "+"). For all other tokens the string corresponds to the token
	 234  // constant name (e.g. for the token IDENT, the string is "IDENT").
	 235  //
	 236  func (tok Token) String() string {
	 237  	s := ""
	 238  	if 0 <= tok && tok < Token(len(tokens)) {
	 239  		s = tokens[tok]
	 240  	}
	 241  	if s == "" {
	 242  		s = "token(" + strconv.Itoa(int(tok)) + ")"
	 243  	}
	 244  	return s
	 245  }
	 246  
	 247  // A set of constants for precedence-based expression parsing.
	 248  // Non-operators have lowest precedence, followed by operators
	 249  // starting with precedence 1 up to unary operators. The highest
	 250  // precedence serves as "catch-all" precedence for selector,
	 251  // indexing, and other operator and delimiter tokens.
	 252  //
	 253  const (
	 254  	LowestPrec	= 0 // non-operators
	 255  	UnaryPrec	 = 6
	 256  	HighestPrec = 7
	 257  )
	 258  
	 259  // Precedence returns the operator precedence of the binary
	 260  // operator op. If op is not a binary operator, the result
	 261  // is LowestPrecedence.
	 262  //
	 263  func (op Token) Precedence() int {
	 264  	switch op {
	 265  	case LOR:
	 266  		return 1
	 267  	case LAND:
	 268  		return 2
	 269  	case EQL, NEQ, LSS, LEQ, GTR, GEQ:
	 270  		return 3
	 271  	case ADD, SUB, OR, XOR:
	 272  		return 4
	 273  	case MUL, QUO, REM, SHL, SHR, AND, AND_NOT:
	 274  		return 5
	 275  	}
	 276  	return LowestPrec
	 277  }
	 278  
	 279  var keywords map[string]Token
	 280  
	 281  func init() {
	 282  	keywords = make(map[string]Token)
	 283  	for i := keyword_beg + 1; i < keyword_end; i++ {
	 284  		keywords[tokens[i]] = i
	 285  	}
	 286  }
	 287  
	 288  // Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
	 289  //
	 290  func Lookup(ident string) Token {
	 291  	if tok, is_keyword := keywords[ident]; is_keyword {
	 292  		return tok
	 293  	}
	 294  	return IDENT
	 295  }
	 296  
	 297  // Predicates
	 298  
	 299  // IsLiteral returns true for tokens corresponding to identifiers
	 300  // and basic type literals; it returns false otherwise.
	 301  //
	 302  func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_end }
	 303  
	 304  // IsOperator returns true for tokens corresponding to operators and
	 305  // delimiters; it returns false otherwise.
	 306  //
	 307  func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end }
	 308  
	 309  // IsKeyword returns true for tokens corresponding to keywords;
	 310  // it returns false otherwise.
	 311  //
	 312  func (tok Token) IsKeyword() bool { return keyword_beg < tok && tok < keyword_end }
	 313  
	 314  // IsExported reports whether name starts with an upper-case letter.
	 315  //
	 316  func IsExported(name string) bool {
	 317  	ch, _ := utf8.DecodeRuneInString(name)
	 318  	return unicode.IsUpper(ch)
	 319  }
	 320  
	 321  // IsKeyword reports whether name is a Go keyword, such as "func" or "return".
	 322  //
	 323  func IsKeyword(name string) bool {
	 324  	// TODO: opt: use a perfect hash function instead of a global map.
	 325  	_, ok := keywords[name]
	 326  	return ok
	 327  }
	 328  
	 329  // IsIdentifier reports whether name is a Go identifier, that is, a non-empty
	 330  // string made up of letters, digits, and underscores, where the first character
	 331  // is not a digit. Keywords are not identifiers.
	 332  //
	 333  func IsIdentifier(name string) bool {
	 334  	for i, c := range name {
	 335  		if !unicode.IsLetter(c) && c != '_' && (i == 0 || !unicode.IsDigit(c)) {
	 336  			return false
	 337  		}
	 338  	}
	 339  	return name != "" && !IsKeyword(name)
	 340  }
	 341
View as plain text