Source file src/pkg/go/token/token.go

     1	// Copyright 2009 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// Package token defines constants representing the lexical tokens of the Go
     6	// programming language and basic operations on tokens (printing, predicates).
     7	//
     8	package token
     9	
    10	import (
    11		"strconv"
    12		"unicode"
    13		"unicode/utf8"
    14	)
    15	
    16	// Token is the set of lexical tokens of the Go programming language.
    17	type Token int
    18	
    19	// The list of tokens.
    20	const (
    21		// Special tokens
    22		ILLEGAL Token = iota
    23		EOF
    24		COMMENT
    25	
    26		literal_beg
    27		// Identifiers and basic type literals
    28		// (these tokens stand for classes of literals)
    29		IDENT  // main
    30		INT    // 12345
    31		FLOAT  // 123.45
    32		IMAG   // 123.45i
    33		CHAR   // 'a'
    34		STRING // "abc"
    35		literal_end
    36	
    37		operator_beg
    38		// Operators and delimiters
    39		ADD // +
    40		SUB // -
    41		MUL // *
    42		QUO // /
    43		REM // %
    44	
    45		AND     // &
    46		OR      // |
    47		XOR     // ^
    48		SHL     // <<
    49		SHR     // >>
    50		AND_NOT // &^
    51	
    52		ADD_ASSIGN // +=
    53		SUB_ASSIGN // -=
    54		MUL_ASSIGN // *=
    55		QUO_ASSIGN // /=
    56		REM_ASSIGN // %=
    57	
    58		AND_ASSIGN     // &=
    59		OR_ASSIGN      // |=
    60		XOR_ASSIGN     // ^=
    61		SHL_ASSIGN     // <<=
    62		SHR_ASSIGN     // >>=
    63		AND_NOT_ASSIGN // &^=
    64	
    65		LAND  // &&
    66		LOR   // ||
    67		ARROW // <-
    68		INC   // ++
    69		DEC   // --
    70	
    71		EQL    // ==
    72		LSS    // <
    73		GTR    // >
    74		ASSIGN // =
    75		NOT    // !
    76	
    77		NEQ      // !=
    78		LEQ      // <=
    79		GEQ      // >=
    80		DEFINE   // :=
    81		ELLIPSIS // ...
    82	
    83		LPAREN // (
    84		LBRACK // [
    85		LBRACE // {
    86		COMMA  // ,
    87		PERIOD // .
    88	
    89		RPAREN    // )
    90		RBRACK    // ]
    91		RBRACE    // }
    92		SEMICOLON // ;
    93		COLON     // :
    94		operator_end
    95	
    96		keyword_beg
    97		// Keywords
    98		BREAK
    99		CASE
   100		CHAN
   101		CONST
   102		CONTINUE
   103	
   104		DEFAULT
   105		DEFER
   106		ELSE
   107		FALLTHROUGH
   108		FOR
   109	
   110		FUNC
   111		GO
   112		GOTO
   113		IF
   114		IMPORT
   115	
   116		INTERFACE
   117		MAP
   118		PACKAGE
   119		RANGE
   120		RETURN
   121	
   122		SELECT
   123		STRUCT
   124		SWITCH
   125		TYPE
   126		VAR
   127		keyword_end
   128	)
   129	
   130	var tokens = [...]string{
   131		ILLEGAL: "ILLEGAL",
   132	
   133		EOF:     "EOF",
   134		COMMENT: "COMMENT",
   135	
   136		IDENT:  "IDENT",
   137		INT:    "INT",
   138		FLOAT:  "FLOAT",
   139		IMAG:   "IMAG",
   140		CHAR:   "CHAR",
   141		STRING: "STRING",
   142	
   143		ADD: "+",
   144		SUB: "-",
   145		MUL: "*",
   146		QUO: "/",
   147		REM: "%",
   148	
   149		AND:     "&",
   150		OR:      "|",
   151		XOR:     "^",
   152		SHL:     "<<",
   153		SHR:     ">>",
   154		AND_NOT: "&^",
   155	
   156		ADD_ASSIGN: "+=",
   157		SUB_ASSIGN: "-=",
   158		MUL_ASSIGN: "*=",
   159		QUO_ASSIGN: "/=",
   160		REM_ASSIGN: "%=",
   161	
   162		AND_ASSIGN:     "&=",
   163		OR_ASSIGN:      "|=",
   164		XOR_ASSIGN:     "^=",
   165		SHL_ASSIGN:     "<<=",
   166		SHR_ASSIGN:     ">>=",
   167		AND_NOT_ASSIGN: "&^=",
   168	
   169		LAND:  "&&",
   170		LOR:   "||",
   171		ARROW: "<-",
   172		INC:   "++",
   173		DEC:   "--",
   174	
   175		EQL:    "==",
   176		LSS:    "<",
   177		GTR:    ">",
   178		ASSIGN: "=",
   179		NOT:    "!",
   180	
   181		NEQ:      "!=",
   182		LEQ:      "<=",
   183		GEQ:      ">=",
   184		DEFINE:   ":=",
   185		ELLIPSIS: "...",
   186	
   187		LPAREN: "(",
   188		LBRACK: "[",
   189		LBRACE: "{",
   190		COMMA:  ",",
   191		PERIOD: ".",
   192	
   193		RPAREN:    ")",
   194		RBRACK:    "]",
   195		RBRACE:    "}",
   196		SEMICOLON: ";",
   197		COLON:     ":",
   198	
   199		BREAK:    "break",
   200		CASE:     "case",
   201		CHAN:     "chan",
   202		CONST:    "const",
   203		CONTINUE: "continue",
   204	
   205		DEFAULT:     "default",
   206		DEFER:       "defer",
   207		ELSE:        "else",
   208		FALLTHROUGH: "fallthrough",
   209		FOR:         "for",
   210	
   211		FUNC:   "func",
   212		GO:     "go",
   213		GOTO:   "goto",
   214		IF:     "if",
   215		IMPORT: "import",
   216	
   217		INTERFACE: "interface",
   218		MAP:       "map",
   219		PACKAGE:   "package",
   220		RANGE:     "range",
   221		RETURN:    "return",
   222	
   223		SELECT: "select",
   224		STRUCT: "struct",
   225		SWITCH: "switch",
   226		TYPE:   "type",
   227		VAR:    "var",
   228	}
   229	
   230	// String returns the string corresponding to the token tok.
   231	// For operators, delimiters, and keywords the string is the actual
   232	// token character sequence (e.g., for the token ADD, the string is
   233	// "+"). For all other tokens the string corresponds to the token
   234	// constant name (e.g. for the token IDENT, the string is "IDENT").
   235	//
   236	func (tok Token) String() string {
   237		s := ""
   238		if 0 <= tok && tok < Token(len(tokens)) {
   239			s = tokens[tok]
   240		}
   241		if s == "" {
   242			s = "token(" + strconv.Itoa(int(tok)) + ")"
   243		}
   244		return s
   245	}
   246	
   247	// A set of constants for precedence-based expression parsing.
   248	// Non-operators have lowest precedence, followed by operators
   249	// starting with precedence 1 up to unary operators. The highest
   250	// precedence serves as "catch-all" precedence for selector,
   251	// indexing, and other operator and delimiter tokens.
   252	//
   253	const (
   254		LowestPrec  = 0 // non-operators
   255		UnaryPrec   = 6
   256		HighestPrec = 7
   257	)
   258	
   259	// Precedence returns the operator precedence of the binary
   260	// operator op. If op is not a binary operator, the result
   261	// is LowestPrecedence.
   262	//
   263	func (op Token) Precedence() int {
   264		switch op {
   265		case LOR:
   266			return 1
   267		case LAND:
   268			return 2
   269		case EQL, NEQ, LSS, LEQ, GTR, GEQ:
   270			return 3
   271		case ADD, SUB, OR, XOR:
   272			return 4
   273		case MUL, QUO, REM, SHL, SHR, AND, AND_NOT:
   274			return 5
   275		}
   276		return LowestPrec
   277	}
   278	
   279	var keywords map[string]Token
   280	
   281	func init() {
   282		keywords = make(map[string]Token)
   283		for i := keyword_beg + 1; i < keyword_end; i++ {
   284			keywords[tokens[i]] = i
   285		}
   286	}
   287	
   288	// Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
   289	//
   290	func Lookup(ident string) Token {
   291		if tok, is_keyword := keywords[ident]; is_keyword {
   292			return tok
   293		}
   294		return IDENT
   295	}
   296	
   297	// Predicates
   298	
   299	// IsLiteral returns true for tokens corresponding to identifiers
   300	// and basic type literals; it returns false otherwise.
   301	//
   302	func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_end }
   303	
   304	// IsOperator returns true for tokens corresponding to operators and
   305	// delimiters; it returns false otherwise.
   306	//
   307	func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end }
   308	
   309	// IsKeyword returns true for tokens corresponding to keywords;
   310	// it returns false otherwise.
   311	//
   312	func (tok Token) IsKeyword() bool { return keyword_beg < tok && tok < keyword_end }
   313	
   314	// IsExported reports whether name starts with an upper-case letter.
   315	//
   316	func IsExported(name string) bool {
   317		ch, _ := utf8.DecodeRuneInString(name)
   318		return unicode.IsUpper(ch)
   319	}
   320	
   321	// IsKeyword reports whether name is a Go keyword, such as "func" or "return".
   322	//
   323	func IsKeyword(name string) bool {
   324		// TODO: opt: use a perfect hash function instead of a global map.
   325		_, ok := keywords[name]
   326		return ok
   327	}
   328	
   329	// IsIdentifier reports whether name is a Go identifier, that is, a non-empty
   330	// string made up of letters, digits, and underscores, where the first character
   331	// is not a digit. Keywords are not identifiers.
   332	//
   333	func IsIdentifier(name string) bool {
   334		for i, c := range name {
   335			if !unicode.IsLetter(c) && c != '_' && (i == 0 || !unicode.IsDigit(c)) {
   336				return false
   337			}
   338		}
   339		return name != "" && !IsKeyword(name)
   340	}
   341	

View as plain text