go lex 源码

golang lex 代码

文件路径：/src/cmd/asm/internal/lex/lex.go

// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package lex implements lexical analysis for the assembler.
package lex

import (
	"fmt"
	"log"
	"os"
	"strings"
	"text/scanner"

	"cmd/internal/src"
)

// A ScanToken represents an input item. It is a simple wrapping of rune, as
// returned by text/scanner.Scanner, plus a couple of extra values.
type ScanToken rune

const (
	// Asm defines some two-character lexemes. We make up
	// a rune/ScanToken value for them - ugly but simple.
	LSH          ScanToken = -1000 - iota // << Left shift.
	RSH                                   // >> Logical right shift.
	ARR                                   // -> Used on ARM for shift type 3, arithmetic right shift.
	ROT                                   // @> Used on ARM for shift type 4, rotate right.
	Include                               // included file started here
	BuildComment                          // //go:build or +build comment
	macroName                             // name of macro that should not be expanded
)

// IsRegisterShift reports whether the token is one of the ARM register shift operators.
func IsRegisterShift(r ScanToken) bool {
	return ROT <= r && r <= LSH // Order looks backwards because these are negative.
}

func (t ScanToken) String() string {
	switch t {
	case scanner.EOF:
		return "EOF"
	case scanner.Ident:
		return "identifier"
	case scanner.Int:
		return "integer constant"
	case scanner.Float:
		return "float constant"
	case scanner.Char:
		return "rune constant"
	case scanner.String:
		return "string constant"
	case scanner.RawString:
		return "raw string constant"
	case scanner.Comment:
		return "comment"
	default:
		return fmt.Sprintf("%q", rune(t))
	}
}

// NewLexer returns a lexer for the named file and the given link context.
func NewLexer(name string) TokenReader {
	input := NewInput(name)
	fd, err := os.Open(name)
	if err != nil {
		log.Fatalf("%s\n", err)
	}
	input.Push(NewTokenizer(name, fd, fd))
	return input
}

// The other files in this directory each contain an implementation of TokenReader.

// A TokenReader is like a reader, but returns lex tokens of type Token. It also can tell you what
// the text of the most recently returned token is, and where it was found.
// The underlying scanner elides all spaces except newline, so the input looks like a stream of
// Tokens; original spacing is lost but we don't need it.
type TokenReader interface {
	// Next returns the next token.
	Next() ScanToken
	// The following methods all refer to the most recent token returned by Next.
	// Text returns the original string representation of the token.
	Text() string
	// File reports the source file name of the token.
	File() string
	// Base reports the position base of the token.
	Base() *src.PosBase
	// SetBase sets the position base.
	SetBase(*src.PosBase)
	// Line reports the source line number of the token.
	Line() int
	// Col reports the source column number of the token.
	Col() int
	// Close does any teardown required.
	Close()
}

// A Token is a scan token plus its string value.
// A macro is stored as a sequence of Tokens with spaces stripped.
type Token struct {
	ScanToken
	text string
}

// Make returns a Token with the given rune (ScanToken) and text representation.
func Make(token ScanToken, text string) Token {
	// If the symbol starts with center dot, as in ·x, rewrite it as ""·x
	if token == scanner.Ident && strings.HasPrefix(text, "\u00B7") {
		text = `""` + text
	}
	// Substitute the substitutes for . and /.
	text = strings.Replace(text, "\u00B7", ".", -1)
	text = strings.Replace(text, "\u2215", "/", -1)
	return Token{ScanToken: token, text: text}
}

func (l Token) String() string {
	return l.text
}

// A Macro represents the definition of a #defined macro.
type Macro struct {
	name   string   // The #define name.
	args   []string // Formal arguments.
	tokens []Token  // Body of macro.
}

// Tokenize turns a string into a list of Tokens; used to parse the -D flag and in tests.
func Tokenize(str string) []Token {
	t := NewTokenizer("command line", strings.NewReader(str), nil)
	var tokens []Token
	for {
		tok := t.Next()
		if tok == scanner.EOF {
			break
		}
		tokens = append(tokens, Make(tok, t.Text()))
	}
	return tokens
}

相关信息

go 源码目录

相关文章

go input 源码

go lex_test 源码

go slice 源码

go stack 源码

go tokenizer 源码