cmd/evm: fix some issues with the evm run command (#28109)

* cmd/evm: improve flags handling

This fixes some issues with flags in cmd/evm. The supported flags did not
actually show up in help output because they weren't categorized. I'm also
adding the VM-related flags to the run command here so they can be given
after the subcommand name. So it can be run like this now:

   ./evm run --code 6001 --debug

* cmd/evm: enable all forks by default in run command

The default genesis was just empty with no forks at all, which is annoying because
contracts will be relying on opcodes introduced in a fork. So this changes the default to
have all forks enabled.

* core/asm: fix some issues in the assembler

This fixes minor bugs in the old assembler:

- It is now possible to have comments on the same line as an instruction.
- Errors for invalid numbers in the jump instruction are reported better
- Line numbers in errors were off by one

cmd/evm/blockrunner.go

@@ -25,7 +25,6 @@ import (
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/eth/tracers/logger"
-	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/tests"
 	"github.com/urfave/cli/v2"
 )
@@ -41,10 +40,7 @@ func blockTestCmd(ctx *cli.Context) error {
 	if len(ctx.Args().First()) == 0 {
 		return errors.New("path-to-test argument required")
 	}
-	// Configure the go-ethereum logger
+
-	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
-	glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name)))
-	log.Root().SetHandler(glogger)
 	var tracer vm.EVMLogger
 	// Configure the EVM logger
 	if ctx.Bool(MachineFlag.Name) {

cmd/evm/main.go

@@ -23,6 +23,7 @@ import (
 	"os"
 
 	"github.com/ethereum/go-ethereum/cmd/evm/internal/t8ntool"
+	"github.com/ethereum/go-ethereum/internal/debug"
 	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/urfave/cli/v2"
 )
@@ -31,99 +32,107 @@ var (
 	DebugFlag = &cli.BoolFlag{
 		Name:     "debug",
 		Usage:    "output full trace logs",
-	}
+		Category: flags.VMCategory,
-	MemProfileFlag = &cli.StringFlag{
-		Name:  "memprofile",
-		Usage: "creates a memory profile at the given path",
-	}
-	CPUProfileFlag = &cli.StringFlag{
-		Name:  "cpuprofile",
-		Usage: "creates a CPU profile at the given path",
 	}
 	StatDumpFlag = &cli.BoolFlag{
 		Name:     "statdump",
 		Usage:    "displays stack and heap memory information",
+		Category: flags.VMCategory,
 	}
 	CodeFlag = &cli.StringFlag{
 		Name:     "code",
 		Usage:    "EVM code",
+		Category: flags.VMCategory,
 	}
 	CodeFileFlag = &cli.StringFlag{
 		Name:     "codefile",
 		Usage:    "File containing EVM code. If '-' is specified, code is read from stdin ",
+		Category: flags.VMCategory,
 	}
 	GasFlag = &cli.Uint64Flag{
 		Name:     "gas",
 		Usage:    "gas limit for the evm",
 		Value:    10000000000,
+		Category: flags.VMCategory,
 	}
 	PriceFlag = &flags.BigFlag{
 		Name:     "price",
 		Usage:    "price set for the evm",
 		Value:    new(big.Int),
+		Category: flags.VMCategory,
 	}
 	ValueFlag = &flags.BigFlag{
 		Name:     "value",
 		Usage:    "value set for the evm",
 		Value:    new(big.Int),
+		Category: flags.VMCategory,
 	}
 	DumpFlag = &cli.BoolFlag{
 		Name:     "dump",
 		Usage:    "dumps the state after the run",
+		Category: flags.VMCategory,
 	}
 	InputFlag = &cli.StringFlag{
 		Name:     "input",
 		Usage:    "input for the EVM",
+		Category: flags.VMCategory,
 	}
 	InputFileFlag = &cli.StringFlag{
 		Name:     "inputfile",
 		Usage:    "file containing input for the EVM",
-	}
+		Category: flags.VMCategory,
-	VerbosityFlag = &cli.IntFlag{
-		Name:  "verbosity",
-		Usage: "sets the verbosity level",
 	}
 	BenchFlag = &cli.BoolFlag{
 		Name:     "bench",
 		Usage:    "benchmark the execution",
+		Category: flags.VMCategory,
 	}
 	CreateFlag = &cli.BoolFlag{
 		Name:     "create",
 		Usage:    "indicates the action should be create rather than call",
+		Category: flags.VMCategory,
 	}
 	GenesisFlag = &cli.StringFlag{
 		Name:     "prestate",
 		Usage:    "JSON file with prestate (genesis) config",
+		Category: flags.VMCategory,
 	}
 	MachineFlag = &cli.BoolFlag{
 		Name:     "json",
 		Usage:    "output trace logs in machine readable format (json)",
+		Category: flags.VMCategory,
 	}
 	SenderFlag = &cli.StringFlag{
 		Name:     "sender",
 		Usage:    "The transaction origin",
+		Category: flags.VMCategory,
 	}
 	ReceiverFlag = &cli.StringFlag{
 		Name:     "receiver",
 		Usage:    "The transaction receiver (execution context)",
+		Category: flags.VMCategory,
 	}
 	DisableMemoryFlag = &cli.BoolFlag{
 		Name:     "nomemory",
 		Value:    true,
 		Usage:    "disable memory output",
+		Category: flags.VMCategory,
 	}
 	DisableStackFlag = &cli.BoolFlag{
 		Name:     "nostack",
 		Usage:    "disable stack output",
+		Category: flags.VMCategory,
 	}
 	DisableStorageFlag = &cli.BoolFlag{
 		Name:     "nostorage",
 		Usage:    "disable storage output",
+		Category: flags.VMCategory,
 	}
 	DisableReturnDataFlag = &cli.BoolFlag{
 		Name:     "noreturndata",
 		Value:    true,
 		Usage:    "enable return data output",
+		Category: flags.VMCategory,
 	}
 )
 
@@ -183,34 +192,38 @@ var blockBuilderCommand = &cli.Command{
 	},
 }
 
-var app = flags.NewApp("the evm command line interface")
+// vmFlags contains flags related to running the EVM.
-
+var vmFlags = []cli.Flag{
-func init() {
-	app.Flags = []cli.Flag{
-		BenchFlag,
-		CreateFlag,
-		DebugFlag,
-		VerbosityFlag,
 	CodeFlag,
 	CodeFileFlag,
+	CreateFlag,
 	GasFlag,
 	PriceFlag,
 	ValueFlag,
-		DumpFlag,
 	InputFlag,
 	InputFileFlag,
-		MemProfileFlag,
-		CPUProfileFlag,
-		StatDumpFlag,
 	GenesisFlag,
-		MachineFlag,
 	SenderFlag,
 	ReceiverFlag,
+}
+
+// traceFlags contains flags that configure tracing output.
+var traceFlags = []cli.Flag{
+	BenchFlag,
+	DebugFlag,
+	DumpFlag,
+	MachineFlag,
+	StatDumpFlag,
 	DisableMemoryFlag,
 	DisableStackFlag,
 	DisableStorageFlag,
 	DisableReturnDataFlag,
-	}
+}
+
+var app = flags.NewApp("the evm command line interface")
+
+func init() {
+	app.Flags = flags.Merge(vmFlags, traceFlags, debug.Flags)
 	app.Commands = []*cli.Command{
 		compileCommand,
 		disasmCommand,
@@ -221,6 +234,14 @@ func init() {
 		transactionCommand,
 		blockBuilderCommand,
 	}
+	app.Before = func(ctx *cli.Context) error {
+		flags.MigrateGlobalFlags(ctx)
+		return debug.Setup(ctx)
+	}
+	app.After = func(ctx *cli.Context) error {
+		debug.Exit()
+		return nil
+	}
 }
 
 func main() {

cmd/evm/runner.go

@@ -24,7 +24,6 @@ import (
 	"math/big"
 	"os"
 	goruntime "runtime"
-	"runtime/pprof"
 	"testing"
 	"time"
 
@@ -34,12 +33,10 @@ import (
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/core/vm/runtime"
 	"github.com/ethereum/go-ethereum/eth/tracers/logger"
 	"github.com/ethereum/go-ethereum/internal/flags"
-	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
@@ -52,6 +49,7 @@ var runCommand = &cli.Command{
 	Usage:       "run arbitrary evm binary",
 	ArgsUsage:   "<code>",
 	Description: `The run command runs arbitrary EVM code.`,
+	Flags:       flags.Merge(vmFlags, traceFlags),
 }
 
 // readGenesis will read the given JSON format genesis file and return
@@ -109,9 +107,6 @@ func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) (output []by
 }
 
 func runCmd(ctx *cli.Context) error {
-	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
-	glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name)))
-	log.Root().SetHandler(glogger)
 	logconfig := &logger.Config{
 		EnableMemory:     !ctx.Bool(DisableMemoryFlag.Name),
 		DisableStack:     ctx.Bool(DisableStackFlag.Name),
@@ -127,7 +122,6 @@ func runCmd(ctx *cli.Context) error {
 		chainConfig *params.ChainConfig
 		sender      = common.BytesToAddress([]byte("sender"))
 		receiver    = common.BytesToAddress([]byte("receiver"))
-		genesisConfig *core.Genesis
 		preimages   = ctx.Bool(DumpFlag.Name)
 		blobHashes  []common.Hash // TODO (MariusVanDerWijden) implement blob hashes in state tests
 	)
@@ -139,30 +133,30 @@ func runCmd(ctx *cli.Context) error {
 	} else {
 		debugLogger = logger.NewStructLogger(logconfig)
 	}
+
+	initialGas := ctx.Uint64(GasFlag.Name)
+	genesisConfig := new(core.Genesis)
+	genesisConfig.GasLimit = initialGas
 	if ctx.String(GenesisFlag.Name) != "" {
-		gen := readGenesis(ctx.String(GenesisFlag.Name))
+		genesisConfig = readGenesis(ctx.String(GenesisFlag.Name))
-		genesisConfig = gen
+		if genesisConfig.GasLimit != 0 {
+			initialGas = genesisConfig.GasLimit
+		}
+	} else {
+		genesisConfig.Config = params.AllEthashProtocolChanges
+	}
+
 	db := rawdb.NewMemoryDatabase()
 	triedb := trie.NewDatabase(db, &trie.Config{
 		Preimages: preimages,
 		HashDB:    hashdb.Defaults,
 	})
 	defer triedb.Close()
-		genesis := gen.MustCommit(db, triedb)
+	genesis := genesisConfig.MustCommit(db, triedb)
 	sdb := state.NewDatabaseWithNodeDB(db, triedb)
 	statedb, _ = state.New(genesis.Root(), sdb, nil)
-		chainConfig = gen.Config
+	chainConfig = genesisConfig.Config
-	} else {
+
-		db := rawdb.NewMemoryDatabase()
-		triedb := trie.NewDatabase(db, &trie.Config{
-			Preimages: preimages,
-			HashDB:    hashdb.Defaults,
-		})
-		defer triedb.Close()
-		sdb := state.NewDatabaseWithNodeDB(db, triedb)
-		statedb, _ = state.New(types.EmptyRootHash, sdb, nil)
-		genesisConfig = new(core.Genesis)
-	}
 	if ctx.String(SenderFlag.Name) != "" {
 		sender = common.HexToAddress(ctx.String(SenderFlag.Name))
 	}
@@ -216,10 +210,6 @@ func runCmd(ctx *cli.Context) error {
 		}
 		code = common.Hex2Bytes(bin)
 	}
-	initialGas := ctx.Uint64(GasFlag.Name)
-	if genesisConfig.GasLimit != 0 {
-		initialGas = genesisConfig.GasLimit
-	}
 	runtimeConfig := runtime.Config{
 		Origin:      sender,
 		State:       statedb,
@@ -236,19 +226,6 @@ func runCmd(ctx *cli.Context) error {
 		},
 	}
 
-	if cpuProfilePath := ctx.String(CPUProfileFlag.Name); cpuProfilePath != "" {
-		f, err := os.Create(cpuProfilePath)
-		if err != nil {
-			fmt.Println("could not create CPU profile: ", err)
-			os.Exit(1)
-		}
-		if err := pprof.StartCPUProfile(f); err != nil {
-			fmt.Println("could not start CPU profile: ", err)
-			os.Exit(1)
-		}
-		defer pprof.StopCPUProfile()
-	}
-
 	if chainConfig != nil {
 		runtimeConfig.ChainConfig = chainConfig
 	} else {
@@ -296,19 +273,6 @@ func runCmd(ctx *cli.Context) error {
 		fmt.Println(string(statedb.Dump(nil)))
 	}
 
-	if memProfilePath := ctx.String(MemProfileFlag.Name); memProfilePath != "" {
-		f, err := os.Create(memProfilePath)
-		if err != nil {
-			fmt.Println("could not create memory profile: ", err)
-			os.Exit(1)
-		}
-		if err := pprof.WriteHeapProfile(f); err != nil {
-			fmt.Println("could not write memory profile: ", err)
-			os.Exit(1)
-		}
-		f.Close()
-	}
-
 	if ctx.Bool(DebugFlag.Name) {
 		if debugLogger != nil {
 			fmt.Fprintln(os.Stderr, "#### TRACE ####")

cmd/evm/staterunner.go

@@ -28,7 +28,6 @@ import (
 	"github.com/ethereum/go-ethereum/core/state/snapshot"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/eth/tracers/logger"
-	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/tests"
 	"github.com/urfave/cli/v2"
 )
@@ -52,11 +51,6 @@ type StatetestResult struct {
 }
 
 func stateTestCmd(ctx *cli.Context) error {
-	// Configure the go-ethereum logger
-	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
-	glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name)))
-	log.Root().SetHandler(glogger)
-
 	// Configure the EVM logger
 	config := &logger.Config{
 		EnableMemory:     !ctx.Bool(DisableMemoryFlag.Name),

core/asm/compiler.go

@@ -17,6 +17,8 @@
 package asm
 
 import (
+	"encoding/hex"
+	"errors"
 	"fmt"
 	"math/big"
 	"os"
@@ -30,7 +32,7 @@ import (
 // and holds the tokens for the program.
 type Compiler struct {
 	tokens []token
-	binary []interface{}
+	out    []byte
 
 	labels map[string]int
 
@@ -50,12 +52,10 @@ func NewCompiler(debug bool) *Compiler {
 // Feed feeds tokens in to ch and are interpreted by
 // the compiler.
 //
-// feed is the first pass in the compile stage as it
+// feed is the first pass in the compile stage as it collects the used labels in the
-// collects the used labels in the program and keeps a
+// program and keeps a program counter which is used to determine the locations of the
-// program counter which is used to determine the locations
+// jump dests. The labels can than be used in the second stage to push labels and
-// of the jump dests. The labels can than be used in the
+// determine the right position.
-// second stage to push labels and determine the right
-// position.
 func (c *Compiler) Feed(ch <-chan token) {
 	var prev token
 	for i := range ch {
@@ -79,7 +79,6 @@ func (c *Compiler) Feed(ch <-chan token) {
 				c.pc++
 			}
 		}
-
 		c.tokens = append(c.tokens, i)
 		prev = i
 	}
@@ -88,12 +87,11 @@ func (c *Compiler) Feed(ch <-chan token) {
 	}
 }
 
-// Compile compiles the current tokens and returns a
+// Compile compiles the current tokens and returns a binary string that can be interpreted
-// binary string that can be interpreted by the EVM
+// by the EVM and an error if it failed.
-// and an error if it failed.
 //
-// compile is the second stage in the compile phase
+// compile is the second stage in the compile phase which compiles the tokens to EVM
-// which compiles the tokens to EVM instructions.
+// instructions.
 func (c *Compiler) Compile() (string, []error) {
 	var errors []error
 	// continue looping over the tokens until
@@ -105,16 +103,8 @@ func (c *Compiler) Compile() (string, []error) {
 	}
 
 	// turn the binary to hex
-	var bin strings.Builder
+	h := hex.EncodeToString(c.out)
-	for _, v := range c.binary {
+	return h, errors
-		switch v := v.(type) {
-		case vm.OpCode:
-			bin.WriteString(fmt.Sprintf("%x", []byte{byte(v)}))
-		case []byte:
-			bin.WriteString(fmt.Sprintf("%x", v))
-		}
-	}
-	return bin.String(), errors
 }
 
 // next returns the next token and increments the
@@ -156,47 +146,73 @@ func (c *Compiler) compileLine() error {
 	return nil
 }
 
-// compileNumber compiles the number to bytes
+// parseNumber compiles the number to bytes
-func (c *Compiler) compileNumber(element token) {
+func parseNumber(tok token) ([]byte, error) {
-	num := math.MustParseBig256(element.text).Bytes()
+	if tok.typ != number {
-	if len(num) == 0 {
+		panic("parseNumber of non-number token")
-		num = []byte{0}
 	}
-	c.pushBin(num)
+	num, ok := math.ParseBig256(tok.text)
+	if !ok {
+		return nil, errors.New("invalid number")
+	}
+	bytes := num.Bytes()
+	if len(bytes) == 0 {
+		bytes = []byte{0}
+	}
+	return bytes, nil
 }
 
 // compileElement compiles the element (push & label or both)
 // to a binary representation and may error if incorrect statements
 // where fed.
 func (c *Compiler) compileElement(element token) error {
-	// check for a jump. jumps must be read and compiled
+	switch {
-	// from right to left.
+	case isJump(element.text):
-	if isJump(element.text) {
+		return c.compileJump(element.text)
+	case isPush(element.text):
+		return c.compilePush()
+	default:
+		c.outputOpcode(toBinary(element.text))
+		return nil
+	}
+}
+
+func (c *Compiler) compileJump(jumpType string) error {
 	rvalue := c.next()
 	switch rvalue.typ {
 	case number:
-			// TODO figure out how to return the error properly
+		numBytes, err := parseNumber(rvalue)
-			c.compileNumber(rvalue)
+		if err != nil {
+			return err
+		}
+		c.outputBytes(numBytes)
+
 	case stringValue:
 		// strings are quoted, remove them.
-			c.pushBin(rvalue.text[1 : len(rvalue.text)-2])
+		str := rvalue.text[1 : len(rvalue.text)-2]
+		c.outputBytes([]byte(str))
+
 	case label:
-			c.pushBin(vm.PUSH4)
+		c.outputOpcode(vm.PUSH4)
 		pos := big.NewInt(int64(c.labels[rvalue.text])).Bytes()
 		pos = append(make([]byte, 4-len(pos)), pos...)
-			c.pushBin(pos)
+		c.outputBytes(pos)
+
 	case lineEnd:
+		// push without argument is supported, it just takes the destination from the stack.
 		c.pos--
+
 	default:
 		return compileErr(rvalue, rvalue.text, "number, string or label")
 	}
 	// push the operation
-		c.pushBin(toBinary(element.text))
+	c.outputOpcode(toBinary(jumpType))
 	return nil
-	} else if isPush(element.text) {
+}
+
+func (c *Compiler) compilePush() error {
 	// handle pushes. pushes are read from left to right.
 	var value []byte
-
 	rvalue := c.next()
 	switch rvalue.typ {
 	case number:
@@ -212,31 +228,32 @@ func (c *Compiler) compileElement(element token) error {
 	default:
 		return compileErr(rvalue, rvalue.text, "number, string or label")
 	}
-
 	if len(value) > 32 {
-			return fmt.Errorf("%d type error: unsupported string or number with size > 32", rvalue.lineno)
+		return fmt.Errorf("%d: string or number size > 32 bytes", rvalue.lineno+1)
 	}
-
+	c.outputOpcode(vm.OpCode(int(vm.PUSH1) - 1 + len(value)))
-		c.pushBin(vm.OpCode(int(vm.PUSH1) - 1 + len(value)))
+	c.outputBytes(value)
-		c.pushBin(value)
-	} else {
-		c.pushBin(toBinary(element.text))
-	}
-
 	return nil
 }
 
 // compileLabel pushes a jumpdest to the binary slice.
 func (c *Compiler) compileLabel() {
-	c.pushBin(vm.JUMPDEST)
+	c.outputOpcode(vm.JUMPDEST)
 }
 
-// pushBin pushes the value v to the binary stack.
+func (c *Compiler) outputOpcode(op vm.OpCode) {
-func (c *Compiler) pushBin(v interface{}) {
 	if c.debug {
-		fmt.Printf("%d: %v\n", len(c.binary), v)
+		fmt.Printf("%d: %v\n", len(c.out), op)
 	}
-	c.binary = append(c.binary, v)
+	c.out = append(c.out, byte(op))
+}
+
+// output pushes the value v to the binary stack.
+func (c *Compiler) outputBytes(b []byte) {
+	if c.debug {
+		fmt.Printf("%d: %x\n", len(c.out), b)
+	}
+	c.out = append(c.out, b...)
 }
 
 // isPush returns whether the string op is either any of
@@ -263,13 +280,13 @@ type compileError struct {
 }
 
 func (err compileError) Error() string {
-	return fmt.Sprintf("%d syntax error: unexpected %v, expected %v", err.lineno, err.got, err.want)
+	return fmt.Sprintf("%d: syntax error: unexpected %v, expected %v", err.lineno, err.got, err.want)
 }
 
 func compileErr(c token, got, want string) error {
 	return compileError{
 		got:    got,
 		want:   want,
-		lineno: c.lineno,
+		lineno: c.lineno + 1,
 	}
 }

core/asm/compiler_test.go

@@ -54,6 +54,14 @@ func TestCompiler(t *testing.T) {
 `,
 			output: "6300000006565b",
 		},
+		{
+			input: `
+	JUMP @label
+label: ;; comment
+	ADD ;; comment
+`,
+			output: "6300000006565b01",
+		},
 	}
 	for _, test := range tests {
 		ch := Lex([]byte(test.input), false)

core/asm/lex_test.go

@@ -72,6 +72,16 @@ func TestLexer(t *testing.T) {
 			input:  "@label123",
 			tokens: []token{{typ: lineStart}, {typ: label, text: "label123"}, {typ: eof}},
 		},
+		// comment after label
+		{
+			input:  "@label123 ;; comment",
+			tokens: []token{{typ: lineStart}, {typ: label, text: "label123"}, {typ: eof}},
+		},
+		// comment after instruction
+		{
+			input:  "push 3 ;; comment\nadd",
+			tokens: []token{{typ: lineStart}, {typ: element, text: "push"}, {typ: number, text: "3"}, {typ: lineEnd, text: "\n"}, {typ: lineStart, lineno: 1}, {typ: element, lineno: 1, text: "add"}, {typ: eof, lineno: 1}},
+		},
 	}
 
 	for _, test := range tests {

core/asm/lexer.go

@@ -42,6 +42,8 @@ type token struct {
 // is able to parse and return.
 type tokenType int
 
+//go:generate go run golang.org/x/tools/cmd/stringer -type tokenType
+
 const (
 	eof              tokenType = iota // end of file
 	lineStart                         // emitted when a line starts
@@ -52,31 +54,13 @@ const (
 	labelDef                          // label definition is emitted when a new label is found
 	number                            // number is emitted when a number is found
 	stringValue                       // stringValue is emitted when a string has been found
-
-	Numbers            = "1234567890"                                           // characters representing any decimal number
-	HexadecimalNumbers = Numbers + "aAbBcCdDeEfF"                               // characters representing any hexadecimal
-	Alpha              = "abcdefghijklmnopqrstuwvxyzABCDEFGHIJKLMNOPQRSTUWVXYZ" // characters representing alphanumeric
 )
 
-// String implements stringer
+const (
-func (it tokenType) String() string {
+	decimalNumbers = "1234567890"                                           // characters representing any decimal number
-	if int(it) > len(stringtokenTypes) {
+	hexNumbers     = decimalNumbers + "aAbBcCdDeEfF"                        // characters representing any hexadecimal
-		return "invalid"
+	alpha          = "abcdefghijklmnopqrstuwvxyzABCDEFGHIJKLMNOPQRSTUWVXYZ" // characters representing alphanumeric
-	}
+)
-	return stringtokenTypes[it]
-}
-
-var stringtokenTypes = []string{
-	eof:              "EOF",
-	lineStart:        "new line",
-	lineEnd:          "end of line",
-	invalidStatement: "invalid statement",
-	element:          "element",
-	label:            "label",
-	labelDef:         "label definition",
-	number:           "number",
-	stringValue:      "string",
-}
 
 // lexer is the basic construct for parsing
 // source code and turning them in to tokens.
@@ -200,7 +184,6 @@ func lexLine(l *lexer) stateFn {
 			l.emit(lineEnd)
 			l.ignore()
 			l.lineno++
-
 			l.emit(lineStart)
 		case r == ';' && l.peek() == ';':
 			return lexComment
@@ -225,6 +208,7 @@ func lexLine(l *lexer) stateFn {
 // of the line and discards the text.
 func lexComment(l *lexer) stateFn {
 	l.acceptRunUntil('\n')
+	l.backup()
 	l.ignore()
 
 	return lexLine
@@ -234,7 +218,7 @@ func lexComment(l *lexer) stateFn {
 // the lex text state function to advance the parsing
 // process.
 func lexLabel(l *lexer) stateFn {
-	l.acceptRun(Alpha + "_" + Numbers)
+	l.acceptRun(alpha + "_" + decimalNumbers)
 
 	l.emit(label)
 
@@ -253,9 +237,9 @@ func lexInsideString(l *lexer) stateFn {
 }
 
 func lexNumber(l *lexer) stateFn {
-	acceptance := Numbers
+	acceptance := decimalNumbers
 	if l.accept("xX") {
-		acceptance = HexadecimalNumbers
+		acceptance = hexNumbers
 	}
 	l.acceptRun(acceptance)
 
@@ -265,7 +249,7 @@ func lexNumber(l *lexer) stateFn {
 }
 
 func lexElement(l *lexer) stateFn {
-	l.acceptRun(Alpha + "_" + Numbers)
+	l.acceptRun(alpha + "_" + decimalNumbers)
 
 	if l.peek() == ':' {
 		l.emit(labelDef)

core/asm/tokentype_string.go

@@ -0,0 +1,31 @@
+// Code generated by "stringer -type tokenType"; DO NOT EDIT.
+
+package asm
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[eof-0]
+	_ = x[lineStart-1]
+	_ = x[lineEnd-2]
+	_ = x[invalidStatement-3]
+	_ = x[element-4]
+	_ = x[label-5]
+	_ = x[labelDef-6]
+	_ = x[number-7]
+	_ = x[stringValue-8]
+}
+
+const _tokenType_name = "eoflineStartlineEndinvalidStatementelementlabellabelDefnumberstringValue"
+
+var _tokenType_index = [...]uint8{0, 3, 12, 19, 35, 42, 47, 55, 61, 72}
+
+func (i tokenType) String() string {
+	if i < 0 || i >= tokenType(len(_tokenType_index)-1) {
+		return "tokenType(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _tokenType_name[_tokenType_index[i]:_tokenType_index[i+1]]
+}