example_test.go

Documentation: text/scanner

		 1  // Copyright 2018 The Go Authors. All rights reserved.
		 2  // Use of this source code is governed by a BSD-style
		 3  // license that can be found in the LICENSE file.
		 4  
		 5  package scanner_test
		 6  
		 7  import (
		 8  	"fmt"
		 9  	"strings"
		10  	"text/scanner"
		11  	"unicode"
		12  )
		13  
		14  func Example() {
		15  	const src = `
		16  // This is scanned code.
		17  if a > 10 {
		18  	someParsable = text
		19  }`
		20  
		21  	var s scanner.Scanner
		22  	s.Init(strings.NewReader(src))
		23  	s.Filename = "example"
		24  	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
		25  		fmt.Printf("%s: %s\n", s.Position, s.TokenText())
		26  	}
		27  
		28  	// Output:
		29  	// example:3:1: if
		30  	// example:3:4: a
		31  	// example:3:6: >
		32  	// example:3:8: 10
		33  	// example:3:11: {
		34  	// example:4:2: someParsable
		35  	// example:4:15: =
		36  	// example:4:17: text
		37  	// example:5:1: }
		38  }
		39  
		40  func Example_isIdentRune() {
		41  	const src = "%var1 var2%"
		42  
		43  	var s scanner.Scanner
		44  	s.Init(strings.NewReader(src))
		45  	s.Filename = "default"
		46  
		47  	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
		48  		fmt.Printf("%s: %s\n", s.Position, s.TokenText())
		49  	}
		50  
		51  	fmt.Println()
		52  	s.Init(strings.NewReader(src))
		53  	s.Filename = "percent"
		54  
		55  	// treat leading '%' as part of an identifier
		56  	s.IsIdentRune = func(ch rune, i int) bool {
		57  		return ch == '%' && i == 0 || unicode.IsLetter(ch) || unicode.IsDigit(ch) && i > 0
		58  	}
		59  
		60  	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
		61  		fmt.Printf("%s: %s\n", s.Position, s.TokenText())
		62  	}
		63  
		64  	// Output:
		65  	// default:1:1: %
		66  	// default:1:2: var1
		67  	// default:1:7: var2
		68  	// default:1:11: %
		69  	//
		70  	// percent:1:1: %var1
		71  	// percent:1:7: var2
		72  	// percent:1:11: %
		73  }
		74  
		75  func Example_mode() {
		76  	const src = `
		77  		// Comment begins at column 5.
		78  
		79  This line should not be included in the output.
		80  
		81  /*
		82  This multiline comment
		83  should be extracted in
		84  its entirety.
		85  */
		86  `
		87  
		88  	var s scanner.Scanner
		89  	s.Init(strings.NewReader(src))
		90  	s.Filename = "comments"
		91  	s.Mode ^= scanner.SkipComments // don't skip comments
		92  
		93  	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
		94  		txt := s.TokenText()
		95  		if strings.HasPrefix(txt, "//") || strings.HasPrefix(txt, "/*") {
		96  			fmt.Printf("%s: %s\n", s.Position, txt)
		97  		}
		98  	}
		99  
	 100  	// Output:
	 101  	// comments:2:5: // Comment begins at column 5.
	 102  	// comments:6:1: /*
	 103  	// This multiline comment
	 104  	// should be extracted in
	 105  	// its entirety.
	 106  	// */
	 107  }
	 108  
	 109  func Example_whitespace() {
	 110  	// tab-separated values
	 111  	const src = `aa	ab	ac	ad
	 112  ba	bb	bc	bd
	 113  ca	cb	cc	cd
	 114  da	db	dc	dd`
	 115  
	 116  	var (
	 117  		col, row int
	 118  		s				scanner.Scanner
	 119  		tsv			[4][4]string // large enough for example above
	 120  	)
	 121  	s.Init(strings.NewReader(src))
	 122  	s.Whitespace ^= 1<<'\t' | 1<<'\n' // don't skip tabs and new lines
	 123  
	 124  	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
	 125  		switch tok {
	 126  		case '\n':
	 127  			row++
	 128  			col = 0
	 129  		case '\t':
	 130  			col++
	 131  		default:
	 132  			tsv[row][col] = s.TokenText()
	 133  		}
	 134  	}
	 135  
	 136  	fmt.Print(tsv)
	 137  
	 138  	// Output:
	 139  	// [[aa ab ac ad] [ba bb bc bd] [ca cb cc cd] [da db dc dd]]
	 140  }
	 141
View as plain text