1// Copyright 2018 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package scanner_test
6
7import (
8	"fmt"
9	"strings"
10	"text/scanner"
11	"unicode"
12)
13
14func Example() {
15	const src = `
16// This is scanned code.
17if a > 10 {
18	someParsable = text
19}`
20
21	var s scanner.Scanner
22	s.Init(strings.NewReader(src))
23	s.Filename = "example"
24	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
25		fmt.Printf("%s: %s\n", s.Position, s.TokenText())
26	}
27
28	// Output:
29	// example:3:1: if
30	// example:3:4: a
31	// example:3:6: >
32	// example:3:8: 10
33	// example:3:11: {
34	// example:4:2: someParsable
35	// example:4:15: =
36	// example:4:17: text
37	// example:5:1: }
38}
39
40func Example_isIdentRune() {
41	const src = "%var1 var2%"
42
43	var s scanner.Scanner
44	s.Init(strings.NewReader(src))
45	s.Filename = "default"
46
47	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
48		fmt.Printf("%s: %s\n", s.Position, s.TokenText())
49	}
50
51	fmt.Println()
52	s.Init(strings.NewReader(src))
53	s.Filename = "percent"
54
55	// treat leading '%' as part of an identifier
56	s.IsIdentRune = func(ch rune, i int) bool {
57		return ch == '%' && i == 0 || unicode.IsLetter(ch) || unicode.IsDigit(ch) && i > 0
58	}
59
60	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
61		fmt.Printf("%s: %s\n", s.Position, s.TokenText())
62	}
63
64	// Output:
65	// default:1:1: %
66	// default:1:2: var1
67	// default:1:7: var2
68	// default:1:11: %
69	//
70	// percent:1:1: %var1
71	// percent:1:7: var2
72	// percent:1:11: %
73}
74
75func Example_mode() {
76	const src = `
77    // Comment begins at column 5.
78
79This line should not be included in the output.
80
81/*
82This multiline comment
83should be extracted in
84its entirety.
85*/
86`
87
88	var s scanner.Scanner
89	s.Init(strings.NewReader(src))
90	s.Filename = "comments"
91	s.Mode ^= scanner.SkipComments // don't skip comments
92
93	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
94		txt := s.TokenText()
95		if strings.HasPrefix(txt, "//") || strings.HasPrefix(txt, "/*") {
96			fmt.Printf("%s: %s\n", s.Position, txt)
97		}
98	}
99
100	// Output:
101	// comments:2:5: // Comment begins at column 5.
102	// comments:6:1: /*
103	// This multiline comment
104	// should be extracted in
105	// its entirety.
106	// */
107}
108
109func Example_whitespace() {
110	// tab-separated values
111	const src = `aa	ab	ac	ad
112ba	bb	bc	bd
113ca	cb	cc	cd
114da	db	dc	dd`
115
116	var (
117		col, row int
118		s        scanner.Scanner
119		tsv      [4][4]string // large enough for example above
120	)
121	s.Init(strings.NewReader(src))
122	s.Whitespace ^= 1<<'\t' | 1<<'\n' // don't skip tabs and new lines
123
124	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
125		switch tok {
126		case '\n':
127			row++
128			col = 0
129		case '\t':
130			col++
131		default:
132			tsv[row][col] = s.TokenText()
133		}
134	}
135
136	fmt.Print(tsv)
137
138	// Output:
139	// [[aa ab ac ad] [ba bb bc bd] [ca cb cc cd] [da db dc dd]]
140}
141