Source file src/text/template/parse/lex_test.go

Documentation: text/template/parse

		 1  // Copyright 2011 The Go Authors. All rights reserved.
		 2  // Use of this source code is governed by a BSD-style
		 3  // license that can be found in the LICENSE file.
		 4  
		 5  package parse
		 6  
		 7  import (
		 8  	"fmt"
		 9  	"testing"
		10  )
		11  
		12  // Make the types prettyprint.
		13  var itemName = map[itemType]string{
		14  	itemError:				"error",
		15  	itemBool:				 "bool",
		16  	itemChar:				 "char",
		17  	itemCharConstant: "charconst",
		18  	itemComment:			"comment",
		19  	itemComplex:			"complex",
		20  	itemDeclare:			":=",
		21  	itemEOF:					"EOF",
		22  	itemField:				"field",
		23  	itemIdentifier:	 "identifier",
		24  	itemLeftDelim:		"left delim",
		25  	itemLeftParen:		"(",
		26  	itemNumber:			 "number",
		27  	itemPipe:				 "pipe",
		28  	itemRawString:		"raw string",
		29  	itemRightDelim:	 "right delim",
		30  	itemRightParen:	 ")",
		31  	itemSpace:				"space",
		32  	itemString:			 "string",
		33  	itemVariable:		 "variable",
		34  
		35  	// keywords
		36  	itemDot:			".",
		37  	itemBlock:		"block",
		38  	itemDefine:	 "define",
		39  	itemElse:		 "else",
		40  	itemIf:			 "if",
		41  	itemEnd:			"end",
		42  	itemNil:			"nil",
		43  	itemRange:		"range",
		44  	itemTemplate: "template",
		45  	itemWith:		 "with",
		46  }
		47  
		48  func (i itemType) String() string {
		49  	s := itemName[i]
		50  	if s == "" {
		51  		return fmt.Sprintf("item%d", int(i))
		52  	}
		53  	return s
		54  }
		55  
		56  type lexTest struct {
		57  	name	string
		58  	input string
		59  	items []item
		60  }
		61  
		62  func mkItem(typ itemType, text string) item {
		63  	return item{
		64  		typ: typ,
		65  		val: text,
		66  	}
		67  }
		68  
		69  var (
		70  	tDot				= mkItem(itemDot, ".")
		71  	tBlock			= mkItem(itemBlock, "block")
		72  	tEOF				= mkItem(itemEOF, "")
		73  	tFor				= mkItem(itemIdentifier, "for")
		74  	tLeft			 = mkItem(itemLeftDelim, "{{")
		75  	tLpar			 = mkItem(itemLeftParen, "(")
		76  	tPipe			 = mkItem(itemPipe, "|")
		77  	tQuote			= mkItem(itemString, `"abc \n\t\" "`)
		78  	tRange			= mkItem(itemRange, "range")
		79  	tRight			= mkItem(itemRightDelim, "}}")
		80  	tRpar			 = mkItem(itemRightParen, ")")
		81  	tSpace			= mkItem(itemSpace, " ")
		82  	raw				 = "`" + `abc\n\t\" ` + "`"
		83  	rawNL			 = "`now is{{\n}}the time`" // Contains newline inside raw quote.
		84  	tRawQuote	 = mkItem(itemRawString, raw)
		85  	tRawQuoteNL = mkItem(itemRawString, rawNL)
		86  )
		87  
		88  var lexTests = []lexTest{
		89  	{"empty", "", []item{tEOF}},
		90  	{"spaces", " \t\n", []item{mkItem(itemText, " \t\n"), tEOF}},
		91  	{"text", `now is the time`, []item{mkItem(itemText, "now is the time"), tEOF}},
		92  	{"text with comment", "hello-{{/* this is a comment */}}-world", []item{
		93  		mkItem(itemText, "hello-"),
		94  		mkItem(itemComment, "/* this is a comment */"),
		95  		mkItem(itemText, "-world"),
		96  		tEOF,
		97  	}},
		98  	{"punctuation", "{{,@% }}", []item{
		99  		tLeft,
	 100  		mkItem(itemChar, ","),
	 101  		mkItem(itemChar, "@"),
	 102  		mkItem(itemChar, "%"),
	 103  		tSpace,
	 104  		tRight,
	 105  		tEOF,
	 106  	}},
	 107  	{"parens", "{{((3))}}", []item{
	 108  		tLeft,
	 109  		tLpar,
	 110  		tLpar,
	 111  		mkItem(itemNumber, "3"),
	 112  		tRpar,
	 113  		tRpar,
	 114  		tRight,
	 115  		tEOF,
	 116  	}},
	 117  	{"empty action", `{{}}`, []item{tLeft, tRight, tEOF}},
	 118  	{"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}},
	 119  	{"block", `{{block "foo" .}}`, []item{
	 120  		tLeft, tBlock, tSpace, mkItem(itemString, `"foo"`), tSpace, tDot, tRight, tEOF,
	 121  	}},
	 122  	{"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}},
	 123  	{"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}},
	 124  	{"raw quote with newline", "{{" + rawNL + "}}", []item{tLeft, tRawQuoteNL, tRight, tEOF}},
	 125  	{"numbers", "{{1 02 0x14 0X14 -7.2i 1e3 1E3 +1.2e-4 4.2i 1+2i 1_2 0x1.e_fp4 0X1.E_FP4}}", []item{
	 126  		tLeft,
	 127  		mkItem(itemNumber, "1"),
	 128  		tSpace,
	 129  		mkItem(itemNumber, "02"),
	 130  		tSpace,
	 131  		mkItem(itemNumber, "0x14"),
	 132  		tSpace,
	 133  		mkItem(itemNumber, "0X14"),
	 134  		tSpace,
	 135  		mkItem(itemNumber, "-7.2i"),
	 136  		tSpace,
	 137  		mkItem(itemNumber, "1e3"),
	 138  		tSpace,
	 139  		mkItem(itemNumber, "1E3"),
	 140  		tSpace,
	 141  		mkItem(itemNumber, "+1.2e-4"),
	 142  		tSpace,
	 143  		mkItem(itemNumber, "4.2i"),
	 144  		tSpace,
	 145  		mkItem(itemComplex, "1+2i"),
	 146  		tSpace,
	 147  		mkItem(itemNumber, "1_2"),
	 148  		tSpace,
	 149  		mkItem(itemNumber, "0x1.e_fp4"),
	 150  		tSpace,
	 151  		mkItem(itemNumber, "0X1.E_FP4"),
	 152  		tRight,
	 153  		tEOF,
	 154  	}},
	 155  	{"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' '本'}}`, []item{
	 156  		tLeft,
	 157  		mkItem(itemCharConstant, `'a'`),
	 158  		tSpace,
	 159  		mkItem(itemCharConstant, `'\n'`),
	 160  		tSpace,
	 161  		mkItem(itemCharConstant, `'\''`),
	 162  		tSpace,
	 163  		mkItem(itemCharConstant, `'\\'`),
	 164  		tSpace,
	 165  		mkItem(itemCharConstant, `'\u00FF'`),
	 166  		tSpace,
	 167  		mkItem(itemCharConstant, `'\xFF'`),
	 168  		tSpace,
	 169  		mkItem(itemCharConstant, `'本'`),
	 170  		tRight,
	 171  		tEOF,
	 172  	}},
	 173  	{"bools", "{{true false}}", []item{
	 174  		tLeft,
	 175  		mkItem(itemBool, "true"),
	 176  		tSpace,
	 177  		mkItem(itemBool, "false"),
	 178  		tRight,
	 179  		tEOF,
	 180  	}},
	 181  	{"dot", "{{.}}", []item{
	 182  		tLeft,
	 183  		tDot,
	 184  		tRight,
	 185  		tEOF,
	 186  	}},
	 187  	{"nil", "{{nil}}", []item{
	 188  		tLeft,
	 189  		mkItem(itemNil, "nil"),
	 190  		tRight,
	 191  		tEOF,
	 192  	}},
	 193  	{"dots", "{{.x . .2 .x.y.z}}", []item{
	 194  		tLeft,
	 195  		mkItem(itemField, ".x"),
	 196  		tSpace,
	 197  		tDot,
	 198  		tSpace,
	 199  		mkItem(itemNumber, ".2"),
	 200  		tSpace,
	 201  		mkItem(itemField, ".x"),
	 202  		mkItem(itemField, ".y"),
	 203  		mkItem(itemField, ".z"),
	 204  		tRight,
	 205  		tEOF,
	 206  	}},
	 207  	{"keywords", "{{range if else end with}}", []item{
	 208  		tLeft,
	 209  		mkItem(itemRange, "range"),
	 210  		tSpace,
	 211  		mkItem(itemIf, "if"),
	 212  		tSpace,
	 213  		mkItem(itemElse, "else"),
	 214  		tSpace,
	 215  		mkItem(itemEnd, "end"),
	 216  		tSpace,
	 217  		mkItem(itemWith, "with"),
	 218  		tRight,
	 219  		tEOF,
	 220  	}},
	 221  	{"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{
	 222  		tLeft,
	 223  		mkItem(itemVariable, "$c"),
	 224  		tSpace,
	 225  		mkItem(itemDeclare, ":="),
	 226  		tSpace,
	 227  		mkItem(itemIdentifier, "printf"),
	 228  		tSpace,
	 229  		mkItem(itemVariable, "$"),
	 230  		tSpace,
	 231  		mkItem(itemVariable, "$hello"),
	 232  		tSpace,
	 233  		mkItem(itemVariable, "$23"),
	 234  		tSpace,
	 235  		mkItem(itemVariable, "$"),
	 236  		tSpace,
	 237  		mkItem(itemVariable, "$var"),
	 238  		mkItem(itemField, ".Field"),
	 239  		tSpace,
	 240  		mkItem(itemField, ".Method"),
	 241  		tRight,
	 242  		tEOF,
	 243  	}},
	 244  	{"variable invocation", "{{$x 23}}", []item{
	 245  		tLeft,
	 246  		mkItem(itemVariable, "$x"),
	 247  		tSpace,
	 248  		mkItem(itemNumber, "23"),
	 249  		tRight,
	 250  		tEOF,
	 251  	}},
	 252  	{"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{
	 253  		mkItem(itemText, "intro "),
	 254  		tLeft,
	 255  		mkItem(itemIdentifier, "echo"),
	 256  		tSpace,
	 257  		mkItem(itemIdentifier, "hi"),
	 258  		tSpace,
	 259  		mkItem(itemNumber, "1.2"),
	 260  		tSpace,
	 261  		tPipe,
	 262  		mkItem(itemIdentifier, "noargs"),
	 263  		tPipe,
	 264  		mkItem(itemIdentifier, "args"),
	 265  		tSpace,
	 266  		mkItem(itemNumber, "1"),
	 267  		tSpace,
	 268  		mkItem(itemString, `"hi"`),
	 269  		tRight,
	 270  		mkItem(itemText, " outro"),
	 271  		tEOF,
	 272  	}},
	 273  	{"declaration", "{{$v := 3}}", []item{
	 274  		tLeft,
	 275  		mkItem(itemVariable, "$v"),
	 276  		tSpace,
	 277  		mkItem(itemDeclare, ":="),
	 278  		tSpace,
	 279  		mkItem(itemNumber, "3"),
	 280  		tRight,
	 281  		tEOF,
	 282  	}},
	 283  	{"2 declarations", "{{$v , $w := 3}}", []item{
	 284  		tLeft,
	 285  		mkItem(itemVariable, "$v"),
	 286  		tSpace,
	 287  		mkItem(itemChar, ","),
	 288  		tSpace,
	 289  		mkItem(itemVariable, "$w"),
	 290  		tSpace,
	 291  		mkItem(itemDeclare, ":="),
	 292  		tSpace,
	 293  		mkItem(itemNumber, "3"),
	 294  		tRight,
	 295  		tEOF,
	 296  	}},
	 297  	{"field of parenthesized expression", "{{(.X).Y}}", []item{
	 298  		tLeft,
	 299  		tLpar,
	 300  		mkItem(itemField, ".X"),
	 301  		tRpar,
	 302  		mkItem(itemField, ".Y"),
	 303  		tRight,
	 304  		tEOF,
	 305  	}},
	 306  	{"trimming spaces before and after", "hello- {{- 3 -}} -world", []item{
	 307  		mkItem(itemText, "hello-"),
	 308  		tLeft,
	 309  		mkItem(itemNumber, "3"),
	 310  		tRight,
	 311  		mkItem(itemText, "-world"),
	 312  		tEOF,
	 313  	}},
	 314  	{"trimming spaces before and after comment", "hello- {{- /* hello */ -}} -world", []item{
	 315  		mkItem(itemText, "hello-"),
	 316  		mkItem(itemComment, "/* hello */"),
	 317  		mkItem(itemText, "-world"),
	 318  		tEOF,
	 319  	}},
	 320  	// errors
	 321  	{"badchar", "#{{\x01}}", []item{
	 322  		mkItem(itemText, "#"),
	 323  		tLeft,
	 324  		mkItem(itemError, "unrecognized character in action: U+0001"),
	 325  	}},
	 326  	{"unclosed action", "{{", []item{
	 327  		tLeft,
	 328  		mkItem(itemError, "unclosed action"),
	 329  	}},
	 330  	{"EOF in action", "{{range", []item{
	 331  		tLeft,
	 332  		tRange,
	 333  		mkItem(itemError, "unclosed action"),
	 334  	}},
	 335  	{"unclosed quote", "{{\"\n\"}}", []item{
	 336  		tLeft,
	 337  		mkItem(itemError, "unterminated quoted string"),
	 338  	}},
	 339  	{"unclosed raw quote", "{{`xx}}", []item{
	 340  		tLeft,
	 341  		mkItem(itemError, "unterminated raw quoted string"),
	 342  	}},
	 343  	{"unclosed char constant", "{{'\n}}", []item{
	 344  		tLeft,
	 345  		mkItem(itemError, "unterminated character constant"),
	 346  	}},
	 347  	{"bad number", "{{3k}}", []item{
	 348  		tLeft,
	 349  		mkItem(itemError, `bad number syntax: "3k"`),
	 350  	}},
	 351  	{"unclosed paren", "{{(3}}", []item{
	 352  		tLeft,
	 353  		tLpar,
	 354  		mkItem(itemNumber, "3"),
	 355  		mkItem(itemError, `unclosed left paren`),
	 356  	}},
	 357  	{"extra right paren", "{{3)}}", []item{
	 358  		tLeft,
	 359  		mkItem(itemNumber, "3"),
	 360  		tRpar,
	 361  		mkItem(itemError, `unexpected right paren U+0029 ')'`),
	 362  	}},
	 363  
	 364  	// Fixed bugs
	 365  	// Many elements in an action blew the lookahead until
	 366  	// we made lexInsideAction not loop.
	 367  	{"long pipeline deadlock", "{{|||||}}", []item{
	 368  		tLeft,
	 369  		tPipe,
	 370  		tPipe,
	 371  		tPipe,
	 372  		tPipe,
	 373  		tPipe,
	 374  		tRight,
	 375  		tEOF,
	 376  	}},
	 377  	{"text with bad comment", "hello-{{/*/}}-world", []item{
	 378  		mkItem(itemText, "hello-"),
	 379  		mkItem(itemError, `unclosed comment`),
	 380  	}},
	 381  	{"text with comment close separated from delim", "hello-{{/* */ }}-world", []item{
	 382  		mkItem(itemText, "hello-"),
	 383  		mkItem(itemError, `comment ends before closing delimiter`),
	 384  	}},
	 385  	// This one is an error that we can't catch because it breaks templates with
	 386  	// minimized JavaScript. Should have fixed it before Go 1.1.
	 387  	{"unmatched right delimiter", "hello-{.}}-world", []item{
	 388  		mkItem(itemText, "hello-{.}}-world"),
	 389  		tEOF,
	 390  	}},
	 391  }
	 392  
	 393  // collect gathers the emitted items into a slice.
	 394  func collect(t *lexTest, left, right string) (items []item) {
	 395  	l := lex(t.name, t.input, left, right, true)
	 396  	for {
	 397  		item := l.nextItem()
	 398  		items = append(items, item)
	 399  		if item.typ == itemEOF || item.typ == itemError {
	 400  			break
	 401  		}
	 402  	}
	 403  	return
	 404  }
	 405  
	 406  func equal(i1, i2 []item, checkPos bool) bool {
	 407  	if len(i1) != len(i2) {
	 408  		return false
	 409  	}
	 410  	for k := range i1 {
	 411  		if i1[k].typ != i2[k].typ {
	 412  			return false
	 413  		}
	 414  		if i1[k].val != i2[k].val {
	 415  			return false
	 416  		}
	 417  		if checkPos && i1[k].pos != i2[k].pos {
	 418  			return false
	 419  		}
	 420  		if checkPos && i1[k].line != i2[k].line {
	 421  			return false
	 422  		}
	 423  	}
	 424  	return true
	 425  }
	 426  
	 427  func TestLex(t *testing.T) {
	 428  	for _, test := range lexTests {
	 429  		items := collect(&test, "", "")
	 430  		if !equal(items, test.items, false) {
	 431  			t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items)
	 432  		}
	 433  	}
	 434  }
	 435  
	 436  // Some easy cases from above, but with delimiters $$ and @@
	 437  var lexDelimTests = []lexTest{
	 438  	{"punctuation", "$$,@%{{}}@@", []item{
	 439  		tLeftDelim,
	 440  		mkItem(itemChar, ","),
	 441  		mkItem(itemChar, "@"),
	 442  		mkItem(itemChar, "%"),
	 443  		mkItem(itemChar, "{"),
	 444  		mkItem(itemChar, "{"),
	 445  		mkItem(itemChar, "}"),
	 446  		mkItem(itemChar, "}"),
	 447  		tRightDelim,
	 448  		tEOF,
	 449  	}},
	 450  	{"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}},
	 451  	{"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}},
	 452  	{"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}},
	 453  	{"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}},
	 454  }
	 455  
	 456  var (
	 457  	tLeftDelim	= mkItem(itemLeftDelim, "$$")
	 458  	tRightDelim = mkItem(itemRightDelim, "@@")
	 459  )
	 460  
	 461  func TestDelims(t *testing.T) {
	 462  	for _, test := range lexDelimTests {
	 463  		items := collect(&test, "$$", "@@")
	 464  		if !equal(items, test.items, false) {
	 465  			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
	 466  		}
	 467  	}
	 468  }
	 469  
	 470  var lexPosTests = []lexTest{
	 471  	{"empty", "", []item{{itemEOF, 0, "", 1}}},
	 472  	{"punctuation", "{{,@%#}}", []item{
	 473  		{itemLeftDelim, 0, "{{", 1},
	 474  		{itemChar, 2, ",", 1},
	 475  		{itemChar, 3, "@", 1},
	 476  		{itemChar, 4, "%", 1},
	 477  		{itemChar, 5, "#", 1},
	 478  		{itemRightDelim, 6, "}}", 1},
	 479  		{itemEOF, 8, "", 1},
	 480  	}},
	 481  	{"sample", "0123{{hello}}xyz", []item{
	 482  		{itemText, 0, "0123", 1},
	 483  		{itemLeftDelim, 4, "{{", 1},
	 484  		{itemIdentifier, 6, "hello", 1},
	 485  		{itemRightDelim, 11, "}}", 1},
	 486  		{itemText, 13, "xyz", 1},
	 487  		{itemEOF, 16, "", 1},
	 488  	}},
	 489  	{"trimafter", "{{x -}}\n{{y}}", []item{
	 490  		{itemLeftDelim, 0, "{{", 1},
	 491  		{itemIdentifier, 2, "x", 1},
	 492  		{itemRightDelim, 5, "}}", 1},
	 493  		{itemLeftDelim, 8, "{{", 2},
	 494  		{itemIdentifier, 10, "y", 2},
	 495  		{itemRightDelim, 11, "}}", 2},
	 496  		{itemEOF, 13, "", 2},
	 497  	}},
	 498  	{"trimbefore", "{{x}}\n{{- y}}", []item{
	 499  		{itemLeftDelim, 0, "{{", 1},
	 500  		{itemIdentifier, 2, "x", 1},
	 501  		{itemRightDelim, 3, "}}", 1},
	 502  		{itemLeftDelim, 6, "{{", 2},
	 503  		{itemIdentifier, 10, "y", 2},
	 504  		{itemRightDelim, 11, "}}", 2},
	 505  		{itemEOF, 13, "", 2},
	 506  	}},
	 507  }
	 508  
	 509  // The other tests don't check position, to make the test cases easier to construct.
	 510  // This one does.
	 511  func TestPos(t *testing.T) {
	 512  	for _, test := range lexPosTests {
	 513  		items := collect(&test, "", "")
	 514  		if !equal(items, test.items, true) {
	 515  			t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items)
	 516  			if len(items) == len(test.items) {
	 517  				// Detailed print; avoid item.String() to expose the position value.
	 518  				for i := range items {
	 519  					if !equal(items[i:i+1], test.items[i:i+1], true) {
	 520  						i1 := items[i]
	 521  						i2 := test.items[i]
	 522  						t.Errorf("\t#%d: got {%v %d %q %d} expected {%v %d %q %d}",
	 523  							i, i1.typ, i1.pos, i1.val, i1.line, i2.typ, i2.pos, i2.val, i2.line)
	 524  					}
	 525  				}
	 526  			}
	 527  		}
	 528  	}
	 529  }
	 530  
	 531  // Test that an error shuts down the lexing goroutine.
	 532  func TestShutdown(t *testing.T) {
	 533  	// We need to duplicate template.Parse here to hold on to the lexer.
	 534  	const text = "erroneous{{define}}{{else}}1234"
	 535  	lexer := lex("foo", text, "{{", "}}", false)
	 536  	_, err := New("root").parseLexer(lexer)
	 537  	if err == nil {
	 538  		t.Fatalf("expected error")
	 539  	}
	 540  	// The error should have drained the input. Therefore, the lexer should be shut down.
	 541  	token, ok := <-lexer.items
	 542  	if ok {
	 543  		t.Fatalf("input was not drained; got %v", token)
	 544  	}
	 545  }
	 546  
	 547  // parseLexer is a local version of parse that lets us pass in the lexer instead of building it.
	 548  // We expect an error, so the tree set and funcs list are explicitly nil.
	 549  func (t *Tree) parseLexer(lex *lexer) (tree *Tree, err error) {
	 550  	defer t.recover(&err)
	 551  	t.ParseName = t.Name
	 552  	t.startParse(nil, lex, map[string]*Tree{})
	 553  	t.parse()
	 554  	t.add()
	 555  	t.stopParse()
	 556  	return t, nil
	 557  }
	 558  

View as plain text