example.go

Documentation: go/doc

		 1  // Copyright 2011 The Go Authors. All rights reserved.
		 2  // Use of this source code is governed by a BSD-style
		 3  // license that can be found in the LICENSE file.
		 4  
		 5  // Extract example functions from file ASTs.
		 6  
		 7  package doc
		 8  
		 9  import (
		10  	"go/ast"
		11  	"go/token"
		12  	"internal/lazyregexp"
		13  	"path"
		14  	"sort"
		15  	"strconv"
		16  	"strings"
		17  	"unicode"
		18  	"unicode/utf8"
		19  )
		20  
		21  // An Example represents an example function found in a test source file.
		22  type Example struct {
		23  	Name				string // name of the item being exemplified (including optional suffix)
		24  	Suffix			string // example suffix, without leading '_' (only populated by NewFromFiles)
		25  	Doc				 string // example function doc string
		26  	Code				ast.Node
		27  	Play				*ast.File // a whole program version of the example
		28  	Comments		[]*ast.CommentGroup
		29  	Output			string // expected output
		30  	Unordered	 bool
		31  	EmptyOutput bool // expect empty output
		32  	Order			 int	// original source code order
		33  }
		34  
		35  // Examples returns the examples found in testFiles, sorted by Name field.
		36  // The Order fields record the order in which the examples were encountered.
		37  // The Suffix field is not populated when Examples is called directly, it is
		38  // only populated by NewFromFiles for examples it finds in _test.go files.
		39  //
		40  // Playable Examples must be in a package whose name ends in "_test".
		41  // An Example is "playable" (the Play field is non-nil) in either of these
		42  // circumstances:
		43  //	 - The example function is self-contained: the function references only
		44  //		 identifiers from other packages (or predeclared identifiers, such as
		45  //		 "int") and the test file does not include a dot import.
		46  //	 - The entire test file is the example: the file contains exactly one
		47  //		 example function, zero test or benchmark functions, and at least one
		48  //		 top-level function, type, variable, or constant declaration other
		49  //		 than the example function.
		50  func Examples(testFiles ...*ast.File) []*Example {
		51  	var list []*Example
		52  	for _, file := range testFiles {
		53  		hasTests := false // file contains tests or benchmarks
		54  		numDecl := 0			// number of non-import declarations in the file
		55  		var flist []*Example
		56  		for _, decl := range file.Decls {
		57  			if g, ok := decl.(*ast.GenDecl); ok && g.Tok != token.IMPORT {
		58  				numDecl++
		59  				continue
		60  			}
		61  			f, ok := decl.(*ast.FuncDecl)
		62  			if !ok || f.Recv != nil {
		63  				continue
		64  			}
		65  			numDecl++
		66  			name := f.Name.Name
		67  			if isTest(name, "Test") || isTest(name, "Benchmark") {
		68  				hasTests = true
		69  				continue
		70  			}
		71  			if !isTest(name, "Example") {
		72  				continue
		73  			}
		74  			if params := f.Type.Params; len(params.List) != 0 {
		75  				continue // function has params; not a valid example
		76  			}
		77  			if f.Body == nil { // ast.File.Body nil dereference (see issue 28044)
		78  				continue
		79  			}
		80  			var doc string
		81  			if f.Doc != nil {
		82  				doc = f.Doc.Text()
		83  			}
		84  			output, unordered, hasOutput := exampleOutput(f.Body, file.Comments)
		85  			flist = append(flist, &Example{
		86  				Name:				name[len("Example"):],
		87  				Doc:				 doc,
		88  				Code:				f.Body,
		89  				Play:				playExample(file, f),
		90  				Comments:		file.Comments,
		91  				Output:			output,
		92  				Unordered:	 unordered,
		93  				EmptyOutput: output == "" && hasOutput,
		94  				Order:			 len(flist),
		95  			})
		96  		}
		97  		if !hasTests && numDecl > 1 && len(flist) == 1 {
		98  			// If this file only has one example function, some
		99  			// other top-level declarations, and no tests or
	 100  			// benchmarks, use the whole file as the example.
	 101  			flist[0].Code = file
	 102  			flist[0].Play = playExampleFile(file)
	 103  		}
	 104  		list = append(list, flist...)
	 105  	}
	 106  	// sort by name
	 107  	sort.Slice(list, func(i, j int) bool {
	 108  		return list[i].Name < list[j].Name
	 109  	})
	 110  	return list
	 111  }
	 112  
	 113  var outputPrefix = lazyregexp.New(`(?i)^[[:space:]]*(unordered )?output:`)
	 114  
	 115  // Extracts the expected output and whether there was a valid output comment
	 116  func exampleOutput(b *ast.BlockStmt, comments []*ast.CommentGroup) (output string, unordered, ok bool) {
	 117  	if _, last := lastComment(b, comments); last != nil {
	 118  		// test that it begins with the correct prefix
	 119  		text := last.Text()
	 120  		if loc := outputPrefix.FindStringSubmatchIndex(text); loc != nil {
	 121  			if loc[2] != -1 {
	 122  				unordered = true
	 123  			}
	 124  			text = text[loc[1]:]
	 125  			// Strip zero or more spaces followed by \n or a single space.
	 126  			text = strings.TrimLeft(text, " ")
	 127  			if len(text) > 0 && text[0] == '\n' {
	 128  				text = text[1:]
	 129  			}
	 130  			return text, unordered, true
	 131  		}
	 132  	}
	 133  	return "", false, false // no suitable comment found
	 134  }
	 135  
	 136  // isTest tells whether name looks like a test, example, or benchmark.
	 137  // It is a Test (say) if there is a character after Test that is not a
	 138  // lower-case letter. (We don't want Testiness.)
	 139  func isTest(name, prefix string) bool {
	 140  	if !strings.HasPrefix(name, prefix) {
	 141  		return false
	 142  	}
	 143  	if len(name) == len(prefix) { // "Test" is ok
	 144  		return true
	 145  	}
	 146  	rune, _ := utf8.DecodeRuneInString(name[len(prefix):])
	 147  	return !unicode.IsLower(rune)
	 148  }
	 149  
	 150  // playExample synthesizes a new *ast.File based on the provided
	 151  // file with the provided function body as the body of main.
	 152  func playExample(file *ast.File, f *ast.FuncDecl) *ast.File {
	 153  	body := f.Body
	 154  
	 155  	if !strings.HasSuffix(file.Name.Name, "_test") {
	 156  		// We don't support examples that are part of the
	 157  		// greater package (yet).
	 158  		return nil
	 159  	}
	 160  
	 161  	// Collect top-level declarations in the file.
	 162  	topDecls := make(map[*ast.Object]ast.Decl)
	 163  	typMethods := make(map[string][]ast.Decl)
	 164  
	 165  	for _, decl := range file.Decls {
	 166  		switch d := decl.(type) {
	 167  		case *ast.FuncDecl:
	 168  			if d.Recv == nil {
	 169  				topDecls[d.Name.Obj] = d
	 170  			} else {
	 171  				if len(d.Recv.List) == 1 {
	 172  					t := d.Recv.List[0].Type
	 173  					tname, _ := baseTypeName(t)
	 174  					typMethods[tname] = append(typMethods[tname], d)
	 175  				}
	 176  			}
	 177  		case *ast.GenDecl:
	 178  			for _, spec := range d.Specs {
	 179  				switch s := spec.(type) {
	 180  				case *ast.TypeSpec:
	 181  					topDecls[s.Name.Obj] = d
	 182  				case *ast.ValueSpec:
	 183  					for _, name := range s.Names {
	 184  						topDecls[name.Obj] = d
	 185  					}
	 186  				}
	 187  			}
	 188  		}
	 189  	}
	 190  
	 191  	// Find unresolved identifiers and uses of top-level declarations.
	 192  	unresolved := make(map[string]bool)
	 193  	var depDecls []ast.Decl
	 194  	hasDepDecls := make(map[ast.Decl]bool)
	 195  
	 196  	var inspectFunc func(ast.Node) bool
	 197  	inspectFunc = func(n ast.Node) bool {
	 198  		switch e := n.(type) {
	 199  		case *ast.Ident:
	 200  			if e.Obj == nil && e.Name != "_" {
	 201  				unresolved[e.Name] = true
	 202  			} else if d := topDecls[e.Obj]; d != nil {
	 203  				if !hasDepDecls[d] {
	 204  					hasDepDecls[d] = true
	 205  					depDecls = append(depDecls, d)
	 206  				}
	 207  			}
	 208  			return true
	 209  		case *ast.SelectorExpr:
	 210  			// For selector expressions, only inspect the left hand side.
	 211  			// (For an expression like fmt.Println, only add "fmt" to the
	 212  			// set of unresolved names, not "Println".)
	 213  			ast.Inspect(e.X, inspectFunc)
	 214  			return false
	 215  		case *ast.KeyValueExpr:
	 216  			// For key value expressions, only inspect the value
	 217  			// as the key should be resolved by the type of the
	 218  			// composite literal.
	 219  			ast.Inspect(e.Value, inspectFunc)
	 220  			return false
	 221  		}
	 222  		return true
	 223  	}
	 224  	ast.Inspect(body, inspectFunc)
	 225  	for i := 0; i < len(depDecls); i++ {
	 226  		switch d := depDecls[i].(type) {
	 227  		case *ast.FuncDecl:
	 228  			// Inspect types of parameters and results. See #28492.
	 229  			if d.Type.Params != nil {
	 230  				for _, p := range d.Type.Params.List {
	 231  					ast.Inspect(p.Type, inspectFunc)
	 232  				}
	 233  			}
	 234  			if d.Type.Results != nil {
	 235  				for _, r := range d.Type.Results.List {
	 236  					ast.Inspect(r.Type, inspectFunc)
	 237  				}
	 238  			}
	 239  
	 240  			// Functions might not have a body. See #42706.
	 241  			if d.Body != nil {
	 242  				ast.Inspect(d.Body, inspectFunc)
	 243  			}
	 244  		case *ast.GenDecl:
	 245  			for _, spec := range d.Specs {
	 246  				switch s := spec.(type) {
	 247  				case *ast.TypeSpec:
	 248  					ast.Inspect(s.Type, inspectFunc)
	 249  
	 250  					depDecls = append(depDecls, typMethods[s.Name.Name]...)
	 251  				case *ast.ValueSpec:
	 252  					if s.Type != nil {
	 253  						ast.Inspect(s.Type, inspectFunc)
	 254  					}
	 255  					for _, val := range s.Values {
	 256  						ast.Inspect(val, inspectFunc)
	 257  					}
	 258  				}
	 259  			}
	 260  		}
	 261  	}
	 262  
	 263  	// Remove predeclared identifiers from unresolved list.
	 264  	for n := range unresolved {
	 265  		if predeclaredTypes[n] || predeclaredConstants[n] || predeclaredFuncs[n] {
	 266  			delete(unresolved, n)
	 267  		}
	 268  	}
	 269  
	 270  	// Use unresolved identifiers to determine the imports used by this
	 271  	// example. The heuristic assumes package names match base import
	 272  	// paths for imports w/o renames (should be good enough most of the time).
	 273  	namedImports := make(map[string]string) // [name]path
	 274  	var blankImports []ast.Spec						 // _ imports
	 275  	for _, s := range file.Imports {
	 276  		p, err := strconv.Unquote(s.Path.Value)
	 277  		if err != nil {
	 278  			continue
	 279  		}
	 280  		if p == "syscall/js" {
	 281  			// We don't support examples that import syscall/js,
	 282  			// because the package syscall/js is not available in the playground.
	 283  			return nil
	 284  		}
	 285  		n := path.Base(p)
	 286  		if s.Name != nil {
	 287  			n = s.Name.Name
	 288  			switch n {
	 289  			case "_":
	 290  				blankImports = append(blankImports, s)
	 291  				continue
	 292  			case ".":
	 293  				// We can't resolve dot imports (yet).
	 294  				return nil
	 295  			}
	 296  		}
	 297  		if unresolved[n] {
	 298  			namedImports[n] = p
	 299  			delete(unresolved, n)
	 300  		}
	 301  	}
	 302  
	 303  	// If there are other unresolved identifiers, give up because this
	 304  	// synthesized file is not going to build.
	 305  	if len(unresolved) > 0 {
	 306  		return nil
	 307  	}
	 308  
	 309  	// Include documentation belonging to blank imports.
	 310  	var comments []*ast.CommentGroup
	 311  	for _, s := range blankImports {
	 312  		if c := s.(*ast.ImportSpec).Doc; c != nil {
	 313  			comments = append(comments, c)
	 314  		}
	 315  	}
	 316  
	 317  	// Include comments that are inside the function body.
	 318  	for _, c := range file.Comments {
	 319  		if body.Pos() <= c.Pos() && c.End() <= body.End() {
	 320  			comments = append(comments, c)
	 321  		}
	 322  	}
	 323  
	 324  	// Strip the "Output:" or "Unordered output:" comment and adjust body
	 325  	// end position.
	 326  	body, comments = stripOutputComment(body, comments)
	 327  
	 328  	// Include documentation belonging to dependent declarations.
	 329  	for _, d := range depDecls {
	 330  		switch d := d.(type) {
	 331  		case *ast.GenDecl:
	 332  			if d.Doc != nil {
	 333  				comments = append(comments, d.Doc)
	 334  			}
	 335  		case *ast.FuncDecl:
	 336  			if d.Doc != nil {
	 337  				comments = append(comments, d.Doc)
	 338  			}
	 339  		}
	 340  	}
	 341  
	 342  	// Synthesize import declaration.
	 343  	importDecl := &ast.GenDecl{
	 344  		Tok:		token.IMPORT,
	 345  		Lparen: 1, // Need non-zero Lparen and Rparen so that printer
	 346  		Rparen: 1, // treats this as a factored import.
	 347  	}
	 348  	for n, p := range namedImports {
	 349  		s := &ast.ImportSpec{Path: &ast.BasicLit{Value: strconv.Quote(p)}}
	 350  		if path.Base(p) != n {
	 351  			s.Name = ast.NewIdent(n)
	 352  		}
	 353  		importDecl.Specs = append(importDecl.Specs, s)
	 354  	}
	 355  	importDecl.Specs = append(importDecl.Specs, blankImports...)
	 356  
	 357  	// Synthesize main function.
	 358  	funcDecl := &ast.FuncDecl{
	 359  		Name: ast.NewIdent("main"),
	 360  		Type: f.Type,
	 361  		Body: body,
	 362  	}
	 363  
	 364  	decls := make([]ast.Decl, 0, 2+len(depDecls))
	 365  	decls = append(decls, importDecl)
	 366  	decls = append(decls, depDecls...)
	 367  	decls = append(decls, funcDecl)
	 368  
	 369  	sort.Slice(decls, func(i, j int) bool {
	 370  		return decls[i].Pos() < decls[j].Pos()
	 371  	})
	 372  
	 373  	sort.Slice(comments, func(i, j int) bool {
	 374  		return comments[i].Pos() < comments[j].Pos()
	 375  	})
	 376  
	 377  	// Synthesize file.
	 378  	return &ast.File{
	 379  		Name:		 ast.NewIdent("main"),
	 380  		Decls:		decls,
	 381  		Comments: comments,
	 382  	}
	 383  }
	 384  
	 385  // playExampleFile takes a whole file example and synthesizes a new *ast.File
	 386  // such that the example is function main in package main.
	 387  func playExampleFile(file *ast.File) *ast.File {
	 388  	// Strip copyright comment if present.
	 389  	comments := file.Comments
	 390  	if len(comments) > 0 && strings.HasPrefix(comments[0].Text(), "Copyright") {
	 391  		comments = comments[1:]
	 392  	}
	 393  
	 394  	// Copy declaration slice, rewriting the ExampleX function to main.
	 395  	var decls []ast.Decl
	 396  	for _, d := range file.Decls {
	 397  		if f, ok := d.(*ast.FuncDecl); ok && isTest(f.Name.Name, "Example") {
	 398  			// Copy the FuncDecl, as it may be used elsewhere.
	 399  			newF := *f
	 400  			newF.Name = ast.NewIdent("main")
	 401  			newF.Body, comments = stripOutputComment(f.Body, comments)
	 402  			d = &newF
	 403  		}
	 404  		decls = append(decls, d)
	 405  	}
	 406  
	 407  	// Copy the File, as it may be used elsewhere.
	 408  	f := *file
	 409  	f.Name = ast.NewIdent("main")
	 410  	f.Decls = decls
	 411  	f.Comments = comments
	 412  	return &f
	 413  }
	 414  
	 415  // stripOutputComment finds and removes the "Output:" or "Unordered output:"
	 416  // comment from body and comments, and adjusts the body block's end position.
	 417  func stripOutputComment(body *ast.BlockStmt, comments []*ast.CommentGroup) (*ast.BlockStmt, []*ast.CommentGroup) {
	 418  	// Do nothing if there is no "Output:" or "Unordered output:" comment.
	 419  	i, last := lastComment(body, comments)
	 420  	if last == nil || !outputPrefix.MatchString(last.Text()) {
	 421  		return body, comments
	 422  	}
	 423  
	 424  	// Copy body and comments, as the originals may be used elsewhere.
	 425  	newBody := &ast.BlockStmt{
	 426  		Lbrace: body.Lbrace,
	 427  		List:	 body.List,
	 428  		Rbrace: last.Pos(),
	 429  	}
	 430  	newComments := make([]*ast.CommentGroup, len(comments)-1)
	 431  	copy(newComments, comments[:i])
	 432  	copy(newComments[i:], comments[i+1:])
	 433  	return newBody, newComments
	 434  }
	 435  
	 436  // lastComment returns the last comment inside the provided block.
	 437  func lastComment(b *ast.BlockStmt, c []*ast.CommentGroup) (i int, last *ast.CommentGroup) {
	 438  	if b == nil {
	 439  		return
	 440  	}
	 441  	pos, end := b.Pos(), b.End()
	 442  	for j, cg := range c {
	 443  		if cg.Pos() < pos {
	 444  			continue
	 445  		}
	 446  		if cg.End() > end {
	 447  			break
	 448  		}
	 449  		i, last = j, cg
	 450  	}
	 451  	return
	 452  }
	 453  
	 454  // classifyExamples classifies examples and assigns them to the Examples field
	 455  // of the relevant Func, Type, or Package that the example is associated with.
	 456  //
	 457  // The classification process is ambiguous in some cases:
	 458  //
	 459  // 	- ExampleFoo_Bar matches a type named Foo_Bar
	 460  // 		or a method named Foo.Bar.
	 461  // 	- ExampleFoo_bar matches a type named Foo_bar
	 462  // 		or Foo (with a "bar" suffix).
	 463  //
	 464  // Examples with malformed names are not associated with anything.
	 465  //
	 466  func classifyExamples(p *Package, examples []*Example) {
	 467  	if len(examples) == 0 {
	 468  		return
	 469  	}
	 470  
	 471  	// Mapping of names for funcs, types, and methods to the example listing.
	 472  	ids := make(map[string]*[]*Example)
	 473  	ids[""] = &p.Examples // package-level examples have an empty name
	 474  	for _, f := range p.Funcs {
	 475  		if !token.IsExported(f.Name) {
	 476  			continue
	 477  		}
	 478  		ids[f.Name] = &f.Examples
	 479  	}
	 480  	for _, t := range p.Types {
	 481  		if !token.IsExported(t.Name) {
	 482  			continue
	 483  		}
	 484  		ids[t.Name] = &t.Examples
	 485  		for _, f := range t.Funcs {
	 486  			if !token.IsExported(f.Name) {
	 487  				continue
	 488  			}
	 489  			ids[f.Name] = &f.Examples
	 490  		}
	 491  		for _, m := range t.Methods {
	 492  			if !token.IsExported(m.Name) {
	 493  				continue
	 494  			}
	 495  			ids[strings.TrimPrefix(m.Recv, "*")+"_"+m.Name] = &m.Examples
	 496  		}
	 497  	}
	 498  
	 499  	// Group each example with the associated func, type, or method.
	 500  	for _, ex := range examples {
	 501  		// Consider all possible split points for the suffix
	 502  		// by starting at the end of string (no suffix case),
	 503  		// then trying all positions that contain a '_' character.
	 504  		//
	 505  		// An association is made on the first successful match.
	 506  		// Examples with malformed names that match nothing are skipped.
	 507  		for i := len(ex.Name); i >= 0; i = strings.LastIndexByte(ex.Name[:i], '_') {
	 508  			prefix, suffix, ok := splitExampleName(ex.Name, i)
	 509  			if !ok {
	 510  				continue
	 511  			}
	 512  			exs, ok := ids[prefix]
	 513  			if !ok {
	 514  				continue
	 515  			}
	 516  			ex.Suffix = suffix
	 517  			*exs = append(*exs, ex)
	 518  			break
	 519  		}
	 520  	}
	 521  
	 522  	// Sort list of example according to the user-specified suffix name.
	 523  	for _, exs := range ids {
	 524  		sort.Slice((*exs), func(i, j int) bool {
	 525  			return (*exs)[i].Suffix < (*exs)[j].Suffix
	 526  		})
	 527  	}
	 528  }
	 529  
	 530  // splitExampleName attempts to split example name s at index i,
	 531  // and reports if that produces a valid split. The suffix may be
	 532  // absent. Otherwise, it must start with a lower-case letter and
	 533  // be preceded by '_'.
	 534  //
	 535  // One of i == len(s) or s[i] == '_' must be true.
	 536  func splitExampleName(s string, i int) (prefix, suffix string, ok bool) {
	 537  	if i == len(s) {
	 538  		return s, "", true
	 539  	}
	 540  	if i == len(s)-1 {
	 541  		return "", "", false
	 542  	}
	 543  	prefix, suffix = s[:i], s[i+1:]
	 544  	return prefix, suffix, isExampleSuffix(suffix)
	 545  }
	 546  
	 547  func isExampleSuffix(s string) bool {
	 548  	r, size := utf8.DecodeRuneInString(s)
	 549  	return size > 0 && unicode.IsLower(r)
	 550  }
	 551
View as plain text