Source file src/index/suffixarray/suffixarray_test.go

Documentation: index/suffixarray

		 1  // Copyright 2010 The Go Authors. All rights reserved.
		 2  // Use of this source code is governed by a BSD-style
		 3  // license that can be found in the LICENSE file.
		 4  
		 5  package suffixarray
		 6  
		 7  import (
		 8  	"bytes"
		 9  	"fmt"
		10  	"io/fs"
		11  	"math/rand"
		12  	"os"
		13  	"path/filepath"
		14  	"regexp"
		15  	"sort"
		16  	"strings"
		17  	"testing"
		18  )
		19  
		20  type testCase struct {
		21  	name		 string	 // name of test case
		22  	source	 string	 // source to index
		23  	patterns []string // patterns to lookup
		24  }
		25  
		26  var testCases = []testCase{
		27  	{
		28  		"empty string",
		29  		"",
		30  		[]string{
		31  			"",
		32  			"foo",
		33  			"(foo)",
		34  			".*",
		35  			"a*",
		36  		},
		37  	},
		38  
		39  	{
		40  		"all a's",
		41  		"aaaaaaaaaa", // 10 a's
		42  		[]string{
		43  			"",
		44  			"a",
		45  			"aa",
		46  			"aaa",
		47  			"aaaa",
		48  			"aaaaa",
		49  			"aaaaaa",
		50  			"aaaaaaa",
		51  			"aaaaaaaa",
		52  			"aaaaaaaaa",
		53  			"aaaaaaaaaa",
		54  			"aaaaaaaaaaa", // 11 a's
		55  			".",
		56  			".*",
		57  			"a+",
		58  			"aa+",
		59  			"aaaa[b]?",
		60  			"aaa*",
		61  		},
		62  	},
		63  
		64  	{
		65  		"abc",
		66  		"abc",
		67  		[]string{
		68  			"a",
		69  			"b",
		70  			"c",
		71  			"ab",
		72  			"bc",
		73  			"abc",
		74  			"a.c",
		75  			"a(b|c)",
		76  			"abc?",
		77  		},
		78  	},
		79  
		80  	{
		81  		"barbara*3",
		82  		"barbarabarbarabarbara",
		83  		[]string{
		84  			"a",
		85  			"bar",
		86  			"rab",
		87  			"arab",
		88  			"barbar",
		89  			"bara?bar",
		90  		},
		91  	},
		92  
		93  	{
		94  		"typing drill",
		95  		"Now is the time for all good men to come to the aid of their country.",
		96  		[]string{
		97  			"Now",
		98  			"the time",
		99  			"to come the aid",
	 100  			"is the time for all good men to come to the aid of their",
	 101  			"to (come|the)?",
	 102  		},
	 103  	},
	 104  
	 105  	{
	 106  		"godoc simulation",
	 107  		"package main\n\nimport(\n		\"rand\"\n		",
	 108  		[]string{},
	 109  	},
	 110  }
	 111  
	 112  // find all occurrences of s in source; report at most n occurrences
	 113  func find(src, s string, n int) []int {
	 114  	var res []int
	 115  	if s != "" && n != 0 {
	 116  		// find at most n occurrences of s in src
	 117  		for i := -1; n < 0 || len(res) < n; {
	 118  			j := strings.Index(src[i+1:], s)
	 119  			if j < 0 {
	 120  				break
	 121  			}
	 122  			i += j + 1
	 123  			res = append(res, i)
	 124  		}
	 125  	}
	 126  	return res
	 127  }
	 128  
	 129  func testLookup(t *testing.T, tc *testCase, x *Index, s string, n int) {
	 130  	res := x.Lookup([]byte(s), n)
	 131  	exp := find(tc.source, s, n)
	 132  
	 133  	// check that the lengths match
	 134  	if len(res) != len(exp) {
	 135  		t.Errorf("test %q, lookup %q (n = %d): expected %d results; got %d", tc.name, s, n, len(exp), len(res))
	 136  	}
	 137  
	 138  	// if n >= 0 the number of results is limited --- unless n >= all results,
	 139  	// we may obtain different positions from the Index and from find (because
	 140  	// Index may not find the results in the same order as find) => in general
	 141  	// we cannot simply check that the res and exp lists are equal
	 142  
	 143  	// check that each result is in fact a correct match and there are no duplicates
	 144  	sort.Ints(res)
	 145  	for i, r := range res {
	 146  		if r < 0 || len(tc.source) <= r {
	 147  			t.Errorf("test %q, lookup %q, result %d (n = %d): index %d out of range [0, %d[", tc.name, s, i, n, r, len(tc.source))
	 148  		} else if !strings.HasPrefix(tc.source[r:], s) {
	 149  			t.Errorf("test %q, lookup %q, result %d (n = %d): index %d not a match", tc.name, s, i, n, r)
	 150  		}
	 151  		if i > 0 && res[i-1] == r {
	 152  			t.Errorf("test %q, lookup %q, result %d (n = %d): found duplicate index %d", tc.name, s, i, n, r)
	 153  		}
	 154  	}
	 155  
	 156  	if n < 0 {
	 157  		// all results computed - sorted res and exp must be equal
	 158  		for i, r := range res {
	 159  			e := exp[i]
	 160  			if r != e {
	 161  				t.Errorf("test %q, lookup %q, result %d: expected index %d; got %d", tc.name, s, i, e, r)
	 162  			}
	 163  		}
	 164  	}
	 165  }
	 166  
	 167  func testFindAllIndex(t *testing.T, tc *testCase, x *Index, rx *regexp.Regexp, n int) {
	 168  	res := x.FindAllIndex(rx, n)
	 169  	exp := rx.FindAllStringIndex(tc.source, n)
	 170  
	 171  	// check that the lengths match
	 172  	if len(res) != len(exp) {
	 173  		t.Errorf("test %q, FindAllIndex %q (n = %d): expected %d results; got %d", tc.name, rx, n, len(exp), len(res))
	 174  	}
	 175  
	 176  	// if n >= 0 the number of results is limited --- unless n >= all results,
	 177  	// we may obtain different positions from the Index and from regexp (because
	 178  	// Index may not find the results in the same order as regexp) => in general
	 179  	// we cannot simply check that the res and exp lists are equal
	 180  
	 181  	// check that each result is in fact a correct match and the result is sorted
	 182  	for i, r := range res {
	 183  		if r[0] < 0 || r[0] > r[1] || len(tc.source) < r[1] {
	 184  			t.Errorf("test %q, FindAllIndex %q, result %d (n == %d): illegal match [%d, %d]", tc.name, rx, i, n, r[0], r[1])
	 185  		} else if !rx.MatchString(tc.source[r[0]:r[1]]) {
	 186  			t.Errorf("test %q, FindAllIndex %q, result %d (n = %d): [%d, %d] not a match", tc.name, rx, i, n, r[0], r[1])
	 187  		}
	 188  	}
	 189  
	 190  	if n < 0 {
	 191  		// all results computed - sorted res and exp must be equal
	 192  		for i, r := range res {
	 193  			e := exp[i]
	 194  			if r[0] != e[0] || r[1] != e[1] {
	 195  				t.Errorf("test %q, FindAllIndex %q, result %d: expected match [%d, %d]; got [%d, %d]",
	 196  					tc.name, rx, i, e[0], e[1], r[0], r[1])
	 197  			}
	 198  		}
	 199  	}
	 200  }
	 201  
	 202  func testLookups(t *testing.T, tc *testCase, x *Index, n int) {
	 203  	for _, pat := range tc.patterns {
	 204  		testLookup(t, tc, x, pat, n)
	 205  		if rx, err := regexp.Compile(pat); err == nil {
	 206  			testFindAllIndex(t, tc, x, rx, n)
	 207  		}
	 208  	}
	 209  }
	 210  
	 211  // index is used to hide the sort.Interface
	 212  type index Index
	 213  
	 214  func (x *index) Len() int					 { return x.sa.len() }
	 215  func (x *index) Less(i, j int) bool { return bytes.Compare(x.at(i), x.at(j)) < 0 }
	 216  func (x *index) Swap(i, j int) {
	 217  	if x.sa.int32 != nil {
	 218  		x.sa.int32[i], x.sa.int32[j] = x.sa.int32[j], x.sa.int32[i]
	 219  	} else {
	 220  		x.sa.int64[i], x.sa.int64[j] = x.sa.int64[j], x.sa.int64[i]
	 221  	}
	 222  }
	 223  
	 224  func (x *index) at(i int) []byte {
	 225  	return x.data[x.sa.get(i):]
	 226  }
	 227  
	 228  func testConstruction(t *testing.T, tc *testCase, x *Index) {
	 229  	if !sort.IsSorted((*index)(x)) {
	 230  		t.Errorf("failed testConstruction %s", tc.name)
	 231  	}
	 232  }
	 233  
	 234  func equal(x, y *Index) bool {
	 235  	if !bytes.Equal(x.data, y.data) {
	 236  		return false
	 237  	}
	 238  	if x.sa.len() != y.sa.len() {
	 239  		return false
	 240  	}
	 241  	n := x.sa.len()
	 242  	for i := 0; i < n; i++ {
	 243  		if x.sa.get(i) != y.sa.get(i) {
	 244  			return false
	 245  		}
	 246  	}
	 247  	return true
	 248  }
	 249  
	 250  // returns the serialized index size
	 251  func testSaveRestore(t *testing.T, tc *testCase, x *Index) int {
	 252  	var buf bytes.Buffer
	 253  	if err := x.Write(&buf); err != nil {
	 254  		t.Errorf("failed writing index %s (%s)", tc.name, err)
	 255  	}
	 256  	size := buf.Len()
	 257  	var y Index
	 258  	if err := y.Read(bytes.NewReader(buf.Bytes())); err != nil {
	 259  		t.Errorf("failed reading index %s (%s)", tc.name, err)
	 260  	}
	 261  	if !equal(x, &y) {
	 262  		t.Errorf("restored index doesn't match saved index %s", tc.name)
	 263  	}
	 264  
	 265  	old := maxData32
	 266  	defer func() {
	 267  		maxData32 = old
	 268  	}()
	 269  	// Reread as forced 32.
	 270  	y = Index{}
	 271  	maxData32 = realMaxData32
	 272  	if err := y.Read(bytes.NewReader(buf.Bytes())); err != nil {
	 273  		t.Errorf("failed reading index %s (%s)", tc.name, err)
	 274  	}
	 275  	if !equal(x, &y) {
	 276  		t.Errorf("restored index doesn't match saved index %s", tc.name)
	 277  	}
	 278  
	 279  	// Reread as forced 64.
	 280  	y = Index{}
	 281  	maxData32 = -1
	 282  	if err := y.Read(bytes.NewReader(buf.Bytes())); err != nil {
	 283  		t.Errorf("failed reading index %s (%s)", tc.name, err)
	 284  	}
	 285  	if !equal(x, &y) {
	 286  		t.Errorf("restored index doesn't match saved index %s", tc.name)
	 287  	}
	 288  
	 289  	return size
	 290  }
	 291  
	 292  func testIndex(t *testing.T) {
	 293  	for _, tc := range testCases {
	 294  		x := New([]byte(tc.source))
	 295  		testConstruction(t, &tc, x)
	 296  		testSaveRestore(t, &tc, x)
	 297  		testLookups(t, &tc, x, 0)
	 298  		testLookups(t, &tc, x, 1)
	 299  		testLookups(t, &tc, x, 10)
	 300  		testLookups(t, &tc, x, 2e9)
	 301  		testLookups(t, &tc, x, -1)
	 302  	}
	 303  }
	 304  
	 305  func TestIndex32(t *testing.T) {
	 306  	testIndex(t)
	 307  }
	 308  
	 309  func TestIndex64(t *testing.T) {
	 310  	maxData32 = -1
	 311  	defer func() {
	 312  		maxData32 = realMaxData32
	 313  	}()
	 314  	testIndex(t)
	 315  }
	 316  
	 317  func TestNew32(t *testing.T) {
	 318  	test(t, func(x []byte) []int {
	 319  		sa := make([]int32, len(x))
	 320  		text_32(x, sa)
	 321  		out := make([]int, len(sa))
	 322  		for i, v := range sa {
	 323  			out[i] = int(v)
	 324  		}
	 325  		return out
	 326  	})
	 327  }
	 328  
	 329  func TestNew64(t *testing.T) {
	 330  	test(t, func(x []byte) []int {
	 331  		sa := make([]int64, len(x))
	 332  		text_64(x, sa)
	 333  		out := make([]int, len(sa))
	 334  		for i, v := range sa {
	 335  			out[i] = int(v)
	 336  		}
	 337  		return out
	 338  	})
	 339  }
	 340  
	 341  // test tests an arbitrary suffix array construction function.
	 342  // Generates many inputs, builds and checks suffix arrays.
	 343  func test(t *testing.T, build func([]byte) []int) {
	 344  	t.Run("ababab...", func(t *testing.T) {
	 345  		// Very repetitive input has numLMS = len(x)/2-1
	 346  		// at top level, the largest it can be.
	 347  		// But maxID is only two (aba and ab$).
	 348  		size := 100000
	 349  		if testing.Short() {
	 350  			size = 10000
	 351  		}
	 352  		x := make([]byte, size)
	 353  		for i := range x {
	 354  			x[i] = "ab"[i%2]
	 355  		}
	 356  		testSA(t, x, build)
	 357  	})
	 358  
	 359  	t.Run("forcealloc", func(t *testing.T) {
	 360  		// Construct a pathological input that forces
	 361  		// recurse_32 to allocate a new temporary buffer.
	 362  		// The input must have more than N/3 LMS-substrings,
	 363  		// which we arrange by repeating an SLSLSLSLSLSL pattern
	 364  		// like ababab... above, but then we must also arrange
	 365  		// for a large number of distinct LMS-substrings.
	 366  		// We use this pattern:
	 367  		// 1 255 1 254 1 253 1 ... 1 2 1 255 2 254 2 253 2 252 2 ...
	 368  		// This gives approximately 2¹⁵ distinct LMS-substrings.
	 369  		// We need to repeat at least one substring, though,
	 370  		// or else the recursion can be bypassed entirely.
	 371  		x := make([]byte, 100000, 100001)
	 372  		lo := byte(1)
	 373  		hi := byte(255)
	 374  		for i := range x {
	 375  			if i%2 == 0 {
	 376  				x[i] = lo
	 377  			} else {
	 378  				x[i] = hi
	 379  				hi--
	 380  				if hi <= lo {
	 381  					lo++
	 382  					if lo == 0 {
	 383  						lo = 1
	 384  					}
	 385  					hi = 255
	 386  				}
	 387  			}
	 388  		}
	 389  		x[:cap(x)][len(x)] = 0 // for sais.New
	 390  		testSA(t, x, build)
	 391  	})
	 392  
	 393  	t.Run("exhaustive2", func(t *testing.T) {
	 394  		// All inputs over {0,1} up to length 21.
	 395  		// Runs in about 10 seconds on my laptop.
	 396  		x := make([]byte, 30)
	 397  		numFail := 0
	 398  		for n := 0; n <= 21; n++ {
	 399  			if n > 12 && testing.Short() {
	 400  				break
	 401  			}
	 402  			x[n] = 0 // for sais.New
	 403  			testRec(t, x[:n], 0, 2, &numFail, build)
	 404  		}
	 405  	})
	 406  
	 407  	t.Run("exhaustive3", func(t *testing.T) {
	 408  		// All inputs over {0,1,2} up to length 14.
	 409  		// Runs in about 10 seconds on my laptop.
	 410  		x := make([]byte, 30)
	 411  		numFail := 0
	 412  		for n := 0; n <= 14; n++ {
	 413  			if n > 8 && testing.Short() {
	 414  				break
	 415  			}
	 416  			x[n] = 0 // for sais.New
	 417  			testRec(t, x[:n], 0, 3, &numFail, build)
	 418  		}
	 419  	})
	 420  }
	 421  
	 422  // testRec fills x[i:] with all possible combinations of values in [1,max]
	 423  // and then calls testSA(t, x, build) for each one.
	 424  func testRec(t *testing.T, x []byte, i, max int, numFail *int, build func([]byte) []int) {
	 425  	if i < len(x) {
	 426  		for x[i] = 1; x[i] <= byte(max); x[i]++ {
	 427  			testRec(t, x, i+1, max, numFail, build)
	 428  		}
	 429  		return
	 430  	}
	 431  
	 432  	if !testSA(t, x, build) {
	 433  		*numFail++
	 434  		if *numFail >= 10 {
	 435  			t.Errorf("stopping after %d failures", *numFail)
	 436  			t.FailNow()
	 437  		}
	 438  	}
	 439  }
	 440  
	 441  // testSA tests the suffix array build function on the input x.
	 442  // It constructs the suffix array and then checks that it is correct.
	 443  func testSA(t *testing.T, x []byte, build func([]byte) []int) bool {
	 444  	defer func() {
	 445  		if e := recover(); e != nil {
	 446  			t.Logf("build %v", x)
	 447  			panic(e)
	 448  		}
	 449  	}()
	 450  	sa := build(x)
	 451  	if len(sa) != len(x) {
	 452  		t.Errorf("build %v: len(sa) = %d, want %d", x, len(sa), len(x))
	 453  		return false
	 454  	}
	 455  	for i := 0; i+1 < len(sa); i++ {
	 456  		if sa[i] < 0 || sa[i] >= len(x) || sa[i+1] < 0 || sa[i+1] >= len(x) {
	 457  			t.Errorf("build %s: sa out of range: %v\n", x, sa)
	 458  			return false
	 459  		}
	 460  		if bytes.Compare(x[sa[i]:], x[sa[i+1]:]) >= 0 {
	 461  			t.Errorf("build %v -> %v\nsa[%d:] = %d,%d out of order", x, sa, i, sa[i], sa[i+1])
	 462  			return false
	 463  		}
	 464  	}
	 465  
	 466  	return true
	 467  }
	 468  
	 469  var (
	 470  	benchdata = make([]byte, 1e6)
	 471  	benchrand = make([]byte, 1e6)
	 472  )
	 473  
	 474  // Of all possible inputs, the random bytes have the least amount of substring
	 475  // repetition, and the repeated bytes have the most. For most algorithms,
	 476  // the running time of every input will be between these two.
	 477  func benchmarkNew(b *testing.B, random bool) {
	 478  	b.ReportAllocs()
	 479  	b.StopTimer()
	 480  	data := benchdata
	 481  	if random {
	 482  		data = benchrand
	 483  		if data[0] == 0 {
	 484  			for i := range data {
	 485  				data[i] = byte(rand.Intn(256))
	 486  			}
	 487  		}
	 488  	}
	 489  	b.StartTimer()
	 490  	b.SetBytes(int64(len(data)))
	 491  	for i := 0; i < b.N; i++ {
	 492  		New(data)
	 493  	}
	 494  }
	 495  
	 496  func makeText(name string) ([]byte, error) {
	 497  	var data []byte
	 498  	switch name {
	 499  	case "opticks":
	 500  		var err error
	 501  		data, err = os.ReadFile("../../testdata/Isaac.Newton-Opticks.txt")
	 502  		if err != nil {
	 503  			return nil, err
	 504  		}
	 505  	case "go":
	 506  		err := filepath.WalkDir("../..", func(path string, info fs.DirEntry, err error) error {
	 507  			if err == nil && strings.HasSuffix(path, ".go") && !info.IsDir() {
	 508  				file, err := os.ReadFile(path)
	 509  				if err != nil {
	 510  					return err
	 511  				}
	 512  				data = append(data, file...)
	 513  			}
	 514  			return nil
	 515  		})
	 516  		if err != nil {
	 517  			return nil, err
	 518  		}
	 519  	case "zero":
	 520  		data = make([]byte, 50e6)
	 521  	case "rand":
	 522  		data = make([]byte, 50e6)
	 523  		for i := range data {
	 524  			data[i] = byte(rand.Intn(256))
	 525  		}
	 526  	}
	 527  	return data, nil
	 528  }
	 529  
	 530  func setBits(bits int) (cleanup func()) {
	 531  	if bits == 32 {
	 532  		maxData32 = realMaxData32
	 533  	} else {
	 534  		maxData32 = -1 // force use of 64-bit code
	 535  	}
	 536  	return func() {
	 537  		maxData32 = realMaxData32
	 538  	}
	 539  }
	 540  
	 541  func BenchmarkNew(b *testing.B) {
	 542  	for _, text := range []string{"opticks", "go", "zero", "rand"} {
	 543  		b.Run("text="+text, func(b *testing.B) {
	 544  			data, err := makeText(text)
	 545  			if err != nil {
	 546  				b.Fatal(err)
	 547  			}
	 548  			if testing.Short() && len(data) > 5e6 {
	 549  				data = data[:5e6]
	 550  			}
	 551  			for _, size := range []int{100e3, 500e3, 1e6, 5e6, 10e6, 50e6} {
	 552  				if len(data) < size {
	 553  					continue
	 554  				}
	 555  				data := data[:size]
	 556  				name := fmt.Sprintf("%dK", size/1e3)
	 557  				if size >= 1e6 {
	 558  					name = fmt.Sprintf("%dM", size/1e6)
	 559  				}
	 560  				b.Run("size="+name, func(b *testing.B) {
	 561  					for _, bits := range []int{32, 64} {
	 562  						if ^uint(0) == 0xffffffff && bits == 64 {
	 563  							continue
	 564  						}
	 565  						b.Run(fmt.Sprintf("bits=%d", bits), func(b *testing.B) {
	 566  							cleanup := setBits(bits)
	 567  							defer cleanup()
	 568  
	 569  							b.SetBytes(int64(len(data)))
	 570  							b.ReportAllocs()
	 571  							for i := 0; i < b.N; i++ {
	 572  								New(data)
	 573  							}
	 574  						})
	 575  					}
	 576  				})
	 577  			}
	 578  		})
	 579  	}
	 580  }
	 581  
	 582  func BenchmarkSaveRestore(b *testing.B) {
	 583  	r := rand.New(rand.NewSource(0x5a77a1)) // guarantee always same sequence
	 584  	data := make([]byte, 1<<20)						 // 1MB of data to index
	 585  	for i := range data {
	 586  		data[i] = byte(r.Intn(256))
	 587  	}
	 588  	for _, bits := range []int{32, 64} {
	 589  		if ^uint(0) == 0xffffffff && bits == 64 {
	 590  			continue
	 591  		}
	 592  		b.Run(fmt.Sprintf("bits=%d", bits), func(b *testing.B) {
	 593  			cleanup := setBits(bits)
	 594  			defer cleanup()
	 595  
	 596  			b.StopTimer()
	 597  			x := New(data)
	 598  			size := testSaveRestore(nil, nil, x)			 // verify correctness
	 599  			buf := bytes.NewBuffer(make([]byte, size)) // avoid growing
	 600  			b.SetBytes(int64(size))
	 601  			b.StartTimer()
	 602  			b.ReportAllocs()
	 603  			for i := 0; i < b.N; i++ {
	 604  				buf.Reset()
	 605  				if err := x.Write(buf); err != nil {
	 606  					b.Fatal(err)
	 607  				}
	 608  				var y Index
	 609  				if err := y.Read(buf); err != nil {
	 610  					b.Fatal(err)
	 611  				}
	 612  			}
	 613  		})
	 614  	}
	 615  }
	 616  

View as plain text