1// Copyright 2019 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package text_test 6 7import ( 8 "fmt" 9 "math" 10 "strings" 11 "testing" 12 "unicode/utf8" 13 14 "github.com/google/go-cmp/cmp" 15 16 "google.golang.org/protobuf/internal/encoding/text" 17 "google.golang.org/protobuf/internal/flags" 18) 19 20var eofErr = text.ErrUnexpectedEOF.Error() 21 22type R struct { 23 // K is expected Kind of the returned Token object from calling Decoder.Read. 24 K text.Kind 25 // E is expected error substring from calling Decoder.Read if set. 26 E string 27 // T contains NT (if K is Name) or ST (if K is Scalar) or nil (others) 28 T interface{} 29 // P is expected Token.Pos if set > 0. 30 P int 31 // RS is expected result from Token.RawString() if not empty. 32 RS string 33} 34 35// NT contains data for checking against a name token. 36type NT struct { 37 K text.NameKind 38 // Sep is true if name token should have separator character, else false. 39 Sep bool 40 // If K is IdentName or TypeName, invoke corresponding getter and compare against this field. 41 S string 42 // If K is FieldNumber, invoke getter and compare against this field. 43 N int32 44} 45 46// ST contains data for checking against a scalar token. 47type ST struct { 48 // checker that is expected to return OK. 49 ok checker 50 // checker that is expected to return not OK. 51 nok checker 52} 53 54// checker provides API for the token wrapper API call types Str, Enum, Bool, 55// Uint64, Uint32, Int64, Int32, Float64, Float32. 56type checker interface { 57 // checkOk checks and expects for token API call to return ok and compare 58 // against implementation-stored value. Returns empty string if success, 59 // else returns error message describing the error. 60 checkOk(text.Token) string 61 // checkNok checks and expects for token API call to return not ok. Returns 62 // empty string if success, else returns error message describing the error. 63 checkNok(text.Token) string 64} 65 66type Str struct { 67 val string 68} 69 70func (s Str) checkOk(tok text.Token) string { 71 got, ok := tok.String() 72 if !ok { 73 return fmt.Sprintf("Token.String() returned not OK for token: %v", tok.RawString()) 74 } 75 if got != s.val { 76 return fmt.Sprintf("Token.String() got %q want %q for token: %v", got, s.val, tok.RawString()) 77 } 78 return "" 79} 80 81func (s Str) checkNok(tok text.Token) string { 82 if _, ok := tok.String(); ok { 83 return fmt.Sprintf("Token.String() returned OK for token: %v", tok.RawString()) 84 } 85 return "" 86} 87 88type Enum struct { 89 val string 90} 91 92func (e Enum) checkOk(tok text.Token) string { 93 got, ok := tok.Enum() 94 if !ok { 95 return fmt.Sprintf("Token.Enum() returned not OK for token: %v", tok.RawString()) 96 } 97 if got != e.val { 98 return fmt.Sprintf("Token.Enum() got %q want %q for token: %v", got, e.val, tok.RawString()) 99 } 100 return "" 101} 102 103func (e Enum) checkNok(tok text.Token) string { 104 if _, ok := tok.Enum(); ok { 105 return fmt.Sprintf("Token.Enum() returned OK for token: %v", tok.RawString()) 106 } 107 return "" 108} 109 110type Bool struct { 111 val bool 112} 113 114func (b Bool) checkOk(tok text.Token) string { 115 got, ok := tok.Bool() 116 if !ok { 117 return fmt.Sprintf("Token.Bool() returned not OK for token: %v", tok.RawString()) 118 } 119 if got != b.val { 120 return fmt.Sprintf("Token.Bool() got %v want %v for token: %v", got, b.val, tok.RawString()) 121 } 122 return "" 123} 124 125func (b Bool) checkNok(tok text.Token) string { 126 if _, ok := tok.Bool(); ok { 127 return fmt.Sprintf("Token.Bool() returned OK for token: %v", tok.RawString()) 128 } 129 return "" 130} 131 132type Uint64 struct { 133 val uint64 134} 135 136func (n Uint64) checkOk(tok text.Token) string { 137 got, ok := tok.Uint64() 138 if !ok { 139 return fmt.Sprintf("Token.Uint64() returned not OK for token: %v", tok.RawString()) 140 } 141 if got != n.val { 142 return fmt.Sprintf("Token.Uint64() got %v want %v for token: %v", got, n.val, tok.RawString()) 143 } 144 return "" 145} 146 147func (n Uint64) checkNok(tok text.Token) string { 148 if _, ok := tok.Uint64(); ok { 149 return fmt.Sprintf("Token.Uint64() returned OK for token: %v", tok.RawString()) 150 } 151 return "" 152} 153 154type Uint32 struct { 155 val uint32 156} 157 158func (n Uint32) checkOk(tok text.Token) string { 159 got, ok := tok.Uint32() 160 if !ok { 161 return fmt.Sprintf("Token.Uint32() returned not OK for token: %v", tok.RawString()) 162 } 163 if got != n.val { 164 return fmt.Sprintf("Token.Uint32() got %v want %v for token: %v", got, n.val, tok.RawString()) 165 } 166 return "" 167} 168 169func (n Uint32) checkNok(tok text.Token) string { 170 if _, ok := tok.Uint32(); ok { 171 return fmt.Sprintf("Token.Uint32() returned OK for token: %v", tok.RawString()) 172 } 173 return "" 174} 175 176type Int64 struct { 177 val int64 178} 179 180func (n Int64) checkOk(tok text.Token) string { 181 got, ok := tok.Int64() 182 if !ok { 183 return fmt.Sprintf("Token.Int64() returned not OK for token: %v", tok.RawString()) 184 } 185 if got != n.val { 186 return fmt.Sprintf("Token.Int64() got %v want %v for token: %v", got, n.val, tok.RawString()) 187 } 188 return "" 189} 190 191func (n Int64) checkNok(tok text.Token) string { 192 if _, ok := tok.Int64(); ok { 193 return fmt.Sprintf("Token.Int64() returned OK for token: %v", tok.RawString()) 194 } 195 return "" 196} 197 198type Int32 struct { 199 val int32 200} 201 202func (n Int32) checkOk(tok text.Token) string { 203 got, ok := tok.Int32() 204 if !ok { 205 return fmt.Sprintf("Token.Int32() returned not OK for token: %v", tok.RawString()) 206 } 207 if got != n.val { 208 return fmt.Sprintf("Token.Int32() got %v want %v for token: %v", got, n.val, tok.RawString()) 209 } 210 return "" 211} 212 213func (n Int32) checkNok(tok text.Token) string { 214 if _, ok := tok.Int32(); ok { 215 return fmt.Sprintf("Token.Int32() returned OK for token: %v", tok.RawString()) 216 } 217 return "" 218} 219 220type Float64 struct { 221 val float64 222} 223 224func (n Float64) checkOk(tok text.Token) string { 225 got, ok := tok.Float64() 226 if !ok { 227 return fmt.Sprintf("Token.Float64() returned not OK for token: %v", tok.RawString()) 228 } 229 if math.Float64bits(got) != math.Float64bits(n.val) { 230 return fmt.Sprintf("Token.Float64() got %v want %v for token: %v", got, n.val, tok.RawString()) 231 } 232 return "" 233} 234 235func (n Float64) checkNok(tok text.Token) string { 236 if _, ok := tok.Float64(); ok { 237 return fmt.Sprintf("Token.Float64() returned OK for token: %v", tok.RawString()) 238 } 239 return "" 240} 241 242type Float32 struct { 243 val float32 244} 245 246func (n Float32) checkOk(tok text.Token) string { 247 got, ok := tok.Float32() 248 if !ok { 249 return fmt.Sprintf("Token.Float32() returned not OK for token: %v", tok.RawString()) 250 } 251 if math.Float32bits(got) != math.Float32bits(n.val) { 252 return fmt.Sprintf("Token.Float32() got %v want %v for token: %v", got, n.val, tok.RawString()) 253 } 254 return "" 255} 256 257func (n Float32) checkNok(tok text.Token) string { 258 if _, ok := tok.Float32(); ok { 259 return fmt.Sprintf("Token.Float32() returned OK for token: %v", tok.RawString()) 260 } 261 return "" 262} 263 264func TestDecoder(t *testing.T) { 265 const space = " \n\r\t" 266 tests := []struct { 267 in string 268 // want is a list of expected Tokens returned from calling Decoder.Read. 269 // An item makes the test code invoke Decoder.Read and compare against 270 // R.K and R.E. If R.K is Name, it compares 271 want []R 272 }{ 273 { 274 in: "", 275 want: []R{{K: text.EOF}}, 276 }, 277 { 278 in: "# comment", 279 want: []R{{K: text.EOF}}, 280 }, 281 { 282 in: space + "# comment" + space, 283 want: []R{{K: text.EOF}}, 284 }, 285 { 286 in: space, 287 want: []R{{K: text.EOF, P: len(space)}}, 288 }, 289 { 290 // Calling Read after EOF will keep returning EOF for 291 // succeeding Read calls. 292 in: space, 293 want: []R{ 294 {K: text.EOF}, 295 {K: text.EOF}, 296 {K: text.EOF}, 297 }, 298 }, 299 { 300 // NUL is an invalid whitespace since C++ uses C-strings. 301 in: "\x00", 302 want: []R{{E: "invalid field name: \x00"}}, 303 }, 304 305 // Field names. 306 { 307 in: "name", 308 want: []R{ 309 {K: text.Name, T: NT{K: text.IdentName, S: "name"}, RS: "name"}, 310 {E: eofErr}, 311 }, 312 }, 313 { 314 in: space + "name:" + space, 315 want: []R{ 316 {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}}, 317 {E: eofErr}, 318 }, 319 }, 320 { 321 in: space + "name" + space + ":" + space, 322 want: []R{ 323 {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}}, 324 {E: eofErr}, 325 }, 326 }, 327 { 328 in: "name # comment", 329 want: []R{ 330 {K: text.Name, T: NT{K: text.IdentName, S: "name"}}, 331 {E: eofErr}, 332 }, 333 }, 334 { 335 // Comments only extend until the newline. 336 in: "# comment \nname", 337 want: []R{ 338 {K: text.Name, T: NT{K: text.IdentName, S: "name"}, P: 11}, 339 }, 340 }, 341 { 342 in: "name # comment \n:", 343 want: []R{ 344 {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}}, 345 }, 346 }, 347 { 348 in: "name123", 349 want: []R{ 350 {K: text.Name, T: NT{K: text.IdentName, S: "name123"}}, 351 }, 352 }, 353 { 354 in: "name_123", 355 want: []R{ 356 {K: text.Name, T: NT{K: text.IdentName, S: "name_123"}}, 357 }, 358 }, 359 { 360 in: "_123", 361 want: []R{ 362 {K: text.Name, T: NT{K: text.IdentName, S: "_123"}}, 363 }, 364 }, 365 { 366 in: ":", 367 want: []R{{E: "syntax error (line 1:1): invalid field name: :"}}, 368 }, 369 { 370 in: "\n\n\n {", 371 want: []R{{E: "syntax error (line 4:2): invalid field name: {"}}, 372 }, 373 { 374 in: "123name", 375 want: []R{{E: "invalid field name: 123name"}}, 376 }, 377 { 378 in: `/`, 379 want: []R{{E: `invalid field name: /`}}, 380 }, 381 { 382 in: `世界`, 383 want: []R{{E: `invalid field name: 世`}}, 384 }, 385 { 386 in: `1a/b`, 387 want: []R{{E: `invalid field name: 1a`}}, 388 }, 389 { 390 in: `1c\d`, 391 want: []R{{E: `invalid field name: 1c`}}, 392 }, 393 { 394 in: "\x84f", 395 want: []R{{E: "invalid field name: \x84"}}, 396 }, 397 { 398 in: "\uFFFDxxx", 399 want: []R{{E: "invalid field name: \uFFFD"}}, 400 }, 401 { 402 in: "-a234567890123456789012345678901234567890abc", 403 want: []R{{E: "invalid field name: -a2345678901234567890123456789012…"}}, 404 }, 405 { 406 in: "[type]", 407 want: []R{ 408 {K: text.Name, T: NT{K: text.TypeName, S: "type"}, RS: "[type]"}, 409 }, 410 }, 411 { 412 // V1 allows this syntax. C++ does not, however, C++ also fails if 413 // field is Any and does not contain '/'. 414 in: "[/type]", 415 want: []R{ 416 {K: text.Name, T: NT{K: text.TypeName, S: "/type"}}, 417 }, 418 }, 419 { 420 in: "[.type]", 421 want: []R{{E: "invalid type URL/extension field name: [.type]"}}, 422 }, 423 { 424 in: "[pkg.Foo.extension_field]", 425 want: []R{ 426 {K: text.Name, T: NT{K: text.TypeName, S: "pkg.Foo.extension_field"}}, 427 }, 428 }, 429 { 430 in: "[domain.com/type]", 431 want: []R{ 432 {K: text.Name, T: NT{K: text.TypeName, S: "domain.com/type"}}, 433 }, 434 }, 435 { 436 in: "[domain.com/pkg.type]", 437 want: []R{ 438 {K: text.Name, T: NT{K: text.TypeName, S: "domain.com/pkg.type"}}, 439 }, 440 }, 441 { 442 in: "[sub.domain.com\x2fpath\x2fto\x2fproto.package.name]", 443 want: []R{ 444 { 445 K: text.Name, 446 T: NT{ 447 K: text.TypeName, 448 S: "sub.domain.com/path/to/proto.package.name", 449 }, 450 RS: "[sub.domain.com\x2fpath\x2fto\x2fproto.package.name]", 451 }, 452 }, 453 }, 454 { 455 // V2 no longer allows a quoted string for the Any type URL. 456 in: `["domain.com/pkg.type"]`, 457 want: []R{{E: `invalid type URL/extension field name: ["`}}, 458 }, 459 { 460 // V2 no longer allows a quoted string for the Any type URL. 461 in: `['domain.com/pkg.type']`, 462 want: []R{{E: `invalid type URL/extension field name: ['`}}, 463 }, 464 { 465 in: "[pkg.Foo.extension_field:", 466 want: []R{{E: "invalid type URL/extension field name: [pkg.Foo.extension_field:"}}, 467 }, 468 { 469 // V2 no longer allows whitespace within identifier "word". 470 in: "[proto.packa ge.field]", 471 want: []R{{E: "invalid type URL/extension field name: [proto.packa g"}}, 472 }, 473 { 474 // V2 no longer allows comments within identifier "word". 475 in: "[proto.packa # comment\n ge.field]", 476 want: []R{{E: "invalid type URL/extension field name: [proto.packa # comment\n g"}}, 477 }, 478 { 479 in: "[proto.package.]", 480 want: []R{{E: "invalid type URL/extension field name: [proto.package."}}, 481 }, 482 { 483 in: "[proto.package/]", 484 want: []R{{E: "invalid type URL/extension field name: [proto.package/"}}, 485 }, 486 { 487 in: `message_field{[bad@]`, 488 want: []R{ 489 {K: text.Name}, 490 {K: text.MessageOpen}, 491 {E: `invalid type URL/extension field name: [bad@`}, 492 }, 493 }, 494 { 495 in: `message_field{[invalid//type]`, 496 want: []R{ 497 {K: text.Name}, 498 {K: text.MessageOpen}, 499 {E: `invalid type URL/extension field name: [invalid//`}, 500 }, 501 }, 502 { 503 in: `message_field{[proto.package.]`, 504 want: []R{ 505 {K: text.Name}, 506 {K: text.MessageOpen}, 507 {E: `invalid type URL/extension field name: [proto.package.`}, 508 }, 509 }, 510 { 511 in: "[proto.package", 512 want: []R{{E: eofErr}}, 513 }, 514 { 515 in: "[" + space + "type" + space + "]" + space + ":", 516 want: []R{ 517 { 518 K: text.Name, 519 T: NT{ 520 K: text.TypeName, 521 Sep: true, 522 S: "type", 523 }, 524 RS: "[" + space + "type" + space + "]", 525 }, 526 }, 527 }, 528 { 529 // Whitespaces/comments are only allowed betweeb 530 in: "[" + space + "domain" + space + "." + space + "com # comment\n" + 531 "/" + "pkg" + space + "." + space + "type" + space + "]", 532 want: []R{ 533 {K: text.Name, T: NT{K: text.TypeName, S: "domain.com/pkg.type"}}, 534 }, 535 }, 536 { 537 in: "42", 538 want: []R{ 539 {K: text.Name, T: NT{K: text.FieldNumber, N: 42}}, 540 }, 541 }, 542 { 543 in: "0x42:", 544 want: []R{{E: "invalid field number: 0x42"}}, 545 }, 546 { 547 in: "042:", 548 want: []R{{E: "invalid field number: 042"}}, 549 }, 550 { 551 in: "123.456:", 552 want: []R{{E: "invalid field number: 123.456"}}, 553 }, 554 { 555 in: "-123", 556 want: []R{{E: "invalid field number: -123"}}, 557 }, 558 { 559 in: "- \t 123.321e6", 560 want: []R{{E: "invalid field number: -123.321e6"}}, 561 }, 562 { 563 in: "-", 564 want: []R{{E: "invalid field name: -"}}, 565 }, 566 { 567 in: "- ", 568 want: []R{{E: "invalid field name: -"}}, 569 }, 570 { 571 in: "- # negative\n 123", 572 want: []R{{E: "invalid field number: -123"}}, 573 }, 574 { 575 // Field number > math.MaxInt32. 576 in: "2147483648:", 577 want: []R{{E: "invalid field number: 2147483648"}}, 578 }, 579 580 // String field value. More string parsing specific testing in 581 // TestUnmarshalString. 582 { 583 in: `name: "hello world"`, 584 want: []R{ 585 {K: text.Name}, 586 { 587 K: text.Scalar, 588 T: ST{ok: Str{"hello world"}, nok: Enum{}}, 589 RS: `"hello world"`, 590 }, 591 {K: text.EOF}, 592 }, 593 }, 594 { 595 in: `name: 'hello'`, 596 want: []R{ 597 {K: text.Name}, 598 {K: text.Scalar, T: ST{ok: Str{"hello"}}}, 599 }, 600 }, 601 { 602 in: `name: "hello'`, 603 want: []R{ 604 {K: text.Name}, 605 {E: eofErr}, 606 }, 607 }, 608 { 609 in: `name: 'hello`, 610 want: []R{ 611 {K: text.Name}, 612 {E: eofErr}, 613 }, 614 }, 615 { 616 // Field name without separator is ok. prototext package will need 617 // to determine that this is not valid for scalar values. 618 in: space + `name` + space + `"hello"` + space, 619 want: []R{ 620 {K: text.Name}, 621 {K: text.Scalar, T: ST{ok: Str{"hello"}}}, 622 }, 623 }, 624 { 625 in: `name'hello'`, 626 want: []R{ 627 {K: text.Name}, 628 {K: text.Scalar, T: ST{ok: Str{"hello"}}}, 629 }, 630 }, 631 { 632 in: `name: ` + space + `"hello"` + space + `,`, 633 want: []R{ 634 {K: text.Name}, 635 {K: text.Scalar, T: ST{ok: Str{"hello"}}}, 636 {K: text.EOF}, 637 }, 638 }, 639 { 640 in: `name` + space + `:` + `"hello"` + space + `;` + space, 641 want: []R{ 642 {K: text.Name}, 643 {K: text.Scalar, T: ST{ok: Str{"hello"}}}, 644 {K: text.EOF}, 645 }, 646 }, 647 { 648 in: `name:"hello" , ,`, 649 want: []R{ 650 {K: text.Name}, 651 {K: text.Scalar}, 652 {E: "(line 1:16): invalid field name: ,"}, 653 }, 654 }, 655 { 656 in: `name:"hello" , ;`, 657 want: []R{ 658 {K: text.Name}, 659 {K: text.Scalar}, 660 {E: "(line 1:16): invalid field name: ;"}, 661 }, 662 }, 663 { 664 in: `name:"hello" name:'world'`, 665 want: []R{ 666 {K: text.Name}, 667 {K: text.Scalar, T: ST{ok: Str{"hello"}}}, 668 {K: text.Name}, 669 {K: text.Scalar, T: ST{ok: Str{"world"}}}, 670 {K: text.EOF}, 671 }, 672 }, 673 { 674 in: `name:"hello", name:"world"`, 675 want: []R{ 676 {K: text.Name}, 677 {K: text.Scalar, T: ST{ok: Str{"hello"}}}, 678 {K: text.Name}, 679 {K: text.Scalar, T: ST{ok: Str{"world"}}}, 680 {K: text.EOF}, 681 }, 682 }, 683 { 684 in: `name:"hello"; name:"world",`, 685 want: []R{ 686 {K: text.Name}, 687 {K: text.Scalar, T: ST{ok: Str{"hello"}}}, 688 {K: text.Name}, 689 {K: text.Scalar, T: ST{ok: Str{"world"}}}, 690 {K: text.EOF}, 691 }, 692 }, 693 { 694 in: `foo:"hello"bar:"world"`, 695 want: []R{ 696 {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "foo"}}, 697 {K: text.Scalar, T: ST{ok: Str{"hello"}}}, 698 {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "bar"}}, 699 {K: text.Scalar, T: ST{ok: Str{"world"}}}, 700 {K: text.EOF}, 701 }, 702 }, 703 { 704 in: `foo:"hello"[bar]:"world"`, 705 want: []R{ 706 {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "foo"}}, 707 {K: text.Scalar, T: ST{ok: Str{"hello"}}}, 708 {K: text.Name, T: NT{K: text.TypeName, Sep: true, S: "bar"}}, 709 {K: text.Scalar, T: ST{ok: Str{"world"}}}, 710 {K: text.EOF}, 711 }, 712 }, 713 { 714 in: `name:"foo"` + space + `"bar"` + space + `'qux'`, 715 want: []R{ 716 {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}}, 717 {K: text.Scalar, T: ST{ok: Str{"foobarqux"}}}, 718 {K: text.EOF}, 719 }, 720 }, 721 { 722 in: `name:"foo"'bar'"qux"`, 723 want: []R{ 724 {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}}, 725 {K: text.Scalar, T: ST{ok: Str{"foobarqux"}}}, 726 {K: text.EOF}, 727 }, 728 }, 729 { 730 in: `name:"foo"` + space + `"bar" # comment` + "\n'qux' # comment", 731 want: []R{ 732 {K: text.Name, T: NT{K: text.IdentName, Sep: true, S: "name"}}, 733 {K: text.Scalar, T: ST{ok: Str{"foobarqux"}}}, 734 {K: text.EOF}, 735 }, 736 }, 737 738 // Lists. 739 { 740 in: `name: [`, 741 want: []R{ 742 {K: text.Name}, 743 {K: text.ListOpen}, 744 {E: eofErr}, 745 }, 746 }, 747 { 748 in: `name: []`, 749 want: []R{ 750 {K: text.Name}, 751 {K: text.ListOpen}, 752 {K: text.ListClose}, 753 {K: text.EOF}, 754 }, 755 }, 756 { 757 in: `name []`, 758 want: []R{ 759 {K: text.Name}, 760 {K: text.ListOpen}, 761 {K: text.ListClose}, 762 {K: text.EOF}, 763 }, 764 }, 765 { 766 in: `name: [,`, 767 want: []R{ 768 {K: text.Name}, 769 {K: text.ListOpen}, 770 {E: `(line 1:8): invalid scalar value: ,`}, 771 }, 772 }, 773 { 774 in: `name: [0`, 775 want: []R{ 776 {K: text.Name}, 777 {K: text.ListOpen}, 778 {K: text.Scalar}, 779 {E: eofErr}, 780 }, 781 }, 782 { 783 in: `name: [` + space + `"hello"` + space + `]` + space, 784 want: []R{ 785 {K: text.Name}, 786 {K: text.ListOpen}, 787 {K: text.Scalar, T: ST{ok: Str{"hello"}}, P: len(space) + 7}, 788 {K: text.ListClose}, 789 {K: text.EOF}, 790 }, 791 }, 792 { 793 in: `name: ["hello",]`, 794 want: []R{ 795 {K: text.Name}, 796 {K: text.ListOpen}, 797 {K: text.Scalar, T: ST{ok: Str{"hello"}}}, 798 {E: `invalid scalar value: ]`}, 799 }, 800 }, 801 { 802 in: `name: ["foo"` + space + `'bar' "qux"]`, 803 want: []R{ 804 {K: text.Name}, 805 {K: text.ListOpen}, 806 {K: text.Scalar, T: ST{ok: Str{"foobarqux"}}}, 807 {K: text.ListClose}, 808 {K: text.EOF}, 809 }, 810 }, 811 { 812 in: `name:` + space + `["foo",` + space + "'bar', # comment\n\n" + `"qux"]`, 813 want: []R{ 814 {K: text.Name}, 815 {K: text.ListOpen}, 816 {K: text.Scalar, T: ST{ok: Str{"foo"}}}, 817 {K: text.Scalar, T: ST{ok: Str{"bar"}}}, 818 {K: text.Scalar, T: ST{ok: Str{"qux"}}}, 819 {K: text.ListClose}, 820 {K: text.EOF}, 821 }, 822 }, 823 824 { 825 // List within list is not allowed. 826 in: `name: [[]]`, 827 want: []R{ 828 {K: text.Name}, 829 {K: text.ListOpen}, 830 {E: `syntax error (line 1:8): invalid scalar value: [`}, 831 }, 832 }, 833 { 834 // List items need to be separated by ,. 835 in: `name: ["foo" true]`, 836 want: []R{ 837 {K: text.Name}, 838 {K: text.ListOpen}, 839 {K: text.Scalar, T: ST{ok: Str{"foo"}}}, 840 {E: `syntax error (line 1:14): unexpected character 't'`}, 841 }, 842 }, 843 { 844 in: `name: ["foo"; "bar"]`, 845 want: []R{ 846 {K: text.Name}, 847 {K: text.ListOpen}, 848 {K: text.Scalar, T: ST{ok: Str{"foo"}}}, 849 {E: `syntax error (line 1:13): unexpected character ';'`}, 850 }, 851 }, 852 { 853 in: `name: ["foo", true, ENUM, 1.0]`, 854 want: []R{ 855 {K: text.Name}, 856 {K: text.ListOpen}, 857 {K: text.Scalar, T: ST{ok: Str{"foo"}}}, 858 {K: text.Scalar, T: ST{ok: Enum{"true"}}}, 859 {K: text.Scalar, T: ST{ok: Enum{"ENUM"}}}, 860 {K: text.Scalar, T: ST{ok: Float32{1.0}}}, 861 {K: text.ListClose}, 862 }, 863 }, 864 865 // Boolean literal values. 866 { 867 in: `name: True`, 868 want: []R{ 869 {K: text.Name}, 870 { 871 K: text.Scalar, 872 T: ST{ok: Bool{true}}, 873 }, 874 {K: text.EOF}, 875 }, 876 }, 877 { 878 in: `name false`, 879 want: []R{ 880 {K: text.Name}, 881 { 882 K: text.Scalar, 883 T: ST{ok: Bool{false}}, 884 }, 885 {K: text.EOF}, 886 }, 887 }, 888 { 889 in: `name: [t, f, True, False, true, false, 1, 0, 0x01, 0x00, 01, 00]`, 890 want: []R{ 891 {K: text.Name}, 892 {K: text.ListOpen}, 893 {K: text.Scalar, T: ST{ok: Bool{true}}}, 894 {K: text.Scalar, T: ST{ok: Bool{false}}}, 895 {K: text.Scalar, T: ST{ok: Bool{true}}}, 896 {K: text.Scalar, T: ST{ok: Bool{false}}}, 897 {K: text.Scalar, T: ST{ok: Bool{true}}}, 898 {K: text.Scalar, T: ST{ok: Bool{false}}}, 899 {K: text.Scalar, T: ST{ok: Bool{true}}}, 900 {K: text.Scalar, T: ST{ok: Bool{false}}}, 901 {K: text.Scalar, T: ST{ok: Bool{true}}}, 902 {K: text.Scalar, T: ST{ok: Bool{false}}}, 903 {K: text.Scalar, T: ST{ok: Bool{true}}}, 904 {K: text.Scalar, T: ST{ok: Bool{false}}}, 905 {K: text.ListClose}, 906 }, 907 }, 908 { 909 // Looks like boolean but not. 910 in: `name: [tRUe, falSE, -1, -0, -0x01, -0x00, -01, -00, 0.0]`, 911 want: []R{ 912 {K: text.Name}, 913 {K: text.ListOpen}, 914 {K: text.Scalar, T: ST{nok: Bool{}}}, 915 {K: text.Scalar, T: ST{nok: Bool{}}}, 916 {K: text.Scalar, T: ST{nok: Bool{}}}, 917 {K: text.Scalar, T: ST{nok: Bool{}}}, 918 {K: text.Scalar, T: ST{nok: Bool{}}}, 919 {K: text.Scalar, T: ST{nok: Bool{}}}, 920 {K: text.Scalar, T: ST{nok: Bool{}}}, 921 {K: text.Scalar, T: ST{nok: Bool{}}}, 922 {K: text.Scalar, T: ST{nok: Bool{}}}, 923 {K: text.ListClose}, 924 }, 925 }, 926 { 927 in: `foo: true[bar] false`, 928 want: []R{ 929 {K: text.Name}, 930 {K: text.Scalar, T: ST{ok: Bool{true}}}, 931 {K: text.Name}, 932 {K: text.Scalar, T: ST{ok: Bool{false}}}, 933 }, 934 }, 935 936 // Enum field values. 937 { 938 in: space + `name: ENUM`, 939 want: []R{ 940 {K: text.Name}, 941 {K: text.Scalar, T: ST{ok: Enum{"ENUM"}}}, 942 }, 943 }, 944 { 945 in: space + `name:[TRUE, FALSE, T, F, t, f]`, 946 want: []R{ 947 {K: text.Name}, 948 {K: text.ListOpen}, 949 {K: text.Scalar, T: ST{ok: Enum{"TRUE"}}}, 950 {K: text.Scalar, T: ST{ok: Enum{"FALSE"}}}, 951 {K: text.Scalar, T: ST{ok: Enum{"T"}}}, 952 {K: text.Scalar, T: ST{ok: Enum{"F"}}}, 953 {K: text.Scalar, T: ST{ok: Enum{"t"}}}, 954 {K: text.Scalar, T: ST{ok: Enum{"f"}}}, 955 {K: text.ListClose}, 956 }, 957 }, 958 { 959 in: `foo: Enum1[bar]:Enum2`, 960 want: []R{ 961 {K: text.Name}, 962 {K: text.Scalar, T: ST{ok: Enum{"Enum1"}}}, 963 {K: text.Name}, 964 {K: text.Scalar, T: ST{ok: Enum{"Enum2"}}}, 965 }, 966 }, 967 { 968 // Invalid enum values. 969 in: `name: [-inf, -foo, "string", 42, 1.0, 0x47]`, 970 want: []R{ 971 {K: text.Name}, 972 {K: text.ListOpen}, 973 {K: text.Scalar, T: ST{nok: Enum{}}}, 974 {K: text.Scalar, T: ST{nok: Enum{}}}, 975 {K: text.Scalar, T: ST{nok: Enum{}}}, 976 {K: text.Scalar, T: ST{nok: Enum{}}}, 977 {K: text.Scalar, T: ST{nok: Enum{}}}, 978 {K: text.Scalar, T: ST{nok: Enum{}}}, 979 {K: text.ListClose}, 980 }, 981 }, 982 { 983 in: `name: true.`, 984 want: []R{ 985 {K: text.Name}, 986 {E: `invalid scalar value: true.`}, 987 }, 988 }, 989 990 // Numeric values. 991 { 992 in: `nums:42 nums:0x2A nums:052`, 993 want: []R{ 994 {K: text.Name}, 995 {K: text.Scalar, T: ST{ok: Uint64{42}}}, 996 {K: text.Name}, 997 {K: text.Scalar, T: ST{ok: Uint64{42}}}, 998 {K: text.Name}, 999 {K: text.Scalar, T: ST{ok: Uint64{42}}}, 1000 }, 1001 }, 1002 { 1003 in: `nums:[-42, -0x2a, -052]`, 1004 want: []R{ 1005 {K: text.Name}, 1006 {K: text.ListOpen}, 1007 {K: text.Scalar, T: ST{nok: Uint64{}}}, 1008 {K: text.Scalar, T: ST{nok: Uint64{}}}, 1009 {K: text.Scalar, T: ST{nok: Uint64{}}}, 1010 {K: text.ListClose}, 1011 }, 1012 }, 1013 { 1014 in: `nums:[-42, -0x2a, -052]`, 1015 want: []R{ 1016 {K: text.Name}, 1017 {K: text.ListOpen}, 1018 {K: text.Scalar, T: ST{ok: Int64{-42}}}, 1019 {K: text.Scalar, T: ST{ok: Int64{-42}}}, 1020 {K: text.Scalar, T: ST{ok: Int64{-42}}}, 1021 {K: text.ListClose}, 1022 }, 1023 }, 1024 { 1025 in: `nums: [0,0x0,00,-9876543210,9876543210,0x0123456789abcdef,-0x0123456789abcdef,01234567,-01234567]`, 1026 want: []R{ 1027 {K: text.Name}, 1028 {K: text.ListOpen}, 1029 {K: text.Scalar, T: ST{ok: Uint64{0}}}, 1030 {K: text.Scalar, T: ST{ok: Int64{0}}}, 1031 {K: text.Scalar, T: ST{ok: Uint64{0}}}, 1032 {K: text.Scalar, T: ST{ok: Int64{-9876543210}}}, 1033 {K: text.Scalar, T: ST{ok: Uint64{9876543210}}}, 1034 {K: text.Scalar, T: ST{ok: Uint64{0x0123456789abcdef}}}, 1035 {K: text.Scalar, T: ST{ok: Int64{-0x0123456789abcdef}}}, 1036 {K: text.Scalar, T: ST{ok: Uint64{01234567}}}, 1037 {K: text.Scalar, T: ST{ok: Int64{-01234567}}}, 1038 {K: text.ListClose}, 1039 }, 1040 }, 1041 { 1042 in: `nums: [0,0x0,00,-876543210,876543210,0x01234,-0x01234,01234567,-01234567]`, 1043 want: []R{ 1044 {K: text.Name}, 1045 {K: text.ListOpen}, 1046 {K: text.Scalar, T: ST{ok: Uint32{0}}}, 1047 {K: text.Scalar, T: ST{ok: Int32{0}}}, 1048 {K: text.Scalar, T: ST{ok: Uint32{0}}}, 1049 {K: text.Scalar, T: ST{ok: Int32{-876543210}}}, 1050 {K: text.Scalar, T: ST{ok: Uint32{876543210}}}, 1051 {K: text.Scalar, T: ST{ok: Uint32{0x01234}}}, 1052 {K: text.Scalar, T: ST{ok: Int32{-0x01234}}}, 1053 {K: text.Scalar, T: ST{ok: Uint32{01234567}}}, 1054 {K: text.Scalar, T: ST{ok: Int32{-01234567}}}, 1055 {K: text.ListClose}, 1056 }, 1057 }, 1058 { 1059 in: `nums: [` + 1060 fmt.Sprintf("%d", uint64(math.MaxUint64)) + `,` + 1061 fmt.Sprintf("%d", uint32(math.MaxUint32)) + `,` + 1062 fmt.Sprintf("%d", int64(math.MaxInt64)) + `,` + 1063 fmt.Sprintf("%d", int64(math.MinInt64)) + `,` + 1064 fmt.Sprintf("%d", int32(math.MaxInt32)) + `,` + 1065 fmt.Sprintf("%d", int32(math.MinInt32)) + 1066 `]`, 1067 want: []R{ 1068 {K: text.Name}, 1069 {K: text.ListOpen}, 1070 {K: text.Scalar, T: ST{ok: Uint64{math.MaxUint64}}}, 1071 {K: text.Scalar, T: ST{ok: Uint32{math.MaxUint32}}}, 1072 {K: text.Scalar, T: ST{ok: Int64{math.MaxInt64}}}, 1073 {K: text.Scalar, T: ST{ok: Int64{math.MinInt64}}}, 1074 {K: text.Scalar, T: ST{ok: Int32{math.MaxInt32}}}, 1075 {K: text.Scalar, T: ST{ok: Int32{math.MinInt32}}}, 1076 {K: text.ListClose}, 1077 }, 1078 }, 1079 { 1080 // Integer exceeds range. 1081 in: `nums: [` + 1082 `18446744073709551616,` + // max uint64 + 1 1083 fmt.Sprintf("%d", uint64(math.MaxUint32+1)) + `,` + 1084 fmt.Sprintf("%d", uint64(math.MaxInt64+1)) + `,` + 1085 `-9223372036854775809,` + // min int64 - 1 1086 fmt.Sprintf("%d", uint64(math.MaxInt32+1)) + `,` + 1087 fmt.Sprintf("%d", int64(math.MinInt32-1)) + `` + 1088 `]`, 1089 want: []R{ 1090 {K: text.Name}, 1091 {K: text.ListOpen}, 1092 {K: text.Scalar, T: ST{nok: Uint64{}}}, 1093 {K: text.Scalar, T: ST{nok: Uint32{}}}, 1094 {K: text.Scalar, T: ST{nok: Int64{}}}, 1095 {K: text.Scalar, T: ST{nok: Int64{}}}, 1096 {K: text.Scalar, T: ST{nok: Int32{}}}, 1097 {K: text.Scalar, T: ST{nok: Int32{}}}, 1098 {K: text.ListClose}, 1099 }, 1100 }, 1101 { 1102 in: `nums: [0xbeefbeef, 0xbeefbeefbeefbeef]`, 1103 want: []R{ 1104 {K: text.Name}, 1105 {K: text.ListOpen}, 1106 { 1107 K: text.Scalar, 1108 T: func() ST { 1109 if flags.ProtoLegacy { 1110 return ST{ok: Int32{-1091584273}} 1111 } 1112 return ST{nok: Int32{}} 1113 }(), 1114 }, 1115 { 1116 K: text.Scalar, 1117 T: func() ST { 1118 if flags.ProtoLegacy { 1119 return ST{ok: Int64{-4688318750159552785}} 1120 } 1121 return ST{nok: Int64{}} 1122 }(), 1123 }, 1124 {K: text.ListClose}, 1125 }, 1126 }, 1127 { 1128 in: `nums: [0.,0f,1f,10f,-0f,-1f,-10f,1.0,0.1e-3,1.5e+5,1e10,.0]`, 1129 want: []R{ 1130 {K: text.Name}, 1131 {K: text.ListOpen}, 1132 {K: text.Scalar, T: ST{ok: Float64{0.0}}}, 1133 {K: text.Scalar, T: ST{ok: Float64{0.0}}}, 1134 {K: text.Scalar, T: ST{ok: Float64{1.0}}}, 1135 {K: text.Scalar, T: ST{ok: Float64{10.0}}}, 1136 {K: text.Scalar, T: ST{ok: Float64{math.Copysign(0, -1)}}}, 1137 {K: text.Scalar, T: ST{ok: Float64{-1.0}}}, 1138 {K: text.Scalar, T: ST{ok: Float64{-10.0}}}, 1139 {K: text.Scalar, T: ST{ok: Float64{1.0}}}, 1140 {K: text.Scalar, T: ST{ok: Float64{0.1e-3}}}, 1141 {K: text.Scalar, T: ST{ok: Float64{1.5e+5}}}, 1142 {K: text.Scalar, T: ST{ok: Float64{1.0e+10}}}, 1143 {K: text.Scalar, T: ST{ok: Float64{0.0}}}, 1144 {K: text.ListClose}, 1145 }, 1146 }, 1147 { 1148 in: `nums: [0.,0f,1f,10f,-0f,-1f,-10f,1.0,0.1e-3,1.5e+5,1e10,.0]`, 1149 want: []R{ 1150 {K: text.Name}, 1151 {K: text.ListOpen}, 1152 {K: text.Scalar, T: ST{ok: Float32{0.0}}}, 1153 {K: text.Scalar, T: ST{ok: Float32{0.0}}}, 1154 {K: text.Scalar, T: ST{ok: Float32{1.0}}}, 1155 {K: text.Scalar, T: ST{ok: Float32{10.0}}}, 1156 {K: text.Scalar, T: ST{ok: Float32{float32(math.Copysign(0, -1))}}}, 1157 {K: text.Scalar, T: ST{ok: Float32{-1.0}}}, 1158 {K: text.Scalar, T: ST{ok: Float32{-10.0}}}, 1159 {K: text.Scalar, T: ST{ok: Float32{1.0}}}, 1160 {K: text.Scalar, T: ST{ok: Float32{0.1e-3}}}, 1161 {K: text.Scalar, T: ST{ok: Float32{1.5e+5}}}, 1162 {K: text.Scalar, T: ST{ok: Float32{1.0e+10}}}, 1163 {K: text.Scalar, T: ST{ok: Float32{0.0}}}, 1164 {K: text.ListClose}, 1165 }, 1166 }, 1167 { 1168 in: `nums: [0.,1f,10F,1e1,1.10]`, 1169 want: []R{ 1170 {K: text.Name}, 1171 {K: text.ListOpen}, 1172 {K: text.Scalar, T: ST{nok: Int64{}}}, 1173 {K: text.Scalar, T: ST{nok: Int64{}}}, 1174 {K: text.Scalar, T: ST{nok: Int64{}}}, 1175 {K: text.Scalar, T: ST{nok: Int64{}}}, 1176 {K: text.Scalar, T: ST{nok: Int64{}}}, 1177 {K: text.ListClose}, 1178 }, 1179 }, 1180 { 1181 in: `nums: [0.,1f,10F,1e1,1.10]`, 1182 want: []R{ 1183 {K: text.Name}, 1184 {K: text.ListOpen}, 1185 {K: text.Scalar, T: ST{nok: Int32{}}}, 1186 {K: text.Scalar, T: ST{nok: Int32{}}}, 1187 {K: text.Scalar, T: ST{nok: Int32{}}}, 1188 {K: text.Scalar, T: ST{nok: Int32{}}}, 1189 {K: text.Scalar, T: ST{nok: Int32{}}}, 1190 {K: text.ListClose}, 1191 }, 1192 }, 1193 { 1194 in: `nums: [0.,1f,10F,1e1,1.10]`, 1195 want: []R{ 1196 {K: text.Name}, 1197 {K: text.ListOpen}, 1198 {K: text.Scalar, T: ST{nok: Uint64{}}}, 1199 {K: text.Scalar, T: ST{nok: Uint64{}}}, 1200 {K: text.Scalar, T: ST{nok: Uint64{}}}, 1201 {K: text.Scalar, T: ST{nok: Uint64{}}}, 1202 {K: text.Scalar, T: ST{nok: Uint64{}}}, 1203 {K: text.ListClose}, 1204 }, 1205 }, 1206 { 1207 in: `nums: [0.,1f,10F,1e1,1.10]`, 1208 want: []R{ 1209 {K: text.Name}, 1210 {K: text.ListOpen}, 1211 {K: text.Scalar, T: ST{nok: Uint32{}}}, 1212 {K: text.Scalar, T: ST{nok: Uint32{}}}, 1213 {K: text.Scalar, T: ST{nok: Uint32{}}}, 1214 {K: text.Scalar, T: ST{nok: Uint32{}}}, 1215 {K: text.Scalar, T: ST{nok: Uint32{}}}, 1216 {K: text.ListClose}, 1217 }, 1218 }, 1219 { 1220 in: `nums: [` + 1221 fmt.Sprintf("%g", math.MaxFloat32) + `,` + 1222 fmt.Sprintf("%g", -math.MaxFloat32) + `,` + 1223 fmt.Sprintf("%g", math.MaxFloat32*2) + `,` + 1224 fmt.Sprintf("%g", -math.MaxFloat32*2) + `,` + 1225 `3.59539e+308,` + // math.MaxFloat64 * 2 1226 `-3.59539e+308,` + // -math.MaxFloat64 * 2 1227 fmt.Sprintf("%d000", uint64(math.MaxUint64)) + 1228 `]`, 1229 want: []R{ 1230 {K: text.Name}, 1231 {K: text.ListOpen}, 1232 {K: text.Scalar, T: ST{ok: Float32{float32(math.MaxFloat32)}}}, 1233 {K: text.Scalar, T: ST{ok: Float32{float32(-math.MaxFloat32)}}}, 1234 {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}}, 1235 {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}}, 1236 {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}}, 1237 {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}}, 1238 {K: text.Scalar, T: ST{ok: Float32{float32(math.MaxUint64) * 1000}}}, 1239 {K: text.ListClose}, 1240 }, 1241 }, 1242 { 1243 in: `nums: [` + 1244 fmt.Sprintf("%g", math.MaxFloat64) + `,` + 1245 fmt.Sprintf("%g", -math.MaxFloat64) + `,` + 1246 `3.59539e+308,` + // math.MaxFloat64 * 2 1247 `-3.59539e+308,` + // -math.MaxFloat64 * 2 1248 fmt.Sprintf("%d000", uint64(math.MaxUint64)) + 1249 `]`, 1250 want: []R{ 1251 {K: text.Name}, 1252 {K: text.ListOpen}, 1253 {K: text.Scalar, T: ST{ok: Float64{math.MaxFloat64}}}, 1254 {K: text.Scalar, T: ST{ok: Float64{-math.MaxFloat64}}}, 1255 {K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}}, 1256 {K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}}, 1257 {K: text.Scalar, T: ST{ok: Float64{float64(math.MaxUint64) * 1000}}}, 1258 {K: text.ListClose}, 1259 }, 1260 }, 1261 { 1262 // -0 is only valid for signed types. It is not valid for unsigned types. 1263 in: `num: [-0, -0]`, 1264 want: []R{ 1265 {K: text.Name}, 1266 {K: text.ListOpen}, 1267 {K: text.Scalar, T: ST{nok: Uint32{}}}, 1268 {K: text.Scalar, T: ST{nok: Uint64{}}}, 1269 {K: text.ListClose}, 1270 }, 1271 }, 1272 { 1273 // -0 is only valid for signed types. It is not valid for unsigned types. 1274 in: `num: [-0, -0]`, 1275 want: []R{ 1276 {K: text.Name}, 1277 {K: text.ListOpen}, 1278 {K: text.Scalar, T: ST{ok: Int32{0}}}, 1279 {K: text.Scalar, T: ST{ok: Int64{0}}}, 1280 {K: text.ListClose}, 1281 }, 1282 }, 1283 { 1284 // Negative zeros on float64 should preserve sign bit. 1285 in: `num: [-0, -.0]`, 1286 want: []R{ 1287 {K: text.Name}, 1288 {K: text.ListOpen}, 1289 {K: text.Scalar, T: ST{ok: Float64{math.Copysign(0, -1)}}}, 1290 {K: text.Scalar, T: ST{ok: Float64{math.Copysign(0, -1)}}}, 1291 {K: text.ListClose}, 1292 }, 1293 }, 1294 { 1295 // Negative zeros on float32 should preserve sign bit. 1296 in: `num: [-0, -.0]`, 1297 want: []R{ 1298 {K: text.Name}, 1299 {K: text.ListOpen}, 1300 {K: text.Scalar, T: ST{ok: Float32{float32(math.Copysign(0, -1))}}}, 1301 {K: text.Scalar, T: ST{ok: Float32{float32(math.Copysign(0, -1))}}}, 1302 {K: text.ListClose}, 1303 }, 1304 }, 1305 { 1306 in: `num: +0`, 1307 want: []R{ 1308 {K: text.Name}, 1309 {E: `invalid scalar value: +`}, 1310 }, 1311 }, 1312 { 1313 in: `num: 01.1234`, 1314 want: []R{ 1315 {K: text.Name}, 1316 {E: `invalid scalar value: 01.1234`}, 1317 }, 1318 }, 1319 { 1320 in: `num: 0x`, 1321 want: []R{ 1322 {K: text.Name}, 1323 {E: `invalid scalar value: 0x`}, 1324 }, 1325 }, 1326 { 1327 in: `num: 0xX`, 1328 want: []R{ 1329 {K: text.Name}, 1330 {E: `invalid scalar value: 0xX`}, 1331 }, 1332 }, 1333 { 1334 in: `num: 0800`, 1335 want: []R{ 1336 {K: text.Name}, 1337 {E: `invalid scalar value: 0800`}, 1338 }, 1339 }, 1340 { 1341 in: `num: 1.`, 1342 want: []R{ 1343 {K: text.Name}, 1344 {K: text.Scalar, T: ST{ok: Float32{1.0}}}, 1345 }, 1346 }, 1347 { 1348 in: `num: -.`, 1349 want: []R{ 1350 {K: text.Name}, 1351 {E: `invalid scalar value: -.`}, 1352 }, 1353 }, 1354 1355 // Float special literal values, case-insensitive match. 1356 { 1357 in: `name:[nan, NaN, Nan, NAN]`, 1358 want: []R{ 1359 {K: text.Name}, 1360 {K: text.ListOpen}, 1361 {K: text.Scalar, T: ST{ok: Float64{math.NaN()}}}, 1362 {K: text.Scalar, T: ST{ok: Float64{math.NaN()}}}, 1363 {K: text.Scalar, T: ST{ok: Float64{math.NaN()}}}, 1364 {K: text.Scalar, T: ST{ok: Float64{math.NaN()}}}, 1365 {K: text.ListClose}, 1366 }, 1367 }, 1368 { 1369 in: `name:[inf, INF, infinity, Infinity, INFinity]`, 1370 want: []R{ 1371 {K: text.Name}, 1372 {K: text.ListOpen}, 1373 {K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}}, 1374 {K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}}, 1375 {K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}}, 1376 {K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}}, 1377 {K: text.Scalar, T: ST{ok: Float64{math.Inf(1)}}}, 1378 {K: text.ListClose}, 1379 }, 1380 }, 1381 { 1382 in: `name:[-inf, -INF, -infinity, -Infinity, -INFinity]`, 1383 want: []R{ 1384 {K: text.Name}, 1385 {K: text.ListOpen}, 1386 {K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}}, 1387 {K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}}, 1388 {K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}}, 1389 {K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}}, 1390 {K: text.Scalar, T: ST{ok: Float64{math.Inf(-1)}}}, 1391 {K: text.ListClose}, 1392 }, 1393 }, 1394 { 1395 in: `name:[nan, NaN, Nan, NAN]`, 1396 want: []R{ 1397 {K: text.Name}, 1398 {K: text.ListOpen}, 1399 {K: text.Scalar, T: ST{ok: Float32{float32(math.NaN())}}}, 1400 {K: text.Scalar, T: ST{ok: Float32{float32(math.NaN())}}}, 1401 {K: text.Scalar, T: ST{ok: Float32{float32(math.NaN())}}}, 1402 {K: text.Scalar, T: ST{ok: Float32{float32(math.NaN())}}}, 1403 {K: text.ListClose}, 1404 }, 1405 }, 1406 { 1407 in: `name:[inf, INF, infinity, Infinity, INFinity]`, 1408 want: []R{ 1409 {K: text.Name}, 1410 {K: text.ListOpen}, 1411 {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}}, 1412 {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}}, 1413 {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}}, 1414 {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}}, 1415 {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(1))}}}, 1416 {K: text.ListClose}, 1417 }, 1418 }, 1419 { 1420 in: `name:[-inf, -INF, -infinity, -Infinity, -INFinity]`, 1421 want: []R{ 1422 {K: text.Name}, 1423 {K: text.ListOpen}, 1424 {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}}, 1425 {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}}, 1426 {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}}, 1427 {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}}, 1428 {K: text.Scalar, T: ST{ok: Float32{float32(math.Inf(-1))}}}, 1429 {K: text.ListClose}, 1430 }, 1431 }, 1432 { 1433 // C++ permits this, but we currently reject this. It is easy to add 1434 // if needed. 1435 in: `name: -nan`, 1436 want: []R{ 1437 {K: text.Name}, 1438 {K: text.Scalar, T: ST{nok: Float64{}}}, 1439 }, 1440 }, 1441 // Messages. 1442 { 1443 in: `m: {}`, 1444 want: []R{ 1445 {K: text.Name}, 1446 {K: text.MessageOpen}, 1447 {K: text.MessageClose}, 1448 {K: text.EOF}, 1449 }, 1450 }, 1451 { 1452 in: `m: <>`, 1453 want: []R{ 1454 {K: text.Name}, 1455 {K: text.MessageOpen}, 1456 {K: text.MessageClose}, 1457 {K: text.EOF}, 1458 }, 1459 }, 1460 { 1461 in: space + `m {` + space + "\n# comment\n" + `}` + space, 1462 want: []R{ 1463 {K: text.Name}, 1464 {K: text.MessageOpen}, 1465 {K: text.MessageClose}, 1466 }, 1467 }, 1468 { 1469 in: `m { foo: < bar: "hello" > }`, 1470 want: []R{ 1471 {K: text.Name, RS: "m"}, 1472 {K: text.MessageOpen}, 1473 1474 {K: text.Name, RS: "foo"}, 1475 {K: text.MessageOpen}, 1476 1477 {K: text.Name, RS: "bar"}, 1478 {K: text.Scalar, T: ST{ok: Str{"hello"}}}, 1479 1480 {K: text.MessageClose}, 1481 1482 {K: text.MessageClose}, 1483 }, 1484 }, 1485 { 1486 in: `list [ <s:"hello">, {s:"world"} ]`, 1487 want: []R{ 1488 {K: text.Name, RS: "list"}, 1489 {K: text.ListOpen}, 1490 1491 {K: text.MessageOpen}, 1492 {K: text.Name, RS: "s"}, 1493 {K: text.Scalar, T: ST{ok: Str{"hello"}}}, 1494 {K: text.MessageClose}, 1495 1496 {K: text.MessageOpen}, 1497 {K: text.Name, RS: "s"}, 1498 {K: text.Scalar, T: ST{ok: Str{"world"}}}, 1499 {K: text.MessageClose}, 1500 1501 {K: text.ListClose}, 1502 {K: text.EOF}, 1503 }, 1504 }, 1505 { 1506 in: `m: { >`, 1507 want: []R{ 1508 {K: text.Name}, 1509 {K: text.MessageOpen}, 1510 {E: `mismatched close character '>'`}, 1511 }, 1512 }, 1513 { 1514 in: `m: <s: "hello"}`, 1515 want: []R{ 1516 {K: text.Name}, 1517 {K: text.MessageOpen}, 1518 1519 {K: text.Name}, 1520 {K: text.Scalar, T: ST{ok: Str{"hello"}}}, 1521 1522 {E: `mismatched close character '}'`}, 1523 }, 1524 }, 1525 { 1526 in: `{}`, 1527 want: []R{{E: `invalid field name: {`}}, 1528 }, 1529 { 1530 in: ` 1531m: { 1532 foo: true; 1533 bar: { 1534 enum: ENUM 1535 list: [ < >, { } ] ; 1536 } 1537 [qux]: "end" 1538} 1539 `, 1540 want: []R{ 1541 {K: text.Name}, 1542 {K: text.MessageOpen}, 1543 1544 {K: text.Name, RS: "foo"}, 1545 {K: text.Scalar, T: ST{ok: Bool{true}}}, 1546 1547 {K: text.Name, RS: "bar"}, 1548 {K: text.MessageOpen}, 1549 1550 {K: text.Name, RS: "enum"}, 1551 {K: text.Scalar, T: ST{ok: Enum{"ENUM"}}}, 1552 1553 {K: text.Name, RS: "list"}, 1554 {K: text.ListOpen}, 1555 {K: text.MessageOpen}, 1556 {K: text.MessageClose}, 1557 {K: text.MessageOpen}, 1558 {K: text.MessageClose}, 1559 {K: text.ListClose}, 1560 1561 {K: text.MessageClose}, 1562 1563 {K: text.Name, RS: "[qux]"}, 1564 {K: text.Scalar, T: ST{ok: Str{"end"}}}, 1565 1566 {K: text.MessageClose}, 1567 {K: text.EOF}, 1568 }, 1569 }, 1570 1571 // Other syntax errors. 1572 { 1573 in: "x: -", 1574 want: []R{ 1575 {K: text.Name}, 1576 {E: `syntax error (line 1:4): invalid scalar value: -`}, 1577 }, 1578 }, 1579 { 1580 in: "x:[\"��\"x", 1581 want: []R{ 1582 {K: text.Name}, 1583 {K: text.ListOpen}, 1584 {K: text.Scalar, T: ST{ok: Str{"��"}}, P: 3}, 1585 {E: `syntax error (line 1:7)`}, 1586 }, 1587 }, 1588 { 1589 in: "x:\n\n[\"������\"x", 1590 want: []R{ 1591 {K: text.Name}, 1592 {K: text.ListOpen}, 1593 {K: text.Scalar, T: ST{ok: Str{"������"}}, P: 5}, 1594 {E: `syntax error (line 3:7)`}, 1595 }, 1596 }, 1597 { 1598 // multi-rune emojis; could be column:8 1599 in: "x:[\"��������\"x", 1600 want: []R{ 1601 {K: text.Name}, 1602 {K: text.ListOpen}, 1603 {K: text.Scalar, T: ST{ok: Str{"��������"}}, P: 3}, 1604 {E: `syntax error (line 1:10)`}, 1605 }, 1606 }, 1607 } 1608 1609 for _, tc := range tests { 1610 t.Run("", func(t *testing.T) { 1611 tc := tc 1612 in := []byte(tc.in) 1613 dec := text.NewDecoder(in[:len(in):len(in)]) 1614 for i, want := range tc.want { 1615 peekTok, peekErr := dec.Peek() 1616 tok, err := dec.Read() 1617 if err != nil { 1618 if want.E == "" { 1619 errorf(t, tc.in, "Read() got unexpected error: %v", err) 1620 } else if !strings.Contains(err.Error(), want.E) { 1621 errorf(t, tc.in, "Read() got %q, want %q", err, want.E) 1622 } 1623 return 1624 } 1625 if want.E != "" { 1626 errorf(t, tc.in, "Read() got nil error, want %q", want.E) 1627 return 1628 } 1629 gotK := tok.Kind() 1630 if gotK != want.K { 1631 errorf(t, tc.in, "Read() got %v, want %v", gotK, want.K) 1632 return 1633 } 1634 checkToken(t, tok, i, want, tc.in) 1635 if !cmp.Equal(tok, peekTok, cmp.Comparer(text.TokenEquals)) { 1636 errorf(t, tc.in, "Peek() %+v != Read() token %+v", peekTok, tok) 1637 } 1638 if err != peekErr { 1639 errorf(t, tc.in, "Peek() error %v != Read() error %v", err, peekErr) 1640 } 1641 } 1642 }) 1643 } 1644} 1645 1646func checkToken(t *testing.T, tok text.Token, idx int, r R, in string) { 1647 // Validate Token.Pos() if R.P is set. 1648 if r.P > 0 { 1649 got := tok.Pos() 1650 if got != r.P { 1651 errorf(t, in, "want#%d: Token.Pos() got %v want %v", idx, got, r.P) 1652 } 1653 } 1654 1655 // Validate Token.RawString if R.RS is set. 1656 if len(r.RS) > 0 { 1657 got := tok.RawString() 1658 if got != r.RS { 1659 errorf(t, in, "want#%d: Token.RawString() got %v want %v", idx, got, r.P) 1660 } 1661 } 1662 1663 // Skip checking for Token details if r.T is not set. 1664 if r.T == nil { 1665 return 1666 } 1667 1668 switch tok.Kind() { 1669 case text.Name: 1670 want := r.T.(NT) 1671 kind := tok.NameKind() 1672 if kind != want.K { 1673 errorf(t, in, "want#%d: Token.NameKind() got %v want %v", idx, kind, want.K) 1674 return 1675 } 1676 switch kind { 1677 case text.IdentName: 1678 got := tok.IdentName() 1679 if got != want.S { 1680 errorf(t, in, "want#%d: Token.IdentName() got %v want %v", idx, got, want.S) 1681 } 1682 case text.TypeName: 1683 got := tok.TypeName() 1684 if got != want.S { 1685 errorf(t, in, "want#%d: Token.TypeName() got %v want %v", idx, got, want.S) 1686 } 1687 case text.FieldNumber: 1688 got := tok.FieldNumber() 1689 if got != want.N { 1690 errorf(t, in, "want#%d: Token.FieldNumber() got %v want %v", idx, got, want.N) 1691 } 1692 } 1693 1694 case text.Scalar: 1695 want := r.T.(ST) 1696 if ok := want.ok; ok != nil { 1697 if err := ok.checkOk(tok); err != "" { 1698 errorf(t, in, "want#%d: %s", idx, err) 1699 } 1700 } 1701 if nok := want.nok; nok != nil { 1702 if err := nok.checkNok(tok); err != "" { 1703 errorf(t, in, "want#%d: %s", idx, err) 1704 } 1705 } 1706 } 1707} 1708 1709func errorf(t *testing.T, in string, fmtStr string, args ...interface{}) { 1710 t.Helper() 1711 vargs := []interface{}{in} 1712 for _, arg := range args { 1713 vargs = append(vargs, arg) 1714 } 1715 t.Errorf("input:\n%s\n~end~\n"+fmtStr, vargs...) 1716} 1717 1718func TestUnmarshalString(t *testing.T) { 1719 tests := []struct { 1720 in string 1721 // want is expected string result. 1722 want string 1723 // err is expected error substring from calling DecodeString if set. 1724 err string 1725 }{ 1726 { 1727 in: func() string { 1728 var b []byte 1729 for i := 0; i < utf8.RuneSelf; i++ { 1730 switch i { 1731 case 0, '\\', '\n', '\'': // these must be escaped, so ignore them 1732 default: 1733 b = append(b, byte(i)) 1734 } 1735 } 1736 return "'" + string(b) + "'" 1737 }(), 1738 want: "\x01\x02\x03\x04\x05\x06\a\b\t\v\f\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[]^_`abcdefghijklmnopqrstuvwxyz{|}~\u007f", 1739 }, 1740 { 1741 in: "'\xde\xad\xbe\xef'", 1742 err: `invalid UTF-8 detected`, 1743 }, 1744 { 1745 // Valid UTF-8 wire encoding, but sub-optimal encoding. 1746 in: "'\xc0\x80'", 1747 err: "invalid UTF-8 detected", 1748 }, 1749 { 1750 // Valid UTF-8 wire encoding, but invalid rune (surrogate pair). 1751 in: "'\xed\xa0\x80'", 1752 err: "invalid UTF-8 detected", 1753 }, 1754 { 1755 // Valid UTF-8 wire encoding, but invalid rune (above max rune). 1756 in: "'\xf7\xbf\xbf\xbf'", 1757 err: "invalid UTF-8 detected", 1758 }, 1759 { 1760 // Valid UTF-8 wire encoding of the RuneError rune. 1761 in: "'\xef\xbf\xbd'", 1762 want: string(utf8.RuneError), 1763 }, 1764 { 1765 in: "'hello\u1234world'", 1766 want: "hello\u1234world", 1767 }, 1768 { 1769 in: `'\"\'\\\?\a\b\n\r\t\v\f\1\12\123\xA\xaB\x12\uAb8f\U0010FFFF'`, 1770 want: "\"'\\?\a\b\n\r\t\v\f\x01\nS\n\xab\x12\uab8f\U0010ffff", 1771 }, 1772 { 1773 in: `str: '\8'`, 1774 err: `invalid escape code "\\8" in string`, 1775 }, 1776 { 1777 in: `'\1x'`, 1778 want: "\001x", 1779 }, 1780 { 1781 in: `'\12x'`, 1782 want: "\012x", 1783 }, 1784 { 1785 in: `'\123x'`, 1786 want: "\123x", 1787 }, 1788 { 1789 in: `'\1234x'`, 1790 want: "\1234x", 1791 }, 1792 { 1793 in: `'\1'`, 1794 want: "\001", 1795 }, 1796 { 1797 in: `'\12'`, 1798 want: "\012", 1799 }, 1800 { 1801 in: `'\123'`, 1802 want: "\123", 1803 }, 1804 { 1805 in: `'\1234'`, 1806 want: "\1234", 1807 }, 1808 { 1809 in: `'\377'`, 1810 want: "\377", 1811 }, 1812 { 1813 // Overflow octal escape. 1814 in: `'\400'`, 1815 err: `invalid octal escape code "\\400" in string`, 1816 }, 1817 { 1818 in: `'\xfx'`, 1819 want: "\x0fx", 1820 }, 1821 { 1822 in: `'\xffx'`, 1823 want: "\xffx", 1824 }, 1825 { 1826 in: `'\xfffx'`, 1827 want: "\xfffx", 1828 }, 1829 { 1830 in: `'\xf'`, 1831 want: "\x0f", 1832 }, 1833 { 1834 in: `'\xff'`, 1835 want: "\xff", 1836 }, 1837 { 1838 in: `'\xfff'`, 1839 want: "\xfff", 1840 }, 1841 { 1842 in: `'\xz'`, 1843 err: `invalid hex escape code "\\x" in string`, 1844 }, 1845 { 1846 in: `'\uPo'`, 1847 err: eofErr, 1848 }, 1849 { 1850 in: `'\uPoo'`, 1851 err: `invalid Unicode escape code "\\uPoo'" in string`, 1852 }, 1853 { 1854 in: `str: '\uPoop'`, 1855 err: `invalid Unicode escape code "\\uPoop" in string`, 1856 }, 1857 { 1858 // Unmatched surrogate pair. 1859 in: `str: '\uDEAD'`, 1860 err: `unexpected EOF`, // trying to reader other half 1861 }, 1862 { 1863 // Surrogate pair with invalid other half. 1864 in: `str: '\uDEAD\u0000'`, 1865 err: `invalid Unicode escape code "\\u0000" in string`, 1866 }, 1867 { 1868 // Properly matched surrogate pair. 1869 in: `'\uD800\uDEAD'`, 1870 want: "��", 1871 }, 1872 { 1873 // Overflow on Unicode rune. 1874 in: `'\U00110000'`, 1875 err: `invalid Unicode escape code "\\U00110000" in string`, 1876 }, 1877 { 1878 in: `'\z'`, 1879 err: `invalid escape code "\\z" in string`, 1880 }, 1881 { 1882 // Strings cannot have NUL literal since C-style strings forbid them. 1883 in: "'\x00'", 1884 err: `invalid character '\x00' in string`, 1885 }, 1886 { 1887 // Strings cannot have newline literal. The C++ permits them if an 1888 // option is specified to allow them. In Go, we always forbid them. 1889 in: "'\n'", 1890 err: `invalid character '\n' in string`, 1891 }, 1892 } 1893 1894 for _, tc := range tests { 1895 t.Run("", func(t *testing.T) { 1896 got, err := text.UnmarshalString(tc.in) 1897 if err != nil { 1898 if tc.err == "" { 1899 errorf(t, tc.in, "UnmarshalString() got unexpected error: %q", err) 1900 } else if !strings.Contains(err.Error(), tc.err) { 1901 errorf(t, tc.in, "UnmarshalString() error got %q, want %q", err, tc.err) 1902 } 1903 return 1904 } 1905 if tc.err != "" { 1906 errorf(t, tc.in, "UnmarshalString() got nil error, want %q", tc.err) 1907 return 1908 } 1909 if got != tc.want { 1910 errorf(t, tc.in, "UnmarshalString()\n[got]\n%s\n[want]\n%s", got, tc.want) 1911 } 1912 }) 1913 } 1914} 1915 1916// Tests line and column number produced by Decoder.Position. 1917func TestPosition(t *testing.T) { 1918 dec := text.NewDecoder([]byte("0123456789\n12345\n789")) 1919 1920 tests := []struct { 1921 pos int 1922 row int 1923 col int 1924 }{ 1925 { 1926 pos: 0, 1927 row: 1, 1928 col: 1, 1929 }, 1930 { 1931 pos: 10, 1932 row: 1, 1933 col: 11, 1934 }, 1935 { 1936 pos: 11, 1937 row: 2, 1938 col: 1, 1939 }, 1940 { 1941 pos: 18, 1942 row: 3, 1943 col: 2, 1944 }, 1945 } 1946 1947 for _, tc := range tests { 1948 t.Run("", func(t *testing.T) { 1949 row, col := dec.Position(tc.pos) 1950 if row != tc.row || col != tc.col { 1951 t.Errorf("Position(%d) got (%d,%d) want (%d,%d)", tc.pos, row, col, tc.row, tc.col) 1952 } 1953 }) 1954 } 1955} 1956