internal/encoding/yaml/goccy: resolve scalar numbers by syntax rules

numberKind mirrored the yaml.v3 resolver through Go's strconv
functions, inheriting artifacts that belong to neither YAML nor CUE:
integers were bounded to 64 bits, so larger decimal integers decoded
as the float form `number & N` and larger hexadecimals fell back to
strings, even though YAML and CUE integers have no size limit, and
values shaped like broken YAML 1.1 octals, such as `0778`, resolved
as floats and needed a carve-out to reach their string interpretation.

Resolve numbers by declared syntax rules instead: the YAML 1.2 core
schema forms extended with the YAML 1.1 forms CUE keeps supporting,
namely leading-zero octals, binary integers, a sign before any base,
and underscore separators. Integers of any size in any base now
decode as CUE integers, the `0778` carve-out disappears since a
decimal integer must not start with a zero digit, and `number & N`
remains only for scalars tagged `!!float` whose value is written as
an integer. The rxAnyOctalYaml11 pattern moves to the goccy encoder
added by a follow-up change, whose quoting still relies on it.

Converting YAML 1.1 octals to CUE form is now also limited to integer
shapes and understands signs. It previously fired on any value with a
leading zero digit, so valid YAML floats such as `01289.5` were
mangled into `0o1289.5` and failed to decode, and it ignored signed
octals such as `-0123`, which then failed to decode as invalid CUE.
Both bugs are shared with the yaml.v3 based decoder, where they
remain; the yamlgoccy experiment covers the transition.

Signed-off-by: Matthew Sackman <matthew@cue.works>
Assisted-by: Claude Code (claude-fable-5)
Change-Id: Ie93e11aa537efceebb91ade0c699acf96d93f8a6
Reviewed-on: https://cue.gerrithub.io/c/cue-lang/cue/+/1244090
TryBot-Result: CUEcueckoo <cueckoo@cuelang.org>
Unity-Result: CUE porcuepine <cue.porcuepine@gmail.com>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>

Author: cuematthew

cmd/cue/cmd/testdata/script/import_yaml_numbers.txtar

@@ -39,7 +39,7 @@ float: !!float 123
 memory: "1Gi"
 cpu:    "100m"
 frac:   "1.5Gi"
-huge:   number & 18446744073709551616
+huge:   18446744073709551616
 exp:    123456e1
 float:  number & 123
 -- expect-import-old-stderr --

internal/encoding/yaml/goccy/decode.go

@@ -1454,16 +1454,24 @@ func infString(n *yast.InfinityNode) string {
 	return "+Inf"
 }
 
-// yamlStyleFloat matches the plain scalars that resolve as floats when
-// they do not parse as integers; see [numberKind].
-var yamlStyleFloat = sync.OnceValue(func() *regexp.Regexp {
-	return regexp.MustCompile(`^[-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)?$`)
+// rxYamlInt matches the plain scalars this package resolves as
+// integers: the YAML 1.2 core schema forms extended with the YAML 1.1
+// forms CUE keeps supporting, namely leading-zero octals, binary
+// integers, and a sign before any base. A decimal integer must not
+// start with a zero digit, so that broken octals such as `0778`
+// resolve as strings. Underscore separators are stripped before
+// matching; see [numberKind].
+var rxYamlInt = sync.OnceValue(func() *regexp.Regexp {
+	return regexp.MustCompile(`^[-+]?(0|[1-9][0-9]*|0b[01]+|0o?[0-7]+|0x[0-9a-fA-F]+)$`)
 })
 
-// rxAnyOctalYaml11 uses the implicit tag resolution regular expression for base-8 integers
-// from YAML's 1.1 spec, but including the 8 and 9 digits which aren't valid for octal integers.
-var rxAnyOctalYaml11 = sync.OnceValue(func() *regexp.Regexp {
-	return regexp.MustCompile(`^[-+]?0[0-9_]+$`)
+// rxYamlFloat matches the plain scalars this package resolves as
+// floats: decimal digits with a dot, an exponent, or both. Requiring
+// the dot or exponent keeps decimal integers, whatever their leading
+// digit, out of the float form. Underscore separators are stripped
+// before matching; see [numberKind].
+var rxYamlFloat = sync.OnceValue(func() *regexp.Regexp {
+	return regexp.MustCompile(`^[-+]?((\.[0-9]+|[0-9]+\.[0-9]*)([eE][-+]?[0-9]+)?|[0-9]+[eE][-+]?[0-9]+)$`)
 })
 
 // specialFloats maps the plain scalar spellings of infinities and NaN
@@ -1477,36 +1485,24 @@ var specialFloats = map[string]string{
 
 // numberKind reports whether this package resolves the plain scalar s
 // as a number, returning token.INT, token.FLOAT, or token.ILLEGAL when
-// s is not a number. Infinities and NaN are handled separately via
-// [specialFloats]. Note that we cannot use CUE's own number syntax to
-// decide, as it is a superset of YAML's: for example, `1Gi` is a valid
-// CUE literal but a string in YAML.
+// s is not a number. The accepted forms are those of [rxYamlInt] and
+// [rxYamlFloat], with no limit on the size of integers. Infinities and
+// NaN are handled separately via [specialFloats]. Note that we cannot
+// use CUE's own number syntax to decide, as it is a superset of
+// YAML's: for example, `1Gi` is a valid CUE literal but a string in
+// YAML.
 func numberKind(s string) token.Token {
-	if s == "" {
+	if s == "" || s[0] == '_' {
 		return token.ILLEGAL
 	}
-	switch c := s[0]; {
-	case c == '.':
-		// A float such as `.5e3`.
-		if _, err := strconv.ParseFloat(s, 64); err == nil {
-			return token.FLOAT
-		}
-	case c == '+' || c == '-' || (c >= '0' && c <= '9'):
-		// Strip underscores, which YAML 1.1 allows as separators.
-		plain := strings.ReplaceAll(s, "_", "")
-		if _, err := strconv.ParseInt(plain, 0, 64); err == nil {
-			return token.INT
-		}
-		if _, err := strconv.ParseUint(plain, 0, 64); err == nil {
-			return token.INT
-		}
-		// A float, such as an integer beyond 64 bits or an exponent
-		// without a decimal point like `123456e1`.
-		if yamlStyleFloat().MatchString(plain) {
-			if _, err := strconv.ParseFloat(plain, 64); err == nil {
-				return token.FLOAT
-			}
-		}
+	// Strip underscores, which YAML 1.1 allows as separators within
+	// numbers.
+	plain := strings.ReplaceAll(s, "_", "")
+	switch {
+	case rxYamlInt().MatchString(plain):
+		return token.INT
+	case rxYamlFloat().MatchString(plain):
+		return token.FLOAT
 	}
 	return token.ILLEGAL
 }
@@ -1529,24 +1525,30 @@ func (d *decoder) scalarString(n *yast.StringNode) (ast.Expr, error) {
 		case token.INT:
 			return d.intExpr(pos, value)
 		case token.FLOAT:
-			// Values which look like YAML 1.1 octal literals but aren't
-			// valid octal integers, such as `0778`, are interpreted as
-			// strings instead, as most YAML decoders do.
-			if !rxAnyOctalYaml11().MatchString(value) {
-				return d.floatExpr(pos, value, true)
-			}
+			return d.floatExpr(pos, value, true)
 		}
 	}
 	return d.quotedString(pos, value), nil
 }
 
-// yaml11OctalToCUE converts a YAML 1.1 octal literal like 0777 to CUE
-// form. Other values are returned unchanged.
+// yaml11OctalToCUE converts a YAML 1.1 octal literal like 0777 or
+// -0777 to CUE form. Other values, such as floats with a leading zero
+// digit like 01289.5, are returned unchanged.
 func yaml11OctalToCUE(value string) string {
-	if len(value) > 1 && value[0] == '0' && value[1] >= '0' && value[1] <= '9' {
-		return "0o" + value[1:]
+	sign, digits := "", value
+	if len(digits) > 0 && (digits[0] == '+' || digits[0] == '-') {
+		sign, digits = digits[:1], digits[1:]
+	}
+	rest, ok := strings.CutPrefix(digits, "0")
+	if !ok || rest == "" {
+		return value
+	}
+	for _, c := range rest {
+		if (c < '0' || c > '7') && c != '_' {
+			return value
+		}
 	}
-	return value
+	return sign + "0o" + rest
 }
 
 func (d *decoder) integer(n *yast.IntegerNode) (ast.Expr, error) {

internal/encoding/yaml/goccy/decode_test.go

@@ -152,9 +152,15 @@ var unmarshalTests = []struct {
 	}, {
 		"octal_yaml11: 02472256",
 		"octal_yaml11: 0o2472256",
+	}, {
+		"octal_yaml11_neg: -02472256",
+		"octal_yaml11_neg: -0o2472256",
 	}, {
 		"octal_yaml12: 0o2472256",
 		"octal_yaml12: 0o2472256",
+	}, {
+		"float_leading_zero: 01289.5",
+		"float_leading_zero: 01289.5",
 	}, {
 		"not_octal_yaml11: 0123456789",
 		`not_octal_yaml11: "0123456789"`,
@@ -400,7 +406,7 @@ null:   1
 	},
 	{
 		"float32_maxuint64+1: 18446744073709551616",
-		`"float32_maxuint64+1": number & 18446744073709551616`,
+		`"float32_maxuint64+1": 18446744073709551616`,
 	},
 
 	// float64
@@ -416,22 +422,24 @@ null:   1
 		"float64_maxuint64: 18446744073709551615",
 		"float64_maxuint64: 18446744073709551615",
 	},
-	// TODO(mvdan): numberKind uses strconv APIs like ParseUint to decide
-	// whether a scalar is a YAML integer or a float.
-	// Integers in CUE aren't limited to 64 bits, so we should arguably not decode
-	// large integers that don't fit in 64 bits as floats via `number &`.
+	// Integers are unbounded, like CUE's, with no 64-bit cliff in any
+	// base.
 	{
 		"float64_maxuint64+1: 18446744073709551616",
-		`"float64_maxuint64+1": number & 18446744073709551616`,
+		`"float64_maxuint64+1": 18446744073709551616`,
 	},
 	{
 		"v: -9223372036854775809",
-		"v: number & -9223372036854775809",
+		"v: -9223372036854775809",
+	},
+	{
+		"v: 36_893_488_147_419_103_232",
+		"v: 36_893_488_147_419_103_232",
 	},
 
 	// Scalars that are valid CUE numbers but not YAML numbers must stay
 	// strings, such as Kubernetes resource quantities with multiplier
-	// suffixes or hexadecimals beyond 64 bits.
+	// suffixes.
 	{
 		"memory: 1Gi",
 		`memory: "1Gi"`,
@@ -450,7 +458,7 @@ null:   1
 	},
 	{
 		"v: 0xFFFFFFFFFFFFFFFFF",
-		`v: "0xFFFFFFFFFFFFFFFFF"`,
+		"v: 0xFFFFFFFFFFFFFFFFF",
 	},
 
 	// Overflow cases.