Promote descending-spawn-priority experiment

Author: moskyb

EXPERIMENTS.md

@@ -47,10 +47,6 @@ We previously made this the default behaviour of the agent (as of v3.63.0) but h
 
 **Status:** Available as an experiment to allow users who have since depended on this behaviour to re-enable it. If you use this feature please let us know so we may better understand your use case.
 
-### `descending-spawn-priority`
-
-When using `--spawn` with `--spawn-with-priority`, the agent assigns ascending priorities to each spawned agent (1, 2, 3, ...). This experiment changes the priorities to be descending (-1, -2, -3, ...) instead. This helps jobs be assigned across all hosts in cases where the value of `--spawn` varies between hosts.
-
 ### `zip-plugins`
 
 Allows plugins to be downloaded as zip archives instead of being cloned from a Git repository. This is useful for plugins hosted as zip files on HTTP(S) URLs.

clicommand/agent_start.go

@@ -98,7 +98,7 @@ type AgentStartConfig struct {
 	Priority          string   `cli:"priority"`
 	Spawn             int      `cli:"spawn"`
 	SpawnPerCPU       int      `cli:"spawn-per-cpu"`
-	SpawnWithPriority bool     `cli:"spawn-with-priority"`
+	SpawnWithPriority string   `cli:"spawn-with-priority"`
 	RedactedVars      []string `cli:"redacted-vars" normalize:"list"`
 	CancelSignal      string   `cli:"cancel-signal"`
 
@@ -666,9 +666,10 @@ var AgentStartCommand = cli.Command{
 			Value:  0,
 			EnvVar: "BUILDKITE_AGENT_SPAWN_PER_CPU",
 		},
-		cli.BoolFlag{
+		cli.StringFlag{
 			Name:   "spawn-with-priority",
-			Usage:  "Assign priorities to every spawned agent (when using --spawn or --spawn-per-cpu) equal to the agent's index (default: false)",
+			Usage:  `Assign priorities to every spawned agent (when using --spawn or --spawn-per-cpu). Pass "static" (1, 1, 1, ...), "ascending" (1, 2, 3, ...), or "descending" (-1, -2, -3, ...). Descending helps jobs be assigned across all hosts when the value of --spawn varies between hosts`,
+			Value:  "static",
 			EnvVar: "BUILDKITE_AGENT_SPAWN_WITH_PRIORITY",
 		},
 		cancelSignalFlag,
@@ -842,6 +843,11 @@ var AgentStartCommand = cli.Command{
 			cfg.PingMode = agent.PingModePollOnly
 		}
 
+		validSpawnWithPriorities := []string{"static", "ascending", "descending"}
+		if !slices.Contains(validSpawnWithPriorities, cfg.SpawnWithPriority) {
+			return fmt.Errorf("invalid spawn-with-priority, must be one of %v", validSpawnWithPriorities)
+		}
+
 		if cfg.VerificationJWKSFile != "" {
 			if !slices.Contains(verificationFailureBehaviors, cfg.VerificationFailureBehavior) {
 				return fmt.Errorf(
@@ -1288,17 +1294,23 @@ var AgentStartCommand = cli.Command{
 			// Handle per-spawn name interpolation, replacing %spawn with the spawn index
 			registerReq.Name = strings.ReplaceAll(cfg.Name, "%spawn", strconv.Itoa(i))
 
-			if cfg.SpawnWithPriority {
-				p := i
-				if experiments.IsEnabled(ctx, experiments.DescendingSpawnPriority) {
-					// This experiment helps jobs be assigned across all hosts
-					// in cases where the value of --spawn varies between hosts.
-					p = -i
-				}
-				l.Info("Assigning priority %d for agent %d", p, i)
-				registerReq.Priority = strconv.Itoa(p)
+			var priority string
+			switch cfg.SpawnWithPriority {
+			case "static":
+				priority = cfg.Priority
+
+			case "ascending":
+				priority = strconv.Itoa(i)
+
+			case "descending":
+				priority = strconv.Itoa(-i)
+
+			default:
+				return fmt.Errorf("unknown spawn-with-priority value %s", cfg.SpawnWithPriority)
 			}
 
+			registerReq.Priority = priority
+
 			// Register the agent with the buildkite API
 			reg, err := client.Register(ctx, registerReq)
 			if err != nil {

internal/experiments/experiments.go

@@ -24,13 +24,13 @@ const (
 const (
 	// Available experiments
 	AgentAPI                       = "agent-api"
-	DescendingSpawnPriority        = "descending-spawn-priority"
 	InterpolationPrefersRuntimeEnv = "interpolation-prefers-runtime-env"
 	PTYRaw                         = "pty-raw"
 	ZipPlugins                     = "zip-plugins"
 
 	// Promoted or removed experiments - un-export these to ensure no new code
 	// can depend on them.
+	descendingSpawnPriority    = "descending-spawn-priority"
 	allowArtifactPathTraversal = "allow-artifact-path-traversal"
 	ansiTimestamps             = "ansi-timestamps"
 	avoidRecursiveTrap         = "avoid-recursive-trap"
@@ -51,13 +51,13 @@ const (
 var (
 	Available = map[string]struct{}{
 		AgentAPI:                       {},
-		DescendingSpawnPriority:        {},
 		InterpolationPrefersRuntimeEnv: {},
 		PTYRaw:                         {},
 		ZipPlugins:                     {},
 	}
 
 	Promoted = map[string]string{
+		descendingSpawnPriority:    "The `descending-spawn-priority` has been replaced with `--spawn-with-priority descending` as of agent v4",
 		ansiTimestamps:             standardPromotionMsg(ansiTimestamps, "v3.48.0"),
 		allowArtifactPathTraversal: "The allow-artifact-path-traversal escape-hatch experiment has been removed as of agent v4, because the path traversal behaviour was insecure",
 		avoidRecursiveTrap:         standardPromotionMsg(avoidRecursiveTrap, "v3.66.0"),