diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index a952d859af..7130762d55 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -168,6 +168,26 @@ function appsec_image_from_tag_mapping(string $tag): string REQUIREMENTS_BLOCK_JSON_PATH: "loader/packaging/block_tests.json" REQUIREMENTS_ALLOW_JSON_PATH: "loader/packaging/allow_tests.json" +"system tests shard selector test": + stage: prepare + image: registry.ddbuild.io/images/mirror/python:3.12-slim-bullseye + tags: [ "arch:amd64" ] + needs: [] + variables: + GIT_SUBMODULE_STRATEGY: none + script: + - python3 .gitlab/tests/test_package_system_tests_sharding.py -v + +"system tests pinning contract test": + stage: prepare + image: registry.ddbuild.io/images/mirror/php:8.2-cli + tags: [ "arch:amd64" ] + needs: [] + variables: + GIT_SUBMODULE_STRATEGY: none + script: + - php .gitlab/tests/test_package_system_tests_pinning.php + # dd-trace-php release packaging "prepare code": @@ -176,6 +196,13 @@ function appsec_image_from_tag_mapping(string $tag): string tags: [ "arch:amd64" ] script: - ./.gitlab/append-build-id.sh + - | + SYSTEM_TESTS_SHA=$(git ls-remote https://github.com/DataDog/system-tests.git refs/heads/main | awk 'NR == 1 { print $1 }') + if ! printf '%s\n' "$SYSTEM_TESTS_SHA" | grep -Eq '^[0-9a-f]{40}$'; then + echo "Failed to resolve a valid system-tests commit: $SYSTEM_TESTS_SHA" + exit 1 + fi + printf 'SYSTEM_TESTS_SHA=%s\n' "$SYSTEM_TESTS_SHA" > system-tests.env # Upgrading composer - composer self-update --no-interaction # Installing dependencies with composer @@ -190,6 +217,8 @@ function appsec_image_from_tag_mapping(string $tag): string paths: - VERSION - ./src/bridge/_generated*.php + reports: + dotenv: system-tests.env - /tmp/vault kv get --format=json "kv/k8s/gitlab-runner/dd-trace-php/datadoghq-api-key" 2>/dev/null | python3 -c "import sys,json;print(json.load(sys.stdin)['data']['data']['key'])" > /tmp/.dd-api-key 2>/dev/null || true - - git clone https://github.com/DataDog/system-tests.git + - | + if ! printf '%s\n' "${SYSTEM_TESTS_SHA:-}" | grep -Eq '^[0-9a-f]{40}$'; then + echo "Missing or invalid SYSTEM_TESTS_SHA: ${SYSTEM_TESTS_SHA:-}" + exit 1 + fi + git init -q system-tests + git -C system-tests remote add origin https://github.com/DataDog/system-tests.git + git -C system-tests fetch --depth=1 origin "$SYSTEM_TESTS_SHA" + git -C system-tests checkout --detach "$SYSTEM_TESTS_SHA" + CHECKED_OUT_SYSTEM_TESTS_SHA=$(git -C system-tests rev-parse HEAD) + if [ "$CHECKED_OUT_SYSTEM_TESTS_SHA" != "$SYSTEM_TESTS_SHA" ]; then + echo "Checked out system-tests $CHECKED_OUT_SYSTEM_TESTS_SHA, expected $SYSTEM_TESTS_SHA" + exit 1 + fi - mv packages/{datadog-setup.php,dd-library-php-*x86_64-linux-gnu.tar.gz} system-tests/binaries - cd system-tests - ./build.sh $BUILD_SH_ARGS @@ -1385,6 +1427,7 @@ function appsec_image_from_tag_mapping(string $tag): string "System Tests: [, tracer-release]": extends: .system_tests timeout: 4h + parallel: 4 variables: BUILD_SH_ARGS: -w php # Expand the DinD loopback volume to avoid running out of disk space. @@ -1400,7 +1443,12 @@ function appsec_image_from_tag_mapping(string $tag): string script: - DD_API_KEY=$(cat /tmp/.dd-api-key 2>/dev/null) || { echo "Failed to fetch DD_API_KEY"; exit 1; } - export DD_API_KEY - - SCENARIOS=$(PYTHONPATH=. venv/bin/python utils/scripts/compute-workflow-parameters.py php -g tracer_release -f json | python3 -c "import sys,json;d=json.load(sys.stdin);s=set();[s.update(v['scenarios']) for v in d.values() if isinstance(v,dict) and 'scenarios' in v];print(' '.join(sorted(s)))") + - | + set -o pipefail + SCENARIOS=$( + PYTHONPATH=. venv/bin/python utils/scripts/compute-workflow-parameters.py php -g tracer_release -f json | + python3 "$CI_PROJECT_DIR/.gitlab/select-system-tests-shard.py" + ) || exit $? - FAILED=""; for S in $SCENARIOS; do echo "=== Running $S ==="; ./run.sh $S || FAILED="$FAILED $S"; done; if [ -n "$FAILED" ]; then echo "Failed scenarios:$FAILED"; exit 1; fi diff --git a/.gitlab/select-system-tests-shard.py b/.gitlab/select-system-tests-shard.py new file mode 100644 index 0000000000..62578ad137 --- /dev/null +++ b/.gitlab/select-system-tests-shard.py @@ -0,0 +1,70 @@ +import json +import os +import sys + + +def fail(message): + raise SystemExit(f"Failed to select tracer-release scenarios: {message}") + + +def validate_scenario_group(group, location): + if not isinstance(group, list) or any( + not isinstance(scenario, str) or not scenario or any(character.isspace() for character in scenario) + for scenario in group + ): + fail(f"expected {location} to be a list of non-empty names without whitespace") + return group + + +def main(): + try: + data = json.load(sys.stdin) + except (json.JSONDecodeError, UnicodeDecodeError) as error: + fail(f"invalid scenario JSON: {error}") + + if not isinstance(data, dict): + fail("expected a JSON object") + + try: + shard_count = int(os.environ["CI_NODE_TOTAL"]) + shard_number = int(os.environ["CI_NODE_INDEX"]) + except (KeyError, ValueError) as error: + fail(f"invalid shard configuration: {error}") + + if shard_count != 4 or not 1 <= shard_number <= shard_count: + fail(f"invalid shard {shard_number} of {shard_count}") + + endtoend_defs = data.get("endtoend_defs") + if not isinstance(endtoend_defs, dict): + fail("expected endtoend_defs to be an object") + + parallel_jobs = endtoend_defs.get("parallel_jobs") + if not isinstance(parallel_jobs, list) or not parallel_jobs: + fail("expected endtoend_defs.parallel_jobs to be a non-empty list") + + scenarios = set() + for index, job in enumerate(parallel_jobs): + if not isinstance(job, dict) or "scenarios" not in job: + fail(f"expected endtoend_defs.parallel_jobs[{index}] to contain scenarios") + scenarios.update( + validate_scenario_group( + job["scenarios"], + f"endtoend_defs.parallel_jobs[{index}].scenarios", + ) + ) + + for name, value in data.items(): + if name in ("endtoend", "endtoend_defs"): + continue + if isinstance(value, dict) and "scenarios" in value: + scenarios.update(validate_scenario_group(value["scenarios"], f"{name}.scenarios")) + + selected = sorted(scenarios)[shard_number - 1::shard_count] + if not selected: + fail(f"shard {shard_number} of {shard_count} is empty") + + print(" ".join(selected)) + + +if __name__ == "__main__": + main() diff --git a/.gitlab/tests/test_package_system_tests_pinning.php b/.gitlab/tests/test_package_system_tests_pinning.php new file mode 100644 index 0000000000..ada149ae3a --- /dev/null +++ b/.gitlab/tests/test_package_system_tests_pinning.php @@ -0,0 +1,137 @@ + system-tests.env", + 'system-tests dotenv creation' +); +require_contains( + $prepare_code, + " artifacts:\n" . + " paths:\n" . + " - VERSION\n" . + " - ./src/bridge/_generated*.php\n" . + " reports:\n" . + " dotenv: system-tests.env", + 'system-tests dotenv artifact report' +); + +$system_tests = generated_definition($configuration, '.system_tests'); +require_contains( + $system_tests, + "- job: \"prepare code\"\n artifacts: true", + 'prepare code artifact dependency' +); +require_contains( + $system_tests, + "if ! printf '%s\\n' \"\${SYSTEM_TESTS_SHA:-}\" | grep -Eq '^[0-9a-f]{40}\$'; then\n" . + " echo \"Missing or invalid SYSTEM_TESTS_SHA: \${SYSTEM_TESTS_SHA:-}\"\n" . + " exit 1\n" . + " fi", + 'checkout revision validation' +); +require_contains( + $system_tests, + "git init -q system-tests\n" . + " git -C system-tests remote add origin https://github.com/DataDog/system-tests.git", + 'system-tests checkout initialization' +); +require_contains( + $system_tests, + 'git -C system-tests fetch --depth=1 origin "$SYSTEM_TESTS_SHA"', + 'exact shallow system-tests fetch' +); +require_contains( + $system_tests, + 'git -C system-tests checkout --detach "$SYSTEM_TESTS_SHA"', + 'detached system-tests checkout' +); +require_contains( + $system_tests, + 'CHECKED_OUT_SYSTEM_TESTS_SHA=$(git -C system-tests rev-parse HEAD)', + 'checked-out system-tests revision lookup' +); +require_contains( + $system_tests, + "if [ \"\$CHECKED_OUT_SYSTEM_TESTS_SHA\" != \"\$SYSTEM_TESTS_SHA\" ]; then\n" . + " echo \"Checked out system-tests \$CHECKED_OUT_SYSTEM_TESTS_SHA, expected \$SYSTEM_TESTS_SHA\"\n" . + " exit 1\n" . + " fi", + 'checked-out system-tests revision mismatch failure' +); + +if (!preg_match_all( + '/^"System Tests: \[[^,\]\n]+, tracer-release\]":$/m', + $configuration, + $tracer_release_jobs +) || count($tracer_release_jobs[0]) !== 25) { + fail('Generated pipeline must contain 25 tracer-release definitions'); +} + +foreach ($tracer_release_jobs[0] as $heading) { + $name = substr($heading, 0, -1); + $definition = generated_definition($configuration, $name); + require_contains($definition, " parallel: 4\n", "$name four-way parallel expansion"); + require_contains($definition, " set -o pipefail\n", "$name scenario pipeline failure detection"); + require_contains( + $definition, + "PYTHONPATH=. venv/bin/python utils/scripts/compute-workflow-parameters.py php -g tracer_release -f json |\n" . + " python3 \"\$CI_PROJECT_DIR/.gitlab/select-system-tests-shard.py\"", + "$name scenario producer-to-selector pipeline" + ); + require_contains($definition, ') || exit $?', "$name selector failure propagation"); +} + +echo "Generated pipeline system-tests pinning contract: OK\n"; diff --git a/.gitlab/tests/test_package_system_tests_sharding.py b/.gitlab/tests/test_package_system_tests_sharding.py new file mode 100644 index 0000000000..7cc19d08ea --- /dev/null +++ b/.gitlab/tests/test_package_system_tests_sharding.py @@ -0,0 +1,109 @@ +import collections +import json +import os +from pathlib import Path +import subprocess +import sys +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +SELECTOR = ROOT / ".gitlab/select-system-tests-shard.py" + + +def canonical_workflow(scenarios): + midpoint = len(scenarios) // 2 + return { + "endtoend_defs": { + "parallel_jobs": [ + {"scenarios": scenarios[:midpoint]}, + {"scenarios": scenarios[midpoint:]}, + ] + }, + "parametric": {"scenarios": []}, + } + + +def run_selector(workflow, shard_number, shard_count="4"): + environment = os.environ.copy() + environment.update(CI_NODE_INDEX=str(shard_number), CI_NODE_TOTAL=str(shard_count)) + return subprocess.run( + [sys.executable, str(SELECTOR)], + input=json.dumps(workflow), + env=environment, + capture_output=True, + text=True, + ) + + +def select_scenarios(scenarios, shard_number, shard_count="4"): + return run_selector(canonical_workflow(scenarios), shard_number, shard_count) + + +class PackageSystemTestsShardingTest(unittest.TestCase): + def test_pinned_canonical_revision_runs_every_scenario_exactly_once(self): + endtoend_scenarios = [f"SCENARIO_{index:03d}" for index in range(96)] + [ + "000_REVISION_2", + "000_REVISION_3_A", + "000_REVISION_3_B", + "000_REVISION_4_A", + "000_REVISION_4_B", + "000_REVISION_4_C", + ] + workflow = canonical_workflow(endtoend_scenarios) + workflow["parametric"]["scenarios"] = ["PARAMETRIC"] + scenarios = endtoend_scenarios + ["PARAMETRIC"] + executions = collections.Counter() + + for shard_number in range(1, 5): + result = run_selector(workflow, shard_number) + self.assertEqual(result.returncode, 0, result.stderr) + executions.update(result.stdout.split()) + + self.assertEqual(executions, collections.Counter({scenario: 1 for scenario in scenarios})) + + def test_staggered_unpinned_revisions_are_not_exactly_once(self): + common = [f"SCENARIO_{index:03d}" for index in range(96)] + revisions = [ + common, + common + ["000_REVISION_2"], + common + ["000_REVISION_3_A", "000_REVISION_3_B"], + common + ["000_REVISION_4_A", "000_REVISION_4_B", "000_REVISION_4_C"], + ] + visible = set().union(*map(set, revisions)) + executions = collections.Counter() + + for shard_number, scenarios in enumerate(revisions, start=1): + result = run_selector(canonical_workflow(scenarios), shard_number) + self.assertEqual(result.returncode, 0, result.stderr) + executions.update(result.stdout.split()) + + self.assertEqual(len(visible), 102) + self.assertNotEqual( + executions, + collections.Counter({scenario: 1 for scenario in visible}), + ) + self.assertTrue(visible - executions.keys()) + + def test_invalid_selection_fails(self): + result = select_scenarios(["ONLY_ONE"], 4) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Failed to select tracer-release scenarios:", result.stderr) + + def test_invalid_canonical_schema_fails(self): + invalid_workflows = [ + {"endtoend": {"scenarios": ["LEGACY_ONLY"]}}, + {"endtoend_defs": {"parallel_jobs": []}}, + {"endtoend_defs": {"parallel_jobs": [{}]}}, + {"endtoend_defs": {"parallel_jobs": [{"scenarios": ["HAS WHITESPACE"]}]}}, + ] + + for workflow in invalid_workflows: + with self.subTest(workflow=workflow): + result = run_selector(workflow, 1) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Failed to select tracer-release scenarios:", result.stderr) + + +if __name__ == "__main__": + unittest.main()