From a9ff382e76a017f1ee817c254b65c42f3c08abf8 Mon Sep 17 00:00:00 2001 From: Laura Calabro Date: Wed, 3 Jun 2026 19:00:10 +0000 Subject: [PATCH 1/2] fix(cdk-assets): re-tag docker image for multi-destination publishes When the same container image asset is published to multiple destinations (e.g. multi-region deployments), the WorkGraphBuilder deduplicates the asset-build node so that build() only runs for one destination. This means docker.tag() is only called for the first destination's ECR URI. When publish() subsequently runs for other destinations, docker push fails with "An image does not exist locally with the tag" because the local tag was never created for those repositories. This fix adds a check in publish() that verifies the destination imageUri exists locally before pushing. If it doesn't, it finds an existing local image with the same imageTag (content hash) and re-tags it for the current destination. This is a lightweight docker tag operation that avoids any rebuild. Fixes multi-region container image deployments failing on second+ region. --- .../cdk-assets-lib/lib/private/docker.ts | 19 +++ .../lib/private/handlers/container-images.ts | 16 +++ .../cdk-assets-lib/test/docker-images.test.ts | 108 ++++++++++++++++++ 3 files changed, 143 insertions(+) diff --git a/packages/@aws-cdk/cdk-assets-lib/lib/private/docker.ts b/packages/@aws-cdk/cdk-assets-lib/lib/private/docker.ts index 279b4a853..c5f20d780 100644 --- a/packages/@aws-cdk/cdk-assets-lib/lib/private/docker.ts +++ b/packages/@aws-cdk/cdk-assets-lib/lib/private/docker.ts @@ -166,6 +166,25 @@ export class Docker { await this.execute(['tag', sourceTag, targetTag]); } + /** + * Find a local image tagged with the given imageTag (the tag portion after the colon). + * Returns the full `repository:tag` string, or undefined if not found. + */ + public async findImageByTag(imageTag: string): Promise { + try { + const configArgs = this.configDir ? ['--config', this.configDir] : []; + const shellEventPublisher = shellEventPublisherFromEventEmitter(this.eventEmitter); + const output = await shell( + [getDockerCmd(), ...configArgs, 'images', '--format', '{{.Repository}}:{{.Tag}}', '--filter', `reference=*/*:${imageTag}`], + { shellEventPublisher, subprocessOutputDestination: 'ignore' }, + ); + const firstLine = output.trim().split('\n')[0]?.trim(); + return firstLine || undefined; + } catch { + return undefined; + } + } + public async push(options: PushOptions) { await this.execute(['push', options.tag], { subprocessOutputDestination: this.subprocessOutputDestination, diff --git a/packages/@aws-cdk/cdk-assets-lib/lib/private/handlers/container-images.ts b/packages/@aws-cdk/cdk-assets-lib/lib/private/handlers/container-images.ts index 6901ea937..34c1c6e2b 100644 --- a/packages/@aws-cdk/cdk-assets-lib/lib/private/handlers/container-images.ts +++ b/packages/@aws-cdk/cdk-assets-lib/lib/private/handlers/container-images.ts @@ -18,6 +18,7 @@ interface ContainerImageAssetHandlerInit { export class ContainerImageAssetHandler implements IAssetHandler { private init?: ContainerImageAssetHandlerInit; + private buildCompleted = false; constructor( private readonly workDir: string, @@ -60,6 +61,7 @@ export class ContainerImageAssetHandler implements IAssetHandler { } await dockerForBuilding.tag(localTagName, initOnce.imageUri); + this.buildCompleted = true; } public async isPublished(): Promise { @@ -93,6 +95,20 @@ export class ContainerImageAssetHandler implements IAssetHandler { return; } + // When the same image asset is published to multiple destinations (e.g. + // multi-region deployments), the work graph deduplicates the build step so + // only one destination gets docker-tagged during build(). If build() was + // not called on this handler, ensure the image is tagged locally for this + // destination by re-tagging from any existing local image that carries the + // same imageTag. + if (!this.buildCompleted && !await dockerForPushing.exists(initOnce.imageUri)) { + const imageTag = this.asset.destination.imageTag; + const existing = await dockerForPushing.findImageByTag(imageTag); + if (existing) { + await dockerForPushing.tag(existing, initOnce.imageUri); + } + } + this.host.emitMessage(EventType.UPLOAD, `Push ${initOnce.imageUri}`); await dockerForPushing.push({ tag: initOnce.imageUri, diff --git a/packages/@aws-cdk/cdk-assets-lib/test/docker-images.test.ts b/packages/@aws-cdk/cdk-assets-lib/test/docker-images.test.ts index 435ceb89d..3346f1292 100644 --- a/packages/@aws-cdk/cdk-assets-lib/test/docker-images.test.ts +++ b/packages/@aws-cdk/cdk-assets-lib/test/docker-images.test.ts @@ -132,6 +132,31 @@ beforeEach(() => { }, }, }), + '/multi-dest/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + dockerImages: { + theAsset: { + source: { + directory: 'dockerdir', + }, + destinations: { + dest1: { + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + repositoryName: 'repo1', + imageTag: 'abcdef', + }, + dest2: { + region: 'eu-south-99', + assumeRoleArn: 'arn:aws:role', + repositoryName: 'repo2', + imageTag: 'abcdef', + }, + }, + }, + }, + }), + '/multi-dest/cdk.out/dockerdir/Dockerfile': 'FROM scratch', '/simple/cdk.out/dockerdir/Dockerfile': 'FROM scratch', '/abs/cdk.out/assets.json': JSON.stringify({ version: Manifest.version(), @@ -844,6 +869,89 @@ test('publishing only', async () => { expect(true).toBeTruthy(); // Expect no exception, satisfy linter }); +test('publish re-tags image when local tag is missing for a destination', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath(mockfs.path('/multi-dest/cdk.out')), { + aws, + throwOnError: true, + }); + + mockEcr.on(DescribeRepositoriesCommand).callsFake((input) => { + const repos: Record = { + repo1: '12345.amazonaws.com/repo1', + repo2: '12345.amazonaws.com/repo2', + }; + const url = repos[input.repositoryNames[0]]; + if (!url) { + throw new Error(`Unexpected repo: ${JSON.stringify(input)}`); + } + return { repositories: [{ repositoryUri: url }] }; + }); + + // Image doesn't exist in either ECR destination + mockEcr.on(DescribeImagesCommand).rejects(err); + + const expectAllSpawns = mockSpawn( + // First destination: build + tag + push (buildCompleted=true, skips exists check) + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'proxy.com'] }, + { commandLine: ['docker', 'inspect', 'cdkasset-theasset'], exitCode: 1 }, + { commandLine: ['docker', 'build', '--tag', 'cdkasset-theasset', '.'], cwd: 'multi-dest/cdk.out/dockerdir' }, + { commandLine: ['docker', 'tag', 'cdkasset-theasset', '12345.amazonaws.com/repo1:abcdef'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/repo1:abcdef'] }, + // Second destination: build finds cached image, tags, then push + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'proxy.com'] }, + { commandLine: ['docker', 'inspect', 'cdkasset-theasset'] }, + { commandLine: ['docker', 'tag', 'cdkasset-theasset', '12345.amazonaws.com/repo2:abcdef'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/repo2:abcdef'] }, + ); + + await pub.publish(); + + expectAllSpawns(); +}); + +test('publish re-tags from existing image when build was not called for destination', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath(mockfs.path('/multi-dest/cdk.out')), { + aws, + throwOnError: true, + buildAssets: false, + publishAssets: true, + }); + + mockEcr.on(DescribeRepositoriesCommand).callsFake((input) => { + const repos: Record = { + repo1: '12345.amazonaws.com/repo1', + repo2: '12345.amazonaws.com/repo2', + }; + const url = repos[input.repositoryNames[0]]; + if (!url) { + throw new Error(`Unexpected repo: ${JSON.stringify(input)}`); + } + return { repositories: [{ repositoryUri: url }] }; + }); + + // Image doesn't exist in either ECR destination + mockEcr.on(DescribeImagesCommand).rejects(err); + + const expectAllSpawns = mockSpawn( + // First destination: publish() finds imageUri doesn't exist locally, finds by tag, re-tags + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'proxy.com'] }, + { commandLine: ['docker', 'inspect', '12345.amazonaws.com/repo1:abcdef'], exitCode: 1 }, + { commandLine: ['docker', 'images', '--format', '{{.Repository}}:{{.Tag}}', '--filter', 'reference=*/*:abcdef'], stdout: 'cdkasset-theasset:abcdef' }, + { commandLine: ['docker', 'tag', 'cdkasset-theasset:abcdef', '12345.amazonaws.com/repo1:abcdef'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/repo1:abcdef'] }, + // Second destination: same flow + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'proxy.com'] }, + { commandLine: ['docker', 'inspect', '12345.amazonaws.com/repo2:abcdef'], exitCode: 1 }, + { commandLine: ['docker', 'images', '--format', '{{.Repository}}:{{.Tag}}', '--filter', 'reference=*/*:abcdef'], stdout: 'cdkasset-theasset:abcdef' }, + { commandLine: ['docker', 'tag', 'cdkasset-theasset:abcdef', '12345.amazonaws.com/repo2:abcdef'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/repo2:abcdef'] }, + ); + + await pub.publish(); + + expectAllSpawns(); +}); + test('overriding the docker command', async () => { process.env.CDK_DOCKER = 'custom'; From 7c9c43a44397de8d711a67042a56e5b9367c9466 Mon Sep 17 00:00:00 2001 From: Laura Calabro Date: Thu, 4 Jun 2026 17:42:48 +0000 Subject: [PATCH 2/2] test(cdk-assets): add integ tests for multi-destination docker asset publishing Add two integration tests that validate the fix for publishing the same docker image asset to multiple destinations: 1. Same-region multi-dest: Deploys two stacks with the same docker image source to the same region. The work graph deduplicates the build node, exercising the re-tag path in publish(). 2. Cross-region: Deploys the same docker image to two different regions, requiring bootstrap of a secondary region. The secondary region is derived from the AWS_REGIONS pool to guarantee it differs from the primary. Gated by CDK_SECONDARY_REGION env var to avoid affecting other tests. --- .../cli-integ/resources/cdk-apps/app/app.js | 26 +++++++++++++++++++ ...eploy-docker-asset-multi-dest.integtest.ts | 8 ++++++ ...loy-docker-asset-multi-region.integtest.ts | 21 +++++++++++++++ .../cdk-assets-lib/test/docker-images.test.ts | 10 +++---- 4 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/deploy/cdk-deploy-docker-asset-multi-dest.integtest.ts create mode 100644 packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/deploy/cdk-deploy-docker-asset-multi-region.integtest.ts diff --git a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js index 0bcf1c019..fe20f0b63 100755 --- a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js +++ b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js @@ -864,6 +864,24 @@ class MultipleDockerAssetsStack extends cdk.Stack { } } +/** + * A stack that uses the same docker image as another stack, to test that the + * same image asset can be published to multiple destinations without rebuilding. + */ +class DockerMultiDestStack extends cdk.Stack { + constructor(parent, id, props) { + super(parent, id, props); + + new docker.DockerImageAsset(this, 'image', { + directory: path.join(__dirname, 'docker') + }); + + new cdk.CfnResource(this, 'Handle', { + type: 'AWS::CloudFormation::WaitConditionHandle' + }); + } +} + /** * A stack that will never succeed deploying (done in a way that CDK cannot detect but CFN will complain about) */ @@ -1081,6 +1099,14 @@ switch (stackSet) { new DockerInUseStack(app, `${stackPrefix}-docker-in-use`); new DockerStackWithCustomFile(app, `${stackPrefix}-docker-with-custom-file`); new MultipleDockerAssetsStack(app, `${stackPrefix}-multiple-docker-assets`); + new DockerMultiDestStack(app, `${stackPrefix}-docker-multi-dest-1`, { env: defaultEnv }); + new DockerMultiDestStack(app, `${stackPrefix}-docker-multi-dest-2`, { env: defaultEnv }); + + if (process.env.CDK_SECONDARY_REGION) { + const secondaryEnv = { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_SECONDARY_REGION }; + new DockerMultiDestStack(app, `${stackPrefix}-docker-multi-region-1`, { env: defaultEnv }); + new DockerMultiDestStack(app, `${stackPrefix}-docker-multi-region-2`, { env: secondaryEnv }); + } new NotificationArnsStack(app, `${stackPrefix}-notification-arns`); diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/deploy/cdk-deploy-docker-asset-multi-dest.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/deploy/cdk-deploy-docker-asset-multi-dest.integtest.ts new file mode 100644 index 000000000..eb531eeb7 --- /dev/null +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/deploy/cdk-deploy-docker-asset-multi-dest.integtest.ts @@ -0,0 +1,8 @@ +import { integTest, withDefaultFixture } from '../../../lib'; + +integTest( + 'deploy same docker asset to multiple stacks', + withDefaultFixture(async (fixture) => { + await fixture.cdkDeploy(['docker-multi-dest-1', 'docker-multi-dest-2']); + }), +); diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/deploy/cdk-deploy-docker-asset-multi-region.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/deploy/cdk-deploy-docker-asset-multi-region.integtest.ts new file mode 100644 index 000000000..52e8752e9 --- /dev/null +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/deploy/cdk-deploy-docker-asset-multi-region.integtest.ts @@ -0,0 +1,21 @@ +import { integTest, withDefaultFixture } from '../../../lib'; + +integTest( + 'deploy same docker asset to multiple regions', + withDefaultFixture(async (fixture) => { + const primaryRegion = fixture.aws.region; + const availableRegions = (process.env.AWS_REGIONS ?? 'us-east-1,us-west-2').split(','); + const secondaryRegion = availableRegions.find((r) => r !== primaryRegion) ?? 'us-west-2'; + + // Bootstrap the secondary region + const account = await fixture.aws.account(); + await fixture.cdk(['bootstrap', '--bootstrap-kms-key-id', 'AWS_MANAGED_KEY', `aws://${account}/${secondaryRegion}`], { + modEnv: { CDK_NEW_BOOTSTRAP: '1' }, + }); + + // Deploy both stacks — same docker source, different target regions + await fixture.cdkDeploy(['docker-multi-region-1', 'docker-multi-region-2'], { + modEnv: { CDK_SECONDARY_REGION: secondaryRegion }, + }); + }), +); diff --git a/packages/@aws-cdk/cdk-assets-lib/test/docker-images.test.ts b/packages/@aws-cdk/cdk-assets-lib/test/docker-images.test.ts index 3346f1292..8c5c26a8e 100644 --- a/packages/@aws-cdk/cdk-assets-lib/test/docker-images.test.ts +++ b/packages/@aws-cdk/cdk-assets-lib/test/docker-images.test.ts @@ -891,14 +891,13 @@ test('publish re-tags image when local tag is missing for a destination', async mockEcr.on(DescribeImagesCommand).rejects(err); const expectAllSpawns = mockSpawn( - // First destination: build + tag + push (buildCompleted=true, skips exists check) + // First destination: login + build + tag + push (buildCompleted=true, skips exists check) { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'proxy.com'] }, { commandLine: ['docker', 'inspect', 'cdkasset-theasset'], exitCode: 1 }, { commandLine: ['docker', 'build', '--tag', 'cdkasset-theasset', '.'], cwd: 'multi-dest/cdk.out/dockerdir' }, { commandLine: ['docker', 'tag', 'cdkasset-theasset', '12345.amazonaws.com/repo1:abcdef'] }, { commandLine: ['docker', 'push', '12345.amazonaws.com/repo1:abcdef'] }, - // Second destination: build finds cached image, tags, then push - { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'proxy.com'] }, + // Second destination: login skipped (same domain), build finds cached image, tags, push { commandLine: ['docker', 'inspect', 'cdkasset-theasset'] }, { commandLine: ['docker', 'tag', 'cdkasset-theasset', '12345.amazonaws.com/repo2:abcdef'] }, { commandLine: ['docker', 'push', '12345.amazonaws.com/repo2:abcdef'] }, @@ -933,14 +932,13 @@ test('publish re-tags from existing image when build was not called for destinat mockEcr.on(DescribeImagesCommand).rejects(err); const expectAllSpawns = mockSpawn( - // First destination: publish() finds imageUri doesn't exist locally, finds by tag, re-tags + // First destination: login + exists check fails + find by tag + re-tag + push { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'proxy.com'] }, { commandLine: ['docker', 'inspect', '12345.amazonaws.com/repo1:abcdef'], exitCode: 1 }, { commandLine: ['docker', 'images', '--format', '{{.Repository}}:{{.Tag}}', '--filter', 'reference=*/*:abcdef'], stdout: 'cdkasset-theasset:abcdef' }, { commandLine: ['docker', 'tag', 'cdkasset-theasset:abcdef', '12345.amazonaws.com/repo1:abcdef'] }, { commandLine: ['docker', 'push', '12345.amazonaws.com/repo1:abcdef'] }, - // Second destination: same flow - { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'proxy.com'] }, + // Second destination: login skipped (same domain), same re-tag flow { commandLine: ['docker', 'inspect', '12345.amazonaws.com/repo2:abcdef'], exitCode: 1 }, { commandLine: ['docker', 'images', '--format', '{{.Repository}}:{{.Tag}}', '--filter', 'reference=*/*:abcdef'], stdout: 'cdkasset-theasset:abcdef' }, { commandLine: ['docker', 'tag', 'cdkasset-theasset:abcdef', '12345.amazonaws.com/repo2:abcdef'] },