fix(cdk-assets): re-tag docker image for multi-destination publishes

When the same container image asset is published to multiple destinations
(e.g. multi-region deployments), the WorkGraphBuilder deduplicates the
asset-build node so that build() only runs for one destination. This
means docker.tag() is only called for the first destination's ECR URI.

When publish() subsequently runs for other destinations, docker push
fails with "An image does not exist locally with the tag" because the
local tag was never created for those repositories.

This fix adds a check in publish() that verifies the destination imageUri
exists locally before pushing. If it doesn't, it finds an existing local
image with the same imageTag (content hash) and re-tags it for the
current destination. This is a lightweight docker tag operation that
avoids any rebuild.

Fixes multi-region container image deployments failing on second+ region.

Author: calabrla

packages/@aws-cdk/cdk-assets-lib/lib/private/docker.ts

@@ -166,6 +166,25 @@ export class Docker {
     await this.execute(['tag', sourceTag, targetTag]);
   }
 
+  /**
+   * Find a local image tagged with the given imageTag (the tag portion after the colon).
+   * Returns the full `repository:tag` string, or undefined if not found.
+   */
+  public async findImageByTag(imageTag: string): Promise<string | undefined> {
+    try {
+      const configArgs = this.configDir ? ['--config', this.configDir] : [];
+      const shellEventPublisher = shellEventPublisherFromEventEmitter(this.eventEmitter);
+      const output = await shell(
+        [getDockerCmd(), ...configArgs, 'images', '--format', '{{.Repository}}:{{.Tag}}', '--filter', `reference=*/*:${imageTag}`],
+        { shellEventPublisher, subprocessOutputDestination: 'ignore' },
+      );
+      const firstLine = output.trim().split('\n')[0]?.trim();
+      return firstLine || undefined;
+    } catch {
+      return undefined;
+    }
+  }
+
   public async push(options: PushOptions) {
     await this.execute(['push', options.tag], {
       subprocessOutputDestination: this.subprocessOutputDestination,

packages/@aws-cdk/cdk-assets-lib/lib/private/handlers/container-images.ts

@@ -18,6 +18,7 @@ interface ContainerImageAssetHandlerInit {
 
 export class ContainerImageAssetHandler implements IAssetHandler {
   private init?: ContainerImageAssetHandlerInit;
+  private buildCompleted = false;
 
   constructor(
     private readonly workDir: string,
@@ -60,6 +61,7 @@ export class ContainerImageAssetHandler implements IAssetHandler {
     }
 
     await dockerForBuilding.tag(localTagName, initOnce.imageUri);
+    this.buildCompleted = true;
   }
 
   public async isPublished(): Promise<boolean> {
@@ -93,6 +95,20 @@ export class ContainerImageAssetHandler implements IAssetHandler {
       return;
     }
 
+    // When the same image asset is published to multiple destinations (e.g.
+    // multi-region deployments), the work graph deduplicates the build step so
+    // only one destination gets docker-tagged during build(). If build() was
+    // not called on this handler, ensure the image is tagged locally for this
+    // destination by re-tagging from any existing local image that carries the
+    // same imageTag.
+    if (!this.buildCompleted && !await dockerForPushing.exists(initOnce.imageUri)) {
+      const imageTag = this.asset.destination.imageTag;
+      const existing = await dockerForPushing.findImageByTag(imageTag);
+      if (existing) {
+        await dockerForPushing.tag(existing, initOnce.imageUri);
+      }
+    }
+
     this.host.emitMessage(EventType.UPLOAD, `Push ${initOnce.imageUri}`);
     await dockerForPushing.push({
       tag: initOnce.imageUri,

packages/@aws-cdk/cdk-assets-lib/test/docker-images.test.ts

@@ -132,6 +132,31 @@ beforeEach(() => {
         },
       },
     }),
+    '/multi-dest/cdk.out/assets.json': JSON.stringify({
+      version: Manifest.version(),
+      dockerImages: {
+        theAsset: {
+          source: {
+            directory: 'dockerdir',
+          },
+          destinations: {
+            dest1: {
+              region: 'us-north-50',
+              assumeRoleArn: 'arn:aws:role',
+              repositoryName: 'repo1',
+              imageTag: 'abcdef',
+            },
+            dest2: {
+              region: 'eu-south-99',
+              assumeRoleArn: 'arn:aws:role',
+              repositoryName: 'repo2',
+              imageTag: 'abcdef',
+            },
+          },
+        },
+      },
+    }),
+    '/multi-dest/cdk.out/dockerdir/Dockerfile': 'FROM scratch',
     '/simple/cdk.out/dockerdir/Dockerfile': 'FROM scratch',
     '/abs/cdk.out/assets.json': JSON.stringify({
       version: Manifest.version(),
@@ -844,6 +869,89 @@ test('publishing only', async () => {
   expect(true).toBeTruthy(); // Expect no exception, satisfy linter
 });
 
+test('publish re-tags image when local tag is missing for a destination', async () => {
+  const pub = new AssetPublishing(AssetManifest.fromPath(mockfs.path('/multi-dest/cdk.out')), {
+    aws,
+    throwOnError: true,
+  });
+
+  mockEcr.on(DescribeRepositoriesCommand).callsFake((input) => {
+    const repos: Record<string, string> = {
+      repo1: '12345.amazonaws.com/repo1',
+      repo2: '12345.amazonaws.com/repo2',
+    };
+    const url = repos[input.repositoryNames[0]];
+    if (!url) {
+      throw new Error(`Unexpected repo: ${JSON.stringify(input)}`);
+    }
+    return { repositories: [{ repositoryUri: url }] };
+  });
+
+  // Image doesn't exist in either ECR destination
+  mockEcr.on(DescribeImagesCommand).rejects(err);
+
+  const expectAllSpawns = mockSpawn(
+    // First destination: build + tag + push (buildCompleted=true, skips exists check)
+    { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'proxy.com'] },
+    { commandLine: ['docker', 'inspect', 'cdkasset-theasset'], exitCode: 1 },
+    { commandLine: ['docker', 'build', '--tag', 'cdkasset-theasset', '.'], cwd: 'multi-dest/cdk.out/dockerdir' },
+    { commandLine: ['docker', 'tag', 'cdkasset-theasset', '12345.amazonaws.com/repo1:abcdef'] },
+    { commandLine: ['docker', 'push', '12345.amazonaws.com/repo1:abcdef'] },
+    // Second destination: build finds cached image, tags, then push
+    { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'proxy.com'] },
+    { commandLine: ['docker', 'inspect', 'cdkasset-theasset'] },
+    { commandLine: ['docker', 'tag', 'cdkasset-theasset', '12345.amazonaws.com/repo2:abcdef'] },
+    { commandLine: ['docker', 'push', '12345.amazonaws.com/repo2:abcdef'] },
+  );
+
+  await pub.publish();
+
+  expectAllSpawns();
+});
+
+test('publish re-tags from existing image when build was not called for destination', async () => {
+  const pub = new AssetPublishing(AssetManifest.fromPath(mockfs.path('/multi-dest/cdk.out')), {
+    aws,
+    throwOnError: true,
+    buildAssets: false,
+    publishAssets: true,
+  });
+
+  mockEcr.on(DescribeRepositoriesCommand).callsFake((input) => {
+    const repos: Record<string, string> = {
+      repo1: '12345.amazonaws.com/repo1',
+      repo2: '12345.amazonaws.com/repo2',
+    };
+    const url = repos[input.repositoryNames[0]];
+    if (!url) {
+      throw new Error(`Unexpected repo: ${JSON.stringify(input)}`);
+    }
+    return { repositories: [{ repositoryUri: url }] };
+  });
+
+  // Image doesn't exist in either ECR destination
+  mockEcr.on(DescribeImagesCommand).rejects(err);
+
+  const expectAllSpawns = mockSpawn(
+    // First destination: publish() finds imageUri doesn't exist locally, finds by tag, re-tags
+    { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'proxy.com'] },
+    { commandLine: ['docker', 'inspect', '12345.amazonaws.com/repo1:abcdef'], exitCode: 1 },
+    { commandLine: ['docker', 'images', '--format', '{{.Repository}}:{{.Tag}}', '--filter', 'reference=*/*:abcdef'], stdout: 'cdkasset-theasset:abcdef' },
+    { commandLine: ['docker', 'tag', 'cdkasset-theasset:abcdef', '12345.amazonaws.com/repo1:abcdef'] },
+    { commandLine: ['docker', 'push', '12345.amazonaws.com/repo1:abcdef'] },
+    // Second destination: same flow
+    { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'proxy.com'] },
+    { commandLine: ['docker', 'inspect', '12345.amazonaws.com/repo2:abcdef'], exitCode: 1 },
+    { commandLine: ['docker', 'images', '--format', '{{.Repository}}:{{.Tag}}', '--filter', 'reference=*/*:abcdef'], stdout: 'cdkasset-theasset:abcdef' },
+    { commandLine: ['docker', 'tag', 'cdkasset-theasset:abcdef', '12345.amazonaws.com/repo2:abcdef'] },
+    { commandLine: ['docker', 'push', '12345.amazonaws.com/repo2:abcdef'] },
+  );
+
+  await pub.publish();
+
+  expectAllSpawns();
+});
+
 test('overriding the docker command', async () => {
   process.env.CDK_DOCKER = 'custom';