images module extension: allow downloading blobs from multiple sources

e2e/cc/toolchain.MODULE.bazel

@@ -153,8 +153,9 @@ filegroup(
 )
 """,
     digest = SYSROOT_DIGESTS["amd64"]["libc++"],
-    registry = "ghcr.io",
-    repository = "malt3/sysroots/libcxx",
+    sources = {
+        "malt3/sysroots/libcxx": ["ghcr.io"],
+    },
     type = "tzst",
 )
 
@@ -168,8 +169,9 @@ filegroup(
 )
 """,
     digest = SYSROOT_DIGESTS["amd64"]["libstdc++"],
-    registry = "ghcr.io",
-    repository = "malt3/sysroots/libstdcxx",
+    sources = {
+        "malt3/sysroots/libstdcxx": ["ghcr.io"],
+    },
     type = "tzst",
 )
 
@@ -183,8 +185,9 @@ filegroup(
 )
 """,
     digest = SYSROOT_DIGESTS["arm64"]["libc++"],
-    registry = "ghcr.io",
-    repository = "malt3/sysroots/libcxx",
+    sources = {
+        "malt3/sysroots/libcxx": ["ghcr.io"],
+    },
     type = "tzst",
 )
 
@@ -198,8 +201,9 @@ filegroup(
 )
 """,
     digest = SYSROOT_DIGESTS["arm64"]["libstdc++"],
-    registry = "ghcr.io",
-    repository = "malt3/sysroots/libstdcxx",
+    sources = {
+        "malt3/sysroots/libstdcxx": ["ghcr.io"],
+    },
     type = "tzst",
 )
 
@@ -209,30 +213,34 @@ pull_blob_file(
     name = "base_image_arm64_libcxx",
     digest = SYSROOT_DIGESTS["arm64"]["libc++"],
     downloaded_file_path = "sysroot.tzst",
-    registry = "ghcr.io",
-    repository = "malt3/sysroots/libcxx",
+    sources = {
+        "malt3/sysroots/libcxx": ["ghcr.io"],
+    },
 )
 
 pull_blob_file(
     name = "base_image_arm64_libstdcxx",
     digest = SYSROOT_DIGESTS["arm64"]["libstdc++"],
     downloaded_file_path = "sysroot.tzst",
-    registry = "ghcr.io",
-    repository = "malt3/sysroots/libstdcxx",
+    sources = {
+        "malt3/sysroots/libstdcxx": ["ghcr.io"],
+    },
 )
 
 pull_blob_file(
     name = "base_image_amd64_libcxx",
     digest = SYSROOT_DIGESTS["amd64"]["libc++"],
     downloaded_file_path = "sysroot.tzst",
-    registry = "ghcr.io",
-    repository = "malt3/sysroots/libcxx",
+    sources = {
+        "malt3/sysroots/libcxx": ["ghcr.io"],
+    },
 )
 
 pull_blob_file(
     name = "base_image_amd64_libstdcxx",
     digest = SYSROOT_DIGESTS["amd64"]["libstdc++"],
     downloaded_file_path = "sysroot.tzst",
-    registry = "ghcr.io",
-    repository = "malt3/sysroots/libstdcxx",
+    sources = {
+        "malt3/sysroots/libstdcxx": ["ghcr.io"],
+    },
 )

img/private/download_blobs.bzl

@@ -9,6 +9,7 @@ upfront during repository fetching.
 load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
 load("//img/private/common:build.bzl", "TOOLCHAIN", "TOOLCHAINS")
 load("//img/private/common:transitions.bzl", "reset_platform_transition")
+load("//img/private/repository_rules:registry.bzl", "get_sources_list")
 
 def _download_blob(ctx, output):
     """Download a layer from a container registry."""
@@ -18,6 +19,9 @@ def _download_blob(ctx, output):
 
     img_toolchain_info = ctx.toolchains[TOOLCHAIN].imgtoolchaininfo
 
+    # Convert sources dict to list of "repository@registry" strings
+    sources_list = get_sources_list(ctx.attr.sources)
+
     # Only set REGISTRY_AUTH_FILE if docker_config_path is non-empty
     docker_config_path = ctx.attr._docker_config_path[BuildSettingInfo].value
     env = {}
@@ -32,13 +36,11 @@ def _download_blob(ctx, output):
             "download-blob",
             "--digest",
             digest,
-            "--repository",
-            ctx.attr.repository,
             "--output",
             output.path,
         ] + [
-            "--registry={}".format(r)
-            for r in ctx.attr.registries
+            "--source={}".format(source)
+            for source in sources_list
         ],
         env = env,
         mnemonic = "DownloadBlob",
@@ -68,11 +70,17 @@ download_blobs = rule(
             doc = "List of digests to download.",
             mandatory = True,
         ),
-        "registries": attr.string_list(
-            doc = "List of registry mirrors used to pull the image.",
-        ),
-        "repository": attr.string(
-            doc = "Repository name of the image.",
+        "sources": attr.string_list_dict(
+            mandatory = True,
+            doc = """Mapping of image repositories to lists of registries that serve them.
+
+Each entry specifies a repository path and the registries that can serve it:
+- Key: The image repository (e.g., "library/ubuntu", "my-project/my-image")
+- Value: List of registries that serve this repository
+
+All repository@registry combinations will be tried (in random order for load distribution).
+
+If a registry list is empty, it defaults to Docker Hub (index.docker.io).""",
         ),
         "_docker_config_path": attr.label(
             default = Label("//img/settings:docker_config_path"),

img/private/extensions/images.bzl

@@ -1,7 +1,7 @@
 """Module extension for pulling container images."""
 
 load("@bazel_skylib//lib:sets.bzl", "sets")
-load("//img/private/extensions:images_helpers.bzl", "build_facts_to_store", "build_image_files_dict", "build_reverse_blob_mappings", "collect_blobs_to_create", "download_and_parse_manifest", "get_registries_from_image", "merge_pull_attrs", "pull_tag_to_struct")
+load("//img/private/extensions:images_helpers.bzl", "build_facts_to_store", "build_image_files_dict", "build_reverse_blob_mappings", "collect_blobs_to_create", "download_and_parse_manifest", "get_merged_sources_from_images", "get_registries_from_image", "merge_pull_attrs", "pull_tag_to_struct")
 load("//img/private/repository_rules:image_repo.bzl", "image_repo")
 load("//img/private/repository_rules:pull_blob.bzl", "pull_blob_file", "pull_manifest_blob")
 
@@ -50,7 +50,7 @@ def _images_impl(ctx):
             if digest not in images_by_digest:
                 images_by_digest[digest] = pull_tag_to_struct(img)
             else:
-                images_by_digest[digest] = merge_pull_attrs(images_by_digest[digest], img, other_is_root = mod.is_root)
+                images_by_digest[digest] = merge_pull_attrs(images_by_digest[digest], pull_tag_to_struct(img), other_is_root = mod.is_root)
             if digest not in digest_visibility:
                 digest_visibility[digest] = []
             visibility_identifier = "{}/{}/{}".format(mod.name, mod.version, img.name or img.repository)
@@ -84,48 +84,34 @@ def _images_impl(ctx):
 
     # Create blob repositories for manifest/index blobs (deduplicated)
     for digest in manifest_blobs.keys():
-        # Use reverse mapping to find source image for this manifest blob
+        # Use reverse mapping to find all source images for this manifest blob
         if digest not in manifest_blob_to_images or len(manifest_blob_to_images[digest]) == 0:
             fail("Could not find source image for manifest/index digest '{}'.".format(digest))
-        source_image_digest = manifest_blob_to_images[digest][0]
-        source_img = images_by_digest[source_image_digest]
 
-        # Get registry info from source image
-        registry = None
-        registries = get_registries_from_image(source_img)
-        if hasattr(source_img, "registry") and source_img.registry:
-            registry = source_img.registry
+        # Build merged sources from all images that serve this blob
+        sources = get_merged_sources_from_images(manifest_blob_to_images[digest], images_by_digest)
 
         repo_name = "blob_{}".format(digest.replace("sha256:", "").replace(":", "_"))
         pull_manifest_blob(
             name = repo_name,
-            registry = registry,
-            registries = registries,
-            repository = source_img.repository,
+            sources = sources,
             digest = digest,
             downloader = downloader,
         )
 
     # Create blob repositories for config/layer blobs (deduplicated, eager)
     for digest in file_blobs.keys():
-        # Use reverse mapping to find source image for this file blob
+        # Use reverse mapping to find all source images for this file blob
         if digest not in file_blob_to_images or len(file_blob_to_images[digest]) == 0:
             fail("Could not find source image for config/layer blob digest '{}'.".format(digest))
-        source_image_digest = file_blob_to_images[digest][0]
-        source_img = images_by_digest[source_image_digest]
 
-        # Get registry info from source image
-        registries = get_registries_from_image(source_img)
-        registry = None
-        if hasattr(source_img, "registry") and source_img.registry:
-            registry = source_img.registry
+        # Build merged sources from all images that serve this blob
+        sources = get_merged_sources_from_images(file_blob_to_images[digest], images_by_digest)
 
         repo_name = "blob_{}".format(digest.replace("sha256:", "").replace(":", "_"))
         pull_blob_file(
             name = repo_name,
-            registry = registry,
-            registries = registries,
-            repository = source_img.repository,
+            sources = sources,
             digest = digest,
             downloaded_file_path = "blob",
             handling = "eager",
@@ -134,24 +120,17 @@ def _images_impl(ctx):
 
     # Create blob repositories for lazy layer blobs (deduplicated, lazy)
     for digest in lazy_file_blobs.keys():
-        # Use reverse mapping to find source image for this file blob
+        # Use reverse mapping to find all source images for this file blob
         if digest not in file_blob_to_images or len(file_blob_to_images[digest]) == 0:
             fail("Could not find source image for lazy layer blob digest '{}'.".format(digest))
-        source_image_digest = file_blob_to_images[digest][0]
-        source_img = images_by_digest[source_image_digest]
 
-        # Get registry info from source image
-        registries = get_registries_from_image(source_img)
-        registry = None
-        if hasattr(source_img, "registry") and source_img.registry:
-            registry = source_img.registry
+        # Build merged sources from all images that serve this blob
+        sources = get_merged_sources_from_images(file_blob_to_images[digest], images_by_digest)
 
         repo_name = "lazy_{}".format(digest.replace("sha256:", "").replace(":", "_"))
         pull_blob_file(
             name = repo_name,
-            registry = registry,
-            registries = registries,
-            repository = source_img.repository,
+            sources = sources,
             digest = digest,
             downloaded_file_path = "blob",
             handling = "lazy",