Support for Configuring Additional Container Images from User Registry to Pulp

common/library/module_utils/input_validation/schema/local_repo_config.json

@@ -2,6 +2,67 @@
   "$schema": "http://json-schema.org/draft-07/schema#",
   "type": "object",
   "properties": {
+    "user_registry": {
+      "type": [
+        "array",
+        "null"
+      ],
+      "items": {
+        "type": "object",
+        "properties": {
+          "host": {
+            "type": "string",
+            "minLength": 1,
+            "pattern": "^[a-zA-Z0-9.-]+:[0-9]+$"
+          },
+          "cert_path": {
+            "type": "string",
+            "pattern": "^$|^[a-zA-Z0-9/\\._-]*\\.crt$"
+          },
+          "key_path": {
+            "type": "string",
+            "pattern": "^$|^[a-zA-Z0-9/\\._-]*\\.key$"
+          }
+        },
+        "required": [
+          "host"
+        ],
+        "allOf": [
+          {
+            "if": {
+              "properties": {
+                "cert_path": {
+                  "minLength": 1
+                }
+              }
+            },
+            "then": {
+              "properties": {
+                "cert_path": {
+                  "pattern": "^[a-zA-Z0-9/\\._-]*\\.crt$"
+                }
+              }
+            }
+          },
+          {
+            "if": {
+              "properties": {
+                "key_path": {
+                  "minLength": 1
+                }
+              }
+            },
+            "then": {
+              "properties": {
+                "key_path": {
+                  "pattern": "^[a-zA-Z0-9/\\._-]*\\.key$"
+                }
+              }
+            }
+          }
+        ]
+      }   
+    },
     "user_repo_url_x86_64": {
       "type": [
         "array",
@@ -1082,4 +1143,4 @@
     "omnia_repo_url_rhel_x86_64"
   ],
   "additionalProperties": false
-}
+}

common/library/module_utils/input_validation/validation_flows/local_repo_validation.py

@@ -1,4 +1,4 @@
-# Copyright 2025 Dell Inc. or its subsidiaries. All Rights Reserved.
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -78,6 +78,22 @@ def validate_local_repo_config(input_file_path, data,
     errors = []
     base_repo_names = []
     local_repo_yml = create_file_path(input_file_path, file_names["local_repo_config"])
+
+    user_registry = data.get("user_registry") 
+    if user_registry:
+        for registry in user_registry:
+            host = registry.get("host")
+            cert_path = registry.get("cert_path")
+            key_path = registry.get("key_path")
+
+            # Validate user_registry certificate and key paths
+            if cert_path and not os.path.exists(cert_path):
+                errors.append(create_error_msg(local_repo_yml, "user_registry", 
+                                             f"Certificate file not found: {cert_path}"))
+
+            if key_path and not os.path.exists(key_path):
+                errors.append(create_error_msg(local_repo_yml, "user_registry", 
+                                             f"Key file not found: {key_path}"))
     repo_names = {}
     sub_result = check_subscription_status(logger)
     logger.info(f"validate_local_repo_config: Subscription status: {sub_result}")
@@ -113,6 +129,50 @@ def validate_local_repo_config(input_file_path, data,
     software_config_file_path = create_file_path(input_file_path, file_names["software_config"])
     software_config_json = load_json(software_config_file_path)
 
+    # Check if additional_packages is enabled and contains image packages
+    additional_packages_enabled = any(sw.get("name") == "additional_packages" for sw in software_config_json.get("softwares", []))
+    if additional_packages_enabled:
+        # Get arch values from additional_packages entry in software_config.json
+        additional_packages_archs = []
+        for software in software_config_json.get("softwares", []):
+            if software.get("name") == "additional_packages":
+                arch_list = software.get("arch", [])
+                additional_packages_archs = arch_list  # Get all archs
+                break
+
+        # Check each arch specific additional_packages.json
+        has_image_packages = False
+        for additional_packages_arch in additional_packages_archs:
+            additional_packages_path = create_file_path(
+                input_file_path,
+                f"config/{additional_packages_arch}/{software_config_json['cluster_os_type']}/{software_config_json['cluster_os_version']}/additional_packages.json"
+            )
+
+            if os.path.exists(additional_packages_path):
+                additional_packages_data = load_json(additional_packages_path)
+                has_image_packages = False
+
+                # Check all sections for image packages
+                for section_name, section_data in additional_packages_data.items():
+                    if isinstance(section_data, dict) and "cluster" in section_data:
+                        cluster_packages = section_data.get("cluster", [])
+
+                        for package in cluster_packages:
+                            if package.get("type") == "image":
+                                has_image_packages = True
+                                break
+
+                    if has_image_packages:
+                        break
+
+        # If any architecture has image packages, user_registry must be defined and not empty
+        if has_image_packages and user_registry is None:
+            errors.append(create_error_msg(
+                local_repo_yml,
+                "user_registry", 
+                "user_registry must be defined when additional_packages.json contains packages of type 'image'"
+            ))
+
     # Extra validation: custom_slurm must have <arch>_slurm_custom in user_repo_url_<arch>
     for sw in software_config_json["softwares"]:
         if sw["name"] == "slurm_custom":

common/library/module_utils/local_repo/config.py

@@ -1,4 +1,4 @@
-# Copyright 2025 Dell Inc. or its subsidiaries. All Rights Reserved.
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -35,8 +35,8 @@
 DEFAULT_STATUS_FILENAME = "status.csv"
 STATUS_CSV_HEADER = 'name,type,status\n'
 SOFTWARE_CSV_HEADER = "name,status"
-USER_REG_CRED_INPUT = "/opt/omnia/input/project_default/user_registry_credential.yml"
-USER_REG_KEY_PATH = "/opt/omnia/input/project_default/.local_repo_credentials_key"
+# USER_REG_CRED_INPUT = "/opt/omnia/input/project_default/user_registry_credential.yml"
+# USER_REG_KEY_PATH = "/opt/omnia/input/project_default/.local_repo_credentials_key"
 # ----------------------------
 # Software tasklist Defaults
 # Used by prepare_tasklist.py
@@ -110,8 +110,10 @@
 
     "create_container_remote_auth": "pulp container remote create --name %s --url %s --upstream-name %s --policy %s --include-tags '%s' --username %s --password '%s'",
 
-    "update_container_remote_auth": "pulp container remote update --name %s --url %s --upstream-name %s --policy %s --include-tags '%s' --username %s --password '%s'"
-
+    "update_container_remote_auth": "pulp container remote update --name %s --url %s --upstream-name %s --policy %s --include-tags '%s' --username %s --password '%s'",
+    "container_distribution_show": "pulp container distribution show --name %s | jq .repository",
+    "show_repository_version": "pulp container repository show --href %s | jq .latest_version_href",
+    "list_image_tags": "pulp show --href /pulp/api/v3/content/container/tags/?repository_version=%s"
 }
 OMNIA_CREDENTIALS_YAML_PATH = "/opt/omnia/input/project_default/omnia_config_credentials.yml"
 OMNIA_CREDENTIALS_VAULT_PATH = "/opt/omnia/input/project_default/.omnia_config_credentials_key"

common/library/module_utils/local_repo/download_image.py

@@ -1,4 +1,4 @@
-# Copyright 2025 Dell Inc. or its subsidiaries. All Rights Reserved.
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -206,22 +206,61 @@ def get_repo_url_and_content(package):
         ValueError: If the package prefix is not supported.
     """
     patterns = {
-         r"^(ghcr\.io)(/.+)": "https://ghcr.io",
-         r"^(docker\.io)(/.+)": "https://registry-1.docker.io",
-         r"^(quay\.io)(/.+)": "https://quay.io",
-         r"^(registry\.k8s\.io)(/.+)": "https://registry.k8s.io",
-         r"^(nvcr\.io)(/.+)": "https://nvcr.io",
-         r"^(public\.ecr\.aws)(/.+)": "https://public.ecr.aws",
-         r"^(gcr\.io)(/.+)": "https://gcr.io"
+        r"^(ghcr\.io)(:\d+)?(/.+)": "https://ghcr.io",
+        r"^(docker\.io)(:\d+)?(/.+)": "https://registry-1.docker.io",
+        r"^(quay\.io)(:\d+)?(/.+)": "https://quay.io",
+        r"^(registry\.k8s\.io)(:\d+)?(/.+)": "https://registry.k8s.io",
+        r"^(nvcr\.io)(:\d+)?(/.+)": "https://nvcr.io",
+        r"^(public\.ecr\.aws)(:\d+)?(/.+)": "https://public.ecr.aws",
+        r"^(gcr\.io)(:\d+)?(/.+)": "https://gcr.io",
     }
     for pattern, repo_url in patterns.items():
         match = re.match(pattern, package)
         if match:
             base_url = repo_url
-            package_content = match.group(2).lstrip("/")  # Remove leading slash
+
+            # If user provided a port, preserve it
+            if match.group(2):
+                base_url = f"{repo_url}{match.group(2)}"
+
+            package_content = match.group(3).lstrip("/")
             return base_url, package_content
 
-    raise ValueError(f"Unsupported package prefix for package: {package}")
+    # fallback for private / IP-based registries
+    match = re.match(r"^(?P<registry>[^/]+)(?P<path>/.*)$", package)
+    if match:
+        return f"https://{match.group('registry')}", match.group("path").lstrip("/")
+
+    raise ValueError(f"Invalid package format: {package}")
+
+
+# def get_repo_url_and_content(package):
+#     """
+#     Get the repository URL and content from a given package.
+#     Parameters:
+#         package (str): The package to extract the URL and content from.
+#     Returns:
+#         tuple: A tuple containing the repository URL and content.
+#     Raises:
+#         ValueError: If the package prefix is not supported.
+#     """
+#     patterns = {
+#          r"^(ghcr\.io)(/.+)": "https://ghcr.io",
+#          r"^(docker\.io)(/.+)": "https://registry-1.docker.io",
+#          r"^(quay\.io)(/.+)": "https://quay.io",
+#          r"^(registry\.k8s\.io)(/.+)": "https://registry.k8s.io",
+#          r"^(nvcr\.io)(/.+)": "https://nvcr.io",
+#          r"^(public\.ecr\.aws)(/.+)": "https://public.ecr.aws",
+#          r"^(gcr\.io)(/.+)": "https://gcr.io"
+#     }
+#     for pattern, repo_url in patterns.items():
+#         match = re.match(pattern, package)
+#         if match:
+#             base_url = repo_url
+#             package_content = match.group(2).lstrip("/")  # Remove leading slash
+#             return base_url, package_content
+
+#     raise ValueError(f"Unsupported package prefix for package: {package}")
 
 def process_image(package, status_file_path, version_variables,
                  user_registries,docker_username, docker_password, logger):
@@ -245,66 +284,79 @@ def process_image(package, status_file_path, version_variables,
     base_url, package_content = get_repo_url_and_content(package['package'])
     package_identifier = None
 
+    # Only check user registries for additional_packages
+    if user_registries and "additional_packages" in status_file_path:
+        result, package_identifier = handle_user_image_registry(
+            package,
+            package_content,
+            version_variables,
+            user_registries,
+            logger
+        )
 
-    if user_registries:
-        result, package_identifier = handle_user_image_registry(package, package_content,
-                                     version_variables, user_registries, logger)
-    # If user registry not found or no user registry given, proceed with public registry
-    if not result:
-        try:
-            repo_name_prefix = "container_repo_"
-            repository_name = f"{repo_name_prefix}{package['package'].replace('/', '_').replace(':', '_')}"
-            remote_name = f"remote_{package['package'].replace('/', '_')}"
-            package_identifier = package['package']
-            # Create container repository
-            with repository_creation_lock:
-                result = create_container_repository(repository_name, logger)
+        if not result:
+            logger.info(f"Image {package['package']} will not be synced to Pulp.")
+            status = "Failed"
+            return status
+
+        else:
+            logger.info(f"Image {package['package']} synced to Pulp.")
+            status = "Success"
+            return status
+
+    try:
+        repo_name_prefix = "container_repo_"
+        repository_name = f"{repo_name_prefix}{package['package'].replace('/', '_').replace(':', '_')}"
+        remote_name = f"remote_{package['package'].replace('/', '_').replace(':', '_')}"
+        package_identifier = package['package']
+
+        # Create container repository
+        with repository_creation_lock:
+            result = create_container_repository(repository_name, logger)
+        if result is False or (isinstance(result, dict) and result.get("returncode", 1) != 0):
+            raise Exception(f"Failed to create repository: {repository_name}")
+
+        # Process digest or tag
+        if "digest" in package:
+            package_identifier += f":{package['digest']}"
+            result = create_container_remote_digest(
+                remote_name, base_url, package_content, policy_type, logger
+            )
             if result is False or (isinstance(result, dict) and result.get("returncode", 1) != 0):
-                raise Exception(f"Failed to create repository: {repository_name}")
-            # Process digest or tag
-            if "digest" in package:
-                package_identifier += f":{package['digest']}"
-                result = create_container_remote_digest(remote_name, base_url,
-                         package_content, policy_type, logger)
-                if result is False or (isinstance(result, dict) and result.get("returncode", 1) != 0):
-                    raise Exception(f"Failed to create remote digest: {remote_name}")
-
-            elif "tag" in package:
-                tag_template = Template(package['tag'])
-                tag_val = tag_template.render(**version_variables)
-                package_identifier += f":{package['tag']}"
-
-                # Only use auth for docker.io images
-                if package['package'].startswith('docker.io/'):
-
-                    with remote_creation_lock:
-                        if docker_username and docker_password:
-                            result = create_container_remote_with_auth(
-                                remote_name, base_url, package_content, policy_type,
-                                tag_val, logger, docker_username, docker_password
-                            )
-                        else:
-                            result = create_container_remote(
-                                remote_name, base_url, package_content, policy_type, tag_val, logger
-                            )
+                raise Exception(f"Failed to create remote digest: {remote_name}")
+
+        elif "tag" in package:
+            tag_template = Template(package['tag'])
+            tag_val = tag_template.render(**version_variables)
+            package_identifier += f":{package['tag']}"
+
+            with remote_creation_lock:
+                if package['package'].startswith('docker.io/') and docker_username and docker_password:
+                    result = create_container_remote_with_auth(
+                        remote_name, base_url, package_content, policy_type,
+                        tag_val, logger, docker_username, docker_password
+                    )
                 else:
-                    # For non-docker.io registries, use unauthenticated access
-                    with remote_creation_lock:
-                        result = create_container_remote(
-                            remote_name, base_url, package_content, policy_type, tag_val, logger
-                        )
-
-                if result is False or (isinstance(result, dict) and result.get("returncode", 1) != 0):
-                    raise Exception(f"Failed to create remote: {remote_name}")
-            # Sync and distribute container repository
-            result = sync_container_repository(repository_name, remote_name, package_content,logger)
+                    result = create_container_remote(
+                        remote_name, base_url, package_content, policy_type, tag_val, logger
+                    )
+
             if result is False or (isinstance(result, dict) and result.get("returncode", 1) != 0):
-                raise Exception(f"Failed to sync repository: {repository_name}")
+                raise Exception(f"Failed to create remote: {remote_name}")
 
-        except Exception as e:
-            status = "Failed"
-            logger.error(f"Failed to process image: {package_identifier}. Error: {e}")
+        # Sync and distribute
+        result = sync_container_repository(
+            repository_name, remote_name, package_content, logger
+        )
+        if result is False or (isinstance(result, dict) and result.get("returncode", 1) != 0):
+            raise Exception(f"Failed to sync repository: {repository_name}")
+
+    except Exception as e:
+        status = "Failed"
+        logger.error(f"Failed to process image: {package_identifier}. Error: {e}")
 
-    write_status_to_file(status_file_path, package_identifier, package['type'], status, logger, file_lock)
+    write_status_to_file(
+        status_file_path, package_identifier, package['type'], status, logger, file_lock
+    )
     logger.info("#" * 30 + f" {process_image.__name__} end " + "#" * 30)
     return status