fix: fixing repodepot support for azlinux3 and ubuntu2404 as well as PMC on Ubuntu

parts/linux/cloud-init/artifacts/init-aks-custom-cloud-mariner.sh

@@ -1,6 +1,27 @@
 #!/bin/bash
 set -x
 mkdir -p /root/AzureCACertificates
+
+IS_MARINER=0
+IS_AZURELINUX=0
+if [[ -f /etc/os-release ]]; then
+        . /etc/os-release
+    if [[ $NAME == *"Mariner"* ]]; then
+        IS_MARINER=1
+    elif [[ $NAME == *"Microsoft Azure Linux"* ]]; then
+        IS_AZURELINUX=1
+    else
+        echo "Unknown Linux distribution"
+        exit 1
+    fi
+else
+    echo "Unsupported operating system"
+    exit 1
+fi
+
+echo "distribution is $distribution"
+echo "Running on $NAME"
+
 # http://168.63.129.16 is a constant for the host's wireserver endpoint
 certs=$(curl "http://168.63.129.16/machine?comp=acmspackage&type=cacertificates&ext=json")
 IFS_backup=$IFS
@@ -17,15 +38,77 @@ cp /root/AzureCACertificates/*.crt /etc/pki/ca-trust/source/anchors/
 
 cloud-init status --wait
 
+function init_mariner_repo_depot {
+    local repodepot_endpoint=$1
+    echo "Adding [extended] repo"
+    cp /etc/yum.repos.d/mariner-extras.repo /etc/yum.repos.d/mariner-extended.repo
+    sed -i -e "s|extras|extended|" /etc/yum.repos.d/mariner-extended.repo
+    sed -i -e "s|Extras|Extended|" /etc/yum.repos.d/mariner-extended.repo
+
+    echo "Adding [nvidia] repo"
+    cp /etc/yum.repos.d/mariner-extras.repo /etc/yum.repos.d/mariner-nvidia.repo
+    sed -i -e "s|extras|nvidia|" /etc/yum.repos.d/mariner-nvidia.repo
+    sed -i -e "s|Extras|Nvidia|" /etc/yum.repos.d/mariner-nvidia.repo
+
+    echo "Adding [cloud-native] repo"
+    cp /etc/yum.repos.d/mariner-extras.repo /etc/yum.repos.d/mariner-cloud-native.repo
+    sed -i -e "s|extras|cloud-native|" /etc/yum.repos.d/mariner-cloud-native.repo
+    sed -i -e "s|Extras|Cloud-Native|" /etc/yum.repos.d/mariner-cloud-native.repo
+
+    echo "Pointing Mariner repos at RepoDepot..."
+    for f in /etc/yum.repos.d/*.repo
+    do
+        sed -i -e "s|https://packages.microsoft.com|${repodepot_endpoint}/mariner/packages.microsoft.com|" $f
+        echo "$f modified."
+    done
+    echo "Mariner repo setup complete."
+}
+
+function init_azurelinux_repo_depot {
+    local repodepot_endpoint=$1
+    repos=("amd" "base" "cloud-native" "extended" "ms-non-oss" "ms-oss" "nvidia")
+
+    # tbd maybe we do this a bit nicer
+    rm -f /etc/yum.repos.d/azurelinux*
+
+    for repo in "${repos[@]}"; do
+        output_file="/etc/yum.repos.d/azurelinux-${repo}.repo"
+        repo_content=(
+            "[azurelinux-official-$repo]"
+            "name=Azure Linux Official $repo \$releasever \$basearch"
+            "baseurl=$repodepot_endpoint/azurelinux/\$releasever/prod/$repo/\$basearch"
+            "gpgkey=file:///etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY"
+            "gpgcheck=1"
+            "repo_gpgcheck=1"
+            "enabled=1"
+            "skip_if_unavailable=True"
+            "sslverify=1"
+        )
+
+        rm -f "$output_file"
+
+        for line in "${repo_content[@]}"; do
+            echo "$line" >> "$output_file"
+        done
+
+        echo "File '$output_file' has been created."
+    done
+}
+
 marinerRepoDepotEndpoint="$(echo "${REPO_DEPOT_ENDPOINT}" | sed 's/\/ubuntu//')"
 if [ -z "$marinerRepoDepotEndpoint" ]; then
   >&2 echo "repo depot endpoint empty while running custom-cloud init script"
 else
-  for f in /etc/yum.repos.d/*.repo
-  do
-      sed -i -e "s|https://packages.microsoft.com|${marinerRepoDepotEndpoint}/mariner/packages.microsoft.com|" "$f"
-      echo "## REPO - $f - MODIFIED"
-  done
+  # logic taken from https://repodepot.azure.com/scripts/cloud-init/setup_repodepot.sh
+  if [ "$IS_MARINER" -eq 1 ]; then
+      echo "Initializing Mariner repo depot settings..."
+      init_mariner_repo_depot ${marinerRepoDepotEndpoint}
+  elif [ "$IS_AZURELINUX" -eq 1 ]; then
+      echo "Initializing Azure Linux repo depot settings..."
+      init_azurelinux_repo_depot ${marinerRepoDepotEndpoint}
+  else
+      echo "No customizations for distribution: $NAME"
+  fi
 fi
 
 # Set the chrony config to use the PHC /dev/ptp0 clock
@@ -58,4 +141,4 @@ EOF
 
 systemctl restart chronyd
 
-#EOF
+#EOF

parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh

@@ -2,12 +2,26 @@
 set -x
 mkdir -p /root/AzureCACertificates
 
-# For Flatcar: systemd timer instead of cron, skip cloud-init/apt ops, chronyd service name).
 IS_FLATCAR=0
-if [ -f /etc/os-release ] && grep -qi '^ID=flatcar' /etc/os-release; then
-  IS_FLATCAR=1
+IS_UBUNTU=0
+if [[ -f /etc/os-release ]]; then
+    . /etc/os-release
+    if [[ $NAME == *"Ubuntu"* ]]; then
+        IS_UBUNTU=1
+    elif [[ $ID == *"flatcar"* ]]; then
+        IS_FLATCAR=1
+    else
+        echo "Unknown Linux distribution"
+        exit 1
+    fi
+else
+    echo "Unsupported operating system"
+    exit 1
 fi
 
+echo "distribution is $distribution"
+echo "Running on $NAME"
+
 # http://168.63.129.16 is a constant for the host's wireserver endpoint
 certs=$(curl "http://168.63.129.16/machine?comp=acmspackage&type=cacertificates&ext=json")
 IFS_backup=$IFS
@@ -41,13 +55,168 @@ if [ "$action" = "ca-refresh" ]; then
     exit
 fi
 
-if [ "$IS_FLATCAR" -eq 0 ]; then
+function init_ubuntu_main_repo_depot {
+    local repodepot_endpoint="$1"
+    # Initialize directory for keys
+    mkdir -p /etc/apt/keyrings
+
+    # This copies the updated bundle to the location used by OpenSSL which is commonly used
+    echo "Copying updated bundle to OpenSSL .pem file..."
+    cp /etc/ssl/certs/ca-certificates.crt /usr/lib/ssl/cert.pem
+    echo "Updated bundle copied."
+
+    # Back up sources.list and sources.list.d contents
+    mkdir -p /etc/apt/backup/
+    if [ -f "/etc/apt/sources.list" ]; then
+        mv /etc/apt/sources.list /etc/apt/backup/
+    fi
+    for sources_file in /etc/apt/sources.list.d/*; do
+        if [ -f "$sources_file" ]; then
+            mv "$sources_file" /etc/apt/backup/
+        fi
+    done
+
+    # Set location of sources file
+    . /etc/os-release
+    aptSourceFile="/etc/apt/sources.list.d/ubuntu.sources"
+
+    # Create main sources file
+    cat <<EOF > /etc/apt/sources.list.d/ubuntu.sources
+
+Types: deb
+URIs: ${repodepot_endpoint}/ubuntu
+Suites: ${VERSION_CODENAME} ${VERSION_CODENAME}-updates ${VERSION_CODENAME}-backports ${VERSION_CODENAME}-security
+Components: main universe restricted multiverse
+Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
+EOF
+
+    # Update the apt sources file using the RepoDepot Ubuntu URL for this cloud. Update it by replacing
+    # all urls with the RepoDepot Ubuntu url
+    ubuntuUrl=${repodepot_endpoint}/ubuntu
+    echo "Converting URLs in $aptSourceFile to RepoDepot URLs..."
+    sed -i "s,https\?://.[^ ]*,$ubuntuUrl,g" $aptSourceFile
+    echo "apt source URLs converted, see new file below:"
+    echo ""
+    echo "-----"
+    cat $aptSourceFile
+    echo "-----"
+    echo ""
+}
+
+function check_url {
+    local url=$1
+    echo "Checking url: $url"
+
+    # Use curl to check the URL and capture both stdout and stderr
+    curl_exit_code=$(curl -s --head --request GET $url)
+    # Check the exit status of curl
+    if [[ $? -ne 0 ]] || echo "$curl_exit_code" | grep -E "404 Not Found" > /dev/null; then
+        echo "ERROR: $url is not available. Please manually check if the url is valid before re-running script"
+        exit 1
+    fi
+}
+
+function write_to_sources_file {
+    local sources_list_d_file=$1
+    local source_uri=$2
+    shift 2
+    local key_paths=("$@")
+
+    sources_file_path="/etc/apt/sources.list.d/${sources_list_d_file}.sources"
+    ubuntuDist=$(lsb_release -c | awk '{print $2}')
+    if [ "$sources_list_d_file" == "microsoft-prod-testing" ]; then
+        ubuntuDist="testing"
+    fi
+
+    tee -a $sources_file_path <<EOF
+
+Types: deb
+URIs: $source_uri
+Suites: $ubuntuDist
+Components: main
+Arch: amd64
+Signed-By: ${key_paths[*]}
+EOF
+}
+
+function add_key_ubuntu {
+    local key_name=$1
+
+    if [[ $no_connectivity == "false" ]]; then
+        key_url="${repodepot_endpoint}/keys/${key_name}"
+        check_url $key_url
+        echo "Adding $key_name key to keyring..."
+        key_data=$(wget -O - $key_url)
+    fi
+    key_path=$(derive_key_paths $key_name)
+    echo "$key_data" | gpg --dearmor | tee $key_path > /dev/null
+    echo "$key_name key added to keyring."
+}
+
+function derive_key_paths {
+    local key_names=("$@")
+    local key_paths=()
+
+    for key_name in "${key_names[@]}"; do
+        key_paths+=("/etc/apt/keyrings/${key_name}.gpg")
+    done
+
+    echo "${key_paths[*]}"
+}
+
+function add_ms_keys {
+    # Add the Microsoft package server keys to keyring.
+    echo "Adding Microsoft keys to keyring..."
+
+    add_key_ubuntu microsoft.asc
+    add_key_ubuntu msopentech.asc
+}
+
+function aptget_update {
+    # If no connectivity, return early and don't bother trying an apt-get update
+    if [[ $no_connectivity == "true" ]]; then
+        return
+    fi
+
+    echo "apt-get updating..."
+    echo "note: depending on how many sources have been added this may take a couple minutes..."
+    if apt-get update | grep -q "404 Not Found"; then
+        echo "ERROR: apt-get update failed to find all sources. Please validate the sources or remove bad sources from your sources and try again."
+        exit 1
+    else
+        echo "apt-get update complete!"
+    fi
+}
+
+function init_ubuntu_pmc_repo_depot {
+    local repodepot_endpoint="$1"
+    # Add Microsoft packages source to the azure specific sources.list.
+    echo "Adding the packages.microsoft.com Ubuntu-$ubuntuRel repo..."
+
+    microsoftPackageSource="$repodepot_endpoint/microsoft/ubuntu/$ubuntuRel/prod"
+    check_url $microsoftPackageSource
+    write_to_sources_file microsoft-prod $microsoftPackageSource $(derive_key_paths microsoft.asc msopentech.asc)
+    write_to_sources_file microsoft-prod-testing $microsoftPackageSource $(derive_key_paths microsoft.asc msopentech.asc)
+    echo "Ubuntu ($ubuntuRel) repo added."
+    echo "Adding packages.microsoft.com keys"
+    add_ms_keys $repodepot_endpoint
+}
+
+if [ "$IS_UBUNTU" -eq 1 ]; then
     (crontab -l ; echo "0 19 * * * $0 ca-refresh") | crontab -
 
     cloud-init status --wait
-    repoDepotEndpoint="${REPO_DEPOT_ENDPOINT}"
-    sudo sed -i "s,http://.[^ ]*,$repoDepotEndpoint,g" /etc/apt/sources.list
-else
+    rootRepoDepotEndpoint="$(echo "${REPO_DEPOT_ENDPOINT}" | sed 's/\/ubuntu//')"
+    # logic taken from https://repodepot.azure.com/scripts/cloud-init/setup_repodepot.sh
+    ubuntuRel=$(lsb_release --release | awk '{print $2}')
+    ubuntuDist=$(lsb_release -c | awk '{print $2}')
+    # initialize archive.ubuntu.com repo
+    init_ubuntu_main_repo_depot ${rootRepoDepotEndpoint}
+    init_ubuntu_pmc_repo_depot ${rootRepoDepotEndpoint}
+    # update apt list
+    echo "Running apt-get update"
+    aptget_update
+elif [ "$IS_FLATCAR" -eq 1 ]; then
     script_path="$(readlink -f "$0")"
     svc="/etc/systemd/system/azure-ca-refresh.service"
     tmr="/etc/systemd/system/azure-ca-refresh.timer"
@@ -79,15 +248,15 @@ fi
 
 # Disable systemd-timesyncd and install chrony and uses local time source
 chrony_conf="/etc/chrony/chrony.conf"
-if [ "$IS_FLATCAR" -eq 0 ]; then
+if [ "$IS_UBUNTU" -eq 1 ]; then
     systemctl stop systemd-timesyncd
     systemctl disable systemd-timesyncd
 
     if [ ! -e "$chrony_conf" ]; then
         apt-get update
         apt-get install chrony -y
     fi
-else
+elif [ "$IS_FLATCAR" -eq 1 ]; then
     rm -f ${chrony_conf}
 fi
 
@@ -139,10 +308,10 @@ refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0
 makestep 1.0 -1
 EOF
 
-if [ "$IS_FLATCAR" -eq 0 ]; then
+if [ "$IS_UBUNTU" -eq 1 ]; then
     systemctl restart chrony
-else
+elif [ "$IS_FLATCAR" -eq 1 ]; then
     systemctl restart chronyd
 fi
 
-#EOF
+#EOF