From ff28a62796594c5d1fdbf5bff789d5a05b759f1d Mon Sep 17 00:00:00 2001 From: Sylvain Boily <4981802+djsly@users.noreply.github.com> Date: Tue, 16 Dec 2025 17:50:31 -0500 Subject: [PATCH] adding new logic for repo depot configuration for all Ubuntu versions and all AzLinux version. --- .../init-aks-custom-cloud-mariner.sh | 95 ++++++++- .../artifacts/init-aks-custom-cloud.sh | 193 ++++++++++++++++-- 2 files changed, 270 insertions(+), 18 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/init-aks-custom-cloud-mariner.sh b/parts/linux/cloud-init/artifacts/init-aks-custom-cloud-mariner.sh index 87395fa2ce5..2f6531fb9ef 100644 --- a/parts/linux/cloud-init/artifacts/init-aks-custom-cloud-mariner.sh +++ b/parts/linux/cloud-init/artifacts/init-aks-custom-cloud-mariner.sh @@ -1,6 +1,27 @@ #!/bin/bash set -x mkdir -p /root/AzureCACertificates + +IS_MARINER=0 +IS_AZURELINUX=0 +if [[ -f /etc/os-release ]]; then + . /etc/os-release + if [[ $NAME == *"Mariner"* ]]; then + IS_MARINER=1 + elif [[ $NAME == *"Microsoft Azure Linux"* ]]; then + IS_AZURELINUX=1 + else + echo "Unknown Linux distribution" + exit 1 + fi +else + echo "Unsupported operating system" + exit 1 +fi + +echo "distribution is $distribution" +echo "Running on $NAME" + # http://168.63.129.16 is a constant for the host's wireserver endpoint certs=$(curl "http://168.63.129.16/machine?comp=acmspackage&type=cacertificates&ext=json") IFS_backup=$IFS @@ -17,15 +38,77 @@ cp /root/AzureCACertificates/*.crt /etc/pki/ca-trust/source/anchors/ cloud-init status --wait +function init_mariner_repo_depot { + local repodepot_endpoint=$1 + echo "Adding [extended] repo" + cp /etc/yum.repos.d/mariner-extras.repo /etc/yum.repos.d/mariner-extended.repo + sed -i -e "s|extras|extended|" /etc/yum.repos.d/mariner-extended.repo + sed -i -e "s|Extras|Extended|" /etc/yum.repos.d/mariner-extended.repo + + echo "Adding [nvidia] repo" + cp /etc/yum.repos.d/mariner-extras.repo /etc/yum.repos.d/mariner-nvidia.repo + sed -i -e "s|extras|nvidia|" /etc/yum.repos.d/mariner-nvidia.repo + sed -i -e "s|Extras|Nvidia|" /etc/yum.repos.d/mariner-nvidia.repo + + echo "Adding [cloud-native] repo" + cp /etc/yum.repos.d/mariner-extras.repo /etc/yum.repos.d/mariner-cloud-native.repo + sed -i -e "s|extras|cloud-native|" /etc/yum.repos.d/mariner-cloud-native.repo + sed -i -e "s|Extras|Cloud-Native|" /etc/yum.repos.d/mariner-cloud-native.repo + + echo "Pointing Mariner repos at RepoDepot..." + for f in /etc/yum.repos.d/*.repo + do + sed -i -e "s|https://packages.microsoft.com|${repodepot_endpoint}/mariner/packages.microsoft.com|" $f + echo "$f modified." + done + echo "Mariner repo setup complete." +} + +function init_azurelinux_repo_depot { + local repodepot_endpoint=$1 + repos=("amd" "base" "cloud-native" "extended" "ms-non-oss" "ms-oss" "nvidia") + + # tbd maybe we do this a bit nicer + rm -f /etc/yum.repos.d/azurelinux* + + for repo in "${repos[@]}"; do + output_file="/etc/yum.repos.d/azurelinux-${repo}.repo" + repo_content=( + "[azurelinux-official-$repo]" + "name=Azure Linux Official $repo \$releasever \$basearch" + "baseurl=$repodepot_endpoint/azurelinux/\$releasever/prod/$repo/\$basearch" + "gpgkey=file:///etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY" + "gpgcheck=1" + "repo_gpgcheck=1" + "enabled=1" + "skip_if_unavailable=True" + "sslverify=1" + ) + + rm -f "$output_file" + + for line in "${repo_content[@]}"; do + echo "$line" >> "$output_file" + done + + echo "File '$output_file' has been created." + done +} + marinerRepoDepotEndpoint="$(echo "${REPO_DEPOT_ENDPOINT}" | sed 's/\/ubuntu//')" if [ -z "$marinerRepoDepotEndpoint" ]; then >&2 echo "repo depot endpoint empty while running custom-cloud init script" else - for f in /etc/yum.repos.d/*.repo - do - sed -i -e "s|https://packages.microsoft.com|${marinerRepoDepotEndpoint}/mariner/packages.microsoft.com|" "$f" - echo "## REPO - $f - MODIFIED" - done + # logic taken from https://repodepot.azure.com/scripts/cloud-init/setup_repodepot.sh + if [ "$IS_MARINER" -eq 1 ]; then + echo "Initializing Mariner repo depot settings..." + init_mariner_repo_depot ${marinerRepoDepotEndpoint} + elif [ "$IS_AZURELINUX" -eq 1 ]; then + echo "Initializing Azure Linux repo depot settings..." + init_azurelinux_repo_depot ${marinerRepoDepotEndpoint} + else + echo "No customizations for distribution: $NAME" + fi fi # Set the chrony config to use the PHC /dev/ptp0 clock @@ -58,4 +141,4 @@ EOF systemctl restart chronyd -#EOF \ No newline at end of file +#EOF diff --git a/parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh b/parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh index a1de8eed0b8..5300d282ef5 100644 --- a/parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh +++ b/parts/linux/cloud-init/artifacts/init-aks-custom-cloud.sh @@ -2,12 +2,26 @@ set -x mkdir -p /root/AzureCACertificates -# For Flatcar: systemd timer instead of cron, skip cloud-init/apt ops, chronyd service name). IS_FLATCAR=0 -if [ -f /etc/os-release ] && grep -qi '^ID=flatcar' /etc/os-release; then - IS_FLATCAR=1 +IS_UBUNTU=0 +if [[ -f /etc/os-release ]]; then + . /etc/os-release + if [[ $NAME == *"Ubuntu"* ]]; then + IS_UBUNTU=1 + elif [[ $ID == *"flatcar"* ]]; then + IS_FLATCAR=1 + else + echo "Unknown Linux distribution" + exit 1 + fi +else + echo "Unsupported operating system" + exit 1 fi +echo "distribution is $distribution" +echo "Running on $NAME" + # http://168.63.129.16 is a constant for the host's wireserver endpoint certs=$(curl "http://168.63.129.16/machine?comp=acmspackage&type=cacertificates&ext=json") IFS_backup=$IFS @@ -41,13 +55,168 @@ if [ "$action" = "ca-refresh" ]; then exit fi -if [ "$IS_FLATCAR" -eq 0 ]; then +function init_ubuntu_main_repo_depot { + local repodepot_endpoint="$1" + # Initialize directory for keys + mkdir -p /etc/apt/keyrings + + # This copies the updated bundle to the location used by OpenSSL which is commonly used + echo "Copying updated bundle to OpenSSL .pem file..." + cp /etc/ssl/certs/ca-certificates.crt /usr/lib/ssl/cert.pem + echo "Updated bundle copied." + + # Back up sources.list and sources.list.d contents + mkdir -p /etc/apt/backup/ + if [ -f "/etc/apt/sources.list" ]; then + mv /etc/apt/sources.list /etc/apt/backup/ + fi + for sources_file in /etc/apt/sources.list.d/*; do + if [ -f "$sources_file" ]; then + mv "$sources_file" /etc/apt/backup/ + fi + done + + # Set location of sources file + . /etc/os-release + aptSourceFile="/etc/apt/sources.list.d/ubuntu.sources" + + # Create main sources file + cat < /etc/apt/sources.list.d/ubuntu.sources + +Types: deb +URIs: ${repodepot_endpoint}/ubuntu +Suites: ${VERSION_CODENAME} ${VERSION_CODENAME}-updates ${VERSION_CODENAME}-backports ${VERSION_CODENAME}-security +Components: main universe restricted multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +EOF + + # Update the apt sources file using the RepoDepot Ubuntu URL for this cloud. Update it by replacing + # all urls with the RepoDepot Ubuntu url + ubuntuUrl=${repodepot_endpoint}/ubuntu + echo "Converting URLs in $aptSourceFile to RepoDepot URLs..." + sed -i "s,https\?://.[^ ]*,$ubuntuUrl,g" $aptSourceFile + echo "apt source URLs converted, see new file below:" + echo "" + echo "-----" + cat $aptSourceFile + echo "-----" + echo "" +} + +function check_url { + local url=$1 + echo "Checking url: $url" + + # Use curl to check the URL and capture both stdout and stderr + curl_exit_code=$(curl -s --head --request GET $url) + # Check the exit status of curl + if [[ $? -ne 0 ]] || echo "$curl_exit_code" | grep -E "404 Not Found" > /dev/null; then + echo "ERROR: $url is not available. Please manually check if the url is valid before re-running script" + exit 1 + fi +} + +function write_to_sources_file { + local sources_list_d_file=$1 + local source_uri=$2 + shift 2 + local key_paths=("$@") + + sources_file_path="/etc/apt/sources.list.d/${sources_list_d_file}.sources" + ubuntuDist=$(lsb_release -c | awk '{print $2}') + if [ "$sources_list_d_file" == "microsoft-prod-testing" ]; then + ubuntuDist="testing" + fi + + tee -a $sources_file_path < /dev/null + echo "$key_name key added to keyring." +} + +function derive_key_paths { + local key_names=("$@") + local key_paths=() + + for key_name in "${key_names[@]}"; do + key_paths+=("/etc/apt/keyrings/${key_name}.gpg") + done + + echo "${key_paths[*]}" +} + +function add_ms_keys { + # Add the Microsoft package server keys to keyring. + echo "Adding Microsoft keys to keyring..." + + add_key_ubuntu microsoft.asc + add_key_ubuntu msopentech.asc +} + +function aptget_update { + # If no connectivity, return early and don't bother trying an apt-get update + if [[ $no_connectivity == "true" ]]; then + return + fi + + echo "apt-get updating..." + echo "note: depending on how many sources have been added this may take a couple minutes..." + if apt-get update | grep -q "404 Not Found"; then + echo "ERROR: apt-get update failed to find all sources. Please validate the sources or remove bad sources from your sources and try again." + exit 1 + else + echo "apt-get update complete!" + fi +} + +function init_ubuntu_pmc_repo_depot { + local repodepot_endpoint="$1" + # Add Microsoft packages source to the azure specific sources.list. + echo "Adding the packages.microsoft.com Ubuntu-$ubuntuRel repo..." + + microsoftPackageSource="$repodepot_endpoint/microsoft/ubuntu/$ubuntuRel/prod" + check_url $microsoftPackageSource + write_to_sources_file microsoft-prod $microsoftPackageSource $(derive_key_paths microsoft.asc msopentech.asc) + write_to_sources_file microsoft-prod-testing $microsoftPackageSource $(derive_key_paths microsoft.asc msopentech.asc) + echo "Ubuntu ($ubuntuRel) repo added." + echo "Adding packages.microsoft.com keys" + add_ms_keys $repodepot_endpoint +} + +if [ "$IS_UBUNTU" -eq 1 ]; then (crontab -l ; echo "0 19 * * * $0 ca-refresh") | crontab - cloud-init status --wait - repoDepotEndpoint="${REPO_DEPOT_ENDPOINT}" - sudo sed -i "s,http://.[^ ]*,$repoDepotEndpoint,g" /etc/apt/sources.list -else + rootRepoDepotEndpoint="$(echo "${REPO_DEPOT_ENDPOINT}" | sed 's/\/ubuntu//')" + # logic taken from https://repodepot.azure.com/scripts/cloud-init/setup_repodepot.sh + ubuntuRel=$(lsb_release --release | awk '{print $2}') + ubuntuDist=$(lsb_release -c | awk '{print $2}') + # initialize archive.ubuntu.com repo + init_ubuntu_main_repo_depot ${rootRepoDepotEndpoint} + init_ubuntu_pmc_repo_depot ${rootRepoDepotEndpoint} + # update apt list + echo "Running apt-get update" + aptget_update +elif [ "$IS_FLATCAR" -eq 1 ]; then script_path="$(readlink -f "$0")" svc="/etc/systemd/system/azure-ca-refresh.service" tmr="/etc/systemd/system/azure-ca-refresh.timer" @@ -79,7 +248,7 @@ fi # Disable systemd-timesyncd and install chrony and uses local time source chrony_conf="/etc/chrony/chrony.conf" -if [ "$IS_FLATCAR" -eq 0 ]; then +if [ "$IS_UBUNTU" -eq 1 ]; then systemctl stop systemd-timesyncd systemctl disable systemd-timesyncd @@ -87,7 +256,7 @@ if [ "$IS_FLATCAR" -eq 0 ]; then apt-get update apt-get install chrony -y fi -else +elif [ "$IS_FLATCAR" -eq 1 ]; then rm -f ${chrony_conf} fi @@ -139,10 +308,10 @@ refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0 makestep 1.0 -1 EOF -if [ "$IS_FLATCAR" -eq 0 ]; then +if [ "$IS_UBUNTU" -eq 1 ]; then systemctl restart chrony -else +elif [ "$IS_FLATCAR" -eq 1 ]; then systemctl restart chronyd fi -#EOF \ No newline at end of file +#EOF