From 209944f7cfef76d3926e12367e5b8b749d3309b1 Mon Sep 17 00:00:00 2001 From: thespad Date: Mon, 30 Dec 2024 20:23:18 +0000 Subject: [PATCH 1/4] Support pulling mod by manifest hash --- README.md | 1 + docker-mods.v3 | 17 +++++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4558e5f..a474510 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ These files are used by Linuxserver build processes to handle mods in our images. Not for end-user consumption. +* **31.12.24:** - Support pulling mods using manifest hash. * **22.12.24:** - Add modcache support. * **26.06.24:** - Add RO and User handlers. * **10.06.24:** - Move lsiown to its own file. Remove support for legacy v2 and hybrid mods. diff --git a/docker-mods.v3 b/docker-mods.v3 index 721e791..8da3c1f 100755 --- a/docker-mods.v3 +++ b/docker-mods.v3 @@ -5,7 +5,7 @@ # Version 3 # 2022-09-25 - Initial Release -MOD_SCRIPT_VER="3.20241223" +MOD_SCRIPT_VER="3.20241231" # Define custom folder paths SCRIPTS_DIR="/custom-cont-init.d" @@ -298,15 +298,20 @@ run_mods() { ENDPOINT="${DOCKER_MOD%%:*}" USERNAME="${DOCKER_MOD%%/*}" REPO="${ENDPOINT#*/}" - TAG="${DOCKER_MOD#*:}" + FULLTAG="${DOCKER_MOD#*:}" + TAG="${FULLTAG%@*}" + TAGSHA="${DOCKER_MOD#*@}" + if [[ ! "${TAGSHA}" =~ sha256 ]]; then + unset TAGSHA + fi if [[ "${TAG}" == "${DOCKER_MOD}" ]]; then TAG="latest" fi - FILENAME="${USERNAME}.${REPO}.${TAG}" + FILENAME="${USERNAME}.${REPO}.${TAG}${TAGSHA:+.${TAGSHA#*:}}" MANIFEST_URL="https://${REGISTRY}/v2/${ENDPOINT}/manifests" BLOB_URL="https://${REGISTRY}/v2/${ENDPOINT}/blobs/" - MOD_UA="Mozilla/5.0 (Linux $(uname -m)) linuxserver.io ${REGISTRY}/${ENDPOINT}:${TAG}" - write_mod_debug "Registry='${REGISTRY}', Repository='${USERNAME}', Image='${ENDPOINT}', Tag='${TAG}'" + MOD_UA="Mozilla/5.0 (Linux $(uname -m)) linuxserver.io ${REGISTRY}/${ENDPOINT}:${TAG}${TAGSHA:+@$TAGSHA}" + write_mod_debug "Registry='${REGISTRY}', Repository='${USERNAME}', Image='${ENDPOINT}', Tag='${TAG}', TagSHA='${TAGSHA:-N/A}'" case "${REGISTRY}" in "lscr.io") AUTH_URL="https://ghcr.io/token?scope=repository%3A${USERNAME}%2F${REPO}%3Apull";; "ghcr.io") AUTH_URL="https://ghcr.io/token?scope=repository%3A${USERNAME}%2F${REPO}%3Apull";; @@ -360,7 +365,7 @@ run_mods() { MOD_OFFLINE="true" else # Determine first and only layer of image - SHALAYER=$(get_blob_sha "${TOKEN}" "${MANIFEST_URL}" "${TAG}" "${ARCH:--amd64}") + SHALAYER=$(get_blob_sha "${TOKEN}" "${MANIFEST_URL}" "${TAGSHA:-$TAG}" "${ARCH:--amd64}") if [[ $? -eq 1 ]]; then write_mod_error "No manifest available for arch ${ARCH:--amd64}, cannot fetch mod" continue From ed2093ab36497c39af2dc65d9ec0f7a55a4be879 Mon Sep 17 00:00:00 2001 From: thespad Date: Mon, 30 Dec 2024 20:25:42 +0000 Subject: [PATCH 2/4] More resilient check --- docker-mods.v3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-mods.v3 b/docker-mods.v3 index d408867..32de337 100755 --- a/docker-mods.v3 +++ b/docker-mods.v3 @@ -301,7 +301,7 @@ run_mods() { FULLTAG="${DOCKER_MOD#*:}" TAG="${FULLTAG%@*}" TAGSHA="${DOCKER_MOD#*@}" - if [[ ! "${TAGSHA}" =~ sha256 ]]; then + if [[ ! "${TAGSHA}" =~ sha256: ]]; then unset TAGSHA fi if [[ "${TAG}" == "${DOCKER_MOD}" ]]; then From 8d069dbd46b7e751740abfeb279b15bbbcdc9ebd Mon Sep 17 00:00:00 2001 From: thespad Date: Tue, 31 Dec 2024 16:25:23 +0000 Subject: [PATCH 3/4] Properly support all SHA manifest formats --- docker-mods.v3 | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/docker-mods.v3 b/docker-mods.v3 index 32de337..301f2d0 100755 --- a/docker-mods.v3 +++ b/docker-mods.v3 @@ -295,29 +295,43 @@ run_mods() { fi ;; esac - ENDPOINT="${DOCKER_MOD%%:*}" - USERNAME="${DOCKER_MOD%%/*}" - REPO="${ENDPOINT#*/}" - FULLTAG="${DOCKER_MOD#*:}" - TAG="${FULLTAG%@*}" - TAGSHA="${DOCKER_MOD#*@}" - if [[ ! "${TAGSHA}" =~ sha256: ]]; then + # Identify what kind of image name format we're working with + if [[ ${DOCKER_MOD} == *:*@* ]]; then + ENDPOINT="${DOCKER_MOD%%:*}" + USERNAME="${DOCKER_MOD%%/*}" + REPO="${ENDPOINT#*/}" + FULLTAG="${DOCKER_MOD#*:}" + TAG="${FULLTAG%@*}" + TAGSHA="${DOCKER_MOD#*@}" + elif [[ ${DOCKER_MOD} == *@* ]]; then + ENDPOINT="${DOCKER_MOD%%@*}" + USERNAME="${DOCKER_MOD%%/*}" + REPO="${ENDPOINT#*/}" + unset FULLTAG + unset TAG + TAGSHA="${DOCKER_MOD#*@}" + elif [[ ${DOCKER_MOD} == *:* ]]; then + ENDPOINT="${DOCKER_MOD%%:*}" + USERNAME="${DOCKER_MOD%%/*}" + REPO="${ENDPOINT#*/}" + unset FULLTAG + TAG="${DOCKER_MOD#*:}" unset TAGSHA fi if [[ "${TAG}" == "${DOCKER_MOD}" ]]; then TAG="latest" fi - FILENAME="${USERNAME}.${REPO}.${TAG}${TAGSHA:+.${TAGSHA#*:}}" + FILENAME="${USERNAME}.${REPO}${TAG:+.${TAG}}${TAGSHA:+.${TAGSHA#*:}}" MANIFEST_URL="https://${REGISTRY}/v2/${ENDPOINT}/manifests" BLOB_URL="https://${REGISTRY}/v2/${ENDPOINT}/blobs/" - MOD_UA="Mozilla/5.0 (Linux $(uname -m)) linuxserver.io ${REGISTRY}/${ENDPOINT}:${TAG}${TAGSHA:+@$TAGSHA}" - write_mod_debug "Registry='${REGISTRY}', Repository='${USERNAME}', Image='${ENDPOINT}', Tag='${TAG}', TagSHA='${TAGSHA:-N/A}'" + MOD_UA="Mozilla/5.0 (Linux $(uname -m)) linuxserver.io ${REGISTRY}/${ENDPOINT}${TAG:+:${TAG}}${TAGSHA:+@${TAGSHA}}" + write_mod_debug "Registry='${REGISTRY}', Repository='${USERNAME}', Image='${ENDPOINT}', Tag='${TAG:-N/A}', TagSHA='${TAGSHA:-N/A}'" case "${REGISTRY}" in "lscr.io") AUTH_URL="https://ghcr.io/token?scope=repository%3A${USERNAME}%2F${REPO}%3Apull";; "ghcr.io") AUTH_URL="https://ghcr.io/token?scope=repository%3A${USERNAME}%2F${REPO}%3Apull";; "quay.io") AUTH_URL="https://quay.io/v2/auth?service=quay.io&scope=repository%3A${USERNAME}%2F${REPO}%3Apull";; "registry-1.docker.io") AUTH_URL="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ENDPOINT}:pull";; - *) AUTH_URL=$(get_auth_url "${MANIFEST_URL}" "${TAG}") + *) AUTH_URL=$(get_auth_url "${MANIFEST_URL}" "${TAG:-${TAGSHA}}") esac # Kill off modification logic if any of the usernames are banned for BANNED in $(curl -s https://raw.githubusercontent.com/linuxserver/docker-mods/master/blacklist.txt); do @@ -344,7 +358,7 @@ run_mods() { write_mod_info "Adding ${DOCKER_MOD} to container" # If we're using lscr try and get the manifest from ghcr, if it fails re-request a token from Docker Hub if [[ "${REGISTRY}" == "lscr.io" ]]; then - if [[ -n $(curl --user-agent "${MOD_UA}" -sLH "Authorization: Bearer ${TOKEN}" "${MANIFEST_URL}/${TAG}" | jq -r '.errors' >/dev/null 2>&1) ]]; then + if [[ -n $(curl --user-agent "${MOD_UA}" -sLH "Authorization: Bearer ${TOKEN}" "${MANIFEST_URL}/${TAG:-${TAGSHA}}" | jq -r '.errors' >/dev/null 2>&1) ]]; then write_mod_debug "Couldn't fetch manifest from ghcr.io, trying docker.io" AUTH_URL="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ENDPOINT}:pull" TOKEN="$( From b3c106b625499f88ce819cb11c012e6a0113916a Mon Sep 17 00:00:00 2001 From: thespad Date: Tue, 31 Dec 2024 19:04:20 +0000 Subject: [PATCH 4/4] Supress curl download output in debug mode --- docker-mods.v3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-mods.v3 b/docker-mods.v3 index 301f2d0..4b20087 100755 --- a/docker-mods.v3 +++ b/docker-mods.v3 @@ -12,7 +12,7 @@ SCRIPTS_DIR="/custom-cont-init.d" SERVICES_DIR="/custom-services.d" if [[ ${DOCKER_MODS_DEBUG_CURL,,} = "true" ]]; then - CURL_NOISE_LEVEL="-v" + CURL_NOISE_LEVEL="-vs" else CURL_NOISE_LEVEL="--silent" fi