From 81e1568b15d4988921b6fe4b3994d6afa041dd0e Mon Sep 17 00:00:00 2001 From: thespad Date: Sun, 22 Dec 2024 14:59:16 +0000 Subject: [PATCH 1/7] Add modcache --- README.md | 1 + docker-mods.v3 | 102 ++++++++++++++++++++++++++++++++----------------- 2 files changed, 68 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index fd93e4f..4558e5f 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ These files are used by Linuxserver build processes to handle mods in our images. Not for end-user consumption. +* **22.12.24:** - Add modcache support. * **26.06.24:** - Add RO and User handlers. * **10.06.24:** - Move lsiown to its own file. Remove support for legacy v2 and hybrid mods. * **13.04.24:** - Let lsiown ignore broken symlinks (requires gnu find). diff --git a/docker-mods.v3 b/docker-mods.v3 index 0844269..ac5d5d0 100755 --- a/docker-mods.v3 +++ b/docker-mods.v3 @@ -5,7 +5,7 @@ # Version 3 # 2022-09-25 - Initial Release -MOD_SCRIPT_VER="3.20241207" +MOD_SCRIPT_VER="3.20241222" # Define custom folder paths SCRIPTS_DIR="/custom-cont-init.d" @@ -120,7 +120,7 @@ create_with_contenv_alias() { # Check for curl curl_check() { if [[ ! -f /usr/bin/curl ]] || [[ ! -f /usr/bin/jq ]]; then - write_mod_info "Curl/JQ was not found on this system for Docker mods installing" + write_mod_info "Curl/JQ was not found on this system and is required for Docker mods, installing..." if [[ -f /usr/bin/apt ]]; then # Ubuntu export DEBIAN_FRONTEND="noninteractive" @@ -154,7 +154,7 @@ write_mod_info() { write_mod_error() { local MSG=$* - echo "[mod-init] (ERROR) $MSG" + echo -e "[mod-init] (ERROR) $MSG" } write_mod_debug() { @@ -164,7 +164,7 @@ write_mod_debug() { # Use different filtering depending on URL get_blob_sha() { - MULTIDIGEST=$(curl -f --retry 10 --retry-max-time 60 --retry-connrefused \ + MULTIDIGEST=$(curl -f --retry 5 --retry-max-time 30 --retry-connrefused \ ${CURL_NOISE_LEVEL} \ --location \ --header "Accept: application/vnd.docker.distribution.manifest.v2+json" \ @@ -194,7 +194,7 @@ get_blob_sha() { write_mod_debug "Mod only has a single arch manifest" >&2 MULTIDIGEST=$(jq -r ".manifests[].digest?" <<< "${MULTIDIGEST}") fi - if DIGEST=$(curl -f --retry 10 --retry-max-time 60 --retry-connrefused \ + if DIGEST=$(curl -f --retry 5 --retry-max-time 30 --retry-connrefused \ ${CURL_NOISE_LEVEL} \ --location \ --header "Accept: application/vnd.docker.distribution.manifest.v2+json" \ @@ -254,6 +254,7 @@ run_mods() { write_mod_info "Running Docker Modification Logic" write_mod_debug "Running in debug mode" write_mod_debug "Mod script version ${MOD_SCRIPT_VER}" + mkdir -p /modcache for DOCKER_MOD in $(echo "${DOCKER_MODS}" | tr '|' '\n'); do # Support alternative endpoints case "${DOCKER_MOD}" in @@ -320,7 +321,7 @@ run_mods() { if [[ -n "${AUTH_URL}" ]]; then # Get registry token for api operations TOKEN="$( - curl -f --retry 10 --retry-max-time 60 --retry-connrefused \ + curl -f --retry 5 --retry-max-time 30 --retry-connrefused \ ${CURL_NOISE_LEVEL} \ "${AUTH_URL}" | jq -r '.token' @@ -335,7 +336,7 @@ run_mods() { write_mod_debug "Couldn't fetch manifest from ghcr.io, trying docker.io" AUTH_URL="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ENDPOINT}:pull" TOKEN="$( - curl -f --retry 10 --retry-max-time 60 --retry-connrefused \ + curl -f --retry 5 --retry-max-time 30 --retry-connrefused \ ${CURL_NOISE_LEVEL} \ "${AUTH_URL}" | jq -r '.token' @@ -347,50 +348,81 @@ run_mods() { fi ARCH=$(get_arch) write_mod_debug "Arch detected as ${ARCH}" - # Determine first and only layer of image - SHALAYER=$(get_blob_sha "${TOKEN}" "${MANIFEST_URL}" "${TAG}" "${ARCH:=-amd64}") - if [[ $? -eq 1 ]]; then - write_mod_error "No manifest available for arch ${ARCH:=-amd64}, cannot fetch mod" - continue - elif [[ -z "${SHALAYER}" ]]; then - write_mod_error "${DOCKER_MOD} digest could not be fetched from ${REGISTRY}" - continue + if [[ -z "${TOKEN}" ]]; then + write_mod_error "Couldn't fetch auth token from ${REGISTRY}, switching to offline mode for ${DOCKER_MOD}" + MOD_OFFLINE="true" + else + # Determine first and only layer of image + SHALAYER=$(get_blob_sha "${TOKEN}" "${MANIFEST_URL}" "${TAG}" "${ARCH:=-amd64}") + if [[ $? -eq 1 ]]; then + write_mod_error "No manifest available for arch ${ARCH:=-amd64}, cannot fetch mod" + continue + elif [[ -z "${SHALAYER}" ]]; then + write_mod_info "${DOCKER_MOD} digest could not be fetched from ${REGISTRY}, checking local cache" + MOD_OFFLINE="true" + fi + write_mod_debug "Mod SHA is ${SHALAYER:-unknown, mod may not exist}" fi # Check if we have allready applied this layer if [[ -f "/${FILENAME}" ]] && [[ "${SHALAYER}" == "$(cat /"${FILENAME}")" ]]; then write_mod_info "${DOCKER_MOD} at ${SHALAYER} has been previously applied skipping" + continue + elif [[ -f "/modcache/${FILENAME}.tar.xz" ]] && [[ "${SHALAYER}" =~ $(sha256sum "/modcache/${FILENAME}.tar.xz" | cut -f1 -d" ") ]]; then + write_mod_info "${DOCKER_MOD} at ${SHALAYER} found in modcache, applying" + elif [[ -f "/modcache/${FILENAME}.tar.xz" ]] && [[ "${MOD_OFFLINE}" = "true" ]]; then + write_mod_info "OFFLINE: ${DOCKER_MOD} found in modcache, applying" + elif [[ ! -f "/modcache/${FILENAME}.tar.xz" ]] && [[ "${MOD_OFFLINE}" = "true" ]]; then + write_mod_error "OFFLINE: ${DOCKER_MOD} not found in modcache, skipping" + continue else write_mod_info "Downloading ${DOCKER_MOD} from ${REGISTRY}" + if [[ -f "/modcache/${FILENAME}.lock" ]]; then + write_mod_info "${DOCKER_MOD} is already being downloaded by another container, waiting..." + for ((i = 5 ; i < 21 ; i=i*2 )); do + sleep $i + if [[ ! -f "/modcache/${FILENAME}.lock" ]]; then + break + elif [[ $i == 20 ]]; then + write_mod_error "${DOCKER_MOD} timed out waiting for lock, skipping\n\tIf no other containers are using this mod you may need to delete /modcache/${FILENAME}.lock" + SKIP_MOD_DOWNLOAD=true + fi + done + fi + if [[ "${SKIP_MOD_DOWNLOAD}" == "true" ]]; then + continue + fi # Download and extract layer to / - curl -f --retry 10 --retry-max-time 60 --retry-all-errors \ + touch "/modcache/${FILENAME}.lock" + curl -f --retry 5 --retry-max-time 30 --retry-all-errors \ ${CURL_NOISE_LEVEL} \ --location \ --header "Authorization: Bearer ${TOKEN}" \ --user-agent "${MOD_UA}" \ "${BLOB_URL}${SHALAYER}" -o \ - "/${FILENAME}.tar.xz" - mkdir -p /tmp/mod - if ! tar -tzf "/${FILENAME}.tar.xz" >/dev/null 2>&1; then + "/modcache/${FILENAME}.tar.xz" + if ! tar -tzf "/modcache/${FILENAME}.tar.xz" >/dev/null 2>&1; then write_mod_error "Invalid tarball, could not download ${DOCKER_MOD} from ${REGISTRY}" + rm "/modcache/${FILENAME}.lock" continue fi - write_mod_info "Installing ${DOCKER_MOD}" - tar xzf "/${FILENAME}.tar.xz" -C /tmp/mod - # Remove any v2 mod elements as they're no longer supported - if [[ -d /tmp/mod/etc/cont-init.d ]]; then - rm -rf /tmp/mod/etc/cont-init.d - fi - if [[ -d /tmp/mod/etc/services.d ]]; then - rm -rf /tmp/mod/etc/services.d - fi - shopt -s dotglob - cp -R /tmp/mod/* / - shopt -u dotglob - rm -rf /tmp/mod - rm -rf "/${FILENAME}.tar.xz" - echo "${SHALAYER}" >"/${FILENAME}" - write_mod_info "${DOCKER_MOD} applied to container" fi + write_mod_info "Installing ${DOCKER_MOD}" + mkdir -p /tmp/mod + tar xzf "/modcache/${FILENAME}.tar.xz" -C /tmp/mod + # Remove any v2 mod elements as they're no longer supported + if [[ -d /tmp/mod/etc/cont-init.d ]]; then + rm -rf /tmp/mod/etc/cont-init.d + fi + if [[ -d /tmp/mod/etc/services.d ]]; then + rm -rf /tmp/mod/etc/services.d + fi + shopt -s dotglob + cp -R /tmp/mod/* / + shopt -u dotglob + rm -rf /tmp/mod + echo "${SHALAYER}" >"/${FILENAME}" + #rm "/modcache/${FILENAME}.lock" + write_mod_info "${DOCKER_MOD} applied to container" done } From 2a605cdb7210e3a368f1a548c73b65e23b049c31 Mon Sep 17 00:00:00 2001 From: thespad Date: Sun, 22 Dec 2024 15:22:08 +0000 Subject: [PATCH 2/7] Uncomment lockfile removal --- docker-mods.v3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-mods.v3 b/docker-mods.v3 index ac5d5d0..3fc4e3c 100755 --- a/docker-mods.v3 +++ b/docker-mods.v3 @@ -421,7 +421,7 @@ run_mods() { shopt -u dotglob rm -rf /tmp/mod echo "${SHALAYER}" >"/${FILENAME}" - #rm "/modcache/${FILENAME}.lock" + rm "/modcache/${FILENAME}.lock" write_mod_info "${DOCKER_MOD} applied to container" done } From 449356a33e990aaf6f946462e2ae1bd0a1c1a65c Mon Sep 17 00:00:00 2001 From: thespad Date: Sun, 22 Dec 2024 15:32:24 +0000 Subject: [PATCH 3/7] Don't try and delete lock file if it doesn't exist --- docker-mods.v3 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker-mods.v3 b/docker-mods.v3 index 3fc4e3c..0d8ec13 100755 --- a/docker-mods.v3 +++ b/docker-mods.v3 @@ -402,7 +402,7 @@ run_mods() { "/modcache/${FILENAME}.tar.xz" if ! tar -tzf "/modcache/${FILENAME}.tar.xz" >/dev/null 2>&1; then write_mod_error "Invalid tarball, could not download ${DOCKER_MOD} from ${REGISTRY}" - rm "/modcache/${FILENAME}.lock" + rm "/modcache/${FILENAME}.lock" || write_mod_error "Failed to delete lock file /modcache/${FILENAME}.lock" continue fi fi @@ -421,7 +421,9 @@ run_mods() { shopt -u dotglob rm -rf /tmp/mod echo "${SHALAYER}" >"/${FILENAME}" - rm "/modcache/${FILENAME}.lock" + if [[ -f "/modcache/${FILENAME}.lock" ]]; then + rm "/modcache/${FILENAME}.lock" + fi write_mod_info "${DOCKER_MOD} applied to container" done } From 299f679e0153542313ec4bfb5fd1b5b9eab5b38c Mon Sep 17 00:00:00 2001 From: thespad Date: Sun, 22 Dec 2024 15:36:08 +0000 Subject: [PATCH 4/7] Error if unable to delete lock file --- docker-mods.v3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-mods.v3 b/docker-mods.v3 index 0d8ec13..c5bb7d9 100755 --- a/docker-mods.v3 +++ b/docker-mods.v3 @@ -422,7 +422,7 @@ run_mods() { rm -rf /tmp/mod echo "${SHALAYER}" >"/${FILENAME}" if [[ -f "/modcache/${FILENAME}.lock" ]]; then - rm "/modcache/${FILENAME}.lock" + rm "/modcache/${FILENAME}.lock" || write_mod_error "Failed to delete lock file /modcache/${FILENAME}.lock" fi write_mod_info "${DOCKER_MOD} applied to container" done From e84b25458cc3721e641547a546a353cc7938c770 Mon Sep 17 00:00:00 2001 From: thespad Date: Sun, 22 Dec 2024 15:49:43 +0000 Subject: [PATCH 5/7] If lockfile found, use cached download --- docker-mods.v3 | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/docker-mods.v3 b/docker-mods.v3 index c5bb7d9..ff033df 100755 --- a/docker-mods.v3 +++ b/docker-mods.v3 @@ -381,30 +381,34 @@ run_mods() { for ((i = 5 ; i < 21 ; i=i*2 )); do sleep $i if [[ ! -f "/modcache/${FILENAME}.lock" ]]; then + SKIP_MOD_DOWNLOAD=true break elif [[ $i == 20 ]]; then write_mod_error "${DOCKER_MOD} timed out waiting for lock, skipping\n\tIf no other containers are using this mod you may need to delete /modcache/${FILENAME}.lock" - SKIP_MOD_DOWNLOAD=true + SKIP_MOD=true fi done fi - if [[ "${SKIP_MOD_DOWNLOAD}" == "true" ]]; then + if [[ "${SKIP_MOD}" == "true" ]]; then continue + elif [[ "${SKIP_MOD_DOWNLOAD}" != "true" ]]; then + # Download and extract layer to / + touch "/modcache/${FILENAME}.lock" + curl -f --retry 5 --retry-max-time 30 --retry-all-errors \ + ${CURL_NOISE_LEVEL} \ + --location \ + --header "Authorization: Bearer ${TOKEN}" \ + --user-agent "${MOD_UA}" \ + "${BLOB_URL}${SHALAYER}" -o \ + "/modcache/${FILENAME}.tar.xz" fi - # Download and extract layer to / - touch "/modcache/${FILENAME}.lock" - curl -f --retry 5 --retry-max-time 30 --retry-all-errors \ - ${CURL_NOISE_LEVEL} \ - --location \ - --header "Authorization: Bearer ${TOKEN}" \ - --user-agent "${MOD_UA}" \ - "${BLOB_URL}${SHALAYER}" -o \ - "/modcache/${FILENAME}.tar.xz" - if ! tar -tzf "/modcache/${FILENAME}.tar.xz" >/dev/null 2>&1; then - write_mod_error "Invalid tarball, could not download ${DOCKER_MOD} from ${REGISTRY}" + fi + if ! tar -tzf "/modcache/${FILENAME}.tar.xz" >/dev/null 2>&1; then + write_mod_error "Invalid tarball for ${DOCKER_MOD}, skipping" + if [[ -f "/modcache/${FILENAME}.lock" ]]; then rm "/modcache/${FILENAME}.lock" || write_mod_error "Failed to delete lock file /modcache/${FILENAME}.lock" - continue fi + continue fi write_mod_info "Installing ${DOCKER_MOD}" mkdir -p /tmp/mod From 08bdfc0b82c830a1f09bef79a11d5d11aafbd3be Mon Sep 17 00:00:00 2001 From: thespad Date: Sun, 22 Dec 2024 19:09:31 +0000 Subject: [PATCH 6/7] Add modmanager hooks, fix indents --- .editorconfig | 2 +- docker-mods.v3 | 61 +++++++++++++++++++++++++++++--------------------- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/.editorconfig b/.editorconfig index a92f7df..08cdc6e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -15,6 +15,6 @@ trim_trailing_whitespace = false indent_style = space indent_size = 2 -[{**.sh,root/etc/cont-init.d/**,root/etc/services.d/**}] +[{docker-mods.*}] indent_style = space indent_size = 4 diff --git a/docker-mods.v3 b/docker-mods.v3 index ff033df..e897b76 100755 --- a/docker-mods.v3 +++ b/docker-mods.v3 @@ -12,9 +12,9 @@ SCRIPTS_DIR="/custom-cont-init.d" SERVICES_DIR="/custom-services.d" if [[ ${DOCKER_MODS_DEBUG_CURL,,} = "true" ]]; then - CURL_NOISE_LEVEL="-v" + CURL_NOISE_LEVEL="-v" else - CURL_NOISE_LEVEL="--silent" + CURL_NOISE_LEVEL="--silent" fi tamper_check() { @@ -353,9 +353,9 @@ run_mods() { MOD_OFFLINE="true" else # Determine first and only layer of image - SHALAYER=$(get_blob_sha "${TOKEN}" "${MANIFEST_URL}" "${TAG}" "${ARCH:=-amd64}") + SHALAYER=$(get_blob_sha "${TOKEN}" "${MANIFEST_URL}" "${TAG}" "${ARCH:--amd64}") if [[ $? -eq 1 ]]; then - write_mod_error "No manifest available for arch ${ARCH:=-amd64}, cannot fetch mod" + write_mod_error "No manifest available for arch ${ARCH:--amd64}, cannot fetch mod" continue elif [[ -z "${SHALAYER}" ]]; then write_mod_info "${DOCKER_MOD} digest could not be fetched from ${REGISTRY}, checking local cache" @@ -379,14 +379,14 @@ run_mods() { if [[ -f "/modcache/${FILENAME}.lock" ]]; then write_mod_info "${DOCKER_MOD} is already being downloaded by another container, waiting..." for ((i = 5 ; i < 21 ; i=i*2 )); do - sleep $i - if [[ ! -f "/modcache/${FILENAME}.lock" ]]; then - SKIP_MOD_DOWNLOAD=true - break - elif [[ $i == 20 ]]; then - write_mod_error "${DOCKER_MOD} timed out waiting for lock, skipping\n\tIf no other containers are using this mod you may need to delete /modcache/${FILENAME}.lock" - SKIP_MOD=true - fi + sleep $i + if [[ ! -f "/modcache/${FILENAME}.lock" ]]; then + SKIP_MOD_DOWNLOAD=true + break + elif [[ $i == 20 ]]; then + write_mod_error "${DOCKER_MOD} timed out waiting for lock, skipping\n\tIf no other containers are using this mod you may need to delete /modcache/${FILENAME}.lock" + SKIP_MOD=true + fi done fi if [[ "${SKIP_MOD}" == "true" ]]; then @@ -410,21 +410,25 @@ run_mods() { fi continue fi - write_mod_info "Installing ${DOCKER_MOD}" - mkdir -p /tmp/mod - tar xzf "/modcache/${FILENAME}.tar.xz" -C /tmp/mod - # Remove any v2 mod elements as they're no longer supported - if [[ -d /tmp/mod/etc/cont-init.d ]]; then - rm -rf /tmp/mod/etc/cont-init.d + if [[ -z "${MODMANAGER_MODONLY}" ]]; then + write_mod_info "Installing ${DOCKER_MOD}" + mkdir -p /tmp/mod + tar xzf "/modcache/${FILENAME}.tar.xz" -C /tmp/mod + # Remove any v2 mod elements as they're no longer supported + if [[ -d /tmp/mod/etc/cont-init.d ]]; then + rm -rf /tmp/mod/etc/cont-init.d + fi + if [[ -d /tmp/mod/etc/services.d ]]; then + rm -rf /tmp/mod/etc/services.d + fi + shopt -s dotglob + cp -R /tmp/mod/* / + shopt -u dotglob + rm -rf /tmp/mod + echo "${SHALAYER}" >"/${FILENAME}" + else + write_mod_debug "Modmanager skipping mod application" fi - if [[ -d /tmp/mod/etc/services.d ]]; then - rm -rf /tmp/mod/etc/services.d - fi - shopt -s dotglob - cp -R /tmp/mod/* / - shopt -u dotglob - rm -rf /tmp/mod - echo "${SHALAYER}" >"/${FILENAME}" if [[ -f "/modcache/${FILENAME}.lock" ]]; then rm "/modcache/${FILENAME}.lock" || write_mod_error "Failed to delete lock file /modcache/${FILENAME}.lock" fi @@ -512,6 +516,11 @@ run_branding() { } # Main script loop +if [[ -n "${MODMANAGER_MODONLY}" ]]; then + run_mods + exit 0 +fi + if grep -qEe ' / \w+ ro' /proc/mounts; then printf '1' > /run/s6/container_environment/LSIO_READ_ONLY_FS LSIO_READ_ONLY_FS=1 From 8f1388cd4f51ca65fb4274e0861ffc0fe846538e Mon Sep 17 00:00:00 2001 From: thespad Date: Sun, 22 Dec 2024 21:28:49 +0000 Subject: [PATCH 7/7] Be consistent --- docker-mods.v3 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-mods.v3 b/docker-mods.v3 index e897b76..a77d2e3 100755 --- a/docker-mods.v3 +++ b/docker-mods.v3 @@ -149,7 +149,7 @@ curl_check() { write_mod_info() { local MSG=$* - echo "[mod-init] $MSG" + echo -e "[mod-init] $MSG" } write_mod_error() { @@ -159,7 +159,7 @@ write_mod_error() { write_mod_debug() { local MSG=$* - if [[ ${DOCKER_MODS_DEBUG,,} = "true" ]]; then echo "[mod-init] (DEBUG) $MSG"; fi + if [[ ${DOCKER_MODS_DEBUG,,} = "true" ]]; then echo -e "[mod-init] (DEBUG) $MSG"; fi } # Use different filtering depending on URL