Merge pull request #992 from linuxserver/mod-scripts-modcache

.editorconfig

@@ -15,6 +15,6 @@ trim_trailing_whitespace = false
 indent_style = space
 indent_size = 2
 
-[{**.sh,root/etc/cont-init.d/**,root/etc/services.d/**}]
+[{docker-mods.*}]
 indent_style = space
 indent_size = 4

README.md

@@ -2,6 +2,7 @@
 
 These files are used by Linuxserver build processes to handle mods in our images. Not for end-user consumption.
 
+* **22.12.24:** - Add modcache support.
 * **26.06.24:** - Add RO and User handlers.
 * **10.06.24:** - Move lsiown to its own file. Remove support for legacy v2 and hybrid mods.
 * **13.04.24:** - Let lsiown ignore broken symlinks (requires gnu find).

docker-mods.v3

@@ -5,16 +5,16 @@
 
 # Version 3
 # 2022-09-25 - Initial Release
-MOD_SCRIPT_VER="3.20241207"
+MOD_SCRIPT_VER="3.20241222"
 
 # Define custom folder paths
 SCRIPTS_DIR="/custom-cont-init.d"
 SERVICES_DIR="/custom-services.d"
 
 if [[ ${DOCKER_MODS_DEBUG_CURL,,} = "true" ]]; then
-  CURL_NOISE_LEVEL="-v"
+    CURL_NOISE_LEVEL="-v"
 else
-  CURL_NOISE_LEVEL="--silent"
+    CURL_NOISE_LEVEL="--silent"
 fi
 
 tamper_check() {
@@ -120,7 +120,7 @@ create_with_contenv_alias() {
 # Check for curl
 curl_check() {
     if [[ ! -f /usr/bin/curl ]] || [[ ! -f /usr/bin/jq ]]; then
-        write_mod_info "Curl/JQ was not found on this system for Docker mods installing"
+        write_mod_info "Curl/JQ was not found on this system and is required for Docker mods, installing..."
         if [[ -f /usr/bin/apt ]]; then
             # Ubuntu
             export DEBIAN_FRONTEND="noninteractive"
@@ -149,22 +149,22 @@ curl_check() {
 
 write_mod_info() {
     local MSG=$*
-    echo "[mod-init] $MSG"
+    echo -e "[mod-init] $MSG"
 }
 
 write_mod_error() {
     local MSG=$*
-    echo "[mod-init] (ERROR) $MSG"
+    echo -e "[mod-init] (ERROR) $MSG"
 }
 
 write_mod_debug() {
     local MSG=$*
-    if [[ ${DOCKER_MODS_DEBUG,,} = "true" ]]; then echo "[mod-init] (DEBUG) $MSG"; fi
+    if [[ ${DOCKER_MODS_DEBUG,,} = "true" ]]; then echo -e "[mod-init] (DEBUG) $MSG"; fi
 }
 
 # Use different filtering depending on URL
 get_blob_sha() {
-    MULTIDIGEST=$(curl  -f --retry 10 --retry-max-time 60 --retry-connrefused \
+    MULTIDIGEST=$(curl  -f --retry 5 --retry-max-time 30 --retry-connrefused \
         ${CURL_NOISE_LEVEL} \
         --location \
         --header "Accept: application/vnd.docker.distribution.manifest.v2+json" \
@@ -194,7 +194,7 @@ get_blob_sha() {
             write_mod_debug "Mod only has a single arch manifest" >&2
             MULTIDIGEST=$(jq -r ".manifests[].digest?" <<< "${MULTIDIGEST}")
         fi
-        if DIGEST=$(curl  -f --retry 10 --retry-max-time 60 --retry-connrefused \
+        if DIGEST=$(curl  -f --retry 5 --retry-max-time 30 --retry-connrefused \
             ${CURL_NOISE_LEVEL} \
             --location \
             --header "Accept: application/vnd.docker.distribution.manifest.v2+json" \
@@ -254,6 +254,7 @@ run_mods() {
     write_mod_info "Running Docker Modification Logic"
     write_mod_debug "Running in debug mode"
     write_mod_debug "Mod script version ${MOD_SCRIPT_VER}"
+    mkdir -p /modcache
     for DOCKER_MOD in $(echo "${DOCKER_MODS}" | tr '|' '\n'); do
         # Support alternative endpoints
         case "${DOCKER_MOD}" in
@@ -320,7 +321,7 @@ run_mods() {
         if [[ -n "${AUTH_URL}" ]]; then
             # Get registry token for api operations
             TOKEN="$(
-                curl -f --retry 10 --retry-max-time 60 --retry-connrefused \
+                curl -f --retry 5 --retry-max-time 30 --retry-connrefused \
                     ${CURL_NOISE_LEVEL} \
                     "${AUTH_URL}" |
                     jq -r '.token'
@@ -335,7 +336,7 @@ run_mods() {
                 write_mod_debug "Couldn't fetch manifest from ghcr.io, trying docker.io"
                 AUTH_URL="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ENDPOINT}:pull"
                 TOKEN="$(
-                    curl -f --retry 10 --retry-max-time 60 --retry-connrefused \
+                    curl -f --retry 5 --retry-max-time 30 --retry-connrefused \
                         ${CURL_NOISE_LEVEL} \
                         "${AUTH_URL}" |
                         jq -r '.token'
@@ -347,35 +348,72 @@ run_mods() {
         fi
         ARCH=$(get_arch)
         write_mod_debug "Arch detected as ${ARCH}"
-        # Determine first and only layer of image
-        SHALAYER=$(get_blob_sha "${TOKEN}" "${MANIFEST_URL}" "${TAG}" "${ARCH:=-amd64}")
-        if [[ $? -eq 1 ]]; then
-            write_mod_error "No manifest available for arch ${ARCH:=-amd64}, cannot fetch mod"
-            continue
-        elif [[ -z "${SHALAYER}" ]]; then
-            write_mod_error "${DOCKER_MOD} digest could not be fetched from ${REGISTRY}"
-            continue
+        if [[ -z "${TOKEN}" ]]; then
+            write_mod_error "Couldn't fetch auth token from ${REGISTRY}, switching to offline mode for ${DOCKER_MOD}"
+            MOD_OFFLINE="true"
+        else
+            # Determine first and only layer of image
+            SHALAYER=$(get_blob_sha "${TOKEN}" "${MANIFEST_URL}" "${TAG}" "${ARCH:--amd64}")
+            if [[ $? -eq 1 ]]; then
+                write_mod_error "No manifest available for arch ${ARCH:--amd64}, cannot fetch mod"
+                continue
+            elif [[ -z "${SHALAYER}" ]]; then
+                write_mod_info "${DOCKER_MOD} digest could not be fetched from ${REGISTRY}, checking local cache"
+                MOD_OFFLINE="true"
+            fi
+            write_mod_debug "Mod SHA is ${SHALAYER:-unknown, mod may not exist}"
         fi
         # Check if we have allready applied this layer
         if [[ -f "/${FILENAME}" ]] && [[ "${SHALAYER}" == "$(cat /"${FILENAME}")" ]]; then
             write_mod_info "${DOCKER_MOD} at ${SHALAYER} has been previously applied skipping"
+            continue
+        elif [[ -f "/modcache/${FILENAME}.tar.xz" ]] && [[ "${SHALAYER}" =~ $(sha256sum "/modcache/${FILENAME}.tar.xz" | cut -f1 -d" ") ]]; then
+            write_mod_info "${DOCKER_MOD} at ${SHALAYER} found in modcache, applying"
+        elif [[ -f "/modcache/${FILENAME}.tar.xz" ]] && [[ "${MOD_OFFLINE}" = "true" ]]; then
+            write_mod_info "OFFLINE: ${DOCKER_MOD} found in modcache, applying"
+        elif [[ ! -f "/modcache/${FILENAME}.tar.xz" ]] && [[ "${MOD_OFFLINE}" = "true" ]]; then
+            write_mod_error "OFFLINE: ${DOCKER_MOD} not found in modcache, skipping"
+            continue
         else
             write_mod_info "Downloading ${DOCKER_MOD} from ${REGISTRY}"
-            # Download and extract layer to /
-            curl -f --retry 10 --retry-max-time 60 --retry-all-errors \
-                ${CURL_NOISE_LEVEL} \
-                --location \
-                --header "Authorization: Bearer ${TOKEN}" \
-                --user-agent "${MOD_UA}" \
-                "${BLOB_URL}${SHALAYER}" -o \
-                "/${FILENAME}.tar.xz"
-            mkdir -p /tmp/mod
-            if ! tar -tzf "/${FILENAME}.tar.xz" >/dev/null 2>&1; then
-                write_mod_error "Invalid tarball, could not download ${DOCKER_MOD} from ${REGISTRY}"
-                continue
+            if [[ -f "/modcache/${FILENAME}.lock" ]]; then
+                write_mod_info "${DOCKER_MOD} is already being downloaded by another container, waiting..."
+                for ((i = 5 ; i < 21 ; i=i*2 )); do
+                    sleep $i
+                    if [[ ! -f "/modcache/${FILENAME}.lock" ]]; then
+                        SKIP_MOD_DOWNLOAD=true
+                        break
+                    elif [[ $i == 20 ]]; then
+                        write_mod_error "${DOCKER_MOD} timed out waiting for lock, skipping\n\tIf no other containers are using this mod you may need to delete /modcache/${FILENAME}.lock"
+                        SKIP_MOD=true
+                    fi
+                done
             fi
+            if [[ "${SKIP_MOD}" == "true" ]]; then
+                continue
+            elif [[ "${SKIP_MOD_DOWNLOAD}" != "true" ]]; then
+                # Download and extract layer to /
+                touch "/modcache/${FILENAME}.lock"
+                curl -f --retry 5 --retry-max-time 30 --retry-all-errors \
+                    ${CURL_NOISE_LEVEL} \
+                    --location \
+                    --header "Authorization: Bearer ${TOKEN}" \
+                    --user-agent "${MOD_UA}" \
+                    "${BLOB_URL}${SHALAYER}" -o \
+                    "/modcache/${FILENAME}.tar.xz"
+            fi
+        fi
+        if ! tar -tzf "/modcache/${FILENAME}.tar.xz" >/dev/null 2>&1; then
+            write_mod_error "Invalid tarball for ${DOCKER_MOD}, skipping"
+            if [[ -f "/modcache/${FILENAME}.lock" ]]; then
+                rm "/modcache/${FILENAME}.lock" || write_mod_error "Failed to delete lock file /modcache/${FILENAME}.lock"
+            fi
+            continue
+        fi
+        if [[ -z "${MODMANAGER_MODONLY}" ]]; then
             write_mod_info "Installing ${DOCKER_MOD}"
-            tar xzf "/${FILENAME}.tar.xz" -C /tmp/mod
+            mkdir -p /tmp/mod
+            tar xzf "/modcache/${FILENAME}.tar.xz" -C /tmp/mod
             # Remove any v2 mod elements as they're no longer supported
             if [[ -d /tmp/mod/etc/cont-init.d ]]; then
                 rm -rf /tmp/mod/etc/cont-init.d
@@ -387,10 +425,14 @@ run_mods() {
             cp -R /tmp/mod/* /
             shopt -u dotglob
             rm -rf /tmp/mod
-            rm -rf "/${FILENAME}.tar.xz"
             echo "${SHALAYER}" >"/${FILENAME}"
-            write_mod_info "${DOCKER_MOD} applied to container"
+        else
+            write_mod_debug "Modmanager skipping mod application"
         fi
+        if [[ -f "/modcache/${FILENAME}.lock" ]]; then
+            rm "/modcache/${FILENAME}.lock" || write_mod_error "Failed to delete lock file /modcache/${FILENAME}.lock"
+        fi
+        write_mod_info "${DOCKER_MOD} applied to container"
     done
 }
 
@@ -474,6 +516,11 @@ run_branding() {
 }
 
 # Main script loop
+if [[ -n "${MODMANAGER_MODONLY}" ]]; then
+    run_mods
+    exit 0
+fi
+
 if grep -qEe ' / \w+ ro' /proc/mounts; then
     printf '1' > /run/s6/container_environment/LSIO_READ_ONLY_FS
     LSIO_READ_ONLY_FS=1