Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions deploy/helm/container-cache/deploy/files/hf.conf
Original file line number Diff line number Diff line change
Expand Up @@ -164,8 +164,13 @@
proxy_ssl_protocols TLSv1.2 TLSv1.3;
proxy_pass_request_headers on;

# Neutralise a malformed "bytes=" Range (see lua/safe-range.lua).
# The range filter runs regardless of caching, so this bypass block
# needs it too.
set_by_lua_file $safe_range /etc/nginx/conf.d/lua/safe-range.lua;

# Forward the client Range untouched so chunk reads work as-is.
proxy_set_header Range $http_range;
proxy_set_header Range $safe_range;

# Initialize upstream_last_modified to prevent warnings
set $upstream_last_modified "";
Expand Down Expand Up @@ -216,9 +221,15 @@

location / {
proxy_pass_request_headers on;

# Neutralise a malformed "bytes=" Range before anything reads
# $http_range (see lua/safe-range.lua). nginx caches $http_range once
# evaluated, so this must stay ahead of the directives below.
set_by_lua_file $safe_range /etc/nginx/conf.d/lua/safe-range.lua;

# Use Range header for file content - enables partial downloads
proxy_set_header Range $http_range;
proxy_cache_key $request_method|$uri|$arg_versionId|$http_range;
proxy_set_header Range $safe_range;
proxy_cache_key $request_method|$uri|$arg_versionId|$safe_range;

# HF-OPTIMIZED: Buffers sized for 10MB median object size
# Buffer profile inherited from proxy-common.conf (aligned fleet-wide).
Expand All @@ -229,7 +240,7 @@
# Consistent-hash routing to the owner pod (see lua/cc-route.lua).
# Route on the same key nginx caches on so each byte-range chunk maps
# to its own owner and a large blob spreads across the tier.
set $cc_hash_key "$request_method|$uri|$arg_versionId|$http_range";
set $cc_hash_key "$request_method|$uri|$arg_versionId|$safe_range";
set $cc_replicas {{ int $.Values.replicaCount }};
rewrite_by_lua_file /etc/nginx/conf.d/lua/cc-route.lua;
{{- end }}
Expand Down
78 changes: 78 additions & 0 deletions deploy/helm/container-cache/deploy/files/lua/safe-range.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
-- SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-- SPDX-License-Identifier: Apache-2.0
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--[[
Neutralises a Range header that names the "bytes" unit but does not parse.

Why this exists. The NGC CLI computes a range of first-byte-pos 0 through
last-byte-pos size-1. For a zero-length file that is 0 through -1, so it sends:

Range: bytes=0--1

RFC 7233 section 2.1 defines last-byte-pos as 1*DIGIT, which admits no sign, so
the byte-range-set does not parse. nginx rejects it at parse time and answers
416. That is conformant: section 3.1 says a server SHOULD send 416 when ranges
are invalid. The NGC origin instead ignores the bad header and returns 200 with
the full representation, which is lenient but permitted.

The result is that a client which downloads successfully from the origin fails
behind this cache. Since we are a transparent intermediary in front of a client
we do not control, we match the origin rather than being stricter than it.

Scope, deliberately narrow:
- Only a header naming the "bytes" unit is considered. nginx already ignores
units it does not understand, so those are left untouched.
- Only a header that fails to PARSE is cleared. A header that parses but is
unsatisfiable or semantically invalid, such as "bytes=100-50", keeps
nginx's 416. That behaviour is correct and the origin rejects it too.

Used via set_by_lua_file so it runs in configuration order alongside the
surrounding `set` directives. That ordering matters: $http_range is cached by
nginx once evaluated, so any `set` that reads it before this runs would pin the
malformed value into the cache key. Callers must use the returned $safe_range,
not $http_range, in proxy_cache_key, proxy_set_header Range and $cc_hash_key.

Returns the original value when it is valid or absent, otherwise an empty
string, having also removed the header so nginx's range filter does not see it.
]]

local range = ngx.var.http_range
if not range or range == "" then
return ""
end

local range_set = range:match("^[Bb][Yy][Tt][Ee][Ss]=(.*)$")
if not range_set then
return range
end

local valid = range_set ~= ""
if valid then
-- byte-range-spec = first-byte-pos "-" [ last-byte-pos ]
-- suffix-byte-range-spec = "-" suffix-length
for spec in (range_set .. ","):gmatch("([^,]*),") do
spec = spec:gsub("^%s*(.-)%s*$", "%1")
if not (spec:match("^%d+%-%d*$") or spec:match("^%-%d+$")) then
valid = false
break
end
end
end

if valid then
return range
end

ngx.req.clear_header("Range")
return ""
13 changes: 10 additions & 3 deletions deploy/helm/container-cache/deploy/files/ngc.conf
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,13 @@
}

location / {
# Neutralise a malformed "bytes=" Range before anything reads
# $http_range. nginx would answer 416; the origin ignores it and
# serves 200. See lua/safe-range.lua. Must stay ahead of every
# directive below that consumes the range, because nginx caches
# $http_range once evaluated.
set_by_lua_file $safe_range /etc/nginx/conf.d/lua/safe-range.lua;

# Initialize upstream_last_modified to prevent warnings
set $upstream_last_modified "";

Expand All @@ -67,14 +74,14 @@
# Consistent-hash routing to the owner pod (see lua/cc-route.lua).
# Route on the same key nginx caches on so each byte-range chunk maps
# to its own owner and a large blob spreads across the tier.
set $cc_hash_key "$request_method|$uri|$arg_versionId|$http_range";
set $cc_hash_key "$request_method|$uri|$arg_versionId|$safe_range";
set $cc_replicas {{ int $.Values.replicaCount }};
rewrite_by_lua_file /etc/nginx/conf.d/lua/cc-route.lua;
{{- end }}

proxy_pass_request_headers on;
proxy_set_header Range $http_range;
proxy_cache_key $request_method|$uri|$arg_versionId|$http_range;
proxy_set_header Range $safe_range;
proxy_cache_key $request_method|$uri|$arg_versionId|$safe_range;
access_by_lua_file /etc/nginx/conf.d/lua/lua-access.lua;
proxy_set_header Host $host;
proxy_pass $scheme://$host;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@
proxy_http_version 1.1;
proxy_pass_request_headers on;
proxy_set_header Host $host;
proxy_set_header Range $http_range;
proxy_set_header Range $safe_range;
proxy_set_header X-NVCF-CC-Relayed "1";
# Declaring any proxy_set_header here cancels inheritance of the
# server-level set, which includes `Connection ""`. Without repeating
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,11 @@
# DNS resolver for Nucleus requests - moderate caching for internal service
resolver nvcf-unbound.dns-proxy.svc.cluster.local valid=900s; # 15min cache - internal services less stable than CDNs
proxy_set_header Host $http_host;
# Client library knows what slices to use and we trust it with http_range
proxy_set_header Range $http_range;
# Neutralise a malformed "bytes=" Range (see lua/safe-range.lua);
# any range that parses is still passed through untouched.
set_by_lua_file $safe_range /etc/nginx/conf.d/lua/safe-range.lua;
# Client library knows what slices to use and we trust it with the range
proxy_set_header Range $safe_range;
# Required headers for Nucleus LFT
proxy_set_header X-OV-URI $http_x_ov_uri;
proxy_set_header Omniverse-Proxy-Scheme $http_omniverse_proxy_scheme;
Expand All @@ -45,7 +48,7 @@
proxy_socket_keepalive on;

# Cache key includes NCA ID and OVC 1 cache parameters
proxy_cache_key $http_nvcf_nca_id|$request_method|$http_omniverse_content_uid|$http_range;
proxy_cache_key $http_nvcf_nca_id|$request_method|$http_omniverse_content_uid|$safe_range;
proxy_pass https://$host;

# From OVC 1
Expand All @@ -65,12 +68,14 @@

# Use Host header
proxy_set_header Host $http_host;
# Neutralise a malformed "bytes=" Range (see lua/safe-range.lua).
set_by_lua_file $safe_range /etc/nginx/conf.d/lua/safe-range.lua;
# Use Range Header
proxy_set_header Range $http_range;
proxy_set_header Range $safe_range;
proxy_pass_request_headers on;

# Cache key includes NCA ID and generic uri with optional omniverse content uid and range
proxy_cache_key $http_nvcf_nca_id|$request_method|$host|$request_uri|$http_omniverse_content_uid|$http_range;
proxy_cache_key $http_nvcf_nca_id|$request_method|$host|$request_uri|$http_omniverse_content_uid|$safe_range;
proxy_pass https://$http_host;
}
}
2 changes: 2 additions & 0 deletions deploy/helm/container-cache/deploy/templates/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ data:
{{ .Files.Get "files/lua/lua-gen-key.lua" | nindent 4 }}
lua-ssl.lua: |-
{{ .Files.Get "files/lua/lua-ssl.lua" | nindent 4 }}
safe-range.lua: |-
{{ .Files.Get "files/lua/safe-range.lua" | nindent 4 }}
{{- if (.Values.consistentHashRouting).enabled }}
cc-route.lua: |-
{{ .Files.Get "files/lua/cc-route.lua" | nindent 4 }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,8 @@ spec:
path: lua-gen-key.lua
- key: lua-ssl.lua
path: lua-ssl.lua
- key: safe-range.lua
path: safe-range.lua
{{- if (.Values.consistentHashRouting).enabled }}
- key: cc-route.lua
path: cc-route.lua
Expand Down
159 changes: 159 additions & 0 deletions deploy/helm/container-cache/tests/range-sanitize-runtime-test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Behavioural test for files/lua/safe-range.lua. Runs the real sanitizer inside
# OpenResty against a mock origin that tolerates a malformed Range the way the
# NGC CDN does, and asserts we match the origin instead of answering 416.
#
# bash tests/range-sanitize-runtime-test.sh
#
# Requires docker and python3. Skips cleanly when docker is unavailable.
set -euo pipefail
CHART_DIR="$(cd "$(dirname "$0")/.." && pwd)/deploy"
LUA_SRC="$CHART_DIR/files/lua/safe-range.lua"
IMAGE="${OPENRESTY_IMAGE:-openresty/openresty:alpine}"
PORT_PROXY=18190
PORT_ORIGIN=18191

command -v docker >/dev/null 2>&1 || { echo "SKIP: docker not available"; exit 0; }
docker info >/dev/null 2>&1 || { echo "SKIP: docker daemon not reachable"; exit 0; }
[ -f "$LUA_SRC" ] || { echo "FAIL: $LUA_SRC missing" >&2; exit 1; }

TMP="$(mktemp -d)"
CONT="ccrange-$$"
cleanup() {
docker rm -f "$CONT" >/dev/null 2>&1 || true
[ -n "${ORIGIN_PID:-}" ] && kill "$ORIGIN_PID" 2>/dev/null || true
rm -rf "$TMP"
}
trap cleanup EXIT
fail() { echo "FAIL: $*" >&2; exit 1; }

mkdir -p "$TMP/lua"
cp "$LUA_SRC" "$TMP/lua/safe-range.lua"

cat > "$TMP/origin.py" <<PY
# Mock NGC/CloudFront origin: ignores a malformed Range, always answers 200.
from http.server import BaseHTTPRequestHandler, HTTPServer
BODIES = {'/empty': b'', '/nonempty': b'A'*1000}
class H(BaseHTTPRequestHandler):
protocol_version = 'HTTP/1.1'
def do_GET(self):
b = BODIES.get(self.path.split('?')[0])
if b is None:
self.send_response(404); self.send_header('Content-Length','0'); self.end_headers(); return
self.send_response(200)
self.send_header('Content-Length', str(len(b)))
self.send_header('Accept-Ranges','bytes')
self.send_header('X-Origin-Saw-Range', self.headers.get('Range') or 'none')
self.end_headers(); self.wfile.write(b)
def log_message(self,*a): pass
HTTPServer(('127.0.0.1',$PORT_ORIGIN), H).serve_forever()
PY

cat > "$TMP/nginx.conf" <<NGINX
worker_processes 1;
events {}
error_log /dev/stderr warn;
http {
proxy_cache_path /cache levels=1:2 keys_zone=z:10m use_temp_path=off;
access_log off;
server {
listen $PORT_PROXY;
location / {
proxy_cache z;
proxy_cache_valid 200 206 1h;
add_header X-Cache \$upstream_cache_status always;
add_header X-Key "\$request_method|\$uri|\$safe_range" always;
set_by_lua_file \$safe_range /etc/lua/safe-range.lua;
set \$cc_hash_key "\$request_method|\$uri|\$safe_range";
proxy_set_header Range \$safe_range;
proxy_cache_key \$request_method|\$uri|\$safe_range;
proxy_pass http://127.0.0.1:$PORT_ORIGIN;
}
}
}
NGINX

python3 "$TMP/origin.py" & ORIGIN_PID=$!
docker run --rm -d --name "$CONT" --network host \
-v "$TMP/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf:ro" \
-v "$TMP/lua:/etc/lua:ro" "$IMAGE" >/dev/null

for _ in $(seq 1 25); do
curl -sf -o /dev/null "http://127.0.0.1:$PORT_PROXY/nonempty" 2>/dev/null && break
sleep 0.4
done
curl -sf -o /dev/null "http://127.0.0.1:$PORT_PROXY/nonempty" \
|| { docker logs "$CONT" 2>&1 | tail -20; fail "proxy did not come up"; }

probe() { # path, range -> "code size"
if [ "$2" = "-" ]; then
curl -sS -o /dev/null -w '%{http_code} %{size_download}' "http://127.0.0.1:$PORT_PROXY$1"
else
curl -sS -o /dev/null -w '%{http_code} %{size_download}' -H "Range: $2" "http://127.0.0.1:$PORT_PROXY$1"
fi
}
expect() { # path, range, want "code size", why
got="$(probe "$1" "$2")"
[ "$got" = "$3" ] || fail "$1 with Range '$2': expected [$3], got [$got] -- $4"
printf ' ok %-12s %-14s -> %s\n' "$1" "$2" "$got"
}
expect_status() { # path, range, want-code, why
# Status only. The body of an error response is nginx's default error page,
# whose size varies between builds (194 bytes in production, 203 in the
# OpenResty test image), so asserting it would break under OPENRESTY_IMAGE.
got="$(probe "$1" "$2")"
code="${got%% *}"
[ "$code" = "$3" ] || fail "$1 with Range '$2': expected status $3, got [$got] -- $4"
printf ' ok %-12s %-14s -> %s (body size not asserted)\n' "$1" "$2" "$code"
}

echo "1. a malformed range is neutralised, matching the origin's 200"
expect /empty "bytes=0--1" "200 0" "the reported NGC CLI failure: zero-length file"
expect /nonempty "bytes=0--1" "200 1000" "the fault is Range parsing, not zero length"

echo "2. valid ranges are untouched"
expect /nonempty "bytes=0-0" "206 1" "single byte range must still work"
expect /nonempty "bytes=0-99" "206 100" "bounded range must still work"
expect /nonempty "bytes=-100" "206 100" "suffix range must still work"
expect /nonempty "bytes=900-" "206 100" "open-ended range must still work"

echo "2b. what the origin actually receives (the sanitizer's real contract)"
saw() { # path, range -> the Range header the origin observed
curl -sS -o /dev/null -D- -H "Range: $2" "http://127.0.0.1:$PORT_PROXY$1?cb=$RANDOM" \
| tr -d '\r' | awk -F': ' 'tolower($1)=="x-origin-saw-range"{print $2; exit}'
}
forwards() { # path, range, expected-at-origin, why
got="$(saw "$1" "$2")"
[ "$got" = "$3" ] || fail "$1 with Range '$2': origin saw '$got', expected '$3' -- $4"
printf ' ok origin saw %-18s for %s\n' "'$got'" "'$2'"
}
forwards /nonempty "bytes=0-9,20-29" "bytes=0-9,20-29" "multi-range must reach the origin unchanged"
forwards /nonempty "bytes=0-99" "bytes=0-99" "valid range must reach the origin unchanged"
forwards /nonempty "bytes=0--1" "none" "malformed range must be stripped before the origin"
forwards /empty "bytes=0--1" "none" "malformed range stripped on the zero-length file too"

echo "3. a parseable but unsatisfiable range keeps nginx's conformant 416"
expect_status /nonempty "bytes=100-50" "416" "RFC 7233 3.1 says SHOULD 416; do not paper over it"
expect /empty "bytes=0-0" "200 0" "zero-length with a valid range is not an error"

echo "4. no range is unaffected"
expect /empty "-" "200 0"
expect /nonempty "-" "200 1000"

echo "5. a sanitized request shares the cache entry with an unranged one"
docker exec "$CONT" sh -c 'rm -rf /cache/*' >/dev/null 2>&1 || true
docker restart "$CONT" >/dev/null
for _ in $(seq 1 25); do curl -sf -o /dev/null "http://127.0.0.1:$PORT_PROXY/nonempty" 2>/dev/null && break; sleep 0.4; done
curl -sS -o /dev/null "http://127.0.0.1:$PORT_PROXY/nonempty"
hdrs="$(curl -sS -o /dev/null -D- -H 'Range: bytes=0--1' "http://127.0.0.1:$PORT_PROXY/nonempty")"
echo "$hdrs" | grep -qi 'X-Cache: HIT' \
|| fail "malformed-range request did not reuse the unranged cache entry (duplicate entry)"
echo "$hdrs" | grep -qi 'X-Key: GET|/nonempty|$' \
|| echo "$hdrs" | grep -qiE 'X-Key: GET\|/nonempty\|[[:space:]]*$' \
|| fail "cache key still carries the malformed range"
echo " ok shares the cache key with the unranged request"

echo "PASS: safe-range.lua behaves correctly against a tolerant origin"
Loading
Loading