fix(stream): derive H.264 level from frame macroblocks, not height

Anamorphic 2.39:1 scaled to 1080 height = ~2586x1080 = 11016 MBs, busting
level 4.1's 8192-MB MaxFS -> nvenc "InitializeEncoder failed: Invalid Level"
(libx264: "frame MB size > level limit") -> 0 segments, session stalls. Most
4K rips are 2.39:1, so HLS playback was silently broken for them.

H264LevelForFrame(w,h) derives the level from the real macroblock count
(max of MB-tier and height-tier). hls.go computes output width and uses it.
16:9 unchanged; anamorphic bumps to 5.0 when needed. Discovered + verified
during the trickplay smoke.
This commit is contained in:
Deivid Soto 2026-06-01 08:29:10 +02:00
parent ea00130d08
commit 8accafbe59
3 changed files with 107 additions and 12 deletions

View file

@ -22,6 +22,7 @@ import (
"fmt" "fmt"
"io" "io"
"log" "log"
"math"
"net/http" "net/http"
"os" "os"
"os/exec" "os/exec"
@ -1184,11 +1185,14 @@ func buildHLSFFmpegArgsAt(cfg HLSSessionConfig, probe *StreamProbe, tmpDir strin
// per session start, polluting logs even though encode succeeds. // per session start, polluting logs even though encode succeeds.
args = append(args, "-vaapi_device", "/dev/dri/renderD128") args = append(args, "-vaapi_device", "/dev/dri/renderD128")
} }
// Derive H.264 level from the actual output height. A fixed "4.0" caps the // Derive H.264 level from the actual output FRAME (width × height), not just
// encoder at 1080p — anything taller (1440p, 4K source on quality=original) // height. A fixed "4.0" caps the encoder at 1080p; deriving by height alone
// fails libx264 with "frame MB size > level limit" and emits unplayable // still under-levels anamorphic content — a 2.39:1 source scaled to 1080
// segments. The output height matches qcap.MaxHeight when the source is // height is ~2586×1080 = 11016 MBs, busting level 4.1's 8192-MB cap, which
// downscaled, otherwise probe.Height (already populated by ffprobe). // fails the encode ("Invalid Level" on nvenc, "frame MB size > level limit"
// on libx264) and stalls the session. The output height matches qcap.MaxHeight
// when the source is downscaled, otherwise probe.Height; the output width is
// the source width scaled by the same factor (the filter chain preserves AR).
qcap := resolveQualityCap(cfg.Quality) qcap := resolveQualityCap(cfg.Quality)
outputHeight := qcap.MaxHeight outputHeight := qcap.MaxHeight
if outputHeight == 0 { if outputHeight == 0 {
@ -1197,7 +1201,11 @@ func buildHLSFFmpegArgsAt(cfg HLSSessionConfig, probe *StreamProbe, tmpDir strin
if outputHeight == 0 || (probe.Height > 0 && probe.Height < outputHeight) { if outputHeight == 0 || (probe.Height > 0 && probe.Height < outputHeight) {
outputHeight = probe.Height outputHeight = probe.Height
} }
args = append(args, "-profile:v", "main", "-level:v", H264LevelForHeight(outputHeight)) outputWidth := probe.Width
if probe.Height > 0 && outputHeight != probe.Height {
outputWidth = int(math.Round(float64(probe.Width) * float64(outputHeight) / float64(probe.Height)))
}
args = append(args, "-profile:v", "main", "-level:v", H264LevelForFrame(outputWidth, outputHeight))
// Bitrate must match the level libx264 actually picks for outputHeight, // Bitrate must match the level libx264 actually picks for outputHeight,
// not the qcap target for the user's requested label. If a user asks for // not the qcap target for the user's requested label. If a user asks for

View file

@ -271,3 +271,60 @@ func H264LevelForHeight(height int) string {
return "6.0" return "6.0"
} }
} }
// h264LevelRank orders level strings so callers can pick the higher of two.
var h264LevelRank = map[string]int{
"3.0": 30, "3.1": 31, "3.2": 32,
"4.0": 40, "4.1": 41, "4.2": 42,
"5.0": 50, "5.1": 51, "6.0": 60,
}
// levelForMacroblocks returns the lowest H.264 level whose MaxFS (frame size in
// macroblocks) covers `mbs`. The height-based H264LevelForHeight tier is correct
// for 16:9, but anamorphic content (2.39:1 cinemascope) scaled to a given height
// has a much wider frame: a 2.39:1 source downscaled to 1080 height becomes
// ~2586×1080 = 11016 MBs, which busts level 4.1's 8192-MB MaxFS. ffmpeg then
// fails the encode — libx264 with "frame MB size > level limit", h264_nvenc with
// "InitializeEncoder failed: invalid param (8): Invalid Level" — and emits zero
// packets (the whole HLS session stalls at "preparando sesión"). MaxFS values
// from the H.264 spec, Table A-1.
func levelForMacroblocks(mbs int) string {
switch {
case mbs <= 1620:
return "3.0"
case mbs <= 3600:
return "3.1"
case mbs <= 5120:
return "3.2"
case mbs <= 8192: // levels 4.0 and 4.1 share MaxFS 8192; pick 4.1 for headroom
return "4.1"
case mbs <= 8704:
return "4.2"
case mbs <= 22080:
return "5.0"
case mbs <= 36864:
return "5.1"
default:
return "6.0"
}
}
// H264LevelForFrame returns the lowest H.264 level that satisfies BOTH the
// height-derived tier (which carries macroblock-rate / fps headroom) and the
// actual frame's macroblock count (which catches anamorphic frames that are far
// wider than 16:9 at a given height). Use this instead of H264LevelForHeight
// wherever the output width is known — it never under-levels an ultra-wide
// frame, and for 16:9 content it returns exactly what H264LevelForHeight does.
func H264LevelForFrame(width, height int) string {
byHeight := H264LevelForHeight(height)
if width <= 0 || height <= 0 {
return byHeight
}
// Macroblocks are 16×16; partial blocks at the edge still count (ceil).
mbs := ((width + 15) / 16) * ((height + 15) / 16)
byMB := levelForMacroblocks(mbs)
if h264LevelRank[byMB] > h264LevelRank[byHeight] {
return byMB
}
return byHeight
}

View file

@ -81,12 +81,12 @@ func TestResolveEncoderProfileHonoursConfiguredPreset(t *testing.T) {
configured string configured string
wantPreset string wantPreset string
}{ }{
{HWAccelNone, "ultrafast", "ultrafast"}, // libx264 honours {HWAccelNone, "ultrafast", "ultrafast"}, // libx264 honours
{HWAccelNone, "medium", "medium"}, // libx264 honours {HWAccelNone, "medium", "medium"}, // libx264 honours
{HWAccelNVENC, "p1", "p3"}, // NVENC ignores, sticks to p3 {HWAccelNVENC, "p1", "p3"}, // NVENC ignores, sticks to p3
{HWAccelNVENC, "veryfast", "p3"}, // NVENC ignores libx264 vocab {HWAccelNVENC, "veryfast", "p3"}, // NVENC ignores libx264 vocab
{HWAccelQSV, "veryslow", "veryfast"}, // QSV ignores, sticks to veryfast {HWAccelQSV, "veryslow", "veryfast"}, // QSV ignores, sticks to veryfast
{HWAccelVideoToolbox, "veryfast", ""}, // VideoToolbox has no preset {HWAccelVideoToolbox, "veryfast", ""}, // VideoToolbox has no preset
} }
for _, tc := range cases { for _, tc := range cases {
got := ResolveEncoderProfile(tc.hw, tc.configured) got := ResolveEncoderProfile(tc.hw, tc.configured)
@ -154,3 +154,33 @@ func TestHWAccelDiagnosticLogLineSoftwareButEncodersFound(t *testing.T) {
} }
} }
func TestH264LevelForFrame(t *testing.T) {
cases := []struct {
name string
width, height int
want string
}{
// 16:9 must match the height-only helper exactly (no regression).
{"720p 16:9", 1280, 720, "4.0"},
{"1080p 16:9", 1920, 1080, "4.1"},
{"1440p 16:9", 2560, 1440, "5.0"},
{"2160p 16:9", 3840, 2160, "5.1"},
// Anamorphic 2.39:1 at 1080 height — the regression: ~2586×1080 = 11016
// MBs busts level 4.1 (8192 MaxFS); must bump to 5.0.
{"1080h anamorphic 2.39:1", 2586, 1080, "5.0"},
// Anamorphic 720 height (1728×720 = 4860 MBs) still fits the 4.0 the
// height floor already picks for fps headroom.
{"720h anamorphic 2.4:1", 1728, 720, "4.0"},
// Source 4K anamorphic (3840×1604) encoded at source: 24240 MBs → 5.1.
{"4K anamorphic source", 3840, 1604, "5.1"},
// Width unknown → fall back to the height-only tier.
{"width unknown", 0, 1080, "4.1"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if got := H264LevelForFrame(c.width, c.height); got != c.want {
t.Errorf("H264LevelForFrame(%d,%d) = %q, want %q", c.width, c.height, got, c.want)
}
})
}
}