From 1cad73b9a761cea2aeb995161fed605eff5bbab9 Mon Sep 17 00:00:00 2001 From: Deivid Soto Date: Sun, 31 May 2026 21:48:34 +0200 Subject: [PATCH] feat(downloads): pre-flight free-disk guard before each download (hueco medio) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CheckDiskSpace (internal/engine/diskspace.go) refuses a download before writing when its expected size wouldn't leave a configurable reserve free, so a download never fills the filesystem to 0 mid-write (which corrupts the partial file). Wired into all three downloaders ahead of any write — torrent (DataDir), debrid (outputDir, resume-aware), usenet (outputDir, fresh only). Reserve from downloads.min_free_disk_mb (default 2048 MiB) via SetMinFreeBytes. The manager treats an InsufficientDiskError as terminal — no source fallback, since another source would fill the same disk — and surfaces the clear message. Best-effort: unknown size or a stat failure doesn't block (ENOSPC stays the backstop). Also hardens formatBytes against an exabyte-scale out-of-bounds panic. --- Docs/plans/unarr-agent-roadmap.md | 8 +++- internal/cmd/daemon.go | 11 +++++- internal/config/config.go | 5 +++ internal/engine/debrid.go | 13 +++++++ internal/engine/diskspace.go | 63 +++++++++++++++++++++++++++++++ internal/engine/diskspace_test.go | 56 +++++++++++++++++++++++++++ internal/engine/manager.go | 8 ++++ internal/engine/torrent.go | 23 ++++++++++- internal/engine/usenet.go | 13 +++++++ 9 files changed, 196 insertions(+), 4 deletions(-) create mode 100644 internal/engine/diskspace.go create mode 100644 internal/engine/diskspace_test.go diff --git a/Docs/plans/unarr-agent-roadmap.md b/Docs/plans/unarr-agent-roadmap.md index 9192c6c..2f2cb3e 100644 --- a/Docs/plans/unarr-agent-roadmap.md +++ b/Docs/plans/unarr-agent-roadmap.md @@ -67,7 +67,7 @@ el pipeline de `prewarm` (ya hace encode de la siguiente ep) — generaliza prew desde la web. Diseño + set de opciones en el estado abajo. ### Huecos medios ⬜ -- Sin gestión de espacio en disco (`Statfs`) → disco lleno revienta a mitad. +- ~~Sin gestión de espacio en disco (`Statfs`)~~ ✅ **Pre-flight de espacio (2026-05-31)** — `CheckDiskSpace` antes de cada descarga (torrent/usenet/debrid) con reserva configurable `downloads.min_free_disk_mb` (default 2048); manager NO hace fallback en disco lleno; aviso web 507 `INSUFFICIENT_DISK` al despachar (torrentclaw). Monitoreo mid-download diferido. Ver estado abajo. - Resume de torrent NO persiste reinicio del daemon (usenet sí). - Sin seeding/ratio lifecycle (flags existen, nadie los aplica). - Reproducir-mientras-baja: readahead estático 5MB, sin playhead→prioridad dinámica. @@ -99,6 +99,12 @@ WireGuard endpoint sin pin · sesión única (1 viewer). - **Rutas localizadas unarr 404 (media)**: bajo `NEXT_PUBLIC_BRAND=unarr` el allowlist `UNARR_PAGE_PREFIXES` solo lista los paths EN (`/library`, `/title`, …), pero next-intl sirve los localizados (`/es/biblioteca`, `/es/descargas`, `/es/perfil`) → 404 al navegar la biblioteca en español. Las páginas EN (`/title/`) funcionan. Hallado durante el smoke de "características del fichero" (2026-05-31). Fix: añadir los pathnames localizados al allowlist o derivarlos del mapeo de next-intl. Ajeno a este hueco. - **Thumbnails — sprites/trickplay (media)**: cerrado solo el camino bajo demanda (N frames en vivo). El scrubber pregenerado (sprite/BIF de toda la timeline, preview al pasar el ratón por la barra) queda como hueco propio: reaprovecharía `/thumbnail` + cacheo en disco del agente. Decidido alcance "solo bajo demanda" con el usuario (2026-05-31). +### Hueco medio — Gestión de espacio en disco (pre-flight) ✅ CERRADO (2026-05-31) +Una descarga ya no llena el disco a 0 a mitad (corrompía el fichero parcial). +- **CLI**: `internal/engine/diskspace.go` — `CheckDiskSpace(dir, need, reserve)` usa `agent.DiskInfo` (Statfs/GetDiskFreeSpaceEx, ya abstraído) y devuelve `*InsufficientDiskError` si `free-need < reserve`; best-effort (need≤0 o stat falla → nil, ENOSPC sigue de backstop). Cableado antes de escribir en los 3 downloaders (torrent: DataDir+totalBytes; debrid: outputDir+restantes; usenet: outputDir+totalBytes solo en fresh). Reserva por `SetMinFreeBytes` desde `downloads.min_free_disk_mb` (default 2048 MiB). `manager` falla sin fallback en disco lleno (otra fuente llena el mismo disco). Fix latente: `formatBytes` paniqueaba ≥1PB (array hasta TB) → +PB/EB+clamp. +- **WEB**: `/api/internal/download` rechaza 507 `INSUFFICIENT_DISK` antes de crear la tarea si `diskFreeBytes - sizeBytes < 2 GiB` (reserva = default agente). Solo single-file torrent + agente online (telemetría de disco ya fluía). Saltado: stream, usenet, episodios (sizeBytes=pack completo → falso reject), agente offline. `DownloadButton` muestra estado `diskfull` (i18n 7 locales, namespace torrent). Bajo unarr el endpoint está fuera del allowlist → unarr solo streamea; el pre-flight del agente cubre sus descargas. +- **Tests/smoke**: Go `diskspace_test` (Statfs real vía TempDir: enough/insufficient/reserve/unknown/bad-dir). Web reject no e2e-smokeable en el dev box (es unarr → endpoint 404); verificado por build+typecheck+lógica. /critico 2 revisores → 2 bugs reales (guard sin `health.online`; falso reject en season packs) + 4 clarity. + ### Hueco medio — Características del fichero + thumbnails bajo demanda ✅ CERRADO (2026-05-31) Panel "ver características del fichero" (ruta + mediainfo completa: codec/HDR/bit-depth/tracks audio+subs/tamaño/duración — ya en DB vía ffprobe, solo faltaba surface) + tira de fotogramas extraídos en vivo por el agente. - **CLI**: `GET /thumbnail?p=&pos=&w=&t=` en el stream server (ffmpeg `-ss ` antes de `-i`, `-frames:v 1`, MJPEG a stdout). Token scope `thumb:` (mismo HMAC que `/stream`/`/hls`; web mintea, agente verifica; vector cross-lang Go↔TS pinneado). Clamp a fichero regular, 404-sin-oracle, timeout 20s. `ffmpegPath` cableado en `daemon.go`. Floor `0.13.0`. diff --git a/internal/cmd/daemon.go b/internal/cmd/daemon.go index 84dc0e1..dcb8b12 100644 --- a/internal/cmd/daemon.go +++ b/internal/cmd/daemon.go @@ -293,6 +293,15 @@ func runDaemonStart() error { // Create debrid downloader debridDl := engine.NewDebridDownloader() + usenetDl := engine.NewUsenetDownloader(agentClient) + + // Pre-flight disk reserve: refuse a download that would leave less than this + // many bytes free, so a download never fills the filesystem to 0 mid-write. + minFreeBytes := int64(cfg.Download.MinFreeDiskMB) << 20 + torrentDl.SetMinFreeBytes(minFreeBytes) + debridDl.SetMinFreeBytes(minFreeBytes) + usenetDl.SetMinFreeBytes(minFreeBytes) + log.Printf("[disk] download free-space reserve: %d MiB", cfg.Download.MinFreeDiskMB) // Create download manager manager := engine.NewManager(engine.ManagerConfig{ @@ -305,7 +314,7 @@ func runDaemonStart() error { TVShowsDir: cfg.Organize.TVShowsDir, OutputDir: cfg.Download.Dir, }, - }, reporter, torrentDl, debridDl, engine.NewUsenetDownloader(agentClient)) + }, reporter, torrentDl, debridDl, usenetDl) // Create persistent stream server streamSrv := engine.NewStreamServer(cfg.Download.StreamPort) diff --git a/internal/config/config.go b/internal/config/config.go index 0aec611..5cab448 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -43,6 +43,7 @@ type DownloadConfig struct { PreferredMethod string `toml:"preferred_method"` PreferredQuality string `toml:"preferred_quality"` // "2160p", "1080p", "720p" — hint for auto-selection MaxConcurrent int `toml:"max_concurrent"` + MinFreeDiskMB int `toml:"min_free_disk_mb"` // refuse a download if it would leave less than this free (reserve to keep the FS healthy); default 2048, 0 = disable MaxDownloadSpeed string `toml:"max_download_speed"` // e.g. "10MB", "500KB", "0" = unlimited MaxUploadSpeed string `toml:"max_upload_speed"` // e.g. "1MB", "0" = unlimited MetadataTimeout string `toml:"metadata_timeout"` // e.g. "1h", "30m", "0" = unlimited (default: "0") @@ -195,6 +196,7 @@ func Default() Config { Download: DownloadConfig{ PreferredMethod: "auto", MaxConcurrent: 3, + MinFreeDiskMB: 2048, // 2 GiB reserve StreamPort: 11818, RequireStreamToken: true, // secure by default; loopback exempt Transcode: TranscodeConfig{ @@ -293,6 +295,9 @@ func applyDefaults(cfg *Config, meta toml.MetaData) { if !meta.IsDefined("downloads", "max_concurrent") { cfg.Download.MaxConcurrent = 3 } + if !meta.IsDefined("downloads", "min_free_disk_mb") { + cfg.Download.MinFreeDiskMB = 2048 // 2 GiB reserve so a download never fills the FS to 0 + } if !meta.IsDefined("downloads", "stream_port") { cfg.Download.StreamPort = 11818 } diff --git a/internal/engine/debrid.go b/internal/engine/debrid.go index fce60dd..49fe8e6 100644 --- a/internal/engine/debrid.go +++ b/internal/engine/debrid.go @@ -27,6 +27,8 @@ var httpClient = &http.Client{ type DebridDownloader struct { activeMu sync.Mutex active map[string]context.CancelFunc + + minFreeBytes int64 // disk reserve for the pre-flight space check (0 = reserve disabled) } // NewDebridDownloader creates a debrid downloader. @@ -36,6 +38,11 @@ func NewDebridDownloader() *DebridDownloader { } } +// SetMinFreeBytes sets the free-space reserve enforced before a download starts. +// Call once at construction; 0 disables the reserve (the size-vs-free check still +// runs). See CheckDiskSpace. +func (d *DebridDownloader) SetMinFreeBytes(n int64) { d.minFreeBytes = n } + func (d *DebridDownloader) Method() DownloadMethod { return MethodDebrid } // Available returns true if the task has a direct HTTPS URL from the server. @@ -167,6 +174,12 @@ func (d *DebridDownloader) Download(ctx context.Context, task *Task, outputDir s log.Printf("[%s] starting debrid download: %s", agent.ShortID(task.ID), fileName) } + // Pre-flight disk-space guard on the bytes still to write (resume subtracts + // what's already on disk). Best-effort; ENOSPC stays the backstop. + if err := CheckDiskSpace(outputDir, totalBytes-startOffset, d.minFreeBytes); err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil { return nil, fmt.Errorf("create directory: %w", err) } diff --git a/internal/engine/diskspace.go b/internal/engine/diskspace.go new file mode 100644 index 0000000..7faf792 --- /dev/null +++ b/internal/engine/diskspace.go @@ -0,0 +1,63 @@ +package engine + +import ( + "errors" + "fmt" + "log" + + "github.com/torrentclaw/unarr/internal/agent" +) + +// InsufficientDiskError is returned by CheckDiskSpace when a download's expected +// size (plus a reserve that keeps the filesystem healthy) won't fit in the free +// space of its target directory. The manager treats it as terminal — it does NOT +// fall back to another source (a different source would fill the same disk) and +// surfaces the message to the web as the task's error. +type InsufficientDiskError struct { + Dir string + Need int64 // bytes the download still needs to write + Free int64 // bytes currently free on Dir's filesystem + Reserve int64 // bytes to keep free after the download +} + +func (e *InsufficientDiskError) Error() string { + return fmt.Sprintf( + "insufficient disk space in %s: need %s + %s reserve, only %s free", + e.Dir, formatBytes(e.Need), formatBytes(e.Reserve), formatBytes(e.Free), + ) +} + +// IsInsufficientDisk reports whether err is (or wraps) an InsufficientDiskError. +func IsInsufficientDisk(err error) bool { + var d *InsufficientDiskError + return errors.As(err, &d) +} + +// CheckDiskSpace fails fast when dir's filesystem can't hold needBytes while +// keeping reserveBytes free. It's the pre-flight guard so a download never fills +// the disk to 0 mid-write (which corrupts the partial file and can wedge the OS). +// +// Best-effort by design: a non-positive needBytes (size unknown) or a failure to +// stat the filesystem returns nil rather than block a download on a guard we +// can't evaluate — the OS-level ENOSPC stays the backstop. +func CheckDiskSpace(dir string, needBytes, reserveBytes int64) error { + if needBytes <= 0 { + return nil // size unknown — nothing to check against + } + free, _, err := agent.DiskInfo(dir) + if err != nil { + log.Printf("[disk] free-space pre-flight skipped for %q: stat error: %v", dir, err) + return nil + } + if free <= 0 { + // Distinct from a stat error: DiskInfo succeeded but reports no free + // space. Don't block on a value we can't trust (0/negative) — log it so a + // genuinely-full disk is visible rather than masked as a generic skip. + log.Printf("[disk] free-space pre-flight skipped for %q: DiskInfo reported non-positive free (%d)", dir, free) + return nil + } + if free-needBytes < reserveBytes { + return &InsufficientDiskError{Dir: dir, Need: needBytes, Free: free, Reserve: reserveBytes} + } + return nil +} diff --git a/internal/engine/diskspace_test.go b/internal/engine/diskspace_test.go new file mode 100644 index 0000000..a24d5be --- /dev/null +++ b/internal/engine/diskspace_test.go @@ -0,0 +1,56 @@ +package engine + +import ( + "path/filepath" + "strings" + "testing" +) + +func TestCheckDiskSpace_Enough(t *testing.T) { + // A tiny need in a real temp dir (huge free space) → nil. + if err := CheckDiskSpace(t.TempDir(), 1024, 0); err != nil { + t.Errorf("expected nil for a 1 KiB need, got %v", err) + } +} + +func TestCheckDiskSpace_Insufficient(t *testing.T) { + // Need more than any real disk has → InsufficientDiskError. + err := CheckDiskSpace(t.TempDir(), 1<<62, 0) + if err == nil { + t.Fatal("expected an error for an impossibly large need") + } + if !IsInsufficientDisk(err) { + t.Errorf("IsInsufficientDisk = false, want true (err=%v)", err) + } + if !strings.Contains(err.Error(), "insufficient disk space") { + t.Errorf("error message = %q, want it to mention insufficient disk space", err.Error()) + } +} + +func TestCheckDiskSpace_ReserveTriggers(t *testing.T) { + // Tiny need but an impossibly large reserve → free-need < reserve → error. + err := CheckDiskSpace(t.TempDir(), 1024, 1<<62) + if !IsInsufficientDisk(err) { + t.Errorf("expected insufficient when reserve exceeds free space, got %v", err) + } +} + +func TestCheckDiskSpace_UnknownSize(t *testing.T) { + // need <= 0 means the size is unknown — the check must be skipped, even with + // an enormous reserve. + if err := CheckDiskSpace(t.TempDir(), 0, 1<<62); err != nil { + t.Errorf("need=0 must skip the check, got %v", err) + } + if err := CheckDiskSpace(t.TempDir(), -5, 1<<62); err != nil { + t.Errorf("negative need must skip the check, got %v", err) + } +} + +func TestCheckDiskSpace_BadDirIsBestEffort(t *testing.T) { + // An unstat-able path → DiskInfo errors → best-effort nil (never block a + // download on a guard we can't evaluate; ENOSPC stays the backstop). + bad := filepath.Join(t.TempDir(), "does", "not", "exist") + if err := CheckDiskSpace(bad, 1<<40, 0); err != nil { + t.Errorf("unstat-able dir must skip the check, got %v", err) + } +} diff --git a/internal/engine/manager.go b/internal/engine/manager.go index 2a07b6f..36ba4a6 100644 --- a/internal/engine/manager.go +++ b/internal/engine/manager.go @@ -344,6 +344,12 @@ func (m *Manager) processTask(ctx context.Context, task *Task) { close(progressCh) if err != nil { + // A full disk is terminal — another source would fill the same disk, so + // skip the fallback and surface the clear message immediately. + if IsInsufficientDisk(err) { + m.fail(ctx, task, err.Error()) + return + } // Try fallback if tryFallback(task, m.downloaders) { log.Printf("[%s] %s failed, trying fallback: %v", agent.ShortID(task.ID), method, err) @@ -386,6 +392,8 @@ func (m *Manager) processTaskRetry(ctx context.Context, task *Task) { close(progressCh) if err != nil { + // No further fallback here — same disk, same outcome — so an + // InsufficientDiskError on the fallback surfaces its message directly. m.fail(ctx, task, err.Error()) return } diff --git a/internal/engine/torrent.go b/internal/engine/torrent.go index efcddbe..b93b6bd 100644 --- a/internal/engine/torrent.go +++ b/internal/engine/torrent.go @@ -90,8 +90,15 @@ type TorrentDownloader struct { activeMu sync.Mutex active map[string]*torrent.Torrent // taskID -> torrent handle + + minFreeBytes int64 // disk reserve for the pre-flight space check (0 = reserve disabled) } +// SetMinFreeBytes sets the free-space reserve enforced before a download starts. +// Call once at construction; 0 disables the reserve (the size-vs-free check still +// runs). See CheckDiskSpace. +func (d *TorrentDownloader) SetMinFreeBytes(n int64) { d.minFreeBytes = n } + // NewTorrentDownloader creates a BitTorrent downloader with a long-lived client. func NewTorrentDownloader(cfg TorrentConfig) (*TorrentDownloader, error) { // MetadataTimeout: 0 = unlimited (wait forever like qBittorrent) @@ -340,6 +347,15 @@ func (d *TorrentDownloader) Download(ctx context.Context, task *Task, outputDir log.Printf("[%s] downloading %s (%s)", task.ID[:8], fileName, formatBytes(totalBytes)) + // 2.5 Pre-flight disk-space guard — refuse before writing rather than fill + // the disk to 0 mid-download (corrupts the partial file). Torrents land in + // DataDir (not the manager's outputDir), so stat DataDir. Conservative: uses + // the full selected size without subtracting pieces a resume may already hold. + if err := CheckDiskSpace(d.cfg.DataDir, totalBytes, d.minFreeBytes); err != nil { + cleanup() + return nil, err + } + // 3. Poll progress with stall detection result, err := d.pollDownload(ctx, t, task, totalBytes, fileName, progressCh) if err != nil { @@ -715,12 +731,15 @@ func formatBytes(b int64) string { if b < unit { return fmt.Sprintf("%d B", b) } + // Cap exp at the last unit so an exabyte-scale value (or a corrupt/huge + // size) can never index past the slice and panic. + units := []string{"KB", "MB", "GB", "TB", "PB", "EB"} div, exp := int64(unit), 0 - for n := b / unit; n >= unit; n /= unit { + for n := b / unit; n >= unit && exp < len(units)-1; n /= unit { div *= unit exp++ } - return fmt.Sprintf("%.1f %s", float64(b)/float64(div), []string{"KB", "MB", "GB", "TB"}[exp]) + return fmt.Sprintf("%.1f %s", float64(b)/float64(div), units[exp]) } // --------------------------------------------------------------------------- diff --git a/internal/engine/usenet.go b/internal/engine/usenet.go index c39be86..4712cb5 100644 --- a/internal/engine/usenet.go +++ b/internal/engine/usenet.go @@ -42,8 +42,15 @@ type UsenetDownloader struct { // Cached NZB search results (from Available → Download) nzbCache map[string]*agent.NzbSearchResult // taskID → best result nzbCacheMu sync.RWMutex + + minFreeBytes int64 // disk reserve for the pre-flight space check (0 = reserve disabled) } +// SetMinFreeBytes sets the free-space reserve enforced before a download starts. +// Call once at construction; 0 disables the reserve (the size-vs-free check still +// runs). See CheckDiskSpace. +func (u *UsenetDownloader) SetMinFreeBytes(n int64) { u.minFreeBytes = n } + // NewUsenetDownloader creates a usenet downloader. // apiClient is used to call the web API for NZB search, download, and credentials. func NewUsenetDownloader(apiClient *agent.Client) *UsenetDownloader { @@ -171,6 +178,12 @@ func (u *UsenetDownloader) Download(ctx context.Context, task *Task, outputDir s if resumed { log.Printf("[%s] resuming usenet download (%d/%d segments completed)", shortID, tracker.TotalCompleted(), totalSegs) + } else { + // Pre-flight disk-space guard on a fresh download (a resume already has + // its partial bytes on disk; ENOSPC stays the backstop there). + if err := CheckDiskSpace(outputDir, totalBytes, u.minFreeBytes); err != nil { + return nil, err + } } // Always flush progress on exit — covers graceful shutdown, SIGTERM,