feat(downloads): pre-flight free-disk guard before each download (hueco medio)
CheckDiskSpace (internal/engine/diskspace.go) refuses a download before writing when its expected size wouldn't leave a configurable reserve free, so a download never fills the filesystem to 0 mid-write (which corrupts the partial file). Wired into all three downloaders ahead of any write — torrent (DataDir), debrid (outputDir, resume-aware), usenet (outputDir, fresh only). Reserve from downloads.min_free_disk_mb (default 2048 MiB) via SetMinFreeBytes. The manager treats an InsufficientDiskError as terminal — no source fallback, since another source would fill the same disk — and surfaces the clear message. Best-effort: unknown size or a stat failure doesn't block (ENOSPC stays the backstop). Also hardens formatBytes against an exabyte-scale out-of-bounds panic.
This commit is contained in:
parent
2be92516c6
commit
1cad73b9a7
9 changed files with 196 additions and 4 deletions
|
|
@ -27,6 +27,8 @@ var httpClient = &http.Client{
|
|||
type DebridDownloader struct {
|
||||
activeMu sync.Mutex
|
||||
active map[string]context.CancelFunc
|
||||
|
||||
minFreeBytes int64 // disk reserve for the pre-flight space check (0 = reserve disabled)
|
||||
}
|
||||
|
||||
// NewDebridDownloader creates a debrid downloader.
|
||||
|
|
@ -36,6 +38,11 @@ func NewDebridDownloader() *DebridDownloader {
|
|||
}
|
||||
}
|
||||
|
||||
// SetMinFreeBytes sets the free-space reserve enforced before a download starts.
|
||||
// Call once at construction; 0 disables the reserve (the size-vs-free check still
|
||||
// runs). See CheckDiskSpace.
|
||||
func (d *DebridDownloader) SetMinFreeBytes(n int64) { d.minFreeBytes = n }
|
||||
|
||||
func (d *DebridDownloader) Method() DownloadMethod { return MethodDebrid }
|
||||
|
||||
// Available returns true if the task has a direct HTTPS URL from the server.
|
||||
|
|
@ -167,6 +174,12 @@ func (d *DebridDownloader) Download(ctx context.Context, task *Task, outputDir s
|
|||
log.Printf("[%s] starting debrid download: %s", agent.ShortID(task.ID), fileName)
|
||||
}
|
||||
|
||||
// Pre-flight disk-space guard on the bytes still to write (resume subtracts
|
||||
// what's already on disk). Best-effort; ENOSPC stays the backstop.
|
||||
if err := CheckDiskSpace(outputDir, totalBytes-startOffset, d.minFreeBytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create directory: %w", err)
|
||||
}
|
||||
|
|
|
|||
63
internal/engine/diskspace.go
Normal file
63
internal/engine/diskspace.go
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
package engine
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/torrentclaw/unarr/internal/agent"
|
||||
)
|
||||
|
||||
// InsufficientDiskError is returned by CheckDiskSpace when a download's expected
|
||||
// size (plus a reserve that keeps the filesystem healthy) won't fit in the free
|
||||
// space of its target directory. The manager treats it as terminal — it does NOT
|
||||
// fall back to another source (a different source would fill the same disk) and
|
||||
// surfaces the message to the web as the task's error.
|
||||
type InsufficientDiskError struct {
|
||||
Dir string
|
||||
Need int64 // bytes the download still needs to write
|
||||
Free int64 // bytes currently free on Dir's filesystem
|
||||
Reserve int64 // bytes to keep free after the download
|
||||
}
|
||||
|
||||
func (e *InsufficientDiskError) Error() string {
|
||||
return fmt.Sprintf(
|
||||
"insufficient disk space in %s: need %s + %s reserve, only %s free",
|
||||
e.Dir, formatBytes(e.Need), formatBytes(e.Reserve), formatBytes(e.Free),
|
||||
)
|
||||
}
|
||||
|
||||
// IsInsufficientDisk reports whether err is (or wraps) an InsufficientDiskError.
|
||||
func IsInsufficientDisk(err error) bool {
|
||||
var d *InsufficientDiskError
|
||||
return errors.As(err, &d)
|
||||
}
|
||||
|
||||
// CheckDiskSpace fails fast when dir's filesystem can't hold needBytes while
|
||||
// keeping reserveBytes free. It's the pre-flight guard so a download never fills
|
||||
// the disk to 0 mid-write (which corrupts the partial file and can wedge the OS).
|
||||
//
|
||||
// Best-effort by design: a non-positive needBytes (size unknown) or a failure to
|
||||
// stat the filesystem returns nil rather than block a download on a guard we
|
||||
// can't evaluate — the OS-level ENOSPC stays the backstop.
|
||||
func CheckDiskSpace(dir string, needBytes, reserveBytes int64) error {
|
||||
if needBytes <= 0 {
|
||||
return nil // size unknown — nothing to check against
|
||||
}
|
||||
free, _, err := agent.DiskInfo(dir)
|
||||
if err != nil {
|
||||
log.Printf("[disk] free-space pre-flight skipped for %q: stat error: %v", dir, err)
|
||||
return nil
|
||||
}
|
||||
if free <= 0 {
|
||||
// Distinct from a stat error: DiskInfo succeeded but reports no free
|
||||
// space. Don't block on a value we can't trust (0/negative) — log it so a
|
||||
// genuinely-full disk is visible rather than masked as a generic skip.
|
||||
log.Printf("[disk] free-space pre-flight skipped for %q: DiskInfo reported non-positive free (%d)", dir, free)
|
||||
return nil
|
||||
}
|
||||
if free-needBytes < reserveBytes {
|
||||
return &InsufficientDiskError{Dir: dir, Need: needBytes, Free: free, Reserve: reserveBytes}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
56
internal/engine/diskspace_test.go
Normal file
56
internal/engine/diskspace_test.go
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
package engine
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCheckDiskSpace_Enough(t *testing.T) {
|
||||
// A tiny need in a real temp dir (huge free space) → nil.
|
||||
if err := CheckDiskSpace(t.TempDir(), 1024, 0); err != nil {
|
||||
t.Errorf("expected nil for a 1 KiB need, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDiskSpace_Insufficient(t *testing.T) {
|
||||
// Need more than any real disk has → InsufficientDiskError.
|
||||
err := CheckDiskSpace(t.TempDir(), 1<<62, 0)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for an impossibly large need")
|
||||
}
|
||||
if !IsInsufficientDisk(err) {
|
||||
t.Errorf("IsInsufficientDisk = false, want true (err=%v)", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "insufficient disk space") {
|
||||
t.Errorf("error message = %q, want it to mention insufficient disk space", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDiskSpace_ReserveTriggers(t *testing.T) {
|
||||
// Tiny need but an impossibly large reserve → free-need < reserve → error.
|
||||
err := CheckDiskSpace(t.TempDir(), 1024, 1<<62)
|
||||
if !IsInsufficientDisk(err) {
|
||||
t.Errorf("expected insufficient when reserve exceeds free space, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDiskSpace_UnknownSize(t *testing.T) {
|
||||
// need <= 0 means the size is unknown — the check must be skipped, even with
|
||||
// an enormous reserve.
|
||||
if err := CheckDiskSpace(t.TempDir(), 0, 1<<62); err != nil {
|
||||
t.Errorf("need=0 must skip the check, got %v", err)
|
||||
}
|
||||
if err := CheckDiskSpace(t.TempDir(), -5, 1<<62); err != nil {
|
||||
t.Errorf("negative need must skip the check, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDiskSpace_BadDirIsBestEffort(t *testing.T) {
|
||||
// An unstat-able path → DiskInfo errors → best-effort nil (never block a
|
||||
// download on a guard we can't evaluate; ENOSPC stays the backstop).
|
||||
bad := filepath.Join(t.TempDir(), "does", "not", "exist")
|
||||
if err := CheckDiskSpace(bad, 1<<40, 0); err != nil {
|
||||
t.Errorf("unstat-able dir must skip the check, got %v", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -344,6 +344,12 @@ func (m *Manager) processTask(ctx context.Context, task *Task) {
|
|||
close(progressCh)
|
||||
|
||||
if err != nil {
|
||||
// A full disk is terminal — another source would fill the same disk, so
|
||||
// skip the fallback and surface the clear message immediately.
|
||||
if IsInsufficientDisk(err) {
|
||||
m.fail(ctx, task, err.Error())
|
||||
return
|
||||
}
|
||||
// Try fallback
|
||||
if tryFallback(task, m.downloaders) {
|
||||
log.Printf("[%s] %s failed, trying fallback: %v", agent.ShortID(task.ID), method, err)
|
||||
|
|
@ -386,6 +392,8 @@ func (m *Manager) processTaskRetry(ctx context.Context, task *Task) {
|
|||
close(progressCh)
|
||||
|
||||
if err != nil {
|
||||
// No further fallback here — same disk, same outcome — so an
|
||||
// InsufficientDiskError on the fallback surfaces its message directly.
|
||||
m.fail(ctx, task, err.Error())
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -90,8 +90,15 @@ type TorrentDownloader struct {
|
|||
|
||||
activeMu sync.Mutex
|
||||
active map[string]*torrent.Torrent // taskID -> torrent handle
|
||||
|
||||
minFreeBytes int64 // disk reserve for the pre-flight space check (0 = reserve disabled)
|
||||
}
|
||||
|
||||
// SetMinFreeBytes sets the free-space reserve enforced before a download starts.
|
||||
// Call once at construction; 0 disables the reserve (the size-vs-free check still
|
||||
// runs). See CheckDiskSpace.
|
||||
func (d *TorrentDownloader) SetMinFreeBytes(n int64) { d.minFreeBytes = n }
|
||||
|
||||
// NewTorrentDownloader creates a BitTorrent downloader with a long-lived client.
|
||||
func NewTorrentDownloader(cfg TorrentConfig) (*TorrentDownloader, error) {
|
||||
// MetadataTimeout: 0 = unlimited (wait forever like qBittorrent)
|
||||
|
|
@ -340,6 +347,15 @@ func (d *TorrentDownloader) Download(ctx context.Context, task *Task, outputDir
|
|||
|
||||
log.Printf("[%s] downloading %s (%s)", task.ID[:8], fileName, formatBytes(totalBytes))
|
||||
|
||||
// 2.5 Pre-flight disk-space guard — refuse before writing rather than fill
|
||||
// the disk to 0 mid-download (corrupts the partial file). Torrents land in
|
||||
// DataDir (not the manager's outputDir), so stat DataDir. Conservative: uses
|
||||
// the full selected size without subtracting pieces a resume may already hold.
|
||||
if err := CheckDiskSpace(d.cfg.DataDir, totalBytes, d.minFreeBytes); err != nil {
|
||||
cleanup()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 3. Poll progress with stall detection
|
||||
result, err := d.pollDownload(ctx, t, task, totalBytes, fileName, progressCh)
|
||||
if err != nil {
|
||||
|
|
@ -715,12 +731,15 @@ func formatBytes(b int64) string {
|
|||
if b < unit {
|
||||
return fmt.Sprintf("%d B", b)
|
||||
}
|
||||
// Cap exp at the last unit so an exabyte-scale value (or a corrupt/huge
|
||||
// size) can never index past the slice and panic.
|
||||
units := []string{"KB", "MB", "GB", "TB", "PB", "EB"}
|
||||
div, exp := int64(unit), 0
|
||||
for n := b / unit; n >= unit; n /= unit {
|
||||
for n := b / unit; n >= unit && exp < len(units)-1; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %s", float64(b)/float64(div), []string{"KB", "MB", "GB", "TB"}[exp])
|
||||
return fmt.Sprintf("%.1f %s", float64(b)/float64(div), units[exp])
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -42,8 +42,15 @@ type UsenetDownloader struct {
|
|||
// Cached NZB search results (from Available → Download)
|
||||
nzbCache map[string]*agent.NzbSearchResult // taskID → best result
|
||||
nzbCacheMu sync.RWMutex
|
||||
|
||||
minFreeBytes int64 // disk reserve for the pre-flight space check (0 = reserve disabled)
|
||||
}
|
||||
|
||||
// SetMinFreeBytes sets the free-space reserve enforced before a download starts.
|
||||
// Call once at construction; 0 disables the reserve (the size-vs-free check still
|
||||
// runs). See CheckDiskSpace.
|
||||
func (u *UsenetDownloader) SetMinFreeBytes(n int64) { u.minFreeBytes = n }
|
||||
|
||||
// NewUsenetDownloader creates a usenet downloader.
|
||||
// apiClient is used to call the web API for NZB search, download, and credentials.
|
||||
func NewUsenetDownloader(apiClient *agent.Client) *UsenetDownloader {
|
||||
|
|
@ -171,6 +178,12 @@ func (u *UsenetDownloader) Download(ctx context.Context, task *Task, outputDir s
|
|||
if resumed {
|
||||
log.Printf("[%s] resuming usenet download (%d/%d segments completed)",
|
||||
shortID, tracker.TotalCompleted(), totalSegs)
|
||||
} else {
|
||||
// Pre-flight disk-space guard on a fresh download (a resume already has
|
||||
// its partial bytes on disk; ENOSPC stays the backstop there).
|
||||
if err := CheckDiskSpace(outputDir, totalBytes, u.minFreeBytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Always flush progress on exit — covers graceful shutdown, SIGTERM,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue