refactor: consolidate the duplicated download and worker-pool code

internal/download
  The ~40-line progress-bar decorator stack was copy-pasted into three download
  paths and had started to drift; it is now newDownloadBar, which returns a nil
  bar when progress is off (so callers use it unconditionally) plus a finish
  func to defer. The three hand-rolled read/write loops that differed only in
  whether they touched a bar become copyWithProgress.

  FileDeezerEncrypted and FileYandexEncrypted were ~90% identical: mkdir, GET,
  status check, create output, bar, cleanup-on-failure, Content-Length check,
  fsync. That scaffolding is now streamEncrypted, and each function supplies
  only its decryption step. Also switches the io.EOF/io.ErrUnexpectedEOF
  comparisons to errors.Is.

internal/app
  Three copies of "coerce a JSON array that may be []any or []map[string]any"
  become anySlice, which also reports whether the value was an array at all so
  callers can still distinguish missing from empty.

  The worker-pool dispatch existed three times and the copies disagreed: two
  counted failures under a mutex and logged inline with the track id, while the
  third collected errors in a channel and logged them all after wg.Wait()
  without the id. runTrackWorkers unifies them on the inline form, which
  reports failures as they happen and names the track, and keeps the mutex held
  across the log so failure lines cannot interleave.

Tests
  The progress-bar paths had no coverage at all, because progress is disabled
  when not attached to a terminal; four new tests force it on via the existing
  STREAMRIP_GO_FORCE_PROGRESS escape hatch and cover the determinate bar, the
  unknown-length spinner, Deezer decryption with a bar attached, and the
  failure path still removing the partial file. internal/download coverage
  53% -> 63%.

  Three cancellation tests pin the Ctrl-C fix. Verified as genuine regression
  tests by reverting the guards: without them the rip walks all 40 tracks of a
  cancelled album, logs a failure for each, and Rip() returns nil.

Adds store.SQLite.IsFailed, the read complement to MarkFailed, used by the
bookkeeping test to confirm failure records still land under a cancelled
context.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
JorenandClaude Opus 5 committed 2026-10-03 17:23:24 +02:00
1 parent 78d4f0316d
commit b5d4d5688d
5 files changed
+629 -436

No files matched your search

+173 -247
View File
@@ -7,6 +7,7 @@ import (
"crypto/cipher"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
@@ -70,11 +71,24 @@ func (d *Downloader) FileVideo(ctx context.Context, sourceURL, outputPath string
return d.file(ctx, sourceURL, outputPath, true, true)
}
func (d *Downloader) FileDeezerEncrypted(ctx context.Context, sourceURL, outputPath, trackID string) error {
// streamEncrypted performs the shared scaffolding for an encrypted download:
// create the output directory, GET the URL, check the status, create the output
// file, attach a progress bar, clean up a partial file on failure, verify the
// byte count against Content-Length and fsync.
//
// decode does the format-specific work of turning the response body into plain
// audio on dst and returns the number of bytes written. FileDeezerEncrypted and
// FileYandexEncrypted were ~90% identical before this.
func (d *Downloader) streamEncrypted(
ctx context.Context,
sourceURL, outputPath string,
decode func(dst io.Writer, src io.Reader, bar *mpb.Bar) (int64, error),
) error {
logDownloadStart(sourceURL, outputPath)
if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil)
if err != nil {
return err
@@ -87,6 +101,7 @@ func (d *Downloader) FileDeezerEncrypted(ctx context.Context, sourceURL, outputP
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("download failed: status=%d", resp.StatusCode)
}
out, err := os.Create(outputPath)
if err != nil {
return err
@@ -99,97 +114,72 @@ func (d *Downloader) FileDeezerEncrypted(ctx context.Context, sourceURL, outputP
}
}()
var bar *mpb.Bar
if d.ProgressEnabled() {
d.barStarted.Store(1)
desc := shortenName(filepath.Base(outputPath), 54)
if resp.ContentLength > 0 {
bar = d.progress.AddBar(
resp.ContentLength,
mpb.PrependDecorators(
decor.Name(desc+" ", decor.WC{W: 56, C: decor.DSyncWidth | decor.DindentRight}),
decor.Percentage(decor.WCSyncWidthR),
),
mpb.AppendDecorators(
decor.CountersKibiByte("% .1f / % .1f", decor.WCSyncWidthR),
decor.Name(" | ", decor.WCSyncWidth),
decor.AverageSpeed(decor.SizeB1024(0), "% .1f", decor.WCSyncWidthR),
decor.Name(" | ETA ", decor.WCSyncWidth),
decor.AverageETA(decor.ET_STYLE_GO, decor.WCSyncWidthR),
),
mpb.BarRemoveOnComplete(),
)
} else {
bar = d.progress.AddSpinner(
0,
mpb.PrependDecorators(
decor.Name(desc+" ", decor.WC{W: 56, C: decor.DSyncWidth | decor.DindentRight}),
),
mpb.AppendDecorators(
decor.CurrentKibiByte("% .1f", decor.WCSyncWidthR),
decor.Name(" | ", decor.WCSyncWidth),
decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncWidthR),
),
mpb.BarRemoveOnComplete(),
)
defer bar.SetTotal(-1, true)
}
defer func() {
if !success && bar != nil {
bar.Abort(true)
}
}()
}
bar, finishBar := d.newDownloadBar(outputPath, resp.ContentLength)
defer func() { finishBar(success) }()
block, err := blowfish.NewCipher(deriveDeezerBlowfishKey(d.blowfishSecret(), trackID))
written, err := decode(out, resp.Body, bar)
if err != nil {
return err
}
buf := make([]byte, deezerBFChunkSize)
dec := make([]byte, deezerBFChunkSize)
chunkIndex := 0
totalRead := int64(0)
for {
n, readErr := io.ReadFull(resp.Body, buf)
if readErr == io.EOF {
break
}
if readErr != nil && readErr != io.ErrUnexpectedEOF {
return readErr
}
chunk := buf[:n]
if chunkIndex%3 == 0 && n == deezerBFChunkSize {
mode := cipher.NewCBCDecrypter(block, deezerBFIV)
mode.CryptBlocks(dec[:n], chunk)
chunk = dec[:n]
}
if _, err = out.Write(chunk); err != nil {
return err
}
totalRead += int64(n)
if bar != nil {
bar.IncrBy(n)
}
chunkIndex++
if readErr == io.ErrUnexpectedEOF {
break
}
}
if resp.ContentLength > 0 && totalRead != resp.ContentLength {
if resp.ContentLength > 0 && written != resp.ContentLength {
return io.ErrUnexpectedEOF
}
if err = out.Sync(); err != nil {
return err
}
success = true
return nil
}
func (d *Downloader) FileYandexEncrypted(ctx context.Context, sourceURL, outputPath, key string) error {
logDownloadStart(sourceURL, outputPath)
if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
// FileDeezerEncrypted downloads a Deezer stream and decrypts it. Deezer
// encrypts every third 2048-byte chunk with Blowfish-CBC; a short final chunk
// is left as-is.
func (d *Downloader) FileDeezerEncrypted(ctx context.Context, sourceURL, outputPath, trackID string) error {
block, err := blowfish.NewCipher(deriveDeezerBlowfishKey(d.blowfishSecret(), trackID))
if err != nil {
return err
}
return d.streamEncrypted(ctx, sourceURL, outputPath, func(dst io.Writer, src io.Reader, bar *mpb.Bar) (int64, error) {
buf := make([]byte, deezerBFChunkSize)
dec := make([]byte, deezerBFChunkSize)
chunkIndex := 0
total := int64(0)
for {
n, readErr := io.ReadFull(src, buf)
if errors.Is(readErr, io.EOF) {
return total, nil
}
if readErr != nil && !errors.Is(readErr, io.ErrUnexpectedEOF) {
return total, readErr
}
chunk := buf[:n]
if chunkIndex%3 == 0 && n == deezerBFChunkSize {
mode := cipher.NewCBCDecrypter(block, deezerBFIV)
mode.CryptBlocks(dec[:n], chunk)
chunk = dec[:n]
}
if _, writeErr := dst.Write(chunk); writeErr != nil {
return total, writeErr
}
total += int64(n)
if bar != nil {
bar.IncrBy(n)
}
chunkIndex++
if errors.Is(readErr, io.ErrUnexpectedEOF) {
return total, nil
}
}
})
}
// FileYandexEncrypted downloads a Yandex stream and decrypts it. Yandex uses
// AES-CTR over the whole body with a zero IV.
func (d *Downloader) FileYandexEncrypted(ctx context.Context, sourceURL, outputPath, key string) error {
keyBytes, err := hex.DecodeString(strings.TrimSpace(key))
if err != nil {
return fmt.Errorf("invalid yandex key: %w", err)
@@ -201,102 +191,11 @@ func (d *Downloader) FileYandexEncrypted(ctx context.Context, sourceURL, outputP
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil)
if err != nil {
return err
}
resp, err := d.http.Do(req)
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("download failed: status=%d", resp.StatusCode)
}
out, err := os.Create(outputPath)
if err != nil {
return err
}
success := false
defer func() {
_ = out.Close()
if !success {
_ = os.Remove(outputPath)
}
}()
var bar *mpb.Bar
if d.ProgressEnabled() {
d.barStarted.Store(1)
desc := shortenName(filepath.Base(outputPath), 54)
if resp.ContentLength > 0 {
bar = d.progress.AddBar(
resp.ContentLength,
mpb.PrependDecorators(
decor.Name(desc+" ", decor.WC{W: 56, C: decor.DSyncWidth | decor.DindentRight}),
decor.Percentage(decor.WCSyncWidthR),
),
mpb.AppendDecorators(
decor.CountersKibiByte("% .1f / % .1f", decor.WCSyncWidthR),
decor.Name(" | ", decor.WCSyncWidth),
decor.AverageSpeed(decor.SizeB1024(0), "% .1f", decor.WCSyncWidthR),
decor.Name(" | ETA ", decor.WCSyncWidth),
decor.AverageETA(decor.ET_STYLE_GO, decor.WCSyncWidthR),
),
mpb.BarRemoveOnComplete(),
)
} else {
bar = d.progress.AddSpinner(
0,
mpb.PrependDecorators(
decor.Name(desc+" ", decor.WC{W: 56, C: decor.DSyncWidth | decor.DindentRight}),
),
mpb.AppendDecorators(
decor.CurrentKibiByte("% .1f", decor.WCSyncWidthR),
decor.Name(" | ", decor.WCSyncWidth),
decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncWidthR),
),
mpb.BarRemoveOnComplete(),
)
defer bar.SetTotal(-1, true)
}
defer func() {
if !success && bar != nil {
bar.Abort(true)
}
}()
}
stream := cipher.NewCTR(block, make([]byte, aes.BlockSize))
reader := &cipher.StreamReader{S: stream, R: resp.Body}
buf := make([]byte, downloadBufferSize)
totalWritten := int64(0)
for {
n, readErr := reader.Read(buf)
if n > 0 {
if _, writeErr := out.Write(buf[:n]); writeErr != nil {
return writeErr
}
totalWritten += int64(n)
if bar != nil {
bar.IncrBy(n)
}
}
if readErr != nil {
if readErr == io.EOF {
break
}
return readErr
}
}
if resp.ContentLength > 0 && totalWritten != resp.ContentLength {
return io.ErrUnexpectedEOF
}
if err = out.Sync(); err != nil {
return err
}
success = true
return nil
return d.streamEncrypted(ctx, sourceURL, outputPath, func(dst io.Writer, src io.Reader, bar *mpb.Bar) (int64, error) {
stream := cipher.NewCTR(block, make([]byte, aes.BlockSize))
return copyWithProgress(dst, &cipher.StreamReader{S: stream, R: src}, bar)
})
}
func (d *Downloader) file(ctx context.Context, sourceURL, outputPath string, allowProgress bool, includeVideo bool) error {
@@ -339,81 +238,22 @@ func (d *Downloader) file(ctx context.Context, sourceURL, outputPath string, all
}
}()
if d.ProgressEnabled() && allowProgress {
d.barStarted.Store(1)
desc := shortenName(filepath.Base(outputPath), 54)
var bar *mpb.Bar
if resp.ContentLength > 0 {
bar = d.progress.AddBar(
resp.ContentLength,
mpb.PrependDecorators(
decor.Name(desc+" ", decor.WC{W: 56, C: decor.DSyncWidth | decor.DindentRight}),
decor.Percentage(decor.WCSyncWidthR),
),
mpb.AppendDecorators(
decor.CountersKibiByte("% .1f / % .1f", decor.WCSyncWidthR),
decor.Name(" | ", decor.WCSyncWidth),
decor.AverageSpeed(decor.SizeB1024(0), "% .1f", decor.WCSyncWidthR),
decor.Name(" | ETA ", decor.WCSyncWidth),
decor.AverageETA(decor.ET_STYLE_GO, decor.WCSyncWidthR),
),
mpb.BarRemoveOnComplete(),
)
} else {
bar = d.progress.AddSpinner(
0,
mpb.PrependDecorators(
decor.Name(desc+" ", decor.WC{W: 56, C: decor.DSyncWidth | decor.DindentRight}),
),
mpb.AppendDecorators(
decor.CurrentKibiByte("% .1f", decor.WCSyncWidthR),
decor.Name(" | ", decor.WCSyncWidth),
decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncWidthR),
),
mpb.BarRemoveOnComplete(),
)
defer bar.SetTotal(-1, true)
}
defer func() {
if !success && bar != nil {
bar.Abort(true)
}
}()
buf := make([]byte, downloadBufferSize)
totalWritten := int64(0)
for {
n, readErr := reader.Read(buf)
if n > 0 {
if _, writeErr := out.Write(buf[:n]); writeErr != nil {
return writeErr
}
totalWritten += int64(n)
bar.IncrBy(n)
}
if readErr != nil {
if readErr == io.EOF {
break
}
return readErr
}
}
if resp.ContentLength > 0 && totalWritten != resp.ContentLength {
return io.ErrUnexpectedEOF
}
if err = out.Sync(); err != nil {
return err
}
} else {
written, copyErr := io.CopyBuffer(out, reader, make([]byte, downloadBufferSize))
if copyErr != nil {
return copyErr
}
if resp.ContentLength > 0 && written != resp.ContentLength {
return io.ErrUnexpectedEOF
}
if err = out.Sync(); err != nil {
return err
}
var bar *mpb.Bar
finishBar := func(bool) {}
if allowProgress {
bar, finishBar = d.newDownloadBar(outputPath, resp.ContentLength)
}
defer func() { finishBar(success) }()
written, err := copyWithProgress(out, reader, bar)
if err != nil {
return err
}
if resp.ContentLength > 0 && written != resp.ContentLength {
return io.ErrUnexpectedEOF
}
if err = out.Sync(); err != nil {
return err
}
success = true
@@ -439,6 +279,92 @@ func (d *Downloader) Logf(format string, args ...any) {
fmt.Print(msg)
}
// copyWithProgress copies src to dst, reporting progress to bar when it is
// non-nil. It replaces three hand-rolled read/write loops that differed only
// in whether they touched a progress bar.
func copyWithProgress(dst io.Writer, src io.Reader, bar *mpb.Bar) (int64, error) {
if bar == nil {
return io.CopyBuffer(dst, src, make([]byte, downloadBufferSize))
}
buf := make([]byte, downloadBufferSize)
total := int64(0)
for {
n, readErr := src.Read(buf)
if n > 0 {
if _, writeErr := dst.Write(buf[:n]); writeErr != nil {
return total, writeErr
}
total += int64(n)
bar.IncrBy(n)
}
if readErr != nil {
if errors.Is(readErr, io.EOF) {
return total, nil
}
return total, readErr
}
}
}
// newDownloadBar creates the progress bar for a download: a determinate bar
// when the content length is known, a spinner when it is not. It returns a nil
// bar when progress display is off, so callers can use the result
// unconditionally, plus a finish func to defer.
//
// This block was copy-pasted into all three download paths; the decorator
// stacks had already started to drift apart.
func (d *Downloader) newDownloadBar(outputPath string, contentLength int64) (*mpb.Bar, func(success bool)) {
if !d.ProgressEnabled() {
return nil, func(bool) {}
}
d.barStarted.Store(1)
desc := shortenName(filepath.Base(outputPath), 54)
if contentLength > 0 {
bar := d.progress.AddBar(
contentLength,
mpb.PrependDecorators(
decor.Name(desc+" ", decor.WC{W: 56, C: decor.DSyncWidth | decor.DindentRight}),
decor.Percentage(decor.WCSyncWidthR),
),
mpb.AppendDecorators(
decor.CountersKibiByte("% .1f / % .1f", decor.WCSyncWidthR),
decor.Name(" | ", decor.WCSyncWidth),
decor.AverageSpeed(decor.SizeB1024(0), "% .1f", decor.WCSyncWidthR),
decor.Name(" | ETA ", decor.WCSyncWidth),
decor.AverageETA(decor.ET_STYLE_GO, decor.WCSyncWidthR),
),
mpb.BarRemoveOnComplete(),
)
return bar, func(success bool) {
if !success {
bar.Abort(true)
}
}
}
bar := d.progress.AddSpinner(
0,
mpb.PrependDecorators(
decor.Name(desc+" ", decor.WC{W: 56, C: decor.DSyncWidth | decor.DindentRight}),
),
mpb.AppendDecorators(
decor.CurrentKibiByte("% .1f", decor.WCSyncWidthR),
decor.Name(" | ", decor.WCSyncWidth),
decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncWidthR),
),
mpb.BarRemoveOnComplete(),
)
// Order matches the original defers, which ran LIFO: abort first, then
// finalise the spinner total.
return bar, func(success bool) {
if !success {
bar.Abort(true)
}
bar.SetTotal(-1, true)
}
}
// logDownloadStart emits the source URL and destination filename when the
// user passed -v or higher. The transport-level logger covers the same
// requests at -vv, but this line gives a friendlier per-track summary.
+143
View File
@@ -1,6 +1,7 @@
package download
import (
"bytes"
"context"
"crypto/aes"
"crypto/cipher"
@@ -12,6 +13,7 @@ import (
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
@@ -367,3 +369,144 @@ func containsArgPair(args []string, key, value string) bool {
}
return false
}
// newProgressDownloader builds a Downloader with the progress bars actually
// enabled. The bar code paths are otherwise never exercised, because tests do
// not run on a terminal; STREAMRIP_GO_FORCE_PROGRESS is the documented escape
// hatch. Output goes to stderr, which `go test` captures.
func newProgressDownloader(t *testing.T) *Downloader {
t.Helper()
t.Setenv("STREAMRIP_GO_FORCE_PROGRESS", "1")
d := NewWithOptions(true, true, 0)
if !d.ProgressEnabled() {
t.Fatal("expected progress to be enabled")
}
t.Cleanup(d.Close)
return d
}
// TestFileWithProgressBar covers the determinate-bar path (Content-Length
// known) through the shared newDownloadBar/copyWithProgress helpers.
func TestFileWithProgressBar(t *testing.T) {
payload := bytes.Repeat([]byte("abcd"), 4096)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Length", strconv.Itoa(len(payload)))
_, _ = w.Write(payload)
}))
defer ts.Close()
d := newProgressDownloader(t)
out := filepath.Join(t.TempDir(), "sub", "a.flac")
if err := d.File(context.Background(), ts.URL, out); err != nil {
t.Fatalf("File() error = %v", err)
}
got, err := os.ReadFile(out)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, payload) {
t.Fatalf("content mismatch: got %d bytes, want %d", len(got), len(payload))
}
}
// TestFileWithSpinnerUnknownLength covers the spinner path, taken when the
// server does not send a Content-Length.
func TestFileWithSpinnerUnknownLength(t *testing.T) {
payload := bytes.Repeat([]byte("xyz"), 3000)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Transfer-Encoding", "chunked")
_, _ = w.Write(payload)
}))
defer ts.Close()
d := newProgressDownloader(t)
out := filepath.Join(t.TempDir(), "a.flac")
if err := d.File(context.Background(), ts.URL, out); err != nil {
t.Fatalf("File() error = %v", err)
}
got, err := os.ReadFile(out)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, payload) {
t.Fatalf("content mismatch: got %d bytes, want %d", len(got), len(payload))
}
}
// TestFileDeezerEncryptedWithProgressBar runs the Deezer decryption through
// the progress-enabled path, so the shared streamEncrypted scaffolding is
// covered with a bar attached as well as without.
func TestFileDeezerEncryptedWithProgressBar(t *testing.T) {
plain := make([]byte, deezerBFChunkSize*4+100)
for i := range plain {
plain[i] = byte((i * 7) % 253)
}
trackID := "123456"
enc := encryptDeezerPayload(t, plain, trackID)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Length", strconv.Itoa(len(enc)))
_, _ = w.Write(enc)
}))
defer ts.Close()
d := newProgressDownloader(t)
out := filepath.Join(t.TempDir(), "a.mp3")
if err := d.FileDeezerEncrypted(context.Background(), ts.URL, out, trackID); err != nil {
t.Fatalf("FileDeezerEncrypted() error = %v", err)
}
got, err := os.ReadFile(out)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, plain) {
t.Fatal("decrypted output does not match the original plaintext")
}
}
// encryptDeezerPayload applies Deezer's scheme: Blowfish-CBC on every third
// full 2048-byte chunk, everything else left in the clear.
func encryptDeezerPayload(t *testing.T, plain []byte, trackID string) []byte {
t.Helper()
block, err := blowfish.NewCipher(deriveDeezerBlowfishKey(defaultDeezerBlowfishSecret, trackID))
if err != nil {
t.Fatal(err)
}
enc := make([]byte, len(plain))
copy(enc, plain)
for i, off := 0, 0; off < len(plain); i, off = i+1, off+deezerBFChunkSize {
end := off + deezerBFChunkSize
if end > len(plain) {
break
}
if i%3 != 0 {
continue
}
mode := cipher.NewCBCEncrypter(block, deezerBFIV)
mode.CryptBlocks(enc[off:end], plain[off:end])
}
return enc
}
// TestProgressBarFailurePathRemovesPartialFile checks that a mid-download
// failure still runs the bar cleanup and removes the partial output, which is
// the behaviour the finishBar(success) defer replaced.
func TestProgressBarFailurePathRemovesPartialFile(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Length", "100000")
_, _ = w.Write(bytes.Repeat([]byte("a"), 10))
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
}))
defer ts.Close()
d := newProgressDownloader(t)
out := filepath.Join(t.TempDir(), "a.flac")
if err := d.File(context.Background(), ts.URL, out); err == nil {
t.Fatal("expected a truncated-response error")
}
if _, err := os.Stat(out); !os.IsNotExist(err) {
t.Fatalf("partial file not removed, stat err = %v", err)
}
}