mirror of
https://git.sr.ht/~joren/streamrip-go
synced 2026-10-16 22:07:30 +02:00
Completes the correctness pass. Download database migration no longer destroys dedupe state The pre-source-column migration carried every legacy row over with source='' and then dropped downloads_legacy, while IsDownloaded matched source exactly. The result was that every previously recorded download looked new, the user silently re-downloaded their whole library, and the original rows were gone. Now the file is copied to <path>.bak-<unixtime> first, downloads_legacy is kept, and IsDownloaded falls back to the sourceless rows (over-matching across sources costs a skipped re-download, which is the cheaper error). Schema state is tracked in PRAGMA user_version so later migrations can be ordered explicitly. Databases already migrated by the old code are detected on open. Also enables WAL and a 10s busy_timeout and caps the pool at one connection, so a concurrent reader no longer gets SQLITE_BUSY with no retry, and builds the DSN through url.URL so a path containing '?' is not parsed as query parameters. --quality now applies to every source applyGlobalConfigOverrides set only qobuz, tidal, deezer and soundcloud, so `rip -q 4 url <beatport-url>` was a silent no-op even though qualityForSource reads Yandex and Beatport. All six are set, with the per-source floor applied and reported rather than silently ignored. config: validation and a fixed reset Load checked nothing but misc.version, so a negative max_connections or a bogus conversion codec was accepted and only surfaced mid-rip. Adds Validate() covering per-source quality ranges, max_connections, requests_per_minute and truncate_to, and normalizes the conversion codec from the file through the same path as --codec. `config reset` assigned one value to both Config.File and Config.Session, so their Secrets slices shared a backing array; it now takes separate copies. Tests cover the shipped defaults and config.toml.example, which is how the over-strict artwork max-width bound in the first draft was caught: -1 is the documented "do not downscale" sentinel. soundcloud: use the shared HTTP client It was the only provider constructing its own http.Client, so it silently ignored downloads.verify_ssl and max_connections and bypassed the transport that redacts credentials from -vv logs. It also had no rate limiter at all. qobuz: mobile fallback could not download a long track http.Client.Timeout bounds the whole request including the body read, so the 30s API client aborted any track that took longer than 30s to stream. The body copy now uses a no-timeout client, as internal/download already does. qobuz: HKDF no longer fails silently hexDecodeOrNil discarded its error, so a non-hex app secret keyed HKDF on nil, derived a wrong KEK and produced silently corrupt audio instead of an error. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
370 lines
11 KiB
Go
370 lines
11 KiB
Go
package download
|
|
|
|
import (
|
|
"context"
|
|
"crypto/aes"
|
|
"crypto/cipher"
|
|
"encoding/hex"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"golang.org/x/crypto/blowfish"
|
|
)
|
|
|
|
func TestDownloaderHasNoClientTimeout(t *testing.T) {
|
|
d := NewWithOptions(true, false, 0)
|
|
if d.http.Timeout != 0 {
|
|
t.Fatalf("http timeout = %v, want 0 (no global timeout)", d.http.Timeout)
|
|
}
|
|
}
|
|
|
|
func TestDownloaderFile(t *testing.T) {
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
_, _ = w.Write([]byte("abc123"))
|
|
}))
|
|
defer ts.Close()
|
|
|
|
d := New()
|
|
out := filepath.Join(t.TempDir(), "x", "a.bin")
|
|
if err := d.File(context.Background(), ts.URL, out); err != nil {
|
|
t.Fatalf("File() error = %v", err)
|
|
}
|
|
|
|
b, err := os.ReadFile(out)
|
|
if err != nil {
|
|
t.Fatalf("ReadFile() error = %v", err)
|
|
}
|
|
if string(b) != "abc123" {
|
|
t.Fatalf("contents = %q, want %q", string(b), "abc123")
|
|
}
|
|
}
|
|
|
|
func TestManifestDetection(t *testing.T) {
|
|
if !isManifestResponse("application/dash+xml", []byte("x")) {
|
|
t.Fatalf("expected dash content-type to be manifest")
|
|
}
|
|
if !isManifestResponse("application/vnd.apple.mpegurl", []byte("x")) {
|
|
t.Fatalf("expected mpegurl content-type to be manifest")
|
|
}
|
|
if !isManifestResponse("application/octet-stream", []byte("<?xml version='1.0'?><MPD></MPD>")) {
|
|
t.Fatalf("expected MPD XML body to be manifest")
|
|
}
|
|
if !isManifestResponse("application/octet-stream", []byte("<MPD type='static'></MPD>")) {
|
|
t.Fatalf("expected MPD body without xml prolog to be manifest")
|
|
}
|
|
if !isManifestResponse("text/plain", []byte("#EXTM3U\n#EXT-X-VERSION:3")) {
|
|
t.Fatalf("expected HLS body to be manifest")
|
|
}
|
|
if isManifestResponse("audio/flac", []byte("fLaC")) {
|
|
t.Fatalf("did not expect flac to be manifest")
|
|
}
|
|
}
|
|
|
|
func TestDeezerBlowfishKeyDerivation(t *testing.T) {
|
|
trackID := "3135556"
|
|
key := deriveDeezerBlowfishKey(trackID)
|
|
if len(key) != 16 {
|
|
t.Fatalf("blowfish key len = %d, want 16", len(key))
|
|
}
|
|
}
|
|
|
|
func TestFileDeezerEncrypted(t *testing.T) {
|
|
trackID := "3135556"
|
|
plain := make([]byte, deezerBFChunkSize+777)
|
|
for i := range plain {
|
|
plain[i] = byte((i * 7) % 251)
|
|
}
|
|
enc := make([]byte, len(plain))
|
|
copy(enc, plain)
|
|
|
|
block, err := blowfish.NewCipher(deriveDeezerBlowfishKey(trackID))
|
|
if err != nil {
|
|
t.Fatalf("cipher error: %v", err)
|
|
}
|
|
cbc := cipher.NewCBCEncrypter(block, deezerBFIV)
|
|
cbc.CryptBlocks(enc[:deezerBFChunkSize], enc[:deezerBFChunkSize])
|
|
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
_, _ = w.Write(enc)
|
|
}))
|
|
defer ts.Close()
|
|
|
|
d := NewWithOptions(true, false, 0)
|
|
out := filepath.Join(t.TempDir(), "x", "a.flac")
|
|
if err = d.FileDeezerEncrypted(context.Background(), ts.URL, out, trackID); err != nil {
|
|
t.Fatalf("FileDeezerEncrypted() error = %v", err)
|
|
}
|
|
|
|
got, err := os.ReadFile(out)
|
|
if err != nil {
|
|
t.Fatalf("ReadFile() error = %v", err)
|
|
}
|
|
if string(got) != string(plain) {
|
|
t.Fatalf("decrypted file mismatch")
|
|
}
|
|
}
|
|
|
|
func TestFileYandexEncrypted(t *testing.T) {
|
|
plain := make([]byte, 8192+333)
|
|
for i := range plain {
|
|
plain[i] = byte((i * 11) % 251)
|
|
}
|
|
keyHex := "00112233445566778899aabbccddeeff"
|
|
key, err := hex.DecodeString(keyHex)
|
|
if err != nil {
|
|
t.Fatalf("DecodeString() error = %v", err)
|
|
}
|
|
block, err := aes.NewCipher(key)
|
|
if err != nil {
|
|
t.Fatalf("NewCipher() error = %v", err)
|
|
}
|
|
enc := make([]byte, len(plain))
|
|
copy(enc, plain)
|
|
stream := cipher.NewCTR(block, make([]byte, aes.BlockSize))
|
|
stream.XORKeyStream(enc, enc)
|
|
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
_, _ = w.Write(enc)
|
|
}))
|
|
defer ts.Close()
|
|
|
|
d := NewWithOptions(true, false, 0)
|
|
out := filepath.Join(t.TempDir(), "x", "a.m4a")
|
|
if err = d.FileYandexEncrypted(context.Background(), ts.URL, out, keyHex); err != nil {
|
|
t.Fatalf("FileYandexEncrypted() error = %v", err)
|
|
}
|
|
|
|
got, err := os.ReadFile(out)
|
|
if err != nil {
|
|
t.Fatalf("ReadFile() error = %v", err)
|
|
}
|
|
if string(got) != string(plain) {
|
|
t.Fatalf("decrypted file mismatch")
|
|
}
|
|
}
|
|
|
|
func TestDownloaderFileTruncatedResponseRemovesPartialFile(t *testing.T) {
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Length", "10")
|
|
_, _ = w.Write([]byte("abc"))
|
|
}))
|
|
defer ts.Close()
|
|
|
|
d := NewWithOptions(true, false, 0)
|
|
out := filepath.Join(t.TempDir(), "x", "a.bin")
|
|
err := d.File(context.Background(), ts.URL, out)
|
|
if err == nil || !errors.Is(err, io.ErrUnexpectedEOF) {
|
|
t.Fatalf("expected unexpected EOF, got %v", err)
|
|
}
|
|
if _, statErr := os.Stat(out); !errors.Is(statErr, os.ErrNotExist) {
|
|
t.Fatalf("expected no partial file, stat err=%v", statErr)
|
|
}
|
|
}
|
|
|
|
func TestFileDeezerEncryptedTruncatedResponseRemovesPartialFile(t *testing.T) {
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Length", "4096")
|
|
_, _ = w.Write([]byte("short"))
|
|
}))
|
|
defer ts.Close()
|
|
|
|
d := NewWithOptions(true, false, 0)
|
|
out := filepath.Join(t.TempDir(), "x", "a.flac")
|
|
err := d.FileDeezerEncrypted(context.Background(), ts.URL, out, "3135556")
|
|
if err == nil || !errors.Is(err, io.ErrUnexpectedEOF) {
|
|
t.Fatalf("expected unexpected EOF, got %v", err)
|
|
}
|
|
if _, statErr := os.Stat(out); !errors.Is(statErr, os.ErrNotExist) {
|
|
t.Fatalf("expected no partial file, stat err=%v", statErr)
|
|
}
|
|
}
|
|
|
|
func TestFileDeezerEncryptedBadStatus(t *testing.T) {
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusForbidden)
|
|
}))
|
|
defer ts.Close()
|
|
|
|
d := NewWithOptions(true, false, 0)
|
|
out := filepath.Join(t.TempDir(), "x", "a.flac")
|
|
err := d.FileDeezerEncrypted(context.Background(), ts.URL, out, "3135556")
|
|
if err == nil || !strings.Contains(err.Error(), "status=403") {
|
|
t.Fatalf("expected status error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestFileYandexEncryptedBadKey(t *testing.T) {
|
|
d := NewWithOptions(true, false, 0)
|
|
out := filepath.Join(t.TempDir(), "x", "a.m4a")
|
|
err := d.FileYandexEncrypted(context.Background(), "https://example.com/file", out, "abcd")
|
|
if err == nil || !strings.Contains(err.Error(), "invalid yandex key length") {
|
|
t.Fatalf("expected key length error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestDownloaderFileContextCancellationRemovesPartialFile(t *testing.T) {
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/octet-stream")
|
|
_, _ = w.Write([]byte("partial-data"))
|
|
if f, ok := w.(http.Flusher); ok {
|
|
f.Flush()
|
|
}
|
|
<-r.Context().Done()
|
|
}))
|
|
defer ts.Close()
|
|
|
|
d := NewWithOptions(true, false, 0)
|
|
out := filepath.Join(t.TempDir(), "x", "cancel.bin")
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Millisecond)
|
|
defer cancel()
|
|
err := d.File(ctx, ts.URL, out)
|
|
if err == nil {
|
|
t.Fatalf("expected context cancellation error")
|
|
}
|
|
if _, statErr := os.Stat(out); !errors.Is(statErr, os.ErrNotExist) {
|
|
t.Fatalf("expected no partial file, stat err=%v", statErr)
|
|
}
|
|
}
|
|
|
|
func TestStreamManifestWithFFmpegMissing(t *testing.T) {
|
|
d := NewWithOptions(true, false, 0)
|
|
t.Setenv("PATH", "")
|
|
err := d.streamManifestWithFFmpeg(context.Background(), "https://example.com/live.m3u8", filepath.Join(t.TempDir(), "out.m4a"), false)
|
|
if err == nil || !strings.Contains(strings.ToLower(err.Error()), "ffmpeg not found") {
|
|
t.Fatalf("expected ffmpeg missing error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestStreamManifestWithFFmpegFailureRemovesPartialFile(t *testing.T) {
|
|
if _, err := exec.LookPath("ffmpeg"); err != nil {
|
|
t.Skip("ffmpeg not installed")
|
|
}
|
|
d := NewWithOptions(true, false, 0)
|
|
out := filepath.Join(t.TempDir(), "out.m4a")
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
err := d.streamManifestWithFFmpeg(ctx, "https://127.0.0.1:1/unreachable.m3u8", out, false)
|
|
if err == nil || !strings.Contains(err.Error(), "ffmpeg stream copy failed") {
|
|
t.Fatalf("expected ffmpeg failure error, got %v", err)
|
|
}
|
|
if _, statErr := os.Stat(out); !errors.Is(statErr, os.ErrNotExist) {
|
|
t.Fatalf("expected no partial file after ffmpeg failure, stat err=%v", statErr)
|
|
}
|
|
}
|
|
|
|
func TestParseFFmpegDurationLine(t *testing.T) {
|
|
totalMS, ok := parseFFmpegDurationLine(" Duration: 00:04:52.57, start: 0.000000, bitrate: 975 kb/s")
|
|
if !ok {
|
|
t.Fatalf("expected duration parse to succeed")
|
|
}
|
|
if want := int64(292570); totalMS != want {
|
|
t.Fatalf("unexpected duration ms: got=%d want=%d", totalMS, want)
|
|
}
|
|
}
|
|
|
|
func TestParseFFmpegDurationBitrateBPS(t *testing.T) {
|
|
bps, ok := parseFFmpegDurationBitrateBPS(" Duration: 00:04:52.57, start: 0.000000, bitrate: 975 kb/s")
|
|
if !ok {
|
|
t.Fatalf("expected bitrate parse to succeed")
|
|
}
|
|
if want := int64(975000); bps != want {
|
|
t.Fatalf("unexpected bitrate: got=%d want=%d", bps, want)
|
|
}
|
|
}
|
|
|
|
func TestParseFFmpegProgressBitrateBPS(t *testing.T) {
|
|
bps, ok := parseFFmpegProgressBitrateBPS("bitrate=1706.8kbits/s")
|
|
if !ok {
|
|
t.Fatalf("expected progress bitrate parse to succeed")
|
|
}
|
|
if want := int64(1706800); bps != want {
|
|
t.Fatalf("unexpected bitrate: got=%d want=%d", bps, want)
|
|
}
|
|
}
|
|
|
|
func TestParseFFmpegTotalSize(t *testing.T) {
|
|
size, ok := parseFFmpegTotalSize("total_size=1234567")
|
|
if !ok {
|
|
t.Fatalf("expected total_size parse to succeed")
|
|
}
|
|
if want := int64(1234567); size != want {
|
|
t.Fatalf("unexpected total_size: got=%d want=%d", size, want)
|
|
}
|
|
}
|
|
|
|
func TestParseFFmpegOutTime(t *testing.T) {
|
|
currentMS, ok := parseFFmpegOutTime("out_time=00:01:02.340000")
|
|
if !ok {
|
|
t.Fatalf("expected out_time parse to succeed")
|
|
}
|
|
if want := int64(62340); currentMS != want {
|
|
t.Fatalf("unexpected out_time ms: got=%d want=%d", currentMS, want)
|
|
}
|
|
}
|
|
|
|
func TestParseClockDurationMSInvalid(t *testing.T) {
|
|
if _, ok := parseClockDurationMS("bad"); ok {
|
|
t.Fatalf("expected invalid duration to fail")
|
|
}
|
|
if _, ok := parseClockDurationMS("00:12"); ok {
|
|
t.Fatalf("expected short duration to fail")
|
|
}
|
|
}
|
|
|
|
func TestEstimateTotalBytesFromBitrate(t *testing.T) {
|
|
total := estimateTotalBytesFromBitrate(10000, 1600000)
|
|
if want := int64(2000000); total != want {
|
|
t.Fatalf("unexpected estimate from bitrate: got=%d want=%d", total, want)
|
|
}
|
|
}
|
|
|
|
func TestEstimateTotalBytesFromProgress(t *testing.T) {
|
|
total := estimateTotalBytesFromProgress(10000, 2500, 500000)
|
|
if want := int64(2000000); total != want {
|
|
t.Fatalf("unexpected estimate from progress: got=%d want=%d", total, want)
|
|
}
|
|
}
|
|
|
|
func TestBuildFFmpegStreamArgsAudioOnly(t *testing.T) {
|
|
args := buildFFmpegStreamArgs("https://example.com/master.m3u8", "/tmp/out.m4a", false)
|
|
if !containsArgPair(args, "-map", "0:a:0") {
|
|
t.Fatalf("expected audio map in args: %v", args)
|
|
}
|
|
if containsArgPair(args, "-map", "0:v:0?") {
|
|
t.Fatalf("did not expect video map in audio-only args: %v", args)
|
|
}
|
|
if containsArgPair(args, "-map", "0") {
|
|
t.Fatalf("did not expect broad map=0 in args: %v", args)
|
|
}
|
|
}
|
|
|
|
func TestBuildFFmpegStreamArgsIncludeVideo(t *testing.T) {
|
|
args := buildFFmpegStreamArgs("https://example.com/master.m3u8", "/tmp/out.mp4", true)
|
|
if !containsArgPair(args, "-map", "0:v:0?") {
|
|
t.Fatalf("expected video map in args: %v", args)
|
|
}
|
|
if !containsArgPair(args, "-map", "0:a:0?") {
|
|
t.Fatalf("expected audio map in args: %v", args)
|
|
}
|
|
if containsArgPair(args, "-map", "0") {
|
|
t.Fatalf("did not expect broad map=0 in args: %v", args)
|
|
}
|
|
}
|
|
|
|
func containsArgPair(args []string, key, value string) bool {
|
|
for i := 0; i+1 < len(args); i++ {
|
|
if args[i] == key && args[i+1] == value {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|