From 945695cea7530c2823e2d55d7880edf49871af79 Mon Sep 17 00:00:00 2001 From: lb-a Date: Wed, 29 Apr 2026 23:53:51 +0200 Subject: [PATCH] performance: tune shared HTTP transport for concurrent CDN downloads --- cmd/rip/lastfm.go | 6 +++--- internal/app/app.go | 9 ++++++++- internal/app/app_test.go | 8 ++++---- internal/download/downloader.go | 10 ++++++---- internal/download/downloader_test.go | 16 ++++++++-------- internal/netutil/http.go | 27 ++++++++++++++++++++++++++- internal/provider/deezer/client.go | 2 +- internal/provider/qobuz/client.go | 2 +- internal/provider/tidal/client.go | 2 +- 9 files changed, 58 insertions(+), 24 deletions(-) diff --git a/cmd/rip/lastfm.go b/cmd/rip/lastfm.go index 38eb74c..2d96fb3 100644 --- a/cmd/rip/lastfm.go +++ b/cmd/rip/lastfm.go @@ -86,7 +86,7 @@ func fetchLastFMPlaylist(ctx context.Context, verifySSL bool, playlistURL string if !isValidLastFMPlaylistURL(playlistURL) { return "", nil, fmt.Errorf("invalid playlist url") } - client := netutil.NewHTTPClient(30*time.Second, verifySSL) + client := netutil.NewHTTPClient(30*time.Second, verifySSL, 0) page1, err := fetchLastFMPlaylistPage(ctx, client, parsed, 1) if err != nil { @@ -123,7 +123,7 @@ func fetchLastFMPlaylist(ctx context.Context, verifySSL bool, playlistURL string } func fetchLastFMPlaylistViaMirror(ctx context.Context, verifySSL bool, playlistURL string) (string, []lastFMTrack, error) { - client := netutil.NewHTTPClient(30*time.Second, verifySSL) + client := netutil.NewHTTPClient(30*time.Second, verifySSL, 0) all := make([]lastFMTrack, 0, 200) title := "" @@ -376,7 +376,7 @@ func fetchSoundcloudOEmbed(ctx context.Context, verifySSL bool, trackURL string) q.Set("url", trackURL) endpoint := "https://soundcloud.com/oembed?" + q.Encode() - client := netutil.NewHTTPClient(20*time.Second, verifySSL) + client := netutil.NewHTTPClient(20*time.Second, verifySSL, 0) req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return nil, err diff --git a/internal/app/app.go b/internal/app/app.go index 58a76f3..a347344 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -114,13 +114,20 @@ func New(cfg *config.Config) (*Main, error) { Config: cfg, Providers: providers, Store: db, - DL: download.NewWithOptions(cfg.Session.Downloads.VerifySSL, cfg.Session.CLI.ProgressBars), + DL: download.NewWithOptions(cfg.Session.Downloads.VerifySSL, cfg.Session.CLI.ProgressBars, downloaderMaxConnsPerHost(cfg.Session.Downloads.MaxConnections)), Tagger: tag.New(), Pending: []media.Pending{}, Media: []media.Media{}, }, nil } +func downloaderMaxConnsPerHost(maxConnections int) int { + if maxConnections > 16 { + return maxConnections + } + return 16 +} + func (m *Main) Close() error { m.DL.Close() artwork.CleanupTempDirs() diff --git a/internal/app/app_test.go b/internal/app/app_test.go index b6dc359..86aed23 100644 --- a/internal/app/app_test.go +++ b/internal/app/app_test.go @@ -309,7 +309,7 @@ func TestTrackRipFailsWhenTaggerReportsMissingFFmpeg(t *testing.T) { "qobuz": &fakeProvider{url: ts.URL}, }, Store: sqlite, - DL: download.NewWithOptions(true, false), + DL: download.NewWithOptions(true, false, 0), Tagger: failingTagger{err: fmt.Errorf("ffmpeg not found: %w", exec.ErrNotFound)}, } @@ -537,7 +537,7 @@ func TestPlaylistRipPipeline(t *testing.T) { "qobuz": &fakePlaylistProvider{url: ts.URL}, }, Store: sqlite, - DL: download.NewWithOptions(true, false), + DL: download.NewWithOptions(true, false, 0), Tagger: noopTagger{}, } @@ -588,7 +588,7 @@ func TestPlaylistRipUsesSourceSubdirectory(t *testing.T) { "qobuz": &fakePlaylistProvider{url: ts.URL}, }, Store: sqlite, - DL: download.NewWithOptions(true, false), + DL: download.NewWithOptions(true, false, 0), Tagger: noopTagger{}, } @@ -773,7 +773,7 @@ func TestRipAlbumUsesResolvedAudioProfileForFolderName(t *testing.T) { "qobuz": fake, }, Store: sqlite, - DL: download.NewWithOptions(true, false), + DL: download.NewWithOptions(true, false, 0), Tagger: noopTagger{}, } diff --git a/internal/download/downloader.go b/internal/download/downloader.go index 9e75bbc..d04f7bc 100644 --- a/internal/download/downloader.go +++ b/internal/download/downloader.go @@ -31,18 +31,20 @@ type Downloader struct { barStarted atomic.Int32 } +const downloadBufferSize = 1 << 20 + func New() *Downloader { - return NewWithOptions(true, true) + return NewWithOptions(true, true, 0) } func NewWithVerifySSL(verifySSL bool) *Downloader { - return NewWithOptions(verifySSL, true) + return NewWithOptions(verifySSL, true, 0) } -func NewWithOptions(verifySSL bool, showProgress bool) *Downloader { +func NewWithOptions(verifySSL bool, showProgress bool, maxConnsPerHost int) *Downloader { forceProgress := strings.EqualFold(os.Getenv("STREAMRIP_GO_FORCE_PROGRESS"), "1") || strings.EqualFold(os.Getenv("STREAMRIP_GO_FORCE_PROGRESS"), "true") interactive := showProgress && (forceProgress || (term.IsTerminal(int(os.Stderr.Fd())) && strings.ToLower(os.Getenv("TERM")) != "dumb")) - d := &Downloader{http: netutil.NewHTTPClient(0, verifySSL), showProgress: interactive} + d := &Downloader{http: netutil.NewHTTPClient(0, verifySSL, maxConnsPerHost), showProgress: interactive} if interactive { d.progress = mpb.New(mpb.WithWidth(40), mpb.WithOutput(os.Stderr)) } diff --git a/internal/download/downloader_test.go b/internal/download/downloader_test.go index 1e8c461..5229d9e 100644 --- a/internal/download/downloader_test.go +++ b/internal/download/downloader_test.go @@ -18,7 +18,7 @@ import ( ) func TestDownloaderHasNoClientTimeout(t *testing.T) { - d := NewWithOptions(true, false) + d := NewWithOptions(true, false, 0) if d.http.Timeout != 0 { t.Fatalf("http timeout = %v, want 0 (no global timeout)", d.http.Timeout) } @@ -95,7 +95,7 @@ func TestFileDeezerEncrypted(t *testing.T) { })) defer ts.Close() - d := NewWithOptions(true, false) + d := NewWithOptions(true, false, 0) out := filepath.Join(t.TempDir(), "x", "a.flac") if err = d.FileDeezerEncrypted(context.Background(), ts.URL, out, trackID); err != nil { t.Fatalf("FileDeezerEncrypted() error = %v", err) @@ -117,7 +117,7 @@ func TestDownloaderFileTruncatedResponseRemovesPartialFile(t *testing.T) { })) defer ts.Close() - d := NewWithOptions(true, false) + d := NewWithOptions(true, false, 0) out := filepath.Join(t.TempDir(), "x", "a.bin") err := d.File(context.Background(), ts.URL, out) if err == nil || !errors.Is(err, io.ErrUnexpectedEOF) { @@ -135,7 +135,7 @@ func TestFileDeezerEncryptedTruncatedResponseRemovesPartialFile(t *testing.T) { })) defer ts.Close() - d := NewWithOptions(true, false) + d := NewWithOptions(true, false, 0) out := filepath.Join(t.TempDir(), "x", "a.flac") err := d.FileDeezerEncrypted(context.Background(), ts.URL, out, "3135556") if err == nil || !errors.Is(err, io.ErrUnexpectedEOF) { @@ -152,7 +152,7 @@ func TestFileDeezerEncryptedBadStatus(t *testing.T) { })) defer ts.Close() - d := NewWithOptions(true, false) + d := NewWithOptions(true, false, 0) out := filepath.Join(t.TempDir(), "x", "a.flac") err := d.FileDeezerEncrypted(context.Background(), ts.URL, out, "3135556") if err == nil || !strings.Contains(err.Error(), "status=403") { @@ -171,7 +171,7 @@ func TestDownloaderFileContextCancellationRemovesPartialFile(t *testing.T) { })) defer ts.Close() - d := NewWithOptions(true, false) + d := NewWithOptions(true, false, 0) out := filepath.Join(t.TempDir(), "x", "cancel.bin") ctx, cancel := context.WithTimeout(context.Background(), 60*time.Millisecond) defer cancel() @@ -185,7 +185,7 @@ func TestDownloaderFileContextCancellationRemovesPartialFile(t *testing.T) { } func TestStreamManifestWithFFmpegMissing(t *testing.T) { - d := NewWithOptions(true, false) + d := NewWithOptions(true, false, 0) t.Setenv("PATH", "") err := d.streamManifestWithFFmpeg(context.Background(), "https://example.com/live.m3u8", filepath.Join(t.TempDir(), "out.m4a"), false) if err == nil || !strings.Contains(strings.ToLower(err.Error()), "ffmpeg not found") { @@ -197,7 +197,7 @@ func TestStreamManifestWithFFmpegFailureRemovesPartialFile(t *testing.T) { if _, err := exec.LookPath("ffmpeg"); err != nil { t.Skip("ffmpeg not installed") } - d := NewWithOptions(true, false) + d := NewWithOptions(true, false, 0) out := filepath.Join(t.TempDir(), "out.m4a") ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() diff --git a/internal/netutil/http.go b/internal/netutil/http.go index 54436b0..1e9e3fe 100644 --- a/internal/netutil/http.go +++ b/internal/netutil/http.go @@ -6,13 +6,38 @@ import ( "time" ) -func NewHTTPClient(timeout time.Duration, verifySSL bool) *http.Client { +const defaultMaxConnsPerHost = 16 + +// NewHTTPClient builds an *http.Client whose transport is tuned for the +// concurrent download workloads this app issues against single CDN hosts. +// +// maxConnsPerHost caps idle keep-alive sockets per host; pass <= 0 to use a +// sensible default. The downloader and provider clients should pass the +// configured concurrency so keep-alive sockets aren't evicted between workers. +func NewHTTPClient(timeout time.Duration, verifySSL bool, maxConnsPerHost int) *http.Client { + if maxConnsPerHost <= 0 { + maxConnsPerHost = defaultMaxConnsPerHost + } + transport := http.DefaultTransport.(*http.Transport).Clone() if transport.TLSClientConfig == nil { transport.TLSClientConfig = &tls.Config{} } transport.TLSClientConfig.InsecureSkipVerify = !verifySSL + transport.MaxIdleConnsPerHost = maxConnsPerHost + if maxIdle := maxConnsPerHost * 4; maxIdle > transport.MaxIdleConns { + transport.MaxIdleConns = maxIdle + } + if transport.MaxIdleConns < 100 { + transport.MaxIdleConns = 100 + } + transport.MaxConnsPerHost = 0 + transport.IdleConnTimeout = 90 * time.Second + transport.WriteBufferSize = 64 * 1024 + transport.ReadBufferSize = 64 * 1024 + transport.ForceAttemptHTTP2 = true + return &http.Client{ Timeout: timeout, Transport: transport, diff --git a/internal/provider/deezer/client.go b/internal/provider/deezer/client.go index d781ac0..5d0190b 100644 --- a/internal/provider/deezer/client.go +++ b/internal/provider/deezer/client.go @@ -62,7 +62,7 @@ type Client struct { } func New(cfg *config.Config) *Client { - httpClient := netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL) + httpClient := netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL, cfg.Session.Downloads.MaxConnections) if jar, err := cookiejar.New(nil); err == nil { httpClient.Jar = jar } diff --git a/internal/provider/qobuz/client.go b/internal/provider/qobuz/client.go index a648b92..7d16db9 100644 --- a/internal/provider/qobuz/client.go +++ b/internal/provider/qobuz/client.go @@ -45,7 +45,7 @@ type Client struct { func New(cfg *config.Config) *Client { return &Client{ cfg: cfg, - http: netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL), + http: netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL, cfg.Session.Downloads.MaxConnections), limiter: ratelimit.New(cfg.Session.Downloads.RequestsPerMinute), baseURL: baseURL, fetchCfg: nil, diff --git a/internal/provider/tidal/client.go b/internal/provider/tidal/client.go index baa4f5b..d411991 100644 --- a/internal/provider/tidal/client.go +++ b/internal/provider/tidal/client.go @@ -63,7 +63,7 @@ type Client struct { func New(cfg *config.Config) *Client { return &Client{ cfg: cfg, - http: netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL), + http: netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL, cfg.Session.Downloads.MaxConnections), limiter: ratelimit.New(cfg.Session.Downloads.RequestsPerMinute), baseURL: baseURL, lyricsAPI: lyricsAPIv1,