performance: tune shared HTTP transport for concurrent CDN downloads

cmd/rip/lastfm.go

@@ -86,7 +86,7 @@ func fetchLastFMPlaylist(ctx context.Context, verifySSL bool, playlistURL string
 	if !isValidLastFMPlaylistURL(playlistURL) {
 		return "", nil, fmt.Errorf("invalid playlist url")
 	}
-	client := netutil.NewHTTPClient(30*time.Second, verifySSL)
+	client := netutil.NewHTTPClient(30*time.Second, verifySSL, 0)
 
 	page1, err := fetchLastFMPlaylistPage(ctx, client, parsed, 1)
 	if err != nil {
@@ -123,7 +123,7 @@ func fetchLastFMPlaylist(ctx context.Context, verifySSL bool, playlistURL string
 }
 
 func fetchLastFMPlaylistViaMirror(ctx context.Context, verifySSL bool, playlistURL string) (string, []lastFMTrack, error) {
-	client := netutil.NewHTTPClient(30*time.Second, verifySSL)
+	client := netutil.NewHTTPClient(30*time.Second, verifySSL, 0)
 	all := make([]lastFMTrack, 0, 200)
 	title := ""
 
@@ -376,7 +376,7 @@ func fetchSoundcloudOEmbed(ctx context.Context, verifySSL bool, trackURL string)
 	q.Set("url", trackURL)
 	endpoint := "https://soundcloud.com/oembed?" + q.Encode()
 
-	client := netutil.NewHTTPClient(20*time.Second, verifySSL)
+	client := netutil.NewHTTPClient(20*time.Second, verifySSL, 0)
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
 	if err != nil {
 		return nil, err

internal/app/app.go

@@ -114,13 +114,20 @@ func New(cfg *config.Config) (*Main, error) {
 		Config:    cfg,
 		Providers: providers,
 		Store:     db,
-		DL:        download.NewWithOptions(cfg.Session.Downloads.VerifySSL, cfg.Session.CLI.ProgressBars),
+		DL:        download.NewWithOptions(cfg.Session.Downloads.VerifySSL, cfg.Session.CLI.ProgressBars, downloaderMaxConnsPerHost(cfg.Session.Downloads.MaxConnections)),
 		Tagger:    tag.New(),
 		Pending:   []media.Pending{},
 		Media:     []media.Media{},
 	}, nil
 }
 
+func downloaderMaxConnsPerHost(maxConnections int) int {
+	if maxConnections > 16 {
+		return maxConnections
+	}
+	return 16
+}
+
 func (m *Main) Close() error {
 	m.DL.Close()
 	artwork.CleanupTempDirs()

internal/app/app_test.go

@@ -309,7 +309,7 @@ func TestTrackRipFailsWhenTaggerReportsMissingFFmpeg(t *testing.T) {
 			"qobuz": &fakeProvider{url: ts.URL},
 		},
 		Store:  sqlite,
-		DL:     download.NewWithOptions(true, false),
+		DL:     download.NewWithOptions(true, false, 0),
 		Tagger: failingTagger{err: fmt.Errorf("ffmpeg not found: %w", exec.ErrNotFound)},
 	}
 
@@ -537,7 +537,7 @@ func TestPlaylistRipPipeline(t *testing.T) {
 			"qobuz": &fakePlaylistProvider{url: ts.URL},
 		},
 		Store:  sqlite,
-		DL:     download.NewWithOptions(true, false),
+		DL:     download.NewWithOptions(true, false, 0),
 		Tagger: noopTagger{},
 	}
 
@@ -588,7 +588,7 @@ func TestPlaylistRipUsesSourceSubdirectory(t *testing.T) {
 			"qobuz": &fakePlaylistProvider{url: ts.URL},
 		},
 		Store:  sqlite,
-		DL:     download.NewWithOptions(true, false),
+		DL:     download.NewWithOptions(true, false, 0),
 		Tagger: noopTagger{},
 	}
 
@@ -773,7 +773,7 @@ func TestRipAlbumUsesResolvedAudioProfileForFolderName(t *testing.T) {
 			"qobuz": fake,
 		},
 		Store:  sqlite,
-		DL:     download.NewWithOptions(true, false),
+		DL:     download.NewWithOptions(true, false, 0),
 		Tagger: noopTagger{},
 	}
 

internal/download/downloader.go

@@ -31,18 +31,20 @@ type Downloader struct {
 	barStarted   atomic.Int32
 }
 
+const downloadBufferSize = 1 << 20
+
 func New() *Downloader {
-	return NewWithOptions(true, true)
+	return NewWithOptions(true, true, 0)
 }
 
 func NewWithVerifySSL(verifySSL bool) *Downloader {
-	return NewWithOptions(verifySSL, true)
+	return NewWithOptions(verifySSL, true, 0)
 }
 
-func NewWithOptions(verifySSL bool, showProgress bool) *Downloader {
+func NewWithOptions(verifySSL bool, showProgress bool, maxConnsPerHost int) *Downloader {
 	forceProgress := strings.EqualFold(os.Getenv("STREAMRIP_GO_FORCE_PROGRESS"), "1") || strings.EqualFold(os.Getenv("STREAMRIP_GO_FORCE_PROGRESS"), "true")
 	interactive := showProgress && (forceProgress || (term.IsTerminal(int(os.Stderr.Fd())) && strings.ToLower(os.Getenv("TERM")) != "dumb"))
-	d := &Downloader{http: netutil.NewHTTPClient(0, verifySSL), showProgress: interactive}
+	d := &Downloader{http: netutil.NewHTTPClient(0, verifySSL, maxConnsPerHost), showProgress: interactive}
 	if interactive {
 		d.progress = mpb.New(mpb.WithWidth(40), mpb.WithOutput(os.Stderr))
 	}

internal/download/downloader_test.go

@@ -18,7 +18,7 @@ import (
 )
 
 func TestDownloaderHasNoClientTimeout(t *testing.T) {
-	d := NewWithOptions(true, false)
+	d := NewWithOptions(true, false, 0)
 	if d.http.Timeout != 0 {
 		t.Fatalf("http timeout = %v, want 0 (no global timeout)", d.http.Timeout)
 	}
@@ -95,7 +95,7 @@ func TestFileDeezerEncrypted(t *testing.T) {
 	}))
 	defer ts.Close()
 
-	d := NewWithOptions(true, false)
+	d := NewWithOptions(true, false, 0)
 	out := filepath.Join(t.TempDir(), "x", "a.flac")
 	if err = d.FileDeezerEncrypted(context.Background(), ts.URL, out, trackID); err != nil {
 		t.Fatalf("FileDeezerEncrypted() error = %v", err)
@@ -117,7 +117,7 @@ func TestDownloaderFileTruncatedResponseRemovesPartialFile(t *testing.T) {
 	}))
 	defer ts.Close()
 
-	d := NewWithOptions(true, false)
+	d := NewWithOptions(true, false, 0)
 	out := filepath.Join(t.TempDir(), "x", "a.bin")
 	err := d.File(context.Background(), ts.URL, out)
 	if err == nil || !errors.Is(err, io.ErrUnexpectedEOF) {
@@ -135,7 +135,7 @@ func TestFileDeezerEncryptedTruncatedResponseRemovesPartialFile(t *testing.T) {
 	}))
 	defer ts.Close()
 
-	d := NewWithOptions(true, false)
+	d := NewWithOptions(true, false, 0)
 	out := filepath.Join(t.TempDir(), "x", "a.flac")
 	err := d.FileDeezerEncrypted(context.Background(), ts.URL, out, "3135556")
 	if err == nil || !errors.Is(err, io.ErrUnexpectedEOF) {
@@ -152,7 +152,7 @@ func TestFileDeezerEncryptedBadStatus(t *testing.T) {
 	}))
 	defer ts.Close()
 
-	d := NewWithOptions(true, false)
+	d := NewWithOptions(true, false, 0)
 	out := filepath.Join(t.TempDir(), "x", "a.flac")
 	err := d.FileDeezerEncrypted(context.Background(), ts.URL, out, "3135556")
 	if err == nil || !strings.Contains(err.Error(), "status=403") {
@@ -171,7 +171,7 @@ func TestDownloaderFileContextCancellationRemovesPartialFile(t *testing.T) {
 	}))
 	defer ts.Close()
 
-	d := NewWithOptions(true, false)
+	d := NewWithOptions(true, false, 0)
 	out := filepath.Join(t.TempDir(), "x", "cancel.bin")
 	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Millisecond)
 	defer cancel()
@@ -185,7 +185,7 @@ func TestDownloaderFileContextCancellationRemovesPartialFile(t *testing.T) {
 }
 
 func TestStreamManifestWithFFmpegMissing(t *testing.T) {
-	d := NewWithOptions(true, false)
+	d := NewWithOptions(true, false, 0)
 	t.Setenv("PATH", "")
 	err := d.streamManifestWithFFmpeg(context.Background(), "https://example.com/live.m3u8", filepath.Join(t.TempDir(), "out.m4a"), false)
 	if err == nil || !strings.Contains(strings.ToLower(err.Error()), "ffmpeg not found") {
@@ -197,7 +197,7 @@ func TestStreamManifestWithFFmpegFailureRemovesPartialFile(t *testing.T) {
 	if _, err := exec.LookPath("ffmpeg"); err != nil {
 		t.Skip("ffmpeg not installed")
 	}
-	d := NewWithOptions(true, false)
+	d := NewWithOptions(true, false, 0)
 	out := filepath.Join(t.TempDir(), "out.m4a")
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()

internal/netutil/http.go

@@ -6,13 +6,38 @@ import (
 	"time"
 )
 
-func NewHTTPClient(timeout time.Duration, verifySSL bool) *http.Client {
+const defaultMaxConnsPerHost = 16
+
+// NewHTTPClient builds an *http.Client whose transport is tuned for the
+// concurrent download workloads this app issues against single CDN hosts.
+//
+// maxConnsPerHost caps idle keep-alive sockets per host; pass <= 0 to use a
+// sensible default. The downloader and provider clients should pass the
+// configured concurrency so keep-alive sockets aren't evicted between workers.
+func NewHTTPClient(timeout time.Duration, verifySSL bool, maxConnsPerHost int) *http.Client {
+	if maxConnsPerHost <= 0 {
+		maxConnsPerHost = defaultMaxConnsPerHost
+	}
+
 	transport := http.DefaultTransport.(*http.Transport).Clone()
 	if transport.TLSClientConfig == nil {
 		transport.TLSClientConfig = &tls.Config{}
 	}
 	transport.TLSClientConfig.InsecureSkipVerify = !verifySSL
 
+	transport.MaxIdleConnsPerHost = maxConnsPerHost
+	if maxIdle := maxConnsPerHost * 4; maxIdle > transport.MaxIdleConns {
+		transport.MaxIdleConns = maxIdle
+	}
+	if transport.MaxIdleConns < 100 {
+		transport.MaxIdleConns = 100
+	}
+	transport.MaxConnsPerHost = 0
+	transport.IdleConnTimeout = 90 * time.Second
+	transport.WriteBufferSize = 64 * 1024
+	transport.ReadBufferSize = 64 * 1024
+	transport.ForceAttemptHTTP2 = true
+
 	return &http.Client{
 		Timeout:   timeout,
 		Transport: transport,

internal/provider/deezer/client.go

@@ -62,7 +62,7 @@ type Client struct {
 }
 
 func New(cfg *config.Config) *Client {
-	httpClient := netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL)
+	httpClient := netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL, cfg.Session.Downloads.MaxConnections)
 	if jar, err := cookiejar.New(nil); err == nil {
 		httpClient.Jar = jar
 	}

internal/provider/qobuz/client.go

@@ -45,7 +45,7 @@ type Client struct {
 func New(cfg *config.Config) *Client {
 	return &Client{
 		cfg:      cfg,
-		http:     netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL),
+		http:     netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL, cfg.Session.Downloads.MaxConnections),
 		limiter:  ratelimit.New(cfg.Session.Downloads.RequestsPerMinute),
 		baseURL:  baseURL,
 		fetchCfg: nil,

internal/provider/tidal/client.go

@@ -63,7 +63,7 @@ type Client struct {
 func New(cfg *config.Config) *Client {
 	return &Client{
 		cfg:       cfg,
-		http:      netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL),
+		http:      netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL, cfg.Session.Downloads.MaxConnections),
 		limiter:   ratelimit.New(cfg.Session.Downloads.RequestsPerMinute),
 		baseURL:   baseURL,
 		lyricsAPI: lyricsAPIv1,