Joren/streamrip-go
1
0
Fork 0
mirror of https://git.sr.ht/~joren/streamrip-go synced 2026-06-17 15:05:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

performance: tune shared HTTP transport for concurrent CDN downloads

Browse Source
This commit is contained in:
lb-a
2026-04-29 23:53:51 +02:00
parent 9e27ba842f
commit 945695cea7
9 changed files with 58 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/rip/lastfm.go
View File

@@ -86,7 +86,7 @@ func fetchLastFMPlaylist(ctx context.Context, verifySSL bool, playlistURL string
if !isValidLastFMPlaylistURL(playlistURL) {
return "", nil, fmt.Errorf("invalid playlist url")
}
client := netutil.NewHTTPClient(30*time.Second, verifySSL)
client := netutil.NewHTTPClient(30*time.Second, verifySSL, 0)
page1, err := fetchLastFMPlaylistPage(ctx, client, parsed, 1)
if err != nil {
@@ -123,7 +123,7 @@ func fetchLastFMPlaylist(ctx context.Context, verifySSL bool, playlistURL string
}
func fetchLastFMPlaylistViaMirror(ctx context.Context, verifySSL bool, playlistURL string) (string, []lastFMTrack, error) {
client := netutil.NewHTTPClient(30*time.Second, verifySSL)
client := netutil.NewHTTPClient(30*time.Second, verifySSL, 0)
all := make([]lastFMTrack, 0, 200)
title := ""
@@ -376,7 +376,7 @@ func fetchSoundcloudOEmbed(ctx context.Context, verifySSL bool, trackURL string)
q.Set("url", trackURL)
endpoint := "https://soundcloud.com/oembed?" + q.Encode()
client := netutil.NewHTTPClient(20*time.Second, verifySSL)
client := netutil.NewHTTPClient(20*time.Second, verifySSL, 0)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return nil, err

9
internal/app/app.go
View File

@@ -114,13 +114,20 @@ func New(cfg *config.Config) (*Main, error) {
Config: cfg,
Providers: providers,
Store: db,
DL: download.NewWithOptions(cfg.Session.Downloads.VerifySSL, cfg.Session.CLI.ProgressBars),
DL: download.NewWithOptions(cfg.Session.Downloads.VerifySSL, cfg.Session.CLI.ProgressBars, downloaderMaxConnsPerHost(cfg.Session.Downloads.MaxConnections)),
Tagger: tag.New(),
Pending: []media.Pending{},
Media: []media.Media{},
}, nil
}
func downloaderMaxConnsPerHost(maxConnections int) int {
if maxConnections > 16 {
return maxConnections
}
return 16
}
func (m *Main) Close() error {
m.DL.Close()
artwork.CleanupTempDirs()

8
internal/app/app_test.go
View File

@@ -309,7 +309,7 @@ func TestTrackRipFailsWhenTaggerReportsMissingFFmpeg(t *testing.T) {
"qobuz": &fakeProvider{url: ts.URL},
},
Store: sqlite,
DL: download.NewWithOptions(true, false),
DL: download.NewWithOptions(true, false, 0),
Tagger: failingTagger{err: fmt.Errorf("ffmpeg not found: %w", exec.ErrNotFound)},
}
@@ -537,7 +537,7 @@ func TestPlaylistRipPipeline(t *testing.T) {
"qobuz": &fakePlaylistProvider{url: ts.URL},
},
Store: sqlite,
DL: download.NewWithOptions(true, false),
DL: download.NewWithOptions(true, false, 0),
Tagger: noopTagger{},
}
@@ -588,7 +588,7 @@ func TestPlaylistRipUsesSourceSubdirectory(t *testing.T) {
"qobuz": &fakePlaylistProvider{url: ts.URL},
},
Store: sqlite,
DL: download.NewWithOptions(true, false),
DL: download.NewWithOptions(true, false, 0),
Tagger: noopTagger{},
}
@@ -773,7 +773,7 @@ func TestRipAlbumUsesResolvedAudioProfileForFolderName(t *testing.T) {
"qobuz": fake,
},
Store: sqlite,
DL: download.NewWithOptions(true, false),
DL: download.NewWithOptions(true, false, 0),
Tagger: noopTagger{},
}

10
internal/download/downloader.go
View File

@@ -31,18 +31,20 @@ type Downloader struct {
barStarted atomic.Int32
}
const downloadBufferSize = 1 << 20
func New() *Downloader {
return NewWithOptions(true, true)
return NewWithOptions(true, true, 0)
}
func NewWithVerifySSL(verifySSL bool) *Downloader {
return NewWithOptions(verifySSL, true)
return NewWithOptions(verifySSL, true, 0)
}
func NewWithOptions(verifySSL bool, showProgress bool) *Downloader {
func NewWithOptions(verifySSL bool, showProgress bool, maxConnsPerHost int) *Downloader {
forceProgress := strings.EqualFold(os.Getenv("STREAMRIP_GO_FORCE_PROGRESS"), "1") || strings.EqualFold(os.Getenv("STREAMRIP_GO_FORCE_PROGRESS"), "true")
interactive := showProgress && (forceProgress || (term.IsTerminal(int(os.Stderr.Fd())) && strings.ToLower(os.Getenv("TERM")) != "dumb"))
d := &Downloader{http: netutil.NewHTTPClient(0, verifySSL), showProgress: interactive}
d := &Downloader{http: netutil.NewHTTPClient(0, verifySSL, maxConnsPerHost), showProgress: interactive}
if interactive {
d.progress = mpb.New(mpb.WithWidth(40), mpb.WithOutput(os.Stderr))
}

16
internal/download/downloader_test.go
View File

@@ -18,7 +18,7 @@ import (
)
func TestDownloaderHasNoClientTimeout(t *testing.T) {
d := NewWithOptions(true, false)
d := NewWithOptions(true, false, 0)
if d.http.Timeout != 0 {
t.Fatalf("http timeout = %v, want 0 (no global timeout)", d.http.Timeout)
}
@@ -95,7 +95,7 @@ func TestFileDeezerEncrypted(t *testing.T) {
}))
defer ts.Close()
d := NewWithOptions(true, false)
d := NewWithOptions(true, false, 0)
out := filepath.Join(t.TempDir(), "x", "a.flac")
if err = d.FileDeezerEncrypted(context.Background(), ts.URL, out, trackID); err != nil {
t.Fatalf("FileDeezerEncrypted() error = %v", err)
@@ -117,7 +117,7 @@ func TestDownloaderFileTruncatedResponseRemovesPartialFile(t *testing.T) {
}))
defer ts.Close()
d := NewWithOptions(true, false)
d := NewWithOptions(true, false, 0)
out := filepath.Join(t.TempDir(), "x", "a.bin")
err := d.File(context.Background(), ts.URL, out)
if err == nil || !errors.Is(err, io.ErrUnexpectedEOF) {
@@ -135,7 +135,7 @@ func TestFileDeezerEncryptedTruncatedResponseRemovesPartialFile(t *testing.T) {
}))
defer ts.Close()
d := NewWithOptions(true, false)
d := NewWithOptions(true, false, 0)
out := filepath.Join(t.TempDir(), "x", "a.flac")
err := d.FileDeezerEncrypted(context.Background(), ts.URL, out, "3135556")
if err == nil || !errors.Is(err, io.ErrUnexpectedEOF) {
@@ -152,7 +152,7 @@ func TestFileDeezerEncryptedBadStatus(t *testing.T) {
}))
defer ts.Close()
d := NewWithOptions(true, false)
d := NewWithOptions(true, false, 0)
out := filepath.Join(t.TempDir(), "x", "a.flac")
err := d.FileDeezerEncrypted(context.Background(), ts.URL, out, "3135556")
if err == nil || !strings.Contains(err.Error(), "status=403") {
@@ -171,7 +171,7 @@ func TestDownloaderFileContextCancellationRemovesPartialFile(t *testing.T) {
}))
defer ts.Close()
d := NewWithOptions(true, false)
d := NewWithOptions(true, false, 0)
out := filepath.Join(t.TempDir(), "x", "cancel.bin")
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Millisecond)
defer cancel()
@@ -185,7 +185,7 @@ func TestDownloaderFileContextCancellationRemovesPartialFile(t *testing.T) {
}
func TestStreamManifestWithFFmpegMissing(t *testing.T) {
d := NewWithOptions(true, false)
d := NewWithOptions(true, false, 0)
t.Setenv("PATH", "")
err := d.streamManifestWithFFmpeg(context.Background(), "https://example.com/live.m3u8", filepath.Join(t.TempDir(), "out.m4a"), false)
if err == nil || !strings.Contains(strings.ToLower(err.Error()), "ffmpeg not found") {
@@ -197,7 +197,7 @@ func TestStreamManifestWithFFmpegFailureRemovesPartialFile(t *testing.T) {
if _, err := exec.LookPath("ffmpeg"); err != nil {
t.Skip("ffmpeg not installed")
}
d := NewWithOptions(true, false)
d := NewWithOptions(true, false, 0)
out := filepath.Join(t.TempDir(), "out.m4a")
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()

27
internal/netutil/http.go
View File

@@ -6,13 +6,38 @@ import (
"time"
)
func NewHTTPClient(timeout time.Duration, verifySSL bool) *http.Client {
const defaultMaxConnsPerHost = 16
// NewHTTPClient builds an *http.Client whose transport is tuned for the
// concurrent download workloads this app issues against single CDN hosts.
//
// maxConnsPerHost caps idle keep-alive sockets per host; pass <= 0 to use a
// sensible default. The downloader and provider clients should pass the
// configured concurrency so keep-alive sockets aren't evicted between workers.
func NewHTTPClient(timeout time.Duration, verifySSL bool, maxConnsPerHost int) *http.Client {
if maxConnsPerHost <= 0 {
maxConnsPerHost = defaultMaxConnsPerHost
}
transport := http.DefaultTransport.(*http.Transport).Clone()
if transport.TLSClientConfig == nil {
transport.TLSClientConfig = &tls.Config{}
}
transport.TLSClientConfig.InsecureSkipVerify = !verifySSL
transport.MaxIdleConnsPerHost = maxConnsPerHost
if maxIdle := maxConnsPerHost * 4; maxIdle > transport.MaxIdleConns {
transport.MaxIdleConns = maxIdle
}
if transport.MaxIdleConns < 100 {
transport.MaxIdleConns = 100
}
transport.MaxConnsPerHost = 0
transport.IdleConnTimeout = 90 * time.Second
transport.WriteBufferSize = 64 * 1024
transport.ReadBufferSize = 64 * 1024
transport.ForceAttemptHTTP2 = true
return &http.Client{
Timeout: timeout,
Transport: transport,

2
internal/provider/deezer/client.go
View File

@@ -62,7 +62,7 @@ type Client struct {
}
func New(cfg *config.Config) *Client {
httpClient := netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL)
httpClient := netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL, cfg.Session.Downloads.MaxConnections)
if jar, err := cookiejar.New(nil); err == nil {
httpClient.Jar = jar
}

2
internal/provider/qobuz/client.go
View File

@@ -45,7 +45,7 @@ type Client struct {
func New(cfg *config.Config) *Client {
return &Client{
cfg: cfg,
http: netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL),
http: netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL, cfg.Session.Downloads.MaxConnections),
limiter: ratelimit.New(cfg.Session.Downloads.RequestsPerMinute),
baseURL: baseURL,
fetchCfg: nil,

2
internal/provider/tidal/client.go
View File

@@ -63,7 +63,7 @@ type Client struct {
func New(cfg *config.Config) *Client {
return &Client{
cfg: cfg,
http: netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL),
http: netutil.NewHTTPClient(30*time.Second, cfg.Session.Downloads.VerifySSL, cfg.Session.Downloads.MaxConnections),
limiter: ratelimit.New(cfg.Session.Downloads.RequestsPerMinute),
baseURL: baseURL,
lyricsAPI: lyricsAPIv1,