performance: tune shared HTTP transport for concurrent CDN downloads

internal/netutil/http.go

@@ -6,13 +6,38 @@ import (
 	"time"
 )
 
-func NewHTTPClient(timeout time.Duration, verifySSL bool) *http.Client {
+const defaultMaxConnsPerHost = 16
+
+// NewHTTPClient builds an *http.Client whose transport is tuned for the
+// concurrent download workloads this app issues against single CDN hosts.
+//
+// maxConnsPerHost caps idle keep-alive sockets per host; pass <= 0 to use a
+// sensible default. The downloader and provider clients should pass the
+// configured concurrency so keep-alive sockets aren't evicted between workers.
+func NewHTTPClient(timeout time.Duration, verifySSL bool, maxConnsPerHost int) *http.Client {
+	if maxConnsPerHost <= 0 {
+		maxConnsPerHost = defaultMaxConnsPerHost
+	}
+
 	transport := http.DefaultTransport.(*http.Transport).Clone()
 	if transport.TLSClientConfig == nil {
 		transport.TLSClientConfig = &tls.Config{}
 	}
 	transport.TLSClientConfig.InsecureSkipVerify = !verifySSL
 
+	transport.MaxIdleConnsPerHost = maxConnsPerHost
+	if maxIdle := maxConnsPerHost * 4; maxIdle > transport.MaxIdleConns {
+		transport.MaxIdleConns = maxIdle
+	}
+	if transport.MaxIdleConns < 100 {
+		transport.MaxIdleConns = 100
+	}
+	transport.MaxConnsPerHost = 0
+	transport.IdleConnTimeout = 90 * time.Second
+	transport.WriteBufferSize = 64 * 1024
+	transport.ReadBufferSize = 64 * 1024
+	transport.ForceAttemptHTTP2 = true
+
 	return &http.Client{
 		Timeout:   timeout,
 		Transport: transport,