Files
streamrip-go/internal/download/downloader.go

704 lines
18 KiB
Go

package download
import (
"bufio"
"context"
"crypto/cipher"
"crypto/md5"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"github.com/vbauerster/mpb/v8"
"github.com/vbauerster/mpb/v8/decor"
"golang.org/x/term"
"streamrip-go/internal/netutil"
"golang.org/x/crypto/blowfish"
)
type Downloader struct {
http *http.Client
showProgress bool
progress *mpb.Progress
barStarted atomic.Int32
}
func New() *Downloader {
return NewWithOptions(true, true)
}
func NewWithVerifySSL(verifySSL bool) *Downloader {
return NewWithOptions(verifySSL, true)
}
func NewWithOptions(verifySSL bool, showProgress bool) *Downloader {
forceProgress := strings.EqualFold(os.Getenv("STREAMRIP_GO_FORCE_PROGRESS"), "1") || strings.EqualFold(os.Getenv("STREAMRIP_GO_FORCE_PROGRESS"), "true")
interactive := showProgress && (forceProgress || (term.IsTerminal(int(os.Stderr.Fd())) && strings.ToLower(os.Getenv("TERM")) != "dumb"))
d := &Downloader{http: netutil.NewHTTPClient(0, verifySSL), showProgress: interactive}
if interactive {
d.progress = mpb.New(mpb.WithWidth(40), mpb.WithOutput(os.Stderr))
}
return d
}
func (d *Downloader) File(ctx context.Context, sourceURL, outputPath string) error {
return d.file(ctx, sourceURL, outputPath, true, false)
}
func (d *Downloader) FileNoProgress(ctx context.Context, sourceURL, outputPath string) error {
return d.file(ctx, sourceURL, outputPath, false, false)
}
func (d *Downloader) FileVideo(ctx context.Context, sourceURL, outputPath string) error {
return d.file(ctx, sourceURL, outputPath, true, true)
}
func (d *Downloader) FileDeezerEncrypted(ctx context.Context, sourceURL, outputPath, trackID string) error {
if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil)
if err != nil {
return err
}
resp, err := d.http.Do(req)
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("download failed: status=%d", resp.StatusCode)
}
out, err := os.Create(outputPath)
if err != nil {
return err
}
success := false
defer func() {
_ = out.Close()
if !success {
_ = os.Remove(outputPath)
}
}()
var bar *mpb.Bar
if d.ProgressEnabled() {
d.barStarted.Store(1)
desc := shortenName(filepath.Base(outputPath), 54)
if resp.ContentLength > 0 {
bar = d.progress.AddBar(
resp.ContentLength,
mpb.PrependDecorators(
decor.Name(desc+" ", decor.WC{W: 56, C: decor.DSyncWidth | decor.DindentRight}),
decor.Percentage(decor.WCSyncWidthR),
),
mpb.AppendDecorators(
decor.CountersKibiByte("% .1f / % .1f", decor.WCSyncWidthR),
decor.Name(" | ", decor.WCSyncWidth),
decor.AverageSpeed(decor.SizeB1024(0), "% .1f", decor.WCSyncWidthR),
decor.Name(" | ETA ", decor.WCSyncWidth),
decor.AverageETA(decor.ET_STYLE_GO, decor.WCSyncWidthR),
),
mpb.BarRemoveOnComplete(),
)
} else {
bar = d.progress.AddSpinner(
0,
mpb.PrependDecorators(
decor.Name(desc+" ", decor.WC{W: 56, C: decor.DSyncWidth | decor.DindentRight}),
),
mpb.AppendDecorators(
decor.CurrentKibiByte("% .1f", decor.WCSyncWidthR),
decor.Name(" | ", decor.WCSyncWidth),
decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncWidthR),
),
mpb.BarRemoveOnComplete(),
)
defer bar.SetTotal(-1, true)
}
}
block, err := blowfish.NewCipher(deriveDeezerBlowfishKey(trackID))
if err != nil {
return err
}
buf := make([]byte, deezerBFChunkSize)
dec := make([]byte, deezerBFChunkSize)
chunkIndex := 0
totalRead := int64(0)
for {
n, readErr := io.ReadFull(resp.Body, buf)
if readErr == io.EOF {
break
}
if readErr != nil && readErr != io.ErrUnexpectedEOF {
return readErr
}
chunk := buf[:n]
if chunkIndex%3 == 0 && n == deezerBFChunkSize {
mode := cipher.NewCBCDecrypter(block, deezerBFIV)
mode.CryptBlocks(dec[:n], chunk)
chunk = dec[:n]
}
if _, err = out.Write(chunk); err != nil {
return err
}
totalRead += int64(n)
if bar != nil {
bar.IncrBy(n)
}
chunkIndex++
if readErr == io.ErrUnexpectedEOF {
break
}
}
if resp.ContentLength > 0 && totalRead != resp.ContentLength {
return io.ErrUnexpectedEOF
}
if err = out.Sync(); err != nil {
return err
}
success = true
return nil
}
func (d *Downloader) file(ctx context.Context, sourceURL, outputPath string, allowProgress bool, includeVideo bool) error {
if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil)
if err != nil {
return err
}
resp, err := d.http.Do(req)
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("download failed: status=%d", resp.StatusCode)
}
reader := bufio.NewReader(resp.Body)
peek, _ := reader.Peek(1024)
if isManifestResponse(resp.Header.Get("Content-Type"), peek) {
_ = resp.Body.Close()
return d.streamManifestWithFFmpeg(ctx, sourceURL, outputPath, includeVideo)
}
out, err := os.Create(outputPath)
if err != nil {
return err
}
success := false
defer func() {
_ = out.Close()
if !success {
_ = os.Remove(outputPath)
}
}()
if d.ProgressEnabled() && allowProgress {
d.barStarted.Store(1)
desc := shortenName(filepath.Base(outputPath), 54)
var bar *mpb.Bar
if resp.ContentLength > 0 {
bar = d.progress.AddBar(
resp.ContentLength,
mpb.PrependDecorators(
decor.Name(desc+" ", decor.WC{W: 56, C: decor.DSyncWidth | decor.DindentRight}),
decor.Percentage(decor.WCSyncWidthR),
),
mpb.AppendDecorators(
decor.CountersKibiByte("% .1f / % .1f", decor.WCSyncWidthR),
decor.Name(" | ", decor.WCSyncWidth),
decor.AverageSpeed(decor.SizeB1024(0), "% .1f", decor.WCSyncWidthR),
decor.Name(" | ETA ", decor.WCSyncWidth),
decor.AverageETA(decor.ET_STYLE_GO, decor.WCSyncWidthR),
),
mpb.BarRemoveOnComplete(),
)
} else {
bar = d.progress.AddSpinner(
0,
mpb.PrependDecorators(
decor.Name(desc+" ", decor.WC{W: 56, C: decor.DSyncWidth | decor.DindentRight}),
),
mpb.AppendDecorators(
decor.CurrentKibiByte("% .1f", decor.WCSyncWidthR),
decor.Name(" | ", decor.WCSyncWidth),
decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncWidthR),
),
mpb.BarRemoveOnComplete(),
)
defer bar.SetTotal(-1, true)
}
buf := make([]byte, 256*1024)
totalWritten := int64(0)
for {
n, readErr := reader.Read(buf)
if n > 0 {
if _, writeErr := out.Write(buf[:n]); writeErr != nil {
return writeErr
}
totalWritten += int64(n)
bar.IncrBy(n)
}
if readErr != nil {
if readErr == io.EOF {
break
}
return readErr
}
}
if resp.ContentLength > 0 && totalWritten != resp.ContentLength {
return io.ErrUnexpectedEOF
}
if err = out.Sync(); err != nil {
return err
}
} else {
written, copyErr := io.Copy(out, reader)
if copyErr != nil {
return copyErr
}
if resp.ContentLength > 0 && written != resp.ContentLength {
return io.ErrUnexpectedEOF
}
if err = out.Sync(); err != nil {
return err
}
}
success = true
return nil
}
func (d *Downloader) Close() {
if d.progress != nil {
d.progress.Wait()
}
}
func (d *Downloader) ProgressEnabled() bool {
return d.showProgress && d.progress != nil
}
func (d *Downloader) Logf(format string, args ...any) {
msg := fmt.Sprintf(format, args...)
if d.ProgressEnabled() && d.barStarted.Load() == 1 {
_, _ = d.progress.Write([]byte(msg))
return
}
fmt.Print(msg)
}
func shortenName(name string, max int) string {
if max <= 0 {
return name
}
r := []rune(name)
if len(r) <= max {
return name
}
if max <= 3 {
return string(r[:max])
}
return string(r[:max-3]) + "..."
}
func (d *Downloader) streamManifestWithFFmpeg(ctx context.Context, sourceURL, outputPath string, includeVideo bool) error {
if _, err := exec.LookPath("ffmpeg"); err != nil {
return fmt.Errorf("ffmpeg not found for manifest stream: %w", err)
}
args := buildFFmpegStreamArgs(sourceURL, outputPath, includeVideo)
if !d.ProgressEnabled() {
cmd := exec.CommandContext(ctx, "ffmpeg", args...)
output, err := cmd.CombinedOutput()
if err != nil {
_ = os.Remove(outputPath)
return fmt.Errorf("ffmpeg stream copy failed: %w: %s", err, string(output))
}
return nil
}
d.barStarted.Store(1)
desc := shortenName(filepath.Base(outputPath), 54)
spinner := d.progress.AddSpinner(
0,
mpb.PrependDecorators(
decor.Name(desc+" ", decor.WC{W: 56, C: decor.DSyncWidth | decor.DindentRight}),
),
mpb.AppendDecorators(
decor.CurrentKibiByte("% .1f", decor.WCSyncWidthR),
decor.Name(" | ", decor.WCSyncWidth),
decor.Elapsed(decor.ET_STYLE_GO, decor.WCSyncWidthR),
),
mpb.BarRemoveOnComplete(),
)
cmd := exec.CommandContext(ctx, "ffmpeg", args...)
stderr, err := cmd.StderrPipe()
if err != nil {
spinner.SetTotal(-1, true)
return fmt.Errorf("ffmpeg stream setup failed: %w", err)
}
if err = cmd.Start(); err != nil {
spinner.SetTotal(-1, true)
return fmt.Errorf("ffmpeg stream setup failed: %w", err)
}
var ffOut strings.Builder
scanDone := make(chan scanState, 1)
go func() {
state := scanState{}
shownBytes := int64(0)
barTotalBytes := int64(0)
newEstimatedBar := func(totalBytes int64) *mpb.Bar {
return d.progress.AddBar(
totalBytes,
mpb.PrependDecorators(
decor.Name(desc+" ", decor.WC{W: 56, C: decor.DSyncWidth | decor.DindentRight}),
decor.Percentage(decor.WCSyncWidthR),
),
mpb.AppendDecorators(
decor.CountersKibiByte("% .1f / % .1f", decor.WCSyncWidthR),
decor.Name(" | ", decor.WCSyncWidth),
decor.AverageSpeed(decor.SizeB1024(0), "% .1f", decor.WCSyncWidthR),
decor.Name(" | ETA ", decor.WCSyncWidth),
decor.AverageETA(decor.ET_STYLE_GO, decor.WCSyncWidthR),
),
mpb.BarRemoveOnComplete(),
)
}
var bar *mpb.Bar
scanner := bufio.NewScanner(stderr)
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
for scanner.Scan() {
line := scanner.Text()
if ffOut.Len() < 256*1024 {
ffOut.WriteString(line)
ffOut.WriteByte('\n')
}
if totalMS, ok := parseFFmpegDurationLine(line); ok && totalMS > 0 {
state.totalMS = totalMS
}
if bitrateBPS, ok := parseFFmpegDurationBitrateBPS(line); ok && bitrateBPS > 0 {
state.bitrateBPS = bitrateBPS
}
if bitrateBPS, ok := parseFFmpegProgressBitrateBPS(line); ok && bitrateBPS > 0 {
state.bitrateBPS = bitrateBPS
}
if totalSize, ok := parseFFmpegTotalSize(line); ok && totalSize > state.currentBytes {
state.currentBytes = totalSize
}
if currentMS, ok := parseFFmpegOutTime(line); ok {
if currentMS > state.currentMS {
state.currentMS = currentMS
}
}
if bar == nil {
estimatedTotal := int64(0)
if state.totalMS > 0 && state.bitrateBPS > 0 {
estimatedTotal = estimateTotalBytesFromBitrate(state.totalMS, state.bitrateBPS)
}
if estimatedTotal == 0 {
estimatedTotal = estimateTotalBytesFromProgress(state.totalMS, state.currentMS, state.currentBytes)
}
if estimatedTotal > 0 {
if estimatedTotal < state.currentBytes {
estimatedTotal = state.currentBytes
}
barTotalBytes = estimatedTotal
spinner.SetTotal(-1, true)
bar = newEstimatedBar(barTotalBytes)
}
}
if state.currentBytes > shownBytes {
delta := state.currentBytes - shownBytes
if bar != nil {
bar.IncrBy(int(delta))
if state.currentBytes > barTotalBytes {
barTotalBytes = state.currentBytes
bar.SetTotal(barTotalBytes, false)
}
} else {
spinner.IncrBy(int(delta))
}
shownBytes = state.currentBytes
}
}
if bar != nil {
if shownBytes < barTotalBytes {
bar.IncrBy(int(barTotalBytes - shownBytes))
}
bar.SetTotal(barTotalBytes, true)
} else {
spinner.SetTotal(-1, true)
}
state.currentBytes = shownBytes
state.scanErr = scanner.Err()
scanDone <- state
}()
waitErr := cmd.Wait()
state := <-scanDone
if waitErr != nil {
_ = os.Remove(outputPath)
if out := strings.TrimSpace(ffOut.String()); out != "" {
return fmt.Errorf("ffmpeg stream copy failed: %w: %s", waitErr, out)
}
return fmt.Errorf("ffmpeg stream copy failed: %w", waitErr)
}
if state.scanErr != nil {
return fmt.Errorf("ffmpeg stream output read failed: %w", state.scanErr)
}
return nil
}
func buildFFmpegStreamArgs(sourceURL, outputPath string, includeVideo bool) []string {
args := []string{
"-y",
"-protocol_whitelist", "file,http,https,tcp,tls,crypto,data",
"-i", sourceURL,
}
if includeVideo {
args = append(args,
"-map", "0:v:0?",
"-map", "0:a:0?",
)
} else {
args = append(args, "-map", "0:a:0")
}
args = append(args, "-c", "copy", "-hide_banner", "-nostats", "-progress", "pipe:2", outputPath)
return args
}
type scanState struct {
totalMS int64
currentMS int64
bitrateBPS int64
currentBytes int64
scanErr error
}
func isManifestResponse(contentType string, peek []byte) bool {
ct := strings.ToLower(contentType)
if strings.Contains(ct, "dash+xml") || strings.Contains(ct, "mpegurl") || strings.Contains(ct, "vnd.apple.mpegurl") {
return true
}
s := strings.TrimSpace(strings.ToLower(string(peek)))
if (strings.HasPrefix(s, "<?xml") && strings.Contains(s, "<mpd")) || strings.HasPrefix(s, "<mpd") {
return true
}
if strings.HasPrefix(s, "#extm3u") {
return true
}
return false
}
func parseFFmpegDurationLine(line string) (int64, bool) {
idx := strings.Index(line, "Duration:")
if idx < 0 {
return 0, false
}
raw := strings.TrimSpace(line[idx+len("Duration:"):])
if raw == "" {
return 0, false
}
if comma := strings.Index(raw, ","); comma >= 0 {
raw = raw[:comma]
}
return parseClockDurationMS(raw)
}
func parseFFmpegOutTime(line string) (int64, bool) {
if !strings.HasPrefix(line, "out_time=") {
return 0, false
}
raw := strings.TrimSpace(strings.TrimPrefix(line, "out_time="))
if raw == "" || strings.EqualFold(raw, "N/A") {
return 0, false
}
return parseClockDurationMS(raw)
}
func parseFFmpegTotalSize(line string) (int64, bool) {
if !strings.HasPrefix(line, "total_size=") {
return 0, false
}
raw := strings.TrimSpace(strings.TrimPrefix(line, "total_size="))
if raw == "" || strings.EqualFold(raw, "N/A") {
return 0, false
}
v, err := strconv.ParseInt(raw, 10, 64)
if err != nil || v < 0 {
return 0, false
}
return v, true
}
func parseFFmpegDurationBitrateBPS(line string) (int64, bool) {
idx := strings.Index(strings.ToLower(line), "bitrate:")
if idx < 0 {
return 0, false
}
raw := strings.TrimSpace(line[idx+len("bitrate:"):])
if raw == "" {
return 0, false
}
return parseFFmpegBitrateBPS(raw)
}
func parseFFmpegProgressBitrateBPS(line string) (int64, bool) {
if !strings.HasPrefix(line, "bitrate=") {
return 0, false
}
raw := strings.TrimSpace(strings.TrimPrefix(line, "bitrate="))
if raw == "" || strings.EqualFold(raw, "N/A") {
return 0, false
}
return parseFFmpegBitrateBPS(raw)
}
func parseFFmpegBitrateBPS(raw string) (int64, bool) {
raw = strings.TrimSpace(raw)
if raw == "" || strings.EqualFold(raw, "N/A") {
return 0, false
}
parts := strings.Fields(strings.ToLower(raw))
if len(parts) == 0 {
return 0, false
}
numStr := parts[0]
unit := ""
if len(parts) > 1 {
unit = parts[1]
} else {
i := 0
for i < len(numStr) {
c := numStr[i]
if (c >= '0' && c <= '9') || c == '.' {
i++
continue
}
break
}
if i < len(numStr) {
unit = numStr[i:]
numStr = numStr[:i]
}
}
num, err := strconv.ParseFloat(strings.TrimSpace(numStr), 64)
if err != nil || num <= 0 {
return 0, false
}
unit = strings.TrimSpace(strings.Trim(unit, ","))
mult := float64(1)
switch unit {
case "kb/s", "kbits/s", "kbit/s", "kbps":
mult = 1000
case "mb/s", "mbits/s", "mbit/s", "mbps":
mult = 1000 * 1000
case "gb/s", "gbits/s", "gbit/s", "gbps":
mult = 1000 * 1000 * 1000
case "b/s", "bit/s", "bits/s", "":
mult = 1
default:
return 0, false
}
bps := int64(num * mult)
if bps <= 0 {
return 0, false
}
return bps, true
}
func estimateTotalBytesFromBitrate(totalMS, bitrateBPS int64) int64 {
if totalMS <= 0 || bitrateBPS <= 0 {
return 0
}
return (totalMS * bitrateBPS) / 8000
}
func estimateTotalBytesFromProgress(totalMS, currentMS, currentBytes int64) int64 {
if totalMS <= 0 || currentMS <= 0 || currentBytes <= 0 {
return 0
}
if currentMS > totalMS {
currentMS = totalMS
}
est := (currentBytes * totalMS) / currentMS
if est < currentBytes {
est = currentBytes
}
return est
}
func parseClockDurationMS(raw string) (int64, bool) {
parts := strings.Split(strings.TrimSpace(raw), ":")
if len(parts) != 3 {
return 0, false
}
hours, err := strconv.ParseInt(parts[0], 10, 64)
if err != nil || hours < 0 {
return 0, false
}
minutes, err := strconv.ParseInt(parts[1], 10, 64)
if err != nil || minutes < 0 {
return 0, false
}
secPart := parts[2]
secRaw, fracRaw, hasFrac := strings.Cut(secPart, ".")
seconds, err := strconv.ParseInt(secRaw, 10, 64)
if err != nil || seconds < 0 {
return 0, false
}
ms := ((hours*60+minutes)*60 + seconds) * 1000
if hasFrac {
if len(fracRaw) > 3 {
fracRaw = fracRaw[:3]
}
for len(fracRaw) < 3 {
fracRaw += "0"
}
fracMS, fracErr := strconv.ParseInt(fracRaw, 10, 64)
if fracErr != nil || fracMS < 0 {
return 0, false
}
ms += fracMS
}
return ms, true
}
const deezerBFChunkSize = 2048
var deezerBFIV = []byte{0, 1, 2, 3, 4, 5, 6, 7}
func deriveDeezerBlowfishKey(trackID string) []byte {
sum := md5.Sum([]byte(trackID))
md5Hex := fmt.Sprintf("%x", sum)
secret := "g4el58wc0zvf9na1"
key := make([]byte, 16)
for i := 0; i < 16; i++ {
key[i] = md5Hex[i] ^ md5Hex[i+16] ^ secret[i]
}
return key
}