From fe6b7c78f6d42639380655ec5c105c4968013935 Mon Sep 17 00:00:00 2001 From: Joren Date: Mon, 7 Oct 2024 12:45:49 +0200 Subject: [PATCH 1/4] Add options for polling, path validation, env variables --- config.toml | 10 +++- src/config.go | 125 +++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 117 insertions(+), 18 deletions(-) diff --git a/config.toml b/config.toml index 0effe73..2539382 100644 --- a/config.toml +++ b/config.toml @@ -1,8 +1,14 @@ +[General] BaseDir = "/mnt/media" -Format = "mkv" +Format = "mkv" TempBaseDir = "/tmp/nre" EnableConsole = true -WatchedFolder = "/mnt/watched" + +[WatchFolder] +Path = "/mnt/watched" +PollingInterval = 10 +UsePolling = false +UseInotify = true [N_m3u8DLRE] Path = "nre" \ No newline at end of file diff --git a/src/config.go b/src/config.go index 3af4b15..fc8a45f 100644 --- a/src/config.go +++ b/src/config.go @@ -4,19 +4,28 @@ import ( "fmt" "io" "os" + "strconv" + "strings" "github.com/BurntSushi/toml" ) type Config struct { - BaseDir string - Format string - TempBaseDir string - N_m3u8DLRE struct { + General struct { + BaseDir string + Format string + TempBaseDir string + EnableConsole bool + } + WatchFolder struct { + Path string + UsePolling bool + UseInotify bool + PollingInterval int + } + N_m3u8DLRE struct { Path string } - EnableConsole bool - WatchedFolder string } var config Config @@ -24,25 +33,109 @@ var config Config func loadConfig() { configFile, err := os.Open("config.toml") if err != nil { - fmt.Println("Error opening config file:", err) - return + logger.LogError("Config", fmt.Sprintf("Error opening config file: %v", err)) + os.Exit(1) } defer configFile.Close() byteValue, _ := io.ReadAll(configFile) if _, err := toml.Decode(string(byteValue), &config); err != nil { - fmt.Println("Error decoding config file:", err) - return + logger.LogError("Config", fmt.Sprintf("Error decoding config file: %v", err)) + os.Exit(1) } - if config.N_m3u8DLRE.Path == "" { - fmt.Println("Error: N_m3u8DL-RE path is not specified in the config file") - return + overrideConfigWithEnv() + + if err := validatePaths(); err != nil { + logger.LogError("Config", fmt.Sprintf("Configuration error: %v", err)) + os.Exit(1) } - if config.WatchedFolder == "" { - fmt.Println("Error: Watched folder is not specified in the config file") - return + if config.WatchFolder.PollingInterval <= 0 { + config.WatchFolder.PollingInterval = 10 + } + + logConfig() +} + +func overrideConfigWithEnv() { + if envBaseDir := os.Getenv("BASE_DIR"); envBaseDir != "" { + config.General.BaseDir = envBaseDir + } + if envFormat := os.Getenv("FORMAT"); envFormat != "" { + config.General.Format = envFormat + } + if envTempBaseDir := os.Getenv("TEMP_BASE_DIR"); envTempBaseDir != "" { + config.General.TempBaseDir = envTempBaseDir + } + if envWatchedFolder := os.Getenv("WATCHED_FOLDER"); envWatchedFolder != "" { + config.WatchFolder.Path = envWatchedFolder + } + if envUsePolling := os.Getenv("USE_POLLING"); envUsePolling != "" { + config.WatchFolder.UsePolling = strings.ToLower(envUsePolling) == "true" + } + if envUseInotify := os.Getenv("USE_INOTIFY"); envUseInotify != "" { + config.WatchFolder.UseInotify = strings.ToLower(envUseInotify) == "true" + } + if envPollingInterval := os.Getenv("POLLING_INTERVAL"); envPollingInterval != "" { + if interval, err := strconv.Atoi(envPollingInterval); err == nil { + config.WatchFolder.PollingInterval = interval + } } } + +func validatePaths() error { + paths := []struct { + name string + path string + }{ + {"BaseDir", config.General.BaseDir}, + } + + for _, p := range paths { + if p.path == "" { + return fmt.Errorf("%s is not specified", p.name) + } + if _, err := os.Stat(p.path); os.IsNotExist(err) { + return fmt.Errorf("%s does not exist: %s", p.name, p.path) + } else if err != nil { + return fmt.Errorf("error accessing %s: %v", p.name, err) + } + } + + if config.WatchFolder.UsePolling || config.WatchFolder.UseInotify { + if config.WatchFolder.Path == "" { + return fmt.Errorf("WatchedFolder is not specified") + } + if _, err := os.Stat(config.WatchFolder.Path); os.IsNotExist(err) { + return fmt.Errorf("WatchedFolder does not exist: %s", config.WatchFolder.Path) + } else if err != nil { + return fmt.Errorf("error accessing WatchedFolder: %v", err) + } + } + + return nil +} + +func logConfig() { + configInfo := fmt.Sprintf(` +Configuration Loaded: + General: + BaseDir: %s + Format: %s + TempBaseDir: %s + EnableConsole: %t + WatchFolder: + Path: %s + UsePolling: %t + UseInotify: %t + PollingInterval: %d + N_m3u8DLRE: + Path: %s +`, config.General.BaseDir, config.General.Format, config.General.TempBaseDir, config.General.EnableConsole, + config.WatchFolder.Path, config.WatchFolder.UsePolling, config.WatchFolder.UseInotify, config.WatchFolder.PollingInterval, + config.N_m3u8DLRE.Path) + + logger.LogInfo("Config", configInfo) +} From c46538a55ff4de5d4141ca136e83e7bc9c06b1d3 Mon Sep 17 00:00:00 2001 From: Joren Date: Mon, 7 Oct 2024 12:46:26 +0200 Subject: [PATCH 2/4] Change the config paths according to new layout --- src/downloaders.go | 8 ++++---- src/handlers.go | 6 +++--- src/main.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/downloaders.go b/src/downloaders.go index c027fb7..21f7f84 100644 --- a/src/downloaders.go +++ b/src/downloaders.go @@ -23,7 +23,7 @@ func removeBOM(input []byte) []byte { func downloadFile(drmdFilename string, item Item, jobInfo *JobInfo) error { logger.LogInfo("Download File", fmt.Sprintf("Starting download for: %s", item.Filename)) - tempDir := filepath.Join(config.TempBaseDir, sanitizeFilename(item.Filename)) + tempDir := filepath.Join(config.General.TempBaseDir, sanitizeFilename(item.Filename)) err := os.MkdirAll(tempDir, 0755) if err != nil { logger.LogError("Download File", fmt.Sprintf("Error creating temporary directory: %v", err)) @@ -132,7 +132,7 @@ func downloadFile(drmdFilename string, item Item, jobInfo *JobInfo) error { for { if outputBuffer.Len() > 0 { message := outputBuffer.Bytes() - if config.EnableConsole { + if config.General.EnableConsole { broadcast(drmdFilename, message) } outputBuffer.Reset() @@ -183,9 +183,9 @@ func getDownloadCommand(item Item, mpdPath string, tempDir string) string { filename := fmt.Sprintf("\"%s\"", sanitizedFilename) command += fmt.Sprintf(" --save-name %s", filename) - command += fmt.Sprintf(" --mux-after-done format=%s", config.Format) + command += fmt.Sprintf(" --mux-after-done format=%s", config.General.Format) - saveDir := config.BaseDir + saveDir := config.General.BaseDir if metadata.Type == "serie" { saveDir = filepath.Join(saveDir, "Series", metadata.Title, metadata.Season) } else { diff --git a/src/handlers.go b/src/handlers.go index 309c4ae..de5889e 100644 --- a/src/handlers.go +++ b/src/handlers.go @@ -386,8 +386,8 @@ var clients = make(map[string]map[*websocket.Conn]bool) var mu sync.Mutex func handleWebSocket(w http.ResponseWriter, r *http.Request) { - fmt.Println(config.EnableConsole) - if !config.EnableConsole { + fmt.Println(config.General.EnableConsole) + if !config.General.EnableConsole { http.Error(w, "Console output is disabled", http.StatusForbidden) return } @@ -428,7 +428,7 @@ func handleWebSocket(w http.ResponseWriter, r *http.Request) { } func broadcast(filename string, message []byte) { - if !config.EnableConsole { + if !config.General.EnableConsole { return } diff --git a/src/main.go b/src/main.go index 6089f4d..c24ed9d 100644 --- a/src/main.go +++ b/src/main.go @@ -84,7 +84,7 @@ func startWebServer() { http.HandleFunc("/clear-completed", handleClearCompleted) http.HandleFunc("/ws", handleWebSocket) - fmt.Println("Starting web server on http://0.0.0.0:8080") + logger.LogInfo("Main", "Starting web server on http://0.0.0.0:8080") http.ListenAndServe(":8080", nil) } From a0496102912e0b6ea98e7870d072e4c5f2694a84 Mon Sep 17 00:00:00 2001 From: Joren Date: Mon, 7 Oct 2024 12:46:38 +0200 Subject: [PATCH 3/4] Implement polling, update readme --- README.md | 41 ++++++++++++++++++++++++++++++++++++----- src/watcher.go | 30 +++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index a4a4808..c07d9fe 100644 --- a/README.md +++ b/README.md @@ -7,17 +7,50 @@ drmdtool is a utility for processing .drmd files using N_m3u8DL-RE. Create a `config.toml` file in the same directory as the drmdtool executable: ```toml +[General] BaseDir = "/path/to/save/downloads" Format = "mkv" TempBaseDir = "/tmp/nre" EnableConsole = true -WatchedFolder = "/path/to/watched/folder" -[N_m3u8DL-RE] +[WatchFolder] +Path = "/path/to/watched/folder" +PollingInterval = 10 +UsePolling = true +UseInotify = false + +[N_m3u8DLRE] Path = "/path/to/N_m3u8DL-RE" ``` -Adjust the paths and format as needed. (mkv, mp4) +### Configuration Options + +- **General** + - `BaseDir`: Directory where downloaded files will be saved. + - `Format`: Output format for the downloaded files (e.g., `mkv`, `mp4`). + - `TempBaseDir`: Temporary directory for intermediate files. + - `EnableConsole`: Boolean to enable or disable console output. + +- **WatchFolder** + - `Path`: Directory to watch for new `.drmd` files. + - `PollingInterval`: Interval in seconds for polling the watch folder. + - `UsePolling`: Boolean to enable or disable folder polling. + - `UseInotify`: Boolean to enable or disable inotify for file watching. + +- **N_m3u8DLRE** + - `Path`: Path to the N_m3u8DL-RE executable. + +### Environment Variable Overrides + +You can override the configuration options using environment variables. The following environment variables are supported: + +- `BASE_DIR`: Overrides `General.BaseDir` +- `FORMAT`: Overrides `General.Format` +- `TEMP_BASE_DIR`: Overrides `General.TempBaseDir` +- `WATCHED_FOLDER`: Overrides `WatchFolder.Path` +- `USE_POLLING`: Overrides `WatchFolder.UsePolling` (set to `true` or `false`) +- `USE_INOTIFY`: Overrides `WatchFolder.UseInotify` (set to `true` or `false`) +- `POLLING_INTERVAL`: Overrides `WatchFolder.PollingInterval` ## Web UI Usage @@ -30,7 +63,6 @@ Adjust the paths and format as needed. (mkv, mp4) 3. Use the interface to upload .drmd files and monitor download progress - ## CLI Usage To process a file directly from the command line: @@ -41,7 +73,6 @@ To process a file directly from the command line: This will download the file and save it in the base directory specified in the config. - # Previews ## Index Page diff --git a/src/watcher.go b/src/watcher.go index fd1d982..660a4dc 100644 --- a/src/watcher.go +++ b/src/watcher.go @@ -13,6 +13,16 @@ import ( ) func watchFolder() { + if config.WatchFolder.UsePolling { + go pollFolder() + } + + if config.WatchFolder.UseInotify { + go inotifyWatch() + } +} + +func inotifyWatch() { watcher, err := fsnotify.NewWatcher() if err != nil { log.Fatal(err) @@ -43,13 +53,31 @@ func watchFolder() { } }() - err = watcher.Add(config.WatchedFolder) + err = watcher.Add(config.WatchFolder.Path) if err != nil { log.Fatal(err) } <-done } +func pollFolder() { + ticker := time.NewTicker(time.Duration(config.WatchFolder.PollingInterval) * time.Second) + defer ticker.Stop() + + for range ticker.C { + files, err := filepath.Glob(filepath.Join(config.WatchFolder.Path, "*.drmd")) + if err != nil { + log.Println("Error polling folder:", err) + continue + } + + for _, file := range files { + fmt.Println("New .drmd detected via polling:", file) + go processWatchedFile(file) + } + } +} + func processWatchedFile(filePath string) { for { initialSize, err := getFileSize(filePath) From b1ba08933afd5e050bca4f97aae7687f507d5507 Mon Sep 17 00:00:00 2001 From: Joren Date: Mon, 7 Oct 2024 12:48:37 +0200 Subject: [PATCH 4/4] Console should also beable to be controlled by env var --- src/config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/config.go b/src/config.go index fc8a45f..1081f12 100644 --- a/src/config.go +++ b/src/config.go @@ -69,6 +69,9 @@ func overrideConfigWithEnv() { if envTempBaseDir := os.Getenv("TEMP_BASE_DIR"); envTempBaseDir != "" { config.General.TempBaseDir = envTempBaseDir } + if envEnableConsole := os.Getenv("ENABLE_CONSOLE"); envEnableConsole != "" { + config.General.EnableConsole = strings.ToLower(envEnableConsole) == "true" + } if envWatchedFolder := os.Getenv("WATCHED_FOLDER"); envWatchedFolder != "" { config.WatchFolder.Path = envWatchedFolder }