Update README.md

Reviewed-on: #9

README.md

@@ -7,15 +7,51 @@ drmdtool is a utility for processing .drmd files using N_m3u8DL-RE.
 Create a `config.toml` file in the same directory as the drmdtool executable:
 
 ```toml
+[General]
 BaseDir = "/path/to/save/downloads"
 Format = "mkv"
 TempBaseDir = "/tmp/nre"
+EnableConsole = true
 
-[N_m3u8DL-RE]
+[WatchFolder]
+Path = "/path/to/watched/folder"
+PollingInterval = 10
+UsePolling = true
+UseInotify = false
+
+[N_m3u8DLRE]
 Path = "/path/to/N_m3u8DL-RE"
 ```
 
-Adjust the paths and format as needed. (mkv, mp4)
+### Configuration Options
+
+- **General**
+  - `BaseDir`: Directory where downloaded files will be saved.
+  - `Format`: Output format for the downloaded files (e.g., `mkv`, `mp4`).
+  - `TempBaseDir`: Temporary directory for intermediate files.
+  - `EnableConsole`: Boolean to enable or disable console output.
+
+- **WatchFolder**
+  - `Path`: Directory to watch for new `.drmd` files.
+  - `PollingInterval`: Interval in seconds for polling the watch folder.
+  - `UsePolling`: Boolean to enable or disable folder polling.
+  - `UseInotify`: Boolean to enable or disable inotify for file watching.
+
+- **N_m3u8DLRE**
+  - `Path`: Path to the N_m3u8DL-RE executable.
+
+### Environment Variable Overrides
+
+You can override the configuration options using environment variables. The following environment variables are supported:
+
+- `BASE_DIR`: Overrides `General.BaseDir`
+- `FORMAT`: Overrides `General.Format`
+- `TEMP_BASE_DIR`: Overrides `General.TempBaseDir`
+- `ENABLE_CONSOLE`: Overrides `General.EnableConsole` (set to `true` or `false`)
+- `WATCHED_FOLDER`: Overrides `WatchFolder.Path`
+- `USE_POLLING`: Overrides `WatchFolder.UsePolling` (set to `true` or `false`)
+- `USE_INOTIFY`: Overrides `WatchFolder.UseInotify` (set to `true` or `false`)
+- `POLLING_INTERVAL`: Overrides `WatchFolder.PollingInterval`
 
 ## Web UI Usage
 
@@ -28,7 +64,6 @@ Adjust the paths and format as needed. (mkv, mp4)
 
 3. Use the interface to upload .drmd files and monitor download progress
 
-
 ## CLI Usage
 
 To process a file directly from the command line:
@@ -39,7 +74,6 @@ To process a file directly from the command line:
 
 This will download the file and save it in the base directory specified in the config.
 
-
 # Previews
 
 ## Index Page

config.toml

@@ -1,6 +1,14 @@
+[General]
 BaseDir = "/mnt/media"
 Format = "mkv"
 TempBaseDir = "/tmp/nre"
+EnableConsole = true
+
+[WatchFolder]
+Path = "/mnt/watched"
+PollingInterval = 10
+UsePolling = false
+UseInotify = true
 
 [N_m3u8DLRE]
 Path = "nre"

src/config.go

@@ -4,14 +4,25 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strconv"
+	"strings"
 
 	"github.com/BurntSushi/toml"
 )
 
 type Config struct {
+	General struct {
 		BaseDir       string
 		Format        string
 		TempBaseDir   string
+		EnableConsole bool
+	}
+	WatchFolder struct {
+		Path            string
+		UsePolling      bool
+		UseInotify      bool
+		PollingInterval int
+	}
 	N_m3u8DLRE struct {
 		Path string
 	}
@@ -22,21 +33,112 @@ var config Config
 func loadConfig() {
 	configFile, err := os.Open("config.toml")
 	if err != nil {
-		fmt.Println("Error opening config file:", err)
-		return
+		logger.LogError("Config", fmt.Sprintf("Error opening config file: %v", err))
+		os.Exit(1)
 	}
 	defer configFile.Close()
 
 	byteValue, _ := io.ReadAll(configFile)
 
 	if _, err := toml.Decode(string(byteValue), &config); err != nil {
-		fmt.Println("Error decoding config file:", err)
-		return
+		logger.LogError("Config", fmt.Sprintf("Error decoding config file: %v", err))
+		os.Exit(1)
 	}
 
-	if config.N_m3u8DLRE.Path == "" {
-		fmt.Println("Error: N_m3u8DL-RE path is not specified in the config file")
-		return
+	overrideConfigWithEnv()
+
+	if err := validatePaths(); err != nil {
+		logger.LogError("Config", fmt.Sprintf("Configuration error: %v", err))
+		os.Exit(1)
 	}
 
+	if config.WatchFolder.PollingInterval <= 0 {
+		config.WatchFolder.PollingInterval = 10
+	}
+
+	logConfig()
+}
+
+func overrideConfigWithEnv() {
+	if envBaseDir := os.Getenv("BASE_DIR"); envBaseDir != "" {
+		config.General.BaseDir = envBaseDir
+	}
+	if envFormat := os.Getenv("FORMAT"); envFormat != "" {
+		config.General.Format = envFormat
+	}
+	if envTempBaseDir := os.Getenv("TEMP_BASE_DIR"); envTempBaseDir != "" {
+		config.General.TempBaseDir = envTempBaseDir
+	}
+	if envEnableConsole := os.Getenv("ENABLE_CONSOLE"); envEnableConsole != "" {
+		config.General.EnableConsole = strings.ToLower(envEnableConsole) == "true"
+	}
+	if envWatchedFolder := os.Getenv("WATCHED_FOLDER"); envWatchedFolder != "" {
+		config.WatchFolder.Path = envWatchedFolder
+	}
+	if envUsePolling := os.Getenv("USE_POLLING"); envUsePolling != "" {
+		config.WatchFolder.UsePolling = strings.ToLower(envUsePolling) == "true"
+	}
+	if envUseInotify := os.Getenv("USE_INOTIFY"); envUseInotify != "" {
+		config.WatchFolder.UseInotify = strings.ToLower(envUseInotify) == "true"
+	}
+	if envPollingInterval := os.Getenv("POLLING_INTERVAL"); envPollingInterval != "" {
+		if interval, err := strconv.Atoi(envPollingInterval); err == nil {
+			config.WatchFolder.PollingInterval = interval
+		}
+	}
+}
+
+func validatePaths() error {
+	paths := []struct {
+		name string
+		path string
+	}{
+		{"BaseDir", config.General.BaseDir},
+	}
+
+	for _, p := range paths {
+		if p.path == "" {
+			return fmt.Errorf("%s is not specified", p.name)
+		}
+		if _, err := os.Stat(p.path); os.IsNotExist(err) {
+			return fmt.Errorf("%s does not exist: %s", p.name, p.path)
+		} else if err != nil {
+			return fmt.Errorf("error accessing %s: %v", p.name, err)
+		}
+	}
+
+	if config.WatchFolder.UsePolling || config.WatchFolder.UseInotify {
+		if config.WatchFolder.Path == "" {
+			return fmt.Errorf("WatchedFolder is not specified")
+		}
+		if _, err := os.Stat(config.WatchFolder.Path); os.IsNotExist(err) {
+			return fmt.Errorf("WatchedFolder does not exist: %s", config.WatchFolder.Path)
+		} else if err != nil {
+			return fmt.Errorf("error accessing WatchedFolder: %v", err)
+		}
+	}
+
+	return nil
+}
+
+func logConfig() {
+	configInfo := fmt.Sprintf(`
+Configuration Loaded:
+  General:
+    BaseDir: %s
+    Format: %s
+    TempBaseDir: %s
+    EnableConsole: %t
+  WatchFolder:
+    Path: %s
+    UsePolling: %t
+    UseInotify: %t
+    PollingInterval: %d
+  N_m3u8DLRE:
+    Path: %s
+`, config.General.BaseDir, config.General.Format, config.General.TempBaseDir, config.General.EnableConsole,
+		config.WatchFolder.Path, config.WatchFolder.UsePolling, config.WatchFolder.UseInotify, config.WatchFolder.PollingInterval,
+		config.N_m3u8DLRE.Path)
+
+	logger.LogInfo("Config", configInfo)
 }

src/downloaders.go

@@ -20,10 +20,10 @@ func removeBOM(input []byte) []byte {
 	return input
 }
 
-func downloadFile(item Item, jobInfo *JobInfo) error {
+func downloadFile(drmdFilename string, item Item, jobInfo *JobInfo) error {
 	logger.LogInfo("Download File", fmt.Sprintf("Starting download for: %s", item.Filename))
 
-	tempDir := filepath.Join(config.TempBaseDir, sanitizeFilename(item.Filename))
+	tempDir := filepath.Join(config.General.TempBaseDir, sanitizeFilename(item.Filename))
 	err := os.MkdirAll(tempDir, 0755)
 	if err != nil {
 		logger.LogError("Download File", fmt.Sprintf("Error creating temporary directory: %v", err))
@@ -131,7 +131,10 @@ func downloadFile(item Item, jobInfo *JobInfo) error {
 	go func() {
 		for {
 			if outputBuffer.Len() > 0 {
-				broadcast(outputBuffer.Bytes())
+				message := outputBuffer.Bytes()
+				if config.General.EnableConsole {
+					broadcast(drmdFilename, message)
+				}
 				outputBuffer.Reset()
 			}
 			time.Sleep(1 * time.Second)
@@ -180,9 +183,9 @@ func getDownloadCommand(item Item, mpdPath string, tempDir string) string {
 	filename := fmt.Sprintf("\"%s\"", sanitizedFilename)
 	command += fmt.Sprintf(" --save-name %s", filename)
 
-	command += fmt.Sprintf(" --mux-after-done format=%s", config.Format)
+	command += fmt.Sprintf(" --mux-after-done format=%s", config.General.Format)
 
-	saveDir := config.BaseDir
+	saveDir := config.General.BaseDir
 	if metadata.Type == "serie" {
 		saveDir = filepath.Join(saveDir, "Series", metadata.Title, metadata.Season)
 	} else {

src/go.mod

@@ -8,9 +8,12 @@ require (
 	github.com/beevik/etree v1.4.1
 )
 
+require golang.org/x/sys v0.4.0 // indirect
+
 require (
 	github.com/asticode/go-astikit v0.20.0 // indirect
 	github.com/asticode/go-astits v1.8.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0
 	github.com/gorilla/websocket v1.5.3
 	golang.org/x/net v0.0.0-20200904194848-62affa334b73 // indirect
 	golang.org/x/text v0.3.2 // indirect

src/go.sum

@@ -10,6 +10,8 @@ github.com/beevik/etree v1.4.1 h1:PmQJDDYahBGNKDcpdX8uPy1xRCwoCGVUiW669MEirVI=
 github.com/beevik/etree v1.4.1/go.mod h1:gPNJNaBGVZ9AwsidazFZyygnd+0pAU38N4D+WemwKNs=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/pkg/exec v0.0.0-20150614095509-0bd164ad2a5a h1:EN123kAtAAE2pg/+TvBsUBZfHCWNNFyL2ZBPPfNWAc0=
@@ -28,6 +30,8 @@ golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

src/handlers.go

@@ -382,19 +382,36 @@ func updateProgress(filename string, value float64, currentFile string) {
 }
 
 var upgrader = websocket.Upgrader{}
-var clients = make(map[*websocket.Conn]bool)
+var clients = make(map[string]map[*websocket.Conn]bool)
 var mu sync.Mutex
 
 func handleWebSocket(w http.ResponseWriter, r *http.Request) {
+	fmt.Println(config.General.EnableConsole)
+	if !config.General.EnableConsole {
+		http.Error(w, "Console output is disabled", http.StatusForbidden)
+		return
+	}
+
+	filename := r.URL.Query().Get("filename")
+	if filename == "" {
+		http.Error(w, "Filename is required", http.StatusBadRequest)
+		return
+	}
+
 	conn, err := upgrader.Upgrade(w, r, nil)
 	if err != nil {
-		fmt.Println("Error while upgrading connection:", err)
+		logger.LogError("WebSocket", fmt.Sprintf("Error while upgrading connection: %v", err))
 		return
 	}
 	defer conn.Close()
 
+	logger.LogInfo("WebSocket", fmt.Sprintf("WebSocket connection established for filename: %s", filename))
+
 	mu.Lock()
-	clients[conn] = true
+	if clients[filename] == nil {
+		clients[filename] = make(map[*websocket.Conn]bool)
+	}
+	clients[filename][conn] = true
 	mu.Unlock()
 
 	for {
@@ -404,18 +421,25 @@ func handleWebSocket(w http.ResponseWriter, r *http.Request) {
 	}
 
 	mu.Lock()
-	delete(clients, conn)
+	delete(clients[filename], conn)
 	mu.Unlock()
+
+	logger.LogInfo("WebSocket", fmt.Sprintf("WebSocket connection closed for filename: %s", filename))
 }
 
-func broadcast(message []byte) {
+func broadcast(filename string, message []byte) {
+	if !config.General.EnableConsole {
+		return
+	}
+
 	mu.Lock()
 	defer mu.Unlock()
 
-	for client := range clients {
+	for client := range clients[filename] {
 		if err := client.WriteMessage(websocket.TextMessage, message); err != nil {
 			client.Close()
-			delete(clients, client)
+			delete(clients[filename], client)
+			logger.LogError("Broadcast", fmt.Sprintf("Error writing message to client: %v", err))
 		}
 	}
 }

src/main.go

@@ -60,6 +60,7 @@ func main() {
 	flag.Parse()
 
 	if *inputFile == "" {
+		go watchFolder()
 		startWebServer()
 	} else {
 		items, err := parseInputFile(*inputFile)
@@ -83,7 +84,7 @@ func startWebServer() {
 	http.HandleFunc("/clear-completed", handleClearCompleted)
 	http.HandleFunc("/ws", handleWebSocket)
 
-	fmt.Println("Starting web server on http://0.0.0.0:8080")
+	logger.LogInfo("Main", "Starting web server on http://0.0.0.0:8080")
 	http.ListenAndServe(":8080", nil)
 }
 

src/templates/progress

@@ -145,9 +145,10 @@
     <div style="display: none;" id="console"></div>
     <script>
         let isPaused = false;
+        const filename = "{{.Filename}}";
 
         function updateProgress() {
-            fetch('/progress?filename={{.Filename}}', {
+            fetch(`/progress?filename=${filename}`, {
                 headers: {
                     'Accept': 'application/json'
                 }
@@ -179,7 +180,7 @@
         }
 
         function abortDownload() {
-            fetch('/abort?filename={{.Filename}}', { method: 'POST' })
+            fetch(`/abort?filename=${filename}`, { method: 'POST' })
                 .then(response => {
                     if (response.ok) {
                         console.log('Abort signal sent. The download will stop soon.');
@@ -190,7 +191,7 @@
         }
 
         function pauseDownload() {
-            fetch('/pause?filename={{.Filename}}', { method: 'POST' })
+            fetch(`/pause?filename=${filename}`, { method: 'POST' })
                 .then(response => {
                     if (response.ok) {
                         console.log('Pause signal sent. The download will pause soon.');
@@ -203,7 +204,7 @@
         }
 
         function resumeDownload() {
-            fetch('/resume?filename={{.Filename}}', { method: 'POST' })
+            fetch(`/resume?filename=${filename}`, { method: 'POST' })
                 .then(response => {
                     if (response.ok) {
                         console.log('Resume signal sent. The download will resume soon.');
@@ -217,13 +218,22 @@
         }
 
         const consoleDiv = document.getElementById('console');
-        const ws = new WebSocket(`ws://${window.location.host}/ws`);
+        const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
+        const ws = new WebSocket(`${protocol}//${window.location.host}/ws?filename=${filename}`);
 
         ws.onmessage = function(event) {
             consoleDiv.textContent += event.data; 
             consoleDiv.scrollTop = consoleDiv.scrollHeight; 
         };
 
+        ws.onclose = function() {
+            console.log('WebSocket connection closed');
+        };
+
+        ws.onerror = function(error) {
+            console.error('WebSocket error:', error);
+        };
+
         document.getElementById('toggle-console').onclick = function() {
             if (consoleDiv.style.display === "none") {
                 consoleDiv.style.display = "block";

src/utils.go

@@ -259,21 +259,25 @@ func processItems(filename string, items []Item) error {
 		select {
 		case <-jobInfo.AbortChan:
 			updateProgress(filename, 100, "Aborted")
+			logger.LogJobState(filename, "aborted")
 			return fmt.Errorf("download aborted")
 		default:
 			if jobInfo.Paused {
 				select {
 				case <-jobInfo.ResumeChan:
 					jobInfo.Paused = false
+					logger.LogJobState(filename, "resumed")
 				case <-jobInfo.AbortChan:
 					updateProgress(filename, 100, "Aborted")
+					logger.LogJobState(filename, "aborted")
 					return fmt.Errorf("download aborted")
 				}
 			}
 			updateProgress(filename, float64(i)/float64(len(items))*100, items[i].Filename)
-			err := downloadFile(items[i], jobInfo)
+			err := downloadFile(filename, items[i], jobInfo)
 			if err != nil {
 				if err.Error() == "download paused" {
+					logger.LogJobState(filename, "paused")
 					removeCompletedEpisodes(filename, items[:i])
 					i--
 					continue
@@ -282,6 +286,7 @@ func processItems(filename string, items []Item) error {
 		}
 	}
 	updateProgress(filename, 100, "")
+	logger.LogJobState(filename, "completed successfully")
 	return nil
 }
 

src/watcher.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+)
+
+func watchFolder() {
+	if config.WatchFolder.UsePolling {
+		go pollFolder()
+	}
+
+	if config.WatchFolder.UseInotify {
+		go inotifyWatch()
+	}
+}
+
+func inotifyWatch() {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer watcher.Close()
+
+	done := make(chan bool)
+
+	go func() {
+		for {
+			select {
+			case event, ok := <-watcher.Events:
+				if !ok {
+					return
+				}
+				if event.Op&fsnotify.Create == fsnotify.Create {
+					if strings.HasSuffix(event.Name, ".drmd") {
+						fmt.Println("New .drmd detected:", event.Name)
+						processWatchedFile(event.Name)
+					}
+				}
+			case err, ok := <-watcher.Errors:
+				if !ok {
+					return
+				}
+				log.Println("Error:", err)
+			}
+		}
+	}()
+
+	err = watcher.Add(config.WatchFolder.Path)
+	if err != nil {
+		log.Fatal(err)
+	}
+	<-done
+}
+
+func pollFolder() {
+	ticker := time.NewTicker(time.Duration(config.WatchFolder.PollingInterval) * time.Second)
+	defer ticker.Stop()
+
+	for range ticker.C {
+		files, err := filepath.Glob(filepath.Join(config.WatchFolder.Path, "*.drmd"))
+		if err != nil {
+			log.Println("Error polling folder:", err)
+			continue
+		}
+
+		for _, file := range files {
+			fmt.Println("New .drmd detected via polling:", file)
+			go processWatchedFile(file)
+		}
+	}
+}
+
+func processWatchedFile(filePath string) {
+	for {
+		initialSize, err := getFileSize(filePath)
+		if err != nil {
+			logger.LogError("Watcher", fmt.Sprintf("Error getting file size: %v", err))
+			return
+		}
+
+		time.Sleep(1 * time.Second)
+
+		currentSize, err := getFileSize(filePath)
+		if err != nil {
+			logger.LogError("Watcher", fmt.Sprintf("Error getting file size: %v", err))
+			return
+		}
+
+		if initialSize == currentSize {
+			break
+		}
+	}
+
+	file, err := os.Open(filePath)
+	if err != nil {
+		logger.LogError("Watcher", fmt.Sprintf("Error opening file: %v", err))
+		return
+	}
+	defer file.Close()
+
+	originalFilename := filepath.Base(filePath)
+	tempFile, err := os.CreateTemp(uploadDir, originalFilename)
+	if err != nil {
+		logger.LogError("Watcher", fmt.Sprintf("Error creating temporary file: %v", err))
+		return
+	}
+	defer tempFile.Close()
+
+	_, err = io.Copy(tempFile, file)
+	if err != nil {
+		logger.LogError("Watcher", fmt.Sprintf("Error copying file: %v", err))
+		return
+	}
+
+	if err := os.Remove(filePath); err != nil {
+		logger.LogError("Watcher", fmt.Sprintf("Error deleting original file: %v", err))
+	}
+
+	items, err := parseInputFile(tempFile.Name())
+	if err != nil {
+		logger.LogError("Watcher", fmt.Sprintf("Error parsing input file: %v", err))
+		return
+	}
+
+	go processItems(filepath.Base(tempFile.Name()), items)
+}
+
+func getFileSize(filePath string) (int64, error) {
+	fileInfo, err := os.Stat(filePath)
+	if err != nil {
+		return 0, err
+	}
+	return fileInfo.Size(), nil
+}