Downloader fixes with videos

For ExternalUrl module items, the Canvas API returns url=null and
html_url=.../module_item_redirect/<id>. We were passing item.URL (empty)
causing the session_token call to return no session_url.

README.md

@@ -21,18 +21,31 @@ go build -o canvasarchiver ./cmd/canvasarchiver
    ./canvasarchiver
    ```
 
+   Or for files-only mode:
+   ```bash
+   ./canvasarchiver -fo
+   ```
+
 2. On first run, you'll be prompted to authenticate:
    - Visit the provided OAuth URL
    - Authorize the application
    - Copy the authorization code back to the terminal
 
-3. Enter your Course ID when prompted
+3. Enter your Course ID when prompted (or use `-me` to download all enrolled courses)
 
 4. The tool will download:
    - Regular course files (to `Course Files/`)
    - Module content (to `Modules/`)
    - Panopto recordings (to `Recordings/`)
 
+### Flags
+
+| Flag | Description |
+|------|-------------|
+| `-fo` | Files only mode - download all files to a single directory without module structure |
+| `-me` | Download all enrolled courses |
+| `-n` | Prefix modules with order numbers `[1]`, `[2]`, etc. |
+
 
 ## Configuration
 

cmd/canvasarchiver/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"flag"
 	"fmt"
 	"net/http"
 	"os"
@@ -12,6 +13,12 @@ import (
 )
 
 func main() {
+	filesOnly := flag.Bool("fo", false, "Files only mode - download all files to a single directory without module structure")
+	videosOnly := flag.Bool("vo", false, "Videos only mode - download only Panopto videos to a single directory")
+	me := flag.Bool("me", false, "Download all enrolled courses")
+	moduleNumbers := flag.Bool("n", false, "Prefix modules with order numbers [1], [2], etc.")
+	flag.Parse()
+
 	httpClient := &http.Client{}
 
 	authenticator := auth.NewAuthenticator(httpClient)
@@ -21,11 +28,31 @@ func main() {
 		return
 	}
 
+	if *me {
+		canvasClient := canvas.NewClient(httpClient, accessToken, "", *filesOnly, *videosOnly, *moduleNumbers)
+		courses, err := canvasClient.GetEnrolledCourses()
+		if err != nil {
+			fmt.Printf("Error fetching courses: %v\n", err)
+			return
+		}
+
+		fmt.Printf("[+] Found %d enrolled courses\n", len(courses))
+		for _, course := range courses {
+			fmt.Printf("  -> Downloading: %s (ID: %d)\n", course.Name, course.ID)
+			downloadCourse(httpClient, accessToken, fmt.Sprintf("%d", course.ID), *filesOnly, *videosOnly, *moduleNumbers)
+		}
+		return
+	}
+
 	var courseID string
 	fmt.Print("Enter Course ID: ")
 	fmt.Scanln(&courseID)
 
-	canvasClient := canvas.NewClient(httpClient, accessToken, courseID)
+	downloadCourse(httpClient, accessToken, courseID, *filesOnly, *videosOnly, *moduleNumbers)
+}
+
+func downloadCourse(httpClient *http.Client, accessToken, courseID string, filesOnly, videosOnly, moduleNumbers bool) {
+	canvasClient := canvas.NewClient(httpClient, accessToken, courseID, filesOnly, videosOnly, moduleNumbers)
 
 	if err := canvasClient.GetCourseInfo(); err != nil {
 		fmt.Printf("Error: %v\n", err)
@@ -40,5 +67,8 @@ func main() {
 
 	canvasClient.DownloadModules(courseRoot)
 
-	panopto.DownloadMainRecordings(httpClient, accessToken, courseID, courseRoot)
+	// Run recordings: always in -vo mode; skipped in -fo mode; normal otherwise.
+	if videosOnly || !filesOnly {
+		panopto.DownloadMainRecordings(httpClient, accessToken, courseID, courseRoot, videosOnly)
+	}
 }

internal/canvas/client.go

@@ -17,17 +17,25 @@ import (
 )
 
 type Client struct {
-	HTTPClient  *http.Client
-	AccessToken string
-	CourseID    string
-	CourseName  string
+	HTTPClient      *http.Client
+	AccessToken     string
+	CourseID        string
+	CourseName      string
+	FilesOnly       bool
+	VideosOnly      bool
+	ModuleNumbers   bool
+	downloadedFiles map[string]bool
 }
 
-func NewClient(httpClient *http.Client, accessToken, courseID string) *Client {
+func NewClient(httpClient *http.Client, accessToken, courseID string, filesOnly, videosOnly, moduleNumbers bool) *Client {
 	return &Client{
-		HTTPClient:  httpClient,
-		AccessToken: accessToken,
-		CourseID:    courseID,
+		HTTPClient:      httpClient,
+		AccessToken:     accessToken,
+		CourseID:        courseID,
+		FilesOnly:       filesOnly,
+		VideosOnly:      videosOnly,
+		ModuleNumbers:   moduleNumbers,
+		downloadedFiles: make(map[string]bool),
 	}
 }
 
@@ -47,7 +55,25 @@ func (c *Client) GetCourseInfo() error {
 	return nil
 }
 
+func (c *Client) GetEnrolledCourses() ([]models.Course, error) {
+	req, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/courses?enrollment_state=active&per_page=100", config.BaseURL), nil)
+	req.Header.Set("Authorization", "Bearer "+c.AccessToken)
+	resp, err := c.HTTPClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var courses []models.Course
+	json.NewDecoder(resp.Body).Decode(&courses)
+	return courses, nil
+}
+
 func (c *Client) DownloadCourseFiles(root string) {
+	if c.VideosOnly {
+		fmt.Println("\n[*] Skipping regular course files (videos only mode)")
+		return
+	}
 	fmt.Println("\n[*] Fetching regular course files...")
 
 	fReq, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/courses/%s/folders?per_page=100", config.BaseURL, c.CourseID), nil)
@@ -80,10 +106,24 @@ func (c *Client) DownloadCourseFiles(root string) {
 	fileCount := 0
 	for _, file := range files {
 
+		if c.FilesOnly {
+			if c.downloadedFiles[file.DisplayName] {
+				continue
+			}
+			c.downloadedFiles[file.DisplayName] = true
+		}
+
 		rawFolderPath := folderMap[file.FolderID]
 		safeFolderPath := utils.SanitizePath(rawFolderPath)
 
-		subDir := filepath.Join(root, "Course Files", safeFolderPath)
+		subDir := root
+		if !c.FilesOnly {
+			if safeFolderPath != "" && strings.ToLower(safeFolderPath) != "course files" {
+				subDir = filepath.Join(root, "Course Files", safeFolderPath)
+			} else {
+				subDir = filepath.Join(root, "Course Files")
+			}
+		}
 		os.MkdirAll(subDir, 0o755)
 		path := filepath.Join(subDir, utils.Sanitize(file.DisplayName))
 
@@ -124,24 +164,44 @@ func (c *Client) DownloadModules(courseRoot string) {
 	json.NewDecoder(resp.Body).Decode(&modules)
 	resp.Body.Close()
 
-	for _, mod := range modules {
-		modBaseDir := filepath.Join(courseRoot, "Modules", utils.Sanitize(mod.Name))
+	for i, mod := range modules {
+		modName := mod.Name
+		if c.ModuleNumbers {
+			modName = fmt.Sprintf("[%d] %s", i+1, mod.Name)
+		}
+
+		// In videos-only mode everything goes flat into courseRoot.
+		// In files-only mode everything goes flat into courseRoot.
+		// Otherwise use the structured Modules/<name> path.
+		modBaseDir := courseRoot
+		if !c.FilesOnly && !c.VideosOnly {
+			modBaseDir = filepath.Join(courseRoot, "Modules", utils.Sanitize(modName))
+		}
 		os.MkdirAll(modBaseDir, 0o755)
-		fmt.Printf("\n[Module] %s\n", mod.Name)
+
+		if !c.FilesOnly && !c.VideosOnly {
+			fmt.Printf("\n[Module] %s\n", modName)
+		} else if c.VideosOnly {
+			fmt.Printf("\n[Module] %s (scanning for videos)\n", modName)
+		}
 
 		subHeaderStack := []string{}
 		lastIndent := 0
 
 		for _, item := range mod.Items {
 
+			// In videos-only mode always download to the flat courseRoot.
 			targetDir := modBaseDir
-			if len(subHeaderStack) > 0 {
+			if len(subHeaderStack) > 0 && !c.FilesOnly && !c.VideosOnly {
 				targetDir = filepath.Join(modBaseDir, filepath.Join(subHeaderStack...))
 			}
 			os.MkdirAll(targetDir, 0o755)
 
 			switch item.Type {
 			case "SubHeader":
+				if c.FilesOnly || c.VideosOnly {
+					continue
+				}
 				currentIndent := item.Indent
 				if currentIndent <= lastIndent && len(subHeaderStack) > 0 {
 					levelsToKeep := currentIndent
@@ -159,14 +219,23 @@ func (c *Client) DownloadModules(courseRoot string) {
 				fmt.Printf("%s--- %s ---\n", indent, item.Title)
 
 			case "File":
+				if c.VideosOnly {
+					continue
+				}
 				c.downloadModuleFile(item, targetDir)
 
 			case "ExternalTool":
+				if c.FilesOnly {
+					continue
+				}
 				indent := strings.Repeat("  ", len(subHeaderStack)+1)
 				fmt.Printf("%s- Found video tool: %s\n", indent, item.Title)
 				panopto.DownloadVideo(c.HTTPClient, c.AccessToken, c.CourseID, targetDir, item.URL, item.Title)
 
 			case "ExternalUrl":
+				if c.FilesOnly {
+					continue
+				}
 				if strings.Contains(item.ExternalURL, "panopto.eu") {
 					indent := strings.Repeat("  ", len(subHeaderStack)+1)
 					fmt.Printf("%s- Found direct video link: %s\n", indent, item.Title)
@@ -175,6 +244,9 @@ func (c *Client) DownloadModules(courseRoot string) {
 				}
 
 			case "Page":
+				if c.FilesOnly {
+					continue
+				}
 				c.searchPageForVideos(item, targetDir)
 			}
 		}
@@ -200,11 +272,18 @@ func (c *Client) downloadModuleFile(item models.ModuleItem, dir string) {
 		return
 	}
 
+	if c.FilesOnly {
+		if c.downloadedFiles[fileMeta.DisplayName] {
+			return
+		}
+		c.downloadedFiles[fileMeta.DisplayName] = true
+	}
+
 	ext := filepath.Ext(fileMeta.DisplayName)
 	origBase := strings.TrimSuffix(fileMeta.DisplayName, ext)
 
 	fileName := fileMeta.DisplayName
-	if !strings.EqualFold(origBase, item.Title) && item.Title != "" {
+	if !c.FilesOnly && !strings.EqualFold(origBase, item.Title) && item.Title != "" {
 		fileName = fmt.Sprintf("%s (%s)%s", origBase, item.Title, ext)
 	}
 

internal/models/models.go

@@ -10,6 +10,7 @@ type TokenResponse struct {
 }
 
 type Course struct {
+	ID   int    `json:"id"`
 	Name string `json:"name"`
 }
 

internal/panopto/downloader.go

@@ -17,6 +17,27 @@ import (
 	"git.directme.in/Joren/CanvasArchiver/internal/utils"
 )
 
+// normalizePanoptoURL converts the query-param form that Canvas stores
+// (List.aspx?folderID=X) to the fragment form that yt-dlp's PanoptoList
+// extractor understands (List.aspx#folderID="X"). Without this yt-dlp
+// ignores the folder filter and downloads the entire Panopto instance.
+func normalizePanoptoURL(rawURL string) string {
+	parsed, err := url.Parse(rawURL)
+	if err != nil {
+		return rawURL
+	}
+	if strings.Contains(parsed.Path, "List.aspx") {
+		folderID := parsed.Query().Get("folderID")
+		if folderID != "" {
+			// Strip query, set fragment: List.aspx#folderID="<id>"
+			parsed.RawQuery = ""
+			parsed.Fragment = fmt.Sprintf(`folderID="%s"`, folderID)
+			return parsed.String()
+		}
+	}
+	return rawURL
+}
+
 func getYoutubeDLCommand() string {
 	exePath, err := os.Executable()
 	if err == nil {
@@ -192,13 +213,35 @@ func DownloadVideo(httpClient *http.Client, accessToken, courseID, modDir, input
 		fmt.Printf("    [*] Downloading video: %s\n", title)
 
 		ytCmd := getYoutubeDLCommand()
-		cmd := exec.Command(ytCmd,
-			"--no-playlist",
-			"--cookies", cookieFile,
-			"--referer", config.BaseURL+"/",
-			"-P", modDir,
-			"-o", utils.Sanitize(title)+".%(ext)s",
-			targetURL)
+
+		// Normalize folder URLs so yt-dlp scopes to the right folder.
+		normalizedURL := normalizePanoptoURL(targetURL)
+
+		// Folder/list URLs are intentional playlists; don't pass --no-playlist.
+		isList := strings.Contains(normalizedURL, "List.aspx")
+		var outputTpl string
+		var args []string
+		if isList {
+			outputTpl = utils.Sanitize(title) + "/%(title)s.%(ext)s"
+			args = []string{
+				"--cookies", cookieFile,
+				"--referer", config.BaseURL + "/",
+				"-P", modDir,
+				"-o", outputTpl,
+				normalizedURL,
+			}
+		} else {
+			outputTpl = utils.Sanitize(title) + ".%(ext)s"
+			args = []string{
+				"--no-playlist",
+				"--cookies", cookieFile,
+				"--referer", config.BaseURL + "/",
+				"-P", modDir,
+				"-o", outputTpl,
+				normalizedURL,
+			}
+		}
+		cmd := exec.Command(ytCmd, args...)
 
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
@@ -212,7 +255,7 @@ func DownloadVideo(httpClient *http.Client, accessToken, courseID, modDir, input
 	}
 }
 
-func DownloadMainRecordings(httpClient *http.Client, accessToken, courseID, root string) {
+func DownloadMainRecordings(httpClient *http.Client, accessToken, courseID, root string, videosOnly bool) {
 	fmt.Println("\n[*] Checking for main Panopto recordings...")
 
 	jar, _ := cookiejar.New(nil)
@@ -300,8 +343,18 @@ func DownloadMainRecordings(httpClient *http.Client, accessToken, courseID, root
 		}
 		os.WriteFile(cookieFile, []byte(cData), 0o644)
 
-		recordingsDir := filepath.Join(root, "Recordings")
-		os.MkdirAll(recordingsDir, 0o755)
+		var downloadDir string
+		var outputTemplate string
+
+		if videosOnly {
+			// Flat: all videos go directly into course root
+			downloadDir = root
+			outputTemplate = "%(title)s.%(ext)s"
+		} else {
+			downloadDir = filepath.Join(root, "Recordings")
+			os.MkdirAll(downloadDir, 0o755)
+			outputTemplate = "%(playlist_title)s/%(title)s.%(ext)s"
+		}
 
 		fmt.Println("[*] Starting yt-dlp download for main recordings...")
 
@@ -310,8 +363,8 @@ func DownloadMainRecordings(httpClient *http.Client, accessToken, courseID, root
 		cmd := exec.Command(ytCmd,
 			"--cookies", cookieFile,
 			"--referer", config.BaseURL+"/",
-			"-P", recordingsDir,
-			"-o", "%(playlist_title)s/%(title)s.%(ext)s",
+			"-P", downloadDir,
+			"-o", outputTemplate,
 			decodedURL,
 		)
 		cmd.Stdout = os.Stdout
@@ -323,3 +376,187 @@ func DownloadMainRecordings(httpClient *http.Client, accessToken, courseID, root
 		fmt.Println("[!] No main recordings available or handshake failed")
 	}
 }
+
+// DownloadExternalPanoptoURL authenticates via the module_item_redirect path
+// (exactly what the Canvas mobile app does for ExternalUrl items) and then
+// runs yt-dlp against the given Panopto URL with the resulting cookies.
+//
+// moduleItemURL – item.URL  (https://canvas.vub.be/api/v1/.../module_item_redirect/<id>)
+// panoptoURL    – item.ExternalURL  (https://vub.cloud.panopto.eu/Panopto/Pages/...)
+func DownloadExternalPanoptoURL(httpClient *http.Client, accessToken, moduleItemURL, panoptoURL, modDir, title string) {
+	fmt.Printf("    [dbg] moduleItemURL: %s\n", moduleItemURL)
+	fmt.Printf("    [dbg] panoptoURL:    %s\n", panoptoURL)
+
+	jar, _ := cookiejar.New(nil)
+	// Manual redirect following so we can track cross-domain hops correctly.
+	noRedirectClient := &http.Client{
+		Jar: jar,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+
+	// Step 1: exchange the module item URL for a Canvas web session URL.
+	returnTo := moduleItemURL + "?display=borderless"
+	sessionTokenURL := config.BaseURL + "/login/session_token?return_to=" + url.QueryEscape(returnTo)
+	fmt.Printf("    [dbg] session_token URL: %s\n", sessionTokenURL)
+	bridgeReq, _ := http.NewRequest("GET", sessionTokenURL, nil)
+	bridgeReq.Header.Set("Authorization", "Bearer "+accessToken)
+	bridgeReq.Header.Set("User-Agent", config.UserAgent)
+	bResp, err := httpClient.Do(bridgeReq)
+	if err != nil {
+		fmt.Printf("    [!] session_token request failed: %v\n", err)
+		return
+	}
+	rawBody, _ := io.ReadAll(bResp.Body)
+	bResp.Body.Close()
+	fmt.Printf("    [dbg] session_token response (%d): %s\n", bResp.StatusCode, string(rawBody))
+
+	var bridgeData struct {
+		SessionURL string `json:"session_url"`
+	}
+	json.Unmarshal(rawBody, &bridgeData)
+
+	if bridgeData.SessionURL == "" {
+		fmt.Printf("    [!] No session_url returned (skipping %s)\n", title)
+		return
+	}
+	fmt.Printf("    [dbg] session_url: %s\n", bridgeData.SessionURL)
+
+	// Step 2: GET session URL → Canvas shows OAuth2 confirm page for Panopto.
+	// We follow redirects manually so cross-domain hops (canvas→panopto) work correctly.
+	currentURL := bridgeData.SessionURL
+	var formHTML string
+	var formFinalURL string
+	for hop := 0; hop < 10; hop++ {
+		hopReq, _ := http.NewRequest("GET", currentURL, nil)
+		hopReq.Header.Set("User-Agent", config.UserAgent)
+		hopResp, err := noRedirectClient.Do(hopReq)
+		if err != nil {
+			fmt.Printf("    [!] Failed hop %d to %s: %v\n", hop, currentURL, err)
+			return
+		}
+		body, _ := io.ReadAll(hopResp.Body)
+		hopResp.Body.Close()
+		fmt.Printf("    [dbg] hop %d: status=%d url=%s body=%d\n", hop, hopResp.StatusCode, currentURL, len(body))
+		if hopResp.StatusCode == 301 || hopResp.StatusCode == 302 || hopResp.StatusCode == 303 {
+			loc, _ := hopResp.Location()
+			if loc == nil {
+				fmt.Printf("    [!] Redirect with no Location header\n")
+				return
+			}
+			currentURL = loc.String()
+			continue
+		}
+		formHTML = string(body)
+		formFinalURL = currentURL
+		break
+	}
+	fmt.Printf("    [dbg] OAuth page final URL: %s, length: %d\n", formFinalURL, len(formHTML))
+
+	if strings.Contains(formHTML, "U hebt geen toegang") || strings.Contains(formHTML, "You do not have access") {
+		fmt.Printf("    [!] Access denied. Skipping %s\n", title)
+		return
+	}
+
+	// Step 3: POST /login/oauth2/accept → Canvas redirects to Panopto Login.aspx?code=...
+	// webClient follows all hops automatically, ending with Panopto setting cookies.
+	action := utils.ResolveAction(formFinalURL, formHTML)
+	formData := utils.ExtractFormFields(formHTML)
+	fmt.Printf("    [dbg] POST action: %s\n", action)
+
+	postReq, _ := http.NewRequest("POST", action, strings.NewReader(formData.Encode()))
+	postReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	postReq.Header.Set("User-Agent", config.UserAgent)
+	postReq.Header.Set("Origin", config.BaseURL)
+	postReq.Header.Set("Referer", formFinalURL)
+	postResp, err := noRedirectClient.Do(postReq)
+	if err != nil {
+		fmt.Printf("    [!] OAuth accept POST failed: %v\n", err)
+		return
+	}
+	postBody, _ := io.ReadAll(postResp.Body)
+	postResp.Body.Close()
+	fmt.Printf("    [dbg] POST response status: %d, body len: %d\n", postResp.StatusCode, len(postBody))
+
+	// Now follow the redirect chain from the POST (canvas → panopto Login.aspx → CookieCheck).
+	if postResp.StatusCode == 302 || postResp.StatusCode == 303 {
+		currentURL2 := ""
+		if loc, _ := postResp.Location(); loc != nil {
+			currentURL2 = loc.String()
+		}
+		for hop := 0; hop < 10 && currentURL2 != ""; hop++ {
+			hopReq, _ := http.NewRequest("GET", currentURL2, nil)
+			hopReq.Header.Set("User-Agent", config.UserAgent)
+			hopResp, err := noRedirectClient.Do(hopReq)
+			if err != nil {
+				fmt.Printf("    [dbg] post-redirect hop %d error: %v\n", hop, err)
+				break
+			}
+			body2, _ := io.ReadAll(hopResp.Body)
+			hopResp.Body.Close()
+			fmt.Printf("    [dbg] post-redirect hop %d: status=%d url=%s body=%d\n", hop, hopResp.StatusCode, currentURL2, len(body2))
+			if hopResp.StatusCode == 301 || hopResp.StatusCode == 302 || hopResp.StatusCode == 303 {
+				if loc, _ := hopResp.Location(); loc != nil {
+					currentURL2 = loc.String()
+					continue
+				}
+			}
+			break
+		}
+	}
+
+	// noRedirectClient.Jar now holds Panopto session cookies from the CookieCheck chain.
+	panoptoDomain, _ := url.Parse("https://vub.cloud.panopto.eu")
+	cookies := noRedirectClient.Jar.Cookies(panoptoDomain)
+	fmt.Printf("    [dbg] Panopto cookies: %d\n", len(cookies))
+	for _, c := range cookies {
+		fmt.Printf("    [dbg]   cookie: %s=%s\n", c.Name, c.Value[:min(20, len(c.Value))])
+	}
+	if len(cookies) == 0 {
+		fmt.Printf("    [!] No Panopto cookies after auth – falling back for: %s\n", title)
+		DownloadVideo(httpClient, accessToken, "", modDir, panoptoURL, title)
+		return
+	}
+
+	cookieFile := filepath.Join(modDir, ".cookies_ext.txt")
+	cData := "# Netscape HTTP Cookie File\n"
+	for _, c := range cookies {
+		cData += fmt.Sprintf(".vub.cloud.panopto.eu\tTRUE\t/\tTRUE\t0\t%s\t%s\n", c.Name, c.Value)
+	}
+	os.WriteFile(cookieFile, []byte(cData), 0o644)
+
+	fmt.Printf("    [*] Downloading: %s\n", title)
+
+	normalizedURL := normalizePanoptoURL(panoptoURL)
+	isList := strings.Contains(normalizedURL, "List.aspx")
+
+	ytCmd := getYoutubeDLCommand()
+	var args []string
+	if isList {
+		args = []string{
+			"--cookies", cookieFile,
+			"--referer", config.BaseURL + "/",
+			"-P", modDir,
+			"-o", utils.Sanitize(title) + "/%(title)s.%(ext)s",
+			normalizedURL,
+		}
+	} else {
+		args = []string{
+			"--no-playlist",
+			"--cookies", cookieFile,
+			"--referer", config.BaseURL + "/",
+			"-P", modDir,
+			"-o", utils.Sanitize(title) + ".%(ext)s",
+			normalizedURL,
+		}
+	}
+
+	cmd := exec.Command(ytCmd, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		fmt.Printf("    [!] yt-dlp failed: %v\n", err)
+	}
+	os.Remove(cookieFile)
+}