feat/videos-only-mode #5
12
README.md
12
README.md
@@ -21,11 +21,16 @@ go build -o canvasarchiver ./cmd/canvasarchiver
|
|||||||
./canvasarchiver
|
./canvasarchiver
|
||||||
```
|
```
|
||||||
|
|
||||||
Or for files-only mode:
|
Or for files-only mode (all files flat, no videos):
|
||||||
```bash
|
```bash
|
||||||
./canvasarchiver -fo
|
./canvasarchiver -fo
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Or for videos-only mode (all Panopto videos flat, no files):
|
||||||
|
```bash
|
||||||
|
./canvasarchiver -vo
|
||||||
|
```
|
||||||
|
|
||||||
2. On first run, you'll be prompted to authenticate:
|
2. On first run, you'll be prompted to authenticate:
|
||||||
- Visit the provided OAuth URL
|
- Visit the provided OAuth URL
|
||||||
- Authorize the application
|
- Authorize the application
|
||||||
@@ -38,11 +43,14 @@ go build -o canvasarchiver ./cmd/canvasarchiver
|
|||||||
- Module content (to `Modules/`)
|
- Module content (to `Modules/`)
|
||||||
- Panopto recordings (to `Recordings/`)
|
- Panopto recordings (to `Recordings/`)
|
||||||
|
|
||||||
|
In `-vo` mode, only videos are downloaded — all into the course root directory (no subdirectories).
|
||||||
|
|
||||||
### Flags
|
### Flags
|
||||||
|
|
||||||
| Flag | Description |
|
| Flag | Description |
|
||||||
|------|-------------|
|
|------|-------------|
|
||||||
| `-fo` | Files only mode - download all files to a single directory without module structure |
|
| `-fo` | Files only — download all files flat into one directory; skips videos and module structure |
|
||||||
|
| `-vo` | Videos only — scan recordings and all module video items, download everything flat into one directory; skips regular files |
|
||||||
| `-me` | Download all enrolled courses |
|
| `-me` | Download all enrolled courses |
|
||||||
| `-n` | Prefix modules with order numbers `[1]`, `[2]`, etc. |
|
| `-n` | Prefix modules with order numbers `[1]`, `[2]`, etc. |
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
filesOnly := flag.Bool("fo", false, "Files only mode - download all files to a single directory without module structure")
|
filesOnly := flag.Bool("fo", false, "Files only mode - download all files to a single directory without module structure")
|
||||||
|
videosOnly := flag.Bool("vo", false, "Videos only mode - download only Panopto videos to a single directory")
|
||||||
me := flag.Bool("me", false, "Download all enrolled courses")
|
me := flag.Bool("me", false, "Download all enrolled courses")
|
||||||
moduleNumbers := flag.Bool("n", false, "Prefix modules with order numbers [1], [2], etc.")
|
moduleNumbers := flag.Bool("n", false, "Prefix modules with order numbers [1], [2], etc.")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
@@ -28,7 +29,7 @@ func main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if *me {
|
if *me {
|
||||||
canvasClient := canvas.NewClient(httpClient, accessToken, "", *filesOnly, *moduleNumbers)
|
canvasClient := canvas.NewClient(httpClient, accessToken, "", *filesOnly, *videosOnly, *moduleNumbers)
|
||||||
courses, err := canvasClient.GetEnrolledCourses()
|
courses, err := canvasClient.GetEnrolledCourses()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("Error fetching courses: %v\n", err)
|
fmt.Printf("Error fetching courses: %v\n", err)
|
||||||
@@ -38,7 +39,7 @@ func main() {
|
|||||||
fmt.Printf("[+] Found %d enrolled courses\n", len(courses))
|
fmt.Printf("[+] Found %d enrolled courses\n", len(courses))
|
||||||
for _, course := range courses {
|
for _, course := range courses {
|
||||||
fmt.Printf(" -> Downloading: %s (ID: %d)\n", course.Name, course.ID)
|
fmt.Printf(" -> Downloading: %s (ID: %d)\n", course.Name, course.ID)
|
||||||
downloadCourse(httpClient, accessToken, fmt.Sprintf("%d", course.ID), *filesOnly, *moduleNumbers)
|
downloadCourse(httpClient, accessToken, fmt.Sprintf("%d", course.ID), *filesOnly, *videosOnly, *moduleNumbers)
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -47,11 +48,11 @@ func main() {
|
|||||||
fmt.Print("Enter Course ID: ")
|
fmt.Print("Enter Course ID: ")
|
||||||
fmt.Scanln(&courseID)
|
fmt.Scanln(&courseID)
|
||||||
|
|
||||||
downloadCourse(httpClient, accessToken, courseID, *filesOnly, *moduleNumbers)
|
downloadCourse(httpClient, accessToken, courseID, *filesOnly, *videosOnly, *moduleNumbers)
|
||||||
}
|
}
|
||||||
|
|
||||||
func downloadCourse(httpClient *http.Client, accessToken, courseID string, filesOnly, moduleNumbers bool) {
|
func downloadCourse(httpClient *http.Client, accessToken, courseID string, filesOnly, videosOnly, moduleNumbers bool) {
|
||||||
canvasClient := canvas.NewClient(httpClient, accessToken, courseID, filesOnly, moduleNumbers)
|
canvasClient := canvas.NewClient(httpClient, accessToken, courseID, filesOnly, videosOnly, moduleNumbers)
|
||||||
|
|
||||||
if err := canvasClient.GetCourseInfo(); err != nil {
|
if err := canvasClient.GetCourseInfo(); err != nil {
|
||||||
fmt.Printf("Error: %v\n", err)
|
fmt.Printf("Error: %v\n", err)
|
||||||
@@ -66,7 +67,8 @@ func downloadCourse(httpClient *http.Client, accessToken, courseID string, files
|
|||||||
|
|
||||||
canvasClient.DownloadModules(courseRoot)
|
canvasClient.DownloadModules(courseRoot)
|
||||||
|
|
||||||
if !filesOnly {
|
// Run recordings: always in -vo mode; skipped in -fo mode; normal otherwise.
|
||||||
panopto.DownloadMainRecordings(httpClient, accessToken, courseID, courseRoot)
|
if videosOnly || !filesOnly {
|
||||||
|
panopto.DownloadMainRecordings(httpClient, accessToken, courseID, courseRoot, videosOnly)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,16 +22,18 @@ type Client struct {
|
|||||||
CourseID string
|
CourseID string
|
||||||
CourseName string
|
CourseName string
|
||||||
FilesOnly bool
|
FilesOnly bool
|
||||||
|
VideosOnly bool
|
||||||
ModuleNumbers bool
|
ModuleNumbers bool
|
||||||
downloadedFiles map[string]bool
|
downloadedFiles map[string]bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewClient(httpClient *http.Client, accessToken, courseID string, filesOnly, moduleNumbers bool) *Client {
|
func NewClient(httpClient *http.Client, accessToken, courseID string, filesOnly, videosOnly, moduleNumbers bool) *Client {
|
||||||
return &Client{
|
return &Client{
|
||||||
HTTPClient: httpClient,
|
HTTPClient: httpClient,
|
||||||
AccessToken: accessToken,
|
AccessToken: accessToken,
|
||||||
CourseID: courseID,
|
CourseID: courseID,
|
||||||
FilesOnly: filesOnly,
|
FilesOnly: filesOnly,
|
||||||
|
VideosOnly: videosOnly,
|
||||||
ModuleNumbers: moduleNumbers,
|
ModuleNumbers: moduleNumbers,
|
||||||
downloadedFiles: make(map[string]bool),
|
downloadedFiles: make(map[string]bool),
|
||||||
}
|
}
|
||||||
@@ -68,6 +70,10 @@ func (c *Client) GetEnrolledCourses() ([]models.Course, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *Client) DownloadCourseFiles(root string) {
|
func (c *Client) DownloadCourseFiles(root string) {
|
||||||
|
if c.VideosOnly {
|
||||||
|
fmt.Println("\n[*] Skipping regular course files (videos only mode)")
|
||||||
|
return
|
||||||
|
}
|
||||||
fmt.Println("\n[*] Fetching regular course files...")
|
fmt.Println("\n[*] Fetching regular course files...")
|
||||||
|
|
||||||
fReq, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/courses/%s/folders?per_page=100", config.BaseURL, c.CourseID), nil)
|
fReq, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/courses/%s/folders?per_page=100", config.BaseURL, c.CourseID), nil)
|
||||||
@@ -164,14 +170,19 @@ func (c *Client) DownloadModules(courseRoot string) {
|
|||||||
modName = fmt.Sprintf("[%d] %s", i+1, mod.Name)
|
modName = fmt.Sprintf("[%d] %s", i+1, mod.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// In videos-only mode everything goes flat into courseRoot.
|
||||||
|
// In files-only mode everything goes flat into courseRoot.
|
||||||
|
// Otherwise use the structured Modules/<name> path.
|
||||||
modBaseDir := courseRoot
|
modBaseDir := courseRoot
|
||||||
if !c.FilesOnly {
|
if !c.FilesOnly && !c.VideosOnly {
|
||||||
modBaseDir = filepath.Join(courseRoot, "Modules", utils.Sanitize(modName))
|
modBaseDir = filepath.Join(courseRoot, "Modules", utils.Sanitize(modName))
|
||||||
}
|
}
|
||||||
os.MkdirAll(modBaseDir, 0o755)
|
os.MkdirAll(modBaseDir, 0o755)
|
||||||
|
|
||||||
if !c.FilesOnly {
|
if !c.FilesOnly && !c.VideosOnly {
|
||||||
fmt.Printf("\n[Module] %s\n", modName)
|
fmt.Printf("\n[Module] %s\n", modName)
|
||||||
|
} else if c.VideosOnly {
|
||||||
|
fmt.Printf("\n[Module] %s (scanning for videos)\n", modName)
|
||||||
}
|
}
|
||||||
|
|
||||||
subHeaderStack := []string{}
|
subHeaderStack := []string{}
|
||||||
@@ -179,15 +190,16 @@ func (c *Client) DownloadModules(courseRoot string) {
|
|||||||
|
|
||||||
for _, item := range mod.Items {
|
for _, item := range mod.Items {
|
||||||
|
|
||||||
|
// In videos-only mode always download to the flat courseRoot.
|
||||||
targetDir := modBaseDir
|
targetDir := modBaseDir
|
||||||
if len(subHeaderStack) > 0 && !c.FilesOnly {
|
if len(subHeaderStack) > 0 && !c.FilesOnly && !c.VideosOnly {
|
||||||
targetDir = filepath.Join(modBaseDir, filepath.Join(subHeaderStack...))
|
targetDir = filepath.Join(modBaseDir, filepath.Join(subHeaderStack...))
|
||||||
}
|
}
|
||||||
os.MkdirAll(targetDir, 0o755)
|
os.MkdirAll(targetDir, 0o755)
|
||||||
|
|
||||||
switch item.Type {
|
switch item.Type {
|
||||||
case "SubHeader":
|
case "SubHeader":
|
||||||
if c.FilesOnly {
|
if c.FilesOnly || c.VideosOnly {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
currentIndent := item.Indent
|
currentIndent := item.Indent
|
||||||
@@ -207,6 +219,9 @@ func (c *Client) DownloadModules(courseRoot string) {
|
|||||||
fmt.Printf("%s--- %s ---\n", indent, item.Title)
|
fmt.Printf("%s--- %s ---\n", indent, item.Title)
|
||||||
|
|
||||||
case "File":
|
case "File":
|
||||||
|
if c.VideosOnly {
|
||||||
|
continue
|
||||||
|
}
|
||||||
c.downloadModuleFile(item, targetDir)
|
c.downloadModuleFile(item, targetDir)
|
||||||
|
|
||||||
case "ExternalTool":
|
case "ExternalTool":
|
||||||
|
|||||||
@@ -17,6 +17,27 @@ import (
|
|||||||
"git.directme.in/Joren/CanvasArchiver/internal/utils"
|
"git.directme.in/Joren/CanvasArchiver/internal/utils"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// normalizePanoptoURL converts the query-param form that Canvas stores
|
||||||
|
// (List.aspx?folderID=X) to the fragment form that yt-dlp's PanoptoList
|
||||||
|
// extractor understands (List.aspx#folderID="X"). Without this yt-dlp
|
||||||
|
// ignores the folder filter and downloads the entire Panopto instance.
|
||||||
|
func normalizePanoptoURL(rawURL string) string {
|
||||||
|
parsed, err := url.Parse(rawURL)
|
||||||
|
if err != nil {
|
||||||
|
return rawURL
|
||||||
|
}
|
||||||
|
if strings.Contains(parsed.Path, "List.aspx") {
|
||||||
|
folderID := parsed.Query().Get("folderID")
|
||||||
|
if folderID != "" {
|
||||||
|
// Strip query, set fragment: List.aspx#folderID="<id>"
|
||||||
|
parsed.RawQuery = ""
|
||||||
|
parsed.Fragment = fmt.Sprintf(`folderID="%s"`, folderID)
|
||||||
|
return parsed.String()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rawURL
|
||||||
|
}
|
||||||
|
|
||||||
func getYoutubeDLCommand() string {
|
func getYoutubeDLCommand() string {
|
||||||
exePath, err := os.Executable()
|
exePath, err := os.Executable()
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -192,13 +213,35 @@ func DownloadVideo(httpClient *http.Client, accessToken, courseID, modDir, input
|
|||||||
fmt.Printf(" [*] Downloading video: %s\n", title)
|
fmt.Printf(" [*] Downloading video: %s\n", title)
|
||||||
|
|
||||||
ytCmd := getYoutubeDLCommand()
|
ytCmd := getYoutubeDLCommand()
|
||||||
cmd := exec.Command(ytCmd,
|
|
||||||
|
// Normalize folder URLs so yt-dlp scopes to the right folder.
|
||||||
|
normalizedURL := normalizePanoptoURL(targetURL)
|
||||||
|
|
||||||
|
// Folder/list URLs are intentional playlists; don't pass --no-playlist.
|
||||||
|
isList := strings.Contains(normalizedURL, "List.aspx")
|
||||||
|
var outputTpl string
|
||||||
|
var args []string
|
||||||
|
if isList {
|
||||||
|
outputTpl = utils.Sanitize(title) + "/%(title)s.%(ext)s"
|
||||||
|
args = []string{
|
||||||
|
"--cookies", cookieFile,
|
||||||
|
"--referer", config.BaseURL + "/",
|
||||||
|
"-P", modDir,
|
||||||
|
"-o", outputTpl,
|
||||||
|
normalizedURL,
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
outputTpl = utils.Sanitize(title) + ".%(ext)s"
|
||||||
|
args = []string{
|
||||||
"--no-playlist",
|
"--no-playlist",
|
||||||
"--cookies", cookieFile,
|
"--cookies", cookieFile,
|
||||||
"--referer", config.BaseURL + "/",
|
"--referer", config.BaseURL + "/",
|
||||||
"-P", modDir,
|
"-P", modDir,
|
||||||
"-o", utils.Sanitize(title)+".%(ext)s",
|
"-o", outputTpl,
|
||||||
targetURL)
|
normalizedURL,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cmd := exec.Command(ytCmd, args...)
|
||||||
|
|
||||||
cmd.Stdout = os.Stdout
|
cmd.Stdout = os.Stdout
|
||||||
cmd.Stderr = os.Stderr
|
cmd.Stderr = os.Stderr
|
||||||
@@ -212,7 +255,7 @@ func DownloadVideo(httpClient *http.Client, accessToken, courseID, modDir, input
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func DownloadMainRecordings(httpClient *http.Client, accessToken, courseID, root string) {
|
func DownloadMainRecordings(httpClient *http.Client, accessToken, courseID, root string, videosOnly bool) {
|
||||||
fmt.Println("\n[*] Checking for main Panopto recordings...")
|
fmt.Println("\n[*] Checking for main Panopto recordings...")
|
||||||
|
|
||||||
jar, _ := cookiejar.New(nil)
|
jar, _ := cookiejar.New(nil)
|
||||||
@@ -300,8 +343,18 @@ func DownloadMainRecordings(httpClient *http.Client, accessToken, courseID, root
|
|||||||
}
|
}
|
||||||
os.WriteFile(cookieFile, []byte(cData), 0o644)
|
os.WriteFile(cookieFile, []byte(cData), 0o644)
|
||||||
|
|
||||||
recordingsDir := filepath.Join(root, "Recordings")
|
var downloadDir string
|
||||||
os.MkdirAll(recordingsDir, 0o755)
|
var outputTemplate string
|
||||||
|
|
||||||
|
if videosOnly {
|
||||||
|
// Flat: all videos go directly into course root
|
||||||
|
downloadDir = root
|
||||||
|
outputTemplate = "%(title)s.%(ext)s"
|
||||||
|
} else {
|
||||||
|
downloadDir = filepath.Join(root, "Recordings")
|
||||||
|
os.MkdirAll(downloadDir, 0o755)
|
||||||
|
outputTemplate = "%(playlist_title)s/%(title)s.%(ext)s"
|
||||||
|
}
|
||||||
|
|
||||||
fmt.Println("[*] Starting yt-dlp download for main recordings...")
|
fmt.Println("[*] Starting yt-dlp download for main recordings...")
|
||||||
|
|
||||||
@@ -310,8 +363,8 @@ func DownloadMainRecordings(httpClient *http.Client, accessToken, courseID, root
|
|||||||
cmd := exec.Command(ytCmd,
|
cmd := exec.Command(ytCmd,
|
||||||
"--cookies", cookieFile,
|
"--cookies", cookieFile,
|
||||||
"--referer", config.BaseURL+"/",
|
"--referer", config.BaseURL+"/",
|
||||||
"-P", recordingsDir,
|
"-P", downloadDir,
|
||||||
"-o", "%(playlist_title)s/%(title)s.%(ext)s",
|
"-o", outputTemplate,
|
||||||
decodedURL,
|
decodedURL,
|
||||||
)
|
)
|
||||||
cmd.Stdout = os.Stdout
|
cmd.Stdout = os.Stdout
|
||||||
@@ -323,3 +376,4 @@ func DownloadMainRecordings(httpClient *http.Client, accessToken, courseID, root
|
|||||||
fmt.Println("[!] No main recordings available or handshake failed")
|
fmt.Println("[!] No main recordings available or handshake failed")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user