Compare commits
23 Commits
v1.0.0
...
ef10711b6e
| Author | SHA1 | Date | |
|---|---|---|---|
|
ef10711b6e
|
|||
|
25729ce492
|
|||
| 04e7d3e0f4 | |||
| 2776d057cd | |||
|
522a8b22f8
|
|||
| ea9d4dc2dc | |||
| 333e784ce9 | |||
| 43392a4132 | |||
|
13063c6cc5
|
|||
|
|
14a71e7dca | ||
| 051b67de51 | |||
|
|
9691ecd7a5 | ||
|
|
b8e6180b35 | ||
|
|
8591ae283e | ||
| 11d9867155 | |||
| 9ffe1283c2 | |||
|
05ed4dd4ed
|
|||
| be63064bee | |||
| cd259e01a3 | |||
|
352315b041
|
|||
| c9fa079893 | |||
| 6cc25602dc | |||
|
fbf75c88b3
|
23
README.md
23
README.md
@@ -21,18 +21,39 @@ go build -o canvasarchiver ./cmd/canvasarchiver
|
|||||||
./canvasarchiver
|
./canvasarchiver
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Or for files-only mode (all files flat, no videos):
|
||||||
|
```bash
|
||||||
|
./canvasarchiver -fo
|
||||||
|
```
|
||||||
|
|
||||||
|
Or for videos-only mode (all Panopto videos flat, no files):
|
||||||
|
```bash
|
||||||
|
./canvasarchiver -vo
|
||||||
|
```
|
||||||
|
|
||||||
2. On first run, you'll be prompted to authenticate:
|
2. On first run, you'll be prompted to authenticate:
|
||||||
- Visit the provided OAuth URL
|
- Visit the provided OAuth URL
|
||||||
- Authorize the application
|
- Authorize the application
|
||||||
- Copy the authorization code back to the terminal
|
- Copy the authorization code back to the terminal
|
||||||
|
|
||||||
3. Enter your Course ID when prompted
|
3. Enter your Course ID when prompted (or use `-me` to download all enrolled courses)
|
||||||
|
|
||||||
4. The tool will download:
|
4. The tool will download:
|
||||||
- Regular course files (to `Course Files/`)
|
- Regular course files (to `Course Files/`)
|
||||||
- Module content (to `Modules/`)
|
- Module content (to `Modules/`)
|
||||||
- Panopto recordings (to `Recordings/`)
|
- Panopto recordings (to `Recordings/`)
|
||||||
|
|
||||||
|
In `-vo` mode, only videos are downloaded — all into the course root directory (no subdirectories).
|
||||||
|
|
||||||
|
### Flags
|
||||||
|
|
||||||
|
| Flag | Description |
|
||||||
|
|------|-------------|
|
||||||
|
| `-fo` | Files only — download all files flat into one directory; skips videos and module structure |
|
||||||
|
| `-vo` | Videos only — scan recordings and all module video items, download everything flat into one directory; skips regular files |
|
||||||
|
| `-me` | Download all enrolled courses |
|
||||||
|
| `-n` | Prefix modules with order numbers `[1]`, `[2]`, etc. |
|
||||||
|
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
@@ -12,6 +13,12 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
filesOnly := flag.Bool("fo", false, "Files only mode - download all files to a single directory without module structure")
|
||||||
|
videosOnly := flag.Bool("vo", false, "Videos only mode - download only Panopto videos to a single directory")
|
||||||
|
me := flag.Bool("me", false, "Download all enrolled courses")
|
||||||
|
moduleNumbers := flag.Bool("n", false, "Prefix modules with order numbers [1], [2], etc.")
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
httpClient := &http.Client{}
|
httpClient := &http.Client{}
|
||||||
|
|
||||||
authenticator := auth.NewAuthenticator(httpClient)
|
authenticator := auth.NewAuthenticator(httpClient)
|
||||||
@@ -21,11 +28,31 @@ func main() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if *me {
|
||||||
|
canvasClient := canvas.NewClient(httpClient, accessToken, "", *filesOnly, *videosOnly, *moduleNumbers)
|
||||||
|
courses, err := canvasClient.GetEnrolledCourses()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error fetching courses: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("[+] Found %d enrolled courses\n", len(courses))
|
||||||
|
for _, course := range courses {
|
||||||
|
fmt.Printf(" -> Downloading: %s (ID: %d)\n", course.Name, course.ID)
|
||||||
|
downloadCourse(httpClient, accessToken, fmt.Sprintf("%d", course.ID), *filesOnly, *videosOnly, *moduleNumbers)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
var courseID string
|
var courseID string
|
||||||
fmt.Print("Enter Course ID: ")
|
fmt.Print("Enter Course ID: ")
|
||||||
fmt.Scanln(&courseID)
|
fmt.Scanln(&courseID)
|
||||||
|
|
||||||
canvasClient := canvas.NewClient(httpClient, accessToken, courseID)
|
downloadCourse(httpClient, accessToken, courseID, *filesOnly, *videosOnly, *moduleNumbers)
|
||||||
|
}
|
||||||
|
|
||||||
|
func downloadCourse(httpClient *http.Client, accessToken, courseID string, filesOnly, videosOnly, moduleNumbers bool) {
|
||||||
|
canvasClient := canvas.NewClient(httpClient, accessToken, courseID, filesOnly, videosOnly, moduleNumbers)
|
||||||
|
|
||||||
if err := canvasClient.GetCourseInfo(); err != nil {
|
if err := canvasClient.GetCourseInfo(); err != nil {
|
||||||
fmt.Printf("Error: %v\n", err)
|
fmt.Printf("Error: %v\n", err)
|
||||||
@@ -40,5 +67,8 @@ func main() {
|
|||||||
|
|
||||||
canvasClient.DownloadModules(courseRoot)
|
canvasClient.DownloadModules(courseRoot)
|
||||||
|
|
||||||
panopto.DownloadMainRecordings(httpClient, accessToken, courseID, courseRoot)
|
// Run recordings: always in -vo mode; skipped in -fo mode; normal otherwise.
|
||||||
|
if videosOnly || !filesOnly {
|
||||||
|
panopto.DownloadMainRecordings(httpClient, accessToken, courseID, courseRoot, videosOnly)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,11 +1,17 @@
|
|||||||
package auth
|
package auth
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bufio"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"git.directme.in/Joren/CanvasArchiver/internal/config"
|
"git.directme.in/Joren/CanvasArchiver/internal/config"
|
||||||
"git.directme.in/Joren/CanvasArchiver/internal/models"
|
"git.directme.in/Joren/CanvasArchiver/internal/models"
|
||||||
@@ -24,28 +30,7 @@ func NewAuthenticator(client *http.Client) *Authenticator {
|
|||||||
func (a *Authenticator) GetAccessToken() (string, error) {
|
func (a *Authenticator) GetAccessToken() (string, error) {
|
||||||
creds, err := LoadCredentials()
|
creds, err := LoadCredentials()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
return a.login()
|
||||||
fmt.Println("--- Initial Canvas Login Required ---")
|
|
||||||
fmt.Printf("Visit: %s/login/oauth2/auth?client_id=%s&response_type=code&redirect_uri=%s\n",
|
|
||||||
config.BaseURL, config.ClientID, url.QueryEscape(config.RedirectURI))
|
|
||||||
fmt.Print("Enter Code: ")
|
|
||||||
var code string
|
|
||||||
fmt.Scanln(&code)
|
|
||||||
|
|
||||||
tr, err := a.doTokenRequest(url.Values{
|
|
||||||
"grant_type": {"authorization_code"},
|
|
||||||
"client_id": {config.ClientID},
|
|
||||||
"client_secret": {config.ClientSecret},
|
|
||||||
"redirect_uri": {config.RedirectURI},
|
|
||||||
"code": {code},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
SaveCredentials(&models.Credentials{RefreshToken: tr.RefreshToken})
|
|
||||||
fmt.Println("[+] Login successful.")
|
|
||||||
return tr.AccessToken, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println("[*] Reusing saved refresh token...")
|
fmt.Println("[*] Reusing saved refresh token...")
|
||||||
@@ -54,11 +39,49 @@ func (a *Authenticator) GetAccessToken() (string, error) {
|
|||||||
"client_id": {config.ClientID},
|
"client_id": {config.ClientID},
|
||||||
"client_secret": {config.ClientSecret},
|
"client_secret": {config.ClientSecret},
|
||||||
"refresh_token": {creds.RefreshToken},
|
"refresh_token": {creds.RefreshToken},
|
||||||
|
"redirect_uri": {config.RedirectURI},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("[!] Refresh failed: %v\n", err)
|
||||||
|
return a.login()
|
||||||
|
}
|
||||||
|
fmt.Println("[+] Session refreshed.")
|
||||||
|
return tr.AccessToken, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *Authenticator) login() (string, error) {
|
||||||
|
codeVerifier, codeChallenge, err := generatePKCEPair()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println("--- Initial Canvas Login Required ---")
|
||||||
|
fmt.Printf("Visit: %s\n", buildAuthURL(codeChallenge))
|
||||||
|
fmt.Print("Enter Code or redirect URL: ")
|
||||||
|
input, err := bufio.NewReader(os.Stdin).ReadString('\n')
|
||||||
|
if err != nil && err != io.EOF {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
code, err := extractCode(input)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
tr, err := a.doTokenRequest(url.Values{
|
||||||
|
"grant_type": {"authorization_code"},
|
||||||
|
"client_id": {config.ClientID},
|
||||||
|
"client_secret": {config.ClientSecret},
|
||||||
|
"redirect_uri": {config.RedirectURI},
|
||||||
|
"code": {code},
|
||||||
|
"code_verifier": {codeVerifier},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
fmt.Println("[+] Session refreshed.")
|
|
||||||
|
SaveCredentials(&models.Credentials{RefreshToken: tr.RefreshToken})
|
||||||
|
fmt.Println("[+] Login successful.")
|
||||||
return tr.AccessToken, nil
|
return tr.AccessToken, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -70,14 +93,75 @@ func (a *Authenticator) doTokenRequest(v url.Values) (*models.TokenResponse, err
|
|||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode != 200 {
|
if resp.StatusCode != 200 {
|
||||||
return nil, fmt.Errorf("token request failed: %d", resp.StatusCode)
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
return nil, fmt.Errorf("token request failed: %d: %s", resp.StatusCode, string(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
var tr models.TokenResponse
|
var tr models.TokenResponse
|
||||||
json.NewDecoder(resp.Body).Decode(&tr)
|
if err := json.NewDecoder(resp.Body).Decode(&tr); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
return &tr, nil
|
return &tr, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func buildAuthURL(codeChallenge string) string {
|
||||||
|
u, _ := url.Parse(config.BaseURL + "/login/oauth2/auth")
|
||||||
|
q := u.Query()
|
||||||
|
q.Set("client_id", config.ClientID)
|
||||||
|
q.Set("response_type", "code")
|
||||||
|
q.Set("mobile", "1")
|
||||||
|
q.Set("purpose", config.OAuthPurpose)
|
||||||
|
q.Set("code_challenge", codeChallenge)
|
||||||
|
q.Set("code_challenge_method", "S256")
|
||||||
|
q.Set("redirect_uri", config.RedirectURI)
|
||||||
|
if config.AuthProvider != "" {
|
||||||
|
q.Set("authentication_provider", config.AuthProvider)
|
||||||
|
}
|
||||||
|
u.RawQuery = q.Encode()
|
||||||
|
return u.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func generatePKCEPair() (string, string, error) {
|
||||||
|
randomBytes := make([]byte, 64)
|
||||||
|
if _, err := rand.Read(randomBytes); err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
verifier := base64.RawURLEncoding.EncodeToString(randomBytes)
|
||||||
|
challengeBytes := sha256.Sum256([]byte(verifier))
|
||||||
|
challenge := base64.RawURLEncoding.EncodeToString(challengeBytes[:])
|
||||||
|
return verifier, challenge, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractCode(input string) (string, error) {
|
||||||
|
input = strings.TrimSpace(input)
|
||||||
|
if input == "" {
|
||||||
|
return "", fmt.Errorf("empty authorization code")
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, field := range strings.Fields(input) {
|
||||||
|
if strings.Contains(field, "code=") {
|
||||||
|
input = field
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
parsedURL, err := url.Parse(input)
|
||||||
|
if err == nil {
|
||||||
|
if code := parsedURL.Query().Get("code"); code != "" {
|
||||||
|
return code, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if values, err := url.ParseQuery(input); err == nil {
|
||||||
|
if code := values.Get("code"); code != "" {
|
||||||
|
return code, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return input, nil
|
||||||
|
}
|
||||||
|
|
||||||
func SaveCredentials(creds *models.Credentials) {
|
func SaveCredentials(creds *models.Credentials) {
|
||||||
data, _ := json.MarshalIndent(creds, "", " ")
|
data, _ := json.MarshalIndent(creds, "", " ")
|
||||||
os.WriteFile(config.CredsFile, data, 0o644)
|
os.WriteFile(config.CredsFile, data, 0o644)
|
||||||
|
|||||||
@@ -17,17 +17,25 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type Client struct {
|
type Client struct {
|
||||||
HTTPClient *http.Client
|
HTTPClient *http.Client
|
||||||
AccessToken string
|
AccessToken string
|
||||||
CourseID string
|
CourseID string
|
||||||
CourseName string
|
CourseName string
|
||||||
|
FilesOnly bool
|
||||||
|
VideosOnly bool
|
||||||
|
ModuleNumbers bool
|
||||||
|
downloadedFiles map[string]bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewClient(httpClient *http.Client, accessToken, courseID string) *Client {
|
func NewClient(httpClient *http.Client, accessToken, courseID string, filesOnly, videosOnly, moduleNumbers bool) *Client {
|
||||||
return &Client{
|
return &Client{
|
||||||
HTTPClient: httpClient,
|
HTTPClient: httpClient,
|
||||||
AccessToken: accessToken,
|
AccessToken: accessToken,
|
||||||
CourseID: courseID,
|
CourseID: courseID,
|
||||||
|
FilesOnly: filesOnly,
|
||||||
|
VideosOnly: videosOnly,
|
||||||
|
ModuleNumbers: moduleNumbers,
|
||||||
|
downloadedFiles: make(map[string]bool),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -47,7 +55,25 @@ func (c *Client) GetCourseInfo() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Client) GetEnrolledCourses() ([]models.Course, error) {
|
||||||
|
req, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/courses?enrollment_state=active&per_page=100", config.BaseURL), nil)
|
||||||
|
req.Header.Set("Authorization", "Bearer "+c.AccessToken)
|
||||||
|
resp, err := c.HTTPClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
var courses []models.Course
|
||||||
|
json.NewDecoder(resp.Body).Decode(&courses)
|
||||||
|
return courses, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Client) DownloadCourseFiles(root string) {
|
func (c *Client) DownloadCourseFiles(root string) {
|
||||||
|
if c.VideosOnly {
|
||||||
|
fmt.Println("\n[*] Skipping regular course files (videos only mode)")
|
||||||
|
return
|
||||||
|
}
|
||||||
fmt.Println("\n[*] Fetching regular course files...")
|
fmt.Println("\n[*] Fetching regular course files...")
|
||||||
|
|
||||||
fReq, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/courses/%s/folders?per_page=100", config.BaseURL, c.CourseID), nil)
|
fReq, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/courses/%s/folders?per_page=100", config.BaseURL, c.CourseID), nil)
|
||||||
@@ -80,10 +106,24 @@ func (c *Client) DownloadCourseFiles(root string) {
|
|||||||
fileCount := 0
|
fileCount := 0
|
||||||
for _, file := range files {
|
for _, file := range files {
|
||||||
|
|
||||||
|
if c.FilesOnly {
|
||||||
|
if c.downloadedFiles[file.DisplayName] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c.downloadedFiles[file.DisplayName] = true
|
||||||
|
}
|
||||||
|
|
||||||
rawFolderPath := folderMap[file.FolderID]
|
rawFolderPath := folderMap[file.FolderID]
|
||||||
safeFolderPath := utils.SanitizePath(rawFolderPath)
|
safeFolderPath := utils.SanitizePath(rawFolderPath)
|
||||||
|
|
||||||
subDir := filepath.Join(root, "Course Files", safeFolderPath)
|
subDir := root
|
||||||
|
if !c.FilesOnly {
|
||||||
|
if safeFolderPath != "" && strings.ToLower(safeFolderPath) != "course files" {
|
||||||
|
subDir = filepath.Join(root, "Course Files", safeFolderPath)
|
||||||
|
} else {
|
||||||
|
subDir = filepath.Join(root, "Course Files")
|
||||||
|
}
|
||||||
|
}
|
||||||
os.MkdirAll(subDir, 0o755)
|
os.MkdirAll(subDir, 0o755)
|
||||||
path := filepath.Join(subDir, utils.Sanitize(file.DisplayName))
|
path := filepath.Join(subDir, utils.Sanitize(file.DisplayName))
|
||||||
|
|
||||||
@@ -124,24 +164,44 @@ func (c *Client) DownloadModules(courseRoot string) {
|
|||||||
json.NewDecoder(resp.Body).Decode(&modules)
|
json.NewDecoder(resp.Body).Decode(&modules)
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
|
|
||||||
for _, mod := range modules {
|
for i, mod := range modules {
|
||||||
modBaseDir := filepath.Join(courseRoot, "Modules", utils.Sanitize(mod.Name))
|
modName := mod.Name
|
||||||
|
if c.ModuleNumbers {
|
||||||
|
modName = fmt.Sprintf("[%d] %s", i+1, mod.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// In videos-only mode everything goes flat into courseRoot.
|
||||||
|
// In files-only mode everything goes flat into courseRoot.
|
||||||
|
// Otherwise use the structured Modules/<name> path.
|
||||||
|
modBaseDir := courseRoot
|
||||||
|
if !c.FilesOnly && !c.VideosOnly {
|
||||||
|
modBaseDir = filepath.Join(courseRoot, "Modules", utils.Sanitize(modName))
|
||||||
|
}
|
||||||
os.MkdirAll(modBaseDir, 0o755)
|
os.MkdirAll(modBaseDir, 0o755)
|
||||||
fmt.Printf("\n[Module] %s\n", mod.Name)
|
|
||||||
|
if !c.FilesOnly && !c.VideosOnly {
|
||||||
|
fmt.Printf("\n[Module] %s\n", modName)
|
||||||
|
} else if c.VideosOnly {
|
||||||
|
fmt.Printf("\n[Module] %s (scanning for videos)\n", modName)
|
||||||
|
}
|
||||||
|
|
||||||
subHeaderStack := []string{}
|
subHeaderStack := []string{}
|
||||||
lastIndent := 0
|
lastIndent := 0
|
||||||
|
|
||||||
for _, item := range mod.Items {
|
for _, item := range mod.Items {
|
||||||
|
|
||||||
|
// In videos-only mode always download to the flat courseRoot.
|
||||||
targetDir := modBaseDir
|
targetDir := modBaseDir
|
||||||
if len(subHeaderStack) > 0 {
|
if len(subHeaderStack) > 0 && !c.FilesOnly && !c.VideosOnly {
|
||||||
targetDir = filepath.Join(modBaseDir, filepath.Join(subHeaderStack...))
|
targetDir = filepath.Join(modBaseDir, filepath.Join(subHeaderStack...))
|
||||||
}
|
}
|
||||||
os.MkdirAll(targetDir, 0o755)
|
os.MkdirAll(targetDir, 0o755)
|
||||||
|
|
||||||
switch item.Type {
|
switch item.Type {
|
||||||
case "SubHeader":
|
case "SubHeader":
|
||||||
|
if c.FilesOnly || c.VideosOnly {
|
||||||
|
continue
|
||||||
|
}
|
||||||
currentIndent := item.Indent
|
currentIndent := item.Indent
|
||||||
if currentIndent <= lastIndent && len(subHeaderStack) > 0 {
|
if currentIndent <= lastIndent && len(subHeaderStack) > 0 {
|
||||||
levelsToKeep := currentIndent
|
levelsToKeep := currentIndent
|
||||||
@@ -159,14 +219,34 @@ func (c *Client) DownloadModules(courseRoot string) {
|
|||||||
fmt.Printf("%s--- %s ---\n", indent, item.Title)
|
fmt.Printf("%s--- %s ---\n", indent, item.Title)
|
||||||
|
|
||||||
case "File":
|
case "File":
|
||||||
|
if c.VideosOnly {
|
||||||
|
continue
|
||||||
|
}
|
||||||
c.downloadModuleFile(item, targetDir)
|
c.downloadModuleFile(item, targetDir)
|
||||||
|
|
||||||
case "ExternalTool":
|
case "ExternalTool":
|
||||||
|
if c.FilesOnly {
|
||||||
|
continue
|
||||||
|
}
|
||||||
indent := strings.Repeat(" ", len(subHeaderStack)+1)
|
indent := strings.Repeat(" ", len(subHeaderStack)+1)
|
||||||
fmt.Printf("%s- Found video tool: %s\n", indent, item.Title)
|
fmt.Printf("%s- Found video tool: %s\n", indent, item.Title)
|
||||||
panopto.DownloadVideo(c.HTTPClient, c.AccessToken, c.CourseID, targetDir, item.URL, item.Title)
|
panopto.DownloadVideo(c.HTTPClient, c.AccessToken, c.CourseID, targetDir, item.URL, item.Title)
|
||||||
|
|
||||||
|
case "ExternalUrl":
|
||||||
|
if c.FilesOnly {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.Contains(item.ExternalURL, "panopto.eu") {
|
||||||
|
indent := strings.Repeat(" ", len(subHeaderStack)+1)
|
||||||
|
fmt.Printf("%s- Found direct video link: %s\n", indent, item.Title)
|
||||||
|
|
||||||
|
panopto.DownloadVideo(c.HTTPClient, c.AccessToken, c.CourseID, targetDir, item.ExternalURL, item.Title)
|
||||||
|
}
|
||||||
|
|
||||||
case "Page":
|
case "Page":
|
||||||
|
if c.FilesOnly {
|
||||||
|
continue
|
||||||
|
}
|
||||||
c.searchPageForVideos(item, targetDir)
|
c.searchPageForVideos(item, targetDir)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -192,11 +272,18 @@ func (c *Client) downloadModuleFile(item models.ModuleItem, dir string) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if c.FilesOnly {
|
||||||
|
if c.downloadedFiles[fileMeta.DisplayName] {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.downloadedFiles[fileMeta.DisplayName] = true
|
||||||
|
}
|
||||||
|
|
||||||
ext := filepath.Ext(fileMeta.DisplayName)
|
ext := filepath.Ext(fileMeta.DisplayName)
|
||||||
origBase := strings.TrimSuffix(fileMeta.DisplayName, ext)
|
origBase := strings.TrimSuffix(fileMeta.DisplayName, ext)
|
||||||
|
|
||||||
fileName := fileMeta.DisplayName
|
fileName := fileMeta.DisplayName
|
||||||
if !strings.EqualFold(origBase, item.Title) && item.Title != "" {
|
if !c.FilesOnly && !strings.EqualFold(origBase, item.Title) && item.Title != "" {
|
||||||
fileName = fmt.Sprintf("%s (%s)%s", origBase, item.Title, ext)
|
fileName = fmt.Sprintf("%s (%s)%s", origBase, item.Title, ext)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,9 @@ const (
|
|||||||
BaseURL = "https://canvas.vub.be"
|
BaseURL = "https://canvas.vub.be"
|
||||||
ClientID = "170000000000044"
|
ClientID = "170000000000044"
|
||||||
ClientSecret = "3sxR3NtgXRfT9KdpWGAFQygq6O9RzLN021h2lAzhHUZEeSQ5XGV41Ddi5iutwW6f"
|
ClientSecret = "3sxR3NtgXRfT9KdpWGAFQygq6O9RzLN021h2lAzhHUZEeSQ5XGV41Ddi5iutwW6f"
|
||||||
RedirectURI = "urn:ietf:wg:oauth:2.0:oob"
|
RedirectURI = "https://sso.canvaslms.com/canvas/login"
|
||||||
|
OAuthPurpose = "CanvasArchiver"
|
||||||
|
AuthProvider = "microsoft"
|
||||||
CredsFile = "credentials.json"
|
CredsFile = "credentials.json"
|
||||||
PanoptoID = "15"
|
PanoptoID = "15"
|
||||||
UserAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Mobile Safari/537.36"
|
UserAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Mobile Safari/537.36"
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ type TokenResponse struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Course struct {
|
type Course struct {
|
||||||
|
ID int `json:"id"`
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -30,10 +31,11 @@ type Module struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ModuleItem struct {
|
type ModuleItem struct {
|
||||||
Title string `json:"title"`
|
Title string `json:"title"`
|
||||||
Type string `json:"type"`
|
Type string `json:"type"`
|
||||||
URL string `json:"url"`
|
URL string `json:"url"`
|
||||||
HTMLURL string `json:"html_url"`
|
HTMLURL string `json:"html_url"`
|
||||||
ContentID int `json:"content_id"`
|
ExternalURL string `json:"external_url"`
|
||||||
Indent int `json:"indent"`
|
ContentID int `json:"content_id"`
|
||||||
|
Indent int `json:"indent"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,6 +17,27 @@ import (
|
|||||||
"git.directme.in/Joren/CanvasArchiver/internal/utils"
|
"git.directme.in/Joren/CanvasArchiver/internal/utils"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// normalizePanoptoURL converts the query-param form that Canvas stores
|
||||||
|
// (List.aspx?folderID=X) to the fragment form that yt-dlp's PanoptoList
|
||||||
|
// extractor understands (List.aspx#folderID="X"). Without this yt-dlp
|
||||||
|
// ignores the folder filter and downloads the entire Panopto instance.
|
||||||
|
func normalizePanoptoURL(rawURL string) string {
|
||||||
|
parsed, err := url.Parse(rawURL)
|
||||||
|
if err != nil {
|
||||||
|
return rawURL
|
||||||
|
}
|
||||||
|
if strings.Contains(parsed.Path, "List.aspx") {
|
||||||
|
folderID := parsed.Query().Get("folderID")
|
||||||
|
if folderID != "" {
|
||||||
|
// Strip query, set fragment: List.aspx#folderID="<id>"
|
||||||
|
parsed.RawQuery = ""
|
||||||
|
parsed.Fragment = fmt.Sprintf(`folderID="%s"`, folderID)
|
||||||
|
return parsed.String()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rawURL
|
||||||
|
}
|
||||||
|
|
||||||
func getYoutubeDLCommand() string {
|
func getYoutubeDLCommand() string {
|
||||||
exePath, err := os.Executable()
|
exePath, err := os.Executable()
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -45,12 +66,13 @@ func DownloadVideo(httpClient *http.Client, accessToken, courseID, modDir, input
|
|||||||
}
|
}
|
||||||
|
|
||||||
var launchURL string
|
var launchURL string
|
||||||
|
isDirectLink := false
|
||||||
|
|
||||||
if strings.Contains(inputURL, "/api/v1/") {
|
if strings.Contains(inputURL, "/api/v1/") {
|
||||||
launchURL = inputURL
|
launchURL = inputURL
|
||||||
} else if strings.Contains(inputURL, "panopto.eu") {
|
|
||||||
launchURL = fmt.Sprintf("%s/api/v1/courses/%s/external_tools/sessionless_launch?url=%s",
|
|
||||||
config.BaseURL, courseID, url.QueryEscape(inputURL))
|
|
||||||
} else {
|
} else {
|
||||||
|
|
||||||
|
isDirectLink = true
|
||||||
launchURL = fmt.Sprintf("%s/api/v1/courses/%s/external_tools/sessionless_launch?id=%s&launch_type=course_navigation",
|
launchURL = fmt.Sprintf("%s/api/v1/courses/%s/external_tools/sessionless_launch?id=%s&launch_type=course_navigation",
|
||||||
config.BaseURL, courseID, config.PanoptoID)
|
config.BaseURL, courseID, config.PanoptoID)
|
||||||
}
|
}
|
||||||
@@ -70,6 +92,7 @@ func DownloadVideo(httpClient *http.Client, accessToken, courseID, modDir, input
|
|||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
|
|
||||||
if launchData.URL == "" {
|
if launchData.URL == "" {
|
||||||
|
fmt.Printf(" [!] No launch URL found (Video skipped)\n")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -84,17 +107,21 @@ func DownloadVideo(httpClient *http.Client, accessToken, courseID, modDir, input
|
|||||||
json.NewDecoder(bResp.Body).Decode(&bridgeData)
|
json.NewDecoder(bResp.Body).Decode(&bridgeData)
|
||||||
bResp.Body.Close()
|
bResp.Body.Close()
|
||||||
|
|
||||||
formReq, _ := http.NewRequest("GET", bridgeData.SessionURL, nil)
|
formResp, err := httpClient.Get(bridgeData.SessionURL)
|
||||||
formReq.Header.Set("User-Agent", config.UserAgent)
|
|
||||||
formResp, err := httpClient.Do(formReq)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
formHTML, _ := io.ReadAll(formResp.Body)
|
formHTMLBytes, _ := io.ReadAll(formResp.Body)
|
||||||
formResp.Body.Close()
|
formResp.Body.Close()
|
||||||
|
formHTML := string(formHTMLBytes)
|
||||||
|
|
||||||
action := utils.ResolveAction(bridgeData.SessionURL, string(formHTML))
|
if strings.Contains(formHTML, "U hebt geen toegang") || strings.Contains(formHTML, "You do not have access") {
|
||||||
formData := utils.ExtractFormFields(string(formHTML))
|
fmt.Printf(" [!] Access denied by Panopto (U hebt geen toegang). Skipping.\n")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
action := utils.ResolveAction(bridgeData.SessionURL, formHTML)
|
||||||
|
formData := utils.ExtractFormFields(formHTML)
|
||||||
|
|
||||||
pReq, _ := http.NewRequest("POST", action, strings.NewReader(formData.Encode()))
|
pReq, _ := http.NewRequest("POST", action, strings.NewReader(formData.Encode()))
|
||||||
pReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
pReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||||
@@ -150,7 +177,31 @@ func DownloadVideo(httpClient *http.Client, accessToken, courseID, modDir, input
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// This is for making sure yt-dlp does not auto-start downloading all videos, when access to a hyperlink is denied
|
||||||
if finalURL != "" && !strings.Contains(finalURL, "NonFatalError") {
|
if finalURL != "" && !strings.Contains(finalURL, "NonFatalError") {
|
||||||
|
|
||||||
|
targetURL := finalURL
|
||||||
|
if isDirectLink {
|
||||||
|
targetURL = inputURL
|
||||||
|
|
||||||
|
checkReq, _ := http.NewRequest("GET", targetURL, nil)
|
||||||
|
checkReq.Header.Set("User-Agent", config.UserAgent)
|
||||||
|
|
||||||
|
checkResp, err := panoptoClient.Do(checkReq)
|
||||||
|
if err == nil {
|
||||||
|
checkResp.Body.Close()
|
||||||
|
|
||||||
|
if checkResp.StatusCode == http.StatusFound || checkResp.StatusCode == http.StatusSeeOther {
|
||||||
|
loc, _ := checkResp.Location()
|
||||||
|
if loc != nil && (strings.Contains(loc.String(), "Login.aspx") || strings.Contains(loc.String(), "Auth")) {
|
||||||
|
fmt.Printf(" [!] Video inaccessible (redirects to Login). Skipping to prevent mass download.\n")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
cookieFile := filepath.Join(modDir, ".cookies_temp.txt")
|
cookieFile := filepath.Join(modDir, ".cookies_temp.txt")
|
||||||
cData := "# Netscape HTTP Cookie File\n"
|
cData := "# Netscape HTTP Cookie File\n"
|
||||||
panoptoDomain, _ := url.Parse("https://vub.cloud.panopto.eu")
|
panoptoDomain, _ := url.Parse("https://vub.cloud.panopto.eu")
|
||||||
@@ -163,13 +214,35 @@ func DownloadVideo(httpClient *http.Client, accessToken, courseID, modDir, input
|
|||||||
|
|
||||||
ytCmd := getYoutubeDLCommand()
|
ytCmd := getYoutubeDLCommand()
|
||||||
|
|
||||||
cmd := exec.Command(ytCmd,
|
// Normalize folder URLs so yt-dlp scopes to the right folder.
|
||||||
"--no-playlist",
|
normalizedURL := normalizePanoptoURL(targetURL)
|
||||||
"--cookies", cookieFile,
|
|
||||||
"--referer", config.BaseURL+"/",
|
// Folder/list URLs are intentional playlists; don't pass --no-playlist.
|
||||||
"-P", modDir,
|
isList := strings.Contains(normalizedURL, "List.aspx")
|
||||||
"-o", utils.Sanitize(title)+".%(ext)s",
|
var outputTpl string
|
||||||
finalURL)
|
var args []string
|
||||||
|
if isList {
|
||||||
|
outputTpl = utils.Sanitize(title) + "/%(title)s.%(ext)s"
|
||||||
|
args = []string{
|
||||||
|
"--cookies", cookieFile,
|
||||||
|
"--referer", config.BaseURL + "/",
|
||||||
|
"-P", modDir,
|
||||||
|
"-o", outputTpl,
|
||||||
|
normalizedURL,
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
outputTpl = utils.Sanitize(title) + ".%(ext)s"
|
||||||
|
args = []string{
|
||||||
|
"--no-playlist",
|
||||||
|
"--cookies", cookieFile,
|
||||||
|
"--referer", config.BaseURL + "/",
|
||||||
|
"-P", modDir,
|
||||||
|
"-o", outputTpl,
|
||||||
|
normalizedURL,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cmd := exec.Command(ytCmd, args...)
|
||||||
|
|
||||||
cmd.Stdout = os.Stdout
|
cmd.Stdout = os.Stdout
|
||||||
cmd.Stderr = os.Stderr
|
cmd.Stderr = os.Stderr
|
||||||
if err := cmd.Run(); err != nil {
|
if err := cmd.Run(); err != nil {
|
||||||
@@ -182,7 +255,7 @@ func DownloadVideo(httpClient *http.Client, accessToken, courseID, modDir, input
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func DownloadMainRecordings(httpClient *http.Client, accessToken, courseID, root string) {
|
func DownloadMainRecordings(httpClient *http.Client, accessToken, courseID, root string, videosOnly bool) {
|
||||||
fmt.Println("\n[*] Checking for main Panopto recordings...")
|
fmt.Println("\n[*] Checking for main Panopto recordings...")
|
||||||
|
|
||||||
jar, _ := cookiejar.New(nil)
|
jar, _ := cookiejar.New(nil)
|
||||||
@@ -270,8 +343,18 @@ func DownloadMainRecordings(httpClient *http.Client, accessToken, courseID, root
|
|||||||
}
|
}
|
||||||
os.WriteFile(cookieFile, []byte(cData), 0o644)
|
os.WriteFile(cookieFile, []byte(cData), 0o644)
|
||||||
|
|
||||||
recordingsDir := filepath.Join(root, "Recordings")
|
var downloadDir string
|
||||||
os.MkdirAll(recordingsDir, 0o755)
|
var outputTemplate string
|
||||||
|
|
||||||
|
if videosOnly {
|
||||||
|
// Flat: all videos go directly into course root
|
||||||
|
downloadDir = root
|
||||||
|
outputTemplate = "%(title)s.%(ext)s"
|
||||||
|
} else {
|
||||||
|
downloadDir = filepath.Join(root, "Recordings")
|
||||||
|
os.MkdirAll(downloadDir, 0o755)
|
||||||
|
outputTemplate = "%(playlist_title)s/%(title)s.%(ext)s"
|
||||||
|
}
|
||||||
|
|
||||||
fmt.Println("[*] Starting yt-dlp download for main recordings...")
|
fmt.Println("[*] Starting yt-dlp download for main recordings...")
|
||||||
|
|
||||||
@@ -280,8 +363,8 @@ func DownloadMainRecordings(httpClient *http.Client, accessToken, courseID, root
|
|||||||
cmd := exec.Command(ytCmd,
|
cmd := exec.Command(ytCmd,
|
||||||
"--cookies", cookieFile,
|
"--cookies", cookieFile,
|
||||||
"--referer", config.BaseURL+"/",
|
"--referer", config.BaseURL+"/",
|
||||||
"-P", recordingsDir,
|
"-P", downloadDir,
|
||||||
"-o", "%(playlist_title)s/%(title)s.%(ext)s",
|
"-o", outputTemplate,
|
||||||
decodedURL,
|
decodedURL,
|
||||||
)
|
)
|
||||||
cmd.Stdout = os.Stdout
|
cmd.Stdout = os.Stdout
|
||||||
@@ -293,3 +376,4 @@ func DownloadMainRecordings(httpClient *http.Client, accessToken, courseID, root
|
|||||||
fmt.Println("[!] No main recordings available or handshake failed")
|
fmt.Println("[!] No main recordings available or handshake failed")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user