feat: add local image downloading functionality

- Add --download-images flag to download images locally with posts
- Add --image-quality flag with high/medium/low options (1456px/848px/424px)
- Add --images-dir flag to customize image directory name
- Support all Substack CDN patterns (substackcdn.com, substack-post-media.s3.amazonaws.com, legacy bucketeer)
- Automatically update HTML/Markdown content to reference local image paths
- Create organized directory structure: {output}/images/{post-slug}/
- Generate filesystem-safe filenames while preserving uniqueness
- Graceful error handling for individual image download failures
- Comprehensive test suite with real Substack HTML integration tests
- Full backwards compatibility maintained

BREAKING CHANGE: None - all existing functionality preserved

Author: alexferrari88

CLAUDE.md

@@ -50,6 +50,13 @@ go mod download
 - Converts HTML to Markdown/Text using external libraries
 - Handles file writing with different formats
 
+### Image Downloader (`lib/images.go`)
+- Downloads images locally from Substack posts
+- Supports multiple image quality levels (high/medium/low)
+- Handles various Substack CDN URL patterns
+- Updates HTML/Markdown content to reference local image paths
+- Creates organized directory structure for downloaded images
+
 ### Commands Structure
 Uses Cobra framework:
 - `download`: Main functionality for downloading posts
@@ -76,6 +83,18 @@ go run . download --url https://example.substack.com --output ./downloads
 go run . download --url https://example.substack.com --verbose --dry-run
 ```
 
+### Downloading posts with images
+```bash
+# Download posts with high-quality images
+go run . download --url https://example.substack.com --download-images --image-quality high --output ./downloads
+
+# Download with medium quality images and custom images directory
+go run . download --url https://example.substack.com --download-images --image-quality medium --images-dir assets --output ./downloads
+
+# Download single post with images in markdown format
+go run . download --url https://example.substack.com/p/post-title --download-images --format md --output ./downloads
+```
+
 ### Building for release
 ```bash
 go build -ldflags="-s -w" -o sbstck-dl .

README.md

@@ -59,12 +59,15 @@ Usage:
   sbstck-dl download [flags]
 
 Flags:
-  --add-source-url      Add the original post URL at the end of the downloaded file
-  -d, --dry-run         Enable dry run
-  -f, --format string   Specify the output format (options: "html", "md", "txt" (default "html")
-  -h, --help            help for download
-  -o, --output string   Specify the download directory (default ".")
-  -u, --url string      Specify the Substack url
+      --add-source-url         Add the original post URL at the end of the downloaded file
+      --download-images        Download images locally and update content to reference local files
+  -d, --dry-run                Enable dry run
+  -f, --format string          Specify the output format (options: "html", "md", "txt" (default "html")
+  -h, --help                   help for download
+      --image-quality string   Image quality to download (options: "high", "medium", "low") (default "high")
+      --images-dir string      Directory name for downloaded images (default "images")
+  -o, --output string          Specify the download directory (default ".")
+  -u, --url string             Specify the Substack url
 
 Global Flags:
       --after string    Download posts published after this date (format: YYYY-MM-DD)
@@ -84,6 +87,49 @@ If you use the `--add-source-url` flag, each downloaded file will have the follo
 
 Where `POST_URL` is the canonical URL of the downloaded post. For HTML format, this will be wrapped in a small paragraph with a link.
 
+#### Downloading Images
+
+Use the `--download-images` flag to download all images from Substack posts locally. This ensures posts remain accessible even if images are deleted from Substack's CDN.
+
+**Features:**
+- Downloads images at optimal quality (high/medium/low)
+- Creates organized directory structure: `{output}/images/{post-slug}/`
+- Updates HTML/Markdown content to reference local image paths
+- Handles all Substack image formats and CDN patterns
+- Graceful error handling for individual image failures
+
+**Examples:**
+
+```bash
+# Download posts with high-quality images (default)
+sbstck-dl download --url https://example.substack.com --download-images
+
+# Download with medium quality images
+sbstck-dl download --url https://example.substack.com --download-images --image-quality medium
+
+# Download with custom images directory name
+sbstck-dl download --url https://example.substack.com --download-images --images-dir assets
+
+# Download single post with images in markdown format
+sbstck-dl download --url https://example.substack.com/p/post-title --download-images --format md
+```
+
+**Image Quality Options:**
+- `high`: 1456px width (best quality, larger files)
+- `medium`: 848px width (balanced quality/size)
+- `low`: 424px width (smaller files, mobile-optimized)
+
+**Directory Structure:**
+```
+output/
+├── 20231201_120000_post-title.html
+└── images/
+    └── post-title/
+        ├── image1_1456x819.jpeg
+        ├── image2_848x636.png
+        └── image3_1272x720.webp
+```
+
 ### Listing posts
 
 ```bash
@@ -125,7 +171,7 @@ sbstck-dl download --url https://example.substack.com --cookie_name substack.sid
 
 - [x] Improve retry logic
 - [ ] Implement loading from config file
-- [ ] Add support for downloading media
+- [x] Add support for downloading images
 - [x] Add tests
 - [x] Add CI
 - [x] Add documentation

cmd/download.go

@@ -15,12 +15,15 @@ import (
 
 // downloadCmd represents the download command
 var (
-	downloadUrl  string
-	format       string
-	outputFolder string
-	dryRun       bool
-	addSourceURL bool
-	downloadCmd  = &cobra.Command{
+	downloadUrl    string
+	format         string
+	outputFolder   string
+	dryRun         bool
+	addSourceURL   bool
+	downloadImages bool
+	imageQuality   string
+	imagesDir      string
+	downloadCmd    = &cobra.Command{
 		Use:   "download",
 		Short: "Download individual posts or the entire public archive",
 		Long:  `You can provide the url of a single post or the main url of the Substack you want to download.`,
@@ -54,9 +57,19 @@ var (
 					fmt.Printf("Writing post to file %s\n", path)
 				}
 
-				err = post.WriteToFile(path, format, addSourceURL)
-				if err != nil {
-					log.Printf("Error writing file %s: %v\n", path, err)
+				if downloadImages {
+					imageQualityEnum := lib.ImageQuality(imageQuality)
+					imageResult, err := post.WriteToFileWithImages(ctx, path, format, addSourceURL, downloadImages, imageQualityEnum, imagesDir, fetcher)
+					if err != nil {
+						log.Printf("Error writing file %s: %v\n", path, err)
+					} else if verbose && imageResult.Success > 0 {
+						fmt.Printf("Downloaded %d images (%d failed) for post %s\n", imageResult.Success, imageResult.Failed, post.Slug)
+					}
+				} else {
+					err = post.WriteToFile(path, format, addSourceURL)
+					if err != nil {
+						log.Printf("Error writing file %s: %v\n", path, err)
+					}
 				}
 
 				if verbose {
@@ -126,9 +139,19 @@ var (
 						fmt.Printf("Writing post to file %s\n", path)
 					}
 
-					err = post.WriteToFile(path, format, addSourceURL)
-					if err != nil {
-						log.Printf("Error writing file %s: %v\n", path, err)
+					if downloadImages {
+						imageQualityEnum := lib.ImageQuality(imageQuality)
+						imageResult, err := post.WriteToFileWithImages(ctx, path, format, addSourceURL, downloadImages, imageQualityEnum, imagesDir, fetcher)
+						if err != nil {
+							log.Printf("Error writing file %s: %v\n", path, err)
+						} else if verbose && imageResult.Success > 0 {
+							fmt.Printf("Downloaded %d images (%d failed) for post %s\n", imageResult.Success, imageResult.Failed, post.Slug)
+						}
+					} else {
+						err = post.WriteToFile(path, format, addSourceURL)
+						if err != nil {
+							log.Printf("Error writing file %s: %v\n", path, err)
+						}
 					}
 				}
 				if verbose {
@@ -146,6 +169,9 @@ func init() {
 	downloadCmd.Flags().StringVarP(&outputFolder, "output", "o", ".", "Specify the download directory")
 	downloadCmd.Flags().BoolVarP(&dryRun, "dry-run", "d", false, "Enable dry run")
 	downloadCmd.Flags().BoolVar(&addSourceURL, "add-source-url", false, "Add the original post URL at the end of the downloaded file")
+	downloadCmd.Flags().BoolVar(&downloadImages, "download-images", false, "Download images locally and update content to reference local files")
+	downloadCmd.Flags().StringVar(&imageQuality, "image-quality", "high", "Image quality to download (options: \"high\", \"medium\", \"low\")")
+	downloadCmd.Flags().StringVar(&imagesDir, "images-dir", "images", "Directory name for downloaded images")
 	downloadCmd.MarkFlagRequired("url")
 }
 

cmd/version.go

@@ -12,7 +12,7 @@ var versionCmd = &cobra.Command{
 	Short: "Print the version number of sbstck-dl",
 	Long:  `Display the current version of the app.`,
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("sbstck-dl v0.4.0")
+		fmt.Println("sbstck-dl v0.5.0")
 	},
 }
 

lib/extractor.go

@@ -127,6 +127,96 @@ func (p *Post) WriteToFile(path string, format string, addSourceURL bool) error
 	return os.WriteFile(path, []byte(content), 0644)
 }
 
+// WriteToFileWithImages writes the Post's content to a file with optional image downloading
+func (p *Post) WriteToFileWithImages(ctx context.Context, path string, format string, addSourceURL bool, 
+	downloadImages bool, imageQuality ImageQuality, imagesDir string, fetcher *Fetcher) (*ImageDownloadResult, error) {
+	
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return nil, err
+	}
+
+	content, err := p.contentForFormat(format, true)
+	if err != nil {
+		return nil, err
+	}
+
+	var imageResult *ImageDownloadResult
+
+	// Download images if requested and format supports it
+	if downloadImages && (format == "html" || format == "md") {
+		outputDir := filepath.Dir(path)
+		imageDownloader := NewImageDownloader(fetcher, outputDir, imagesDir, imageQuality)
+		
+		// Only process HTML content for image downloading
+		htmlContent := content
+		if format == "md" {
+			// For markdown, we need to work with the original HTML
+			htmlContent = p.BodyHTML
+		}
+		
+		imageResult, err = imageDownloader.DownloadImages(ctx, htmlContent, p.Slug)
+		if err != nil {
+			return nil, fmt.Errorf("failed to download images: %w", err)
+		}
+
+		// Update content based on format
+		if format == "html" {
+			content = imageResult.UpdatedHTML
+			// Re-add title if needed
+			if strings.HasPrefix(content, "<h1>") {
+				// Title already included
+			} else {
+				content = fmt.Sprintf("<h1>%s</h1>\n\n%s", p.Title, imageResult.UpdatedHTML)
+			}
+		} else if format == "md" {
+			// Convert updated HTML to markdown
+			updatedContent, err := mdConverter.ConvertString(imageResult.UpdatedHTML)
+			if err != nil {
+				return nil, fmt.Errorf("failed to convert updated HTML to markdown: %w", err)
+			}
+			content = fmt.Sprintf("# %s\n\n%s", p.Title, updatedContent)
+		}
+	} else if downloadImages && format == "txt" {
+		// For text format, we can't embed images, but we can still download them
+		outputDir := filepath.Dir(path)
+		imageDownloader := NewImageDownloader(fetcher, outputDir, imagesDir, imageQuality)
+		
+		imageResult, err = imageDownloader.DownloadImages(ctx, p.BodyHTML, p.Slug)
+		if err != nil {
+			return nil, fmt.Errorf("failed to download images: %w", err)
+		}
+		// Keep original text content since we can't embed images in text format
+	}
+
+	// Add source URL if requested
+	if addSourceURL && p.CanonicalUrl != "" {
+		sourceLine := fmt.Sprintf("\n\noriginal content: %s", p.CanonicalUrl)
+
+		// Adjust formatting slightly for HTML
+		if format == "html" {
+			sourceLine = fmt.Sprintf("<p style=\"margin-top: 2em; font-size: small; color: grey;\">original content: <a href=\"%s\">%s</a></p>", p.CanonicalUrl, p.CanonicalUrl)
+		}
+		content += sourceLine
+	}
+
+	// Write the file
+	if err := os.WriteFile(path, []byte(content), 0644); err != nil {
+		return imageResult, err
+	}
+
+	// Return empty result if no image downloading was performed
+	if imageResult == nil {
+		imageResult = &ImageDownloadResult{
+			Images:      []ImageInfo{},
+			UpdatedHTML: content,
+			Success:     0,
+			Failed:      0,
+		}
+	}
+
+	return imageResult, nil
+}
+
 // PostWrapper wraps a Post object for JSON unmarshaling.
 type PostWrapper struct {
 	Post Post `json:"post"`