Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions adapters/folder/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,9 @@ func (ifc *ImportFolderCmd) run(cmd *cobra.Command, args []string, app *app.Appl
const icloudMetadataExt = ".csv"

func (ifc *ImportFolderCmd) Browse(ctx context.Context) chan *assets.Group {
gOut := make(chan *assets.Group)
// Buffered channel to prevent scanner goroutines from blocking on slow consumers
// Buffer size of 10000 allows scanning to complete independently of upload speed
gOut := make(chan *assets.Group, 10000)
go func() {
defer func() {
close(gOut)
Expand All @@ -126,7 +128,8 @@ func (ifc *ImportFolderCmd) Browse(ctx context.Context) chan *assets.Group {
func (ifc *ImportFolderCmd) concurrentParseDir(ctx context.Context, fsys fs.FS, dir string, gOut chan *assets.Group) {
ifc.wg.Add(1)
ctx, cancel := context.WithCancelCause(ctx)
go ifc.pool.Submit(func() {
// Submit directly - pool.Submit() already handles goroutines internally
ifc.pool.Submit(func() {
defer ifc.wg.Done()
err := ifc.parseDir(ctx, fsys, dir, gOut)
if err != nil {
Expand Down
80 changes: 37 additions & 43 deletions internal/worker/worker.go
Original file line number Diff line number Diff line change
@@ -1,59 +1,53 @@
package worker

import (
"sync"
)

// Task represents a unit of work to be processed by the worker pool.
type Task func()

// Pool manages a pool of worker goroutines.
type Pool struct {
tasks chan Task
wg sync.WaitGroup
quit chan struct{}
closed bool
// Semaphore limits concurrent execution using a buffered channel.
// Unlike a worker pool, it allows unlimited goroutines but only N run concurrently.
// This prevents deadlock in recursive directory scanning scenarios.
type Semaphore struct {
sem chan struct{}
}

// NewPool creates a new Pool with a specified number of workers.
func NewPool(numWorkers int) *Pool {
pool := &Pool{
tasks: make(chan Task),
quit: make(chan struct{}),
// NewSemaphore creates a new Semaphore with the specified concurrency limit.
func NewSemaphore(limit int) *Semaphore {
return &Semaphore{
sem: make(chan struct{}, limit),
}
}

for i := 0; i < numWorkers; i++ {
pool.wg.Add(1)
go pool.worker()
}
// Acquire acquires a semaphore slot, blocking if limit is reached.
func (s *Semaphore) Acquire() {
s.sem <- struct{}{}
}

// Release releases a semaphore slot, allowing another goroutine to proceed.
func (s *Semaphore) Release() {
<-s.sem
}

return pool
// Pool is kept for backward compatibility but now uses Semaphore internally.
type Pool struct {
semaphore *Semaphore
}

// worker is the function that each worker goroutine runs.
func (p *Pool) worker() {
defer p.wg.Done()
for {
select {
case task := <-p.tasks:
task()
case <-p.quit:
return
}
// NewPool creates a new Pool that uses a Semaphore for concurrency control.
func NewPool(numWorkers int) *Pool {
return &Pool{
semaphore: NewSemaphore(numWorkers),
}
}

// Submit adds a task to the worker pool.
func (p *Pool) Submit(task Task) {
p.tasks <- task
// Submit executes a task with semaphore-based concurrency control.
// The task runs in its own goroutine but is limited by the semaphore.
func (p *Pool) Submit(task func()) {
go func() {
p.semaphore.Acquire()
defer p.semaphore.Release()
task()
}()
}

// Stop stops all the workers and waits for them to finish.
// Stop is a no-op for backward compatibility.
// With semaphore approach, there are no workers to stop.
func (p *Pool) Stop() {
if !p.closed {
close(p.quit)
p.wg.Wait()
close(p.tasks)
p.closed = true
}
// No-op: semaphore doesn't need cleanup
}