Skip to content

Commit 76c29ee

Browse files
authored
Add SQLite storage implementation with SQLC (#192)
* Implement SQLite with SQLC * Use in tests and fix * Create AdditionalQueries to enable SQL filtering * Add golang-migrate * Add storage-migrate command * Remove TODOs * Fix linting * Update integration test to use SQL * Specify time in args to ListActiveGardens * Add docker test task * Update DATETIME handling
1 parent 4382bc4 commit 76c29ee

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+3428
-196
lines changed

Taskfile.yml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,19 @@ tasks:
1919
cmds:
2020
- go test -short -race -covermode=atomic -coverprofile=coverage.out -coverpkg=./... {{.CLI_ARGS}} ./...
2121

22+
unit-test-docker:
23+
aliases: ["utd"]
24+
desc: Run unit tests for Go app in Ubuntu Docker container
25+
dir: ./garden-app
26+
cmds:
27+
- |
28+
docker run --rm \
29+
-v $(pwd):/workspace \
30+
-v go-mod-cache:/go/pkg/mod \
31+
-w /workspace \
32+
golang:latest \
33+
go test -short -race -covermode=atomic -coverprofile=coverage.out -coverpkg=./... {{.CLI_ARGS}} ./...
34+
2235
lint:
2336
desc: Run linting for Go app
2437
dir: ./garden-app

deploy/docker-compose.yml

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -99,14 +99,6 @@ services:
9999
profiles:
100100
- demo
101101

102-
redis:
103-
image: redis
104-
ports:
105-
- "6379:6379"
106-
profiles:
107-
- test
108-
- run-local
109-
110102
volumes:
111103
influxdb:
112104
grafana:

garden-app/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ coverage.out
66
integration_coverage.out
77
cover.html
88
gardens_kv.yaml
9+
garden.db

garden-app/cmd/root.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ func Execute() {
2727

2828
command.AddCommand(migrateCommand)
2929

30+
command.AddCommand(storageMigrateCommand)
31+
3032
viper.SetEnvPrefix("GARDEN_APP")
3133
viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
3234
viper.AutomaticEnv()

garden-app/cmd/storage_migrate.go

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
package cmd
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"log/slog"
7+
8+
"github.com/calvinmclean/automated-garden/garden-app/pkg/storage"
9+
"github.com/calvinmclean/babyapi"
10+
"github.com/spf13/cobra"
11+
)
12+
13+
var storageMigrateCommand = &cobra.Command{
14+
Use: "storage-migrate",
15+
Short: "Migrate all resources from KV storage to SQL storage",
16+
RunE: func(cmd *cobra.Command, _ []string) error {
17+
slog.Info("Starting storage migration from KV to SQL")
18+
19+
// Parse source (KV) storage config
20+
sourceDriver, _ := cmd.Flags().GetString("source-driver")
21+
sourceDSN, _ := cmd.Flags().GetString("source-dsn")
22+
sourceFilename, _ := cmd.Flags().GetString("source-filename")
23+
24+
sourceConfig := storage.Config{
25+
Driver: sourceDriver,
26+
Options: map[string]any{},
27+
}
28+
29+
if sourceDSN != "" {
30+
sourceConfig.Options["data_source_name"] = sourceDSN
31+
}
32+
if sourceFilename != "" {
33+
sourceConfig.Options["filename"] = sourceFilename
34+
}
35+
36+
// Parse destination (SQL) storage config
37+
destDSN, _ := cmd.Flags().GetString("dest-dsn")
38+
39+
destConfig := storage.Config{
40+
Driver: "sqlite",
41+
Options: map[string]any{
42+
"data_source_name": destDSN,
43+
"disable_migrations": false,
44+
},
45+
}
46+
47+
// Create source (KV) client
48+
sourceClient, err := storage.NewClient(sourceConfig)
49+
if err != nil {
50+
return fmt.Errorf("error creating source storage client: %w", err)
51+
}
52+
53+
// Create destination (SQL) client
54+
destClient, err := storage.NewClient(destConfig)
55+
if err != nil {
56+
return fmt.Errorf("error creating destination storage client: %w", err)
57+
}
58+
59+
ctx := context.Background()
60+
61+
// Migrate Gardens
62+
slog.Info("Migrating Gardens")
63+
gardens, err := sourceClient.Gardens.Search(ctx, "", babyapi.EndDatedQueryParam(true))
64+
if err != nil {
65+
return fmt.Errorf("error getting Gardens from source: %w", err)
66+
}
67+
slog.Info(fmt.Sprintf("Found %d Gardens to migrate", len(gardens)))
68+
for _, g := range gardens {
69+
if err := destClient.Gardens.Set(ctx, g); err != nil {
70+
return fmt.Errorf("error saving Garden to destination: %w", err)
71+
}
72+
}
73+
slog.Info(fmt.Sprintf("Successfully migrated %d Gardens", len(gardens)))
74+
75+
// Migrate Zones (need to get per-garden since zones are nested under gardens)
76+
slog.Info("Migrating Zones")
77+
zoneCount := 0
78+
for _, g := range gardens {
79+
zones, err := sourceClient.Zones.Search(ctx, g.GetID(), babyapi.EndDatedQueryParam(true))
80+
if err != nil {
81+
return fmt.Errorf("error getting Zones for garden %s: %w", g.GetID(), err)
82+
}
83+
for _, z := range zones {
84+
if err := destClient.Zones.Set(ctx, z); err != nil {
85+
return fmt.Errorf("error saving Zone to destination: %w", err)
86+
}
87+
}
88+
zoneCount += len(zones)
89+
}
90+
slog.Info(fmt.Sprintf("Successfully migrated %d Zones", zoneCount))
91+
92+
// Migrate WaterSchedules
93+
slog.Info("Migrating WaterSchedules")
94+
waterSchedules, err := sourceClient.WaterSchedules.Search(ctx, "", babyapi.EndDatedQueryParam(true))
95+
if err != nil {
96+
return fmt.Errorf("error getting WaterSchedules from source: %w", err)
97+
}
98+
slog.Info(fmt.Sprintf("Found %d WaterSchedules to migrate", len(waterSchedules)))
99+
for _, ws := range waterSchedules {
100+
if err := destClient.WaterSchedules.Set(ctx, ws); err != nil {
101+
return fmt.Errorf("error saving WaterSchedule to destination: %w", err)
102+
}
103+
}
104+
slog.Info(fmt.Sprintf("Successfully migrated %d WaterSchedules", len(waterSchedules)))
105+
106+
// Migrate WeatherClientConfigs
107+
slog.Info("Migrating WeatherClientConfigs")
108+
weatherClients, err := sourceClient.WeatherClientConfigs.Search(ctx, "", babyapi.EndDatedQueryParam(true))
109+
if err != nil {
110+
return fmt.Errorf("error getting WeatherClientConfigs from source: %w", err)
111+
}
112+
slog.Info(fmt.Sprintf("Found %d WeatherClientConfigs to migrate", len(weatherClients)))
113+
for _, wc := range weatherClients {
114+
if err := destClient.WeatherClientConfigs.Set(ctx, wc); err != nil {
115+
return fmt.Errorf("error saving WeatherClientConfig to destination: %w", err)
116+
}
117+
}
118+
slog.Info(fmt.Sprintf("Successfully migrated %d WeatherClientConfigs", len(weatherClients)))
119+
120+
// Migrate NotificationClientConfigs
121+
slog.Info("Migrating NotificationClientConfigs")
122+
notificationClients, err := sourceClient.NotificationClientConfigs.Search(ctx, "", babyapi.EndDatedQueryParam(true))
123+
if err != nil {
124+
return fmt.Errorf("error getting NotificationClientConfigs from source: %w", err)
125+
}
126+
slog.Info(fmt.Sprintf("Found %d NotificationClientConfigs to migrate", len(notificationClients)))
127+
for _, nc := range notificationClients {
128+
if err := destClient.NotificationClientConfigs.Set(ctx, nc); err != nil {
129+
return fmt.Errorf("error saving NotificationClientConfig to destination: %w", err)
130+
}
131+
}
132+
slog.Info(fmt.Sprintf("Successfully migrated %d NotificationClientConfigs", len(notificationClients)))
133+
134+
// Migrate WaterRoutines
135+
slog.Info("Migrating WaterRoutines")
136+
waterRoutines, err := sourceClient.WaterRoutines.Search(ctx, "", babyapi.EndDatedQueryParam(true))
137+
if err != nil {
138+
return fmt.Errorf("error getting WaterRoutines from source: %w", err)
139+
}
140+
slog.Info(fmt.Sprintf("Found %d WaterRoutines to migrate", len(waterRoutines)))
141+
for _, wr := range waterRoutines {
142+
if err := destClient.WaterRoutines.Set(ctx, wr); err != nil {
143+
return fmt.Errorf("error saving WaterRoutine to destination: %w", err)
144+
}
145+
}
146+
slog.Info(fmt.Sprintf("Successfully migrated %d WaterRoutines", len(waterRoutines)))
147+
148+
slog.Info("Storage migration completed successfully")
149+
return nil
150+
},
151+
}
152+
153+
func init() {
154+
storageMigrateCommand.Flags().String("source-driver", "hashmap", "Source storage driver (hashmap or redis)")
155+
storageMigrateCommand.Flags().String("source-dsn", "", "Source Redis DSN (e.g., 'localhost:6379')")
156+
storageMigrateCommand.Flags().String("source-filename", "", "Source hashmap filename (e.g., 'storage.json')")
157+
storageMigrateCommand.Flags().String("dest-dsn", "garden.db", "Destination SQLite DSN")
158+
}

garden-app/go.mod

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
module github.com/calvinmclean/automated-garden/garden-app
22

3-
go 1.24
3+
go 1.24.0
44

55
//replace github.com/calvinmclean/babyapi => ../../babyapi
66

@@ -12,6 +12,7 @@ require (
1212
github.com/eclipse/paho.mqtt.golang v1.4.3
1313
github.com/go-chi/render v1.0.3
1414
github.com/go-co-op/gocron v1.35.2
15+
github.com/golang-migrate/migrate/v4 v4.19.1
1516
github.com/gregdel/pushover v1.3.0
1617
github.com/influxdata/influxdb-client-go/v2 v2.12.3
1718
github.com/mark3labs/mcp-go v0.33.0
@@ -24,12 +25,13 @@ require (
2425
github.com/slok/go-http-metrics v0.10.0
2526
github.com/spf13/cobra v1.8.1
2627
github.com/spf13/viper v1.17.0
27-
github.com/stretchr/testify v1.9.0
28+
github.com/stretchr/testify v1.10.0
2829
github.com/tarmac-project/hord v0.6.0
2930
github.com/tarmac-project/hord/drivers/hashmap v0.6.0
3031
github.com/tarmac-project/hord/drivers/redis v0.6.0
3132
gopkg.in/dnaeon/go-vcr.v4 v4.0.1
3233
gopkg.in/yaml.v3 v3.0.1
34+
modernc.org/sqlite v1.46.1
3335
)
3436

3537
require (
@@ -46,11 +48,12 @@ require (
4648
github.com/beorn7/perks v1.0.1 // indirect
4749
github.com/buger/jsonparser v1.1.1 // indirect
4850
github.com/bytedance/sonic v1.10.2 // indirect
49-
github.com/cespare/xxhash/v2 v2.2.0 // indirect
51+
github.com/cespare/xxhash/v2 v2.3.0 // indirect
5052
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
5153
github.com/chenzhuoyu/iasm v0.9.0 // indirect
5254
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
5355
github.com/deepmap/oapi-codegen v1.15.0 // indirect
56+
github.com/dustin/go-humanize v1.0.1 // indirect
5457
github.com/fatih/structs v1.1.0 // indirect
5558
github.com/flosch/pongo2/v4 v4.0.2 // indirect
5659
github.com/fsnotify/fsnotify v1.6.0 // indirect
@@ -95,25 +98,28 @@ require (
9598
github.com/mailgun/raymond/v2 v2.0.48 // indirect
9699
github.com/mailru/easyjson v0.7.7 // indirect
97100
github.com/mattn/go-colorable v0.1.13 // indirect
98-
github.com/mattn/go-isatty v0.0.19 // indirect
101+
github.com/mattn/go-isatty v0.0.20 // indirect
99102
github.com/mattn/go-runewidth v0.0.15 // indirect
103+
github.com/mattn/go-sqlite3 v1.14.34 // indirect
100104
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
101105
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
102106
github.com/microcosm-cc/bluemonday v1.0.26 // indirect
103107
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
104108
github.com/modern-go/reflect2 v1.0.2 // indirect
109+
github.com/ncruces/go-strftime v1.0.0 // indirect
105110
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
106111
github.com/pkg/errors v0.9.1 // indirect
107112
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
108113
github.com/prometheus/client_model v0.5.0 // indirect
109114
github.com/prometheus/common v0.44.0 // indirect
110115
github.com/prometheus/procfs v0.12.0 // indirect
116+
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
111117
github.com/rivo/uniseg v0.4.4 // indirect
112118
github.com/russross/blackfriday/v2 v2.1.0 // indirect
113119
github.com/sagikazarmark/locafero v0.3.0 // indirect
114120
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
115121
github.com/schollz/closestmatch v2.1.0+incompatible // indirect
116-
github.com/sirupsen/logrus v1.8.1 // indirect
122+
github.com/sirupsen/logrus v1.9.3 // indirect
117123
github.com/sourcegraph/conc v0.3.0 // indirect
118124
github.com/spf13/afero v1.10.0 // indirect
119125
github.com/spf13/cast v1.7.1 // indirect
@@ -134,14 +140,17 @@ require (
134140
go.uber.org/atomic v1.11.0 // indirect
135141
go.uber.org/multierr v1.11.0 // indirect
136142
golang.org/x/arch v0.5.0 // indirect
137-
golang.org/x/crypto v0.14.0 // indirect
138-
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect
139-
golang.org/x/net v0.17.0 // indirect
140-
golang.org/x/sync v0.7.0 // indirect
141-
golang.org/x/sys v0.14.0 // indirect
142-
golang.org/x/term v0.13.0 // indirect
143-
golang.org/x/text v0.13.0 // indirect
144-
golang.org/x/time v0.3.0 // indirect
145-
google.golang.org/protobuf v1.31.0 // indirect
143+
golang.org/x/crypto v0.45.0 // indirect
144+
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
145+
golang.org/x/net v0.47.0 // indirect
146+
golang.org/x/sync v0.18.0 // indirect
147+
golang.org/x/sys v0.38.0 // indirect
148+
golang.org/x/term v0.37.0 // indirect
149+
golang.org/x/text v0.31.0 // indirect
150+
golang.org/x/time v0.12.0 // indirect
151+
google.golang.org/protobuf v1.36.7 // indirect
146152
gopkg.in/ini.v1 v1.67.0 // indirect
153+
modernc.org/libc v1.67.6 // indirect
154+
modernc.org/mathutil v1.7.1 // indirect
155+
modernc.org/memory v1.11.0 // indirect
147156
)

0 commit comments

Comments
 (0)