-
Notifications
You must be signed in to change notification settings - Fork 66
Granular object type filtering for pg to pg snapshot and streaming #735
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
tsg
wants to merge
5
commits into
main
Choose a base branch
from
object_type_filtering
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
5 commits
Select commit
Hold shift + click to select a range
16d3d20
Granular object type filtering for pg to pg snapshot and streaming
tsg ab6b294
Tests fix
tsg e4494a2
Merge branch 'main' into object_type_filtering
tsg e979d8f
Merge branch 'main' into object_type_filtering
tsg 946548c
Merge branch 'main' into object_type_filtering
tsg File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
193 changes: 193 additions & 0 deletions
193
pkg/snapshot/generator/postgres/schema/pgdumprestore/object_type_filter.go
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,193 @@ | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| package pgdumprestore | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "bytes" | ||
| "fmt" | ||
| "regexp" | ||
| "strings" | ||
| ) | ||
|
|
||
| // objectTypeCategory maps user-facing category names to pg_dump TOC Type values. | ||
| var objectTypeCategories = map[string][]string{ | ||
| "tables": {"TABLE", "DEFAULT"}, | ||
| "sequences": {"SEQUENCE", "SEQUENCE OWNED BY"}, | ||
| "types": {"TYPE", "DOMAIN"}, | ||
| "indexes": {"INDEX"}, | ||
| "constraints": {"CONSTRAINT", "FK CONSTRAINT"}, | ||
| "functions": {"FUNCTION", "AGGREGATE", "PROCEDURE"}, | ||
| "views": {"VIEW"}, | ||
| "materialized_views": {"MATERIALIZED VIEW"}, | ||
| "triggers": {"TRIGGER"}, | ||
| "event_triggers": {"EVENT TRIGGER"}, | ||
| "policies": {"POLICY", "ROW SECURITY"}, | ||
| "rules": {"RULE"}, | ||
| "comments": {"COMMENT"}, | ||
| "extensions": {"EXTENSION"}, | ||
| "collations": {"COLLATION"}, | ||
| "text_search": {"TEXT SEARCH CONFIGURATION", "TEXT SEARCH DICTIONARY", "TEXT SEARCH PARSER", "TEXT SEARCH TEMPLATE"}, | ||
| } | ||
|
|
||
| // tocHeaderRegex matches pg_dump TOC comment headers like: | ||
| // -- Name: my_func; Type: FUNCTION; Schema: public; Owner: postgres | ||
| var tocHeaderRegex = regexp.MustCompile(`^--\s*Name:.*;\s*Type:\s*([^;]+)\s*;`) | ||
|
|
||
| // parseTOCHeader extracts the object Type value from a pg_dump TOC comment header line. | ||
| // Returns the type string and true if the line is a TOC header, or ("", false) otherwise. | ||
| func parseTOCHeader(line string) (string, bool) { | ||
| matches := tocHeaderRegex.FindStringSubmatch(line) | ||
| if len(matches) < 2 { | ||
| return "", false | ||
| } | ||
| return strings.TrimSpace(matches[1]), true | ||
| } | ||
|
|
||
| // objectTypeFilter determines which pg_dump object types should be excluded | ||
| // based on user-specified include or exclude category lists. | ||
| type objectTypeFilter struct { | ||
| excludedTypes map[string]struct{} | ||
| // categories tracks which user-facing categories are excluded for | ||
| // higher-level checks (e.g., skipping sequence dump step). | ||
| excludedCategories map[string]struct{} | ||
| } | ||
|
|
||
| // newObjectTypeFilter creates an objectTypeFilter from include/exclude category lists. | ||
| // Only one of include or exclude can be set (not both). | ||
| // Returns nil if neither is set (no filtering). | ||
| func newObjectTypeFilter(include, exclude []string) (*objectTypeFilter, error) { | ||
| if len(include) > 0 && len(exclude) > 0 { | ||
| return nil, fmt.Errorf("include_object_types and exclude_object_types cannot both be set") | ||
| } | ||
|
|
||
| if len(include) == 0 && len(exclude) == 0 { | ||
| return nil, nil | ||
| } | ||
|
|
||
| f := &objectTypeFilter{ | ||
| excludedTypes: make(map[string]struct{}), | ||
| excludedCategories: make(map[string]struct{}), | ||
| } | ||
|
|
||
| if len(include) > 0 { | ||
| // Validate all included categories | ||
| includedSet := make(map[string]struct{}, len(include)) | ||
| for _, cat := range include { | ||
| if _, ok := objectTypeCategories[cat]; !ok { | ||
| return nil, fmt.Errorf("unknown object type category: %q", cat) | ||
| } | ||
| includedSet[cat] = struct{}{} | ||
| } | ||
| // Exclude everything NOT in the include list | ||
| for cat, types := range objectTypeCategories { | ||
| if _, included := includedSet[cat]; !included { | ||
| f.excludedCategories[cat] = struct{}{} | ||
| for _, t := range types { | ||
| f.excludedTypes[t] = struct{}{} | ||
| } | ||
| } | ||
| } | ||
| } else { | ||
| // Validate and exclude the specified categories | ||
| for _, cat := range exclude { | ||
| types, ok := objectTypeCategories[cat] | ||
| if !ok { | ||
| return nil, fmt.Errorf("unknown object type category: %q", cat) | ||
| } | ||
| f.excludedCategories[cat] = struct{}{} | ||
| for _, t := range types { | ||
| f.excludedTypes[t] = struct{}{} | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return f, nil | ||
| } | ||
|
|
||
| // isExcluded returns true if the given pg_dump Type value should be excluded. | ||
| // SCHEMA type is never excluded (required for namespace resolution). | ||
| func (f *objectTypeFilter) isExcluded(pgdumpType string) bool { | ||
| if f == nil { | ||
| return false | ||
| } | ||
| // SCHEMA is always included | ||
| if pgdumpType == "SCHEMA" { | ||
| return false | ||
| } | ||
| _, excluded := f.excludedTypes[pgdumpType] | ||
| return excluded | ||
| } | ||
|
|
||
| // isCategoryExcluded returns true if the given user-facing category is excluded. | ||
| func (f *objectTypeFilter) isCategoryExcluded(category string) bool { | ||
| if f == nil { | ||
| return false | ||
| } | ||
| _, excluded := f.excludedCategories[category] | ||
| return excluded | ||
| } | ||
|
|
||
| // cleanupStatementPrefixes maps SQL cleanup statement prefixes (from pg_dump | ||
| // --clean --if-exists output) to the object type category they belong to. | ||
| var cleanupStatementPrefixes = map[string]string{ | ||
| "DROP POLICY": "policies", | ||
| "DROP TRIGGER": "triggers", | ||
| "DROP RULE": "rules", | ||
| "DROP INDEX": "indexes", | ||
| "DROP FUNCTION": "functions", | ||
| "DROP AGGREGATE": "functions", | ||
| "DROP PROCEDURE": "functions", | ||
| "DROP VIEW": "views", | ||
| "DROP MATERIALIZED VIEW": "materialized_views", | ||
| "DROP TEXT SEARCH": "text_search", | ||
| "DROP COLLATION": "collations", | ||
| "DROP EXTENSION": "extensions", | ||
| "DROP EVENT TRIGGER": "event_triggers", | ||
| "DROP SEQUENCE": "sequences", | ||
| "DROP TABLE": "tables", | ||
| "DROP TYPE": "types", | ||
| "DROP DOMAIN": "types", | ||
| "DROP SCHEMA": "schemas", | ||
| "COMMENT ON": "comments", | ||
| } | ||
|
|
||
| // filterCleanupDump removes lines from a cleanup dump that belong to excluded | ||
| // object type categories. This prevents errors like "relation does not exist" | ||
| // when DROP POLICY/TRIGGER/RULE statements reference tables that don't yet | ||
| // exist on the target. | ||
| func (f *objectTypeFilter) filterCleanupDump(cleanupDump []byte) []byte { | ||
| if f == nil { | ||
| return cleanupDump | ||
| } | ||
|
|
||
| scanner := bufio.NewScanner(bytes.NewReader(cleanupDump)) | ||
| var filtered strings.Builder | ||
| for scanner.Scan() { | ||
| line := scanner.Text() | ||
| if f.shouldSkipCleanupLine(line) { | ||
| continue | ||
| } | ||
| filtered.WriteString(line) | ||
| filtered.WriteString("\n") | ||
| } | ||
| return []byte(filtered.String()) | ||
| } | ||
|
|
||
| // shouldSkipCleanupLine returns true if a cleanup dump line should be skipped | ||
| // because it belongs to an excluded object type category. | ||
| func (f *objectTypeFilter) shouldSkipCleanupLine(line string) bool { | ||
| if f == nil { | ||
| return false | ||
| } | ||
| for prefix, cat := range cleanupStatementPrefixes { | ||
| if strings.HasPrefix(line, prefix) { | ||
| // SCHEMA is never excluded | ||
| if cat == "schemas" { | ||
| return false | ||
| } | ||
| return f.isCategoryExcluded(cat) | ||
| } | ||
| } | ||
| return false | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This logic is identical to the stuff in
ddl_object_type_filter.go, maybe extract shared logic to a generic helper.