-
Notifications
You must be signed in to change notification settings - Fork 12
Import Snowflake Cortex Analyst measures and enrichment keys #200
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from 10 commits
e985426
9124e4b
b7f0127
6a14c81
b19d987
f718051
6640a93
2284417
55c8517
642cde3
a322a39
b68e9e2
71d5216
7cf5af0
da115d3
87ea415
c684a99
d20f85d
5ef649c
fca4603
0ffb3a8
ef6e760
4db2970
7cb7c30
892f84e
976b382
3d47659
924caf0
7245d1a
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -33,7 +33,8 @@ pub struct SidemanticConfig { | |
| pub sql_metrics: Option<String>, | ||
| #[serde(default)] | ||
| pub sql_segments: Option<String>, | ||
| /// Graph-level metadata payload (round-trips format-specific state such as OSI). | ||
| /// Graph-level metadata payload (round-trips format-specific state such as OSI, | ||
| /// and Snowflake Cortex top-level sections from the Python native export). | ||
| #[serde(default)] | ||
| pub metadata: Option<serde_json::Value>, | ||
| } | ||
|
|
@@ -132,6 +133,15 @@ pub struct DimensionConfig { | |
| pub metadata: Option<serde_json::Value>, | ||
| #[serde(default)] | ||
| pub meta: Option<serde_json::Value>, | ||
| /// Alternative names (e.g. Snowflake Cortex Analyst, Cube). | ||
| #[serde(default)] | ||
| pub synonyms: Option<Vec<String>>, | ||
| /// Representative sample values for this dimension. | ||
| #[serde(default)] | ||
| pub sample_values: Option<Vec<String>>, | ||
| /// Linked Cortex Search service name (Snowflake Cortex Analyst). | ||
| #[serde(default)] | ||
| pub cortex_search_service_name: Option<String>, | ||
|
Comment on lines
+140
to
+144
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Fresh evidence in this patch is that Rust now accepts these native enrichment keys, but Useful? React with 👍 / 👎. |
||
| pub format: Option<String>, | ||
| pub value_format_name: Option<String>, | ||
| pub parent: Option<String>, | ||
|
|
@@ -190,6 +200,9 @@ pub struct MetricConfig { | |
| pub metadata: Option<serde_json::Value>, | ||
| #[serde(default)] | ||
| pub meta: Option<serde_json::Value>, | ||
| /// Alternative names (e.g. Snowflake Cortex Analyst, Cube). | ||
| #[serde(default)] | ||
| pub synonyms: Option<Vec<String>>, | ||
| #[serde(default = "default_public")] | ||
| pub public: bool, | ||
| } | ||
|
|
@@ -1492,6 +1505,61 @@ models: | |
| ); | ||
| } | ||
|
|
||
| #[test] | ||
| fn test_native_contract_accepts_snowflake_enrichment_fields() { | ||
| // Native YAML produced by Python `export-native` after a Snowflake import | ||
| // carries root `metadata`, dimension synonyms/sample_values/cortex search, | ||
| // and metric synonyms. The Rust native loader must accept (not reject) it. | ||
| let yaml = r#" | ||
| metadata: | ||
| snowflake: | ||
| verified_queries: | ||
| - name: total revenue | ||
| custom_instructions: Prefer revenue. | ||
| models: | ||
| - name: orders | ||
| table: orders | ||
| dimensions: | ||
| - name: status | ||
| type: categorical | ||
| synonyms: [state] | ||
| sample_values: ["1001", "1002"] | ||
| cortex_search_service_name: status_search | ||
| metrics: | ||
| - name: revenue | ||
| agg: sum | ||
| sql: amount | ||
| synonyms: [total revenue] | ||
| "#; | ||
|
|
||
| let config: SidemanticConfig = serde_yaml::from_str(yaml).unwrap(); | ||
|
|
||
| assert_eq!( | ||
| config.metadata.as_ref().unwrap()["snowflake"]["custom_instructions"], | ||
| "Prefer revenue." | ||
| ); | ||
|
|
||
| let dim = &config.models[0].dimensions[0]; | ||
| assert_eq!(dim.synonyms.as_deref(), Some(&["state".to_string()][..])); | ||
| assert_eq!( | ||
| dim.sample_values.as_deref(), | ||
| Some(&["1001".to_string(), "1002".to_string()][..]) | ||
| ); | ||
| assert_eq!( | ||
| dim.cortex_search_service_name.as_deref(), | ||
| Some("status_search") | ||
| ); | ||
|
|
||
| let metric = &config.models[0].metrics[0]; | ||
| assert_eq!( | ||
| metric.synonyms.as_deref(), | ||
| Some(&["total revenue".to_string()][..]) | ||
| ); | ||
|
|
||
| // The config must still convert into the internal model without error. | ||
| config.into_parts().unwrap(); | ||
| } | ||
|
|
||
| #[test] | ||
| fn test_parse_many_to_many_relationship_fields() { | ||
| let yaml = r#" | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -29,6 +29,7 @@ | |
| "models", | ||
| "metrics", | ||
| "parameters", | ||
| "metadata", | ||
|
nicosuave marked this conversation as resolved.
|
||
| "sql_metrics", | ||
| "sql_segments", | ||
| } | ||
|
|
@@ -79,6 +80,9 @@ | |
| "label", | ||
| "metadata", | ||
| "meta", | ||
| "synonyms", | ||
| "sample_values", | ||
| "cortex_search_service_name", | ||
| "format", | ||
| "value_format_name", | ||
| "parent", | ||
|
|
@@ -118,6 +122,7 @@ | |
| "periods", | ||
| "retention_granularity", | ||
| "granularity", | ||
| "synonyms", | ||
| "inner_metrics", | ||
| "entity_dimensions", | ||
| "having", | ||
|
|
@@ -387,6 +392,11 @@ def parse(self, source: str | Path) -> SemanticGraph: | |
| validate_native_format_version(data) | ||
| reject_unknown_fields(data, ROOT_FIELDS, "root", source_path=source_path) | ||
|
|
||
| # Preserve graph-level metadata (e.g. Snowflake Cortex top-level sections). | ||
| graph_metadata = data.get("metadata") | ||
| if isinstance(graph_metadata, dict): | ||
| graph.metadata.update(graph_metadata) | ||
|
Comment on lines
+409
to
+411
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When this accepts root Useful? React with 👍 / 👎. |
||
|
|
||
| # Parse models | ||
| for model_def in data.get("models") or []: | ||
| model = self._parse_model(model_def, source_path=source_path) | ||
|
|
@@ -484,6 +494,9 @@ def export(self, graph: SemanticGraph, output_path: str | Path) -> None: | |
| if graph.parameters: | ||
| data["parameters"] = [self._export_parameter(parameter) for parameter in graph.parameters.values()] | ||
|
|
||
| if graph.metadata: | ||
| data["metadata"] = graph.metadata | ||
|
nicosuave marked this conversation as resolved.
|
||
|
|
||
| output_path.parent.mkdir(parents=True, exist_ok=True) | ||
|
|
||
| with open(output_path, "w") as f: | ||
|
|
@@ -561,6 +574,9 @@ def _parse_model(self, model_def: dict, *, source_path: Path | None = None) -> M | |
| parent=dim_def.get("parent"), | ||
| metadata=dim_def.get("metadata"), | ||
| meta=dim_def.get("meta"), | ||
| synonyms=dim_def.get("synonyms"), | ||
| sample_values=dim_def.get("sample_values"), | ||
| cortex_search_service_name=dim_def.get("cortex_search_service_name"), | ||
| window=dim_def.get("window"), | ||
| ) | ||
| dimensions.append(dimension) | ||
|
|
@@ -781,6 +797,7 @@ def _parse_metric( | |
| "value_format_name", | ||
| "drill_fields", | ||
| "non_additive_dimension", | ||
| "synonyms", | ||
|
nicosuave marked this conversation as resolved.
|
||
| "meta", | ||
| "public", | ||
| ]: | ||
|
|
@@ -928,6 +945,12 @@ def _export_model(self, model: Model) -> dict: | |
| dim_def["metadata"] = dim.metadata | ||
| if dim.meta: | ||
| dim_def["meta"] = dim.meta | ||
| if dim.synonyms: | ||
| dim_def["synonyms"] = dim.synonyms | ||
| if dim.sample_values: | ||
| dim_def["sample_values"] = dim.sample_values | ||
| if dim.cortex_search_service_name: | ||
| dim_def["cortex_search_service_name"] = dim.cortex_search_service_name | ||
|
nicosuave marked this conversation as resolved.
|
||
| if dim.format: | ||
| dim_def["format"] = dim.format | ||
| if dim.value_format_name: | ||
|
|
@@ -966,6 +989,8 @@ def _export_model(self, model: Model) -> dict: | |
| measure_def["metadata"] = measure.metadata | ||
| if measure.meta: | ||
| measure_def["meta"] = measure.meta | ||
| if measure.synonyms: | ||
| measure_def["synonyms"] = measure.synonyms | ||
| if not measure.public: | ||
| measure_def["public"] = measure.public | ||
| if measure.format: | ||
|
|
@@ -1089,6 +1114,8 @@ def _export_metric(self, measure: Metric, graph) -> dict: | |
| result["metadata"] = measure.metadata | ||
| if measure.meta: | ||
| result["meta"] = measure.meta | ||
| if measure.synonyms: | ||
| result["synonyms"] = measure.synonyms | ||
| if not measure.public: | ||
| result["public"] = measure.public | ||
|
|
||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.