Skip to content

Commit d23a4c2

Browse files
committed
Fix a bug where dataset will not load if s3 prefix starts with a digit
Signed-off-by: Chinmay More <chinmay.s.more@gmail.com>
1 parent 180f37b commit d23a4c2

4 files changed

Lines changed: 14 additions & 4 deletions

File tree

api/src/main/java/marquez/service/models/NodeId.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ private String[] parts(int expectedParts, String expectedType) {
222222
return parts;
223223
} else {
224224
// try to avoid matching colons in URIs- e.g., scheme://authority and host:port patterns
225-
Pattern p = Pattern.compile("(?:" + ID_DELIM + "(?!//|\\d+))");
225+
Pattern p = Pattern.compile("(?:" + ID_DELIM + "(?!//|\\d+(?:/|$)))");
226226
Matcher matcher = p.matcher(value);
227227
String[] returnParts = new String[expectedParts];
228228

api/src/test/java/marquez/service/models/NodeIdTest.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ public void testJobWithVersion(String namespace, String job) {
7171
value = {
7272
"my-namespace$my-dataset",
7373
"gs://bucket$/path/to/data",
74+
"s3://mybucket$3d801.temp/table",
7475
"postgresql://hostname:5432/database$my_table",
7576
"my-namespace$my_struct<a:bigint,b:bigint,c:string>"
7677
},
@@ -95,6 +96,8 @@ public void testDataset(String namespace, String dataset) {
9596
value = {
9697
"my-namespace$my-dataset#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
9798
"gs://bucket$/path/to/data#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
99+
"s3://mybucket$3d801.temp/table#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
100+
"s3://mybucket$/3d801.temp/table#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
98101
"postgresql://hostname:5432/database$my_table#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
99102
"my-namespace$my_struct<a:bigint,b:bigint,c:string>#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
100103
},

clients/java/src/main/java/marquez/client/models/NodeId.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ private String[] parts(int expectedParts, String expectedType) {
150150
return parts;
151151
} else {
152152
// try to avoid matching colons in URIs- e.g., scheme://authority and host:port patterns
153-
Pattern p = Pattern.compile("(?:" + ID_DELIM + "(?!//|\\d+))");
153+
Pattern p = Pattern.compile("(?:" + ID_DELIM + "(?!//|\\d+(?:/|$)))");
154154
Matcher matcher = p.matcher(value);
155155
String[] returnParts = new String[expectedParts];
156156

clients/java/src/test/java/marquez/client/models/NodeIdTest.java

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ public class NodeIdTest {
2121
value = {
2222
"my-namespace$my-dataset",
2323
"gs://bucket$/path/to/data",
24+
"s3://mybucket$3d801.temp/table",
2425
"postgresql://hostname:5432/database$my_table",
2526
"my-namespace$my_struct<a:bigint,b:bigint,c:string>"
2627
},
@@ -38,7 +39,9 @@ public void testDataset(String namespace, String dataset) {
3839
value = {
3940
"my-namespace$my-dataset$colA",
4041
"gs://bucket$/path/to/data$colA",
41-
"gs://bucket$/path/to/data$col_A"
42+
"gs://bucket$/path/to/data$col_A",
43+
"s3://mybucket$3d801.temp/table$colA",
44+
"s3://mybucket$3d801.temp/table$col_A"
4245
},
4346
delimiter = '$')
4447
public void testDatasetField(String namespace, String dataset, String field) {
@@ -90,6 +93,8 @@ public void testJobWithVersion(String namespace, String job) {
9093
value = {
9194
"my-namespace$my-dataset#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
9295
"gs://bucket$/path/to/data#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
96+
"s3://mybucket$3d801.temp/table#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
97+
"s3://mybucket$/3d801.temp/table#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
9398
"postgresql://hostname:5432/database$my_table#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
9499
"my-namespace$my_struct<a:bigint,b:bigint,c:string>#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
95100
},
@@ -113,7 +118,9 @@ public void testDatasetWithVersion(String namespace, String dataset) {
113118
value = {
114119
"my-namespace$my-dataset$colA#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
115120
"gs://bucket$/path/to/data$colA#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
116-
"gs://bucket$/path/to/data$col_A#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
121+
"gs://bucket$/path/to/data$col_A#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
122+
"s3://mybucket$3d801.temp/table$colA#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
123+
"s3://mybucket$/3d801.temp/table$col_A#aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"
117124
},
118125
delimiter = '$')
119126
public void testDatasetFieldWithVersion(

0 commit comments

Comments
 (0)