Skip to content

Improvements for PR #100 - ignore_metadata for assert_approx_df_equality #184

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
alexott wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Improvements for PR #100 - ignore_metadata for assert_approx_df_equality #184

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d8b4e51
Improvements for PR #100 - ignore_metadata for assert_approx_df_equality
alexott Apr 12, 2026
daabb8a
unify order of arguments with `assert_df_equality`
alexott Apr 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion chispa/dataframe_comparer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ def assert_approx_df_equality(
allow_nan_equality: bool = False,
ignore_column_order: bool = False,
ignore_row_order: bool = False,
ignore_metadata: bool = False,
ignore_columns: list[str] | None = None,
formats: FormattingConfig | None = None,
) -> None:
Expand All @@ -144,7 +145,7 @@ def assert_approx_df_equality(
df1 = reduce(lambda acc, fn: fn(acc), transforms, df1)
df2 = reduce(lambda acc, fn: fn(acc), transforms, df2)

assert_schema_equality(df1.schema, df2.schema, ignore_nullable)
assert_schema_equality(df1.schema, df2.schema, ignore_nullable, ignore_metadata)

if precision != 0:
assert_generic_rows_equality(
Expand Down
13 changes: 13 additions & 0 deletions tests/test_dataframe_comparer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,19 @@ def it_throws_when_dfs_are_not_same_with_ignored_columns(spark: SparkSession):
with pytest.raises(DataFramesNotEqualError):
assert assert_approx_df_equality(df1, df2, 0.1, ignore_columns=["name"])

def it_can_ignore_metadata(spark: SparkSession):
schema1 = StructType([
StructField("num", IntegerType(), True, {"comment": "a"}),
StructField("name", StringType(), True),
])
schema2 = StructType([
StructField("num", IntegerType(), True, {"comment": "b"}),
StructField("name", StringType(), True),
])
df1 = spark.createDataFrame([(1, "jose"), (2, "li")], schema=schema1)
df2 = spark.createDataFrame([(1, "jose"), (2, "li")], schema=schema2)
assert_approx_df_equality(df1, df2, 0.1, ignore_metadata=True)

def it_does_not_throw_with_struct_columns_and_ignore_row_order(spark: SparkSession):
data1 = [((1.0, "jose"),), ((1.1, "li"),)]
df1 = spark.createDataFrame(data1, ["person"])
Expand Down
Loading