Skip to content

Commit cc14430

Browse files
committed
feat(python/sedonadb): add DataFrame.group_by + GroupedDataFrame.agg
Grouped aggregation on top of the registry-driven function dispatch (apache#885) and the global-aggregation binding (apache#887). API: df.group_by("k").agg(total=sd.funcs.sum(sd.col("v"))) df.group_by("k1", "k2").agg( sd.funcs.sum(col("x")).alias("sum_x"), n=sd.funcs.count(col("y")), ) df.group_by(col("x") + col("y")).agg(...) df.group_by(col("k"), "other_key").agg(...) - `df.group_by(*keys)` — varargs of `str | Expr`. Strings auto-promote to `col(name)`; arbitrary `Expr` values are accepted as computed group keys. Empty keys → ValueError; non-str/non-Expr → TypeError. - Returns a new `GroupedDataFrame` — a thin holder for the parent df plus the resolved group exprs. Single method `.agg(*exprs, **named_exprs)` with the same shape as `DataFrame.agg`. Pure Python — the Rust `InternalDataFrame::aggregate(group_exprs, agg_exprs)` from apache#887 already handles the grouped case; this PR just populates `group_exprs` when constructing the aggregation. The `GroupedDataFrame` intermediate is kept minimal (one method beyond `__init__`) so it stays a clean place to add convenience aggregates (`count`, `size`, etc.) later without polluting `DataFrame`. Tests: 12 covering single/multi string keys, Expr keys, computed Expr keys, mixed str/Expr, positional + kwarg agg, lazy return type, and the empty/bad-type error paths for both `group_by` and its `.agg`.
1 parent ea969b7 commit cc14430

2 files changed

Lines changed: 243 additions & 0 deletions

File tree

python/sedonadb/python/sedonadb/dataframe.py

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -596,6 +596,50 @@ def agg(self, *exprs: Expr, **named_exprs: Expr) -> "DataFrame":
596596
self._options,
597597
)
598598

599+
def group_by(self, *keys: Union[str, Expr]) -> "GroupedDataFrame":
600+
"""Group rows by one or more keys for aggregation.
601+
602+
Returns a `GroupedDataFrame` whose `.agg(...)` method runs the
603+
aggregation. Strings are auto-promoted to column references
604+
(same pattern as `sort`); arbitrary `Expr` values are accepted
605+
as computed group keys.
606+
607+
Args:
608+
*keys: One or more `str` column names or `Expr` group keys.
609+
At least one is required.
610+
611+
Examples:
612+
613+
>>> sd = sedona.db.connect()
614+
>>> df = sd.sql(
615+
... "SELECT * FROM (VALUES ('a', 1), ('a', 2), ('b', 3)) AS t(k, v)"
616+
... )
617+
>>> df.group_by("k").agg(total=sd.funcs.sum(sd.col("v"))).sort("k").show()
618+
┌──────┬───────┐
619+
│ k ┆ total │
620+
│ utf8 ┆ int64 │
621+
╞══════╪═══════╡
622+
│ a ┆ 3 │
623+
├╌╌╌╌╌╌┼╌╌╌╌╌╌╌┤
624+
│ b ┆ 3 │
625+
└──────┴───────┘
626+
"""
627+
if not keys:
628+
raise ValueError("group_by() requires at least one key")
629+
630+
coerced: List[Expr] = []
631+
for k in keys:
632+
if isinstance(k, Expr):
633+
coerced.append(k)
634+
elif isinstance(k, str):
635+
coerced.append(_col(k))
636+
else:
637+
raise TypeError(
638+
f"group_by() expects str or Expr arguments, got {type(k).__name__}"
639+
)
640+
641+
return GroupedDataFrame(self, coerced)
642+
599643
def limit(self, n: Optional[int], /, *, offset: int = 0) -> "DataFrame":
600644
"""Limit result to n rows starting at offset
601645
@@ -1226,6 +1270,60 @@ def _scan_collected_default(ctx_impl, obj, schema, options):
12261270
return _scan_default(ctx_impl, obj, schema, options).to_memtable()
12271271

12281272

1273+
class GroupedDataFrame:
1274+
"""A `DataFrame` partitioned by one or more group keys.
1275+
1276+
Produced by `DataFrame.group_by(...)`. The only public method is
1277+
`agg(...)`, which runs the aggregation and returns a new
1278+
`DataFrame` with one row per group. The class exists as a step in
1279+
the chain so that future convenience aggregates (e.g. `count()`,
1280+
`size()`) can land here without polluting `DataFrame`.
1281+
"""
1282+
1283+
__slots__ = ("_df", "_group_exprs")
1284+
1285+
def __init__(self, df: DataFrame, group_exprs: List[Expr]):
1286+
self._df = df
1287+
self._group_exprs = group_exprs
1288+
1289+
def agg(self, *exprs: Expr, **named_exprs: Expr) -> DataFrame:
1290+
"""Aggregate within each group.
1291+
1292+
Same signature as `DataFrame.agg`: positional aggregate `Expr`s
1293+
and/or keyword aggregates where the keyword is the output
1294+
column name.
1295+
1296+
Args:
1297+
*exprs: Positional aggregate expressions.
1298+
**named_exprs: Keyword aggregate expressions; each keyword
1299+
becomes the output alias.
1300+
"""
1301+
if not exprs and not named_exprs:
1302+
raise ValueError("agg() requires at least one aggregate expression")
1303+
1304+
for e in exprs:
1305+
if not isinstance(e, Expr):
1306+
raise TypeError(f"agg() expects Expr arguments, got {type(e).__name__}")
1307+
1308+
all_exprs: List[Expr] = list(exprs)
1309+
for name, e in named_exprs.items():
1310+
if not isinstance(e, Expr):
1311+
raise TypeError(
1312+
f"agg() expects Expr keyword values, got {type(e).__name__} "
1313+
f"for keyword {name!r}"
1314+
)
1315+
all_exprs.append(e.alias(name))
1316+
1317+
return DataFrame(
1318+
self._df._ctx,
1319+
self._df._impl.aggregate(
1320+
[g._impl for g in self._group_exprs],
1321+
[e._impl for e in all_exprs],
1322+
),
1323+
self._df._options,
1324+
)
1325+
1326+
12291327
def _scan_geopandas(ctx_impl, obj, schema, options):
12301328
return _scan_collected_default(
12311329
ctx_impl, obj.to_arrow(geometry_encoding="WKB"), schema, options
Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Tests for DataFrame.group_by(*keys).agg(*exprs, **named_exprs).
19+
# Aggregate exprs come from `con.funcs.<name>(args)` via the function
20+
# registry (#885); the Rust binding is shared with `df.agg`.
21+
22+
import pandas as pd
23+
import pandas.testing as pdt
24+
import pytest
25+
26+
from sedonadb.dataframe import DataFrame, GroupedDataFrame
27+
from sedonadb.expr import col
28+
29+
30+
def _sorted(df: pd.DataFrame, *by: str) -> pd.DataFrame:
31+
# Group output is unordered. Sort to compare deterministically.
32+
return df.sort_values(list(by)).reset_index(drop=True)
33+
34+
35+
def test_group_by_single_key_string(con):
36+
df = con.create_data_frame(
37+
pd.DataFrame({"k": ["a", "a", "b", "b"], "v": [1, 2, 3, 4]})
38+
)
39+
out = df.group_by("k").agg(total=con.funcs.sum(col("v"))).to_pandas()
40+
pdt.assert_frame_equal(
41+
_sorted(out, "k"),
42+
pd.DataFrame({"k": ["a", "b"], "total": [3, 7]}),
43+
)
44+
45+
46+
def test_group_by_returns_grouped_dataframe(con):
47+
df = con.create_data_frame(pd.DataFrame({"k": ["a"], "v": [1]}))
48+
g = df.group_by("k")
49+
assert isinstance(g, GroupedDataFrame)
50+
51+
52+
def test_group_by_multiple_keys(con):
53+
df = con.create_data_frame(
54+
pd.DataFrame(
55+
{
56+
"k1": ["a", "a", "a", "b"],
57+
"k2": ["x", "x", "y", "y"],
58+
"v": [1, 2, 3, 4],
59+
}
60+
)
61+
)
62+
out = df.group_by("k1", "k2").agg(total=con.funcs.sum(col("v"))).to_pandas()
63+
expected = pd.DataFrame(
64+
{"k1": ["a", "a", "b"], "k2": ["x", "y", "y"], "total": [3, 3, 4]}
65+
)
66+
pdt.assert_frame_equal(_sorted(out, "k1", "k2"), expected)
67+
68+
69+
def test_group_by_expr_key(con):
70+
# group_by(col("k")) and group_by("k") should produce the same plan.
71+
df = con.create_data_frame(pd.DataFrame({"k": ["a", "a", "b"], "v": [1, 2, 3]}))
72+
out = df.group_by(col("k")).agg(total=con.funcs.sum(col("v"))).to_pandas()
73+
pdt.assert_frame_equal(
74+
_sorted(out, "k"),
75+
pd.DataFrame({"k": ["a", "b"], "total": [3, 3]}),
76+
)
77+
78+
79+
def test_group_by_computed_expr_key(con):
80+
# Group by an arithmetic expression — rows whose x+y matches are in
81+
# the same group. (1,9), (4,6), (5,5) all sum to 10.
82+
df = con.create_data_frame(pd.DataFrame({"x": [1, 4, 5, 2], "y": [9, 6, 5, 3]}))
83+
out = (
84+
df.group_by((col("x") + col("y")).alias("xy"))
85+
.agg(n=con.funcs.count(col("x")))
86+
.to_pandas()
87+
)
88+
pdt.assert_frame_equal(
89+
_sorted(out, "xy"),
90+
pd.DataFrame({"xy": [5, 10], "n": [1, 3]}),
91+
)
92+
93+
94+
def test_group_by_mixed_string_and_expr(con):
95+
df = con.create_data_frame(pd.DataFrame({"k": ["a", "a", "b"], "v": [1, 2, 3]}))
96+
out = df.group_by("k", col("v") > 1).agg(n=con.funcs.count(col("v"))).to_pandas()
97+
# Three distinct (k, v>1) tuples: (a, false), (a, true), (b, true).
98+
assert len(out) == 3
99+
assert sorted(out["n"].tolist()) == [1, 1, 1]
100+
101+
102+
def test_group_by_agg_positional_and_kwarg(con):
103+
df = con.create_data_frame(pd.DataFrame({"k": ["a", "a", "b"], "v": [1, 2, 3]}))
104+
out = (
105+
df.group_by("k")
106+
.agg(
107+
con.funcs.sum(col("v")).alias("sum_v"),
108+
n=con.funcs.count(col("v")),
109+
)
110+
.to_pandas()
111+
)
112+
pdt.assert_frame_equal(
113+
_sorted(out, "k"),
114+
pd.DataFrame({"k": ["a", "b"], "sum_v": [3, 3], "n": [2, 1]}),
115+
)
116+
117+
118+
def test_group_by_agg_returns_lazy_dataframe(con):
119+
df = con.create_data_frame(pd.DataFrame({"k": ["a"], "v": [1]}))
120+
out = df.group_by("k").agg(total=con.funcs.sum(col("v")))
121+
assert isinstance(out, DataFrame)
122+
123+
124+
def test_group_by_empty_raises(con):
125+
df = con.create_data_frame(pd.DataFrame({"k": ["a"], "v": [1]}))
126+
with pytest.raises(ValueError, match="at least one key"):
127+
df.group_by()
128+
129+
130+
def test_group_by_bad_key_type_raises(con):
131+
df = con.create_data_frame(pd.DataFrame({"k": ["a"], "v": [1]}))
132+
with pytest.raises(TypeError, match="str or Expr"):
133+
df.group_by(123)
134+
135+
136+
def test_grouped_agg_empty_raises(con):
137+
df = con.create_data_frame(pd.DataFrame({"k": ["a"], "v": [1]}))
138+
with pytest.raises(ValueError, match="at least one aggregate expression"):
139+
df.group_by("k").agg()
140+
141+
142+
def test_grouped_agg_non_expr_raises(con):
143+
df = con.create_data_frame(pd.DataFrame({"k": ["a"], "v": [1]}))
144+
with pytest.raises(TypeError, match="agg\\(\\) expects Expr arguments"):
145+
df.group_by("k").agg("not an expr")

0 commit comments

Comments
 (0)