pyproject.toml

[project]
name = "stickler-eval"
version = "0.6.0"
description = "Structured object comparison and evaluation library"
authors = [
    {name = "Spencer Romo", email = "sromo@amazon.com"},
    {name = "Ayushi Haria", email = "ayuharia@amazon.com"},
    {name = "Aditya Addepalli", email = "adiadd@amazon.com"},
    {name = "Ryan Razkenari", email = "razken@amazon.com"}
]
readme = "README.md"
license = "Apache-2.0"
classifiers = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Programming Language :: Python :: 3.14",
    "Topic :: Scientific/Engineering",
    "Topic :: Software Development :: Libraries :: Python Modules",
]
requires-python = ">=3.10"

# The comparison engine plus confidence calibration, which is core
# functionality rather than a peripheral add-on: knowing whether a model's
# self-reported confidence tracks correctness is part of evaluating it.
# Genuinely peripheral modules (document splitting, HTML reporting) declare
# their own extras below.
dependencies = [
    "pydantic>=2.11.0,<3.0.0",
    "rapidfuzz>=3.4.0,<4.0.0",
    "munkres>=1.1.4",
    "numpy>=1.26.0,<3.0.0",
    "jsonschema>=4.17.3,<5.0.0",
    "python-dateutil>=2.8.2,<3.0.0",
    # AUROC is the only metric that needs it; the other three calibration
    # metrics are pure Python. Issue #216 replaces this with a numpy
    # implementation, after which scikit-learn leaves the dependency set.
    # Floor is 1.7.2 rather than 1.8.0 because 1.8.0 requires Python 3.11+.
    "scikit-learn>=1.7.2,<2.0.0",
]

# pip-compatible extras (for `pip install -e ".[dev]"`)
# Note: the `dev` extra must be kept in sync with [dependency-groups].dev below
[project.optional-dependencies]
dev = [
    "pytest>=7.0.0",
    "pytest-xdist>=3.0.0",
    "coverage>=7.0.0",
    "beautifulsoup4>=4.14.2",
    # Used by the memory-measurement assertions in the stress and deep-nesting
    # tests. Not imported anywhere in src/.
    "psutil>=5.9.6,<7.0.0",
    # The suite exercises every optional module, so it needs every extra's
    # dependencies even though they are not required at runtime.
    "jinja2>=3.1.6,<3.2.0",
    "pandas>=2.1.1,<3.0.0",
    "scipy>=1.12.0,<2.0.0",
    "scikit-learn>=1.7.2,<2.0.0",
    "ruff>=0.14.10",
    "mkdocs",
    "mkdocs-material",
    "mkdocs-awesome-nav",
    "mkdocstrings-python",
    "pymdown-extensions",
    # Used by examples/notebooks/Review_Efficiency_Exploration.ipynb.
    "matplotlib>=3.8.0",
]
llm = [
    # >=1.14.0: the structured_output_model= kwarg on Agent.__call__ used by the
    # docs and examples landed in 1.14.0. On 1.0-1.13 __call__ swallows it via
    # **kwargs, so no structured output is requested and the call fails later
    # with AttributeError on .structured_output.
    #
    # No upper bound below 2.0: LLMComparator only uses Agent(model=,
    # system_prompt=, callback_handler=) and strands.models.Model, all verified
    # present through 1.50.2. A ceiling should name a real incompatibility, not
    # the newest release that existed when it was written. Capping at <2.0.0
    # made stickler[llm] uninstallable alongside strands-agents-evals, which
    # requires >=1.42.0.
    "strands-agents>=1.14.0,<2.0.0",
    # LLMComparator renders its prompt with a jinja2 Template.
    "jinja2>=3.1.6,<3.2.0",
]
bert = [
    "evaluate>=0.4.0",
    "torch>=2.2.0",
    "bert-score>=0.3.0",
]
semantic = [
    "boto3>=1.28.0",
    # SemanticComparator's cosine similarity uses scipy.spatial.
    "scipy>=1.12.0,<2.0.0",
]
# Confidence calibration is part of the core dependency set, so this extra is
# empty. Kept as a no-op so `pip install "stickler-eval[confidence]"` keeps
# working for anyone who already pinned it.
confidence = []
# Document packet splitting: clustering and agreement metrics.
docsplit = [
    "pandas>=2.1.1,<3.0.0",
    "scipy>=1.12.0,<2.0.0",
    "scikit-learn>=1.7.2,<2.0.0",
]
# HTML reporting: ProcessEvaluation renders markdown tables from DataFrames.
reporting = [
    "pandas>=2.1.1,<3.0.0",
]
# Everything except `bert`, whose ML stack (torch, transformers) is large
# enough that pulling it in unasked is a surprise. Written as self-referential
# extras so it cannot drift out of sync with the extras it aggregates.
all = [
    "stickler-eval[llm,semantic,confidence,docsplit,reporting]",
]

[project.urls]
Homepage = "https://github.com/awslabs/stickler"
Repository = "https://github.com/awslabs/stickler"
Documentation = "https://awslabs.github.io/stickler/"
Issues = "https://github.com/awslabs/stickler/issues"

[tool.setuptools]
package-dir = {"" = "src"}

[tool.setuptools.packages.find]
where = ["src"]

[tool.setuptools.package-data]
stickler = ["reporting/html/styling/*.css", "reporting/html/interactive/*.js"]

[tool.bandit]
exclude_dirs = ["tests"]

[build-system]
requires = ["setuptools >= 61.0", "wheel"]
build-backend = "setuptools.build_meta"

# uv-native dependency groups (for `uv sync`)
# Keep in sync with [project.optional-dependencies] above
[dependency-groups]
dev = [
    {include-group = "test"},
    {include-group = "lint"},
    {include-group = "docs"},
]
test = [
    "pytest>=7.0.0",
    "pytest-xdist>=3.0.0",
    "coverage>=7.0.0",
    "beautifulsoup4>=4.14.2",
    # Used by the memory-measurement assertions in the stress and deep-nesting
    # tests. Not imported anywhere in src/.
    "psutil>=5.9.6,<7.0.0",
    # Used by examples/notebooks/Review_Efficiency_Exploration.ipynb.
    "matplotlib>=3.8.0",
    # The suite exercises every optional module, so it needs every extra's
    # dependencies even though they are not required at runtime.
    "jinja2>=3.1.6,<3.2.0",
    "pandas>=2.1.1,<3.0.0",
    "scipy>=1.12.0,<2.0.0",
    "scikit-learn>=1.7.2,<2.0.0",
]
lint = [
    "ruff>=0.14.10",
]
docs = [
    "mkdocs",
    "mkdocs-material",
    "mkdocs-awesome-nav",
    "mkdocstrings-python",
    "pymdown-extensions",
]

[tool.uv]
# Dependency cooldown: wait 7 days before resolving newly published package
# versions to mitigate supply-chain attacks via freshly compromised releases.
# https://docs.astral.sh/uv/concepts/resolution/#dependency-cooldowns
# Note: keep `exclude-newer` as the first key in this table; the semgrep rule
# package_managers.uv.uv-missing-dependency-cooldown only matches it there.
exclude-newer = "7 days"
default-groups = ["dev"]

[tool.pytest.ini_options]
testpaths = ["tests"]
python_files = ["test_*.py"]
python_classes = ["Test*"]
python_functions = ["test_*"]
addopts = ["-v", "--tb=short"]
markers = [
    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
]