CommonCrawl-CreativeCommons/pyproject.toml at main · BramVanroy/CommonCrawl-CreativeCommons · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
[build-system]
requires = ["setuptools >= 64", "setuptools_scm>=8", "wheel"]
build-backend = "setuptools.build_meta"

[project]
name = "c5"
dynamic = ["version"]
authors = [
    {name = "Bram Vanroy", email = "bram.vanroy@ivdnt.org"},
]
maintainers = [
    {name = "Bram Vanroy", email = "bram.vanroy@ivdnt.org"},
]
description = "Software to build the Common Crawl Creative Commons Corpus (C5)"
readme = "README.md"
keywords = ["data creation", "creative commons", "data annotation", "common crawl"]
license = "Apache-2.0"
license-files = ["LICEN[CS]E*"]
classifiers = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "Topic :: Scientific/Engineering",
    "Topic :: Text Processing",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Operating System :: OS Independent",
]

requires-python = ">=3.11"
dependencies = [
    "beautifulsoup4>=4.7.0,<5",
    "datasets>=4,<5",
    "datatrove[io,s3,processing,multilingual,cli]@git+https://github.com/huggingface/datatrove@v0.6.0",
    "duckdb>1,<2",
    "huggingface_hub[hf_transfer,hf_xet]",
    "html5lib>1,<2",
    "kenlm",
    "lxml>5,<6",
    "lxml_html_clean",
    "pip",
    "pydantic>=2.7,<3",
    "tldextract>5.0,<6.0",
    # Running into some issues with _Sentinel errors in pydantic
    "typing_extensions>=4.14,<4.15",
    "pydantic>=2.11,<2.12",
    "transformers>=4.55.2,<5",
    # required by datatrove but versions >=0.22 have trouble installing
    "kiwipiepy>=0.21,<=0.22",
]

[project.optional-dependencies]
dev = [
    "pytest",
    "ruff"
]

[project.urls]
homepage = "https://github.com/BramVanroy/CommonCrawl-CreativeCommons"
documentation = "https://github.com/BramVanroy/CommonCrawl-CreativeCommons"
issues = "https://github.com/BramVanroy/CommonCrawl-CreativeCommons/issues"
repository = "https://github.com/BramVanroy/CommonCrawl-CreativeCommons.git"


[project.scripts]

# TOOLS
[tool.setuptools]
include-package-data = true

[tool.setuptools.packages.find]
where = ["src"]

[tool.setuptools_scm]
version_file = "src/c5/version.py"

[tool.ruff]
line-length = 119

[tool.ruff.lint]
ignore = ["C901", "E501", "E741", "F402", "F823", "E402"]
select = ["C", "E", "F", "W", "I"]

# Ignore import violations in all `__init__.py` files.
[tool.ruff.lint.per-file-ignores]
"__init__.py" = ["E402", "F401", "F403", "F811"]

[tool.ruff.lint.isort]
lines-after-imports = 2
known-first-party = ["c5"]

[tool.ruff.format]
# Like Black, use double quotes for strings.
quote-style = "double"

# Like Black, indent with spaces, rather than tabs.
indent-style = "space"

# Like Black, respect magic trailing commas.
skip-magic-trailing-comma = false

# Like Black, automatically detect the appropriate line ending.
line-ending = "auto"