From 91315f9b8e7037fc92884e120bf7a737c8065d66 Mon Sep 17 00:00:00 2001 From: Lars-Dominik Braun Date: Sat, 9 Mar 2019 15:29:51 +0100 Subject: [PATCH 1/6] Move requirements.txt to setup.py Advantages: - Single-command install (pip install .) - Package can only be installed with known-good dependencies --- .travis.yml | 5 ++--- doc/install.rst | 4 ---- requirements-sphinx.txt | 12 ------------ requirements.txt | 14 -------------- setup.py | 22 ++++++++++++---------- 5 files changed, 14 insertions(+), 43 deletions(-) delete mode 100644 requirements.txt diff --git a/.travis.yml b/.travis.yml index 737a4c73..cd1914f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,14 +20,13 @@ branches: # command to install dependencies install: - - travis_retry pip install -q -r requirements.txt; + - travis_retry pip install '.[resmon]' - travis_retry pip install -q coverage python-coveralls warcat youtube-dl # command to run tests script: - - pip install . -t thematrix/ --no-dependencies - - cd thematrix/ && nosetests --with-coverage --cover-package=wpull --cover-branches + - nosetests --with-coverage --cover-package=wpull --cover-branches after_success: diff --git a/doc/install.rst b/doc/install.rst index 1febbf93..1204bf38 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -58,10 +58,6 @@ case, use the manual install. Manual Install ============== -Install the dependencies known to work with Wpull:: - - pip3 install -r https://raw2.github.com/chfoo/wpull/master/requirements.txt - Install Wpull from GitHub:: pip3 install git+https://github.com/chfoo/wpull.git#egg=wpull diff --git a/requirements-sphinx.txt b/requirements-sphinx.txt index 995d297e..8b422e1e 100644 --- a/requirements-sphinx.txt +++ b/requirements-sphinx.txt @@ -1,15 +1,3 @@ -# Should be the same for requirements.txt: -chardet>=2.0.1,<=2.3 -dnspython3==1.12 -html5lib>=0.999,<1.0 -# lxml>=3.1.0,<=3.5 # except for this because it requires building C libs -namedlist>=1.3,<=1.7 -psutil>=2.0,<=4.2 -sqlalchemy>=0.9,<=1.0.13 -tornado>=3.2.2,<5.0 -typing>=3.5,<=3.5.1 -yapsy==1.11.223 - # for sphinx doc only: sphinx-argparse<=0.1.15 sphinxcontrib-napoleon>=0.2.6,<=0.5.0 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index e9fbfb52..00000000 --- a/requirements.txt +++ /dev/null @@ -1,14 +0,0 @@ -# Absolutely known to work versions only: -chardet>=2.0.1,<=2.3 -dnspython3==1.12 -html5lib>=0.999,<1.0 -lxml>=3.1.0,<=3.5 -namedlist>=1.3,<=1.7 -psutil>=2.0,<=4.2 -sqlalchemy>=0.9,<=1.0.13 -tornado>=3.2.2,<5.0 -typing>=3.5,<=3.5.1 -yapsy==1.11.223 - -# Keep me sorted and update the other files :) - diff --git a/setup.py b/setup.py index 775472a7..78ca3f04 100644 --- a/setup.py +++ b/setup.py @@ -93,23 +93,25 @@ def get_version(): 'wpull3=wpull.application.main:main', ], }, + extras_require={ + 'resmon': ['psutil>=2.0,<=4.2'], + } ) -# Do not add version ranges unless absolutely required! -# See also requirements.txt setup_kwargs['install_requires'] = [ - 'chardet', - 'dnspython3', - 'html5lib', - 'namedlist', - 'sqlalchemy', - 'tornado', - 'yapsy', + 'chardet>=2.0.1,<=2.3', + 'dnspython3==1.12', + 'html5lib>=0.999,<1.0', + 'lxml>=3.1.0,<=3.5', + 'namedlist>=1.3,<=1.7', + 'sqlalchemy>=0.9,<=1.0.13', + 'tornado>=3.2.2,<5.0', + 'yapsy==1.11.223', ] if sys.version_info < (3, 5): - setup_kwargs['install_requires'].append('typing') + setup_kwargs['install_requires'].append('typing>=3.5,<=3.5.1') if __name__ == '__main__': From 3b7d797abbe97ec0deee4de6bb7637f3df57b28e Mon Sep 17 00:00:00 2001 From: Lars-Dominik Braun Date: Sat, 9 Mar 2019 15:44:51 +0100 Subject: [PATCH 2/6] Move testing to setup.py as well warcat and youtube-dl actually need to be installed, but tests_require only downloads the egg and sets sys.path. --- .travis.yml | 4 ++-- setup.cfg | 6 ++++++ setup.py | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 setup.cfg diff --git a/.travis.yml b/.travis.yml index cd1914f1..f750f39c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,12 +21,12 @@ branches: # command to install dependencies install: - travis_retry pip install '.[resmon]' - - travis_retry pip install -q coverage python-coveralls warcat youtube-dl + - travis_retry pip install warcat youtube-dl # command to run tests script: - - nosetests --with-coverage --cover-package=wpull --cover-branches + - python setup.py test after_success: diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..e2d65c91 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,6 @@ +[aliases] +test=nosetests +[nosetests] +with-coverage=1 +cover-package=wpull +cover-branches=1 diff --git a/setup.py b/setup.py index 78ca3f04..793d80e5 100644 --- a/setup.py +++ b/setup.py @@ -95,7 +95,10 @@ def get_version(): }, extras_require={ 'resmon': ['psutil>=2.0,<=4.2'], - } + }, + setup_requires=['nose>=1.0'], + # XXX: for some odd reason this specific coverage version is required + tests_require=['coverage==4.0.3', 'python-coveralls'], ) From 6b54b61d6c57162607a6084d06cbd0fb05c04a15 Mon Sep 17 00:00:00 2001 From: Lars-Dominik Braun Date: Sat, 9 Mar 2019 16:10:48 +0100 Subject: [PATCH 3/6] Exclude broken html5lib/tornado versions See https://github.com/ArchiveTeam/wpull/pull/402#issue-227999981 and https://github.com/ArchiveTeam/wpull/pull/393#issuecomment-431101750 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 793d80e5..c20c2c3b 100644 --- a/setup.py +++ b/setup.py @@ -105,11 +105,11 @@ def get_version(): setup_kwargs['install_requires'] = [ 'chardet>=2.0.1,<=2.3', 'dnspython3==1.12', - 'html5lib>=0.999,<1.0', + 'html5lib>=0.999,<=0.9999999', 'lxml>=3.1.0,<=3.5', 'namedlist>=1.3,<=1.7', 'sqlalchemy>=0.9,<=1.0.13', - 'tornado>=3.2.2,<5.0', + 'tornado>=3.2.2,<4.5.3', 'yapsy==1.11.223', ] From 5927464a35c580bde2cbe0e46c9eae1a6fa61e05 Mon Sep 17 00:00:00 2001 From: Lars-Dominik Braun Date: Thu, 14 Mar 2019 09:37:42 +0100 Subject: [PATCH 4/6] Upgrade dependencies chardet: Works fine with 3.0.4. No API changes namedlist: No new release since 1.7 sqlalchemy: Works fine with 1.3.1. NO UNIT TESTS! yapsy: Version 1.12.0 works. --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index c20c2c3b..1213de41 100644 --- a/setup.py +++ b/setup.py @@ -103,14 +103,14 @@ def get_version(): setup_kwargs['install_requires'] = [ - 'chardet>=2.0.1,<=2.3', + 'chardet>=2.0.1', 'dnspython3==1.12', 'html5lib>=0.999,<=0.9999999', 'lxml>=3.1.0,<=3.5', - 'namedlist>=1.3,<=1.7', - 'sqlalchemy>=0.9,<=1.0.13', + 'namedlist>=1.3', + 'sqlalchemy>=0.9', 'tornado>=3.2.2,<4.5.3', - 'yapsy==1.11.223', + 'yapsy>=1.11.223', ] if sys.version_info < (3, 5): From ab989dbf706f1c1195ad2cf8d82049fbb412d014 Mon Sep 17 00:00:00 2001 From: Lars-Dominik Braun Date: Thu, 14 Mar 2019 09:44:47 +0100 Subject: [PATCH 5/6] setup.py: Document supported Python versions Requires pip>=9 or setuptools>=24 --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 1213de41..f06916e4 100644 --- a/setup.py +++ b/setup.py @@ -81,6 +81,7 @@ def get_version(): 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Internet :: File Transfer Protocol (FTP)', 'Topic :: Internet :: WWW/HTTP', 'Topic :: System :: Archiving', @@ -99,6 +100,7 @@ def get_version(): setup_requires=['nose>=1.0'], # XXX: for some odd reason this specific coverage version is required tests_require=['coverage==4.0.3', 'python-coveralls'], + python_requires='>=3.4,<3.7', ) @@ -118,6 +120,7 @@ def get_version(): if __name__ == '__main__': + # this check is for old versions of pip/setuptools if sys.version_info[0] < 3: raise Exception('Sorry, Python 2 is not supported.') From f859bf835be9ded070229e68f0126384bc841e0f Mon Sep 17 00:00:00 2001 From: Lars-Dominik Braun Date: Thu, 14 Mar 2019 10:01:31 +0100 Subject: [PATCH 6/6] travis: Ignore required Python version We want automated tests. Even for unsupported versions. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f750f39c..fe2e87ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,7 @@ branches: # command to install dependencies install: - - travis_retry pip install '.[resmon]' + - travis_retry pip install --ignore-requires-python '.[resmon]' - travis_retry pip install warcat youtube-dl