From 235ae3c93f8812aea7e5b034a7565d91a7be2534 Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Fri, 1 Dec 2023 19:47:10 +0100 Subject: [PATCH 01/11] Env variable for faster Python imports (#1175) New `ESSENTIA_PYTHON_NODOC` env variable for faster Python package imports. Skip loading an instance of each algorithm to populate __doc__ and __struct__ fields in its wrapper class, when this varaible equals to 'True', 'true' or '1'. --- src/python/essentia/standard.py | 21 +++++++++++++++++---- src/python/essentia/streaming.py | 20 ++++++++++++++++---- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/python/essentia/standard.py b/src/python/essentia/standard.py index ea0e7d059..15f184852 100644 --- a/src/python/essentia/standard.py +++ b/src/python/essentia/standard.py @@ -22,15 +22,28 @@ import sys as _sys from ._essentia import keys as algorithmNames, info as algorithmInfo from copy import copy +from os import getenv + + +# Whether to skip loading algorithms for reading their metadata (faster import). +ESSENTIA_PYTHON_NODOC = getenv('ESSENTIA_PYTHON_NODOC', False) +ESSENTIA_PYTHON_NODOC = (ESSENTIA_PYTHON_NODOC == 'True' or + ESSENTIA_PYTHON_NODOC == 'true' or + ESSENTIA_PYTHON_NODOC == '1') # given an essentia algorithm name, create the corresponding class def _create_essentia_class(name, moduleName = __name__): essentia.log.debug(essentia.EPython, 'Creating essentia.standard class: %s' % name) - _algoInstance = _essentia.Algorithm(name) - _algoDoc = _algoInstance.getDoc() - _algoStruct = _algoInstance.getStruct() - del _algoInstance + + if not ESSENTIA_PYTHON_NODOC or name == "FrameCutter": + _algoInstance = _essentia.Algorithm(name) + _algoDoc = _algoInstance.getDoc() + _algoStruct = _algoInstance.getStruct() + del _algoInstance + else: + _algoDoc = None + _algoStruct = None class Algo(_essentia.Algorithm): __doc__ = _algoDoc diff --git a/src/python/essentia/streaming.py b/src/python/essentia/streaming.py index e8e458fa9..0fdbec29e 100644 --- a/src/python/essentia/streaming.py +++ b/src/python/essentia/streaming.py @@ -21,6 +21,14 @@ import sys as _sys from . import common as _c from ._essentia import skeys as algorithmNames, sinfo as algorithmInfo +from os import getenv + + +# Whether to skip loading algorithms for reading their metadata (faster import). +ESSENTIA_PYTHON_NODOC = getenv('ESSENTIA_PYTHON_NODOC', False) +ESSENTIA_PYTHON_NODOC = (ESSENTIA_PYTHON_NODOC == 'True' or + ESSENTIA_PYTHON_NODOC == 'true' or + ESSENTIA_PYTHON_NODOC == '1') # Used as a place-holder for sources and sinks, implements the right shift # operator @@ -139,10 +147,14 @@ def totalProduced(self): def _create_streaming_algo(givenname): essentia.log.debug(essentia.EPython, 'Creating essentia.streaming class: %s' % givenname) - _algoInstance = _essentia.StreamingAlgorithm(givenname) - _algoDoc = _algoInstance.getDoc() - _algoStruct = _algoInstance.getStruct() - del _algoInstance + if not ESSENTIA_PYTHON_NODOC or givenname == 'FrameCutter': + _algoInstance = _essentia.StreamingAlgorithm(givenname) + _algoDoc = _algoInstance.getDoc() + _algoStruct = _algoInstance.getStruct() + del _algoInstance + else: + _algoDoc = None + _algoStruct = None class StreamingAlgo(_essentia.StreamingAlgorithm): __doc__ = _algoDoc From afc8c62aa2c3f60cbc696a925b303a749e2ec167 Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Mon, 4 Dec 2023 16:12:46 +0100 Subject: [PATCH 02/11] Fix wrong indent --- src/python/essentia/standard.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/python/essentia/standard.py b/src/python/essentia/standard.py index 15f184852..d148b855e 100644 --- a/src/python/essentia/standard.py +++ b/src/python/essentia/standard.py @@ -35,8 +35,7 @@ def _create_essentia_class(name, moduleName = __name__): essentia.log.debug(essentia.EPython, 'Creating essentia.standard class: %s' % name) - - if not ESSENTIA_PYTHON_NODOC or name == "FrameCutter": + if not ESSENTIA_PYTHON_NODOC or name == "FrameCutter": _algoInstance = _essentia.Algorithm(name) _algoDoc = _algoInstance.getDoc() _algoStruct = _algoInstance.getStruct() From 3e3873149ca4bebb8bc7e19f46beb154cb649550 Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Tue, 5 Dec 2023 21:49:32 +0100 Subject: [PATCH 03/11] Faster Python imports with prepackaged algo metadata Instead of using an environmental variable ESSENTIA_PYTHON_NODOC, prepare the metadata at the build stage. Load on import. --- src/python/essentia/meta.py | 35 ++++++++++++++++++++++++++++++++ src/python/essentia/standard.py | 34 +++++++++++++++---------------- src/python/essentia/streaming.py | 33 +++++++++++++++--------------- src/python/wscript | 20 ++++++++++++++++++ 4 files changed, 87 insertions(+), 35 deletions(-) create mode 100644 src/python/essentia/meta.py diff --git a/src/python/essentia/meta.py b/src/python/essentia/meta.py new file mode 100644 index 000000000..88484c553 --- /dev/null +++ b/src/python/essentia/meta.py @@ -0,0 +1,35 @@ +from json import dump +from os.path import join +import essentia +from ._essentia import Algorithm, StreamingAlgorithm, keys, skeys + + +def _metadata_standard(): + meta = {} + for name in keys(): + essentia.log.debug(essentia.EPython, 'Loading __doc__ and __struct__ metadata for essentia.standard class: %s' % name) + _algoInstance = Algorithm(name) + meta[name] = {} + meta[name]['__doc__'] = _algoInstance.getDoc() + meta[name]['__struct__'] = _algoInstance.getStruct() + del _algoInstance + return meta + + +def _metadata_streaming(): + meta = {} + for name in skeys(): + essentia.log.debug(essentia.EPython, 'Loading __doc__ and __struct__ metadata for essentia.streaming class: %s' % name) + _algoInstance = StreamingAlgorithm(name) + meta[name] = {} + meta[name]['__doc__'] = _algoInstance.getDoc() + meta[name]['__struct__'] = _algoInstance.getStruct() + del _algoInstance + return meta + + +def _extract_metadata(filedir): + """ Loads algorithms' metadata (__doc__ and __struct__) from the C extension + and stores it to files in a filedir""" + dump(_metadata_standard(), open(join(filedir, 'standard.meta.json'), 'w')) + dump(_metadata_streaming(), open(join(filedir, 'streaming.meta.json'), 'w')) diff --git a/src/python/essentia/standard.py b/src/python/essentia/standard.py index d148b855e..34f113898 100644 --- a/src/python/essentia/standard.py +++ b/src/python/essentia/standard.py @@ -22,27 +22,16 @@ import sys as _sys from ._essentia import keys as algorithmNames, info as algorithmInfo from copy import copy -from os import getenv +import os.path +import json -# Whether to skip loading algorithms for reading their metadata (faster import). -ESSENTIA_PYTHON_NODOC = getenv('ESSENTIA_PYTHON_NODOC', False) -ESSENTIA_PYTHON_NODOC = (ESSENTIA_PYTHON_NODOC == 'True' or - ESSENTIA_PYTHON_NODOC == 'true' or - ESSENTIA_PYTHON_NODOC == '1') - # given an essentia algorithm name, create the corresponding class -def _create_essentia_class(name, moduleName = __name__): +def _create_essentia_class(name, meta, moduleName = __name__): essentia.log.debug(essentia.EPython, 'Creating essentia.standard class: %s' % name) - if not ESSENTIA_PYTHON_NODOC or name == "FrameCutter": - _algoInstance = _essentia.Algorithm(name) - _algoDoc = _algoInstance.getDoc() - _algoStruct = _algoInstance.getStruct() - del _algoInstance - else: - _algoDoc = None - _algoStruct = None + _algoDoc = meta[name]['__doc__'] + _algoStruct = meta[name]['__struct__'] class Algo(_essentia.Algorithm): __doc__ = _algoDoc @@ -147,8 +136,17 @@ def __str__(self): # load all classes into python def _reloadAlgorithms(moduleName = __name__): - for name in _essentia.keys(): - _create_essentia_class(name, moduleName) + meta_file = 'standard.meta.json' + essentia.log.debug(essentia.EPython, f'Loading __doc__ and __struct__ metadata for essentia.standard from {meta_file}') + # Looking for a metadata file in the same directory as `standard.py` + dir_path = os.path.dirname(os.path.realpath(__file__)) + file_path = os.path.join(dir_path, meta_file) + with open(file_path, 'r') as f: + meta = json.load(f) + + for name in algorithmNames(): + _create_essentia_class(name, meta, moduleName) + _reloadAlgorithms() diff --git a/src/python/essentia/streaming.py b/src/python/essentia/streaming.py index 0fdbec29e..f3acbbf59 100644 --- a/src/python/essentia/streaming.py +++ b/src/python/essentia/streaming.py @@ -21,15 +21,10 @@ import sys as _sys from . import common as _c from ._essentia import skeys as algorithmNames, sinfo as algorithmInfo -from os import getenv +import os.path +import json -# Whether to skip loading algorithms for reading their metadata (faster import). -ESSENTIA_PYTHON_NODOC = getenv('ESSENTIA_PYTHON_NODOC', False) -ESSENTIA_PYTHON_NODOC = (ESSENTIA_PYTHON_NODOC == 'True' or - ESSENTIA_PYTHON_NODOC == 'true' or - ESSENTIA_PYTHON_NODOC == '1') - # Used as a place-holder for sources and sinks, implements the right shift # operator class _StreamConnector: @@ -144,17 +139,11 @@ def totalProduced(self): -def _create_streaming_algo(givenname): +def _create_streaming_algo(givenname, meta): essentia.log.debug(essentia.EPython, 'Creating essentia.streaming class: %s' % givenname) - if not ESSENTIA_PYTHON_NODOC or givenname == 'FrameCutter': - _algoInstance = _essentia.StreamingAlgorithm(givenname) - _algoDoc = _algoInstance.getDoc() - _algoStruct = _algoInstance.getStruct() - del _algoInstance - else: - _algoDoc = None - _algoStruct = None + _algoDoc = meta[givenname]['__doc__'] + _algoStruct = meta[givenname]['__struct__'] class StreamingAlgo(_essentia.StreamingAlgorithm): __doc__ = _algoDoc @@ -207,11 +196,21 @@ def configure(self, **kwargs): # load all streaming algorithms into module def _reloadStreamingAlgorithms(): + meta_file = 'streaming.meta.json' + essentia.log.debug(essentia.EPython, f'Loading __doc__ and __struct__ metadata for essentia.streaming from {meta_file}') + # Looking for a metadata file in the same directory as `streaming.py` + dir_path = os.path.dirname(os.path.realpath(__file__)) + file_path = os.path.join(dir_path, meta_file) + with open(file_path, 'r') as f: + meta = json.load(f) + for name in algorithmNames(): - _create_streaming_algo(name) + _create_streaming_algo(name, meta) + _reloadStreamingAlgorithms() + # This subclass provides some more functionality for VectorInput class VectorInput(_essentia.VectorInput): __doc__ = 'VectorInput v1.0\n\n\n'+\ diff --git a/src/python/wscript b/src/python/wscript index 1bf9147aa..56d707b89 100644 --- a/src/python/wscript +++ b/src/python/wscript @@ -5,6 +5,7 @@ from __future__ import print_function import distutils.sysconfig import os +import sys def options(ctx): @@ -58,5 +59,24 @@ def build(ctx): use = ctx.env.USE_LIBS if ctx.env.ONLY_PYTHON else 'essentia ' #+ ctx.env.USE_LIBS ) + # Create local python package folder. + print("Precompute algorithms' help metadata (__doc__ and __struct__)") + PYTHONPATH = 'build/python' + os.system(f'rm -rf {PYTHONPATH}') + os.system(f'mkdir -p {PYTHONPATH}') + os.system(f'cp -r src/python/essentia {PYTHONPATH}/') + + # TODO These filenames only work for Linux. + os.system(f'cp build/src/python/_essentia*.so {PYTHONPATH}/essentia') + os.system(f'cp build/src/libessentia.so {PYTHONPATH}/essentia') + + pythonpath = os.path.abspath(PYTHONPATH) + ldpath = os.path.join(pythonpath, 'essentia') + python_code = f'from essentia.meta import _extract_metadata; _extract_metadata(\"{PYTHONPATH}/essentia\")' + os.system(f"PYTHONPATH={pythonpath} LD_LIBRARY_PATH={ldpath} {sys.executable} -c '{python_code}'") + print("Algorithms' metadata created") + ctx.install_files('${PYTHONDIR}', ctx.path.ant_glob('essentia/**/*.py'), relative_trick=True) + ctx.install_files('${PYTHONDIR}/essentia', ctx.path.parent.parent.ant_glob(f'{PYTHONPATH}/essentia/*.meta.json'), + relative_trick=False) From 60f70b8c77a0ca30bb66fe635eb1edc4867cff52 Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Tue, 12 Dec 2023 17:23:30 +0100 Subject: [PATCH 04/11] Use Python 3 equivalent for `iteritems` Use Python 3 equivalent for `iteritems` to avoid using Python 2 to 3 transitional package six. --- src/python/essentia/common.py | 5 ++--- src/python/essentia/standard.py | 3 +-- src/python/essentia/streaming.py | 3 +-- src/python/essentia/translate.py | 36 ++++++++++++++++---------------- test/src/QA/qa_test.py | 2 +- 5 files changed, 23 insertions(+), 26 deletions(-) diff --git a/src/python/essentia/common.py b/src/python/essentia/common.py index ad540d4b9..7ce7ca2f3 100644 --- a/src/python/essentia/common.py +++ b/src/python/essentia/common.py @@ -16,7 +16,6 @@ # version 3 along with this program. If not, see http://www.gnu.org/licenses/ import numpy -from six import iteritems from . import _essentia @@ -231,7 +230,7 @@ def determineEdt(obj): firstType = None allKeysAreStrings = True allTypesEqual = True - for key, val in iteritems(obj): + for key, val in iter(obj.items()): if not isinstance(key, str): allKeysAreStrings = False break @@ -360,7 +359,7 @@ def __init__(self, poolRep=None): elif isinstance(poolRep, dict): self.cppPool = _essentia.Pool() - for key, val in iteritems(poolRep): + for key, val in iter(poolRep.items()): for v in val: self.add(key, v) diff --git a/src/python/essentia/standard.py b/src/python/essentia/standard.py index 34f113898..cca1da690 100644 --- a/src/python/essentia/standard.py +++ b/src/python/essentia/standard.py @@ -15,7 +15,6 @@ # You should have received a copy of the Affero GNU General Public License # version 3 along with this program. If not, see http://www.gnu.org/licenses/ -from six import iteritems from . import _essentia import essentia from . import common as _c @@ -46,7 +45,7 @@ def __init__(self, **kwargs): def configure(self, **kwargs): # verify that all types match and do any necessary conversions - for name, val in iteritems(kwargs): + for name, val in iter(kwargs.items()): goalType = self.paramType(name) if type(val).__module__ == 'numpy': diff --git a/src/python/essentia/streaming.py b/src/python/essentia/streaming.py index f3acbbf59..a197f40e6 100644 --- a/src/python/essentia/streaming.py +++ b/src/python/essentia/streaming.py @@ -15,7 +15,6 @@ # You should have received a copy of the Affero GNU General Public License # version 3 along with this program. If not, see http://www.gnu.org/licenses/ -from six import iteritems from . import _essentia import essentia import sys as _sys @@ -175,7 +174,7 @@ def __init__(self, **kwargs): def configure(self, **kwargs): # verify that all types match and do any necessary conversions - for name, val in iteritems(kwargs): + for name, val in iter(kwargs.items()): goalType = self.paramType(name) try: convertedVal = _c.convertData(val, goalType) diff --git a/src/python/essentia/translate.py b/src/python/essentia/translate.py index 684d7d9f9..0401f0fd1 100644 --- a/src/python/essentia/translate.py +++ b/src/python/essentia/translate.py @@ -65,8 +65,8 @@ def __init__(self, default_value=None): # to configure that parameter, and the configure_log def find_edt(composite_param_name, marker_obj, configure_log): # find inner algorithm and inner parameter name that this composite_param will configure - for inner_algo_name, properties in configure_log.iteritems(): - for inner_param_name, value in properties['parameters'].iteritems(): + for inner_algo_name, properties in iter(configure_log.items()): + for inner_param_name, value in iter(properties['parameters'].items()): if marker_obj == value: return properties['instance'].paramType(inner_param_name) @@ -76,7 +76,7 @@ def find_edt(composite_param_name, marker_obj, configure_log): # given a reference to an inner algorithm and the configure_log, returns the name of the algo (use # lower() if referring to the member var name) def inner_algo_name(instance, configure_log): - for algo_name, properties in configure_log.iteritems(): + for algo_name, properties in iter(configure_log.items()): if instance == properties['instance']: return algo_name @@ -145,8 +145,8 @@ def generate_dot_algo_label(algo_inst, indent=''): def generate_dot_network(configure_log, composite_algo_inst): # make connections dot_code ='\n// connecting the network\n' - for algo_name, properties in configure_log.iteritems(): - for left_connector, right_connectors in properties['instance'].connections.iteritems(): + for algo_name, properties in iter(configure_log.items()): + for left_connector, right_connectors in iter(properties['instance'].connections.items()): for right_connector in right_connectors: if isinstance(right_connector, streaming._StreamConnector): dot_code += ' _'+inner_algo_name(left_connector.output_algo, configure_log).lower()+':'+left_connector.name+'_o:e'+' -> '+\ @@ -157,13 +157,13 @@ def generate_dot_network(configure_log, composite_algo_inst): ' _'+inneralgoname+':'+left_connector.name+'_o:e'+' -> nowhere_'+inneralgoname+';\n' # make connections from floating inputs - for name, connector in composite_algo_inst.inputs.iteritems(): + for name, connector in iter(composite_algo_inst.inputs.items()): innerinputname = connector.name inneralgoname = inner_algo_name(connector.input_algo, configure_log).lower() dot_code += ' '+name+':e -> _'+inneralgoname+':'+innerinputname+'_i:w;\n' # make connections from floating outputs - for name, connector in composite_algo_inst.outputs.iteritems(): + for name, connector in iter(composite_algo_inst.outputs.items()): inneroutputname = connector.name inneralgoname = inner_algo_name(connector.output_algo, configure_log).lower() dot_code += ' _'+inneralgoname+':'+inneroutputname+'_o:e -> '+name+':w;\n' @@ -180,7 +180,7 @@ def generate_dot_cluster(configure_log, clustername, composite_algo_inst): ' label='+clustername+';\n\n' # for each algo in the cluster, declare it in dot: - for algo_name, properties in configure_log.iteritems(): + for algo_name, properties in iter(configure_log.items()): dot_code += generate_dot_algo(algo_name, properties['instance']) # create the connections @@ -241,7 +241,7 @@ def dummy_configure(self, **kwargs): # itself kwargs_no_markers = dict(kwargs) - for key, value in kwargs.iteritems(): + for key, value in iter(kwargs.items()): if value in marker_objs.values(): if value.default_value == None: del kwargs_no_markers[key] @@ -396,11 +396,11 @@ class '''+composite_algo.__name__+''' : public essentia::streaming::AlgorithmCom cpp_code += 'void ' + composite_algo.__name__ + '::createInnerNetwork() {\n' # declare inputs - for input_alias, connector in algo_inst.inputs.iteritems(): + for input_alias, connector in iter(algo_inst.inputs.items()): input_owner_name = None input_name = None - for algo_name, properties in zip(sorted_algos, sorted_params): #configure_log.iteritems(): + for algo_name, properties in zip(sorted_algos, sorted_params): #iter(configure_log.items()): if properties['instance'] == connector.input_algo: input_owner_name = algo_name input_name = connector.name @@ -419,7 +419,7 @@ class '''+composite_algo.__name__+''' : public essentia::streaming::AlgorithmCom output_owner_name = None output_name = None - for algo_name, properties in zip(sorted_algos, sorted_params): #configure_log.iteritems(): + for algo_name, properties in zip(sorted_algos, sorted_params): #iter(configure_log.items()): if properties['instance'] == connector.output_algo: output_owner_name = algo_name output_name = connector.name @@ -433,8 +433,8 @@ class '''+composite_algo.__name__+''' : public essentia::streaming::AlgorithmCom cpp_code += '\n' # make connections - for algo_name, properties in zip(sorted_algos, sorted_params): #configure_log.iteritems(): - for left_connector, right_connectors in properties['instance'].connections.iteritems(): + for algo_name, properties in zip(sorted_algos, sorted_params): #iter(configure_log.items()): + for left_connector, right_connectors in iter(properties['instance'].connections.items()): for right_connector in right_connectors: if isinstance(right_connector, streaming._StreamConnector): cpp_code += ' connect( _'+\ @@ -471,11 +471,11 @@ class '''+composite_algo.__name__+''' : public essentia::streaming::AlgorithmCom cpp_code += '\n' # configure inner algorithms - for algo_name, properties in zip(sorted_algos, sorted_params): #configure_log.iteritems(): + for algo_name, properties in zip(sorted_algos, sorted_params): #iter(configure_log.items()): # skip if inner algorithm wasn't configured explicitly if not properties['parameters']: continue - for param_name, value in properties['parameters'].iteritems(): + for param_name, value in iter(properties['parameters'].items()): type = common.determineEdt(value) if 'LIST' in str(type) or 'VECTOR' in str(type): if type in [common.Edt.VECTOR_STRING]: @@ -491,11 +491,11 @@ class '''+composite_algo.__name__+''' : public essentia::streaming::AlgorithmCom cpp_code += ' _'+algo_name.lower()+'->configure(' - for param_name, value in properties['parameters'].iteritems(): + for param_name, value in iter(properties['parameters'].items()): if isinstance(value, MarkerObject): # figure out which composite param it is composite_param_name = None - for marker_name, marker_obj in marker_objs.iteritems(): + for marker_name, marker_obj in iter(marker_objs.items()): if marker_obj == value: composite_param_name = marker_name break diff --git a/test/src/QA/qa_test.py b/test/src/QA/qa_test.py index c5989294c..23d9621b0 100644 --- a/test/src/QA/qa_test.py +++ b/test/src/QA/qa_test.py @@ -291,7 +291,7 @@ def score_all(self): gt = self.solutions.get((gt_name, key_inst), None) if gt is not None: if key_wrap != gt_name: - for key_metric, metric in self.metrics.iteritems(): + for key_metric, metric in iter(self.metrics.items()): self.score(key_wrap, key_inst, solution, key_metric, metric, gt) continue if self.verbose: From 3ae0e0e9818fff2a631e586f77a887ffa00f183f Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Wed, 13 Dec 2023 12:03:43 +0100 Subject: [PATCH 05/11] Get rid of six package in docs and setup.py --- .github/workflows/build-docs.yml | 2 +- doc/sphinxdoc/installing.rst | 2 +- doc/sphinxdoc/machine_learning.rst | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 6f06a7095..04fa8ab5c 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -20,7 +20,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y build-essential libeigen3-dev libyaml-dev libfftw3-dev libavcodec-dev libavformat-dev libavutil-dev libswresample-dev libsamplerate0-dev libtag1-dev libchromaprint-dev python3-dev python3-numpy-dev python3-numpy python3-yaml python3-six + sudo apt-get install -y build-essential libeigen3-dev libyaml-dev libfftw3-dev libavcodec-dev libavformat-dev libavutil-dev libswresample-dev libsamplerate0-dev libtag1-dev libchromaprint-dev python3-dev python3-numpy-dev python3-numpy python3-yaml sudo apt-get install -y doxygen python3-pip pandoc pip3 install sphinx pyparsing sphinxcontrib-doxylink docutils jupyter sphinxprettysearchresults sphinx-toolbox # Install TensorFlow diff --git a/doc/sphinxdoc/installing.rst b/doc/sphinxdoc/installing.rst index bd5c08bfd..aedbc2420 100644 --- a/doc/sphinxdoc/installing.rst +++ b/doc/sphinxdoc/installing.rst @@ -55,7 +55,7 @@ You can install those dependencies on a Debian/Ubuntu system from official repos In order to use Python 3 bindings for the library, you might also need to install python3-dev, python3-numpy-dev (or python3-numpy on Ubuntu) and python3-yaml for YAML support in python:: - sudo apt-get install python3-dev python3-numpy-dev python3-numpy python3-yaml python3-six + sudo apt-get install python3-dev python3-numpy-dev python3-numpy python3-yaml Note that, depending on the version of Essentia, different versions of ``libav*`` and ``libtag1-dev`` packages are required. See `release notes for official releases `_. diff --git a/doc/sphinxdoc/machine_learning.rst b/doc/sphinxdoc/machine_learning.rst index d2dcc0215..2d8412cde 100644 --- a/doc/sphinxdoc/machine_learning.rst +++ b/doc/sphinxdoc/machine_learning.rst @@ -59,7 +59,7 @@ Install the `dependencies =1.8.2', 'six'] +setup_requires = ['numpy>=1.8.2'] install_requires = setup_requires + ['pyyaml'] # Require tensorflow for the package essentia-tensorflow From a6a40afcf213536ac94a814e14a5ea62d03e57f3 Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Wed, 13 Dec 2023 12:19:16 +0100 Subject: [PATCH 06/11] Remove preprocessor checks for PY_MAJOR_VERSION>=3 Remove legacy code for Python 2. --- src/python/essentia.cpp | 23 --------------- src/python/pyalgorithm.cpp | 5 ---- src/python/pystreamingalgorithm.cpp | 5 ---- src/python/python3.h | 2 -- src/python/pytypes/pypool.cpp | 5 ---- src/python/pyvectorinput.cpp | 5 ---- src/python/typewrapper.h | 46 ----------------------------- 7 files changed, 91 deletions(-) diff --git a/src/python/essentia.cpp b/src/python/essentia.cpp index 300159c77..2280a591b 100644 --- a/src/python/essentia.cpp +++ b/src/python/essentia.cpp @@ -77,26 +77,17 @@ init_essentia() { PyType_Ready(&VectorVectorStereoSampleType) < 0) { cerr << "Unable to instantiate Essentia's wrapper types." << endl; -#if PY_MAJOR_VERSION >= 3 return NULL; -#else - return; -#endif } // import the NumPy C api int numpy_error = _import_array(); if (numpy_error) { cerr << "Unable to import NumPy C API from Essentia module. Error code = " << numpy_error << endl; -#if PY_MAJOR_VERSION >= 3 return NULL; -#else - return; -#endif } -#if PY_MAJOR_VERSION >= 3 static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "_essentia", /* m_name */ @@ -108,22 +99,12 @@ init_essentia() { NULL, /* m_clear */ NULL, /* m_free */ }; -#endif -#if PY_MAJOR_VERSION >= 3 Essentia__Module = PyModule_Create(&moduledef); -#else - Essentia__Module = Py_InitModule3("_essentia", Essentia__Methods, - "Module that allows access to essentia plugins and algorithms."); -#endif if (Essentia__Module == NULL) { cerr << "Error loading _essentia python/C module" << endl; -#if PY_MAJOR_VERSION >= 3 return NULL; -#else - return; -#endif } // insert the Algorithm class @@ -143,9 +124,5 @@ init_essentia() { essentia::init(); E_DEBUG(EPyBindings, "Successfully initialized _essentia python/C module"); -#if PY_MAJOR_VERSION >= 3 return Essentia__Module; -#else - return; -#endif } diff --git a/src/python/pyalgorithm.cpp b/src/python/pyalgorithm.cpp index a8b9bd653..289ab4637 100644 --- a/src/python/pyalgorithm.cpp +++ b/src/python/pyalgorithm.cpp @@ -509,12 +509,7 @@ static PyMethodDef PyAlgorithm_methods[] = { static PyTypeObject PyAlgorithmType = { -#if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(NULL, 0) -#else - PyObject_HEAD_INIT(NULL) - 0, // ob_size -#endif "essentia.standard.Algorithm", // tp_name sizeof(PyAlgorithm), // tp_basicsize 0, // tp_itemsize diff --git a/src/python/pystreamingalgorithm.cpp b/src/python/pystreamingalgorithm.cpp index 7ecafb429..5fb6c3932 100644 --- a/src/python/pystreamingalgorithm.cpp +++ b/src/python/pystreamingalgorithm.cpp @@ -398,12 +398,7 @@ static PyMethodDef PyStreamingAlgorithm_methods[] = { }; static PyTypeObject PyStreamingAlgorithmType = { -#if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(NULL, 0) -#else - PyObject_HEAD_INIT(NULL) - 0, // ob_size -#endif "essentia.streaming.Algorithm", // tp_name sizeof(PyStreamingAlgorithm), // tp_basicsize 0, // tp_itemsize diff --git a/src/python/python3.h b/src/python/python3.h index ac464bc97..eb013c874 100644 --- a/src/python/python3.h +++ b/src/python/python3.h @@ -23,7 +23,6 @@ // Defines for Python3 wrapper -#if PY_MAJOR_VERSION >= 3 #define PyString_AsString PyUnicode_AsUTF8 #define PyString_AS_STRING PyUnicode_AsUTF8 #define PyString_Check PyUnicode_Check @@ -35,6 +34,5 @@ #define PyInt_Check PyLong_Check #define init_essentia PyInit__essentia -#endif #endif // ESSENTIA_PYTHON3_H diff --git a/src/python/pytypes/pypool.cpp b/src/python/pytypes/pypool.cpp index 6c2303898..43d2da10c 100644 --- a/src/python/pytypes/pypool.cpp +++ b/src/python/pytypes/pypool.cpp @@ -53,12 +53,7 @@ PyMethodDef PyPool_methods[] = { PyTypeObject PyPoolType = { -#if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(NULL, 0) -#else - PyObject_HEAD_INIT(NULL) - 0, // ob_size -#endif "essentia.Pool", // tp_name sizeof(PyPool), // tp_basicsize 0, // tp_itemsize diff --git a/src/python/pyvectorinput.cpp b/src/python/pyvectorinput.cpp index 7a2a092e7..c8fcdb6a9 100644 --- a/src/python/pyvectorinput.cpp +++ b/src/python/pyvectorinput.cpp @@ -92,12 +92,7 @@ static int vectorinput_init(PyStreamingAlgorithm* self, PyObject *args, PyObject } static PyTypeObject PyVectorInputType = { -#if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(NULL, 0) -#else - PyObject_HEAD_INIT(NULL) - 0, // ob_size -#endif "essentia.streaming.VectorInput", // tp_name sizeof(PyStreamingAlgorithm), // tp_basicsize 0, // tp_itemsize diff --git a/src/python/typewrapper.h b/src/python/typewrapper.h index a31088d4d..73bf83d21 100644 --- a/src/python/typewrapper.h +++ b/src/python/typewrapper.h @@ -74,7 +74,6 @@ class className { \ #define DECLARE_PYTHON_TYPE(type) \ extern PyTypeObject type##Type; -#if PY_MAJOR_VERSION >= 3 #define DEFINE_PYTHON_TYPE(type) \ PyTypeObject type##Type = { \ PyVarObject_HEAD_INIT(NULL, 0) \ @@ -116,50 +115,5 @@ PyTypeObject type##Type = { \ 0, /* tp_alloc */ \ type::make_new, /* tp_new */ \ } -#else -#define DEFINE_PYTHON_TYPE(type) \ -PyTypeObject type##Type = { \ - PyObject_HEAD_INIT(NULL) \ - 0, /* ob_size */ \ - "essentia." #type, /* tp_name */ \ - sizeof(type), /* tp_basicsize */ \ - 0, /* tp_itemsize */ \ - type::dealloc, /* tp_dealloc */ \ - 0, /* tp_print */ \ - 0, /* tp_getattr */ \ - 0, /* tp_setattr */ \ - 0, /* tp_compare */ \ - 0, /* tp_repr */ \ - 0, /* tp_as_number */ \ - 0, /* tp_as_sequence */ \ - 0, /* tp_as_mapping */ \ - 0, /* tp_hash */ \ - 0, /* tp_call */ \ - 0, /* tp_str */ \ - 0, /* tp_getattro */ \ - 0, /* tp_setattro */ \ - 0, /* tp_as_buffer */ \ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ \ - #type " objects", /* tp_doc */ \ - 0, /* tp_traverse */ \ - 0, /* tp_clear */ \ - 0, /* tp_richcompare */ \ - 0, /* tp_weaklistoffset */ \ - 0, /* tp_iter */ \ - 0, /* tp_iternext */ \ - 0, /* tp_methods */ \ - 0, /* tp_members */ \ - 0, /* tp_getset */ \ - 0, /* tp_base */ \ - 0, /* tp_dict */ \ - 0, /* tp_descr_get */ \ - 0, /* tp_descr_set */ \ - 0, /* tp_dictoffset */ \ - type::init, /* tp_init */ \ - 0, /* tp_alloc */ \ - type::make_new, /* tp_new */ \ -} -#endif - #endif // ESSENTIA_PYTHON_TYPEWRAPPER_H From 8dbe00c74db8add3dcbdc051847d8e5c54481594 Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Wed, 13 Dec 2023 14:17:37 +0100 Subject: [PATCH 07/11] Fix warning for a duplicate essentia.pc task --- src/wscript | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/wscript b/src/wscript index 651ee1c82..fe5e91f53 100644 --- a/src/wscript +++ b/src/wscript @@ -550,11 +550,11 @@ def build(ctx): install_path = '${PREFIX}/lib' ) - ctx(source='../essentia.pc.in', **ctx.env.pcfile_opts) + # pkgconfig file + ctx(source='../essentia.pc.in', **ctx.env.pcfile_opts, install_path='${PREFIX}/lib/pkgconfig') ctx.add_group() - # install headers if asked to headers = ctx.path.ant_glob('essentia/**/*.h') for h in headers: @@ -563,12 +563,6 @@ def build(ctx): # version.h ctx.install_files('${PREFIX}/include/essentia', 'version.h') - # install pkgconfig file - #pc = ctx.path.ant_glob('essentia/build/essentia.pc') #TODO remove - #ctx.install_files('${PREFIX}/lib/pkgconfig', pc) #TODO remove - ctx.install_files('${PREFIX}/lib/pkgconfig', '../build/essentia.pc') - - if ctx.env.EXAMPLE_LIST or ctx.env.WITH_VAMP: ctx.recurse('examples') From 29d35d58244b486ab781332ae9a1c03a8b386309 Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Wed, 13 Dec 2023 23:18:01 +0100 Subject: [PATCH 08/11] Rework wscript for algos' metadata --- src/python/essentia/meta.py | 6 ++-- src/python/wscript | 58 +++++++++++++++++++++++-------------- 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/src/python/essentia/meta.py b/src/python/essentia/meta.py index 88484c553..f8d1f72c3 100644 --- a/src/python/essentia/meta.py +++ b/src/python/essentia/meta.py @@ -28,8 +28,8 @@ def _metadata_streaming(): return meta -def _extract_metadata(filedir): +def _extract_metadata(filepath_standard, filepath_streaming): """ Loads algorithms' metadata (__doc__ and __struct__) from the C extension and stores it to files in a filedir""" - dump(_metadata_standard(), open(join(filedir, 'standard.meta.json'), 'w')) - dump(_metadata_streaming(), open(join(filedir, 'streaming.meta.json'), 'w')) + dump(_metadata_standard(), open(filepath_standard, 'w')) + dump(_metadata_streaming(), open(filepath_streaming, 'w')) diff --git a/src/python/wscript b/src/python/wscript index 56d707b89..7d8c31fe6 100644 --- a/src/python/wscript +++ b/src/python/wscript @@ -49,34 +49,48 @@ def build(ctx): # system python path os.path.join(ctx.env.PYTHONDIR, 'numpy', 'core', 'include') ] - pybindings = ctx.shlib( - source = ctx.path.ant_glob('essentia.cpp parsing.cpp pytypes/*.cpp'), - target = '_essentia', + ctx.shlib( + source = ctx.path.ant_glob('essentia.cpp parsing.cpp pytypes/*.cpp'), + target = '_essentia', features = 'pyext', includes = NUMPY_INCPATH + [ '.', 'pytypes' ] + (ctx.env.INCLUDES_ESSENTIA if ctx.env.ONLY_PYTHON else adjust(ctx.env.INCLUDES, '..')), cxxflags = [ '-w' ], install_path = '${PYTHONDIR}/essentia', - use = ctx.env.USE_LIBS if ctx.env.ONLY_PYTHON else 'essentia ' #+ ctx.env.USE_LIBS + use = ctx.env.USE_LIBS if ctx.env.ONLY_PYTHON else 'essentia ', #+ ctx.env.USE_LIBS + name = 'pybindings' ) - # Create local python package folder. - print("Precompute algorithms' help metadata (__doc__ and __struct__)") - PYTHONPATH = 'build/python' - os.system(f'rm -rf {PYTHONPATH}') - os.system(f'mkdir -p {PYTHONPATH}') - os.system(f'cp -r src/python/essentia {PYTHONPATH}/') - - # TODO These filenames only work for Linux. - os.system(f'cp build/src/python/_essentia*.so {PYTHONPATH}/essentia') - os.system(f'cp build/src/libessentia.so {PYTHONPATH}/essentia') - - pythonpath = os.path.abspath(PYTHONPATH) - ldpath = os.path.join(pythonpath, 'essentia') - python_code = f'from essentia.meta import _extract_metadata; _extract_metadata(\"{PYTHONPATH}/essentia\")' - os.system(f"PYTHONPATH={pythonpath} LD_LIBRARY_PATH={ldpath} {sys.executable} -c '{python_code}'") - print("Algorithms' metadata created") + ctx.add_group() + ctx(rule=create_algo_metadata, + target=['standard.meta.json', 'streaming.meta.json'], + name='pybindings_algo_meta') ctx.install_files('${PYTHONDIR}', ctx.path.ant_glob('essentia/**/*.py'), relative_trick=True) - ctx.install_files('${PYTHONDIR}/essentia', ctx.path.parent.parent.ant_glob(f'{PYTHONPATH}/essentia/*.meta.json'), - relative_trick=False) + ctx.install_files('${PYTHONDIR}/essentia', ['standard.meta.json', 'streaming.meta.json']) + + +def create_algo_metadata(task): + output_paths = (task.outputs[0].abspath(), task.outputs[1].abspath()) + python_code = 'from essentia.meta import _extract_metadata; _extract_metadata("%s", "%s")' % output_paths + if task.env.ONLY_PYTHON: + ldpath = task.env.LIBPATH_ESSENTIA[0] + else: + ldpath = "../build/src" + + command = """ +echo "Precompute algorithms' help metadata (__doc__ and __struct__)" +echo `pwd` +echo "Using libessentia.so at %s" +set -x +rm -rf ../build/python +mkdir -p ../build/python +cp -r ../src/python/essentia ../build/python/ +cp ../build/src/python/_essentia*.so ../build/python/essentia/ +cp %s/libessentia.so ../build/python/essentia/ + +PYTHONPATH=../build/python LD_LIBRARY_PATH=../build/python/essentia %s -c '%s' +set +x +echo "Algorithms' metadata created" +""" % (ldpath, ldpath, sys.executable, python_code) + task.exec_command(command) From 767ee1cb25d2d4f0a3267bcd25cb1c7c1929b928 Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Thu, 14 Dec 2023 20:02:11 +0100 Subject: [PATCH 09/11] Rework wscript for algos' metadata (2) --- src/python/wscript | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/python/wscript b/src/python/wscript index 7d8c31fe6..f7f1ba8dd 100644 --- a/src/python/wscript +++ b/src/python/wscript @@ -77,7 +77,8 @@ def create_algo_metadata(task): ldpath = task.env.LIBPATH_ESSENTIA[0] else: ldpath = "../build/src" - + + # TODO Use DYLD_LIBRARY_PATH on MacOS command = """ echo "Precompute algorithms' help metadata (__doc__ and __struct__)" echo `pwd` @@ -87,10 +88,10 @@ rm -rf ../build/python mkdir -p ../build/python cp -r ../src/python/essentia ../build/python/ cp ../build/src/python/_essentia*.so ../build/python/essentia/ -cp %s/libessentia.so ../build/python/essentia/ +#cp %s/libessentia.{so,a} ../build/python/essentia/ -PYTHONPATH=../build/python LD_LIBRARY_PATH=../build/python/essentia %s -c '%s' +PYTHONPATH=../build/python LD_LIBRARY_PATH=%s %s -c '%s' set +x echo "Algorithms' metadata created" -""" % (ldpath, ldpath, sys.executable, python_code) +""" % (ldpath, ldpath, ldpath, sys.executable, python_code) task.exec_command(command) From b9615f7bf65f9c60ce38d2eabc407d6299d59da2 Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Thu, 14 Dec 2023 20:10:16 +0100 Subject: [PATCH 10/11] cibuildwheel: set to max verbosity --- pyproject-tensorflow.toml | 4 ++++ pyproject.toml | 2 ++ 2 files changed, 6 insertions(+) diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index e4759a621..2d3e9de70 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -1,3 +1,7 @@ +[tool.cibuildwheel] +build-verbosity = 3 + + [tool.cibuildwheel.linux] manylinux-x86_64-image = "mtgupf/essentia-builds:manylinux2014_x86_64" diff --git a/pyproject.toml b/pyproject.toml index 0c4f4c08b..d62583f60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,6 @@ [tool.cibuildwheel] +build-verbosity = 3 + [tool.cibuildwheel.linux] manylinux-x86_64-image = "mtgupf/essentia-builds:manylinux2014_x86_64" From 314cfbd1396303a2b27fda451f586c155abcc981 Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Fri, 15 Dec 2023 02:36:15 +0100 Subject: [PATCH 11/11] Rework wscript for algos' metadata (3) --- src/python/wscript | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/python/wscript b/src/python/wscript index f7f1ba8dd..b550427e4 100644 --- a/src/python/wscript +++ b/src/python/wscript @@ -78,7 +78,12 @@ def create_algo_metadata(task): else: ldpath = "../build/src" - # TODO Use DYLD_LIBRARY_PATH on MacOS + if sys.platform.startswith('linux'): + ldpath_varname = 'LD_LIBRARY_PATH' + elif sys.platform == 'darwin': + ldpath_varname = 'DYLD_LIBRARY_PATH' + #TODO win32 + command = """ echo "Precompute algorithms' help metadata (__doc__ and __struct__)" echo `pwd` @@ -90,8 +95,8 @@ cp -r ../src/python/essentia ../build/python/ cp ../build/src/python/_essentia*.so ../build/python/essentia/ #cp %s/libessentia.{so,a} ../build/python/essentia/ -PYTHONPATH=../build/python LD_LIBRARY_PATH=%s %s -c '%s' +PYTHONPATH=../build/python %s=%s %s -c '%s' set +x echo "Algorithms' metadata created" -""" % (ldpath, ldpath, ldpath, sys.executable, python_code) +""" % (ldpath, ldpath, ldpath_varname, ldpath, sys.executable, python_code) task.exec_command(command)