In depth upgrades

.env

@@ -1 +1 @@
-PYTHONPATH="src"
+PYTHONPATH="src"

.github/workflows/build.yml

@@ -18,10 +18,10 @@ jobs:
         with:
           ref: main
 
-      - name: Setting up Python 3.11
+      - name: Setting up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.12'
           cache: pip
           cache-dependency-path: |
             ci/requirements.txt

.github/workflows/ci-cd.yml

@@ -26,6 +26,10 @@ jobs:
     uses: ./.github/workflows/build.yml
     secrets: inherit
 
+  standalone:
+    needs: [tests]
+    uses: ./.github/workflows/standalone.yml
+
   docs:
     needs: [release]  # get (release) version number first
     uses: ./.github/workflows/docs.yml

.github/workflows/coverage.yml

@@ -21,10 +21,10 @@ jobs:
         with:
           ref: main
 
-      - name: Setting up Python 3.11
+      - name: Setting up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.12'
           cache: pip
           cache-dependency-path: |
             ci/requirements.txt

.github/workflows/docs.yml

@@ -16,10 +16,10 @@ jobs:
           fetch-depth: 0
           ref: main
 
-      - name: Setting up Python 3.11
+      - name: Setting up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.12'
           cache: pip
           cache-dependency-path: |
             ci/requirements.txt

.github/workflows/release.yml

@@ -25,10 +25,10 @@ jobs:
           git config user.name "${GITHUB_ACTOR}"
           git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
 
-      - name: Setting up Python 3.11
+      - name: Setting up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.12'
           cache: pip
           cache-dependency-path: |
             ci/requirements.txt

.github/workflows/standalone.yml

@@ -0,0 +1,60 @@
+name: Standalone
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  standalone:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+
+    steps:
+      - name: Checking out the repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setting up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: |
+            ci/requirements.txt
+
+      - name: Install required system packages
+        shell: sh
+        run: |
+          PLATFORM="$(echo "$RUNNER_OS" | tr '[:upper:]' '[:lower:]')"
+          REQFN="ci/requirements_$PLATFORM.txt"
+          if [ "$PLATFORM" = "linux" ] && [ -f "$REQFN" ]; then
+            sudo apt-get -y install $(sed -e '/^\s*#/d' "$REQFN")
+          elif [ "$PLATFORM" = "windows" ] && [ -f "$REQFN" ]; then
+            choco install $(sed -e '/^\s*#/d' "$REQFN")
+          else
+            echo "No extra standalone system packages declared for $PLATFORM."
+          fi
+
+      - name: Install dependencies
+        shell: sh
+        run: |
+          [ -d "/c/miniconda" ] && /c/miniconda/condabin/activate.bat
+          python -m pip install --upgrade pip
+          python -m pip install --progress-bar=off -r ci/requirements.txt
+
+      - name: Build standalone bundles
+        shell: sh
+        run: tox -e standalone -v
+
+      - name: Upload standalone artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: standalone-${{ matrix.os }}
+          path: |
+            dist/standalone/*.zip
+            dist/standalone/*/README_STANDALONE.txt
+            dist/standalone/*/build_info.json

.github/workflows/tests.yml

@@ -16,14 +16,14 @@ jobs:
       matrix:
         python_arch: ['x64']
         python:
-          - [3, 11]
           - [3, 12]
           - [3, 13]
+          - [3, 14]
         # os: ['ubuntu', 'windows', 'macos']
         os: ['ubuntu']
     env:
-      PYTHON: ${{ join(matrix.python, '.') }}  # e.g. '3.11'
-      TOX_ENV: ${{ format('{0}{1}', 'py', join(matrix.python, '')) }}  # e.g. 'py311'
+      PYTHON: ${{ join(matrix.python, '.') }}  # e.g. '3.12'
+      TOX_ENV: ${{ format('{0}{1}', 'py', join(matrix.python, '')) }}  # e.g. 'py312'
     runs-on: ${{ format('{0}-latest', matrix.os) }}
     name: "Test py-${{ join(matrix.python, '.') }} (${{ matrix.os }})"
     steps:
@@ -32,7 +32,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          ref: main
 
       - name: Setting up Python ${{ env.PYTHON }}
         uses: actions/setup-python@v5

.gitignore

@@ -1,6 +1,13 @@
 *.py[cod]
 __pycache__
 
+# random files
+S2870 BSA THF 1 1 d.h5
+test.nxs
+testdata/test_nexus_io.nxs
+testdata/test.yaml
+.vscode/settings.json
+
 # C extensions
 *.so
 

.pre-commit-config.yaml

@@ -10,15 +10,9 @@ repos:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: debug-statements
-  - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.8
     hooks:
-      - id: isort
-  - repo: https://github.com/psf/black
-    rev: 23.1.0
-    hooks:
-      - id: black
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
-    hooks:
-      - id: flake8
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format

CHANGELOG.md

@@ -2,11 +2,11 @@
 
 ## v1.0.6 (2025-07-25)
 
-### Bug fixes 
+### Bug fixes
 
 * Project Config: declare Python 3.11 compat. updates GitHub Action tests as well ([`eaeaae8`](https://github.com/BAMresearch/McSAS3/commit/eaeaae83b111a2b0bb1a524ec19e7142fc69bcec))
 
-### Code style 
+### Code style
 
 * __main__: reformat long lines ([`606fc2f`](https://github.com/BAMresearch/McSAS3/commit/606fc2fa62d2b758eedac951a61fb8df6d02c948))
 

MANIFEST.in

@@ -1,11 +1,14 @@
 graft docs
 exclude docs/reference/autosummary/*
+graft design_documentation
 graft src
 graft ci
 graft tests
 graft testdata
+exclude testdata/test.yaml
 exclude testdata/quickstartdemo1_fitResult.*
 graft example_configurations
+graft tools
 
 include .editorconfig
 include .vscode/*.json

README.md

@@ -31,7 +31,8 @@ Due to an issue with sasmodels when using OpenCL: if you see problems with the f
 2. There are launchers that can work from the command line, for optimization and (separately) histogramming. These use minimal configuration files for setting up the different parts of the code. Adjust these for your output files and optimization requirements, and then you can use these to automatically provide a McSAS3 analysis for every measurement.
 3. Currently, it reads three-column ascii / CSV files, or NeXus/HDF5 files. example read configurations are provided.
 4. Observability limits are not included yet
-5. A GUI is not available (yet).
+5. A separate GUI client exists in the sibling `McSAS3GUI` repository. The core documentation here
+   focuses on the maintained CLI and canonical Python workflow APIs.
 6. Some bugs remain. Feel free to add bugs to the issues. They will be fixed as time permits.
 
 ## Installation
@@ -58,6 +59,51 @@ To run the optimizer from the command line using the test settings and test data
 
 This is, of course, a mere test case. The result should look like the Figure shown earlier.
 
+### Python API
+
+The supported Python entry point is the canonical `ProcessingData` workflow API. For scripts or
+notebooks, prefer the top-level `mcsas3` workflow functions:
+
+```python
+from pathlib import Path
+
+from mcsas3 import (
+    STAGE_CLIPPED,
+    load_result_processing_data,
+    optimize_processing_data,
+    prepare_1d_processing_data_from_file,
+    selected_bundle_from_processing,
+)
+
+processing = prepare_1d_processing_data_from_file(
+    Path("testdata", "quickstartdemo1.csv"),
+    csvargs={"sep": ";", "header": None, "names": ["Q", "I", "ISigma"]},
+    nbins=100,
+    analysis_stage=STAGE_CLIPPED,
+)
+
+optimize_processing_data(
+    processing,
+    Path("result.h5"),
+    modelName="mcsas_sphere",
+    fitParameterLimits={"radius": "auto"},
+    staticParameters={"background": 0.0, "scale": 1.0, "sld": 33.4, "sld_solvent": 0.0},
+    maxIter=1000,
+    convCrit=1.0,
+    nRep=2,
+    nCores=1,
+)
+
+restored = load_result_processing_data(Path("result.h5"))
+selected_bundle = selected_bundle_from_processing(restored)
+q = selected_bundle["Q"].signal
+intensity = selected_bundle["signal"].signal
+```
+
+This keeps the public path on canonical `ProcessingData` / `DataBundle` objects. For reusable
+clipping, omission, rebinning, and 2D reconstruction helpers, use `mcsas3.preprocessing`. If you
+are updating older notebooks or scripts, see the migration notes in the user documentation.
+
 To do the same for real measurements, you need to configure McSAS3 by supplying it with three configuration files (two for the optimization, one for the histogramming):
 
 ### Data read configuration file
@@ -66,7 +112,10 @@ This file contains the parameters necessary to read a data file. The example fil
 
 ```yaml
     --- # configuration used to read files into McSAS3. this is assumed to be a 1D file in csv format
-    # Note that the units are assumed to be 1/(m sr) for I and 1/nm for Q
+    # Internal canonical units are 1/(m sr) for I and 1/nm for Q.
+    # Override them here when the source file uses different units.
+    QUnits: "1 / angstrom"
+    IUnits: "1 / centimeter / steradian"
     nbins: 100
     dataRange:
       - 0.0 # minimum
@@ -82,14 +131,17 @@ This file contains the parameters necessary to read a data file. The example fil
 
 Here, *nbins* is the number of binned datapoints to apply to the data clipped to within the dataRange Q limits. We normally rebin the data to reduce the number of datapoints used for the optimization procedure. Typically 100 datapoints per decade is more than sufficient. The uncertainties are propagated and means calculated from the datapoints within a bin.
 
-The *csvargs* is the dictionary of options passed on to the Pandas.from_csv function. The thus loaded columns should at least contain columns named 'Q', 'I', and 'ISigma' (the uncertainty on I).
+The *csvargs* is the dictionary of options passed on to `pandas.read_csv()`. The loaded columns
+should at least contain columns named `Q`, `I`, and `ISigma` (the uncertainty on `I`).
 
 You can also directly load NeXus or HDF5 files, for example you can directly load the processed files that come out of the DAWN software package. The file read configuration for a NeXus or HDF5 file is slightly different. The reader can follow either the 'default' attributes to the data to use, or you can supply a dictionary of HDF5 paths to the datasets to fit (this is the more robust option). For example:
 
 ```yaml
     --- # configuration used to read nexus files into McSAS3. this is assumed to be a 1D file in nexus
-    # Note that the units are assumed to be 1/(m sr) for I and 1/nm for Q
-    # if necessary, the paths to the datasets can be indicated.
+    # Internal canonical units are 1/(m sr) for I and 1/nm for Q.
+    # if necessary, the paths to the datasets can be indicated, and units can be overridden.
+    QUnits: "1 / angstrom"
+    IUnits: "1 / centimeter / steradian"
     nbins: 100
     dataRange:
       - 0.0 # minimum
@@ -162,6 +214,28 @@ For each histogramming range, histogram-independent population statistics are al
 
 https://BAMresearch.github.io/McSAS3
 
+The docs now also cover:
+
+- quickstart workflows for the maintained CLI and canonical Python API
+- upgrade notes for older notebooks and scripts
+- generated module-structure diagrams
+- release delivery for Python packages and standalone CLI bundles
+
+## Project structure
+
+The maintained code structure is documented in the design docs, including a generated Mermaid
+module dependency diagram:
+
+- [canonical data contract](design_documentation/canonical_data_contract.md)
+- [upgrade plan](design_documentation/upgrade_plan.md)
+- [generated module dependency diagram](design_documentation/generated_module_dependencies.md)
+
+To regenerate the dependency diagram after structural changes, run:
+
+```bash
+./.venv/bin/python tools/generate_dependency_diagram.py
+```
+
 ## Development
 
 ### Testing
@@ -183,4 +257,3 @@ Run all tests with:
 Update the project configuration from the *copier* template:
 
     copier update --trust --skip-answered
-

design_documentation/README.md

@@ -0,0 +1,19 @@
+# McSAS3 Design Documentation
+
+This directory captures the current internal shape of McSAS3 and the most important
+refactor target currently in scope: replacing the internal measurement containers with the
+MoDaCor `ProcessingData` / `DataBundle` / `BaseData` model.
+
+Documents in this directory:
+
+- `upgrade_plan.md`: living stepwise upgrade plan for McSAS3 and coordinated McSAS3GUI work.
+- `generated_module_dependencies.md`: generated Mermaid diagram of current top-level module
+  dependencies in `src/mcsas3`; regenerate it with `python tools/generate_dependency_diagram.py`.
+- `current_architecture.md`: historical package layout and runtime flow captured before the final
+  `McData*` retirement; preserved for migration context.
+- `canonical_data_contract.md`: agreed canonical `ProcessingData` stage names, bundle keys,
+  default units, and current canonical workflow rules.
+- `modacor_data_model_migration.md`: historical migration rationale and staging notes from the
+  period when `McData*` still existed in the core repo.
+
+This is an internal engineering baseline, not user-facing documentation.