diff --git a/.github/workflows/autoblack.yml b/.github/workflows/autoblack.yml deleted file mode 100644 index 761222b..0000000 --- a/.github/workflows/autoblack.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Check / auto apply Black -on: - push: - branches: - - master -jobs: - black: - name: Check / auto apply black - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Check files using the black formatter - uses: psf/black@stable - id: black - with: - options: "." - continue-on-error: true - - shell: pwsh - id: check_files_changed - run: | - # Diff HEAD with the previous commit - $diff = git diff - $HasDiff = $diff.Length -gt 0 - Write-Host "::set-output name=files_changed::$HasDiff" - - name: Create Pull Request - if: steps.check_files_changed.outputs.files_changed == 'true' - uses: peter-evans/create-pull-request@v6 - with: - token: ${{ secrets.GITHUB_TOKEN }} - title: "Format Python code with Black" - commit-message: ":art: Format Python code with Black" - body: | - This pull request uses the [psf/black](https://github.com/psf/black) formatter. - base: ${{ github.head_ref }} # Creates pull request onto pull request or commit branch - branch: actions/black diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..3cd07b0 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,26 @@ +name: Lint with Mypy, Ruff +on: + push: + branches: [main] + pull_request: + + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e . + pip install --group lint + - name: Run mypy + run: mypy ynh-dev ynh_dev + - uses: astral-sh/ruff-action@v3 + with: + src: ynh-dev ynh_dev diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml deleted file mode 100644 index d4e239c..0000000 --- a/.github/workflows/shellcheck.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Run Shellcheck on push and PR - -on: - push: - pull_request: - -jobs: - shellcheck: - name: Shellcheck - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Run ShellCheck - uses: ludeeus/action-shellcheck@master - with: - additional_files: 'deploy.sh ynh-dev' diff --git a/.gitignore b/.gitignore index 4ca7c29..17a88b8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,9 @@ +# Python stuff +__pycache__/ +.*_cache/ +uv.lock +.python-version + # Apps *_ynh @@ -11,6 +17,7 @@ Vagrantfile moulinette yunohost yunohost-admin +yunohost-portal ssowat # Folders diff --git a/README.md b/README.md index db16be6..a5eddbd 100644 --- a/README.md +++ b/README.md @@ -249,7 +249,7 @@ In packages like `yunohost`, you have automated non-regression tests at your dis > In such case, you may initiate or attach the container with a specific name, like: > > ```bash -> ./ynh-dev start bookworm ynh-test +> ./ynh-dev -d bookworm -v core-tests -b stable start > ``` > > And run `yunohost tools postinstall` like for the other container. @@ -293,7 +293,8 @@ It could be due to bridge conflict (for example if you have incus installed too) This [ticket](https://github.com/YunoHost/issues/issues/1664) could help. If you have docker and incus, and your dns resolution inside incus container does not work at all, you can try: -``` + +```bash sudo iptables -I DOCKER-USER -i incusbr0 -o eno1 -j ACCEPT ``` @@ -316,9 +317,9 @@ Depending on what you want to achieve, you might want to run the postinstall rig Deploy a `ynh-dev` folder at the root of the filesystem with: ```bash -cd / -curl https://raw.githubusercontent.com/yunohost/ynh-dev/master/deploy.sh | bash +git clone https://github.com/yunohost/ynh-dev /ynh-dev cd /ynh-dev +./ynh-dev init ``` ### 3. Develop and test diff --git a/custom-catalog/catalog_manager.py b/custom-catalog/catalog_manager.py deleted file mode 100755 index 09baab7..0000000 --- a/custom-catalog/catalog_manager.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/python3 - -import sys -import os -import json -import toml -import yaml -import time -from collections import OrderedDict - -CATALOG_LIST_PATH = "/etc/yunohost/apps_catalog.yml" -assert os.path.exists( - CATALOG_LIST_PATH -), f"Catalog list yaml file '{CATALOG_LIST_PATH} does not exists" - -now = time.time() -my_env = os.environ.copy() -my_env["GIT_TERMINAL_PROMPT"] = "0" - -DEFAULT_APPS_FOLDER = "/ynh-dev/custom-catalog/" -DEFAULT_APP_BRANCH = "master" - - -def build(folder=DEFAULT_APPS_FOLDER): - assert os.path.exists(folder), f"'{folder}' doesn't exists." - - app_list_path = os.path.join(folder, "apps.json") - assert os.path.exists(app_list_path), "no 'apps.json' app list found." - - with open(app_list_path) as f: - app_list = json.load(f) - - apps = {} - fail = False - - for app, infos in app_list.items(): - app = app.lower() - try: - app_dict = build_app_dict(app, infos, folder) - except Exception as e: - print(f"[\033[1m\033[31mFAIL\033[00m] Processing {app} failed: {str(e)}") - fail = True - continue - - apps[app_dict["id"]] = app_dict - - # We also remove the app install question and resources parts which aint needed anymore by webadmin etc (or at least we think ;P) - for app in apps.values(): - if "manifest" in app and "install" in app["manifest"]: - del app["manifest"]["install"] - if "manifest" in app and "resources" in app["manifest"]: - del app["manifest"]["resources"] - - output_file = os.path.join(folder, "catalog.json") - data = { - "apps": apps, - "from_api_version": 3, - } - - with open(output_file, "w") as f: - f.write(json.dumps(data, sort_keys=True, indent=2)) - - if fail: - sys.exit(1) - - -def build_app_dict(app, infos, folder): - app_folder = os.path.join(folder, app + "_ynh") - - # Build the dict with all the infos - manifest_toml = os.path.join(app_folder, "manifest.toml") - manifest_json = os.path.join(app_folder, "manifest.json") - if os.path.exists(manifest_toml): - with open(manifest_toml) as f: - manifest = toml.load(f, _dict=OrderedDict) - else: - with open(manifest_json) as f: - manifest = json.load(f, _dict=OrderedDict) - - return { - "id": app, - "git": { - "branch": infos.get("branch", DEFAULT_APP_BRANCH), - "revision": infos.get("revision", "HEAD"), - "url": f"file://{app_folder}", - }, - "lastUpdate": now, - "manifest": manifest, - "state": infos.get("state", "notworking"), - "level": infos.get("level", -1), - "maintained": infos.get("maintained", True), - # "high_quality": infos.get("high_quality", False), - # "featured": infos.get("featured", False), - "category": infos.get("category", None), - "subtags": infos.get("subtags", []), - "potential_alternative_to": infos.get("potential_alternative_to", []), - "antifeatures": list( - set( - list(manifest.get("antifeatures", {}).keys()) - + infos.get("antifeatures", []) - ) - ), - } - - -def reset(): - with open(CATALOG_LIST_PATH, "w") as f: - catalog_list = [{"id": "default", "url": "https://app.yunohost.org/default/"}] - yaml.safe_dump(catalog_list, f, default_flow_style=False) - - -def add(): - with open(CATALOG_LIST_PATH) as f: - catalog_list = yaml.load(f, Loader=yaml.FullLoader) - ids = [catalog["id"] for catalog in catalog_list] - if "custom" not in ids: - catalog_list.append({"id": "custom", "url": None}) - with open(CATALOG_LIST_PATH, "w") as f: - yaml.safe_dump(catalog_list, f, default_flow_style=False) - - -def override(): - with open(CATALOG_LIST_PATH, "w") as f: - catalog_list = [{"id": "custom", "url": None}] - yaml.safe_dump(catalog_list, f, default_flow_style=False) diff --git a/deploy.sh b/deploy.sh deleted file mode 100755 index e3ff08c..0000000 --- a/deploy.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -set -x - -git clone https://github.com/yunohost/ynh-dev -cd ./ynh-dev || exit 1 -git clone https://github.com/YunoHost/moulinette -git clone https://github.com/YunoHost/yunohost -git clone https://github.com/YunoHost/yunohost-admin -git clone https://github.com/YunoHost/SSOwat ssowat -git clone https://github.com/YunoHost/yunohost-portal - -mkdir -p apps - -set +x - -echo " " -echo "---------------------------------------------------------------------" -echo "Done ! You should cd into 'ynh-dev' then check out './ynh-dev --help'" -echo "---------------------------------------------------------------------" -echo " " diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ab7a947 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,86 @@ +[project] +name = "ynh-dev" +version = "2.0" +description = "Yunohost dev environment manager" +readme = "README.md" +authors = [ + {name = "YunoHost", email = "yunohost@yunohost.org"} +] +requires-python = ">=3.11" + +dependencies = [ + "pyinotify", +] + + +[dependency-groups] +lint = [ + "ruff", + "mypy>=1.18", + "types-toml", + "types-PyYAML", + "types-psutil", +] + +[tool.uv] +package = false + +[[tool.mypy.overrides]] +module = ["pyinotify.*"] +follow_untyped_imports = true + +[tool.ruff] +line-length = 120 + +[tool.ruff.lint] +select = [ + "YTT", # flake8-2020 + "ANN", # flake8-annotations + "BLE", # flake8-blind-except + "B", # flake8-bugbear + "A", # flake8-builtins + "C4", # flake8-comprehensions + "DTZ", # flake8-datetimez + "ISC", # flake8-implicit-str-concat + "ICN", # flake8-import-conventions + # "LOG", # flake8-logging + # "G", # flake8-logging-format + "PIE", # flake8-pie + "Q", # flake8-quotes + "RET", # flake8-return + "SIM", # flake8-simplify + "SLOT", # flake8-slots + "PTH", # flake8-use-pathlib + "FLY", # flynt + "I", # isort + "N", # pep8-naming + "PERF", # Perflint + "E", # pycodestyle + "W", # pycodestyle + "F", # Pyflakes + "PL", # Pylint + "UP", # pyupgrade + "FURB", # refurb + "RUF", # Ruff-specific rules + "TRY", # tryceratops + # "D", +] +ignore = [ + "ANN401", # any-type + "COM812", # missing-trailing-comma + "D203", # incorrect-blank-line-before-class + "PLR0911", # too-many-return-statements + "PLR0912", # too-many-branches + "PLR0913", # too-many-arguments + "PLR0915", # too-many-statements + "PLR2004", # magic-value-comparison + "UP009", # utf8-encoding-declaration + "TRY003", # raise-vanilla-args + "D100", # undocumented-public-module + "D104", # undocumented-public-package + "D105", # undocumented-magic-method + "D200", # unnecessary-multiline-docstring + "D212", # multi-line-summary-first-line + "D401", # non-imperative-mood + "FURB171", +] diff --git a/ynh-dev b/ynh-dev index 84dc326..634fd25 100755 --- a/ynh-dev +++ b/ynh-dev @@ -1,645 +1,16 @@ -#!/usr/bin/env bash +#!/usr/bin/env python3 -# shellcheck disable=SC2155,SC2034,SC2164 +import os -function show_usage() { - cat < None: + if "container" in os.environ: + from ynh_dev.ynh_dev_guest import main_container # noqa: PLC0415 + main_container() + else: + from ynh_dev.ynh_dev_host import main_host # noqa: PLC0415 + main_host() - start [DIST] [NAME] [YNH_BRANCH] (Create and) starts a box (DIST=bookworm, NAME=ynh-dev and YNH_BRANCH=unstable by default) - attach [DIST] [NAME] [YNH_BRANCH] Attach an already started box (DIST=bookworm, NAME=ynh-dev and YNH_BRANCH=unstable by default) - destroy [DIST] [NAME] [YNH_BRANCH] Destroy the ynh-dev box (DIST=bookworm, NAME=ynh-dev and YNH_BRANCH=unstable by default) - rebuild [DIST] [NAME] [YNH_BRANCH] Rebuild a fresh, up-to-date box (DIST=bookworm, NAME=ynh-dev and YNH_BRANCH=unstable by default) - ${BLUE}Inside the dev instance${NORMAL} - ${BLUE}=======================${NORMAL} - - ip Give the ip of the guest container - use-git [PKG] Use Git repositories from dev environment path - lint [PKG] Lint source code from core or app packages. - e.g. ./ynh-dev lint yunohost - e.g. ./ynh-dev lint nextcloud_ynh - test [PKG] Deploy, update and run tests for some packages - Tests for single modules and functions can ran with - e.g. ./ynh-dev test yunohost/appurl:urlavailable - api Start yunohost-api and yunohost-portal-api in debug mode in the current terminal, and auto-restart them when code changes - catalog - build Rebuild the custom catalog - add Add the custom catalog in Yunohost catalog list - override Override default catalog with the custom catalog - reset Reset the catalog list to Yunohost's default - - rebuild-api-doc Rebuild the API swagger doc -EOF -} - -function main() -{ - local ACTION="$1" - local ARGUMENTS=("${@:2}") - - [ -z "$ACTION" ] && show_usage && exit 0 - - case "${ACTION}" in - - help|-h|--help) show_usage "${ARGUMENTS[@]}" ;; - - start|--start) start_ynhdev "${ARGUMENTS[@]}" ;; - attach|--attach) attach_ynhdev "${ARGUMENTS[@]}" ;; - destroy|--destroy) destroy_ynhdev "${ARGUMENTS[@]}" ;; - rebuild|--rebuild) rebuild_ynhdev "${ARGUMENTS[@]}" ;; - - ip|--ip) show_vm_ip "${ARGUMENTS[@]}" ;; - use-git|--use-git) use_git "${ARGUMENTS[@]}" ;; - api|--api) api "${ARGUMENTS[@]}" ;; - lint|--lint) run_linters "${ARGUMENTS[@]}" ;; - test|--test) run_tests "${ARGUMENTS[@]}" ;; - - catalog|--catalog) catalog "${ARGUMENTS[@]}" ;; - - rebuild-api-doc|--rebuild-api-doc) rebuild_api_doc "${ARGUMENTS[@]}" ;; - - *) critical "Unknown action ${ACTION}." ;; - esac -} - -################################################################## -# Misc helpers # -################################################################## - -readonly NORMAL=$(printf '\033[0m') -readonly BOLD=$(printf '\033[1m') -readonly faint=$(printf '\033[2m') -readonly UNDERLINE=$(printf '\033[4m') -readonly NEGATIVE=$(printf '\033[7m') -readonly RED=$(printf '\033[31m') -readonly GREEN=$(printf '\033[32m') -readonly ORANGE=$(printf '\033[33m') -readonly BLUE=$(printf '\033[34m') -readonly YELLOW=$(printf '\033[93m') -readonly WHITE=$(printf '\033[39m') - -function success() -{ - local msg=${1} - echo "[${BOLD}${GREEN} OK ${NORMAL}] ${msg}" -} - -function info() -{ - local msg=${1} - echo "[${BOLD}${BLUE}INFO${NORMAL}] ${msg}" -} - -function warn() -{ - local msg=${1} - echo "[${BOLD}${ORANGE}WARN${NORMAL}] ${msg}" 2>&1 -} - -function error() -{ - local msg=${1} - echo "[${BOLD}${RED}FAIL${NORMAL}] ${msg}" 2>&1 -} - -function critical() -{ - local msg=${1} - echo "[${BOLD}${RED}CRIT${NORMAL}] ${msg}" 2>&1 - exit 1 -} - -function assert_inside_vm() { - [ -d /etc/yunohost ] || critical "There's no YunoHost in there. Are you sure that you are inside the container ?" -} - -function assert_yunohost_is_installed() { - [ -e /etc/yunohost/installed ] || critical "YunoHost is not yet properly installed. Rerun this after post-install." -} - -function create_sym_link() { - local DEST=$1 - local LINK=$2 - # Remove current sources if not a symlink - [ -L "$LINK" ] || rm -rf "$LINK" - # Symlink from Git repository - ln -sfn "$DEST" "$LINK" -} - -function prepare_cache_and_deps() { - local DEV_PATH="$1" - local CACHE_PATH="$2" - - mkdir -p "$CACHE_PATH" - # create_sym_link "$DEV_PATH/.env" "$CACHE_PATH/.env" - create_sym_link "$CACHE_PATH/node_modules" "$DEV_PATH/node_modules" - create_sym_link "$DEV_PATH/package.json" "$CACHE_PATH/package.json" - create_sym_link "$DEV_PATH/yarn.lock" "$CACHE_PATH/yarn.lock" - - # Vite require node v14 to parse modern syntax - local DISTRO="$(lsb_release -s -c)" - - # install yarn if not already - if [[ $(dpkg-query -W -f='${Status}' yarn 2>/dev/null | grep -c "ok installed") -eq 0 ]]; - then - info "Installing yarn…" - apt update - apt install yarn - fi - - pushd "$CACHE_PATH" - # Install dependencies with yarn forced to lock file versions (equivalent to `npm ci`) - info "Installing dependencies ... (this may take a while)" - yarn install --frozen-lockfile - popd -} - -################################################################## -# Actions # -################################################################## - -function check_incus_setup() -{ - # Check incus is installed somehow - if ! command -v incus &>/dev/null; then - critical "You need to have Incus installed for ynh-dev to be usable from the host machine. Refer to the README to know how to install it." - fi - if ! id -nG "$(whoami)" | grep -qw "incus-admin" && [ ! "$(id -u)" -eq 0 ]; then - critical "You need to be in the incus-admin group!" - fi - - ip a | grep -q incusbr0 \ - || warn "There is no 'incusbr0' interface... Did you ran 'incus admin init' ?" - - set_incus_remote -} - -function set_incus_remote() -{ - # Check jq is installed somehow - if ! command -v jq &>/dev/null; then - critical "You need jq installed for ynh-dev" - fi - - remote_url=$(incus remote list -f json | jq '.yunohost.Addr') - if [[ "${remote_url}" == *"devbaseimgs"* ]]; then - incus remote remove yunohost - remote_url=null - fi - if [[ "$remote_url" == "null" ]]; then - incus remote add yunohost https://repo.yunohost.org/incus --protocol simplestreams --public - fi -} - -function start_ynhdev() -{ - check_incus_setup - - local DIST=${1:-bookworm} - local YNH_BRANCH=${3:-unstable} - local BOX=${2:-ynh-dev}-${DIST}-${YNH_BRANCH} - - if ! incus info "$BOX" &>/dev/null - then - if ! incus image info "$BOX-base" &>/dev/null - then - LXC_BASE="yunohost/$DIST-$YNH_BRANCH/dev" - incus launch "yunohost:$LXC_BASE" "$BOX" -c security.nesting=true -c security.privileged=true \ - || critical "Failed to launch the container ?" - else - incus launch "$BOX-base" "$BOX" -c security.nesting=true -c security.privileged=true - fi - incus config device add "$BOX" ynhdev-shared-folder disk path=/ynh-dev source="$(readlink -f "$(pwd)")" - info "Now attaching to the container" - else - info "Attaching to existing container" - fi - - incus exec "$BOX" dhclient - - attach_ynhdev "$@" -} - -function attach_ynhdev() -{ - check_incus_setup - local DIST=${1:-bookworm} - local YNH_BRANCH=${3:-unstable} - local BOX=${2:-ynh-dev}-${DIST}-${YNH_BRANCH} - incus start "$BOX" 2>/dev/null || true - incus exec "$BOX" --cwd /ynh-dev -- /bin/bash -} - -function destroy_ynhdev() -{ - check_incus_setup - local DIST=${1:-bookworm} - local YNH_BRANCH=${3:-unstable} - local BOX=${2:-ynh-dev}-${DIST}-${YNH_BRANCH} - incus stop "$BOX" - incus delete "$BOX" -} - -function rebuild_ynhdev() -{ - check_incus_setup - - local DIST=${1:-bookworm} - local YNH_BRANCH=${3:-unstable} - local BOX=${2:-ynh-dev}-${DIST}-${YNH_BRANCH} - - set -x - incus info "$BOX-rebuild" >/dev/null && incus delete "$BOX-rebuild" --force - incus launch "images:debian/$DIST/amd64" "$BOX-rebuild" -c security.nesting=true -c security.privileged=true - sleep 5 - incus exec "$BOX-rebuild" -- apt install curl -y - INSTALL_SCRIPT="https://install.yunohost.org/$DIST" - incus exec "$BOX-rebuild" -- /bin/bash -c "curl $INSTALL_SCRIPT | bash -s -- -a -d $YNH_BRANCH" - incus stop "$BOX-rebuild" - incus publish "$BOX-rebuild" --alias "$BOX-base" - set +x -} - -function show_vm_ip() -{ - assert_inside_vm - hostname --all-ip-addresses | tr ' ' '\n' -} - -function use_git() -{ - assert_inside_vm - local PACKAGES=("$@") - - if [ "${#PACKAGES[@]}" -eq 0 ]; then - PACKAGES=('moulinette' 'ssowat' 'yunohost' 'yunohost-admin') - fi - - for i in "${!PACKAGES[@]}"; - do - case ${PACKAGES[i]} in - ssowat) - create_sym_link "/ynh-dev/ssowat" "/usr/share/ssowat" - local ssowat_conf_file="/etc/nginx/conf.d/ssowat.conf" - if ! grep -q lua_package_path $ssowat_conf_file; then - local current_ssowatconf=$(cat $ssowat_conf_file) - echo "lua_package_path '/ynh-dev/ZeroBraneStudio/lualibs/?/?.lua;/ynh-dev/ZeroBraneStudio/lualibs/?.lua;;';" > $ssowat_conf_file - echo "lua_package_cpath '/ynh-dev/ZeroBraneStudio/bin/linux/x64/clibs/?.so;;';" >> $ssowat_conf_file - echo "$current_ssowatconf" >> $ssowat_conf_file - fi - if [ ! -d "/ynh-dev/ZeroBraneStudio" ]; then - info "If you want to debug ssowat, you can clone https://github.com/pkulchenko/ZeroBraneStudio into the ynh-dev directory of your host," - info "open it, and open a file at the root of the ssowat project in ynh-dev directory, click on \"Project -> Project Directory -> Set From Current File\"." - info "You can start the remote debugger with \"Project -> Start Debugger Server\"." - info "Add the line \"require(\"mobdebug\").start('THE_IP_OF_YOUR_HOST_IN_THE_CONTAINER_NETWORK')\" at the top of the file access.lua and reload nginx in your container with \"systemctl reload nginx\"." - info "After that you should be able to debug ssowat \o/. The debugger should pause the first time it reaches the line \"require(\"mobdebug\").start('...')\", but you can add breakpoints where needed." - info "More info here: http://notebook.kulchenko.com/zerobrane/debugging-openresty-nginx-lua-scripts-with-zerobrane-studio and here: https://github.com/pkulchenko/MobDebug." - fi - - success "Now using Git repository for SSOwat" - ;; - moulinette) - create_sym_link "/ynh-dev/moulinette/locales" "/usr/share/moulinette/locale" - create_sym_link "/ynh-dev/moulinette/moulinette" "/usr/lib/python3/dist-packages/moulinette" - success "Now using Git repository for Moulinette" - ;; - yunohost) - - while IFS= read -r -d '' FILE; do - create_sym_link "$FILE" "/usr/bin/${FILE##*/}" - done < <(find /ynh-dev/yunohost/bin/ -mindepth 1 -maxdepth 1 -print0) - while IFS= read -r -d '' FILE; do - create_sym_link "$FILE" "/usr/share/yunohost/${FILE##*/}" - done < <(find /ynh-dev/yunohost/share/ -mindepth 1 -maxdepth 1 -print0) - - create_sym_link "/ynh-dev/yunohost/hooks" "/usr/share/yunohost/hooks" - create_sym_link "/ynh-dev/yunohost/helpers/helpers" "/usr/share/yunohost/helpers" - while IFS= read -r -d '' HELPER_DIR; do - create_sym_link "$HELPER_DIR" "/usr/share/yunohost/${HELPER_DIR##*/}" - done < <(find /ynh-dev/yunohost/helpers/ -mindepth 1 -maxdepth 1 -name "helpers*.d" -print0) - create_sym_link "/ynh-dev/yunohost/conf" "/usr/share/yunohost/conf" - create_sym_link "/ynh-dev/yunohost/locales" "/usr/share/yunohost/locales" - create_sym_link "/ynh-dev/yunohost/src" "/usr/lib/python3/dist-packages/yunohost" - - python3 "/ynh-dev/yunohost/doc/generate_bash_completion.py" -o "/ynh-dev/yunohost/doc/bash-completion.sh" - create_sym_link "/ynh-dev/yunohost/doc/bash-completion.sh" "/etc/bash_completion.d/yunohost" - - success "Now using Git repository for YunoHost" - - ;; - yunohost-admin) - - local DEV_PATH="/ynh-dev/yunohost-admin/app" - local CACHE_PATH="/var/cache/ynh-dev/yunohost-admin" - - create_sym_link "/ynh-dev/yunohost-admin/app/.env" "/var/cache/ynh-dev/yunohost-admin/.env" - prepare_cache_and_deps "$DEV_PATH" "$CACHE_PATH" - - cd "$CACHE_PATH" - - # Inject container ip in .env file - # Used by vite to expose itself on network and proxy api requests. - IP=$(hostname -I | tr ' ' '\n' | grep "\.") - echo "VITE_IP=$IP" > .env - - # Allow port 8080 in config file or else the dev server will stop working after postinstall - if [[ ! -e /etc/yunohost/installed ]] - then - python3 - </dev/null || critical "You should first run: apt install inotify-tools" - - function kill_and_restart_api() - { - systemctl --quiet is-active yunohost-api && systemctl stop yunohost-api || true - systemctl --quiet is-active yunohost-portal-api && systemctl stop yunohost-portal-api || true - for PID in $(pgrep -f yunohost-api) $(pgrep -f yunohost-portal-api); do kill "$PID"; done - yunohost-api --debug & - yunohost-portal-api --debug & - } - - info "Now monitoring for changes in python files, restarting yunohost-api and yunohost-portal-api when changes occur!" - - kill_and_restart_api - trap 'for PID in $(pgrep -f yunohost-api) $(pgrep -f yunohost-portal-api); do kill $PID; done; exit' SIGINT - - while { inotifywait --quiet -r -e modify /ynh-dev/yunohost/share /ynh-dev/yunohost/locales/ /ynh-dev/yunohost/src /ynh-dev/moulinette/moulinette --exclude "(test_|\.pyc)" || true; } - do - echo "" - echo "==========================" - info "Restarting services" - echo "==========================" - echo "" - kill_and_restart_api - done - -} - -function install_tox() -{ - if ! type "/root/.local/bin/tox" > /dev/null 2>&1; then - info "> Installing tox ..." - apt-get install pipx -y - pipx install tox - fi -} -function install_package_linter() -{ - if [ ! -f /ynh-dev/package_linter/package_linter.py ] > /dev/null 2>&1 ; then - pushd /ynh-dev - git clone https://github.com/YunoHost/package_linter - cd package_linter - python -m venv ./venv - /ynh-dev/package_linter/venv/bin/pip install -r requirements.txt - popd - fi -} -function run_linters() -{ - assert_inside_vm - local PACKAGES=("$@") - for PACKAGE in "${PACKAGES[@]}"; do - case $PACKAGE in - yunohost) - install_tox - pushd /ynh-dev/yunohost - /root/.local/bin/tox run - /root/.local/bin/tox run -e py311-mypy - popd - ;; - moulinette) - install_tox - pushd /ynh-dev/moulinette - /root/.local/bin/tox run - /root/.local/bin/tox run -e py311-mypy - popd - ;; - ssowat|yunohost-portal|yunohost-admin) - echo "Linter not implemented for $PACKAGE" - ;; - *) - install_package_linter - pushd "/ynh-dev/apps/$PACKAGE" - /ynh-dev/package_linter/venv/bin/python3 /ynh-dev/package_linter/package_linter.py "/ynh-dev/apps/$PACKAGE" - popd - ;; - - esac - done -} - -function run_tests() -{ - assert_inside_vm - local PACKAGES=("$@") - for PACKAGE in "${PACKAGES[@]}"; do - TEST_FUNCTION=$(echo "$PACKAGE" | tr '/:' ' ' | awk '{print $3}') - TEST_MODULE=$(echo "$PACKAGE" | tr '/:' ' ' | awk '{print $2}') - PACKAGE=$(echo "$PACKAGE" | tr '/:' ' ' | awk '{print $1}') - - case $PACKAGE in - yunohost) - # Pytest and tests dependencies - if ! type "pytest" > /dev/null 2>&1; then - info "> Installing pytest ..." - apt-get update - apt-get install python3-pip -y - pip3 install pytest pytest-sugar pytest-cov --break-system-packages - fi - for DEP in pytest-mock requests-mock mock; do - if [ -z "$(pip3 show $DEP)" ]; then - info "Installing $DEP with pip3" - pip3 install $DEP --break-system-packages - fi - done - - # ./src/tests is being moved to ./tests, this small patch supports both paths - if [[ -e "/ynh-dev/yunohost/tests/conftest.py" ]]; then - tests_parentdir=/ynh-dev/yunohost - else - tests_parentdir=/ynh-dev/yunohost/src - fi - - # Apps for test - cd "$tests_parentdir/tests" - [ -d "apps" ] || git clone https://github.com/YunoHost/test_apps ./apps - cd apps - git pull > /dev/null 2>&1 - - # Run tests - info "Running tests for YunoHost" - [ -e "/etc/yunohost/installed" ] || critical "You should run postinstallation before running tests :s." - - testpath=tests - if [[ -n "$TEST_MODULE" ]]; then - testpath="${testpath}/test_${TEST_MODULE}.py" - if [[ -n "$TEST_FUNCTION" ]]; then - testpath="${testpath}::test_${TEST_FUNCTION}" - fi - fi - cd "$tests_parentdir" - pytest "$testpath" - ;; - esac - done -} - -function catalog() -{ - assert_inside_vm - local ACTION="$1" - local CUSTOM_APPS_FOLDER=${2:-"/ynh-dev/custom-catalog"} - local CUSTOM_CAT_PATH="${CUSTOM_APPS_FOLDER}/catalog.json" - local CACHE_FOLDER="/var/cache/yunohost/repo" - - cd /ynh-dev/custom-catalog/ - - case "${ACTION}" in - build) - info "Rebuilding custom app catalog" - python3 -c "from catalog_manager import build; build(folder='${CUSTOM_APPS_FOLDER}')" && success "Successfully build custom catalog list in '${CUSTOM_CAT_PATH}'" - ;; - add) - info "Injecting custom catalog in YunoHost catalog list" - create_sym_link "${CUSTOM_CAT_PATH}" "${CACHE_FOLDER}/custom.json" - python3 -c "from catalog_manager import add; add()" && success "Custom catalog '${CUSTOM_CAT_PATH}' added to catalog list" - ;; - override) - info "Overriding default catalog by custom one" - create_sym_link "${CUSTOM_CAT_PATH}" "${CACHE_FOLDER}/custom.json" - python3 -c "from catalog_manager import override; override()" && success "Default catalog is now overrided by '$CUSTOM_CAT_PATH'" - ;; - reset) - info "Reseting to YunoHost default catalog list" - [ -e "$CACHE_FOLDER/custom.json" ] && rm "$CACHE_FOLDER/custom.json" - python3 -c "from catalog_manager import reset; reset()" || exit 1 - success "Returned to default" - ;; - *) - critical "Unknown catalog action '${ACTION}'." - ;; - esac -} - -function rebuild_api_doc() -{ - test -d yunohost || critical "Expected to find a 'yunohost' folder ?" - - cd yunohost/doc - - if ! test -d swagger - then - # Download swagger ui - info "Downloading swagger UI" - curl -L https://github.com/swagger-api/swagger-ui/archive/refs/tags/v4.15.5.tar.gz | tar -xvz swagger-ui-4.15.5/dist/; - mv swagger-ui-4.15.5/dist/ swagger - rmdir swagger-ui-4.15.5 - fi - - info "Rebuild swagger json/js according to actionsmap" - python3 generate_api_doc.py - success "You should now open yunohost/doc/api.html using your favorite browser" -} - -main "$@" +if __name__ == "__main__": + main() diff --git a/ynh_dev/__init__.py b/ynh_dev/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/custom-catalog/apps.json.example b/ynh_dev/apps.json.example similarity index 100% rename from custom-catalog/apps.json.example rename to ynh_dev/apps.json.example diff --git a/ynh_dev/catalog_manager.py b/ynh_dev/catalog_manager.py new file mode 100755 index 0000000..e6e46bf --- /dev/null +++ b/ynh_dev/catalog_manager.py @@ -0,0 +1,110 @@ +#!/usr/bin/python3 + +import json +import os +import sys +import time +from collections import OrderedDict +from pathlib import Path +from typing import Any + +import toml +import yaml + +my_env = os.environ.copy().update({"GIT_TERMINAL_PROMPT": "0"}) + + +class Catalog: + def __init__(self) -> None: + self.CATALOG_LIST_PATH = Path("/etc/yunohost/apps_catalog.yml") + self.DEFAULT_APPS_FOLDER = Path("/ynh-dev/custom-catalog/") + self.DEFAULT_APP_BRANCH = "master" + self.DEFAULT_CATALOG = [{"id": "default", "url": "https://app.yunohost.org/default/"}] + # assert self.CATALOG_LIST_PATH.exists(), f"Catalog list yaml file '{self.CATALOG_LIST_PATH} does not exists" + + def build(self, folder: Path | None = None) -> None: + folder = folder or self.DEFAULT_APPS_FOLDER + assert folder.exists(), f"'{folder}' doesn't exist." + apps_list_path = folder / "apps.json" + assert apps_list_path.exists(), "no 'apps.json' app list found." + apps_list = json.load(apps_list_path.open()) + + apps = {} + fail = False + + for app_, infos in apps_list.items(): + app = app_.lower() + try: + app_dict = self.build_app_dict(app, infos, folder) + except (OSError, json.JSONDecodeError, toml.TomlDecodeError) as e: + print(f"[\033[1m\033[31mFAIL\033[00m] Processing {app} failed: {e!s}") + fail = True + continue + + apps[app_dict["id"]] = app_dict + + # We also remove the app install question and resources parts which aint + # needed anymore by webadmin etc (or at least we think ;P) + for app in apps.values(): + if "manifest" in app and "install" in app["manifest"]: + del app["manifest"]["install"] + if "manifest" in app and "resources" in app["manifest"]: + del app["manifest"]["resources"] + + data = { + "apps": apps, + "from_api_version": 3, + } + + output_file = folder / "catalog.json" + json.dump(data, output_file.open("w"), sort_keys=True, indent=2) + + if fail: + sys.exit(1) + + def build_app_dict(self, app: str, infos: dict[str, Any], folder: Path) -> dict[str, Any]: + app_folder = folder / f"{app}_ynh" + + # Build the dict with all the infos + manifest_toml = app_folder / "manifest.toml" + manifest_json = app_folder / "manifest.json" + if manifest_toml.exists(): + manifest = toml.load(manifest_toml.open(), _dict=OrderedDict) + else: + manifest = json.load(manifest_json.open(), _dict=OrderedDict) + + return { + "id": app, + "git": { + "branch": infos.get("branch", self.DEFAULT_APP_BRANCH), + "revision": infos.get("revision", "HEAD"), + "url": f"file://{app_folder}", + }, + "lastUpdate": time.time(), + "manifest": manifest, + "state": infos.get("state", "notworking"), + "level": infos.get("level", -1), + "maintained": infos.get("maintained", True), + # "high_quality": infos.get("high_quality", False), + # "featured": infos.get("featured", False), + "category": infos.get("category"), + "subtags": infos.get("subtags", []), + "potential_alternative_to": infos.get("potential_alternative_to", []), + "antifeatures": list(set(list(manifest.get("antifeatures", {}).keys()) + infos.get("antifeatures", []))), + } + + def reset(self) -> None: + yaml.safe_dump(self.DEFAULT_CATALOG, self.CATALOG_LIST_PATH.open("w"), default_flow_style=False) + + def add(self) -> None: + if not self.CATALOG_LIST_PATH.exists(): + self.reset() + catalog_list = yaml.safe_load(self.CATALOG_LIST_PATH.open()) + ids = [catalog["id"] for catalog in catalog_list] + if "custom" not in ids: + catalog_list.append({"id": "custom", "url": None}) + yaml.safe_dump(catalog_list, self.CATALOG_LIST_PATH.open("w"), default_flow_style=False) + + def override(self) -> None: + catalog_list = [{"id": "custom", "url": None}] + yaml.safe_dump(catalog_list, self.CATALOG_LIST_PATH.open("w"), default_flow_style=False) diff --git a/ynh_dev/libincus.py b/ynh_dev/libincus.py new file mode 100644 index 0000000..b9726ca --- /dev/null +++ b/ynh_dev/libincus.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 + +import json +import logging +import os +import platform +import shutil +import subprocess +from pathlib import Path +from typing import Any + + +class Incus: + def __init__(self) -> None: + pass + + def arch(self) -> str: + plat = platform.machine() + if plat in ["x86_64", "amd64"]: + return "amd64" + if plat in ["arm64", "aarch64"]: + return "arm64" + if plat in ["armhf"]: + return "armhf" + raise RuntimeError(f"Unknown platform {plat}!") + + def _run(self, *args: str, interactive: bool = False, **kwargs: Any) -> str: + command = ["incus", *args] + if interactive: + subprocess.run(command, **kwargs, stdin=subprocess.PIPE, capture_output=False, check=True) + result = "" + else: + result = subprocess.check_output(command, **kwargs).decode("utf-8") + return result + + def _run_logged_prefixed(self, *args: str, prefix: str = "", **kwargs: Any) -> None: + command = ["incus", *args] + + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs) + assert process.stdout + with process.stdout: + for line in iter(process.stdout.readline, b""): # b'\n'-separated lines + linestr = line if isinstance(line, str) else line.decode("utf-8") + logging.debug("%s%s", prefix, linestr.rstrip("\n")) + exitcode = process.wait() # 0 means success + if exitcode: + raise RuntimeError(f"Could not run {' '.join(command)}") + + def instance_stopped(self, name: str) -> bool: + assert self.instance_exists(name) + res = json.loads(self._run("info", name)) + return str(res["Status"]) == "STOPPED" + + def instance_exists(self, name: str) -> bool: + res = json.loads(self._run("list", "-f", "json")) + instance_names = [instance["name"] for instance in res] + return name in instance_names + + def instance_start(self, name: str) -> None: + self._run("start", name) + + def instance_stop(self, name: str) -> None: + self._run("stop", name) + + def instance_delete(self, name: str) -> None: + self._run("delete", name) + + def launch(self, image_name: str, instance_name: str, *args: str) -> None: + self._run("launch", image_name, instance_name, *args) + + def push_file(self, instance_name: str, file: Path, target: str) -> None: + self._run("file", "push", str(file), f"{instance_name}{target}") + os.sync() + + def execute(self, instance_name: str, *args: str, exec_: bool = False, cwd: str | None = None) -> None: + cwd_args = ["--cwd", cwd] if cwd else [] + incus_args: list[str] = ["exec", instance_name, *cwd_args, "--", *args] + if exec_: + incus = shutil.which("incus") + assert incus + os.execv(incus, ["incus", *incus_args]) + else: + self._run_logged_prefixed(*incus_args, prefix=" In container |\t") + + def publish(self, instance_name: str, image_alias: str, properties: dict[str, str]) -> None: + properties_list = [f"{key}={value}" for key, value in properties.items()] + self._run("publish", instance_name, "--alias", image_alias, *properties_list) + + def image_export(self, image_alias: str, image_target: str, target_dir: Path) -> None: + self._run("image", "export", image_alias, image_target, cwd=target_dir) + + def image_exists(self, alias: str) -> bool: + res = json.loads(self._run("image", "list", "-f", "json")) + image_aliases = [alias["name"] for image in res for alias in image["aliases"]] + return alias in image_aliases + + def image_alias_exists(self, alias: str) -> bool: + res = json.loads(self._run("image", "alias", "list", "-f", "json")) + aliases = [alias["name"] for alias in res] + return alias in aliases + + def image_delete(self, alias: str) -> None: + self._run("image", "delete", alias) + + def image_download(self, alias: str) -> None: + if self.image_alias_exists(alias): + self.image_delete(alias) + self._run("image", "copy", alias, "local:", "--copy-aliases", "--auto-update", interactive=True) + + def remotes(self) -> dict[str, dict[str, str]]: + return json.loads(self._run("remote", "list", "-f", "json")) + + def remote_add(self, name: str, url: str, public: bool, protocol: str) -> None: + self._run( + "remote", + "add", + name, + url, + "--protocol", + protocol, + *(["--public"] if public else []), + ) diff --git a/ynh_dev/ynh_dev_guest.py b/ynh_dev/ynh_dev_guest.py new file mode 100644 index 0000000..7e84a34 --- /dev/null +++ b/ynh_dev/ynh_dev_guest.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python3 + +import argparse +import atexit +import json +import shutil +import subprocess +import textwrap +from pathlib import Path +from typing import Any + +import psutil +import pyinotify +import yaml + +from .catalog_manager import Catalog + + +def ips() -> list[str]: + return subprocess.check_output(["hostname", "--all-ip-addresses"]).decode("utf-8").split() + + +def install_yarn() -> None: + print("Installing dependencies ... (this may take a while)") + if not shutil.which("yarn"): + subprocess.check_call(["apt", "install", "yarnpkg"]) + + +def install_tox() -> Path: + if not shutil.which("tox"): + subprocess.check_call(["apt", "install", "pipx"]) + subprocess.check_call(["pipx", "install", "tox"]) + return Path("/root/.local/bin/tox") + + +def install_pytest() -> None: + if not shutil.which("pytest"): + subprocess.check_call(["apt", "install", "python3-pip"]) + + +def clone_or_pull(url: str, path: Path) -> None: + if path.exists(): + subprocess.check_call(["git", "pull"], cwd=path) + else: + subprocess.check_call(["git", "clone", url, str(path)]) + + +def install_package_linter() -> None: + package_linter_dir = Path("/ynh-dev/package_linter") + clone_or_pull("https://github.com/YunoHost/package_linter", package_linter_dir) + subprocess.check_call(["python", "-m", "venv", "venv"], cwd=package_linter_dir) + subprocess.check_call(["venv/bin/pip", "install", "-r", "requirements.txt"], cwd=package_linter_dir) + + +def symlink(target: Path, link: Path) -> None: + if link.is_symlink(): + link.unlink() + if link.exists(): + if link.is_dir(): + shutil.rmtree(link) + else: + link.unlink() + link.symlink_to(target) + + +class SSOWat: + def __init__(self, path: Path) -> None: + self.path = path + self.name = "ssowat" + + def use(self, root: Path) -> None: + symlink(self.path, root / "usr/share/ssowat") + debugger_dir = self.path.parent / "ZeroBraneStudio" + ssowat_conf_file = root / "etc/nginx/conf.d/ssowat.conf" + if "lua_package_path" not in (content := ssowat_conf_file.read_text()): + ssowat_conf_file.write_text(f"""\ + lua_package_path '{debugger_dir}/lualibs/?/?.lua;{debugger_dir}/lualibs/?.lua;;'; + lua_package_cpath '{debugger_dir}/bin/linux/x64/clibs/?.so;;'; + {content} + """) + + if not debugger_dir.exists(): + print( + textwrap.dedent(""" + If you want to debug ssowat, you can clone https://github.com/pkulchenko/ZeroBraneStudio + into the ynh-dev directory of your host, open it, and open a file at the root of the ssowat + project in ynh-dev directory, click on "Project -> Project Directory -> Set From Current File". + You can start the remote debugger with "Project -> Start Debugger Server". + Add the line "require("mobdebug").start('THE_IP_OF_YOUR_HOST_IN_THE_CONTAINER_NETWORK')" at + the top of the file access.lua and reload nginx in your container with "systemctl reload nginx". + After that you should be able to debug ssowat \\o/. The debugger should pause the first time it + reaches the line "require("mobdebug").start('...')", but you can add breakpoints where needed. + More info here: http://notebook.kulchenko.com/zerobrane/debugging-openresty-nginx-lua-scripts-with-zerobrane-studio + and here: https://github.com/pkulchenko/MobDebug. + """) + ) + + def dev(self, root: Path) -> None: + self.use(root) + + def lint(self) -> None: + print(f"Linker not implemented for {self.name}") + + +class Moulinette: + def __init__(self, path: Path) -> None: + self.path = path + self.name = "moulinette" + + def use(self, root: Path) -> None: + symlink(self.path / "locales", root / "usr/share/moulinette/locale") + symlink(self.path / "moulinette", root / "usr/lib/python3/dist-packages/moulinette") + + def dev(self, root: Path) -> None: + self.use(root) + + def lint(self) -> None: + tox = install_tox() + subprocess.run([str(tox), "run"], cwd=self.path, check=False) + subprocess.run([str(tox), "run", "-e", "py311-mypy"], cwd=self.path, check=False) + + +class YunoHost: + def __init__(self, path: Path) -> None: + self.path = path + self.name = "yunohost" + + def use(self, root: Path) -> None: + for file in (self.path / "bin").iterdir(): + symlink(file, root / file.name) + for file in (self.path / "share").iterdir(): + symlink(file, root / "usr/share/yunohost" / file.name) + for file in (self.path / "helpers").iterdir(): + symlink(file, root / "usr/share/yunohost" / file.name) + + symlink(self.path / "hooks", root / "usr/share/yunohost/hooks") + symlink(self.path / "conf", root / "usr/share/yunohost/conf") + symlink(self.path / "locales", root / "usr/share/yunohost/locales") + symlink(self.path / "src", root / "usr/lib/python3/dist-packages/yunohost") + + subprocess.check_call( + [self.path / "doc/generate_bash_completion.py", "-o", root / "etc/bash_completion.d/yunohost"] + ) + subprocess.check_call( + [self.path / "doc/generate_zsh_completion.py", "-o", root / "usr/share/zsh/vendor-completions/_yunohost"] + ) + + def dev(self, root: Path) -> None: + self.use(root) + self.run_api() + + def run_api(self) -> None: + services = ["yunohost-api", "yunohost-portal-api"] + + def kill_api() -> None: + for service in services: + if subprocess.run(["systemctl", "--quiet", "is-active", service], check=False).returncode == 0: + subprocess.run(["systemctl", "stop", service], check=False) + for proc in psutil.process_iter(): + if any(service in arg for arg in proc.cmdline()): + proc.kill() + proc.wait() + + def start_api_debug() -> None: + for service in services: + subprocess.Popen([service, "--debug"], close_fds=True) + + def restart_api() -> None: + print() + print("==========================") + print("Restarting services") + print("==========================") + print() + kill_api() + start_api_debug() + + def wait_inotify() -> pyinotify.Notifier: + wm = pyinotify.WatchManager() + notifier = pyinotify.Notifier(wm) + excl = pyinotify.ExcludeFilter(["^test_.*", ".*\\.pyc$"]) + mask = pyinotify.IN_MODIFY # type: ignore + for path in ["share", "locales", "src"]: + wm.add_watch(f"/ynh-dev/yunohost/{path}", mask, exclude_filter=excl) + for path in ["moulinette"]: + wm.add_watch(f"/ynh-dev/moulinette/{path}", mask, exclude_filter=excl) + return notifier + + print( + "Monitoring for changes in python files, restarting yunohost-api and " + "yunohost-portal-api when changes occur!" + ) + restart_api() + atexit.register(kill_api) + + while wait_inotify().check_events(): + restart_api() + + def lint(self) -> None: + tox = install_tox() + subprocess.run([str(tox), "run"], cwd=self.path, check=False) + subprocess.run([str(tox), "run", "-e", "py311-mypy"], cwd=self.path, check=False) + + def test(self) -> None: + test_apps_dir = self.path / "tests" / "apps" + clone_or_pull("https://github.com/YunoHost/test_apps", test_apps_dir) + + +class YunoHostAdmin: + def __init__(self, path: Path) -> None: + self.path = path + self.name = "yunohost-admin" + + def build(self, root: Path) -> None: + dev_path = self.path / "app" + target = root / "usr/share/yunohost/admin" + target_bkp = target.with_name(target.name + "-bkp") + if not target_bkp.exists(): + target.rename(target_bkp) + + subprocess.check_call(["yarnpkg", "build"], cwd=dev_path) + symlink(dev_path / "dist", target) + print(f"App built and available at https://{ips()[0]}/yunohost/admin") + + def dev(self, root: Path) -> None: + dev_path = self.path / "app" + cache_path = root / "var/cache/ynh-dev/yunohost-admin" + cache_path.mkdir(parents=True, exist_ok=True) + symlink(dev_path / ".env", cache_path / ".env") + symlink(cache_path / "node_modules", dev_path / "node_modules") + symlink(dev_path / "package.json", cache_path / "package.json") + symlink(dev_path / "yarn.lock", cache_path / "yarn.lock") + + install_yarn() + subprocess.check_call(["yarnpkg", "install", "--frozen-lockfile"], cwd=cache_path) + + # Inject container ip in .env file + # Used by vite to expose itself on network and proxy api requests. + (dev_path / ".env").open("w").write(f"VITE_IP={ips()[0]}\n") + + installed = Path("/etc/yunohost/installed").exists() + if installed: + subprocess.check_call(["yunohost", "firewall", "allow", "TCP", "8080"]) + else: + firewall = Path("/etc/yunohost/firewall.yml") + assert firewall.exists(), f"Firewall yaml file {firewall} does not exists ?" + settings = yaml.safe_load(firewall.open("r")) + if not settings.get(8080, {}).get("open", False): + settings[8080] = {"open": True, "comment": "yunohost-admin dev vite"} + yaml.safe_dump(settings, firewall.open("w"), default_flow_style=False) + + print("Now running dev server") + subprocess.run(["yarnpkg", "dev", "--host"], cwd=dev_path, check=False) + + def lint(self) -> None: + print(f"Linker not implemented for {self.name}") + + def test(self) -> None: + pass + # # Pytest and tests dependencies + # if ! type "pytest" > /dev/null 2>&1; then + # info "> Installing pytest ..." + # apt-get update + # apt-get install python3-pip -y + # pip3 install pytest pytest-sugar pytest-cov --break-system-packages + # fi + # for DEP in pytest-mock requests-mock mock; do + # if [ -z "$(pip3 show $DEP)" ]; then + # info "Installing $DEP with pip3" + # pip3 install $DEP --break-system-packages + # fi + # done + + # # ./src/tests is being moved to ./tests, this small patch supports both paths + # if [[ -e "/ynh-dev/yunohost/tests/conftest.py" ]]; then + # tests_parentdir=/ynh-dev/yunohost + # else + # tests_parentdir=/ynh-dev/yunohost/src + # fi + + # # Apps for test + # cd "$tests_parentdir/tests" + # [ -d "apps" ] || git clone https://github.com/YunoHost/test_apps ./apps + # cd apps + # git pull > /dev/null 2>&1 + + # # Run tests + # info "Running tests for YunoHost" + # [ -e "/etc/yunohost/installed" ] || critical "You should run postinstallation before running tests :s." + + # testpath=tests + # if [[ -n "$TEST_MODULE" ]]; then + # testpath="${testpath}/test_${TEST_MODULE}.py" + # if [[ -n "$TEST_FUNCTION" ]]; then + # testpath="${testpath}::test_${TEST_FUNCTION}" + # fi + # fi + # cd "$tests_parentdir" + # pytest "$testpath" + + +class YunoHostPortal: + def __init__(self, path: Path) -> None: + self.path = path + self.name = "yunohost-portal" + + def use(self, root: Path) -> None: + dev_path = self.path + target = root / "usr/share/yunohost/portal" + target_bkp = target.with_name(target.name + "-bkp") + if not target_bkp.exists(): + target.rename(target_bkp) + + subprocess.check_call(["yarnpkg", "generate"], cwd=dev_path) + symlink(dev_path / ".output" / "public", target) + print(f"App built and available at https://{ips()[0]}/yunohost/sso") + + def dev(self, root: Path) -> None: + dev_path = self.path + env_file = dev_path / ".env" + cache_path = root / "var/cache/ynh-dev/yunohost-portal" + cache_path.mkdir(parents=True, exist_ok=True) + symlink(env_file, cache_path / ".env") + symlink(cache_path / "node_modules", dev_path / "node_modules") + symlink(dev_path / "package.json", cache_path / "package.json") + symlink(dev_path / "yarn.lock", cache_path / "yarn.lock") + + if not env_file.is_file(): + ip = ips()[0] + domain = json.loads(subprocess.check_output(["yunohost", "domain", "main-domain", "--output-as=json"]))[ + "current_main_domain" + ] + print( + textwrap.dedent(f"""\ + There's no 'yunohost-portal/.env' file. + + Based on your current main domain (but you can use any domain added on your YunoHost instance) + the file should look like: + + NUXT_PUBLIC_API_IP=\"$MAIN_DOMAIN\" + + If not already, add your instance's IP into '/etc/yunohost/.portal-api-allowed-cors-origins' + to avoid CORS issues and make sure to add a redirection in your host's '/etc/hosts' which, + based on your instance ip and main domain, would be: + + {ip} {domain} + """) + ) + + install_yarn() + subprocess.check_call(["yarnpkg", "install", "--frozen-lockfile"], cwd=cache_path) + + subprocess.check_call(["yunohost", "firewall", "allow", "TCP", "3000"]) + subprocess.check_call(["yunohost", "firewall", "allow", "TCP", "24678"]) + + print("Now running dev server") + subprocess.run(["yarnpkg", "dev", "--host"], cwd=dev_path, check=False) + + def lint(self) -> None: + print(f"Linker not implemented for {self.name}") + + +def test_app(app: Path) -> None: + install_package_linter() + subprocess.check_call( + [ + "/ynh-dev/package_linter/venv/bin/python3", + "/ynh-dev/package_linter/package_linter.py", + str(app), + ] + ) + + +PROJECTS: dict[str, Any] = { + "moulinette": Moulinette, + "ssowat": SSOWat, + "yunohost": YunoHost, + "yunohost-admin": YunoHostAdmin, + "yunohost-portal": YunoHostPortal, +} + + +def main_container() -> None: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(title="container actions", required=True, dest="action") + action = sub.add_parser("ip", help="Give the ip of the guest container") + action = sub.add_parser("use-git", help="Use Git repositories from dev environment path") + action.add_argument("components", type=str, nargs="+", choices=PROJECTS) + action = sub.add_parser("use-git-dev", help="Use Git repositories from dev environment path and start dev server") + action.add_argument("components", type=str, nargs="+", choices=PROJECTS) + action = sub.add_parser("lint", help="Lint source code from core or app packages.") + action.add_argument("components", type=str, nargs="+", choices=PROJECTS) + action = sub.add_parser("test", help="Deploy, update and run tests for some packages") + action.add_argument("components", type=str, nargs="+", choices=PROJECTS) + + sub.add_parser( + "api", + help=""" + Start yunohost-api and yunohost-portal-api in debug mode in the current terminal, + and auto-restart them when code changes + """, + ) + sub.add_parser("rebuild-api-doc", help="Rebuild the API swagger doc") + + catalog = sub.add_parser("catalog") + catalog_sub = catalog.add_subparsers(dest="catalog_action", required=True) + catalog_sub.add_parser("build", help="Rebuild the custom catalog") + catalog_sub.add_parser("add", help="Add the custom catalog in Yunohost catalog list") + catalog_sub.add_parser("override", help="Override default catalog with the custom catalog") + catalog_sub.add_parser("reset", help="Reset the catalog list to Yunohost's default") + + args = parser.parse_args() + + match args.action: + case "ip": + print("\n".join(ips())) + case "use-git": + for arg in args.components: + project = PROJECTS[arg](f"/ynh-dev/{arg}") + project.use(Path("/")) + case "use-git-dev": + for arg in args.components: + project = PROJECTS[arg](f"/ynh-dev/{arg}") + project.dev(Path("/")) + case "lint": + for arg in args.components: + project = PROJECTS[arg](f"/ynh-dev/{arg}") + project.lint() + case "test": + for arg in args.components: + if arg in PROJECTS: + project = PROJECTS[arg](f"/ynh-dev/{arg}") + project.test() + else: + test_app(Path(f"/ynh-dev/apps/{args.component}")) + case "catalog": + getattr(Catalog(), args.catalog_action)() + + +if __name__ == "__main__": + main_container() diff --git a/ynh_dev/ynh_dev_host.py b/ynh_dev/ynh_dev_host.py new file mode 100644 index 0000000..dd4c923 --- /dev/null +++ b/ynh_dev/ynh_dev_host.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 + +import argparse +import grp +import logging +import os +import shutil +import subprocess +import sys +from pathlib import Path + +from .libincus import Incus + +DISTS = ["bullseye", "bookworm", "trixie"] +VARIANTS = ["appci", "before-install", "build-and-lint", "core-tests", "demo", "dev"] +BRANCHES = ["stable", "testing", "unstable"] + +YNH_DEV_DIR = Path(__file__).parent.parent + + +def main_host() -> None: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(title="host actions", required=True, dest="action") + sub.add_parser("init", help="Download source repositories") + sub.add_parser("start", help="(Create and) starts a box") + sub.add_parser("attach", help="Attach an already started box") + sub.add_parser("destroy", help="Destroy the ynh-dev box") + + parser.add_argument("-d", "--dist", type=str, choices=DISTS, default="bookworm") + parser.add_argument("-v", "--variant", type=str, choices=VARIANTS, default="core-tests") + parser.add_argument("-b", "--ynh_branch", type=str, choices=BRANCHES, default="stable") + parser.add_argument("-n", "--name", type=str) + + args = parser.parse_args() + logging.getLogger().setLevel(logging.INFO) + + check_incus_setup() + ensure_incus_remote() + + image = f"yunohost/{args.dist}-{args.ynh_branch}/{args.variant}" + container = f"ynh-dev-{args.dist}-{args.ynh_branch}-{args.variant}{f'-{args.name}' if args.name else ''}" + + incus = Incus() + + match args.action: + case "init": + init() + case "start": + if not incus.image_exists(image): + logging.info(f"Downloading {image}...") + incus.image_download("yunohost:" + image) + if incus.instance_exists(container): + logging.warning(f"Container {container} is already started!") + + else: + logging.info(f"Launching {image} -> {container}...") + incus.launch(image, container, "-c", "security.nesting=true", "-c", "security.privileged=true") + incus._run( + "config", + "device", + "add", + container, + "ynh-dev-shared-folder", + "disk", + "path=/ynh-dev", + f"source={YNH_DEV_DIR}", + ) + + incus.execute(container, "dhclient") + attach(incus, container) + + case "attach": + attach(incus, container) + + case "destroy": + incus.instance_stop(container) + incus.instance_delete(container) + + + +def clone_or_pull(url: str, path: Path) -> None: + if path.exists(): + subprocess.check_call(["git", "pull"], cwd=path) + else: + subprocess.check_call(["git", "clone", url, str(path)]) + + +def init() -> None: + clone_or_pull("https://github.com/YunoHost/moulinette", YNH_DEV_DIR / "moulinette") + clone_or_pull("https://github.com/YunoHost/yunohost", YNH_DEV_DIR / "yunohost") + clone_or_pull("https://github.com/YunoHost/yunohost-admin", YNH_DEV_DIR / "yunohost-admin") + clone_or_pull("https://github.com/YunoHost/SSOwat ssowat", YNH_DEV_DIR / "SSOwat ssowat") + clone_or_pull("https://github.com/YunoHost/yunohost-portal", YNH_DEV_DIR / "yunohost-portal") + (YNH_DEV_DIR / "apps").mkdir(exist_ok=True) + + +def attach(incus: Incus, container: str) -> None: + logging.info(f"Attaching to {container}.") + incus.execute(container, "/bin/bash", cwd="/ynh-dev", exec_=True) + + +def check_incus_setup() -> None: + if shutil.which("incus") is None: + logging.error( + "You need to have Incus installed for ynh-dev to be usable from the host machine. " + "Refer to the README to know how to install it." + ) + sys.exit(1) + incus_group = grp.getgrnam("incus-admin").gr_gid + if incus_group not in os.getgroups(): + logging.error("You need to be in the incus-admin group!") + sys.exit(1) + + if "incusbr0" not in subprocess.check_output(["ip", "addr"]).decode("utf-8"): + logging.warning("There is no 'incusbr0' interface... Did you ran 'incus admin init' ?") + + +def ensure_incus_remote() -> None: + remote_url = "https://repo.yunohost.org/incus/" + if ynh_remote := Incus().remotes().get("yunohost"): + if (url := ynh_remote["Addr"]) != remote_url: + logging.error(f"Remote yunohost has url {url} instead of {remote_url}!") + sys.exit(1) + return + Incus().remote_add("yunohost", remote_url, True, "simplestreams") + + +if __name__ == "__main__": + main_host()