From 6045f0d012767744e16f69fd500d551794e69688 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 11 Mar 2026 02:36:53 -0700 Subject: [PATCH 01/17] Add ArchiveBox as a new community app ArchiveBox is a self-hosted internet archiving solution that saves websites as HTML, PDF, screenshots, WARC, and more. This adds it to the TrueNAS community app catalog with optional Sonic full-text search backend support. Co-Authored-By: Claude Opus 4.6 --- ix-dev/community/archivebox/README.md | 3 + ix-dev/community/archivebox/app.yaml | 37 ++ ix-dev/community/archivebox/ix_values.yaml | 12 + ix-dev/community/archivebox/questions.yaml | 623 ++++++++++++++++++ .../archivebox/templates/docker-compose.yaml | 64 ++ .../templates/test_values/basic-values.yaml | 38 ++ 6 files changed, 777 insertions(+) create mode 100644 ix-dev/community/archivebox/README.md create mode 100644 ix-dev/community/archivebox/app.yaml create mode 100644 ix-dev/community/archivebox/ix_values.yaml create mode 100644 ix-dev/community/archivebox/questions.yaml create mode 100644 ix-dev/community/archivebox/templates/docker-compose.yaml create mode 100644 ix-dev/community/archivebox/templates/test_values/basic-values.yaml diff --git a/ix-dev/community/archivebox/README.md b/ix-dev/community/archivebox/README.md new file mode 100644 index 00000000000..5ca95d821c6 --- /dev/null +++ b/ix-dev/community/archivebox/README.md @@ -0,0 +1,3 @@ +# ArchiveBox + +[ArchiveBox](https://archivebox.io) is a powerful, self-hosted internet archiving solution to collect, save, and view websites offline. diff --git a/ix-dev/community/archivebox/app.yaml b/ix-dev/community/archivebox/app.yaml new file mode 100644 index 00000000000..ce73b50fca2 --- /dev/null +++ b/ix-dev/community/archivebox/app.yaml @@ -0,0 +1,37 @@ +app_version: 0.7.3 +capabilities: [] +categories: +- productivity +date_added: '2026-03-11' +description: ArchiveBox is a powerful, self-hosted internet archiving solution to + collect, save, and view websites offline. Save pages as HTML, PDF, screenshots, + WARC, and more. +home: https://archivebox.io +host_mounts: [] +icon: https://media.sys.truenas.net/apps/archivebox/icons/icon.png +keywords: +- archiving +- bookmarks +- web +- wayback +- preservation +lib_version: 2.2.8 +lib_version_hash: "" +maintainers: +- email: dev@truenas.com + name: truenas + url: https://www.truenas.com/ +name: archivebox +run_as_context: +- description: Container [archivebox] runs as root user and group. + gid: 0 + group_name: Host group is [root] + uid: 0 + user_name: Host user is [root] +screenshots: [] +sources: +- https://archivebox.io +- https://github.com/ArchiveBox/ArchiveBox +title: ArchiveBox +train: community +version: 1.0.0 diff --git a/ix-dev/community/archivebox/ix_values.yaml b/ix-dev/community/archivebox/ix_values.yaml new file mode 100644 index 00000000000..a52ebcf2b15 --- /dev/null +++ b/ix-dev/community/archivebox/ix_values.yaml @@ -0,0 +1,12 @@ +images: + image: + repository: archivebox/archivebox + tag: 0.7.3 + sonic_image: + repository: archivebox/sonic + tag: latest + +consts: + archivebox_container_name: archivebox + sonic_container_name: sonic + perms_container_name: permissions diff --git a/ix-dev/community/archivebox/questions.yaml b/ix-dev/community/archivebox/questions.yaml new file mode 100644 index 00000000000..e532965f727 --- /dev/null +++ b/ix-dev/community/archivebox/questions.yaml @@ -0,0 +1,623 @@ +groups: + - name: ArchiveBox Configuration + description: Configure ArchiveBox + - name: Network Configuration + description: Configure Network for ArchiveBox + - name: Storage Configuration + description: Configure Storage for ArchiveBox + - name: Labels Configuration + description: Configure Labels for ArchiveBox + - name: Resources Configuration + description: Configure Resources for ArchiveBox + +questions: + - variable: TZ + group: ArchiveBox Configuration + label: Timezone + schema: + type: string + default: Etc/UTC + required: true + $ref: + - definitions/timezone + - variable: archivebox + label: "" + group: ArchiveBox Configuration + schema: + type: dict + attrs: + - variable: admin_username + label: Admin Username + description: The admin username for ArchiveBox (only used on first run). + schema: + type: string + default: "admin" + required: true + - variable: admin_password + label: Admin Password + description: The admin password for ArchiveBox (only used on first run). + schema: + type: string + default: "" + required: true + private: true + - variable: public_index + label: Public Index + description: Allow anonymous users to view the snapshot list. + schema: + type: boolean + default: true + - variable: public_snapshots + label: Public Snapshots + description: Allow anonymous users to view snapshot content. + schema: + type: boolean + default: true + - variable: public_add_view + label: Public Add View + description: Allow anonymous users to submit new URLs to archive. + schema: + type: boolean + default: false + - variable: search_backend_enabled + label: Enable Sonic Search + description: | + Enable the Sonic full-text search backend for faster searching.
+ This will deploy an additional Sonic container. + schema: + type: boolean + default: true + - variable: search_backend_password + label: Sonic Search Password + description: Password for the Sonic search backend. + schema: + type: string + default: "" + required: true + private: true + show_if: [["search_backend_enabled", "=", true]] + - variable: additional_envs + label: Additional Environment Variables + description: | + Additional environment variables for ArchiveBox.
+ See https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration for all options. + schema: + type: list + default: [] + items: + - variable: env + label: Environment Variable + schema: + type: dict + attrs: + - variable: name + label: Name + schema: + type: string + required: true + - variable: value + label: Value + schema: + type: string + + - variable: network + label: "" + group: Network Configuration + schema: + type: dict + attrs: + - variable: web_port + label: WebUI Port + schema: + type: dict + attrs: + - variable: bind_mode + label: Port Bind Mode + description: | + The port bind mode.
+ - Publish: The port will be published on the host for external access.
+ - Expose: The port will be exposed for inter-container communication.
+ - None: The port will not be exposed or published.
+ Note: If the Dockerfile defines an EXPOSE directive, + the port will still be exposed for inter-container communication regardless of this setting. + schema: + type: string + default: "published" + enum: + - value: "published" + description: Publish port on the host for external access + - value: "exposed" + description: Expose port for inter-container communication + - value: "" + description: None + - variable: port_number + label: Port Number + schema: + type: int + default: 30820 + min: 1 + max: 65535 + required: true + - variable: host_ips + label: Host IPs + description: IPs on the host to bind this port + schema: + type: list + show_if: [["bind_mode", "=", "published"]] + default: [] + items: + - variable: host_ip + label: Host IP + schema: + type: string + required: true + $ref: + - definitions/node_bind_ip + - variable: networks + label: Networks + description: The docker networks to join + schema: + type: list + default: [] + items: + - variable: network + label: Network + schema: + type: dict + attrs: + - variable: name + label: Name + description: | + The name of the network to join.
+ The network must already exist. + schema: + type: string + default: "" + required: true + - variable: containers + label: Containers + description: The containers to add to this network. + schema: + type: list + items: + - variable: container + label: Container + schema: + type: dict + attrs: + - variable: name + label: Container Name + schema: + type: string + required: true + enum: + - value: archivebox + description: archivebox + - value: sonic + description: sonic + - variable: config + label: Container Network Configuration + schema: + type: dict + attrs: + - variable: aliases + label: Aliases (Optional) + description: The network aliases to use for this container on this network. + schema: + type: list + default: [] + items: + - variable: alias + label: Alias + schema: + type: string + - variable: interface_name + label: Interface Name (Optional) + description: The network interface name to use for this network + schema: + type: string + - variable: mac_address + label: MAC Address (Optional) + description: The MAC address to use for this network interface. + schema: + type: string + - variable: ipv4_address + label: IPv4 Address (Optional) + description: The IPv4 address to use for this network interface. + schema: + type: string + - variable: ipv6_address + label: IPv6 Address (Optional) + description: The IPv6 address to use for this network interface. + schema: + type: string + - variable: gw_priority + label: Gateway Priority (Optional) + description: Indicates the priority of the gateway for this network interface. + schema: + type: int + "null": true + - variable: priority + label: Priority (Optional) + description: Indicates in which order Compose connects the service's containers to its networks. + schema: + type: int + "null": true + + - variable: storage + label: "" + group: Storage Configuration + schema: + type: dict + attrs: + - variable: data + label: ArchiveBox Data Storage + description: Stores all archived data, configuration, and the SQLite database. + schema: + type: dict + attrs: + - variable: type + label: Type + description: | + ixVolume: Is dataset created automatically by the system.
+ Host Path: Is a path that already exists on the system. + schema: + type: string + required: true + default: "ix_volume" + enum: + - value: "host_path" + description: Host Path (Path that already exists on the system) + - value: "ix_volume" + description: ixVolume (Dataset created automatically by the system) + - variable: ix_volume_config + label: ixVolume Configuration + description: The configuration for the ixVolume dataset. + schema: + type: dict + show_if: [["type", "=", "ix_volume"]] + $ref: + - "normalize/ix_volume" + attrs: + - variable: acl_enable + label: Enable ACL + description: Enable ACL for the storage. + schema: + type: boolean + default: false + - variable: dataset_name + label: Dataset Name + description: The name of the dataset to use for storage. + schema: + type: string + required: true + hidden: true + default: "data" + - variable: acl_entries + label: ACL Configuration + schema: + type: dict + show_if: [["acl_enable", "=", true]] + attrs: [] + - variable: host_path_config + label: Host Path Configuration + schema: + type: dict + show_if: [["type", "=", "host_path"]] + attrs: + - variable: acl_enable + label: Enable ACL + description: Enable ACL for the storage. + schema: + type: boolean + default: false + - variable: acl + label: ACL Configuration + schema: + type: dict + show_if: [["acl_enable", "=", true]] + attrs: [] + $ref: + - "normalize/acl" + - variable: path + label: Host Path + description: The host path to use for storage. + schema: + type: hostpath + show_if: [["acl_enable", "=", false]] + required: true + - variable: sonic_data + label: Sonic Search Data Storage + description: Stores the Sonic full-text search index. + schema: + type: dict + attrs: + - variable: type + label: Type + description: | + ixVolume: Is dataset created automatically by the system.
+ Host Path: Is a path that already exists on the system. + schema: + type: string + required: true + default: "ix_volume" + enum: + - value: "host_path" + description: Host Path (Path that already exists on the system) + - value: "ix_volume" + description: ixVolume (Dataset created automatically by the system) + - variable: ix_volume_config + label: ixVolume Configuration + description: The configuration for the ixVolume dataset. + schema: + type: dict + show_if: [["type", "=", "ix_volume"]] + $ref: + - "normalize/ix_volume" + attrs: + - variable: acl_enable + label: Enable ACL + description: Enable ACL for the storage. + schema: + type: boolean + default: false + - variable: dataset_name + label: Dataset Name + description: The name of the dataset to use for storage. + schema: + type: string + required: true + hidden: true + default: "sonic_data" + - variable: acl_entries + label: ACL Configuration + schema: + type: dict + show_if: [["acl_enable", "=", true]] + attrs: [] + - variable: host_path_config + label: Host Path Configuration + schema: + type: dict + show_if: [["type", "=", "host_path"]] + attrs: + - variable: acl_enable + label: Enable ACL + description: Enable ACL for the storage. + schema: + type: boolean + default: false + - variable: acl + label: ACL Configuration + schema: + type: dict + show_if: [["acl_enable", "=", true]] + attrs: [] + $ref: + - "normalize/acl" + - variable: path + label: Host Path + description: The host path to use for storage. + schema: + type: hostpath + show_if: [["acl_enable", "=", false]] + required: true + - variable: additional_storage + label: Additional Storage + schema: + type: list + default: [] + items: + - variable: storageEntry + label: Storage Entry + schema: + type: dict + attrs: + - variable: type + label: Type + description: | + ixVolume: Is dataset created automatically by the system.
+ Host Path: Is a path that already exists on the system.
+ SMB Share: Is a SMB share that is mounted to as a volume.
+ NFS Share: Is a NFS share that is mounted to as a volume. + schema: + type: string + required: true + default: "ix_volume" + enum: + - value: "host_path" + description: Host Path (Path that already exists on the system) + - value: "ix_volume" + description: ixVolume (Dataset created automatically by the system) + - value: "cifs" + description: SMB/CIFS Share (Mounts a volume to a SMB share) + - value: "nfs" + description: NFS Share (Mounts a volume to a NFS share) + - variable: read_only + label: Read Only + description: Mount the volume as read only. + schema: + type: boolean + default: false + - variable: mount_path + label: Mount Path + description: The path inside the container to mount the storage. + schema: + type: path + required: true + - variable: host_path_config + label: Host Path Configuration + schema: + type: dict + show_if: [["type", "=", "host_path"]] + attrs: + - variable: acl_enable + label: Enable ACL + description: Enable ACL for the storage. + schema: + type: boolean + default: false + - variable: acl + label: ACL Configuration + schema: + type: dict + show_if: [["acl_enable", "=", true]] + attrs: [] + $ref: + - "normalize/acl" + - variable: path + label: Host Path + description: The host path to use for storage. + schema: + type: hostpath + show_if: [["acl_enable", "=", false]] + required: true + - variable: ix_volume_config + label: ixVolume Configuration + description: The configuration for the ixVolume dataset. + schema: + type: dict + show_if: [["type", "=", "ix_volume"]] + $ref: + - "normalize/ix_volume" + attrs: + - variable: acl_enable + label: Enable ACL + description: Enable ACL for the storage. + schema: + type: boolean + default: false + - variable: dataset_name + label: Dataset Name + description: The name of the dataset to use for storage. + schema: + type: string + required: true + default: "storage_entry" + - variable: acl_entries + label: ACL Configuration + schema: + type: dict + show_if: [["acl_enable", "=", true]] + attrs: [] + $ref: + - "normalize/acl" + - variable: cifs_config + label: SMB Configuration + description: The configuration for the SMB dataset. + schema: + type: dict + show_if: [["type", "=", "cifs"]] + attrs: + - variable: server + label: Server + description: The server to mount the SMB share. + schema: + type: string + required: true + - variable: path + label: Path + description: The path to mount the SMB share. + schema: + type: string + required: true + - variable: username + label: Username + description: The username to use for the SMB share. + schema: + type: string + required: true + - variable: password + label: Password + description: The password to use for the SMB share. + schema: + type: string + required: true + private: true + - variable: domain + label: Domain + description: The domain to use for the SMB share. + schema: + type: string + - variable: nfs_config + label: NFS Configuration + description: The configuration for the NFS dataset. + schema: + type: dict + show_if: [["type", "=", "nfs"]] + attrs: + - variable: server + label: Server + description: The server to mount the NFS share. + schema: + type: string + required: true + - variable: path + label: Path + description: The path to mount the NFS share. + schema: + type: string + required: true + - variable: labels + label: "" + group: Labels Configuration + schema: + type: list + default: [] + items: + - variable: label + label: Label + schema: + type: dict + attrs: + - variable: key + label: Key + schema: + type: string + required: true + - variable: value + label: Value + schema: + type: string + required: true + - variable: containers + label: Containers + description: Containers where the label should be applied + schema: + type: list + items: + - variable: container + label: Container + schema: + type: string + required: true + enum: + - value: archivebox + description: archivebox + - value: sonic + description: sonic + - variable: resources + label: "" + group: Resources Configuration + schema: + type: dict + attrs: + - variable: limits + label: Limits + schema: + type: dict + attrs: + - variable: cpus + label: CPUs + description: CPUs limit for ArchiveBox. + schema: + type: int + default: 2 + required: true + - variable: memory + label: Memory (in MB) + description: Memory limit for ArchiveBox. + schema: + type: int + default: 4096 + required: true diff --git a/ix-dev/community/archivebox/templates/docker-compose.yaml b/ix-dev/community/archivebox/templates/docker-compose.yaml new file mode 100644 index 00000000000..c2b2c0e1fcb --- /dev/null +++ b/ix-dev/community/archivebox/templates/docker-compose.yaml @@ -0,0 +1,64 @@ +{% set tpl = ix_lib.base.render.Render(values) %} + +{% set archivebox_net = tpl.networks.create_internal("archivebox-net") %} + +{% set c = tpl.add_container(values.consts.archivebox_container_name, "image") %} +{% do c.add_network(archivebox_net) %} +{% set perm_container = tpl.deps.perms(values.consts.perms_container_name) %} + +{# Health check on the web UI port #} +{% do c.healthcheck.set_test("curl", {"port": values.network.web_port.port_number, "path": "/"}) %} + +{# Core environment variables #} +{% do c.environment.add_env("ALLOWED_HOSTS", "*") %} +{% do c.environment.add_env("PUBLIC_INDEX", values.archivebox.public_index) %} +{% do c.environment.add_env("PUBLIC_SNAPSHOTS", values.archivebox.public_snapshots) %} +{% do c.environment.add_env("PUBLIC_ADD_VIEW", values.archivebox.public_add_view) %} + +{# Admin credentials (only used on first run) #} +{% do c.environment.add_env("ADMIN_USERNAME", values.archivebox.admin_username) %} +{% do c.environment.add_env("ADMIN_PASSWORD", values.archivebox.admin_password) %} + +{# Sonic search backend configuration #} +{% if values.archivebox.search_backend_enabled %} + {% set sonic = tpl.add_container(values.consts.sonic_container_name, "sonic_image") %} + {% do sonic.set_user(568, 568) %} + {% do sonic.add_network(archivebox_net) %} + {% do sonic.healthcheck.set_test("netcat", {"port": 1491}) %} + {% do sonic.environment.add_env("SEARCH_BACKEND_PASSWORD", values.archivebox.search_backend_password) %} + {% do sonic.add_storage("/var/lib/sonic/store", values.storage.sonic_data) %} + + {% do c.depends.add_dependency(values.consts.sonic_container_name, "service_healthy") %} + {% do c.environment.add_env("SEARCH_BACKEND_ENGINE", "sonic") %} + {% do c.environment.add_env("SEARCH_BACKEND_HOST_NAME", values.consts.sonic_container_name) %} + {% do c.environment.add_env("SEARCH_BACKEND_PASSWORD", values.archivebox.search_backend_password) %} + + {% do perm_container.add_or_skip_action("sonic_data", values.storage.sonic_data, {"uid": 568, "gid": 568, "mode": "check"}) %} +{% endif %} + +{# Additional user-defined environment variables #} +{% do c.environment.add_user_envs(values.archivebox.additional_envs) %} + +{# Port mapping #} +{% do c.add_port(values.network.web_port) %} + +{# Storage: main data directory #} +{% do c.add_storage("/data", values.storage.data) %} + +{# Additional storage mounts #} +{% for store in values.storage.additional_storage %} + {% do c.add_storage(store.mount_path, store) %} +{% endfor %} + +{# Permissions container #} +{% if perm_container.has_actions() %} + {% do perm_container.activate() %} + {% if values.archivebox.search_backend_enabled %} + {% do sonic.depends.add_dependency(values.consts.perms_container_name, "service_completed_successfully") %} + {% endif %} +{% endif %} + +{# Portal for TrueNAS UI #} +{% do tpl.portals.add(values.network.web_port) %} + +{{ tpl.render() | tojson }} diff --git a/ix-dev/community/archivebox/templates/test_values/basic-values.yaml b/ix-dev/community/archivebox/templates/test_values/basic-values.yaml new file mode 100644 index 00000000000..ba1c24d5084 --- /dev/null +++ b/ix-dev/community/archivebox/templates/test_values/basic-values.yaml @@ -0,0 +1,38 @@ +resources: + limits: + cpus: 2.0 + memory: 4096 + +TZ: Etc/UTC + +archivebox: + admin_username: admin + admin_password: testpassword123 + public_index: true + public_snapshots: true + public_add_view: false + search_backend_enabled: true + search_backend_password: SonicTestPassword + additional_envs: [] + +network: + web_port: + bind_mode: published + port_number: 8080 + +ix_volumes: + data: /opt/tests/mnt/archivebox/data + sonic_data: /opt/tests/mnt/archivebox/sonic-data + +storage: + data: + type: ix_volume + ix_volume_config: + dataset_name: data + create_host_path: true + sonic_data: + type: ix_volume + ix_volume_config: + dataset_name: sonic_data + create_host_path: true + additional_storage: [] From cee422408ab3fb1210d82c5200d592d0d011533e Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 11 Mar 2026 02:55:48 -0700 Subject: [PATCH 02/17] Add CSRF_TRUSTED_ORIGINS and common config options Expose CSRF_TRUSTED_ORIGINS explicitly (defaults to localhost:{port}) so login/API works when accessed via TrueNAS hostname. Also add TIMEOUT, CHECK_SSL_VALIDITY, and SAVE_ARCHIVE_DOT_ORG as first-class config options in the UI. Co-Authored-By: Claude Opus 4.6 --- ix-dev/community/archivebox/questions.yaml | 35 +++++++++++++++++++ .../archivebox/templates/docker-compose.yaml | 12 +++++++ .../templates/test_values/basic-values.yaml | 4 +++ 3 files changed, 51 insertions(+) diff --git a/ix-dev/community/archivebox/questions.yaml b/ix-dev/community/archivebox/questions.yaml index e532965f727..f25b9f9eec2 100644 --- a/ix-dev/community/archivebox/questions.yaml +++ b/ix-dev/community/archivebox/questions.yaml @@ -53,12 +53,47 @@ questions: schema: type: boolean default: true + - variable: csrf_trusted_origins + label: CSRF Trusted Origins + description: | + Comma-separated list of trusted origins for CSRF protection.
+ Must match the URL(s) you use to access ArchiveBox (e.g. http://truenas.local:30820).
+ If left empty, it defaults to http://localhost:{port}. + schema: + type: string + default: "" - variable: public_add_view label: Public Add View description: Allow anonymous users to submit new URLs to archive. schema: type: boolean default: false + - variable: timeout + label: Timeout + description: | + Maximum time in seconds to wait for each archive method to complete.
+ Increase to 120+ if you see many slow downloads timing out. + schema: + type: int + default: 60 + min: 10 + max: 3600 + - variable: check_ssl_validity + label: Check SSL Validity + description: | + Enforce strict SSL certificate checking.
+ Set to false to allow saving URLs with expired or self-signed certificates. + schema: + type: boolean + default: true + - variable: save_archive_dot_org + label: Save to Archive.org + description: | + Submit all archived URLs to Archive.org's Wayback Machine.
+ Set to false to disable this behavior. + schema: + type: boolean + default: true - variable: search_backend_enabled label: Enable Sonic Search description: | diff --git a/ix-dev/community/archivebox/templates/docker-compose.yaml b/ix-dev/community/archivebox/templates/docker-compose.yaml index c2b2c0e1fcb..4c8dfe7dd9a 100644 --- a/ix-dev/community/archivebox/templates/docker-compose.yaml +++ b/ix-dev/community/archivebox/templates/docker-compose.yaml @@ -15,6 +15,18 @@ {% do c.environment.add_env("PUBLIC_SNAPSHOTS", values.archivebox.public_snapshots) %} {% do c.environment.add_env("PUBLIC_ADD_VIEW", values.archivebox.public_add_view) %} +{# CSRF trusted origins - required for login/API to work from non-localhost #} +{% if values.archivebox.csrf_trusted_origins %} + {% do c.environment.add_env("CSRF_TRUSTED_ORIGINS", values.archivebox.csrf_trusted_origins) %} +{% else %} + {% do c.environment.add_env("CSRF_TRUSTED_ORIGINS", "http://localhost:%d"|format(values.network.web_port.port_number)) %} +{% endif %} + +{# Archiving behavior #} +{% do c.environment.add_env("TIMEOUT", values.archivebox.timeout) %} +{% do c.environment.add_env("CHECK_SSL_VALIDITY", values.archivebox.check_ssl_validity) %} +{% do c.environment.add_env("SAVE_ARCHIVE_DOT_ORG", values.archivebox.save_archive_dot_org) %} + {# Admin credentials (only used on first run) #} {% do c.environment.add_env("ADMIN_USERNAME", values.archivebox.admin_username) %} {% do c.environment.add_env("ADMIN_PASSWORD", values.archivebox.admin_password) %} diff --git a/ix-dev/community/archivebox/templates/test_values/basic-values.yaml b/ix-dev/community/archivebox/templates/test_values/basic-values.yaml index ba1c24d5084..dc7f702ddda 100644 --- a/ix-dev/community/archivebox/templates/test_values/basic-values.yaml +++ b/ix-dev/community/archivebox/templates/test_values/basic-values.yaml @@ -8,6 +8,10 @@ TZ: Etc/UTC archivebox: admin_username: admin admin_password: testpassword123 + csrf_trusted_origins: "" + timeout: 60 + check_ssl_validity: true + save_archive_dot_org: true public_index: true public_snapshots: true public_add_view: false From 1966e7f08c119c8ee32958df4d9f44abda966376 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 11 Mar 2026 03:09:19 -0700 Subject: [PATCH 03/17] Add USER_AGENT, COOKIES_FILE, and CHROME_USER_DATA_DIR options Expose these as explicit config options in the TrueNAS UI so users can customize browser identity, pass authentication cookies, and use persistent Chrome profiles for archiving authenticated content. Co-Authored-By: Claude Opus 4.6 --- ix-dev/community/archivebox/questions.yaml | 28 +++++++++++++++++++ .../archivebox/templates/docker-compose.yaml | 9 ++++++ .../templates/test_values/basic-values.yaml | 3 ++ 3 files changed, 40 insertions(+) diff --git a/ix-dev/community/archivebox/questions.yaml b/ix-dev/community/archivebox/questions.yaml index f25b9f9eec2..0a855004920 100644 --- a/ix-dev/community/archivebox/questions.yaml +++ b/ix-dev/community/archivebox/questions.yaml @@ -94,6 +94,34 @@ questions: schema: type: boolean default: true + - variable: user_agent + label: User Agent + description: | + Custom browser User-Agent string to use when fetching pages.
+ Useful to avoid being blocked as a bot by some websites.
+ Leave empty to use the default. + schema: + type: string + default: "" + - variable: cookies_file + label: Cookies File Path + description: | + Path to a Netscape-format cookies.txt file inside the container.
+ Used to pass login sessions to ArchiveBox for archiving authenticated content.
+ Mount the file via Additional Storage and reference its container path here (e.g. /data/cookies.txt). + schema: + type: string + default: "" + - variable: chrome_user_data_dir + label: Chrome User Data Dir + description: | + Path to a Chrome user data directory inside the container.
+ Allows using a persistent Chrome profile with saved logins, extensions, etc.
+ Mount the directory via Additional Storage and reference its container path here
+ (e.g. /data/personas/Default/chrome_profile). + schema: + type: string + default: "" - variable: search_backend_enabled label: Enable Sonic Search description: | diff --git a/ix-dev/community/archivebox/templates/docker-compose.yaml b/ix-dev/community/archivebox/templates/docker-compose.yaml index 4c8dfe7dd9a..95bdca09839 100644 --- a/ix-dev/community/archivebox/templates/docker-compose.yaml +++ b/ix-dev/community/archivebox/templates/docker-compose.yaml @@ -26,6 +26,15 @@ {% do c.environment.add_env("TIMEOUT", values.archivebox.timeout) %} {% do c.environment.add_env("CHECK_SSL_VALIDITY", values.archivebox.check_ssl_validity) %} {% do c.environment.add_env("SAVE_ARCHIVE_DOT_ORG", values.archivebox.save_archive_dot_org) %} +{% if values.archivebox.user_agent %} + {% do c.environment.add_env("USER_AGENT", values.archivebox.user_agent) %} +{% endif %} +{% if values.archivebox.cookies_file %} + {% do c.environment.add_env("COOKIES_FILE", values.archivebox.cookies_file) %} +{% endif %} +{% if values.archivebox.chrome_user_data_dir %} + {% do c.environment.add_env("CHROME_USER_DATA_DIR", values.archivebox.chrome_user_data_dir) %} +{% endif %} {# Admin credentials (only used on first run) #} {% do c.environment.add_env("ADMIN_USERNAME", values.archivebox.admin_username) %} diff --git a/ix-dev/community/archivebox/templates/test_values/basic-values.yaml b/ix-dev/community/archivebox/templates/test_values/basic-values.yaml index dc7f702ddda..4c953d916f9 100644 --- a/ix-dev/community/archivebox/templates/test_values/basic-values.yaml +++ b/ix-dev/community/archivebox/templates/test_values/basic-values.yaml @@ -12,6 +12,9 @@ archivebox: timeout: 60 check_ssl_validity: true save_archive_dot_org: true + user_agent: "" + cookies_file: "" + chrome_user_data_dir: "" public_index: true public_snapshots: true public_add_view: false From 59cd9a5c032d4d54aa89dd19f60ecb8b46492bc7 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 11 Mar 2026 03:52:47 -0700 Subject: [PATCH 04/17] Default CSRF_TRUSTED_ORIGINS to localhost and truenas hostnames Co-Authored-By: Claude Opus 4.6 --- ix-dev/community/archivebox/templates/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ix-dev/community/archivebox/templates/docker-compose.yaml b/ix-dev/community/archivebox/templates/docker-compose.yaml index 95bdca09839..ef156979440 100644 --- a/ix-dev/community/archivebox/templates/docker-compose.yaml +++ b/ix-dev/community/archivebox/templates/docker-compose.yaml @@ -19,7 +19,7 @@ {% if values.archivebox.csrf_trusted_origins %} {% do c.environment.add_env("CSRF_TRUSTED_ORIGINS", values.archivebox.csrf_trusted_origins) %} {% else %} - {% do c.environment.add_env("CSRF_TRUSTED_ORIGINS", "http://localhost:%d"|format(values.network.web_port.port_number)) %} + {% do c.environment.add_env("CSRF_TRUSTED_ORIGINS", "http://localhost:%d,http://truenas:%d"|format(values.network.web_port.port_number, values.network.web_port.port_number)) %} {% endif %} {# Archiving behavior #} From 78944efe37a564f2fe595da51ea83672bcad7398 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 11 Mar 2026 14:52:24 -0700 Subject: [PATCH 05/17] remove some config surface area and add scheduler container --- ix-dev/community/archivebox/app.yaml | 26 ++- ix-dev/community/archivebox/item.yaml | 20 +++ ix-dev/community/archivebox/ix_values.yaml | 5 + ix-dev/community/archivebox/questions.yaml | 93 ++++------- .../archivebox/templates/docker-compose.yaml | 156 +++++++++++------- .../templates/test_values/basic-values.yaml | 5 +- 6 files changed, 180 insertions(+), 125 deletions(-) create mode 100644 ix-dev/community/archivebox/item.yaml diff --git a/ix-dev/community/archivebox/app.yaml b/ix-dev/community/archivebox/app.yaml index ce73b50fca2..bd2b4adb1b3 100644 --- a/ix-dev/community/archivebox/app.yaml +++ b/ix-dev/community/archivebox/app.yaml @@ -10,28 +10,42 @@ home: https://archivebox.io host_mounts: [] icon: https://media.sys.truenas.net/apps/archivebox/icons/icon.png keywords: -- archiving -- bookmarks - web +- internet +- bookmarks +- links +- rss +- archive +- archiving - wayback - preservation +- scraping +- chrome lib_version: 2.2.8 -lib_version_hash: "" +lib_version_hash: 76b40b42163deb26edbad5f973f1bd69b68bd85625bcd8247545194d99b9f912 maintainers: +- email: truenas-yaml@archivebox.io + name: Nick Sweeting + url: https://github.com/ArchiveBox/ArchiveBox - email: dev@truenas.com name: truenas url: https://www.truenas.com/ name: archivebox run_as_context: -- description: Container [archivebox] runs as root user and group. +- description: Container [archivebox] starts as root but internally drops privileges + to whichever user owns the /data dir. gid: 0 group_name: Host group is [root] uid: 0 user_name: Host user is [root] -screenshots: [] +screenshots: +- https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2 +- https://github.com/ArchiveBox/ArchiveBox/assets/511499/dad2bc51-e7e5-484e-bb26-f956ed692d16 +- https://github.com/ArchiveBox/ArchiveBox/assets/511499/ace0954a-ddac-4520-9d18-1c77b1ec50b2 +- https://github.com/ArchiveBox/ArchiveBox/assets/511499/8d67382c-e0ce-4286-89f7-7915f09b930c sources: -- https://archivebox.io - https://github.com/ArchiveBox/ArchiveBox +- https://archivebox.io title: ArchiveBox train: community version: 1.0.0 diff --git a/ix-dev/community/archivebox/item.yaml b/ix-dev/community/archivebox/item.yaml new file mode 100644 index 00000000000..a22ef27b235 --- /dev/null +++ b/ix-dev/community/archivebox/item.yaml @@ -0,0 +1,20 @@ +categories: +- productivity +icon_url: https://media.sys.truenas.net/apps/archivebox/icons/icon.png +screenshots: +- https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2 +- https://github.com/ArchiveBox/ArchiveBox/assets/511499/dad2bc51-e7e5-484e-bb26-f956ed692d16 +- https://github.com/ArchiveBox/ArchiveBox/assets/511499/ace0954a-ddac-4520-9d18-1c77b1ec50b2 +- https://github.com/ArchiveBox/ArchiveBox/assets/511499/8d67382c-e0ce-4286-89f7-7915f09b930c +tags: +- web +- internet +- bookmarks +- links +- rss +- archive +- archiving +- wayback +- preservation +- scraping +- chrome diff --git a/ix-dev/community/archivebox/ix_values.yaml b/ix-dev/community/archivebox/ix_values.yaml index a52ebcf2b15..0824a59dfb0 100644 --- a/ix-dev/community/archivebox/ix_values.yaml +++ b/ix-dev/community/archivebox/ix_values.yaml @@ -2,11 +2,16 @@ images: image: repository: archivebox/archivebox tag: 0.7.3 + container_utils_image: + repository: ixsystems/container-utils + tag: 1.0.2 sonic_image: repository: archivebox/sonic tag: latest consts: archivebox_container_name: archivebox + scheduler_container_name: archivebox_scheduler sonic_container_name: sonic perms_container_name: permissions + internal_web_port: 8000 diff --git a/ix-dev/community/archivebox/questions.yaml b/ix-dev/community/archivebox/questions.yaml index 0a855004920..b7f9099a3b0 100644 --- a/ix-dev/community/archivebox/questions.yaml +++ b/ix-dev/community/archivebox/questions.yaml @@ -26,50 +26,50 @@ questions: schema: type: dict attrs: + - variable: public_url + label: Public URL + description: | + The URL users will use to access this ArchiveBox server (e.g. http://archivebox.example.com:30820).
+ This value is used to fill the CSRF_TRUSTED_ORIGINS and ALLOWED_HOSTS env vars if they are not set. + schema: + type: string + default: "" + required: true - variable: admin_username label: Admin Username - description: The admin username for ArchiveBox (only used on first run). + description: The initial admin username for ArchiveBox (only used on first run). schema: type: string default: "admin" required: true - variable: admin_password label: Admin Password - description: The admin password for ArchiveBox (only used on first run). + description: The initial admin password for ArchiveBox (only used on first run). schema: type: string default: "" required: true private: true - variable: public_index - label: Public Index - description: Allow anonymous users to view the snapshot list. + label: Public Index Visibility + description: Allow anonymous/non-logged-in users to view the archived URLs list. schema: type: boolean - default: true + default: false - variable: public_snapshots - label: Public Snapshots - description: Allow anonymous users to view snapshot content. + label: Public Snapshot Contents + description: Allow anonymous/non-logged-in users to view archived snapshot contents. schema: type: boolean - default: true - - variable: csrf_trusted_origins - label: CSRF Trusted Origins - description: | - Comma-separated list of trusted origins for CSRF protection.
- Must match the URL(s) you use to access ArchiveBox (e.g. http://truenas.local:30820).
- If left empty, it defaults to http://localhost:{port}. - schema: - type: string - default: "" + default: false - variable: public_add_view label: Public Add View - description: Allow anonymous users to submit new URLs to archive. + description: Allow anonymous/non-logged-in users to submit new URLs to archive. schema: type: boolean default: false - variable: timeout - label: Timeout + label: Extraction Time Limit description: | Maximum time in seconds to wait for each archive method to complete.
Increase to 120+ if you see many slow downloads timing out. @@ -78,67 +78,43 @@ questions: default: 60 min: 10 max: 3600 - - variable: check_ssl_validity - label: Check SSL Validity - description: | - Enforce strict SSL certificate checking.
- Set to false to allow saving URLs with expired or self-signed certificates. - schema: - type: boolean - default: true - variable: save_archive_dot_org label: Save to Archive.org description: | - Submit all archived URLs to Archive.org's Wayback Machine.
- Set to false to disable this behavior. + Submit all archived URLs to Archive.org's Wayback Machine. schema: type: boolean - default: true + default: false - variable: user_agent label: User Agent description: | Custom browser User-Agent string to use when fetching pages.
Useful to avoid being blocked as a bot by some websites.
- Leave empty to use the default. + Leave empty to use the default ArchiveBox/vX.X.X user agent. schema: type: string default: "" - variable: cookies_file label: Cookies File Path description: | - Path to a Netscape-format cookies.txt file inside the container.
+ Path to a Netscape-format cookies.txt file mounted inside the container /data volume.
Used to pass login sessions to ArchiveBox for archiving authenticated content.
- Mount the file via Additional Storage and reference its container path here (e.g. /data/cookies.txt). + Mount the file via Additional Storage and reference its container path here (e.g. /data/cookies.txt).
+ See https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration/#cookies_file for more information. schema: type: string default: "" - variable: chrome_user_data_dir label: Chrome User Data Dir description: | - Path to a Chrome user data directory inside the container.
+ Path to a Chrome user data directory inside the container /data volume.
Allows using a persistent Chrome profile with saved logins, extensions, etc.
Mount the directory via Additional Storage and reference its container path here
- (e.g. /data/personas/Default/chrome_profile). + (e.g. /data/personas/Default/chrome_profile).
+ See https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile for more information. schema: type: string default: "" - - variable: search_backend_enabled - label: Enable Sonic Search - description: | - Enable the Sonic full-text search backend for faster searching.
- This will deploy an additional Sonic container. - schema: - type: boolean - default: true - - variable: search_backend_password - label: Sonic Search Password - description: Password for the Sonic search backend. - schema: - type: string - default: "" - required: true - private: true - show_if: [["search_backend_enabled", "=", true]] - variable: additional_envs label: Additional Environment Variables description: | @@ -256,8 +232,8 @@ questions: enum: - value: archivebox description: archivebox - - value: sonic - description: sonic + - value: archivebox_scheduler + description: archivebox_scheduler - variable: config label: Container Network Configuration schema: @@ -657,8 +633,8 @@ questions: enum: - value: archivebox description: archivebox - - value: sonic - description: sonic + - value: archivebox_scheduler + description: archivebox_scheduler - variable: resources label: "" group: Resources Configuration @@ -672,15 +648,16 @@ questions: attrs: - variable: cpus label: CPUs - description: CPUs limit for ArchiveBox. + description: CPUs limit for ArchiveBox (at least 2 recommended). schema: type: int default: 2 required: true - variable: memory label: Memory (in MB) - description: Memory limit for ArchiveBox. + description: Memory limit for ArchiveBox (at least 1536MB recommended). schema: type: int default: 4096 + min: 512 required: true diff --git a/ix-dev/community/archivebox/templates/docker-compose.yaml b/ix-dev/community/archivebox/templates/docker-compose.yaml index ef156979440..0c48d1d10f1 100644 --- a/ix-dev/community/archivebox/templates/docker-compose.yaml +++ b/ix-dev/community/archivebox/templates/docker-compose.yaml @@ -1,82 +1,124 @@ {% set tpl = ix_lib.base.render.Render(values) %} +{% set public_url = tpl.funcs.url_to_dict(values.archivebox.public_url, True) %} +{% set allowed_host = public_url.host %} +{% set sonic_password = "archivebox" %} +{% set user_env = namespace( + allowed_hosts=false, + csrf_trusted_origins=false, + check_ssl_validity=false +) %} +{% if public_url.port %} + {% set allowed_host = "%s:%d"|format(public_url.host, public_url.port) %} +{% endif %} +{% set csrf_trusted_origin = "%s://%s"|format(public_url.scheme, allowed_host) %} +{% for item in values.archivebox.additional_envs %} + {% if item.name == "ALLOWED_HOSTS" %} + {% set user_env.allowed_hosts = true %} + {% elif item.name == "CSRF_TRUSTED_ORIGINS" %} + {% set user_env.csrf_trusted_origins = true %} + {% elif item.name == "CHECK_SSL_VALIDITY" %} + {% set user_env.check_ssl_validity = true %} + {% endif %} +{% endfor %} -{% set archivebox_net = tpl.networks.create_internal("archivebox-net") %} +{% macro add_archivebox_shared_env(container) %} + {# Archiving behavior shared by the web and scheduler containers #} + {% do container.environment.add_env("TIMEOUT", values.archivebox.timeout) %} + {% if not user_env.check_ssl_validity %} + {% do container.environment.add_env("CHECK_SSL_VALIDITY", false) %} + {% endif %} + {% do container.environment.add_env("SAVE_ARCHIVE_DOT_ORG", values.archivebox.save_archive_dot_org) %} + {% if values.archivebox.user_agent %} + {% do container.environment.add_env("USER_AGENT", values.archivebox.user_agent) %} + {% endif %} + {% if values.archivebox.cookies_file %} + {% do container.environment.add_env("COOKIES_FILE", values.archivebox.cookies_file) %} + {% endif %} + {% if values.archivebox.chrome_user_data_dir %} + {% do container.environment.add_env("CHROME_USER_DATA_DIR", values.archivebox.chrome_user_data_dir) %} + {% endif %} -{% set c = tpl.add_container(values.consts.archivebox_container_name, "image") %} -{% do c.add_network(archivebox_net) %} -{% set perm_container = tpl.deps.perms(values.consts.perms_container_name) %} + {# Additional user-defined environment variables #} + {% do container.environment.add_user_envs(values.archivebox.additional_envs) %} +{% endmacro %} -{# Health check on the web UI port #} -{% do c.healthcheck.set_test("curl", {"port": values.network.web_port.port_number, "path": "/"}) %} +{% macro add_archivebox_web_env(container) %} + {# Core environment variables #} + {% if not user_env.allowed_hosts %} + {% do container.environment.add_env("ALLOWED_HOSTS", allowed_host) %} + {% endif %} + {% do container.environment.add_env("PUBLIC_INDEX", values.archivebox.public_index) %} + {% do container.environment.add_env("PUBLIC_SNAPSHOTS", values.archivebox.public_snapshots) %} + {% do container.environment.add_env("PUBLIC_ADD_VIEW", values.archivebox.public_add_view) %} + {% if not user_env.csrf_trusted_origins %} + {% do container.environment.add_env("CSRF_TRUSTED_ORIGINS", csrf_trusted_origin) %} + {% endif %} -{# Core environment variables #} -{% do c.environment.add_env("ALLOWED_HOSTS", "*") %} -{% do c.environment.add_env("PUBLIC_INDEX", values.archivebox.public_index) %} -{% do c.environment.add_env("PUBLIC_SNAPSHOTS", values.archivebox.public_snapshots) %} -{% do c.environment.add_env("PUBLIC_ADD_VIEW", values.archivebox.public_add_view) %} + {# Admin credentials (only used on first run) #} + {% do container.environment.add_env("ADMIN_USERNAME", values.archivebox.admin_username) %} + {% do container.environment.add_env("ADMIN_PASSWORD", values.archivebox.admin_password) %} +{% endmacro %} -{# CSRF trusted origins - required for login/API to work from non-localhost #} -{% if values.archivebox.csrf_trusted_origins %} - {% do c.environment.add_env("CSRF_TRUSTED_ORIGINS", values.archivebox.csrf_trusted_origins) %} -{% else %} - {% do c.environment.add_env("CSRF_TRUSTED_ORIGINS", "http://localhost:%d,http://truenas:%d"|format(values.network.web_port.port_number, values.network.web_port.port_number)) %} -{% endif %} +{% set archivebox_net = tpl.networks.create_internal("archivebox-net") %} -{# Archiving behavior #} -{% do c.environment.add_env("TIMEOUT", values.archivebox.timeout) %} -{% do c.environment.add_env("CHECK_SSL_VALIDITY", values.archivebox.check_ssl_validity) %} -{% do c.environment.add_env("SAVE_ARCHIVE_DOT_ORG", values.archivebox.save_archive_dot_org) %} -{% if values.archivebox.user_agent %} - {% do c.environment.add_env("USER_AGENT", values.archivebox.user_agent) %} -{% endif %} -{% if values.archivebox.cookies_file %} - {% do c.environment.add_env("COOKIES_FILE", values.archivebox.cookies_file) %} -{% endif %} -{% if values.archivebox.chrome_user_data_dir %} - {% do c.environment.add_env("CHROME_USER_DATA_DIR", values.archivebox.chrome_user_data_dir) %} -{% endif %} +{% set archivebox = tpl.add_container(values.consts.archivebox_container_name, "image") %} +{% do archivebox.add_network(archivebox_net) %} +{% set scheduler = tpl.add_container(values.consts.scheduler_container_name, "image") %} +{% do scheduler.add_network(archivebox_net) %} +{% do scheduler.set_command(["schedule", "--foreground", "--update", "--every=day"]) %} +{% set perm_container = tpl.deps.perms(values.consts.perms_container_name) %} -{# Admin credentials (only used on first run) #} -{% do c.environment.add_env("ADMIN_USERNAME", values.archivebox.admin_username) %} -{% do c.environment.add_env("ADMIN_PASSWORD", values.archivebox.admin_password) %} +{# Health checks #} +{% do archivebox.healthcheck.set_test("curl", { + "port": values.consts.internal_web_port, + "path": "/health/", + "host": "127.0.0.1", + "headers": [["Host", allowed_host]] +}) %} +{% do scheduler.healthcheck.set_test("pgrep", {"process": "archivebox schedule --foreground"}) %} -{# Sonic search backend configuration #} -{% if values.archivebox.search_backend_enabled %} - {% set sonic = tpl.add_container(values.consts.sonic_container_name, "sonic_image") %} - {% do sonic.set_user(568, 568) %} - {% do sonic.add_network(archivebox_net) %} - {% do sonic.healthcheck.set_test("netcat", {"port": 1491}) %} - {% do sonic.environment.add_env("SEARCH_BACKEND_PASSWORD", values.archivebox.search_backend_password) %} - {% do sonic.add_storage("/var/lib/sonic/store", values.storage.sonic_data) %} - - {% do c.depends.add_dependency(values.consts.sonic_container_name, "service_healthy") %} - {% do c.environment.add_env("SEARCH_BACKEND_ENGINE", "sonic") %} - {% do c.environment.add_env("SEARCH_BACKEND_HOST_NAME", values.consts.sonic_container_name) %} - {% do c.environment.add_env("SEARCH_BACKEND_PASSWORD", values.archivebox.search_backend_password) %} - - {% do perm_container.add_or_skip_action("sonic_data", values.storage.sonic_data, {"uid": 568, "gid": 568, "mode": "check"}) %} -{% endif %} +{% do add_archivebox_shared_env(archivebox) %} +{% do add_archivebox_web_env(archivebox) %} +{% do add_archivebox_shared_env(scheduler) %} -{# Additional user-defined environment variables #} -{% do c.environment.add_user_envs(values.archivebox.additional_envs) %} +{# Sonic search backend configuration #} +{% set sonic = tpl.add_container(values.consts.sonic_container_name, "sonic_image") %} +{% do sonic.set_user(568, 568) %} +{% do sonic.add_network(archivebox_net) %} +{% do sonic.healthcheck.set_test("netcat", {"port": 1491}) %} +{% do sonic.environment.add_env("SEARCH_BACKEND_PASSWORD", sonic_password) %} +{% do sonic.add_storage("/var/lib/sonic/store", values.storage.sonic_data) %} + +{% do archivebox.depends.add_dependency(values.consts.sonic_container_name, "service_healthy") %} +{% do scheduler.depends.add_dependency(values.consts.sonic_container_name, "service_healthy") %} +{% do archivebox.environment.add_env("SEARCH_BACKEND_ENGINE", "sonic") %} +{% do archivebox.environment.add_env("SEARCH_BACKEND_HOST_NAME", values.consts.sonic_container_name) %} +{% do archivebox.environment.add_env("SEARCH_BACKEND_PASSWORD", sonic_password) %} +{% do scheduler.environment.add_env("SEARCH_BACKEND_ENGINE", "sonic") %} +{% do scheduler.environment.add_env("SEARCH_BACKEND_HOST_NAME", values.consts.sonic_container_name) %} +{% do scheduler.environment.add_env("SEARCH_BACKEND_PASSWORD", sonic_password) %} + +{% do perm_container.add_or_skip_action("sonic_data", values.storage.sonic_data, {"uid": 568, "gid": 568, "mode": "check"}) %} {# Port mapping #} -{% do c.add_port(values.network.web_port) %} +{% do archivebox.add_port(values.network.web_port, {"container_port": values.consts.internal_web_port}) %} {# Storage: main data directory #} -{% do c.add_storage("/data", values.storage.data) %} +{% do archivebox.add_storage("/data", values.storage.data) %} +{% do scheduler.add_storage("/data", values.storage.data) %} {# Additional storage mounts #} {% for store in values.storage.additional_storage %} - {% do c.add_storage(store.mount_path, store) %} + {% do archivebox.add_storage(store.mount_path, store) %} + {% do scheduler.add_storage(store.mount_path, store) %} {% endfor %} {# Permissions container #} {% if perm_container.has_actions() %} {% do perm_container.activate() %} - {% if values.archivebox.search_backend_enabled %} - {% do sonic.depends.add_dependency(values.consts.perms_container_name, "service_completed_successfully") %} - {% endif %} + {% do archivebox.depends.add_dependency(values.consts.perms_container_name, "service_completed_successfully") %} + {% do scheduler.depends.add_dependency(values.consts.perms_container_name, "service_completed_successfully") %} + {% do sonic.depends.add_dependency(values.consts.perms_container_name, "service_completed_successfully") %} {% endif %} {# Portal for TrueNAS UI #} diff --git a/ix-dev/community/archivebox/templates/test_values/basic-values.yaml b/ix-dev/community/archivebox/templates/test_values/basic-values.yaml index 4c953d916f9..b42e50382d2 100644 --- a/ix-dev/community/archivebox/templates/test_values/basic-values.yaml +++ b/ix-dev/community/archivebox/templates/test_values/basic-values.yaml @@ -8,9 +8,8 @@ TZ: Etc/UTC archivebox: admin_username: admin admin_password: testpassword123 - csrf_trusted_origins: "" + public_url: http://archivebox.example.com:8080 timeout: 60 - check_ssl_validity: true save_archive_dot_org: true user_agent: "" cookies_file: "" @@ -18,8 +17,6 @@ archivebox: public_index: true public_snapshots: true public_add_view: false - search_backend_enabled: true - search_backend_password: SonicTestPassword additional_envs: [] network: From 97ff2bb64d6addc7a4c4d282ba9bcbe5fdfdabc7 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 11 Mar 2026 15:01:29 -0700 Subject: [PATCH 06/17] use matching internal and external port number for clarity --- ix-dev/community/archivebox/ix_values.yaml | 1 - ix-dev/community/archivebox/templates/docker-compose.yaml | 5 +++-- .../archivebox/templates/test_values/basic-values.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ix-dev/community/archivebox/ix_values.yaml b/ix-dev/community/archivebox/ix_values.yaml index 0824a59dfb0..7957d3b14e2 100644 --- a/ix-dev/community/archivebox/ix_values.yaml +++ b/ix-dev/community/archivebox/ix_values.yaml @@ -14,4 +14,3 @@ consts: scheduler_container_name: archivebox_scheduler sonic_container_name: sonic perms_container_name: permissions - internal_web_port: 8000 diff --git a/ix-dev/community/archivebox/templates/docker-compose.yaml b/ix-dev/community/archivebox/templates/docker-compose.yaml index 0c48d1d10f1..3104e83e4fa 100644 --- a/ix-dev/community/archivebox/templates/docker-compose.yaml +++ b/ix-dev/community/archivebox/templates/docker-compose.yaml @@ -63,6 +63,7 @@ {% set archivebox = tpl.add_container(values.consts.archivebox_container_name, "image") %} {% do archivebox.add_network(archivebox_net) %} +{% do archivebox.set_command(["server", "--init", "0.0.0.0:%d"|format(values.network.web_port.port_number)]) %} {% set scheduler = tpl.add_container(values.consts.scheduler_container_name, "image") %} {% do scheduler.add_network(archivebox_net) %} {% do scheduler.set_command(["schedule", "--foreground", "--update", "--every=day"]) %} @@ -70,7 +71,7 @@ {# Health checks #} {% do archivebox.healthcheck.set_test("curl", { - "port": values.consts.internal_web_port, + "port": values.network.web_port.port_number, "path": "/health/", "host": "127.0.0.1", "headers": [["Host", allowed_host]] @@ -101,7 +102,7 @@ {% do perm_container.add_or_skip_action("sonic_data", values.storage.sonic_data, {"uid": 568, "gid": 568, "mode": "check"}) %} {# Port mapping #} -{% do archivebox.add_port(values.network.web_port, {"container_port": values.consts.internal_web_port}) %} +{% do archivebox.add_port(values.network.web_port) %} {# Storage: main data directory #} {% do archivebox.add_storage("/data", values.storage.data) %} diff --git a/ix-dev/community/archivebox/templates/test_values/basic-values.yaml b/ix-dev/community/archivebox/templates/test_values/basic-values.yaml index b42e50382d2..7a07187f77a 100644 --- a/ix-dev/community/archivebox/templates/test_values/basic-values.yaml +++ b/ix-dev/community/archivebox/templates/test_values/basic-values.yaml @@ -8,7 +8,7 @@ TZ: Etc/UTC archivebox: admin_username: admin admin_password: testpassword123 - public_url: http://archivebox.example.com:8080 + public_url: http://archivebox.example.com:30820 timeout: 60 save_archive_dot_org: true user_agent: "" @@ -22,7 +22,7 @@ archivebox: network: web_port: bind_mode: published - port_number: 8080 + port_number: 30820 ix_volumes: data: /opt/tests/mnt/archivebox/data From 1eb6d10c2075d0a30c0ddbdc49ad6dc018727beb Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Thu, 12 Mar 2026 18:46:42 +0200 Subject: [PATCH 07/17] initial cleanup --- cspell.config.yaml | 1 + ix-dev/community/archivebox/app.yaml | 5 +- ix-dev/community/archivebox/ix_values.yaml | 7 +- ix-dev/community/archivebox/questions.yaml | 75 +++++---- .../archivebox/templates/docker-compose.yaml | 152 +++++++----------- .../templates/test_values/basic-values.yaml | 4 + 6 files changed, 108 insertions(+), 136 deletions(-) diff --git a/cspell.config.yaml b/cspell.config.yaml index 1adc136d1e7..a5dd7a7c367 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -13,6 +13,7 @@ words: - apikey - arcadiatechnology - archisteamfarm + - archivebox - archiver - arti - asigra diff --git a/ix-dev/community/archivebox/app.yaml b/ix-dev/community/archivebox/app.yaml index bd2b4adb1b3..7e9487e9118 100644 --- a/ix-dev/community/archivebox/app.yaml +++ b/ix-dev/community/archivebox/app.yaml @@ -2,7 +2,7 @@ app_version: 0.7.3 capabilities: [] categories: - productivity -date_added: '2026-03-11' +date_added: '2026-03-12' description: ArchiveBox is a powerful, self-hosted internet archiving solution to collect, save, and view websites offline. Save pages as HTML, PDF, screenshots, WARC, and more. @@ -24,9 +24,6 @@ keywords: lib_version: 2.2.8 lib_version_hash: 76b40b42163deb26edbad5f973f1bd69b68bd85625bcd8247545194d99b9f912 maintainers: -- email: truenas-yaml@archivebox.io - name: Nick Sweeting - url: https://github.com/ArchiveBox/ArchiveBox - email: dev@truenas.com name: truenas url: https://www.truenas.com/ diff --git a/ix-dev/community/archivebox/ix_values.yaml b/ix-dev/community/archivebox/ix_values.yaml index 7957d3b14e2..2db33599d2a 100644 --- a/ix-dev/community/archivebox/ix_values.yaml +++ b/ix-dev/community/archivebox/ix_values.yaml @@ -7,10 +7,13 @@ images: tag: 1.0.2 sonic_image: repository: archivebox/sonic - tag: latest + tag: 1.4.9 consts: archivebox_container_name: archivebox - scheduler_container_name: archivebox_scheduler + scheduler_container_name: scheduler sonic_container_name: sonic perms_container_name: permissions + + data_path: /data + internal_sonic_port: 1491 diff --git a/ix-dev/community/archivebox/questions.yaml b/ix-dev/community/archivebox/questions.yaml index b7f9099a3b0..2ac62a15d54 100644 --- a/ix-dev/community/archivebox/questions.yaml +++ b/ix-dev/community/archivebox/questions.yaml @@ -1,6 +1,8 @@ groups: - name: ArchiveBox Configuration description: Configure ArchiveBox + - name: User and Group Configuration + description: Configure User and Group for ArchiveBox - name: Network Configuration description: Configure Network for ArchiveBox - name: Storage Configuration @@ -20,6 +22,7 @@ questions: required: true $ref: - definitions/timezone + - variable: archivebox label: "" group: ArchiveBox Configuration @@ -29,10 +32,10 @@ questions: - variable: public_url label: Public URL description: | - The URL users will use to access this ArchiveBox server (e.g. http://archivebox.example.com:30820).
+ The URL users will use to access this ArchiveBox server (e.g. http://archivebox.example.com:30387).
This value is used to fill the CSRF_TRUSTED_ORIGINS and ALLOWED_HOSTS env vars if they are not set. schema: - type: string + type: uri default: "" required: true - variable: admin_username @@ -40,7 +43,7 @@ questions: description: The initial admin username for ArchiveBox (only used on first run). schema: type: string - default: "admin" + default: "" required: true - variable: admin_password label: Admin Password @@ -80,8 +83,7 @@ questions: max: 3600 - variable: save_archive_dot_org label: Save to Archive.org - description: | - Submit all archived URLs to Archive.org's Wayback Machine. + description: Submit all archived URLs to Archive.org's Wayback Machine. schema: type: boolean default: false @@ -94,27 +96,6 @@ questions: schema: type: string default: "" - - variable: cookies_file - label: Cookies File Path - description: | - Path to a Netscape-format cookies.txt file mounted inside the container /data volume.
- Used to pass login sessions to ArchiveBox for archiving authenticated content.
- Mount the file via Additional Storage and reference its container path here (e.g. /data/cookies.txt).
- See https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration/#cookies_file for more information. - schema: - type: string - default: "" - - variable: chrome_user_data_dir - label: Chrome User Data Dir - description: | - Path to a Chrome user data directory inside the container /data volume.
- Allows using a persistent Chrome profile with saved logins, extensions, etc.
- Mount the directory via Additional Storage and reference its container path here
- (e.g. /data/personas/Default/chrome_profile).
- See https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile for more information. - schema: - type: string - default: "" - variable: additional_envs label: Additional Environment Variables description: | @@ -138,6 +119,28 @@ questions: label: Value schema: type: string + - variable: run_as + label: "" + group: User and Group Configuration + schema: + type: dict + attrs: + - variable: user + label: User ID + description: The user id that ArchiveBox files will be owned by. + schema: + type: int + min: 568 + default: 568 + required: true + - variable: group + label: Group ID + description: The group id that ArchiveBox files will be owned by. + schema: + type: int + min: 568 + default: 568 + required: true - variable: network label: "" @@ -173,7 +176,7 @@ questions: label: Port Number schema: type: int - default: 30820 + default: 30387 min: 1 max: 65535 required: true @@ -232,8 +235,10 @@ questions: enum: - value: archivebox description: archivebox - - value: archivebox_scheduler - description: archivebox_scheduler + - value: scheduler + description: scheduler + - value: sonic + description: sonic - variable: config label: Container Network Configuration schema: @@ -290,7 +295,7 @@ questions: type: dict attrs: - variable: data - label: ArchiveBox Data Storage + label: Data Storage description: Stores all archived data, configuration, and the SQLite database. schema: type: dict @@ -447,7 +452,7 @@ questions: type: list default: [] items: - - variable: storageEntry + - variable: storage_entry label: Storage Entry schema: type: dict @@ -633,8 +638,11 @@ questions: enum: - value: archivebox description: archivebox - - value: archivebox_scheduler - description: archivebox_scheduler + - value: scheduler + description: scheduler + - value: sonic + description: sonic + - variable: resources label: "" group: Resources Configuration @@ -659,5 +667,4 @@ questions: schema: type: int default: 4096 - min: 512 required: true diff --git a/ix-dev/community/archivebox/templates/docker-compose.yaml b/ix-dev/community/archivebox/templates/docker-compose.yaml index 3104e83e4fa..a871f78c132 100644 --- a/ix-dev/community/archivebox/templates/docker-compose.yaml +++ b/ix-dev/community/archivebox/templates/docker-compose.yaml @@ -1,120 +1,81 @@ {% set tpl = ix_lib.base.render.Render(values) %} -{% set public_url = tpl.funcs.url_to_dict(values.archivebox.public_url, True) %} -{% set allowed_host = public_url.host %} -{% set sonic_password = "archivebox" %} -{% set user_env = namespace( - allowed_hosts=false, - csrf_trusted_origins=false, - check_ssl_validity=false -) %} -{% if public_url.port %} - {% set allowed_host = "%s:%d"|format(public_url.host, public_url.port) %} -{% endif %} -{% set csrf_trusted_origin = "%s://%s"|format(public_url.scheme, allowed_host) %} -{% for item in values.archivebox.additional_envs %} - {% if item.name == "ALLOWED_HOSTS" %} - {% set user_env.allowed_hosts = true %} - {% elif item.name == "CSRF_TRUSTED_ORIGINS" %} - {% set user_env.csrf_trusted_origins = true %} - {% elif item.name == "CHECK_SSL_VALIDITY" %} - {% set user_env.check_ssl_validity = true %} - {% endif %} -{% endfor %} - -{% macro add_archivebox_shared_env(container) %} - {# Archiving behavior shared by the web and scheduler containers #} - {% do container.environment.add_env("TIMEOUT", values.archivebox.timeout) %} - {% if not user_env.check_ssl_validity %} - {% do container.environment.add_env("CHECK_SSL_VALIDITY", false) %} - {% endif %} - {% do container.environment.add_env("SAVE_ARCHIVE_DOT_ORG", values.archivebox.save_archive_dot_org) %} - {% if values.archivebox.user_agent %} - {% do container.environment.add_env("USER_AGENT", values.archivebox.user_agent) %} - {% endif %} - {% if values.archivebox.cookies_file %} - {% do container.environment.add_env("COOKIES_FILE", values.archivebox.cookies_file) %} - {% endif %} - {% if values.archivebox.chrome_user_data_dir %} - {% do container.environment.add_env("CHROME_USER_DATA_DIR", values.archivebox.chrome_user_data_dir) %} - {% endif %} - - {# Additional user-defined environment variables #} - {% do container.environment.add_user_envs(values.archivebox.additional_envs) %} -{% endmacro %} - -{% macro add_archivebox_web_env(container) %} - {# Core environment variables #} - {% if not user_env.allowed_hosts %} - {% do container.environment.add_env("ALLOWED_HOSTS", allowed_host) %} - {% endif %} - {% do container.environment.add_env("PUBLIC_INDEX", values.archivebox.public_index) %} - {% do container.environment.add_env("PUBLIC_SNAPSHOTS", values.archivebox.public_snapshots) %} - {% do container.environment.add_env("PUBLIC_ADD_VIEW", values.archivebox.public_add_view) %} - {% if not user_env.csrf_trusted_origins %} - {% do container.environment.add_env("CSRF_TRUSTED_ORIGINS", csrf_trusted_origin) %} - {% endif %} - - {# Admin credentials (only used on first run) #} - {% do container.environment.add_env("ADMIN_USERNAME", values.archivebox.admin_username) %} - {% do container.environment.add_env("ADMIN_PASSWORD", values.archivebox.admin_password) %} -{% endmacro %} {% set archivebox_net = tpl.networks.create_internal("archivebox-net") %} +{% set perm_container = tpl.deps.perms(values.consts.perms_container_name) %} +{% set perms_config = {"uid": values.run_as.user, "gid": values.run_as.group, "mode": "check"}%} + {% set archivebox = tpl.add_container(values.consts.archivebox_container_name, "image") %} {% do archivebox.add_network(archivebox_net) %} -{% do archivebox.set_command(["server", "--init", "0.0.0.0:%d"|format(values.network.web_port.port_number)]) %} + {% set scheduler = tpl.add_container(values.consts.scheduler_container_name, "image") %} {% do scheduler.add_network(archivebox_net) %} -{% do scheduler.set_command(["schedule", "--foreground", "--update", "--every=day"]) %} -{% set perm_container = tpl.deps.perms(values.consts.perms_container_name) %} - -{# Health checks #} -{% do archivebox.healthcheck.set_test("curl", { - "port": values.network.web_port.port_number, - "path": "/health/", - "host": "127.0.0.1", - "headers": [["Host", allowed_host]] -}) %} -{% do scheduler.healthcheck.set_test("pgrep", {"process": "archivebox schedule --foreground"}) %} - -{% do add_archivebox_shared_env(archivebox) %} -{% do add_archivebox_web_env(archivebox) %} -{% do add_archivebox_shared_env(scheduler) %} -{# Sonic search backend configuration #} {% set sonic = tpl.add_container(values.consts.sonic_container_name, "sonic_image") %} -{% do sonic.set_user(568, 568) %} {% do sonic.add_network(archivebox_net) %} -{% do sonic.healthcheck.set_test("netcat", {"port": 1491}) %} -{% do sonic.environment.add_env("SEARCH_BACKEND_PASSWORD", sonic_password) %} -{% do sonic.add_storage("/var/lib/sonic/store", values.storage.sonic_data) %} -{% do archivebox.depends.add_dependency(values.consts.sonic_container_name, "service_healthy") %} -{% do scheduler.depends.add_dependency(values.consts.sonic_container_name, "service_healthy") %} -{% do archivebox.environment.add_env("SEARCH_BACKEND_ENGINE", "sonic") %} -{% do archivebox.environment.add_env("SEARCH_BACKEND_HOST_NAME", values.consts.sonic_container_name) %} -{% do archivebox.environment.add_env("SEARCH_BACKEND_PASSWORD", sonic_password) %} -{% do scheduler.environment.add_env("SEARCH_BACKEND_ENGINE", "sonic") %} -{% do scheduler.environment.add_env("SEARCH_BACKEND_HOST_NAME", values.consts.sonic_container_name) %} -{% do scheduler.environment.add_env("SEARCH_BACKEND_PASSWORD", sonic_password) %} +{% set sonic_password = tpl.funcs.secure_string(32) %} +{% set containers = [archivebox, scheduler] %} +{% for c in containers %} + {% do c.add_caps(["CHOWN", "SETGID", "SETUID", "FOWNER", "DAC_OVERRIDE"]) %} + {% do c.depends.add_dependency(values.consts.sonic_container_name, "service_healthy") %} + + {% do c.environment.add_env("DATA_DIR", values.archivebox.timeout) %} + {% do c.environment.add_env("TIMEOUT", values.archivebox.timeout) %} + {% do c.environment.add_env("SAVE_ARCHIVE_DOT_ORG", values.archivebox.save_archive_dot_org) %} + {% if values.archivebox.user_agent %} + {% do c.environment.add_env("USER_AGENT", values.archivebox.user_agent) %} + {% endif %} + + {% do c.environment.add_env("SEARCH_BACKEND_ENGINE", "sonic") %} + {% do c.environment.add_env("SEARCH_BACKEND_PORT", values.consts.internal_sonic_port) %} + {% do c.environment.add_env("SEARCH_BACKEND_HOST_NAME", values.consts.sonic_container_name) %} + {% do c.environment.add_env("SEARCH_BACKEND_PASSWORD", sonic_password) %} + {% do c.environment.add_user_envs(values.archivebox.additional_envs) %} + + {% do c.add_storage(values.consts.data_path, values.storage.data) %} +{% endfor %} -{% do perm_container.add_or_skip_action("sonic_data", values.storage.sonic_data, {"uid": 568, "gid": 568, "mode": "check"}) %} -{# Port mapping #} +{# TODO: Fix this +{% set public_url = tpl.funcs.url_to_dict(values.archivebox.public_url, True) %} +{% set allowed_host = public_url.host %} + +{% do container.environment.add_env("ALLOWED_HOSTS", allowed_host) %} +{% do container.environment.add_env("CSRF_TRUSTED_ORIGINS", csrf_trusted_origin) %} +{% do container.environment.add_env("CHECK_SSL_VALIDITY", csrf_trusted_origin) %} + +"headers": [["Host", allowed_host]] +#} + +{% do archivebox.set_command(["server", "--init", "0.0.0.0:%d"|format(values.network.web_port.port_number)]) %} +{% do archivebox.healthcheck.set_test("curl", {"port": values.network.web_port.port_number, "path": "/health/"}) %} +{% do archivebox.environment.add_env("PUBLIC_INDEX", values.archivebox.public_index) %} +{% do archivebox.environment.add_env("PUBLIC_SNAPSHOTS", values.archivebox.public_snapshots) %} +{% do archivebox.environment.add_env("PUBLIC_ADD_VIEW", values.archivebox.public_add_view) %} +{% do archivebox.environment.add_env("ADMIN_USERNAME", values.archivebox.admin_username) %} +{% do archivebox.environment.add_env("ADMIN_PASSWORD", values.archivebox.admin_password) %} + {% do archivebox.add_port(values.network.web_port) %} -{# Storage: main data directory #} -{% do archivebox.add_storage("/data", values.storage.data) %} -{% do scheduler.add_storage("/data", values.storage.data) %} +{% do scheduler.set_command(["schedule", "--foreground", "--update", "--every=day"]) %} +{% do scheduler.healthcheck.set_test("pgrep", {"process": "archivebox schedule --foreground"}) %} +{% do scheduler.depends.add_dependency(values.consts.archivebox_container_name, "service_healthy") %} + +{% do sonic.set_user(values.run_as.user, values.run_as.group) %} +{% do sonic.healthcheck.set_test("tcp", {"port": 1491}) %} +{% do sonic.environment.add_env("SEARCH_BACKEND_PASSWORD", sonic_password) %} +{% do sonic.environment.add_env("SEARCH_BACKEND_PORT", values.consts.internal_sonic_port) %} + +{% do sonic.add_storage("/var/lib/sonic/store", values.storage.sonic_data) %} +{% do perm_container.add_or_skip_action("sonic_data", values.storage.sonic_data, perms_config) %} -{# Additional storage mounts #} {% for store in values.storage.additional_storage %} {% do archivebox.add_storage(store.mount_path, store) %} {% do scheduler.add_storage(store.mount_path, store) %} + {% do perm_container.add_or_skip_action(store.mount_path, store.size, perms_config) %} {% endfor %} -{# Permissions container #} {% if perm_container.has_actions() %} {% do perm_container.activate() %} {% do archivebox.depends.add_dependency(values.consts.perms_container_name, "service_completed_successfully") %} @@ -122,7 +83,6 @@ {% do sonic.depends.add_dependency(values.consts.perms_container_name, "service_completed_successfully") %} {% endif %} -{# Portal for TrueNAS UI #} {% do tpl.portals.add(values.network.web_port) %} {{ tpl.render() | tojson }} diff --git a/ix-dev/community/archivebox/templates/test_values/basic-values.yaml b/ix-dev/community/archivebox/templates/test_values/basic-values.yaml index 7a07187f77a..fc4dfafe2d2 100644 --- a/ix-dev/community/archivebox/templates/test_values/basic-values.yaml +++ b/ix-dev/community/archivebox/templates/test_values/basic-values.yaml @@ -24,6 +24,10 @@ network: bind_mode: published port_number: 30820 +run_as: + user: 568 + group: 568 + ix_volumes: data: /opt/tests/mnt/archivebox/data sonic_data: /opt/tests/mnt/archivebox/sonic-data From c28257e92daae4ae2213bbf1bf25a7e08099fee9 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Thu, 12 Mar 2026 18:46:47 +0200 Subject: [PATCH 08/17] add lib --- .../templates/library/base_v2_2_8/__init__.py | 0 .../templates/library/base_v2_2_8/configs.py | 89 ++ .../library/base_v2_2_8/container.py | 498 ++++++++ .../templates/library/base_v2_2_8/depends.py | 34 + .../templates/library/base_v2_2_8/deploy.py | 24 + .../templates/library/base_v2_2_8/deps.py | 62 + .../library/base_v2_2_8/deps_elastic.py | 99 ++ .../library/base_v2_2_8/deps_mariadb.py | 95 ++ .../library/base_v2_2_8/deps_meilisearch.py | 89 ++ .../library/base_v2_2_8/deps_memcached.py | 70 ++ .../library/base_v2_2_8/deps_mongodb.py | 101 ++ .../library/base_v2_2_8/deps_perms.py | 137 +++ .../library/base_v2_2_8/deps_postgres.py | 260 ++++ .../library/base_v2_2_8/deps_redis.py | 94 ++ .../library/base_v2_2_8/deps_solr.py | 89 ++ .../library/base_v2_2_8/deps_tika.py | 67 + .../templates/library/base_v2_2_8/device.py | 31 + .../base_v2_2_8/device_cgroup_rules.py | 54 + .../templates/library/base_v2_2_8/devices.py | 71 ++ .../templates/library/base_v2_2_8/dns.py | 79 ++ .../library/base_v2_2_8/docker_client.py | 45 + .../library/base_v2_2_8/environment.py | 119 ++ .../templates/library/base_v2_2_8/error.py | 4 + .../templates/library/base_v2_2_8/expose.py | 31 + .../library/base_v2_2_8/extra_hosts.py | 33 + .../library/base_v2_2_8/formatter.py | 26 + .../library/base_v2_2_8/functions.py | 251 ++++ .../library/base_v2_2_8/healthcheck.py | 297 +++++ .../templates/library/base_v2_2_8/labels.py | 29 + .../templates/library/base_v2_2_8/networks.py | 246 ++++ .../templates/library/base_v2_2_8/notes.py | 303 +++++ .../templates/library/base_v2_2_8/portals.py | 73 ++ .../templates/library/base_v2_2_8/ports.py | 147 +++ .../templates/library/base_v2_2_8/render.py | 101 ++ .../library/base_v2_2_8/resources.py | 115 ++ .../templates/library/base_v2_2_8/restart.py | 25 + .../library/base_v2_2_8/security_opts.py | 52 + .../templates/library/base_v2_2_8/storage.py | 125 ++ .../templates/library/base_v2_2_8/sysctls.py | 38 + .../library/base_v2_2_8/tests/__init__.py | 0 .../base_v2_2_8/tests/test_build_image.py | 51 + .../library/base_v2_2_8/tests/test_configs.py | 71 ++ .../base_v2_2_8/tests/test_container.py | 536 ++++++++ .../library/base_v2_2_8/tests/test_depends.py | 54 + .../library/base_v2_2_8/tests/test_deps.py | 1083 +++++++++++++++++ .../library/base_v2_2_8/tests/test_device.py | 150 +++ .../tests/test_device_cgroup_rules.py | 79 ++ .../library/base_v2_2_8/tests/test_dns.py | 64 + .../base_v2_2_8/tests/test_environment.py | 219 ++++ .../library/base_v2_2_8/tests/test_expose.py | 46 + .../base_v2_2_8/tests/test_extra_hosts.py | 57 + .../base_v2_2_8/tests/test_formatter.py | 13 + .../base_v2_2_8/tests/test_functions.py | 190 +++ .../base_v2_2_8/tests/test_healthcheck.py | 419 +++++++ .../library/base_v2_2_8/tests/test_labels.py | 88 ++ .../base_v2_2_8/tests/test_networks.py | 329 +++++ .../library/base_v2_2_8/tests/test_notes.py | 381 ++++++ .../library/base_v2_2_8/tests/test_portal.py | 93 ++ .../library/base_v2_2_8/tests/test_ports.py | 383 ++++++ .../library/base_v2_2_8/tests/test_render.py | 37 + .../base_v2_2_8/tests/test_resources.py | 140 +++ .../library/base_v2_2_8/tests/test_restart.py | 57 + .../base_v2_2_8/tests/test_security_opts.py | 91 ++ .../library/base_v2_2_8/tests/test_sysctls.py | 62 + .../base_v2_2_8/tests/test_validations.py | 132 ++ .../library/base_v2_2_8/tests/test_volumes.py | 746 ++++++++++++ .../templates/library/base_v2_2_8/tmpfs.py | 75 ++ .../library/base_v2_2_8/truenas_client.py | 66 + .../library/base_v2_2_8/validations.py | 366 ++++++ .../library/base_v2_2_8/volume_mount.py | 87 ++ .../library/base_v2_2_8/volume_mount_types.py | 43 + .../library/base_v2_2_8/volume_sources.py | 108 ++ .../library/base_v2_2_8/volume_types.py | 130 ++ .../templates/library/base_v2_2_8/volumes.py | 61 + 74 files changed, 10610 insertions(+) create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/__init__.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/configs.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/container.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/depends.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/deploy.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/deps.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_elastic.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_mariadb.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_meilisearch.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_memcached.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_mongodb.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_perms.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_postgres.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_redis.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_solr.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_tika.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/device.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/device_cgroup_rules.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/devices.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/dns.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/docker_client.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/environment.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/error.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/expose.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/extra_hosts.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/formatter.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/functions.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/healthcheck.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/labels.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/networks.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/notes.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/portals.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/ports.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/render.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/resources.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/restart.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/security_opts.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/storage.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/sysctls.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/__init__.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_build_image.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_configs.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_container.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_depends.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_deps.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_device.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_device_cgroup_rules.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_dns.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_environment.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_expose.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_extra_hosts.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_formatter.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_functions.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_healthcheck.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_labels.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_networks.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_notes.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_portal.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_ports.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_render.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_resources.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_restart.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_security_opts.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_sysctls.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_validations.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_volumes.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/tmpfs.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/truenas_client.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/validations.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/volume_mount.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/volume_mount_types.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/volume_sources.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/volume_types.py create mode 100644 ix-dev/community/archivebox/templates/library/base_v2_2_8/volumes.py diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/__init__.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/configs.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/configs.py new file mode 100644 index 00000000000..9de4148db0a --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/configs.py @@ -0,0 +1,89 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class Configs: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._configs: dict[str, dict] = {} + + def add(self, name: str, data: str): + if not isinstance(data, str): + raise RenderError(f"Expected [data] to be a string, got [{type(data)}]") + + if data == "": + raise RenderError(f"Expected [data] to be non-empty for config [{name}]") + + if name not in self._configs: + self._configs[name] = {"name": name, "data": data} + return + + if data == self._configs[name]["data"]: + return + + raise RenderError(f"Config [{name}] already added with different data") + + def has_configs(self): + return bool(self._configs) + + def render(self): + return { + c["name"]: {"content": escape_dollar(c["data"])} + for c in sorted(self._configs.values(), key=lambda c: c["name"]) + } + + +class ContainerConfigs: + def __init__(self, render_instance: "Render", configs: Configs): + self._render_instance = render_instance + self.top_level_configs: Configs = configs + self.container_configs: set[ContainerConfig] = set() + + def add(self, name: str, data: str, target: str, mode: str = ""): + self.top_level_configs.add(name, data) + + if target == "": + raise RenderError(f"Expected [target] to be set for config [{name}]") + if mode != "": + mode = valid_octal_mode_or_raise(mode) + + if target in [c.target for c in self.container_configs]: + raise RenderError(f"Target [{target}] already used for another config") + target = valid_fs_path_or_raise(target) + self.container_configs.add(ContainerConfig(self._render_instance, name, target, mode)) + + def has_configs(self): + return bool(self.container_configs) + + def render(self): + return [c.render() for c in sorted(self.container_configs, key=lambda c: c.source)] + + +class ContainerConfig: + def __init__(self, render_instance: "Render", source: str, target: str, mode: str): + self._render_instance = render_instance + self.source = source + self.target = target + self.mode = mode + + def render(self): + result: dict[str, str | int] = { + "source": self.source, + "target": self.target, + } + + if self.mode: + result["mode"] = int(self.mode, 8) + + return result diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/container.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/container.py new file mode 100644 index 00000000000..b9ed8869a51 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/container.py @@ -0,0 +1,498 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .device_cgroup_rules import DeviceCGroupRules + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .expose import Expose + from .extra_hosts import ExtraHosts + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .networks import ContainerNetworks + from .ports import Ports + from .restart import RestartPolicy + from .tmpfs import Tmpfs + from .validations import ( + valid_cap_or_raise, + valid_cgroup_or_raise, + valid_ipc_mode_or_raise, + valid_mac_or_raise, + valid_network_mode_or_raise, + valid_pid_mode_or_raise, + valid_port_bind_mode_or_raise, + valid_port_mode_or_raise, + valid_pull_policy_or_raise, + ) + from .security_opts import SecurityOpts + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from device_cgroup_rules import DeviceCGroupRules + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from expose import Expose + from extra_hosts import ExtraHosts + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from networks import ContainerNetworks + from ports import Ports + from restart import RestartPolicy + from tmpfs import Tmpfs + from validations import ( + valid_cap_or_raise, + valid_cgroup_or_raise, + valid_ipc_mode_or_raise, + valid_mac_or_raise, + valid_network_mode_or_raise, + valid_pid_mode_or_raise, + valid_port_bind_mode_or_raise, + valid_port_mode_or_raise, + valid_pull_policy_or_raise, + ) + from security_opts import SecurityOpts + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._extra_hosts: ExtraHosts = ExtraHosts(self._render_instance) + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: SecurityOpts = SecurityOpts(self._render_instance) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance, self) + self._tmpfs: Tmpfs = Tmpfs(self._render_instance, self) + self._ipc_mode: str | None = None + self._pid_mode: str | None = None + self.mac_address: str | None = None + self._cgroup: str | None = None + self._device_cgroup_rules: DeviceCGroupRules = DeviceCGroupRules(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: ContainerNetworks = ContainerNetworks(self._render_instance) + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels() + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + self.expose: Expose = Expose(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + self._auto_add_networks() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _auto_add_networks(self): + networks = self._render_instance.values.get("network", {}).get("networks", []) + if not networks: + return + + for network in networks: + containers = network.get("containers", []) + if not containers: + raise RenderError(f'Network [{network.get("name", "")}] must have at least one container') + for container in containers: + if self._name != container["name"]: + continue + self.add_network(network["name"], container.get("config", {})) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def name(self) -> str: + return self._name + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + line = line.strip() if line else "" + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = escape_dollar(dockerfile) + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_extra_host(self, host: str, ip: str): + self._extra_hosts.add_host(host, ip) + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_additional_groups(self) -> list[int | str]: + result = [] + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + result.append(44) # video + result.append(107) # render + return result + + def get_current_groups(self) -> list[str]: + result = [str(g) for g in self._group_add] + result.extend([str(g) for g in self.get_additional_groups()]) + return result + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_ipc_mode(self, ipc_mode: str): + self._ipc_mode = valid_ipc_mode_or_raise(ipc_mode, self._render_instance.container_names()) + + def set_pid_mode(self, mode: str = ""): + self._pid_mode = valid_pid_mode_or_raise(mode, self._render_instance.container_names()) + + def add_device_cgroup_rule(self, dev_grp_rule: str): + self._device_cgroup_rules.add_rule(dev_grp_rule) + + def set_cgroup(self, cgroup: str): + self._cgroup = valid_cgroup_or_raise(cgroup) + + def set_mac(self, mac_address: str): + valid_mac_or_raise(mac_address) + self.mac_address = mac_address + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, key: str, value: str | bool | None = None, arg: str | None = None): + self._security_opt.add_opt(key, value, arg) + + def remove_security_opt(self, key: str): + self._security_opt.remove_opt(key) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def add_port(self, port_config: dict | None = None, dev_config: dict | None = None): + port_config = port_config or {} + dev_config = dev_config or {} + # Merge port_config and dev_config (dev_config has precedence) + config = port_config | dev_config + bind_mode = valid_port_bind_mode_or_raise(config.get("bind_mode", "")) + # Skip port if its neither published nor exposed + if not bind_mode: + return + + # Collect port config + mode = valid_port_mode_or_raise(config.get("mode", "ingress")) + host_port = config.get("port_number", 0) + container_port = config.get("container_port", 0) or host_port + protocol = config.get("protocol", "tcp") + host_ips = config.get("host_ips") or ["0.0.0.0", "::"] + if not isinstance(host_ips, list): + raise RenderError(f"Expected [host_ips] to be a list, got [{host_ips}]") + + if bind_mode == "published": + for host_ip in host_ips: + self.ports._add_port( + host_port, container_port, {"protocol": protocol, "host_ip": host_ip, "mode": mode} + ) + elif bind_mode == "exposed": + self.expose.add_port(container_port, protocol) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_network(self, network: str, config: dict = {}): + self.networks.add(self._name, network, config) + + def add_storage(self, mount_path: str, config: "IxStorage"): + if config.get("type", "") == "tmpfs": + self._tmpfs.add(mount_path, config) + else: + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_udev(self, read_only: bool = True, mount_path: str = "/run/udev"): + self._storage._add_udev(read_only, mount_path) + + def add_tun_device(self): + self.devices._add_tun_device() + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def add_usb_bus(self): + self.devices.add_usb_bus() + + def setup_as_helper(self, profile: str = "low", disable_network: bool = True): + self.restart.set_policy("on-failure", 1) + self.healthcheck.disable() + self.remove_devices() + if profile: + self.deploy.resources.set_profile(profile) + if disable_network: + self.set_network_mode("none") + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks.has_items(): + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._ipc_mode is not None: + result["ipc"] = self._ipc_mode + + if self._pid_mode is not None: + result["pid"] = self._pid_mode + + if self._device_cgroup_rules.has_rules(): + result["device_cgroup_rules"] = self._device_cgroup_rules.render() + + if self._cgroup is not None: + result["cgroup"] = self._cgroup + + if self.mac_address is not None: + result["mac_address"] = self.mac_address + + if self._extra_hosts.has_hosts(): + result["extra_hosts"] = self._extra_hosts.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + for g in self.get_additional_groups(): + self.add_group(g) + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt.has_opts(): + result["security_opt"] = self._security_opt.render() + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self.expose.has_ports(): + result["expose"] = self.expose.render() + + if self.networks.has_items(): + result["networks"] = self.networks.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + if self._tmpfs.has_tmpfs(): + result["tmpfs"] = self._tmpfs.render() + + return result diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/depends.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/depends.py new file mode 100644 index 00000000000..4e057cf0859 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/depends.py @@ -0,0 +1,34 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_depend_condition_or_raise +except ImportError: + from error import RenderError + from validations import valid_depend_condition_or_raise + + +class Depends: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._dependencies: dict[str, str] = {} + + def add_dependency(self, name: str, condition: str): + condition = valid_depend_condition_or_raise(condition) + if name in self._dependencies.keys(): + raise RenderError(f"Dependency [{name}] already added") + if name not in self._render_instance.container_names(): + raise RenderError( + f"Dependency [{name}] not found in defined containers. " + f"Available containers: [{', '.join(self._render_instance.container_names())}]" + ) + self._dependencies[name] = condition + + def has_dependencies(self): + return len(self._dependencies) > 0 + + def render(self): + return {d: {"condition": c} for d, c in self._dependencies.items()} diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/deploy.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deploy.py new file mode 100644 index 00000000000..894dbc643b9 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deploy.py @@ -0,0 +1,24 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .resources import Resources +except ImportError: + from resources import Resources + + +class Deploy: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self.resources: Resources = Resources(self._render_instance) + + def has_deploy(self): + return self.resources.has_resources() + + def render(self): + if self.resources.has_resources(): + return {"resources": self.resources.render()} + + return {} diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps.py new file mode 100644 index 00000000000..52cfceb3ec9 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps.py @@ -0,0 +1,62 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_elastic import ElasticSearchContainer, ElasticConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_meilisearch import MeilisearchContainer, MeiliConfig + from .deps_memcached import MemcachedContainer, MemcachedConfig + from .deps_mongodb import MongoDBContainer, MongoDBConfig + from .deps_perms import PermsContainer + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_solr import SolrContainer, SolrConfig + from .deps_tika import TikaContainer, TikaConfig +except ImportError: + from deps_elastic import ElasticSearchContainer, ElasticConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_meilisearch import MeilisearchContainer, MeiliConfig + from deps_memcached import MemcachedContainer, MemcachedConfig + from deps_mongodb import MongoDBContainer, MongoDBConfig + from deps_perms import PermsContainer + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_solr import SolrContainer, SolrConfig + from deps_tika import TikaContainer, TikaConfig + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) + + def mongodb(self, name: str, image: str, config: MongoDBConfig, perms_instance: PermsContainer): + return MongoDBContainer(self._render_instance, name, image, config, perms_instance) + + def meilisearch(self, name: str, image: str, config: MeiliConfig, perms_instance: PermsContainer): + return MeilisearchContainer(self._render_instance, name, image, config, perms_instance) + + def elasticsearch(self, name: str, image: str, config: ElasticConfig, perms_instance: PermsContainer): + return ElasticSearchContainer(self._render_instance, name, image, config, perms_instance) + + def solr(self, name: str, image: str, config: SolrConfig, perms_instance: PermsContainer): + return SolrContainer(self._render_instance, name, image, config, perms_instance) + + def tika(self, name: str, image: str, config: TikaConfig): + return TikaContainer(self._render_instance, name, image, config) + + def memcached(self, name: str, image: str, config: MemcachedConfig): + return MemcachedContainer(self._render_instance, name, image, config) diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_elastic.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_elastic.py new file mode 100644 index 00000000000..99a0886b7ba --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_elastic.py @@ -0,0 +1,99 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + + +class ElasticConfig(TypedDict): + password: str + node_name: str + port: NotRequired[int] + volume: "IxStorage" + + +SUPPORTED_REPOS = ["elasticsearch"] + + +class ElasticSearchContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: ElasticConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/usr/share/elasticsearch/data" + + for key in ("password", "node_name", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for ElasticSearch") + + c = self._render_instance.add_container(name, image) + + c.set_user(1000, 1000) + basic_auth_header = self._render_instance.funcs["basic_auth_header"]("elastic", config["password"]) + c.healthcheck.set_test( + "curl", + { + "port": self.get_port(), + "path": "/_cluster/health?local=true", + "headers": [("Authorization", basic_auth_header)], + }, + ) + c.remove_devices() + c.set_grace_period(60) + c.add_storage(self._data_dir, config["volume"]) + + c.environment.add_env("ELASTIC_PASSWORD", config["password"]) + c.environment.add_env("http.port", self.get_port()) + c.environment.add_env("path.data", self._data_dir) + c.environment.add_env("path.repo", self.get_snapshots_dir()) + c.environment.add_env("node.name", config["node_name"]) + c.environment.add_env("discovery.type", "single-node") + c.environment.add_env("xpack.security.enabled", True) + c.environment.add_env("xpack.security.transport.ssl.enabled", False) + + perms_instance.add_or_skip_action( + f"{self._name}_elastic_data", config["volume"], {"uid": 1000, "gid": 1000, "mode": "check"} + ) + + self._get_repo(image) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository") + if not repo: + raise RenderError("Could not determine repo") + if repo not in SUPPORTED_REPOS: + raise RenderError( + f"Unsupported repo [{repo}] for elastic search. Supported repos: {', '.join(SUPPORTED_REPOS)}" + ) + return repo + + def get_port(self): + return self._config.get("port") or 9200 + + def get_url(self): + return f"http://{self._name}:{self.get_port()}" + + def get_snapshots_dir(self): + return f"{self._data_dir}/snapshots" diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_mariadb.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_mariadb.py new file mode 100644 index 00000000000..b65752310ae --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_mariadb.py @@ -0,0 +1,95 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +SUPPORTED_REPOS = ["mariadb"] + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(self.get_port()) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + self._get_repo(image) + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb", {"password": root_password}) + c.remove_devices() + c.set_grace_period(60) + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository") + if not repo: + raise RenderError("Could not determine repo") + if repo not in SUPPORTED_REPOS: + raise RenderError(f"Unsupported repo [{repo}] for mariadb. Supported repos: {', '.join(SUPPORTED_REPOS)}") + return repo + + def get_url(self, variant: str): + addr = f"{self._name}:{self.get_port()}" + urls = { + "jdbc": f"jdbc:mariadb://{addr}/{self._config['database']}", + } + + if variant not in urls: + raise RenderError(f"Expected [variant] to be one of [{', '.join(urls.keys())}], got [{variant}]") + return urls[variant] + + def get_port(self): + return self._config.get("port") or 3306 + + @property + def container(self): + return self._container diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_meilisearch.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_meilisearch.py new file mode 100644 index 00000000000..7dbf10e8722 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_meilisearch.py @@ -0,0 +1,89 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + + +class MeiliConfig(TypedDict): + master_key: str + port: NotRequired[int] + volume: "IxStorage" + + +SUPPORTED_REPOS = ["getmeili/meilisearch"] + + +class MeilisearchContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MeiliConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/meili_data" + + for key in ("master_key", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for meilisearch") + + c = self._render_instance.add_container(name, image) + + user, group = 568, 568 + run_as = self._render_instance.values.get("run_as") + if run_as: + user = run_as["user"] or user # Avoids running as root + group = run_as["group"] or group # Avoids running as root + + c.set_user(user, group) + c.healthcheck.set_test("curl", {"port": self.get_port(), "path": "/health"}) + c.remove_devices() + c.set_grace_period(60) + c.add_storage(self._data_dir, config["volume"]) + + c.environment.add_env("MEILI_HTTP_ADDR", f"0.0.0.0:{self.get_port()}") + c.environment.add_env("MEILI_NO_ANALYTICS", True) + c.environment.add_env("MEILI_EXPERIMENTAL_DUMPLESS_UPGRADE", True) + c.environment.add_env("MEILI_MASTER_KEY", config["master_key"]) + + perms_instance.add_or_skip_action( + f"{self._name}_meili_data", config["volume"], {"uid": user, "gid": group, "mode": "check"} + ) + + self._get_repo(image) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository") + if not repo: + raise RenderError("Could not determine repo") + if repo not in SUPPORTED_REPOS: + raise RenderError( + f"Unsupported repo [{repo}] for meilisearch. Supported repos: {', '.join(SUPPORTED_REPOS)}" + ) + return repo + + def get_port(self): + return self._config.get("port") or 7700 + + def get_url(self): + return f"http://{self._name}:{self.get_port()}" diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_memcached.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_memcached.py new file mode 100644 index 00000000000..7d40470fbf1 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_memcached.py @@ -0,0 +1,70 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + + +try: + from .error import RenderError +except ImportError: + from error import RenderError + + +class MemcachedConfig(TypedDict): + port: NotRequired[int] + memory_mb: NotRequired[int] + + +SUPPORTED_REPOS = ["memcached"] + + +class MemcachedContainer: + + def __init__(self, render_instance: "Render", name: str, image: str, config: MemcachedConfig): + self._render_instance = render_instance + self._name = name + self._config = config + + c = self._render_instance.add_container(name, image) + + user, group = 568, 568 + run_as = self._render_instance.values.get("run_as") + if run_as: + user = run_as["user"] or user # Avoids running as root + group = run_as["group"] or group # Avoids running as root + + c.set_user(user, group) + c.healthcheck.set_test("tcp", {"port": self.get_port()}) + c.remove_devices() + c.set_grace_period(60) + + mem = self._config.get("memory_mb") or 256 + c.set_command(["-p", str(self.get_port()), "-m", f"{mem}M"]) + + self._get_repo(image) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository") + if not repo: + raise RenderError("Could not determine repo") + if repo not in SUPPORTED_REPOS: + raise RenderError(f"Unsupported repo [{repo}] for tika. Supported repos: {', '.join(SUPPORTED_REPOS)}") + return repo + + def get_port(self): + return self._config.get("port") or 11211 + + def get_address(self): + return f"{self._name}:{self.get_port()}" diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_mongodb.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_mongodb.py new file mode 100644 index 00000000000..ded64848482 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_mongodb.py @@ -0,0 +1,101 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + + +class MongoDBConfig(TypedDict): + user: str + password: str + database: str + volume: "IxStorage" + + +SUPPORTED_REPOS = ["mongo"] + + +class MongoDBContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MongoDBConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/data/db" + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mongodb") + + c = self._render_instance.add_container(name, image) + + user, group = 568, 568 + run_as = self._render_instance.values.get("run_as") + if run_as: + user = run_as["user"] or user # Avoids running as root + group = run_as["group"] or group # Avoids running as root + + c.set_user(user, group) + c.healthcheck.set_test("mongodb", {"db": config["database"]}) + c.remove_devices() + c.set_grace_period(60) + c.add_storage(self._data_dir, config["volume"]) + + c.environment.add_env("MONGO_INITDB_ROOT_USERNAME", config["user"]) + c.environment.add_env("MONGO_INITDB_ROOT_PASSWORD", config["password"]) + c.environment.add_env("MONGO_INITDB_DATABASE", config["database"]) + + perms_instance.add_or_skip_action( + f"{self._name}_mongodb_data", config["volume"], {"uid": user, "gid": group, "mode": "check"} + ) + + self._get_repo(image) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository") + if not repo: + raise RenderError("Could not determine repo") + if repo not in SUPPORTED_REPOS: + raise RenderError(f"Unsupported repo [{repo}] for mongodb. Supported repos: {', '.join(SUPPORTED_REPOS)}") + return repo + + def get_port(self): + return self._config.get("port") or 27017 + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self.get_port()}" + db = self._config["database"] + + urls = { + "mongodb": f"mongodb://{creds}@{addr}/{db}", + "host_port": addr, + } + + if variant not in urls: + raise RenderError(f"Expected [variant] to be one of [{', '.join(urls.keys())}], got [{variant}]") + return urls[variant] diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_perms.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_perms.py new file mode 100644 index 00000000000..989e824759c --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_perms.py @@ -0,0 +1,137 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + read_only = volume_config.get("read_only", False) + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "read_only": read_only, + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "container_utils_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/permissions.py"]) + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + if not parsed["action_data"]["read_only"]: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_postgres.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_postgres.py new file mode 100644 index 00000000000..9ec13f479fa --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_postgres.py @@ -0,0 +1,260 @@ +import re +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + additional_options: NotRequired[dict[str, str]] + + +MAX_POSTGRES_VERSION = 18 +SUPPORTED_REPOS = [ + "postgres", + "postgis/postgis", + "paradedb/paradedb", + "pgvector/pgvector", + "timescale/timescaledb", + "ghcr.io/oss-apps/postgres", + "ghcr.io/immich-app/postgres", +] +SUPPORTED_UPGRADE_REPOS = [ + "postgres", + "postgis/postgis", + "paradedb/paradedb", + "pgvector/pgvector", + # "timescale/timescaledb", // Currently NOT supported for upgrades + "ghcr.io/oss-apps/postgres", + "ghcr.io/immich-app/postgres", +] + + +def get_major_version(variant: str, tag: str): + # Handle digest pins by taking only the tag part before the @ + tag = tag.split("@")[0] + if variant == "postgres": + # 17.7-bookworm + regex = re.compile(r"^\d+\.\d+-\w+") + + def oper(x): + return x.split(".")[0] + + elif variant == "postgis/postgis": + # 17-3.5 + regex = re.compile(r"^\d+\-\d+\.\d+") + + def oper(x): + return x.split("-")[0] + + elif variant == "pgvector/pgvector": + # 0.8.1-pg17-trixie + regex = re.compile(r"^\d+\.\d+\.\d+\-pg\d+(\-\w+)?") + + def oper(x): + parts = x.split("-") + return parts[1].lstrip("pg") + + elif variant == "ghcr.io/immich-app/postgres": + # 15-vectorchord0.4.3 + regex = re.compile(r"^\d+\-vectorchord\d+\.\d+\.\d+") + + def oper(x): + return x.split("-")[0] + + elif variant == "timescale/timescaledb": + # 2.24.0-pg18 + regex = re.compile(r"^\d+\.\d+\.\d+-pg\d+") + + def oper(x): + parts = x.split("-") + return parts[1].lstrip("pg") + + elif variant == "paradedb/paradedb": + # 0.21.8-pg18 + regex = re.compile(r"^\d+\.\d+\.\d+-pg\d+") + + def oper(x): + parts = x.split("-") + return parts[1].lstrip("pg") + + elif variant == "ghcr.io/oss-apps/postgres": + # 18.0-trixie + regex = re.compile(r"^\d+\.\d+-\w+") + + def oper(x): + return x.split(".")[0] + + if not regex.match(tag): + raise RenderError(f"Could not determine major version from tag [{tag}] for variant [{variant}]") + + return oper(tag) + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = None + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + self._data_dir = "/var/lib/postgresql" + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self.get_port()) + + # TODO: Set some defaults for ZFS Optimizations (Need to check if applies on updates) + # https://vadosware.io/post/everything-ive-seen-on-optimizing-postgres-on-zfs-on-linux/ + + opts = [] + for k, v in config.get("additional_options", {}).items(): + opts.extend(["-c", f"{k}={v}"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "PGPORT": port, + } + + c = self._render_instance.add_container(name, image) + containers = [c] + + c.healthcheck.set_test("postgres", {"user": config["user"], "db": config["database"], "port": port}) + c.set_shm_size_mb(256) + c.remove_devices() + c.set_grace_period(60) + + if opts: + c.set_command(opts) + + target_major_version = self._get_target_version(image) + # This is the new format upstream Postgres uses/suggests. + # E.g., for Postgres 17, the data dir is /var/lib/postgresql/17/docker + common_variables.update({"PGDATA": f"{self._data_dir}/{target_major_version}/docker"}) + + repo = self._get_repo(image) + if repo in SUPPORTED_UPGRADE_REPOS: + upg = self._render_instance.add_container(self._upgrade_name, "postgres_upgrade_image") + + self._upgrade_container = upg + containers.append(upg) + + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.restart.set_policy("on-failure", 1) + upg.healthcheck.disable() + upg.setup_as_helper(profile="medium") + upg.environment.add_env("TARGET_VERSION", target_major_version) + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + for container in containers: + # TODO: We can now use 568:568 (or any user/group). + # Need to first plan a migration path for the existing users. + container.set_user(999, 999) + container.remove_devices() + container.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + container.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository") + if not repo: + raise RenderError("Could not determine repo") + if repo not in SUPPORTED_REPOS: + raise RenderError(f"Unsupported repo [{repo}] for postgres. Supported repos: {', '.join(SUPPORTED_REPOS)}") + return repo + + def _get_target_version(self, image): + repo = self._get_repo(image) + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = str(images[image].get("tag", "")) + target_major_version = get_major_version(repo, tag) + + try: + # Make sure we end up with an integer + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_port(self): + return self._config.get("port") or 5432 + + def get_url(self, variant: str): + raw_user = self._config["user"] + raw_password = self._config["password"] + + user = urllib.parse.quote_plus(raw_user) + password = urllib.parse.quote_plus(raw_password) + creds = f"{user}:{password}" + addr = f"{self._name}:{self.get_port()}" + db = self._config["database"] + + if variant == "dotnet_pgsql": + for char in ["'", ";"]: + if char in raw_password: + raise RenderError(f"Password cannot contain [{char}] for Dotnet Postgres Password") + + urls = { + "postgres": f"postgres://{creds}@{addr}/{db}?sslmode=disable", + "postgresql": f"postgresql://{creds}@{addr}/{db}?sslmode=disable", + "postgresql_no_creds": f"postgresql://{addr}/{db}?sslmode=disable", + "jdbc": f"jdbc:postgresql://{addr}/{db}", + "dotnet_pgsql": f"Host={addr};Database={db};Username={raw_user};Password={raw_password}", + "host_port": addr, + } + + if variant not in urls: + raise RenderError(f"Expected [variant] to be one of [{', '.join(urls.keys())}], got [{variant}]") + return urls[variant] diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_redis.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_redis.py new file mode 100644 index 00000000000..691e76b2614 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_redis.py @@ -0,0 +1,94 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +SUPPORTED_REPOS = ["redis", "valkey/valkey"] + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self.get_port()) + self._get_repo(image) + + user, group = 568, 568 + run_as = self._render_instance.values.get("run_as") + if run_as: + user = run_as["user"] or user # Avoids running as root + group = run_as["group"] or group # Avoids running as root + c = self._render_instance.add_container(name, image) + c.set_user(user, group) + c.remove_devices() + c.set_grace_period(60) + c.healthcheck.set_test("redis", {"password": config["password"]}) + + cmd = [] + cmd.extend(["--port", str(port)]) + cmd.extend(["--requirepass", config["password"]]) + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.set_command(cmd) + + c.add_storage("/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": user, "gid": group, "mode": "check"} + ) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository") + if not repo: + raise RenderError("Could not determine repo") + if repo not in SUPPORTED_REPOS: + raise RenderError(f"Unsupported repo [{repo}] for redis. Supported repos: {', '.join(SUPPORTED_REPOS)}") + return repo + + def get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self.get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_solr.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_solr.py new file mode 100644 index 00000000000..521e2c2f674 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_solr.py @@ -0,0 +1,89 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired, List + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + + +class SolrConfig(TypedDict): + core: str + modules: NotRequired[List[str]] + port: NotRequired[int] + volume: "IxStorage" + + +SUPPORTED_REPOS = ["solr"] + + +class SolrContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: SolrConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/solr" + + for key in ("core", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for solr") + + c = self._render_instance.add_container(name, image) + + user, group = 568, 568 + run_as = self._render_instance.values.get("run_as") + if run_as: + user = run_as["user"] or user # Avoids running as root + group = run_as["group"] or group # Avoids running as root + + c.set_user(user, group) + c.healthcheck.set_test("curl", {"port": self.get_port(), "path": f"/solr/{config['core']}/admin/ping"}) + c.remove_devices() + c.set_grace_period(60) + c.add_storage(self._data_dir, config["volume"]) + + c.set_command(["solr-precreate", config["core"]]) + + c.environment.add_env("SOLR_PORT", self.get_port()) + if modules := config.get("modules"): + c.environment.add_env("SOLR_MODULES", ",".join(modules)) + + perms_instance.add_or_skip_action( + f"{self._name}_solr_data", config["volume"], {"uid": user, "gid": group, "mode": "check"} + ) + + self._get_repo(image) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository") + if not repo: + raise RenderError("Could not determine repo") + if repo not in SUPPORTED_REPOS: + raise RenderError(f"Unsupported repo [{repo}] for solr. Supported repos: {', '.join(SUPPORTED_REPOS)}") + return repo + + def get_port(self): + return self._config.get("port") or 8983 + + def get_url(self): + return f"http://{self._name}:{self.get_port()}/solr/{self._config['core']}" diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_tika.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_tika.py new file mode 100644 index 00000000000..f967cbe2383 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/deps_tika.py @@ -0,0 +1,67 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + + +try: + from .error import RenderError +except ImportError: + from error import RenderError + + +class TikaConfig(TypedDict): + port: NotRequired[int] + + +SUPPORTED_REPOS = ["apache/tika"] + + +class TikaContainer: + def __init__(self, render_instance: "Render", name: str, image: str, config: TikaConfig): + self._render_instance = render_instance + self._name = name + self._config = config + + c = self._render_instance.add_container(name, image) + + user, group = 568, 568 + run_as = self._render_instance.values.get("run_as") + if run_as: + user = run_as["user"] or user # Avoids running as root + group = run_as["group"] or group # Avoids running as root + + c.set_user(user, group) + c.healthcheck.set_test("wget", {"port": self.get_port(), "path": "/tika", "spider": False}) + c.remove_devices() + c.set_grace_period(60) + + c.set_command(["--port", str(self.get_port())]) + + self._get_repo(image) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository") + if not repo: + raise RenderError("Could not determine repo") + if repo not in SUPPORTED_REPOS: + raise RenderError(f"Unsupported repo [{repo}] for tika. Supported repos: {', '.join(SUPPORTED_REPOS)}") + return repo + + def get_port(self): + return self._config.get("port") or 9998 + + def get_url(self): + return f"http://{self._name}:{self.get_port()}" diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/device.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/device.py new file mode 100644 index 00000000000..bfe97097cba --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/device.py @@ -0,0 +1,31 @@ +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise, allowed_device_or_raise, valid_cgroup_perm_or_raise +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise, allowed_device_or_raise, valid_cgroup_perm_or_raise + + +class Device: + def __init__(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + hd = valid_fs_path_or_raise(host_device.rstrip("/")) + cd = valid_fs_path_or_raise(container_device.rstrip("/")) + if not hd or not cd: + raise RenderError( + "Expected [host_device] and [container_device] to be set. " + f"Got host_device [{host_device}] and container_device [{container_device}]" + ) + + cgroup_perm = valid_cgroup_perm_or_raise(cgroup_perm) + if not allow_disallowed: + hd = allowed_device_or_raise(hd) + + self.cgroup_perm: str = cgroup_perm + self.host_device: str = hd + self.container_device: str = cd + + def render(self): + result = f"{self.host_device}:{self.container_device}" + if self.cgroup_perm: + result += f":{self.cgroup_perm}" + return result diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/device_cgroup_rules.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/device_cgroup_rules.py new file mode 100644 index 00000000000..dcccfee773f --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/device_cgroup_rules.py @@ -0,0 +1,54 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_device_cgroup_rule_or_raise +except ImportError: + from error import RenderError + from validations import valid_device_cgroup_rule_or_raise + + +class DeviceCGroupRule: + def __init__(self, rule: str): + rule = valid_device_cgroup_rule_or_raise(rule) + parts = rule.split(" ") + major, minor = parts[1].split(":") + + self._type = parts[0] + self._major = major + self._minor = minor + self._permissions = parts[2] + + def get_key(self): + return f"{self._type}_{self._major}_{self._minor}" + + def render(self): + return f"{self._type} {self._major}:{self._minor} {self._permissions}" + + +class DeviceCGroupRules: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._rules: set[DeviceCGroupRule] = set() + self._track_rule_combos: set[str] = set() + + def add_rule(self, rule: str): + dev_group_rule = DeviceCGroupRule(rule) + if dev_group_rule in self._rules: + raise RenderError(f"Device Group Rule [{rule}] already added") + + rule_key = dev_group_rule.get_key() + if rule_key in self._track_rule_combos: + raise RenderError(f"Device Group Rule [{rule}] has already been added for this device group") + + self._rules.add(dev_group_rule) + self._track_rule_combos.add(rule_key) + + def has_rules(self): + return len(self._rules) > 0 + + def render(self): + return sorted([rule.render() for rule in self._rules]) diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/devices.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/devices.py new file mode 100644 index 00000000000..168e98d0326 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/devices.py @@ -0,0 +1,71 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def _add_tun_device(self): + self.add_device("/dev/net/tun", "/dev/net/tun", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/dns.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/dns.py new file mode 100644 index 00000000000..d3ae7b19fa4 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/dns.py @@ -0,0 +1,79 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import allowed_dns_opt_or_raise +except ImportError: + from error import RenderError + from validations import allowed_dns_opt_or_raise + + +class Dns: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._dns_options: set[str] = set() + self._dns_searches: set[str] = set() + self._dns_nameservers: set[str] = set() + + self._auto_add_dns_opts_from_values() + self._auto_add_dns_searches_from_values() + self._auto_add_dns_nameservers_from_values() + + def _get_dns_opt_keys(self): + return [self._get_key_from_opt(opt) for opt in self._dns_options] + + def _get_key_from_opt(self, opt): + return opt.split(":")[0] + + def _auto_add_dns_opts_from_values(self): + values = self._render_instance.values + for dns_opt in values.get("network", {}).get("dns_opts", []): + self.add_dns_opt(dns_opt) + + def _auto_add_dns_searches_from_values(self): + values = self._render_instance.values + for dns_search in values.get("network", {}).get("dns_searches", []): + self.add_dns_search(dns_search) + + def _auto_add_dns_nameservers_from_values(self): + values = self._render_instance.values + for dns_nameserver in values.get("network", {}).get("dns_nameservers", []): + self.add_dns_nameserver(dns_nameserver) + + def add_dns_search(self, dns_search): + if dns_search in self._dns_searches: + raise RenderError(f"DNS Search [{dns_search}] already added") + self._dns_searches.add(dns_search) + + def add_dns_nameserver(self, dns_nameserver): + if dns_nameserver in self._dns_nameservers: + raise RenderError(f"DNS Nameserver [{dns_nameserver}] already added") + self._dns_nameservers.add(dns_nameserver) + + def add_dns_opt(self, dns_opt): + # eg attempts:3 + key = allowed_dns_opt_or_raise(self._get_key_from_opt(dns_opt)) + if key in self._get_dns_opt_keys(): + raise RenderError(f"DNS Option [{key}] already added") + self._dns_options.add(dns_opt) + + def has_dns_opts(self): + return len(self._dns_options) > 0 + + def has_dns_searches(self): + return len(self._dns_searches) > 0 + + def has_dns_nameservers(self): + return len(self._dns_nameservers) > 0 + + def render_dns_searches(self): + return sorted(self._dns_searches) + + def render_dns_opts(self): + return sorted(self._dns_options) + + def render_dns_nameservers(self): + return sorted(self._dns_nameservers) diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/docker_client.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/docker_client.py new file mode 100644 index 00000000000..52f24552510 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/docker_client.py @@ -0,0 +1,45 @@ +import docker +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import is_truenas_system +except ImportError: + from error import RenderError + from validations import is_truenas_system + + +class DockerClient: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._network_names: set[str] = set() + + try: + self.client = docker.from_env() + except Exception: + if is_truenas_system(): + raise RenderError("Docker client could not be initialized.") + self.client = None + + self._auto_fetch_networks() + + def _auto_fetch_networks(self): + if self.client is None: + return + try: + networks = self.client.networks.list() + except Exception: + if is_truenas_system(): + raise RenderError("Could not fetch networks from Docker client.") + return + + for network in networks: + if network.name is None: + continue + self._network_names.add(network.name) + + def network_exists(self, network_name: str) -> bool: + return network_name in self._network_names diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/environment.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/environment.py new file mode 100644 index 00000000000..d5fe816b19d --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/environment.py @@ -0,0 +1,119 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + self._skip_id_variables: bool = render_instance.values.get("skip_id_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def remove_auto_env(self, name: str): + if name in self._auto_variables.keys(): + del self._auto_variables[name] + return + raise RenderError(f"Environment variable [{name}] is not defined.") + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/error.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/error.py new file mode 100644 index 00000000000..aef48d3b025 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/error.py @@ -0,0 +1,4 @@ +class RenderError(Exception): + """Base class for exceptions in this module.""" + + pass diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/expose.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/expose.py new file mode 100644 index 00000000000..a3ac0aec590 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/expose.py @@ -0,0 +1,31 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_port_or_raise, valid_port_protocol_or_raise +except ImportError: + from error import RenderError + from validations import valid_port_or_raise, valid_port_protocol_or_raise + + +class Expose: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._ports: set[str] = set() + + def add_port(self, port: int, protocol: str = "tcp"): + port = valid_port_or_raise(port) + protocol = valid_port_protocol_or_raise(protocol) + key = f"{port}/{protocol}" + if key in self._ports: + raise RenderError(f"Exposed port [{port}/{protocol}] already added") + self._ports.add(key) + + def has_ports(self): + return len(self._ports) > 0 + + def render(self): + return sorted(self._ports) diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/extra_hosts.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/extra_hosts.py new file mode 100644 index 00000000000..eaad3bed26e --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/extra_hosts.py @@ -0,0 +1,33 @@ +import ipaddress +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError +except ImportError: + from error import RenderError + + +class ExtraHosts: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._extra_hosts: dict[str, str] = {} + + def add_host(self, host: str, ip: str): + if not ip == "host-gateway": + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}] for host [{host}]") + + if host in self._extra_hosts: + raise RenderError(f"Host [{host}] already added with [{self._extra_hosts[host]}]") + self._extra_hosts[host] = ip + + def has_hosts(self): + return len(self._extra_hosts) > 0 + + def render(self): + return {host: ip for host, ip in self._extra_hosts.items()} diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/formatter.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/formatter.py new file mode 100644 index 00000000000..24e882f47a9 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/formatter.py @@ -0,0 +1,26 @@ +import json +import hashlib + + +def escape_dollar(text: str) -> str: + return text.replace("$", "$$") + + +def get_hashed_name_for_volume(prefix: str, config: dict): + config_hash = hashlib.sha256(json.dumps(config).encode("utf-8")).hexdigest() + return f"{prefix}_{config_hash}" + + +def get_hash_with_prefix(prefix: str, data: str): + return f"{prefix}_{hashlib.sha256(data.encode('utf-8')).hexdigest()}" + + +def merge_dicts_no_overwrite(dict1, dict2): + overlapping_keys = dict1.keys() & dict2.keys() + if overlapping_keys: + raise ValueError(f"Merging of dicts failed. Overlapping keys: {overlapping_keys}") + return {**dict1, **dict2} + + +def get_image_with_hashed_data(image: str, data: str): + return get_hash_with_prefix(f"ix-{image}", data) diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/functions.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/functions.py new file mode 100644 index 00000000000..b2906202cba --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/functions.py @@ -0,0 +1,251 @@ +import re +import copy +import yaml +import bcrypt +import secrets +import urllib.parse +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _to_yaml(self, data): + return yaml.dump(data) + + def _bcrypt_hash(self, password: str, rounds: int = 12): + if rounds < 4 or rounds > 31: + raise RenderError("bcrypt rounds must be between 4 and 31") + password_bytes = password.encode("utf-8") + if len(password_bytes) > 72: + raise RenderError(f"Expected bcrypt password to be at most 72 bytes. Got {len(password_bytes)} bytes.") + hashed = bcrypt.hashpw(password_bytes, bcrypt.gensalt(rounds=rounds)).decode("utf-8") + return hashed + + def _htpasswd(self, username: str, password: str, rounds: int = 12): + hashed = self._bcrypt_hash(password, rounds=rounds) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length)[:length] + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + lower_str_value = str(value).lower() + if lower_str_value in ["true", "false"]: + return lower_str_value == "true" + + try: + float_value = float(value) + if float_value.is_integer(): + return int(float_value) + else: + return float(value) + except ValueError: + pass + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _deep_merge(self, dict1: dict, dict2: dict): + """ + Deep merge: recursively merges nested dictionaries. + Values from dict2 override values from dict1. + Nested dicts are merged recursively rather than replaced. + """ + result = dict1.copy() + + for key, value in dict2.items(): + if key in result and isinstance(result[key], dict) and isinstance(value, dict): + # Both values are dicts - merge them recursively + result[key] = self._deep_merge(result[key], value) + else: + # Either not both dicts, or key doesn't exist - use dict2's value + result[key] = value + + return result + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _url_to_dict(self, url: str, v6_brackets: bool = False): + try: + # Try parsing as-is first + parsed = urllib.parse.urlparse(url) + + # If we didn't get a hostname, try with http:// prefix + if not parsed.hostname: + parsed = urllib.parse.urlparse(f"http://{url}") + + # Final check that we have a valid result + if not parsed.hostname: + raise RenderError( + f"Failed to parse URL [{url}]. Ensure it is a valid URL with a hostname and optional port." + ) + + result = { + "netloc": parsed.netloc, + "scheme": parsed.scheme, + "host": parsed.hostname, + "port": parsed.port, + "path": parsed.path, + } + if v6_brackets and parsed.hostname and ":" in parsed.hostname: + result["host"] = f"[{parsed.hostname}]" + result["host_no_brackets"] = parsed.hostname + + return result + + except Exception: + raise RenderError( + f"Failed to parse URL [{url}]. Ensure it is a valid URL with a hostname and optional port." + ) + + def _require_unique(self, values, key, split_char=""): + new_values = [] + for value in values: + new_values.append(value.split(split_char)[0] if split_char else value) + + if len(new_values) != len(set(new_values)): + raise RenderError(f"Expected values in [{key}] to be unique, but got [{', '.join(values)}]") + + def _require_no_reserved(self, values, key, reserved, split_char="", starts_with=False): + new_values = [] + for value in values: + new_values.append(value.split(split_char)[0] if split_char else value) + + if starts_with: + for arg in new_values: + for reserved_value in reserved: + if arg.startswith(reserved_value): + raise RenderError(f"Value [{reserved_value}] is reserved and cannot be set in [{key}]") + return + + for reserved_value in reserved: + if reserved_value in new_values: + raise RenderError(f"Value [{reserved_value}] is reserved and cannot be set in [{key}]") + + def _url_encode(self, string): + return urllib.parse.quote_plus(string) + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def has_amd_gpu(self): + gpus = self._render_instance.values.get("resources", {}).get("gpus", {}) + if gpus.get("use_all_gpus", False) and gpus.get("kfd_device_exists", False): + return True + return False + + def has_nvidia_gpu(self): + gpus = self._render_instance.values.get("resources", {}).get("gpus", {}) + if gpus.get("nvidia_gpu_selection", {}): + for gpu in gpus["nvidia_gpu_selection"].values(): + if gpu.get("use_gpu", False): + return True + return False + + def func_map(self): + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "deep_merge": self._deep_merge, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + "require_unique": self._require_unique, + "require_no_reserved": self._require_no_reserved, + "url_encode": self._url_encode, + "url_to_dict": self._url_to_dict, + "to_yaml": self._to_yaml, + "has_amd_gpu": self.has_amd_gpu, + "has_nvidia_gpu": self.has_nvidia_gpu, + } diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/healthcheck.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/healthcheck.py new file mode 100644 index 00000000000..5952dadd5b4 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/healthcheck.py @@ -0,0 +1,297 @@ +import json +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 30 + self._timeout_sec: int = 5 + self._retries: int = 5 + self._start_period_sec: int = 15 + self._start_interval_sec: int = 2 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if isinstance(test, list): + if test[0] == "CMD" and any(t.startswith("$") for t in test): + raise RenderError(f"Healthcheck with 'CMD' cannot contain shell variables '{test}'") + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def set_start_interval(self, start_interval: int): + self._start_interval_sec = start_interval + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "retries": self._retries, + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "start_period": f"{self._start_period_sec}s", + "start_interval": f"{self._start_interval_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> list[str]: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + "mongodb": mongodb_test, + "pidof": pidof_test, + "pgrep": pgrep_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if key not in config: + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> list[str]: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + method = get_key(config, "method", "GET", False) + data = get_key(config, "data", None, False) + + cmd = ["CMD", "curl", "--request", method, "--silent", "--output", "/dev/null", "--show-error", "--fail"] + + if scheme == "https": + cmd.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + cmd.extend(["--header", f"{header[0]}: {header[1]}"]) + + if data is not None: + cmd.extend(["--data", json.dumps(data)]) + + cmd.append(f"{scheme}://{host}:{port}{path}") + return cmd + + +def wget_test(config: dict) -> list[str]: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + method = get_key(config, "method", "", False) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + spider = get_key(config, "spider", True, False) + data = get_key(config, "data", None, False) + busybox = get_key(config, "busybox", False, False) + + cmd = ["CMD", "wget", "--quiet"] + if method: + if busybox: + raise RenderError("Cannot use method with busybox's wget") + cmd.extend(["--method", method]) + + if spider: + cmd.append("--spider") + else: + cmd.extend(["-O", "/dev/null"]) + + if scheme == "https": + cmd.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + cmd.extend(["--header", f"{header[0]}: {header[1]}"]) + + if data is not None: + if busybox: + cmd.extend(["--post-data", json.dumps(data)]) + else: + cmd.extend(["--body-data", json.dumps(data)]) + + cmd.append(f"{scheme}://{host}:{port}{path}") + + return cmd + + +def http_test(config: dict) -> list[str]: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + hc = f"""{{ printf "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&0; grep "HTTP" | grep -q "200"; }} 0<>/dev/tcp/{host}/{port}""" # noqa + return ["CMD-SHELL", f"/bin/bash -c '{hc}'"] + + +def netcat_test(config: dict) -> list[str]: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + udp_mode = get_key(config, "udp", False, False) + cmd = ["CMD", "nc", "-z", "-w", "1"] + + if udp_mode: + cmd.append("-u") + + cmd.extend([host, str(port)]) + + return cmd + + +def tcp_test(config: dict) -> list[str]: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return ["CMD", "timeout", "1", "bash", "-c", f"cat < /dev/null > /dev/tcp/{host}/{port}"] + + +def redis_test(config: dict) -> list[str]: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + password = get_key(config, "password", None, False) + cmd = ["CMD", "redis-cli", "-h", host, "-p", str(port)] + + if password: + cmd.extend(["-a", password]) + + cmd.append("ping") + + return cmd + + +def postgres_test(config: dict) -> list[str]: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + user = get_key(config, "user", None, True) + db = get_key(config, "db", None, True) + + return ["CMD", "pg_isready", "-h", host, "-p", str(port), "-U", user, "-d", db] + + +def mariadb_test(config: dict) -> list[str]: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + password = get_key(config, "password", None, True) + + return [ + "CMD", + "mariadb-admin", + "--user=root", + f"--host={host}", + f"--port={port}", + f"--password={password}", + "ping", + ] + + +def mongodb_test(config: dict) -> list[str]: + config = config or {} + port = get_key(config, "port", 27017, False) + host = get_key(config, "host", "127.0.0.1", False) + db = get_key(config, "db", None, True) + + return [ + "CMD", + "mongosh", + "--host", + host, + "--port", + str(port), + db, + "--eval", + 'db.adminCommand("ping")', + "--quiet", + ] + + +def pidof_test(config: dict) -> list[str]: + config = config or {} + process = get_key(config, "process", None, True) + + # -s - Single shot, return 0 if at least one process found + return ["CMD", "pidof", "-s", process] + + +def pgrep_test(config: dict) -> list[str]: + config = config or {} + process = get_key(config, "process", None, True) + + # -f - Match against full process name + return ["CMD", "pgrep", "-f", process] diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/labels.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/labels.py new file mode 100644 index 00000000000..756ea664c18 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/labels.py @@ -0,0 +1,29 @@ +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_label_key_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_label_key_or_raise + + +class Labels: + def __init__(self): + self._labels: dict[str, str] = {} + + def add_label(self, key: str, value: str): + key = valid_label_key_or_raise(key) + + if key in self._labels.keys(): + raise RenderError(f"Label [{key}] already added") + + self._labels[key] = escape_dollar(str(value)) + + def has_labels(self) -> bool: + return bool(self._labels) + + def render(self) -> dict[str, str]: + if not self.has_labels(): + return {} + return {label: value for label, value in sorted(self._labels.items())} diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/networks.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/networks.py new file mode 100644 index 00000000000..fa5fb79cad2 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/networks.py @@ -0,0 +1,246 @@ +from typing import TYPE_CHECKING +from dataclasses import dataclass + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .labels import Labels + from .validations import valid_mac_or_raise, valid_ip_or_raise +except ImportError: + from error import RenderError + from labels import Labels + from validations import valid_mac_or_raise, valid_ip_or_raise + + +class Networks: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._networks: dict[str, Network] = {} + + def create_internal(self, name: str) -> str: + """Creates an internal network. Only used for internal communication between containers.""" + name = "ix-internal-" + name + + if self.exists(name): + raise RenderError(f"Network [{name}] already exists.") + + net_config = { + "external": False, + "enable_ipv4": True, + # We disable IPv6 for internal networks, because there is an upstream bug, + # which any ipv6 network takes priority over ipv4 networks even with gw_priority set. + # https://github.com/moby/moby/issues/51999 + "enable_ipv6": False, + "labels": {"tn.network.internal": "true"}, + } + + self._networks[name] = Network(name, net_config) + return name + + def register(self, name: str): + """ + Adds a top level network. Network must already exist, for example created via Docker CLI. + This is used to reference external networks that are not managed by the renderer. + """ + if not self._render_instance.docker.network_exists(name): + raise RenderError(f"Network [{name}] has to be created before it can be used. ie via Docker CLI") + + if self.exists(name): + raise RenderError(f"Network [{name}] already exists.") + + # We only need to define the network as external, + # the lifecycle of the network is managed outside of the renderer, + # via the Docker CLI or other tools. + self._networks[name] = Network(name, {"external": True}) + + def has_items(self): + return bool(self._networks) + + def exists(self, name: str): + return name in self._networks + + def render(self): + result: dict = {} + for name in sorted(self._networks.keys()): + result[name] = self._networks[name].render() + return result + + +@dataclass +class NetworkConfig: + name: str | None = None + """If set, this name will be used instead of the generated name.""" + external: bool | None = None + """If true, this network is managed externally.""" + enable_ipv6: bool | None = None + """If true, this network will have IPv6 enabled.""" + enable_ipv4: bool | None = None + """If true, this network will have IPv4 enabled.""" + labels: Labels | None = None + """If set, this will be added as labels to the network.""" + + def __post_init__(self): + if isinstance(self.labels, dict): + labels_dict = self.labels + self.labels = Labels() + for key, value in labels_dict.items(): + self.labels.add_label(key, value) + + if self.enable_ipv4 is not None and self.enable_ipv6 is not None: + # If both explicitly set to false, we should error out + if not self.enable_ipv4 and not self.enable_ipv6: + raise RenderError(f"Network [{self.name}] cannot have both IPv4 and IPv6 disabled") + + +class Network: + def __init__(self, name: str, config: dict = {}): + self._name = name + self._config = NetworkConfig(**config) + + def render(self): + result: dict = {} + if self._config.name is not None: + result["name"] = self._config.name + if self._config.external is not None: + result["external"] = self._config.external + # FIXME: Re-enable after few months. this flag is added on compose v2.33.1 + # TrueNAS 25.04 still uses docker-compose v2.32.3 + # if self._config.enable_ipv4 is not None: + # result["enable_ipv4"] = self._config.enable_ipv4 + if self._config.enable_ipv6 is not None: + result["enable_ipv6"] = self._config.enable_ipv6 + if self._config.labels and self._config.labels.has_labels(): + result["labels"] = self._config.labels.render() + + return result + + +@dataclass +class ContainerNetworkConfig: + interface_name: str | None = None + """If set, this will be the name of the network interface inside the container.""" + mac_address: str | None = None + """If set, this will be the MAC address of the network interface inside the container.""" + ipv4_address: str | None = None + """If set, this will be the IPv4 address of the network interface inside the container.""" + ipv6_address: str | None = None + """If set, this will be the IPv6 address of the network interface inside the container.""" + gw_priority: int | None = None + """The network with the highest gw_priority is selected as the default gateway for the service container.""" + priority: int | None = None + """Indicates in which order Compose connects the service’s containers to its networks.""" + aliases: list[str] | None = None + """If set, this will be the list of DNS aliases for the network interface inside the container.""" + + +class ContainerNetworks: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._networks: dict[str, ContainerNetwork] = {} + + def add(self, container_name: str, net_name: str, config: dict = {}): + if not self._render_instance.networks.exists(net_name): + raise RenderError( + f"Network [{net_name}] must be first registered. " + "This is probably a bug in the renderer, please report it." + ) + + if self.exists(net_name): + raise RenderError(f"Network [{net_name}] already added to the container [{container_name}].") + + net = ContainerNetwork(net_name, config) + for existing_net in self._networks.values(): + if net._config.interface_name and existing_net._config.interface_name: + if net._config.interface_name == existing_net._config.interface_name: + raise RenderError( + f"Network [{net_name}] cannot have the same interface name " + f"[{net._config.interface_name}] as network [{existing_net._name}]" + f" in container [{container_name}]" + ) + if net._config.mac_address and existing_net._config.mac_address: + if net._config.mac_address == existing_net._config.mac_address: + raise RenderError( + f"Network [{net_name}] cannot have the same MAC address " + f"[{net._config.mac_address}] as network [{existing_net._name}]" + f" in container [{container_name}]" + ) + if net._config.ipv4_address and existing_net._config.ipv4_address: + if net._config.ipv4_address == existing_net._config.ipv4_address: + raise RenderError( + f"Network [{net_name}] cannot have the same IPv4 address " + f"[{net._config.ipv4_address}] as network [{existing_net._name}]" + f" in container [{container_name}]" + ) + if net._config.ipv6_address and existing_net._config.ipv6_address: + if net._config.ipv6_address == existing_net._config.ipv6_address: + raise RenderError( + f"Network [{net_name}] cannot have the same IPv6 address " + f"[{net._config.ipv6_address}] as network [{existing_net._name}]" + f" in container [{container_name}]" + ) + if isinstance(net._config.gw_priority, int) and isinstance(existing_net._config.gw_priority, int): + if net._config.gw_priority == existing_net._config.gw_priority: + raise RenderError( + f"Network [{net_name}] cannot have the same gateway priority " + f"[{net._config.gw_priority}] as network [{existing_net._name}]" + f" in container [{container_name}]" + ) + if isinstance(net._config.priority, int) and isinstance(existing_net._config.priority, int): + if net._config.priority == existing_net._config.priority: + raise RenderError( + f"Network [{net_name}] cannot have the same priority " + f"[{net._config.priority}] as network [{existing_net._name}]" + f" in container [{container_name}]" + ) + if net._config.aliases and existing_net._config.aliases: + overlapping_aliases = set(net._config.aliases) & set(existing_net._config.aliases) + if overlapping_aliases: + raise RenderError( + f"Network [{net_name}] cannot have the same aliases " + f"[{', '.join(overlapping_aliases)}] as network [{existing_net._name}]" + f" in container [{container_name}]" + ) + + self._networks[net_name] = net + + def has_items(self): + return bool(self._networks) + + def exists(self, name: str): + return name in self._networks + + def render(self): + result: dict = {} + for name in sorted(self._networks.keys()): + result[name] = self._networks[name].render() + return result + + +class ContainerNetwork: + def __init__(self, name: str, config: dict = {}): + self._name = name + self._config = ContainerNetworkConfig(**config) + + def render(self): + result: dict = {} + if self._config.interface_name: + result["interface_name"] = self._config.interface_name + if self._config.ipv4_address: + result["ipv4_address"] = valid_ip_or_raise(self._config.ipv4_address) + if self._config.ipv6_address: + result["ipv6_address"] = valid_ip_or_raise(self._config.ipv6_address) + if self._config.mac_address: + result["mac_address"] = valid_mac_or_raise(self._config.mac_address) + if isinstance(self._config.gw_priority, int): + result["gw_priority"] = self._config.gw_priority + if isinstance(self._config.priority, int): + result["priority"] = self._config.priority + if self._config.aliases: + if len(self._config.aliases) != len(set(self._config.aliases)): + raise RenderError( + f"Network [{self._name}] cannot have duplicate aliases " f"[{', '.join(self._config.aliases)}]" + ) + result["aliases"] = self._config.aliases + return result diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/notes.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/notes.py new file mode 100644 index 00000000000..c33c43516ad --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/notes.py @@ -0,0 +1,303 @@ +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +SHORT_LIVED = "short-lived" + + +@dataclass +class Security: + header: str + items: list[str] + + +class Notes: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._app_name: str = "" + self._app_train: str = "" + self._info: list[str] = [] + self._warnings: list[str] = [] + self._deprecations: list[str] = [] + self._security: dict[str, list[Security]] = {} + self._header: str = "" + self._body: str = "" + self._footer: str = "" + + self._auto_set_app_name() + self._auto_set_app_train() + self._auto_set_header() + self._auto_set_footer() + + def _is_enterprise_train(self): + if self._app_train == "enterprise": + return True + + def _auto_set_app_name(self): + app_name = self._render_instance.values.get("ix_context", {}).get("app_metadata", {}).get("title", "") + self._app_name = app_name or "" + + def _auto_set_app_train(self): + app_train = self._render_instance.values.get("ix_context", {}).get("app_metadata", {}).get("train", "") + self._app_train = app_train or "" + + def _auto_set_header(self): + self._header = f"# {self._app_name}\n\n" + + def _auto_set_footer(self): + url = "https://github.com/truenas/apps" + if self._is_enterprise_train(): + url = "https://ixsystems.atlassian.net" + footer = "## Bug Reports and Feature Requests\n\n" + footer += "If you find a bug in this app or have an idea for a new feature, please file an issue at\n" + footer += f"{url}\n" + self._footer = footer + + def add_info(self, info: str): + self._info.append(info) + + def add_warning(self, warning: str): + self._warnings.append(warning) + + def _prepend_warning(self, warning: str): + self._warnings.insert(0, warning) + + def add_deprecation(self, deprecation: str): + self._deprecations.append(deprecation) + + def set_body(self, body: str): + self._body = body + + def get_pretty_host_mount(self, hm: str) -> tuple[str, bool]: + hm = hm.rstrip("/") + mapping = { + "/dev": "Device Directory", + "/dev/bus/usb": "USB Devices", + "/dev/net/tun": "TUN Device", + "/dev/snd": "Sound Device", + "/dev/fuse": "Fuse Device", + "/dev/hugepages": "Huge Pages", + "/dev/cpu": "CPU Device", + "/dev/uinput": "UInput Device", + "/dev/dvb": "DVB Devices", + "/dev/dri": "DRI Device", + "/dev/kfd": "AMD GPU Device", + "/etc/os-release": "OS Release File", + "/etc/group": "Group File", + "/etc/passwd": "Password File", + "/etc/hostname": "Hostname File", + "/lib/modules": "Kernel Modules", + "/proc": "Process Information", + "/sys": "System Information", + "/var/run/docker.sock": "Docker Socket", + "/var/run/utmp": "UTMP", + "/var/run/dbus": "DBus Socket", + "/run/udev": "Udev Socket", + } + if hm in mapping: + return f"{mapping[hm]} ({hm})", True + + hm = hm + "/" + starters = ("/dev/", "/proc/", "/sys/", "/etc/", "/lib/") + if any(hm.startswith(s) for s in starters): + return hm.rstrip("/"), True + + return "", False + + def get_group_name_from_id(self, group_id: int | str) -> str: + mapping = { + 0: "root", + 20: "dialout", + 24: "cdrom", + 29: "audio", + 568: "apps", + 999: "docker", + } + if group_id in mapping: + return mapping[group_id] + return str(group_id) + + def scan_containers(self): + for name, c in self._render_instance._containers.items(): + if self._security.get(name) is None: + self._security[name] = [] + + if c.restart._policy == "on-failure": + self._security[name].append(Security(header=SHORT_LIVED, items=[])) + + if c._grace_period and c._grace_period > 60: + self.add_warning( + f"Container [{name}] has a grace period of [{c._grace_period}] seconds. " + "TrueNAS waits a maximum of 60 seconds for docker engine to stop " + "during system reboot/shutdown. If the container needs the full configured grace period, " + "manually stop it before reboot/shutdown to ensure the full wait time is honored." + ) + + if c._privileged: + self._security[name].append( + Security( + header="Privileged mode is enabled", + items=[ + "Has the same level of control as a system administrator", + "Can access and modify any part of your TrueNAS system", + ], + ) + ) + + networks = [] + for net in c.networks._networks.values(): + networks.append(net._name) + if networks: + self._security[name].append(Security(header="Joined networks", items=networks)) + + run_as_sec_items = [] + user, group = c._user.split(":") if c._user else [-1, -1] + if user in ["0", -1]: + user = "root" if user == "0" else "unknown" + if group in ["0", -1]: + group = "root" if group == "0" else "unknown" + run_as_sec_items.append(f"User: {user}") + run_as_sec_items.append(f"Group: {group}") + groups = [self.get_group_name_from_id(g) for g in c._group_add] + if groups: + groups_str = ", ".join(sorted(groups)) + run_as_sec_items.append(f"Supplementary Groups: {groups_str}") + self._security[name].append(Security("Running user/group(s)", run_as_sec_items)) + + if c._ipc_mode == "host": + self._security[name].append( + Security( + header="Host IPC namespace is enabled", + items=[ + "Container can access the inter-process communication mechanisms of the host", + "Allows communication with other processes on the host under particular circumstances", + ], + ) + ) + if c._pid_mode == "host": + self._security[name].append( + Security( + header="Host PID namespace is enabled", + items=[ + "Container can see and interact with all host processes", + "Potential for privilege escalation or process manipulation", + ], + ) + ) + if c._cgroup == "host": + self._security[name].append( + Security( + header="Host cgroup namespace is enabled", + items=[ + "Container shares control groups with the host system", + "Can bypass resource limits and isolation boundaries", + ], + ) + ) + if "no-new-privileges=true" not in c._security_opt.render(): + self._security[name].append( + Security( + header="Security option [no-new-privileges] is not set", + items=[ + "Processes can gain additional privileges through setuid/setgid binaries", + "Can potentially allow privilege escalation attacks within the container", + ], + ) + ) + + host_mounts = [] + for dev in c.devices._devices: + pretty, _ = self.get_pretty_host_mount(dev.host_device) + host_mounts.append(f"{pretty} - ({dev.cgroup_perm or 'Read/Write'})") + + for vm in c.storage._volume_mounts: + if vm.volume_mount_spec.get("type", "") == "bind": + source = vm.volume_mount_spec.get("source", "") + read_only = vm.volume_mount_spec.get("read_only", False) + pretty, is_host_mount = self.get_pretty_host_mount(source) + if is_host_mount: + host_mounts.append(f"{pretty} - ({'Read Only' if read_only else 'Read/Write'})") + + if host_mounts: + self._security[name].append( + Security( + header="Passing Host Files, Devices, or Sockets into the Container", items=sorted(host_mounts) + ) + ) + if c._tty: + self._prepend_warning( + f"Container [{name}] is running with a TTY, " + "Logs do not appear correctly in the UI due to an [upstream bug]" + "(https://github.com/docker/docker-py/issues/1394)" + ) + self._security = {k: v for k, v in self._security.items() if v} + + def render(self): + self.scan_containers() + + result = self._header + + if self._warnings: + result += "## Warnings\n\n" + for warning in self._warnings: + result += f"- {warning}\n" + result += "\n" + + if self._deprecations: + result += "## Deprecations\n\n" + for deprecation in self._deprecations: + result += f"- {deprecation}\n" + result += "\n" + + if self._info: + result += "## Info\n\n" + for info in self._info: + result += f"- {info}\n" + result += "\n" + + if self._security: + result += "## Security\n\n" + result += "**Read the following security precautions to ensure" + result += " that you wish to continue using this application.**\n\n" + + def render_security(container_name: str, security: list[Security]) -> str: + output = "---\n\n" + output += f"### Container: [{container_name}]" + if any(sec.header == SHORT_LIVED for sec in security): + output += "\n\n**This container is short-lived.**" + output += "\n\n" + for sec in [s for s in security if s.header != SHORT_LIVED]: + output += f"#### {sec.header}\n\n" + for item in sec.items: + output += f"- {item}\n" + if sec.items: + output += "\n" + return output + + sec_list = [] + sec_short_lived_list = [] + for container_name, security in self._security.items(): + if any(sec.header == SHORT_LIVED for sec in security): + sec_short_lived_list.append((container_name, security)) + continue + sec_list.append((container_name, security)) + + sec_list = sorted(sec_list, key=lambda x: x[0]) + sec_short_lived_list = sorted(sec_short_lived_list, key=lambda x: x[0]) + + joined_sec_list = [*sec_list, *sec_short_lived_list] + for idx, item in enumerate(joined_sec_list): + container, sec = item + result += render_security(container, sec) + # If its the last container, add a final --- + if idx == len(joined_sec_list) - 1: + result += "---\n\n" + + if self._body: + result += self._body.strip() + "\n\n" + + result += self._footer + + return result diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/portals.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/portals.py new file mode 100644 index 00000000000..00362ebc231 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/portals.py @@ -0,0 +1,73 @@ +from typing import TYPE_CHECKING + +import copy + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_portal_scheme_or_raise, valid_http_path_or_raise, valid_port_or_raise +except ImportError: + from error import RenderError + from validations import valid_portal_scheme_or_raise, valid_http_path_or_raise, valid_port_or_raise + + +class Portals: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._portals: set[Portal] = set() + + def add(self, port: dict, config: dict | None = None): + config = copy.deepcopy((config or {})) + port = copy.deepcopy((port or {})) + # If its not published, portal does not make sense + if port.get("bind_mode", "") != "published": + return + + name = config.get("name", "Web UI") + + if name in [p._name for p in self._portals]: + raise RenderError(f"Portal [{name}] already added") + + host = config.get("host", None) + host_ips = port.get("host_ips", []) + if not isinstance(host_ips, list): + raise RenderError("Expected [host_ips] to be a list of strings") + + # Remove wildcard IPs + if "::" in host_ips: + host_ips.remove("::") + if "0.0.0.0" in host_ips: + host_ips.remove("0.0.0.0") + + # If host is not set, use the first host_ip (if it exists) + if not host and len(host_ips) >= 1: + host = host_ips[0] + + config["host"] = host + if not config.get("port"): + config["port"] = port.get("port_number", 0) + + self._portals.add(Portal(name, config)) + + def render(self): + return [p.render() for _, p in sorted([(p._name, p) for p in self._portals])] + + +class Portal: + def __init__(self, name: str, config: dict): + self._name = name + self._scheme = valid_portal_scheme_or_raise(config.get("scheme", "http")) + self._host = config.get("host", "0.0.0.0") or "0.0.0.0" + self._port = valid_port_or_raise(config.get("port", 0)) + self._path = valid_http_path_or_raise(config.get("path", "/")) + + def render(self): + return { + "name": self._name, + "scheme": self._scheme, + "host": self._host, + "port": self._port, + "path": self._path, + } diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/ports.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/ports.py new file mode 100644 index 00000000000..87e6ded9515 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/ports.py @@ -0,0 +1,147 @@ +import ipaddress +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import ( + valid_ip_or_raise, + valid_port_mode_or_raise, + valid_port_or_raise, + valid_port_protocol_or_raise, + ) +except ImportError: + from error import RenderError + from validations import ( + valid_ip_or_raise, + valid_port_mode_or_raise, + valid_port_or_raise, + valid_port_protocol_or_raise, + ) + + +class Ports: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._ports: dict[str, dict] = {} + + def _gen_port_key(self, host_port: int, host_ip: str, proto: str, ip_family: int) -> str: + return f"{host_port}_{host_ip}_{proto}_{ip_family}" + + def _is_wildcard_ip(self, ip: str) -> bool: + return ip in ["0.0.0.0", "::"] + + def _get_opposite_wildcard(self, ip: str) -> str: + return "0.0.0.0" if ip == "::" else "::" + + def _get_sort_key(self, p: dict) -> str: + return f"{p['published']}_{p['target']}_{p['protocol']}_{p.get('host_ip', '_')}" + + def _is_ports_same(self, port1: dict, port2: dict) -> bool: + return ( + port1["published"] == port2["published"] + and port1["target"] == port2["target"] + and port1["protocol"] == port2["protocol"] + and port1.get("host_ip", "_") == port2.get("host_ip", "_") + ) + + def _has_opposite_family_port(self, port_config: dict, wildcard_ports: dict) -> bool: + comparison_port = port_config.copy() + comparison_port["host_ip"] = self._get_opposite_wildcard(port_config["host_ip"]) + for p in wildcard_ports.values(): + if self._is_ports_same(comparison_port, p): + return True + return False + + def _check_port_conflicts(self, port_config: dict, ip_family: int) -> None: + host_port = port_config["published"] + host_ip = port_config["host_ip"] + proto = port_config["protocol"] + + key = self._gen_port_key(host_port, host_ip, proto, ip_family) + + if key in self._ports.keys(): + raise RenderError(f"Port [{host_port}/{proto}/ipv{ip_family}] already added for [{host_ip}]") + + wildcard_ip = "0.0.0.0" if ip_family == 4 else "::" + if host_ip != wildcard_ip: + # Check if there is a port with same details but with wildcard IP of the same family + wildcard_key = self._gen_port_key(host_port, wildcard_ip, proto, ip_family) + if wildcard_key in self._ports.keys(): + raise RenderError( + f"Cannot bind port [{host_port}/{proto}/ipv{ip_family}] to [{host_ip}], " + f"already bound to [{wildcard_ip}]" + ) + else: + # We are adding a port with wildcard IP + # Check if there is a port with same details but with specific IP of the same family + for p in self._ports.values(): + # Skip if the port is not for the same family + if ip_family != ipaddress.ip_address(p["host_ip"]).version: + continue + + # Make a copy of the port config + search_port = p.copy() + # Replace the host IP with wildcard IP + search_port["host_ip"] = wildcard_ip + # If the ports match, means that a port for specific IP is already added + # and we are trying to add it again with wildcard IP. Raise an error + if self._is_ports_same(search_port, port_config): + raise RenderError( + f"Cannot bind port [{host_port}/{proto}/ipv{ip_family}] to [{host_ip}], " + f"already bound to [{p['host_ip']}]" + ) + + def _add_port(self, host_port: int, container_port: int, config: dict | None = None): + config = config or {} + host_port = valid_port_or_raise(host_port) + container_port = valid_port_or_raise(container_port) + proto = valid_port_protocol_or_raise(config.get("protocol", "tcp")) + mode = valid_port_mode_or_raise(config.get("mode", "ingress")) + + host_ip = valid_ip_or_raise(config.get("host_ip", "")) + ip = ipaddress.ip_address(host_ip) + + port_config = { + "published": host_port, + "target": container_port, + "protocol": proto, + "mode": mode, + "host_ip": host_ip, + } + self._check_port_conflicts(port_config, ip.version) + + key = self._gen_port_key(host_port, host_ip, proto, ip.version) + self._ports[key] = port_config + # After all the local validations, lets validate the port with the TrueNAS API + self._render_instance.client.validate_ip_port_combo(host_ip, host_port) + + def has_ports(self): + return len(self._ports) > 0 + + def render(self): + specific_ports = [] + wildcard_ports = {} + + for port_config in self._ports.values(): + if self._is_wildcard_ip(port_config["host_ip"]): + wildcard_ports[id(port_config)] = port_config.copy() + else: + specific_ports.append(port_config.copy()) + + processed_ports = specific_ports.copy() + for wild_port in wildcard_ports.values(): + processed_port = wild_port.copy() + + # Check if there's a matching wildcard port for the opposite IP family + has_opposite_family = self._has_opposite_family_port(wild_port, wildcard_ports) + + if has_opposite_family: + processed_port.pop("host_ip") + + if processed_port not in processed_ports: + processed_ports.append(processed_port) + + return sorted(processed_ports, key=self._get_sort_key) diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/render.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/render.py new file mode 100644 index 00000000000..9a27de577fb --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/render.py @@ -0,0 +1,101 @@ +import copy + +try: + from .configs import Configs + from .container import Container + from .deps import Deps + from .docker_client import DockerClient + from .error import RenderError + from .functions import Functions + from .networks import Networks + from .notes import Notes + from .portals import Portals + from .truenas_client import TNClient + from .volumes import Volumes +except ImportError: + from configs import Configs + from container import Container + from deps import Deps + from docker_client import DockerClient + from error import RenderError + from functions import Functions + from networks import Networks + from notes import Notes + from portals import Portals + from truenas_client import TNClient + from volumes import Volumes + + +class Render(object): + def __init__(self, values): + self._containers: dict[str, Container] = {} + self.values = values + # Make a copy after we inject the images + self._original_values: dict = copy.deepcopy(self.values) + + self.deps: Deps = Deps(self) + + self.client: TNClient = TNClient(render_instance=self) + self.docker: DockerClient = DockerClient(render_instance=self) + + self.configs = Configs(render_instance=self) + self.funcs = Functions(render_instance=self).func_map() + self.portals: Portals = Portals(render_instance=self) + self.notes: Notes = Notes(render_instance=self) + self.networks: Networks = Networks(render_instance=self) + self.volumes = Volumes(render_instance=self) + + self._auto_add_networks() + + def _auto_add_networks(self): + networks = self.values.get("network", {}).get("networks", []) + if not networks: + return + + for network in networks: + self.networks.register(network["name"]) + + def container_names(self): + return list(self._containers.keys()) + + def add_container(self, name: str, image: str): + name = name.strip() + if not name: + raise RenderError("Container name cannot be empty") + container = Container(self, name, image) + if name in self._containers: + raise RenderError(f"Container {name} already exists.") + self._containers[name] = container + return container + + def render(self): + if self.values != self._original_values: + raise RenderError("Values have been modified since the renderer was created.") + + if not self._containers: + raise RenderError("No containers added.") + + result: dict = { + "x-notes": self.notes.render(), + "x-portals": self.portals.render(), + "services": {c._name: c.render() for c in self._containers.values()}, + } + + # Make sure that after services are rendered + # there are no labels that target a non-existent container + # This is to prevent typos + for label in self.values.get("labels", []): + for c in label.get("containers", []): + if c not in self.container_names(): + raise RenderError(f"Label [{label['key']}] references container [{c}] which does not exist") + + if self.volumes.has_volumes(): + result["volumes"] = self.volumes.render() + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self.networks.has_items(): + result["networks"] = self.networks.render() + + return result diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/resources.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/resources.py new file mode 100644 index 00000000000..733f43bb6f7 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/resources.py @@ -0,0 +1,115 @@ +import re +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +DEFAULT_CPUS = 2.0 +DEFAULT_MEMORY = 4096 + + +class Resources: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._limits: dict = {} + self._reservations: dict = {} + self._nvidia_ids: set[str] = set() + self._auto_add_cpu_from_values() + self._auto_add_memory_from_values() + self._auto_add_gpus_from_values() + + def _set_cpu(self, cpus: Any): + c = str(cpus) + if not re.match(r"^[1-9][0-9]*(\.[0-9]+)?$", c): + raise RenderError(f"Expected cpus to be a number or a float (minimum 1.0), got [{cpus}]") + self._limits.update({"cpus": c}) + + def _set_memory(self, memory: Any): + m = str(memory) + if not re.match(r"^[1-9][0-9]*$", m): + raise RenderError(f"Expected memory to be a number, got [{memory}]") + self._limits.update({"memory": f"{m}M"}) + + def _auto_add_cpu_from_values(self): + resources = self._render_instance.values.get("resources", {}) + self._set_cpu(resources.get("limits", {}).get("cpus", DEFAULT_CPUS)) + + def _auto_add_memory_from_values(self): + resources = self._render_instance.values.get("resources", {}) + self._set_memory(resources.get("limits", {}).get("memory", DEFAULT_MEMORY)) + + def _auto_add_gpus_from_values(self): + resources = self._render_instance.values.get("resources", {}) + gpus = resources.get("gpus", {}).get("nvidia_gpu_selection", {}) + if not gpus: + return + + for pci, gpu in gpus.items(): + if gpu.get("use_gpu", False): + if not gpu.get("uuid"): + raise RenderError(f"Expected [uuid] to be set for GPU in slot [{pci}] in [nvidia_gpu_selection]") + self._nvidia_ids.add(gpu["uuid"]) + + if self._nvidia_ids: + if not self._reservations: + self._reservations["devices"] = [] + self._reservations["devices"].append( + { + "capabilities": ["gpu"], + "driver": "nvidia", + "device_ids": sorted(self._nvidia_ids), + } + ) + + # This is only used on ix-app that we allow + # disabling cpus and memory. GPUs are only added + # if the user has requested them. + def remove_cpus_and_memory(self): + self._limits.pop("cpus", None) + self._limits.pop("memory", None) + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._reservations.pop("devices", None) + + def set_profile(self, profile: str): + cpu, memory = profile_mapping(profile) + self._set_cpu(cpu) + self._set_memory(memory) + + def has_resources(self): + return len(self._limits) > 0 or len(self._reservations) > 0 + + def has_gpus(self): + gpu_devices = [d for d in self._reservations.get("devices", []) if "gpu" in d["capabilities"]] + return len(gpu_devices) > 0 + + def render(self): + result = {} + if self._limits: + result["limits"] = self._limits + if self._reservations: + result["reservations"] = self._reservations + + return result + + +def profile_mapping(profile: str): + profiles = { + "low": (1, 512), + "medium": (2, 1024), + } + + if profile not in profiles: + raise RenderError( + f"Resource profile [{profile}] is not valid. Valid options are: [{', '.join(profiles.keys())}]" + ) + + return profiles[profile] diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/restart.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/restart.py new file mode 100644 index 00000000000..2f6281af487 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/restart.py @@ -0,0 +1,25 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .validations import valid_restart_policy_or_raise +except ImportError: + from validations import valid_restart_policy_or_raise + + +class RestartPolicy: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._policy: str = "unless-stopped" + self._maximum_retry_count: int = 0 + + def set_policy(self, policy: str, maximum_retry_count: int = 0): + self._policy = valid_restart_policy_or_raise(policy, maximum_retry_count) + self._maximum_retry_count = maximum_retry_count + + def render(self): + if self._policy == "on-failure" and self._maximum_retry_count > 0: + return f"{self._policy}:{self._maximum_retry_count}" + return self._policy diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/security_opts.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/security_opts.py new file mode 100644 index 00000000000..ba288d2b906 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/security_opts.py @@ -0,0 +1,52 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_security_opt_or_raise +except ImportError: + from error import RenderError + from validations import valid_security_opt_or_raise + + +class SecurityOpt: + def __init__(self, opt: str, value: str | bool | None = None, arg: str | None = None): + self._opt: str = valid_security_opt_or_raise(opt) + self._value = str(value).lower() if isinstance(value, bool) else value + self._arg: str | None = arg + + def render(self): + result = self._opt + if self._value is not None: + result = f"{result}={self._value}" + if self._arg is not None: + result = f"{result}:{self._arg}" + return result + + +class SecurityOpts: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._opts: dict[str, SecurityOpt] = dict() + self.add_opt("no-new-privileges", True) + + def add_opt(self, key: str, value: str | bool | None, arg: str | None = None): + if key in self._opts: + raise RenderError(f"Security Option [{key}] already added") + self._opts[key] = SecurityOpt(key, value, arg) + + def remove_opt(self, key: str): + if key not in self._opts: + raise RenderError(f"Security Option [{key}] not found") + del self._opts[key] + + def has_opts(self): + return len(self._opts) > 0 + + def render(self): + result = [] + for opt in sorted(self._opts.values(), key=lambda o: o._opt): + result.append(opt.render()) + return result diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/storage.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/storage.py new file mode 100644 index 00000000000..6f2f69a52de --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/storage.py @@ -0,0 +1,125 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from container import Container + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + uid: NotRequired[int] + gid: NotRequired[int] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._container_instance = container_instance + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if self.is_defined(mount_path): + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + if self._container_instance._tmpfs.is_defined(mount_path): + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def is_defined(self, mount_path: str): + return mount_path in [m.mount_path for m in self._volume_mounts] + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_udev(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/run/udev", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/sysctls.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/sysctls.py new file mode 100644 index 00000000000..e6b8469f3bf --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/__init__.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_build_image.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_build_image.py new file mode 100644 index 00000000000..debb0be96b3 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_build_image.py @@ -0,0 +1,51 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_build_image_with_from(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.build_image(["FROM test_image"]) + + +def test_build_image_with_from_with_whitespace(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.build_image([" FROM test_image"]) + + +def test_build_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.build_image(["RUN echo hello", None, "", "RUN echo world", "RUN echo $MY_VAR > /tmp/test.txt"]) + output = render.render() + assert ( + output["services"]["test_container"]["image"] + == "ix-nginx:latest_bb420e21d704f6aaaa9c671c4698cc4ae1a004333bd74780911cd7df6918487b" + ) + assert output["services"]["test_container"]["build"] == { + "tags": ["ix-nginx:latest_bb420e21d704f6aaaa9c671c4698cc4ae1a004333bd74780911cd7df6918487b"], + "dockerfile_inline": """FROM nginx:latest +RUN echo hello +RUN echo world +RUN echo $$MY_VAR > /tmp/test.txt +""", + } diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_configs.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_configs.py new file mode 100644 index 00000000000..16d663726e0 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_configs.py @@ -0,0 +1,71 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_duplicate_config_with_different_data(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "test_data", "/some/path") + with pytest.raises(Exception): + c1.configs.add("test_config", "test_data2", "/some/path") + + +def test_add_config_with_empty_target(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.configs.add("test_config", "test_data", "") + + +def test_add_config_with_empty_data(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.configs.add("test_config", "", "/some/path") + + +def test_add_duplicate_target(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "test_data", "/some/path") + with pytest.raises(Exception): + c1.configs.add("test_config2", "test_data2", "/some/path") + + +def test_add_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "$test_data", "/some/path") + output = render.render() + assert output["configs"]["test_config"]["content"] == "$$test_data" + assert output["services"]["test_container"]["configs"] == [{"source": "test_config", "target": "/some/path"}] + + +def test_add_config_with_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.configs.add("test_config", "test_data", "/some/path", "0777") + output = render.render() + assert output["configs"]["test_config"]["content"] == "test_data" + assert output["services"]["test_container"]["configs"] == [ + {"source": "test_config", "target": "/some/path", "mode": 511} + ] diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_container.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_container.py new file mode 100644 index 00000000000..1fe84900bdb --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_container.py @@ -0,0 +1,536 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] + + +def test_add_ports(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_port({"port_number": 8081, "container_port": 8080, "bind_mode": "published"}) + c1.add_port({"port_number": 8082, "container_port": 8080, "bind_mode": "published", "protocol": "udp"}) + c1.add_port({"port_number": 8083, "container_port": 8080, "bind_mode": "exposed"}) + c1.add_port({"port_number": 8084, "container_port": 8080, "bind_mode": ""}) + c1.add_port( + {"port_number": 9091, "container_port": 9091, "bind_mode": "published"}, + {"container_port": 9092, "protocol": "udp"}, + ) + output = render.render() + assert output["services"]["test_container"]["ports"] == [ + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress"}, + {"published": 8082, "target": 8080, "protocol": "udp", "mode": "ingress"}, + {"published": 9091, "target": 9092, "protocol": "udp", "mode": "ingress"}, + ] + assert output["services"]["test_container"]["expose"] == ["8080/tcp"] + + +def test_add_ports_with_invalid_host_ips(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_port({"port_number": 8081, "container_port": 8080, "bind_mode": "published", "host_ips": "invalid"}) + + +def test_add_ports_with_empty_host_ips(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_port({"port_number": 8081, "container_port": 8080, "bind_mode": "published", "host_ips": []}) + output = render.render() + assert output["services"]["test_container"]["ports"] == [ + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress"} + ] + + +def test_set_ipc_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_ipc_mode("host") + output = render.render() + assert output["services"]["test_container"]["ipc"] == "host" + + +def test_set_ipc_empty_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_ipc_mode("") + output = render.render() + assert output["services"]["test_container"]["ipc"] == "" + + +def test_set_ipc_mode_with_invalid_ipc_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_ipc_mode("invalid") + + +def test_set_ipc_mode_with_container_ipc_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c2 = render.add_container("test_container2", "test_image") + c2.healthcheck.disable() + c1.set_ipc_mode("container:test_container2") + output = render.render() + assert output["services"]["test_container"]["ipc"] == "container:test_container2" + + +def test_set_ipc_mode_with_container_ipc_mode_and_invalid_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_ipc_mode("container:invalid") + + +def test_set_pid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_pid_mode("host") + output = render.render() + assert output["services"]["test_container"]["pid"] == "host" + + +def test_set_pid_empty_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_pid_mode("") + output = render.render() + assert output["services"]["test_container"]["pid"] == "" + + +def test_set_pid_mode_with_invalid_pid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_pid_mode("invalid") + + +def test_set_pid_mode_with_container_pid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c2 = render.add_container("test_container2", "test_image") + c2.healthcheck.disable() + c1.set_pid_mode("container:test_container2") + output = render.render() + assert output["services"]["test_container"]["pid"] == "container:test_container2" + + +def test_set_pid_mode_with_container_pid_mode_and_invalid_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_pid_mode("container:invalid") + + +def test_set_cgroup(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_cgroup("host") + output = render.render() + assert output["services"]["test_container"]["cgroup"] == "host" + + +def test_set_cgroup_invalid(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_cgroup("invalid") + + +def test_set_mac_invalid(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_mac("invalid_mac") + + +def test_set_mac_valid(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_mac("00:00:00:00:00:00") + output = render.render() + assert output["services"]["test_container"]["mac_address"] == "00:00:00:00:00:00" + + +def test_setup_as_helper(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.setup_as_helper() + output = render.render() + assert output["services"]["test_container"]["restart"] == "on-failure:1" + assert output["services"]["test_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["healthcheck"]["disable"] is True + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "1" + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "512M" + assert "devices" not in output["services"]["test_container"] + + +def test_setup_as_helper_med_profile(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.setup_as_helper(profile="medium") + output = render.render() + assert output["services"]["test_container"]["restart"] == "on-failure:1" + assert output["services"]["test_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["healthcheck"]["disable"] is True + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "2" + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "1024M" + assert "devices" not in output["services"]["test_container"] + + +def test_setup_as_helper_no_profile(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.setup_as_helper(profile="") + output = render.render() + assert output["services"]["test_container"]["restart"] == "on-failure:1" + assert output["services"]["test_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["healthcheck"]["disable"] is True + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert "devices" not in output["services"]["test_container"] + + +def test_setup_as_helper_with_net(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.setup_as_helper(disable_network=False) + output = render.render() + assert output["services"]["test_container"]["restart"] == "on-failure:1" + assert output["services"]["test_container"]["healthcheck"]["disable"] is True + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "1" + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "512M" + assert "devices" not in output["services"]["test_container"] + + +def test_container_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + assert c1.name() == "test_container" diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_depends.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_depends.py new file mode 100644 index 00000000000..a1d83739273 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_depends.py @@ -0,0 +1,54 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_dependency(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c2 = render.add_container("test_container2", "test_image") + c1.healthcheck.disable() + c2.healthcheck.disable() + c1.depends.add_dependency("test_container2", "service_started") + output = render.render() + assert output["services"]["test_container"]["depends_on"]["test_container2"] == {"condition": "service_started"} + + +def test_add_dependency_invalid_condition(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + render.add_container("test_container2", "test_image") + with pytest.raises(Exception): + c1.depends.add_dependency("test_container2", "invalid_condition") + + +def test_add_dependency_missing_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.depends.add_dependency("test_container2", "service_started") + + +def test_add_dependency_duplicate(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + render.add_container("test_container2", "test_image") + c1.depends.add_dependency("test_container2", "service_started") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.depends.add_dependency("test_container2", "service_started") diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_deps.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_deps.py new file mode 100644 index 00000000000..73541d7e836 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_deps.py @@ -0,0 +1,1083 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + }, + "container_utils_image": { + "repository": "ixsystems/container-utils", + "tag": "1.0.0", + }, + "postgres_upgrade_image": { + "repository": "ixsystems/postgres-upgrade", + "tag": "1.0.0", + }, + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "pg_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_unsupported_repo(mock_values): + mock_values["images"]["pg_image"] = {"repository": "unsupported_repo", "tag": "16.6-bookworm"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + with pytest.raises(Exception): + render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16.6-bookworm"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + "port": 5000, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5000/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["stop_grace_period"] == "60s" + assert output["services"]["pg_container"]["image"] == "postgres:16.6-bookworm" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": [ + "CMD", + "pg_isready", + "-h", + "127.0.0.1", + "-p", + "5000", + "-U", + "test_user", + "-d", + "test_database", + ], + "interval": "30s", + "timeout": "5s", + "retries": 5, + "start_period": "15s", + "start_interval": "2s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "PGPORT": "5000", + "PGDATA": "/var/lib/postgresql/16/docker", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_postgres_options(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16.6-bookworm"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + "additional_options": {"maintenance_work_mem": "1024MB", "max_connections": "100"}, + }, + perms_container, + ) + + output = render.render() + assert output["services"]["pg_container"]["command"] == [ + "-c", + "maintenance_work_mem=1024MB", + "-c", + "max_connections=100", + ] + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "redis_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_unsupported_repo(mock_values): + mock_values["images"]["redis_image"] = {"repository": "unsupported_repo", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + with pytest.raises(Exception): + render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "redis_container", + "redis_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "valkey/valkey", "tag": "latest"} + mock_values["run_as"] = {"user": 0, "group": 0} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["stop_grace_period"] == "60s" + assert output["services"]["redis_container"]["image"] == "valkey/valkey:latest" + assert output["services"]["redis_container"]["user"] == "568:568" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": [ + "CMD", + "redis-cli", + "-h", + "127.0.0.1", + "-p", + "6379", + "-a", + "test&password@", + "ping", + ], + "interval": "30s", + "timeout": "5s", + "retries": 5, + "start_period": "15s", + "start_interval": "2s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "REDIS_PASSWORD": "test&password@", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "mariadb_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb_unsupported_repo(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "unsupported_repo", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + with pytest.raises(Exception): + render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["stop_grace_period"] == "60s" + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": [ + "CMD", + "mariadb-admin", + "--user=root", + "--host=127.0.0.1", + "--port=3306", + "--password=test_password", + "ping", + ], + "interval": "30s", + "timeout": "5s", + "retries": 5, + "start_period": "15s", + "start_interval": "2s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17.7-bookworm"} + mock_values["images"]["redis_image"] = {"repository": "valkey/valkey", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + read_only_volume = {"type": "volume", "read_only": True, "volume_config": {"volume_name": "test_read_only_volume", "auto_permissions": True}} # noqa + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + c1.add_storage("/some/path10", read_only_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data10", read_only_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + # fmt: off + content = [ + {"read_only": False, "mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"read_only": False, "mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"read_only": False, "mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"read_only": False, "mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"read_only": False, "mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"read_only": True, "mount_path": "/mnt/permission/data10", "is_temporary": False, "identifier": "data10", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"read_only": False, "mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"read_only": False, "mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 568, "gid": 568, "chmod": None}, # noqa + {"read_only": False, "mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + assert output["services"]["test_perms_container"]["entrypoint"] == ["python3", "/script/permissions.py"] + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "pg_container", + "pg_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16.6-bookworm"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["PGDATA"] == "/var/lib/postgresql/16/docker" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["image"] == "ixsystems/postgres-upgrade:1.0.0" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] + + +def test_postgres_version_with_digest_pin(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "17.7-bookworm@sha256:1234567890"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pgup = output["services"]["postgres_container_upgrade"] + assert pgup["environment"]["TARGET_VERSION"] == "17" + + +def test_add_mongodb(mock_values): + mock_values["images"]["mongodb_image"] = {"repository": "mongo", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mongodb( + "mongodb_container", + "mongodb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mongodb_container"] + assert "reservations" not in output["services"]["mongodb_container"]["deploy"]["resources"] + assert output["services"]["mongodb_container"]["stop_grace_period"] == "60s" + assert output["services"]["mongodb_container"]["image"] == "mongo:latest" + assert output["services"]["mongodb_container"]["user"] == "568:568" + assert output["services"]["mongodb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mongodb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mongodb_container"]["healthcheck"] == { + "test": [ + "CMD", + "mongosh", + "--host", + "127.0.0.1", + "--port", + "27017", + "test_database", + "--eval", + 'db.adminCommand("ping")', + "--quiet", + ], + "interval": "30s", + "timeout": "5s", + "retries": 5, + "start_period": "15s", + "start_interval": "2s", + } + assert output["services"]["mongodb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/data/db", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mongodb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MONGO_INITDB_ROOT_USERNAME": "test_user", + "MONGO_INITDB_ROOT_PASSWORD": "test_password", + "MONGO_INITDB_DATABASE": "test_database", + } + assert output["services"]["mongodb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mongodb_unsupported_repo(mock_values): + mock_values["images"]["mongo_image"] = {"repository": "unsupported_repo", "tag": "7"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + with pytest.raises(Exception): + render.deps.mongodb( + "mongo_container", + "mongo_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + +def test_add_meilisearch(mock_values): + mock_values["images"]["meili_image"] = {"repository": "getmeili/meilisearch", "tag": "v1.17.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.meilisearch( + "meili_container", + "meili_image", + { + "master_key": "test_master_key", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["meili_container"] + assert "reservations" not in output["services"]["meili_container"]["deploy"]["resources"] + assert output["services"]["meili_container"]["stop_grace_period"] == "60s" + assert output["services"]["meili_container"]["image"] == "getmeili/meilisearch:v1.17.0" + assert output["services"]["meili_container"]["user"] == "568:568" + assert output["services"]["meili_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["meili_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["meili_container"]["healthcheck"] == { + "test": [ + "CMD", + "curl", + "--request", + "GET", + "--silent", + "--output", + "/dev/null", + "--show-error", + "--fail", + "http://127.0.0.1:7700/health", + ], + "interval": "30s", + "timeout": "5s", + "retries": 5, + "start_period": "15s", + "start_interval": "2s", + } + assert output["services"]["meili_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/meili_data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["meili_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MEILI_MASTER_KEY": "test_master_key", + "MEILI_HTTP_ADDR": "0.0.0.0:7700", + "MEILI_NO_ANALYTICS": "true", + "MEILI_EXPERIMENTAL_DUMPLESS_UPGRADE": "true", + } + assert output["services"]["meili_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_meilisearch_unsupported_repo(mock_values): + mock_values["images"]["meili_image"] = {"repository": "unsupported_repo", "tag": "7"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + with pytest.raises(Exception): + render.deps.meilisearch( + "meili_container", + "meili_image", + { + "master_key": "test_master_key", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + +def test_add_elasticsearch(mock_values): + mock_values["images"]["elastic_image"] = { + "repository": "elasticsearch", + "tag": "9.1.2", + } + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.elasticsearch( + "elastic_container", + "elastic_image", + { + "password": "test_password", + "node_name": "some_test_node", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["elastic_container"] + assert "reservations" not in output["services"]["elastic_container"]["deploy"]["resources"] + assert output["services"]["elastic_container"]["stop_grace_period"] == "60s" + assert output["services"]["elastic_container"]["image"] == "elasticsearch:9.1.2" + assert output["services"]["elastic_container"]["user"] == "1000:1000" + assert output["services"]["elastic_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["elastic_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["elastic_container"]["healthcheck"] == { + "test": [ + "CMD", + "curl", + "--request", + "GET", + "--silent", + "--output", + "/dev/null", + "--show-error", + "--fail", + "--header", + "Authorization: Basic ZWxhc3RpYzp0ZXN0X3Bhc3N3b3Jk", + "http://127.0.0.1:9200/_cluster/health?local=true", + ], # noqa + "interval": "30s", + "timeout": "5s", + "retries": 5, + "start_period": "15s", + "start_interval": "2s", + } + assert output["services"]["elastic_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/usr/share/elasticsearch/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["elastic_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ELASTIC_PASSWORD": "test_password", + "http.port": "9200", + "path.data": "/usr/share/elasticsearch/data", + "path.repo": "/usr/share/elasticsearch/data/snapshots", + "node.name": "some_test_node", + "discovery.type": "single-node", + "xpack.security.enabled": "true", + "xpack.security.transport.ssl.enabled": "false", + } + assert output["services"]["elastic_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_elasticsearch_unsupported_repo(mock_values): + mock_values["images"]["elastic_image"] = {"repository": "unsupported_repo", "tag": "7"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + with pytest.raises(Exception): + render.deps.elasticsearch( + "elastic_container", + "elastic_image", + { + "password": "test_password", + "node_name": "some_test_node", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + +def test_add_solr(mock_values): + mock_values["images"]["solr_image"] = {"repository": "solr", "tag": "9.9.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.solr( + "solr_container", + "solr_image", + { + "core": "test_core", + "modules": ["analysis-extras", "some-other-module"], + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["solr_container"] + assert "reservations" not in output["services"]["solr_container"]["deploy"]["resources"] + assert output["services"]["solr_container"]["stop_grace_period"] == "60s" + assert output["services"]["solr_container"]["image"] == "solr:9.9.0" + assert output["services"]["solr_container"]["user"] == "568:568" + assert output["services"]["solr_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["solr_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["solr_container"]["healthcheck"] == { + "test": [ + "CMD", + "curl", + "--request", + "GET", + "--silent", + "--output", + "/dev/null", + "--show-error", + "--fail", + "http://127.0.0.1:8983/solr/test_core/admin/ping", + ], + "interval": "30s", + "timeout": "5s", + "retries": 5, + "start_period": "15s", + "start_interval": "2s", + } + assert output["services"]["solr_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/solr", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["solr_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "SOLR_PORT": "8983", + "SOLR_MODULES": "analysis-extras,some-other-module", + } + assert output["services"]["solr_container"]["command"] == ["solr-precreate", "test_core"] + assert output["services"]["solr_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_solr_unsupported_repo(mock_values): + mock_values["images"]["solr_image"] = {"repository": "unsupported_repo", "tag": "7"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + with pytest.raises(Exception): + render.deps.solr( + "solr_container", + "solr_image", + { + "core": "test_core", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + +def test_add_tika(mock_values): + mock_values["images"]["tika_image"] = {"repository": "apache/tika", "tag": "3.2.3.0-full"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + render.deps.tika( + "tika_container", + "tika_image", + { + "port": 10999, + }, + ) + output = render.render() + assert "devices" not in output["services"]["tika_container"] + assert "reservations" not in output["services"]["tika_container"]["deploy"]["resources"] + assert output["services"]["tika_container"]["stop_grace_period"] == "60s" + assert output["services"]["tika_container"]["image"] == "apache/tika:3.2.3.0-full" + assert output["services"]["tika_container"]["user"] == "568:568" + assert output["services"]["tika_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["tika_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["tika_container"]["healthcheck"] == { + "test": [ + "CMD", + "wget", + "--quiet", + "-O", + "/dev/null", + "http://127.0.0.1:10999/tika", + ], + "interval": "30s", + "timeout": "5s", + "retries": 5, + "start_period": "15s", + "start_interval": "2s", + } + assert output["services"]["tika_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + } + assert output["services"]["tika_container"]["command"] == ["--port", "10999"] + + +def test_add_tika_unsupported_repo(mock_values): + mock_values["images"]["tika_image"] = {"repository": "unsupported_repo", "tag": "7"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.tika( + "tika_container", + "tika_image", + {}, + ) + + +def test_add_memcached(mock_values): + mock_values["images"]["memcached_image"] = {"repository": "memcached", "tag": "1.6.40"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + render.deps.memcached( + "memcached_container", + "memcached_image", + { + "port": 10999, + "memory_mb": 512, + }, + ) + output = render.render() + assert "devices" not in output["services"]["memcached_container"] + assert "reservations" not in output["services"]["memcached_container"]["deploy"]["resources"] + assert output["services"]["memcached_container"]["stop_grace_period"] == "60s" + assert output["services"]["memcached_container"]["image"] == "memcached:1.6.40" + assert output["services"]["memcached_container"]["user"] == "568:568" + assert output["services"]["memcached_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["memcached_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["memcached_container"]["healthcheck"] == { + "test": [ + "CMD", + "timeout", + "1", + "bash", + "-c", + "cat < /dev/null > /dev/tcp/127.0.0.1/10999", + ], + "interval": "30s", + "timeout": "5s", + "retries": 5, + "start_period": "15s", + "start_interval": "2s", + } + assert output["services"]["memcached_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + } + assert output["services"]["memcached_container"]["command"] == ["-p", "10999", "-m", "512M"] + + +def test_add_memcached_unsupported_repo(mock_values): + mock_values["images"]["memcached_image"] = {"repository": "unsupported_repo", "tag": "7"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.memcached( + "memcached_container", + "memcached_image", + {}, + ) diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_device.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_device.py new file mode 100644 index 00000000000..2e71daa5a0a --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_device.py @@ -0,0 +1,150 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") + + +def test_add_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_add_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/net/tun:/dev/net/tun"] diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_device_cgroup_rules.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_device_cgroup_rules.py new file mode 100644 index 00000000000..581fe82017f --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_device_cgroup_rules.py @@ -0,0 +1,79 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_device_cgroup_rule(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_device_cgroup_rule("c 13:* rwm") + c1.add_device_cgroup_rule("b 10:20 rwm") + output = render.render() + assert output["services"]["test_container"]["device_cgroup_rules"] == [ + "b 10:20 rwm", + "c 13:* rwm", + ] + + +def test_device_cgroup_rule_duplicate(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_device_cgroup_rule("c 13:* rwm") + with pytest.raises(Exception): + c1.add_device_cgroup_rule("c 13:* rwm") + + +def test_device_cgroup_rule_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_device_cgroup_rule("c 13:* rwm") + with pytest.raises(Exception): + c1.add_device_cgroup_rule("c 13:* rm") + + +def test_device_cgroup_rule_invalid_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_device_cgroup_rule("d 10:20 rwm") + + +def test_device_cgroup_rule_invalid_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_device_cgroup_rule("a 10:20 rwd") + + +def test_device_cgroup_rule_invalid_format(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_device_cgroup_rule("a 10 20 rwd") + + +def test_device_cgroup_rule_invalid_format_missing_major(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_device_cgroup_rule("a 10 rwd") diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_dns.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_dns.py new file mode 100644 index 00000000000..fe6b21e34f3 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_dns.py @@ -0,0 +1,64 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_dns_opts(mock_values): + mock_values["network"] = {"dns_opts": ["attempts:3", "opt1", "opt2"]} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["dns_opt"] == ["attempts:3", "opt1", "opt2"] + + +def test_auto_add_dns_searches(mock_values): + mock_values["network"] = {"dns_searches": ["search1", "search2"]} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["dns_search"] == ["search1", "search2"] + + +def test_auto_add_dns_nameservers(mock_values): + mock_values["network"] = {"dns_nameservers": ["nameserver1", "nameserver2"]} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["dns"] == ["nameserver1", "nameserver2"] + + +def test_add_duplicate_dns_nameservers(mock_values): + mock_values["network"] = {"dns_nameservers": ["nameserver1", "nameserver1"]} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_add_duplicate_dns_searches(mock_values): + mock_values["network"] = {"dns_searches": ["search1", "search1"]} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_add_duplicate_dns_opts(mock_values): + mock_values["network"] = {"dns_opts": ["attempts:3", "attempts:5"]} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_environment.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_environment.py new file mode 100644 index 00000000000..f6a794b3664 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_environment.py @@ -0,0 +1,219 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + mock_values["run_as"] = {"user": "1000", "group": "1000"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs == { + "NVIDIA_VISIBLE_DEVICES": "void", + } + + +def test_remove_auto_env(mock_values): + mock_values["run_as"] = {"user": "1000", "group": "1000"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.remove_auto_env("UID") + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert "UID" not in envs + + +def test_remove_env_not_defined(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.remove_auto_env("NOT_DEFINED") + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_expose.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_expose.py new file mode 100644 index 00000000000..b8724d75484 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_expose.py @@ -0,0 +1,46 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_expose_ports(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.expose.add_port(8081) + c1.expose.add_port(8081, "udp") + c1.expose.add_port(8082, "udp") + output = render.render() + assert output["services"]["test_container"]["expose"] == ["8081/tcp", "8081/udp", "8082/udp"] + + +def test_add_duplicate_expose_ports(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.expose.add_port(8081) + with pytest.raises(Exception): + c1.expose.add_port(8081) + + +def test_add_expose_ports_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.expose.add_port(8081) + output = render.render() + assert "expose" not in output["services"]["test_container"] diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_extra_hosts.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_extra_hosts.py new file mode 100644 index 00000000000..35230be16ee --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_extra_hosts.py @@ -0,0 +1,57 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_extra_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_extra_host("test_host", "127.0.0.1") + c1.add_extra_host("test_host2", "127.0.0.2") + c1.add_extra_host("host.docker.internal", "host-gateway") + output = render.render() + assert output["services"]["test_container"]["extra_hosts"] == { + "host.docker.internal": "host-gateway", + "test_host": "127.0.0.1", + "test_host2": "127.0.0.2", + } + + +def test_add_duplicate_extra_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_extra_host("test_host", "127.0.0.1") + with pytest.raises(Exception): + c1.add_extra_host("test_host", "127.0.0.2") + + +def test_add_extra_host_with_ipv6(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_extra_host("test_host", "::1") + output = render.render() + assert output["services"]["test_container"]["extra_hosts"] == {"test_host": "::1"} + + +def test_add_extra_host_with_invalid_ip(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_extra_host("test_host", "invalid_ip") diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_formatter.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_formatter.py new file mode 100644 index 00000000000..843cf65d2e2 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_formatter.py @@ -0,0 +1,13 @@ +from formatter import escape_dollar + + +def test_escape_dollar(): + cases = [ + {"input": "test", "expected": "test"}, + {"input": "$test", "expected": "$$test"}, + {"input": "$$test", "expected": "$$$$test"}, + {"input": "$$$test", "expected": "$$$$$$test"}, + {"input": "$test$", "expected": "$$test$$"}, + ] + for case in cases: + assert escape_dollar(case["input"]) == case["expected"] diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_functions.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_functions.py new file mode 100644 index 00000000000..c378cad50ad --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_functions.py @@ -0,0 +1,190 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + mock_values["resources"] = { + "gpus": { + "use_all_gpus": True, + "kfd_device_exists": True, + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + }, + } + } + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "auto_cast", "values": ["TrUe"], "expected": True}, + {"func": "auto_cast", "values": ["FaLsE"], "expected": False}, + {"func": "auto_cast", "values": ["0.2"], "expected": 0.2}, + {"func": "auto_cast", "values": [True], "expected": True}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + { + "func": "bcrypt_hash", + "values": ["a" * 72], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + { + "func": "bcrypt_hash", + "values": ["a" * 73], + "expect_raise": True, + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + { + "func": "htpasswd", + "values": ["my_user", "my_pass", 10], + "expect_regex": r"^my_user:\$2b\$10\$[a-zA-Z0-9-_\.\/]+$", + }, + { + "func": "htpasswd", + "values": ["my_user", "a" * 73], + "expect_raise": True, + }, + { + "func": "htpasswd", + "values": ["my_user", "my_pass", 32], + "expect_raise": True, + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + { + "func": "deep_merge", + "values": [{"a": 1, "b": {"b1": 2}}, {"b": {"b2": 3}}], + "expected": {"a": 1, "b": {"b1": 2, "b2": 3}}, + }, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + {"func": "require_unique", "values": [["a=1", "b=2", "c"], "values.key", "="], "expected": None}, + { + "func": "require_unique", + "values": [["a=1", "b=2", "b=3"], "values.key", "="], + "expect_raise": True, + }, + { + "func": "require_no_reserved", + "values": [["a=1", "b=2", "c"], "values.key", ["d"], "="], + "expected": None, + }, + { + "func": "require_no_reserved", + "values": [["a=1", "b=2", "c"], "values.key", ["a"], "="], + "expect_raise": True, + }, + { + "func": "require_no_reserved", + "values": [["a=1", "b=2", "c"], "values.key", ["b"], "=", True], + "expect_raise": True, + }, + { + "func": "url_encode", + "values": ["7V!@@%%63r@a5#e!2X9!68g4b"], + "expected": "7V%21%40%40%25%2563r%40a5%23e%212X9%2168g4b", + }, + { + "func": "url_to_dict", + "values": ["192.168.1.1:8080"], + "expected": { + "host": "192.168.1.1", + "port": 8080, + "scheme": "http", + "netloc": "192.168.1.1:8080", + "path": "", + }, + }, + { + "func": "url_to_dict", + "values": ["[::]:8080"], + "expected": {"host": "::", "port": 8080, "scheme": "http", "netloc": "[::]:8080", "path": ""}, + }, + { + "func": "url_to_dict", + "values": ["[::]:8080/abc/", True], + "expected": { + "host": "[::]", + "port": 8080, + "host_no_brackets": "::", + "scheme": "http", + "netloc": "[::]:8080", + "path": "/abc/", + }, + }, + { + "func": "to_yaml", + "values": [{"a": 1, "b": 2}], + "expected": "a: 1\nb: 2\n", + }, + {"func": "has_amd_gpu", "values": [], "expected": True}, + {"func": "has_nvidia_gpu", "values": [], "expected": True}, + ] + + for test in tests: + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_healthcheck.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_healthcheck.py new file mode 100644 index 00000000000..742efb1f893 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_healthcheck.py @@ -0,0 +1,419 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "30s", + "timeout": "5s", + "retries": 5, + "start_period": "15s", + "start_interval": "2s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "1"], + "interval": "30s", + "timeout": "5s", + "retries": 5, + "start_period": "15s", + "start_interval": "2s", + } + + +def test_CMD_with_var_should_fail(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "123$567"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + c1.healthcheck.set_start_interval(5) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "123$$567"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + "start_interval": "5s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD-SHELL", + f"""/bin/bash -c '{{ printf "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&0; grep "HTTP" | grep -q "200"; }} 0<>/dev/tcp/127.0.0.1/8080'""", # noqa + ] + + +def test_curl_healthcheck_as_CMD(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "data": {"test": "val"}, "exec_type": "CMD"}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD", + "curl", + "--request", + "GET", + "--silent", + "--output", + "/dev/null", + "--show-error", + "--fail", + "--data", + '{"test": "val"}', + "http://127.0.0.1:8080/health", + ] + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "data": {"test": "val"}}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD", + "curl", + "--request", + "GET", + "--silent", + "--output", + "/dev/null", + "--show-error", + "--fail", + "--data", + '{"test": "val"}', + "http://127.0.0.1:8080/health", + ] + + +def test_curl_healthcheck_with_headers_and_method_and_data(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test( + "curl", {"port": 8080, "path": "/health", "method": "POST", "headers": [("X-Test", "some-value")], "data": {}} + ) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD", + "curl", + "--request", + "POST", + "--silent", + "--output", + "/dev/null", + "--show-error", + "--fail", + "--header", + "X-Test: some-value", + "--data", + "{}", + "http://127.0.0.1:8080/health", + ] + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD", + "wget", + "--quiet", + "--spider", + "http://127.0.0.1:8080/health", + ] + + +def test_wget_healthcheck_no_spider(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health", "spider": False}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD", + "wget", + "--quiet", + "-O", + "/dev/null", + "http://127.0.0.1:8080/health", + ] + + +def test_wget_healthcheck_data(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test( + "wget", {"port": 8080, "path": "/health", "spider": False, "data": {"test": "val"}, "method": "POST"} + ) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD", + "wget", + "--quiet", + "--method", + "POST", + "-O", + "/dev/null", + "--body-data", + '{"test": "val"}', + "http://127.0.0.1:8080/health", + ] + + +def test_wget_healthcheck_data_busybox(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test( + "wget", {"port": 8080, "path": "/health", "spider": False, "data": {"test": "val"}, "busybox": True} + ) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD", + "wget", + "--quiet", + "-O", + "/dev/null", + "--post-data", + '{"test": "val"}', + "http://127.0.0.1:8080/health", + ] + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD", + "nc", + "-z", + "-w", + "1", + "127.0.0.1", + "8080", + ] + + +def test_netcat_udp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080, "udp": True}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD", + "nc", + "-z", + "-w", + "1", + "-u", + "127.0.0.1", + "8080", + ] + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD", + "timeout", + "1", + "bash", + "-c", + "cat < /dev/null > /dev/tcp/127.0.0.1/8080", + ] + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis", {"password": "test"}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD", + "redis-cli", + "-h", + "127.0.0.1", + "-p", + "6379", + "-a", + "test", + "ping", + ] + + +def test_redis_healthcheck_no_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis", {"password": ""}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD", + "redis-cli", + "-h", + "127.0.0.1", + "-p", + "6379", + "ping", + ] + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres", {"user": "test-user", "db": "test-db"}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD", + "pg_isready", + "-h", + "127.0.0.1", + "-p", + "5432", + "-U", + "test-user", + "-d", + "test-db", + ] + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb", {"password": "test-pass"}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD", + "mariadb-admin", + "--user=root", + "--host=127.0.0.1", + "--port=3306", + "--password=test-pass", + "ping", + ] + + +def test_mongodb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mongodb", {"db": "test-db"}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD", + "mongosh", + "--host", + "127.0.0.1", + "--port", + "27017", + "test-db", + "--eval", + 'db.adminCommand("ping")', + "--quiet", + ] + + +def test_pidof(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("pidof", {"process": "some-process"}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD", + "pidof", + "-s", + "some-process", + ] + + +def test_pgrep(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("pgrep", {"process": "some-process"}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == [ + "CMD", + "pgrep", + "-f", + "some-process", + ] diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_labels.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_labels.py new file mode 100644 index 00000000000..ffa21eceac5 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_labels.py @@ -0,0 +1,88 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_disallowed_label(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.labels.add_label("com.docker.compose.service", "test_service") + + +def test_add_duplicate_label(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.labels.add_label("my.custom.label", "test_value") + with pytest.raises(Exception): + c1.labels.add_label("my.custom.label", "test_value1") + + +def test_add_label_on_non_existing_container(mock_values): + mock_values["labels"] = [ + { + "key": "my.custom.label1", + "value": "test_value1", + "containers": ["test_container", "test_container2"], + }, + ] + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.render() + + +def test_add_label(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.labels.add_label("my.custom.label1", "test_value1") + c1.labels.add_label("my.custom.label2", "test_value2") + output = render.render() + assert output["services"]["test_container"]["labels"] == { + "my.custom.label1": "test_value1", + "my.custom.label2": "test_value2", + } + + +def test_auto_add_labels(mock_values): + mock_values["labels"] = [ + { + "key": "my.custom.label1", + "value": "test_value1", + "containers": ["test_container", "test_container2"], + }, + { + "key": "my.custom.label2", + "value": "test_value2", + "containers": ["test_container"], + }, + ] + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c2 = render.add_container("test_container2", "test_image") + c1.healthcheck.disable() + c2.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["labels"] == { + "my.custom.label1": "test_value1", + "my.custom.label2": "test_value2", + } + assert output["services"]["test_container2"]["labels"] == { + "my.custom.label1": "test_value1", + } diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_networks.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_networks.py new file mode 100644 index 00000000000..c90a63f64d8 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_networks.py @@ -0,0 +1,329 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_internal_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c2 = render.add_container("test_container2", "test_image") + c2.healthcheck.disable() + net1 = render.networks.create_internal("test_network1") + net2 = render.networks.create_internal("test_network2") + c1.add_network(net1) + c1.add_network(net2) + c2.add_network(net1) + output = render.render() + assert output["networks"] == { + "ix-internal-test_network1": { + "external": False, + # "enable_ipv4": True, + "enable_ipv6": False, + "labels": {"tn.network.internal": "true"}, + }, + "ix-internal-test_network2": { + "external": False, + # "enable_ipv4": True, + "enable_ipv6": False, + "labels": {"tn.network.internal": "true"}, + }, + } + assert output["services"]["test_container"]["networks"] == { + "ix-internal-test_network1": {}, + "ix-internal-test_network2": {}, + } + assert output["services"]["test_container2"]["networks"] == { + "ix-internal-test_network1": {}, + } + + +def test_add_external_network(mock_values): + render = Render(mock_values) + # Mock the network names (No Docker client in tests) + render.docker._network_names = set(["test_network1", "test_network2"]) + + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c2 = render.add_container("test_container2", "test_image") + c2.healthcheck.disable() + render.networks.register("test_network1") + render.networks.register("test_network2") + c1.add_network("test_network1") + c1.add_network("test_network2") + c2.add_network("test_network1") + output = render.render() + assert output["networks"] == { + "test_network1": {"external": True}, + "test_network2": {"external": True}, + } + assert output["services"]["test_container"]["networks"] == { + "test_network1": {}, + "test_network2": {}, + } + assert output["services"]["test_container2"]["networks"] == { + "test_network1": {}, + } + + +def test_add_both_internal_and_external_network(mock_values): + render = Render(mock_values) + # Mock the network names (No Docker client in tests) + render.docker._network_names = set(["test_network2"]) + + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c2 = render.add_container("test_container2", "test_image") + c2.healthcheck.disable() + net1 = render.networks.create_internal("test_network1") + render.networks.register("test_network2") + c1.add_network(net1) + c1.add_network("test_network2") + c2.add_network(net1) + output = render.render() + assert output["networks"] == { + "ix-internal-test_network1": { + "external": False, + # "enable_ipv4": True, + "enable_ipv6": False, + "labels": {"tn.network.internal": "true"}, + }, + "test_network2": {"external": True}, + } + assert output["services"]["test_container"]["networks"] == { + "ix-internal-test_network1": {}, + "test_network2": {}, + } + assert output["services"]["test_container2"]["networks"] == { + "ix-internal-test_network1": {}, + } + + +def test_add_network_with_config(mock_values): + render = Render(mock_values) + # Mock the network names (No Docker client in tests) + render.docker._network_names = set(["test_network1"]) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + net = render.networks.create_internal("test_network1") + c1.add_network( + net, + { + "interface_name": "eth0", + "ipv4_address": "192.168.1.10", + "mac_address": "00:11:22:33:44:55", + "gw_priority": 1, + "priority": 2, + }, + ) + + render.networks.register("test_network1") + c1.add_network( + "test_network1", + { + "interface_name": "eth1", + "ipv4_address": "192.168.1.11", + "mac_address": "00:11:22:33:44:56", + "gw_priority": 3, + "priority": 4, + }, + ) + + output = render.render() + assert output["services"]["test_container"]["networks"] == { + "ix-internal-test_network1": { + "interface_name": "eth0", + "ipv4_address": "192.168.1.10", + "mac_address": "00:11:22:33:44:55", + "gw_priority": 1, + "priority": 2, + }, + "test_network1": { + "interface_name": "eth1", + "ipv4_address": "192.168.1.11", + "mac_address": "00:11:22:33:44:56", + "gw_priority": 3, + "priority": 4, + }, + } + + +def test_auto_add_networks(mock_values): + render = Render(mock_values) + + render.docker._network_names = set(["test_network1", "test_network2"]) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c2 = render.add_container("test_container2", "test_image") + c2.healthcheck.disable() + + net = render.networks.create_internal("test_network1") + c1.add_network(net) + c2.add_network(net) + + # Hack around the fact that docker client is not available in tests + new_net = render.values.get("network", {}) + new_net["networks"] = [ + { + "name": "test_network1", + "containers": [ + {"name": "test_container", "config": {"interface_name": "eth0"}}, + {"name": "test_container2", "config": {"interface_name": "eth0"}}, + ], + }, + { + "name": "test_network2", + "containers": [ + {"name": "test_container", "config": {"interface_name": "eth1", "aliases": ["alias1"]}}, + ], + }, + ] + render.values["network"] = new_net + render._original_values["network"] = new_net + + render._auto_add_networks() + for container in render._containers.values(): + container._auto_add_networks() + # End hack + + output = render.render() + assert output["networks"] == { + "ix-internal-test_network1": { + "external": False, + # "enable_ipv4": True, + "enable_ipv6": False, + "labels": {"tn.network.internal": "true"}, + }, + "test_network2": {"external": True}, + "test_network1": {"external": True}, + } + assert output["services"]["test_container"]["networks"] == { + "ix-internal-test_network1": {}, + "test_network2": { + "interface_name": "eth1", + "aliases": ["alias1"], + }, + "test_network1": { + "interface_name": "eth0", + }, + } + assert output["services"]["test_container2"]["networks"] == { + "ix-internal-test_network1": {}, + "test_network1": { + "interface_name": "eth0", + }, + } + + +def test_add_network_with_duplicate_interface_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + net = render.networks.create_internal("test_network1") + c1.add_network(net, {"interface_name": "eth0"}) + with pytest.raises(Exception): + c1.add_network(net, {"interface_name": "eth0"}) + + +def test_add_network_with_duplicate_mac_address(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + net = render.networks.create_internal("test_network1") + c1.add_network(net, {"mac_address": "00:11:22:33:44:55"}) + with pytest.raises(Exception): + c1.add_network(net, {"mac_address": "00:11:22:33:44:55"}) + + +def test_add_network_with_duplicate_ipv4_address(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + net = render.networks.create_internal("test_network1") + c1.add_network(net, {"ipv4_address": "192.168.1.10"}) + with pytest.raises(Exception): + c1.add_network(net, {"ipv4_address": "192.168.1.10"}) + + +def test_add_network_with_duplicate_ipv6_address(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + net = render.networks.create_internal("test_network1") + c1.add_network(net, {"ipv6_address": "2001:db8:85a3:8d3:1319:8a2e:370:7348"}) + with pytest.raises(Exception): + c1.add_network(net, {"ipv6_address": "2001:db8:85a3:8d3:1319:8a2e:370:7348"}) + + +def test_add_network_with_duplicate_gateway_priority(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + net = render.networks.create_internal("test_network1") + c1.add_network(net, {"gw_priority": 1}) + with pytest.raises(Exception): + c1.add_network(net, {"gw_priority": 1}) + + +def test_add_network_with_duplicate_priority(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + net = render.networks.create_internal("test_network1") + c1.add_network(net, {"priority": 1}) + with pytest.raises(Exception): + c1.add_network(net, {"priority": 1}) + + +def test_add_duplicate_internal_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + render.networks.create_internal("test_network1") + with pytest.raises(Exception): + render.networks.create_internal("test_network1") + + +def test_add_duplicate_external_network(mock_values): + render = Render(mock_values) + # Mock the network names (No Docker client in tests) + render.docker._network_names = set(["test_network1"]) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + render.networks.register("test_network1") + with pytest.raises(Exception): + render.networks.register("test_network1") + + +def test_add_duplicate_internal_external_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + render.networks.create_internal("test_network1") + with pytest.raises(Exception): + render.networks.register("ix-internal-test_network1") + + +def test_add_network_with_duplicate_aliases(mock_values): + render = Render(mock_values) + # Mock the network names (No Docker client in tests) + render.docker._network_names = set(["test_network1", "test_network2"]) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + render.networks.register("test_network1") + render.networks.register("test_network2") + c1.add_network("test_network1", {"aliases": ["alias1", "alias2"]}) + with pytest.raises(Exception): + c1.add_network("test_network2", {"aliases": ["alias2", "alias3"]}) diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_notes.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_notes.py new file mode 100644 index 00000000000..b8e1fbb0947 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_notes.py @@ -0,0 +1,381 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "ix_context": { + "app_metadata": { + "name": "test_app", + "title": "Test App", + "train": "enterprise", + } + }, + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_notes(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Test App + +## Security + +**Read the following security precautions to ensure that you wish to continue using this application.** + +--- + +### Container: [test_container] + +#### Running user/group(s) + +- User: unknown +- Group: unknown +- Supplementary Groups: apps + +--- + +## Bug Reports and Feature Requests + +If you find a bug in this app or have an idea for a new feature, please file an issue at +https://ixsystems.atlassian.net +""" + ) + + +def test_notes_on_non_enterprise_train(mock_values): + mock_values["ix_context"]["app_metadata"]["train"] = "community" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(568, 568) + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Test App + +## Security + +**Read the following security precautions to ensure that you wish to continue using this application.** + +--- + +### Container: [test_container] + +#### Running user/group(s) + +- User: 568 +- Group: 568 +- Supplementary Groups: apps + +--- + +## Bug Reports and Feature Requests + +If you find a bug in this app or have an idea for a new feature, please file an issue at +https://github.com/truenas/apps +""" + ) + + +def test_notes_with_warnings(mock_values): + render = Render(mock_values) + render.notes.add_warning("this is not properly configured. fix it now!") + render.notes.add_warning("that is not properly configured. fix it later!") + c1 = render.add_container("test_container", "test_image") + c1.set_user(568, 568) + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Test App + +## Warnings + +- this is not properly configured. fix it now! +- that is not properly configured. fix it later! + +## Security + +**Read the following security precautions to ensure that you wish to continue using this application.** + +--- + +### Container: [test_container] + +#### Running user/group(s) + +- User: 568 +- Group: 568 +- Supplementary Groups: apps + +--- + +## Bug Reports and Feature Requests + +If you find a bug in this app or have an idea for a new feature, please file an issue at +https://ixsystems.atlassian.net +""" + ) + + +def test_notes_with_deprecations(mock_values): + render = Render(mock_values) + render.notes.add_deprecation("this is will be removed later. fix it now!") + render.notes.add_deprecation("that is will be removed later. fix it later!") + c1 = render.add_container("test_container", "test_image") + c1.set_user(568, 568) + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Test App + +## Deprecations + +- this is will be removed later. fix it now! +- that is will be removed later. fix it later! + +## Security + +**Read the following security precautions to ensure that you wish to continue using this application.** + +--- + +### Container: [test_container] + +#### Running user/group(s) + +- User: 568 +- Group: 568 +- Supplementary Groups: apps + +--- + +## Bug Reports and Feature Requests + +If you find a bug in this app or have an idea for a new feature, please file an issue at +https://ixsystems.atlassian.net +""" + ) + + +def test_notes_with_body(mock_values): + render = Render(mock_values) + render.notes.set_body( + """## Additional info + +Some info +some other info. +""" + ) + c1 = render.add_container("test_container", "test_image") + c1.set_user(568, 568) + c1.healthcheck.disable() + output = render.render() + assert ( + output["x-notes"] + == """# Test App + +## Security + +**Read the following security precautions to ensure that you wish to continue using this application.** + +--- + +### Container: [test_container] + +#### Running user/group(s) + +- User: 568 +- Group: 568 +- Supplementary Groups: apps + +--- + +## Additional info + +Some info +some other info. + +## Bug Reports and Feature Requests + +If you find a bug in this app or have an idea for a new feature, please file an issue at +https://ixsystems.atlassian.net +""" + ) + + +def test_notes_all(mock_values): + render = Render(mock_values) + render.notes.add_warning("this is not properly configured. fix it now!") + render.notes.add_warning("that is not properly configured. fix it later!") + render.notes.add_deprecation("this is will be removed later. fix it now!") + render.notes.add_deprecation("that is will be removed later. fix it later!") + render.notes.add_info("some info") + render.notes.add_info("some other info") + render.notes.set_body( + """## Additional info + +Some info +some other info. +""" + ) + net1 = render.networks.create_internal("test_network1") + net2 = render.networks.create_internal("test_network2") + + c1 = render.add_container("test_container", "test_image") + c1.add_network(net1) + c1.add_network(net2) + c1.healthcheck.disable() + c1.set_privileged(True) + c1.set_user(0, 0) + c1.add_group(0) + c1.set_ipc_mode("host") + c1.set_pid_mode("host") + c1.set_cgroup("host") + c1.set_tty(True) + c1.set_grace_period(61) + c1.remove_security_opt("no-new-privileges") + c1.add_docker_socket() + c1.add_tun_device() + c1.add_usb_bus() + c1.add_snd_device() + c1.devices.add_device("/dev/null", "/dev/null", "rwm") + c1.add_storage("/etc/os-release", {"type": "host_path", "host_path_config": {"path": "/etc/os-release"}}) + c1.restart.set_policy("on-failure", 1) + + c2 = render.add_container("test_container2", "test_image") + c2.healthcheck.disable() + c2.set_user(568, 568) + + c3 = render.add_container("test_container3", "test_image") + c3.healthcheck.disable() + c3.restart.set_policy("on-failure", 1) + c3.set_user(568, 568) + + output = render.render() + assert ( + output["x-notes"] + == """# Test App + +## Warnings + +- Container [test_container] is running with a TTY, Logs do not appear correctly in the UI due to an [upstream bug](https://github.com/docker/docker-py/issues/1394) +- this is not properly configured. fix it now! +- that is not properly configured. fix it later! +- Container [test_container] has a grace period of [61] seconds. TrueNAS waits a maximum of 60 seconds for docker engine to stop during system reboot/shutdown. If the container needs the full configured grace period, manually stop it before reboot/shutdown to ensure the full wait time is honored. + +## Deprecations + +- this is will be removed later. fix it now! +- that is will be removed later. fix it later! + +## Info + +- some info +- some other info + +## Security + +**Read the following security precautions to ensure that you wish to continue using this application.** + +--- + +### Container: [test_container2] + +#### Running user/group(s) + +- User: 568 +- Group: 568 +- Supplementary Groups: apps + +--- + +### Container: [test_container] + +**This container is short-lived.** + +#### Privileged mode is enabled + +- Has the same level of control as a system administrator +- Can access and modify any part of your TrueNAS system + +#### Joined networks + +- ix-internal-test_network1 +- ix-internal-test_network2 + +#### Running user/group(s) + +- User: root +- Group: root +- Supplementary Groups: apps, audio, docker, root + +#### Host IPC namespace is enabled + +- Container can access the inter-process communication mechanisms of the host +- Allows communication with other processes on the host under particular circumstances + +#### Host PID namespace is enabled + +- Container can see and interact with all host processes +- Potential for privilege escalation or process manipulation + +#### Host cgroup namespace is enabled + +- Container shares control groups with the host system +- Can bypass resource limits and isolation boundaries + +#### Security option [no-new-privileges] is not set + +- Processes can gain additional privileges through setuid/setgid binaries +- Can potentially allow privilege escalation attacks within the container + +#### Passing Host Files, Devices, or Sockets into the Container + +- /dev/null - (rwm) +- Docker Socket (/var/run/docker.sock) - (Read Only) +- OS Release File (/etc/os-release) - (Read/Write) +- Sound Device (/dev/snd) - (Read/Write) +- TUN Device (/dev/net/tun) - (Read/Write) +- USB Devices (/dev/bus/usb) - (Read/Write) + +--- + +### Container: [test_container3] + +**This container is short-lived.** + +#### Running user/group(s) + +- User: 568 +- Group: 568 +- Supplementary Groups: apps + +--- + +## Additional info + +Some info +some other info. + +## Bug Reports and Feature Requests + +If you find a bug in this app or have an idea for a new feature, please file an issue at +https://ixsystems.atlassian.net +""" # noqa + ) diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_portal.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_portal.py new file mode 100644 index 00000000000..27cd2051fa6 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_portal.py @@ -0,0 +1,93 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_no_portals(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["x-portals"] == [] + + +def test_add_portal_with_host_ips(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + port1 = {"bind_mode": "published", "port_number": 8080, "host_ips": ["1.2.3.4", "5.6.7.8"]} + port2 = {"bind_mode": "published", "port_number": 8081, "host_ips": ["::", "0.0.0.0"]} + port3 = {"bind_mode": "published", "port_number": 8081, "host_ips": ["1.2.3.4"]} + port3 = {"bind_mode": "published", "port_number": 8081, "host_ips": ["1.2.3.4"]} + port4 = {"bind_mode": "exposed", "port_number": 1234, "host_ips": ["1.2.3.4"]} + render.portals.add(port1) + render.portals.add(port1, {"name": "test1", "host": "my-host.com"}) + render.portals.add(port2, {"name": "test2"}) + render.portals.add(port3, {"name": "test3", "port": None}) + render.portals.add(port3, {"name": "test4", "port": 1234}) + render.portals.add(port4, {"name": "test5", "port": 1234}) + output = render.render() + assert output["x-portals"] == [ + {"name": "Web UI", "scheme": "http", "host": "1.2.3.4", "port": 8080, "path": "/"}, + {"name": "test1", "scheme": "http", "host": "my-host.com", "port": 8080, "path": "/"}, + {"name": "test2", "scheme": "http", "host": "0.0.0.0", "port": 8081, "path": "/"}, + {"name": "test3", "scheme": "http", "host": "1.2.3.4", "port": 8081, "path": "/"}, + {"name": "test4", "scheme": "http", "host": "1.2.3.4", "port": 1234, "path": "/"}, + ] + + +def test_add_duplicate_portal(mock_values): + render = Render(mock_values) + port = {"bind_mode": "published", "port_number": 8080, "host_ips": ["1.2.3.4", "5.6.7.8"]} + render.portals.add(port) + with pytest.raises(Exception): + render.portals.add(port) + + +def test_add_duplicate_portal_with_explicit_name(mock_values): + render = Render(mock_values) + port = {"bind_mode": "published", "port_number": 8080, "host_ips": ["1.2.3.4", "5.6.7.8"]} + render.portals.add(port, {"name": "Some Portal"}) + with pytest.raises(Exception): + render.portals.add(port, {"name": "Some Portal"}) + + +def test_add_portal_with_invalid_scheme(mock_values): + render = Render(mock_values) + port = {"bind_mode": "published", "port_number": 8080, "host_ips": ["1.2.3.4", "5.6.7.8"]} + with pytest.raises(Exception): + render.portals.add(port, {"scheme": "invalid_scheme"}) + + +def test_add_portal_with_invalid_path(mock_values): + render = Render(mock_values) + port = {"bind_mode": "published", "port_number": 8080, "host_ips": ["1.2.3.4", "5.6.7.8"]} + with pytest.raises(Exception): + render.portals.add(port, {"path": "invalid_path"}) + + +def test_add_portal_with_invalid_path_double_slash(mock_values): + render = Render(mock_values) + port = {"bind_mode": "published", "port_number": 8080, "host_ips": ["1.2.3.4", "5.6.7.8"]} + with pytest.raises(Exception): + render.portals.add(port, {"path": "/some//path"}) + + +def test_add_portal_with_invalid_port(mock_values): + render = Render(mock_values) + port = {"bind_mode": "published", "port_number": 8080, "host_ips": ["1.2.3.4", "5.6.7.8"]} + with pytest.raises(Exception): + render.portals.add(port, {"port": -1}) diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_ports.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_ports.py new file mode 100644 index 00000000000..c073d6b2ce8 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_ports.py @@ -0,0 +1,383 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +tests = [ + { + "name": "add_ports_should_work", + "inputs": [ + { + "values": ({"bind_mode": "published", "port_number": 8081}, {"container_port": 8080}), + "expect_error": False, + }, + { + "values": ( + {"bind_mode": "published", "port_number": 8082}, + {"container_port": 8080, "protocol": "udp"}, + ), + "expect_error": False, + }, + ], + "expected": [ + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress"}, + {"published": 8082, "target": 8080, "protocol": "udp", "mode": "ingress"}, + ], + }, + { + "name": "add_duplicate_ports_should_fail", + "inputs": [ + { + "values": ({"bind_mode": "published", "port_number": 8081}, {"container_port": 8080}), + "expect_error": False, + }, + { + "values": ({"bind_mode": "published", "port_number": 8081}, {"container_port": 8080}), + "expect_error": True, + }, + ], + }, + { + "name": "adding_duplicate_port_different_protocol_should_work", + "inputs": [ + { + "values": ({"bind_mode": "published", "port_number": 8081}, {"container_port": 8080}), + "expect_error": False, + }, + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "protocol": "udp"}, + ), + "expect_error": False, + }, + ], + "expected": [ + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress"}, + {"published": 8081, "target": 8080, "protocol": "udp", "mode": "ingress"}, + ], + }, + { + "name": "adding_same_port_for_both_wildcard_families_should_work", + "inputs": [ + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["0.0.0.0"]}, + ), + "expect_error": False, + }, + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["::"]}, + ), + "expect_error": False, + }, + ], + "expected": [ + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress"}, + ], + }, + { + "name": "adding_duplicate_port_for_v4_ip_and_v4_wildcard_should_fail", + "inputs": [ + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["192.168.1.10"]}, + ), + "expect_error": False, + }, + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["0.0.0.0"]}, + ), + "expect_error": True, + }, + ], + }, + { + "name": "adding_duplicate_port_for_v4_wildcard_and_v4_ip_should_fail", + "inputs": [ + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["0.0.0.0"]}, + ), + "expect_error": False, + }, + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["192.168.1.10"]}, + ), + "expect_error": True, + }, + ], + }, + { + "name": "adding_duplicate_port_for_v4_wildcard_and_v6_ip_should_work", + "inputs": [ + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["192.168.1.10"]}, + ), + "expect_error": False, + }, + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["fd00:1234:5678:abcd::10"]}, + ), + "expect_error": False, + }, + ], + "expected": [ + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.10"}, + { + "published": 8081, + "target": 8080, + "protocol": "tcp", + "mode": "ingress", + "host_ip": "fd00:1234:5678:abcd::10", + }, + ], + }, + { + "name": "adding_duplicate_port_for_v6_wildcard_and_v4_ip_should_work", + "inputs": [ + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["::"]}, + ), + "expect_error": False, + }, + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["192.168.1.10"]}, + ), + "expect_error": False, + }, + ], + "expected": [ + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.10"}, + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "::"}, + ], + }, + { + "name": "adding_duplicate_port_for_v6_wildcard_and_v6_ip_should_fail", + "inputs": [ + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["::"]}, + ), + "expect_error": False, + }, + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["fd00:1234:5678:abcd::10"]}, + ), + "expect_error": True, + }, + ], + }, + { + "name": "adding_duplicate_port_for_v6_ip_and_v6_wildcard_should_fail", + "inputs": [ + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["fd00:1234:5678:abcd::10"]}, + ), + "expect_error": False, + }, + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["::"]}, + ), + "expect_error": True, + }, + ], + }, + { + "name": "adding_duplicate_port_with_different_v4_ip_should_work", + "inputs": [ + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["192.168.1.10"]}, + ), + "expect_error": False, + }, + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["192.168.1.11"]}, + ), + "expect_error": False, + }, + ], + "expected": [ + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.10"}, + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.11"}, + ], + }, + { + "name": "adding_port_with_invalid_protocol_should_fail", + "inputs": [ + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "protocol": "invalid_protocol"}, + ), + "expect_error": True, + }, + ], + }, + { + "name": "adding_port_with_invalid_mode_should_fail", + "inputs": [ + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "mode": "invalid_mode"}, + ), + "expect_error": True, + }, + ], + }, + { + "name": "adding_port_with_invalid_ip_should_fail", + "inputs": [ + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["invalid_ip"]}, + ), + "expect_error": True, + }, + ], + }, + { + "name": "adding_port_with_invalid_port_number_should_fail", + "inputs": [ + {"values": ({"bind_mode": "published", "port_number": -1}, {"container_port": 8080}), "expect_error": True}, + ], + }, + { + "name": "adding_port_with_invalid_container_port_should_fail", + "inputs": [ + {"values": ({"bind_mode": "published", "port_number": 8081}, {"container_port": -1}), "expect_error": True}, + ], + }, + { + "name": "adding_duplicate_ports_with_different_host_ip_should_work", + "inputs": [ + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["192.168.1.10"]}, + ), + "expect_error": False, + }, + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["192.168.1.10"], "protocol": "udp"}, + ), + "expect_error": False, + }, + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["192.168.1.11"]}, + ), + "expect_error": False, + }, + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["192.168.1.11"], "protocol": "udp"}, + ), + "expect_error": False, + }, + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["fd00:1234:5678:abcd::10"]}, + ), + "expect_error": False, + }, + { + "values": ( + {"bind_mode": "published", "port_number": 8081}, + {"container_port": 8080, "host_ips": ["fd00:1234:5678:abcd::11"]}, + ), + "expect_error": False, + }, + ], + # fmt: off + "expected": [ + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.10"}, + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "192.168.1.11"}, + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "fd00:1234:5678:abcd::10"}, # noqa + {"published": 8081, "target": 8080, "protocol": "tcp", "mode": "ingress", "host_ip": "fd00:1234:5678:abcd::11"}, # noqa + {"published": 8081, "target": 8080, "protocol": "udp", "mode": "ingress", "host_ip": "192.168.1.10"}, + {"published": 8081, "target": 8080, "protocol": "udp", "mode": "ingress", "host_ip": "192.168.1.11"}, + ], + # fmt: on + }, +] + + +@pytest.mark.parametrize("test", tests) +def test_ports(test): + mock_values = { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + errored = False + for input in test["inputs"]: + if input["expect_error"]: + with pytest.raises(Exception): + c1.add_port(*input["values"]) + errored = True + else: + c1.add_port(*input["values"]) + + errored = True if [i["expect_error"] for i in test["inputs"]].count(True) > 0 else False + if errored: + return + + output = render.render() + assert output["services"]["test_container"]["ports"] == test["expected"] diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_render.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_render.py new file mode 100644 index 00000000000..60dc00679e6 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_render.py @@ -0,0 +1,37 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_values_cannot_be_modified(mock_values): + render = Render(mock_values) + render.values["test"] = "test" + with pytest.raises(Exception): + render.render() + + +def test_duplicate_containers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_no_containers(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_resources.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_resources.py new file mode 100644 index 00000000000..cd83d164e5e --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_resources.py @@ -0,0 +1,140 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_automatically_add_cpu(mock_values): + mock_values["resources"] = {"limits": {"cpus": 1.0}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "1.0" + + +def test_invalid_cpu(mock_values): + mock_values["resources"] = {"limits": {"cpus": "invalid"}} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_automatically_add_memory(mock_values): + mock_values["resources"] = {"limits": {"memory": 1024}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "1024M" + + +def test_invalid_memory(mock_values): + mock_values["resources"] = {"limits": {"memory": "invalid"}} + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_automatically_add_gpus(mock_values): + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + devices = output["services"]["test_container"]["deploy"]["resources"]["reservations"]["devices"] + assert len(devices) == 1 + assert devices[0] == { + "capabilities": ["gpu"], + "driver": "nvidia", + "device_ids": ["uuid_0", "uuid_1"], + } + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_gpu_without_uuid(mock_values): + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_remove_cpus_and_memory_with_gpus(mock_values): + mock_values["resources"] = {"gpus": {"nvidia_gpu_selection": {"pci_slot_0": {"uuid": "uuid_1", "use_gpu": True}}}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.remove_cpus_and_memory() + output = render.render() + assert "limits" not in output["services"]["test_container"]["deploy"]["resources"] + devices = output["services"]["test_container"]["deploy"]["resources"]["reservations"]["devices"] + assert len(devices) == 1 + assert devices[0] == { + "capabilities": ["gpu"], + "driver": "nvidia", + "device_ids": ["uuid_1"], + } + + +def test_remove_cpus_and_memory(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.remove_cpus_and_memory() + output = render.render() + assert "deploy" not in output["services"]["test_container"] + + +def test_remove_devices(mock_values): + mock_values["resources"] = {"gpus": {"nvidia_gpu_selection": {"pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}}}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.remove_devices() + output = render.render() + assert "reservations" not in output["services"]["test_container"]["deploy"]["resources"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_set_profile(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.deploy.resources.set_profile("low") + output = render.render() + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["cpus"] == "1" + assert output["services"]["test_container"]["deploy"]["resources"]["limits"]["memory"] == "512M" + + +def test_set_profile_invalid_profile(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.deploy.resources.set_profile("invalid_profile") diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_restart.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_restart.py new file mode 100644 index 00000000000..06b29755904 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_restart.py @@ -0,0 +1,57 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_invalid_restart_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.restart.set_policy("invalid_policy") + + +def test_valid_restart_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.restart.set_policy("on-failure") + output = render.render() + assert output["services"]["test_container"]["restart"] == "on-failure" + + +def test_valid_restart_policy_with_maximum_retry_count(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.restart.set_policy("on-failure", 10) + output = render.render() + assert output["services"]["test_container"]["restart"] == "on-failure:10" + + +def test_invalid_restart_policy_with_maximum_retry_count(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.restart.set_policy("on-failure", maximum_retry_count=-1) + + +def test_invalid_restart_policy_with_maximum_retry_count_and_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.restart.set_policy("always", maximum_retry_count=10) diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_security_opts.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_security_opts.py new file mode 100644 index 00000000000..d918ed959b1 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_security_opts.py @@ -0,0 +1,91 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("apparmor", "unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == ["apparmor=unconfined", "no-new-privileges=true"] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges", True) + + +def test_add_empty_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("", True) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt_boolean(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + c1.add_security_opt("no-new-privileges", False) + output = render.render() + assert output["services"]["test_container"]["security_opt"] == ["no-new-privileges=false"] + + +def test_add_security_opt_arg(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("label", "type", "svirt_apache_t") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "label=type:svirt_apache_t", + "no-new-privileges=true", + ] + + +def test_add_security_opt_with_invalid_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("invalid") + + +def test_add_security_opt_with_opt_containing_value(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges=true") + with pytest.raises(Exception): + c1.add_security_opt("apparmor:unconfined") diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_sysctls.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_sysctls.py new file mode 100644 index 00000000000..c9414044eaa --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_validations.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_validations.py new file mode 100644 index 00000000000..f0986ce9a56 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_validations.py @@ -0,0 +1,132 @@ +import pytest +from unittest.mock import patch + +from pathlib import Path +from validations import is_allowed_path, RESTRICTED, RESTRICTED_IN + + +def mock_resolve(self): + # Don't modify paths that are from RESTRICTED list initialization + if str(self) in [str(p) for p in RESTRICTED]: + return self + + # For symlinks that point to restricted paths, return the target path + # without stripping /private/ + if str(self).endswith("symlink_restricted"): + return Path("/home") # Return the actual restricted target + + # For other paths, strip /private/ if present + return Path(str(self).removeprefix("/private/")) + + +@pytest.mark.parametrize( + "test_path, expected", + [ + # Non-restricted path (should be valid) + ("/tmp/somefile", True), + # Exactly /mnt (restricted_in) + ("/mnt", False), + # Exactly / (restricted_in) + ("/", False), + # Subdirectory inside /mnt/.ix-apps (restricted) + ("/mnt/.ix-apps/something", False), + # A path that is a restricted directory exactly + ("/home", False), + ("/var/log", False), + ("/mnt/.ix-apps", False), + ("/data", False), + # Subdirectory inside e.g. /data + ("/data/subdir", False), + # Not an obviously restricted path + ("/usr/local/share", True), + # Another system path likely not in restricted list + ("/opt/myapp", True), + ], +) +@patch.object(Path, "resolve", mock_resolve) +def test_is_allowed_path_direct(test_path, expected): + """Test direct paths against the is_allowed_path function.""" + assert is_allowed_path(test_path) == expected + + +@patch.object(Path, "resolve", mock_resolve) +def test_is_allowed_path_ix_volume(): + """Test that IX volumes are not allowed""" + assert is_allowed_path("/mnt/.ix-apps/something", True) + + +@patch.object(Path, "resolve", mock_resolve) +def test_is_allowed_path_symlink(tmp_path): + """ + Test that a symlink pointing to a restricted directory is detected as invalid, + and a symlink pointing to an allowed directory is valid. + """ + # Create a real (allowed) directory and a restricted directory in a temp location + allowed_dir = tmp_path / "allowed_dir" + allowed_dir.mkdir() + + restricted_dir = tmp_path / "restricted_dir" + restricted_dir.mkdir() + + # We will simulate that "restricted_dir" is actually a symlink link pointing to e.g. "/var/log" + # or we create a subdir to match the restricted pattern. + # For demonstration, let's just patch it to a path in the restricted list. + real_restricted_path = Path("/home") # This is one of the restricted directories + + # Create symlinks to test + symlink_allowed = tmp_path / "symlink_allowed" + symlink_restricted = tmp_path / "symlink_restricted" + + # Point the symlinks + symlink_allowed.symlink_to(allowed_dir) + symlink_restricted.symlink_to(real_restricted_path) + + assert is_allowed_path(str(symlink_allowed)) is True + assert is_allowed_path(str(symlink_restricted)) is False + + +def test_is_allowed_path_nested_symlink(tmp_path): + """ + Test that even a nested symlink that eventually resolves into restricted + directories is seen as invalid. + """ + # e.g., Create 2 symlinks that chain to /root + link1 = tmp_path / "link1" + link2 = tmp_path / "link2" + + # link2 -> /root + link2.symlink_to(Path("/root")) + # link1 -> link2 + link1.symlink_to(link2) + + assert is_allowed_path(str(link1)) is False + + +def test_is_allowed_path_nonexistent(tmp_path): + """ + Test a path that does not exist at all. The code calls .resolve() which will + give the absolute path, but if it's not restricted, it should still be valid. + """ + nonexistent = tmp_path / "this_does_not_exist" + assert is_allowed_path(str(nonexistent)) is True + + +@pytest.mark.parametrize( + "test_path", + list(RESTRICTED), +) +@patch.object(Path, "resolve", mock_resolve) +def test_is_allowed_path_restricted_list(test_path): + """Test that all items in the RESTRICTED list are invalid.""" + assert is_allowed_path(test_path) is False + + +@pytest.mark.parametrize( + "test_path", + list(RESTRICTED_IN), +) +def test_is_allowed_path_restricted_in_list(test_path): + """ + Test that items in RESTRICTED_IN are invalid. + """ + assert is_allowed_path(test_path) is False diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_volumes.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_volumes.py new file mode 100644 index 00000000000..e6311afa365 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tests/test_volumes.py @@ -0,0 +1,746 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + c1.add_storage("/some/other/path", {"type": "tmpfs", "tmpfs_config": {"size": 100}}) + c1.add_storage( + "/some/other/path2", {"type": "tmpfs", "tmpfs_config": {"size": 100, "mode": "0777", "uid": 1000, "gid": 1000}} + ) + output = render.render() + assert output["services"]["test_container"]["tmpfs"] == [ + "/some/other/path2:gid=1000,mode=0777,size=104857600,uid=1000", + "/some/other/path:size=104857600", + "/some/path", + ] + + +def test_add_tmpfs_with_existing_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "volume", "volume_config": {"volume_name": "test_volume"}}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs", "tmpfs_config": {"size": 100}}) + + +def test_add_volume_with_existing_tmpfs(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs", "tmpfs_config": {"size": 100}}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "volume", "volume_config": {"volume_name": "test_volume"}}) + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_udev(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_udev() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/run/udev", + "target": "/run/udev", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_udev_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_udev(read_only=False) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/run/udev", + "target": "/run/udev", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_host_path_with_disallowed_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_host_path_without_disallowed_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/mnt", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/mnt", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/tmpfs.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tmpfs.py new file mode 100644 index 00000000000..fd611cd7b00 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/tmpfs.py @@ -0,0 +1,75 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from container import Container + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise, valid_octal_mode_or_raise +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise, valid_octal_mode_or_raise + + +class Tmpfs: + + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._tmpfs: dict = {} + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if self.is_defined(mount_path): + raise RenderError(f"Tmpfs mount path [{mount_path}] already added") + + if self._container_instance.storage.is_defined(mount_path): + raise RenderError(f"Tmpfs mount path [{mount_path}] already used for another volume mount") + + mount_config = config.get("tmpfs_config", {}) + size = mount_config.get("size", None) + mode = mount_config.get("mode", None) + uid = mount_config.get("uid", None) + gid = mount_config.get("gid", None) + + if size is not None: + if not isinstance(size, int): + raise RenderError(f"Expected [size] to be an integer for [tmpfs] type, got [{size}]") + if not size > 0: + raise RenderError(f"Expected [size] to be greater than 0 for [tmpfs] type, got [{size}]") + # Convert Mebibytes to Bytes + size = size * 1024 * 1024 + + if mode is not None: + mode = valid_octal_mode_or_raise(mode) + + if uid is not None and not isinstance(uid, int): + raise RenderError(f"Expected [uid] to be an integer for [tmpfs] type, got [{uid}]") + + if gid is not None and not isinstance(gid, int): + raise RenderError(f"Expected [gid] to be an integer for [tmpfs] type, got [{gid}]") + + self._tmpfs[mount_path] = {} + if size is not None: + self._tmpfs[mount_path]["size"] = str(size) + if mode is not None: + self._tmpfs[mount_path]["mode"] = str(mode) + if uid is not None: + self._tmpfs[mount_path]["uid"] = str(uid) + if gid is not None: + self._tmpfs[mount_path]["gid"] = str(gid) + + def is_defined(self, mount_path: str): + return mount_path in self._tmpfs + + def has_tmpfs(self): + return bool(self._tmpfs) + + def render(self): + result = [] + for mount_path, config in self._tmpfs.items(): + opts = sorted([f"{k}={v}" for k, v in config.items()]) + result.append(f"{mount_path}:{','.join(opts)}" if opts else mount_path) + return sorted(result) diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/truenas_client.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/truenas_client.py new file mode 100644 index 00000000000..dc157ed7ad8 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/truenas_client.py @@ -0,0 +1,66 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import is_truenas_system +except ImportError: + from error import RenderError + from validations import is_truenas_system + + +# Import based on system detection +if is_truenas_system(): + from truenas_api_client import Client as TrueNASClient + + try: + # 25.04 and later + from truenas_api_client.exc import ValidationErrors + except ImportError: + # 24.10 and earlier + from truenas_api_client import ValidationErrors +else: + # Mock classes for non-TrueNAS systems + class TrueNASClient: + def call(self, *args, **kwargs): + return None + + class ValidationErrors(Exception): + def __init__(self, errors): + self.errors = errors + + +class TNClient: + def __init__(self, render_instance: "Render"): + self.client = TrueNASClient() + self._render_instance = render_instance + self._app_name: str = self._render_instance.values.get("ix_context", {}).get("app_name", "") or "unknown" + + def validate_ip_port_combo(self, ip: str, port: int) -> None: + # Example of an error messages: + # The port is being used by following services: 1) "0.0.0.0:80" used by WebUI Service + # The port is being used by following services: 1) "0.0.0.0:9998" used by Applications ('$app_name' application) + try: + self.client.call("port.validate_port", f"render.{self._app_name}.schema", port, ip, None, True) + except ValidationErrors as e: + err_str = str(e) + # If the IP:port combo appears more than once in the error message, + # means that the port is used by more than one service/app. + # This shouldn't happen in a well-configured system. + # Notice that the ip portion is not included check, + # because input might be a specific IP, but another service or app + # might be using the same port on a wildcard IP + if err_str.count(f':{port}" used by') > 1: + raise RenderError(err_str) from None + + # If the error complains about the current app, we ignore it + # This is to handle cases where the app is being updated or edited + if f"Applications ('{self._app_name}' application)" in err_str: + # During upgrade, we want to ignore the error if it is related to the current app + return + + raise RenderError(err_str) from None + except Exception: + pass diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/validations.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/validations.py new file mode 100644 index 00000000000..be4cf9ba96b --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/validations.py @@ -0,0 +1,366 @@ +import os +import re +import ipaddress +from pathlib import Path + + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +MAC_ADDR_REGEX = re.compile(r"^([0-9A-Fa-f]{2}:){5}([0-9A-Fa-f]{2})$") + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") +RESTRICTED_IN: tuple[Path, ...] = (Path("/mnt"), Path("/")) +RESTRICTED: tuple[Path, ...] = ( + Path("/mnt/.ix-apps"), + Path("/data"), + Path("/var/db"), + Path("/root"), + Path("/conf"), + Path("/audit"), + Path("/var/run/middleware"), + Path("/home"), + Path("/boot"), + Path("/var/log"), +) + + +def is_truenas_system(): + """Check if we're running on a TrueNAS system""" + return "truenas" in os.uname().release + + +def valid_label_key_or_raise(key: str): + if not key: + raise RenderError("Label key cannot be empty") + if key.startswith("com.docker.compose"): + raise RenderError(f"Label [{key}] cannot start with [com.docker.compose] as it is reserved") + return key + + +def valid_mac_or_raise(mac: str): + if MAC_ADDR_REGEX.match(mac): + return mac + raise RenderError(f"Invalid MAC Address [{mac}], valid format is either XX:XX:XX:XX:XX:XX") + + +def valid_security_opt_or_raise(opt: str): + if ":" in opt or "=" in opt: + raise RenderError(f"Security Option [{opt}] cannot contain [:] or [=]. Pass value as an argument") + valid_opts = ["apparmor", "no-new-privileges", "seccomp", "systempaths", "label"] + if opt not in valid_opts: + raise RenderError(f"Security Option [{opt}] is not valid. Valid options are: [{', '.join(valid_opts)}]") + + return opt + + +def valid_port_bind_mode_or_raise(status: str): + valid_statuses = ("published", "exposed", "") + if status not in valid_statuses: + raise RenderError(f"Invalid port status [{status}]") + return status + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_ipc_mode_or_raise(ipc_mode: str, containers: list[str]): + valid_modes = ("", "host", "private", "shareable", "none") + if ipc_mode in valid_modes: + return ipc_mode + if ipc_mode.startswith("container:"): + if ipc_mode[10:] not in containers: + raise RenderError(f"IPC mode [{ipc_mode}] is not valid. Container [{ipc_mode[10:]}] does not exist") + return ipc_mode + raise RenderError(f"IPC mode [{ipc_mode}] is not valid. Valid options are: [{', '.join(valid_modes)}]") + + +def valid_pid_mode_or_raise(ipc_mode: str, containers: list[str]): + valid_modes = ("", "host") + if ipc_mode in valid_modes: + return ipc_mode + if ipc_mode.startswith("container:"): + if ipc_mode[10:] not in containers: + raise RenderError(f"PID mode [{ipc_mode}] is not valid. Container [{ipc_mode[10:]}] does not exist") + return ipc_mode + raise RenderError(f"PID mode [{ipc_mode}] is not valid. Valid options are: [{', '.join(valid_modes)}]") + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'", "#"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def valid_cgroup_or_raise(cgroup: str): + valid_cgroup = ("host", "private") + if cgroup not in valid_cgroup: + raise RenderError(f"Cgroup [{cgroup}] is not valid. Valid options are: [{', '.join(valid_cgroup)}]") + return cgroup + + +def valid_device_cgroup_rule_or_raise(dev_grp_rule: str): + parts = dev_grp_rule.split(" ") + if len(parts) != 3: + raise RenderError( + f"Device Group Rule [{dev_grp_rule}] is not valid. Expected format is [ : ]" + ) + + valid_types = ("a", "b", "c") + if parts[0] not in valid_types: + raise RenderError( + f"Device Group Rule [{dev_grp_rule}] is not valid. Expected type to be one of [{', '.join(valid_types)}]" + f" but got [{parts[0]}]" + ) + + major, minor = parts[1].split(":") + for part in (major, minor): + if part != "*" and not part.isdigit(): + raise RenderError( + f"Device Group Rule [{dev_grp_rule}] is not valid. Expected major and minor to be digits" + f" or [*] but got [{major}] and [{minor}]" + ) + + valid_cgroup_perm_or_raise(parts[2]) + + return dev_grp_rule + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def is_allowed_path(input_path: str, is_ix_volume: bool = False) -> bool: + """ + Validates that the given path (after resolving symlinks) is not + one of the restricted paths or within those restricted directories. + + Returns True if the path is allowed, False otherwise. + """ + # Resolve the path to avoid symlink bypasses + real_path = Path(input_path).resolve() + for restricted in RESTRICTED if not is_ix_volume else [r for r in RESTRICTED if r != Path("/mnt/.ix-apps")]: + if real_path.is_relative_to(restricted): + return False + + return real_path not in RESTRICTED_IN + + +def allowed_fs_host_path_or_raise(path: str, is_ix_volume: bool = False): + if not is_allowed_path(path, is_ix_volume): + raise RenderError(f"Path [{path}] is not allowed to be mounted.") + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd", "/dev/net/tun"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/volume_mount.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/volume_mount.py new file mode 100644 index 00000000000..92cb8eb6e6d --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/volume_mount.py @@ -0,0 +1,87 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .formatter import merge_dicts_no_overwrite + from .volume_mount_types import BindMountType, VolumeMountType + from .volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource +except ImportError: + from error import RenderError + from formatter import merge_dicts_no_overwrite + from volume_mount_types import BindMountType, VolumeMountType + from volume_sources import HostPathSource, IxVolumeSource, CifsSource, NfsSource, VolumeSource + + +class VolumeMount: + def __init__(self, render_instance: "Render", mount_path: str, config: "IxStorage"): + self._render_instance = render_instance + self.mount_path: str = mount_path + + storage_type: str = config.get("type", "") + if not storage_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match storage_type: + case "host_path": + spec_type = "bind" + mount_config = config.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + mount_type_specific_definition = BindMountType(self._render_instance, mount_config).render() + source = HostPathSource(self._render_instance, mount_config).get() + case "ix_volume": + spec_type = "bind" + mount_config = config.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + mount_type_specific_definition = BindMountType(self._render_instance, mount_config).render() + source = IxVolumeSource(self._render_instance, mount_config).get() + case "nfs": + spec_type = "volume" + mount_config = config.get("nfs_config") + if mount_config is None: + raise RenderError("Expected [nfs_config] to be set for [nfs] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = NfsSource(self._render_instance, mount_config).get() + case "cifs": + spec_type = "volume" + mount_config = config.get("cifs_config") + if mount_config is None: + raise RenderError("Expected [cifs_config] to be set for [cifs] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = CifsSource(self._render_instance, mount_config).get() + case "volume": + spec_type = "volume" + mount_config = config.get("volume_config") + if mount_config is None: + raise RenderError("Expected [volume_config] to be set for [volume] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = VolumeSource(self._render_instance, mount_config).get() + case "temporary": + spec_type = "volume" + mount_config = config.get("volume_config") + if mount_config is None: + raise RenderError("Expected [volume_config] to be set for [temporary] type.") + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = VolumeSource(self._render_instance, mount_config).get() + case "anonymous": + spec_type = "volume" + mount_config = config.get("volume_config") or {} + mount_type_specific_definition = VolumeMountType(self._render_instance, mount_config).render() + source = None + case _: + raise RenderError(f"Storage type [{storage_type}] is not supported for volume mounts.") + + common_spec = {"type": spec_type, "target": self.mount_path, "read_only": config.get("read_only", False)} + if source is not None: + common_spec["source"] = source + self._render_instance.volumes.add_volume(source, storage_type, mount_config) # type: ignore + + self.volume_mount_spec = merge_dicts_no_overwrite(common_spec, mount_type_specific_definition) + + def render(self) -> dict: + return self.volume_mount_spec diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/volume_mount_types.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/volume_mount_types.py new file mode 100644 index 00000000000..1bf089789ec --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/volume_mount_types.py @@ -0,0 +1,43 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorageVolumeConfig, IxStorageBindLikeConfigs + + +try: + from .validations import valid_host_path_propagation +except ImportError: + from validations import valid_host_path_propagation + + +class BindMountType: + def __init__(self, render_instance: "Render", config: "IxStorageBindLikeConfigs"): + self._render_instance = render_instance + self.spec: dict = {} + + propagation = valid_host_path_propagation(config.get("propagation", "rprivate")) + create_host_path = config.get("create_host_path", False) + + self.spec: dict = { + "bind": { + "create_host_path": create_host_path, + "propagation": propagation, + } + } + + def render(self) -> dict: + """Render the bind mount specification.""" + return self.spec + + +class VolumeMountType: + def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): + self._render_instance = render_instance + self.spec: dict = {} + + self.spec: dict = {"volume": {"nocopy": config.get("nocopy", False)}} + + def render(self) -> dict: + """Render the volume mount specification.""" + return self.spec diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/volume_sources.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/volume_sources.py new file mode 100644 index 00000000000..dcfce44b753 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/volume_sources.py @@ -0,0 +1,108 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorageHostPathConfig, IxStorageIxVolumeConfig, IxStorageVolumeConfig + +try: + from .error import RenderError + from .formatter import get_hashed_name_for_volume + from .validations import valid_fs_path_or_raise, allowed_fs_host_path_or_raise +except ImportError: + from error import RenderError + from formatter import get_hashed_name_for_volume + from validations import valid_fs_path_or_raise, allowed_fs_host_path_or_raise + + +class HostPathSource: + def __init__(self, render_instance: "Render", config: "IxStorageHostPathConfig"): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + + path = "" + if config.get("acl_enable", False): + acl_path = config.get("acl", {}).get("path") + if not acl_path: + raise RenderError("Expected [host_path_config.acl.path] to be set for [host_path] type.") + path = valid_fs_path_or_raise(acl_path) + else: + path = valid_fs_path_or_raise(config.get("path", "")) + + path = path.rstrip("/") + self.source = allowed_fs_host_path_or_raise(path) + + def get(self): + return self.source + + +class IxVolumeSource: + def __init__(self, render_instance: "Render", config: "IxStorageIxVolumeConfig"): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + dataset_name = config.get("dataset_name") + if not dataset_name: + raise RenderError("Expected [ix_volume_config.dataset_name] to be set for [ix_volume] type.") + + ix_volumes = self._render_instance.values.get("ix_volumes", {}) + if dataset_name not in ix_volumes: + available = ", ".join(ix_volumes.keys()) + raise RenderError( + f"Expected the key [{dataset_name}] to be set in [ix_volumes] for [ix_volume] type. " + f"Available keys: [{available}]." + ) + + path = valid_fs_path_or_raise(ix_volumes[dataset_name].rstrip("/")) + self.source = allowed_fs_host_path_or_raise(path, True) + + def get(self): + return self.source + + +class CifsSource: + def __init__(self, render_instance: "Render", config: dict): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [cifs_config] to be set for [cifs] type.") + self.source = get_hashed_name_for_volume("cifs", config) + + def get(self): + return self.source + + +class NfsSource: + def __init__(self, render_instance: "Render", config: dict): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [nfs_config] to be set for [nfs] type.") + self.source = get_hashed_name_for_volume("nfs", config) + + def get(self): + return self.source + + +class VolumeSource: + def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): + self._render_instance = render_instance + self.source: str = "" + + if not config: + raise RenderError("Expected [volume_config] to be set for [volume] type.") + + volume_name: str = config.get("volume_name", "") + if not volume_name: + raise RenderError("Expected [volume_config.volume_name] to be set for [volume] type.") + + self.source = volume_name + + def get(self): + return self.source diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/volume_types.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/volume_types.py new file mode 100644 index 00000000000..489d14c0169 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/volume_types.py @@ -0,0 +1,130 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorageNfsConfig, IxStorageCifsConfig, IxStorageVolumeConfig + + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_fs_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_fs_path_or_raise + + +class NfsVolume: + def __init__(self, render_instance: "Render", config: "IxStorageNfsConfig"): + self._render_instance = render_instance + + if not config: + raise RenderError("Expected [nfs_config] to be set for [nfs] type") + + required_keys = ["server", "path"] + for key in required_keys: + if not config.get(key): + raise RenderError(f"Expected [{key}] to be set for [nfs] type") + + opts = [f"addr={config['server']}"] + cfg_options = config.get("options") + if cfg_options: + if not isinstance(cfg_options, list): + raise RenderError("Expected [nfs_config.options] to be a list for [nfs] type") + + tracked_keys: set[str] = set() + disallowed_opts = ["addr"] + for opt in cfg_options: + if not isinstance(opt, str): + raise RenderError("Options for [nfs] type must be a list of strings.") + + key = opt.split("=")[0] + if key in tracked_keys: + raise RenderError(f"Option [{key}] already added for [nfs] type.") + if key in disallowed_opts: + raise RenderError(f"Option [{key}] is not allowed for [nfs] type.") + opts.append(opt) + tracked_keys.add(key) + + opts.sort() + + path = valid_fs_path_or_raise(config["path"].rstrip("/")) + self.volume_spec = { + "driver_opts": { + "type": "nfs", + "device": f":{path}", + "o": f"{','.join([escape_dollar(opt) for opt in opts])}", + }, + } + + def get(self): + return self.volume_spec + + +class CifsVolume: + def __init__(self, render_instance: "Render", config: "IxStorageCifsConfig"): + self._render_instance = render_instance + self.volume_spec: dict = {} + + if not config: + raise RenderError("Expected [cifs_config] to be set for [cifs] type") + + required_keys = ["server", "path", "username", "password"] + for key in required_keys: + if not config.get(key): + raise RenderError(f"Expected [{key}] to be set for [cifs] type") + + opts = [ + "noperm", + f"user={config['username']}", + f"password={config['password']}", + ] + + domain = config.get("domain") + if domain: + opts.append(f"domain={domain}") + + cfg_options = config.get("options") + if cfg_options: + if not isinstance(cfg_options, list): + raise RenderError("Expected [cifs_config.options] to be a list for [cifs] type") + + tracked_keys: set[str] = set() + disallowed_opts = ["user", "password", "domain", "noperm"] + for opt in cfg_options: + if not isinstance(opt, str): + raise RenderError("Options for [cifs] type must be a list of strings.") + + key = opt.split("=")[0] + if key in tracked_keys: + raise RenderError(f"Option [{key}] already added for [cifs] type.") + if key in disallowed_opts: + raise RenderError(f"Option [{key}] is not allowed for [cifs] type.") + opts.append(opt) + tracked_keys.add(key) + opts.sort() + + server = config["server"].lstrip("/") + path = config["path"].strip("/") + path = valid_fs_path_or_raise("/" + path).lstrip("/") + + self.volume_spec = { + "driver_opts": { + "type": "cifs", + "device": f"//{server}/{path}", + "o": f"{','.join([escape_dollar(opt) for opt in opts])}", + }, + } + + def get(self): + return self.volume_spec + + +class DockerVolume: + def __init__(self, render_instance: "Render", config: "IxStorageVolumeConfig"): + self._render_instance = render_instance + self.volume_spec: dict = {} + + def get(self): + return self.volume_spec diff --git a/ix-dev/community/archivebox/templates/library/base_v2_2_8/volumes.py b/ix-dev/community/archivebox/templates/library/base_v2_2_8/volumes.py new file mode 100644 index 00000000000..1a21d464588 --- /dev/null +++ b/ix-dev/community/archivebox/templates/library/base_v2_2_8/volumes.py @@ -0,0 +1,61 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + + +try: + from .error import RenderError + from .storage import IxStorageVolumeLikeConfigs + from .volume_types import NfsVolume, CifsVolume, DockerVolume +except ImportError: + from error import RenderError + from storage import IxStorageVolumeLikeConfigs + from volume_types import NfsVolume, CifsVolume, DockerVolume + + +class Volumes: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volumes: dict[str, Volume] = {} + + def add_volume(self, source: str, storage_type: str, config: "IxStorageVolumeLikeConfigs"): + # This method can be called many times from the volume mounts + # Only add the volume if it is not already added, but dont raise an error + if source == "": + raise RenderError(f"Volume source [{source}] cannot be empty") + + if source in self._volumes: + return + + self._volumes[source] = Volume(self._render_instance, storage_type, config) + + def has_volumes(self) -> bool: + return bool(self._volumes) + + def render(self): + return {name: v.render() for name, v in sorted(self._volumes.items()) if v.render() is not None} + + +class Volume: + def __init__( + self, + render_instance: "Render", + storage_type: str, + config: "IxStorageVolumeLikeConfigs", + ): + self._render_instance = render_instance + self.volume_spec: dict | None = {} + + match storage_type: + case "nfs": + self.volume_spec = NfsVolume(self._render_instance, config).get() # type: ignore + case "cifs": + self.volume_spec = CifsVolume(self._render_instance, config).get() # type: ignore + case "volume" | "temporary": + self.volume_spec = DockerVolume(self._render_instance, config).get() # type: ignore + case _: + self.volume_spec = None + + def render(self): + return self.volume_spec From da523fb89ade7a8f26f74d8a7c88876c95d5be39 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Thu, 12 Mar 2026 18:49:39 +0200 Subject: [PATCH 09/17] rm --- ix-dev/community/archivebox/templates/docker-compose.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/ix-dev/community/archivebox/templates/docker-compose.yaml b/ix-dev/community/archivebox/templates/docker-compose.yaml index a871f78c132..0d96059f756 100644 --- a/ix-dev/community/archivebox/templates/docker-compose.yaml +++ b/ix-dev/community/archivebox/templates/docker-compose.yaml @@ -65,7 +65,6 @@ {% do sonic.set_user(values.run_as.user, values.run_as.group) %} {% do sonic.healthcheck.set_test("tcp", {"port": 1491}) %} {% do sonic.environment.add_env("SEARCH_BACKEND_PASSWORD", sonic_password) %} -{% do sonic.environment.add_env("SEARCH_BACKEND_PORT", values.consts.internal_sonic_port) %} {% do sonic.add_storage("/var/lib/sonic/store", values.storage.sonic_data) %} {% do perm_container.add_or_skip_action("sonic_data", values.storage.sonic_data, perms_config) %} From 5178e62a18aabc3562d06cc96f23a9d509b52be8 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Thu, 12 Mar 2026 18:58:20 +0200 Subject: [PATCH 10/17] genmeta --- ix-dev/community/archivebox/app.yaml | 36 +++++++++++++++++++++------ ix-dev/community/archivebox/item.yaml | 10 ++++---- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/ix-dev/community/archivebox/app.yaml b/ix-dev/community/archivebox/app.yaml index 7e9487e9118..446bb747b45 100644 --- a/ix-dev/community/archivebox/app.yaml +++ b/ix-dev/community/archivebox/app.yaml @@ -1,5 +1,16 @@ app_version: 0.7.3 -capabilities: [] +capabilities: +- description: ArchiveBox, Scheduler are able to change file ownership arbitrarily + name: CHOWN +- description: ArchiveBox, Scheduler are able to bypass file permission checks + name: DAC_OVERRIDE +- description: ArchiveBox, Scheduler are able to bypass permission checks for file + operations + name: FOWNER +- description: ArchiveBox, Scheduler are able to change group ID of processes + name: SETGID +- description: ArchiveBox, Scheduler are able to change user ID of processes + name: SETUID categories: - productivity date_added: '2026-03-12' @@ -8,7 +19,7 @@ description: ArchiveBox is a powerful, self-hosted internet archiving solution t WARC, and more. home: https://archivebox.io host_mounts: [] -icon: https://media.sys.truenas.net/apps/archivebox/icons/icon.png +icon: https://media.sys.truenas.net/apps/archivebox/icons/icon.svg keywords: - web - internet @@ -29,17 +40,26 @@ maintainers: url: https://www.truenas.com/ name: archivebox run_as_context: -- description: Container [archivebox] starts as root but internally drops privileges - to whichever user owns the /data dir. +- description: Container [archivebox] runs as root user and group. gid: 0 group_name: Host group is [root] uid: 0 user_name: Host user is [root] +- description: Container [scheduler] runs as root user and group. + gid: 0 + group_name: Host group is [root] + uid: 0 + user_name: Host user is [root] +- description: Container [sonic] can run as any non-root user and group. + gid: 568 + group_name: Host group is [apps] + uid: 568 + user_name: Host user is [apps] screenshots: -- https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2 -- https://github.com/ArchiveBox/ArchiveBox/assets/511499/dad2bc51-e7e5-484e-bb26-f956ed692d16 -- https://github.com/ArchiveBox/ArchiveBox/assets/511499/ace0954a-ddac-4520-9d18-1c77b1ec50b2 -- https://github.com/ArchiveBox/ArchiveBox/assets/511499/8d67382c-e0ce-4286-89f7-7915f09b930c +- https://media.sys.truenas.net/apps/archivebox/screenshots/screenshot1.png +- https://media.sys.truenas.net/apps/archivebox/screenshots/screenshot2.png +- https://media.sys.truenas.net/apps/archivebox/screenshots/screenshot3.png +- https://media.sys.truenas.net/apps/archivebox/screenshots/screenshot4.png sources: - https://github.com/ArchiveBox/ArchiveBox - https://archivebox.io diff --git a/ix-dev/community/archivebox/item.yaml b/ix-dev/community/archivebox/item.yaml index a22ef27b235..3503824caa3 100644 --- a/ix-dev/community/archivebox/item.yaml +++ b/ix-dev/community/archivebox/item.yaml @@ -1,11 +1,11 @@ categories: - productivity -icon_url: https://media.sys.truenas.net/apps/archivebox/icons/icon.png +icon_url: https://media.sys.truenas.net/apps/archivebox/icons/icon.svg screenshots: -- https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2 -- https://github.com/ArchiveBox/ArchiveBox/assets/511499/dad2bc51-e7e5-484e-bb26-f956ed692d16 -- https://github.com/ArchiveBox/ArchiveBox/assets/511499/ace0954a-ddac-4520-9d18-1c77b1ec50b2 -- https://github.com/ArchiveBox/ArchiveBox/assets/511499/8d67382c-e0ce-4286-89f7-7915f09b930c +- https://media.sys.truenas.net/apps/archivebox/screenshots/screenshot1.png +- https://media.sys.truenas.net/apps/archivebox/screenshots/screenshot2.png +- https://media.sys.truenas.net/apps/archivebox/screenshots/screenshot3.png +- https://media.sys.truenas.net/apps/archivebox/screenshots/screenshot4.png tags: - web - internet From de8b975ba4ccf087d25e6ecb0df0e9613037a871 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Thu, 12 Mar 2026 19:11:15 +0200 Subject: [PATCH 11/17] sonic hc fix spam --- ix-dev/community/archivebox/templates/docker-compose.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ix-dev/community/archivebox/templates/docker-compose.yaml b/ix-dev/community/archivebox/templates/docker-compose.yaml index 0d96059f756..2c806a2c93e 100644 --- a/ix-dev/community/archivebox/templates/docker-compose.yaml +++ b/ix-dev/community/archivebox/templates/docker-compose.yaml @@ -63,7 +63,8 @@ {% do scheduler.depends.add_dependency(values.consts.archivebox_container_name, "service_healthy") %} {% do sonic.set_user(values.run_as.user, values.run_as.group) %} -{% do sonic.healthcheck.set_test("tcp", {"port": 1491}) %} +{# 05D3 == 1491 in hex (sonic port) #} +{% do sonic.healthcheck.set_custom_test(["CMD", "bash", "-c", "grep -q '00000000:05D3' /proc/net/tcp"]) %} {% do sonic.environment.add_env("SEARCH_BACKEND_PASSWORD", sonic_password) %} {% do sonic.add_storage("/var/lib/sonic/store", values.storage.sonic_data) %} From d142e9112301a2f6c24d7a26a7521ce044f57002 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Thu, 12 Mar 2026 19:17:04 +0200 Subject: [PATCH 12/17] those need additional storage to set, so doing half the work is not really great. Let user handle it completely instead --- .../archivebox/templates/test_values/basic-values.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ix-dev/community/archivebox/templates/test_values/basic-values.yaml b/ix-dev/community/archivebox/templates/test_values/basic-values.yaml index fc4dfafe2d2..89f616aa434 100644 --- a/ix-dev/community/archivebox/templates/test_values/basic-values.yaml +++ b/ix-dev/community/archivebox/templates/test_values/basic-values.yaml @@ -12,8 +12,6 @@ archivebox: timeout: 60 save_archive_dot_org: true user_agent: "" - cookies_file: "" - chrome_user_data_dir: "" public_index: true public_snapshots: true public_add_view: false From 4385c697fa7e41563f1cdd901760ccfdb5dacb73 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Thu, 12 Mar 2026 19:22:36 +0200 Subject: [PATCH 13/17] keep it minimal for now. expose later --- ix-dev/community/archivebox/questions.yaml | 30 ++++--------------- .../archivebox/templates/docker-compose.yaml | 15 ---------- .../templates/test_values/basic-values.yaml | 2 -- 3 files changed, 6 insertions(+), 41 deletions(-) diff --git a/ix-dev/community/archivebox/questions.yaml b/ix-dev/community/archivebox/questions.yaml index 2ac62a15d54..593cd97106f 100644 --- a/ix-dev/community/archivebox/questions.yaml +++ b/ix-dev/community/archivebox/questions.yaml @@ -29,15 +29,6 @@ questions: schema: type: dict attrs: - - variable: public_url - label: Public URL - description: | - The URL users will use to access this ArchiveBox server (e.g. http://archivebox.example.com:30387).
- This value is used to fill the CSRF_TRUSTED_ORIGINS and ALLOWED_HOSTS env vars if they are not set. - schema: - type: uri - default: "" - required: true - variable: admin_username label: Admin Username description: The initial admin username for ArchiveBox (only used on first run). @@ -71,6 +62,12 @@ questions: schema: type: boolean default: false + - variable: save_archive_dot_org + label: Save to Archive.org + description: Submit all archived URLs to Archive.org's Wayback Machine. + schema: + type: boolean + default: false - variable: timeout label: Extraction Time Limit description: | @@ -81,21 +78,6 @@ questions: default: 60 min: 10 max: 3600 - - variable: save_archive_dot_org - label: Save to Archive.org - description: Submit all archived URLs to Archive.org's Wayback Machine. - schema: - type: boolean - default: false - - variable: user_agent - label: User Agent - description: | - Custom browser User-Agent string to use when fetching pages.
- Useful to avoid being blocked as a bot by some websites.
- Leave empty to use the default ArchiveBox/vX.X.X user agent. - schema: - type: string - default: "" - variable: additional_envs label: Additional Environment Variables description: | diff --git a/ix-dev/community/archivebox/templates/docker-compose.yaml b/ix-dev/community/archivebox/templates/docker-compose.yaml index 2c806a2c93e..b7adcc58fae 100644 --- a/ix-dev/community/archivebox/templates/docker-compose.yaml +++ b/ix-dev/community/archivebox/templates/docker-compose.yaml @@ -23,10 +23,6 @@ {% do c.environment.add_env("DATA_DIR", values.archivebox.timeout) %} {% do c.environment.add_env("TIMEOUT", values.archivebox.timeout) %} {% do c.environment.add_env("SAVE_ARCHIVE_DOT_ORG", values.archivebox.save_archive_dot_org) %} - {% if values.archivebox.user_agent %} - {% do c.environment.add_env("USER_AGENT", values.archivebox.user_agent) %} - {% endif %} - {% do c.environment.add_env("SEARCH_BACKEND_ENGINE", "sonic") %} {% do c.environment.add_env("SEARCH_BACKEND_PORT", values.consts.internal_sonic_port) %} {% do c.environment.add_env("SEARCH_BACKEND_HOST_NAME", values.consts.sonic_container_name) %} @@ -37,17 +33,6 @@ {% endfor %} -{# TODO: Fix this -{% set public_url = tpl.funcs.url_to_dict(values.archivebox.public_url, True) %} -{% set allowed_host = public_url.host %} - -{% do container.environment.add_env("ALLOWED_HOSTS", allowed_host) %} -{% do container.environment.add_env("CSRF_TRUSTED_ORIGINS", csrf_trusted_origin) %} -{% do container.environment.add_env("CHECK_SSL_VALIDITY", csrf_trusted_origin) %} - -"headers": [["Host", allowed_host]] -#} - {% do archivebox.set_command(["server", "--init", "0.0.0.0:%d"|format(values.network.web_port.port_number)]) %} {% do archivebox.healthcheck.set_test("curl", {"port": values.network.web_port.port_number, "path": "/health/"}) %} {% do archivebox.environment.add_env("PUBLIC_INDEX", values.archivebox.public_index) %} diff --git a/ix-dev/community/archivebox/templates/test_values/basic-values.yaml b/ix-dev/community/archivebox/templates/test_values/basic-values.yaml index 89f616aa434..e246c848072 100644 --- a/ix-dev/community/archivebox/templates/test_values/basic-values.yaml +++ b/ix-dev/community/archivebox/templates/test_values/basic-values.yaml @@ -8,10 +8,8 @@ TZ: Etc/UTC archivebox: admin_username: admin admin_password: testpassword123 - public_url: http://archivebox.example.com:30820 timeout: 60 save_archive_dot_org: true - user_agent: "" public_index: true public_snapshots: true public_add_view: false From c2204760db59a06297af18c1e2576b2e82d4efc5 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Thu, 12 Mar 2026 19:25:52 +0200 Subject: [PATCH 14/17] huh --- ix-dev/community/archivebox/templates/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ix-dev/community/archivebox/templates/docker-compose.yaml b/ix-dev/community/archivebox/templates/docker-compose.yaml index b7adcc58fae..f673ce2efba 100644 --- a/ix-dev/community/archivebox/templates/docker-compose.yaml +++ b/ix-dev/community/archivebox/templates/docker-compose.yaml @@ -58,7 +58,7 @@ {% for store in values.storage.additional_storage %} {% do archivebox.add_storage(store.mount_path, store) %} {% do scheduler.add_storage(store.mount_path, store) %} - {% do perm_container.add_or_skip_action(store.mount_path, store.size, perms_config) %} + {% do perm_container.add_or_skip_action(store.mount_path, store, perms_config) %} {% endfor %} {% if perm_container.has_actions() %} From 041fb8250a0dfbdfa5ab8265d125f28843c46b20 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Thu, 12 Mar 2026 19:29:05 +0200 Subject: [PATCH 15/17] ugh --- ix-dev/community/archivebox/templates/docker-compose.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ix-dev/community/archivebox/templates/docker-compose.yaml b/ix-dev/community/archivebox/templates/docker-compose.yaml index f673ce2efba..dabd33947a5 100644 --- a/ix-dev/community/archivebox/templates/docker-compose.yaml +++ b/ix-dev/community/archivebox/templates/docker-compose.yaml @@ -20,7 +20,7 @@ {% do c.add_caps(["CHOWN", "SETGID", "SETUID", "FOWNER", "DAC_OVERRIDE"]) %} {% do c.depends.add_dependency(values.consts.sonic_container_name, "service_healthy") %} - {% do c.environment.add_env("DATA_DIR", values.archivebox.timeout) %} + {% do c.environment.add_env("DATA_DIR", values.consts.data_path) %} {% do c.environment.add_env("TIMEOUT", values.archivebox.timeout) %} {% do c.environment.add_env("SAVE_ARCHIVE_DOT_ORG", values.archivebox.save_archive_dot_org) %} {% do c.environment.add_env("SEARCH_BACKEND_ENGINE", "sonic") %} @@ -32,7 +32,6 @@ {% do c.add_storage(values.consts.data_path, values.storage.data) %} {% endfor %} - {% do archivebox.set_command(["server", "--init", "0.0.0.0:%d"|format(values.network.web_port.port_number)]) %} {% do archivebox.healthcheck.set_test("curl", {"port": values.network.web_port.port_number, "path": "/health/"}) %} {% do archivebox.environment.add_env("PUBLIC_INDEX", values.archivebox.public_index) %} From d063cd907e954d2932d95968df93b28ad94c3151 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Thu, 12 Mar 2026 19:36:48 +0200 Subject: [PATCH 16/17] Add TODO for ALLOWED_HOSTS in docker-compose.yaml --- ix-dev/community/archivebox/templates/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ix-dev/community/archivebox/templates/docker-compose.yaml b/ix-dev/community/archivebox/templates/docker-compose.yaml index dabd33947a5..c83263a91c4 100644 --- a/ix-dev/community/archivebox/templates/docker-compose.yaml +++ b/ix-dev/community/archivebox/templates/docker-compose.yaml @@ -14,6 +14,7 @@ {% set sonic = tpl.add_container(values.consts.sonic_container_name, "sonic_image") %} {% do sonic.add_network(archivebox_net) %} +{# TODO: Add ALLOWED_HOSTS and append 127.0.0.1 so curl will work for hc #} {% set sonic_password = tpl.funcs.secure_string(32) %} {% set containers = [archivebox, scheduler] %} {% for c in containers %} From b9c5fa4458b288a37a276ac08e17d07d82e111c2 Mon Sep 17 00:00:00 2001 From: Stavros Kois Date: Thu, 12 Mar 2026 19:46:31 +0200 Subject: [PATCH 17/17] port --- ix-dev/community/archivebox/questions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ix-dev/community/archivebox/questions.yaml b/ix-dev/community/archivebox/questions.yaml index 593cd97106f..72536c1524d 100644 --- a/ix-dev/community/archivebox/questions.yaml +++ b/ix-dev/community/archivebox/questions.yaml @@ -158,7 +158,7 @@ questions: label: Port Number schema: type: int - default: 30387 + default: 30392 min: 1 max: 65535 required: true