feat(elastic_security): add _source field filter to list_detection_si… #9499
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Python tests | |
| on: | |
| push: | |
| branches: ["main"] | |
| paths: | |
| - tracecat/** | |
| - registry/** | |
| - tests/** | |
| - packages/tracecat-ee/** | |
| - packages/tracecat-registry/** | |
| - pyproject.toml | |
| - uv.lock | |
| - docker-compose.dev.yml | |
| - .env.example | |
| - env.sh | |
| - .github/workflows/test-python.yml | |
| pull_request: | |
| branches: ["main", "staging"] | |
| paths: | |
| - tracecat/** | |
| - registry/** | |
| - tests/** | |
| - packages/tracecat-ee/** | |
| - packages/tracecat-registry/** | |
| - pyproject.toml | |
| - uv.lock | |
| - docker-compose.dev.yml | |
| - .env.example | |
| - env.sh | |
| - .github/workflows/test-python.yml | |
| permissions: | |
| contents: read | |
| env: | |
| UV_SYSTEM_PYTHON: 1 | |
| jobs: | |
| test-custom-registry-install: | |
| runs-on: blacksmith-8vcpu-ubuntu-2204 | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Install uv | |
| uses: useblacksmith/setup-uv@f4588471335f14dcb60198856207b916701a9e9f # v4 | |
| with: | |
| version: "0.9.7" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| pyproject.toml | |
| uv.lock | |
| - name: Set up Python 3.12 | |
| uses: useblacksmith/setup-python@943b05a7c4ca70c2360391137527a37bee33fb1d # v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Clone custom registry starter | |
| run: git clone https://github.com/TracecatHQ/custom-integrations-starter-kit | |
| - name: Install dependencies | |
| working-directory: custom-integrations-starter-kit | |
| run: | | |
| uv venv | |
| env -u UV_SYSTEM_PYTHON uv pip install --python .venv/bin/python . | |
| .venv/bin/python -c "import custom_actions" | |
| test-all: | |
| runs-on: blacksmith-8vcpu-ubuntu-2204 | |
| timeout-minutes: 60 | |
| strategy: | |
| matrix: | |
| test_group: | |
| - unit | |
| - registry | |
| - temporal | |
| # - ee # TODO: re-enable this when we have tests | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Install uv | |
| uses: useblacksmith/setup-uv@f4588471335f14dcb60198856207b916701a9e9f # v4 | |
| with: | |
| version: "0.9.7" | |
| enable-cache: true | |
| cache-dependency-glob: | | |
| pyproject.toml | |
| uv.lock | |
| - name: Set up Python 3.12 | |
| uses: useblacksmith/setup-python@943b05a7c4ca70c2360391137527a37bee33fb1d # v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Disable IPv6 (Blacksmith VMs lack IPv6 routing) | |
| run: | | |
| sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 | |
| sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 | |
| - name: Run environment setup script | |
| run: | | |
| echo "y | |
| localhost | |
| n | |
| test@tracecat.com" | bash env.sh | |
| - name: Start core Docker services and install dependencies | |
| run: | | |
| set -euo pipefail | |
| compose_files=(-f docker-compose.dev.yml) | |
| services=(postgres_db minio redis) | |
| uv sync --frozen & | |
| uv_sync_pid=$! | |
| cleanup_uv_sync() { | |
| if kill -0 "$uv_sync_pid" 2>/dev/null; then | |
| kill "$uv_sync_pid" 2>/dev/null || true | |
| fi | |
| } | |
| trap cleanup_uv_sync EXIT | |
| if [ "${{ matrix.test_group }}" = "temporal" ]; then | |
| services=(temporal postgres_db minio redis) | |
| cat > /tmp/tracecat-ci-postgres.yml <<'YAML' | |
| services: | |
| postgres_db: | |
| command: ["postgres", "-c", "max_connections=500"] | |
| YAML | |
| compose_files+=(-f /tmp/tracecat-ci-postgres.yml) | |
| fi | |
| docker compose "${compose_files[@]}" up -d "${services[@]}" | |
| echo "Waiting for services to become healthy..." | |
| deadline=$((SECONDS + 300)) | |
| while true; do | |
| statuses=$( | |
| docker compose "${compose_files[@]}" ps --format json | | |
| jq -r 'if type == "array" then .[] else . end | select(.Health != null and .Health != "") | .Health' | |
| ) | |
| total=$(echo "$statuses" | grep -c . || true) | |
| healthy=$(echo "$statuses" | grep -c "^healthy$" || true) | |
| if [ "$total" -eq 0 ]; then | |
| echo "No services with health checks found yet, waiting..." | |
| elif [ "$healthy" -lt "$total" ]; then | |
| echo "Waiting for services to be healthy ($healthy/$total ready)..." | |
| else | |
| echo "All $total services with health checks are healthy" | |
| break | |
| fi | |
| if [ "$SECONDS" -ge "$deadline" ]; then | |
| echo "Timed out waiting for Docker services to become healthy" | |
| docker compose "${compose_files[@]}" ps | |
| exit 124 | |
| fi | |
| sleep 5 | |
| done | |
| echo "Waiting for Python environment sync..." | |
| wait "$uv_sync_pid" | |
| trap - EXIT | |
| - name: Register Temporal search attributes | |
| if: matrix.test_group == 'temporal' | |
| run: | | |
| set -euo pipefail | |
| attrs=( | |
| TracecatTriggerType | |
| TracecatTriggeredByUserId | |
| TracecatWorkspaceId | |
| TracecatAlias | |
| TracecatCorrelationId | |
| TracecatExecutionType | |
| ) | |
| add_args=() | |
| for attr in "${attrs[@]}"; do | |
| add_args+=(--name "$attr" --type Keyword) | |
| done | |
| docker compose -f docker-compose.dev.yml exec -T temporal \ | |
| tctl --address temporal:7233 --namespace default --auto_confirm \ | |
| admin cluster add-search-attributes "${add_args[@]}" | |
| for _ in {1..30}; do | |
| output=$(docker compose -f docker-compose.dev.yml exec -T temporal \ | |
| tctl --address temporal:7233 --namespace default \ | |
| admin cluster get-search-attributes --print_json true) | |
| missing=() | |
| for attr in "${attrs[@]}"; do | |
| if [[ "$output" != *"$attr"* ]]; then | |
| missing+=("$attr") | |
| fi | |
| done | |
| if [ "${#missing[@]}" -eq 0 ]; then | |
| echo "Temporal search attributes registered" | |
| exit 0 | |
| fi | |
| echo "Waiting for Temporal search attributes: ${missing[*]}" | |
| sleep 2 | |
| done | |
| echo "Timed out waiting for Temporal search attributes: ${missing[*]}" | |
| exit 1 | |
| - name: Run tests | |
| env: | |
| TRACECAT__SYSTEM_PATH: "/usr/local/bin:/usr/bin:/bin" | |
| TRACECAT__EXECUTOR_BACKEND: "direct" | |
| TRACECAT__WORKFLOW_RETURN_STRATEGY: "context" | |
| TRACECAT__UNSAFE_DISABLE_SM_MASKING: "true" | |
| # xdist workers share the same CI Postgres container; keep each | |
| # worker's service-session pool small enough to avoid exhausting it. | |
| TRACECAT__DB_POOL_SIZE: "1" | |
| TRACECAT__DB_MAX_OVERFLOW: "2" | |
| run: | | |
| if [ "${{ matrix.test_group }}" = "ee" ]; then | |
| uv run pytest packages/tracecat-ee/tests/ -m "not live_secret" -ra | |
| elif [ "${{ matrix.test_group }}" = "temporal" ]; then | |
| # Temporal tests run with compression enabled and parallelization | |
| # Worker-specific task queues in conftest.py enable safe parallel execution | |
| TRACECAT_TEST_API_MODE=inprocess \ | |
| TRACECAT__CONTEXT_COMPRESSION_ENABLED=true \ | |
| TRACECAT__CONTEXT_COMPRESSION_THRESHOLD_KB=0 \ | |
| uv run pytest tests/temporal -m "not live_secret and not requires_api" -n auto -ra -o faulthandler_timeout=300 | |
| # API-backed temporal tests start a local API process against the | |
| # test DB. Run them serially to avoid startup races under xdist. | |
| TRACECAT_TEST_API_MODE=inprocess \ | |
| TRACECAT__CONTEXT_COMPRESSION_ENABLED=true \ | |
| TRACECAT__CONTEXT_COMPRESSION_THRESHOLD_KB=0 \ | |
| uv run pytest tests/temporal -m "requires_api and not live_secret" -ra -o faulthandler_timeout=300 | |
| else | |
| uv run pytest tests/${{ matrix.test_group }} -m "not live_secret" -n auto -ra -o faulthandler_timeout=300 | |
| fi | |
| - name: Show Docker logs on failure | |
| if: failure() | |
| run: | | |
| echo "=== Docker Service Status ===" | |
| docker compose -f docker-compose.dev.yml ps | |
| echo "=== Postgres Logs ===" | |
| docker compose -f docker-compose.dev.yml logs postgres_db --tail=200 | |
| echo "=== MinIO Logs ===" | |
| docker compose -f docker-compose.dev.yml logs minio --tail=200 | |
| echo "=== Redis Logs ===" | |
| docker compose -f docker-compose.dev.yml logs redis --tail=200 | |
| if [ "${{ matrix.test_group }}" = "temporal" ]; then | |
| echo "=== Temporal Logs ===" | |
| docker compose -f docker-compose.dev.yml logs temporal --tail=200 | |
| fi |