[ci] refactor ci #3232
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: pr_run_test | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - 'README.md' | |
| - 'README_zh-CN.md' | |
| - 'docs/**' | |
| - 'configs/**' | |
| - 'tools/**' | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '56 22 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CONDA_ENV: pr_regression | |
| HF_DATASETS_OFFLINE: 1 | |
| HF_EVALUATE_OFFLINE: 1 | |
| TRANSFORMERS_OFFLINE: 1 | |
| VLLM_USE_MODELSCOPE: false | |
| LMDEPLOY_USE_MODELSCOPE: false | |
| HF_HUB_OFFLINE: 1 | |
| CONDA_PATH: /mnt/shared-storage-user/opencompass-shared/qa-llm-cicd/miniconda3 | |
| REPORT_ROOT: /mnt/shared-storage-user/opencompass-shared/qa-llm-cicd/eval_report/prtest | |
| BASELINE_DIR: mock-api-baseline | |
| COMPASS_DATA_CACHE: /mnt/shared-storage-user/auto-eval-pipeline/opencompass/llmeval/compass_data_cache | |
| HF_DATASETS_CACHE: /mnt/shared-storage-user/opencompass-shared/qa-llm-cicd/hf_cache | |
| HF_HUB_CACHE: /mnt/shared-storage-gpfs2/gpfs2-shared-public/huggingface/hub | |
| KUBEBRAIN_CLUSTER_ENTRY: https://h.pjlab.org.cn | |
| # Must match rjob --namespace below (used by submit/get/logs/stop). | |
| KUBEBRAIN_NAMESPACE: ailab-llmeval | |
| SCORE_MIN: '75' | |
| SCORE_MAX: '80' | |
| jobs: | |
| cmd_test: | |
| runs-on: yidian_cu12 | |
| timeout-minutes: 45 | |
| env: | |
| JOB_NAME: pr-test-${{ github.run_id }}-cmd-${{ github.run_attempt }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Prepare - Install opencompass | |
| run: | | |
| . ${{env.CONDA_PATH}}/bin/activate | |
| conda activate ${{env.CONDA_ENV}} | |
| python3 -m pip uninstall opencompass -y | |
| python3 -m pip install .[full] | |
| conda info --envs | |
| pip list | |
| lmdeploy check_env | |
| - name: Run test | |
| run: | | |
| . ${{env.CONDA_PATH}}/bin/activate | |
| conda activate ${{env.CONDA_ENV}} | |
| pip list | |
| NS=${{ env.KUBEBRAIN_NAMESPACE }} | |
| rjob submit --metadata-name=${{ env.JOB_NAME }} --charged-group=llmeval_gpu --private-machine=group --group=llmeval_gpu --namespace=${NS} --gpu=2 --cpu=32 --memory=32568 --private-machine=group --image=registry.h.pjlab.org.cn/ailab-puyu/xpuyu:torch-2.6.0-45d96d5f-0607 --env=COMPASS_DATA_CACHE=/mnt/shared-storage-user/auto-eval-pipeline/opencompass/llmeval/compass_data_cache --env=TIKTOKEN_CACHE_DIR=/mnt/shared-storage-user/auto-eval-pipeline/opencompass/llmeval/share_tiktoken --env=HF_ENDPOINT=https://hf-mirror.com --env=HF_DATASETS_CACHE=${{env.HF_DATASETS_CACHE}} --env=HF_HUB_CACHE=/mnt/shared-storage-user/large-model-center-share-weights/hf_hub --env=CUDA_MODULE_LOADING=EAGER --env=HF_DATASETS_OFFLINE=1 --env=TRANSFORMERS_OFFLINE=1 --env=HF_EVALUATE_OFFLINE=1 --env=HF_HUB_OFFLINE=1 --mount=gpfs://gpfs1/qa-llm-cicd:/mnt/shared-storage-user/qa-llm-cicd --mount=gpfs://gpfs1/opencompass-shared:/mnt/shared-storage-user/opencompass-shared --mount=gpfs://gpfs1/auto-eval-pipeline:/mnt/shared-storage-user/auto-eval-pipeline --mount=gpfs://gpfs1/large-model-center-share-weights:/mnt/shared-storage-user/large-model-center-share-weights -- bash -exc 'OUT="${{env.REPORT_ROOT}}/${{ github.run_id }}"; set +e; /mnt/shared-storage-user/opencompass-shared/qa-llm-cicd/pr_test_new.sh "$OUT"; status=$?; chmod -R a+rwX "$OUT" 2>/dev/null || true; exit $status' | |
| for i in {1..300}; do | |
| current_status=$(rjob get --namespace=${NS} ${{ env.JOB_NAME }} | grep -oP 'rjob [^:]+: \K[^ ]+' || true) | |
| if [[ $current_status == "Succeeded" ]]; then | |
| echo "Task succeeded" | |
| rjob logs job --namespace=${NS} ${{ env.JOB_NAME }} -n 100 | |
| exit 0 | |
| elif [[ $current_status == "Failed" || $current_status == "Stopped" ]]; then | |
| echo "Task failed or stopped, fetching logs" | |
| rjob logs job --namespace=${NS} ${{ env.JOB_NAME }} -n 100 | |
| exit 1 | |
| fi | |
| sleep 6 | |
| done | |
| echo "Timed out waiting for rjob ${{ env.JOB_NAME }} (last status=${current_status:-unknown})" | |
| rjob logs job --namespace=${NS} ${{ env.JOB_NAME }} -n 100 || true | |
| exit 1 | |
| - name: Get result | |
| run: | | |
| . ${{env.CONDA_PATH}}/bin/activate | |
| conda activate ${{env.CONDA_ENV}} | |
| python autotest/utils/check_cmd_scores.py \ | |
| "${{env.REPORT_ROOT}}/${{ github.run_id }}" \ | |
| --min "${{env.SCORE_MIN}}" \ | |
| --max "${{env.SCORE_MAX}}" | |
| - name: Cleanup output on success | |
| run: | | |
| # Only delete this job's outputs; keep sibling mock_api_test dirs under the same run_id. | |
| RUN_DIR="${{env.REPORT_ROOT}}/${{ github.run_id }}" | |
| for d in "$RUN_DIR"/regression_result{1,2,3}; do | |
| if [[ -d "$d" ]]; then | |
| echo "Removing successful cmd output: $d" | |
| rm -rf "$d" | |
| fi | |
| done | |
| rmdir "$RUN_DIR" 2>/dev/null || true | |
| - name: Uninstall opencompass | |
| if: always() | |
| run: | | |
| . ${{env.CONDA_PATH}}/bin/activate | |
| conda activate ${{env.CONDA_ENV}} | |
| python3 -m pip uninstall opencompass -y | |
| conda info --envs | |
| rjob stop --namespace=${{ env.KUBEBRAIN_NAMESPACE }} ${{ env.JOB_NAME }} | |
| mock_api_test: | |
| runs-on: yidian_cu12 | |
| timeout-minutes: 120 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - func_type: chat_obj_fullbench_v1 | |
| name: chat-v1 | |
| memory: 32568 | |
| cpu: 16 | |
| - func_type: chat_obj_fullbench_v2 | |
| name: chat-v2 | |
| memory: 32568 | |
| cpu: 16 | |
| - func_type: chat_sub_fullbench | |
| name: chat-sub | |
| # memory: 3072 | |
| memory: 32568 | |
| cpu: 2 | |
| - func_type: chat_longtext_fullbench | |
| name: chat-longtext | |
| memory: 65136 | |
| cpu: 16 | |
| env: | |
| JOB_NAME: pr-test-${{ github.run_id }}-api-${{ matrix.name }}-${{ github.run_attempt }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Prepare - Install opencompass | |
| run: | | |
| . ${{env.CONDA_PATH}}/bin/activate | |
| conda activate ${{env.CONDA_ENV}} | |
| python3 -m pip uninstall opencompass -y | |
| python3 -m pip install .[full] | |
| conda info --envs | |
| pip list | |
| lmdeploy check_env | |
| - name: Run test | |
| run: | | |
| . ${{env.CONDA_PATH}}/bin/activate | |
| conda activate ${{env.CONDA_ENV}} | |
| pip list | |
| NS=${{ env.KUBEBRAIN_NAMESPACE }} | |
| rjob submit --metadata-name=${{ env.JOB_NAME }} --charged-group=llmeval_gpu --private-machine=group --group=llmeval_gpu --namespace=${NS} --gpu=0 --cpu=${{ matrix.cpu }} --memory=${{ matrix.memory }} --private-machine=group --image=registry.h.pjlab.org.cn/ailab-puyu-puyu_gpu/lmdeploy:v0.12.0-cu12.8 --env=COMPASS_DATA_CACHE=/mnt/shared-storage-user/auto-eval-pipeline/opencompass/llmeval/compass_data_cache --env=TIKTOKEN_CACHE_DIR=/mnt/shared-storage-user/auto-eval-pipeline/opencompass/llmeval/share_tiktoken --env=HF_ENDPOINT=https://hf-mirror.com --env=HF_DATASETS_CACHE=${{env.HF_DATASETS_CACHE}} --env=HF_HUB_CACHE=/mnt/shared-storage-gpfs2/gpfs2-shared-public/huggingface/hub --env=CUDA_MODULE_LOADING=EAGER --env=HF_DATASETS_OFFLINE=1 --env=TRANSFORMERS_OFFLINE=1 --env=HF_EVALUATE_OFFLINE=1 --env=HF_HUB_OFFLINE=1 --env=REPORT_DIR=${{env.REPORT_ROOT}} --env=CHAT_TYPE=${{matrix.func_type}} --mount=gpfs://gpfs1/qa-llm-cicd:/mnt/shared-storage-user/qa-llm-cicd --mount=gpfs://gpfs1/opencompass-shared:/mnt/shared-storage-user/opencompass-shared --mount=gpfs://gpfs1/auto-eval-pipeline:/mnt/shared-storage-user/auto-eval-pipeline --mount=gpfs://gpfs2/gpfs2-shared-public:/mnt/shared-storage-gpfs2/gpfs2-shared-public -- bash -exc 'source ${{env.CONDA_PATH}}/bin/activate; conda activate ${{env.CONDA_ENV}}; conda env list; cd ${{github.workspace}}; ln -s /mnt/shared-storage-user/auto-eval-pipeline/opencompass/llmeval/compass_data_cache/data .; python /mnt/shared-storage-user/opencompass-shared/qa-llm-cicd/mock_chat_api.py --type winrate --port 26333 > mock_${{matrix.name}}.log 2>&1 & sleep 3; OUT="${{env.REPORT_ROOT}}/${{ github.run_id }}/${{matrix.func_type}}"; set +e; opencompass autotest/all/${{matrix.func_type}}.py --work-dir "$OUT" --reuse; status=$?; chmod -R a+rwX "$OUT" 2>/dev/null || true; exit $status' | |
| for i in {1..300}; do | |
| current_status=$(rjob get --namespace=${NS} ${{ env.JOB_NAME }} | grep -oP 'rjob [^:]+: \K[^ ]+' || true) | |
| if [[ $current_status == "Succeeded" ]]; then | |
| echo "Task succeeded" | |
| rjob logs job --namespace=${NS} ${{ env.JOB_NAME }} -n 100 | |
| exit 0 | |
| elif [[ $current_status == "Failed" || $current_status == "Stopped" ]]; then | |
| echo "Task failed or stopped, fetching logs" | |
| rjob logs job --namespace=${NS} ${{ env.JOB_NAME }} -n 100 | |
| exit 1 | |
| fi | |
| sleep 6 | |
| done | |
| echo "Timed out waiting for rjob ${{ env.JOB_NAME }} (last status=${current_status:-unknown})" | |
| rjob logs job --namespace=${NS} ${{ env.JOB_NAME }} -n 100 || true | |
| exit 1 | |
| - name: Compare predictions with baseline | |
| run: | | |
| . ${{env.CONDA_PATH}}/bin/activate | |
| conda activate ${{env.CONDA_ENV}} | |
| CURRENT="${{env.REPORT_ROOT}}/${{ github.run_id }}/${{matrix.func_type}}" | |
| BASELINE="${{env.REPORT_ROOT}}/${{env.BASELINE_DIR}}/${{matrix.func_type}}" | |
| echo "Current run: $CURRENT" | |
| echo "Baseline: $BASELINE" | |
| if [[ ! -d "$CURRENT" ]]; then | |
| echo "Current run output not found: $CURRENT" | |
| exit 1 | |
| fi | |
| if [[ ! -d "$BASELINE" ]]; then | |
| echo "Baseline not found: $BASELINE" | |
| echo "Upload golden predictions to REPORT_ROOT/BASELINE_DIR/<func_type> on shared storage." | |
| exit 1 | |
| fi | |
| python autotest/utils/compare_results.py compare_results \ | |
| "$CURRENT" "$BASELINE" predictions | |
| python autotest/utils/compare_results.py compare_results \ | |
| "$CURRENT" "$BASELINE" results | |
| python autotest/utils/compare_results.py compare_results \ | |
| "$CURRENT" "$BASELINE" summary | |
| - name: Cleanup output on success | |
| run: | | |
| CURRENT="${{env.REPORT_ROOT}}/${{ github.run_id }}/${{matrix.func_type}}" | |
| BASELINE_ROOT="${{env.REPORT_ROOT}}/${{env.BASELINE_DIR}}" | |
| # Safety: never touch golden baseline. | |
| if [[ "$CURRENT" == "$BASELINE_ROOT" || "$CURRENT" == "$BASELINE_ROOT"/* ]]; then | |
| echo "Refusing to delete baseline path: $CURRENT" | |
| exit 1 | |
| fi | |
| if [[ -d "$CURRENT" ]]; then | |
| echo "Removing successful mock output: $CURRENT" | |
| rm -rf "$CURRENT" | |
| else | |
| echo "No output dir to remove: $CURRENT" | |
| fi | |
| rmdir "${{env.REPORT_ROOT}}/${{ github.run_id }}" 2>/dev/null || true | |
| - name: Uninstall opencompass | |
| if: always() | |
| run: | | |
| . ${{env.CONDA_PATH}}/bin/activate | |
| conda activate ${{env.CONDA_ENV}} | |
| python3 -m pip uninstall opencompass -y | |
| conda info --envs | |
| rjob stop --namespace=${{ env.KUBEBRAIN_NAMESPACE }} ${{ env.JOB_NAME }} |