Skip to content

Commit 93eb758

Browse files
vamahajavamahaja
committed
feat: add Pulp upload scripts and pipeline integration
Add an optional path to publish built RPM/DEB artifacts to Pulp alongside the existing Chacra flow. - Introduce a `PULP_UPLOAD` boolean parameter (default false) on the `ceph-dev` job. - Add `scripts/setup_pulp.sh` to install `pulp-cli` (with pulp-cli-deb) via uv and write client config from Jenkins credentials. - Add `scripts/pulp_upload.sh` to upload discovered packages, attach them to the target repository, create a publication, and publish a distribution with project/version/ref/arch/sha labels. - Update the Jenkinsfile to install/configure the Pulp client when forcing a rebuild with Pulp upload while Chacra already has artifacts, and to invoke `pulp_upload.sh` after `chacra_upload.sh` when `THROWAWAY` is not `true` and `PULP_UPLOAD` is `true`. Signed-off-by: Vaibhav Mahajan <vamahaja@redhat.com>
1 parent 27fa20b commit 93eb758

4 files changed

Lines changed: 168 additions & 0 deletions

File tree

‎ceph-dev-pipeline/build/Jenkinsfile‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -349,6 +349,15 @@ pipeline {
349349
println("Skipping compilation since chacra already has artifacts. To override, use THROWAWAY=true (to skip this check) or FORCE=true (to re-upload artifacts).")
350350
build_matrix["${DIST}_${ARCH}"] = false
351351
}
352+
if ( chacractl_rc != 0 && env.FORCE == "true" && env.PULP_UPLOAD == "true" ) {
353+
withCredentials([
354+
usernamePassword(credentialsId: 'pulp-auth', usernameVariable: 'PULP_USERNAME', passwordVariable: 'PULP_PASSWORD')
355+
]) {
356+
sh './scripts/setup_pulp.sh'
357+
}
358+
} else {
359+
println("Skipping Pulp upload since chacra already has artifacts. To override, use FORCE=true.")
360+
}
352361
}
353362
}
354363
}
@@ -587,6 +596,22 @@ pipeline {
587596
export OS_PKG_TYPE="${os.pkg_type}"
588597
if [ "$THROWAWAY" != "true" ]; then ./scripts/chacra_upload.sh; fi
589598
"""
599+
600+
sh """#!/bin/bash
601+
if [ "$THROWAWAY" != "true" ] && [ "$PULP_UPLOAD" == "true" ]; then
602+
export CEPH_VERSION="${env.VERSION}"
603+
export OS_NAME="${os.name}"
604+
export OS_VERSION="${os.version}"
605+
export OS_VERSION_NAME="${os.version_name}"
606+
export OS_PKG_TYPE="${os.pkg_type}"
607+
export SHA1="${env.SHA1}"
608+
export FLAVOR="${env.FLAVOR}"
609+
610+
./scripts/pulp_upload.sh
611+
else
612+
echo "Skipping pulp upload because PULP_UPLOAD=$PULP_UPLOAD"
613+
fi
614+
"""
590615
}
591616
}
592617
post {

‎ceph-dev-pipeline/config/definitions/ceph-dev-pipeline.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,11 @@
6464
description: "Whether to push new binaries to Chacra if some are already present"
6565
default: false
6666

67+
- bool:
68+
name: PULP_UPLOAD
69+
description: "Whether to upload packages to Pulp"
70+
default: false
71+
6772
- choice:
6873
name: FLAVOR
6974
choices:

‎scripts/pulp_upload.sh‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
#!/bin/bash
2+
set -ex
3+
4+
log() {
5+
echo "[pulp_upload] $*" >&2
6+
}
7+
8+
PULP_PROJECT="ceph"
9+
SHORT_SHA1=${SHA1: -8}
10+
11+
BRANCH="main"
12+
if [[ $CEPH_VERSION =~ ^20.* ]]; then
13+
BRANCH="tentacle"
14+
elif [[ $CEPH_VERSION =~ ^19.* ]]; then
15+
BRANCH="squid"
16+
elif [[ $CEPH_VERSION =~ ^18.* ]]; then
17+
BRANCH="reef"
18+
elif [[ $CEPH_VERSION =~ ^17.* ]]; then
19+
BRANCH="quincy"
20+
elif [[ $CEPH_VERSION =~ ^16.* ]]; then
21+
BRANCH="pacific"
22+
else:
23+
log "ERROR: Unsupported version '${VERSION}'"
24+
exit 1
25+
fi
26+
27+
REPO_NAME="${PULP_PROJECT}-${BRANCH}-${OS_NAME}-${OS_VERSION}-${ARCH}"
28+
REPO_ENDPOINT="${PULP_PROJECT}/${BRANCH}/${SHA1}/${OS_NAME}/${OS_VERSION}"
29+
REPO_ENDPOINT="${REPO_ENDPOINT}/flavors/${FLAVOR}/${ARCH}"
30+
31+
log "Starting Pulp upload (OS_PKG_TYPE=${OS_PKG_TYPE})"
32+
log "Repository: ${REPO_NAME}; base_path: ${REPO_ENDPOINT}"
33+
34+
if [ "$OS_PKG_TYPE" = "rpm" ]; then
35+
rpm_release=$(grep Release dist/ceph/ceph.spec | \
36+
sed 's/Release:[ \t]*//g' | cut -d '%' -f 1)
37+
rpm_version=$(grep Version dist/ceph/ceph.spec | \
38+
sed 's/Version:[ \t]*//g')
39+
40+
PACKAGE_MANAGER_VERSION="$rpm_version-$rpm_release"
41+
log "Package version label (RPM): ${PACKAGE_MANAGER_VERSION}"
42+
43+
log "Discovering RPM artifacts under" \
44+
"${WORKSPACE}/dist/ceph/rpmbuild/ (SRPMS, RPMS)"
45+
while IFS= read -r rpm_file; do
46+
[ -z "$rpm_file" ] && continue
47+
pulp rpm content -t package upload --repository "${REPO_NAME}" \
48+
--file "$rpm_file"
49+
done < <(
50+
find "${WORKSPACE}/dist/ceph/rpmbuild/SRPMS" \
51+
"${WORKSPACE}/dist/ceph/rpmbuild/RPMS" \
52+
-name "*.rpm"
53+
)
54+
log "Finished RPM uploads (repository=${REPO_NAME})"
55+
elif [ "$OS_PKG_TYPE" = "deb" ]; then
56+
PACKAGE_MANAGER_VERSION="${VERSION}-1${OS_VERSION_NAME}"
57+
log "Package version label (DEB): ${PACKAGE_MANAGER_VERSION}"
58+
59+
log "Discovering DEB artifacts under ${WORKSPACE}/dist/ceph/" \
60+
"(.changes, .deb, .ddeb, .dsc, .gz)"
61+
while IFS= read -r deb_file; do
62+
[ -z "$deb_file" ] && continue
63+
pulp deb content -t package upload --repository "${REPO_NAME}" \
64+
--file "$deb_file"
65+
done < <(
66+
find "${WORKSPACE}/dist/ceph/" -regextype egrep \
67+
-regex ".*(\.changes|\.deb|\.ddeb|\.dsc|ceph.*\.gz)$" |
68+
grep -vE "(Packages|Sources|Contents)" || true
69+
)
70+
log "Finished DEB uploads (repository=${REPO_NAME})"
71+
else
72+
log "ERROR: Unsupported OS_PKG_TYPE='${OS_PKG_TYPE}' (expected rpm or deb)"
73+
exit 1
74+
fi
75+
76+
FINAL_VERSION=$(pulp "${OS_PKG_TYPE}" repository version list \
77+
--repository "$REPO_NAME" --limit 1 | jq -r '.[0].number')
78+
log "Found final version ${FINAL_VERSION}"
79+
80+
if [ "$OS_PKG_TYPE" == "deb" ]; then
81+
PUB_HREF=$(
82+
pulp deb publication create --repository "$REPO_NAME" \
83+
--version "$FINAL_VERSION" --simple "true" --structured "false" \
84+
| jq -r '.pulp_href'
85+
)
86+
log "Created DEB publication ${PUB_HREF}"
87+
else
88+
PUB_HREF=$(
89+
pulp "${OS_PKG_TYPE}" publication create --repository "$REPO_NAME" \
90+
--version "$FINAL_VERSION" | jq -r '.pulp_href'
91+
)
92+
log "Created RPM publication ${PUB_HREF}"
93+
fi
94+
95+
DIST_NAME="dist-${REPO_NAME}-${SHORT_SHA1}"
96+
LABELS="project=${PULP_PROJECT},"
97+
LABELS="${LABELS}version=${PACKAGE_MANAGER_VERSION},"
98+
LABELS="${LABELS}ref=${BRANCH},"
99+
LABELS="${LABELS}arch=${ARCH}"
100+
LABELS="${LABELS},sha1=${SHA1}"
101+
102+
log "Creating distribution ${DIST_NAME} " \
103+
"with base_path=${REPO_ENDPOINT} labels=${LABELS}"
104+
pulp "${OS_PKG_TYPE}" distribution create \
105+
--name "${DIST_NAME}" \
106+
--repository "${REPO_NAME}" \
107+
--base-path "${REPO_ENDPOINT}" \
108+
--publication "${PUB_HREF}"
109+
110+
log "Updating distribution ${DIST_NAME} with publication" \
111+
"publication=${PUB_HREF} and labels=${LABELS}"
112+
pulp "${OS_PKG_TYPE}" distribution update \
113+
--name "${DIST_NAME}" \
114+
--publication "${PUB_HREF}" \
115+
--labels "${LABELS}"
116+
117+
log "Pulp upload and distribution publish completed (repository=${REPO_NAME})"

‎scripts/setup_pulp.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/bash -ex
2+
3+
echo "Setting up Pulp client"
4+
source "$WORKSPACE/scripts/setup_uv.sh"
5+
6+
PULP_SERVER_URL="https://pulp.apps.pok.os.sepia.ceph.com/pulp/api/v3/"
7+
8+
~/.local/bin/uv tool install pulp-cli --with pulp-cli-deb
9+
10+
echo "Configuring Pulp client"
11+
pulp config create --base-url "${PULP_SERVER_URL}" \
12+
--username "${PULP_USERNAME}" --password "${PULP_PASSWORD}" \
13+
--no-verify-ssl --overwrite
14+
15+
echo "Checking Pulp client status"
16+
if ! pulp status; then
17+
echo "Error: Pulp client is not configured correctly."
18+
exit 1
19+
fi
20+
21+
echo "Pulp client setup complete"

0 commit comments

Comments
 (0)