diff --git a/comparisons/compareProducts.sh b/comparisons/compareProducts.sh index ff802222343..8ee0e754e01 100755 --- a/comparisons/compareProducts.sh +++ b/comparisons/compareProducts.sh @@ -1,5 +1,17 @@ #!/bin/bash +function RunedmEventSize() +{ + if [ -e "${1}.edmEventSize" ] ; then + cp "${1}.edmEventSize" "${2}" + else + case "$1" in + *.root ) edmEventSize -v "$1" > "$2" ;; + *.rntpl ) edmRNTupleTempStorage --average "$1" > "$2" ;; + esac + fi +} + fA=`echo $1` if [ ! -f "${fA}" ]; then echo ${fA} does not exist @@ -33,21 +45,12 @@ fi echo "Checking process ${procF} ${fA} and ${fB} with useUnpacked=${useUnpacked} (if above ${absMin} or ${dptMin}%):" case `basename $fA` in - step*.root) + step*.root | step*.rntpl ) ds=`date -u +%s.%N` os=os.${ds} - if [ -e ${fA}.edmEventSize ] ; then - cp ${fA}.edmEventSize ${os} & - else - edmEventSize -v ${fA} > ${os} & - fi - ns=ns.${ds} - if [ -e ${fB}.edmEventSize ] ; then - cp ${fB}.edmEventSize ${ns} & - else - edmEventSize -v ${fB} > ${ns} & - fi + RunedmEventSize ${fA} ${os} & + RunedmEventSize ${fB} ${ns} & wait grep ${procF} ${os} ${ns} | sed -e "s/${os}:/os /g;s/${ns}:/ns /g" | absMin=${absMin} dptMin=${dptMin} useUnpacked=${useUnpacked} awk -f compareProducts.awk diff --git a/pr_testing/run-pr-comparisons b/pr_testing/run-pr-comparisons index ae2e8bcb992..c94b3898d86 100755 --- a/pr_testing/run-pr-comparisons +++ b/pr_testing/run-pr-comparisons @@ -81,7 +81,14 @@ rm -f $WORKSPACE/ALL_DONE if [ "X$COMPARISON_RELEASE" = "X" ] ; then COMPARISON_RELEASE=$CMSSW_VERSION; fi if [ "$REAL_ARCH" = "" ] ; then REAL_ARCH="${RELVAL_REAL_ARCH}"; fi PR_NUM=$(echo ${PULL_REQUEST} | md5sum | sed 's| .*||' | cut -c27-33) -BASELINE_DIR=ib-baseline-tests/$COMPARISON_RELEASE/$COMPARISON_ARCH/$REAL_ARCH/matrix${TEST_FLAVOR}-results +if [ "${BASELINE_TEST_FLAVOR}" == "" ] ; then + BASELINE_TEST_FLAVOR="${TEST_FLAVOR}" + case "${TEST_FLAVOR}" in + rntuple ) BASELINE_TEST_FLAVOR="";; + * ) ;; + esac +fi +BASELINE_DIR=ib-baseline-tests/$COMPARISON_RELEASE/$COMPARISON_ARCH/$REAL_ARCH/matrix${BASELINE_TEST_FLAVOR}-results PR_BASELINE_JOBDIR=pull-request-integration/${UPLOAD_UNIQ_ID} PR_BASELINE_DIR=${PR_BASELINE_JOBDIR}/runTheMatrix${UC_TEST_FLAVOR}-results JENKINS_ARTIFACTS_URL="https://cmssdt.cern.ch/SDT/jenkins-artifacts" @@ -155,11 +162,19 @@ echo "Done Downloading `date`" ${NANO_TEST} && process_nano nano $WORKSPACE/data/PR-${PR_NUM} set +x -for r in $(find $WORKSPACE/data/PR-${PR_NUM} -name 'step*.root' -type f | grep -v 'inDQM.root') ; do +for r in $(find $WORKSPACE/data/PR-${PR_NUM} -name 'step*.root' -o -name 'step*.rntpl' -type f | grep -v 'inDQM.root') ; do while [ $(jobs -p | wc -l) -ge ${NUM_PROC} ] ; do sleep 0.1 ; done echo "Running edmEventSize and edmProvDump: $r" - (edmEventSize -v $r > $r.edmEventSize || true) & - (edmProvDump $r > $r.edmProvDump || true) & + case $r in + *.rntpl ) + (edmRNTupleTempStorage --average $r > $r.edmEventSize || true) & + (edmRNTupleTempProvDump $r > $r.edmProvDump || true) & + ;; + *.root ) + (edmEventSize -v $r > $r.edmEventSize || true) & + (edmProvDump $r > $r.edmProvDump || true) & + ;; + esac done set -x jobs @@ -281,12 +296,20 @@ fi echo "DQM_PID: ${DQM_PID}" #Run product size comparison script for Phase2 workflow -mkdir compareProducts && pushd compareProducts +mkdir compareProducts +pushd compareProducts cp $CMS_BOT_DIR/comparisons/compareProducts.* ./ - ./compareProducts.sh $WORKSPACE/data/PR-$PR_NUM/136.874_*/step3.root $WORKSPACE/data/$COMPARISON_RELEASE/136.874_*/step3.root _ 100 10 > products_AOD.log || true - ./compareProducts.sh $WORKSPACE/data/PR-$PR_NUM/136.874_*/step3_inMINIAOD.root $WORKSPACE/data/$COMPARISON_RELEASE/136.874_*/step3_inMINIAOD.root _ 100 10 > products_MINIAOD.log || true - mv products_AOD.log $WORKSPACE/upload/products_AOD.log - mv products_MINIAOD.log $WORKSPACE/upload/products_MINIAOD.log + for wf in 136.874 ; do + for step in step3 ; do + for stype in "" "_inMINIAOD" ; do + fA=$(find $WORKSPACE/data/PR-$PR_NUM/${wf}_* -name ${step}${stype}.root -o -name ${step}${stype}.rntpl -type f | grep "/${step}.") + fB=$(find $WORKSPACE/data/$COMPARISON_RELEASE/${wf}_* -name ${step}${stype}.root -o -name ${step}${stype}.rntpl -type f | grep "/${step}.") + aod_log="${stype}" + [ "${stype}" == "" ] && aod_log="_AOD" + ./compareProducts.sh "$fA" "$fB" _ 100 10 > ${WORKSPACE}/upload/products${aod_log}.log || true + done + done + done popd set +x diff --git a/pr_testing/test_multiple_prs.sh b/pr_testing/test_multiple_prs.sh index c97d10150e6..d80f10bf11c 100755 --- a/pr_testing/test_multiple_prs.sh +++ b/pr_testing/test_multiple_prs.sh @@ -1528,7 +1528,14 @@ if [ "X$DO_SHORT_MATRIX" = Xtrue ]; then if [ $(echo ${ENABLE_BOT_TESTS} | tr ',' ' ' | tr ' ' '\n' | grep "^${uc_tn}$" | wc -l) -gt 0 ] ; then prop_file="$WORKSPACE/run-relvals-${tn}.prop" cp $WORKSPACE/test-env.txt ${prop_file} - echo "DO_COMPARISON=false" >> ${prop_file} + if [ "${tn}" == "rntuple" ] ; then + echo "DO_COMPARISON=$DO_COMPARISON" >> ${prop_file} + echo "COMPARISON_REL=${COMPARISON_REL}" >> ${prop_file} + echo "COMPARISON_ARCH=${COMPARISON_ARCH}" >> ${prop_file} + echo "REAL_ARCH=${RELVAL_REAL_ARCH}" >> ${prop_file} + else + echo "DO_COMPARISON=false" >> ${prop_file} + fi echo "MATRIX_TIMEOUT=$MATRIX_TIMEOUT" >> ${prop_file} WF1=$(echo "${WF_COMMON}" | sed 's|;.*||') WF2="$(get_pr_relval_args $DO_COMPARISON _${uc_tn} | sed 's|.*;||')"