File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -424,6 +424,31 @@ function wait_warmup_ready() {
424424 kubectl wait --for=condition=Ready --timeout=180s pod/${dataset_name} -warmup > /dev/null || panic " warmup pod ${dataset_name} -warmup is not ready"
425425}
426426
427+ function wait_single_mount_data_ready() {
428+ dataset_name=$1
429+ deadline=180
430+ elapsed=0
431+
432+ while true ; do
433+ found_file=$( kubectl exec " ${dataset_name} -warmup" -- /bin/sh -c ' find /data -maxdepth 3 -type f -name testfile 2>/dev/null | head -n 1 || true' 2> /dev/null)
434+ if [[ -n " $found_file " ]]; then
435+ value=$( kubectl exec " ${dataset_name} -warmup" -- /bin/sh -c " cat \" $found_file \" 2>/dev/null || true" 2> /dev/null)
436+ if [[ " $value " == " helloworld" ]]; then
437+ syslog " Found single-mount data ready in warmup pod at $found_file "
438+ break
439+ fi
440+ fi
441+
442+ elapsed=$( expr $elapsed + 5)
443+ if [[ " $elapsed " -ge " $deadline " ]]; then
444+ kubectl exec " ${dataset_name} -warmup" -- /bin/sh -c ' ls -al /data >&2 || true; find /data -maxdepth 3 \( -type d -o -type f \) >&2 || true' > /dev/null 2>&1 || true
445+ panic " timeout waiting for single-mount data to become visible in warmup pod"
446+ fi
447+
448+ sleep 5
449+ done
450+ }
451+
427452function create_job() {
428453 job_file=$1
429454 job_name=$2
@@ -476,6 +501,9 @@ function run_scenario() {
476501 create_warmup_pod $dataset_name
477502 wait_runtime_stable $dataset_name
478503 wait_warmup_ready $dataset_name
504+ if [[ " $dataset_name " == " $s3_dataset_name " ]]; then
505+ wait_single_mount_data_ready $dataset_name
506+ fi
479507 create_job $job_file $job_name
480508 wait_job_completed $job_name
481509 cleanup_scenario $dataset_file $dataset_name $job_file
You can’t perform that action at this time.
0 commit comments