Do not crash if Slurm returns invalid remaining timelimit

Author: Kobzol

crates/hyperqueue/src/common/manager/slurm.rs

@@ -1,5 +1,6 @@
 use crate::common::manager::common::format_duration;
 use crate::common::utils::time::parse_hms_time;
+use anyhow::Context;
 use std::process::Command;
 use std::time::Duration;
 use tako::Map;
@@ -32,13 +33,15 @@ pub fn parse_remaining_timelimit(output: &str) -> anyhow::Result<Duration> {
         items
             .get("RunTime")
             .ok_or_else(|| anyhow::anyhow!("RunTime entry not found"))?,
-    )?;
+    )
+    .context("Cannot parse Slurm runtime")?;
 
     let time_limit = parse_slurm_duration(
         items
             .get("TimeLimit")
             .ok_or_else(|| anyhow::anyhow!("TimeLimit entry not found"))?,
-    )?;
+    )
+    .context("Cannot parse Slurm timelimit")?;
 
     if time_limit < run_time {
         anyhow::bail!("Slurm: TimeLimit is smaller than RunTime");

crates/hyperqueue/src/worker/bootstrap.rs

@@ -177,8 +177,13 @@ pub fn try_get_slurm_info() -> anyhow::Result<ManagerInfo> {
             anyhow!("SLURM_JOB_ID/SLURM_JOBID not found. The process is not running under SLURM")
         })?;
 
-    let duration = slurm::get_remaining_timelimit(&manager_job_id)
-        .expect("Could not get remaining time from scontrol");
+    let time_limit = match slurm::get_remaining_timelimit(&manager_job_id) {
+        Ok(duration) => Some(duration),
+        Err(error) => {
+            log::warn!("Cannot get remaining worker timelimit from Slurm: {error:?}");
+            None
+        }
+    };
 
     let max_memory_mb = std::env::var("SLURM_MEM_PER_NODE")
         .ok()
@@ -189,7 +194,7 @@ pub fn try_get_slurm_info() -> anyhow::Result<ManagerInfo> {
     Ok(ManagerInfo {
         manager: ManagerType::Slurm,
         allocation_id: manager_job_id,
-        time_limit: Some(duration),
+        time_limit,
         max_memory_mb,
     })
 }