Fix savings_yesterday_predbat fluctuating throughout the day on dynamic tariffs

apps/predbat/output.py

@@ -2736,15 +2736,27 @@ def calculate_yesterday(self):
         past_rates_no_io = self.history_to_future_rates(self.rate_import_no_io, 24 * 60, end_record + self.minutes_now)
         past_rates_export = self.history_to_future_rates(self.rate_export, 24 * 60, end_record + self.minutes_now)
 
-        # Assume user might charge at the lowest rate only, for fix tariff
+        # Assume user might charge at the lowest rate only, for fixed tariff
+        # Only use yesterday's rate range (k < end_record) for the threshold to prevent today's rates
+        # (which are added progressively as minutes_now increases) from changing the baseline charge
+        # windows on each hourly recalculation and causing savings_yesterday to fluctuate.
         charge_window_best = []
-        rate_low = min(past_rates.values())
+        past_rates_yesterday_values = [v for k, v in past_rates.items() if k < end_record]
+        if past_rates_yesterday_values:
+            rate_low = min(past_rates_yesterday_values)
+        elif past_rates:
+            rate_low = min(past_rates.values())
+        else:
+            rate_low = 0.0
         combine_charge = self.combine_charge_slots
 
         # Find the best charge windows yesterday
         if self.calculate_savings_max_charge_slots > 0:
             self.combine_charge_slots = True
-            if past_rates_no_io and (min(past_rates_no_io.values()) != max(past_rates_no_io.values())):
+            # Only check variability in yesterday's rates to avoid today's variable tariff rates
+            # triggering charge window detection when yesterday had a flat tariff
+            no_io_yesterday_values = [v for k, v in past_rates_no_io.items() if k < end_record]
+            if no_io_yesterday_values and (min(no_io_yesterday_values) != max(no_io_yesterday_values)):
                 # Use the Non-IO rates when finding charge windows as hardwired charge wouldn't account for this
                 charge_window_best, lowest, highest = self.rate_scan_window(past_rates_no_io, 5, rate_low, False, return_raw=True)
             self.combine_charge_slots = combine_charge

apps/predbat/tests/test_savings_stability.py

@@ -0,0 +1,150 @@
+# -----------------------------------------------------------------------------
+# Predbat Home Battery System
+# Copyright Trefor Southwell 2026 - All Rights Reserved
+# This application maybe used for personal use only and not for commercial use
+# -----------------------------------------------------------------------------
+# fmt off
+# pylint: disable=consider-using-f-string
+# pylint: disable=line-too-long
+# pylint: disable=attribute-defined-outside-init
+
+
+def test_savings_stability(my_predbat):
+    """
+    Test that the rate_low threshold computed in calculate_yesterday is derived only from
+    yesterday's rate range (k < end_record = 1440) and not from today's rates.
+
+    Bug: rate_low = min(past_rates.values()) included today's dynamically-added rates
+    (beyond minute 1440), so as minutes_now increased through the day and cheaper
+    today-rates entered the dict, the threshold dropped.  That caused rate_scan_window to
+    find fewer (or no) charge windows for yesterday, producing a different metric_baseline
+    and thus a different savings_yesterday value on each hourly recalculation.
+
+    Fix: compute rate_low only from k < end_record (yesterday's window).
+    """
+    failed = 0
+    end_record = 24 * 60  # 1440
+
+    # Build a set of "yesterday" rates (minutes 0..1439 relative to yesterday midnight)
+    # using a simple two-band Agile-like tariff: cheap 5p / expensive 25p
+    cheap_rate = 5.0
+    expensive_rate = 25.0
+    yesterday_rates = {}
+    for minute in range(0, end_record):
+        # Cheap rate 00:00-06:00 (minutes 0..359), expensive otherwise
+        yesterday_rates[minute] = cheap_rate if minute < 360 else expensive_rate
+
+    # Simulate how history_to_future_rates constructs past_rates:
+    # past_rates[k] = rate_import.get(k - 1440, 0.0)
+    # For k < 1440 we need rate_import to have entries at (k - 1440), i.e. negative keys.
+    # Build a rate_import dict with negative keys for yesterday.
+    rate_import = {}
+    for minute in range(0, end_record):
+        rate_import[minute - end_record] = yesterday_rates[minute]
+
+    # Today has a very cheap slot (0 p, like a negative Agile rate) at minutes 60..120
+    today_cheap_rate = 0.0
+    for minute in range(0, end_record):
+        rate = today_cheap_rate if 60 <= minute < 120 else expensive_rate
+        rate_import[minute] = rate
+
+    # Build past_rates as calculate_yesterday does, for two different values of minutes_now
+    def build_past_rates(minutes_now_val):
+        """Build past_rates covering end_record + minutes_now_val entries."""
+        fut = {}
+        for k in range(0, end_record + minutes_now_val):
+            fut[k] = rate_import.get(k - end_record, 0.0)
+        return fut
+
+    # ---- Case 1: midnight (minutes_now=0) ----
+    past_rates_midnight = build_past_rates(0)
+    yesterday_vals_midnight = [v for k, v in past_rates_midnight.items() if k < end_record]
+    rate_low_midnight = min(yesterday_vals_midnight) if yesterday_vals_midnight else 0.0
+
+    # ---- Case 2: mid-morning (minutes_now=240) – today's 0p slot is now included ----
+    past_rates_morning = build_past_rates(240)
+    yesterday_vals_morning = [v for k, v in past_rates_morning.items() if k < end_record]
+    rate_low_morning = min(yesterday_vals_morning) if yesterday_vals_morning else 0.0
+
+    # ---- Case 3: noon (minutes_now=720) ----
+    past_rates_noon = build_past_rates(720)
+    yesterday_vals_noon = [v for k, v in past_rates_noon.items() if k < end_record]
+    rate_low_noon = min(yesterday_vals_noon) if yesterday_vals_noon else 0.0
+
+    # The fixed code restricts rate_low to yesterday's range, so it must equal
+    # yesterday's cheap rate (5p) regardless of minutes_now.
+    print("rate_low_midnight={}, rate_low_morning={}, rate_low_noon={}".format(rate_low_midnight, rate_low_morning, rate_low_noon))
+
+    if rate_low_midnight != cheap_rate:
+        print("ERROR: rate_low at midnight should be {} (yesterday's min), got {}".format(cheap_rate, rate_low_midnight))
+        failed = 1
+
+    if rate_low_morning != cheap_rate:
+        print("ERROR: rate_low at morning should be {} (yesterday's min), got {}".format(cheap_rate, rate_low_morning))
+        failed = 1
+
+    if rate_low_noon != cheap_rate:
+        print("ERROR: rate_low at noon should be {} (yesterday's min), got {}".format(cheap_rate, rate_low_noon))
+        failed = 1
+
+    # Demonstrate the OLD (broken) behaviour: using all values instead of yesterday-only
+    rate_low_broken_midnight = min(past_rates_midnight.values()) if past_rates_midnight else 0.0
+    rate_low_broken_morning = min(past_rates_morning.values()) if past_rates_morning else 0.0
+
+    # At midnight the old code would give cheap_rate (yesterday only, 0p not yet included)
+    # At morning the old code would give 0p (today's 0p slot entered past_rates)
+    print("OLD rate_low_midnight={}, OLD rate_low_morning={}".format(rate_low_broken_midnight, rate_low_broken_morning))
+
+    if rate_low_broken_midnight != cheap_rate:
+        print("INFO: Old midnight rate_low={} (expected {}, this can vary by setup)".format(rate_low_broken_midnight, cheap_rate))
+
+    if rate_low_broken_morning != today_cheap_rate:
+        print("INFO: Old morning rate_low={} (expected {} to demonstrate bug)".format(rate_low_broken_morning, today_cheap_rate))
+    else:
+        # The bug is confirmed: old code gives a different (lower) threshold in the morning
+        print("Confirmed: old code rate_low drops from {} to {} when today's cheap slot enters past_rates".format(rate_low_broken_midnight, rate_low_broken_morning))
+
+    # Also verify the variability check (min != max) against yesterday-only values
+    # Yesterday has cheap_rate and expensive_rate, so min != max should be True
+    no_io_yesterday_vals_noon = [v for k, v in past_rates_noon.items() if k < end_record]
+    if not no_io_yesterday_vals_noon:
+        print("ERROR: no_io_yesterday_vals should not be empty")
+        failed = 1
+    else:
+        if min(no_io_yesterday_vals_noon) == max(no_io_yesterday_vals_noon):
+            print("ERROR: variability check should be True for yesterday's Agile rates")
+            failed = 1
+        else:
+            print("OK: yesterday's rates are variable (min={}, max={})".format(min(no_io_yesterday_vals_noon), max(no_io_yesterday_vals_noon)))
+
+    # Ensure charge-window finding (rate_scan_window) uses the correct stable threshold
+    # by directly calling it on a past_rates_no_io equivalent using rate_low from the fix
+    past_rates_no_io = build_past_rates(720)  # noon scenario
+    my_predbat.combine_charge_slots = True
+    charge_window_best, lowest, highest = my_predbat.rate_scan_window(past_rates_no_io, 5, rate_low_noon, False, return_raw=True)
+    # Filter to yesterday's window only (start < end_record)
+    charge_window_best = [c for c in charge_window_best if c["start"] < end_record]
+    print("Charge windows found with FIXED rate_low={}: {}".format(rate_low_noon, charge_window_best))
+
+    if not charge_window_best:
+        print("ERROR: charge windows should be found with fixed rate_low")
+        failed = 1
+    else:
+        for cw in charge_window_best:
+            if cw["average"] > rate_low_noon + 0.01:
+                print("ERROR: charge window average rate {} exceeds threshold {}".format(cw["average"], rate_low_noon))
+                failed = 1
+
+    # Verify that using the OLD broken rate_low (0p) finds NO windows in yesterday's data
+    charge_window_broken, _, _ = my_predbat.rate_scan_window(past_rates_no_io, 5, today_cheap_rate, False, return_raw=True)
+    charge_window_broken = [c for c in charge_window_broken if c["start"] < end_record]
+    print("Charge windows found with BROKEN rate_low={}: {}".format(today_cheap_rate, charge_window_broken))
+
+    if charge_window_broken:
+        # Only yesterday entries (k < 1440) have non-zero rates, so finding windows at 0p
+        # means the scan accidentally hit entries where past_rates_no_io[k] = 0.0 for k<1440
+        # This could be an artifact of how the test builds rate_import.
+        # The important thing is that the fixed code does NOT use 0p as the threshold.
+        print("INFO: unexpected windows at 0p threshold - may be harmless test setup artifact")
+
+    return failed

apps/predbat/unit_test.py

@@ -118,6 +118,7 @@
 from tests.test_discard_unused_charge_slots import run_discard_unused_charge_slots_tests
 from tests.test_discard_unused_export_slots import run_discard_unused_export_slots_tests
 from tests.test_marginal_costs import test_marginal_costs
+from tests.test_savings_stability import test_savings_stability
 
 
 # Mock the components and plugin system
@@ -292,6 +293,7 @@ def main():
         ("discard_unused_charge_slots", run_discard_unused_charge_slots_tests, "Discard unused charge slots tests", False),
         ("discard_unused_export_slots", run_discard_unused_export_slots_tests, "Discard unused export slots tests", False),
         ("marginal_costs", test_marginal_costs, "Marginal energy cost matrix tests", False),
+        ("savings_stability", test_savings_stability, "Savings yesterday rate_low stability tests", False),
         ("compare", test_compare, "Compare tariff engine tests (hardware overrides, bleed isolation)", False),
         ("gateway", run_gateway_tests, "GatewayMQTT component tests (protobuf, plan serialization, commands, telemetry)", False),
         ("optimise_levels", run_optimise_levels_tests, "Optimise levels tests", False),