Lazy load journal and account report pages

frontend/src/api/index.ts

@@ -224,7 +224,7 @@ const filters_conversion_interval = [
 export const get_account_report = define_endpoint(
   "account_report",
   account_report_validator,
-  [...filters_conversion_interval, "a", "r"],
+  [...filters_conversion_interval, "a", "r", "page"],
 );
 export const get_balance_sheet = define_endpoint(
   "balance_sheet",

frontend/src/api/validators.ts

@@ -168,6 +168,7 @@ export const tree_report_validator = object({
 export const account_report_validator = object({
   charts: chart_validator,
   journal: optional(string),
+  total_pages: optional(number),
   dates: optional(array(date_range)),
   interval_balances: optional(array(account_hierarchy_validator)),
   budgets: optional(record(array(account_budget))),

frontend/src/reports/accounts/AccountReport.svelte

@@ -16,6 +16,8 @@
     report_type,
     charts,
     journal,
+    load_page,
+    total_pages,
     interval_balances,
     dates,
     budgets,
@@ -82,6 +84,8 @@
   {#if report_type === "journal" && journal != null}
     <JournalTable
       {journal}
+      {load_page}
+      {total_pages}
       initial_sort={["date", "desc"]}
       show_change_and_balance={true}
     />

frontend/src/reports/accounts/index.ts

@@ -6,6 +6,8 @@ import type { NonRelativeUrlPathError } from "../../helpers.ts";
 import { getUrlPath } from "../../helpers.ts";
 import { fragment_from_string } from "../../lib/dom.ts";
 import { err, ok, type Result } from "../../lib/result.ts";
+import { log_error } from "../../log.ts";
+import { notify_err } from "../../notifications.ts";
 import { getURLFilters } from "../../stores/filters.ts";
 import { Route } from "../route.ts";
 import AccountReport from "./AccountReport.svelte";
@@ -20,6 +22,8 @@ export interface AccountReportProps {
   report_type: AccountReportType;
   charts: ParsedFavaChart[];
   journal: DocumentFragment | null;
+  load_page: ((page: number) => Promise<DocumentFragment | null>) | null;
+  total_pages: number;
   interval_balances: AccountTreeNode[] | null;
   dates: { begin: Date; end: Date }[] | null;
   budgets: Record<string, AccountBudget[]> | null;
@@ -52,16 +56,44 @@ export const account_report = new Route<AccountReportProps>(
   async (url) => {
     const account = get_account_from_url(url).unwrap();
     const report_type = to_report_type(url.searchParams.get("r"));
-    const { charts, journal, interval_balances, dates, budgets } =
+    const filters = getURLFilters(url);
+    const { charts, journal, total_pages, interval_balances, dates, budgets } =
       await get_account_report({
-        ...getURLFilters(url),
+        ...filters,
         a: account,
         r: report_type,
+        page: 1,
       });
 
+    let error_shown = false;
+    const load_page = async (page: number) => {
+      return get_account_report({
+        ...filters,
+        a: account,
+        r: report_type,
+        page,
+      }).then(
+        (res) =>
+          res.journal != null ? fragment_from_string(res.journal) : null,
+        (error: unknown) => {
+          log_error(
+            `Failed to fetch account journal page ${page.toString()}`,
+            error,
+          );
+          if (!error_shown) {
+            notify_err(new Error("Failed to fetch some account journal pages"));
+            error_shown = true;
+          }
+          return null;
+        },
+      );
+    };
+
     return {
       charts,
       journal: journal != null ? fragment_from_string(journal) : null,
+      load_page: journal != null ? load_page : null,
+      total_pages: journal != null ? (total_pages ?? 1) : 1,
       interval_balances,
       dates,
       budgets,

frontend/src/reports/journal/Journal.svelte

@@ -2,12 +2,14 @@
   import type { JournalReportProps } from "./index.ts";
   import JournalTable from "./JournalTable.svelte";
 
-  let { journal, initial_sort, all_pages }: JournalReportProps = $props();
+  let { journal, initial_sort, load_page, total_pages }: JournalReportProps =
+    $props();
 </script>
 
 <JournalTable
   {journal}
   {initial_sort}
-  {all_pages}
+  {load_page}
+  {total_pages}
   show_change_and_balance={false}
 />

frontend/src/reports/journal/JournalTable.svelte

@@ -20,17 +20,19 @@
   import { sort_journal } from "./sort.ts";
 
   interface Props {
-    all_pages?: Promise<DocumentFragment | null>[];
     initial_sort: JournalSort;
     journal: DocumentFragment;
+    load_page?: ((page: number) => Promise<DocumentFragment | null>) | null;
     show_change_and_balance: boolean;
+    total_pages?: number;
   }
 
   let {
-    all_pages = [],
     initial_sort,
     journal,
+    load_page,
     show_change_and_balance,
+    total_pages = 1,
   }: Props = $props();
 
   let ol: HTMLOListElement | undefined = $state();
@@ -97,7 +99,7 @@
   {ondragenter}
   {@attach (node: HTMLOListElement) => {
     void journal;
-    untrack(() => {
+    return untrack(() => {
       const sort = $journal_sort;
       node.innerHTML = "";
       node.append(journal);
@@ -107,21 +109,62 @@
       if (needs_sorting) {
         sort_journal(node, sort);
       }
-      if (all_pages.length > 0) {
-        loading_state
-          .run(async () => {
-            for (const page of all_pages) {
-              const fragment = await page;
-              if (fragment) {
-                node.append(fragment);
-              }
+      if (!load_page || total_pages <= 1) {
+        return undefined;
+      }
+
+      const load_journal_page = load_page;
+      const scroll_container = node.closest("article");
+      let next_page = 2;
+      let loading_next_page = false;
+
+      function load_if_near_bottom() {
+        const distance_to_bottom =
+          scroll_container == null
+            ? node.getBoundingClientRect().bottom - window.innerHeight
+            : scroll_container.scrollHeight -
+              scroll_container.scrollTop -
+              scroll_container.clientHeight;
+
+        if (distance_to_bottom < 1000) {
+          load_next_page().catch(log_error);
+        }
+      }
+
+      async function load_next_page() {
+        if (loading_next_page || next_page > total_pages) {
+          return;
+        }
+        loading_next_page = true;
+        try {
+          await loading_state.run(async () => {
+            const fragment = await load_journal_page(next_page);
+            next_page += 1;
+            if (fragment) {
+              node.append(fragment);
             }
-            if (needs_sorting) {
-              sort_journal(node, sort);
+            const current_sort = $journal_sort;
+            if (!shallow_equal(current_sort, initial_sort)) {
+              sort_journal(node, current_sort);
             }
-          })
-          .catch(log_error);
+          });
+        } finally {
+          loading_next_page = false;
+        }
+        requestAnimationFrame(load_if_near_bottom);
       }
+
+      const scroll_target = scroll_container ?? window;
+      scroll_target.addEventListener("scroll", load_if_near_bottom, {
+        passive: true,
+      });
+      window.addEventListener("resize", load_if_near_bottom);
+      requestAnimationFrame(load_if_near_bottom);
+
+      return () => {
+        scroll_target.removeEventListener("scroll", load_if_near_bottom);
+        window.removeEventListener("resize", load_if_near_bottom);
+      };
     });
   }}
 ></ol>

frontend/src/reports/journal/index.ts

@@ -1,4 +1,3 @@
-import { range } from "d3-array";
 import { get as store_get } from "svelte/store";
 
 import { get_journal_page } from "../../api/index.ts";
@@ -15,7 +14,8 @@ import Journal from "./Journal.svelte";
 export interface JournalReportProps {
   journal: DocumentFragment;
   initial_sort: JournalSort;
-  all_pages: Promise<DocumentFragment | null>[];
+  load_page: (page: number) => Promise<DocumentFragment | null>;
+  total_pages: number;
 }
 
 export const journal = new Route<JournalReportProps>(
@@ -34,8 +34,7 @@ export const journal = new Route<JournalReportProps>(
     });
 
     let error_shown = false;
-    const pages = range(2, total_pages + 1);
-    const all_pages = pages.map(async (page) => {
+    const load_page = async (page: number) => {
       return get_journal_page({ ...filters, page, order }).then(
         (res) => fragment_from_string(res.journal),
         (error: unknown) => {
@@ -47,12 +46,13 @@ export const journal = new Route<JournalReportProps>(
           return null;
         },
       );
-    });
+    };
 
     return {
       journal: fragment_from_string(journal),
       initial_sort: ["date", order],
-      all_pages,
+      load_page,
+      total_pages,
     };
   },
   () => _("Journal"),

src/fava/core/__init__.py

@@ -254,7 +254,7 @@ def account_is_closed(self, account_name: str) -> bool:
     def paginate_journal(
         self,
         page: int,
-        per_page: int = 1000,
+        per_page: int = 500,
         order: Literal["asc", "desc"] = "desc",
     ) -> JournalPage | None:
         """Get entries for a journal page with pagination info.

src/fava/core/tree.py

@@ -248,10 +248,12 @@ def cap(self, options: BeancountOptions) -> None:
 
         unrealized_gains = -self.get("").balance_children
         # Insert unrealized gains.
-        self.insert(
-            equity + ":" + options["account_unrealized_gains"],
-            unrealized_gains,
-        )
+        unrealized_gains_account = options.get("account_unrealized_gains")
+        if unrealized_gains_account is not None:
+            self.insert(
+                equity + ":" + unrealized_gains_account,
+                unrealized_gains,
+            )
 
         # Transfer Income and Expenses
         self.insert(

src/fava/json_api.py

@@ -15,6 +15,7 @@
 from http import HTTPStatus
 from inspect import Parameter
 from inspect import signature
+from math import ceil
 from pathlib import Path
 from pprint import pformat
 from typing import Any
@@ -757,6 +758,7 @@ class AccountReportJournal:
 
     charts: Sequence[ChartData]
     journal: str
+    total_pages: int
 
 
 @dataclass(frozen=True)
@@ -776,6 +778,7 @@ def get_account_report() -> AccountReportJournal | AccountReportTree:
 
     account_name = request.args.get("a", "")
     subreport = request.args.get("r")
+    page_number = int(request.args.get("page", "1"))
 
     charts = [
         ChartApi.account_balance(account_name),
@@ -842,17 +845,32 @@ def get_account_report() -> AccountReportJournal | AccountReportTree:
     journal_table_contents = get_template_attribute(
         "_journal_table.html", "journal_table_contents"
     )
-    entries = reversed(
-        g.ledger.account_journal(
-            g.filtered,
-            account_name,
-            g.conv,
-            with_children=g.ledger.fava_options.account_journal_include_children,
+    all_entries = list(
+        reversed(
+            g.ledger.account_journal(
+                g.filtered,
+                account_name,
+                g.conv,
+                with_children=(
+                    g.ledger.fava_options.account_journal_include_children
+                ),
+            )
         )
     )
+    per_page = 500
+    total_pages = max(1, ceil(len(all_entries) / per_page))
+    if page_number > total_pages:
+        raise NotFoundError
+    page_entries = all_entries[
+        (page_number - 1) * per_page : page_number * per_page
+    ]
     return AccountReportJournal(
         charts,
-        journal=journal_table_contents(entries, show_change_and_balance=True),
+        journal=journal_table_contents(
+            page_entries,
+            show_change_and_balance=True,
+        ),
+        total_pages=total_pages,
     )
 
 

tests/__snapshots__/test_json_api-test_api-account_report_off_by_one_journal.json

@@ -39,5 +39,6 @@
       "type": "bar"
     }
   ],
+  "total_pages": 1,
   "journal": "<li class='transaction  cleared'>\n        <p>\n        <span class='datecell' data-sort-value='4'><a href='#context-ENTRY_HASH'>2022-01-01</a></span>\n        <span class='flag'>*</span>\n        <span class='description'>\n            <strong class='payee'></strong>Buy\n            \n        </span>\n        <span class='indicators'><span></span><span></span></span>\n            <span class='change num'><span title='USD'>1 USD</span><br></span>\n          <span class='num'><span title='USD'>101 USD</span><br></span>\n        </p>\n        \n          <ul class='postings'>\n        <li>\n            <p>\n                <span class='datecell'></span>\n                <span class='flag'></span>\n                <span class='description'><a href='/off-by-one/account/Assets:Cash/?conversion=at_value&amp;interval=day'>Assets:Cash</a></span>\n                <span class='num'>-100 USD</span>\n                <span class='num'> </span>\n                <span class='num'></span>\n            </p>\n            \n        </li>\n        <li>\n            <p>\n                <span class='datecell'></span>\n                <span class='flag'></span>\n                <span class='description'><a href='/off-by-one/account/Assets:Commodity/?conversion=at_value&amp;interval=day'>Assets:Commodity</a></span>\n                <span class='num'>1 COM</span>\n                <span class='num'>100 USD, 2022-01-01</span>\n                <span class='num'></span>\n            </p>\n            \n        </li>\n        </ul>\n    </li><li class='transaction  cleared'>\n        <p>\n        <span class='datecell' data-sort-value='3'><a href='#context-ENTRY_HASH'>2022-01-01</a></span>\n        <span class='flag'>*</span>\n        <span class='description'>\n            <strong class='payee'></strong>Transfer\n            \n        </span>\n        <span class='indicators'><span></span><span></span></span>\n            <span class='change num'><span title='USD'>100 USD</span><br></span>\n          <span class='num'><span title='USD'>100 USD</span><br></span>\n        </p>\n        \n          <ul class='postings'>\n        <li>\n            <p>\n                <span class='datecell'></span>\n                <span class='flag'></span>\n                <span class='description'><a href='/off-by-one/account/Assets:Cash/?conversion=at_value&amp;interval=day'>Assets:Cash</a></span>\n                <span class='num'>100 USD</span>\n                <span class='num'> </span>\n                <span class='num'></span>\n            </p>\n            \n        </li>\n        <li>\n            <p>\n                <span class='datecell'></span>\n                <span class='flag'></span>\n                <span class='description'><a href='/off-by-one/account/Income:All/?conversion=at_value&amp;interval=day'>Income:All</a></span>\n                <span class='num'>-100 USD</span>\n                <span class='num'> </span>\n                <span class='num'></span>\n            </p>\n            \n        </li>\n        </ul>\n    </li><li class='open  '>\n        <p>\n        <span class='datecell' data-sort-value='1'><a href='#context-ENTRY_HASH'>2022-01-01</a></span>\n        <span class='flag'>Open</span>\n        <span class='description'>\n            <a href='/off-by-one/account/Assets:Cash/?conversion=at_value&amp;interval=day'>Assets:Cash</a>\n        </span>\n        <span class='indicators'></span>\n          <span class='num'></span>\n        </p>\n        \n    </li><li class='open  '>\n        <p>\n        <span class='datecell' data-sort-value='0'><a href='#context-ENTRY_HASH'>2022-01-01</a></span>\n        <span class='flag'>Open</span>\n        <span class='description'>\n            <a href='/off-by-one/account/Assets:Commodity/?conversion=at_value&amp;interval=day'>Assets:Commodity</a>\n        </span>\n        <span class='indicators'></span>\n          <span class='num'></span>\n        </p>\n        \n    </li>"
 }

tests/test_core_tree.py

@@ -1,10 +1,12 @@
 from __future__ import annotations
 
+from typing import cast
 from typing import TYPE_CHECKING
 
 from fava.core.tree import Tree
 
 if TYPE_CHECKING:  # pragma: no cover
+    from fava.beans.types import BeancountOptions
     from fava.core import FavaLedger
 
     from .conftest import SnapshotFunc
@@ -55,3 +57,13 @@ def test_tree_cap(example_ledger: FavaLedger, snapshot: SnapshotFunc) -> None:
     tree.cap(example_ledger.options)
 
     snapshot({n.name: n.balance.to_strings() for n in tree.values()})
+
+
+def test_tree_cap_without_unrealized_gains_account(
+    example_ledger: FavaLedger,
+) -> None:
+    tree = Tree(example_ledger.all_entries)
+    options = dict(example_ledger.options)
+    options["account_unrealized_gains"] = None
+
+    tree.cap(cast("BeancountOptions", options))