diff --git a/RdmpCohortBuildHealthBoardBreakdown/INSTALL.md b/RdmpCohortBuildHealthBoardBreakdown/INSTALL.md new file mode 100644 index 0000000000..7c8984f2da --- /dev/null +++ b/RdmpCohortBuildHealthBoardBreakdown/INSTALL.md @@ -0,0 +1,62 @@ +# RdmpCohortBuildHealthBoardBreakdown plugin (RDMP 9.2.3) + +Reproduces the Cohort Builder's per-set / per-container count tree (the FinalCount and cumulative +running totals shown as UNION/INTERSECT/EXCEPT are applied) **split by region** (e.g. Scottish health +board), plus an unfiltered national total. Saved as a wide CSV. + +Built against the **released RDMP 9.2.3**. Do not use on a different major.minor RDMP. + +## How it works (cache-only, cross-server safe) + +It builds the cohort **once** (populating the query cache), then recomposes every count point from the +cached per-set identifier tables and splits each by the region column with one GROUP BY per node. It +never re-runs the source catalogues per region, and never touches the source servers after the single +build, only the query-cache server (which is why the demography catalogue must be on the same server as +the query cache). The region-to-name/node mapping is read from a user-supplied lookup table, so nothing +is hard-coded. + +## Requirements + +- The cohort identification configuration must have a **query caching server** configured (the + breakdown works only on cached results; it refuses otherwise). +- The **demography catalogue** (with a CHI IsExtractionIdentifier column and the region column) must be + on the **same SQL server as the query cache** (the command checks and refuses if not). +- A **lookup table** mapping each region code to a name and node, with columns: + `Region` (the code as it appears in the demography data), `HB_Name` (display name), `SafeHaven_Region` + (grouping node, may be NULL). An `HB_Code` column may be present but is ignored. One row per code. + +## Install + +**GUI:** RDMP desktop, Plugins node, *Add Plugin* (or drag `RdmpCohortBuildHealthBoardBreakdown.rdmp` +onto it), restart RDMP. **Or** drop the `.rdmp` next to `rdmp.exe` / +`ResearchDataManagementPlatform.exe`. + +Confirm (CLI): `rdmp.exe cmd ListSupportedCommands` lists `ExportCohortBuildHealthBoardBreakdown`. + +## Use + +**GUI:** right-click a Cohort Identification Configuration, choose the export command. It prompts for the +demography catalogue, the region column, and the lookup table. + +**CLI:** the inputs are RDMP objects, mapped by id: +``` +rdmp.exe cmd ExportCohortBuildHealthBoardBreakdown \ + CohortIdentificationConfiguration: Catalogue: ColumnInfo: TableInfo: out.csv +``` +`out.csv` and the timeout are optional; the object arguments are required (no hard-coded defaults). + +## Output (wide format) + +One row per count point (name written once), a `Metric` column (Final and Cumulative), a `Total` column +(RDMP's national number), one column per region recognised by the lookup, then `Other` (present codes the +lookup does not recognise) and `NotKnown` (not in demography, or NULL region). The column header is +repeated above two percentage rows: `% of final cohort` and `% of demography` (each region's share of the +whole demography population, a cohort-vs-population sanity check). Regions + Other + NotKnown reconcile to +Total on every row. + +## Validation + +Verified end-to-end on a deterministic synthetic fixture (top EXCEPT over an inclusion INTERSECT minus +four exclusion sets, cohort partitioned across regions, driven by a synthetic lookup table): every +national and per-region FinalCount and cumulative was asserted cell-by-cell, the unfiltered column equals +RDMP's own CohortCompiler counts, and the regions sum to national at every node. diff --git a/RdmpCohortBuildHealthBoardBreakdown/README.md b/RdmpCohortBuildHealthBoardBreakdown/README.md new file mode 100644 index 0000000000..16ebcfcd07 --- /dev/null +++ b/RdmpCohortBuildHealthBoardBreakdown/README.md @@ -0,0 +1,46 @@ +# RdmpCohortBuildHealthBoardBreakdown (RDMP 9.2.3 plugin) + +Reproduces the Cohort Builder's per-set / per-container count tree (the `FinalCount` and cumulative +running totals shown as UNION/INTERSECT/EXCEPT are applied) **split by region** (e.g. Scottish health +board), plus an unfiltered national total, and writes it to a wide CSV. The region-to-name/node mapping +is read from a user-supplied lookup table (nothing is hard-coded). + +This folder is a self-contained package: the ready-to-install plugin, install/usage notes, and the source. + +## Contents + +| Path | What it is | +|---|---| +| `RdmpCohortBuildHealthBoardBreakdown.rdmp` | the built plugin (drop into RDMP / add via the Plugins node) | +| `INSTALL.md` | install + usage (GUI right-click and CLI), including the lookup-table schema | +| `src/` | plugin source (command, report, region lookup, models, UI hook, csproj, nuspec) | + +## How it works (in one paragraph) + +It builds the national cohort once (which populates RDMP's query cache), then recomposes every count +point purely from the cached per-set identifier tables and splits each by the region column with one +`GROUP BY` per node, all regions at once. No per-region rebuild, and no hits on the source catalogues +after the single build (so it is cross-server safe). Inputs are RDMP objects (an `ICatalogue` for +demography, a `ColumnInfo` for the region column, a `TableInfo` for the lookup); requires a query-caching +server, with the demography catalogue on the same server as the cache. + +## Output + +Wide CSV: one row per count point (name once), a `Metric` column (Final + Cumulative), a `Total` +column (RDMP's national number), one column per region recognised by the lookup, then `Other` (present +codes the lookup does not recognise) and `NotKnown` (not in demography / null region). The column header +is repeated above a `% of final cohort` row and a `% of demography` row (each region's share of the whole +demography population, for a cohort-vs-population sanity check). Regions + Other + NotKnown reconcile to +Total on every row. + +## Validation + +Verified end-to-end against a deterministic synthetic fixture (top EXCEPT over an inclusion INTERSECT +minus four exclusion sets, driven by a synthetic lookup table): every national and per-region +`FinalCount` / cumulative is asserted cell-by-cell, the unfiltered column equals RDMP's own +`CohortCompiler` counts, and the regions sum to national at every node. + +## Build from source (optional) + +`src/` builds against RDMP 9.2.3 (`Rdmp.Core`, `Private=false`). Package the resulting DLL + nuspec into +a `.rdmp` zip (`` at root, DLL under `lib/net10.0/`). diff --git a/RdmpCohortBuildHealthBoardBreakdown/RdmpCohortBuildHealthBoardBreakdown.rdmp b/RdmpCohortBuildHealthBoardBreakdown/RdmpCohortBuildHealthBoardBreakdown.rdmp new file mode 100644 index 0000000000..35b27ee104 Binary files /dev/null and b/RdmpCohortBuildHealthBoardBreakdown/RdmpCohortBuildHealthBoardBreakdown.rdmp differ diff --git a/RdmpCohortBuildHealthBoardBreakdown/src/CohortBuildBreakdownModels.cs b/RdmpCohortBuildHealthBoardBreakdown/src/CohortBuildBreakdownModels.cs new file mode 100644 index 0000000000..5347eaeb91 --- /dev/null +++ b/RdmpCohortBuildHealthBoardBreakdown/src/CohortBuildBreakdownModels.cs @@ -0,0 +1,88 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using System.Collections.Generic; + +namespace Rdmp.Core.CohortCreation; + +/// +/// One count point of a cohort build tree (a cohort set or a container) with the per-region counts that +/// were computed for it. Used by . +/// +public sealed class CohortBuildBreakdownNode +{ + /// Tree walk order (stable, used to order rows). + public int Seq { get; } + + /// "Cohort Set" or "Container". + public string Type { get; } + + public string Name { get; } + + /// Name of the parent container (empty for the root). + public string Container { get; } + + /// UNION / INTERSECT / EXCEPT for containers; empty for sets. + public string SetOperation { get; } + + public int DisplayOrder { get; } + + /// RDMP's own count for this node (the unfiltered/national total). + public int FinalUnfiltered { get; } + + /// RDMP's own cumulative within the parent container; null if not applicable. + public int? CumulativeUnfiltered { get; } + + /// Region code -> final count (every present code; the GROUP BY Region result). + public IReadOnlyDictionary FinalByRegion { get; } + + /// Region code -> cumulative count; null when this node has no cumulative. + public IReadOnlyDictionary CumulativeByRegion { get; } + + public CohortBuildBreakdownNode(int seq, string type, string name, string container, string setOperation, + int displayOrder, int finalUnfiltered, int? cumulativeUnfiltered, + IReadOnlyDictionary finalByRegion, IReadOnlyDictionary cumulativeByRegion) + { + Seq = seq; + Type = type ?? ""; + Name = name ?? ""; + Container = container ?? ""; + SetOperation = setOperation ?? ""; + DisplayOrder = displayOrder; + FinalUnfiltered = finalUnfiltered; + CumulativeUnfiltered = cumulativeUnfiltered; + FinalByRegion = finalByRegion ?? new Dictionary(); + CumulativeByRegion = cumulativeByRegion; + } +} + +/// +/// The Total / per-region / Other / NotKnown split of one node+metric, relative to a +/// . holds the counts for codes present in the lookup; +/// sums present codes absent from the lookup; is the residual +/// (not in demography, or NULL region). +/// +public sealed class CohortBuildBreakdownBuckets +{ + public int Total { get; } + + /// Region code -> count, for codes recognised by the lookup. + public IReadOnlyDictionary Regions { get; } + + /// Sum of present region codes that the lookup does not recognise. + public int Other { get; } + + /// Total - recognised - Other = not in demography + NULL region. + public int NotKnown { get; } + + public CohortBuildBreakdownBuckets(int total, IReadOnlyDictionary regions, int other, int notKnown) + { + Total = total; + Regions = regions ?? new Dictionary(); + Other = other; + NotKnown = notKnown; + } +} diff --git a/RdmpCohortBuildHealthBoardBreakdown/src/CohortBuildHealthBoardBreakdownPluginUserInterface.cs b/RdmpCohortBuildHealthBoardBreakdown/src/CohortBuildHealthBoardBreakdownPluginUserInterface.cs new file mode 100644 index 0000000000..82c66c46c0 --- /dev/null +++ b/RdmpCohortBuildHealthBoardBreakdown/src/CohortBuildHealthBoardBreakdownPluginUserInterface.cs @@ -0,0 +1,24 @@ +// Surfaces the cohort build health board breakdown in the RDMP desktop GUI (right-click a cohort +// identification configuration). The same command class is auto-discovered for the CLI +// (`rdmp cmd ExportCohortBuildHealthBoardBreakdown`), so one definition serves both. + +using System.Collections.Generic; +using Rdmp.Core; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.Curation.Data.Cohort; + +namespace RdmpCohortBuildHealthBoardBreakdown; + +public class CohortBuildHealthBoardBreakdownPluginUserInterface : PluginUserInterface +{ + public CohortBuildHealthBoardBreakdownPluginUserInterface(IBasicActivateItems itemActivator) : base(itemActivator) + { + } + + public override IEnumerable GetAdditionalRightClickMenuItems(object o) + { + if (o is CohortIdentificationConfiguration cic) + yield return new ExecuteCommandExportCohortBuildHealthBoardBreakdown(BasicActivator, cic); + } +} diff --git a/RdmpCohortBuildHealthBoardBreakdown/src/CohortBuildHealthBoardBreakdownReport.cs b/RdmpCohortBuildHealthBoardBreakdown/src/CohortBuildHealthBoardBreakdownReport.cs new file mode 100644 index 0000000000..e56f509db6 --- /dev/null +++ b/RdmpCohortBuildHealthBoardBreakdown/src/CohortBuildHealthBoardBreakdownReport.cs @@ -0,0 +1,151 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; + +namespace Rdmp.Core.CohortCreation; + +/// +/// Projects a cohort build's count tree (the per-set / per-container FinalCount and cumulative +/// running totals shown in the Cohort Builder) split by region into a WIDE CSV: one row per +/// (count-point x metric), the container/set name written once, a Total column (RDMP's own +/// national count), one column per region recognised by the supplied , an +/// Other column (present codes the lookup does not recognise) and a NotKnown residual +/// (patients not in demography / NULL region). Two bottom rows give each region's share of the final +/// cohort and of the whole demography population (a sanity check). Regions + Other + NotKnown reconcile +/// to Total on every row. +/// +public static class CohortBuildHealthBoardBreakdownReport +{ + public const string OtherColumn = "Other"; + public const string NotKnownColumn = "NotKnown"; + public const string PercentMetric = "% of final cohort"; + + /// Label for the reference row: each region's share of the whole demography population. + public const string DemographyPercentMetric = "% of demography"; + + /// A resolved output column (a region recognised by the lookup). + private sealed record RegionColumn(string Code, string Name, string Node); + + /// + /// Splits one node's region counts into Total / recognised-regions / Other / NotKnown using + /// . is the GROUP BY Region result (every present + /// code); is RDMP's own count. + /// + public static CohortBuildBreakdownBuckets Split(int total, IReadOnlyDictionary byRegion, + RegionLookup lookup) + { + var regions = new Dictionary(System.StringComparer.OrdinalIgnoreCase); + var other = 0; + foreach (var (code, n) in byRegion) + if (lookup.Contains(code)) + regions[code] = n; + else + other += n; // present but not recognised by the lookup + + return new CohortBuildBreakdownBuckets(total, regions, other, total - regions.Values.Sum() - other); + } + + /// The recognised regions that appear anywhere, ordered by node (nulls last) then name. + private static List RegionColumns(IEnumerable nodes, + CohortBuildBreakdownBuckets demographyReference, RegionLookup lookup) => + nodes + .SelectMany(n => n.FinalByRegion.Keys.Concat(n.CumulativeByRegion?.Keys ?? Enumerable.Empty())) + .Concat(demographyReference?.Regions.Keys ?? Enumerable.Empty()) + .Where(lookup.Contains) + .GroupBy(code => code, System.StringComparer.OrdinalIgnoreCase) + .Select(g => new RegionColumn(g.Key, lookup.NameOf(g.Key), lookup.NodeOf(g.Key))) + .OrderBy(c => string.IsNullOrEmpty(c.Node) ? 1 : 0) // nodeless regions (e.g. non-Scottish) last + .ThenBy(c => c.Node, System.StringComparer.OrdinalIgnoreCase) + .ThenBy(c => c.Name, System.StringComparer.OrdinalIgnoreCase) + .ToList(); + + /// + /// Builds the wide CSV (data rows per node+metric, then a % of final cohort row and, when + /// is supplied, a % of demography row underneath it as + /// a cohort-vs-population sanity check). + /// + public static string ToCsv(IReadOnlyList nodes, RegionLookup lookup, + CohortBuildBreakdownBuckets demographyReference = null) + { + var ordered = nodes.OrderBy(n => n.Seq).ToList(); + var columns = RegionColumns(ordered, demographyReference, lookup); + + var header = new List { "Order", "Type", "Name", "Container", "SetOperation", "Metric", "Total" }; + header.AddRange(columns.Select(c => c.Name)); + header.Add(OtherColumn); + header.Add(NotKnownColumn); + + var sb = new StringBuilder(); + sb.AppendLine(string.Join(",", header.Select(Escape))); + + foreach (var n in ordered) + { + AppendCountRow(sb, n, columns, "Final", Split(n.FinalUnfiltered, n.FinalByRegion, lookup)); + if (n.CumulativeUnfiltered.HasValue && n.CumulativeByRegion != null) + AppendCountRow(sb, n, columns, "Cumulative", + Split(n.CumulativeUnfiltered.Value, n.CumulativeByRegion, lookup)); + } + + // bottom: % of final cohort (root node's Final), then % of demography, after a blank separator + var root = ordered.FirstOrDefault(n => string.IsNullOrEmpty(n.Container)) ?? ordered.FirstOrDefault(); + if (root != null && root.FinalUnfiltered > 0) + { + sb.AppendLine(); + sb.AppendLine(string.Join(",", header.Select(Escape))); // repeat header so % aligns to each region + AppendPercentRow(sb, PercentMetric, columns, Split(root.FinalUnfiltered, root.FinalByRegion, lookup)); + if (demographyReference != null) + AppendPercentRow(sb, DemographyPercentMetric, columns, demographyReference); + } + + return sb.ToString(); + } + + private static void AppendPercentRow(StringBuilder sb, string label, List columns, + CohortBuildBreakdownBuckets b) + { + double Pct(int v) => b.Total == 0 ? 0 : v * 100.0 / b.Total; + var cells = new List { "", "", label, "", "", label, Fmt(b.Total == 0 ? 0 : 100.0) }; + cells.AddRange(columns.Select(c => Fmt(Pct(b.Regions.TryGetValue(c.Code, out var v) ? v : 0)))); + cells.Add(Fmt(Pct(b.Other))); + cells.Add(Fmt(Pct(b.NotKnown))); + sb.AppendLine(string.Join(",", cells.Select(Escape))); + } + + private static void AppendCountRow(StringBuilder sb, CohortBuildBreakdownNode n, List columns, + string metric, CohortBuildBreakdownBuckets b) + { + var cells = new List + { + n.DisplayOrder.ToString(CultureInfo.InvariantCulture), + n.Type, n.Name, n.Container, n.SetOperation, metric, + b.Total.ToString(CultureInfo.InvariantCulture) + }; + cells.AddRange(columns.Select(c => + (b.Regions.TryGetValue(c.Code, out var v) ? v : 0).ToString(CultureInfo.InvariantCulture))); + cells.Add(b.Other.ToString(CultureInfo.InvariantCulture)); + cells.Add(b.NotKnown.ToString(CultureInfo.InvariantCulture)); + sb.AppendLine(string.Join(",", cells.Select(Escape))); + } + + public static void WriteCsv(string path, IReadOnlyList nodes, RegionLookup lookup, + CohortBuildBreakdownBuckets demographyReference = null) => + File.WriteAllText(path, ToCsv(nodes, lookup, demographyReference)); + + private static string Fmt(double d) => d.ToString("0.0", CultureInfo.InvariantCulture); + + private static string Escape(string field) + { + field ??= ""; + if (field.Contains(',') || field.Contains('"') || field.Contains('\n') || field.Contains('\r')) + return $"\"{field.Replace("\"", "\"\"")}\""; + return field; + } +} diff --git a/RdmpCohortBuildHealthBoardBreakdown/src/ExecuteCommandExportCohortBuildHealthBoardBreakdown.cs b/RdmpCohortBuildHealthBoardBreakdown/src/ExecuteCommandExportCohortBuildHealthBoardBreakdown.cs new file mode 100644 index 0000000000..2395de8e62 --- /dev/null +++ b/RdmpCohortBuildHealthBoardBreakdown/src/ExecuteCommandExportCohortBuildHealthBoardBreakdown.cs @@ -0,0 +1,360 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading; +using FAnsi; +using FAnsi.Discovery; +using FAnsi.Discovery.QuerySyntax; +using Rdmp.Core.CohortCreation; +using Rdmp.Core.CohortCreation.Execution; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Aggregation; +using Rdmp.Core.Curation.Data.Cohort; +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.Core.QueryCaching.Aggregation; +using Rdmp.Core.ReusableLibraryCode.DataAccess; + +namespace Rdmp.Core.CommandExecution.AtomicCommands; + +/// +/// Reproduces the Cohort Builder's per-set / per-container count tree (the FinalCount and cumulative +/// running totals shown as UNION/INTERSECT/EXCEPT are applied) split by a region column, using a +/// user-supplied lookup table to name/group the regions. Operates purely on the query cache: it builds +/// the cohort once to populate the per-set cache tables, then recomposes every count point from those +/// cache tables and splits it by region with one GROUP BY per node (all regions at once). +/// +public class ExecuteCommandExportCohortBuildHealthBoardBreakdown : BasicCommandExecution +{ + private readonly CohortIdentificationConfiguration _cic; + private ICatalogue _demographyCatalogue; + private ColumnInfo _regionColumn; + private TableInfo _groupLookup; + private readonly int _timeout; + private FileInfo _toFile; + + private ExtractionInformation _idEi; + private IQuerySyntaxHelper _syntax; + private DiscoveredDatabase _cacheDb; + private CachedAggregateConfigurationResultsManager _cacheManager; + private RegionLookup _lookup; + private string _demogTable; + private string _demogId; + private string _regionName; + + private readonly Dictionary _setCacheTable = new(); + private readonly Dictionary<(bool isContainer, int id), (int final, int? cumulative)> _baseline = new(); + + public ExecuteCommandExportCohortBuildHealthBoardBreakdown(IBasicActivateItems activator, + [DemandsInitialization("The cohort identification configuration whose build tree to break down")] + CohortIdentificationConfiguration cic, + [DemandsInitialization("Demography catalogue that provides the patient identifier and the region column")] + ICatalogue demographyCatalogue = null, + [DemandsInitialization("The region column to group the breakdown by")] + ColumnInfo regionColumn = null, + [DemandsInitialization("Lookup table mapping region code to a name and node (columns: Region, HB_Name, SafeHaven_Region)")] + TableInfo groupLookup = null, + [DemandsInitialization("CSV file to write. Defaults to -build-breakdown.csv in the current directory")] + FileInfo toFile = null, + [DemandsInitialization("Per-query command timeout in seconds", DefaultValue = 5000)] + int timeout = 5000) : base(activator) + { + _cic = cic; + _demographyCatalogue = demographyCatalogue; + _regionColumn = regionColumn; + _groupLookup = groupLookup; + _timeout = timeout; + _toFile = toFile; + + if (_cic == null) + { + SetImpossible("No CohortIdentificationConfiguration was supplied"); + return; + } + + if (_cic.RootCohortAggregateContainer_ID == null) + { + SetImpossible($"'{_cic}' has no root container to run"); + return; + } + + if (_cic.QueryCachingServer_ID == null) + SetImpossible($"'{_cic}' has no query caching server - this breakdown works only on cached results"); + + // the demography catalogue / region column / lookup table are resolved (and prompted for, in the + // GUI) in Execute, so the command can be added to a right-click menu with only the cohort selected. + } + + /// Resolves the RDMP-object inputs, prompting the user for any not supplied. Returns false on cancel. + private bool ResolveInputs() + { + _demographyCatalogue ??= SelectOne("Demography catalogue (provides the patient identifier + region column)", + BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects()); + if (_demographyCatalogue == null) + return Fail("No demography catalogue was supplied"); + + _idEi = _demographyCatalogue.GetAllExtractionInformation(ExtractionCategory.Any) + .FirstOrDefault(e => e.IsExtractionIdentifier); + if (_idEi == null) + return Fail($"'{_demographyCatalogue}' has no IsExtractionIdentifier column to join the cohort on"); + + _regionColumn ??= SelectOne("Region column to group by", + _demographyCatalogue.GetAllExtractionInformation(ExtractionCategory.Any) + .Select(e => e.ColumnInfo).Where(c => c != null).Distinct().ToArray()); + if (_regionColumn == null) + return Fail("No region column was supplied"); + + _groupLookup ??= SelectOne("Region lookup table (Region, HB_Name, SafeHaven_Region)", + BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects()); + if (_groupLookup == null) + return Fail("No region lookup table was supplied"); + + // co-location: the recompose + GROUP BY join runs on the cache server, so demography must be there + var cacheServer = _cic.QueryCachingServer.Server; + var demogServer = _regionColumn.TableInfo.Server; + if (!string.IsNullOrWhiteSpace(cacheServer) && !string.IsNullOrWhiteSpace(demogServer) + && !string.Equals(cacheServer.Trim(), demogServer.Trim(), StringComparison.OrdinalIgnoreCase)) + return Fail($"Region column is on server '{demogServer}' but the query cache is on '{cacheServer}'; " + + "the breakdown joins on the cache server, so they must be the same server."); + + return true; + } + + private T SelectOne(string prompt, T[] available) where T : class => + available.Length > 0 && BasicActivator.SelectObject(prompt, available, out var selected) ? selected : null; + + private bool Fail(string reason) + { + BasicActivator.Show(reason); + return false; + } + + public override void Execute() + { + base.Execute(); + + if (!ResolveInputs()) + return; + + _toFile ??= BasicActivator.IsInteractive + ? BasicActivator.SelectFile("Path to write build breakdown to", "Build breakdown", "*.csv") + : new FileInfo(Path.Combine(Environment.CurrentDirectory, $"{Sanitise(_cic.Name)}-build-breakdown.csv")); + + if (_toFile == null) + return; + + _syntax = _regionColumn.GetQuerySyntaxHelper(); + _cacheDb = _cic.QueryCachingServer.Discover(DataAccessContext.InternalDataProcessing); + _cacheManager = new CachedAggregateConfigurationResultsManager(_cic.QueryCachingServer); + _demogTable = _regionColumn.TableInfo.Name; + _demogId = _syntax.EnsureWrapped(_idEi.GetRuntimeName()); + _regionName = _syntax.EnsureWrapped(_regionColumn.GetRuntimeName()); + + // load the (user-defined) region -> name/node mapping from the lookup table + _lookup = RegionLookup.LoadFrom(_groupLookup.Discover(DataAccessContext.InternalDataProcessing), _timeout); + + // 1. Build once: populates every per-set cache table and gives the baseline (unfiltered) counts. + var compiler = new CohortCompiler(BasicActivator, _cic) { IncludeCumulativeTotals = true }; + var runner = new CohortCompilerRunner(compiler, _timeout) { RunSubcontainers = true }; + runner.Run(new CancellationToken()); + + var crashed = compiler.Tasks.Keys.Where(t => t.State == CompilationState.Crashed).ToList(); + if (crashed.Any()) + { + SetImpossible($"{crashed.Count} task(s) crashed during the build - cannot produce a reliable breakdown"); + BasicActivator.Show($"Build failed: {crashed[0].CrashMessage?.Message}"); + return; + } + + foreach (var task in compiler.Tasks.Keys) + { + var isContainer = task switch + { + AggregationContainerTask => true, + AggregationTask => false, + _ => (bool?)null // skip joinables / plugin tasks + }; + if (isContainer == null || task.Child == null) + continue; + _baseline[(isContainer.Value, task.Child.ID)] = (task.FinalRowCount, task.CumulativeRowCount); + } + + // 2. Walk the tree, recomposing each count point from the cache and splitting by region. + var nodes = new List(); + var seq = 0; + Walk(_cic.RootCohortAggregateContainer, null, 0, nodes, ref seq); + + // reference: each region's share of the WHOLE demography population (a sanity-check row) + var demographyReference = ComputeDemographyReference(); + + CohortBuildHealthBoardBreakdownReport.WriteCsv(_toFile.FullName, nodes, _lookup, demographyReference); + + // reconciliation note: recognised-region counts should never exceed the unfiltered total + var drift = nodes.Count(n => + n.FinalByRegion.Where(kv => _lookup.Contains(kv.Key)).Sum(kv => kv.Value) > n.FinalUnfiltered); + var summary = $"Exported build breakdown to {_toFile.FullName} ({nodes.Count} count points)"; + if (drift > 0) + summary += $" - WARNING: {drift} node(s) have region counts exceeding the unfiltered total (check keys)"; + BasicActivator.Show(summary); + } + + private void Walk(CohortAggregateContainer container, CohortAggregateContainer parent, int indexInParent, + List nodes, ref int seq) + { + // container node row (cumulative is within its parent) + var (cFinal, cCum) = _baseline.TryGetValue((true, container.ID), out var cb) ? cb : (0, null); + IReadOnlyDictionary cCumByRegion = null; + if (parent != null && indexInParent > 0 && cCum.HasValue) + cCumByRegion = RunRegionCounts(CumulativeSql(parent, indexInParent)); + + nodes.Add(new CohortBuildBreakdownNode(seq++, "Container", CleanName(container.Name), + CleanName(parent?.Name), container.Operation.ToString(), container.Order, cFinal, + parent != null && indexInParent > 0 ? cCum : null, + RunRegionCounts(IdSql(container)), cCumByRegion)); + + var kids = EnabledOrdered(container); + for (var i = 0; i < kids.Count; i++) + { + switch (kids[i]) + { + case AggregateConfiguration agg: + var (aFinal, aCum) = _baseline.TryGetValue((false, agg.ID), out var ab) ? ab : (0, null); + IReadOnlyDictionary aCumByRegion = null; + if (i > 0 && aCum.HasValue) + aCumByRegion = RunRegionCounts(CumulativeSql(container, i)); + + nodes.Add(new CohortBuildBreakdownNode(seq++, "Cohort Set", CleanName(agg.Name), + CleanName(container.Name), "", agg.Order, aFinal, i > 0 ? aCum : null, + RunRegionCounts(CachedSetSql(agg)), aCumByRegion)); + break; + + case CohortAggregateContainer sub: + Walk(sub, container, i, nodes, ref seq); + break; + } + } + } + + // --- identifier-list SQL composed purely from the per-set cache tables --- + + private string IdSql(IOrderable node) => node switch + { + AggregateConfiguration agg => CachedSetSql(agg), + CohortAggregateContainer c => Compose(c, EnabledOrdered(c)), + _ => throw new NotSupportedException(node.GetType().Name) + }; + + private string CumulativeSql(CohortAggregateContainer container, int upToInclusive) => + Compose(container, EnabledOrdered(container).Take(upToInclusive + 1).ToList()); + + private string Compose(CohortAggregateContainer container, IReadOnlyList children) + { + var op = $"\n{SetOperationSql(container.Operation, _syntax.DatabaseType)}\n"; + return string.Join(op, children.Select(ch => $"({IdSql(ch)})")); + } + + /// + /// Renders a container's set operation for the target DBMS (Oracle spells EXCEPT as MINUS) - the + /// same mapping RDMP uses in CohortQueryBuilderResult.GetSetOperationSql. + /// + public static string SetOperationSql(SetOperation operation, DatabaseType dbType) => operation switch + { + SetOperation.UNION => "UNION", + SetOperation.INTERSECT => "INTERSECT", + SetOperation.EXCEPT => dbType == DatabaseType.Oracle ? "MINUS" : "EXCEPT", + _ => throw new ArgumentOutOfRangeException(nameof(operation), operation, null) + }; + + private string CachedSetSql(AggregateConfiguration agg) + { + if (!_setCacheTable.TryGetValue(agg.ID, out var t)) + { + var table = _cacheManager.GetLatestResultsTableUnsafe(agg, + AggregateOperation.IndexedExtractionIdentifierList) as DiscoveredTable; + if (table == null) + throw new Exception($"Cohort set '{agg.Name}' has no cached identifier list - the build did not cache it"); + var col = table.DiscoverColumns()[0].GetRuntimeName(); + t = (table.GetFullyQualifiedName(), _syntax.EnsureWrapped(col)); + _setCacheTable[agg.ID] = t; + } + + return $"SELECT {t.col} id FROM {t.fqn}"; + } + + private List EnabledOrdered(CohortAggregateContainer container) => + container.GetOrderedContents().Where(o => o switch + { + AggregateConfiguration a => !a.IsDisabled, + CohortAggregateContainer c => !c.IsDisabled, + _ => true + }).ToList(); + + // --- run a GROUP BY Region join (on the cache server) for one count point --- + + private IReadOnlyDictionary RunRegionCounts(string idListSql) + { + var sql = + $"SELECT d.{_regionName} region, COUNT(DISTINCT i.id) n\n" + + $"FROM (\n{idListSql}\n) i\n" + + $"INNER JOIN {_demogTable} d ON d.{_demogId} = i.id\n" + + $"GROUP BY d.{_regionName}"; + + return ReadRegionCounts(sql, out _); + } + + /// + /// The whole demography population split by region (the reference/background distribution). Total + /// includes NULL-region rows so captures them. + /// + private CohortBuildBreakdownBuckets ComputeDemographyReference() + { + var sql = + $"SELECT d.{_regionName} region, COUNT(DISTINCT d.{_demogId}) n\n" + + $"FROM {_demogTable} d\n" + + $"GROUP BY d.{_regionName}"; + + var byRegion = ReadRegionCounts(sql, out var total); + return CohortBuildHealthBoardBreakdownReport.Split(total, byRegion, _lookup); + } + + /// Runs a "region, count" query on the cache server; returns non-null regions and the grand total. + private Dictionary ReadRegionCounts(string sql, out int total) + { + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + total = 0; + using var con = _cacheDb.Server.GetConnection(); + con.Open(); + using var cmd = _cacheDb.Server.GetCommand(sql, con); + cmd.CommandTimeout = _timeout; + using var r = cmd.ExecuteReader(); + while (r.Read()) + { + var n = Convert.ToInt32(r.GetValue(1)); + total += n; // includes any NULL-region group in the denominator + if (!r.IsDBNull(0)) + result[r.GetValue(0).ToString()] = n; + } + + return result; + } + + // RDMP prefixes cohort set names with "cic__" (EnsureNamingConvention); cloning a cohort across + // CICs stacks them (e.g. cic_18286_cic_18284_cic_17950_People in SHARE...). Strip them for display. + private static readonly Regex CicPrefix = new(@"^(cic_\d+_)+", RegexOptions.Compiled); + + public static string CleanName(string name) => string.IsNullOrEmpty(name) ? "" : CicPrefix.Replace(name, ""); + + private static string Sanitise(string name) + { + foreach (var c in Path.GetInvalidFileNameChars()) + name = name.Replace(c, '_'); + return name; + } +} diff --git a/RdmpCohortBuildHealthBoardBreakdown/src/RdmpCohortBuildHealthBoardBreakdown.csproj b/RdmpCohortBuildHealthBoardBreakdown/src/RdmpCohortBuildHealthBoardBreakdown.csproj new file mode 100644 index 0000000000..c9a7da7e32 --- /dev/null +++ b/RdmpCohortBuildHealthBoardBreakdown/src/RdmpCohortBuildHealthBoardBreakdown.csproj @@ -0,0 +1,23 @@ + + + + net10.0 + enable + disable + RdmpCohortBuildHealthBoardBreakdown + false + + false + + + + + + false + + + + diff --git a/RdmpCohortBuildHealthBoardBreakdown/src/RdmpCohortBuildHealthBoardBreakdown.nuspec b/RdmpCohortBuildHealthBoardBreakdown/src/RdmpCohortBuildHealthBoardBreakdown.nuspec new file mode 100644 index 0000000000..73e4c581c6 --- /dev/null +++ b/RdmpCohortBuildHealthBoardBreakdown/src/RdmpCohortBuildHealthBoardBreakdown.nuspec @@ -0,0 +1,17 @@ + + + + RdmpCohortBuildHealthBoardBreakdown + 0.0.1 + HIC + Reproduces the Cohort Builder's per-set / per-container count tree (FinalCount and + cumulative running totals) split by Scottish health board, working purely on the query cache: + builds the cohort once, then recomposes every count point from the cached per-set tables and + splits it by SHARE_Demography.Region. CLI: ExportCohortBuildHealthBoardBreakdown; GUI: + right-click a cohort identification configuration. + + + + + + diff --git a/RdmpCohortBuildHealthBoardBreakdown/src/RegionLookup.cs b/RdmpCohortBuildHealthBoardBreakdown/src/RegionLookup.cs new file mode 100644 index 0000000000..f8c290ce45 --- /dev/null +++ b/RdmpCohortBuildHealthBoardBreakdown/src/RegionLookup.cs @@ -0,0 +1,77 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using System; +using System.Collections.Generic; +using FAnsi.Discovery; + +namespace Rdmp.Core.CohortCreation; + +/// +/// A user-defined mapping from a region code to a display name and a grouping node, loaded at runtime +/// from a lookup table (rather than hard-coded). The lookup table is expected to have the columns named +/// by , and ; a NULL +/// is allowed (e.g. non-Scottish / administrative codes that still have a name +/// but no safe-haven node). Codes absent from the table are treated as unmapped ("Other"). +/// +public sealed class RegionLookup +{ + /// Column holding the region code that appears in the demography data. + public const string RegionColumn = "Region"; + + /// Column holding the display name for a region code. + public const string NameColumn = "HB_Name"; + + /// Column holding the grouping node (used to order columns); may be NULL. + public const string NodeColumn = "SafeHaven_Region"; + + private readonly Dictionary _map; + + public RegionLookup(IReadOnlyDictionary map) => + _map = new Dictionary(map, StringComparer.OrdinalIgnoreCase); + + /// True if the code is present in the lookup (a recognised region). + public bool Contains(string code) => code != null && _map.ContainsKey(code.Trim()); + + /// Display name for the code, or null if unmapped. + public string NameOf(string code) => + code != null && _map.TryGetValue(code.Trim(), out var v) ? v.Name : null; + + /// Grouping node for the code (may be null even for a mapped code), or null if unmapped. + public string NodeOf(string code) => + code != null && _map.TryGetValue(code.Trim(), out var v) ? v.Node : null; + + /// + /// Reads the mapping from a lookup (columns / + /// / ). Rows with a NULL/blank region code are skipped; + /// a NULL name falls back to the code; a NULL node is preserved. + /// + public static RegionLookup LoadFrom(DiscoveredTable table, int timeout) + { + var syntax = table.Database.Server.GetQuerySyntaxHelper(); + var sql = + $"SELECT {syntax.EnsureWrapped(RegionColumn)}, {syntax.EnsureWrapped(NameColumn)}, " + + $"{syntax.EnsureWrapped(NodeColumn)} FROM {table.GetFullyQualifiedName()}"; + + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + using var con = table.Database.Server.GetConnection(); + con.Open(); + using var cmd = table.Database.Server.GetCommand(sql, con); + cmd.CommandTimeout = timeout; + using var r = cmd.ExecuteReader(); + while (r.Read()) + { + var code = r[0] == DBNull.Value ? null : r[0].ToString()?.Trim(); + if (string.IsNullOrEmpty(code)) + continue; + var name = r[1] == DBNull.Value ? code : r[1].ToString(); + var node = r[2] == DBNull.Value ? null : r[2].ToString(); + map[code] = (name, node); + } + + return new RegionLookup(map); + } +}