perf(calendar): pre-filter ICS text before parsing

Drop simple one-off events outside the configured date window before
passing the ICS text to node-ical. Recurring events and overrides are
kept, because their actual instances need the normal expansion logic.

The pre-filter is conservative: if an event cannot be classified cheaply,
it stays in the feed and the parser handles it as before. This keeps the
optimization local while reducing parser work for calendars with many old
non-recurring events.

Ref #4103

Author: KristjanESPERANTO

cspell.config.json

@@ -257,6 +257,7 @@
 		"pubdate",
 		"radokristof",
 		"rajniszp",
+		"RDATE",
 		"rebuilded",
 		"Reis",
 		"rejas",
@@ -364,7 +365,8 @@
 		"xxxe",
 		"Ybbet",
 		"yearmatch",
-		"yearmatchgroup"
+		"yearmatchgroup",
+		"YYYYMMDDTHHMMSSZ"
 	],
 	"ignorePaths": [
 		"css/roboto.css",

defaultmodules/calendar/calendarfetcher.js

@@ -1,6 +1,6 @@
 const ical = require("node-ical");
 const Log = require("logger");
-const CalendarFetcherUtils = require("./calendarfetcherutils");
+const { filterEvents, preFilterICSText } = require("./calendarfetcherutils");
 const HTTPFetcher = require("#http_fetcher");
 
 /**
@@ -52,11 +52,15 @@ class CalendarFetcher {
 	async #handleResponse (response) {
 		try {
 			const responseData = await response.text();
-			const parsed = await ical.async.parseICS(responseData);
+			const filteredData = preFilterICSText(responseData, {
+				includePastEvents: this.includePastEvents,
+				maximumNumberOfDays: this.maximumNumberOfDays
+			});
+			const parsed = await ical.async.parseICS(filteredData);
 
 			Log.debug(`Parsed iCal data from ${this.url} with ${Object.keys(parsed).length} entries.`);
 
-			this.events = CalendarFetcherUtils.filterEvents(parsed, {
+			this.events = filterEvents(parsed, {
 				excludedEvents: this.excludedEvents,
 				includePastEvents: this.includePastEvents,
 				maximumEntries: this.maximumEntries,

defaultmodules/calendar/calendarfetcherutils.js

@@ -6,8 +6,110 @@ const ical = require("node-ical");
 
 const Log = require("logger");
 
+/**
+ * Unfold RFC 5545 continuation lines before scanning event fields.
+ * @param {string} text Raw ICS text.
+ * @returns {string} Text with folded lines joined.
+ */
+function unfoldICS (text) {
+	return text.replace(/\r?\n[ \t]/g, "");
+}
+
+/**
+ * Get the end date implied by a DURATION value.
+ * @param {string} startYYYYMMDD Event start date.
+ * @param {string} duration ICS DURATION value.
+ * @returns {string} End date as YYYYMMDD.
+ */
+function getDurationEndDate (startYYYYMMDD, duration) {
+	const match = (/^P(?:(\d+)W)?(?:(\d+)D)?/).exec(duration);
+	if (!match) return startYYYYMMDD;
+	const days = (parseInt(match[1] || "0", 10) * 7) + parseInt(match[2] || "0", 10);
+	if (!days) return startYYYYMMDD;
+	return moment(startYYYYMMDD, "YYYYMMDD").add(days, "days").format("YYYYMMDD");
+}
+
+/**
+ * Decide whether a raw VEVENT block should be kept for parsing.
+ * @param {string} block Raw VEVENT block.
+ * @param {string} pastDateStr Inclusive lower date bound as YYYYMMDD.
+ * @param {string} futureDateStr Inclusive upper date bound as YYYYMMDD.
+ * @returns {boolean} True if the block should be parsed.
+ */
+function shouldKeepEventBlock (block, pastDateStr, futureDateStr) {
+	const unfolded = unfoldICS(block);
+	if ((/(?:^|\n)(?:RRULE|RDATE|RECURRENCE-ID)[;:]/i).test(unfolded)) return true;
+
+	// Matches DTSTART:20221215T..., DTSTART;TZID=...:20221215T..., DTSTART;VALUE=DATE:20221215
+	const startMatch = (/\nDTSTART[;:][^\r\n]*?(\d{8})/).exec(unfolded);
+	if (!startMatch) return true; // no DTSTART — let the parser deal with it
+
+	const startDateStr = startMatch[1];
+	const endMatch = (/\nDTEND[;:][^\r\n]*?(\d{8})/).exec(unfolded);
+	let endDateStr = endMatch ? endMatch[1] : startDateStr;
+	if (!endMatch) {
+		const durMatch = (/\nDURATION:([^\r\n]+)/).exec(unfolded);
+		if (durMatch) endDateStr = getDurationEndDate(startDateStr, durMatch[1].trim());
+	}
+
+	// YYYYMMDD sorts lexicographically, so plain string compare is fine.
+	return endDateStr >= pastDateStr && startDateStr <= futureDateStr;
+}
+
 const CalendarFetcherUtils = {
 
+	/**
+	 * Drops simple out-of-range VEVENT blocks before parsing the ICS text.
+	 * Recurring events and overrides are kept because their instances need expansion.
+	 * @param {string} icsText - Raw ICS text from the calendar feed.
+	 * @param {object} config - Needs includePastEvents (boolean) and maximumNumberOfDays (number).
+	 * @returns {string} Filtered ICS text, ready for ical.parseICS().
+	 */
+	preFilterICSText (icsText, config) {
+		const today = moment().startOf("day");
+		const pastDateStr = (config.includePastEvents ? today.clone().subtract(config.maximumNumberOfDays, "days") : today).format("YYYYMMDD");
+		const futureDateStr = today.clone().add(config.maximumNumberOfDays, "days").format("YYYYMMDD");
+
+		const firstEventIndex = icsText.indexOf("BEGIN:VEVENT");
+		if (firstEventIndex === -1) return icsText;
+
+		const parts = [icsText.slice(0, firstEventIndex)];
+		let pos = firstEventIndex;
+		let kept = 0;
+		let dropped = 0;
+
+		while (pos < icsText.length) {
+			const blockStart = icsText.indexOf("BEGIN:VEVENT", pos);
+			if (blockStart === -1) {
+				parts.push(icsText.slice(pos));
+				break;
+			}
+
+			const endIdx = icsText.indexOf("END:VEVENT", blockStart);
+			if (endIdx === -1) {
+				// Malformed ICS — keep the rest and let the parser surface it.
+				parts.push(icsText.slice(pos));
+				break;
+			}
+
+			const lineEnd = icsText.indexOf("\n", endIdx);
+			const blockEnd = lineEnd !== -1 ? lineEnd + 1 : icsText.length;
+			const block = icsText.slice(blockStart, blockEnd);
+
+			if (shouldKeepEventBlock(block, pastDateStr, futureDateStr)) {
+				parts.push(block);
+				kept++;
+			} else {
+				dropped++;
+			}
+
+			pos = blockEnd;
+		}
+
+		Log.debug(`ICS pre-filter: kept ${kept} events, dropped ${dropped} events.`);
+		return parts.join("");
+	},
+
 	/**
 	 * Determine based on the title of an event if it should be excluded from the list of events
 	 * @param {object} config the global config

tests/unit/modules/default/calendar/calendar_fetcher_utils_spec.js

@@ -497,4 +497,147 @@ END:VCALENDAR`);
 			expect(filteredEvents[0].location).toBe("Berlin");
 		});
 	});
+
+	describe("preFilterICSText", () => {
+		const makeICS = (events) => {
+			const header = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\n";
+			const footer = "END:VCALENDAR\r\n";
+			return header + events.join("") + footer;
+		};
+
+		const makeEvent = (dtstart, dtend, summary, extra = "") => {
+			return `BEGIN:VEVENT\r\nDTSTART:${dtstart}\r\nDTEND:${dtend}\r\nSUMMARY:${summary}\r\n${extra ? `${extra}\r\n` : ""}END:VEVENT\r\n`;
+		};
+
+		// Format: YYYYMMDDTHHMMSSZ
+		const dateStr = (m) => `${m.clone().startOf("day").format("YYYYMMDD")}T120000Z`;
+
+		const now = moment();
+		const past = now.clone().subtract(400, "days");
+		const recent = now.clone().subtract(1, "days");
+		const tomorrow = now.clone().add(1, "days");
+		const future = now.clone().add(400, "days");
+
+		const preFilterConfig = { includePastEvents: false, maximumNumberOfDays: 365 };
+
+		it("should return unchanged ICS if there are no VEVENT blocks", () => {
+			const input = "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nEND:VCALENDAR\r\n";
+			const result = CalendarFetcherUtils.preFilterICSText(input, preFilterConfig);
+			expect(result).toBe(input);
+		});
+
+		it("should keep events within the date range", () => {
+			const icsText = makeICS([makeEvent(dateStr(recent), dateStr(tomorrow), "Recent event")]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).toContain("Recent event");
+		});
+
+		it("should drop non-recurring events that ended in the distant past", () => {
+			const icsText = makeICS([makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "Old event")]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).not.toContain("Old event");
+		});
+
+		it("should drop non-recurring events that start in the distant future", () => {
+			const icsText = makeICS([makeEvent(dateStr(future), dateStr(future.clone().add(1, "days")), "Far future event")]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).not.toContain("Far future event");
+		});
+
+		it("should always keep events with RRULE regardless of DTSTART", () => {
+			const icsText = makeICS([makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "Birthday", "RRULE:FREQ=YEARLY")]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).toContain("Birthday");
+		});
+
+		it("should keep recurring events even when property names are lowercase", () => {
+			const icsText = makeICS([makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "Lowercase recurrence", "rrule:FREQ=YEARLY")]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).toContain("Lowercase recurrence");
+		});
+
+		it("should always keep events with RDATE regardless of DTSTART", () => {
+			const icsText = makeICS([makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "RDATE event", `RDATE:${dateStr(tomorrow)}`)]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).toContain("RDATE event");
+		});
+
+		it("should always keep events with RECURRENCE-ID", () => {
+			const icsText = makeICS([makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "Override", `RECURRENCE-ID:${dateStr(past)}`)]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).toContain("Override");
+		});
+
+		it("should handle DTSTART with TZID parameter (out of range)", () => {
+			const dateOnlyStr = `${past.clone().startOf("day").format("YYYYMMDD")}T120000`;
+			const event = `BEGIN:VEVENT\r\nDTSTART;TZID=America/New_York:${dateOnlyStr}\r\nDTEND;TZID=America/New_York:${dateOnlyStr}\r\nSUMMARY:Old TZ event\r\nEND:VEVENT\r\n`;
+			const icsText = makeICS([event]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).not.toContain("Old TZ event");
+		});
+
+		it("should handle DTSTART with TZID parameter (in range)", () => {
+			const startStr = `${tomorrow.clone().startOf("day").format("YYYYMMDD")}T120000`;
+			const event = `BEGIN:VEVENT\r\nDTSTART;TZID=America/New_York:${startStr}\r\nDTEND;TZID=America/New_York:${startStr}\r\nSUMMARY:Upcoming TZ event\r\nEND:VEVENT\r\n`;
+			const icsText = makeICS([event]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).toContain("Upcoming TZ event");
+		});
+
+		it("should drop events whose DTSTART + DURATION still lies in the past", () => {
+			const event = `BEGIN:VEVENT\r\nDTSTART:${dateStr(past)}\r\nDURATION:P1D\r\nSUMMARY:Short past event\r\nEND:VEVENT\r\n`;
+			const result = CalendarFetcherUtils.preFilterICSText(makeICS([event]), preFilterConfig);
+			expect(result).not.toContain("Short past event");
+		});
+
+		it("should keep events whose DURATION extends into the window", () => {
+			const event = `BEGIN:VEVENT\r\nDTSTART:${dateStr(past)}\r\nDURATION:P800D\r\nSUMMARY:Long running event\r\nEND:VEVENT\r\n`;
+			const result = CalendarFetcherUtils.preFilterICSText(makeICS([event]), preFilterConfig);
+			expect(result).toContain("Long running event");
+		});
+
+		it("should handle folded DTSTART lines (RFC 5545 line folding)", () => {
+			const dateOnlyStr = `${past.clone().startOf("day").format("YYYYMMDD")}T120000`;
+			// Fold after the parameter list — a CRLF + space continues the previous line.
+			const event = `BEGIN:VEVENT\r\nDTSTART;TZID=Europe/Berlin\r\n :${dateOnlyStr}\r\nSUMMARY:Folded event\r\nEND:VEVENT\r\n`;
+			const result = CalendarFetcherUtils.preFilterICSText(makeICS([event]), preFilterConfig);
+			expect(result).not.toContain("Folded event");
+		});
+
+		it("should handle all-day events with VALUE=DATE format", () => {
+			const recentDate = recent.clone().startOf("day").format("YYYYMMDD");
+			const event = `BEGIN:VEVENT\r\nDTSTART;VALUE=DATE:${recentDate}\r\nDTEND;VALUE=DATE:${recent.clone().add(1, "days").startOf("day").format("YYYYMMDD")}\r\nSUMMARY:All day event\r\nEND:VEVENT\r\n`;
+			const icsText = makeICS([event]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).toContain("All day event");
+		});
+
+		it("should keep multiple events selectively", () => {
+			const icsText = makeICS([
+				makeEvent(dateStr(past), dateStr(past.clone().add(1, "days")), "Old event"),
+				makeEvent(dateStr(recent), dateStr(tomorrow), "Current event"),
+				makeEvent(dateStr(future), dateStr(future.clone().add(1, "days")), "Far future event")
+			]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).not.toContain("Old event");
+			expect(result).toContain("Current event");
+			expect(result).not.toContain("Far future event");
+		});
+
+		it("should include past events when includePastEvents is true", () => {
+			const recentPast = now.clone().subtract(100, "days");
+			const icsText = makeICS([makeEvent(dateStr(recentPast), dateStr(recentPast.clone().add(1, "days")), "Past event")]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, { includePastEvents: true, maximumNumberOfDays: 365 });
+			expect(result).toContain("Past event");
+		});
+
+		it("should preserve the ICS header and footer", () => {
+			const icsText = makeICS([makeEvent(dateStr(recent), dateStr(tomorrow), "Event")]);
+			const result = CalendarFetcherUtils.preFilterICSText(icsText, preFilterConfig);
+			expect(result).toContain("BEGIN:VCALENDAR");
+			expect(result).toContain("VERSION:2.0");
+			expect(result).toContain("END:VCALENDAR");
+		});
+	});
+
 });