Sync internal changes with open source core

bin/copyDb.ts

@@ -162,7 +162,7 @@
 	const sourceDbisDb = rootStore.dbisDb;
 	const sourceAuditStore = rootStore.auditStore;
 	const targetEnv = open(new OpenEnvironmentObject(targetDatabasePath));
 	const targetDbisDb = targetEnv.openDB(INTERNAL_DBIS_NAME);
 	let written;
 	let outstandingWrites = 0;
 	// we use a single transaction to get a snapshot, also we can't use snapshot: false on dupsort dbs
@@ -188,15 +188,15 @@
 			// we want to directly copy bytes so we don't have the overhead of
 			// encoding and decoding
 			dbiInit.encoding = 'binary';
 			dbiInit.compression = existingCompression;
 			//dbiInit.keyEncoding = 'binary';
 			const sourceDbi = rootStore.openDB(key, dbiInit);
 			sourceDbi.decoder = null;
 			sourceDbi.decoderCopies = false;
 			sourceDbi.encoding = 'binary';
 			dbiInit.compression = newCompression;
 			const targetDbi = targetEnv.openDB(key, dbiInit);
 			targetDbi.encoder = null;
 			console.log('copying', key, 'from', sourceDatabase, 'to', targetDatabasePath);
 			await copyDbi(sourceDbi, targetDbi, isPrimary, transaction);
 		}
@@ -209,6 +209,7 @@
 		async function copyDbi(sourceDbi, targetDbi, isPrimary, transaction) {
 			let recordsCopied = 0;
 			let bytesCopied = 0;
+			let skippedRecord = 0;
 			let retries = 10000000;
 			let start = null;
 			while (retries-- > 0) {
@@ -217,13 +218,26 @@
 						try {
 							start = key;
 							const { value, version } = sourceDbi.getEntry(key, { transaction });
+							// deleted entries should be 13 bytes long (8 for timestamp, 4 bytes for flags, 1 byte of the encoding of null)
+							if (value?.length < 14 && isPrimary) {
+								skippedRecord++;
+								continue;
+							}
 							written = targetDbi.put(key, value, isPrimary ? version : undefined);
 							recordsCopied++;
 							if (transaction.openTimer) transaction.openTimer = 0; // reset the timer, don't want it to time out
 							bytesCopied += (key?.length || 10) + value.length;
 							if (outstandingWrites++ > 5000) {
 								await written;
-								console.log('copied', recordsCopied, 'entries', bytesCopied, 'bytes');
+								console.log(
+									'copied',
+									recordsCopied,
+									'entries, skipped',
+									skippedRecord,
+									'delete records,',
+									bytesCopied,
+									'bytes'
+								);
 								outstandingWrites = 0;
 							}
 						} catch (error) {
@@ -238,7 +252,15 @@
 							);
 						}
 					}
-					console.log('finish copying, copied', recordsCopied, 'entries', bytesCopied, 'bytes');
+					console.log(
+						'finish copying, copied',
+						recordsCopied,
+						'entries, skipped',
+						skippedRecord,
+						'delete records,',
+						bytesCopied,
+						'bytes'
+					);
 					return;
 				} catch (error) {
 					// try to resume with a bigger key

components/Application.ts

@@ -80,7 +80,7 @@
 	if (application.payload) {
 		// Given a payload, create a Readable from the Buffer or string
 		tarball = Readable.from(
 			application.payload instanceof Buffer ? application.payload : Buffer.from(application.payload, 'base64')
 		);
 	} else {
 		// Given a package, there are a a couple options
@@ -119,12 +119,12 @@
 			}
 		} else {
 			// Given a package, resolve using `npm pack` (downloads the package as a tarball and writes the path to stdout)
-			const { stdout: tarballFilePath } = await nonInteractiveSpawn(
-				application.name,
-				'npm',
-				['pack', application.packageIdentifier],
-				parentDirPath
-			);
+			const {
+				stdout: tarballFilePath,
+				code,
+				stderr,
+			} = await nonInteractiveSpawn(application.name, 'npm', ['pack', application.packageIdentifier], parentDirPath);
+			if (code !== 0) throw new Error(`Failed to download package ${application.packageIdentifier}: ${stderr}`);
 			tarballPath = join(parentDirPath, tarballFilePath.trim());
 			// Create a Readable from the tarball
 			tarball = createReadStream(tarballPath);

components/EntryHandler.ts

@@ -111,7 +111,7 @@
 						urlPath,
 					};
 					this.emit('all', entry);
 					this.emit(event, entry);
 				});
 				break;
 			}
@@ -139,7 +139,7 @@
 					urlPath,
 				};
 				this.emit('all', entry);
 				this.emit(event, entry);
 				break;
 			}
 		}
@@ -164,7 +164,12 @@
 				cwd: this.#component.directory,
 				persistent: false,
 				ignored: (path) => {
-					return path !== this.#component.directory && allowedBases.every((base) => !path.startsWith(base));
+					const normalizedPath = path.replace(/\\/g, '/');
+					const normalizedBases = allowedBases.map(base => base.replace(/\\/g, '/'));
+					return (
+						normalizedPath !== this.#component.directory.replace(/\\/g, '/') &&
+						normalizedBases.every((base) => !normalizedPath.startsWith(base))
+					);
 				},
 			})
 			.on('all', this.#handleAll.bind(this))

components/Scope.ts

@@ -158,11 +158,13 @@ export class Scope extends EventEmitter {
 		};
 	}
 
-	#getFilesOption(): FilesOption | FileAndURLPathConfig | undefined {
+	#getFilesOption(): FileAndURLPathConfig | undefined {
 		const config = this.options.getAll();
-		if (config && typeof config === 'object' && config !== null && !Array.isArray(config) && 'files' in config) {
-			// TODO maybe some validation here?
-			return config.files as FilesOption | FileAndURLPathConfig;
+		if (config && typeof config === 'object' && config !== null && !Array.isArray(config) && 'files' in config /*&& validate config.files*/) {
+			return {
+				files: config.files as FilesOption,
+				urlPath: config.urlPath as string | undefined,
+			};
 		}
 		return undefined;
 	}

config/configHelpers.ts

@@ -0,0 +1,11 @@
+import * as path from 'node:path';
+import * as hdbTerms from '../utility/hdbTerms.ts';
+
+/**
+ * Get the backup directory path for the given Harper root
+ * @param hdbRoot - Harper root path
+ * @returns Full path to backup directory
+ */
+export function getBackupDirPath(hdbRoot: string): string {
+	return path.join(hdbRoot, hdbTerms.BACKUP_DIR_NAME);
+}

config/configUtils.js

@@ -13,6 +13,7 @@ const _ = require('lodash');
 const { handleHDBError } = require('../utility/errors/hdbError.js');
 const { HTTP_STATUS_CODES, HDB_ERROR_MSGS } = require('../utility/errors/commonErrors.js');
 const { server } = require('../server/Server.ts');
+const { getBackupDirPath } = require('./configHelpers.ts');
 
 const { DATABASES_PARAM_CONFIG, CONFIG_PARAMS, CONFIG_PARAM_MAP } = hdbTerms;
 const UNINIT_GET_CONFIG_ERR = 'Unable to get config value because config is uninitialized';
@@ -121,6 +122,11 @@ function createConfigFile(args, skipFsValidation = false) {
 
 	// Validates config doc and if required sets default values for some parameters.
 	validateConfig(configDoc, skipFsValidation);
+
+	// Apply HARPER_DEFAULT_CONFIG and HARPER_SET_CONFIG environment variables
+	// Must be called AFTER rootPath is set in configDoc
+	// Mutates configDoc in place
+	applyRuntimeEnvVarConfig(configDoc, null, { isInstall: true });
 	const configObj = configDoc.toJSON();
 	flatConfigObj = flattenConfig(configObj);
 
@@ -270,6 +276,9 @@ function initConfig(force = false) {
 
 		checkForUpdatedConfig(configDoc, configFilePath);
 
+		// Apply HARPER_DEFAULT_CONFIG and HARPER_SET_CONFIG environment variables
+		applyRuntimeEnvVarConfig(configDoc, configFilePath);
+
 		// Validates config doc and if required sets default values for some parameters.
 		validateConfig(configDoc);
 		const configObj = configDoc.toJSON();
@@ -545,8 +554,7 @@ function updateConfigValue(
 function backupConfigFile(configPath, hdbRoot) {
 	try {
 		const backupFolderPath = path.join(
-			hdbRoot,
-			'backup',
+			getBackupDirPath(hdbRoot),
 			`${new Date(Date.now()).toISOString().replaceAll(':', '-')}-${hdbTerms.HDB_CONFIG_FILE}.bak`
 		);
 		fs.copySync(configPath, backupFolderPath);
@@ -706,6 +714,75 @@ function parseYamlDoc(filePath) {
 	return YAML.parseDocument(fs.readFileSync(filePath, 'utf8'), { simpleKeys: true });
 }
 
+/**
+ * Apply HARPER_DEFAULT_CONFIG and HARPER_SET_CONFIG environment variables at runtime
+ *
+ * This function performs the following:
+ * 1. Loads configuration state to track sources
+ * 2. Detects user edits (drift) to protect them from HARPER_DEFAULT_CONFIG
+ * 3. Applies HARPER_DEFAULT_CONFIG (respects user edits)
+ * 4. Applies HARPER_SET_CONFIG (overrides everything)
+ * 5. Handles deletions when keys removed from env vars
+ * 6. Saves updated state and persists changes to config file (if configFilePath provided)
+ *
+ * NOTE: This function performs multiple conversions (YAML → JSON → YAML) which is not
+ * efficient but provides clear separation of concerns. The conversions are necessary
+ * to handle YAML structure conflicts (e.g., when a boolean like 'threads: true' needs
+ * to become an object like 'threads: {count: 4}').
+ *
+ * @param {Document} configDoc - YAML document to modify (mutated in place)
+ * @param {string} [configFilePath] - Path to config file (optional, skips file write if not provided)
+ * @param {Object} [options] - Options to pass to applyRuntimeEnvConfig (e.g., {isInstall: true})
+ */
+function applyRuntimeEnvVarConfig(configDoc, configFilePath, options = {}) {
+	const defaultEnvValue = process.env.HARPER_DEFAULT_CONFIG;
+	const setEnvValue = process.env.HARPER_SET_CONFIG;
+
+	// No env vars set, skip entirely (zero overhead)
+	if (!defaultEnvValue && !setEnvValue) return;
+
+	const { applyRuntimeEnvConfig } = require('./harperConfigEnvVars.ts');
+
+	// Get rootPath for state file location
+	const rootPath = configDoc.getIn(['rootPath']);
+	if (!rootPath) {
+		logger.warn('Cannot apply runtime env config: rootPath not found in config');
+		return;
+	}
+
+	// Convert to JSON for processing
+	const configObj = configDoc.toJSON();
+
+	try {
+		// Apply env vars with source tracking and drift detection
+		applyRuntimeEnvConfig(configObj, rootPath, options);
+
+		// Convert back to YAML document and write to file
+		const mergedDoc = YAML.parseDocument(YAML.stringify(configObj), { simpleKeys: true });
+		Object.assign(configDoc, mergedDoc);
+	} catch (error) {
+		logger.error(`Failed to apply runtime env config: ${error.message}`);
+		throw error;
+	}
+
+	// We're done here if no config file to write to
+	if (!configFilePath) {
+		return;
+	}
+
+	// Persist changes to file
+	try {
+		if (configDoc.errors?.length > 0) {
+			throw new Error(`Error parsing harperdb-config.yaml: ${configDoc.errors}`);
+		}
+		fs.writeFileSync(configFilePath, String(configDoc));
+		logger.debug('Config file updated with runtime env var values');
+	} catch (error) {
+		logger.error(`Failed to write config file after applying runtime env vars: ${error.message}`);
+		throw error;
+	}
+}
+
 /**
  * This function reads config settings from old settings file(before 4.0.0), aligns old keys to new keys, gets old
  * values, and updates the in-memory object.