From d4e0d80afaf4ecf3b89fa5e99f4547be46ec295e Mon Sep 17 00:00:00 2001 From: Vikrant Kumar Date: Mon, 29 Jun 2026 14:44:51 +0530 Subject: [PATCH 1/4] feat(#10706): add bulk operation document model Introduce the @medic/bulk-operations shared library holding the document model that the bulk operation framework shares across delete, move and merge: the log document (medic-logs) read by the polling endpoint, and the per-action documents (medic-sentinel) that Sentinel processes in batches. buildBulkOperation assembles both from per-action operation lists. The per-item params live in a base64 json attachment so advancing an action's cursor does not rewrite the whole list. --- shared-libs/bulk-operations/package.json | 11 +++ shared-libs/bulk-operations/src/index.js | 98 +++++++++++++++++++++ shared-libs/bulk-operations/test/index.js | 100 ++++++++++++++++++++++ 3 files changed, 209 insertions(+) create mode 100644 shared-libs/bulk-operations/package.json create mode 100644 shared-libs/bulk-operations/src/index.js create mode 100644 shared-libs/bulk-operations/test/index.js diff --git a/shared-libs/bulk-operations/package.json b/shared-libs/bulk-operations/package.json new file mode 100644 index 00000000000..24d1de4831d --- /dev/null +++ b/shared-libs/bulk-operations/package.json @@ -0,0 +1,11 @@ +{ + "name": "@medic/bulk-operations", + "version": "1.0.0", + "description": "Shared document model for the CHT-Core bulk operation framework (delete, move, merge)", + "main": "src/index.js", + "scripts": { + "test": "nyc --nycrcPath='../nyc.config.js' mocha ./test" + }, + "author": "", + "license": "Apache-2.0" +} diff --git a/shared-libs/bulk-operations/src/index.js b/shared-libs/bulk-operations/src/index.js new file mode 100644 index 00000000000..d84078cce83 --- /dev/null +++ b/shared-libs/bulk-operations/src/index.js @@ -0,0 +1,98 @@ +const { v7: uuid } = require('uuid'); + +/** + * Document model for the bulk operation framework. + * + * A bulk operation is recorded with two kinds of document: + * - a single LOG document (stored in medic-logs) that tracks overall status and is read by the polling endpoint. + * - one ACTION document per action type (stored in medic-sentinel) that Sentinel processes in batches. + * + * Delete, move and merge all express their work as a set of these actions. + */ + +const LOG_ID_PREFIX = 'bulk-operation:'; +const ACTION_ID_PREFIX = 'bulk-operation-action:'; + +const ACTIONS = { + ARCHIVE: 'archive', + SET_CONTACT: 'set-contact', + DELETE_USER: 'delete-user', +}; + +const STATUSES = { + QUEUED: 'queued', + COMPLETED: 'completed', + FAILED: 'failed', +}; + +// The per-item params for each action live in an attachment rather than inline, so updating the +// action document's cursor as Sentinel works through the batches does not rewrite the whole list. +const OPERATIONS_ATTACHMENT = 'operations'; +const OPERATIONS_CONTENT_TYPE = 'application/json'; + +const getOperationUuid = (operationId) => operationId.slice(LOG_ID_PREFIX.length); + +const generateOperationId = () => `${LOG_ID_PREFIX}${uuid()}`; + +const generateActionId = (operationId) => `${ACTION_ID_PREFIX}${getOperationUuid(operationId)}:${uuid()}`; + +const encodeOperations = (operations) => ({ + content_type: OPERATIONS_CONTENT_TYPE, + data: Buffer.from(JSON.stringify(operations)).toString('base64'), +}); + +const decodeOperations = (attachment) => JSON.parse(Buffer.from(attachment.data, 'base64').toString()); + +const buildActionDoc = (operationId, action, operations) => ({ + _id: generateActionId(operationId), + bulk_operation_id: operationId, + action, + cursor: 0, + total: operations.length, + _attachments: { + [OPERATIONS_ATTACHMENT]: encodeOperations(operations), + }, +}); + +const buildLogAction = (action, totalChangesCount, date) => ({ + status: STATUSES.QUEUED, + action, + updated_date: date, + total_changes_count: totalChangesCount, +}); + +/** + * Builds the documents that make up a single bulk operation. + * @param {Array<{action: string, operations: Object[]}>} actionOperations - one entry per action type, each holding + * its list of per-item params + * @param {Date} date - the operation start date, also the initial updated_date of every action + * @returns {{log: Object, actions: Object[]}} the log document (for medic-logs) and the action documents + * (for medic-sentinel); the log's `actions` map is keyed by each action document's `_id` + */ +const buildBulkOperation = (actionOperations, date) => { + const operationId = generateOperationId(); + const actions = actionOperations.map(({ action, operations }) => buildActionDoc(operationId, action, operations)); + + const logActions = {}; + actions.forEach((actionDoc) => { + logActions[actionDoc._id] = buildLogAction(actionDoc.action, actionDoc.total, date); + }); + + const log = { + _id: operationId, + start_date: date, + actions: logActions, + }; + + return { log, actions }; +}; + +module.exports = { + LOG_ID_PREFIX, + ACTION_ID_PREFIX, + ACTIONS, + STATUSES, + OPERATIONS_ATTACHMENT, + buildBulkOperation, + decodeOperations, +}; diff --git a/shared-libs/bulk-operations/test/index.js b/shared-libs/bulk-operations/test/index.js new file mode 100644 index 00000000000..c4fc5074008 --- /dev/null +++ b/shared-libs/bulk-operations/test/index.js @@ -0,0 +1,100 @@ +const chai = require('chai'); +const { validate: isUuid } = require('uuid'); + +const service = require('../src/index'); + +const expect = chai.expect; + +describe('bulk-operations document model', () => { + describe('constants', () => { + it('exposes the action types', () => { + expect(service.ACTIONS).to.deep.equal({ + ARCHIVE: 'archive', + SET_CONTACT: 'set-contact', + DELETE_USER: 'delete-user', + }); + }); + + it('exposes the statuses', () => { + expect(service.STATUSES).to.deep.equal({ + QUEUED: 'queued', + COMPLETED: 'completed', + FAILED: 'failed', + }); + }); + + it('exposes the document id prefixes', () => { + expect(service.LOG_ID_PREFIX).to.equal('bulk-operation:'); + expect(service.ACTION_ID_PREFIX).to.equal('bulk-operation-action:'); + }); + }); + + describe('buildBulkOperation', () => { + const date = new Date('2026-06-29T12:00:00.000Z'); + + it('builds a log document with a uuid-v7 operation id', () => { + const { log } = service.buildBulkOperation([{ action: 'archive', operations: [{ id: 'a' }] }], date); + + expect(log._id.startsWith('bulk-operation:')).to.equal(true); + expect(isUuid(log._id.slice('bulk-operation:'.length))).to.equal(true); + expect(log.start_date).to.equal(date); + }); + + it('builds one action document per action type, linked back to the operation', () => { + const groups = [ + { action: 'archive', operations: [{ id: 'person' }, { id: 'report' }] }, + { action: 'set-contact', operations: [{ id: 'place', current_contact_id: 'person' }] }, + { action: 'delete-user', operations: [{ id: 'org.couchdb.user:chw' }] }, + ]; + + const { log, actions } = service.buildBulkOperation(groups, date); + + expect(actions).to.have.length(3); + actions.forEach((actionDoc, i) => { + expect(actionDoc._id.startsWith('bulk-operation-action:')).to.equal(true); + // the action id embeds the operation's uuid, so the two ids stay parseable together + expect(actionDoc._id).to.include(log._id.slice('bulk-operation:'.length)); + expect(actionDoc.bulk_operation_id).to.equal(log._id); + expect(actionDoc.action).to.equal(groups[i].action); + expect(actionDoc.cursor).to.equal(0); + expect(actionDoc.total).to.equal(groups[i].operations.length); + expect(service.decodeOperations(actionDoc._attachments.operations)).to.deep.equal(groups[i].operations); + }); + }); + + it('stores the operation params in a base64 json attachment', () => { + const operations = [{ id: 'place', current_contact_id: 'person' }]; + const { actions } = service.buildBulkOperation([{ action: 'set-contact', operations }], date); + + const attachment = actions[0]._attachments.operations; + expect(attachment.content_type).to.equal('application/json'); + expect(attachment.data).to.be.a('string'); + expect(service.decodeOperations(attachment)).to.deep.equal(operations); + }); + + it('cross-links each action document to its entry in the log', () => { + const groups = [ + { action: 'archive', operations: [{ id: 'person' }] }, + { action: 'set-contact', operations: [{ id: 'place' }, { id: 'place2' }] }, + ]; + + const { log, actions } = service.buildBulkOperation(groups, date); + + expect(Object.keys(log.actions)).to.deep.equal(actions.map(actionDoc => actionDoc._id)); + actions.forEach((actionDoc) => { + const logAction = log.actions[actionDoc._id]; + expect(logAction.status).to.equal('queued'); + expect(logAction.action).to.equal(actionDoc.action); + expect(logAction.updated_date).to.equal(date); + expect(logAction.total_changes_count).to.equal(actionDoc.total); + }); + }); + + it('generates a distinct operation id on each call', () => { + const first = service.buildBulkOperation([{ action: 'archive', operations: [] }], date); + const second = service.buildBulkOperation([{ action: 'archive', operations: [] }], date); + + expect(first.log._id).to.not.equal(second.log._id); + }); + }); +}); From 25f8e201f979873c037edae3319c24f21f00b1e9 Mon Sep 17 00:00:00 2001 From: Vikrant Kumar Date: Mon, 29 Jun 2026 15:04:25 +0530 Subject: [PATCH 2/4] build(#10706): add @medic/bulk-operations to package-lock --- package-lock.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/package-lock.json b/package-lock.json index 53a1d09fb49..f16699ff667 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5750,6 +5750,10 @@ "resolved": "shared-libs/bulk-docs-utils", "link": true }, + "node_modules/@medic/bulk-operations": { + "resolved": "shared-libs/bulk-operations", + "link": true + }, "node_modules/@medic/calendar-interval": { "resolved": "shared-libs/calendar-interval", "link": true @@ -44275,6 +44279,11 @@ "version": "1.0.0", "license": "Apache-2.0" }, + "shared-libs/bulk-operations": { + "name": "@medic/bulk-operations", + "version": "1.0.0", + "license": "Apache-2.0" + }, "shared-libs/calendar-interval": { "name": "@medic/calendar-interval", "version": "1.0.0", From f0318edd68f68b4447024e6ecf8e2561321c2c5e Mon Sep 17 00:00:00 2001 From: Vikrant Kumar Date: Mon, 29 Jun 2026 16:00:54 +0530 Subject: [PATCH 3/4] feat(#10706): add bulk operation status endpoint Add GET /api/v1/bulk-operations/{id}, which returns the bulk operation log document from the medic-logs database so a caller can poll the progress of a delete, move or merge it started. The service only resolves ids carrying the bulk-operation prefix, so the endpoint cannot be used to read the other log documents that share the database. Online users only; missing or non-bulk-operation ids return 404. --- api/src/controllers/bulk-operations.js | 66 ++++++++++++++++++ api/src/routing.js | 3 + api/src/services/bulk-operations.js | 25 +++++++ .../mocha/controllers/bulk-operations.spec.js | 68 +++++++++++++++++++ .../mocha/services/bulk-operations.spec.js | 64 +++++++++++++++++ 5 files changed, 226 insertions(+) create mode 100644 api/src/controllers/bulk-operations.js create mode 100644 api/src/services/bulk-operations.js create mode 100644 api/tests/mocha/controllers/bulk-operations.spec.js create mode 100644 api/tests/mocha/services/bulk-operations.spec.js diff --git a/api/src/controllers/bulk-operations.js b/api/src/controllers/bulk-operations.js new file mode 100644 index 00000000000..b4c88da9a8d --- /dev/null +++ b/api/src/controllers/bulk-operations.js @@ -0,0 +1,66 @@ +const service = require('../services/bulk-operations'); +const serverUtils = require('../server-utils'); +const auth = require('../auth'); + +/** + * @openapi + * tags: + * - name: Bulk operations + * description: Status of long-running bulk operations (delete, move, merge) + */ +module.exports = { + v1: { + /** + * @openapi + * /api/v1/bulk-operations/{id}: + * get: + * summary: Get the status of a bulk operation + * operationId: v1BulkOperationIdGet + * description: > + * Returns the log document for a bulk operation, including the per-action status and the + * count of changes applied so far. Used to poll the progress of an operation that was + * started through one of the bulk endpoints. + * tags: [Bulk operations] + * x-since: 5.1.0 + * parameters: + * - in: path + * name: id + * required: true + * schema: + * type: string + * description: The id of the bulk operation, as returned when it was started. + * responses: + * '200': + * description: The bulk operation log + * content: + * application/json: + * schema: + * type: object + * properties: + * _id: + * type: string + * description: The bulk operation id. + * start_date: + * type: string + * format: date-time + * description: When the operation was started. + * actions: + * type: object + * description: Per-action status, keyed by action id. + * '401': + * $ref: '#/components/responses/Unauthorized' + * '403': + * $ref: '#/components/responses/Forbidden' + * '404': + * $ref: '#/components/responses/NotFound' + */ + get: serverUtils.doOrError(async (req, res) => { + await auth.assertPermissions(req, { isOnline: true }); + const log = await service.getLog(req.params.id); + if (!log) { + return serverUtils.error({ status: 404, message: 'Bulk operation not found' }, req, res); + } + res.json(log); + }) + } +}; diff --git a/api/src/routing.js b/api/src/routing.js index 370f0f2cf3a..a3f55b50ac6 100644 --- a/api/src/routing.js +++ b/api/src/routing.js @@ -46,6 +46,7 @@ const { people, places } = require('@medic/contacts')(config, db, dataContext); const upgrade = require('./controllers/upgrade'); const settings = require('./controllers/settings'); const bulkDocs = require('./controllers/bulk-docs'); +const bulkOperations = require('./controllers/bulk-operations'); const monitoring = require('./controllers/monitoring'); const africasTalking = require('./controllers/africas-talking'); const rapidPro = require('./controllers/rapidpro'); @@ -478,6 +479,8 @@ app.get('/api/v1/monitoring', deprecation.deprecate('/api/v2/monitoring'), monit app.get('/api/v2/monitoring', monitoring.getV2); app.get('/api/v1/impact', impact.v1.get); +app.get('/api/v1/bulk-operations/:id', bulkOperations.v1.get); + app.post('/api/v1/upgrade', jsonParser, upgrade.upgrade); app.post('/api/v1/upgrade/stage', jsonParser, upgrade.stage); app.post('/api/v1/upgrade/complete', jsonParser, upgrade.complete); diff --git a/api/src/services/bulk-operations.js b/api/src/services/bulk-operations.js new file mode 100644 index 00000000000..392bbf05d86 --- /dev/null +++ b/api/src/services/bulk-operations.js @@ -0,0 +1,25 @@ +const db = require('../db'); +const { LOG_ID_PREFIX } = require('@medic/bulk-operations'); + +// Reads only bulk operation log documents from the medic-logs database. The prefix guard keeps the +// endpoint from returning the other kinds of log documents that share this database. +const getLog = async (id) => { + if (!id || !id.startsWith(LOG_ID_PREFIX)) { + return null; + } + + try { + const log = await db.medicLogs.get(id); + delete log._rev; + return log; + } catch (err) { + if (err.status === 404) { + return null; + } + throw err; + } +}; + +module.exports = { + getLog, +}; diff --git a/api/tests/mocha/controllers/bulk-operations.spec.js b/api/tests/mocha/controllers/bulk-operations.spec.js new file mode 100644 index 00000000000..fbfd2a2547c --- /dev/null +++ b/api/tests/mocha/controllers/bulk-operations.spec.js @@ -0,0 +1,68 @@ +const sinon = require('sinon'); +const chai = require('chai'); + +const serverUtils = require('../../../src/server-utils'); +const controller = require('../../../src/controllers/bulk-operations'); +const service = require('../../../src/services/bulk-operations'); +const auth = require('../../../src/auth'); +const { PermissionError } = require('../../../src/errors'); + +describe('Bulk operations controller', () => { + let req; + let res; + + beforeEach(() => { + sinon.stub(serverUtils, 'error'); + req = { params: { id: 'bulk-operation:abc' } }; + res = { json: sinon.stub() }; + }); + + afterEach(() => sinon.restore()); + + describe('v1 get', () => { + it('returns the bulk operation log for an authorised online user', () => { + const log = { _id: 'bulk-operation:abc', start_date: 'date', actions: {} }; + sinon.stub(auth, 'assertPermissions').resolves(); + sinon.stub(service, 'getLog').resolves(log); + + return controller.v1.get(req, res).then(() => { + chai.expect(auth.assertPermissions.calledOnceWithExactly(req, { isOnline: true })).to.equal(true); + chai.expect(service.getLog.calledOnceWithExactly('bulk-operation:abc')).to.equal(true); + chai.expect(res.json.calledOnceWithExactly(log)).to.equal(true); + chai.expect(serverUtils.error.called).to.equal(false); + }); + }); + + it('returns a 404 when the operation is not found', () => { + sinon.stub(auth, 'assertPermissions').resolves(); + sinon.stub(service, 'getLog').resolves(null); + + return controller.v1.get(req, res).then(() => { + chai.expect(res.json.called).to.equal(false); + chai.expect(serverUtils.error.calledOnce).to.equal(true); + chai.expect(serverUtils.error.args[0][0]).to.deep.equal({ status: 404, message: 'Bulk operation not found' }); + }); + }); + + it('does not reach the service when the user is not permitted', () => { + sinon.stub(auth, 'assertPermissions').rejects(new PermissionError('Insufficient privileges')); + sinon.stub(service, 'getLog').resolves({}); + + return controller.v1.get(req, res).then(() => { + chai.expect(service.getLog.called).to.equal(false); + chai.expect(res.json.called).to.equal(false); + chai.expect(serverUtils.error.calledOnce).to.equal(true); + }); + }); + + it('handles a service rejection gracefully', () => { + sinon.stub(auth, 'assertPermissions').resolves(); + sinon.stub(service, 'getLog').rejects(new Error('db down')); + + return controller.v1.get(req, res).then(() => { + chai.expect(res.json.called).to.equal(false); + chai.expect(serverUtils.error.calledOnce).to.equal(true); + }); + }); + }); +}); diff --git a/api/tests/mocha/services/bulk-operations.spec.js b/api/tests/mocha/services/bulk-operations.spec.js new file mode 100644 index 00000000000..e6043e22315 --- /dev/null +++ b/api/tests/mocha/services/bulk-operations.spec.js @@ -0,0 +1,64 @@ +const chai = require('chai'); +const sinon = require('sinon'); + +const db = require('../../../src/db'); +const service = require('../../../src/services/bulk-operations'); + +describe('Bulk operations service', () => { + afterEach(() => sinon.restore()); + + describe('getLog', () => { + it('returns the log document without the couch _rev', () => { + const doc = { + _id: 'bulk-operation:abc', + _rev: '1-xyz', + start_date: 'date', + actions: { 'bulk-operation-action:abc:1': { status: 'queued' } }, + }; + sinon.stub(db.medicLogs, 'get').resolves(doc); + + return service.getLog('bulk-operation:abc').then((log) => { + chai.expect(db.medicLogs.get.calledOnceWithExactly('bulk-operation:abc')).to.equal(true); + chai.expect(log).to.deep.equal({ + _id: 'bulk-operation:abc', + start_date: 'date', + actions: { 'bulk-operation-action:abc:1': { status: 'queued' } }, + }); + chai.expect(log._rev).to.equal(undefined); + }); + }); + + it('returns null when the operation does not exist', () => { + sinon.stub(db.medicLogs, 'get').rejects({ status: 404 }); + + return service.getLog('bulk-operation:missing').then((log) => { + chai.expect(log).to.equal(null); + }); + }); + + it('does not query the database for an id that is not a bulk operation', () => { + const get = sinon.stub(db.medicLogs, 'get'); + + return Promise + .all([ + service.getLog(undefined), + service.getLog(''), + service.getLog('upgrade_log:something'), + service.getLog('some-other-doc'), + ]) + .then((results) => { + chai.expect(results).to.deep.equal([null, null, null, null]); + chai.expect(get.called).to.equal(false); + }); + }); + + it('rethrows errors that are not a 404', () => { + sinon.stub(db.medicLogs, 'get').rejects({ status: 500 }); + + return service + .getLog('bulk-operation:boom') + .then(() => chai.expect.fail('should have thrown')) + .catch((err) => chai.expect(err.status).to.equal(500)); + }); + }); +}); From b8db4cd1ec107cf5484a74b25a033b0aad4b062c Mon Sep 17 00:00:00 2001 From: Vikrant Kumar Date: Mon, 29 Jun 2026 16:09:27 +0530 Subject: [PATCH 4/4] refactor(#10706): use optional chaining in bulk operation id guard --- api/src/services/bulk-operations.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/services/bulk-operations.js b/api/src/services/bulk-operations.js index 392bbf05d86..332384932dd 100644 --- a/api/src/services/bulk-operations.js +++ b/api/src/services/bulk-operations.js @@ -4,7 +4,7 @@ const { LOG_ID_PREFIX } = require('@medic/bulk-operations'); // Reads only bulk operation log documents from the medic-logs database. The prefix guard keeps the // endpoint from returning the other kinds of log documents that share this database. const getLog = async (id) => { - if (!id || !id.startsWith(LOG_ID_PREFIX)) { + if (!id?.startsWith(LOG_ID_PREFIX)) { return null; }