feat(#10706): add bulk operation status endpoint

api/src/controllers/bulk-operations.js

@@ -0,0 +1,66 @@
+const service = require('../services/bulk-operations');
+const serverUtils = require('../server-utils');
+const auth = require('../auth');
+
+/**
+ * @openapi
+ * tags:
+ *   - name: Bulk operations
+ *     description: Status of long-running bulk operations (delete, move, merge)
+ */
+module.exports = {
+  v1: {
+    /**
+     * @openapi
+     * /api/v1/bulk-operations/{id}:
+     *   get:
+     *     summary: Get the status of a bulk operation
+     *     operationId: v1BulkOperationIdGet
+     *     description: >
+     *       Returns the log document for a bulk operation, including the per-action status and the
+     *       count of changes applied so far. Used to poll the progress of an operation that was
+     *       started through one of the bulk endpoints.
+     *     tags: [Bulk operations]
+     *     x-since: 5.1.0
+     *     parameters:
+     *       - in: path
+     *         name: id
+     *         required: true
+     *         schema:
+     *           type: string
+     *         description: The id of the bulk operation, as returned when it was started.
+     *     responses:
+     *       '200':
+     *         description: The bulk operation log
+     *         content:
+     *           application/json:
+     *             schema:
+     *               type: object
+     *               properties:
+     *                 _id:
+     *                   type: string
+     *                   description: The bulk operation id.
+     *                 start_date:
+     *                   type: string
+     *                   format: date-time
+     *                   description: When the operation was started.
+     *                 actions:
+     *                   type: object
+     *                   description: Per-action status, keyed by action id.
+     *       '401':
+     *         $ref: '#/components/responses/Unauthorized'
+     *       '403':
+     *         $ref: '#/components/responses/Forbidden'
+     *       '404':
+     *         $ref: '#/components/responses/NotFound'
+     */
+    get: serverUtils.doOrError(async (req, res) => {
+      await auth.assertPermissions(req, { isOnline: true });
+      const log = await service.getLog(req.params.id);
+      if (!log) {
+        return serverUtils.error({ status: 404, message: 'Bulk operation not found' }, req, res);
+      }
+      res.json(log);
+    })
+  }
+};

api/src/routing.js

@@ -46,6 +46,7 @@ const { people, places } = require('@medic/contacts')(config, db, dataContext);
 const upgrade = require('./controllers/upgrade');
 const settings = require('./controllers/settings');
 const bulkDocs = require('./controllers/bulk-docs');
+const bulkOperations = require('./controllers/bulk-operations');
 const monitoring = require('./controllers/monitoring');
 const africasTalking = require('./controllers/africas-talking');
 const rapidPro = require('./controllers/rapidpro');
@@ -478,6 +479,8 @@ app.get('/api/v1/monitoring', deprecation.deprecate('/api/v2/monitoring'), monit
 app.get('/api/v2/monitoring', monitoring.getV2);
 app.get('/api/v1/impact', impact.v1.get);
 
+app.get('/api/v1/bulk-operations/:id', bulkOperations.v1.get);
+
 app.post('/api/v1/upgrade', jsonParser, upgrade.upgrade);
 app.post('/api/v1/upgrade/stage', jsonParser, upgrade.stage);
 app.post('/api/v1/upgrade/complete', jsonParser, upgrade.complete);

api/src/services/bulk-operations.js

@@ -0,0 +1,25 @@
+const db = require('../db');
+const { LOG_ID_PREFIX } = require('@medic/bulk-operations');
+
+// Reads only bulk operation log documents from the medic-logs database. The prefix guard keeps the
+// endpoint from returning the other kinds of log documents that share this database.
+const getLog = async (id) => {
+  if (!id?.startsWith(LOG_ID_PREFIX)) {
+    return null;
+  }
+
+  try {
+    const log = await db.medicLogs.get(id);
+    delete log._rev;
+    return log;
+  } catch (err) {
+    if (err.status === 404) {
+      return null;
+    }
+    throw err;
+  }
+};
+
+module.exports = {
+  getLog,
+};

api/tests/mocha/controllers/bulk-operations.spec.js

@@ -0,0 +1,68 @@
+const sinon = require('sinon');
+const chai = require('chai');
+
+const serverUtils = require('../../../src/server-utils');
+const controller = require('../../../src/controllers/bulk-operations');
+const service = require('../../../src/services/bulk-operations');
+const auth = require('../../../src/auth');
+const { PermissionError } = require('../../../src/errors');
+
+describe('Bulk operations controller', () => {
+  let req;
+  let res;
+
+  beforeEach(() => {
+    sinon.stub(serverUtils, 'error');
+    req = { params: { id: 'bulk-operation:abc' } };
+    res = { json: sinon.stub() };
+  });
+
+  afterEach(() => sinon.restore());
+
+  describe('v1 get', () => {
+    it('returns the bulk operation log for an authorised online user', () => {
+      const log = { _id: 'bulk-operation:abc', start_date: 'date', actions: {} };
+      sinon.stub(auth, 'assertPermissions').resolves();
+      sinon.stub(service, 'getLog').resolves(log);
+
+      return controller.v1.get(req, res).then(() => {
+        chai.expect(auth.assertPermissions.calledOnceWithExactly(req, { isOnline: true })).to.equal(true);
+        chai.expect(service.getLog.calledOnceWithExactly('bulk-operation:abc')).to.equal(true);
+        chai.expect(res.json.calledOnceWithExactly(log)).to.equal(true);
+        chai.expect(serverUtils.error.called).to.equal(false);
+      });
+    });
+
+    it('returns a 404 when the operation is not found', () => {
+      sinon.stub(auth, 'assertPermissions').resolves();
+      sinon.stub(service, 'getLog').resolves(null);
+
+      return controller.v1.get(req, res).then(() => {
+        chai.expect(res.json.called).to.equal(false);
+        chai.expect(serverUtils.error.calledOnce).to.equal(true);
+        chai.expect(serverUtils.error.args[0][0]).to.deep.equal({ status: 404, message: 'Bulk operation not found' });
+      });
+    });
+
+    it('does not reach the service when the user is not permitted', () => {
+      sinon.stub(auth, 'assertPermissions').rejects(new PermissionError('Insufficient privileges'));
+      sinon.stub(service, 'getLog').resolves({});
+
+      return controller.v1.get(req, res).then(() => {
+        chai.expect(service.getLog.called).to.equal(false);
+        chai.expect(res.json.called).to.equal(false);
+        chai.expect(serverUtils.error.calledOnce).to.equal(true);
+      });
+    });
+
+    it('handles a service rejection gracefully', () => {
+      sinon.stub(auth, 'assertPermissions').resolves();
+      sinon.stub(service, 'getLog').rejects(new Error('db down'));
+
+      return controller.v1.get(req, res).then(() => {
+        chai.expect(res.json.called).to.equal(false);
+        chai.expect(serverUtils.error.calledOnce).to.equal(true);
+      });
+    });
+  });
+});

api/tests/mocha/services/bulk-operations.spec.js

@@ -0,0 +1,64 @@
+const chai = require('chai');
+const sinon = require('sinon');
+
+const db = require('../../../src/db');
+const service = require('../../../src/services/bulk-operations');
+
+describe('Bulk operations service', () => {
+  afterEach(() => sinon.restore());
+
+  describe('getLog', () => {
+    it('returns the log document without the couch _rev', () => {
+      const doc = {
+        _id: 'bulk-operation:abc',
+        _rev: '1-xyz',
+        start_date: 'date',
+        actions: { 'bulk-operation-action:abc:1': { status: 'queued' } },
+      };
+      sinon.stub(db.medicLogs, 'get').resolves(doc);
+
+      return service.getLog('bulk-operation:abc').then((log) => {
+        chai.expect(db.medicLogs.get.calledOnceWithExactly('bulk-operation:abc')).to.equal(true);
+        chai.expect(log).to.deep.equal({
+          _id: 'bulk-operation:abc',
+          start_date: 'date',
+          actions: { 'bulk-operation-action:abc:1': { status: 'queued' } },
+        });
+        chai.expect(log._rev).to.equal(undefined);
+      });
+    });
+
+    it('returns null when the operation does not exist', () => {
+      sinon.stub(db.medicLogs, 'get').rejects({ status: 404 });
+
+      return service.getLog('bulk-operation:missing').then((log) => {
+        chai.expect(log).to.equal(null);
+      });
+    });
+
+    it('does not query the database for an id that is not a bulk operation', () => {
+      const get = sinon.stub(db.medicLogs, 'get');
+
+      return Promise
+        .all([
+          service.getLog(undefined),
+          service.getLog(''),
+          service.getLog('upgrade_log:something'),
+          service.getLog('some-other-doc'),
+        ])
+        .then((results) => {
+          chai.expect(results).to.deep.equal([null, null, null, null]);
+          chai.expect(get.called).to.equal(false);
+        });
+    });
+
+    it('rethrows errors that are not a 404', () => {
+      sinon.stub(db.medicLogs, 'get').rejects({ status: 500 });
+
+      return service
+        .getLog('bulk-operation:boom')
+        .then(() => chai.expect.fail('should have thrown'))
+        .catch((err) => chai.expect(err.status).to.equal(500));
+    });
+  });
+});

shared-libs/bulk-operations/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "@medic/bulk-operations",
+  "version": "1.0.0",
+  "description": "Shared document model for the CHT-Core bulk operation framework (delete, move, merge)",
+  "main": "src/index.js",
+  "scripts": {
+    "test": "nyc --nycrcPath='../nyc.config.js' mocha ./test"
+  },
+  "author": "",
+  "license": "Apache-2.0"
+}

shared-libs/bulk-operations/src/index.js

@@ -0,0 +1,98 @@
+const { v7: uuid } = require('uuid');
+
+/**
+ * Document model for the bulk operation framework.
+ *
+ * A bulk operation is recorded with two kinds of document:
+ *  - a single LOG document (stored in medic-logs) that tracks overall status and is read by the polling endpoint.
+ *  - one ACTION document per action type (stored in medic-sentinel) that Sentinel processes in batches.
+ *
+ * Delete, move and merge all express their work as a set of these actions.
+ */
+
+const LOG_ID_PREFIX = 'bulk-operation:';
+const ACTION_ID_PREFIX = 'bulk-operation-action:';
+
+const ACTIONS = {
+  ARCHIVE: 'archive',
+  SET_CONTACT: 'set-contact',
+  DELETE_USER: 'delete-user',
+};
+
+const STATUSES = {
+  QUEUED: 'queued',
+  COMPLETED: 'completed',
+  FAILED: 'failed',
+};
+
+// The per-item params for each action live in an attachment rather than inline, so updating the
+// action document's cursor as Sentinel works through the batches does not rewrite the whole list.
+const OPERATIONS_ATTACHMENT = 'operations';
+const OPERATIONS_CONTENT_TYPE = 'application/json';
+
+const getOperationUuid = (operationId) => operationId.slice(LOG_ID_PREFIX.length);
+
+const generateOperationId = () => `${LOG_ID_PREFIX}${uuid()}`;
+
+const generateActionId = (operationId) => `${ACTION_ID_PREFIX}${getOperationUuid(operationId)}:${uuid()}`;
+
+const encodeOperations = (operations) => ({
+  content_type: OPERATIONS_CONTENT_TYPE,
+  data: Buffer.from(JSON.stringify(operations)).toString('base64'),
+});
+
+const decodeOperations = (attachment) => JSON.parse(Buffer.from(attachment.data, 'base64').toString());
+
+const buildActionDoc = (operationId, action, operations) => ({
+  _id: generateActionId(operationId),
+  bulk_operation_id: operationId,
+  action,
+  cursor: 0,
+  total: operations.length,
+  _attachments: {
+    [OPERATIONS_ATTACHMENT]: encodeOperations(operations),
+  },
+});
+
+const buildLogAction = (action, totalChangesCount, date) => ({
+  status: STATUSES.QUEUED,
+  action,
+  updated_date: date,
+  total_changes_count: totalChangesCount,
+});
+
+/**
+ * Builds the documents that make up a single bulk operation.
+ * @param {Array<{action: string, operations: Object[]}>} actionOperations - one entry per action type, each holding
+ *   its list of per-item params
+ * @param {Date} date - the operation start date, also the initial updated_date of every action
+ * @returns {{log: Object, actions: Object[]}} the log document (for medic-logs) and the action documents
+ *   (for medic-sentinel); the log's `actions` map is keyed by each action document's `_id`
+ */
+const buildBulkOperation = (actionOperations, date) => {
+  const operationId = generateOperationId();
+  const actions = actionOperations.map(({ action, operations }) => buildActionDoc(operationId, action, operations));
+
+  const logActions = {};
+  actions.forEach((actionDoc) => {
+    logActions[actionDoc._id] = buildLogAction(actionDoc.action, actionDoc.total, date);
+  });
+
+  const log = {
+    _id: operationId,
+    start_date: date,
+    actions: logActions,
+  };
+
+  return { log, actions };
+};
+
+module.exports = {
+  LOG_ID_PREFIX,
+  ACTION_ID_PREFIX,
+  ACTIONS,
+  STATUSES,
+  OPERATIONS_ATTACHMENT,
+  buildBulkOperation,
+  decodeOperations,
+};