From e5065e5937b7166406ce8b3e8a934615d5a7493f Mon Sep 17 00:00:00 2001 From: martinb Date: Sat, 20 Feb 2021 13:41:22 +0100 Subject: [PATCH 1/3] introduce support for jsonlines input files --- .gitignore | 1 + package-lock.json | 22 ++++++++++++++++++---- package.json | 1 + src/getopts.js | 12 +++++++++++- src/jfq.js | 33 ++++++++++++++++++++++++++++++--- 5 files changed, 61 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index ed4598e7..0d4d8c3c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules bin +.playground diff --git a/package-lock.json b/package-lock.json index 88d8aa10..8d922fad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5613,7 +5613,8 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/growly/-/growly-1.3.0.tgz", "integrity": "sha1-8QdIy+dq+WS3yWyTxrzCivEgwIE=", - "dev": true + "dev": true, + "optional": true }, "har-schema": { "version": "2.0.0", @@ -6062,6 +6063,7 @@ "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", "dev": true, + "optional": true, "requires": { "is-docker": "^2.0.0" } @@ -7963,6 +7965,11 @@ "type-check": "~0.3.2" } }, + "line-reader": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/line-reader/-/line-reader-0.4.0.tgz", + "integrity": "sha1-F+RIGNoKwzVnW6MAlU+U72cOZv0=" + }, "lines-and-columns": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.1.6.tgz", @@ -8044,6 +8051,7 @@ "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "dev": true, + "optional": true, "requires": { "yallist": "^4.0.0" } @@ -8330,6 +8338,7 @@ "resolved": "https://registry.npmjs.org/node-notifier/-/node-notifier-8.0.1.tgz", "integrity": "sha512-BvEXF+UmsnAfYfoapKM9nGxnP+Wn7P91YfXmrKnfcYCx6VBeoN5Ez5Ogck6I8Bi5k4RlpqRYaw75pAwzX9OphA==", "dev": true, + "optional": true, "requires": { "growly": "^1.3.0", "is-wsl": "^2.2.0", @@ -8344,6 +8353,7 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz", "integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==", "dev": true, + "optional": true, "requires": { "lru-cache": "^6.0.0" } @@ -8352,13 +8362,15 @@ "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "dev": true + "dev": true, + "optional": true }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", "dev": true, + "optional": true, "requires": { "isexe": "^2.0.0" } @@ -9351,7 +9363,8 @@ "version": "0.1.1", "resolved": "https://registry.npmjs.org/shellwords/-/shellwords-0.1.1.tgz", "integrity": "sha512-vFwSUfQvqybiICwZY5+DAWIPLKsWO31Q91JSKl3UYv+K5c2QRPzn0qzec6QPu1Qc9eHYItiP3NdJqNVqetYAww==", - "dev": true + "dev": true, + "optional": true }, "side-channel": { "version": "1.0.3", @@ -10516,7 +10529,8 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "dev": true, + "optional": true }, "yargs": { "version": "15.4.1", diff --git a/package.json b/package.json index bc120d6a..df877678 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "js-yaml": "^4.0.0", "json-colorizer": "^2.2.2", "jsonata": "^1.8.4", + "line-reader": "^0.4.0", "parse-json": "^5.2.0", "read-input": "^0.3.1" }, diff --git a/src/getopts.js b/src/getopts.js index c86ed8da..cfecc389 100644 --- a/src/getopts.js +++ b/src/getopts.js @@ -10,6 +10,7 @@ export default async argv => { .option('-a, --accept-yaml', 'YAML input') .option('-p, --plain-text', 'Do not decorate output') .option('-q, --query-file ', 'JSONata query file') + .option('-l, --jsonlines', 'JSON Lines Input') .parse(argv) .opts() @@ -19,10 +20,19 @@ export default async argv => { const yamlIn = !!options.acceptYaml const plainText = !!options.plainText const queryFile = options.queryFile + const jsonlines = !!options.jsonlines const files = program.args.slice(0) + if (yamlIn && jsonlines) { + throw new Error('--accept-yaml and --jsonlines can not be set together') + } + + if (jsonlines && !ndjson) { + throw new Error('--jsonlines requires --ndjson (Newline Delimited JSON) as output') + } + const query = await getQuery(queryFile, files) - return { query, files, ndjson, json, yamlOut, yamlIn, plainText } + return { query, files, ndjson, json, yamlOut, yamlIn, plainText, jsonlines } } const exists = async (path) => { diff --git a/src/jfq.js b/src/jfq.js index 845eaca0..776a41e8 100644 --- a/src/jfq.js +++ b/src/jfq.js @@ -2,12 +2,13 @@ import colorize from 'json-colorizer' import jsonata from 'jsonata' import parseJson from 'parse-json' import readInput from 'read-input' +import * as lineReader from 'line-reader' import getopts from './getopts' import YAML from 'js-yaml' -const main = async () => { - const { files, ndjson, json, yamlIn, yamlOut, query, plainText } = await getopts(process.argv) - const evaluator = parseQuery(query) +const fullFileMode = async (opts, evaluator) => { + const { files, ndjson, json, yamlIn, yamlOut, plainText } = opts + const data = await readInput(files) data.files.forEach(file => { @@ -22,6 +23,32 @@ const main = async () => { }) } +const streamingMode = async (opts, evaluator) => { + const { files, ndjson, json, plainText } = opts + if (files && files.length > 1) { + console.error('streamingMode / jsonlines only works with one file/stream') + process.exit(1) + } + const input = (files && files[0]) || process.stdin + lineReader.eachLine(input, (line, last, continueCb) => { + const lineInput = parseJson(line) + const result = evaluator.evaluate(lineInput) + const output = formatJson(result, ndjson, json, plainText) + console.log(output) + continueCb() + }) +} + +const main = async () => { + const opts = await getopts(process.argv) + const evaluator = parseQuery(opts.query) + if (!opts.jsonlines) { + await fullFileMode(opts, evaluator) + } else { + await streamingMode(opts, evaluator) + } +} + const parseQuery = query => { try { return jsonata(query) From cc3a76c09e7bbd22ff08e9aa51ee0886f7ab7f48 Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Fri, 5 Mar 2021 00:00:18 +0100 Subject: [PATCH 2/3] wip on refactoring according to pr --- babel.config.js | 12 ++++++++++++ package.json | 2 +- src/__tests__/getopts.js | 9 +++++++++ src/getopts.js | 34 +++++++++++++++++++++++++++------- src/jfq.js | 2 +- 5 files changed, 50 insertions(+), 9 deletions(-) create mode 100644 babel.config.js diff --git a/babel.config.js b/babel.config.js new file mode 100644 index 00000000..c76a0d02 --- /dev/null +++ b/babel.config.js @@ -0,0 +1,12 @@ +module.exports = { + presets: [ + [ + '@babel/preset-env', + { + targets: { + node: 'current' + } + } + ] + ] +} diff --git a/package.json b/package.json index df877678..8ea1aeb1 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "lint": "standard --verbose --fix", "posttest": "ls -1 *.md | xargs -t -n 1 markdown-link-check", "prepublishOnly": "npm run build", - "pretest": "npm run lint && npm run build", + "pretest": "rm -f ./bin/jfq.js && npm run lint && npm run build", "test": "jest" }, "repository": { diff --git a/src/__tests__/getopts.js b/src/__tests__/getopts.js index c1c694eb..950e10a0 100644 --- a/src/__tests__/getopts.js +++ b/src/__tests__/getopts.js @@ -4,12 +4,21 @@ import getopts from '../getopts.js' const fakeArgv = (...opts) => ([process.execPath, '/tmp/fakePath.js', ...opts]) describe('getting command line options', () => { + describe('when --yaml is used together with --jsonlines-input', () => { + it('should throw', () => { + expect(() => { + getopts(fakeArgv('--yaml', '--jsonlines-input', '--jsonlines-output')) + }).toThrow(Error) + }) + }) + describe('when there are no options', () => { it('returns default values', async () => { const res = await getopts(fakeArgv()) expect(res.query).toBe('$') expect(res.files).toEqual([]) expect(res.ndjson).toBe(false) + expect(res.jsonlinesInput).toBe(false) }) }) diff --git a/src/getopts.js b/src/getopts.js index cfecc389..8754991a 100644 --- a/src/getopts.js +++ b/src/getopts.js @@ -10,29 +10,49 @@ export default async argv => { .option('-a, --accept-yaml', 'YAML input') .option('-p, --plain-text', 'Do not decorate output') .option('-q, --query-file ', 'JSONata query file') - .option('-l, --jsonlines', 'JSON Lines Input') + .option('-l, --jsonlines-input', 'JSON Lines Input') + .option('-k, --jsonlines-output', 'JSON Lines Output') .parse(argv) .opts() - const ndjson = !!options.ndjson + const ndjsonOpt = !!options.ndjson + const jsonlinesOutputOpt = !!options.jsonlinesOutput + + if (ndjsonOpt && jsonlinesOutputOpt) { + throw new Error('can only set --ndjson or --jsonlines-output') + } + const json = !!options.json const yamlOut = !!options.yaml const yamlIn = !!options.acceptYaml const plainText = !!options.plainText const queryFile = options.queryFile - const jsonlines = !!options.jsonlines + // two aliases + const jsonlinesInput = !!options.jsonlinesInput + const jsonlinesOutput = ndjsonOpt || jsonlinesOutputOpt + const ndjson = jsonlinesOutput const files = program.args.slice(0) - if (yamlIn && jsonlines) { + if (yamlIn && jsonlinesInput) { throw new Error('--accept-yaml and --jsonlines can not be set together') } - if (jsonlines && !ndjson) { - throw new Error('--jsonlines requires --ndjson (Newline Delimited JSON) as output') + if (jsonlinesInput && !jsonlinesOutput) { + throw new Error('--jsonlines-input requires --jsonlines-output/ndjson as output') } const query = await getQuery(queryFile, files) - return { query, files, ndjson, json, yamlOut, yamlIn, plainText, jsonlines } + return { + query, + files, + ndjson, + json, + yamlOut, + yamlIn, + plainText, + jsonlinesInput, + jsonlinesOutput + } } const exists = async (path) => { diff --git a/src/jfq.js b/src/jfq.js index 776a41e8..e4d8ebb8 100644 --- a/src/jfq.js +++ b/src/jfq.js @@ -42,7 +42,7 @@ const streamingMode = async (opts, evaluator) => { const main = async () => { const opts = await getopts(process.argv) const evaluator = parseQuery(opts.query) - if (!opts.jsonlines) { + if (!opts.jsonlinesInput) { await fullFileMode(opts, evaluator) } else { await streamingMode(opts, evaluator) From 06ff3f33b64165e04c8d08b11129fe80c938fafc Mon Sep 17 00:00:00 2001 From: Martin Braun Date: Fri, 5 Mar 2021 00:11:13 +0100 Subject: [PATCH 3/3] minor --- rollup.config.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rollup.config.js b/rollup.config.js index f6d245cc..8f8c256a 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -20,6 +20,7 @@ export default { 'json-colorizer', 'js-yaml', 'parse-json', - 'read-input' + 'read-input', + 'line-reader' ] }