diff --git a/configure.ac b/configure.ac index bc1d27317a..9a986e432a 100644 --- a/configure.ac +++ b/configure.ac @@ -40,6 +40,7 @@ if test "$USE_MAINTAINER_MODE" = yes; then fi AC_CHECK_FUNCS(memmem) +AC_CHECK_FUNCS(mkstemp) AC_CHECK_HEADER("sys/cygwin.h", [have_cygwin=1;]) AC_CHECK_HEADER("shlwapi.h",[have_shlwapi=1;]) diff --git a/docs/content/manual/dev/manual.yml b/docs/content/manual/dev/manual.yml index 257236c1b5..571c68775c 100644 --- a/docs/content/manual/dev/manual.yml +++ b/docs/content/manual/dev/manual.yml @@ -227,6 +227,25 @@ sections: like awk's -f option. This changes the filter argument to be interpreted as a filename, instead of the source of a program. + * `-o` / `--output-file filename`: + + Write output values to the named file instead of standard out. + The output is first written to a temporary file in the same + directory, then renamed into place on success. This makes it + safe to use an input file as the output file (e.g., + `jq '.x += 1' f.json -o f.json`). + + The outputs from `--debug-dump-disasm` and `--debug-trace` are + still written to standard out, so this option can be used to + separate them. + + * `-O` / `--clobber-output filename`: + + Like `-o`, but truncates the output file immediately instead + of writing to a temporary file first. This is faster, but + the output file must not also be an input file, and partial + output may be left behind if jq is interrupted. + * `-L directory` / `--library-path directory`: Prepend `directory` to the search list for modules. If this diff --git a/jq.1.prebuilt b/jq.1.prebuilt index 6b5931a83e..ddf1950ef7 100644 --- a/jq.1.prebuilt +++ b/jq.1.prebuilt @@ -1,5 +1,5 @@ . -.TH "JQ" "1" "May 2025" "" "" +.TH "JQ" "1" "February 2026" "" "" . .SH "NAME" \fBjq\fR \- Command\-line JSON processor @@ -169,6 +169,21 @@ Use the \fBapplication/json\-seq\fR MIME type scheme for separating JSON texts i Read the filter from a file rather than from a command line, like awk\'s \-f option\. This changes the filter argument to be interpreted as a filename, instead of the source of a program\. . .TP +\fB\-o\fR / \fB\-\-output\-file filename\fR: +. +.IP +Write output values to the named file instead of standard out\. The output is first written to a temporary file in the same directory, then renamed into place on success\. This makes it safe to use an input file as the output file (e\.g\., \fBjq \'\.x += 1\' f\.json \-o f\.json\fR)\. +. +.IP +The outputs from \fB\-\-debug\-dump\-disasm\fR and \fB\-\-debug\-trace\fR are still written to standard out, so this option can be used to separate them\. +. +.TP +\fB\-O\fR / \fB\-\-clobber\-output filename\fR: +. +.IP +Like \fB\-o\fR, but truncates the output file immediately instead of writing to a temporary file first\. This is faster, but the output file must not also be an input file, and partial output may be left behind if jq is interrupted\. +. +.TP \fB\-L directory\fR / \fB\-\-library\-path directory\fR: . .IP diff --git a/src/main.c b/src/main.c index 384e66d2b3..c26e762b45 100644 --- a/src/main.c +++ b/src/main.c @@ -1,13 +1,16 @@ #include #include #include +#include #include +#include #ifdef HAVE_SETLOCALE #include #endif #include #include #include +#include #include #ifdef WIN32 @@ -92,6 +95,8 @@ static void usage(int code, int keep_it_short) { " an array;\n" " --seq parse input/output as application/json-seq;\n" " -f, --from-file load the filter from a file;\n" + " -o, --output-file file output to file instead of stdout;\n" + " -O, --clobber-output file like -o but truncates file immediately;\n" " -L, --library-path dir search modules from the directory;\n" " --arg name value set $name to the string value;\n" " --argjson name value set $name to the JSON value;\n" @@ -143,22 +148,24 @@ static int isoption(const char** text, char shortopt, const char* longopt, int i } enum { - SLURP = 1, - RAW_INPUT = 2, - PROVIDE_NULL = 4, - RAW_OUTPUT = 8, - RAW_OUTPUT0 = 16, - ASCII_OUTPUT = 32, - COLOR_OUTPUT = 64, - NO_COLOR_OUTPUT = 128, - SORTED_OUTPUT = 256, - FROM_FILE = 512, - RAW_NO_LF = 1024, - UNBUFFERED_OUTPUT = 2048, - EXIT_STATUS = 4096, - SEQ = 16384, - /* debugging only */ - DUMP_DISASM = 32768, + SLURP = 1 << 0, + RAW_INPUT = 1 << 1, + PROVIDE_NULL = 1 << 2, + RAW_OUTPUT = 1 << 3, + RAW_OUTPUT0 = 1 << 4, + ASCII_OUTPUT = 1 << 5, + COLOR_OUTPUT = 1 << 6, + NO_COLOR_OUTPUT = 1 << 7, + SORTED_OUTPUT = 1 << 8, + FROM_FILE = 1 << 9, + RAW_NO_LF = 1 << 10, + UNBUFFERED_OUTPUT = 1 << 11, + EXIT_STATUS = 1 << 12, + SEQ = 1 << 13, + DUMP_DISASM = 1 << 14, /* debugging only */ +#ifdef WIN32 + BINARY_OUTPUT = 1 << 15, +#endif }; enum { @@ -172,14 +179,14 @@ enum { #define jq_exit_with_status(r) exit(abs(r)) #define jq_exit(r) exit( r > 0 ? r : 0 ) -static int process(jq_state *jq, jv value, int flags, int dumpopts, int options) { +static int process(jq_state *jq, jv value, FILE *ofile, int flags, int dumpopts, int options) { int ret = JQ_OK_NO_OUTPUT; // No valid results && -e -> exit(4) jq_start(jq, value, flags); jv result; while (jv_is_valid(result = jq_next(jq))) { if ((options & RAW_OUTPUT) && jv_get_kind(result) == JV_KIND_STRING) { if (options & ASCII_OUTPUT) { - jv_dumpf(jv_copy(result), stdout, JV_PRINT_ASCII); + jv_dumpf(jv_copy(result), ofile, JV_PRINT_ASCII); } else if ((options & RAW_OUTPUT0) && strlen(jv_string_value(result)) != (unsigned long)jv_string_length_bytes(jv_copy(result))) { jv_free(result); result = jv_invalid_with_msg(jv_string( @@ -187,7 +194,7 @@ static int process(jq_state *jq, jv value, int flags, int dumpopts, int options) break; } else { priv_fwrite(jv_string_value(result), jv_string_length_bytes(jv_copy(result)), - stdout, dumpopts & JV_PRINT_ISATTY); + ofile, dumpopts & JV_PRINT_ISATTY); } ret = JQ_OK; jv_free(result); @@ -197,15 +204,15 @@ static int process(jq_state *jq, jv value, int flags, int dumpopts, int options) else ret = JQ_OK; if (options & SEQ) - priv_fwrite("\036", 1, stdout, dumpopts & JV_PRINT_ISATTY); - jv_dump(result, dumpopts); + priv_fwrite("\036", 1, ofile, dumpopts & JV_PRINT_ISATTY); + jv_dumpf(result, ofile, dumpopts); } if (!(options & RAW_NO_LF)) - priv_fwrite("\n", 1, stdout, dumpopts & JV_PRINT_ISATTY); + priv_fwrite("\n", 1, ofile, dumpopts & JV_PRINT_ISATTY); if (options & RAW_OUTPUT0) - priv_fwrite("\0", 1, stdout, dumpopts & JV_PRINT_ISATTY); + priv_fwrite("\0", 1, ofile, dumpopts & JV_PRINT_ISATTY); if (options & UNBUFFERED_OUTPUT) - fflush(stdout); + fflush(ofile); } if (jq_halted(jq)) { // jq program invoked `halt` or `halt_error` @@ -268,6 +275,84 @@ static void stderr_cb(void *data, jv input) { jv_free(input); } +static char * volatile ofile_tmp_to_clean = NULL; + +static void cleanup_tmp(void) { + if (ofile_tmp_to_clean) { + unlink(ofile_tmp_to_clean); + ofile_tmp_to_clean = NULL; + } +} + +static void cleanup_tmp_signal(int sig) { + if (ofile_tmp_to_clean) + unlink(ofile_tmp_to_clean); + signal(sig, SIG_DFL); + raise(sig); +} + +/* + * Create a temp file suitable for rename-into-place over dest. + * Tries a predictable name first (.jq-.tmp), falling back + * to mkstemp(.jq-XXXXXX) if the predictable name already exists. + * Returns fd on success (and sets *tmp_path to a malloc'd string), + * or -1 on failure (with errno set, *tmp_path unchanged). + */ +static int open_temp_for_rename(const char *dest, char **tmp_path) { + char *dtmp = strdup(dest); + char *btmp = strdup(dest); + if (!dtmp || !btmp) { + free(dtmp); + free(btmp); + return -1; + } + const char *dir = dirname(dtmp); + const char *base = basename(btmp); + int pred_len = snprintf(NULL, 0, "%s/.jq-%s.tmp", dir, base) + 1; + int mks_len = snprintf(NULL, 0, "%s/.jq-XXXXXX", dir) + 1; + size_t len = pred_len > mks_len ? pred_len : mks_len; + char *tmp = malloc(len); + if (!tmp) { + free(dtmp); + free(btmp); + return -1; + } + snprintf(tmp, len, "%s/.jq-%s.tmp", dir, base); + int fd = open(tmp, O_CREAT | O_WRONLY | O_EXCL, 0666); + if (fd == -1 && errno == EEXIST) { + snprintf(tmp, len, "%s/.jq-XXXXXX", dir); + fd = mkstemp(tmp); + } + free(dtmp); + free(btmp); + if (fd == -1) { + int e = errno; + free(tmp); + errno = e; + return -1; + } + *tmp_path = tmp; + return fd; +} + +static void install_cleanup_handlers(void) { + atexit(cleanup_tmp); +#ifndef WIN32 + static const int sigs[] = { SIGABRT, SIGHUP, SIGINT, SIGPIPE, SIGQUIT, SIGTERM }; + struct sigaction sa, old; + for (size_t i = 0; i < sizeof(sigs)/sizeof(sigs[0]); i++) { + sigaction(sigs[i], NULL, &old); + if (old.sa_handler != SIG_IGN) { + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = cleanup_tmp_signal; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + sigaction(sigs[i], &sa, NULL); + } + } +#endif +} + #ifdef WIN32 int umain(int argc, char* argv[]); @@ -294,6 +379,9 @@ int main(int argc, char* argv[]) { int compiled = 0; int parser_flags = 0; int nfiles = 0; + FILE *ofile = stdout; + char *ofile_name = NULL; /* final destination path */ + char *ofile_tmp = NULL; /* temp file to rename into place */ int last_result = -1; /* -1 = no result, 0=null or false, 1=true */ int badwrite; int options = 0; @@ -309,13 +397,6 @@ int main(int argc, char* argv[]) { onig_set_parse_depth_limit(1024); #endif -#ifdef __OpenBSD__ - if (pledge("stdio rpath", NULL) == -1) { - perror("pledge"); - exit(JQ_ERROR_SYSTEM); - } -#endif - #ifdef WIN32 jv_tsd_dtoa_ctx_init(); fflush(stdout); @@ -404,6 +485,82 @@ int main(int argc, char* argv[]) { options |= PROVIDE_NULL; } else if (isoption(&text, 'f', "from-file", is_short)) { options |= FROM_FILE; + } else if (isoption(&text, 'O', "clobber-output", is_short)) { + options |= NO_COLOR_OUTPUT; + if (i >= argc - 1) { + fprintf(stderr, "jq: --clobber-output takes one parameter (e.g. --clobber-output filename)\n"); + die(); + } + if (ofile != stdout) { + fclose(ofile); + if (ofile_tmp) { + unlink(ofile_tmp); + ofile_tmp_to_clean = NULL; + free(ofile_tmp); + ofile_tmp = NULL; + } + free(ofile_name); + ofile_name = NULL; + } + ofile = fopen(argv[i+1], "w"); + if (!ofile) { + fprintf(stderr, "jq: Could not open --clobber-output %s: %s\n", argv[i+1], strerror(errno)); + die(); + } +#ifdef WIN32 + _setmode(fileno(ofile), options & BINARY_OUTPUT ? _O_BINARY : _O_TEXT | _O_U8TEXT); +#endif + i += 1; // skip the next argument + } else if (isoption(&text, 'o', "output-file", is_short)) { + options |= NO_COLOR_OUTPUT; + if (i >= argc - 1) { + fprintf(stderr, "jq: --output-file takes one parameter (e.g. --output-file filename)\n"); + die(); + } + if (ofile != stdout) { + fclose(ofile); + if (ofile_tmp) { + unlink(ofile_tmp); + ofile_tmp_to_clean = NULL; + free(ofile_tmp); + ofile_tmp = NULL; + } + free(ofile_name); + ofile_name = NULL; + } + ofile_name = strdup(argv[i+1]); + if (!ofile_name) { + fprintf(stderr, "jq: error: out of memory\n"); + die(); + } + { + int fd = open_temp_for_rename(ofile_name, &ofile_tmp); + if (fd == -1) { + fprintf(stderr, "jq: Could not create temp file for --output-file %s: %s\n", + argv[i+1], strerror(errno)); + free(ofile_name); + ofile_name = NULL; + die(); + } + ofile = fdopen(fd, "w"); + if (!ofile) { + fprintf(stderr, "jq: Could not open --output-file %s: %s\n", + argv[i+1], strerror(errno)); + close(fd); + unlink(ofile_tmp); + free(ofile_tmp); + ofile_tmp = NULL; + free(ofile_name); + ofile_name = NULL; + die(); + } + ofile_tmp_to_clean = ofile_tmp; + install_cleanup_handlers(); + } +#ifdef WIN32 + _setmode(fileno(ofile), options & BINARY_OUTPUT ? _O_BINARY : _O_TEXT | _O_U8TEXT); +#endif + i += 1; // skip the next argument } else if (isoption(&text, 'L', "library-path", is_short)) { if (jv_get_kind(lib_search_paths) == JV_KIND_NULL) lib_search_paths = jv_array(); @@ -419,9 +576,14 @@ int main(int argc, char* argv[]) { } } else if (isoption(&text, 'b', "binary", is_short)) { #ifdef WIN32 + options |= BINARY_OUTPUT; + if (ofile != stdout) { + fflush(ofile); + _setmode(fileno(ofile), _O_BINARY); + } fflush(stdout); fflush(stderr); - _setmode(fileno(stdin), _O_BINARY); + _setmode(fileno(stdin), _O_BINARY); _setmode(fileno(stdout), _O_BINARY); _setmode(fileno(stderr), _O_BINARY); #endif @@ -537,7 +699,7 @@ int main(int argc, char* argv[]) { } #ifdef USE_ISATTY - if (isatty(STDOUT_FILENO)) { + if (ofile == stdout && isatty(STDOUT_FILENO)) { #ifndef WIN32 dumpopts |= JV_PRINT_ISATTY | JV_PRINT_COLOR; #else @@ -590,12 +752,22 @@ int main(int argc, char* argv[]) { jq_set_attr(jq, jv_string("VERSION_DIR"), jv_string_fmt("%.*s-master", (int)(strchr(JQ_VERSION, '-') - JQ_VERSION), JQ_VERSION)); #ifdef USE_ISATTY - if (!program && !(options & FROM_FILE) && (!isatty(STDOUT_FILENO) || !isatty(STDIN_FILENO))) + if (!program && !(options & FROM_FILE) && (ofile != stdout || !isatty(STDOUT_FILENO) || !isatty(STDIN_FILENO))) program = "."; #endif if (!program) usage(2, 1); +#ifdef __OpenBSD__ + { + const char *promises = ofile_tmp ? "stdio rpath cpath" : "stdio rpath"; + if (pledge(promises, NULL) == -1) { + perror("pledge"); + exit(JQ_ERROR_SYSTEM); + } + } +#endif + if (options & FROM_FILE) { char *program_origin = strdup(program); if (program_origin == NULL) { @@ -664,13 +836,13 @@ int main(int argc, char* argv[]) { jq_util_input_add_input(input_state, "-"); if (options & PROVIDE_NULL) { - ret = process(jq, jv_null(), jq_flags, dumpopts, options); + ret = process(jq, jv_null(), ofile, jq_flags, dumpopts, options); } else { jv value; while (jq_util_input_errors(input_state) == 0 && (jv_is_valid((value = jq_util_input_next_input(input_state))) || jv_invalid_has_msg(jv_copy(value)))) { if (jv_is_valid(value)) { - ret = process(jq, value, jq_flags, dumpopts, options); + ret = process(jq, value, ofile, jq_flags, dumpopts, options); if (ret <= 0 && ret != JQ_OK_NO_OUTPUT) last_result = (ret != JQ_OK_NULL_KIND); if (jq_halted(jq)) @@ -696,15 +868,45 @@ int main(int argc, char* argv[]) { ret = JQ_ERROR_SYSTEM; out: + // Close input files before renaming output (needed on Windows where + // rename() fails if the destination is open). + jq_util_input_free(&input_state); + badwrite = ferror(stdout); - if (fclose(stdout)!=0 || badwrite) { - fprintf(stderr,"jq: error: writing output failed: %s\n", strerror(errno)); + if (fclose(stdout) != 0 || badwrite) { + fprintf(stderr, "jq: error: writing output failed: %s\n", strerror(errno)); ret = JQ_ERROR_SYSTEM; } + if (ofile != stdout) { + int ofile_err = 0; + badwrite = ferror(ofile); + if (fclose(ofile) != 0 || badwrite) { + fprintf(stderr, "jq: error: writing output failed: %s\n", strerror(errno)); + ret = JQ_ERROR_SYSTEM; + ofile_err = 1; + } + if (ofile_tmp) { + if (ofile_err) { + unlink(ofile_tmp); + } else if ( +#ifdef WIN32 + // Windows rename() won't replace an existing file + unlink(ofile_name) != 0 && errno != ENOENT ? 1 : +#endif + rename(ofile_tmp, ofile_name) != 0) { + fprintf(stderr, "jq: error: could not rename %s to %s: %s\n", + ofile_tmp, ofile_name, strerror(errno)); + unlink(ofile_tmp); + ret = JQ_ERROR_SYSTEM; + } + ofile_tmp_to_clean = NULL; + free(ofile_tmp); + free(ofile_name); + } + } jv_free(ARGS); jv_free(program_arguments); - jq_util_input_free(&input_state); jq_teardown(&jq); if (options & EXIT_STATUS) { diff --git a/src/util.c b/src/util.c index bcb86da836..86898b4782 100644 --- a/src/util.c +++ b/src/util.c @@ -79,6 +79,27 @@ FILE *fopen(const char *fname, const char *mode) { } #endif +#ifndef HAVE_MKSTEMP +int mkstemp(char *template) { + size_t len = strlen(template); + int tries=5; + int fd; + + // mktemp() truncates template when it fails + char *s = alloca(len + 1); + assert(s != NULL); + strcpy(s, template); + + do { + // Restore template + strcpy(template, s); + (void) mktemp(template); + fd = open(template, O_CREAT | O_EXCL | O_RDWR, 0600); + } while (fd == -1 && tries-- > 0); + return fd; +} +#endif + jv expand_path(jv path) { assert(jv_get_kind(path) == JV_KIND_STRING); const char *pstr = jv_string_value(path); diff --git a/src/util.h b/src/util.h index 6fdef3837e..45b354282b 100644 --- a/src/util.h +++ b/src/util.h @@ -13,6 +13,10 @@ #include "jv.h" +#ifndef HAVE_MKSTEMP +int mkstemp(char *template); +#endif + jv expand_path(jv); jv get_home(void); jv jq_realpath(jv); diff --git a/tests/shtest b/tests/shtest index e241d4a5c8..1de3c45f34 100755 --- a/tests/shtest +++ b/tests/shtest @@ -807,6 +807,104 @@ printf '[\n {\n "a": 1\n }\n]\n' > $d/expected $JQ --indent 6 -n "[{a:1}]" > $d/out cmp $d/out $d/expected +# Simple --output-file +printf '{"wide":"👋"}' > $d/expected +$JQ -j -c -o $d/out -n '{wide:"👋"}' +cmp $d/out $d/expected + +# --output-file creates a file +rm $d/out +echo '"create"' > $d/expected +$JQ -o $d/out -n '"create"' +cmp $d/out $d/expected + +# Program errors are not written to --output-file +echo 42 > $d/expected +echo 'jq: error (at ): error!' > $d/expected_err +! $JQ -o $d/out -n '42, ("error!" | error)' 2> $d/err +cmp $d/out $d/expected +cmp $d/err $d/expected_err + +# Disassembly is not written to --output-file +echo 42 > $d/expected +printf '0000 TOP\n0001 LOADK 42\n0003 RET\n\n' > $d/expected_disasm +$JQ --debug-dump-disasm -o $d/out -n 42 > $d/disasm +cmp $d/out $d/expected +cmp $d/disasm $d/expected_disasm + +# Debug trace is not written to --output-file +echo 42 > $d/expected +printf '0000 TOP\t\n0001 LOADK 42\tnull\n0003 RET\t42\n0003 RET\t\t\n' > $d/expected_trace +$JQ --debug-trace -o $d/out -n 42 > $d/trace +cmp $d/out $d/expected +cmp $d/trace $d/expected_trace + +# --output-file is one of the inputs (temp file allows safe read) +printf '1\n2\n' > $d/out +printf '4\n5\n' > $d/expected +$JQ -o $d/out '.+3' $d/out +cmp $d/out $d/expected + +# --output-file with --from-file (program file not clobbered) +echo hello > $d/out +cat > $d/expected_err <<'EOF' +jq: error: hello/0 is not defined at , line 1, column 1: + hello + ^^^^^ +jq: 1 compile error +EOF +! $JQ -f -o $d/out $d/out 2> $d/err +diff $d/err $d/expected_err + +# --output-file before and after --rawfile (input not clobbered) +printf hello > $d/out +echo '"hello"' > $d/expected +$JQ -o $d/out --rawfile arg $d/out -n '$arg' +diff $d/out $d/expected +printf hello > $d/out +echo '"hello"' > $d/expected +$JQ --rawfile arg $d/out -o $d/out -n '$arg' +diff $d/out $d/expected + +# --output-file before and after --slurpfile (input not clobbered) +printf '"hello"\n"world"' > $d/out +echo '["hello","world"]' > $d/expected +$JQ -o $d/out --slurpfile arg $d/out -nc '$arg' +diff $d/out $d/expected +printf '"hello"\n"world"' > $d/out +echo '["hello","world"]' > $d/expected +$JQ --slurpfile arg $d/out -o $d/out -nc '$arg' +diff $d/out $d/expected + +# --output-file before and after --binary +printf '1\n2\n' > $d/expected +$JQ_NO_B -o $d/out --binary -n '1,2' +cmp $d/out $d/expected +$JQ_NO_B --binary -o $d/out -n '1,2' +cmp $d/out $d/expected + +# --output-file before and after --color-output +echo '"hello"' > $d/expected +$JQ -o $d/out --color-output -n '"hello"' +cmp $d/out $d/expected +$JQ --color-output -o $d/out -n '"hello"' +cmp $d/out $d/expected + +# --output-file without a filter uses . as the filter +printf '[\n 1,\n 2\n]\n' > $d/expected +echo '[1,2]' | $JQ -o $d/out +cmp $d/out $d/expected + +# --output-file handles failure (nonexistent parent directory) +! $JQ -o $d/notfound/out -n . 2> $d/err +grep "jq: Could not create temp file for --output-file .*/notfound/out" $d/err > /dev/null + +# --output-file does not allow directories +mkdir $d/dir +! $JQ -o $d/dir -n . 2> $d/err +grep "jq: error: could not rename .* to .*/dir" $d/err > /dev/null +rmdir $d/dir + if ! $msys && ! $mingw; then # Test handling of timezones -- #2429, #2475 if ! r=$(TZ=Asia/Tokyo $JQ -rn '1731627341 | strflocaltime("%F %T %z %Z")') \