Add docs and validation for chunked disk vector formats; update utils, tests, and add vignette about chunking

Author: e-kotov

R/inspire_grid.R

@@ -54,7 +54,7 @@
 #'   This automatic limiting can be overridden by setting `options(gridmaker.tile_multiplier)`.
 #'   **Note:** Parallel processing is not supported when `output_type = "spatraster"`.
 #'   Raster output will always run sequentially.
-#' @param max_memory_gb A numeric value. Maximum memory in gigabytes to use for grid creation. Default is NULL, in which case there is an automatic limit of available system memory. The available memory detection may fail on certain HPC (High Performance Computing) systems where jobs are allocated a fixed amount of memory that is less than the total system memory of the allocated node.
+#' @param max_memory_gb A numeric value. Maximum memory in gigabytes to use for grid creation. Default is `NULL`, in which case there is an automatic limit based on **available free system memory** (not total system RAM). Using this argument allows manual override, which is recommended on certain HPC (High Performance Computing) systems where jobs are allocated a fixed amount of memory that is less than the total free memory of the allocated node.
 #' @inheritParams inspire_grid_params
 #'
 #' @return If \code{dsn} is \code{NULL} (the default), an \code{sf} object, \code{data.frame},

R/inspire_grid_params.R

@@ -30,6 +30,21 @@
 #'   `"path/to/grid.tif"` for raster data) or a database connection string.
 #'   If \code{dsn} is provided, the grid is written to the specified location
 #'   instead of being returned as an object.
+#'
+#'   **Supported vector formats for chunked disk writes:**
+#'   \itemize{
+#'     \item `.gpkg` (GeoPackage) - **Recommended** - Best balance of speed, compatibility, and modern features
+#'     \item `.shp` (Shapefile) - Widely used, fast writes, but has limitations (10-char field names, 2GB limit)
+#'     \item `.geojson`, `.json` (GeoJSON) - Web-friendly, works but slower for large grids
+#'     \item `.geojsonl`, `.geojsonseq` (GeoJSONSeq) - Newline-delimited GeoJSON
+#'     \item `.sqlite` (SQLite/SpatiaLite) - Database format (GeoPackage is built on SQLite)
+#'     \item `.fgb` (FlatGeobuf) - Cloud-optimized format
+#'     \item `.gdb` (OpenFileGDB) - ESRI FileGDB format
+#'     \item `.csv`, `.tsv`, `.txt` (for dataframe output only)
+#'   }
+#'
+#'
+#'   Other formats not listed have not been tested and will generate a warning.
 #' @param layer The name of the grid layer, passed directly to `sf::st_write`.
 #'   Its interpretation depends on the destination driver. For a GeoPackage
 #'   file, this will be the layer name. If \code{dsn} is a file path and `layer` is
@@ -39,6 +54,7 @@
 #'   When writing to spatial files via \code{dsn}, these are passed to \code{\link[sf]{st_write}}.
 #'   For \code{output_type = "spatraster"} writing, these are passed to \code{\link[terra]{writeRaster}}.
 #'   For streaming backends (`mirai` or sequential), this can include \code{max_cells_per_chunk} to control memory usage.
+#' @param max_memory_gb A numeric value. Maximum memory in gigabytes to use for grid creation. Default is `NULL`, in which case there is an automatic limit based on **available free system memory** (not total system RAM).
 #'
 #' @name inspire_grid_params
 #' @keywords internal

R/utils.R

@@ -343,7 +343,9 @@ regex_match <- function(text, pattern, i = NULL, ...) {
 #' @keywords internal
 #' @noRd
 validate_disk_compatibility <- function(output_type, dsn) {
-  if (is.null(dsn)) return(TRUE)
+  if (is.null(dsn)) {
+    return(TRUE)
+  }
 
   ext <- tolower(tools::file_ext(dsn))
   is_text <- ext %in% c("csv", "tsv", "txt")
@@ -352,6 +354,29 @@ validate_disk_compatibility <- function(output_type, dsn) {
   is_raster <- output_type == "spatraster"
   is_raster_format <- ext %in% c("tif", "tiff", "nc", "img", "asc", "grd")
 
+  # Vector formats that support append (required for chunked disk writes)
+  # Empirically tested and confirmed to work:
+  # - GeoPackage (.gpkg) and SQLite (.sqlite) - excellent append support
+  # - Shapefile (.shp) - supports append (but has other limitations like field name length)
+  # - GeoJSON (.geojson, .json) - supports append
+  # - FlatGeobuf (.fgb) - cloud-optimized, supports append
+  # - OpenFileGDB (.gdb) - ESRI FileGDB, supports append
+  # - GeoJSONSeq (.geojsonl, .geojsonseq) - newline-delimited GeoJSON, supports append
+  append_safe_vector_formats <- c(
+    "gpkg",
+    "sqlite",
+    "shp",
+    "geojson",
+    "json",
+    "fgb",
+    "gdb",
+    "geojsonl",
+    "geojsonseq"
+  )
+
+  # Formats explicitly confirmed to NOT support append
+  no_append_formats <- c("kml", "gml")
+
   # 1. Prevent Dataframe -> Spatial Vector Format (e.g. gpkg, shp)
   if (is_dataframe && !is_text && !is_raster_format) {
     stop(
@@ -374,10 +399,42 @@ validate_disk_compatibility <- function(output_type, dsn) {
     )
   }
 
-  # 3. Check for readr availability if text output is requested
+  # 3. Validate vector format supports append (required for chunked disk writes)
+  if (is_spatial_vector && !is_text) {
+    if (ext %in% no_append_formats) {
+      # Explicitly unsupported formats
+      stop(
+        sprintf(
+          "Output type '%s' cannot be written to '.%s' format.\n  The '.%s' format does not support appending to existing files.\n  Supported vector formats: %s\n  Or generate the grid in memory (dsn = NULL) and save manually.",
+          output_type,
+          ext,
+          ext,
+          paste0(".", append_safe_vector_formats, collapse = ", ")
+        ),
+        call. = FALSE
+      )
+    } else if (!ext %in% append_safe_vector_formats) {
+      # Unknown/untested formats - provide a warning but more permissive
+      warning(
+        sprintf(
+          "Output type '%s' with '.%s' format has not been tested for append support.\n  Tested formats: %s\n  The operation may fail if this format does not support appending.",
+          output_type,
+          ext,
+          paste0(".", append_safe_vector_formats, collapse = ", ")
+        ),
+        call. = FALSE,
+        immediate. = TRUE
+      )
+    }
+  }
+
+  # 4. Check for readr availability if text output is requested
   if (is_text) {
     if (!requireNamespace("readr", quietly = TRUE)) {
-      stop("Package 'readr' is required to write to .csv/.tsv/.txt files. Please install it.", call. = FALSE)
+      stop(
+        "Package 'readr' is required to write to .csv/.tsv/.txt files. Please install it.",
+        call. = FALSE
+      )
     }
   }
 
@@ -405,7 +462,6 @@ write_grid_chunk <- function(chunk, dsn, layer, append, quiet, ...) {
 
   # --- Text/Delimited Output (readr) ---
   if (ext %in% c("csv", "tsv", "txt")) {
-
     # Drop geometry if it exists (e.g. user asked for sf_polygons but wrote to .csv)
     if (inherits(chunk, "sf")) {
       chunk <- sf::st_drop_geometry(chunk)
@@ -435,7 +491,6 @@ write_grid_chunk <- function(chunk, dsn, layer, append, quiet, ...) {
     )
 
     do.call(readr::write_delim, call_args)
-
   } else {
     # --- Spatial Output (sf) ---
     # sf::st_write accepts '...' for driver specific options

_pkgdown.yml

@@ -38,3 +38,14 @@ navbar:
 home:
   title: 'gridmaker: Create INSPIRE-compliant grids with IDs'
   description: 'Create INSPIRE-compliant grids with IDs'
+
+articles:
+  - title: "Get started"
+    navbar: "Get Started"
+    contents:
+      - gridmaker
+
+  - title: "Tutorials"
+    navbar: "Tutorials"
+    contents:
+      - grid-to-disk

man/inspire_grid.Rd

@@ -32,7 +32,10 @@ test_that("inspire_grid_from_extent streams correctly to disk with mirai backend
   # 3. GENERATE REFERENCE GRID (IN-MEMORY) ----
   # This is the "ground truth" that we will compare against.
   # It is run sequentially to ensure deterministic output.
-  grid_in_memory <- do.call(inspire_grid_from_extent, c(common_args, list(parallel = FALSE)))
+  grid_in_memory <- do.call(
+    inspire_grid_from_extent,
+    c(common_args, list(parallel = FALSE))
+  )
 
   # 4. RUN STREAMING GRID CREATION (ON-DISK) ----
   # Set up a 2-core mirai backend for the test
@@ -96,7 +99,10 @@ test_that("inspire_grid_from_extent handles `layer` argument correctly for disk
   dir.create(temp_dir)
   withr::defer(unlink(temp_dir, recursive = TRUE, force = TRUE))
 
-  simple_extent <- sf::st_bbox(c(xmin = 0, ymin = 0, xmax = 100, ymax = 100), crs = 3035)
+  simple_extent <- sf::st_bbox(
+    c(xmin = 0, ymin = 0, xmax = 100, ymax = 100),
+    crs = 3035
+  )
 
   # Test 1: When layer is NULL, it defaults from dsn and gives a message
   dsn_default <- file.path(temp_dir, "default.gpkg")
@@ -157,7 +163,7 @@ test_that("inspire_grid_from_extent returns dsn invisibly when writing to disk",
   expect_true(file.exists(temp_dsn))
 })
 
-test_that("validate_disk_compatibility throws correct errors", {
+test_that("validate_disk_compatibility validates formats correctly", {
   # Error: Dataframe -> GPKG
   expect_error(
     validate_disk_compatibility("dataframe", "test.gpkg"),
@@ -168,10 +174,38 @@ test_that("validate_disk_compatibility throws correct errors", {
   skip_if_not_installed("readr")
   expect_true(validate_disk_compatibility("dataframe", "test.csv"))
 
-  # Success: SF -> GPKG
+  # Success: Tested vector formats that support append
   expect_true(validate_disk_compatibility("sf_polygons", "test.gpkg"))
+  expect_true(validate_disk_compatibility("sf_polygons", "test.sqlite"))
+  expect_true(validate_disk_compatibility("sf_polygons", "test.shp"))
+  expect_true(validate_disk_compatibility("sf_polygons", "test.geojson"))
+  expect_true(validate_disk_compatibility("sf_polygons", "test.json"))
+
+  # Error: KML and GML explicitly cannot append
+  expect_error(
+    validate_disk_compatibility("sf_polygons", "test.kml"),
+    "cannot be written to '.kml' format"
+  )
+
+  expect_error(
+    validate_disk_compatibility("sf_polygons", "test.gml"),
+    "cannot be written to '.gml' format"
+  )
+
+  # Success: Newly added tested formats
+  expect_true(validate_disk_compatibility("sf_polygons", "test.fgb"))
+  expect_true(validate_disk_compatibility("sf_polygons", "test.gdb"))
+  expect_true(validate_disk_compatibility("sf_polygons", "test.geojsonl"))
+  expect_true(validate_disk_compatibility("sf_polygons", "test.geojsonseq"))
+
+  # Warning: Truly untested format (e.g., MapInfo TAB)
+  expect_warning(
+    validate_disk_compatibility("sf_polygons", "test.tab"),
+    "has not been tested for append support"
+  )
 })
 
+
 test_that("inspire_grid_from_ids writes dataframe to CSV correctly (with chunking)", {
   skip_if_not_installed("readr")
   skip_if_not_installed("sf")
@@ -208,7 +242,10 @@ test_that("inspire_grid_from_extent streams to CSV (dropping geometry)", {
   # Setup: Create a grid that will definitely be chunked (small RAM limit sim or just standard stream)
   # We use standard stream_grid_sequential via inspire_grid_from_extent by not setting parallel
 
-  simple_extent <- sf::st_bbox(c(xmin = 0, ymin = 0, xmax = 20000, ymax = 20000), crs = 3035)
+  simple_extent <- sf::st_bbox(
+    c(xmin = 0, ymin = 0, xmax = 20000, ymax = 20000),
+    crs = 3035
+  )
   tmp_csv <- tempfile(fileext = ".csv")
   on.exit(unlink(tmp_csv), add = TRUE)
 
@@ -266,7 +303,14 @@ test_that("CSV writing respects extra arguments (e.g. na string)", {
   chunk <- data.frame(a = c(1, NA), b = c("x", "y"))
 
   # Write with custom NA string
-  write_grid_chunk(chunk, tmp_csv, layer = NULL, append = FALSE, quiet = TRUE, na = "MISSING")
+  write_grid_chunk(
+    chunk,
+    tmp_csv,
+    layer = NULL,
+    append = FALSE,
+    quiet = TRUE,
+    na = "MISSING"
+  )
 
   # Read back raw text to verify "MISSING" is there
   lines <- readLines(tmp_csv)
@@ -287,7 +331,14 @@ test_that("CSV writing respects extra arguments (e.g. na string)", {
   # Verify pass-through of 'quote' arg
   tmp_csv3 <- tempfile(fileext = ".csv")
   on.exit(unlink(tmp_csv3), add = TRUE)
-  write_grid_chunk(chunk, tmp_csv3, layer = NULL, append = FALSE, quiet = TRUE, quote = "all")
+  write_grid_chunk(
+    chunk,
+    tmp_csv3,
+    layer = NULL,
+    append = FALSE,
+    quiet = TRUE,
+    quote = "all"
+  )
   lines3 <- readLines(tmp_csv3)
   # Look for quoted character values like "x" (quote = "all" quotes character columns)
   expect_true(any(grepl('"x"', lines3)))

man/inspire_grid_params.Rd

@@ -2,3 +2,5 @@
 *.R
 
 /.quarto/
+
+**/*.quarto_ipynb