Paginate query responses and document assets

README.md

@@ -102,6 +102,39 @@ df = pandas.DataFrame.from_dict(response.json())
 df
 ```
 
+## Query
+`query` accepts POST requests with the following parameters:
+- `attribute` - an EJSCREEN attribute in EJAM syntax, such as `pctlowinc` or `pctunemployed`
+- `value` - a decimal cutoff from 0 to 1; numeric-like strings are coerced and invalid values are rejected
+- `page` - a positive whole-number page to return. Default = 1
+- `limit` - rows per page. Default = 100; maximum = 500
+
+`query` returns a JSON object with:
+- `results` - the EJAM output rows for the requested page
+- `pagination` - metadata with `page`, `limit`, `total_rows`, `total_pages`, `returned_rows`, `has_next_page`, and `has_previous_page`
+
+Pagination is 1-based. For example, with `limit = 100`, `page = 2` returns rows 101-200 from the query results. If `page` is beyond the available results, `results` is empty and `pagination` still reports the total row and page counts.
+
+### Examples
+```
+data = {"attribute": "pctlowinc", "value": 0.95, "page": 1, "limit": 100}
+
+# Execute query
+import requests
+import pandas
+url = "https://ejamapi-84652557241.us-central1.run.app/query"
+response = requests.post(url, json=data)
+
+# Load response as Pandas dataframe
+payload = response.json()
+df = pandas.DataFrame.from_dict(payload["results"])
+df
+
+# Request the next page
+data["page"] = payload["pagination"]["page"] + 1
+response = requests.post(url, json=data)
+```
+
 ## Handoff (launch the EJAM app pre-loaded)
 
 Two endpoints let an external app (e.g. EJScreen) hand a set of selected places to the full EJAM app without hitting URL-length limits (important for polygons):

query_pagination.R

@@ -0,0 +1,86 @@
+QUERY_DEFAULT_LIMIT <- 100L
+QUERY_MAX_LIMIT <- 500L
+
+query_positive_whole_number <- function(x, name, max_value = NULL) {
+  x <- suppressWarnings(as.numeric(x))
+  if (
+    length(x) != 1 ||
+    is.na(x) ||
+    !is.finite(x) ||
+    x < 1 ||
+    x != floor(x) ||
+    x > .Machine$integer.max ||
+    (!is.null(max_value) && x > max_value)
+  ) {
+    if (!is.null(max_value)) {
+      stop(
+        sprintf("%s must be a positive whole number no larger than %d.", name, max_value),
+        call. = FALSE
+      )
+    }
+    stop(sprintf("%s must be a positive whole number.", name), call. = FALSE)
+  }
+  as.integer(x)
+}
+
+paginate_query_results <- function(
+  results,
+  page = 1,
+  limit = QUERY_DEFAULT_LIMIT,
+  max_limit = QUERY_MAX_LIMIT
+) {
+  page <- query_positive_whole_number(page, "page")
+  limit <- query_positive_whole_number(limit, "limit", max_value = max_limit)
+
+  total_rows <- nrow(results)
+  total_pages <- if (total_rows == 0L) 0L else ceiling(total_rows / limit)
+  start_idx <- ((page - 1L) * limit) + 1L
+  end_idx <- min(page * limit, total_rows)
+
+  if (start_idx > total_rows) {
+    paginated_results <- results[0, , drop = FALSE]
+  } else {
+    paginated_results <- results[start_idx:end_idx, , drop = FALSE]
+  }
+
+  list(
+    results = paginated_results,
+    pagination = list(
+      page = page,
+      limit = limit,
+      total_rows = as.integer(total_rows),
+      total_pages = as.integer(total_pages),
+      returned_rows = as.integer(nrow(paginated_results)),
+      has_next_page = page < total_pages,
+      has_previous_page = total_pages > 0L && page > 1L
+    )
+  )
+}
+
+query_endpoint_response <- function(
+  attribute,
+  value,
+  page,
+  limit,
+  res,
+  pctile_fun = pctile_x_is_hit_by_score,
+  blockgroupstats_data = blockgroupstats,
+  error_handler = function(message) list(error = message)
+) {
+  value <- suppressWarnings(as.numeric(value))
+  if (length(value) != 1 || is.na(value) || value < 0 || value > 1) {
+    res$status <- 400
+    return(error_handler("value must be a numeric cutoff from 0 to 1."))
+  }
+
+  these <- pctile_fun(attribute, cutoff = value)
+  results <- blockgroupstats_data[these, , drop = FALSE]
+
+  tryCatch(
+    paginate_query_results(results, page = page, limit = limit),
+    error = function(e) {
+      res$status <- 400
+      error_handler(e$message)
+    }
+  )
+}

rest_controller.r

@@ -6,6 +6,8 @@ library(geojsonsf)
 library(jsonlite)
 library(sf)
 
+source("query_pagination.R", local = TRUE)
+
 ############################# #
 #* @apiTitle API for EJAM / EJSCREEN Data, Analysis, and Reports
 #*
@@ -209,19 +211,22 @@ function(sites = NULL, shape = NULL, fips = NULL, buffer = 0, radius = NULL, geo
 # /query ####
 
 #* Return EJAM analysis data as JSON based on attribute query
+#* Query responses are paginated. Use page and limit to request subsequent pages.
 #* @tag Data
 #* @param attribute An EJSCREEN attribute, in EJAM syntax (e.g. pctunemployed)
 #* @param value A decimal, 0-1, representing a cutoff/threshold; returns blockgroups whose percentile rank for the attribute is larger (e.g. pctunemployed > .9)
+#* @param page Positive whole-number page to return. Defaults to 1.
+#* @param limit Positive whole-number rows per page. Defaults to 100 and cannot exceed 500.
 #* @post /query
-function(attribute = "pctunemployed", value=.9, res) {
-  value <- suppressWarnings(as.numeric(value))
-  if (length(value) != 1 || is.na(value) || value < 0 || value > 1) {
-    res$status <- 400
-    return(handle_error("value must be a numeric cutoff from 0 to 1."))
-  }
-  these <- pctile_x_is_hit_by_score(attribute, cutoff = value)
-  results <- blockgroupstats[these,]
-  return (results)
+function(attribute = "pctunemployed", value = .9, page = 1, limit = QUERY_DEFAULT_LIMIT, res) {
+  query_endpoint_response(
+    attribute = attribute,
+    value = value,
+    page = page,
+    limit = limit,
+    res = res,
+    error_handler = handle_error
+  )
 }
 
 # /report ####

tests/test-query-pagination.R

@@ -0,0 +1,111 @@
+library(testthat)
+
+source("query_pagination.R")
+
+sample_results <- data.frame(
+  bgid = sprintf("%03d", 1:12),
+  pctlowinc = seq(0.91, 1.02, length.out = 12),
+  stringsAsFactors = FALSE
+)
+
+test_that("query pagination returns the requested 1-based page with metadata", {
+  payload <- paginate_query_results(sample_results, page = "2", limit = "5")
+
+  expect_equal(payload$results$bgid, sprintf("%03d", 6:10))
+  expect_equal(payload$pagination$page, 2L)
+  expect_equal(payload$pagination$limit, 5L)
+  expect_equal(payload$pagination$total_rows, 12L)
+  expect_equal(payload$pagination$total_pages, 3L)
+  expect_equal(payload$pagination$returned_rows, 5L)
+  expect_true(payload$pagination$has_next_page)
+  expect_true(payload$pagination$has_previous_page)
+})
+
+test_that("query pagination returns an empty page beyond available results", {
+  payload <- paginate_query_results(sample_results, page = 4, limit = 5)
+
+  expect_equal(nrow(payload$results), 0L)
+  expect_equal(names(payload$results), names(sample_results))
+  expect_equal(payload$pagination$page, 4L)
+  expect_equal(payload$pagination$total_pages, 3L)
+  expect_equal(payload$pagination$returned_rows, 0L)
+  expect_false(payload$pagination$has_next_page)
+  expect_true(payload$pagination$has_previous_page)
+})
+
+test_that("query pagination metadata is consistent for empty result sets", {
+  payload <- paginate_query_results(sample_results[0, ], page = 2, limit = 5)
+
+  expect_equal(nrow(payload$results), 0L)
+  expect_equal(payload$pagination$total_rows, 0L)
+  expect_equal(payload$pagination$total_pages, 0L)
+  expect_equal(payload$pagination$returned_rows, 0L)
+  expect_false(payload$pagination$has_next_page)
+  expect_false(payload$pagination$has_previous_page)
+})
+
+test_that("query pagination validates page and limit inputs", {
+  expect_error(
+    paginate_query_results(sample_results, page = 0, limit = 5),
+    "page must be a positive whole number",
+    fixed = TRUE
+  )
+  expect_error(
+    paginate_query_results(sample_results, page = 1, limit = 501),
+    "limit must be a positive whole number no larger than 500",
+    fixed = TRUE
+  )
+  expect_error(
+    paginate_query_results(sample_results, page = 1.5, limit = 5),
+    "page must be a positive whole number",
+    fixed = TRUE
+  )
+  expect_error(
+    paginate_query_results(sample_results, page = .Machine$integer.max + 1, limit = 5),
+    "page must be a positive whole number",
+    fixed = TRUE
+  )
+})
+
+test_that("query endpoint response paginates selected EJAM rows", {
+  selected <- c(TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, FALSE, FALSE)
+  pctile_fun <- function(attribute, cutoff) {
+    expect_equal(attribute, "pctlowinc")
+    expect_equal(cutoff, 0.95)
+    selected
+  }
+  res <- new.env(parent = emptyenv())
+
+  payload <- query_endpoint_response(
+    attribute = "pctlowinc",
+    value = "0.95",
+    page = 2,
+    limit = 4,
+    res = res,
+    pctile_fun = pctile_fun,
+    blockgroupstats_data = sample_results
+  )
+
+  expect_equal(payload$results$bgid, sprintf("%03d", 5:8))
+  expect_equal(payload$pagination$total_rows, 10L)
+  expect_equal(payload$pagination$total_pages, 3L)
+  expect_null(res$status)
+})
+
+test_that("query endpoint response maps invalid inputs to 400 errors", {
+  pctile_fun <- function(attribute, cutoff) rep(TRUE, nrow(sample_results))
+  res <- new.env(parent = emptyenv())
+
+  payload <- query_endpoint_response(
+    attribute = "pctlowinc",
+    value = "0.95",
+    page = 1,
+    limit = 501,
+    res = res,
+    pctile_fun = pctile_fun,
+    blockgroupstats_data = sample_results
+  )
+
+  expect_equal(res$status, 400)
+  expect_equal(payload$error, "limit must be a positive whole number no larger than 500.")
+})