Ingest IUCN Red List expert ranges → global 0.05° cells

Published

2026-07-14 08:12:07

IUCN Red List expert ranges for marine taxa onto the global 0.05° cell grid — the non-bird complement to AquaMaps/BOTW. Source: the 24 marine taxonomic-group downloads under raw/iucnredlist.org/ (corals, fish, sharks, marine mammals, turtles, seagrasses, mangroves…).

Per species (id_no): union its extant polygons (presence ∈ {1,2,3}), capture the whole range (no land mask), value = msens::compute_er_score( "IUCN:{category}") (never hard-coded; CR=50/EN=25/VU=5/NT=2/LC,DD=1). mdl_key = "rng_iucn|{id_no}". Species are filtered to IUCN’s own marine = true flag — authoritative for non-birds (e.g. it drops freshwater turtles from the TURTLES group) — so the plain-MAMMALS terrestrial species are excluded too.

Parallel (mclapply, per group in memory), resumable; REDO_INGEST=1 rebuilds all.

1 Design

Figure 1: IUCN expert ranges → whole-range extinction-risk value on the global 0.05° grid

2 Setup

Code

librarian::shelf(
  arrow, DBI, dplyr, duckdb, fs, glue, here, jsonlite, logger, readr, sf, terra,
  tibble, MarineSensitivity/msens, quiet = T)
source(here("libs/paths.R"))   # dir_raw, dir_big_v, cellid_tif, ver
source(here("libs/vars.R"))    # redo_ingest
options(readr.show_col_types = F)
sf_use_s2(FALSE)

Spherical geometry (s2) switched off

Code

ds_key    <- "rng_iucn"
# consolidated, indexed IUCN ranges on ~/_big (built once from the raw group shapefiles by
# scripts/build_iucn_gpkg.R; already filtered to presence 1:3 / marine=true / category-ok, INDEX
# on id_no). Reads off Drive and a per-species geometry read is an index hit.
iucn_gpkg <- glue("{dir_derived}/iucnredlist.org/rng_iucn_all.gpkg")
dir_dist  <- glue("{dir_big_v}/marine-atlas/dist/dataset={ds_key}")
manifest  <- here("data/manifests/ingest_rng_iucn.json")
n_workers <- max(1L, parallel::detectCores())
dir_create(c(dir_dist, path_dir(manifest)))
stopifnot("run build_cell_grid.qmd first" = file_exists(cellid_tif),
          "build rng_iucn_all.gpkg first" = file_exists(iucn_gpkg))

if (redo_ingest && dir_exists(dir_dist)) { dir_delete(dir_dist); dir_create(dir_dist) }

3 Species → cells (per-species geometry read from the indexed gpkg)

Code

# species list from the gpkg attributes (one row per id_no); geometry read on demand below.
sp_all <- st_read(iucn_gpkg, quiet = TRUE,
                  query = "SELECT id_no, sci_name, class, category, grp FROM rng_iucn") |>
  st_drop_geometry() |>
  distinct(id_no, sci_name, class, category, grp) |>
  mutate(er_score = compute_er_score(paste0("IUCN:", category)),
         mdl_key  = mdl_key_raw(ds_key, id_no), group = grp)
write_csv(sp_all, glue("{dir_dist}/../model_{ds_key}.csv"))

done <- as.integer(path_ext_remove(path_file(dir_ls(dir_dist, glob = "*.parquet"))))
todo <- sp_all |> filter(!id_no %in% done)
log_info("IUCN: {nrow(sp_all)} marine species, {nrow(todo)} to rasterize ({length(done)} exist)")

one <- function(i) {
  r <- todo[i, ]
  # per-species read is an id_no index hit. Do NOT st_union here — cells_from_ranges
  # rasterizes the multiple presence polygons directly (terra unions the touched cells),
  # and a pre-union costs as much as the rasterize itself on multi-polygon ranges.
  ply_sp <- st_read(iucn_gpkg, quiet = TRUE,
                    query = sprintf("SELECT geom FROM rng_iucn WHERE id_no = %d", r$id_no))
  d <- tryCatch(cells_from_ranges(ply_sp, cellid_tif, value = r$er_score),
                error = function(e) NULL)
  if (is.null(d) || nrow(d) == 0) return("empty")
  msens::write_atlas_parquet(
    tibble::tibble(mdl_key = r$mdl_key, cell_id = d$cell_id, val = d$val),
    fs::path(dir_dist, r$id_no, ext = "parquet")); "done"
}
invisible(parallel::mclapply(seq_len(nrow(todo)),
  function(i) tryCatch(one(i), error = function(e) "error"),
  mc.cores = n_workers, mc.preschedule = FALSE))
glue("{nrow(sp_all)} IUCN marine species across {length(unique(sp_all$group))} groups")

6355 IUCN marine species across 21 groups

4 Outputs + manifest

Code

pq   <- dir_ls(dir_dist, glob = "*.parquet")
con  <- dbConnect(duckdb())
smry <- dbGetQuery(con, glue(
  "SELECT count(DISTINCT mdl_key) n_models, count(*) n_cells, min(val) v_min, max(val) v_max
     FROM read_parquet('{dir_dist}/*.parquet')"))
# order-independent content fingerprint of the on-disk surface (not the ingest run)
h <- msens::hash_parquet(glue("{dir_dist}/*.parquet"), con)
dbDisconnect(con, shutdown = TRUE)
msens::report_table(smry, caption = "rng_iucn model_cell surface")

rng_iucn model_cell surface
n_models	n_cells	v_min	v_max
6246	1435630286	1	50

Code

# content-addressed manifest: deterministic, no wall-clock, no machine paths ->
# downstream targets re-run only when this surface's content actually changes
msens::write_manifest(
  manifest, target = "ingest_rng_iucn", content_hash = h,
  stats = list(ds_key = ds_key, ver = ver, n_species = nrow(sp_all),
               n_models = smry$n_models, n_cells = smry$n_cells,
               v_min = smry$v_min, v_max = smry$v_max, n_parquet = length(pq)),
  force = msens::force_target("ingest_rng_iucn"))

--- title: "Ingest IUCN Red List expert ranges → global 0.05° cells" msens: target_name: ingest_rng_iucn workflow_type: ingest dependency: [build_cell_grid] output: data/manifests/ingest_rng_iucn.json dataset: {ds_key: rng_iucn, response_type: range, source_authority: IUCN, temporal_interval: static, native_format: vector} editor_options: chunk_output_type: console --- IUCN Red List **expert ranges** for marine taxa onto the **global 0.05° cell grid** — the non-bird complement to AquaMaps/BOTW. Source: the 24 marine taxonomic-group downloads under `raw/iucnredlist.org/` (corals, fish, sharks, marine mammals, turtles, seagrasses, mangroves…). Per species (`id_no`): union its **extant** polygons (`presence ∈ {1,2,3}`), capture the **whole range** (no land mask), value = `msens::compute_er_score( "IUCN:{category}")` (never hard-coded; CR=50/EN=25/VU=5/NT=2/LC,DD=1). `mdl_key = "rng_iucn|{id_no}"`. Species are filtered to IUCN's own **`marine = true`** flag — authoritative for non-birds (e.g. it drops freshwater turtles from the TURTLES group) — so the plain-`MAMMALS` terrestrial species are excluded too. Parallel (mclapply, per group in memory), resumable; `REDO_INGEST=1` rebuilds all. ## Design ```{mermaid} %%| label: fig-design %%| fig-cap: "IUCN expert ranges → whole-range extinction-risk value on the global 0.05° grid" flowchart LR src["rng_iucn_all.gpkg (~/_big) consolidated + INDEX on id_no (pre-filtered presence/marine/category)"] --> er["val = compute_er_score('IUCN:cat') CR50 EN25 VU5 NT2 LC/DD1"] er --> ee["per-species geom (id_no index hit) → cells_from_ranges() vs 0.05° cellid COG"] ee --> pq[("dist/dataset=rng_iucn/{id_no}.parquet (mdl_key, cell_id, val) · V2/zstd")] pq --> mf["hash_parquet → content-addressed manifest"] ``` ## Setup ```{r} #| label: setup librarian::shelf( arrow, DBI, dplyr, duckdb, fs, glue, here, jsonlite, logger, readr, sf, terra, tibble, MarineSensitivity/msens, quiet = T) source(here("libs/paths.R")) # dir_raw, dir_big_v, cellid_tif, ver source(here("libs/vars.R")) # redo_ingest options(readr.show_col_types = F) sf_use_s2(FALSE) ds_key <- "rng_iucn" # consolidated, indexed IUCN ranges on ~/_big (built once from the raw group shapefiles by # scripts/build_iucn_gpkg.R; already filtered to presence 1:3 / marine=true / category-ok, INDEX # on id_no). Reads off Drive and a per-species geometry read is an index hit. iucn_gpkg <- glue("{dir_derived}/iucnredlist.org/rng_iucn_all.gpkg") dir_dist <- glue("{dir_big_v}/marine-atlas/dist/dataset={ds_key}") manifest <- here("data/manifests/ingest_rng_iucn.json") n_workers <- max(1L, parallel::detectCores()) dir_create(c(dir_dist, path_dir(manifest))) stopifnot("run build_cell_grid.qmd first" = file_exists(cellid_tif), "build rng_iucn_all.gpkg first" = file_exists(iucn_gpkg)) if (redo_ingest && dir_exists(dir_dist)) { dir_delete(dir_dist); dir_create(dir_dist) } ``` ## Species → cells (per-species geometry read from the indexed gpkg) ```{r} #| label: interpolate # species list from the gpkg attributes (one row per id_no); geometry read on demand below. sp_all <- st_read(iucn_gpkg, quiet = TRUE, query = "SELECT id_no, sci_name, class, category, grp FROM rng_iucn") |> st_drop_geometry() |> distinct(id_no, sci_name, class, category, grp) |> mutate(er_score = compute_er_score(paste0("IUCN:", category)), mdl_key = mdl_key_raw(ds_key, id_no), group = grp) write_csv(sp_all, glue("{dir_dist}/../model_{ds_key}.csv")) done <- as.integer(path_ext_remove(path_file(dir_ls(dir_dist, glob = "*.parquet")))) todo <- sp_all |> filter(!id_no %in% done) log_info("IUCN: {nrow(sp_all)} marine species, {nrow(todo)} to rasterize ({length(done)} exist)") one <- function(i) { r <- todo[i, ] # per-species read is an id_no index hit. Do NOT st_union here — cells_from_ranges # rasterizes the multiple presence polygons directly (terra unions the touched cells), # and a pre-union costs as much as the rasterize itself on multi-polygon ranges. ply_sp <- st_read(iucn_gpkg, quiet = TRUE, query = sprintf("SELECT geom FROM rng_iucn WHERE id_no = %d", r$id_no)) d <- tryCatch(cells_from_ranges(ply_sp, cellid_tif, value = r$er_score), error = function(e) NULL) if (is.null(d) || nrow(d) == 0) return("empty") msens::write_atlas_parquet( tibble::tibble(mdl_key = r$mdl_key, cell_id = d$cell_id, val = d$val), fs::path(dir_dist, r$id_no, ext = "parquet")); "done" } invisible(parallel::mclapply(seq_len(nrow(todo)), function(i) tryCatch(one(i), error = function(e) "error"), mc.cores = n_workers, mc.preschedule = FALSE)) glue("{nrow(sp_all)} IUCN marine species across {length(unique(sp_all$group))} groups") ``` ## Outputs + manifest ```{r} #| label: outputs pq <- dir_ls(dir_dist, glob = "*.parquet") con <- dbConnect(duckdb()) smry <- dbGetQuery(con, glue( "SELECT count(DISTINCT mdl_key) n_models, count(*) n_cells, min(val) v_min, max(val) v_max FROM read_parquet('{dir_dist}/*.parquet')")) # order-independent content fingerprint of the on-disk surface (not the ingest run) h <- msens::hash_parquet(glue("{dir_dist}/*.parquet"), con) dbDisconnect(con, shutdown = TRUE) msens::report_table(smry, caption = "rng_iucn model_cell surface") # content-addressed manifest: deterministic, no wall-clock, no machine paths -> # downstream targets re-run only when this surface's content actually changes msens::write_manifest( manifest, target = "ingest_rng_iucn", content_hash = h, stats = list(ds_key = ds_key, ver = ver, n_species = nrow(sp_all), n_models = smry$n_models, n_cells = smry$n_cells, v_min = smry$v_min, v_max = smry$v_max, n_parquet = length(pq)), force = msens::force_target("ingest_rng_iucn")) ```