Publish a browsable index for the public bucket (storage.marinesensitivity.org)

The marine-atlas is public, but nobody can look at it. S3 serves objects, not directories: a URL ending in / 404s unless an object literally has that key, and anonymous ListBucket is denied on this bucket (verified — 403 AccessDenied), so a browser cannot enumerate anything. The only way in is to already know an object’s exact key.

This generates a real index.html for each directory, with credentials, at publish time. A Caddy vhost (server/caddy/Caddyfile) then rewrites folder URLs to those objects, so storage.marinesensitivity.org/marine-atlas/v8/ browses the way people expect.

Named storage rather than s3 so the URL survives a move to another provider.

Machine-only trees are summarized, not walked. serve/model_cell/ alone is ~17,765 Hive partitions and cog/ is tens of thousands of content-addressed rasters. A page each would be tens of thousands of documents nobody reads, so those subtrees are shown at their parent with a count and marked not browsable — rather than linked to a page that would never exist.

1 Setup

librarian::shelf(dplyr, glue, knitr, logger, yaml, MarineSensitivity/msens, quiet = TRUE)
source(here::here("libs/paths.R"))

do_s3   <- Sys.getenv("STORAGE_INDEX_NO_S3") == ""
bucket  <- "s3://oceanmetrics.io-public"
site    <- "https://storage.marinesensitivity.org"
obj_url <- msens::atlas_base_url()                       # path-style HTTPS, dotted bucket
obj_url <- sub("/marine-atlas$", "", obj_url)            # bucket root
dir_out <- path.expand(glue("{dir_big_v}/storage-index"))

unlink(dir_out, recursive = TRUE); dir.create(dir_out, recursive = TRUE)
log_info("bucket={bucket} site={site} push_s3={do_s3}")

2 List the bucket

# ONLY the published atlas. The bucket also holds `backups/` -- including a
# 3.1 GB database dump -- which is technically public but should not be
# advertised: indexing turns "readable if you know the exact key" into
# "discoverable and crawlable". The Caddy vhost enforces the same restriction,
# so a page outside marine-atlas cannot be reached through the browse host even
# by exact path.
objs <- msens::s3_list_all(bucket, prefix = "marine-atlas")
log_info("{format(nrow(objs), big.mark=',')} objects under marine-atlas/, ",
         "{round(sum(objs$size)/1024^3, 1)} GB")

# A RESTRICTED release (`access` in versions.json -- a pre-release under review,
# see build_version_manifest.qmd) is not advertised here. Its objects stay
# exactly where the serving stack reads them; what changes is discoverability:
# no page lists its directory, no page is generated inside it, and any page an
# earlier run left there is removed in the publish chunk. "Readable if you know
# the exact key" is the whole of what remains, which is the presentation-only
# scope the review gate was given (data-level restriction is the separate
# restricted-datasets plan).
restricted <- msens::atlas_versions(refresh = TRUE) |>
  filter(access == "restricted") |> pull(ver)
if (length(restricted)) {
  pat      <- sprintf("^marine-atlas/(%s)/", paste(restricted, collapse = "|"))
  n_before <- nrow(objs)
  objs     <- objs |> filter(!grepl(pat, key))
  log_info("restricted release(s) {paste(restricted, collapse = ', ')}: ",
           "{format(n_before - nrow(objs), big.mark = ',')} objects withheld from the index")
} else log_info("no restricted release; every version is indexed")

objs |>
  mutate(top = sub("/.*$", "", key)) |>
  count(top, name = "objects", wt = NULL) |>
  left_join(objs |> mutate(top = sub("/.*$", "", key)) |>
              group_by(top) |> summarise(gb = round(sum(size)/1024^3, 2), .groups = "drop"),
            by = "top") |>
  arrange(desc(objects)) |>
  knitr::kable(caption = "objects by top-level prefix")

objects by top-level prefix

top	objects	gb
marine-atlas	23809	1.96

3 Generate the pages

# Per-directory explanations, rendered above each listing. A file listing says
# WHAT is there but never what it means, how it was made, or how to reach it
# without a browser -- STAC search, DuckDB over Parquet, GDAL /vsicurl.
readme <- yaml::read_yaml(here::here("data/storage_readme.yml"))
log_info("{length(readme)} directory README(s)")

ix <- msens::build_storage_index(objs, site_url = site, obj_url = obj_url, readme = readme)
log_info("{nrow(ix)} index page(s)")

# publish the SAME text as README.md beside the data, so `aws s3 cp` and `curl`
# users get the explanation the browser shows
for (k in names(readme)) {
  f <- file.path(dir_out, if (k == ".") "README.md" else file.path(k, "README.md"))
  dir.create(dirname(f), recursive = TRUE, showWarnings = FALSE)
  writeLines(readme[[k]], f)
}

for (i in seq_len(nrow(ix))) {
  f <- file.path(dir_out, ix$key[i])
  dir.create(dirname(f), recursive = TRUE, showWarnings = FALSE)
  writeLines(ix$html[i], f)
}

knitr::kable(head(ix["key"], 15), caption = "generated pages (first 15)")

generated pages (first 15)

key
index.html
marine-atlas/index.html
marine-atlas/cog/index.html
marine-atlas/cog/global05/index.html
marine-atlas/cog/usa05/index.html
marine-atlas/grid/index.html
marine-atlas/grid/global05/index.html
marine-atlas/grid/usa05/index.html
marine-atlas/v1/index.html
marine-atlas/v1/tables/index.html
marine-atlas/v2/index.html
marine-atlas/v2/tables/index.html
marine-atlas/v3/index.html
marine-atlas/v3/tables/index.html
marine-atlas/v4/index.html

4 Publish + verify

if (do_s3 && nrow(ix)) {
  args <- c("s3", "sync", shQuote(dir_out), shQuote(bucket),
            "--exclude", shQuote("*"),
            "--include", shQuote("*index.html"), "--include", shQuote("*README.md"),
            # regenerated whenever the bucket changes; a cached copy shows a
            # listing that no longer matches what is there
            "--cache-control", shQuote("no-cache"),
            "--only-show-errors", "--no-progress")
  out <- system2("aws", args, stdout = TRUE, stderr = TRUE)
  if (!is.null(attr(out, "status")) && attr(out, "status") != 0)
    stop("aws s3 sync failed: ", paste(out, collapse = "\n"))

  # verify the DEEPEST page, not the shallowest: CalCOFI's generator reported
  # success while every nested page silently stayed local, because a zero exit
  # status does not mean the objects are there
  deepest <- ix$key[which.max(lengths(strsplit(ix$key, "/", fixed = TRUE)))]
  url <- sprintf("%s/%s", obj_url, deepest)
  code <- tryCatch({ con <- url(url, open = "rb"); close(con); 200L }, error = function(e) 404L)
  stopifnot("deepest index page is not readable after upload" = code == 200L)
  log_info("verified {url}")

  # restricted releases: remove the pages an earlier (public) run generated
  # inside them, then PROVE the folder URL no longer resolves -- the check the
  # Caddy rewrite would otherwise happily serve a stale page through
  for (rv in restricted) {
    args <- c("s3", "rm", shQuote(glue("{bucket}/marine-atlas/{rv}/")), "--recursive",
              "--exclude", shQuote("*"),
              "--include", shQuote("*index.html"), "--include", shQuote("*README.md"),
              "--only-show-errors")
    out <- system2("aws", args, stdout = TRUE, stderr = TRUE)
    if (!is.null(attr(out, "status")) && attr(out, "status") != 0)
      stop("aws s3 rm failed for ", rv, ": ", paste(out, collapse = "\n"))
    gone <- sprintf("%s/marine-atlas/%s/index.html", obj_url, rv)
    code <- tryCatch({ con <- url(gone, open = "rb"); close(con); 200L }, error = function(e) 404L)
    stopifnot("a restricted release still has an index page after removal" = code != 200L)
    log_info("restricted {rv}: index pages removed; {gone} no longer resolves")
  }
} else {
  log_info("STORAGE_INDEX_NO_S3 set - staged locally at {dir_out}")
}

5 Target manifest

msens::write_manifest(
  here::here("data/manifests/publish_storage_index.json"),
  target       = "publish_storage_index",
  content_hash = digest::digest(ix$key, algo = "xxhash64"),
  stats = list(n_objects = nrow(objs), n_pages = nrow(ix),
               gb = round(sum(objs$size) / 1024^3, 2)),
  force = msens::force_target("publish_storage_index"))