Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: pull-data-from-MDEB-data-hub
name: pull data from MDAT data hub

on:
workflow_dispatch:
Expand Down Expand Up @@ -32,29 +32,26 @@ jobs:
with:
r-version: '4.2.0'


- name: Install system dependencies
uses: r-lib/actions/setup-r-dependencies@v2

- name: Install command line packages
run: |
sudo apt update
sudo apt-get install libcurl4-openssl-dev libgit2-dev libudunits2-dev
# sudo apt-get install libgdal-dev libcurl4-gnutls-dev libgit2-dev libudunits2-dev libharfbuzz-dev libfribidi-dev
shell: bash



- name: Install packages
run: |
install.packages(c("remotes","here","httr2","jsonlite","dplyr","glue"))
remotes::install_github("NEFSC/NEFSC-Spatial")
shell: Rscript {0}

- name: Pull MDEB data and save to repo
- name: Pull MDAT data and save to repo
run: |
options(timeout = 60 * 60 * 6)
source(here::here("data-raw/get_mdeb_for_package.R"))
get_mdeb_for_package(overwrite = TRUE)
source(here::here("data-raw/get_mdat_for_package.R"))
get_mdat_for_package(overwrite = TRUE)
shell: Rscript {0}

# Create the Pull Request from dev if changes are detected
Expand Down
18 changes: 9 additions & 9 deletions data-raw/find_mdeb_data.R → data-raw/find_mdat_data.R
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
#' Find available data hosted on the MDEB (Marine Development and Ecology Branch) GIS Data Hub
#' Find available data hosted on the MDAT (Marine Development and Advanced Technology Program) GIS Data Hub
#' hosted on NOAA's Geoplatform
#'
#' @description
#' This function retrieves the data available on the
#' \href{https://mdeb-nefsc-noaa.hub.arcgis.com/}{MDEB GIS Data Hub}.
#' \href{https://mdeb-nefsc-noaa.hub.arcgis.com/}{MDAT GIS Data Hub}.
#'
#' @return A tibble with the following columns:
#' \item{title}{Name of the NOAA Geoplatform feature service}
Expand All @@ -13,21 +13,21 @@
#'
#' @examples
#' # Retrieve a list of available data
#' mdeb_data <- find_mdeb_data()
#' mdeb_data
#' mdat_data <- find_mdat_data()
#' mdat_data
#'
#' @export

find_mdeb_data <- function() {
# url of the MDEB GIS Data Hub search API
mdeb_hub <- "https://mdeb-nefsc-noaa.hub.arcgis.com/api/search/v1/collections/all/items"
find_mdat_data <- function() {
# url of the MDAT GIS Data Hub search API
mdat_hub <- "https://mdeb-nefsc-noaa.hub.arcgis.com/api/search/v1/collections/all/items"
# scrape json to get all feature services
catalog_items <- mdeb_hub |>
catalog_items <- mdat_hub |>
jsonlite::fromJSON()
# count the number of available services
num_items <- catalog_items$numberMatched
# pull all feature service items (API default = 10)
catalog_items_v2 <- paste0(mdeb_hub, "?limit=", num_items) |>
catalog_items_v2 <- paste0(mdat_hub, "?limit=", num_items) |>
jsonlite::fromJSON()
# extract metadata properties
available_datasets <- catalog_items_v2$features$properties |>
Expand Down
22 changes: 11 additions & 11 deletions data-raw/get_mdeb_data.R → data-raw/get_mdat_data.R
Original file line number Diff line number Diff line change
@@ -1,45 +1,45 @@
#' Get data from the MDEB (Marine Development and Ecology Branch) GIS Data Hub
#' hosted on NOAA's Geoplatform
#' Get data from the MDAT (Marine Development and Advanced Technology Program) GIS Data Hub
#' hosted on NOAA's GeoPlatform
#'
#' @description
#' This function retrieves data from the \href{https://mdeb-nefsc-noaa.hub.arcgis.com/}{MDEB GIS Data Hub}.
#' This function retrieves data from the \href{https://mdeb-nefsc-noaa.hub.arcgis.com/}{MDAT GIS Data Hub}.
#'
#' @param dataset_name Name of the dataset to retrieve.
#' @param dataset_id ID of the dataset layer to retrieve.
#'
#' @section Available data:
#' The list of available data and corresponding REST URLs
#' can be found by running `find_mdeb_data()`.
#' can be found by running `find_mdat_data()`.
#'
#' @return An sf data.frame-like object
#'
#' @examples
#' # Pull EcoMon Survey strata from MDEB GIS Data Hub
#' ecomon_strata <- get_mdeb_data(dataset_name = "Ecosystem Monitoring Survey", dataset_id = 1)
#' # Pull EcoMon Survey strata from MDAT GIS Data Hub
#' ecomon_strata <- get_mdat_data(dataset_name = "Ecosystem Monitoring Survey", dataset_id = 1)
#' # Plot the data
#' plot(ecomon_strata)
#'
#' @export

get_mdeb_data <- function(dataset_name = NULL, dataset_id = NULL) {
get_mdat_data <- function(dataset_name = NULL, dataset_id = NULL) {
# check if dataset_name, dataset_id is NULL
if (is.null(dataset_name) || is.null(dataset_id)) {
message(
"Neither dataset_name or id can be NULL.
To find the list of available datasets, please use run
find_mdeb_data()."
find_mdat_data()."
)
return()
}

allData <- find_mdeb_data()
allData <- find_mdat_data()
set <- allData |>
dplyr::filter(title == dataset_name, id == dataset_id)

if (nrow(set) == 0) {
stop(
"Data set not found. To find the list of available datasets, please use run
find_mdeb_data()"
"Data set not found. To find the list of available datasets, please run
find_mdat_data()"
)
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,24 +1,24 @@
#' Pull all content from MDEB data hub and create sf objects
#' Pull all content from MDAT data hub and create sf objects
#'
#' Pulls the data and converts to sf, then saves as part of the package
#'
#'
#'

source(here::here("data-raw/find_mdeb_data.R"))
source(here::here("data-raw/get_mdeb_data.R"))
source(here::here("data-raw/generate_roxygen_mdeb.R"))
source(here::here("data-raw/find_mdat_data.R"))
source(here::here("data-raw/get_mdat_data.R"))
source(here::here("data-raw/generate_roxygen_mdat.R"))

get_mdeb_for_package <- function(overwrite = FALSE) {
# find all the data set on the MDEB HUB
datasets <- find_mdeb_data()
get_mdat_for_package <- function(overwrite = FALSE) {
# find all the data set on the MDAT HUB
datasets <- find_mdat_data()

# loop through each and save as rda file.
# also create documentation for each dataset
for (ids in 1:nrow(datasets)) {
dataset <- datasets[ids, ]
message("Processing: ", dataset$title)
data <- get_mdeb_data(dataset$title, dataset$id)
data <- get_mdat_data(dataset$title, dataset$id)
if (!is.null(data)) {
# Save the data as an R object in the package
dataset_name <- tolower(gsub("SMIT_", "", dataset$name))
Expand Down Expand Up @@ -52,7 +52,7 @@ get_mdeb_for_package <- function(overwrite = FALSE) {
)
}
# Generate Roxygen data doc
generate_roxygen_mdeb(
generate_roxygen_mdat(
dataset_name = paste0(dataset_name, '.rda'),
title = dataset$title,
description = modified_dataset_description
Expand Down
4 changes: 2 additions & 2 deletions pkgdown/_pkgdown.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ navbar:
articles:
text: Articles
menu:
- text: MDEB Datasets
href: articles/mdeb_datasets.html
- text: MDAT Datasets
href: articles/mdat_datasets.html
- text: Team Members
href: articles/teamMembers.html

Expand Down
Loading