Consolidate weekly refresh into single orchestrator script
All checks were successful
Deploy stAndrews / deploy (push) Successful in 5s
All checks were successful
Deploy stAndrews / deploy (push) Successful in 5s
refresh_all.R downloads both SCPA files once, then sources update_owners.R and update_sales.R. Single cron job at 11pm Sunday.
This commit is contained in:
35
data-raw/refresh_all.R
Normal file
35
data-raw/refresh_all.R
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
# refresh_all.R
|
||||||
|
# Weekly orchestrator. Downloads SCPA data files once, then updates
|
||||||
|
# owners and sales. Run via cron; container restart handled by cron.
|
||||||
|
|
||||||
|
options(timeout = 300)
|
||||||
|
|
||||||
|
cat("=== stAndrews weekly refresh", format(Sys.time()), "===\n")
|
||||||
|
|
||||||
|
# ── Download SCPA Public.xlsx ─────────────────────────────────────────────────
|
||||||
|
cat("Downloading SCPA Public.xlsx...\n")
|
||||||
|
download.file(
|
||||||
|
url = "https://www.sc-pa.com/downloads/SCPA%20Public.xlsx",
|
||||||
|
destfile = "./data-raw/property/SCPA Public.xlsx",
|
||||||
|
mode = "wb"
|
||||||
|
)
|
||||||
|
cat("Done.\n")
|
||||||
|
|
||||||
|
# ── Download SCPA_Parcels_Sales_CSV.zip ───────────────────────────────────────
|
||||||
|
cat("Downloading SCPA_Parcels_Sales_CSV.zip...\n")
|
||||||
|
download.file(
|
||||||
|
url = "https://www.sc-pa.com/downloads/SCPA_Parcels_Sales_CSV.zip",
|
||||||
|
destfile = "./data-raw/property/SCPA_Parcels_Sales_CSV.zip",
|
||||||
|
mode = "wb"
|
||||||
|
)
|
||||||
|
cat("Done.\n")
|
||||||
|
|
||||||
|
# ── Update owners ─────────────────────────────────────────────────────────────
|
||||||
|
cat("\n--- update_owners.R ---\n")
|
||||||
|
source("./data-raw/update_owners.R")
|
||||||
|
|
||||||
|
# ── Update sales ──────────────────────────────────────────────────────────────
|
||||||
|
cat("\n--- update_sales.R ---\n")
|
||||||
|
source("./data-raw/update_sales.R")
|
||||||
|
|
||||||
|
cat("\n=== Refresh complete", format(Sys.time()), "===\n")
|
||||||
@@ -22,13 +22,6 @@ subdivisions <- c(
|
|||||||
# load geometry lookup (static) ----
|
# load geometry lookup (static) ----
|
||||||
geometry_lookup <- readRDS("./data-raw/addresses/geometry_lookup.rds")
|
geometry_lookup <- readRDS("./data-raw/addresses/geometry_lookup.rds")
|
||||||
|
|
||||||
# download fresh scpa data ----
|
|
||||||
download.file(
|
|
||||||
url = "https://www.sc-pa.com/downloads/SCPA%20Public.xlsx",
|
|
||||||
destfile = "./data-raw/property/SCPA Public.xlsx",
|
|
||||||
mode = "wb"
|
|
||||||
)
|
|
||||||
|
|
||||||
# load and clean scpa data ----
|
# load and clean scpa data ----
|
||||||
owners_raw <-
|
owners_raw <-
|
||||||
readxl::read_xlsx(
|
readxl::read_xlsx(
|
||||||
|
|||||||
@@ -17,16 +17,8 @@ subdivisions <- c(
|
|||||||
|
|
||||||
geometry_lookup <- readRDS("./data-raw/addresses/geometry_lookup.rds")
|
geometry_lookup <- readRDS("./data-raw/addresses/geometry_lookup.rds")
|
||||||
|
|
||||||
# ── Download and extract Sarasota.csv ─────────────────────────────────────────
|
# ── Load Sarasota.csv from cached zip ────────────────────────────────────────
|
||||||
zip_path <- "./data-raw/property/SCPA_Parcels_Sales_CSV.zip"
|
csv_con <- unz("./data-raw/property/SCPA_Parcels_Sales_CSV.zip", "Parcel_Sales_CSV/Sarasota.csv")
|
||||||
options(timeout = 300)
|
|
||||||
download.file(
|
|
||||||
url = "https://www.sc-pa.com/downloads/SCPA_Parcels_Sales_CSV.zip",
|
|
||||||
destfile = zip_path,
|
|
||||||
mode = "wb"
|
|
||||||
)
|
|
||||||
|
|
||||||
csv_con <- unz(zip_path, "Parcel_Sales_CSV/Sarasota.csv")
|
|
||||||
|
|
||||||
# ── Load and filter ───────────────────────────────────────────────────────────
|
# ── Load and filter ───────────────────────────────────────────────────────────
|
||||||
sales <-
|
sales <-
|
||||||
|
|||||||
Reference in New Issue
Block a user