Add deployment pipeline and clean up repo
- Add Dockerfile, docker-compose.yml, .dockerignore, .env (port 3842) - Add Caddyfile.snippet for analytics gateway import pattern - Add .gitea/workflows/deploy.yaml for act_runner SSH deploy - Untrack sensitive/data files (SCPA xlsx, owners.rds) - Add renv lockfile and infrastructure files - Reorganize data-raw scripts and add SarasotaCounty boundary data - Move www assets to www/images/, add docs PDFs
This commit is contained in:
@@ -0,0 +1 @@
|
||||
UTF-8
|
||||
BIN
data-raw/SarasotaCountyBoundary/SarasotaCountyBoundary.dbf
Normal file
BIN
data-raw/SarasotaCountyBoundary/SarasotaCountyBoundary.dbf
Normal file
Binary file not shown.
@@ -0,0 +1 @@
|
||||
PROJCS["NAD_1983_HARN_StatePlane_Florida_West_FIPS_0902_Feet",GEOGCS["GCS_North_American_1983_HARN",DATUM["D_North_American_1983_HARN",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",656166.667],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",-82.0],PARAMETER["Scale_Factor",0.999941177],PARAMETER["Latitude_Of_Origin",24.3333333333333],UNIT["US survey foot",0.304800609601219]]
|
||||
BIN
data-raw/SarasotaCountyBoundary/SarasotaCountyBoundary.shp
Normal file
BIN
data-raw/SarasotaCountyBoundary/SarasotaCountyBoundary.shp
Normal file
Binary file not shown.
BIN
data-raw/SarasotaCountyBoundary/SarasotaCountyBoundary.shx
Normal file
BIN
data-raw/SarasotaCountyBoundary/SarasotaCountyBoundary.shx
Normal file
Binary file not shown.
71
data-raw/SarasotaCountyBoundary/SarasotaCountyBoundary.xml
Normal file
71
data-raw/SarasotaCountyBoundary/SarasotaCountyBoundary.xml
Normal file
@@ -0,0 +1,71 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?><MD_Metadata xmlns:gts="http://www.isotc211.org/2005/gts" xmlns:gco="http://www.isotc211.org/2005/gco" xmlns:xalan="http://xml.apache.org/xalan" xmlns:srv="http://www.isotc211.org/2005/srv" xmlns:gmx="http://www.isotc211.org/2005/gmx" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:gml="http://www.opengis.net/gml" xmlns="http://www.isotc211.org/2005/gmd">
|
||||
<characterSet>
|
||||
<MD_CharacterSetCode codeList="http://www.isotc211.org/2005/resources/Codelist/gmxCodelists.xml#MD_CharacterSetCode" codeListValue="utf8" codeSpace="ISOTC211/19115">utf8</MD_CharacterSetCode>
|
||||
</characterSet>
|
||||
<hierarchyLevel>
|
||||
<MD_ScopeCode codeList="http://www.isotc211.org/2005/resources/Codelist/gmxCodelists.xml#MD_ScopeCode" codeListValue="dataset" codeSpace="ISOTC211/19115">dataset</MD_ScopeCode>
|
||||
</hierarchyLevel>
|
||||
<hierarchyLevelName>
|
||||
<gco:CharacterString>dataset</gco:CharacterString>
|
||||
</hierarchyLevelName>
|
||||
<contact gco:nilReason="missing"/>
|
||||
<dateStamp gco:nilReason="missing"/>
|
||||
<metadataStandardName>
|
||||
<gco:CharacterString>ISO 19139 Geographic Information - Metadata - Implementation Specification</gco:CharacterString>
|
||||
</metadataStandardName>
|
||||
<metadataStandardVersion>
|
||||
<gco:CharacterString>2007</gco:CharacterString>
|
||||
</metadataStandardVersion>
|
||||
<identificationInfo>
|
||||
<MD_DataIdentification>
|
||||
<citation>
|
||||
<CI_Citation>
|
||||
<title>
|
||||
<gco:CharacterString>SarasotaCountyBoundary</gco:CharacterString>
|
||||
</title>
|
||||
<date gco:nilReason="missing"/>
|
||||
</CI_Citation>
|
||||
</citation>
|
||||
<abstract>
|
||||
<gco:CharacterString>The Sarasota County Boundary layer contains both Sarasota County and municipal boundary lines and areas.</gco:CharacterString>
|
||||
</abstract>
|
||||
<purpose>
|
||||
<gco:CharacterString>The Sarasota County Boundary layer contains both Sarasota County and municipal boundary lines and areas.</gco:CharacterString>
|
||||
</purpose>
|
||||
<descriptiveKeywords>
|
||||
<MD_Keywords>
|
||||
<keyword>
|
||||
<gco:CharacterString>Cadastral</gco:CharacterString>
|
||||
</keyword>
|
||||
<keyword>
|
||||
<gco:CharacterString>Planning</gco:CharacterString>
|
||||
</keyword>
|
||||
</MD_Keywords>
|
||||
</descriptiveKeywords>
|
||||
<language gco:nilReason="missing"/>
|
||||
<characterSet>
|
||||
<MD_CharacterSetCode codeList="http://www.isotc211.org/2005/resources/Codelist/gmxCodelists.xml#MD_CharacterSetCode" codeListValue="utf8" codeSpace="ISOTC211/19115">utf8</MD_CharacterSetCode>
|
||||
</characterSet>
|
||||
<extent>
|
||||
<EX_Extent>
|
||||
<geographicElement>
|
||||
<EX_GeographicBoundingBox>
|
||||
<westBoundLongitude>
|
||||
<gco:Decimal>-82.6425729822934</gco:Decimal>
|
||||
</westBoundLongitude>
|
||||
<eastBoundLongitude>
|
||||
<gco:Decimal>-82.0565543313639</gco:Decimal>
|
||||
</eastBoundLongitude>
|
||||
<southBoundLatitude>
|
||||
<gco:Decimal>26.9440430628277</gco:Decimal>
|
||||
</southBoundLatitude>
|
||||
<northBoundLatitude>
|
||||
<gco:Decimal>27.391160931026</gco:Decimal>
|
||||
</northBoundLatitude>
|
||||
</EX_GeographicBoundingBox>
|
||||
</geographicElement>
|
||||
</EX_Extent>
|
||||
</extent>
|
||||
</MD_DataIdentification>
|
||||
</identificationInfo>
|
||||
</MD_Metadata>
|
||||
24
data-raw/create_geometry_lookup.R
Normal file
24
data-raw/create_geometry_lookup.R
Normal file
@@ -0,0 +1,24 @@
|
||||
# create_geometry_lookup.R
|
||||
# One-time script. Extracts account_number -> geometry from the manually
|
||||
# corrected QGIS file and saves as a stable lookup table.
|
||||
# Re-run only if geometry ever needs to be corrected.
|
||||
# Output: data-raw/addresses/geometry_lookup.rds
|
||||
|
||||
library(sf)
|
||||
library(dplyr)
|
||||
|
||||
raw <- st_read(
|
||||
"./data-raw/addresses/owners_moved.gpkg",
|
||||
layer = "owners_raw",
|
||||
quiet = TRUE
|
||||
)
|
||||
cat("Columns:", paste(names(raw), collapse = ", "), "\n")
|
||||
|
||||
lookup <- raw |>
|
||||
select(account_number)
|
||||
|
||||
cat("Lookup rows:", nrow(lookup), "\n")
|
||||
cat("Any duplicate account numbers:", anyDuplicated(lookup$account_number) > 0, "\n")
|
||||
|
||||
saveRDS(lookup, "./data-raw/addresses/geometry_lookup.rds")
|
||||
cat("Saved to data-raw/addresses/geometry_lookup.rds\n")
|
||||
15
data-raw/create_venice_bndry.R
Normal file
15
data-raw/create_venice_bndry.R
Normal file
@@ -0,0 +1,15 @@
|
||||
library(rmapshaper)
|
||||
library(dplyr)
|
||||
library(sf)
|
||||
library(leaflet)
|
||||
library(sfheaders)
|
||||
|
||||
df <- st_read("./data-raw/SarasotaCountyBoundary/") %>%
|
||||
filter(municipali == "CV") %>%
|
||||
filter(acreage > 2500) %>%
|
||||
ms_simplify(keep = .025) %>%
|
||||
mutate(geometry = sfheaders::sf_remove_holes(geometry)) %>%
|
||||
sf::st_transform(crs = 4326) %>%
|
||||
mutate(label = "City of Venice", .before = boundarycl) %>%
|
||||
saveRDS(., file = "./data/venice.rds")
|
||||
|
||||
145
data-raw/main.R
Normal file
145
data-raw/main.R
Normal file
@@ -0,0 +1,145 @@
|
||||
#libraries ----
|
||||
library("leaflet")
|
||||
library("readxl")
|
||||
library("janitor")
|
||||
library("dplyr")
|
||||
library("tidyr")
|
||||
library("tidygeocoder")
|
||||
library("sf")
|
||||
|
||||
# choose columns ---
|
||||
cols <- c(
|
||||
"account_number",
|
||||
"owner_1",
|
||||
"owner_2",
|
||||
"subdivision",
|
||||
"situs_address_property_address",
|
||||
"situs_city",
|
||||
"situs_state",
|
||||
"situs_zip_code",
|
||||
"homestead_exemption_yes_or_no"
|
||||
)
|
||||
|
||||
# filter to St. Andrews (388) ----
|
||||
df <-
|
||||
readxl::read_xlsx(
|
||||
path = "./data-raw/property/SCPA Public.xlsx",
|
||||
n_max = Inf,
|
||||
.name_repair = ~janitor::make_clean_names(.x)
|
||||
) %>%
|
||||
filter(
|
||||
subdivision %in% c("8120", "8113", "8171", "8195", "8221",
|
||||
"8163", "8240", "8159", "8149", "8110", "8254", "8215", "8143")
|
||||
) %>%
|
||||
#select(cols) %>%
|
||||
rename(
|
||||
situs_address = situs_address_property_address,
|
||||
homestead = homestead_exemption_yes_or_no
|
||||
) %>%
|
||||
filter(!grepl("^0", situs_address)) %>%
|
||||
drop_na(situs_address)
|
||||
|
||||
# create location ----
|
||||
df1 <-
|
||||
df %>%
|
||||
separate(
|
||||
situs_address,
|
||||
into = c("street_no", "street", "suffix", "unit", "bldg", "no"),
|
||||
sep = "\\s+"
|
||||
) %>%
|
||||
mutate(unit = ifelse(is.na(bldg), NA, unit)) %>%
|
||||
mutate(location = paste(
|
||||
street_no, street, suffix, ",",
|
||||
situs_city, situs_state, situs_zip_code,
|
||||
sep = " ")) %>%
|
||||
mutate(location = gsub(" ,", ",", location)) %>%
|
||||
mutate(label = paste(street_no, street, suffix, sep = " ")) %>%
|
||||
arrange(account_number)
|
||||
|
||||
# geotag unique location (205) ----
|
||||
if(!file.exists("./data-raw/geotagged_street_addresses.rds")) {
|
||||
addresses <- df1 %>% select(location) %>% distinct()
|
||||
lat_longs <- addresses %>%
|
||||
geocode(
|
||||
location,
|
||||
method = 'google',
|
||||
lat = lat,
|
||||
long = lng
|
||||
)
|
||||
saveRDS(lat_longs, "./data-raw/geotagged_street_addresses.rds")
|
||||
} else {
|
||||
lat_longs <- readRDS("./data-raw/geotagged_street_addresses.rds")
|
||||
}
|
||||
|
||||
# append coords ----
|
||||
df2 <-
|
||||
df1 %>%
|
||||
left_join(lat_longs[, !names(lat_longs) %in% "account_number"],
|
||||
by = c("location" = "location"),
|
||||
relationship = "many-to-many") %>%
|
||||
distinct() %>%
|
||||
st_as_sf(coords = c("lng", "lat"), crs = 4326)
|
||||
|
||||
|
||||
# write as esri shp file for qgis ----
|
||||
sf::st_write(
|
||||
df2,
|
||||
"./data-raw/addresses/owners_raw.gpkg",
|
||||
driver = "GPKG",
|
||||
delete_dsn = TRUE
|
||||
)
|
||||
|
||||
#####
|
||||
# adjust points in QGIS
|
||||
####
|
||||
|
||||
# after QGIS, shp --> rds ----
|
||||
owners_moved <- sf::st_read(
|
||||
"./data-raw/addresses/owners_moved.gpkg",
|
||||
layer = "owners_raw")
|
||||
owners_moved %>%
|
||||
leaflet() %>%
|
||||
addTiles() %>%
|
||||
addMarkers()
|
||||
saveRDS(owners_moved, "./data/owners.rds")
|
||||
|
||||
# add plats ----
|
||||
plats <- sf::st_read("./data/plats/plats.shp")
|
||||
owners <- readRDS("./data/owners.rds")
|
||||
# display ----
|
||||
library("leafpop")
|
||||
m <-
|
||||
leaflet() %>%
|
||||
addTiles() %>%
|
||||
addPolygons(
|
||||
data = plats,
|
||||
color = "red",
|
||||
weight = 2,
|
||||
opacity = 0.5,
|
||||
fillOpacity = 0.2,
|
||||
label = ~sub_name,
|
||||
group = "Subdivisions"
|
||||
) %>%
|
||||
addMarkers(
|
||||
data = owners,
|
||||
lat = ~lat,
|
||||
lng = ~lng,
|
||||
popup = popupTable(
|
||||
owners,
|
||||
zcol = c(
|
||||
"account_number",
|
||||
"location",
|
||||
"owner_1",
|
||||
"owner_2"
|
||||
)
|
||||
),
|
||||
group = "Owners"
|
||||
) %>%
|
||||
addLayersControl(
|
||||
overlayGroups = c("Subdivisions", "Owners"), # Specify overlay groups
|
||||
options = layersControlOptions(collapsed = FALSE) # Control options
|
||||
)
|
||||
m
|
||||
# add condo layer
|
||||
|
||||
|
||||
Binary file not shown.
BIN
data-raw/sarco/building_footprints/building_footprints.gpkg
Normal file
BIN
data-raw/sarco/building_footprints/building_footprints.gpkg
Normal file
Binary file not shown.
56
data-raw/sarco/create_building_footprints.R
Normal file
56
data-raw/sarco/create_building_footprints.R
Normal file
@@ -0,0 +1,56 @@
|
||||
# create_building_footprints.R
|
||||
# Pull Sarasota County building footprints clipped to St. Andrews boundary.
|
||||
# Source: https://ags3.scgov.net/server/rest/services/Hosted/BuildingFootprint/FeatureServer/0
|
||||
# Output: data-raw/sarco/building_footprints/building_footprints.gpkg
|
||||
# No esri2sf needed — queries ArcGIS REST API directly via GeoJSON URL.
|
||||
|
||||
library(sf)
|
||||
library(dplyr)
|
||||
|
||||
# load st. andrews subdivision boundary ----
|
||||
plats <- st_read("./data/plats/plats.shp", quiet = TRUE) |> st_transform(4326)
|
||||
boundary <- st_union(plats)
|
||||
bb <- st_bbox(boundary)
|
||||
|
||||
# build arcgis rest query url ----
|
||||
base_url <- "https://ags3.scgov.net/server/rest/services/Hosted/BuildingFootprint/FeatureServer/0/query"
|
||||
geometry <- paste(bb["xmin"], bb["ymin"], bb["xmax"], bb["ymax"], sep = ",")
|
||||
params <- paste0(
|
||||
"?where=1=1",
|
||||
"&geometry=", geometry,
|
||||
"&geometryType=esriGeometryEnvelope",
|
||||
"&inSR=4326",
|
||||
"&spatialRel=esriSpatialRelIntersects",
|
||||
"&outFields=*",
|
||||
"&returnGeometry=true",
|
||||
"&f=geojson"
|
||||
)
|
||||
url <- paste0(base_url, params)
|
||||
cat("URL:\n", url, "\n\n")
|
||||
|
||||
# fetch ----
|
||||
cat("Querying feature service...\n")
|
||||
footprints_raw <- st_read(url, quiet = TRUE)
|
||||
cat("Raw rows:", nrow(footprints_raw), "\n")
|
||||
cat("Raw fields:", paste(names(footprints_raw), collapse = ", "), "\n\n")
|
||||
|
||||
# clip to exact subdivision boundary ----
|
||||
footprints <- footprints_raw |>
|
||||
st_transform(4326) |>
|
||||
st_filter(boundary)
|
||||
|
||||
# inspect ----
|
||||
cat("Rows:", nrow(footprints), "\n")
|
||||
cat("\nFields:\n")
|
||||
print(names(footprints))
|
||||
cat("\nSample rows:\n")
|
||||
print(head(st_drop_geometry(footprints)))
|
||||
|
||||
# save ----
|
||||
st_write(
|
||||
footprints,
|
||||
"./data-raw/sarco/building_footprints/building_footprints.gpkg",
|
||||
delete_dsn = TRUE
|
||||
)
|
||||
|
||||
cat("\nSaved to data-raw/sarco/building_footprints/building_footprints.gpkg\n")
|
||||
82
data-raw/update_owners.R
Normal file
82
data-raw/update_owners.R
Normal file
@@ -0,0 +1,82 @@
|
||||
# update_owners.R
|
||||
# Weekly update script. Reads fresh SCPA property data, joins to stable
|
||||
# geometry lookup by account_number, saves data/owners.rds.
|
||||
# Only the SCPA xlsx needs to be replaced to refresh ownership data.
|
||||
# Input: data-raw/property/SCPA Public.xlsx (replace weekly)
|
||||
# data-raw/addresses/geometry_lookup.rds (static)
|
||||
# Output: data/owners.rds
|
||||
|
||||
library(readxl)
|
||||
library(janitor)
|
||||
library(dplyr)
|
||||
library(stringr)
|
||||
library(sf)
|
||||
|
||||
subdivisions <- c(
|
||||
"8120", "8113", "8171", "8195", "8221",
|
||||
"8163", "8240", "8159", "8149", "8110", "8254", "8215", "8143"
|
||||
)
|
||||
|
||||
# load geometry lookup (static) ----
|
||||
geometry_lookup <- readRDS("./data-raw/addresses/geometry_lookup.rds")
|
||||
|
||||
# download fresh scpa data ----
|
||||
download.file(
|
||||
url = "https://www.sc-pa.com/downloads/SCPA%20Public.xlsx",
|
||||
destfile = "./data-raw/property/SCPA Public.xlsx",
|
||||
mode = "wb"
|
||||
)
|
||||
|
||||
# load and clean scpa data ----
|
||||
owners_raw <-
|
||||
readxl::read_xlsx(
|
||||
path = "./data-raw/property/SCPA Public.xlsx",
|
||||
n_max = Inf,
|
||||
.name_repair = ~janitor::make_clean_names(.x)
|
||||
) |>
|
||||
filter(subdivision %in% subdivisions) |>
|
||||
rename(
|
||||
situs_address = situs_address_property_address,
|
||||
homestead = homestead_exemption_yes_or_no
|
||||
) |>
|
||||
filter(!is.na(situs_address)) |>
|
||||
filter(!grepl("^0", situs_address)) |>
|
||||
mutate(
|
||||
# extract clean street address (before multiple spaces / unit suffix)
|
||||
label = str_trim(str_extract(situs_address, "^\\d+\\s+\\S+\\s+\\S+")),
|
||||
location = paste0(label, ", Venice FL")
|
||||
) |>
|
||||
select(account_number, owner_1, owner_2, subdivision, homestead, label, location)
|
||||
|
||||
# join to geometry ----
|
||||
owners <- owners_raw |>
|
||||
inner_join(geometry_lookup, by = "account_number") |>
|
||||
st_as_sf(sf_column_name = "geom")
|
||||
|
||||
# report any unmatched records ----
|
||||
n_unmatched <- nrow(owners_raw) - nrow(owners)
|
||||
if (n_unmatched > 0) {
|
||||
cat("WARNING:", n_unmatched, "records had no matching geometry and were dropped.\n")
|
||||
missing <- anti_join(owners_raw, st_drop_geometry(geometry_lookup), by = "account_number")
|
||||
print(missing)
|
||||
}
|
||||
|
||||
# report most recent sale date in st. andrews ----
|
||||
latest_sale <-
|
||||
readxl::read_xlsx(
|
||||
path = "./data-raw/property/SCPA Public.xlsx",
|
||||
n_max = Inf,
|
||||
.name_repair = ~janitor::make_clean_names(.x)
|
||||
) |>
|
||||
filter(subdivision %in% subdivisions) |>
|
||||
select(account_number, owner_1, contains("sale")) |>
|
||||
filter(!is.na(account_number)) |>
|
||||
mutate(last_sale_date = as.Date(last_sale_date, format = "%m/%d/%Y")) |>
|
||||
arrange(desc(last_sale_date)) |>
|
||||
head(1)
|
||||
|
||||
cat("Owners written:", nrow(owners), "\n")
|
||||
attr(owners, "last_sale_date") <- latest_sale$last_sale_date
|
||||
saveRDS(owners, "./data/owners.rds")
|
||||
cat("Saved to data/owners.rds\n")
|
||||
cat("Most recent sale date:", format(latest_sale$last_sale_date, "%B %d, %Y"), "\n")
|
||||
Reference in New Issue
Block a user