From c58224afa3cd3de88409ec1642aa24d358b22b74 Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Wed, 14 May 2025 18:08:19 -0700 Subject: [PATCH 01/22] setup fastbuild auto test --- .github/workflows/fast-build.yml | 59 +++++++++++++++++++ uwpostgis-foundry/Dockerfile | 4 +- .../src/alpinedb/download_state.sh | 15 ++--- 3 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/fast-build.yml diff --git a/.github/workflows/fast-build.yml b/.github/workflows/fast-build.yml new file mode 100644 index 0000000..1266f73 --- /dev/null +++ b/.github/workflows/fast-build.yml @@ -0,0 +1,59 @@ +name: Fastbuild postgis images + +on: + push: + branches: + - "5-speed-up-builder" + pull_request: + branches: + - "5-speed-up-builder" + +concurrency: + group: ${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +env: + # dynamic doesn't work, only static + BUILD_NUM: $(date +%Y%m%d%H%M%S) + REPOSITORY: + +jobs: + + setup-postgis-states: + runs-on: self-hosted + outputs: + matrix: ${{ steps.matrix.outputs.value }} + + steps: + - id: matrix + run: | + echo 'value=["ak", "al", "ar", "az", "ca", "co", "ct", "dc", "de", "fl", "ga", "hi", "ia", "id", "il", "in", "ks", "ky", "la", "ma", "md", "me", "mi", "mn", "mo", "ms", "mt", "nc", "nd", "ne", "nh", "nj", "nm", "nv", "ny", "oh", "ok", "or", "pa", "ri", "sc", "sd", "tn", "tx", "ut", "va", "vt", "wa", "wi", "wv", "wy", "pr"]' >> $GITHUB_OUTPUT + - run: | + echo "${{ steps.matrix.outputs.value }}" + + build-postgis-states: + runs-on: self-hosted + needs: setup-postgis-states + continue-on-error: true + strategy: + matrix: + value: ${{fromJSON(needs.setup-postgis-states.outputs.matrix)}} + + steps: + - uses: actions/checkout@v4 + - name: import BUILD_NUM from file + run: echo "BUILD_NUM=$(cat ~/BUILD_NUM)" >> $GITHUB_ENV + - name: print env + run: env + - name: Build the postgis image + run: docker build --shm-size 4g --load --cpu-quota $(( $(nproc --all)*100000 )) --build-arg state_var=${{ matrix.value }} uwpostgis-foundry --platform linux/amd64 --tag genoa-container-registry.washington.palantircloud.com/postgis-${{ matrix.value }}:${{ env.BUILD_NUM }} + - name: Run Trivy CVE vulnerability scanner + uses: aquasecurity/trivy-action@0.25.0 + with: + image-ref: genoa-container-registry.washington.palantircloud.com/postgis-${{ matrix.value }}:${{ env.BUILD_NUM }} + format: 'table' + exit-code: '1' + ignore-unfixed: true + vuln-type: 'os,library' + severity: 'CRITICAL,HIGH,MEDIUM' + diff --git a/uwpostgis-foundry/Dockerfile b/uwpostgis-foundry/Dockerfile index acea6ee..b70b8b8 100644 --- a/uwpostgis-foundry/Dockerfile +++ b/uwpostgis-foundry/Dockerfile @@ -1,5 +1,6 @@ -FROM postgis/postgis:16-3.5-alpine AS buildtime_init_builder +FROM postgis/postgis:17-3.5-alpine AS buildtime_init_builder +# build vars ARG state_var ARG TIGER_DOMAIN ARG POSTGRES_DB=geocoder @@ -7,6 +8,7 @@ ARG POSTGRES_USER=clad_svc ARG POSTGRES_PASSWORD=not_on_gitlab ARG GEOCODER_YEAR=2020 +# runtime vars ENV STATES=${state_var} ENV TIGER_DOMAIN=${TIGER_DOMAIN} ENV PGDATA=/pgdata diff --git a/uwpostgis-foundry/src/alpinedb/download_state.sh b/uwpostgis-foundry/src/alpinedb/download_state.sh index 6cb6077..e078a9d 100755 --- a/uwpostgis-foundry/src/alpinedb/download_state.sh +++ b/uwpostgis-foundry/src/alpinedb/download_state.sh @@ -12,6 +12,7 @@ usage(){ [[ -z ${YEAR} ]] && YEAR="2020" [[ -z ${GISDATA} ]] && GISDATA="/tmp" #"/gisdata" [[ -z ${OUTDIR} ]] && OUTDIR="${GISDATA}/gisdata/" +[[ -z ${DOWNLOADTOOL} ]] && DOWNLOADTOOL="wget --no-check-certificate --mirror --reject=html --no-verbose " [[ -z ${UNZIPTOOL} ]] && UNZIPTOOL=unzip while getopts "d:s:y:?" arg; do @@ -73,7 +74,7 @@ load_state_data () { for i in "${files[@]}" do - wget $BASEURL/ADDR/$i --no-verbose --mirror + ${DOWNLOADTOOL} $BASEURL/ADDR/$i done cd $GISDATA/$BASEPATH/ADDR @@ -87,7 +88,7 @@ load_state_data () { # Place ############# cd $GISDATA - wget $BASEURL/PLACE/tl_${YEAR}_${FIPS}_place.zip --mirror --reject=html --no-verbose + ${DOWNLOADTOOL} $BASEURL/PLACE/tl_${YEAR}_${FIPS}_place.zip cd $GISDATA/$BASEPATH/PLACE for z in tl_${YEAR}_${FIPS}*_place.zip ; @@ -99,7 +100,7 @@ load_state_data () { # Cousub ############# cd $GISDATA - wget $BASEURL/COUSUB/tl_${YEAR}_${FIPS}_cousub.zip --mirror --reject=html --no-verbose + ${DOWNLOADTOOL} $BASEURL/COUSUB/tl_${YEAR}_${FIPS}_cousub.zip cd $GISDATA/$BASEPATH/COUSUB for z in tl_${YEAR}_${FIPS}*_cousub.zip ; @@ -111,7 +112,7 @@ load_state_data () { # Tract ############# cd $GISDATA - wget $BASEURL/TRACT/tl_${YEAR}_${FIPS}_tract.zip --mirror --reject=html --no-verbose + ${DOWNLOADTOOL} $BASEURL/TRACT/tl_${YEAR}_${FIPS}_tract.zip cd $GISDATA/$BASEPATH/TRACT for z in tl_${YEAR}_${FIPS}*_tract.zip ; @@ -127,7 +128,7 @@ load_state_data () { for i in "${files[@]}" do - wget $BASEURL/FACES/$i --no-verbose --mirror + ${DOWNLOADTOOL} $BASEURL/FACES/$i done cd $GISDATA/$BASEPATH/FACES/ @@ -145,7 +146,7 @@ load_state_data () { for i in "${files[@]}" do - wget $BASEURL/FEATNAMES/$i --no-verbose --mirror + ${DOWNLOADTOOL} $BASEURL/FEATNAMES/$i done cd $GISDATA/$BASEPATH/FEATNAMES/ @@ -164,7 +165,7 @@ load_state_data () { for i in "${files[@]}" do - wget $BASEURL/EDGES/$i --no-verbose --mirror + ${DOWNLOADTOOL} $BASEURL/EDGES/$i done cd $GISDATA/$BASEPATH/EDGES From 366cc07e758ee093a3bbb043baab32187e9d67fb Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Tue, 4 Nov 2025 16:20:21 -0800 Subject: [PATCH 02/22] Refactor Dockerfile for improved structure and clarity --- uwpostgis-foundry/Dockerfile | 50 ++++++++++++------------------------ 1 file changed, 16 insertions(+), 34 deletions(-) diff --git a/uwpostgis-foundry/Dockerfile b/uwpostgis-foundry/Dockerfile index b70b8b8..8d421d7 100644 --- a/uwpostgis-foundry/Dockerfile +++ b/uwpostgis-foundry/Dockerfile @@ -1,3 +1,6 @@ +# https://stackoverflow.com/questions/34751814/build-postgres-docker-container-with-initial-schema +# Initialize the database during the build +# PART 1 FROM postgis/postgis:17-3.5-alpine AS buildtime_init_builder # build vars @@ -15,39 +18,34 @@ ENV PGDATA=/pgdata SHELL ["/bin/bash", "-c"] -RUN apk add wget unzip postgis libintl +RUN apk add libintl postgis unzip wget # Make data dir -RUN mkdir -p /gisdata \ - && chmod 777 -R /gisdata - -# download national data first +RUN mkdir -m 777 /gisdata COPY ./src/alpinedb/download_national.sh . -RUN ./download_national.sh - -#download state data in separate layer COPY ./src/alpinedb/download_state.sh . COPY ./src/alpinedb/1-load_data.sh /docker-entrypoint-initdb.d/ RUN echo "docker_temp_server_stop && exit 0" > /docker-entrypoint-initdb.d/900-exit_before_boot.sh +# download national data first +RUN ./download_national.sh # force build to not use cache from a previous state(s) build: ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache +# download state data in separate layer RUN ./download_state.sh -# https://stackoverflow.com/questions/34751814/build-postgres-docker-container-with-initial-schema -# Initialize the database during the build -# PART 1 RUN bash -x docker-entrypoint.sh postgres - -FROM alpine:latest AS buildtime_init - -RUN adduser --uid 5001 --disabled-password user +# PART 2 +## copy database from initialization layer +FROM postgis/postgis:17-3.5-alpine AS buildtime_init +RUN apk upgrade --no-cache +## run apk to install key tools +RUN apk add bash libcurl postgis py3-pip py3-psycopg unzip vim wget ## image rebuild from scratch CMD ["/bin/sh"] -RUN /bin/sh -c set -eux; addgroup -g 70 -S postgres; adduser -u 70 -S -D -G postgres -H -h /var/lib/postgresql -s /bin/sh postgres; install --verbose --directory --owner postgres --group postgres --mode 1777 /var/lib/postgresql # buildkit - +#RUN /bin/sh -c set -eux; addgroup -g 70 -S postgres; adduser -u 70 -S -D -G postgres -H -h /var/lib/postgresql -s /bin/sh postgres; install --verbose --directory --owner postgres --group postgres --mode 1777 /var/lib/postgresql # buildkit ENV LANG=en_US.utf8 ENV PG_MAJOR=16 ENV PG_VERSION=16.9 @@ -57,30 +55,19 @@ ENV POSTGIS_SHA256=71d8f5d06debec0bdd94525a5a1d42417ed2a1f65d0070a1a3e3e381b606e ENV PGDATA=/pgdata_init RUN /bin/sh -c set -eux; wget -O postgresql.tar.bz2 "https://ftp.postgresql.org/pub/source/v$PG_VERSION/postgresql-$PG_VERSION.tar.bz2"; echo "$PG_SHA256 *postgresql.tar.bz2" | sha256sum -c -; mkdir -p /usr/src/postgresql; tar --extract --file postgresql.tar.bz2 --directory /usr/src/postgresql --strip-components 1 ; rm postgresql.tar.bz2; apk add --no-cache --virtual .build-deps llvm15-dev clang15 bison coreutils dpkg-dev dpkg flex g++ gcc krb5-dev libc-dev libedit-dev libxml2-dev libxslt-dev linux-headers make openldap-dev openssl-dev perl-dev perl-ipc-run perl-utils python3-dev tcl-dev util-linux-dev zlib-dev icu-dev lz4-dev zstd-dev ; cd /usr/src/postgresql; awk '$1 == "#define" && $2 == "DEFAULT_PGSOCKET_DIR" && $3 == "\"/tmp\"" { $3 = "\"/var/run/postgresql\""; print; next } { print }' src/include/pg_config_manual.h > src/include/pg_config_manual.h.new; grep '/var/run/postgresql' src/include/pg_config_manual.h.new; mv src/include/pg_config_manual.h.new src/include/pg_config_manual.h; gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; wget -O config/config.guess 'https://git.savannah.gnu.org/cgit/config.git/plain/config.guess?id=7d3d27baf8107b630586c962c057e22149653deb'; wget -O config/config.sub 'https://git.savannah.gnu.org/cgit/config.git/plain/config.sub?id=7d3d27baf8107b630586c962c057e22149653deb'; export LLVM_CONFIG="/usr/lib/llvm15/bin/llvm-config"; export CLANG=clang-15; ./configure --enable-option-checking=fatal --build="$gnuArch" --enable-integer-datetimes --enable-thread-safety --enable-tap-tests --disable-rpath --with-uuid=e2fs --with-pgport=5432 --with-system-tzdata=/usr/share/zoneinfo --prefix=/usr/local --with-includes=/usr/local/include --with-libraries=/usr/local/lib --with-gssapi --with-ldap --with-tcl --with-perl --with-python --with-openssl --with-libxml --with-libxslt --with-icu --with-llvm --with-lz4 --with-zstd ; make -j "$(nproc)" world-bin; make install-world-bin; make -C contrib install; runDeps="$( scanelf --needed --nobanner --format '%n#p' --recursive /usr/local | tr ',' '\n' | sort -u | awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' | grep -v -e perl -e python -e tcl )"; apk add --no-cache --virtual .postgresql-rundeps $runDeps bash tzdata zstd icu-data-full $([ "$(apk --print-arch)" != 'ppc64le' ] && echo 'nss_wrapper') ; apk del --no-network .build-deps; cd /; rm -rf /usr/src/postgresql /usr/local/share/doc /usr/local/share/man ; postgres --version # buildkit - RUN /bin/sh -c set -eux; cp -v /usr/local/share/postgresql/postgresql.conf.sample /usr/local/share/postgresql/postgresql.conf.sample.orig; sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/local/share/postgresql/postgresql.conf.sample; grep -F "listen_addresses = '*'" /usr/local/share/postgresql/postgresql.conf.sample # buildkit - RUN mkdir -m 777 /var/run/postgresql RUN chown -R postgres:postgres /var/run/postgresql # buildkit - RUN mkdir -m 777 /pgdata_init RUN chown -R postgres:postgres /pgdata_init # buildkit - CMD ["postgres"] - RUN /bin/sh -c set -eux && apk add --no-cache --virtual .fetch-deps ca-certificates openssl tar && wget -O postgis.tar.gz "https://github.com/postgis/postgis/archive/${POSTGIS_VERSION}.tar.gz" && echo "${POSTGIS_SHA256} *postgis.tar.gz" | sha256sum -c - && mkdir -p /usr/src/postgis && tar --extract --file postgis.tar.gz --directory /usr/src/postgis --strip-components 1 && rm postgis.tar.gz && apk add --no-cache --virtual .build-deps gdal-dev geos-dev proj-dev proj-util sfcgal-dev llvm15-dev clang15 autoconf automake cunit-dev file g++ gcc gettext-dev git json-c-dev libtool libxml2-dev make pcre2-dev perl protobuf-c-dev && cd /usr/src/postgis && gettextize && ./autogen.sh && ./configure --enable-lto && make -j$(nproc) && make install && projsync --system-directory --file ch_swisstopo_CHENyx06_ETRS && projsync --system-directory --file us_noaa_eshpgn && projsync --system-directory --file us_noaa_prvi && projsync --system-directory --file us_noaa_wmhpgn && mkdir /tempdb && chown -R postgres:postgres /tempdb && su postgres -c 'pg_ctl -D /tempdb init' && su postgres -c 'pg_ctl -D /tempdb -c -l /tmp/logfile -o '-F' start ' && cd regress && make -j$(nproc) check RUNTESTFLAGS=--extension PGUSER=postgres && su postgres -c 'psql -c "CREATE EXTENSION IF NOT EXISTS postgis;"' && su postgres -c 'psql -c "CREATE EXTENSION IF NOT EXISTS postgis_raster;"' && su postgres -c 'psql -c "CREATE EXTENSION IF NOT EXISTS postgis_sfcgal;"' && su postgres -c 'psql -c "CREATE EXTENSION IF NOT EXISTS fuzzystrmatch; --needed for postgis_tiger_geocoder "' && su postgres -c 'psql -c "CREATE EXTENSION IF NOT EXISTS address_standardizer;"' && su postgres -c 'psql -c "CREATE EXTENSION IF NOT EXISTS address_standardizer_data_us;"' && su postgres -c 'psql -c "CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder;"' && su postgres -c 'psql -c "CREATE EXTENSION IF NOT EXISTS postgis_topology;"' && su postgres -c 'psql -t -c "SELECT version();"' >> /_pgis_full_version.txt && su postgres -c 'psql -t -c "SELECT PostGIS_Full_Version();"' >> /_pgis_full_version.txt && su postgres -c 'psql -t -c "\dx"' >> /_pgis_full_version.txt && su postgres -c 'pg_ctl -D /tempdb --mode=immediate stop' && rm -rf /tempdb && rm -rf /tmp/logfile && rm -rf /tmp/pgis_reg && apk add --no-cache --virtual .postgis-rundeps gdal geos proj sfcgal json-c libstdc++ pcre2 protobuf-c ca-certificates && cd / && rm -rf /usr/src/postgis && apk del .fetch-deps .build-deps && cat /_pgis_full_version.txt # buildkit ## image rebuild end - -## run apk instead on alpine to install key tools -RUN apk update -RUN apk add wget unzip python3 py3-pip py3-psycopg vim bash postgis libcurl - -RUN apk upgrade --no-cache - SHELL ["/bin/bash", "-c"] # Foundry customizations +RUN adduser --uid 5001 --disabled-password user RUN mkdir -p /opt/palantir/sidecars/shared-volumes/shared/ RUN chown 5001 /opt/palantir/sidecars/shared-volumes/shared/ ENV SHARED_DIR=/opt/palantir/sidecars/shared-volumes/shared @@ -98,15 +85,10 @@ WORKDIR /opt/palantir/sidecars/shared-volumes/shared/ RUN chown -R 5001:5001 /tmp /foundry_venv /var/run/postgresql -# https://stackoverflow.com/questions/34751814/build-postgres-docker-container-with-initial-schema -# Initialize the database during the build -# PART 2 -## copy database from initialization layer RUN mkdir -m 700 /pgdata RUN chown 5001:5001 /pgdata COPY --chown=5001:5001 --chmod=700 --from=buildtime_init_builder /pgdata /pgdata ENV PGDATA=/pgdata USER 5001 - ENTRYPOINT entrypoint -c "/foundry_venv/bin/python3 /process_csv.py" From c9b9c6737702e7c1a8a7e75ffba2ff406f643ef2 Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Tue, 4 Nov 2025 16:23:56 -0800 Subject: [PATCH 03/22] Refactor GitHub Actions workflow for builds Updated branch names and added clear-cache job to optimize Docker builds. Upgraded Trivy action version for vulnerability scanning. --- .github/workflows/fast-build.yml | 51 +++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/.github/workflows/fast-build.yml b/.github/workflows/fast-build.yml index 1266f73..4474773 100644 --- a/.github/workflows/fast-build.yml +++ b/.github/workflows/fast-build.yml @@ -3,10 +3,10 @@ name: Fastbuild postgis images on: push: branches: - - "5-speed-up-builder" + - "5-speed-up-builds" pull_request: branches: - - "5-speed-up-builder" + - "5-speed-up-builds" concurrency: group: ${{ github.head_ref || github.ref_name }} @@ -16,11 +16,49 @@ env: # dynamic doesn't work, only static BUILD_NUM: $(date +%Y%m%d%H%M%S) REPOSITORY: + TIGER_DOMAIN: clad-github-builder.rit.uw.edu jobs: + clear-cache: + runs-on: self-hosted + + steps: + - name: Clear docker cache + run: docker system prune -af --volumes + - name: Delete docker volumes not included in system prune + run: docker volume prune -af + # run: docker buildx prune -a --verbose + - name: export BUILD_NUM to file + run: date +%Y%m%d%H%M%S > ~/BUILD_NUM + + + + build-degauss-foundry: + runs-on: self-hosted + needs: clear-cache + + steps: + - uses: actions/checkout@v5 + - name: import BUILD_NUM from file + run: echo "BUILD_NUM=$(cat ~/BUILD_NUM)" >> $GITHUB_ENV + - name: print env + run: env | sort + - name: Build the degauss-foundry image + run: docker build --load --cpu-quota $(( $(nproc --all)*100000 )) degauss-foundry --platform linux/amd64 --tag genoa-container-registry.washington.palantircloud.com/degauss-foundry:${{ env.BUILD_NUM }} + - name: Run Trivy CVE vulnerability scanner + uses: aquasecurity/trivy-action@0.33.1 + with: + image-ref: genoa-container-registry.washington.palantircloud.com/degauss-foundry:${{ env.BUILD_NUM }} + format: 'table' + exit-code: '1' + ignore-unfixed: true + vuln-type: 'os,library' + severity: 'CRITICAL,HIGH,MEDIUM' + setup-postgis-states: runs-on: self-hosted + needs: clear-cache outputs: matrix: ${{ steps.matrix.outputs.value }} @@ -40,15 +78,15 @@ jobs: value: ${{fromJSON(needs.setup-postgis-states.outputs.matrix)}} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: import BUILD_NUM from file run: echo "BUILD_NUM=$(cat ~/BUILD_NUM)" >> $GITHUB_ENV - name: print env - run: env + run: env | sort - name: Build the postgis image - run: docker build --shm-size 4g --load --cpu-quota $(( $(nproc --all)*100000 )) --build-arg state_var=${{ matrix.value }} uwpostgis-foundry --platform linux/amd64 --tag genoa-container-registry.washington.palantircloud.com/postgis-${{ matrix.value }}:${{ env.BUILD_NUM }} + run: docker build --shm-size 4g --load --cpu-quota $(( $(nproc --all)*100000 )) --build-arg TIGER_DOMAIN=clad-github-builder.rit.uw.edu --build-arg state_var=${{ matrix.value }} uwpostgis-foundry --platform linux/amd64 --tag genoa-container-registry.washington.palantircloud.com/postgis-${{ matrix.value }}:${{ env.BUILD_NUM }} - name: Run Trivy CVE vulnerability scanner - uses: aquasecurity/trivy-action@0.25.0 + uses: aquasecurity/trivy-action@0.33.1 with: image-ref: genoa-container-registry.washington.palantircloud.com/postgis-${{ matrix.value }}:${{ env.BUILD_NUM }} format: 'table' @@ -56,4 +94,3 @@ jobs: ignore-unfixed: true vuln-type: 'os,library' severity: 'CRITICAL,HIGH,MEDIUM' - From c4234e379d36988e9881d03bc9be1f4050ab74a6 Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Tue, 4 Nov 2025 16:25:04 -0800 Subject: [PATCH 04/22] Change base image to alpine for buildtime_init --- uwpostgis-foundry/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uwpostgis-foundry/Dockerfile b/uwpostgis-foundry/Dockerfile index 8d421d7..5ebb354 100644 --- a/uwpostgis-foundry/Dockerfile +++ b/uwpostgis-foundry/Dockerfile @@ -38,7 +38,7 @@ RUN bash -x docker-entrypoint.sh postgres # PART 2 ## copy database from initialization layer -FROM postgis/postgis:17-3.5-alpine AS buildtime_init +FROM alpine:latest AS buildtime_init RUN apk upgrade --no-cache ## run apk to install key tools RUN apk add bash libcurl postgis py3-pip py3-psycopg unzip vim wget From a4202a46718f622eaf328dd618404aa99ae6c209 Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Wed, 5 Nov 2025 14:40:53 -0800 Subject: [PATCH 05/22] Update pip install command to use >= for versions Upgrade pip dependencies to allow for version upgrades. --- uwpostgis-foundry/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/uwpostgis-foundry/Dockerfile b/uwpostgis-foundry/Dockerfile index 5ebb354..f9958dd 100644 --- a/uwpostgis-foundry/Dockerfile +++ b/uwpostgis-foundry/Dockerfile @@ -42,6 +42,7 @@ FROM alpine:latest AS buildtime_init RUN apk upgrade --no-cache ## run apk to install key tools RUN apk add bash libcurl postgis py3-pip py3-psycopg unzip vim wget +RUN python3 -m pip freeze | sed -i 's/==/>=/g' | xargs python3 -m pip install --upgrade ## image rebuild from scratch CMD ["/bin/sh"] From a62150e9d2f6178a2d37aa75c2b2a967ad162674 Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Wed, 5 Nov 2025 15:44:16 -0800 Subject: [PATCH 06/22] Modify pip install command in Dockerfile Updated pip install command to use --break-system-packages. --- uwpostgis-foundry/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uwpostgis-foundry/Dockerfile b/uwpostgis-foundry/Dockerfile index f9958dd..d777fa4 100644 --- a/uwpostgis-foundry/Dockerfile +++ b/uwpostgis-foundry/Dockerfile @@ -42,7 +42,7 @@ FROM alpine:latest AS buildtime_init RUN apk upgrade --no-cache ## run apk to install key tools RUN apk add bash libcurl postgis py3-pip py3-psycopg unzip vim wget -RUN python3 -m pip freeze | sed -i 's/==/>=/g' | xargs python3 -m pip install --upgrade +RUN python3 -m pip freeze | sed -i 's/==/>=/g' | xargs python3 -m pip install --upgrade --break-system-packages ## image rebuild from scratch CMD ["/bin/sh"] From b11992b5351dd2a2c5d9b229335702cc36cfa5b7 Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Wed, 5 Nov 2025 16:04:47 -0800 Subject: [PATCH 07/22] Fix pip upgrade command --- uwpostgis-foundry/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uwpostgis-foundry/Dockerfile b/uwpostgis-foundry/Dockerfile index d777fa4..1325d8b 100644 --- a/uwpostgis-foundry/Dockerfile +++ b/uwpostgis-foundry/Dockerfile @@ -42,7 +42,7 @@ FROM alpine:latest AS buildtime_init RUN apk upgrade --no-cache ## run apk to install key tools RUN apk add bash libcurl postgis py3-pip py3-psycopg unzip vim wget -RUN python3 -m pip freeze | sed -i 's/==/>=/g' | xargs python3 -m pip install --upgrade --break-system-packages +RUN python3 -m pip freeze | sed 's/==/>=/g' | xargs python3 -m pip install --upgrade --break-system-packages ## image rebuild from scratch CMD ["/bin/sh"] From b5a9ecac15b8ea198dbe47e57806622b35e02ccf Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Mon, 24 Nov 2025 16:00:25 -0800 Subject: [PATCH 08/22] Upgrade pip in Dockerfile Upgrade pip to the latest version during build. --- uwpostgis-foundry/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/uwpostgis-foundry/Dockerfile b/uwpostgis-foundry/Dockerfile index 1325d8b..87dc82a 100644 --- a/uwpostgis-foundry/Dockerfile +++ b/uwpostgis-foundry/Dockerfile @@ -42,6 +42,7 @@ FROM alpine:latest AS buildtime_init RUN apk upgrade --no-cache ## run apk to install key tools RUN apk add bash libcurl postgis py3-pip py3-psycopg unzip vim wget +RUN python3 -m pip install --upgrade --break-system-packages pip RUN python3 -m pip freeze | sed 's/==/>=/g' | xargs python3 -m pip install --upgrade --break-system-packages ## image rebuild from scratch From 6d3d718343d5619b0d15af49b74ba953f6f0a853 Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Tue, 25 Nov 2025 13:55:57 -0800 Subject: [PATCH 09/22] Update Dockerfile to use latest R minimal image --- degauss-foundry/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/degauss-foundry/Dockerfile b/degauss-foundry/Dockerfile index 1929147..ea8735f 100644 --- a/degauss-foundry/Dockerfile +++ b/degauss-foundry/Dockerfile @@ -1,4 +1,4 @@ -FROM rhub/r-minimal:4.5.0 +FROM rhub/r-minimal:latest # DeGAUSS container metadata ENV degauss_name="geocoder" @@ -53,7 +53,7 @@ WORKDIR /app # install required version of renv RUN R --quiet -e "install.packages('remotes', repos = 'https://packagemanager.rstudio.com/all/__linux__/focal/latest')" -RUN R --quiet -e "remotes::install_github('rstudio/renv@0.15.4')" +RUN R --quiet -e "remotes::install_github('rstudio/renv@0.15.5')" COPY renv.lock . RUN R --quiet -e "renv::restore()" From 1f24f89ffcde4fb62836342e369ede957f414355 Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Tue, 25 Nov 2025 14:29:50 -0800 Subject: [PATCH 10/22] fail back to test r-minimal:4.5.1 --- degauss-foundry/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/degauss-foundry/Dockerfile b/degauss-foundry/Dockerfile index ea8735f..34d930a 100644 --- a/degauss-foundry/Dockerfile +++ b/degauss-foundry/Dockerfile @@ -1,4 +1,4 @@ -FROM rhub/r-minimal:latest +FROM rhub/r-minimal:4.5.1 # DeGAUSS container metadata ENV degauss_name="geocoder" From debce09cc7c5be5a3fc437e54a0d3d7db8f96ae1 Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Tue, 25 Nov 2025 14:35:02 -0800 Subject: [PATCH 11/22] revert back to r-minimal:4.5.0 --- degauss-foundry/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/degauss-foundry/Dockerfile b/degauss-foundry/Dockerfile index 34d930a..2d26e5a 100644 --- a/degauss-foundry/Dockerfile +++ b/degauss-foundry/Dockerfile @@ -1,4 +1,4 @@ -FROM rhub/r-minimal:4.5.1 +FROM rhub/r-minimal:4.5.0 # DeGAUSS container metadata ENV degauss_name="geocoder" From 396139ec3b0ff9da016193c59064478fabe2eacb Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Tue, 25 Nov 2025 15:49:39 -0800 Subject: [PATCH 12/22] Increase timeout for vulnerability checks in fast-build.yml Added timeout setting to vulnerability checks in fast-build workflow. --- .github/workflows/fast-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/fast-build.yml b/.github/workflows/fast-build.yml index 4474773..020d2df 100644 --- a/.github/workflows/fast-build.yml +++ b/.github/workflows/fast-build.yml @@ -53,6 +53,7 @@ jobs: format: 'table' exit-code: '1' ignore-unfixed: true + timeout: '1h' vuln-type: 'os,library' severity: 'CRITICAL,HIGH,MEDIUM' @@ -92,5 +93,6 @@ jobs: format: 'table' exit-code: '1' ignore-unfixed: true + timeout: '1h' vuln-type: 'os,library' severity: 'CRITICAL,HIGH,MEDIUM' From 0858e4a9e270255ed02646b46e58f1f47bdf3239 Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Tue, 2 Dec 2025 19:48:58 -0800 Subject: [PATCH 13/22] Upgrade pip and dependencies in foundry_venv --- uwpostgis-foundry/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uwpostgis-foundry/Dockerfile b/uwpostgis-foundry/Dockerfile index 87dc82a..2252dc1 100644 --- a/uwpostgis-foundry/Dockerfile +++ b/uwpostgis-foundry/Dockerfile @@ -76,7 +76,9 @@ ENV SHARED_DIR=/opt/palantir/sidecars/shared-volumes/shared RUN python3 -m venv foundry_venv COPY requirements.txt ./ +RUN /foundry_venv/bin/pip install --upgrade pip RUN /foundry_venv/bin/pip install --no-cache-dir -r requirements.txt +RUN /foundry_venv/bin/pip freeze | sed 's/--.*//' | xargs /foundry_venv/bin/pip install --upgrade ADD entrypoint.py /usr/bin/entrypoint RUN chmod +x /usr/bin/entrypoint From 930e0a36ffa89adc040ed5a75b6b55c11216b375 Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Tue, 2 Dec 2025 20:26:47 -0800 Subject: [PATCH 14/22] Extend trivy timeout --- .github/workflows/docker-image.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 849594d..fbb19c6 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -55,6 +55,7 @@ jobs: format: 'table' exit-code: '1' ignore-unfixed: true + timeout: '1h' vuln-type: 'os,library' severity: 'CRITICAL,HIGH,MEDIUM' @@ -94,6 +95,7 @@ jobs: format: 'table' exit-code: '1' ignore-unfixed: true + timeout: '1h' vuln-type: 'os,library' severity: 'CRITICAL,HIGH,MEDIUM' From bd593e0746650fea036ae533f30e081ae4049a8a Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Tue, 9 Dec 2025 16:00:57 -0800 Subject: [PATCH 15/22] Update geocoder database source in Dockerfile --- degauss-foundry/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/degauss-foundry/Dockerfile b/degauss-foundry/Dockerfile index 2d26e5a..2bec6d8 100644 --- a/degauss-foundry/Dockerfile +++ b/degauss-foundry/Dockerfile @@ -61,7 +61,7 @@ RUN R --quiet -e "renv::restore()" COPY entrypointCSV.R . COPY geocode.rb . -ADD https://geomarker.s3.amazonaws.com/geocoder_2021.db /opt/geocoder.db +ADD https://colebrokamp-dropbox.s3.amazonaws.com/geocoder.db /opt/geocoder.db # Foundry customizations part 1 RUN adduser --uid 5001 --disabled-password user From ec3f6dc2c1d1b19b8fe86804a33e775499ae52d6 Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Tue, 9 Dec 2025 17:29:32 -0800 Subject: [PATCH 16/22] split compile from end build --- degauss-foundry/Dockerfile | 99 ++-- .../{entrypointCSV.R => entrypoint.R} | 5 +- degauss-foundry/entrypoint.py | 2 - degauss-foundry/renv.lock | 449 ++++++++++-------- 4 files changed, 321 insertions(+), 234 deletions(-) rename degauss-foundry/{entrypointCSV.R => entrypoint.R} (98%) mode change 100755 => 100644 diff --git a/degauss-foundry/Dockerfile b/degauss-foundry/Dockerfile index 2bec6d8..830d025 100644 --- a/degauss-foundry/Dockerfile +++ b/degauss-foundry/Dockerfile @@ -1,8 +1,43 @@ -FROM rhub/r-minimal:4.5.0 +# https://stackoverflow.com/questions/34751814/build-postgres-docker-container-with-initial-schema +# compile the gem for the build +# PART 1 +FROM rocker/r-ver:4.4.1 + +RUN apt-get update && apt-get install -y \ + bison \ + flex \ + gnupg \ + libcurl4-openssl-dev \ + libsqlite3-dev \ + libssh2-1-dev \ + libssl-dev \ + libxml2-dev \ + make \ + pkg-config \ + ruby-full \ + sqlite3 \ + software-properties-common + +RUN gem install sqlite3 json Text + +RUN mkdir /app +WORKDIR /app + +COPY Makefile.ruby . +COPY /src ./src +COPY /lib ./lib +COPY /gemspec ./gemspec + +RUN make -f Makefile.ruby install + + +# PART 2 +## copy build from initialization layer +FROM rhub/r-minimal:4.5.0 AS buildtime_init # DeGAUSS container metadata ENV degauss_name="geocoder" -ENV degauss_version="3.3.0" +ENV degauss_version="3.4.0" ENV degauss_description="geocodes" ENV degauss_argument="valid_geocode_score_threshold [default: 0.5]" @@ -12,57 +47,39 @@ LABEL "org.degauss.version"="${degauss_version}" LABEL "org.degauss.description"="${degauss_description}" LABEL "org.degauss.argument"="${degauss_argument}" +RUN mkdir /app +WORKDIR /app + +COPY --from=buildtime_init_builder Geocoder-US-2.0.4.gem Geocoder-US-2.0.4.gem +ADD https://colebrokamp-dropbox.s3.amazonaws.com/geocoder.db /opt/geocoder.db + RUN apk add \ - make \ - sqlite \ - sqlite-dev \ - sqlite-libs \ - flex \ - ruby-full \ - ruby-dev \ bison \ - gnupg \ - gcc \ + flex \ g++ \ + gcc \ + gnupg \ + linux-headers \ musl-dev \ - python3 \ py3-pip \ - vim \ - linux-headers - -RUN gem install sqlite3 -RUN gem install json Text - + ruby \ + sqlite \ + vim RUN apk upgrade --no-cache -RUN mkdir /app -WORKDIR /app - - - - -COPY Makefile.ruby . -COPY /src ./src -COPY /lib ./lib -COPY /gemspec ./gemspec - -RUN make -f Makefile.ruby install \ - && gem install Geocoder-US-2.0.4.gem - -WORKDIR /app +RUN gem install json sqlite3 Text +RUN gem install Geocoder-US-2.0.4.gem # install required version of renv -RUN R --quiet -e "install.packages('remotes', repos = 'https://packagemanager.rstudio.com/all/__linux__/focal/latest')" -RUN R --quiet -e "remotes::install_github('rstudio/renv@0.15.5')" +RUN R --quiet -e "install.packages('remotes', repos = c(CRAN = 'https://packagemanager.posit.co/cran/latest'))" +RUN R --quiet -e "remotes::install_github('rstudio/renv@v1.0.7')" COPY renv.lock . -RUN R --quiet -e "renv::restore()" +RUN R --quiet -e "renv::restore(repos = c(CRAN = 'https://packagemanager.posit.co/cran/latest'))" -COPY entrypointCSV.R . +COPY entrypoint.R . COPY geocode.rb . -ADD https://colebrokamp-dropbox.s3.amazonaws.com/geocoder.db /opt/geocoder.db - # Foundry customizations part 1 RUN adduser --uid 5001 --disabled-password user RUN chown -R 5001:5001 /tmp /opt /app @@ -73,12 +90,10 @@ RUN chown -R 5001:5001 /tmp /opt /app ADD entrypoint.py /usr/bin/entrypoint RUN chmod +x /usr/bin/entrypoint - RUN mkdir -p /opt/palantir/sidecars/shared-volumes/shared/ RUN chown 5001 /opt/palantir/sidecars/shared-volumes/shared/ ENV SHARED_DIR=/opt/palantir/sidecars/shared-volumes/shared USER 5001 WORKDIR /opt/palantir/sidecars/shared-volumes/shared/ - -ENTRYPOINT entrypoint -c "/app/entrypointCSV.R /opt/palantir/sidecars/shared-volumes/shared/infile.csv all" +ENTRYPOINT entrypoint -c "/app/entrypoint.R /opt/palantir/sidecars/shared-volumes/shared/infile.csv all" diff --git a/degauss-foundry/entrypointCSV.R b/degauss-foundry/entrypoint.R old mode 100755 new mode 100644 similarity index 98% rename from degauss-foundry/entrypointCSV.R rename to degauss-foundry/entrypoint.R index 41835c3..9132750 --- a/degauss-foundry/entrypointCSV.R +++ b/degauss-foundry/entrypoint.R @@ -14,9 +14,7 @@ opt <- docopt::docopt(doc) if (is.null(opt$score_threshold)) opt$score_threshold <- 0.5 d <- readr::read_csv(opt$filename, show_col_types = FALSE) - -## 8/23 - FOR FOUNDRY PROCESS -# d <- readr::read_csv('/opt/palantir/sidecars/shared-volumes/shared/infile.csv') +# d <- readr::read_csv('test/my_address_file.csv') # d <- readr::read_csv('test/my_address_file_missing.csv') ## must contain character column called address @@ -160,4 +158,3 @@ if (opt$score_threshold != "all") { ) knitr::kable(geocode_summary %>% dplyr::select(geocode_result, `n (%)`)) } - diff --git a/degauss-foundry/entrypoint.py b/degauss-foundry/entrypoint.py index 7a0e861..33be44c 100644 --- a/degauss-foundry/entrypoint.py +++ b/degauss-foundry/entrypoint.py @@ -12,13 +12,11 @@ args = parser.parse_args() the_command = args.command.split(" ") - def run_process(exe): "Define a function for running commands and capturing stdout line by line" p = subprocess.Popen(exe, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) return iter(p.stdout.readline, b"") - start_flag_fname = "/opt/palantir/sidecars/shared-volumes/shared/start_flag" done_flag_fname = "/opt/palantir/sidecars/shared-volumes/shared/done_flag" close_flag_fname = "/opt/palantir/sidecars/shared-volumes/shared/close_flag" diff --git a/degauss-foundry/renv.lock b/degauss-foundry/renv.lock index bc69a91..67814a8 100644 --- a/degauss-foundry/renv.lock +++ b/degauss-foundry/renv.lock @@ -1,12 +1,7 @@ { "R": { - "Version": "4.2.2", - "Repositories": [ - { - "Name": "CRAN", - "URL": "https://cran.rstudio.com" - } - ] + "Version": "4.4.1", + "Repositories": [] }, "Packages": { "R6": { @@ -14,69 +9,88 @@ "Version": "2.5.1", "Source": "Repository", "Repository": "CRAN", - "Hash": "470851b6d5d0ac559e9d01bb352b4021", - "Requirements": [] + "Requirements": [ + "R" + ], + "Hash": "470851b6d5d0ac559e9d01bb352b4021" }, "bit": { "Package": "bit", - "Version": "4.5.0", + "Version": "4.0.5", "Source": "Repository", "Repository": "CRAN", - "Hash": "d242abec29412ce988848d0294b208fd", - "Requirements": [] + "Requirements": [ + "R" + ], + "Hash": "d242abec29412ce988848d0294b208fd" }, "bit64": { "Package": "bit64", - "Version": "4.5.2", + "Version": "4.0.5", "Source": "Repository", "Repository": "CRAN", - "Hash": "9fe98599ca456d6552421db0d6772d8f", "Requirements": [ - "bit" - ] + "R", + "bit", + "methods", + "stats", + "utils" + ], + "Hash": "9fe98599ca456d6552421db0d6772d8f" }, "cachem": { "Package": "cachem", - "Version": "1.0.5", + "Version": "1.1.0", "Source": "Repository", "Repository": "CRAN", - "Hash": "5346f76a33eb7417812c270b04a5581b", "Requirements": [ "fastmap", "rlang" - ] + ], + "Hash": "cd9a672193789068eb5a2aad65a0dedf" }, "cli": { "Package": "cli", - "Version": "3.5.0", + "Version": "3.6.3", "Source": "Repository", "Repository": "CRAN", - "Hash": "eb9fc121ad9a1075c471107ef185be46", - "Requirements": [] + "Requirements": [ + "R", + "utils" + ], + "Hash": "b21916dd77a27642b447374a5d30ecf3" }, "clipr": { "Package": "clipr", "Version": "0.8.0", "Source": "Repository", "Repository": "CRAN", - "Hash": "3f038e5ac7f41d4ac41ce658c85e3042", - "Requirements": [] + "Requirements": [ + "utils" + ], + "Hash": "3f038e5ac7f41d4ac41ce658c85e3042" }, "cpp11": { "Package": "cpp11", - "Version": "0.4.3", + "Version": "0.4.7", "Source": "Repository", "Repository": "CRAN", - "Hash": "ed588261931ee3be2c700d22e94a29ab", - "Requirements": [] + "Requirements": [ + "R" + ], + "Hash": "5a295d7d963cc5035284dcdbaf334f4e" }, "crayon": { "Package": "crayon", - "Version": "1.5.2", + "Version": "1.5.3", "Source": "Repository", "Repository": "CRAN", - "Hash": "e8a1e41acf02548751f45c718d55aa6a", - "Requirements": [] + "Requirements": [ + "grDevices", + "methods", + "utils" + ], + "Hash": "859d96e65ef198fd43e82b9628d593ef" }, "dht": { "Package": "dht", @@ -84,12 +98,12 @@ "Source": "GitHub", "RemoteType": "github", "RemoteHost": "api.github.com", - "RemoteRepo": "dht", "RemoteUsername": "degauss-org", - "RemoteRef": "HEAD", - "RemoteSha": "4d10437461416e688ea259a37c2ce3b5bae96238", - "Hash": "b16ae4484d2ef4a83738f6069c7868d3", + "RemoteRepo": "dht", + "RemoteRef": "master", + "RemoteSha": "f0b59bbb91697a55074d065642ed012b652e7d31", "Requirements": [ + "R", "cli", "dplyr", "fs", @@ -104,233 +118,245 @@ "tidyselect", "whisker", "withr" - ] + ], + "Hash": "05547abc78751624749fb01f2a2e49b9" }, "digest": { "Package": "digest", "Version": "0.6.37", "Source": "Repository", "Repository": "CRAN", - "Hash": "a0cbe758a531d054b537d16dff4d58a1", - "Requirements": [] + "Requirements": [ + "R", + "utils" + ], + "Hash": "33698c4b3127fc9f506654607fb73676" }, "docopt": { "Package": "docopt", "Version": "0.7.1", "Source": "Repository", "Repository": "CRAN", - "Hash": "e9eeef7931ee99ca0093f3f20b88e09b", - "Requirements": [] + "Requirements": [ + "methods" + ], + "Hash": "e9eeef7931ee99ca0093f3f20b88e09b" }, "dplyr": { "Package": "dplyr", - "Version": "1.0.10", + "Version": "1.1.4", "Source": "Repository", "Repository": "CRAN", - "Hash": "539412282059f7f0c07295723d23f987", "Requirements": [ + "R", "R6", + "cli", "generics", "glue", "lifecycle", "magrittr", + "methods", "pillar", "rlang", "tibble", "tidyselect", + "utils", "vctrs" - ] - }, - "ellipsis": { - "Package": "ellipsis", - "Version": "0.3.2", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "bb0eec2fe32e88d9e2836c2f73ea2077", - "Requirements": [ - "rlang" - ] + ], + "Hash": "fedd9d00c2944ff00a0e2696ccf048ec" }, "evaluate": { "Package": "evaluate", - "Version": "0.14", + "Version": "0.24.0", "Source": "Repository", "Repository": "CRAN", - "Hash": "ec8ca05cffcc70569eaaad8469d2a3a7", - "Requirements": [] + "Requirements": [ + "R", + "methods" + ], + "Hash": "a1066cbc05caee9a4bf6d90f194ff4da" }, "fansi": { "Package": "fansi", - "Version": "1.0.3", + "Version": "1.0.6", "Source": "Repository", "Repository": "CRAN", - "Hash": "83a8afdbe71839506baa9f90eebad7ec", - "Requirements": [] + "Requirements": [ + "R", + "grDevices", + "utils" + ], + "Hash": "962174cf2aeb5b9eea581522286a911f" }, "fastmap": { "Package": "fastmap", - "Version": "1.1.0", + "Version": "1.2.0", "Source": "Repository", "Repository": "CRAN", - "Hash": "77bd60a6157420d4ffa93b27cf6a58b8", - "Requirements": [] + "Hash": "aa5e1cd11c2d15497494c5292d7ffcc8" }, "fs": { "Package": "fs", - "Version": "1.5.2", + "Version": "1.6.4", "Source": "Repository", "Repository": "CRAN", - "Hash": "7c89603d81793f0d5486d91ab1fc6f1d", - "Requirements": [] + "Requirements": [ + "R", + "methods" + ], + "Hash": "15aeb8c27f5ea5161f9f6a641fafd93a" }, "generics": { "Package": "generics", "Version": "0.1.3", "Source": "Repository", "Repository": "CRAN", - "Hash": "15e9634c0fcd294799e9b2e929ed1b86", - "Requirements": [] + "Requirements": [ + "R", + "methods" + ], + "Hash": "15e9634c0fcd294799e9b2e929ed1b86" }, "glue": { "Package": "glue", - "Version": "1.6.2", + "Version": "1.7.0", "Source": "Repository", "Repository": "CRAN", - "Hash": "4f2596dfb05dac67b9dc558e5c6fba2e", - "Requirements": [] + "Requirements": [ + "R", + "methods" + ], + "Hash": "e0b3a53876554bd45879e596cdb10a52" }, "highr": { "Package": "highr", - "Version": "0.8", + "Version": "0.11", "Source": "Repository", "Repository": "CRAN", - "Hash": "4dc5bb88961e347a0f4d8aad597cbfac", - "Requirements": [] + "Requirements": [ + "R", + "xfun" + ], + "Hash": "d65ba49117ca223614f71b60d85b8ab7" }, "hms": { "Package": "hms", - "Version": "1.1.2", + "Version": "1.1.3", "Source": "Repository", "Repository": "CRAN", - "Hash": "41100392191e1244b887878b533eea91", "Requirements": [ - "ellipsis", "lifecycle", + "methods", "pkgconfig", "rlang", "vctrs" - ] + ], + "Hash": "b59377caa7ed00fa41808342002138f9" }, "jsonlite": { "Package": "jsonlite", - "Version": "1.7.2", + "Version": "1.8.8", "Source": "Repository", "Repository": "CRAN", - "Hash": "98138e0994d41508c7a6b84a0600cfcb", - "Requirements": [] + "Requirements": [ + "methods" + ], + "Hash": "e1b9c55281c5adc4dd113652d9e26768" }, "knitr": { "Package": "knitr", - "Version": "1.31", + "Version": "1.48", "Source": "Repository", "Repository": "CRAN", - "Hash": "c3994c036d19fc22c5e2a209c8298bfb", "Requirements": [ + "R", "evaluate", "highr", - "markdown", - "stringr", + "methods", + "tools", "xfun", "yaml" - ] + ], + "Hash": "acf380f300c721da9fde7df115a5f86f" }, "lifecycle": { "Package": "lifecycle", - "Version": "1.0.3", + "Version": "1.0.4", "Source": "Repository", "Repository": "CRAN", - "Hash": "001cecbeac1cff9301bdc3775ee46a86", "Requirements": [ + "R", "cli", "glue", "rlang" - ] + ], + "Hash": "b8552d117e1b808b09a832f589b79035" }, "magrittr": { "Package": "magrittr", "Version": "2.0.3", "Source": "Repository", "Repository": "CRAN", - "Hash": "7ce2733a9826b3aeb1775d56fd305472", - "Requirements": [] + "Requirements": [ + "R" + ], + "Hash": "7ce2733a9826b3aeb1775d56fd305472" }, "mappp": { "Package": "mappp", "Version": "1.0.0", "Source": "Repository", "Repository": "CRAN", - "Hash": "969ed5fab0c1752ddd0a43f789c9ac4e", "Requirements": [ "memoise", + "parallel", "parallelly", "pbmcapply", "progress", "purrr", "rlang" - ] - }, - "markdown": { - "Package": "markdown", - "Version": "1.1", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "61e4a10781dd00d7d81dd06ca9b94e95", - "Requirements": [ - "mime", - "xfun" - ] + ], + "Hash": "969ed5fab0c1752ddd0a43f789c9ac4e" }, "memoise": { "Package": "memoise", - "Version": "2.0.0", + "Version": "2.0.1", "Source": "Repository", "Repository": "CRAN", - "Hash": "a0bc51650201a56d00a4798523cc91b3", "Requirements": [ "cachem", "rlang" - ] - }, - "mime": { - "Package": "mime", - "Version": "0.10", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "26fa77e707223e1ce042b2b5d09993dc", - "Requirements": [] + ], + "Hash": "e2817ccf4a065c5d9d7f2cfbe7c1d78c" }, "parallelly": { "Package": "parallelly", - "Version": "1.25.0", + "Version": "1.38.0", "Source": "Repository", "Repository": "CRAN", - "Hash": "bc53006c11a08ba955ccbf85b586875f", - "Requirements": [] + "Requirements": [ + "parallel", + "tools", + "utils" + ], + "Hash": "6e8b139c1904f5e9e14c69db64453bbe" }, "pbmcapply": { "Package": "pbmcapply", - "Version": "1.5.0", + "Version": "1.5.1", "Source": "Repository", "Repository": "CRAN", - "Hash": "4332529a23c4995d743ef489be54a839", - "Requirements": [] + "Requirements": [ + "parallel", + "utils" + ], + "Hash": "bf173fb42b60f8f404f7a9f9c3f95d0d" }, "pillar": { "Package": "pillar", - "Version": "1.8.1", + "Version": "1.9.0", "Source": "Repository", "Repository": "CRAN", - "Hash": "f2316df30902c81729ae9de95ad5a608", "Requirements": [ "cli", "fansi", @@ -338,67 +364,78 @@ "lifecycle", "rlang", "utf8", + "utils", "vctrs" - ] + ], + "Hash": "15da5a8412f317beeee6175fbc76f4bb" }, "pkgconfig": { "Package": "pkgconfig", "Version": "2.0.3", "Source": "Repository", "Repository": "CRAN", - "Hash": "01f28d4278f15c76cddbea05899c5d6f", - "Requirements": [] + "Requirements": [ + "utils" + ], + "Hash": "01f28d4278f15c76cddbea05899c5d6f" }, "prettyunits": { "Package": "prettyunits", - "Version": "1.1.1", + "Version": "1.2.0", "Source": "Repository", "Repository": "CRAN", - "Hash": "95ef9167b75dde9d2ccc3c7528393e7e", - "Requirements": [] + "Requirements": [ + "R" + ], + "Hash": "6b01fc98b1e86c4f705ce9dcfd2f57c7" }, "progress": { "Package": "progress", - "Version": "1.2.2", + "Version": "1.2.3", "Source": "Repository", "Repository": "CRAN", - "Hash": "14dc9f7a3c91ebb14ec5bb9208a07061", "Requirements": [ + "R", "R6", "crayon", "hms", "prettyunits" - ] + ], + "Hash": "f4625e061cb2865f111b47ff163a5ca6" }, "ps": { "Package": "ps", - "Version": "1.7.2", + "Version": "1.7.7", "Source": "Repository", "Repository": "CRAN", - "Hash": "68dd03d98a5efd1eb3012436de45ba83", - "Requirements": [] + "Requirements": [ + "R", + "utils" + ], + "Hash": "878b467580097e9c383acbb16adab57a" }, "purrr": { "Package": "purrr", - "Version": "1.0.0", + "Version": "1.0.2", "Source": "Repository", "Repository": "CRAN", - "Hash": "1ad491d27989ec6c26a2918ad6df116b", "Requirements": [ + "R", "cli", "lifecycle", "magrittr", "rlang", "vctrs" - ] + ], + "Hash": "1cba04a4e9414bdefc9dcaa99649a8dc" }, "readr": { "Package": "readr", - "Version": "2.1.3", + "Version": "2.1.5", "Source": "Repository", "Repository": "CRAN", - "Hash": "2dfbfc673ccb3de3d8836b4b3bd23d14", "Requirements": [ + "R", "R6", "cli", "clipr", @@ -406,43 +443,61 @@ "crayon", "hms", "lifecycle", + "methods", "rlang", "tibble", "tzdb", + "utils", "vroom" - ] + ], + "Hash": "9de96463d2117f6ac49980577939dfb3" }, "renv": { "Package": "renv", - "Version": "0.15.4", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "c1078316e1d4f70275fc1ea60c0bc431", - "Requirements": [] + "Version": "1.0.7", + "Source": "GitHub", + "RemoteType": "github", + "RemoteHost": "api.github.com", + "RemoteRepo": "renv", + "RemoteUsername": "rstudio", + "RemoteRef": "v1.0.7", + "RemoteSha": "265a03869234f07c9171b6829d09427577e19baa", + "Requirements": [ + "utils" + ], + "Hash": "c9546c795efae077e4d11bce9eb9d059" }, "rlang": { "Package": "rlang", - "Version": "1.0.6", + "Version": "1.1.4", "Source": "Repository", "Repository": "CRAN", - "Hash": "4ed1f8336c8d52c3e750adcdc57228a7", - "Requirements": [] + "Requirements": [ + "R", + "utils" + ], + "Hash": "3eec01f8b1dee337674b2e34ab1f9bc1" }, "stringi": { "Package": "stringi", "Version": "1.8.4", "Source": "Repository", "Repository": "CRAN", - "Hash": "a68b980681bcbc84c7a67003fa796bfb", - "Requirements": [] + "Requirements": [ + "R", + "stats", + "tools", + "utils" + ], + "Hash": "39e1144fd75428983dc3f63aa53dfa91" }, "stringr": { "Package": "stringr", - "Version": "1.5.0", + "Version": "1.5.1", "Source": "Repository", "Repository": "CRAN", - "Hash": "671a4d384ae9d32fc47a14e98bfa3dc8", "Requirements": [ + "R", "cli", "glue", "lifecycle", @@ -450,97 +505,109 @@ "rlang", "stringi", "vctrs" - ] + ], + "Hash": "960e2ae9e09656611e0b8214ad543207" }, "tibble": { "Package": "tibble", - "Version": "3.1.8", + "Version": "3.2.1", "Source": "Repository", "Repository": "CRAN", - "Hash": "56b6934ef0f8c68225949a8672fe1a8f", "Requirements": [ + "R", "fansi", "lifecycle", "magrittr", + "methods", "pillar", "pkgconfig", "rlang", + "utils", "vctrs" - ] + ], + "Hash": "a84e2cc86d07289b3b6f5069df7a004c" }, "tidyr": { "Package": "tidyr", - "Version": "1.2.1", + "Version": "1.3.1", "Source": "Repository", "Repository": "CRAN", - "Hash": "cdb403db0de33ccd1b6f53b83736efa8", "Requirements": [ + "R", + "cli", "cpp11", "dplyr", - "ellipsis", "glue", "lifecycle", "magrittr", "purrr", "rlang", + "stringr", "tibble", "tidyselect", + "utils", "vctrs" - ] + ], + "Hash": "915fb7ce036c22a6a33b5a8adb712eb1" }, "tidyselect": { "Package": "tidyselect", - "Version": "1.2.0", + "Version": "1.2.1", "Source": "Repository", "Repository": "CRAN", - "Hash": "79540e5fcd9e0435af547d885f184fd5", "Requirements": [ + "R", "cli", "glue", "lifecycle", "rlang", "vctrs", "withr" - ] + ], + "Hash": "829f27b9c4919c16b593794a6344d6c0" }, "tzdb": { "Package": "tzdb", - "Version": "0.3.0", + "Version": "0.4.0", "Source": "Repository", "Repository": "CRAN", - "Hash": "b2e1cbce7c903eaf23ec05c58e59fb5e", "Requirements": [ + "R", "cpp11" - ] + ], + "Hash": "f561504ec2897f4d46f0c7657e488ae1" }, "utf8": { "Package": "utf8", - "Version": "1.2.2", + "Version": "1.2.4", "Source": "Repository", "Repository": "CRAN", - "Hash": "c9c462b759a5cc844ae25b5942654d13", - "Requirements": [] + "Requirements": [ + "R" + ], + "Hash": "62b65c52671e6665f803ff02954446e9" }, "vctrs": { "Package": "vctrs", - "Version": "0.5.1", + "Version": "0.6.5", "Source": "Repository", "Repository": "CRAN", - "Hash": "970324f6572b4fd81db507b5d4062cb0", "Requirements": [ + "R", "cli", "glue", "lifecycle", "rlang" - ] + ], + "Hash": "c03fa420630029418f7e6da3667aac4a" }, "vroom": { "Package": "vroom", - "Version": "1.6.0", + "Version": "1.6.5", "Source": "Repository", "Repository": "CRAN", - "Hash": "64f81fdead6e0d250fb041e175d123ab", "Requirements": [ + "R", "bit64", "cli", "cpp11", @@ -548,46 +615,56 @@ "glue", "hms", "lifecycle", + "methods", "progress", "rlang", + "stats", "tibble", "tidyselect", "tzdb", "vctrs", "withr" - ] + ], + "Hash": "390f9315bc0025be03012054103d227c" }, "whisker": { "Package": "whisker", "Version": "0.4.1", "Source": "Repository", "Repository": "CRAN", - "Hash": "c6abfa47a46d281a7d5159d0a8891e88", - "Requirements": [] + "Hash": "c6abfa47a46d281a7d5159d0a8891e88" }, "withr": { "Package": "withr", - "Version": "2.5.0", + "Version": "3.0.1", "Source": "Repository", "Repository": "CRAN", - "Hash": "c0e49a9760983e81e55cdd9be92e7182", - "Requirements": [] + "Requirements": [ + "R", + "grDevices", + "graphics" + ], + "Hash": "07909200e8bbe90426fbfeb73e1e27aa" }, "xfun": { "Package": "xfun", - "Version": "0.23", + "Version": "0.47", "Source": "Repository", "Repository": "CRAN", - "Hash": "791a57f43c887111490851dcd166d344", - "Requirements": [] + "Requirements": [ + "R", + "grDevices", + "stats", + "tools" + ], + "Hash": "36ab21660e2d095fef0d83f689e0477c" }, "yaml": { "Package": "yaml", - "Version": "2.2.1", + "Version": "2.3.10", "Source": "Repository", "Repository": "CRAN", - "Hash": "2826c5d9efb0a88f657c7a679c7106db", - "Requirements": [] + "Hash": "51dab85c6c98e50a18d7551e9d49f76c" } } -} +} \ No newline at end of file From 14c73ff5b0f2b2c1cbdf504a3487f46c7bfdfa89 Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Sat, 13 Dec 2025 09:58:35 -0800 Subject: [PATCH 17/22] separate gem compile and remove unused dependencies --- degauss-foundry/Dockerfile | 77 +-- degauss-foundry/lib/geocoder/us/database.rb | 4 +- degauss-foundry/renv.lock | 670 -------------------- degauss-foundry/src/liblwgeom/vsprintf.c | 4 +- 4 files changed, 31 insertions(+), 724 deletions(-) delete mode 100644 degauss-foundry/renv.lock diff --git a/degauss-foundry/Dockerfile b/degauss-foundry/Dockerfile index 830d025..654c723 100644 --- a/degauss-foundry/Dockerfile +++ b/degauss-foundry/Dockerfile @@ -1,8 +1,17 @@ # https://stackoverflow.com/questions/34751814/build-postgres-docker-container-with-initial-schema # compile the gem for the build # PART 1 -FROM rocker/r-ver:4.4.1 +FROM rocker/r-ver AS compiler +# copy statics +RUN mkdir /app +WORKDIR /app +COPY Makefile.ruby . +COPY /src ./src +COPY /lib ./lib +COPY /gemspec ./gemspec + +# install dependencies RUN apt-get update && apt-get install -y \ bison \ flex \ @@ -17,23 +26,16 @@ RUN apt-get update && apt-get install -y \ ruby-full \ sqlite3 \ software-properties-common - RUN gem install sqlite3 json Text +RUN apt-get dist-upgrade -yq -RUN mkdir /app -WORKDIR /app - -COPY Makefile.ruby . -COPY /src ./src -COPY /lib ./lib -COPY /gemspec ./gemspec - -RUN make -f Makefile.ruby install +# build gem +RUN make -f Makefile.ruby install # PART 2 -## copy build from initialization layer -FROM rhub/r-minimal:4.5.0 AS buildtime_init +## ADD build from initialization layer +FROM rhub/r-minimal # DeGAUSS container metadata ENV degauss_name="geocoder" @@ -47,51 +49,28 @@ LABEL "org.degauss.version"="${degauss_version}" LABEL "org.degauss.description"="${degauss_description}" LABEL "org.degauss.argument"="${degauss_argument}" +# copy statics RUN mkdir /app WORKDIR /app - -COPY --from=buildtime_init_builder Geocoder-US-2.0.4.gem Geocoder-US-2.0.4.gem +COPY --from=compiler /app/Geocoder-US-2.0.4.gem Geocoder-US-2.0.4.gem ADD https://colebrokamp-dropbox.s3.amazonaws.com/geocoder.db /opt/geocoder.db - -RUN apk add \ - bison \ - flex \ - g++ \ - gcc \ - gnupg \ - linux-headers \ - musl-dev \ - py3-pip \ - ruby \ - sqlite \ - vim -RUN apk upgrade --no-cache - -RUN gem install json sqlite3 Text -RUN gem install Geocoder-US-2.0.4.gem - -# install required version of renv -RUN R --quiet -e "install.packages('remotes', repos = c(CRAN = 'https://packagemanager.posit.co/cran/latest'))" -RUN R --quiet -e "remotes::install_github('rstudio/renv@v1.0.7')" - -COPY renv.lock . -RUN R --quiet -e "renv::restore(repos = c(CRAN = 'https://packagemanager.posit.co/cran/latest'))" - COPY entrypoint.R . COPY geocode.rb . -# Foundry customizations part 1 -RUN adduser --uid 5001 --disabled-password user -RUN chown -R 5001:5001 /tmp /opt /app +# install entrypoint.R dependencies +RUN apk add py3-pip ruby +RUN gem install Geocoder-US-2.0.4.gem +RUN R --quiet -e "install.packages(c('digest','knitr','mappp','remotes'), repos=c(CRAN='https://packagemanager.posit.co/cran/latest'))" +RUN R --quiet -e "remotes::install_github('degauss-org/dht')" +RUN apk upgrade --no-cache -# Foundry customizations pt2 -# RUN adduser --uid 5001 --disabled-password user -# RUN chown -R 5001:5001 /tmp /opt /app +# Foundry customizations +COPY --chmod=0755 entrypoint.py /usr/bin/entrypoint +RUN adduser --uid 5001 --disabled-password user +RUN chown -R 5001:5001 /tmp /app -ADD entrypoint.py /usr/bin/entrypoint -RUN chmod +x /usr/bin/entrypoint RUN mkdir -p /opt/palantir/sidecars/shared-volumes/shared/ -RUN chown 5001 /opt/palantir/sidecars/shared-volumes/shared/ +RUN chown -R 5001:5001 /opt ENV SHARED_DIR=/opt/palantir/sidecars/shared-volumes/shared USER 5001 diff --git a/degauss-foundry/lib/geocoder/us/database.rb b/degauss-foundry/lib/geocoder/us/database.rb index d7e44ac..4239c60 100644 --- a/degauss-foundry/lib/geocoder/us/database.rb +++ b/degauss-foundry/lib/geocoder/us/database.rb @@ -39,7 +39,7 @@ def initialize (filename, options = {}) :create => false, :dbtype => 1} options = defaults.merge options raise ArgumentError, "can't find database #{filename}" \ - unless options[:create] or File.exist? filename + unless options[:create] or File.exists? filename @db = SQLite3::Database.new( filename ) @st = {} @dbtype = options[:dbtype] @@ -630,7 +630,7 @@ def clean_record! (record) unless record[:score].nil? record.keys.each {|k| record[k] = "" if record[k].nil? } # clean up nils record.delete :components unless @debug - record.delete_if {|k,v| k.is_a? Integer or + record.delete_if {|k,v| k.is_a? Fixnum or [:geometry, :side, :tlid, :fid, :fid1, :fid2, :street_phone, :city_phone, :fromhn, :tohn, :paflag, :flipped, :street_score, :city_score, :priority, :fips_class, :fips_place, :status].include? k} diff --git a/degauss-foundry/renv.lock b/degauss-foundry/renv.lock deleted file mode 100644 index 67814a8..0000000 --- a/degauss-foundry/renv.lock +++ /dev/null @@ -1,670 +0,0 @@ -{ - "R": { - "Version": "4.4.1", - "Repositories": [] - }, - "Packages": { - "R6": { - "Package": "R6", - "Version": "2.5.1", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R" - ], - "Hash": "470851b6d5d0ac559e9d01bb352b4021" - }, - "bit": { - "Package": "bit", - "Version": "4.0.5", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R" - ], - "Hash": "d242abec29412ce988848d0294b208fd" - }, - "bit64": { - "Package": "bit64", - "Version": "4.0.5", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "bit", - "methods", - "stats", - "utils" - ], - "Hash": "9fe98599ca456d6552421db0d6772d8f" - }, - "cachem": { - "Package": "cachem", - "Version": "1.1.0", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "fastmap", - "rlang" - ], - "Hash": "cd9a672193789068eb5a2aad65a0dedf" - }, - "cli": { - "Package": "cli", - "Version": "3.6.3", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "utils" - ], - "Hash": "b21916dd77a27642b447374a5d30ecf3" - }, - "clipr": { - "Package": "clipr", - "Version": "0.8.0", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "utils" - ], - "Hash": "3f038e5ac7f41d4ac41ce658c85e3042" - }, - "cpp11": { - "Package": "cpp11", - "Version": "0.4.7", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R" - ], - "Hash": "5a295d7d963cc5035284dcdbaf334f4e" - }, - "crayon": { - "Package": "crayon", - "Version": "1.5.3", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "grDevices", - "methods", - "utils" - ], - "Hash": "859d96e65ef198fd43e82b9628d593ef" - }, - "dht": { - "Package": "dht", - "Version": "1.2.3", - "Source": "GitHub", - "RemoteType": "github", - "RemoteHost": "api.github.com", - "RemoteUsername": "degauss-org", - "RemoteRepo": "dht", - "RemoteRef": "master", - "RemoteSha": "f0b59bbb91697a55074d065642ed012b652e7d31", - "Requirements": [ - "R", - "cli", - "dplyr", - "fs", - "glue", - "magrittr", - "prettyunits", - "ps", - "purrr", - "readr", - "stringr", - "tidyr", - "tidyselect", - "whisker", - "withr" - ], - "Hash": "05547abc78751624749fb01f2a2e49b9" - }, - "digest": { - "Package": "digest", - "Version": "0.6.37", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "utils" - ], - "Hash": "33698c4b3127fc9f506654607fb73676" - }, - "docopt": { - "Package": "docopt", - "Version": "0.7.1", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "methods" - ], - "Hash": "e9eeef7931ee99ca0093f3f20b88e09b" - }, - "dplyr": { - "Package": "dplyr", - "Version": "1.1.4", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "R6", - "cli", - "generics", - "glue", - "lifecycle", - "magrittr", - "methods", - "pillar", - "rlang", - "tibble", - "tidyselect", - "utils", - "vctrs" - ], - "Hash": "fedd9d00c2944ff00a0e2696ccf048ec" - }, - "evaluate": { - "Package": "evaluate", - "Version": "0.24.0", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "methods" - ], - "Hash": "a1066cbc05caee9a4bf6d90f194ff4da" - }, - "fansi": { - "Package": "fansi", - "Version": "1.0.6", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "grDevices", - "utils" - ], - "Hash": "962174cf2aeb5b9eea581522286a911f" - }, - "fastmap": { - "Package": "fastmap", - "Version": "1.2.0", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "aa5e1cd11c2d15497494c5292d7ffcc8" - }, - "fs": { - "Package": "fs", - "Version": "1.6.4", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "methods" - ], - "Hash": "15aeb8c27f5ea5161f9f6a641fafd93a" - }, - "generics": { - "Package": "generics", - "Version": "0.1.3", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "methods" - ], - "Hash": "15e9634c0fcd294799e9b2e929ed1b86" - }, - "glue": { - "Package": "glue", - "Version": "1.7.0", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "methods" - ], - "Hash": "e0b3a53876554bd45879e596cdb10a52" - }, - "highr": { - "Package": "highr", - "Version": "0.11", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "xfun" - ], - "Hash": "d65ba49117ca223614f71b60d85b8ab7" - }, - "hms": { - "Package": "hms", - "Version": "1.1.3", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "lifecycle", - "methods", - "pkgconfig", - "rlang", - "vctrs" - ], - "Hash": "b59377caa7ed00fa41808342002138f9" - }, - "jsonlite": { - "Package": "jsonlite", - "Version": "1.8.8", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "methods" - ], - "Hash": "e1b9c55281c5adc4dd113652d9e26768" - }, - "knitr": { - "Package": "knitr", - "Version": "1.48", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "evaluate", - "highr", - "methods", - "tools", - "xfun", - "yaml" - ], - "Hash": "acf380f300c721da9fde7df115a5f86f" - }, - "lifecycle": { - "Package": "lifecycle", - "Version": "1.0.4", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "cli", - "glue", - "rlang" - ], - "Hash": "b8552d117e1b808b09a832f589b79035" - }, - "magrittr": { - "Package": "magrittr", - "Version": "2.0.3", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R" - ], - "Hash": "7ce2733a9826b3aeb1775d56fd305472" - }, - "mappp": { - "Package": "mappp", - "Version": "1.0.0", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "memoise", - "parallel", - "parallelly", - "pbmcapply", - "progress", - "purrr", - "rlang" - ], - "Hash": "969ed5fab0c1752ddd0a43f789c9ac4e" - }, - "memoise": { - "Package": "memoise", - "Version": "2.0.1", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "cachem", - "rlang" - ], - "Hash": "e2817ccf4a065c5d9d7f2cfbe7c1d78c" - }, - "parallelly": { - "Package": "parallelly", - "Version": "1.38.0", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "parallel", - "tools", - "utils" - ], - "Hash": "6e8b139c1904f5e9e14c69db64453bbe" - }, - "pbmcapply": { - "Package": "pbmcapply", - "Version": "1.5.1", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "parallel", - "utils" - ], - "Hash": "bf173fb42b60f8f404f7a9f9c3f95d0d" - }, - "pillar": { - "Package": "pillar", - "Version": "1.9.0", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "cli", - "fansi", - "glue", - "lifecycle", - "rlang", - "utf8", - "utils", - "vctrs" - ], - "Hash": "15da5a8412f317beeee6175fbc76f4bb" - }, - "pkgconfig": { - "Package": "pkgconfig", - "Version": "2.0.3", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "utils" - ], - "Hash": "01f28d4278f15c76cddbea05899c5d6f" - }, - "prettyunits": { - "Package": "prettyunits", - "Version": "1.2.0", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R" - ], - "Hash": "6b01fc98b1e86c4f705ce9dcfd2f57c7" - }, - "progress": { - "Package": "progress", - "Version": "1.2.3", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "R6", - "crayon", - "hms", - "prettyunits" - ], - "Hash": "f4625e061cb2865f111b47ff163a5ca6" - }, - "ps": { - "Package": "ps", - "Version": "1.7.7", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "utils" - ], - "Hash": "878b467580097e9c383acbb16adab57a" - }, - "purrr": { - "Package": "purrr", - "Version": "1.0.2", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "cli", - "lifecycle", - "magrittr", - "rlang", - "vctrs" - ], - "Hash": "1cba04a4e9414bdefc9dcaa99649a8dc" - }, - "readr": { - "Package": "readr", - "Version": "2.1.5", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "R6", - "cli", - "clipr", - "cpp11", - "crayon", - "hms", - "lifecycle", - "methods", - "rlang", - "tibble", - "tzdb", - "utils", - "vroom" - ], - "Hash": "9de96463d2117f6ac49980577939dfb3" - }, - "renv": { - "Package": "renv", - "Version": "1.0.7", - "Source": "GitHub", - "RemoteType": "github", - "RemoteHost": "api.github.com", - "RemoteRepo": "renv", - "RemoteUsername": "rstudio", - "RemoteRef": "v1.0.7", - "RemoteSha": "265a03869234f07c9171b6829d09427577e19baa", - "Requirements": [ - "utils" - ], - "Hash": "c9546c795efae077e4d11bce9eb9d059" - }, - "rlang": { - "Package": "rlang", - "Version": "1.1.4", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "utils" - ], - "Hash": "3eec01f8b1dee337674b2e34ab1f9bc1" - }, - "stringi": { - "Package": "stringi", - "Version": "1.8.4", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "stats", - "tools", - "utils" - ], - "Hash": "39e1144fd75428983dc3f63aa53dfa91" - }, - "stringr": { - "Package": "stringr", - "Version": "1.5.1", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "cli", - "glue", - "lifecycle", - "magrittr", - "rlang", - "stringi", - "vctrs" - ], - "Hash": "960e2ae9e09656611e0b8214ad543207" - }, - "tibble": { - "Package": "tibble", - "Version": "3.2.1", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "fansi", - "lifecycle", - "magrittr", - "methods", - "pillar", - "pkgconfig", - "rlang", - "utils", - "vctrs" - ], - "Hash": "a84e2cc86d07289b3b6f5069df7a004c" - }, - "tidyr": { - "Package": "tidyr", - "Version": "1.3.1", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "cli", - "cpp11", - "dplyr", - "glue", - "lifecycle", - "magrittr", - "purrr", - "rlang", - "stringr", - "tibble", - "tidyselect", - "utils", - "vctrs" - ], - "Hash": "915fb7ce036c22a6a33b5a8adb712eb1" - }, - "tidyselect": { - "Package": "tidyselect", - "Version": "1.2.1", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "cli", - "glue", - "lifecycle", - "rlang", - "vctrs", - "withr" - ], - "Hash": "829f27b9c4919c16b593794a6344d6c0" - }, - "tzdb": { - "Package": "tzdb", - "Version": "0.4.0", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "cpp11" - ], - "Hash": "f561504ec2897f4d46f0c7657e488ae1" - }, - "utf8": { - "Package": "utf8", - "Version": "1.2.4", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R" - ], - "Hash": "62b65c52671e6665f803ff02954446e9" - }, - "vctrs": { - "Package": "vctrs", - "Version": "0.6.5", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "cli", - "glue", - "lifecycle", - "rlang" - ], - "Hash": "c03fa420630029418f7e6da3667aac4a" - }, - "vroom": { - "Package": "vroom", - "Version": "1.6.5", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "bit64", - "cli", - "cpp11", - "crayon", - "glue", - "hms", - "lifecycle", - "methods", - "progress", - "rlang", - "stats", - "tibble", - "tidyselect", - "tzdb", - "vctrs", - "withr" - ], - "Hash": "390f9315bc0025be03012054103d227c" - }, - "whisker": { - "Package": "whisker", - "Version": "0.4.1", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "c6abfa47a46d281a7d5159d0a8891e88" - }, - "withr": { - "Package": "withr", - "Version": "3.0.1", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "grDevices", - "graphics" - ], - "Hash": "07909200e8bbe90426fbfeb73e1e27aa" - }, - "xfun": { - "Package": "xfun", - "Version": "0.47", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "R", - "grDevices", - "stats", - "tools" - ], - "Hash": "36ab21660e2d095fef0d83f689e0477c" - }, - "yaml": { - "Package": "yaml", - "Version": "2.3.10", - "Source": "Repository", - "Repository": "CRAN", - "Hash": "51dab85c6c98e50a18d7551e9d49f76c" - } - } -} \ No newline at end of file diff --git a/degauss-foundry/src/liblwgeom/vsprintf.c b/degauss-foundry/src/liblwgeom/vsprintf.c index e4c3dfd..abad18c 100644 --- a/degauss-foundry/src/liblwgeom/vsprintf.c +++ b/degauss-foundry/src/liblwgeom/vsprintf.c @@ -147,9 +147,7 @@ lw_vasprintf (result, format, args) va_list temp; /* Use va_copy for compatibility with both 32 and 64 bit args */ - // __va_copy(temp, args); - - va_copy(temp, args); + __va_copy(temp, args); return int_vasprintf (result, format, &temp); } From d589dac5459b50a674ad116889452e728e4f2940 Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Sat, 13 Dec 2025 10:32:18 -0800 Subject: [PATCH 18/22] remove deprecated functions and use modern equivalents --- degauss-foundry/lib/geocoder/us/database.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/degauss-foundry/lib/geocoder/us/database.rb b/degauss-foundry/lib/geocoder/us/database.rb index 4239c60..141ecba 100644 --- a/degauss-foundry/lib/geocoder/us/database.rb +++ b/degauss-foundry/lib/geocoder/us/database.rb @@ -39,7 +39,7 @@ def initialize (filename, options = {}) :create => false, :dbtype => 1} options = defaults.merge options raise ArgumentError, "can't find database #{filename}" \ - unless options[:create] or File.exists? filename + unless options[:create] or File.exist? filename @db = SQLite3::Database.new( filename ) @st = {} @dbtype = options[:dbtype] @@ -153,8 +153,7 @@ def execute_statement (st, *params) result = st.execute(*params) columns = result.columns.map {|c| c.to_sym} result.each {|row| - rows << Hash[*(columns.zip(row).flatten)]} - + rows << columns.zip(row).to_h} end if @debug runtime = format("%.3f", Time.now - start) @@ -630,7 +629,7 @@ def clean_record! (record) unless record[:score].nil? record.keys.each {|k| record[k] = "" if record[k].nil? } # clean up nils record.delete :components unless @debug - record.delete_if {|k,v| k.is_a? Fixnum or + record.delete_if {|k,v| k.is_a? Integer or [:geometry, :side, :tlid, :fid, :fid1, :fid2, :street_phone, :city_phone, :fromhn, :tohn, :paflag, :flipped, :street_score, :city_score, :priority, :fips_class, :fips_place, :status].include? k} From b570a16471894232b1cb96fac908fadcbe61df7b Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Sat, 13 Dec 2025 10:36:37 -0800 Subject: [PATCH 19/22] fit typo --- degauss-foundry/lib/geocoder/us.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/degauss-foundry/lib/geocoder/us.rb b/degauss-foundry/lib/geocoder/us.rb index a39bc81..a3904ba 100644 --- a/degauss-foundry/lib/geocoder/us.rb +++ b/degauss-foundry/lib/geocoder/us.rb @@ -7,7 +7,7 @@ # General usage is as follows: # # >> require 'geocoder/us' -# >> db = Geocoder::US::Database.new("/opt/tiger/geocoder.db") +# >> db = Geocoder::US::Database.new("/opt/geocoder.db") # >> p db.geocode("1600 Pennsylvania Av, Washington DC") # # [{:pretyp=>"", :street=>"Pennsylvania", :sufdir=>"NW", :zip=>"20502", From 36963ecf51c8791b54b5870bc87b8be51ecc3ff4 Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Sat, 13 Dec 2025 12:41:50 -0800 Subject: [PATCH 20/22] add test address file --- test/address_test.csv | 202 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 test/address_test.csv diff --git a/test/address_test.csv b/test/address_test.csv new file mode 100644 index 0000000..57e830a --- /dev/null +++ b/test/address_test.csv @@ -0,0 +1,202 @@ +id,address +55000100280,"2854 ROSEANN LN GREEN TOWNSHIP, OH 45239" +9800060045,"407 SOUTHVIEW AV CINCINNATI, OH 45219" +59100090241,"909 GRETNA LN FOREST PARK, OH 45240" +55001310120,"P.O. BOX 12345 GREEN TOWNSHIP, OH 45238" +4100010061,"PO 12345 CINCINNATI, OH 45208" +9000010083,"3333 BURNET AVE CINCINNATI, OH 45229" +9000010086,"3333 BURNET AVE SYRACUSE, NY 13206" +21600460082,"222 E CENTRAL PKWY CINCINNATI, OH 45202" +67100240229,foreign +2800040021,verify +59004140168,unknown +50002200628, +67100040004,NA +5100020177,"4506 CAMBERWELL RD CINCINNATI, OH 45209" +55000100212,"5585 FAIRWOOD RD GREEN TOWNSHIP, OH 45239" +51000810328,"6628 JULY CT COLERAIN TOWNSHIP, OH 45239" +61201400371,"5126 BRASHER AV BLUE ASH, OH 45242" +19200650054,"3708 TAPPAN AV CINCINNATI, OH 45223" +60000110098,"11924 7TH AV SYCAMORE TOWNSHIP, OH 45249" +60001000117,"7401 KENNEDY LN SYCAMORE TOWNSHIP, OH 45242" +50002700262,"7085 JEANNIE AV ANDERSON TOWNSHIP, OH 45230" +60002110309,"6120 ST REGIS DR SYCAMORE TOWNSHIP, OH 45236" +22100190292,"4162 CHAMBERS ST CINCINNATI, OH 45223" +61201110164,"17 CARPENTERS RIDGE RD BLUE ASH, OH 45241" +52500160078,"6639 APACHE CIR MADEIRA, OH 45243" +50003700268,"6296 CRITTENDEN DR ANDERSON TOWNSHIP, OH 45244" +17900770440,"1029 RUTLEDGE AV CINCINNATI, OH 45205" +60002310072,"8462 WICKLOW AV SYCAMORE TOWNSHIP, OH 45236" +61200600204,"9354 FLORAL AV BLUE ASH, OH 45242" +51000710151,"8145 BLANCHETTA DR COLERAIN TOWNSHIP, OH 45239" +52700100025,"6827 MT VERNON AV MARIEMONT, OH 45227" +22500030012,"2385 MONTANA AV CINCINNATI, OH 45211" +51000700019,"3053 BANNING RD COLERAIN TOWNSHIP, OH 45239" +65100520093,"4228 FRANKLIN AV NORWOOD, OH 45212" +14900120025,"2388 MARYLAND AV CINCINNATI, OH 45204" +59500020243,"1532 SOUTHRIDGE LN NORTH COLLEGE HILL, OH 45231" +60800100075,"4104 CARRIAGELITE DR SHARONVILLE, OH 45241" +51001010270,"3657 VERNIER DR COLERAIN TOWNSHIP, OH 45251" +2200040012,"1307 DILLON AV CINCINNATI, OH 45208" +55000640056,"4370 AIRYMONT CT GREEN TOWNSHIP, OH 45211" +59600100129,"11 ALBION LN GLENDALE, OH 45246" +67100160157,"2152 BOLSER DR READING, OH 45215" +54001200027,"6513 RAPID RUN RD DELHI TOWNSHIP, OH 45233" +60000930179,"5849 BAYBERRY DR SYCAMORE TOWNSHIP, OH 45242" +02040A150084,"1628 MINION AV CINCINNATI, OH 45205" +5400040073,"3329 GRAYDON AV CINCINNATI, OH 45207" +59400100515,"955 SCHUMARD AV LINCOLN HEIGHTS, OH 45215" +60200080044,"4229 SIBLEY AV SILVERTON, OH 45236" +51001010052,"3630 POOLE RD COLERAIN TOWNSHIP, OH 45251" +9500050041,"2131 VINE ST CINCINNATI, OH 45202" +4100010010,"3611 SHAW AV CINCINNATI, OH 45208" +51000410420,"9982 DUNRAVEN DR COLERAIN TOWNSHIP, OH 45251" +24700060285,"1135 WIONNA AV CINCINNATI, OH 45224" +61201500517,"10960 E ALLENHURST BLVD BLUE ASH, OH 45241" +59002310126,"1119 GARNOA DR SPRINGFIELD TOWNSHIP, OH 45231" +3700030043,"6406 ROE ST CINCINNATI, OH 45227" +59500080299,"1947 STERLING AV NORTH COLLEGE HILL, OH 45239" +11500070009,"3989 BEECHWOOD AV CINCINNATI, OH 45229" +55001610086,"5921 WERK RD GREEN TOWNSHIP, OH 45248" +13100070229,"1922 CATALINA AV CINCINNATI, OH 45237" +5800010049,"3518 HUDSON AV CINCINNATI, OH 45207" +51000240123,"2740 GREENBROOK LN COLERAIN TOWNSHIP, OH 45251" +55002010182,"5979 GAINES RD GREEN TOWNSHIP, OH 45247" +54000600136,"5358 LILIBET CT DELHI TOWNSHIP, OH 45238" +17900750537,"1292 MCKEONE AV CINCINNATI, OH 45205" +1900020125,"1338 HERLIN PL CINCINNATI, OH 45208" +50002810394,"6996 QUEENSWAY LN ANDERSON TOWNSHIP, OH 45230" +52100020167,"2 MELISSA CT TERRACE PARK, OH 45174" +24500060047,"28 WOODSDALE AV CINCINNATI, OH 45216" +57000500301,"7841 SURREYWOOD DR MIAMI TOWNSHIP, OH 45052" +55000410311,"5286 SIDNEY RD GREEN TOWNSHIP, OH 45238" +52002140068,"5731 MONNING PL COLUMBIA TOWNSHIP, OH 45227" +59002820347,"12065 HAZELHURST LN SPRINGFIELD TOWNSHIP, OH 45240" +100050097,"6254 CRESTVIEW PL CINCINNATI, OH 45230" +54000330120,"5056 FRANCISVIEW DR DELHI TOWNSHIP, OH 45238" +18000800026,"4736 GLENWAY AV CINCINNATI, OH 45238" +51000320424,"2609 MERRITTVIEW LN COLERAIN TOWNSHIP, OH 45231" +59900700299,"850 CLEARFIELD LN SPRINGDALE, OH 45240" +24800010022,"3020 WEST TOWER AV CINCINNATI, OH 45238" +17900780057,"4056 VINEDALE AV CINCINNATI, OH 45205" +52001720079,"6831 WINDWARD AV COLUMBIA TOWNSHIP, OH 45227" +17900750214,"1048 SUNSET AV CINCINNATI, OH 45205" +11900010082,"1739 NORTHCUTT AV CINCINNATI, OH 45237" +55002430066,"6673 POWNER FARM DR GREEN TOWNSHIP, OH 45248" +60800050170,"3511 BEEKLEY WOODS DR SHARONVILLE, OH 45241" +7200020282,"1227 MARTIN DR CINCINNATI, OH 45202" +8800070090,"246 EARNSHAW AV CINCINNATI, OH 45219" +62902310012,"9400 CUNNINGHAM RD INDIAN HILL, OH 45243" +4000020104,"3751 HYDE PARK AV CINCINNATI, OH 45209" +54000400362,"4993 DUEBBER DR DELHI TOWNSHIP, OH 45238" +61201120040,"5 MUIRFIELD LN BLUE ASH, OH 45241" +55001810077,"4211 RACE RD GREEN TOWNSHIP, OH 45211" +50100100063,"5027 VILLAGE DR NEWTOWN, OH 45244" +60800300205,"5089 LORD ALFRED CT SHARONVILLE, OH 45241" +59100080228,"11754 HAMLET RD FOREST PARK, OH 45240" +17900750148,"1236 SLIKER AV CINCINNATI, OH 45205" +51000810258,"3257 BLUEACRES DR COLERAIN TOWNSHIP, OH 45239" +4200020020,"3549 BURCH AV CINCINNATI, OH 45208" +57000100143,"7554 BUFFALO RIDGE RD MIAMI TOWNSHIP, OH 45002" +59100030081,"510 BRUNSWICK DR FOREST PARK, OH 45240" +62100080047,"10 HIGHRIDGE DR LOVELAND, OH 45140" +3600040321,"4514 WHETSEL AV CINCINNATI, OH 45227" +57001700095,"3139 BRUNSMAN WY MIAMI TOWNSHIP, OH 45052" +62100070171,"509 NAVAHO DR LOVELAND, OH 45140" +59003930243,"10572 FARMHILL CT SPRINGFIELD TOWNSHIP, OH 45231" +55002320114,"6268 SPRINGMYER DR GREEN TOWNSHIP, OH 45248" +57000200016,"7547 BUFFALO RIDGE RD MIAMI TOWNSHIP, OH 45002" +55000100575,"3194 BALSAMRIDGE DR GREEN TOWNSHIP, OH 45239" +59100090432,"11528 GAFFNEY PL FOREST PARK, OH 45240" +61100110179,"10261 FALLING WATERS LN EVENDALE, OH 45241" +51000530339,"2595 CORNWALL DR COLERAIN TOWNSHIP, OH 45231" +51000730015,"7181 PIPPIN RD COLERAIN TOWNSHIP, OH 45239" +61100800227,"9636 REXFORD DR EVENDALE, OH 45241" +2400030126,"4456 EASTERN AV CINCINNATI, OH 45226" +60800070174,"10945 CONESTOGA CT SHARONVILLE, OH 45241" +11200020024,"630 GREENWOOD AV CINCINNATI, OH 45229" +51000740315,"6691 ACRE DR COLERAIN TOWNSHIP, OH 45239" +20200320176,"1553 KNOX ST CINCINNATI, OH 45214" +55000410216,"5153 RALPH AV GREEN TOWNSHIP, OH 45238" +24700050244,"7921 CHERRYWOOD CT CINCINNATI, OH 45224" +55002320075,"4019 EBENEZER RD GREEN TOWNSHIP, OH 45248" +59003910105,"1882 ROOSEVELT AV SPRINGFIELD TOWNSHIP, OH 45240" +23200020096,"6207 SAVANNAH AV CINCINNATI, OH 45224" +58200070157,"111 BANK AV SAINT BERNARD, OH 45217" +11400020023,"3895 SPRING HOUSE LN CINCINNATI, OH 45217" +59003710280,"9386 MONTORO DR SPRINGFIELD TOWNSHIP, OH 45231" +60800010039,"3307 HAGEMAN AV SHARONVILLE, OH 45241" +59002410156,"904 W MC KELVEY RD SPRINGFIELD TOWNSHIP, OH 45231" +52001000061,"4342 ASHLEY OAKS DR COLUMBIA TOWNSHIP, OH 45227" +20400170164,"1842 WYOMING AV CINCINNATI, OH 45205" +59700800005,"2 BRADNOR PL GREENHILLS, OH 45218" +58200130279,"90 ANGELS WY SAINT BERNARD, OH 45217" +60000111194,"11961 1ST AV SYCAMORE TOWNSHIP, OH 45249" +9800040154,"518 FORTUNE AV CINCINNATI, OH 45219" +21200620068,"3201 STANHOPE AV CINCINNATI, OH 45211" +65100410066,"3917 CATHERINE AV NORWOOD, OH 45212" +56000900081,"9960 CAROLINA TRACE RD HARRISON TOWNSHIP, OH 45030" +59600020095,"332 E SHARON RD GLENDALE, OH 45246" +53001000019,"10101 HAMILTON CLEVES RD CROSBY TOWNSHIP, OH 45030" +55000820460,"6076 LAGRANGE LN GREEN TOWNSHIP, OH 45239" +67100260206,"1325 FUHRMAN RD READING, OH 45215" +60300250056,"8831 WELLERSTATION DR MONTGOMERY, OH 45249" +55100130024,"4222 CHURCHVIEW LN CHEVIOT, OH 45211" +55100070083,"4054 HARDING AV CHEVIOT, OH 45211" +51000420446,"2916 WILLOW RIDGE DR COLERAIN TOWNSHIP, OH 45251" +59002500206,"10506 HADLEY RD SPRINGFIELD TOWNSHIP, OH 45218" +54001200680,"905 BRAEMORE LN DELHI TOWNSHIP, OH 45233" +54000420149,"335 GLEN OAKS DR DELHI TOWNSHIP, OH 45238" +21000730135,"3071 PENROSE PL CINCINNATI, OH 45211" +56100040040,"304 BROOKS LN HARRISON, OH 45030" +65100080039,"5114 GLOBE AV NORWOOD, OH 45212" +20200400358,"1637 PULTE ST CINCINNATI, OH 45225" +60300050094,"8001 MONTE DR MONTGOMERY, OH 45242" +50100130018,"3380 IVY HILLS BLVD NEWTOWN, OH 45244" +16700020157,"7411 GRACELY DR CINCINNATI, OH 45233" +65100400026,"3945 FOREST AV NORWOOD, OH 45212" +65100410095,"3905 AVILLA PL NORWOOD, OH 45212" +21200670036,"3324 ROBINET DR CINCINNATI, OH 45238" +57200070231,"23 TIMBERLINE CT CLEVES, OH 45002" +62002010167,"8524 WHISPERWOODS LN SYMMES TOWNSHIP, OH 45249" +54000630102,"775 IVYHILL DR DELHI TOWNSHIP, OH 45238" +65100110162,"2619 SHERIDAN DR NORWOOD, OH 45212" +52700200252,"3832 INDIANVIEW AV MARIEMONT, OH 45227" +20300300018,"1655 WAVERLY AV CINCINNATI, OH 45214" +50000420115,"1692 TONOPAH DR ANDERSON TOWNSHIP, OH 45255" +65100040079,"5621 ALVINA AV NORWOOD, OH 45212" +52500250067,"6029 CHEROKEE DR MADEIRA, OH 45243" +100040270,"2085 AUTUMNHILL CT CINCINNATI, OH 45230" +51000110447,"2537 RETFORD DR COLERAIN TOWNSHIP, OH 45231" +55001130165,"4072 SIMCA LN GREEN TOWNSHIP, OH 45211" +5500060041,"2922 HACKBERRY ST CINCINNATI, OH 45206" +55000620224,"3490 CENTURION DR GREEN TOWNSHIP, OH 45211" +50001440012,"8103 CABINET CIR ANDERSON TOWNSHIP, OH 45244" +59002230110,"8774 CONSTANCE LN SPRINGFIELD TOWNSHIP, OH 45231" +60002320132,"4028 LIMERICK AV SYCAMORE TOWNSHIP, OH 45236" +61201700308,"11174 JARDIN PL BLUE ASH, OH 45241" +52001710431,"6634 MURRAY AV COLUMBIA TOWNSHIP, OH 45227" +60002000130,"4133 TREBOR DR SYCAMORE TOWNSHIP, OH 45236" +60800320302,"5462 VICTORIAN WY SHARONVILLE, OH 45241" +55002520302,"6215 KINGOAK DR GREEN TOWNSHIP, OH 45248" +200030031,"6522 WALDORF PL CINCINNATI, OH 45230" +60300010229,"80 WEST ST MONTGOMERY, OH 45242" +51001110141,"9858 CAPSTAN DR COLERAIN TOWNSHIP, OH 45251" +52601100145,"5178 ROLLMAN ESTATES DR AMBERLEY VILLAGE, OH 45236" +51000440160,"10056 STURGEON LN COLERAIN TOWNSHIP, OH 45251" +62100210082,"106 PHEASANT WOODS CT LOVELAND, OH 45140" +52500240012,"5687 WHETSEL AV MADEIRA, OH 45227" +11500010197,"3876 VINE ST CINCINNATI, OH 45217" +56100070246,"143 FLINTSTONE DR HARRISON, OH 45030" +59900420081,"341 CHERRY ST SPRINGDALE, OH 45246" +59100120068,"11547 ISLANDALE DR FOREST PARK, OH 45240" +60000140255,"12001 STILLWIND DR SYCAMORE TOWNSHIP, OH 45249" +24500010135,"8423 DIXIE AV CINCINNATI, OH 45216" +9400050062,"150 MULBERRY ST CINCINNATI, OH 45202" +12100020081,"5649 LAWNDALE PL CINCINNATI, OH 45212" +55002430015,"6595 BRIDGETOWN RD GREEN TOWNSHIP, OH 45248" +59004120073,"1696 CLAYBURN CIR SPRINGFIELD TOWNSHIP, OH 45240" +21100680080,"3114 MANNING AV CINCINNATI, OH 45211" +61200800338,"9500 RAVEN LN BLUE ASH, OH 45242" +59001801015,"9620 PEPPER CIR SPRINGFIELD TOWNSHIP, OH 45231" +55000610256,"4331 BOUDINOT AV GREEN TOWNSHIP, OH 45211" \ No newline at end of file From f80611821bf4303767f94434070a1043bbdd5aeb Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Sat, 13 Dec 2025 16:37:13 -0800 Subject: [PATCH 21/22] user postgis containers instead of compiling from scratch --- uwpostgis-foundry/Dockerfile | 87 ++++++++++++------------------------ 1 file changed, 29 insertions(+), 58 deletions(-) diff --git a/uwpostgis-foundry/Dockerfile b/uwpostgis-foundry/Dockerfile index 2252dc1..5e6c5b8 100644 --- a/uwpostgis-foundry/Dockerfile +++ b/uwpostgis-foundry/Dockerfile @@ -1,10 +1,16 @@ # https://stackoverflow.com/questions/34751814/build-postgres-docker-container-with-initial-schema # Initialize the database during the build # PART 1 -FROM postgis/postgis:17-3.5-alpine AS buildtime_init_builder +FROM postgis/postgis:17-3.5-alpine AS db_creator +# copy statics +RUN mkdir -m 777 /gisdata +COPY ./src/alpinedb/download_national.sh . +COPY ./src/alpinedb/download_state.sh . +COPY ./src/alpinedb/1-load_data.sh /docker-entrypoint-initdb.d/ + +# national data first # build vars -ARG state_var ARG TIGER_DOMAIN ARG POSTGRES_DB=geocoder ARG POSTGRES_USER=clad_svc @@ -12,59 +18,34 @@ ARG POSTGRES_PASSWORD=not_on_gitlab ARG GEOCODER_YEAR=2020 # runtime vars -ENV STATES=${state_var} ENV TIGER_DOMAIN=${TIGER_DOMAIN} ENV PGDATA=/pgdata SHELL ["/bin/bash", "-c"] - RUN apk add libintl postgis unzip wget -# Make data dir -RUN mkdir -m 777 /gisdata -COPY ./src/alpinedb/download_national.sh . -COPY ./src/alpinedb/download_state.sh . -COPY ./src/alpinedb/1-load_data.sh /docker-entrypoint-initdb.d/ RUN echo "docker_temp_server_stop && exit 0" > /docker-entrypoint-initdb.d/900-exit_before_boot.sh - -# download national data first RUN ./download_national.sh -# force build to not use cache from a previous state(s) build: -ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache -# download state data in separate layer + +# state data second +# ARG forces build to not use cache from a previous state(s) build: +ARG state_var +ENV STATES=${state_var} RUN ./download_state.sh RUN bash -x docker-entrypoint.sh postgres # PART 2 -## copy database from initialization layer -FROM alpine:latest AS buildtime_init -RUN apk upgrade --no-cache -## run apk to install key tools -RUN apk add bash libcurl postgis py3-pip py3-psycopg unzip vim wget -RUN python3 -m pip install --upgrade --break-system-packages pip -RUN python3 -m pip freeze | sed 's/==/>=/g' | xargs python3 -m pip install --upgrade --break-system-packages +FROM postgis/postgis:17-3.5-alpine -## image rebuild from scratch -CMD ["/bin/sh"] -#RUN /bin/sh -c set -eux; addgroup -g 70 -S postgres; adduser -u 70 -S -D -G postgres -H -h /var/lib/postgresql -s /bin/sh postgres; install --verbose --directory --owner postgres --group postgres --mode 1777 /var/lib/postgresql # buildkit -ENV LANG=en_US.utf8 -ENV PG_MAJOR=16 -ENV PG_VERSION=16.9 -ENV PG_SHA256=07c00fb824df0a0c295f249f44691b86e3266753b380c96f633c3311e10bd005 -ENV POSTGIS_VERSION=3.5.2 -ENV POSTGIS_SHA256=71d8f5d06debec0bdd94525a5a1d42417ed2a1f65d0070a1a3e3e381b606ee35 -ENV PGDATA=/pgdata_init +# copy statics +RUN mkdir -m 700 /pgdata +ENV PGDATA=/pgdata -RUN /bin/sh -c set -eux; wget -O postgresql.tar.bz2 "https://ftp.postgresql.org/pub/source/v$PG_VERSION/postgresql-$PG_VERSION.tar.bz2"; echo "$PG_SHA256 *postgresql.tar.bz2" | sha256sum -c -; mkdir -p /usr/src/postgresql; tar --extract --file postgresql.tar.bz2 --directory /usr/src/postgresql --strip-components 1 ; rm postgresql.tar.bz2; apk add --no-cache --virtual .build-deps llvm15-dev clang15 bison coreutils dpkg-dev dpkg flex g++ gcc krb5-dev libc-dev libedit-dev libxml2-dev libxslt-dev linux-headers make openldap-dev openssl-dev perl-dev perl-ipc-run perl-utils python3-dev tcl-dev util-linux-dev zlib-dev icu-dev lz4-dev zstd-dev ; cd /usr/src/postgresql; awk '$1 == "#define" && $2 == "DEFAULT_PGSOCKET_DIR" && $3 == "\"/tmp\"" { $3 = "\"/var/run/postgresql\""; print; next } { print }' src/include/pg_config_manual.h > src/include/pg_config_manual.h.new; grep '/var/run/postgresql' src/include/pg_config_manual.h.new; mv src/include/pg_config_manual.h.new src/include/pg_config_manual.h; gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; wget -O config/config.guess 'https://git.savannah.gnu.org/cgit/config.git/plain/config.guess?id=7d3d27baf8107b630586c962c057e22149653deb'; wget -O config/config.sub 'https://git.savannah.gnu.org/cgit/config.git/plain/config.sub?id=7d3d27baf8107b630586c962c057e22149653deb'; export LLVM_CONFIG="/usr/lib/llvm15/bin/llvm-config"; export CLANG=clang-15; ./configure --enable-option-checking=fatal --build="$gnuArch" --enable-integer-datetimes --enable-thread-safety --enable-tap-tests --disable-rpath --with-uuid=e2fs --with-pgport=5432 --with-system-tzdata=/usr/share/zoneinfo --prefix=/usr/local --with-includes=/usr/local/include --with-libraries=/usr/local/lib --with-gssapi --with-ldap --with-tcl --with-perl --with-python --with-openssl --with-libxml --with-libxslt --with-icu --with-llvm --with-lz4 --with-zstd ; make -j "$(nproc)" world-bin; make install-world-bin; make -C contrib install; runDeps="$( scanelf --needed --nobanner --format '%n#p' --recursive /usr/local | tr ',' '\n' | sort -u | awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' | grep -v -e perl -e python -e tcl )"; apk add --no-cache --virtual .postgresql-rundeps $runDeps bash tzdata zstd icu-data-full $([ "$(apk --print-arch)" != 'ppc64le' ] && echo 'nss_wrapper') ; apk del --no-network .build-deps; cd /; rm -rf /usr/src/postgresql /usr/local/share/doc /usr/local/share/man ; postgres --version # buildkit -RUN /bin/sh -c set -eux; cp -v /usr/local/share/postgresql/postgresql.conf.sample /usr/local/share/postgresql/postgresql.conf.sample.orig; sed -ri "s!^#?(listen_addresses)\s*=\s*\S+.*!\1 = '*'!" /usr/local/share/postgresql/postgresql.conf.sample; grep -F "listen_addresses = '*'" /usr/local/share/postgresql/postgresql.conf.sample # buildkit -RUN mkdir -m 777 /var/run/postgresql -RUN chown -R postgres:postgres /var/run/postgresql # buildkit -RUN mkdir -m 777 /pgdata_init -RUN chown -R postgres:postgres /pgdata_init # buildkit -CMD ["postgres"] -RUN /bin/sh -c set -eux && apk add --no-cache --virtual .fetch-deps ca-certificates openssl tar && wget -O postgis.tar.gz "https://github.com/postgis/postgis/archive/${POSTGIS_VERSION}.tar.gz" && echo "${POSTGIS_SHA256} *postgis.tar.gz" | sha256sum -c - && mkdir -p /usr/src/postgis && tar --extract --file postgis.tar.gz --directory /usr/src/postgis --strip-components 1 && rm postgis.tar.gz && apk add --no-cache --virtual .build-deps gdal-dev geos-dev proj-dev proj-util sfcgal-dev llvm15-dev clang15 autoconf automake cunit-dev file g++ gcc gettext-dev git json-c-dev libtool libxml2-dev make pcre2-dev perl protobuf-c-dev && cd /usr/src/postgis && gettextize && ./autogen.sh && ./configure --enable-lto && make -j$(nproc) && make install && projsync --system-directory --file ch_swisstopo_CHENyx06_ETRS && projsync --system-directory --file us_noaa_eshpgn && projsync --system-directory --file us_noaa_prvi && projsync --system-directory --file us_noaa_wmhpgn && mkdir /tempdb && chown -R postgres:postgres /tempdb && su postgres -c 'pg_ctl -D /tempdb init' && su postgres -c 'pg_ctl -D /tempdb -c -l /tmp/logfile -o '-F' start ' && cd regress && make -j$(nproc) check RUNTESTFLAGS=--extension PGUSER=postgres && su postgres -c 'psql -c "CREATE EXTENSION IF NOT EXISTS postgis;"' && su postgres -c 'psql -c "CREATE EXTENSION IF NOT EXISTS postgis_raster;"' && su postgres -c 'psql -c "CREATE EXTENSION IF NOT EXISTS postgis_sfcgal;"' && su postgres -c 'psql -c "CREATE EXTENSION IF NOT EXISTS fuzzystrmatch; --needed for postgis_tiger_geocoder "' && su postgres -c 'psql -c "CREATE EXTENSION IF NOT EXISTS address_standardizer;"' && su postgres -c 'psql -c "CREATE EXTENSION IF NOT EXISTS address_standardizer_data_us;"' && su postgres -c 'psql -c "CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder;"' && su postgres -c 'psql -c "CREATE EXTENSION IF NOT EXISTS postgis_topology;"' && su postgres -c 'psql -t -c "SELECT version();"' >> /_pgis_full_version.txt && su postgres -c 'psql -t -c "SELECT PostGIS_Full_Version();"' >> /_pgis_full_version.txt && su postgres -c 'psql -t -c "\dx"' >> /_pgis_full_version.txt && su postgres -c 'pg_ctl -D /tempdb --mode=immediate stop' && rm -rf /tempdb && rm -rf /tmp/logfile && rm -rf /tmp/pgis_reg && apk add --no-cache --virtual .postgis-rundeps gdal geos proj sfcgal json-c libstdc++ pcre2 protobuf-c ca-certificates && cd / && rm -rf /usr/src/postgis && apk del .fetch-deps .build-deps && cat /_pgis_full_version.txt # buildkit -## image rebuild end +RUN apk upgrade --no-cache +RUN apk add py3-pip +RUN python3 -m pip install --break-system-packages --no-cache-dir --upgrade pip +RUN python3 -m pip freeze | sed 's/==/>=/g' | xargs python3 -m pip install --break-system-packages --no-cache-dir --upgrade SHELL ["/bin/bash", "-c"] @@ -74,25 +55,15 @@ RUN mkdir -p /opt/palantir/sidecars/shared-volumes/shared/ RUN chown 5001 /opt/palantir/sidecars/shared-volumes/shared/ ENV SHARED_DIR=/opt/palantir/sidecars/shared-volumes/shared -RUN python3 -m venv foundry_venv -COPY requirements.txt ./ -RUN /foundry_venv/bin/pip install --upgrade pip -RUN /foundry_venv/bin/pip install --no-cache-dir -r requirements.txt -RUN /foundry_venv/bin/pip freeze | sed 's/--.*//' | xargs /foundry_venv/bin/pip install --upgrade +COPY requirements.txt / +COPY --chmod=0755 process_csv.py / +COPY --chmod=0755 entrypoint.py /usr/bin/entrypoint +RUN pip install --break-system-packages --no-cache-dir -r /requirements.txt -ADD entrypoint.py /usr/bin/entrypoint -RUN chmod +x /usr/bin/entrypoint - -COPY process_csv.py ./ +# copy database from db_creator +COPY --chmod=700 --from=db_creator /pgdata /pgdata +RUN chown -R 5001:5001 /tmp /var/run/postgresql /pgdata WORKDIR /opt/palantir/sidecars/shared-volumes/shared/ - -RUN chown -R 5001:5001 /tmp /foundry_venv /var/run/postgresql - -RUN mkdir -m 700 /pgdata -RUN chown 5001:5001 /pgdata -COPY --chown=5001:5001 --chmod=700 --from=buildtime_init_builder /pgdata /pgdata -ENV PGDATA=/pgdata - USER 5001 -ENTRYPOINT entrypoint -c "/foundry_venv/bin/python3 /process_csv.py" +ENTRYPOINT entrypoint -c "/process_csv.py" From d51ff760cbe0e60d0d4e9df3f2cf3c0c259661b9 Mon Sep 17 00:00:00 2001 From: Miles Raymond Date: Sat, 13 Dec 2025 20:01:38 -0800 Subject: [PATCH 22/22] remove gosu with CVEs --- uwpostgis-foundry/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uwpostgis-foundry/Dockerfile b/uwpostgis-foundry/Dockerfile index 5e6c5b8..5e5a210 100644 --- a/uwpostgis-foundry/Dockerfile +++ b/uwpostgis-foundry/Dockerfile @@ -37,6 +37,8 @@ RUN bash -x docker-entrypoint.sh postgres # PART 2 FROM postgis/postgis:17-3.5-alpine +# not used, old, and has CVEs +RUN rm -f /usr/local/bin/gosu # copy statics RUN mkdir -m 700 /pgdata