5 speed up builds

.github/workflows/fast-build.yml

@@ -0,0 +1,98 @@
+name: Fastbuild postgis images
+
+on:
+  push:
+    branches:
+    - "5-speed-up-builds"
+  pull_request:
+    branches:
+    - "5-speed-up-builds"
+
+concurrency:
+  group: ${{ github.head_ref || github.ref_name }}
+  cancel-in-progress: true
+
+env:
+  # dynamic doesn't work, only static
+  BUILD_NUM: $(date +%Y%m%d%H%M%S)
+  REPOSITORY:
+  TIGER_DOMAIN: clad-github-builder.rit.uw.edu
+
+jobs:
+
+  clear-cache:
+    runs-on: self-hosted
+
+    steps:
+    - name: Clear docker cache
+      run: docker system prune -af --volumes
+    - name: Delete docker volumes not included in system prune
+      run: docker volume prune -af
+      # run: docker buildx prune -a --verbose
+    - name: export BUILD_NUM to file
+      run: date +%Y%m%d%H%M%S > ~/BUILD_NUM
+
+
+
+  build-degauss-foundry:
+    runs-on: self-hosted
+    needs: clear-cache
+
+    steps:
+    - uses: actions/checkout@v5
+    - name: import BUILD_NUM from file
+      run: echo "BUILD_NUM=$(cat ~/BUILD_NUM)" >> $GITHUB_ENV
+    - name: print env
+      run: env | sort
+    - name: Build the degauss-foundry image
+      run: docker build --load --cpu-quota $(( $(nproc --all)*100000 )) degauss-foundry --platform linux/amd64 --tag genoa-container-registry.washington.palantircloud.com/degauss-foundry:${{ env.BUILD_NUM }}
+    - name: Run Trivy CVE vulnerability scanner
+      uses: aquasecurity/trivy-action@0.33.1
+      with:
+        image-ref: genoa-container-registry.washington.palantircloud.com/degauss-foundry:${{ env.BUILD_NUM }}
+        format: 'table'
+        exit-code: '1'
+        ignore-unfixed: true
+        timeout: '1h'
+        vuln-type: 'os,library'
+        severity: 'CRITICAL,HIGH,MEDIUM'
+
+  setup-postgis-states:
+    runs-on: self-hosted
+    needs: clear-cache
+    outputs:
+      matrix: ${{ steps.matrix.outputs.value }}
+
+    steps:
+    - id: matrix
+      run: |
+        echo 'value=["ak", "al", "ar", "az", "ca", "co", "ct", "dc", "de", "fl", "ga", "hi", "ia", "id", "il", "in", "ks", "ky", "la", "ma", "md", "me", "mi", "mn", "mo", "ms", "mt", "nc", "nd", "ne", "nh", "nj", "nm", "nv", "ny", "oh", "ok", "or", "pa", "ri", "sc", "sd", "tn", "tx", "ut", "va", "vt", "wa", "wi", "wv", "wy", "pr"]' >> $GITHUB_OUTPUT
+    - run: |
+        echo "${{ steps.matrix.outputs.value }}"
+
+  build-postgis-states:
+    runs-on: self-hosted
+    needs: setup-postgis-states
+    continue-on-error: true
+    strategy:
+      matrix:
+        value: ${{fromJSON(needs.setup-postgis-states.outputs.matrix)}}
+
+    steps:
+    - uses: actions/checkout@v5
+    - name: import BUILD_NUM from file
+      run: echo "BUILD_NUM=$(cat ~/BUILD_NUM)" >> $GITHUB_ENV
+    - name: print env
+      run: env | sort
+    - name: Build the postgis image
+      run: docker build --shm-size 4g --load --cpu-quota $(( $(nproc --all)*100000 )) --build-arg TIGER_DOMAIN=clad-github-builder.rit.uw.edu --build-arg state_var=${{ matrix.value }} uwpostgis-foundry --platform linux/amd64 --tag genoa-container-registry.washington.palantircloud.com/postgis-${{ matrix.value }}:${{ env.BUILD_NUM }}
+    - name: Run Trivy CVE vulnerability scanner
+      uses: aquasecurity/trivy-action@0.33.1
+      with:
+        image-ref: genoa-container-registry.washington.palantircloud.com/postgis-${{ matrix.value }}:${{ env.BUILD_NUM }}
+        format: 'table'
+        exit-code: '1'
+        ignore-unfixed: true
+        timeout: '1h'
+        vuln-type: 'os,library'
+        severity: 'CRITICAL,HIGH,MEDIUM'

degauss-foundry/Dockerfile

@@ -3,7 +3,7 @@ RUN apk upgrade --no-cache
 
 # DeGAUSS container metadata
 ENV degauss_name="geocoder"
-ENV degauss_version="3.3.0"
+ENV degauss_version="3.4.0"
 ENV degauss_description="geocodes"
 ENV degauss_argument="valid_geocode_score_threshold [default: 0.5]"
 

degauss-foundry/entrypointCSV.R → degauss-foundry/entrypoint.R

@@ -14,9 +14,7 @@ opt <- docopt::docopt(doc)
 if (is.null(opt$score_threshold)) opt$score_threshold <- 0.5
 
 d <- readr::read_csv(opt$filename, show_col_types = FALSE)
-
-## 8/23 - FOR FOUNDRY PROCESS
-# d <- readr::read_csv('/opt/palantir/sidecars/shared-volumes/shared/infile.csv')
+# d <- readr::read_csv('test/my_address_file.csv')
 # d <- readr::read_csv('test/my_address_file_missing.csv')
 
 ## must contain character column called address
@@ -160,4 +158,3 @@ if (opt$score_threshold != "all") {
   )
   knitr::kable(geocode_summary %>% dplyr::select(geocode_result, `n (%)`))
 }
-

degauss-foundry/entrypoint.py

@@ -12,13 +12,11 @@
 args = parser.parse_args()
 the_command = args.command.split(" ")
 
-
 def run_process(exe):
     "Define a function for running commands and capturing stdout line by line"
     p = subprocess.Popen(exe, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     return iter(p.stdout.readline, b"")
 
-
 start_flag_fname = "/opt/palantir/sidecars/shared-volumes/shared/start_flag"
 done_flag_fname = "/opt/palantir/sidecars/shared-volumes/shared/done_flag"
 close_flag_fname = "/opt/palantir/sidecars/shared-volumes/shared/close_flag"

degauss-foundry/lib/geocoder/us.rb

@@ -7,7 +7,7 @@
 # General usage is as follows:
 #
 #  >> require 'geocoder/us'
-#  >> db = Geocoder::US::Database.new("/opt/tiger/geocoder.db")
+#  >> db = Geocoder::US::Database.new("/opt/geocoder.db")
 #  >> p db.geocode("1600 Pennsylvania Av, Washington DC")
 #
 #  [{:pretyp=>"", :street=>"Pennsylvania", :sufdir=>"NW", :zip=>"20502",

degauss-foundry/lib/geocoder/us/database.rb

@@ -153,8 +153,7 @@ def execute_statement (st, *params)
         result = st.execute(*params)
         columns = result.columns.map {|c| c.to_sym}
         result.each {|row|
-           rows << Hash[*(columns.zip(row).flatten)]}
-
+           rows << columns.zip(row).to_h}
       end
       if @debug
         runtime = format("%.3f", Time.now - start)