Kamal2

.dockerignore

@@ -45,3 +45,50 @@
 # Ignore Docker-related files
 /.dockerignore
 /Dockerfile*
+
+# SAPI
+/coverage
+/rdoc
+/private
+/public/system
+/public/downloads/*.pdf
+#LaTeX
+/public/latex/*.aux
+/public/latex/*.out
+/public/latex/*.log
+/public/latex/*.gz
+/public/latex/index.pdf
+/public/latex/history.pdf
+/public/sitemap*
+
+#checklist downloads
+/public/downloads/checklist/*.*
+
+#exports csvs
+/public/downloads/documents/*.csv
+/public/downloads/quotas/*.csv
+/public/downloads/cites_listings/*.csv
+/public/downloads/cites_suspensions/*.csv
+/public/downloads/eu_listings/*.csv
+/public/downloads/eu_decisions/*.csv
+/public/downloads/cms_listings/*.csv
+/public/downloads/checklist/*.pdf
+/public/downloads/checklist/*.csv
+/public/downloads/checklist/*.json
+/public/downloads/taxon_concepts_names/*.csv
+/public/downloads/synonyms_and_trade_names/*.csv
+/public/downloads/taxon_concepts_distributions/*.csv
+/public/downloads/shipments/*.csv
+/public/downloads/comptab/*.csv
+/public/downloads/gross_exports/*.csv
+/public/downloads/gross_imports/*.csv
+/public/downloads/net_exports/*.csv
+/public/downloads/net_imports/*.csv
+/public/downloads/trade_download_stats/*.csv
+/public/downloads/species_reference_output/*.csv
+/public/downloads/standard_reference_output/*.csv
+/public/downloads/common_names/*.csv
+/public/downloads/iucn_mappings/*.csv
+/public/downloads/cms_mappings/*.csv
+/public/downloads/orphaned_taxon_concepts/*.csv
+/public/uploads/*

.kamal/hooks/docker-setup.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Docker set up on $KAMAL_HOSTS..."

.kamal/hooks/post-app-boot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Booted app version $KAMAL_VERSION on $KAMAL_HOSTS..."

.kamal/hooks/post-deploy.sample

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# A sample post-deploy hook
+#
+# These environment variables are available:
+# KAMAL_RECORDED_AT
+# KAMAL_PERFORMER
+# KAMAL_VERSION
+# KAMAL_HOSTS
+# KAMAL_ROLES (if set)
+# KAMAL_DESTINATION (if set)
+# KAMAL_RUNTIME
+
+echo "$KAMAL_PERFORMER deployed $KAMAL_VERSION to $KAMAL_DESTINATION in $KAMAL_RUNTIME seconds"

.kamal/hooks/post-proxy-reboot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Rebooted kamal-proxy on $KAMAL_HOSTS"

.kamal/hooks/pre-app-boot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Booting app version $KAMAL_VERSION on $KAMAL_HOSTS..."

.kamal/hooks/pre-build

@@ -0,0 +1,51 @@
+#!/bin/sh
+
+# A sample pre-build hook
+#
+# Checks:
+# 1. We have a clean checkout
+# 2. A remote is configured
+# 3. The branch has been pushed to the remote
+# 4. The version we are deploying matches the remote
+#
+# These environment variables are available:
+# KAMAL_RECORDED_AT
+# KAMAL_PERFORMER
+# KAMAL_VERSION
+# KAMAL_HOSTS
+# KAMAL_ROLES (if set)
+# KAMAL_DESTINATION (if set)
+
+if [ -n "$(git status --porcelain)" ]; then
+  echo "Git checkout is not clean, aborting..." >&2
+  git status --porcelain >&2
+  exit 1
+fi
+
+first_remote=$(git remote)
+
+if [ -z "$first_remote" ]; then
+  echo "No git remote set, aborting..." >&2
+  exit 1
+fi
+
+current_branch=$(git branch --show-current)
+
+if [ -z "$current_branch" ]; then
+  echo "Not on a git branch, aborting..." >&2
+  exit 1
+fi
+
+remote_head=$(git ls-remote $first_remote --tags $current_branch | cut -f1)
+
+if [ -z "$remote_head" ]; then
+  echo "Branch not pushed to remote, aborting..." >&2
+  exit 1
+fi
+
+if [ "$KAMAL_VERSION" != "$remote_head" ]; then
+  echo "Version ($KAMAL_VERSION) does not match remote HEAD ($remote_head), aborting..." >&2
+  exit 1
+fi
+
+exit 0

.kamal/hooks/pre-connect.sample

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+# A sample pre-connect check
+#
+# Warms DNS before connecting to hosts in parallel
+#
+# These environment variables are available:
+# KAMAL_RECORDED_AT
+# KAMAL_PERFORMER
+# KAMAL_VERSION
+# KAMAL_HOSTS
+# KAMAL_ROLES (if set)
+# KAMAL_DESTINATION (if set)
+# KAMAL_RUNTIME
+
+hosts = ENV["KAMAL_HOSTS"].split(",")
+results = nil
+max = 3
+
+elapsed = Benchmark.realtime do
+  results = hosts.map do |host|
+    Thread.new do
+      tries = 1
+
+      begin
+        Socket.getaddrinfo(host, 0, Socket::AF_UNSPEC, Socket::SOCK_STREAM, nil, Socket::AI_CANONNAME)
+      rescue SocketError
+        if tries < max
+          puts "Retrying DNS warmup: #{host}"
+          tries += 1
+          sleep rand
+          retry
+        else
+          puts "DNS warmup failed: #{host}"
+          host
+        end
+      end
+
+      tries
+    end
+  end.map(&:value)
+end
+
+retries = results.sum - hosts.size
+nopes = results.count { |r| r == max }
+
+puts "Prewarmed %d DNS lookups in %.2f sec: %d retries, %d failures" % [ hosts.size, elapsed, retries, nopes ]

.kamal/hooks/pre-deploy.sample

@@ -0,0 +1,122 @@
+#!/usr/bin/env ruby
+
+# A sample pre-deploy hook
+#
+# Checks the Github status of the build, waiting for a pending build to complete for up to 720 seconds.
+#
+# Fails unless the combined status is "success"
+#
+# These environment variables are available:
+# KAMAL_RECORDED_AT
+# KAMAL_PERFORMER
+# KAMAL_VERSION
+# KAMAL_HOSTS
+# KAMAL_COMMAND
+# KAMAL_SUBCOMMAND
+# KAMAL_ROLES (if set)
+# KAMAL_DESTINATION (if set)
+
+# Only check the build status for production deployments
+if ENV["KAMAL_COMMAND"] == "rollback" || ENV["KAMAL_DESTINATION"] != "production"
+  exit 0
+end
+
+require "bundler/inline"
+
+# true = install gems so this is fast on repeat invocations
+gemfile(true, quiet: true) do
+  source "https://rubygems.org"
+
+  gem "octokit"
+  gem "faraday-retry"
+end
+
+MAX_ATTEMPTS = 72
+ATTEMPTS_GAP = 10
+
+def exit_with_error(message)
+  $stderr.puts message
+  exit 1
+end
+
+class GithubStatusChecks
+  attr_reader :remote_url, :git_sha, :github_client, :combined_status
+
+  def initialize
+    @remote_url = github_repo_from_remote_url
+    @git_sha = `git rev-parse HEAD`.strip
+    @github_client = Octokit::Client.new(access_token: ENV["GITHUB_TOKEN"])
+    refresh!
+  end
+
+  def refresh!
+    @combined_status = github_client.combined_status(remote_url, git_sha)
+  end
+
+  def state
+    combined_status[:state]
+  end
+
+  def first_status_url
+    first_status = combined_status[:statuses].find { |status| status[:state] == state }
+    first_status && first_status[:target_url]
+  end
+
+  def complete_count
+    combined_status[:statuses].count { |status| status[:state] != "pending"}
+  end
+
+  def total_count
+    combined_status[:statuses].count
+  end
+
+  def current_status
+    if total_count > 0
+      "Completed #{complete_count}/#{total_count} checks, see #{first_status_url} ..."
+    else
+      "Build not started..."
+    end
+  end
+
+  private
+    def github_repo_from_remote_url
+      url = `git config --get remote.origin.url`.strip.delete_suffix(".git")
+      if url.start_with?("https://github.com/")
+        url.delete_prefix("https://github.com/")
+      elsif url.start_with?("git@github.com:")
+        url.delete_prefix("git@github.com:")
+      else
+        url
+      end
+    end
+end
+
+
+$stdout.sync = true
+
+begin
+  puts "Checking build status..."
+
+  attempts = 0
+  checks = GithubStatusChecks.new
+
+  loop do
+    case checks.state
+    when "success"
+      puts "Checks passed, see #{checks.first_status_url}"
+      exit 0
+    when "failure"
+      exit_with_error "Checks failed, see #{checks.first_status_url}"
+    when "pending"
+      attempts += 1
+    end
+
+    exit_with_error "Checks are still pending, gave up after #{MAX_ATTEMPTS * ATTEMPTS_GAP} seconds" if attempts == MAX_ATTEMPTS
+
+    puts checks.current_status
+    sleep(ATTEMPTS_GAP)
+    checks.refresh!
+  end
+rescue Octokit::NotFound
+  exit_with_error "Build status could not be found"
+end

.kamal/hooks/pre-proxy-reboot.sample

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "Rebooting kamal-proxy on $KAMAL_HOSTS..."

.kamal/secrets-common

@@ -0,0 +1,39 @@
+# Minimal Secrets Template - Backend Rails API Kamal Deployment
+# Secrets defined here are available for reference under registry/password, env/secret, builder/secrets,
+# and accessories/*/env/secret in config/deploy.yml. All secrets should be pulled from either
+# password manager, ENV, or a file. DO NOT ENTER RAW CREDENTIALS HERE! This file needs to be safe for git.
+
+# Registry Configuration (ALWAYS REQUIRED)
+KAMAL_REGISTRY_USERNAME=$KAMAL_REGISTRY_USERNAME
+KAMAL_REGISTRY_PASSWORD=$KAMAL_REGISTRY_PASSWORD
+
+# Rails Configuration (REQUIRED)
+RAILS_MASTER_KEY=$RAILS_MASTER_KEY
+
+# Database Configuration (special, 2 database)
+SAPI_DATABASE_HOST=$SAPI_DATABASE_HOST
+SAPI_DATABASE_NAME=$SAPI_DATABASE_NAME
+SAPI_DATABASE_USERNAME=$SAPI_DATABASE_USERNAME
+SAPI_DATABASE_PASSWORD=$SAPI_DATABASE_PASSWORD
+SAPI_DATABASE_PORT=$SAPI_DATABASE_PORT
+
+CAPTIVE_BREEDING_DATABASE_HOST=$CAPTIVE_BREEDING_DATABASE_HOST
+CAPTIVE_BREEDING_DATABASE_NAME=$CAPTIVE_BREEDING_DATABASE_NAME
+CAPTIVE_BREEDING_DATABASE_USERNAME=$CAPTIVE_BREEDING_DATABASE_USERNAME
+CAPTIVE_BREEDING_DATABASE_PASSWORD=$CAPTIVE_BREEDING_DATABASE_PASSWORD
+CAPTIVE_BREEDING_DATABASE_PORT=$CAPTIVE_BREEDING_DATABASE_PORT
+
+# Mail Configuration
+MAIL_USERNAME=$MAIL_USERNAME
+MAIL_PASSWORD=$MAIL_PASSWORD
+
+# AWS Configuration
+AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
+AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
+AWS_REGION=$AWS_REGION
+
+# Redis for Sidekiq
+SAPI_SIDEKIQ_REDIS_URL=$SAPI_SIDEKIQ_REDIS_URL
+
+# Redis for cache
+SAPI_SIDEKIQ_REDIS_CACHE_URL=$SAPI_SIDEKIQ_REDIS_CACHE_URL

.node-version

@@ -0,0 +1 @@
+18.20.8

Dockerfile

@@ -4,21 +4,40 @@
 # or SUS-ORS project.
 
 # Dockerfile
-FROM ruby:3.2.5
+FROM ruby:3.2.5-slim
 
 # Rails and SAPI has some additional dependencies, e.g. rake requires a JS
 # runtime, so attempt to get these from apt, where possible
 # socat is just for binding ports within docker, not needed for the application
-RUN apt-get update && apt-get install -y --force-yes \
+RUN apt-get update && apt-get install --no-install-recommends -y --force-yes \
+  # ?
   libsodium-dev libgmp3-dev libssl-dev \
+  # Editor
+  vim nano \
+  # PSQL
   libpq-dev postgresql-client \
-  nodejs \
+  # node js
+  curl xz-utils \
+  # For minio, local s3, development only.
   socat \
+  # latex (huge file size)
   texlive-latex-base texlive-fonts-recommended texlive-fonts-extra texlive-latex-extra \
-  ;
+  # Clean up
+  && rm -rf /var/lib/apt/lists/*
 # NB: Postgres client from Debian is 9.4 - not sure if this is acceptable
 
-RUN mkdir /SAPI
+# Install Node.js 18.20.8 manually
+ARG NODE_VERSION=18.20.8
+ARG TARGETARCH
+# Map Docker TARGETARCH to Node.js archive name
+RUN case "$TARGETARCH" in \
+  amd64) NODE_ARCH=x64 ;; \
+  arm64) NODE_ARCH=arm64 ;; \
+  *) echo "Unsupported architecture: $TARGETARCH"; exit 1 ;; \
+  esac && \
+  curl -fsSL https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.xz \
+  | tar -xJ -C /usr/local --strip-components=1
+
 WORKDIR /SAPI
 
 #