get Custom gems working again

.github/workflows/openstudio-server-tests.yml

@@ -10,8 +10,8 @@ on: [push, pull_request]
 
 env:
   USE_TESTING_TIMEOUTS: "true"
-  OPENSTUDIO_VERSION: 3.9.0
-  OPENSTUDIO_VERSION_SHA: c77fbb9569
+  OPENSTUDIO_VERSION: 3.10.0
+  OPENSTUDIO_VERSION_SHA: 86d7e215a1
   OPENSTUDIO_VERSION_EXT: ""
   DOCKER_COMPOSE_VERSION: 1.21.1
   BUNDLE_WITHOUT: native_ext
@@ -105,7 +105,7 @@ jobs:
     - name: docker
       shell: bash 
       run: |
-          export OPENSTUDIO_TAG=3.9.0
+          export OPENSTUDIO_TAG=3.10.0
           sed -i -E "s/.git//g" .dockerignore
           docker volume create --name=osdata
           docker images --all
@@ -114,8 +114,8 @@ jobs:
           docker-compose -f docker-compose.test.yml pull
           docker-compose -f docker-compose.test.yml build --build-arg OPENSTUDIO_VERSION=$OPENSTUDIO_TAG
           docker-compose -f docker-compose.test.yml up -d
-          # set SKIP_URBANOPT_ALGO=true to skip UrbanOpt algo tests
-          docker-compose exec -e SKIP_URBANOPT_ALGO=false -T web /usr/local/bin/run-server-tests
+          #set SKIP_URBANOPT_ALGO=true to skip UrbanOpt algo tests
+          docker-compose exec -e SKIP_URBANOPT_ALGO=true -T web /usr/local/bin/run-server-tests
           docker-compose stop
           git checkout -- .dockerignore && git checkout -- Dockerfile
       env: 
@@ -130,8 +130,8 @@ jobs:
     - name: docker-upload
       if: | 
         github.ref == 'refs/heads/master' || 
-        github.ref == 'refs/heads/develop'
-#        github.ref == 'refs/heads/arm-hack'
+        github.ref == 'refs/heads/develop' ||
+        github.ref == 'refs/heads/custom_gems'
       shell: bash
       run: ./docker/deployment/scripts/deploy_docker_github_actions.sh
       env: 

.github/workflows/security-scan.yml

@@ -10,8 +10,8 @@ on: [push, pull_request]
 
 env:
   USE_TESTING_TIMEOUTS: "true"
-  OPENSTUDIO_VERSION: 3.9.0
-  OPENSTUDIO_VERSION_SHA: c77fbb9569
+  OPENSTUDIO_VERSION: 3.10.0
+  OPENSTUDIO_VERSION_SHA: 86d7e215a1
   OPENSTUDIO_VERSION_EXT: ""
   DOCKER_COMPOSE_VERSION: 1.21.1
   BUNDLE_WITHOUT: native_ext
@@ -31,7 +31,7 @@ jobs:
     - name: docker
       shell: bash 
       run: |
-          export OPENSTUDIO_TAG=3.9.0
+          export OPENSTUDIO_TAG=3.10.0
           sed -i -E "s/.git//g" .dockerignore
           docker volume create --name=osdata
           docker images --all

Dockerfile

@@ -4,8 +4,8 @@
 # NOTES:            Currently this is one big dockerfile and non-optimal.
 
 #may include suffix
-ARG OPENSTUDIO_VERSION=3.9.0
-FROM nrel/openstudio:3.9.0 AS base
+ARG OPENSTUDIO_VERSION=3.10.0
+FROM nrel/openstudio:3.10.0 AS base
 ARG OPENSTUDIO_VERSION
 
 ENV DEBIAN_FRONTEND=noninteractive

Gemfile

@@ -2,10 +2,10 @@ source 'http://rubygems.org'
 ruby '~>3.2.0'
 
 gem 'colored', '~> 1.2'
-gem 'git', '~> 1.12.0'
+gem 'git', '~> 1.13.2'
 gem 'rake', '~> 13.2.1'
 gem 'ffi', '1.17.1'
-gem 'addressable', '2.8.1'
+gem 'addressable', '~> 2.8'
 group :development, :test do
   gem 'rest-client', '~> 2.1.0'
   gem 'rspec', '~> 3.13'

appveyor.yml

@@ -4,8 +4,8 @@ image: Visual Studio 2019
 
 environment:
   USE_TESTING_TIMEOUTS: "true"
-  OPENSTUDIO_VERSION: 3.9.0
-  OPENSTUDIO_VERSION_SHA: c77fbb9569
+  OPENSTUDIO_VERSION: 3.10.0
+  OPENSTUDIO_VERSION_SHA: 86d7e215a1
   OPENSTUDIO_VERSION_EXT: ""
   OPENSTUDIO_TEST_EXE: C:\projects\openstudio\bin\openstudio.exe
   BUILD_TYPE: "test"

ci/appveyor/setup.cmd

@@ -12,12 +12,42 @@ set OS_INSTALL_NAME=OpenStudio-%OPENSTUDIO_VERSION%%OPENSTUDIO_VERSION_EXT%+%OPE
 echo Install name is %OS_INSTALL_NAME%
 
 REM Download and Install OpenStudio
-curl -SLO --insecure https://github.com/NREL/OpenStudio/releases/download/v%OPENSTUDIO_VERSION%%OPENSTUDIO_VERSION_EXT%/%OS_INSTALL_NAME%
+curl -fSLO --insecure https://github.com/NREL/OpenStudio/releases/download/v%OPENSTUDIO_VERSION%%OPENSTUDIO_VERSION_EXT%/%OS_INSTALL_NAME%
+if %ERRORLEVEL% neq 0 (
+  echo ERROR: Failed to download "%OS_INSTALL_NAME%" from "https://github.com/NREL/OpenStudio/releases/download/v%OPENSTUDIO_VERSION%%OPENSTUDIO_VERSION_EXT%/%OS_INSTALL_NAME%"
+  exit /b 1
+)
 dir .
 
+REM  “Unblock” the file so Windows does not refuse to execute it
+powershell -Command "Unblock-File -Path '%OS_INSTALL_NAME%'"
+
 REM Execute the OpenStudio installer
-%OS_INSTALL_NAME% --script ci/appveyor/install-windows.qs
-move C:\openstudio C:\projects\openstudio
+REM %OS_INSTALL_NAME% --script ci/appveyor/install-windows.qs
+REM  3) Run the OpenStudio installer in “quiet” mode, pointing to our QScript
+echo Launching installer…
+REM  Use “.\” to ensure we’re running the downloaded EXE in the current directory
+start "" /wait "%OS_INSTALL_NAME%" --script ci/appveyor/install-windows.qs
+if %ERRORLEVEL% neq 0 (
+  echo.
+  echo ERROR: OpenStudio installer "%OS_INSTALL_NAME%" returned error code %ERRORLEVEL%. Aborting.
+  exit /b 1
+)
+
+REM move C:\openstudio C:\projects\openstudio
+REM  4) Move the default “C:\openstudio” install directory into the projects dir
+if exist C:\openstudio (
+  move /Y C:\openstudio C:\projects\openstudio
+  if %ERRORLEVEL% neq 0 (
+    echo.
+    echo ERROR: Could not move “C:\openstudio” to “C:\projects\openstudio”. Check permissions.
+    exit /b 1
+  )
+) else (
+  echo.
+  echo ERROR: After running the installer, “C:\openstudio” was not found. Aborting.
+  exit /b 1
+)
 dir C:\projects\openstudio
 
 REM Cleanup installer
@@ -26,6 +56,11 @@ del %OS_INSTALL_NAME%
 REM Show Ruby version and OpenStudio version
 ruby -v
 openstudio openstudio_version
+if %ERRORLEVEL% neq 0 (
+  echo.
+  echo ERROR: “openstudio openstudio_version” failed. Perhaps OpenStudio wasn’t installed correctly?
+  exit /b 1
+)
 
 REM Install essential Ruby gems needed for the environment setup
 echo Installing essential gems...

ci/github-actions/install_openstudio.sh

@@ -1,10 +1,10 @@
 #!/usr/bin/env bash
-
+set -euo pipefail
 # This script assumes you are running as a superuser.
 
 OPENSTUDIO_VERSION=$1
 OPENSTUDIO_SHA=$2
-OPENSTUDIO_VERSION_EXT=$3
+OPENSTUDIO_VERSION_EXT="${3:-}"
 
 if [ ! -z ${OPENSTUDIO_VERSION} ] && [ ! -z ${OPENSTUDIO_SHA} ]; then
     # OPENSTUDIO_VERSION_EXT may be empty
@@ -14,13 +14,19 @@ if [ ! -z ${OPENSTUDIO_VERSION} ] && [ ! -z ${OPENSTUDIO_SHA} ]; then
     #OPENSTUDIO_DOWNLOAD_BASE_URL=https://github.com/NREL/OpenStudio/releases/download/v3.8.0
     OPENSTUDIO_DOWNLOAD_BASE_URL=https://github.com/NREL/OpenStudio/releases/download/v${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}
     OPENSTUDIO_DOWNLOAD_URL=$OPENSTUDIO_DOWNLOAD_BASE_URL/$OPENSTUDIO_DOWNLOAD_FILENAME
-
+    echo "Installing OpenStudio ${OPENSTUDIO_DOWNLOAD_FILENAME}"
+    echo "→ Downloading OpenStudio from: ${OPENSTUDIO_DOWNLOAD_URL}"
     # copying this from the docker-openstudio dockerfile
     apt-get update && apt-get install -y curl vim gdebi-core libgmp-dev libffi-dev build-essential zlib1g-dev vim git locales sudo
     export OPENSTUDIO_DOWNLOAD_URL
 
     echo "OpenStudio Package Download URL is ${OPENSTUDIO_DOWNLOAD_URL}"
-    curl -SLO $OPENSTUDIO_DOWNLOAD_URL
+    # explicit -f to fail on 404/500, -s for silent, -S to show errors, -L to follow redirects
+    if ! curl -fsSL -o "${OPENSTUDIO_DOWNLOAD_FILENAME}" "${OPENSTUDIO_DOWNLOAD_URL}"; then
+      echo "ERROR: Failed to download '${OPENSTUDIO_DOWNLOAD_FILENAME}'"
+      echo "  from: ${OPENSTUDIO_DOWNLOAD_URL}"
+      exit 1
+    fi
     # Verify that the download was successful (not access denied XML from s3)
     grep -v -q "<Code>AccessDenied</Code>" ${OPENSTUDIO_DOWNLOAD_FILENAME}
     gdebi -n $OPENSTUDIO_DOWNLOAD_FILENAME

ci/github-actions/setup.sh

@@ -48,7 +48,15 @@ else
         export OS_NAME_WITH_PLUS=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}+${OPENSTUDIO_VERSION_SHA}-Darwin-x86_64
         #curl -SL --insecure https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/develop/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz
         #curl -SL --insecure https://github.com/NREL/OpenStudio/releases/download/v3.8.0/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz
-        curl -SL --insecure https://github.com/NREL/OpenStudio/releases/download/v${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz
+        #curl -SL --insecure https://github.com/NREL/OpenStudio/releases/download/v${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz
+        URL="https://github.com/NREL/OpenStudio/releases/download/v${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}/${OS_NAME}.tar.gz"
+        FILENAME="${OS_NAME_WITH_PLUS}.tar.gz"
+
+        echo "→ Downloading OpenStudio tarball from: ${URL}"
+        if ! curl -fsSL --insecure "${URL}" -o "${FILENAME}"; then
+          echo "ERROR: Failed to download '${FILENAME}' from '${URL}'" >&2
+          exit 1
+        fi
         # OSX downloads with %2B but installs with + sign. These are the encoded chars in url strings.
         #hdiutil attach ${OS_NAME}.dmg
         #sed -i -e "s|REPLACEME|$HOME/openstudio|" ci/github-actions/install-mac.qs

docker/deployment/scripts/deploy_docker_github_actions.sh

@@ -14,8 +14,8 @@ elif [ "${GITHUB_REF}" == "refs/heads/master" ]; then
 # Currently setting this to setup_github_actions to test upload. 
 elif [ "${GITHUB_REF}" == "refs/heads/setup_github_actions" ]; then
     IMAGETAG=experimental
-elif [ "${GITHUB_REF}" == "refs/heads/3.9.0" ]; then
-     IMAGETAG="3.9.0"
+elif [ "${GITHUB_REF}" == "refs/heads/custom_gems" ]; then
+     IMAGETAG="3.10.0"
 fi
 
 if [ "${IMAGETAG}" != "skip" ]; then

docker/server/run-server-tests.sh

@@ -21,7 +21,7 @@ done
 #cd /opt/openstudio/server && bundle exec rspec; (( exit_status = exit_status || $? ))
 # Run only the algorithm specs. The other features/*_spec files should probably disappear and capybara/gecko
 # can be removed.
-#cd /opt/openstudio/server && bundle exec rspec spec/features/docker_stack_custom_gems.rb; (( exit_status = exit_status || $? ))
+cd /opt/openstudio/server && bundle exec rspec spec/features/docker_stack_custom_gems.rb; (( exit_status = exit_status || $? ))
 cd /opt/openstudio/server && bundle exec rspec spec/features/docker_stack_test_apis_spec.rb; (( exit_status = exit_status || $? ))
 cd /opt/openstudio/server && bundle exec rspec spec/features/docker_stack_algo_spec.rb; (( exit_status = exit_status || $? ))
 cd /opt/openstudio/server && bundle exec rspec spec/features/docker_stack_requeue_spec.rb; (( exit_status = exit_status || $? ))

local_setup_scripts/win64/rebuild_sr_no_rm.sh

@@ -6,7 +6,7 @@ sleep 5
 docker volume rm -f osdata || true
 docker volume rm -f dbdata || true
 #docker image rm 127.0.0.1:5000/openstudio-server -f
-docker build . -t="127.0.0.1:5000/openstudio-server" --build-arg OPENSTUDIO_VERSION=3.9.0
+docker build . -t="127.0.0.1:5000/openstudio-server" --build-arg OPENSTUDIO_VERSION=3.10.0
 docker push 127.0.0.1:5000/openstudio-server
 cd docker/R
 #docker image rm 127.0.0.1:5000/openstudio-rserve -f

server/Gemfile

@@ -73,7 +73,7 @@ gem 'mail', '= 2.7.1'
 # these 4 gems are needed until mail > 2.8.0 can be used
 gem 'net-smtp', '~> 0.3.4'
 gem 'net-pop', '~> 0.1.2'
-gem 'net-imap', '~> 0.3.9'
+gem 'net-imap', '~> 0.4.10'
 gem 'date', '3.3.3'
 # don't try to install sassc 2.
 gem 'roo', '~> 2.8.3'
@@ -84,13 +84,13 @@ gem 'roo', '~> 2.8.3'
 gem 'openstudio-workflow', '= 2.4.0'
 #gem 'openstudio-workflow', :github => 'NREL/OpenStudio-workflow-gem', :ref => 'ruby_322'
 #gem 'openstudio-analysis', :github => 'NREL/OpenStudio-analysis-gem', :ref => 'ruby_322'
-gem 'openstudio-analysis', '= 1.4.0'
+gem 'openstudio-analysis', '= 1.5.2'
 #gem 'bcl', :github => 'NREL/bcl-gem', :ref => 'develop'
 #gem 'openstudio_measure_tester', :github => 'NREL/OpenStudio-measure-tester-gem', :ref => 'develop'
 
 # Remove urbanopt cli gems for minor for releases as they use different versions of ext gems
 gem 'json-schema', '~> 4.3.1'
-gem 'urbanopt-cli', '= 1.0.1'
+#gem 'urbanopt-cli', '= 1.0.1'
 #gem 'urbanopt-reopt', '= 0.9.0'
 
 ## End commonly updated gems

server/app/jobs/dj_jobs/run_simulate_data_point.rb

@@ -695,17 +695,26 @@ def run_bundle_gems
 
       log_path = "#{analysis_dir}/bundle.log"
       # set ENVs to nil
-      oscli_env_unset = Hash[Utility::Oss::ENV_VARS_TO_UNSET_FOR_OSCLI.collect{|x| [x,nil]}]
+      #oscli_env_unset = Hash[Utility::Oss::ENV_VARS_TO_UNSET_FOR_OSCLI.collect{|x| [x,nil]}]
+      install_env = {
+          "BUNDLE_GEMFILE" => "#{analysis_dir}/Gemfile",
+          "BUNDLE_PATH"    => "#{analysis_dir}/gems",
+          "RUBYOPT"        => nil,
+          "BUNDLER_SETUP"  => nil
+        }
       @sim_logger.info "Bundle config setup"
       create_config_file
       @sim_logger.info "Bundle config command complete"
-      @sim_logger.info "oscli_env_unset: #{oscli_env_unset}"      
-      cmd = "pwd && printenv && GEM_HOME=#{analysis_dir}/gems bundle install"
+
+      @sim_logger.info "install_env: #{install_env}"      
+      cmd = "bundle _#{Bundler::VERSION}_ install --gemfile=#{analysis_dir}/Gemfile --path=#{analysis_dir}/gems"
       @sim_logger.info "Bundle install command: #{cmd}"
-      pid = Process.spawn(oscli_env_unset, cmd, [:err, :out] => [log_path, 'w'])
+      pid = Process.spawn(install_env, cmd, [:err, :out] => [log_path, 'w'], chdir: analysis_dir)
+      #pid = Process.spawn(oscli_env_unset, cmd, [:err, :out] => [log_path, 'w'], chdir: analysis_dir)
       Process.wait pid
       @sim_logger.info "gem installation complete"
       @sim_logger.info "bundle.log output: #{File.read(log_path).lines}"
+
       @sim_logger.info "replace Bundle config w orig"      
       replace_config_file
       @sim_logger.info "replace Bundle config w orig complete" 

server/app/lib/openstudio_server/version.rb

@@ -4,9 +4,9 @@
 # *******************************************************************************
 
 module OpenstudioServer
-  Version = '3.9.0'.freeze
+  Version = '3.10.0'.freeze
   # format should be ^.*\-{1}[a-z]+[0-9]+
   # for example: -rc1, -beta6, -customusecase0
   VERSION_EXT = ''.freeze # with preceding - or +
-  OS_SHA = 'c77fbb9569'.freeze
+  OS_SHA = '86d7e215a1'.freeze
 end

server/app/lib/utility/oss.rb

@@ -12,7 +12,8 @@ class Oss
       'BUNDLER_ORIG_PATH',
       'BUNDLER_ORIG_MANPATH',
       'GEM_PATH',
-      'GEM_HOME'
+      'GEM_HOME',
+      'BUNDLER_SETUP'
       #   'BUNDLE_WITHOUT' # This now needs to be set BUNDLE_WITHOUT=native_ext
     ].freeze
     # return command to run openstudio cli on current platform