Merge pull request #640 from NREL/develop

Update to OS SDK 3.3.0-rc2

.github/workflows/openstudio-server-tests.yml

@@ -10,9 +10,9 @@ on: [push, pull_request]
 
 env:
   USE_TESTING_TIMEOUTS: "true"
-  OPENSTUDIO_VERSION: 3.2.1 
-  OPENSTUDIO_VERSION_SHA: bdbdbc9da6
-  OPENSTUDIO_VERSION_EXT: ""
+  OPENSTUDIO_VERSION: 3.3.0 
+  OPENSTUDIO_VERSION_SHA: 45b36b8d4c
+  OPENSTUDIO_VERSION_EXT: "-rc2"
   DOCKER_COMPOSE_VERSION: 1.21.1
   BUNDLE_WITHOUT: native_ext
 
@@ -96,7 +96,7 @@ jobs:
     - name: docker
       shell: bash 
       run: |
-          export OPENSTUDIO_TAG=3.2.1
+          export OPENSTUDIO_TAG=3.3.0-rc2
           sed -i -E "s/.git//g" .dockerignore
           docker volume create --name=osdata
           docker images --all
@@ -118,12 +118,24 @@ jobs:
       if: ${{ failure() }}  
       shell: bash 
       run: ./ci/github-actions/print_logs.sh
+    - name: Run docker vulnerability scanner
+      uses: aquasecurity/trivy-action@master
+      with:
+        # the image should have been built and tagged in the previous steps
+        image-ref: 'nrel/openstudio-server:latest'
+        format: 'template'
+        template: '@/contrib/sarif.tpl'
+        output: 'trivy-results.sarif'
+        severity: 'CRITICAL,HIGH'
+    - name: Upload container scan results to GitHub Security tab
+      uses: github/codeql-action/upload-sarif@v1
+      with:
+        sarif_file: 'trivy-results.sarif'
     - name: docker-upload
       if: | 
         github.ref == 'refs/heads/master' || 
-        github.ref == 'refs/heads/develop' || 
-        github.ref == 'refs/heads/setup_github_actions' 
-      shell: bash 
+        github.ref == 'refs/heads/develop'
+      shell: bash
       run: ./docker/deployment/scripts/deploy_docker_github_actions.sh
       env: 
         CI: true

.ruby-version

@@ -1 +1 @@
-2.5.1
+2.7.2

Dockerfile

@@ -4,7 +4,7 @@
 # NOTES:            Currently this is one big dockerfile and non-optimal.
 
 #may include suffix
-ARG OPENSTUDIO_VERSION=3.2.1
+ARG OPENSTUDIO_VERSION=3.3.0-rc2
 FROM nrel/openstudio:$OPENSTUDIO_VERSION as base
 MAINTAINER Nicholas Long [email protected]
 
@@ -158,7 +158,6 @@ RUN echo "Running in testing environment - Installing Firefox and Gecko Driver"
 
 COPY /docker/server/run-server-tests.sh /usr/local/bin/run-server-tests
 RUN chmod +x /usr/local/bin/run-server-tests
-
 # Test adding the git repo to the container for coveralls
 # The #TEST# will be removed in the travis test script to be run in the test container
 #TEST#COPY .git /opt/openstudio/.git

appveyor.yml

@@ -4,9 +4,9 @@ image: Visual Studio 2019
 
 environment:
   USE_TESTING_TIMEOUTS: "true"
-  OPENSTUDIO_VERSION: 3.2.1
-  OPENSTUDIO_VERSION_SHA: bdbdbc9da6
-  OPENSTUDIO_VERSION_EXT: ""
+  OPENSTUDIO_VERSION: 3.3.0
+  OPENSTUDIO_VERSION_SHA: 45b36b8d4c
+  OPENSTUDIO_VERSION_EXT: "-rc2"
   OPENSTUDIO_TEST_EXE: C:\projects\openstudio\bin\openstudio.exe
   BUILD_TYPE: "test"
   SKIP_COVERALLS: "true"

ci/appveyor/setup.cmd

@@ -7,7 +7,7 @@ REM install develop build
 set OS_INSTALL_NAME=OpenStudio-%OPENSTUDIO_VERSION%%OPENSTUDIO_VERSION_EXT%%%2B%OPENSTUDIO_VERSION_SHA%-Windows.exe
 echo Install name is %OS_INSTALL_NAME%
 REM curl -SLO --insecure https://openstudio-builds.s3.amazonaws.com/%OPENSTUDIO_VERSION%/%OS_INSTALL_NAME%
-curl -SLO --insecure https://openstudio-builds.s3.amazonaws.com/3.2.1/%OS_INSTALL_NAME%
+curl -SLO --insecure https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/3.3.0-rc2/%OS_INSTALL_NAME%
 dir .
 REM Install OpenStudio
 %OS_INSTALL_NAME% --script ci/appveyor/install-windows.qs

ci/github-actions/install_openstudio.sh

@@ -10,13 +10,13 @@ if [ ! -z ${OPENSTUDIO_VERSION} ] && [ ! -z ${OPENSTUDIO_SHA} ]; then
     # OPENSTUDIO_VERSION_EXT may be empty
     OPENSTUDIO_DOWNLOAD_FILENAME=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}%2B${OPENSTUDIO_SHA}-Ubuntu-18.04.deb
     echo "Installing OpenStudio ${OPENSTUDIO_DOWNLOAD_FILENAME}"
-    OPENSTUDIO_DOWNLOAD_BASE_URL=https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/3.2.1-rc/
+    OPENSTUDIO_DOWNLOAD_BASE_URL=https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/3.3.0-rc2
     OPENSTUDIO_DOWNLOAD_URL=$OPENSTUDIO_DOWNLOAD_BASE_URL/$OPENSTUDIO_DOWNLOAD_FILENAME
 
     # copying this from the docker-openstudio dockerfile
     apt-get update && apt-get install -y curl vim gdebi-core libgmp-dev libffi-dev build-essential zlib1g-dev vim git locales sudo
     #export OPENSTUDIO_DOWNLOAD_URL=https://openstudio-builds.s3.amazonaws.com/${OPENSTUDIO_VERSION}/$OPENSTUDIO_DOWNLOAD_FILENAME
-    export OPENSTUDIO_DOWNLOAD_URL=https://openstudio-builds.s3.amazonaws.com/3.2.1/$OPENSTUDIO_DOWNLOAD_FILENAME
+    export OPENSTUDIO_DOWNLOAD_URL=https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/3.3.0-rc2/$OPENSTUDIO_DOWNLOAD_FILENAME
 
     echo "OpenStudio Package Download URL is ${OPENSTUDIO_DOWNLOAD_URL}"
     curl -SLO $OPENSTUDIO_DOWNLOAD_URL

ci/github-actions/setup.sh

@@ -37,7 +37,7 @@ else
         export OS_NAME=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}%2B${OPENSTUDIO_VERSION_SHA}-Darwin
         export OS_NAME_WITH_PLUS=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}+${OPENSTUDIO_VERSION_SHA}-Darwin
         #curl -SL --insecure https://openstudio-builds.s3.amazonaws.com/${OPENSTUDIO_VERSION}/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz
-        curl -SL --insecure https://openstudio-builds.s3.amazonaws.com/3.2.1/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz
+        curl -SL --insecure  https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/3.3.0-rc2/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz
         # OSX downloads with %2B but installs with + sign. These are the encoded chars in url strings.
         #hdiutil attach ${OS_NAME}.dmg
         #sed -i -e "s|REPLACEME|$HOME/openstudio|" ci/github-actions/install-mac.qs

docker/R/lib/sobol.R

@@ -155,6 +155,7 @@ for (i in 1:ncol(vars)){
 }
 if(!all(boundary_check)){
   print('SOLUTION SPACE OUT OF BOUNDS, CHECK Grid Jump and Level Values and/or re-run')
+  print(paste("boundary_check:",boundary_check))
   stop(options("show.error.messages"=TRUE),"SOLUTION SPACE OUT OF BOUNDS, CHECK Grid Jump and Level Values and/or re-run")
 }
 print("bounds are satisfied, continuing...")

local_setup_scripts/docker-compose.yml

@@ -42,6 +42,7 @@ services:
       - REDIS_URL=redis://:openstudio@queue:6379
       - MONGO_USER=openstudio
       - MONGO_PASSWORD=openstudio
+      - SECRET_KEY_BASE=c4ab6d293e4bf52ee92e8dda6e16dc9b5448d0c5f7908ee40c66736d515f3c29142d905b283d73e5e9cef6b13cd8e38be6fd3b5e25d00f35b259923a86c7c473
     volumes:
       - osdata:/mnt/openstudio
     depends_on:
@@ -63,6 +64,7 @@ services:
       - REDIS_URL=redis://:openstudio@queue:6379
       - MONGO_USER=openstudio
       - MONGO_PASSWORD=openstudio
+      - SECRET_KEY_BASE=c4ab6d293e4bf52ee92e8dda6e16dc9b5448d0c5f7908ee40c66736d515f3c29142d905b283d73e5e9cef6b13cd8e38be6fd3b5e25d00f35b259923a86c7c473
     volumes:
       - osdata:/mnt/openstudio
     depends_on:
@@ -86,6 +88,7 @@ services:
       - REDIS_URL=redis://:openstudio@queue:6379
       - MONGO_USER=openstudio
       - MONGO_PASSWORD=openstudio
+      - SECRET_KEY_BASE=c4ab6d293e4bf52ee92e8dda6e16dc9b5448d0c5f7908ee40c66736d515f3c29142d905b283d73e5e9cef6b13cd8e38be6fd3b5e25d00f35b259923a86c7c473
     volumes:
       - /mnt/openstudio
     depends_on:
@@ -120,6 +123,7 @@ services:
       - REDIS_URL=redis://:openstudio@queue:6379
       - MONGO_USER=openstudio
       - MONGO_PASSWORD=openstudio
+      - SECRET_KEY_BASE=c4ab6d293e4bf52ee92e8dda6e16dc9b5448d0c5f7908ee40c66736d515f3c29142d905b283d73e5e9cef6b13cd8e38be6fd3b5e25d00f35b259923a86c7c473
 volumes:
   osdata:
     external: true

local_setup_scripts/rebuild_sr.sh

@@ -5,8 +5,8 @@ while [ $(docker ps -q | wc -l) != 1 ]; do sleep 5; done
 sleep 5
 docker volume rm -f osdata || true
 docker volume rm -f dbdata || true
-#docker image rm 127.0.0.1:5000/openstudio-server -f
-docker build . -t="127.0.0.1:5000/openstudio-server" --build-arg OPENSTUDIO_VERSION=3.1.0
+docker image rm 127.0.0.1:5000/openstudio-server -f
+docker build . -t="127.0.0.1:5000/openstudio-server" --build-arg OPENSTUDIO_VERSION=3.2.0
 docker push 127.0.0.1:5000/openstudio-server
 cd docker/R/
 #docker image rm 127.0.0.1:5000/openstudio-rserve -f

local_setup_scripts/win64/docker-compose.yml

@@ -36,6 +36,7 @@ services:
       - REDIS_URL=redis://:openstudio@queue:6379
       - MONGO_USER=openstudio
       - MONGO_PASSWORD=openstudio
+      - SECRET_KEY_BASE=c4ab6d293e4bf52ee92e8dda6e16dc9b5448d0c5f7908ee40c66736d515f3c29142d905b283d73e5e9cef6b13cd8e38be6fd3b5e25d00f35b259923a86c7c473
     volumes:
       - osdata:/mnt/openstudio
     depends_on:
@@ -54,6 +55,7 @@ services:
       - REDIS_URL=redis://:openstudio@queue:6379
       - MONGO_USER=openstudio
       - MONGO_PASSWORD=openstudio
+      - SECRET_KEY_BASE=c4ab6d293e4bf52ee92e8dda6e16dc9b5448d0c5f7908ee40c66736d515f3c29142d905b283d73e5e9cef6b13cd8e38be6fd3b5e25d00f35b259923a86c7c473
     volumes:
       - osdata:/mnt/openstudio
     depends_on:
@@ -74,6 +76,7 @@ services:
       - REDIS_URL=redis://:openstudio@queue:6379
       - MONGO_USER=openstudio
       - MONGO_PASSWORD=openstudio
+      - SECRET_KEY_BASE=c4ab6d293e4bf52ee92e8dda6e16dc9b5448d0c5f7908ee40c66736d515f3c29142d905b283d73e5e9cef6b13cd8e38be6fd3b5e25d00f35b259923a86c7c473
     volumes:
       - /mnt/openstudio
     depends_on:
@@ -102,6 +105,7 @@ services:
       - REDIS_URL=redis://:openstudio@queue:6379
       - MONGO_USER=openstudio
       - MONGO_PASSWORD=openstudio
+      - SECRET_KEY_BASE=c4ab6d293e4bf52ee92e8dda6e16dc9b5448d0c5f7908ee40c66736d515f3c29142d905b283d73e5e9cef6b13cd8e38be6fd3b5e25d00f35b259923a86c7c473
 volumes:
   osdata:
     external: true

local_setup_scripts/win64/rebuild_sr.sh

@@ -6,7 +6,7 @@ sleep 5
 docker volume rm -f osdata || true
 docker volume rm -f dbdata || true
 docker image rm 127.0.0.1:5000/openstudio-server -f
-docker build . -t="127.0.0.1:5000/openstudio-server" --build-arg OPENSTUDIO_VERSION=3.1.0
+docker build . -t="127.0.0.1:5000/openstudio-server" --build-arg OPENSTUDIO_VERSION=3.2.0
 docker push 127.0.0.1:5000/openstudio-server
 cd docker/R
 #docker image rm 127.0.0.1:5000/openstudio-rserve -f

local_setup_scripts/win64/rebuild_sr_no_rm.sh

@@ -6,7 +6,7 @@ sleep 5
 docker volume rm -f osdata || true
 docker volume rm -f dbdata || true
 #docker image rm 127.0.0.1:5000/openstudio-server -f
-docker build . -t="127.0.0.1:5000/openstudio-server" --build-arg OPENSTUDIO_VERSION=3.1.0
+docker build . -t="127.0.0.1:5000/openstudio-server" --build-arg OPENSTUDIO_VERSION=3.2.0
 docker push 127.0.0.1:5000/openstudio-server
 cd docker/R
 #docker image rm 127.0.0.1:5000/openstudio-rserve -f

server/Gemfile

@@ -67,19 +67,16 @@ gem 'sassc', '~> 2.4.0'
 ## Commonly update gems for testing and development
 
 gem 'openstudio-workflow', '= 2.2.0'
-
 #gem 'openstudio-analysis', :github => 'NREL/OpenStudio-analysis-gem', :ref => '270'
 gem 'openstudio-analysis', '= 1.2.0'
 
 gem 'urbanopt-cli', '= 0.6.0'
-gem 'urbanopt-reporting', '= 0.4.0'
-
+gem 'urbanopt-reopt', '= 0.6.0'
 
 ## End commonly updated gems
 
 gem 'openstudio-aws'
 
-
 # libxml 3.2.0 failing on windows
 gem 'bson', '~> 4.12.0' # bson 4.6.0 requires ruby >= 2.3.0
 #gem 'libxml-ruby'
@@ -93,9 +90,9 @@ gem 'rubyXL', '~> 3.4.17'
 
 # linux based js runtime libraries
 if RUBY_PLATFORM =~ /linux/
-  gem 'execjs'
-  gem 'libv8'
-  gem 'therubyracer'
+  gem 'execjs', '=2.7.0'
+  gem 'libv8', '=3.16.14.19'
+  gem 'therubyracer', '=0.12.3'
 elsif RUBY_PLATFORM =~ /darwin/
   gem 'execjs', github: 'NREL/execjs'
 end
@@ -119,7 +116,7 @@ group :development, :test do
   gem 'capybara', '~> 3.35.3'
   gem 'coveralls', '0.7.1', require: false
   gem 'public_suffix', '~> 4.0.6'
-  gem 'puma', '~> 5.2.2' # used by capybara
+  gem 'puma', '~> 5.4.0' # used by capybara
   gem 'rspec', '~> 3.10.0'
   gem 'rspec-rails', '~> 5.0.1'
   gem 'rspec-retry', '~> 0.6.2'

server/app/jobs/dj_jobs/run_simulate_data_point.rb

@@ -254,14 +254,20 @@ def perform
             @sim_logger.error "Workflow #{osw_path} failed with error #{e}"
             run_result = :errored
           ensure
-            if uo_simulation_log
+            if (!uo_simulation_log.nil? && File.exist?(uo_simulation_log))
               @sim_logger.info "UrbanOpt simulation output: #{File.read(uo_simulation_log)}"
+            else
+              @sim_logger.warn "UrbanOpt simulation output: #{uo_simulation_log} does not exist"            
             end
-            if uo_process_log
+            if (!uo_process_log.nil? && File.exist?(uo_process_log))
               @sim_logger.info "UrbanOpt process output: #{File.read(uo_process_log)}"
+            else
+              @sim_logger.warn "UrbanOpt process output: #{uo_process_log} does not exist"
             end
-            if process_log
+            if (!process_log.nil? && File.exist?(process_log))
               @sim_logger.info "Oscli output: #{File.read(process_log)}"
+            else
+              @sim_logger.warn "OSCLI output: #{process_log} does not exist"
             end
             #docker_log = File.join(APP_CONFIG['rails_log_path'], 'docker.log')
             #if File.exist? docker_log
@@ -324,7 +330,7 @@ def perform
           end
           if @data_point.analysis.download_osm
             @sim_logger.info 'downloading in.OSM'
-            report_file = "#{simulation_dir}/in.osm"
+            report_file = "#{run_dir}/in.osm"
             uploads_successful << upload_file(report_file, 'OpenStudio Model', 'model', 'application/osm') if File.exist?(report_file)
           else
             @sim_logger.info "NOT downloading in.OSM since download_osm value is: #{@data_point.analysis.download_osm}"