diff --git a/.github/workflows/openstudio-server-tests.yml b/.github/workflows/openstudio-server-tests.yml index aa43743bd..5e699c055 100644 --- a/.github/workflows/openstudio-server-tests.yml +++ b/.github/workflows/openstudio-server-tests.yml @@ -10,9 +10,9 @@ on: [push, pull_request] env: USE_TESTING_TIMEOUTS: "true" - OPENSTUDIO_VERSION: 3.2.1 - OPENSTUDIO_VERSION_SHA: bdbdbc9da6 - OPENSTUDIO_VERSION_EXT: "" + OPENSTUDIO_VERSION: 3.3.0 + OPENSTUDIO_VERSION_SHA: 45b36b8d4c + OPENSTUDIO_VERSION_EXT: "-rc2" DOCKER_COMPOSE_VERSION: 1.21.1 BUNDLE_WITHOUT: native_ext @@ -96,7 +96,7 @@ jobs: - name: docker shell: bash run: | - export OPENSTUDIO_TAG=3.2.1 + export OPENSTUDIO_TAG=3.3.0-rc2 sed -i -E "s/.git//g" .dockerignore docker volume create --name=osdata docker images --all @@ -118,12 +118,24 @@ jobs: if: ${{ failure() }} shell: bash run: ./ci/github-actions/print_logs.sh + - name: Run docker vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + # the image should have been built and tagged in the previous steps + image-ref: 'nrel/openstudio-server:latest' + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH' + - name: Upload container scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: 'trivy-results.sarif' - name: docker-upload if: | github.ref == 'refs/heads/master' || - github.ref == 'refs/heads/develop' || - github.ref == 'refs/heads/setup_github_actions' - shell: bash + github.ref == 'refs/heads/develop' + shell: bash run: ./docker/deployment/scripts/deploy_docker_github_actions.sh env: CI: true diff --git a/.ruby-version b/.ruby-version index 4fd0fe3cd..37c2961c2 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -2.5.1 \ No newline at end of file +2.7.2 diff --git a/Dockerfile b/Dockerfile index 4e7a0a3ab..1835bb763 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ # NOTES: Currently this is one big dockerfile and non-optimal. #may include suffix -ARG OPENSTUDIO_VERSION=3.2.1 +ARG OPENSTUDIO_VERSION=3.3.0-rc2 FROM nrel/openstudio:$OPENSTUDIO_VERSION as base MAINTAINER Nicholas Long nicholas.long@nrel.gov @@ -158,7 +158,6 @@ RUN echo "Running in testing environment - Installing Firefox and Gecko Driver" COPY /docker/server/run-server-tests.sh /usr/local/bin/run-server-tests RUN chmod +x /usr/local/bin/run-server-tests - # Test adding the git repo to the container for coveralls # The #TEST# will be removed in the travis test script to be run in the test container #TEST#COPY .git /opt/openstudio/.git diff --git a/appveyor.yml b/appveyor.yml index fc3624073..76ac94a4a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,9 +4,9 @@ image: Visual Studio 2019 environment: USE_TESTING_TIMEOUTS: "true" - OPENSTUDIO_VERSION: 3.2.1 - OPENSTUDIO_VERSION_SHA: bdbdbc9da6 - OPENSTUDIO_VERSION_EXT: "" + OPENSTUDIO_VERSION: 3.3.0 + OPENSTUDIO_VERSION_SHA: 45b36b8d4c + OPENSTUDIO_VERSION_EXT: "-rc2" OPENSTUDIO_TEST_EXE: C:\projects\openstudio\bin\openstudio.exe BUILD_TYPE: "test" SKIP_COVERALLS: "true" diff --git a/ci/appveyor/setup.cmd b/ci/appveyor/setup.cmd index 935f0ab11..f3d5b1c4d 100644 --- a/ci/appveyor/setup.cmd +++ b/ci/appveyor/setup.cmd @@ -7,7 +7,7 @@ REM install develop build set OS_INSTALL_NAME=OpenStudio-%OPENSTUDIO_VERSION%%OPENSTUDIO_VERSION_EXT%%%2B%OPENSTUDIO_VERSION_SHA%-Windows.exe echo Install name is %OS_INSTALL_NAME% REM curl -SLO --insecure https://openstudio-builds.s3.amazonaws.com/%OPENSTUDIO_VERSION%/%OS_INSTALL_NAME% -curl -SLO --insecure https://openstudio-builds.s3.amazonaws.com/3.2.1/%OS_INSTALL_NAME% +curl -SLO --insecure https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/3.3.0-rc2/%OS_INSTALL_NAME% dir . REM Install OpenStudio %OS_INSTALL_NAME% --script ci/appveyor/install-windows.qs diff --git a/ci/github-actions/install_openstudio.sh b/ci/github-actions/install_openstudio.sh index b84ec89c0..46fbdf2c0 100755 --- a/ci/github-actions/install_openstudio.sh +++ b/ci/github-actions/install_openstudio.sh @@ -10,13 +10,13 @@ if [ ! -z ${OPENSTUDIO_VERSION} ] && [ ! -z ${OPENSTUDIO_SHA} ]; then # OPENSTUDIO_VERSION_EXT may be empty OPENSTUDIO_DOWNLOAD_FILENAME=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}%2B${OPENSTUDIO_SHA}-Ubuntu-18.04.deb echo "Installing OpenStudio ${OPENSTUDIO_DOWNLOAD_FILENAME}" - OPENSTUDIO_DOWNLOAD_BASE_URL=https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/3.2.1-rc/ + OPENSTUDIO_DOWNLOAD_BASE_URL=https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/3.3.0-rc2 OPENSTUDIO_DOWNLOAD_URL=$OPENSTUDIO_DOWNLOAD_BASE_URL/$OPENSTUDIO_DOWNLOAD_FILENAME # copying this from the docker-openstudio dockerfile apt-get update && apt-get install -y curl vim gdebi-core libgmp-dev libffi-dev build-essential zlib1g-dev vim git locales sudo #export OPENSTUDIO_DOWNLOAD_URL=https://openstudio-builds.s3.amazonaws.com/${OPENSTUDIO_VERSION}/$OPENSTUDIO_DOWNLOAD_FILENAME - export OPENSTUDIO_DOWNLOAD_URL=https://openstudio-builds.s3.amazonaws.com/3.2.1/$OPENSTUDIO_DOWNLOAD_FILENAME + export OPENSTUDIO_DOWNLOAD_URL=https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/3.3.0-rc2/$OPENSTUDIO_DOWNLOAD_FILENAME echo "OpenStudio Package Download URL is ${OPENSTUDIO_DOWNLOAD_URL}" curl -SLO $OPENSTUDIO_DOWNLOAD_URL diff --git a/ci/github-actions/setup.sh b/ci/github-actions/setup.sh index 9945e2a2e..505fc2980 100755 --- a/ci/github-actions/setup.sh +++ b/ci/github-actions/setup.sh @@ -37,7 +37,7 @@ else export OS_NAME=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}%2B${OPENSTUDIO_VERSION_SHA}-Darwin export OS_NAME_WITH_PLUS=OpenStudio-${OPENSTUDIO_VERSION}${OPENSTUDIO_VERSION_EXT}+${OPENSTUDIO_VERSION_SHA}-Darwin #curl -SL --insecure https://openstudio-builds.s3.amazonaws.com/${OPENSTUDIO_VERSION}/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz - curl -SL --insecure https://openstudio-builds.s3.amazonaws.com/3.2.1/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz + curl -SL --insecure https://openstudio-ci-builds.s3-us-west-2.amazonaws.com/3.3.0-rc2/${OS_NAME}.tar.gz -o $OS_NAME_WITH_PLUS.tar.gz # OSX downloads with %2B but installs with + sign. These are the encoded chars in url strings. #hdiutil attach ${OS_NAME}.dmg #sed -i -e "s|REPLACEME|$HOME/openstudio|" ci/github-actions/install-mac.qs diff --git a/docker/R/lib/sobol.R b/docker/R/lib/sobol.R index e03dac787..1958b0ce7 100644 --- a/docker/R/lib/sobol.R +++ b/docker/R/lib/sobol.R @@ -155,6 +155,7 @@ for (i in 1:ncol(vars)){ } if(!all(boundary_check)){ print('SOLUTION SPACE OUT OF BOUNDS, CHECK Grid Jump and Level Values and/or re-run') + print(paste("boundary_check:",boundary_check)) stop(options("show.error.messages"=TRUE),"SOLUTION SPACE OUT OF BOUNDS, CHECK Grid Jump and Level Values and/or re-run") } print("bounds are satisfied, continuing...") diff --git a/local_setup_scripts/docker-compose.yml b/local_setup_scripts/docker-compose.yml index df45f28f1..a2932f249 100644 --- a/local_setup_scripts/docker-compose.yml +++ b/local_setup_scripts/docker-compose.yml @@ -42,6 +42,7 @@ services: - REDIS_URL=redis://:openstudio@queue:6379 - MONGO_USER=openstudio - MONGO_PASSWORD=openstudio + - SECRET_KEY_BASE=c4ab6d293e4bf52ee92e8dda6e16dc9b5448d0c5f7908ee40c66736d515f3c29142d905b283d73e5e9cef6b13cd8e38be6fd3b5e25d00f35b259923a86c7c473 volumes: - osdata:/mnt/openstudio depends_on: @@ -63,6 +64,7 @@ services: - REDIS_URL=redis://:openstudio@queue:6379 - MONGO_USER=openstudio - MONGO_PASSWORD=openstudio + - SECRET_KEY_BASE=c4ab6d293e4bf52ee92e8dda6e16dc9b5448d0c5f7908ee40c66736d515f3c29142d905b283d73e5e9cef6b13cd8e38be6fd3b5e25d00f35b259923a86c7c473 volumes: - osdata:/mnt/openstudio depends_on: @@ -86,6 +88,7 @@ services: - REDIS_URL=redis://:openstudio@queue:6379 - MONGO_USER=openstudio - MONGO_PASSWORD=openstudio + - SECRET_KEY_BASE=c4ab6d293e4bf52ee92e8dda6e16dc9b5448d0c5f7908ee40c66736d515f3c29142d905b283d73e5e9cef6b13cd8e38be6fd3b5e25d00f35b259923a86c7c473 volumes: - /mnt/openstudio depends_on: @@ -120,6 +123,7 @@ services: - REDIS_URL=redis://:openstudio@queue:6379 - MONGO_USER=openstudio - MONGO_PASSWORD=openstudio + - SECRET_KEY_BASE=c4ab6d293e4bf52ee92e8dda6e16dc9b5448d0c5f7908ee40c66736d515f3c29142d905b283d73e5e9cef6b13cd8e38be6fd3b5e25d00f35b259923a86c7c473 volumes: osdata: external: true diff --git a/local_setup_scripts/rebuild_sr.sh b/local_setup_scripts/rebuild_sr.sh index fd8672160..8a0c6ce63 100755 --- a/local_setup_scripts/rebuild_sr.sh +++ b/local_setup_scripts/rebuild_sr.sh @@ -5,8 +5,8 @@ while [ $(docker ps -q | wc -l) != 1 ]; do sleep 5; done sleep 5 docker volume rm -f osdata || true docker volume rm -f dbdata || true -#docker image rm 127.0.0.1:5000/openstudio-server -f -docker build . -t="127.0.0.1:5000/openstudio-server" --build-arg OPENSTUDIO_VERSION=3.1.0 +docker image rm 127.0.0.1:5000/openstudio-server -f +docker build . -t="127.0.0.1:5000/openstudio-server" --build-arg OPENSTUDIO_VERSION=3.2.0 docker push 127.0.0.1:5000/openstudio-server cd docker/R/ #docker image rm 127.0.0.1:5000/openstudio-rserve -f diff --git a/local_setup_scripts/win64/docker-compose.yml b/local_setup_scripts/win64/docker-compose.yml index 964098f84..831ae15f8 100644 --- a/local_setup_scripts/win64/docker-compose.yml +++ b/local_setup_scripts/win64/docker-compose.yml @@ -36,6 +36,7 @@ services: - REDIS_URL=redis://:openstudio@queue:6379 - MONGO_USER=openstudio - MONGO_PASSWORD=openstudio + - SECRET_KEY_BASE=c4ab6d293e4bf52ee92e8dda6e16dc9b5448d0c5f7908ee40c66736d515f3c29142d905b283d73e5e9cef6b13cd8e38be6fd3b5e25d00f35b259923a86c7c473 volumes: - osdata:/mnt/openstudio depends_on: @@ -54,6 +55,7 @@ services: - REDIS_URL=redis://:openstudio@queue:6379 - MONGO_USER=openstudio - MONGO_PASSWORD=openstudio + - SECRET_KEY_BASE=c4ab6d293e4bf52ee92e8dda6e16dc9b5448d0c5f7908ee40c66736d515f3c29142d905b283d73e5e9cef6b13cd8e38be6fd3b5e25d00f35b259923a86c7c473 volumes: - osdata:/mnt/openstudio depends_on: @@ -74,6 +76,7 @@ services: - REDIS_URL=redis://:openstudio@queue:6379 - MONGO_USER=openstudio - MONGO_PASSWORD=openstudio + - SECRET_KEY_BASE=c4ab6d293e4bf52ee92e8dda6e16dc9b5448d0c5f7908ee40c66736d515f3c29142d905b283d73e5e9cef6b13cd8e38be6fd3b5e25d00f35b259923a86c7c473 volumes: - /mnt/openstudio depends_on: @@ -102,6 +105,7 @@ services: - REDIS_URL=redis://:openstudio@queue:6379 - MONGO_USER=openstudio - MONGO_PASSWORD=openstudio + - SECRET_KEY_BASE=c4ab6d293e4bf52ee92e8dda6e16dc9b5448d0c5f7908ee40c66736d515f3c29142d905b283d73e5e9cef6b13cd8e38be6fd3b5e25d00f35b259923a86c7c473 volumes: osdata: external: true diff --git a/local_setup_scripts/win64/rebuild_sr.sh b/local_setup_scripts/win64/rebuild_sr.sh index 2d504e0b0..e26acc558 100644 --- a/local_setup_scripts/win64/rebuild_sr.sh +++ b/local_setup_scripts/win64/rebuild_sr.sh @@ -6,7 +6,7 @@ sleep 5 docker volume rm -f osdata || true docker volume rm -f dbdata || true docker image rm 127.0.0.1:5000/openstudio-server -f -docker build . -t="127.0.0.1:5000/openstudio-server" --build-arg OPENSTUDIO_VERSION=3.1.0 +docker build . -t="127.0.0.1:5000/openstudio-server" --build-arg OPENSTUDIO_VERSION=3.2.0 docker push 127.0.0.1:5000/openstudio-server cd docker/R #docker image rm 127.0.0.1:5000/openstudio-rserve -f diff --git a/local_setup_scripts/win64/rebuild_sr_no_rm.sh b/local_setup_scripts/win64/rebuild_sr_no_rm.sh index d1520ebfa..359382a1d 100644 --- a/local_setup_scripts/win64/rebuild_sr_no_rm.sh +++ b/local_setup_scripts/win64/rebuild_sr_no_rm.sh @@ -6,7 +6,7 @@ sleep 5 docker volume rm -f osdata || true docker volume rm -f dbdata || true #docker image rm 127.0.0.1:5000/openstudio-server -f -docker build . -t="127.0.0.1:5000/openstudio-server" --build-arg OPENSTUDIO_VERSION=3.1.0 +docker build . -t="127.0.0.1:5000/openstudio-server" --build-arg OPENSTUDIO_VERSION=3.2.0 docker push 127.0.0.1:5000/openstudio-server cd docker/R #docker image rm 127.0.0.1:5000/openstudio-rserve -f diff --git a/server/Gemfile b/server/Gemfile index 5901be548..2d82b52d2 100644 --- a/server/Gemfile +++ b/server/Gemfile @@ -67,19 +67,16 @@ gem 'sassc', '~> 2.4.0' ## Commonly update gems for testing and development gem 'openstudio-workflow', '= 2.2.0' - #gem 'openstudio-analysis', :github => 'NREL/OpenStudio-analysis-gem', :ref => '270' gem 'openstudio-analysis', '= 1.2.0' gem 'urbanopt-cli', '= 0.6.0' -gem 'urbanopt-reporting', '= 0.4.0' - +gem 'urbanopt-reopt', '= 0.6.0' ## End commonly updated gems gem 'openstudio-aws' - # libxml 3.2.0 failing on windows gem 'bson', '~> 4.12.0' # bson 4.6.0 requires ruby >= 2.3.0 #gem 'libxml-ruby' @@ -93,9 +90,9 @@ gem 'rubyXL', '~> 3.4.17' # linux based js runtime libraries if RUBY_PLATFORM =~ /linux/ - gem 'execjs' - gem 'libv8' - gem 'therubyracer' + gem 'execjs', '=2.7.0' + gem 'libv8', '=3.16.14.19' + gem 'therubyracer', '=0.12.3' elsif RUBY_PLATFORM =~ /darwin/ gem 'execjs', github: 'NREL/execjs' end @@ -119,7 +116,7 @@ group :development, :test do gem 'capybara', '~> 3.35.3' gem 'coveralls', '0.7.1', require: false gem 'public_suffix', '~> 4.0.6' - gem 'puma', '~> 5.2.2' # used by capybara + gem 'puma', '~> 5.4.0' # used by capybara gem 'rspec', '~> 3.10.0' gem 'rspec-rails', '~> 5.0.1' gem 'rspec-retry', '~> 0.6.2' diff --git a/server/app/jobs/dj_jobs/run_simulate_data_point.rb b/server/app/jobs/dj_jobs/run_simulate_data_point.rb index be0c8f21e..409cee7f6 100644 --- a/server/app/jobs/dj_jobs/run_simulate_data_point.rb +++ b/server/app/jobs/dj_jobs/run_simulate_data_point.rb @@ -254,14 +254,20 @@ def perform @sim_logger.error "Workflow #{osw_path} failed with error #{e}" run_result = :errored ensure - if uo_simulation_log + if (!uo_simulation_log.nil? && File.exist?(uo_simulation_log)) @sim_logger.info "UrbanOpt simulation output: #{File.read(uo_simulation_log)}" + else + @sim_logger.warn "UrbanOpt simulation output: #{uo_simulation_log} does not exist" end - if uo_process_log + if (!uo_process_log.nil? && File.exist?(uo_process_log)) @sim_logger.info "UrbanOpt process output: #{File.read(uo_process_log)}" + else + @sim_logger.warn "UrbanOpt process output: #{uo_process_log} does not exist" end - if process_log + if (!process_log.nil? && File.exist?(process_log)) @sim_logger.info "Oscli output: #{File.read(process_log)}" + else + @sim_logger.warn "OSCLI output: #{process_log} does not exist" end #docker_log = File.join(APP_CONFIG['rails_log_path'], 'docker.log') #if File.exist? docker_log @@ -324,7 +330,7 @@ def perform end if @data_point.analysis.download_osm @sim_logger.info 'downloading in.OSM' - report_file = "#{simulation_dir}/in.osm" + report_file = "#{run_dir}/in.osm" uploads_successful << upload_file(report_file, 'OpenStudio Model', 'model', 'application/osm') if File.exist?(report_file) else @sim_logger.info "NOT downloading in.OSM since download_osm value is: #{@data_point.analysis.download_osm}" diff --git a/server/app/jobs/dj_jobs/urban_opt.rb b/server/app/jobs/dj_jobs/urban_opt.rb index b6dfe7c8f..db680e33a 100644 --- a/server/app/jobs/dj_jobs/urban_opt.rb +++ b/server/app/jobs/dj_jobs/urban_opt.rb @@ -104,7 +104,24 @@ def run_urbanopt(uo_simulation_log, uo_process_log) if !File.exist?(feature_file) raise "feature_file does not exist: #{feature_file}" end + scenario_file_override = "#{simulation_dir}/urbanopt/scenario_file_override.json" scenario_file = "#{simulation_dir}/urbanopt/#{@data_point.analysis.scenario_file}.csv" + #check if scenario_file_override exists + if File.exist?(scenario_file_override) + @sim_logger.info "found scenario_file_override.json file, parsing" + #parse file and look for scenario file + scenario_parse = JSON.parse(File.read(scenario_file_override), symbolize_names: true) + if scenario_parse['scenario_file'.to_sym] + if File.exist?("#{simulation_dir}/urbanopt/#{scenario_parse['scenario_file'.to_sym]}.csv") + @sim_logger.info "replacing scenario_file with #{simulation_dir}/urbanopt/#{scenario_parse['scenario_file'.to_sym]}.csv" + scenario_file = "#{simulation_dir}/urbanopt/#{scenario_parse['scenario_file'.to_sym]}.csv" + @sim_logger.info "updating analysis model @data_point.analysis.scenario_file: #{@data_point.analysis.scenario_file} to #{scenario_parse['scenario_file'.to_sym]}" + @data_point.analysis.scenario_file = scenario_parse['scenario_file'.to_sym] + end + else + raise ":scenario_file does not exist in: #{scenario_file_override}" + end + end if !File.exist?(scenario_file) raise "scenario_file does not exist: #{scenario_file}" end @@ -121,8 +138,61 @@ def run_urbanopt(uo_simulation_log, uo_process_log) raise "UrbanOpt run returned error code #{$?.exitstatus}" end - #run uo-cli process - cmd = "uo process --default --feature #{feature_file} --scenario #{scenario_file}" + #check run_status.json + run_status_json = "#{simulation_dir}/urbanopt/run/#{@data_point.analysis.scenario_file.downcase}/run_status.json" + @sim_logger.info "run_status_json location: #{run_status_json}" + if File.exist?(run_status_json) + run_status_result = JSON.parse(File.read(run_status_json), symbolize_names: true) + if !run_status_result[:results].nil? + if run_status_result[:results].any?{|hash| hash[:status] == 'Failed'} + @sim_logger.error "run_status.json has failures: #{run_status_result}" + raise "run_status.json has failures: #{run_status_result}" + else + @sim_logger.info "run_status_json no Failed: #{run_status_result}" + end + else + @sim_logger.error "run_status.json does not have results: #{run_status_result}" + raise "run_status.json does not have results: #{run_status_result}" + end + else + @sim_logger.error "run_status.json does not exist at: #{run_status_json}" + raise "run_status.json does not exist at: #{run_status_json}" + end + + #run uo-cli process + @sim_logger.info "Run ReOpt is: #{@data_point.analysis.reopt}" + if @data_point.analysis.reopt + #check for API key + #check ENV GEM_DEVELOPER_KEY + if !ENV['GEM_DEVELOPER_KEY'].nil? && !ENV['GEM_DEVELOPER_KEY'].empty? + @sim_logger.info "GEM_DEVELOPER_KEY is not empty" + #check reopt_key.txt file + elsif File.exist?("#{simulation_dir}/urbanopt/reopt_key.json") + @sim_logger.info "reopt_key.json is found" + key_json = JSON.parse(File.read("#{simulation_dir}/urbanopt/reopt_key.json"), symbolize_names: true) + ENV['GEM_DEVELOPER_KEY'] = key_json[:GEM_DEVELOPER_KEY] + if !ENV['GEM_DEVELOPER_KEY'].empty? + @sim_logger.info "GEM_DEVELOPER_KEY is not empty after reading reopt_key.json" + else + @sim_logger.error "GEM_DEVELOPER_KEY is empty after reading reopt_key.json" + raise "GEM_DEVELOPER_KEY is empty after reading reopt_key.json" + end + else + @sim_logger.error "reopt_key.json is NOT found and ENV not set" + raise "reopt_key.txt is NOT found and ENV not set" + end + + reopt_type = @data_point.analysis.reopt_type + @sim_logger.info "reopt_type is: #{reopt_type}" + if ['scenario', 'feature'].include?(reopt_type) + cmd = "uo process --reopt-#{reopt_type} --feature #{feature_file} --scenario #{scenario_file}" + else + @sim_logger.error "reopt_type is not scenario or feature its: #{reopt_type}" + raise "reopt_type is not scenario or feature its: #{reopt_type}" + end + else + cmd = "uo process --default --feature #{feature_file} --scenario #{scenario_file}" + end #uo_process_log = File.join(simulation_dir, 'urbanopt_process.log') @sim_logger.info "Running UrbanOpt workflow using cmd #{cmd} and writing log to #{uo_process_log}" pid = Process.spawn(cmd, [:err, :out] => [uo_process_log, 'w']) @@ -131,7 +201,8 @@ def run_urbanopt(uo_simulation_log, uo_process_log) end if $?.exitstatus != 0 - raise "UrbanOpt process returned error code #{$?.exitstatus}" + #raise "UrbanOpt process returned error code #{$?.exitstatus}" + @sim_logger.error "UrbanOpt process returned error code #{$?.exitstatus}" end #Run OSSCLI --postprocess_only to run reporting measures in UrbanOpt workflow if ReportingMeasure's are present in workflow @sim_logger.info "@data_point.analysis.problem['workflow'].empty?: #{@data_point.analysis.problem['workflow'].empty?}" @@ -173,10 +244,21 @@ def run_urbanopt(uo_simulation_log, uo_process_log) #uo_results[:feature_reports][0][:reporting_periods][0][:natural_gas] # # Save the objective functions + analysis = Analysis.find(@data_point.analysis.id) + objs = analysis.variables.where(objective_function: true) + #if objs + # objs.each do |variable| if @data_point.analysis.output_variables @data_point.analysis.output_variables.each do |variable| + @sim_logger.info "VARIABLE: #{variable}" uo_result = {} report_index = nil + reports_file = variable[:report_file] + @sim_logger.info "reports_file: #{reports_file} :#{variable[:report_file]}" + if reports_file.nil? + @sim_logger.error "reports_file is missing from output_variable: #{variable[:name]}." + raise "reports_file is missing from output_variable: #{variable[:name]}" + end if variable[:objective_function] @sim_logger.info "found variable[:objective_function]: #{variable[:objective_function]}" if variable[:report] == 'feature_reports' @@ -184,7 +266,7 @@ def run_urbanopt(uo_simulation_log, uo_process_log) if variable[:report_id] && variable[:reporting_periods] && variable[:var_name] @sim_logger.info "found variable[:report_id]:#{variable[:report_id]}, variable[:reporting_periods]:#{variable[:reporting_periods]}, variable[:var_name]: #{variable[:var_name]}." #get feature_reports results - uo_results_file = "#{simulation_dir}/urbanopt/run/#{@data_point.analysis.scenario_file}/default_scenario_report.json" + uo_results_file = "#{simulation_dir}/urbanopt/run/#{@data_point.analysis.scenario_file.downcase}/#{reports_file}.json" if File.exist? uo_results_file uo_result = JSON.parse(File.read(uo_results_file), symbolize_names: true) report_index = uo_result[variable[:report].to_sym].index {|h| h[:id] == variable[:report_id].to_s } if uo_result[variable[:report].to_sym] @@ -195,7 +277,9 @@ def run_urbanopt(uo_simulation_log, uo_process_log) if variable[:var_name] == "end_uses" #check end_uses and category exist if variable[:end_use] && variable[:end_use_category] && uo_result[variable[:report].to_sym][report_index][:reporting_periods][variable[:reporting_periods]][variable[:var_name].to_sym].has_key?(variable[:end_use].to_sym) if variable[:end_use] && variable[:end_use_category] && uo_result[variable[:report].to_sym][report_index][:reporting_periods][variable[:reporting_periods]][variable[:var_name].to_sym][variable[:end_use].to_sym].has_key?(variable[:end_use_category].to_sym) - results[variable[:name].split(".")[0]] = { "#{variable[:end_use]}_#{variable[:end_use_category]}".to_sym => uo_result[variable[:report].to_sym][report_index][:reporting_periods][variable[:reporting_periods]][variable[:var_name].to_sym][variable[:end_use].to_sym][variable[:end_use_category].to_sym], "applicable" => true } + results[variable[:name].split(/\./)[0].to_sym] = {} + results[variable[:name].split(/\./)[0].to_sym]["#{variable[:end_use]}_#{variable[:end_use_category]}".to_sym] = uo_result[variable[:report].to_sym][report_index][:reporting_periods][variable[:reporting_periods]][variable[:var_name].to_sym][variable[:end_use].to_sym][variable[:end_use_category].to_sym] + results[variable[:name].split(/\./)[0].to_sym][:applicable] = true else raise "MISSING output variable[:end_use_category]:#{variable[:end_use_category]}, when output variable[:var_name]:#{variable[:var_name]}, output variable[:end_use]:#{variable[:end_use]}" @sim_logger.error "MISSING output variable[:end_use_category]:#{variable[:end_use_category]}, when output variable[:var_name]:#{variable[:var_name]}, output variable[:end_use]:#{variable[:end_use]}" @@ -205,7 +289,9 @@ def run_urbanopt(uo_simulation_log, uo_process_log) @sim_logger.error "MISSING output variable[:end_use]:#{variable[:end_use]}, when output variable[:var_name]:#{variable[:var_name]}" end else #not end_uses - results[variable[:name].split(".")[0]] = { variable[:var_name].to_sym => uo_result[variable[:report].to_sym][report_index][:reporting_periods][variable[:reporting_periods]][variable[:var_name].to_sym], "applicable" => true } + results[variable[:name].split(/\./)[0].to_sym] = {} + results[variable[:name].split(/\./)[0].to_sym][variable[:var_name].to_sym] = uo_result[variable[:report].to_sym][report_index][:reporting_periods][variable[:reporting_periods]][variable[:var_name].to_sym] + results[variable[:name].split(/\./)[0].to_sym][:applicable] = true end else raise "Could not find output variable[:var_name]: #{variable[:var_name]} in reporting period: #{variable[:reporting_periods]}." @@ -220,8 +306,8 @@ def run_urbanopt(uo_simulation_log, uo_process_log) @sim_logger.error "Could not find the index for report id: #{variable[:report_id]} for report: #{variable[:report]}." end else - raise "Could not find results #{uo_results_file}" - @sim_logger.error "Could not find results #{uo_results_file}" + raise "Could not find results file: #{uo_results_file}" + @sim_logger.error "Could not find results file: #{uo_results_file}" end else raise "MISSING output variable[:report_id]:#{variable[:report_id]}, variable[:reporting_periods]:#{variable[:reporting_periods]}, variable[:var_name]: #{variable[:var_name]}." @@ -229,10 +315,10 @@ def run_urbanopt(uo_simulation_log, uo_process_log) end elsif variable[:report] == 'scenario_report' @sim_logger.info "found variable[:report]: #{variable[:report]}" - if variable[:reporting_periods] && variable[:var_name] + if (variable[:reporting_periods] && variable[:var_name]) @sim_logger.info "found variable[:reporting_periods]:#{variable[:reporting_periods]}, variable[:var_name]: #{variable[:var_name]}." - #get feature_reports results - uo_results_file = "#{simulation_dir}/urbanopt/run/#{@data_point.analysis.scenario_file}/default_scenario_report.json" + #get scenario_report results + uo_results_file = "#{simulation_dir}/urbanopt/run/#{@data_point.analysis.scenario_file.downcase}/#{reports_file}.json" if File.exist? uo_results_file uo_result = JSON.parse(File.read(uo_results_file), symbolize_names: true) if !uo_result[variable[:report].to_sym][:reporting_periods][variable[:reporting_periods]].nil? #reporting_periods exist @@ -241,7 +327,10 @@ def run_urbanopt(uo_simulation_log, uo_process_log) if variable[:var_name] == "end_uses" if variable[:end_use] && variable[:end_use_category] && uo_result[variable[:report].to_sym][:reporting_periods][variable[:reporting_periods]][variable[:var_name].to_sym].has_key?(variable[:end_use].to_sym) if variable[:end_use] && variable[:end_use_category] && uo_result[variable[:report].to_sym][:reporting_periods][variable[:reporting_periods]][variable[:var_name].to_sym][variable[:end_use].to_sym].has_key?(variable[:end_use_category].to_sym) - results[variable[:name].split(".")[0]] = { "#{variable[:end_use]}_#{variable[:end_use_category]}".to_sym => uo_result[variable[:report].to_sym][:reporting_periods][variable[:reporting_periods]][variable[:var_name].to_sym][variable[:end_use].to_sym][variable[:end_use_category].to_sym], "applicable" => true } + @sim_logger.info "setting results keys: #{variable[:name].split(/\./)[0]} AND: #{variable[:end_use]}_#{variable[:end_use_category]}" + results[variable[:name].split(/\./)[0].to_sym] = {} + results[variable[:name].split(/\./)[0].to_sym]["#{variable[:end_use]}_#{variable[:end_use_category]}".to_sym] = uo_result[variable[:report].to_sym][:reporting_periods][variable[:reporting_periods]][variable[:var_name].to_sym][variable[:end_use].to_sym][variable[:end_use_category].to_sym] + results[variable[:name].split(/\./)[0].to_sym][:applicable] = true else raise "MISSING output variable[:end_use_category]:#{variable[:end_use_category]}, when output variable[:var_name]:#{variable[:var_name]}, output variable[:end_use]:#{variable[:end_use]}" @sim_logger.error "MISSING output variable[:end_use_category]:#{variable[:end_use_category]}, when output variable[:var_name]:#{variable[:var_name]}, output variable[:end_use]:#{variable[:end_use]}" @@ -250,8 +339,22 @@ def run_urbanopt(uo_simulation_log, uo_process_log) raise "MISSING output variable[:end_use]:#{variable[:end_use]}, when output variable[:var_name]:#{variable[:var_name]}" @sim_logger.error "MISSING output variable[:end_use]:#{variable[:end_use]}, when output variable[:var_name]:#{variable[:var_name]}" end - else #not end_uses - results[variable[:name].split(".")[0]] = { variable[:var_name].to_sym => uo_result[variable[:report].to_sym][:reporting_periods][variable[:reporting_periods]][variable[:var_name].to_sym], "applicable" => true } + #check for comfort_result + elsif variable[:var_name] == "comfort_result" + if variable[:comfort_result_category] && uo_result[variable[:report].to_sym][:reporting_periods][variable[:reporting_periods]][variable[:var_name].to_sym].has_key?(variable[:comfort_result_category].to_sym) + @sim_logger.info "setting comfort_results keys: #{variable[:name].split(/\./)[0]} AND: #{variable[:var_name]}_#{variable[:comfort_result_category]}" + results[variable[:name].split(/\./)[0].to_sym] = {} + results[variable[:name].split(/\./)[0].to_sym]["#{variable[:var_name]}_#{variable[:comfort_result_category]}".to_sym] = uo_result[variable[:report].to_sym][:reporting_periods][variable[:reporting_periods]][variable[:var_name].to_sym][variable[:comfort_result_category].to_sym] + results[variable[:name].split(/\./)[0].to_sym][:applicable] = true + else + raise "MISSING output variable[:comfort_result_category]:#{variable[:comfort_result_category]}, when output variable[:var_name]:#{variable[:var_name]}" + @sim_logger.error "MISSING output variable[:comfort_result_category]:#{variable[:comfort_result_category]}, when output variable[:var_name]:#{variable[:var_name]}" + end + #not end_uses + else + results[variable[:name].split(/\./)[0].to_sym] = {} + results[variable[:name].split(/\./)[0].to_sym][variable[:var_name].to_sym] = uo_result[variable[:report].to_sym][:reporting_periods][variable[:reporting_periods]][variable[:var_name].to_sym] + results[variable[:name].split(/\./)[0].to_sym][:applicable] = true end else raise "Could not find output variable[:var_name]: #{variable[:var_name]} in reporting period: #{variable[:reporting_periods]}." @@ -262,27 +365,111 @@ def run_urbanopt(uo_simulation_log, uo_process_log) @sim_logger.error "Could not find output reporting period: #{variable[:reporting_periods]}." end else - raise "Could not find results #{uo_results_file}" - @sim_logger.error "Could not find results #{uo_results_file}" + raise "Could not find results file: #{uo_results_file}" + @sim_logger.error "Could not find results file: #{uo_results_file}" + end + #no reporting_period + elsif variable[:var_name] #TODO add check on distributed_generation_category + @sim_logger.info "found variable[:var_name]: #{variable[:var_name]} with NO REPORTING_PERIODS." + #get scenario_report results + #scenario_optimization.json + uo_results_file = "#{simulation_dir}/urbanopt/run/#{@data_point.analysis.scenario_file.downcase}/#{reports_file}.json" + if File.exist? uo_results_file + uo_result = JSON.parse(File.read(uo_results_file), symbolize_names: true) + + if uo_result[variable[:report].to_sym].has_key?(variable[:var_name].to_sym) # has var_name? + + if variable[:distributed_generation_category] && uo_result[variable[:report].to_sym][variable[:var_name].to_sym].has_key?(variable[:distributed_generation_category].to_sym) + @sim_logger.info "setting results keys: #{variable[:name].split(/\./)[0]} AND: #{variable[:var_name]}_#{variable[:distributed_generation_category]}" + results[variable[:name].split(/\./)[0].to_sym] = {} + results[variable[:name].split(/\./)[0].to_sym]["#{variable[:var_name]}_#{variable[:distributed_generation_category]}".to_sym] = uo_result[variable[:report].to_sym][variable[:var_name].to_sym][variable[:distributed_generation_category].to_sym] + results[variable[:name].split(/\./)[0].to_sym][:applicable] = true + else + raise "MISSING output variable[:distributed_generation_category]:#{variable[:distributed_generation_category]}, when output variable[:var_name]:#{variable[:var_name]}" + @sim_logger.error "MISSING output variable[:distributed_generation_category]:#{variable[:distributed_generation_category]}, when output variable[:var_name]:#{variable[:var_name]}" + end + + + else + raise "Could not find output variable[:var_name]: #{variable[:var_name]}." + @sim_logger.error "Could not find output variable[:var_name]: #{variable[:var_name]}." + end + + else + raise "Could not find results file: #{uo_results_file}" + @sim_logger.error "Could not find results file: #{uo_results_file}" end else raise "MISSING output variable[:reporting_periods]:#{variable[:reporting_periods]}, variable[:var_name]: #{variable[:var_name]}." @sim_logger.error "MISSING output variable[:reporting_periods]:#{variable[:reporting_periods]}, variable[:var_name]: #{variable[:var_name]}." end + elsif variable[:report] == 'reopt_report' + @sim_logger.info "found variable[:report]: #{variable[:report]}" + if variable[:reopt_category] && variable[:var_name] + @sim_logger.info "found variable[:reopt_category]:#{variable[:reopt_category]}, variable[:var_name]: #{variable[:var_name]}." + #get scenario_report results + uo_results_file = "#{simulation_dir}/urbanopt/run/#{@data_point.analysis.scenario_file.downcase}/reopt/scenario_report_#{@data_point.analysis.scenario_file.downcase}_reopt_run.json" + if File.exist? uo_results_file + @sim_logger.info "found REopt output json file" + uo_result = JSON.parse(File.read(uo_results_file), symbolize_names: true) + if uo_result[0].nil? #this checks if reopt json is formatted correctly + if !uo_result[:outputs][:Scenario][:Site][variable[:reopt_category].to_sym].nil? #reopt_category exist + if uo_result[:outputs][:Scenario][:Site][variable[:reopt_category].to_sym].has_key?(variable[:var_name].to_sym) #reopt_category has var_name? + if !uo_result[:outputs][:Scenario][:Site][variable[:reopt_category].to_sym][variable[:var_name].to_sym].nil? + results[variable[:name].split(/\./)[0].to_sym] = {} + results[variable[:name].split(/\./)[0].to_sym][variable[:var_name].to_sym] = uo_result[:outputs][:Scenario][:Site][variable[:reopt_category].to_sym][variable[:var_name].to_sym] + results[variable[:name].split(/\./)[0].to_sym][:applicable] = true + @sim_logger.info "setting results to: #{uo_result[:outputs][:Scenario][:Site][variable[:reopt_category].to_sym][variable[:var_name].to_sym]}" + else + results[variable[:name].split(/\./)[0].to_sym] = {} + results[variable[:name].split(/\./)[0].to_sym][variable[:var_name].to_sym] = @data_point.analysis.problem['algorithm']['failed_f_value'] + results[variable[:name].split(/\./)[0].to_sym][:applicable] = true + @sim_logger.error "setting results to FAILED_F_VALUE: #{@data_point.analysis.problem['algorithm']['failed_f_value']}" + end + else + raise "Could not find output variable[:var_name]: #{variable[:var_name]} in reopt_category: #{variable[:reopt_category]}." + @sim_logger.error "Could not find output variable[:var_name]: #{variable[:var_name]} in reopt_category: #{variable[:reopt_category]}." + end + else + raise "Could not find output reopt_category: #{variable[:reopt_category]}." + @sim_logger.error "Could not find output reopt_category: #{variable[:reopt_category]}." + end + else + @sim_logger.error "REopt Error: #{uo_result}." + results[variable[:name].split(/\./)[0].to_sym] = {} + results[variable[:name].split(/\./)[0].to_sym][variable[:var_name].to_sym] = @data_point.analysis.problem['algorithm']['failed_f_value'] + results[variable[:name].split(/\./)[0].to_sym][:applicable] = true + end + else + #raise "Could not find results file: #{uo_results_file}" + @sim_logger.error "Could not find results file: #{uo_results_file}" + end + else + raise "MISSING output variable[:reopt_category]:#{variable[:reopt_category]}, variable[:var_name]: #{variable[:var_name]}." + @sim_logger.error "MISSING output variable[:reopt_category]:#{variable[:reopt_category]}, variable[:var_name]: #{variable[:var_name]}." + end + #not feature_report, scenario_report or reopt_report else - raise "output variable '#{variable[:name]}' :report is not scenario_report or feature_reports. :report = '#{variable[:report]}'." - @sim_logger.error "output variable '#{variable[:name]}' :report is not scenario_report or feature_reports. :report = '#{variable[:report]}'." + raise "output variable '#{variable[:name]}' :report is not scenario_report or feature_reports or reopt_report. :report = '#{variable[:report]}'." + @sim_logger.error "output variable '#{variable[:name]}' :report is not scenario_report or feature_reports or reopt_report. :report = '#{variable[:report]}'." end @sim_logger.info "Looking in output variable #{variable[:name]} for objective function [#{variable[:report]}][#{variable[:report_id]}]{#{variable[:reporting_periods]}}[#{variable[:var_name]}]" # look for the objective function key and make sure that it is not nil. False is an okay obj function. + # this is similiar code to whats in workflow-gem but not called because of non-OS workflow. # check if "#{variable[:end_use]}_#{variable[:end_use_category]}" == variable[:name] somewhere?? -BLB - # results = {:"ffce3f6b-023a-46ab-89f2-4f8c692719dd"=>{:electricity=>39869197.34679705, :applicable=>true},:'uuid'...} - if !results[variable[:name].split(".")[0]].nil? && !results[variable[:name].split(".")[0]][variable[:name].split(".")[1].to_sym].nil? - #objective_functions["objective_function_#{variable[:objective_function_index] + 1}"] = results[variable[:name].split(".")[0]][variable[:var_name].to_sym] #no end_uses - objective_functions["objective_function_#{variable[:objective_function_index] + 1}"] = results[variable[:name].split(".")[0]][variable[:name].split(".")[1].to_sym] #end_uses_end_use_category + # results = {:"ffce3f6b-023a-46ab-89f2-4f8c692719dd"=>{:electricity_kwh=>39869197.34679705, :applicable=>true},:'uuid'...} + if !variable[:name].split(/\./)[0].nil? && !variable[:name].split(/\./)[1].nil? && !results[variable[:name].split(/\./)[0].to_sym].nil? && !results[variable[:name].split(/\./)[0].to_sym][variable[:name].split(/\./)[1].to_sym].nil? + #objective_functions["objective_function_#{variable[:objective_function_index] + 1}"] = results[variable[:name].split(/\./)[0]][variable[:var_name].to_sym] #no end_uses + if results[variable[:name].split(/\./)[0].to_sym][variable[:name].split(/\./)[1].to_sym].present? + @sim_logger.info "setting objective function #{variable[:name]} to: #{results[variable[:name].split(/\./)[0].to_sym][variable[:name].split(/\./)[1].to_sym]}" + objective_functions["objective_function_#{variable[:objective_function_index] + 1}"] = results[variable[:name].split(/\./)[0].to_sym][variable[:name].split(/\./)[1].to_sym] #end_uses_end_use_category + else + objective_functions["objective_function_#{variable[:objective_function_index] + 1}"] = @data_point.analysis.problem['algorithm']['failed_f_value'] + @sim_logger.error "No results for objective function #{variable[:name]}. results are NULL/not present. most likely REopt error." + end if variable[:objective_function_target] @sim_logger.info "Found objective function target for #{variable[:name]}" objective_functions["objective_function_target_#{variable[:objective_function_index] + 1}"] = variable[:objective_function_target].to_f @@ -297,12 +484,11 @@ def run_urbanopt(uo_simulation_log, uo_process_log) end else #make raise an option to continue with failures?? - raise "No results for objective function #{variable[:name]}" - @sim_logger.error "No results for objective function #{variable[:name]} in #{__FILE__} at #{__LINE__}" - objective_functions["objective_function_#{variable[:objective_function_index] + 1}"] = Float::MAX - objective_functions["objective_function_target_#{variable[:objective_function_index] + 1}"] = nil - objective_functions["scaling_factor_#{variable[:objective_function_index] + 1}"] = nil - objective_functions["objective_function_group_#{variable[:objective_function_index] + 1}"] = nil + #raise "No results for objective function #{variable[:name]}" + @sim_logger.error "No results for objective function #{variable[:name]}" + objective_functions["objective_function_#{variable[:objective_function_index] + 1}"] = @data_point.analysis.problem['algorithm']['failed_f_value'] + objective_functions["objective_function_target_#{variable[:objective_function_index] + 1}"] = variable[:objective_function_target].to_f + objective_functions["objective_function_group_#{variable[:objective_function_index] + 1}"] = variable[:objective_function_group].to_f end end end @@ -341,9 +527,9 @@ def run_urbanopt(uo_simulation_log, uo_process_log) #copy results to "#{simulation_dir}/urbanopt/run/reports/*" reports_dir = "#{simulation_dir}/reports/" - @sim_logger.info "copying #{simulation_dir}/urbanopt/run/#{@data_point.analysis.scenario_file}/*.{html,json,csv} to #{reports_dir}" + @sim_logger.info "copying #{simulation_dir}/urbanopt/run/#{@data_point.analysis.scenario_file.downcase}/*.{html,json,csv} to #{reports_dir}" FileUtils.mkdir_p reports_dir unless Dir.exist? reports_dir - Dir["#{simulation_dir}/urbanopt/run/#{@data_point.analysis.scenario_file}/*.{html,json,csv}"].each { |file| FileUtils.cp(file, reports_dir) } + Dir["#{simulation_dir}/urbanopt/run/#{@data_point.analysis.scenario_file.downcase}/*.{html,json,csv}"].each { |file| FileUtils.cp(file, reports_dir) } #zip reports @sim_logger.info "zipping up: #{simulation_dir}/urbanopt/run" @@ -417,4 +603,4 @@ def put_into_archive(disk_file_path, zipfile, zipfile_path) end end -end \ No newline at end of file +end diff --git a/server/app/lib/openstudio_server/version.rb b/server/app/lib/openstudio_server/version.rb index c111e61f7..e3c665988 100644 --- a/server/app/lib/openstudio_server/version.rb +++ b/server/app/lib/openstudio_server/version.rb @@ -34,9 +34,9 @@ # ******************************************************************************* module OpenstudioServer - VERSION = '3.2.1'.freeze + VERSION = '3.3.0'.freeze # format should be ^.*\-{1}[a-z]+[0-9]+ # for example: -rc1, -beta6, -customusecase0 - VERSION_EXT = ''.freeze # with preceding - or + - OS_SHA = 'bdbdbc9da6'.freeze + VERSION_EXT = '-rc2'.freeze # with preceding - or + + OS_SHA = '45b36b8d4c'.freeze end diff --git a/server/app/models/analysis.rb b/server/app/models/analysis.rb index 5353f9953..def4468c5 100644 --- a/server/app/models/analysis.rb +++ b/server/app/models/analysis.rb @@ -74,7 +74,9 @@ class Analysis field :urbanopt_variables, type: Array, default: [] #array of UrbanOpt variables field :feature_file, type: String, default: '' # name of UrbanOpt Feature_file.json file field :scenario_file, type: String, default: '' # name of UrbanOpt Scenario.csv file - + field :reopt, type: Boolean, default: false # is ReOpt run? + field :reopt_type, type: String, default: '' # name of ReOpt run scenario or feature + # Temp location for these vas field :samples, type: Integer diff --git a/server/app/models/variable.rb b/server/app/models/variable.rb index 0bd3de9b0..6cb5a7688 100644 --- a/server/app/models/variable.rb +++ b/server/app/models/variable.rb @@ -75,12 +75,16 @@ class Variable field :mapper, type: String # UrbanOpt Mapper name field :uo_measure, type: String # UrbanOpt Measure name field :uo_variable, type: Boolean, default: false # UrbanOpt variable flag + field :report_file, type: String, default: 'default_scenario_report' # UrbanOpt output report name. field :report, type: String, default: 'scenario_report' # UrbanOpt output report name. either: scenario_report/feature_reports field :report_id, type: String, default: '' # UrbanOpt output report :id. either scenario id name or feature report id number field :reporting_periods, type: Integer, default: 0 # UrbanOpt output reporting_periods array index + field :reopt_category, type: String, default: '' # UrbanOpt output reopt_category field :var_name, type: String, default: '' # UrbanOpt output name, ex natural_gas field :end_use, type: String, default: '' # UrbanOpt output end_uses, ex electricity, natural_gas, district_cooling, etc field :end_use_category, type: String, default: '' # UrbanOpt output end_use category, ex heating, cooling, fans, etc + field :comfort_result_category, type: String, default: '' # UrbanOpt output comfort_result_category, ex time_setpoint_not_met_during_occupied_cooling + field :distributed_generation_category, type: String, default: '' # UrbanOpt output distributed_generation_category, ex total_solar_pv_kw # Relationships belongs_to :analysis, index: true @@ -136,6 +140,18 @@ def self.create_output_variable(analysis_id, json) else raise "var_name == end_uses but end_use and end_use_category are missing. check OSA output_variables" end + elsif json['var_name'] == 'comfort_result' #if var_name is comfort_result then name is uuid.comfort_result_comfort_result_category + if json['comfort_result_category'] + json['name'] = "#{SecureRandom.uuid}.#{json['var_name']}_#{json['comfort_result_category']}" + else + raise "var_name == comfort_result but comfort_result and comfort_result_category are missing. check OSA output_variables" + end + elsif json['var_name'] == 'distributed_generation' #if var_name is distributed_generation then name is uuid.distributed_generation_distributed_generation_category + if json['distributed_generation_category'] + json['name'] = "#{SecureRandom.uuid}.#{json['var_name']}_#{json['distributed_generation_category']}" + else + raise "var_name == distributed_generation but distributed_generation and distributed_generation_category are missing. check OSA output_variables" + end else json['name'] = "#{SecureRandom.uuid}.#{json['var_name']}" end @@ -180,14 +196,17 @@ def self.create_output_variable(analysis_id, json) var['objective_function_target'] = json['objective_function_target'] if json['objective_function_target'] var['scaling_factor'] = json['scaling_factor'] if json['scaling_factor'] var['objective_function_group'] = json['objective_function_group'] if json['objective_function_group'] - #set these for UrbanOpt + #set these for UrbanOpt + var.report_file = json['report_file'] if json['report_file'] var['report'] = json['report'] if json['report'] var['report_id'] = json['report_id'] if json['report_id'] var['reporting_periods'] = json['reporting_periods'] if json['reporting_periods'] + var['reopt_category'] = json['reopt_category'] if json['reopt_category'] var['var_name'] = json['var_name'] if json['var_name'] var['end_use'] = json['end_use'] if json['end_use'] var['end_use_category'] = json['end_use_category'] if json['end_use_category'] - + var['comfort_result_category'] = json['comfort_result_category'] if json['comfort_result_category'] + var['distributed_generation_category'] = json['distributed_generation_category'] if json['distributed_generation_category'] var.save! logger.info("output variable: '#{var.to_json}'") var diff --git a/server/spec/features/openstudio_algo_spec.rb b/server/spec/features/openstudio_algo_spec.rb index 26897c49a..fca2eaf2c 100644 --- a/server/spec/features/openstudio_algo_spec.rb +++ b/server/spec/features/openstudio_algo_spec.rb @@ -101,6 +101,22 @@ it 'run cli_test with -z arg', :cli_test, js: true do # setup expected results nsga_nrel = [ + { electricity_consumption_cvrmse: 26.7913, + electricity_consumption_nmbe: 26.2248, + natural_gas_consumption_cvrmse: 77.9983, + natural_gas_consumption_nmbe: 52.0805}, + { electricity_consumption_cvrmse: 22.5502, + electricity_consumption_nmbe: 21.9411, + natural_gas_consumption_cvrmse: 82.5275, + natural_gas_consumption_nmbe: 56.044}, + { electricity_consumption_cvrmse: 81.9164, + electricity_consumption_nmbe: -84.8456, + natural_gas_consumption_cvrmse: 42.5082, + natural_gas_consumption_nmbe: 20.1261}, + { electricity_consumption_cvrmse: 21.0098, + electricity_consumption_nmbe: 20.2345, + natural_gas_consumption_cvrmse: 75.7722, + natural_gas_consumption_nmbe: 50.3806}, { electricity_consumption_cvrmse: 82.2784, electricity_consumption_nmbe: -85.2081, natural_gas_consumption_cvrmse: 45.9709, @@ -245,6 +261,14 @@ it 'run spea_nrel analysis', :spea_nrel, js: true do # setup expected results spea_nrel = [ + { electricity_consumption_cvrmse: 21.0098, + electricity_consumption_nmbe: 20.2345, + natural_gas_consumption_cvrmse: 75.7722, + natural_gas_consumption_nmbe: 50.3806}, + { electricity_consumption_cvrmse: 81.9164, + electricity_consumption_nmbe: -84.8456, + natural_gas_consumption_cvrmse: 42.5082, + natural_gas_consumption_nmbe: 20.1261}, { electricity_consumption_cvrmse: 82.2784, electricity_consumption_nmbe: -85.2081, natural_gas_consumption_cvrmse: 45.9709, @@ -374,6 +398,14 @@ it 'run pso analysis', :pso, js: true do # setup expected results pso = [ + { electricity_consumption_cvrmse: 8.0803, + electricity_consumption_nmbe: 5.1654, + natural_gas_consumption_cvrmse: 66.0755, + natural_gas_consumption_nmbe: -51.2337}, + { electricity_consumption_cvrmse: 42.9781, + electricity_consumption_nmbe: -43.9098, + natural_gas_consumption_cvrmse: 107.3213, + natural_gas_consumption_nmbe: 76.7204}, { electricity_consumption_cvrmse: 8.2663, electricity_consumption_nmbe: 5.4441, natural_gas_consumption_cvrmse: 62.8722, @@ -503,6 +535,14 @@ it 'run rgenoud analysis', :rgenoud, js: true do # setup expected results rgenoud = [ + { electricity_consumption_cvrmse: 59.2834, + electricity_consumption_nmbe: -60.8204, + natural_gas_consumption_cvrmse: 160.4416, + natural_gas_consumption_nmbe: -131.9299}, + { electricity_consumption_cvrmse: 31.7376, + electricity_consumption_nmbe: -32.3214, + natural_gas_consumption_cvrmse: 30.4114, + natural_gas_consumption_nmbe: -11.5533}, { electricity_consumption_cvrmse: 31.5474, electricity_consumption_nmbe: -32.1146, natural_gas_consumption_cvrmse: 29.0854, @@ -632,6 +672,18 @@ it 'run sobol analysis', :sobol, js: true do # setup expected results sobol = [ + { electricity_consumption_cvrmse: 43.6849, + electricity_consumption_nmbe: -44.8533, + natural_gas_consumption_cvrmse: 28.8513, + natural_gas_consumption_nmbe: -1.7651}, + { electricity_consumption_cvrmse: 20.2406, + electricity_consumption_nmbe: 18.5516, + natural_gas_consumption_cvrmse: 60.4654, + natural_gas_consumption_nmbe: -48.584}, + { electricity_consumption_cvrmse: 16.0515, + electricity_consumption_nmbe: 13.4244, + natural_gas_consumption_cvrmse: 115.6455, + natural_gas_consumption_nmbe: -95.4922}, { electricity_consumption_cvrmse: 16.3251, electricity_consumption_nmbe: 13.8002, natural_gas_consumption_cvrmse: 111.2924, @@ -765,6 +817,14 @@ it 'run lhs analysis', :lhs, js: true do # setup expected results lhs = [ + { electricity_consumption_cvrmse: 90.7999, + electricity_consumption_nmbe: -94.0458, + natural_gas_consumption_cvrmse: 41.7615, + natural_gas_consumption_nmbe: -22.5611}, + { electricity_consumption_cvrmse: 25.985, + electricity_consumption_nmbe: 25.6706, + natural_gas_consumption_cvrmse: 112.2071, + natural_gas_consumption_nmbe: 79.9222}, { electricity_consumption_cvrmse: 25.6768, electricity_consumption_nmbe: 25.3392, natural_gas_consumption_cvrmse: 113.6430, @@ -898,6 +958,18 @@ it 'run lhs_discrete analysis', :lhs_discrete, js: true do # setup expected results lhs = [ + { electricity_consumption_cvrmse: 37.239, + electricity_consumption_nmbe: -38.1152, + natural_gas_consumption_cvrmse: 149.9269, + natural_gas_consumption_nmbe: -122.1284}, + { electricity_consumption_cvrmse: 37.239, + electricity_consumption_nmbe: -38.1152, + natural_gas_consumption_cvrmse: 205.2895, + natural_gas_consumption_nmbe: -165.6525}, + { electricity_consumption_cvrmse: 37.6145, + electricity_consumption_nmbe: -38.4922, + natural_gas_consumption_cvrmse: 205.2724, + natural_gas_consumption_nmbe: -165.6411}, { electricity_consumption_cvrmse: 37.2908, electricity_consumption_nmbe: -38.1508, natural_gas_consumption_cvrmse: 199.5555, @@ -1036,6 +1108,18 @@ it 'run morris analysis', :morris, js: true do # setup expected results morris = [ + { electricity_consumption_cvrmse: 23.8448, + electricity_consumption_nmbe: 22.541, + natural_gas_consumption_cvrmse: 135.533, + natural_gas_consumption_nmbe: -112.4872}, + { electricity_consumption_cvrmse: 89.4788, + electricity_consumption_nmbe: -93.057, + natural_gas_consumption_cvrmse: 83.4571, + natural_gas_consumption_nmbe: -64.117}, + { electricity_consumption_cvrmse: 87.0986, + electricity_consumption_nmbe: -90.5346, + natural_gas_consumption_cvrmse: 42.6231, + natural_gas_consumption_nmbe: -25.7106}, { electricity_consumption_cvrmse: 87.3964, electricity_consumption_nmbe: -90.8389, natural_gas_consumption_cvrmse: 40.0277, @@ -1169,11 +1253,15 @@ it 'run single_run analysis', :single_run, js: true do # setup expected results single_run = [ - { electricity_consumption_cvrmse: 34.1625, - electricity_consumption_nmbe: -34.8359, - natural_gas_consumption_cvrmse: 156.1840, - natural_gas_consumption_nmbe: -125.5732 } - ] + { electricity_consumption_cvrmse: 34.4823, + electricity_consumption_nmbe: -35.1756, + natural_gas_consumption_cvrmse: 161.8161, + natural_gas_consumption_nmbe: -130.0589}, + { electricity_consumption_cvrmse: 34.1625, + electricity_consumption_nmbe: -34.8359, + natural_gas_consumption_cvrmse: 156.1840, + natural_gas_consumption_nmbe: -125.5732 } + ] # setup bad results single_run_bad = [ { electricity_consumption_cvrmse: 0, diff --git a/server/spec/features/openstudio_urbanopt_algo_spec.rb b/server/spec/features/openstudio_urbanopt_algo_spec.rb index f78ec87f9..c5918e05a 100644 --- a/server/spec/features/openstudio_urbanopt_algo_spec.rb +++ b/server/spec/features/openstudio_urbanopt_algo_spec.rb @@ -95,8 +95,8 @@ natural_gas_kwh: 21731513.8888 } ] single_run_round = [ - { electricity_kwh: 19300000, - natural_gas_kwh: 21800000 } + { electricity_kwh: 19000000, + natural_gas_kwh: 22000000 } ] # setup bad results single_run_bad = [ @@ -118,7 +118,7 @@ analysis_id = analysis[:_id] status = 'queued' - timeout_seconds = 480 + timeout_seconds = 920 begin ::Timeout.timeout(timeout_seconds) do while status != 'completed' @@ -231,7 +231,7 @@ end sim = sim_result.slice(:electricity_kwh, :natural_gas_kwh) expect(sim.size).to eq(2) - sim = sim.transform_values { |x| x.round(-5) } + sim = sim.transform_values { |x| x.round(-6) } compare = single_run_round.include?(sim) expect(compare).to be true @@ -256,10 +256,10 @@ }] objectives_round = [{ - objective_function_1: 19300000, + objective_function_1: 19000000, objective_function_target_1: 0, objective_function_group_1: 1, - objective_function_2: 21800000, + objective_function_2: 22000000, objective_function_target_2: 0, objective_function_group_2: 2, objective_function_3: 500000, @@ -281,7 +281,11 @@ if key.to_s.include?("target") || key.to_s.include?("group") obj_json[key] = value.to_i else - obj_json[key] = value.round(-5) + if Math.log10(value) > 6 + obj_json[key] = value.round(-6) + else + obj_json[key] = value.round(-5) + end end end diff --git a/server/spec/files/URBANopt_NSGA.json b/server/spec/files/URBANopt_NSGA.json index 188c64f08..627933f63 100644 --- a/server/spec/files/URBANopt_NSGA.json +++ b/server/spec/files/URBANopt_NSGA.json @@ -109,9 +109,6 @@ "run_workflow_timeout": 28800, "upload_results_timeout": 28800, "initialize_worker_timeout": 28800, - "server_scripts": { - "worker_initialization": "./scripts/worker_initialization/initialize.sh" - }, "feature_file": "example_project", "scenario_file": "highefficiency_scenario", "urbanopt_variables": [ diff --git a/server/spec/files/URBANopt_NSGA.zip b/server/spec/files/URBANopt_NSGA.zip index dc1b6ab1f..24e2f268d 100644 Binary files a/server/spec/files/URBANopt_NSGA.zip and b/server/spec/files/URBANopt_NSGA.zip differ diff --git a/server/spec/files/URBANopt_NSGA_spaces_reopt.json b/server/spec/files/URBANopt_NSGA_spaces_reopt.json index 18fd517b0..f17f08054 100644 --- a/server/spec/files/URBANopt_NSGA_spaces_reopt.json +++ b/server/spec/files/URBANopt_NSGA_spaces_reopt.json @@ -16,7 +16,7 @@ "report": "feature_reports", "report_id": "1", "reporting_periods": 0, - "var_name": "electricity", + "var_name": "electricity_kwh", "visualize": true, "export": true, "variable_type": "double" @@ -33,7 +33,7 @@ "report": "feature_reports", "report_id": "1", "reporting_periods": 0, - "var_name": "natural_gas", + "var_name": "natural_gas_kwh", "visualize": true, "export": true, "variable_type": "double" @@ -51,7 +51,7 @@ "report_id": "1", "reporting_periods": 0, "var_name": "end_uses", - "end_use": "electricity", + "end_use": "electricity_kwh", "end_use_category": "fans", "visualize": true, "export": true, @@ -70,7 +70,7 @@ "report_id": "highefficiency_scenario", "reporting_periods": 0, "var_name": "end_uses", - "end_use": "electricity", + "end_use": "electricity_kwh", "end_use_category": "fans", "visualize": true, "export": true, @@ -366,9 +366,6 @@ "run_workflow_timeout": 28800, "upload_results_timeout": 28800, "initialize_worker_timeout": 28800, - "server_scripts": { - "worker_initialization": "./scripts/worker_initialization/initialize.sh" - }, "feature_file": "example_project", "scenario_file": "highefficiency_scenario", "urbanopt_variables": [ diff --git a/server/spec/files/URBANopt_NSGA_spaces_reopt.zip b/server/spec/files/URBANopt_NSGA_spaces_reopt.zip index 5ea02f538..7186dde31 100644 Binary files a/server/spec/files/URBANopt_NSGA_spaces_reopt.zip and b/server/spec/files/URBANopt_NSGA_spaces_reopt.zip differ diff --git a/server/spec/files/URBANopt_fast.zip b/server/spec/files/URBANopt_fast.zip index db3cfdd7b..fb67754db 100644 Binary files a/server/spec/files/URBANopt_fast.zip and b/server/spec/files/URBANopt_fast.zip differ diff --git a/server/spec/files/URBANopt_singlerun.json b/server/spec/files/URBANopt_singlerun.json index c836fd257..d408227df 100644 --- a/server/spec/files/URBANopt_singlerun.json +++ b/server/spec/files/URBANopt_singlerun.json @@ -1,7 +1,7 @@ { "analysis": { - "display_name": "URBANopt_NSGA", - "name": "URBANopt_NSGA", + "display_name": "URBANopt_singlerun", + "name": "URBANopt_singlerun", "urbanopt": true, "output_variables": [ { @@ -13,6 +13,7 @@ "display_name": "electricity", "display_name_short": "electricity", "metadata_id": null, + "report_file": "default_scenario_report", "report": "feature_reports", "report_id": "1", "reporting_periods": 0, @@ -30,6 +31,7 @@ "display_name": "natural_gas", "display_name_short": "natural_gas", "metadata_id": null, + "report_file": "default_scenario_report", "report": "feature_reports", "report_id": "1", "reporting_periods": 0, @@ -47,6 +49,7 @@ "display_name": "electricity_fans", "display_name_short": "electricity_fans", "metadata_id": null, + "report_file": "default_scenario_report", "report": "feature_reports", "report_id": "1", "reporting_periods": 0, @@ -66,6 +69,7 @@ "display_name": "electricity_fans", "display_name_short": "electricity_fans", "metadata_id": null, + "report_file": "default_scenario_report", "report": "scenario_report", "report_id": "highefficiency_scenario", "reporting_periods": 0, diff --git a/server/spec/files/URBANopt_singlerun_fast.json b/server/spec/files/URBANopt_singlerun_fast.json index 29eeb633d..6ae180ba3 100644 --- a/server/spec/files/URBANopt_singlerun_fast.json +++ b/server/spec/files/URBANopt_singlerun_fast.json @@ -1,7 +1,7 @@ { "analysis": { - "display_name": "UrbanOpt_NSGA", - "name": "UrbanOpt_NSGA", + "display_name": "URBANopt_singlerun_fast", + "name": "URBANopt_singlerun_fast", "urbanopt": true, "output_variables": [ {