From 6e0d888d946a0d6a6db7856fcf84730a8118bb82 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 12 Mar 2024 12:51:40 -0400 Subject: [PATCH 1/3] add python 3.12 ci job --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d37a88bb..0291cf65 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,8 @@ jobs: - linux: py39-oldestdeps-cov-xdist - linux: py39-xdist - linux: py310-xdist - - linux: py311-cov-xdist + - linux: py311-xdist + - linux: py312-cov-xdist coverage: codecov - macos: py311-xdist test_downstream: From a053db73ca2bfb41b851baf46da94c53429d7b19 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 12 Mar 2024 12:58:33 -0400 Subject: [PATCH 2/3] use get_context for multiprocessing --- src/stcal/jump/jump.py | 7 ++++--- src/stcal/ramp_fitting/ols_fit.py | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/stcal/jump/jump.py b/src/stcal/jump/jump.py index a047ce6e..03405c08 100644 --- a/src/stcal/jump/jump.py +++ b/src/stcal/jump/jump.py @@ -391,9 +391,10 @@ def detect_jumps( ), ) log.info("Creating %d processes for jump detection ", n_slices) - pool = multiprocessing.Pool(processes=n_slices) -######### JUST FOR DEBUGGING ######################### -# pool = multiprocessing.Pool(processes=1) + ctx = multiprocessing.get_context("forkserver") + pool = ctx.Pool(processes=n_slices) + ######### JUST FOR DEBUGGING ######################### + # pool = ctx.Pool(processes=1) # Starts each slice in its own process. Starmap allows more than one # parameter to be passed. real_result = pool.starmap(twopt.find_crs, slices) diff --git a/src/stcal/ramp_fitting/ols_fit.py b/src/stcal/ramp_fitting/ols_fit.py index a52e2fba..66d19b76 100644 --- a/src/stcal/ramp_fitting/ols_fit.py +++ b/src/stcal/ramp_fitting/ols_fit.py @@ -1,10 +1,10 @@ #! /usr/bin/env python import logging +import multiprocessing import time import warnings from multiprocessing import cpu_count -from multiprocessing.pool import Pool import numpy as np @@ -160,7 +160,8 @@ def ols_ramp_fit_multiprocessing( ramp_data, buffsize, save_opt, readnoise_2d, gain_2d, weighting, number_slices ) - pool = Pool(processes=number_slices) + ctx = multiprocessing.get_context("forkserver") + pool = ctx.Pool(processes=number_slices) pool_results = pool.starmap(ols_ramp_fit_single, slices) pool.close() pool.join() From 987e19fa1e0b20142f3c08274628518eddbbaa31 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 12 Mar 2024 13:04:32 -0400 Subject: [PATCH 3/3] add changelog entry --- CHANGES.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index cf7bc1f8..6b1162d6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,16 @@ Changes to API -------------- +jump +~~~~ + +- Switch multiprocessing method to ``fork_server``. [#249] + +ramp_fitting +~~~~~~~~~~~~ + +- Switch multiprocessing method to ``fork_server``. [#249] + Bug Fixes ---------