Skip to content

Commit

Permalink
Merge pull request #125 from nansencenter/sentinel3_syntool_converter
Browse files Browse the repository at this point in the history 
Sentinel3 Syntool converter
  • Loading branch information
@aperrin66
aperrin66 authored Dec 11, 2024
2 parents 198fec6 + c90f029 commit 513ae01
Show file tree
Hide file tree
Showing 2 changed files with 149 additions and 48 deletions.
66 changes: 50 additions & 16 deletions geospaas_processing/converters/syntool/converter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ def ingest(self, in_file, out_dir, options, **kwargs):
process.stdout, process.stderr)
return results

def post_ingest(self, results, out_dir, **kwargs):
def post_ingest(self, results, out_dir, kwargs):
"""Post-ingestion step, the default is to create a "features"
directory containing some metadata about what was generated
"""
Expand Down Expand Up @@ -116,11 +116,6 @@ class BasicSyntoolConverter(SyntoolConverter):
"""Syntool converter using pre-set configuration files"""

PARAMETER_SELECTORS = (
ParameterSelector(
matches=lambda d: re.match(r'^S3[AB]_OL_2_WFR.*$', d.entry_id),
converter_type='sentinel3_olci_l2',
converter_options={'channels': 'CHL_OC4ME'},
ingest_parameter_files='ingest_geotiff_4326_tiles'),
ParameterSelector(
matches=lambda d: re.match(r'^S3[AB]_SL_1_RBT.*$', d.entry_id),
converter_type='sentinel3_slstr_bt',
Expand Down Expand Up @@ -235,24 +230,21 @@ def parse_converter_args(self, kwargs):
converter_args.extend(self.parse_converter_options(kwargs))
return converter_args

def run(self, in_file, out_dir, **kwargs):
"""Transforms a file into a Syntool-displayable format using
the syntool-converter and syntool-ingestor tools
"""
results_dir = kwargs.pop('results_dir')
# syntool-converter
def run_conversion(self, in_file, out_dir, kwargs):
"""Run the Syntool converter on the input file"""
if self.converter_type is not None:
converted_files = self.convert(in_file, out_dir,
self.parse_converter_args(kwargs),
**kwargs)
else:
converted_files = (in_file,)
return [Path(out_dir, converted_file) for converted_file in converted_files]

# syntool-ingestor
def run_ingestion(self, converted_paths, results_dir, kwargs):
"""Run the Syntool ingestor on the conversion results"""
ingestor_config = Path(kwargs.pop('ingestor_config', 'parameters/3413.ini'))
results = []
for converted_file in converted_files:
converted_path = Path(out_dir, converted_file)
for converted_path in converted_paths:
for ingest_config in self.find_ingest_config(converted_path):
results.extend(self.ingest(
converted_path, results_dir, [
Expand All @@ -264,8 +256,16 @@ def run(self, in_file, out_dir, **kwargs):
os.remove(converted_path)
except IsADirectoryError:
shutil.rmtree(converted_path)
return results

self.post_ingest(results, results_dir, **kwargs)
def run(self, in_file, out_dir, **kwargs):
"""Transforms a file into a Syntool-displayable format using
the syntool-converter and syntool-ingestor tools
"""
results_dir = kwargs.pop('results_dir')
converted_paths = self.run_conversion(in_file, out_dir, kwargs)
results = self.run_ingestion(converted_paths, results_dir, kwargs)
self.post_ingest(results, results_dir, kwargs)
return results


Expand Down Expand Up @@ -318,6 +318,40 @@ def ingest(self, in_file, out_dir, options, **kwargs):
return results


@SyntoolConversionManager.register()
class Sentinel3OLCISyntoolConverter(BasicSyntoolConverter):
"""Syntool converter for Sentinel-3 datasets"""
PARAMETER_SELECTORS = (
ParameterSelector(
matches=lambda d: re.match(r'^S3[AB]_OL_2_WFR.*$', d.entry_id),
converter_type='sentinel3_olci_l2',
converter_options={'channels': ['CHL_OC4ME', 'true_rgb', 'false_rgb']},
ingest_parameter_files='ingest_geotiff_4326_tiles'),
)

def run_conversion(self, in_file, out_dir, kwargs):
"""Allow to give a list of channels, which will cause the
conversion to be executed for each channel
"""
channels = kwargs['converter_options']['channels']
results = []
if isinstance(channels, str):
channels = [channels]
for channel in channels:
edited_kwargs = kwargs.copy()
edited_kwargs['converter_options']['channels'] = channel
try:
results.extend(super().run_conversion(in_file, out_dir, edited_kwargs))
except ConversionError as error:
if isinstance(error.__cause__, SystemExit):
# The sentinel3_olci_l2 converter exits without error
# message if the data quality is not satisfactory.
# We allow the conversion to continue for other bands.
logger.warning("Syntool conversion failed for %s", in_file, exc_info=True)
continue
return results


@SyntoolConversionManager.register()
class CustomReaderSyntoolConverter(BasicSyntoolConverter):
"""Syntool converter using cutom readers. The converter_type
Expand Down
131 changes: 99 additions & 32 deletions tests/converters/test_syntool_converters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ def test_post_ingest(self):
result_dir = Path(tmp_dir, 'result')
result_dir.mkdir()
syntool_converter.SyntoolConverter().post_ingest(
['result'], tmp_dir, dataset=mock_dataset)
['result'], tmp_dir, {'dataset': mock_dataset})
with open(result_dir / 'features' / 'data_access.ini', 'r', encoding='utf-8') as handle:
contents = handle.read()
self.assertEqual(
Expand Down Expand Up @@ -219,24 +219,43 @@ def test_parse_converter_args(self):
converter.parse_converter_args({'converter_options': {'ham': 'egg'}}),
['-t', 'foo', '-opt', 'bar=baz', 'ham=egg'])

def test_run(self):
"""Test running the conversion and ingestion"""
def test_run_conversion(self):
"""Test calling the Syntool converter on the input file"""
converter = syntool_converter.BasicSyntoolConverter(
converter_type='foo',
ingest_parameter_files='bar')
with mock.patch.object(converter, 'convert',
return_value=['conv1.tiff', 'conv2']) as mock_convert, \
mock.patch.object(converter, 'ingest',
return_value=['conv1.tiff', 'conv2']) as mock_convert:
results = converter.run_conversion('in.nc', 'out',
{'converter_options': {'baz': 'quz'}})
mock_convert.assert_called_once_with('in.nc', 'out', ['-t', 'foo', '-opt', 'baz=quz'])
self.assertListEqual(results, [Path('out', 'conv1.tiff'), Path('out', 'conv2')])

def test_run_conversion_no_converter(self):
"""If no converter is set, run_conversion() should return the path to
the input file
"""
converter = syntool_converter.BasicSyntoolConverter(
converter_type=None,
ingest_parameter_files='bar')
with mock.patch.object(converter, 'convert',) as mock_convert:
results = converter.run_conversion('in.nc', 'out', {})
mock_convert.assert_not_called()
self.assertListEqual(results, [Path('out', 'in.nc')])

def test_run_ingestion(self):
"""Test calling the Syntool ingestor on the conversion output
file(s)
"""
converter = syntool_converter.BasicSyntoolConverter(
converter_type='foo',
ingest_parameter_files='bar')
with mock.patch.object(converter, 'ingest',
side_effect=[['ingested_dir1'], ['ingested_dir2']]) as mock_ingest, \
mock.patch.object(converter, 'post_ingest') as mock_post_ingest, \
mock.patch('os.remove', side_effect=(None, IsADirectoryError)) as mock_remove, \
mock.patch('shutil.rmtree') as mock_rmtree:
converter.run(
in_file='in.nc',
out_dir='out',
results_dir='results',
converter_options={'baz': 'quz'})
mock_convert.assert_called_once_with('in.nc', 'out', ['-t', 'foo', '-opt', 'baz=quz'])
converter.run_ingestion(
[Path('out', 'conv1.tiff'), Path('out', 'conv2')], 'results', {})
mock_ingest.assert_has_calls([
mock.call(
Path('out', 'conv1.tiff'),
Expand All @@ -249,36 +268,35 @@ def test_run(self):
['--config', Path('parameters/3413.ini'),
'--options-file', converter.PARAMETERS_DIR / 'bar']),
])
mock_post_ingest.assert_called_once_with(['ingested_dir1', 'ingested_dir2'], 'results')
mock_remove.assert_has_calls((
mock.call(Path('out', 'conv1.tiff')),
mock.call(Path('out', 'conv2'))))
mock_rmtree.assert_called_once_with(Path('out', 'conv2'))

def test_run_no_converter(self):
"""If no converter is set, convert() should return the path to
the input file
"""
def test_run(self):
"""Test running the conversion and ingestion"""
converter = syntool_converter.BasicSyntoolConverter(
converter_type=None,
converter_type='foo',
ingest_parameter_files='bar')
with mock.patch.object(converter, 'convert',) as mock_convert, \
mock.patch.object(converter, 'ingest',
return_value=['ingested_dir1']) as mock_ingest, \
mock.patch.object(converter, 'post_ingest') as mock_post_ingest, \
mock.patch('os.remove') as mock_remove:
converter.run(
with mock.patch.object(converter, 'run_conversion',
return_value=[Path('out', 'conv1.tiff'),
Path('out', 'conv2')]) as mock_conversion, \
mock.patch.object(converter, 'run_ingestion',
return_value=['ingested_dir1', 'ingested_dir2']) as mock_ingestion, \
mock.patch.object(converter, 'post_ingest') as mock_post_ingest:
results = converter.run(
in_file='in.nc',
out_dir='out',
results_dir='results')
mock_convert.assert_not_called()
mock_ingest.assert_called_once_with(
Path('out', 'in.nc'),
results_dir='results',
converter_options={'baz': 'quz'})
self.assertListEqual(results, ['ingested_dir1', 'ingested_dir2'])
mock_conversion.assert_called_with('in.nc', 'out', {'converter_options': {'baz': 'quz'}})
mock_ingestion.assert_called_with(
[Path('out', 'conv1.tiff'), Path('out', 'conv2')],
'results',
['--config', Path('parameters/3413.ini'),
'--options-file', converter.PARAMETERS_DIR / 'bar'])
mock_post_ingest.assert_called_once_with(['ingested_dir1'], 'results')
mock_remove.assert_called_once_with(Path('out', 'in.nc'))
{'converter_options': {'baz': 'quz'}})
mock_post_ingest.assert_called_with(
['ingested_dir1', 'ingested_dir2'], 'results', {'converter_options': {'baz': 'quz'}})


class Sentinel1SyntoolConverterTestCase(unittest.TestCase):
Expand Down Expand Up @@ -353,6 +371,55 @@ def test_ingest(self):
[ingested_dir / 'ingested_hh'])


class Sentinel3OLCISyntoolConverterTestCase(unittest.TestCase):
"""Tests for the Sentinel1SyntoolConverter class"""

def test_run_conversion_multi_channels(self):
"""Test that the converter is run for every channel in
converter options
"""
converter = syntool_converter.Sentinel3OLCISyntoolConverter(
converter_type='foo',
ingest_parameter_files='bar')
with mock.patch.object(converter, 'convert',
side_effect=[['conv1.tiff'], ['conv2.tiff']]) as mock_convert:
results = converter.run_conversion('in.nc', 'out',
{'converter_options': {'channels': ['baz', 'qux']}})
mock_convert.assert_has_calls((
mock.call('in.nc', 'out', ['-t', 'foo', '-opt', 'channels=baz']),
mock.call('in.nc', 'out', ['-t', 'foo', '-opt', 'channels=qux'])
))
self.assertListEqual(results, [Path('out', 'conv1.tiff'), Path('out', 'conv2.tiff')])

def test_run_conversion_single_channels(self):
"""Test that the converter is run once if channels is a string
"""
converter = syntool_converter.Sentinel3OLCISyntoolConverter(
converter_type='foo',
ingest_parameter_files='bar')
with mock.patch.object(converter, 'convert',
return_value=['conv1.tiff', 'conv2.tiff']) as mock_convert:
results = converter.run_conversion('in.nc', 'out',
{'converter_options': {'channels': 'baz,qux'}})
mock_convert.assert_called_with('in.nc', 'out', ['-t', 'foo', '-opt', 'channels=baz,qux'])
self.assertListEqual(results, [Path('out', 'conv1.tiff'), Path('out', 'conv2.tiff')])

def test_run_conversion_system_exit(self):
"""Test that SystemExit exceptions do not interrupt the
conversion process but simply return empty results
"""
error = syntool_converter.ConversionError()
error.__cause__ = SystemExit()
converter = syntool_converter.Sentinel3OLCISyntoolConverter(
converter_type='foo',
ingest_parameter_files='bar')
with mock.patch.object(converter, 'convert', side_effect=[error, ['conv2.tiff']]):
with self.assertLogs(level=logging.WARNING):
results = converter.run_conversion(
'in.nc', 'out', {'converter_options': {'channels': ['baz', 'qux']}})
self.assertListEqual(results, [Path('out', 'conv2.tiff')])


class CustomReaderSyntoolConverterTestCase(unittest.TestCase):
"""Tests for the CustomReaderSyntoolConverter class"""

Expand Down

0 comments on commit 513ae01

Please sign in to comment.