BUG: ignore manifest lines that do not contain s3 URLs (#139)

This came up while trying to use manifests generated using Looker Studio
ImagingDataCommons · Nov 25, 2024 · 8b9d192 · 8b9d192
1 parent 417ffe0
commit 8b9d192
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/idc_index/index.py b/idc_index/index.py
@@ -836,7 +836,9 @@ def _validate_update_manifest_and_get_download_size(
                 REGEXP_EXTRACT(manifest_cp_cmd, '(?:.*?\\/){{3}}([^\\/?#]+)', 1) AS manifest_crdc_series_uuid,
                 REGEXP_EXTRACT(manifest_cp_cmd, 's3://\\S+') AS s3_url,
             FROM
-                manifest_df )
+                manifest_df
+            WHERE
+                REGEXP_EXTRACT(manifest_cp_cmd, 's3://\\S+') IS NOT NULL)
             SELECT
                 seriesInstanceuid,
                 index_crdc_series_uuid,