Merge pull request #81 from GeoscienceAustralia/upgrade/GA_docker_image

Upgrade/ga docker image

Author: abradley60

Docker/Dockerfile

@@ -29,10 +29,11 @@ rm $HOME/miniconda.sh
 ENV PATH=${CONDA_PREFIX}/bin:${PATH}
 RUN ${CONDA_PREFIX}/bin/conda init bash
 
-# Download the opera-adt/RTC code - UPDATE VERSION AND TAG HERE IF NEEDED
-ARG RTC_VERSION=RTC-1.0.4
-ARG RTC_VERSION_TAG=v1.0.4
-RUN  curl -sSL https://github.com/opera-adt/RTC/archive/refs/tags/$RTC_VERSION_TAG.tar.gz -o $RTC_VERSION.tar.gz &&\
+# Download and install the Geoscience Australia fork of the opera/RTC project
+# UPDATE VERSION AND TAG HERE IF NEEDED
+ARG RTC_VERSION=RTC-1.0.4-GA-prod
+ARG RTC_VERSION_TAG=v1.0.4-GA-prod
+RUN  curl -sSL https://github.com/GeoscienceAustralia/RTC/archive/refs/tags/$RTC_VERSION_TAG.tar.gz -o $RTC_VERSION.tar.gz &&\
 tar -xvf $RTC_VERSION.tar.gz -C OPERA &&\
 rm $RTC_VERSION.tar.gz
 

docs/workflows/aws.md

@@ -68,7 +68,7 @@ The AWS pipeline runs using a docker container. At runtime, the script [run_aws_
 --skip_upload_to_s3=false
 --scene_data_source="CDSE"
 --orbit_data_source="CDSE"
---validate_stac=false
+--skip_validate_stac=false
 # Required inputs for linking RTC_S1_STATIC to RTC_S1
 # Assumes that a RTC_S1_STATIC products exist for all RTC_S1 bursts being processed
 --link_static_layers=false           
@@ -90,7 +90,7 @@ The AWS pipeline runs using a docker container. At runtime, the script [run_aws_
 - `skip_upload_to_s3` -> Make the products, but skip uploading them to S3.
 - `scene_data_source` -> Where to download the scene slc file. Either `ASF` or `CDSE`. The default is `CDSE`.
 - `orbit_data_source` -> Where to download the orbit files. Either `ASF` or `CDSE`. The default is `CDSE`.
-- `validate_stac` -> Whether to validate the created STAC doc within the code. If the stac is invalid, products will not be uploaded.
+- `skip_validate_stac` -> To skip validation of the created STAC doc within the code. If the stac is invalid, products will not be uploaded.
 - `link_static_layers` -> Flag to link RTC_S1_STATIC to RTC_S1
 - `linked_static_layers_s3_bucket` -> bucket where RTC_S1_STATIC stored
 - `linked_static_layers_s3_project_folder` -> folder within bucket where RTC_S1_STATIC stored
@@ -99,9 +99,9 @@ The AWS pipeline runs using a docker container. At runtime, the script [run_aws_
 
 **Final paths of products**:
 
-- **RTC_S1** -> final path will be `s3_bucket/s3_project_folder/collection/burst_id/burst_year/burst_month/burst_day/*files*`
-- **RTC_S1_STATIC** -> final path will be `s3_bucket/s3_project_folder/collection/burst_id/*files*`
-
+Final product output paths follow the following structure. odc-product is determined by factors such as the input scene polarisations.
+-  **RTC_S1** -> s3_bucket/s3_project_folder/c{collection_number}/collection/odc_product_name/burst_id/year/month/day/*files
+-  **RTC_S1_STATIC** -> s3_bucket/s3_project_folder/c{collection_number}/collection/odc_product_name/burst_id/*files
 
 ## Environment Variables
 

sar_pipeline/aws/cli.py

@@ -28,6 +28,7 @@
 )
 from sar_pipeline.utils.s3upload import push_files_in_folder_to_s3
 from sar_pipeline.utils.general import log_timing
+from sar_pipeline.utils.spatial import write_burst_geometries_to_geojson
 
 from dem_handler.dem.cop_glo30 import get_cop30_dem_for_bounds
 from dem_handler.dem.rema import get_rema_dem_for_bounds
@@ -171,6 +172,13 @@
     help="Where to download the scene from.",
 )
 @click.option("--make-folders", required=False, default=True, help="Create folders")
+@click.option(
+    "--save-burst-geometries",
+    required=False,
+    default=False,
+    is_flag=True,
+    help="Save the burst geometries to a geojson",
+)
 @log_timing
 def get_data_for_scene_and_make_run_config(
     scene,
@@ -194,6 +202,7 @@ def get_data_for_scene_and_make_run_config(
     scene_data_source,
     orbit_data_source,
     make_folders,
+    save_burst_geometries,
 ):
     """Download the required data for the RTC/opera and create a configuration
     file for the run that points to appropriate files and has the required settings
@@ -329,6 +338,13 @@ def get_data_for_scene_and_make_run_config(
     burst_geoms_to_process = [
         all_scene_burst_info[id_]["geometry"] for id_ in burst_id_list_to_process
     ]
+    # write the geometries to a geojson. Useful for debugging if needed
+    if save_burst_geometries:
+        write_burst_geometries_to_geojson(
+            burst_id_list_to_process,
+            burst_geoms_to_process,
+            out_folder / f"{scene}_burst_geoms.json",
+        )
     combined_burst_geom = shapely.ops.unary_union(burst_geoms_to_process)
     logger.info(f"The scene shape is : {scene_polygon}")
     logger.info(f"The scene bounds are : {scene_polygon.bounds}")
@@ -595,6 +611,7 @@ def make_rtc_opera_stac_and_upload_bursts(
         if link_static_layers and product == "RTC_S1":
             # link to static layer metadata is in the .h5 file
             # use this to map assets to the file
+            logger.info("Linking static layers to product")
             burst_stac_manager.add_linked_static_layer_assets_and_link()
         stac_filename = "metadata.json"
         burst_stac_manager.add_self_link(filename=stac_filename)
@@ -608,7 +625,9 @@ def make_rtc_opera_stac_and_upload_bursts(
                 burst_stac_manager.item.validate()
                 logger.info("STAC is valid.")
             except Exception as e:
-                logger.error(f"STAC validation failed: {e}")
+                logger.error(
+                    f"STAC validation failed, correct error or run without --validate-stac flag.\n{e}"
+                )
                 raise
         else:
             logger.warning(

sar_pipeline/aws/metadata/stac.py

@@ -363,11 +363,11 @@ def add_properties_from_h5(self):
         )
         self.item.properties["sarard:ionospheric_correction_applied"] = False
 
-        # TODO fill with study result values
-        self.item.properties["sarard:geometric_accuracy_ALE"] = "TODO"
-        self.item.properties["sarard:geometric_accuracy_rmse"] = "TODO"
-        self.item.properties["sarard:geometric_accuracy_range"] = "TODO"
-        self.item.properties["sarard:geometric_accuracy_azimuth"] = "TODO"
+        # TODO when official doccument, link the study supporting these values
+        self.item.properties["sarard:geometric_accuracy_ALE"] = 2.94
+        self.item.properties["sarard:geometric_accuracy_rmse"] = 3.08
+        self.item.properties["sarard:geometric_accuracy_range"] = 1.63
+        self.item.properties["sarard:geometric_accuracy_azimuth"] = 1.92
 
         # add the storage stac extension properties
         self.item.properties["storage:schemes"] = {
@@ -647,6 +647,7 @@ def add_linked_static_layer_assets_and_link(self):
         burst_static_layer_stac_url = os.path.join(
             static_layer_url, self.burst_id, "metadata.json"
         )
+        logger.info(f"Static layer url: {burst_static_layer_stac_url}")
 
         try:
             # Send HTTP GET request

sar_pipeline/utils/spatial.py

@@ -3,8 +3,11 @@
 from shapely import wkt
 from shapely.geometry import mapping, box, Polygon
 from shapely import segmentize
+import json
+from pathlib import Path
 import logging
 
+
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
@@ -136,3 +139,44 @@ def reproject_bbox_to_geometry(
     segmentized_geometry = segmentize(bbox_geometry, max_segment_length=segment_length)
     transformed_geometry = transform_polygon(segmentized_geometry, src_crs, trg_crs)
     return transformed_geometry
+
+
+def write_burst_geometries_to_geojson(burst_id_list, burst_geometry_list, save_path):
+    """
+    Write burst geometries and their IDs to a GeoJSON file.
+
+    Parameters
+    ----------
+    burst_id_list : list of str
+        List of burst identifiers (e.g., "t007_014545_iw2").
+    burst_geometry_list : list of shapely.geometry.BaseGeometry
+        List of corresponding geometries for each burst ID. Must be in the same order as `burst_id_list`.
+    save_path : str or Path
+        Path to save the resulting GeoJSON file. Will be overwritten if it exists.
+
+    Raises
+    ------
+    ValueError
+        If `burst_id_list` and `burst_geometry_list` have different lengths.
+
+    """
+    if len(burst_id_list) != len(burst_geometry_list):
+        raise ValueError(
+            "burst_id_list and burst_geometry_list must have the same length."
+        )
+
+    features = []
+    for burst_id, geom in zip(burst_id_list, burst_geometry_list):
+        features.append(
+            {
+                "type": "Feature",
+                "geometry": mapping(geom),
+                "properties": {"burst_id": burst_id},
+            }
+        )
+
+    geojson_obj = {"type": "FeatureCollection", "features": features}
+
+    Path(save_path).parent.mkdir(parents=True, exist_ok=True)
+    with open(save_path, "w", encoding="utf-8") as f:
+        json.dump(geojson_obj, f, indent=2)

scripts/run_aws_pipeline.sh

@@ -15,9 +15,9 @@ make_existing_products=false
 skip_upload_to_s3=false
 scene_data_source="ASF"
 orbit_data_source="ASF"
-validate_stac=false
+skip_validate_stac=false
 ## -- WORKFLOW INPUTS TO LINK RTC_S1_STATIC in RTC_S1 metadata--
-# Assumes that a RTC_S1_STATIC products exist for all RTC_S1 bursts being processed
+# if link_static_layers, RTC_S1_STATIC products must exist for all RTC_S1 bursts being processed
 link_static_layers=false
 linked_static_layers_s3_bucket="deant-data-public-dev"
 linked_static_layers_s3_project_folder="ga_s1"
@@ -47,7 +47,7 @@ while [[ "$#" -gt 0 ]]; do
         --linked_static_layers_s3_project_folder) linked_static_layers_s3_project_folder="$2"; shift 2 ;;
         --scene_data_source) scene_data_source="$2"; shift 2 ;;
         --orbit_data_source) orbit_data_source="$2"; shift 2 ;;
-        --validate_stac) validate_stac=true; shift ;;
+        --skip_validate_stac) skip_validate_stac=true; shift ;;
         --burst_id_list)
             shift
             if [[ $# -eq 1 && -f "$1" ]]; then
@@ -112,7 +112,7 @@ echo make_existing_products : "$make_existing_products"
 echo skip_upload_to_s3 : "$skip_upload_to_s3"
 echo scene_data_source : "$scene_data_source"
 echo orbit_data_source : "$orbit_data_source"
-echo validate_stac : "$validate_stac"
+echo skip_validate_stac : "$skip_validate_stac"
 
 # warn the user about linking static layers
 if [[ "$link_static_layers" = true && "$product" = "RTC_S1" ]]; then
@@ -244,7 +244,7 @@ if [ "$link_static_layers" = true ] ; then
     # The url link to static layers is read in from results .h5 file
     cmd+=( --link-static-layers)
 fi
-if [ "$validate_stac" = true ] ; then
+if [ "$skip_validate_stac" != true ] ; then
     # validate the stac doc within the code
     cmd+=( --validate-stac)
 fi

tests/sar_pipeline/test_full_aws_docker_run.py

@@ -68,8 +68,8 @@
     except:
         raise FileExistsError(
             "Could not find env.secret file at project root containing required environment variables for run. "
-            "Create this file with required variables or ensure environment is configured correctly "
-            "(for example when running automated tests on GitHub)"
+            "Create this file with required variables OR ensure environment is configured correctly "
+            "(e.g. when running automated tests on GitHub)"
         )
 
 
@@ -86,7 +86,7 @@ def build_image():
             "-t",
             f"sar-pipeline:{docker_tag}",
             "-f",
-            "Docker/Dockerfile",
+            "Docker/isce3-v0.24.4.Dockerfile",
             ".",
         ],
         stdout=sys.stdout,
@@ -100,29 +100,68 @@ def build_image():
 
 def test_docker_with_args():
     logging.info(f"Running full process, this may take a while...")
+    logging.info(f"Static layers will be produced and linked to backscatter data.")
     run_dt = str(datetime.now()).replace(" ", "_")
     test_name = Path(__file__).stem
     test_s3_bucket = "deant-data-public-dev"
     test_s3_project_folder = f"TMP/sar-pipeline/{run_dt}/{test_name}"
     logging.info(f"Uploading outputs to : {test_s3_bucket}/{test_s3_project_folder}")
     docker_tag = re.sub(r"[^a-zA-Z0-9_.-]", "-", sar_pipeline.__version__)
+
+    logging.info(f"RUN 1: Producing Static Layers")
+    cmd = [
+        "docker",
+        "run",
+        "--rm",
+        *ENV_VARS,
+        f"sar-pipeline:{docker_tag}",
+        "--scene",
+        "S1A_IW_SLC__1SSH_20220101T124744_20220101T124814_041267_04E7A2_1DAD",
+        "--burst_id_list",
+        "t070_149815_iw3",
+        "--product",
+        "RTC_S1_STATIC",
+        "--collection",
+        "s1_rtc_static_c1",
+        "--s3_bucket",
+        test_s3_bucket,
+        "--s3_project_folder",
+        test_s3_project_folder,
+    ]
     result = subprocess.run(
-        [
-            "docker",
-            "run",
-            "--rm",
-            *ENV_VARS,
-            f"sar-pipeline:{docker_tag}",
-            "--scene",
-            "S1A_IW_SLC__1SSH_20220101T124744_20220101T124814_041267_04E7A2_1DAD",
-            "--burst_id_list",
-            "t070_149815_iw3",
-            "--s3_bucket",
-            test_s3_bucket,
-            "--s3_project_folder",
-            test_s3_project_folder,
-            "--validate_stac",
-        ],
+        cmd,
+        stdout=sys.stdout,
+        stderr=sys.stderr,
+        text=True,
+    )
+    assert (
+        result.returncode == 0
+    ), f"Non-zero exit code: {result.returncode}\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
+
+    logging.info(f"RUN 2: Producing Backscatter and Linking to Static Layers")
+    cmd = [
+        "docker",
+        "run",
+        "--rm",
+        *ENV_VARS,
+        f"sar-pipeline:{docker_tag}",
+        "--scene",
+        "S1A_IW_SLC__1SSH_20220101T124744_20220101T124814_041267_04E7A2_1DAD",
+        "--burst_id_list",
+        "t070_149815_iw3",
+        "--s3_bucket",
+        test_s3_bucket,
+        "--s3_project_folder",
+        test_s3_project_folder,
+        "--link_static_layers",
+        "--linked_static_layers_s3_project_folder",
+        test_s3_project_folder,
+        "--linked_static_layers_collection",
+        "s1_rtc_static_c1",
+    ]
+
+    result = subprocess.run(
+        cmd,
         stdout=sys.stdout,
         stderr=sys.stderr,
         text=True,