CI update for v4 OED release (#211)

* test pr list build artifact * Fix unexpected column name change * Add build step to post excel on PR's * Add script to update rel links * Update release script for test * check * Fix * release test working
OasisLMF · Oct 28, 2024 · b40bb0a · b40bb0a
1 parent 2d98b84
commit b40bb0a
Show file tree

Hide file tree

Showing 7 changed files with 157 additions and 27 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -9,9 +9,12 @@ on:
         required: false
         type: string
     outputs:
-      spec_filename:
+      json_filename:
         description: "OED JSON specification"
         value: ${{ jobs.build.outputs.oed_json_spec }}
+      excel_filename:
+        description: "OED EXCEL specification"
+        value: ${{ jobs.build.outputs.oed_excel_spec }}
 
 jobs:
   build:
@@ -36,7 +39,7 @@ jobs:
       uses: actions/checkout@v3
       if: inputs.ods_branch == ''
       with:
-        ref: ${{ github.ref_name }}
+        ref: ${{ github.ref }}
 
     - name: Clone (workflow_call)
       uses: actions/checkout@v3
@@ -58,18 +61,18 @@ jobs:
       run: ./utils/gen-excel.py --source-csv-dir ${{ env.SPEC_PATH_IN }} --output-path ${{ env.EXCEL_PATH_OUT }}
 
     - name: Store JSON
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: extracted_spec
         path: ${{ env.JSON_PATH_OUT }}
-        retention-days: 5
+        retention-days: 14
 
     - name: Store Excel
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: excel_spec
         path: ${{ env.EXCEL_PATH_OUT }}
-        retention-days: 5
+        retention-days: 14
 
     - name: Build output
       id: build_result

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -74,6 +74,14 @@ jobs:
         GIT_EMAIL: ${{ secrets.BUILD_GIT_EMAIL }}
         GIT_USERNAME: ${{ secrets.BUILD_GIT_USERNAME }}
 
+    - name: Update Readme
+      env:
+        GITHUB_TOKEN: ${{ secrets.BUILD_GIT_TOKEN }}
+      run: |
+        ./utils/update-readme-release.sh ${{ env.RELEASE_TAG }}
+        git add ./README.md
+        git commit -m 'Update Readme.md'
+
     - name: Tag Release
       env:
         GITHUB_TOKEN: ${{ secrets.BUILD_GIT_TOKEN }}
@@ -82,11 +90,18 @@ jobs:
         git tag ${{ env.RELEASE_TAG }}
 
     - name: Download JSON spec
-      uses: actions/download-artifact@v4.1.7
+      uses: actions/download-artifact@v4
       with:
         name: extracted_spec
         path: ${{ github.workspace }}/
 
+
+    - name: Download EXCEL spec
+      uses: actions/download-artifact@v4
+      with:
+        name: excel_spec
+        path: ${{ github.workspace }}/
+
     # --- Create Release --- #
     - name: Push changes
       run: |
@@ -105,16 +120,27 @@ jobs:
         draft: false
         prerelease: ${{ env.PRE_RELEASE }}
 
-    # --- Attach build assest --- #
+    # --- Attach build assests --- #
     - name: Upload JSON Spec
       id: upload-source-release-asset
       uses: actions/upload-release-asset@v1
       env:
         GITHUB_TOKEN: ${{ secrets.BUILD_GIT_TOKEN }}
       with:
         upload_url: ${{ steps.create_release.outputs.upload_url }}
-        asset_path: ${{ github.workspace }}/${{ needs.build.outputs.spec_filename }}
-        asset_name: ${{ needs.build.outputs.spec_filename }}
+        asset_path: ${{ github.workspace }}/${{ needs.build.outputs.json_filename }}
+        asset_name: ${{ needs.build.outputs.json_filename }}
+        asset_content_type: application/octet-stream
+
+    - name: Upload EXCEL Spec
+      id: upload-excel-release-asset
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.BUILD_GIT_TOKEN }}
+      with:
+        upload_url: ${{ steps.create_release.outputs.upload_url }}
+        asset_path: ${{ github.workspace }}/${{ needs.build.outputs.excel_filename }}
+        asset_name: ${{ needs.build.outputs.excel_filename }}
         asset_content_type: application/octet-stream
 
     # --- Slack notify --- #

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -2,6 +2,7 @@ name: OpenData Testing
 
 on:
   push:
+  pull_request:
   workflow_dispatch:
     inputs:
       ods_tools_branch:
@@ -21,9 +22,72 @@ jobs:
     uses: OasisLMF/ODS_Tools/.github/workflows/build.yml@main
     secrets: inherit
     with:
-      oed_spec_json: ${{ needs.build_spec.outputs.spec_filename }}
+      oed_spec_json: ${{ needs.build_spec.outputs.json_filename }}
       ods_branch: ${{ github.event_name != 'workflow_dispatch' && 'main' ||  inputs.ods_tools_branch }}
 
+
+  attach_pr:
+    if: github.event_name == 'pull_request'
+    name: Update PR
+    needs: build_spec
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Github context
+      run:   echo "$GITHUB_CONTEXT"
+      shell: bash
+      env:
+       GITHUB_CONTEXT: ${{ toJson(github) }}
+
+    - name: Get Artifact URL & PR Info
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+        OWNER: ${{ github.repository_owner }}
+        REPO: ${{ github.event.repository.name }}
+        JOB_ID: ${{ github.run_id }}
+        PR_NUMBER: ${{ github.event.number }}
+        HEAD_SHA: ${{ github.sha }}
+      run: |
+         echo "Job ID: $JOB_ID"
+         echo "JOB_ID=$JOB_ID" >> "$GITHUB_ENV"
+
+         EXCEL_ID=$(gh api "/repos/OasisLMF/ODS_OpenExposureData/actions/artifacts" --jq '.artifacts[] | select(.workflow_run.id == '$JOB_ID') | select(.expired == false) | select(.name | startswith("excel_spec")) | .id')
+         echo "EXCEL ID: $EXCEL_ID"
+         echo "EXCEL_ID=$EXCEL_ID" >> "$GITHUB_ENV"
+
+         JSON_ID=$(gh api "/repos/OasisLMF/ODS_OpenExposureData/actions/artifacts" --jq '.artifacts[] | select(.workflow_run.id == '$JOB_ID') | select(.expired == false) | select(.name | startswith("extracted_spec")) | .id')
+         echo "JSON ID: $JSON_ID"
+         echo "JSON_ID=$JSON_ID" >> "$GITHUB_ENV"
+
+         echo "PR Number: $PR_NUMBER"
+         echo "PR_NUMBER=$PR_NUMBER" >> "$GITHUB_ENV"
+
+         echo "Head sha: $HEAD_SHA"
+         echo "HEAD_SHA=$HEAD_SHA" >> "$GITHUB_ENV"
+
+    - name: Update Comment
+      env:
+        JOB_PATH: "${{ github.server_url }}/${{ github.repository }}/actions/\
+          runs/${{ env.JOB_ID }}"
+        HEAD_SHA: ${{ env.HEAD_SHA }}
+      uses: peter-evans/create-or-update-comment@v2
+      with:
+        issue-number: ${{ env.PR_NUMBER }}
+        body: |-
+          ## Build Preview
+
+          You can find files attached to the below linked Workflow Run URL (Logs).
+          Please note that files only stay for around 14 days!
+
+          | Name      | Link
+          ------------|-------------------------------------------------------
+          | Commit    | ${{ env.HEAD_SHA }}
+          | Build      | ${{ env.JOB_PATH }}
+          | Excel File | [excel_spec.zip](https://github.com/OasisLMF/ODS_OpenExposureData/actions/runs/${{ env.JOB_ID }}/artifacts/${{ env.EXCEL_ID }})
+          | JSON File | [extracted_spec.zip](https://github.com/OasisLMF/ODS_OpenExposureData/actions/runs/${{ env.JOB_ID }}/artifacts/${{ env.JSON_ID }})
+
+          [badge]: https://img.shields.io/badge/Build-Success!-3fb950?logo=github&style=for-the-badge
+
   test:
     name: Run Pytest
     runs-on: ubuntu-latest
@@ -45,7 +109,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Download package
-      uses: actions/download-artifact@v4.1.7
+      uses: actions/download-artifact@v4
       with:
         name: bin_package
         path: ${{ github.workspace }}/

diff --git a/README.md b/README.md
@@ -70,17 +70,17 @@ https://oasislmf.org/open-data-standards
 
 ## Releases 
 
-| Version |  Excel Spec | JSON Spec | Release Notes | 
-| ------- | ----------- | --------- | ------------- |
-| 2.0.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/2.0.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/2.0.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/2.0.0) | 
-| 2.1.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/2.1.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/2.1.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/2.1.0) |
-| 2.2.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/2.2.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/2.2.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/2.2.0) |
-| 2.3.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/2.3.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/2.3.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/2.3.0) |
-| 3.0.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/3.0.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/3.0.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/3.0.0) |
-| 3.1.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/3.1.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/3.1.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/3.1.0) |
-| 3.2.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/3.2.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/3.2.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/3.2.0) |
-| 3.3.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/3.3.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/3.3.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/3.3.0) |
-| 3.4.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/3.4.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/3.4.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/3.4.0) |
+ | Version |  Excel Spec | JSON Spec | Release Notes | 
+ | ------- | ----------- | --------- | ------------- |
+ | 2.0.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/2.0.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/2.0.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/2.0.0) | 
+ | 2.1.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/2.1.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/2.1.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/2.1.0) |
+ | 2.2.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/2.2.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/2.2.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/2.2.0) |
+ | 2.3.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/2.3.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/2.3.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/2.3.0) |
+ | 3.0.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/3.0.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/3.0.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/3.0.0) |
+ | 3.1.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/3.1.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/3.1.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/3.1.0) |
+ | 3.2.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/3.2.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/3.2.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/3.2.0) |
+ | 3.3.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/3.3.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/3.3.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/3.3.0) |
+ | 3.4.0 | [OpenExposureData_Spec.xlsx](https://github.com/OasisLMF/ODS_OpenExposureData/raw/3.4.0/OpenExposureData/Docs/OpenExposureData_Spec.xlsx) | [OpenExposureData_Spec.json](https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/3.4.0/OpenExposureData_Spec.json) | [Release Page](https://github.com/OasisLMF/ODS_OpenExposureData/releases/tag/3.4.0) |
 <!-- NEW VERSION TAG -->
 
 

diff --git a/utils/extract_spec.py b/utils/extract_spec.py
@@ -230,7 +230,7 @@ def get_occupancy(occupancy_df):
             [
                 "Category",
                 "OED Code",
-                "AIR code",
+                "CEDE code",
                 "Name",
                 "Description",
                 "Code Range",
@@ -256,7 +256,7 @@ def get_construction(construction_df):
             [
                 "Category",
                 "OED Code",
-                "AIR code",
+                "CEDE code",
                 "Name",
                 "Description",
                 "Code Range",

diff --git a/utils/gen-json.py b/utils/gen-json.py
@@ -192,7 +192,7 @@ def get_occupancy(occupancy_df):
             [
                 "Category",
                 "OED Code",
-                "AIR code",
+                "CEDE code",
                 "Name",
                 "Description",
                 "Code Range",
@@ -218,7 +218,7 @@ def get_construction(construction_df):
             [
                 "Category",
                 "OED Code",
-                "AIR code",
+                "CEDE code",
                 "Name",
                 "Description",
                 "Code Range",

diff --git a/utils/update-readme-release.sh b/utils/update-readme-release.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+BASE_URL="https://github.com/OasisLMF/ODS_OpenExposureData/releases"
+REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
+
+README_PATH=$REPO_DIR/README.md
+README_MATCH='<!-- NEW VERSION TAG -->'
+
+function usage {
+  echo
+  echo "Usage: $0 <release-tag>"
+  echo
+  echo "This script appends release assest links to the repos README.md file"
+  echo " where '<release-tag>' is the github tag linked to a released version of OED"
+  echo
+  echo ""
+  exit 0
+}
+
+function check_input {
+    input_var=$1
+
+    VALID=$(echo $input_var | grep -oPc "^(\d+)\.(\d+)\.(\d+)$")
+    if [[ ! "$VALID" == 1 ]]; then
+        echo "Release Tag is not given or invalid Semvar, input_var=$input_var"
+        usage
+    fi
+}
+
+check_input $1
+
+TAG=$1
+RELEASE_LINK_EXCEL="| $TAG | [OpenExposureData_Spec.xlsx]($BASE_URL/download/$TAG/OpenExposureData_Spec.xlsx )"
+RELEASE_LINK_JSON=" | [OpenExposureData_Spec.json]($BASE_URL/download/$TAG/OpenExposureData_Spec.json )"
+RELEASE_LINK_NOTE=" | [Release Page]($BASE_URL/tag/$TAG) |"
+
+sed -i "/$README_MATCH/i\ $RELEASE_LINK_EXCEL$RELEASE_LINK_JSON$RELEASE_LINK_NOTE" $README_PATH