feat: Build fragment images with only changed content

Previously we always added all FBC operators to fragment image. This
however showed a downside when pipeline is manually restarted. Due to a
restart the pipeline can include content to fragment that is outdated or
even removed.

This change detects which catalog operators have been updated and only
add these to the fragment. It reduces a image size and avoid race
condition when pipeline is manually rebooted.

JIRA: ISV-6459

Signed-off-by: Ales Raszka <[email protected]>

Author: Allda

.github/workflows/build-and-test.yml

@@ -5,6 +5,7 @@ on:  # yamllint disable-line rule:truthy
   push:
     branches:
       - main
+      - ISV-6495
   pull_request:
     types: [opened, synchronize, reopened, labeled]
   workflow_dispatch:
@@ -92,7 +93,7 @@ jobs:
           - isv-fbc-catalog
 
       fail-fast: false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v5
 
@@ -105,6 +106,8 @@ jobs:
         run: |
           # Install python packages needed from ansible
           pipx inject ansible-core jmespath openshift pygithub
+          pipx list
+          ansible --version
 
           # Add certificates to trusted list
           sudo cp  operator-pipeline-images/certs/* /usr/local/share/ca-certificates
@@ -134,7 +137,7 @@ jobs:
             ansible/vaults/integration-tests/ci-pipeline-github-ssh-key
 
       - name: Run the integration tests ansible playbook
-        uses: dawidd6/action-ansible-playbook@v4
+        uses: dawidd6/action-ansible-playbook@v5
         with:
           playbook: playbooks/operator-pipeline-integration-tests.yml
           directory: ./ansible

.github/workflows/deploy.yml

@@ -37,7 +37,7 @@ jobs:
           # Install python packages needed from ansible
           pipx inject ansible-core jmespath openshift pygithub
       - name: Deploy dev environment
-        uses: dawidd6/action-ansible-playbook@v4
+        uses: dawidd6/action-ansible-playbook@v5
         with:
           playbook: playbooks/deploy.yml
           directory: ./ansible
@@ -67,7 +67,7 @@ jobs:
           # Install python packages needed from ansible
           pipx inject ansible-core jmespath openshift pygithub
       - name: Deploy qa environment
-        uses: dawidd6/action-ansible-playbook@v4
+        uses: dawidd6/action-ansible-playbook@v5
         with:
           playbook: playbooks/deploy.yml
           directory: ./ansible
@@ -98,7 +98,7 @@ jobs:
           # Install python packages needed from ansible
           pipx inject ansible-core jmespath openshift pygithub
       - name: Deploy stage environment
-        uses: dawidd6/action-ansible-playbook@v4
+        uses: dawidd6/action-ansible-playbook@v5
         with:
           playbook: playbooks/deploy.yml
           directory: ./ansible
@@ -128,7 +128,7 @@ jobs:
           # Install python packages needed from ansible
           pipx inject ansible-core jmespath openshift pygithub
       - name: Deploy prod environment
-        uses: dawidd6/action-ansible-playbook@v4
+        uses: dawidd6/action-ansible-playbook@v5
         with:
           playbook: playbooks/deploy.yml
           directory: ./ansible

ansible/roles/operator-pipeline/tasks/index-img-bootstrap-signing.yml

@@ -11,7 +11,7 @@
       metadata:
         name: "{{ oc_index_bootstrap_namespace }}"
         annotations:
-          operator.tekton.dev/prune.keep: "{{ tekton_pruner_keep | string }}"
+          operator.tekton.dev/prune.keep: "{{ tekton_pruner_keep | int }}"
 
 - name: Create Tekton access role
   tags:

ansible/roles/operator-pipeline/tasks/index-img-signing.yml

@@ -11,7 +11,7 @@
       metadata:
         name: "{{ oc_signing_namespace }}"
         annotations:
-          operator.tekton.dev/prune.keep: "{{ tekton_pruner_keep | string }}"
+          operator.tekton.dev/prune.keep: "{{ tekton_pruner_keep | int }}"
 
 - name: Include signing secrets
   ansible.builtin.include_tasks: tasks/index-img-signing-secrets.yml

ansible/roles/operator-pipeline/tasks/ocp_namespace.yml

@@ -10,4 +10,4 @@
       metadata:
         name: "{{ oc_namespace }}"
         annotations:
-          operator.tekton.dev/prune.keep: "{{ tekton_pruner_keep | string }}"
+          operator.tekton.dev/prune.keep: "{{ tekton_pruner_keep | int }}"

ansible/roles/operator-pipeline/tasks/webhook-dispatcher.yml

@@ -44,7 +44,7 @@
   delay: 3
 
 - name: Create a webhook dispatcher service account
-  ansible.builtin.include_tasks: tasks/webhook-dispatcher-sa.yml
+  ansible.builtin.import_tasks: tasks/webhook-dispatcher-sa.yml
 
 - name: Create a webhook dispatcher config map
   kubernetes.core.k8s:
@@ -78,6 +78,7 @@
   ansible.builtin.uri:
     url: "{{ operator_pipeline_webhook_dispatcher_base_url }}/api/v1/status/db"
     method: GET
+    status_code: 200
   register: webhook_dispatcher_status
   until: webhook_dispatcher_status.status == 200
   retries: 20

ansible/roles/operator-pipeline/templates/openshift/pipelines/operator-hosted-pipeline.yml

@@ -463,8 +463,8 @@ spec:
           value: "$(params.pipeline_image)"
         - name: commit_sha
           value: $(params.git_commit)
-        - name: affected_catalogs
-          value: "$(tasks.detect-changes.results.catalogs_with_added_or_modified_operators)"
+        - name: affected_catalog_operators
+          value: "$(tasks.detect-changes.results.added_or_modified_catalog_operators)"
         - name: fragment_repository
           value: "$(params.registry)/$(params.image_namespace)/catalog"
       workspaces:

ansible/roles/operator-pipeline/templates/openshift/pipelines/operator-release-pipeline.yml

@@ -401,48 +401,6 @@ spec:
         - name: registry-credentials
           workspace: registry-serve-credentials
 
-    # Automatically add bundle to FBC based on operators release config file
-    - name: add-bundle-to-fbc
-      runAfter:
-        - publish-pyxis-data
-      taskRef:
-        kind: Task
-        name: add-bundle-to-fbc
-      when:
-        - input: "$(tasks.detect-changes.results.added_bundle)"
-          operator: notin
-          values: [""]
-      params:
-        - name: pipeline_image
-          value: "$(params.pipeline_image)"
-        - name: bundle_pullspec
-          value: "$(tasks.copy-bundle-image-to-released-registry.results.image_pullspec)"
-        - name: operator_name
-          value: "$(tasks.detect-changes.results.added_operator)"
-        - name: operator_version
-          value: "$(tasks.detect-changes.results.added_bundle)"
-        - name: git_base_branch
-          value: "$(params.git_base_branch)"
-        - name: github_origin_pr_url
-          value: "$(params.git_pr_url)"
-        - name: github_token_secret_name
-          value: "$(params.github_token_secret_name)"
-        - name: github_token_secret_key
-          value: "$(params.github_token_secret_key)"
-        - name: create_pull_request
-          value: "true"
-      workspaces:
-        - name: source
-          workspace: repository
-          subPath: src
-        - name: output
-          workspace: results
-          subPath: summary
-        - name: ssh-directory
-          workspace: ssh-dir
-        - name: registry-credentials
-          workspace: registry-pull-credentials
-
     # Get the bundle versions to build index
     - name: get-supported-versions
       runAfter:
@@ -480,8 +438,8 @@ spec:
           value: "$(params.pipeline_image)"
         - name: commit_sha
           value: $(params.git_commit)
-        - name: affected_catalogs
-          value: "$(tasks.detect-changes.results.catalogs_with_added_or_modified_operators)"
+        - name: affected_catalog_operators
+          value: "$(tasks.detect-changes.results.added_or_modified_catalog_operators)"
         - name: fragment_repository
           value: "$(params.registry)/$(params.image_namespace)/catalog"
       workspaces:
@@ -650,6 +608,47 @@ spec:
         - name: results
           workspace: results
 
+    - name: add-bundle-to-fbc
+      runAfter:
+        - copy-bundle-image-to-released-registry
+      taskRef:
+        kind: Task
+        name: add-bundle-to-fbc
+      when:
+        - input: "$(tasks.detect-changes.results.added_bundle)"
+          operator: notin
+          values: [""]
+      params:
+        - name: pipeline_image
+          value: "$(params.pipeline_image)"
+        - name: bundle_pullspec
+          value: "$(tasks.copy-bundle-image-to-released-registry.results.image_pullspec)"
+        - name: operator_name
+          value: "$(tasks.detect-changes.results.added_operator)"
+        - name: operator_version
+          value: "$(tasks.detect-changes.results.added_bundle)"
+        - name: git_base_branch
+          value: "$(params.git_base_branch)"
+        - name: github_origin_pr_url
+          value: "$(params.git_pr_url)"
+        - name: github_token_secret_name
+          value: "$(params.github_token_secret_name)"
+        - name: github_token_secret_key
+          value: "$(params.github_token_secret_key)"
+        - name: create_pull_request
+          value: "true"
+      workspaces:
+        - name: source
+          workspace: repository
+          subPath: src
+        - name: output
+          workspace: results
+          subPath: summary
+        - name: ssh-directory
+          workspace: ssh-dir
+        - name: registry-credentials
+          workspace: registry-pull-credentials
+
   finally:
 
     # Release the acquired resource

ansible/roles/operator-pipeline/templates/openshift/tasks/build-fragment-images.yml

@@ -7,8 +7,8 @@ spec:
   params:
     - name: pipeline_image
 
-    - name: affected_catalogs
-      description: Comma separated list of updated catalogs
+    - name: affected_catalog_operators
+      description: Comma separated list of updated operator catalogs
 
     - name: fragment_repository
       description: A repository where fragments are pushed
@@ -34,7 +34,7 @@ spec:
         #! /usr/bin/env bash
         set -xe
 
-        if [[ "$(params.affected_catalogs)" == "" ]]; then
+        if [[ "$(params.affected_catalog_operators)" == "" ]]; then
           echo "No affected catalogs, skipping fragment image build"
           exit 0
         fi
@@ -45,7 +45,7 @@ spec:
         fi
 
         build-fragment-images \
-          --catalog-names "$(params.affected_catalogs)" \
+          --catalog-operators "$(params.affected_catalog_operators)" \
           --repository-destination "$(params.fragment_repository)" \
           --tag-suffix "fragment-$(params.commit_sha)" \
           --verbose $EXTRA_ARGS

ansible/roles/operator-pipeline/templates/openshift/tasks/parse-repo-changes.yml

@@ -38,6 +38,10 @@ spec:
       description: |
         Comma separated list of operators added/updated/deleted inside
         a catalogs directory structure
+    - name: added_or_modified_catalog_operators
+      description: |
+        Comma separated list of operators added or modified inside a catalogs
+        directory structure
     - name: deleted_catalog_operators
       description: |
         Comma separated list of operators deleted from the catalogs
@@ -102,5 +106,8 @@ spec:
         affected_catalog_operators="$(jq -r '.affected_catalog_operators | join(",")' < changes.json)"
         echo -n $affected_catalog_operators > "$(results.affected_catalog_operators.path)"
 
+        added_or_modified_catalog_operators="$(jq -r '.added_or_modified_catalog_operators | join(",")' < changes.json)"
+        echo -n $added_or_modified_catalog_operators > "$(results.added_or_modified_catalog_operators.path)"
+
         deleted_catalog_operators="$(jq -r '.deleted_catalog_operators | join(",")' < changes.json)"
         echo -n $deleted_catalog_operators > "$(results.deleted_catalog_operators.path)"