Move send_message to non-blocking thread #6089
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| branches: ["*"] | |
| push: | |
| branches: ["main"] | |
| concurrency: | |
| group: ci-${{ github.workflow }}-${{ github.actor }}-${{ github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| static-analysis: | |
| name: Prospector Static Analysis | |
| runs-on: ubuntu-22.04 | |
| env: | |
| DJANGO_SETTINGS_MODULE: onadata.settings.github_actions_test | |
| strategy: | |
| fail-fast: false | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| architecture: "x64" | |
| cache: "pip" | |
| cache-dependency-path: | | |
| requirements/base.pip | |
| requirements/dev.pip | |
| requirements/azure.pip | |
| - name: Update apt sources | |
| run: sudo apt-get update | |
| - name: Install APT requirements | |
| run: sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev libxml2-dev libxslt-dev | |
| - name: Setup SSH Agent and add Github to known hosts | |
| env: | |
| SSH_AUTH_SOCK: /tmp/ssh-agent.sock | |
| run: | | |
| ssh-agent -a $SSH_AUTH_SOCK >> /dev/null | |
| ssh-add - <<< "${{ secrets.SSH_PRIVATE_KEY }}" | |
| mkdir -p ~/.ssh | |
| ssh-keyscan github.com > ~/.ssh/known_hosts | |
| - name: Install Pip requirements | |
| env: | |
| SSH_AUTH_SOCK: /tmp/ssh-agent.sock | |
| run: | | |
| pip install -U pip | |
| pip install --upgrade wheel setuptools | |
| pip install -r requirements/base.pip | |
| pip install -r requirements/dev.pip | |
| pip install -r requirements/azure.pip | |
| pip install PyYAML django-redis ${{ secrets.ECR_OPTIONAL_PACKAGES }} | |
| - name: Install linting tools | |
| run: pip install prospector==1.14.1 pylint==3.3.4 | |
| - name: Run Prospector | |
| run: prospector -X -s veryhigh onadata | |
| unit-tests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_path: | |
| - [ | |
| " Django Unit Tests (Libraries, Main, RestServices, SMS Support, Viewer, Messaging)", | |
| "python manage.py test onadata/libs onadata/apps/main onadata/apps/restservice onadata/apps/sms_support onadata/apps/viewer onadata/apps/messaging --noinput --timing --settings=onadata.settings.github_actions_test --verbosity=2 --parallel=4", | |
| ] | |
| - [ | |
| "Django Unit Tests API", | |
| "python manage.py test onadata/apps/api --noinput --timing --settings=onadata.settings.github_actions_test --verbosity=2 --parallel=4", | |
| ] | |
| - [ | |
| "Django Unit Tests Logger", | |
| "python manage.py test onadata/apps/logger --noinput --timing --settings=onadata.settings.github_actions_test --verbosity=2 --parallel=4", | |
| ] | |
| name: "${{ matrix.test_path[0] }}" | |
| runs-on: ubuntu-22.04 | |
| needs: static-analysis | |
| env: | |
| DJANGO_SETTINGS_MODULE: onadata.settings.github_actions_test | |
| services: | |
| postgres: | |
| image: postgis/postgis:15-3.5 | |
| env: | |
| POSTGRES_PASSWORD: onadata | |
| POSTGRES_DB: onadata | |
| POSTGRES_USER: onadata | |
| ports: | |
| - 5432:5432 | |
| # Set health checks to wait until postgres has started | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: "adopt" | |
| java-version: "8" | |
| - name: Setup SSH Agent and add Github to known hosts | |
| env: | |
| SSH_AUTH_SOCK: /tmp/ssh-agent.sock | |
| run: | | |
| ssh-agent -a $SSH_AUTH_SOCK >> /dev/null | |
| ssh-add - <<< "${{ secrets.SSH_PRIVATE_KEY }}" | |
| mkdir -p ~/.ssh | |
| ssh-keyscan github.com > ~/.ssh/known_hosts | |
| - name: Setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| architecture: "x64" | |
| cache: "pip" | |
| cache-dependency-path: | | |
| requirements/base.pip | |
| requirements/dev.pip | |
| requirements/azure.pip | |
| - name: Update apt sources | |
| run: sudo apt-get update | |
| - name: Install APT requirements | |
| run: sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev libxml2-dev libxslt-dev | |
| - name: Install Pip requirements | |
| env: | |
| SSH_AUTH_SOCK: /tmp/ssh-agent.sock | |
| run: | | |
| pip install -U pip | |
| pip install --upgrade wheel setuptools | |
| pip install -r requirements/base.pip | |
| pip install -r requirements/dev.pip | |
| pip install -r requirements/azure.pip | |
| pip install PyYAML django-redis ${{ secrets.ECR_OPTIONAL_PACKAGES }} | |
| - name: Run tests | |
| run: | | |
| ${{ matrix.test_path[1] }} | |
| security-check: | |
| name: Trivy Security Checks | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Check if repository is public | |
| id: check_visibility | |
| run: echo "IS_PUBLIC_REPO=$(if [ ${{ github.event.repository.private }} = false ]; then echo true; else echo false; fi)" >> $GITHUB_ENV | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup SSH Agent and add Github to known hosts | |
| env: | |
| SSH_AUTH_SOCK: /tmp/ssh-agent.sock | |
| run: | | |
| ssh-agent -a $SSH_AUTH_SOCK >> /dev/null | |
| ssh-add - <<< "${{ secrets.SSH_PRIVATE_KEY }}" | |
| mkdir -p ~/.ssh | |
| ssh-keyscan github.com > ~/.ssh/known_hosts | |
| - name: Update apt sources | |
| run: sudo apt-get update | |
| - name: Get the branch name | |
| id: get-branch-name | |
| if: github.event_name == 'push' | |
| run: echo "version=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV | |
| - name: Build Docker image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: ./docker/onadata-uwsgi/Dockerfile.ubuntu | |
| platforms: linux/amd64 | |
| push: false | |
| tags: | | |
| onaio/onadata:${{ github.head_ref || github.base_ref || env.version }} | |
| cache-from: type=registry,ref=onaio/onadata:${{ github.head_ref || github.base_ref || env.version }} | |
| cache-to: type=inline | |
| ssh: | | |
| default=/tmp/ssh-agent.sock | |
| build-args: | | |
| optional_packages=PyYAML django-redis ${{ secrets.ECR_OPTIONAL_PACKAGES }} | |
| - name: Run Trivy vulnerability scanner | |
| uses: aquasecurity/[email protected] | |
| if: github.event_name == 'pull_request' || github.event_name == 'push' | |
| with: | |
| image-ref: onaio/onadata:${{ github.head_ref || github.base_ref || env.version }} | |
| format: sarif | |
| ignore-unfixed: false | |
| severity: "CRITICAL,HIGH" | |
| scanners: "vuln" | |
| output: "trivy_results.sarif" | |
| - name: Install SARIF tools | |
| if: github.event_name == 'pull_request' || github.event_name == 'push' | |
| run: pip install sarif-tools | |
| - name: Convert SARIF to HTML | |
| if: github.event_name == 'pull_request' || github.event_name == 'push' | |
| run: sarif html -o trivy_results.html trivy_results.sarif | |
| - name: Upload Trivy HTML report as artifact | |
| uses: actions/upload-artifact@v4 | |
| if: github.event_name == 'pull_request' || github.event_name == 'push' | |
| with: | |
| name: trivy-html-report | |
| path: trivy_results.html | |
| retention-days: 30 | |
| - name: Upload vulnerability scan results | |
| uses: github/codeql-action/upload-sarif@v3 | |
| if: (github.event_name == 'push' || github.event_name == 'pull_request') && env.IS_PUBLIC_REPO == 'true' | |
| with: | |
| sarif_file: "trivy_results.sarif" | |
| - name: Create summary of trivy issues | |
| if: github.event_name == 'push' || github.event_name == 'pull_request' | |
| run: | | |
| summary=$(jq -r '.runs[0].tool.driver.rules as $rules | [.runs[0].results[] | $rules[.ruleIndex].properties.tags[] | select(. == "HIGH" or . == "CRITICAL" or . == "MEDIUM" or . == "LOW" or . == "UNKNOWN")] | group_by(.) | map({Severity: .[0], Count: length}) | .[] | [.Severity, .Count] | join(": ")' trivy_results.sarif | awk 'NR > 1 { printf(" | ") } {printf "%s",$0}') | |
| if [ -z "$summary" ] | |
| then | |
| summary="0 Issues" | |
| fi | |
| echo "SUMMARY=$summary" >> $GITHUB_ENV | |
| - name: Extract detailed vulnerabilities | |
| if: github.event_name == 'push' || github.event_name == 'pull_request' | |
| run: | | |
| details=$(jq -r ' | |
| .runs[0].tool.driver.rules as $rules | | |
| .runs[0].results[0:10] | | |
| map( | |
| . as $result | | |
| $rules[.ruleIndex] as $rule | | |
| ($rule.properties.tags[] | select(. == "HIGH" or . == "CRITICAL" or . == "MEDIUM" or . == "LOW")) as $severity | | |
| ($result.locations[0].message.text | split(":")[0] | if contains("site-packages/") then split("site-packages/")[1] elif contains("usr/lib/") then split("usr/lib/")[1] else (split("/") | .[-4:] | join("/")) end) as $location | | |
| ($result.message.text | capture("Package: (?<pkg>[^\n]+)") | .pkg) as $package | | |
| ($result.message.text | capture("Installed Version: (?<ver>[^\n]+)") | .ver) as $installed | | |
| ($result.message.text | capture("Fixed Version: (?<fix>[^\n]+)") | .fix // "Not available") as $fixed | | |
| (if $severity == "CRITICAL" then "🔴" elif $severity == "HIGH" then "🟠" elif $severity == "MEDIUM" then "🟡" else "⚪" end) as $icon | | |
| "\($icon) *\($result.ruleId)* (\($severity))\nPackage: `\($package)@\($installed)`\nLocation: `\($location)`\nFixed: `\($fixed)`\n<\($rule.helpUri)|View Details>\n" | |
| ) | join("\n") | |
| ' trivy_results.sarif) | |
| if [ -z "$details" ] | |
| then | |
| details="No vulnerabilities found in the scanned severity levels." | |
| fi | |
| # Escape for GitHub Actions environment | |
| echo "DETAILS<<EOF" >> $GITHUB_ENV | |
| echo "$details" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - name: Send Slack Notification | |
| uses: slackapi/[email protected] | |
| if: github.event_name == 'push' || github.event_name == 'pull_request' | |
| with: | |
| payload: | | |
| { | |
| "text": "Trivy scan results for ${{ github.head_ref || github.base_ref || env.version }}", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "[Ona Data] Trivy scan results for ${{ github.head_ref || github.base_ref || env.version }}: ${{ env.SUMMARY }}" | |
| } | |
| }, | |
| { | |
| "type": "divider" | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "🔍 *Found Vulnerabilities (Top 10):*" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": ${{ toJSON(env.DETAILS) }} | |
| } | |
| }, | |
| { | |
| "type": "divider" | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "📊 *View Full Reports:*\n• <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Download HTML Report>\n• <https://github.com/${{ github.repository }}/security/code-scanning?query=branch:${{ github.head_ref || github.base_ref || env.version }}+is:open++|GitHub Code Scanning>" | |
| } | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK |