diff --git a/examples-copier/.gitignore b/examples-copier/.gitignore new file mode 100644 index 0000000..21aa038 --- /dev/null +++ b/examples-copier/.gitignore @@ -0,0 +1,50 @@ +# Binaries +examples-copier +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool +*.out + +# Dependency directories +vendor/ + +# Go workspace file +go.work + +# Environment files with secrets +env.yaml +.env +.env.local +.env.production +.env.*.local + +# Private keys +*.pem +*.key + +# IDE files +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# OS files +.DS_Store +Thumbs.db + +# Logs +*.log + +# Temporary files +tmp/ +temp/ + +env.yaml diff --git a/examples-copier/Makefile b/examples-copier/Makefile index 3ea26a1..b586e4c 100644 --- a/examples-copier/Makefile +++ b/examples-copier/Makefile @@ -60,7 +60,16 @@ test-webhook: # Test with example payload test-webhook-example: test-webhook @echo "Testing with example payload..." - @./test-webhook -payload test-payloads/example-pr-merged.json + @if [ -z "$$WEBHOOK_SECRET" ]; then \ + echo "Fetching webhook secret from Secret Manager..."; \ + export WEBHOOK_SECRET=$$(gcloud secrets versions access latest --secret=webhook-secret 2>/dev/null); \ + fi; \ + if [ -n "$$WEBHOOK_SECRET" ]; then \ + ./test-webhook -payload test-payloads/example-pr-merged.json -secret "$$WEBHOOK_SECRET"; \ + else \ + echo "Warning: WEBHOOK_SECRET not set, sending without signature"; \ + ./test-webhook -payload test-payloads/example-pr-merged.json; \ + fi # Test with real PR (requires PR, OWNER, REPO variables) test-webhook-pr: test-webhook @@ -69,7 +78,16 @@ test-webhook-pr: test-webhook echo "Usage: make test-webhook-pr PR=123 OWNER=myorg REPO=myrepo"; \ exit 1; \ fi - @./test-webhook -pr $(PR) -owner $(OWNER) -repo $(REPO) + @if [ -z "$$WEBHOOK_SECRET" ]; then \ + echo "Fetching webhook secret from Secret Manager..."; \ + export WEBHOOK_SECRET=$$(gcloud secrets versions access latest --secret=webhook-secret 2>/dev/null); \ + fi; \ + if [ -n "$$WEBHOOK_SECRET" ]; then \ + ./test-webhook -pr $(PR) -owner $(OWNER) -repo $(REPO) -secret "$$WEBHOOK_SECRET"; \ + else \ + echo "Warning: WEBHOOK_SECRET not set, sending without signature"; \ + ./test-webhook -pr $(PR) -owner $(OWNER) -repo $(REPO); \ + fi # Test with real PR using helper script test-pr: diff --git a/examples-copier/QUICK-REFERENCE.md b/examples-copier/QUICK-REFERENCE.md index 783b80c..3d83912 100644 --- a/examples-copier/QUICK-REFERENCE.md +++ b/examples-copier/QUICK-REFERENCE.md @@ -382,6 +382,29 @@ curl http://localhost:8080/health curl http://localhost:8080/metrics | jq ``` +## Deployment + +### Google Cloud Quick Commands + +```bash +# Deploy (env.yaml is included via 'includes' directive in app.yaml) +gcloud app deploy app.yaml + +# View logs +gcloud app logs tail -s default + +# Check health +curl https://github-copy-code-examples.appspot.com/health + +# List secrets +gcloud secrets list + +# Grant access +./grant-secret-access.sh +``` + + + ## File Locations ``` @@ -390,7 +413,9 @@ examples-copier/ ├── MIGRATION-GUIDE.md # Migration from legacy ├── QUICK-REFERENCE.md # This file ├── REFACTORING-SUMMARY.md # Feature details -├── DEPLOYMENT-GUIDE.md # Deployment instructions +├── docs/ +│ ├── DEPLOYMENT.md # Deployment guide +│ └── DEPLOYMENT-CHECKLIST.md # Deployment checklist ├── TESTING-SUMMARY.md # Test documentation ├── configs/ │ ├── .env # Environment config diff --git a/examples-copier/README.md b/examples-copier/README.md index 54b0ae9..1dbb97c 100644 --- a/examples-copier/README.md +++ b/examples-copier/README.md @@ -66,7 +66,8 @@ GITHUB_INSTALLATION_ID=789012 # Google Cloud GCP_PROJECT_ID=your-project -PEM_KEY_NAME=projects/123/secrets/CODE_COPIER_PEM/versions/latest +PEM_KEY_NAME=projects/123/secrets//versions/latest +WEBHOOK_SECRET_NAME=projects/123/secrets/webhook-secret # Application Settings PORT=8080 @@ -415,7 +416,7 @@ container := NewServiceContainer(config) ## Deployment -See [DEPLOYMENT-GUIDE.md](DEPLOYMENT-GUIDE.md) for complete deployment instructions. +See [DEPLOYMENT.md](./docs/DEPLOYMENT.md) for complete deployment guide and [DEPLOYMENT-CHECKLIST.md](./docs/DEPLOYMENT-CHECKLIST.md) for step-by-step checklist. ### Google Cloud App Engine @@ -444,7 +445,8 @@ docker run -p 8080:8080 --env-file .env examples-copier - **[Configuration Guide](docs/CONFIGURATION-GUIDE.md)** - Complete configuration reference ⭐ NEW - **[Pattern Matching Guide](docs/PATTERN-MATCHING-GUIDE.md)** - Pattern matching with examples - **[Local Testing](docs/LOCAL-TESTING.md)** - Test locally before deploying -- **[Deployment Guide](docs/DEPLOYMENT-GUIDE.md)** - Deploy to production +- **[Deployment Guide](docs/DEPLOYMENT.md)** - Deploy to production +- **[Deployment Checklist](docs/DEPLOYMENT-CHECKLIST.md)** - Step-by-step deployment checklist ### Reference diff --git a/examples-copier/app.go b/examples-copier/app.go index 37b1d7d..ac99a28 100644 --- a/examples-copier/app.go +++ b/examples-copier/app.go @@ -39,6 +39,17 @@ func main() { os.Exit(1) } + // Load secrets from Secret Manager if not directly provided + if err := services.LoadWebhookSecret(config); err != nil { + fmt.Printf("❌ Error loading webhook secret: %v\n", err) + os.Exit(1) + } + + if err := services.LoadMongoURI(config); err != nil { + fmt.Printf("❌ Error loading MongoDB URI: %v\n", err) + os.Exit(1) + } + // Override dry-run from command line if dryRun { config.DryRun = true diff --git a/examples-copier/app.yaml b/examples-copier/app.yaml index 9e4885f..4b2ee75 100644 --- a/examples-copier/app.yaml +++ b/examples-copier/app.yaml @@ -3,3 +3,6 @@ runtime_config: operating_system: "ubuntu22" runtime_version: "1.23" env: flex + +includes: + - env.yaml diff --git a/examples-copier/cmd/test-webhook/main.go b/examples-copier/cmd/test-webhook/main.go index c3754be..e3bbeb2 100644 --- a/examples-copier/cmd/test-webhook/main.go +++ b/examples-copier/cmd/test-webhook/main.go @@ -20,7 +20,7 @@ func main() { prNumber := flag.Int("pr", 0, "PR number to fetch from GitHub") owner := flag.String("owner", "", "Repository owner") repo := flag.String("repo", "", "Repository name") - webhookURL := flag.String("url", "http://localhost:8080/webhook", "Webhook URL") + webhookURL := flag.String("url", "http://localhost:8080/events", "Webhook URL") secret := flag.String("secret", "", "Webhook secret for signature") payloadFile := flag.String("payload", "", "Path to custom payload JSON file") dryRun := flag.Bool("dry-run", false, "Print payload without sending") @@ -95,7 +95,7 @@ Options: -pr int PR number to fetch from GitHub -owner string Repository owner (required with -pr) -repo string Repository name (required with -pr) - -url string Webhook URL (default: http://localhost:8080/webhook) + -url string Webhook URL (default: http://localhost:8080/events) -secret string Webhook secret for HMAC signature -payload string Path to custom payload JSON file -dry-run Print payload without sending @@ -117,7 +117,7 @@ Examples: # Send to production with secret test-webhook -pr 123 -owner myorg -repo myrepo \ - -url https://myapp.appspot.com/webhook \ + -url https://myapp.appspot.com/events \ -secret "my-webhook-secret" Environment Variables: diff --git a/examples-copier/config.json b/examples-copier/config.json deleted file mode 100644 index f66e3a5..0000000 --- a/examples-copier/config.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "source_directory": "examples", - "target_repo": "mongodb/target-repo", - "target_branch": "main", - "target_directory": "docs/code-examples", - "recursive_copy": true, - "copier_commit_strategy": "pr", - "pr_title": "Update code examples", - "commit_message": "Sync examples from source repository", - "merge_without_review": false - } -] - diff --git a/examples-copier/configs/.env.example b/examples-copier/configs/.env.example deleted file mode 100644 index 19b62cb..0000000 --- a/examples-copier/configs/.env.example +++ /dev/null @@ -1,54 +0,0 @@ -# Sample .env for the Docs Examples Copier -# Required identifiers for the GitHub App installation -GITHUB_APP_ID="" -INSTALLATION_ID="" -# Optional (not used for JWT auth): OAuth client ID of your GitHub App -# GITHUB_APP_CLIENT_ID="" - -# Source repository (where config.json and deprecated_examples.json live) -REPO_OWNER="mongodb" -REPO_NAME="docs-code-examples" -SRC_BRANCH="main" - -# Author information for commits to the source repo -COMMITER_NAME="Copier Bot" -COMMITER_EMAIL="bot@example.com" - -# Web server configuration -PORT="8080" # omit or empty to use default 8080 -WEBSERVER_PATH="/webhook" # webhook route path - -# File names/paths in the source repo -CONFIG_FILE="config.json" -DEPRECATION_FILE="deprecated_examples.json" - -# Logging / GCP (optional) -GOOGLE_CLOUD_PROJECT_ID="github-copy-code-examples" -COPIER_LOG_NAME="copy-copier-log" -# Disable GCP logging in local/dev if desired (string boolean) -# COPIER_DISABLE_CLOUD_LOGGING="true" -# Enable debug logging either by LOG_LEVEL=debug or COPIER_DEBUG=true -# LOG_LEVEL="debug" -# COPIER_DEBUG="true" - -# Authentication: Secret Manager vs local -# If running locally/tests, you can skip GCP Secret Manager and provide the key via env -# SKIP_SECRET_MANAGER="true" -# GITHUB_APP_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----\n" -# or base64-encoded: -# GITHUB_APP_PRIVATE_KEY_B64="" -# Otherwise, when using GCP Secret Manager, set the fully-qualified secret name: -# PEM_NAME="projects/1234567890/secrets/CODE_COPIER_PEM/versions/latest" - -# BEHAVIOR DEFAULTS -# Global default commit message if not set at config level -DEFAULT_COMMIT_MESSAGE="Automatic copy of updated code examples" -# Default copy behavior if not set at config level (string booleans: "true" or "false") -# If "true", all files in source repo are recursively copied from path, otherwise only the top-level examples. -DEFAULT_RECURSIVE_COPY="true" -# Default write strategy if not set at config level: "direct" (commit directly) or "pr" (PR created) -# If "pr", the DEFAULT_PR_MERGE setting applies. -COPIER_COMMIT_STRATEGY="pr" -# When COPIER_COMMIT_STRATEGY is set to "pr", default PR merge behavior if not set at config level (string booleans: "true" or "false") -# If "true", PRs merge automatically if no conflicts, otherwise PR created but not merged. -DEFAULT_PR_MERGE="false" \ No newline at end of file diff --git a/examples-copier/configs/.env.example.new b/examples-copier/configs/.env.example.new deleted file mode 100644 index 5f6fa02..0000000 --- a/examples-copier/configs/.env.example.new +++ /dev/null @@ -1,97 +0,0 @@ -# ============================================================================= -# REQUIRED CONFIGURATION -# ============================================================================= - -# GitHub App Configuration (Required) -GITHUB_APP_CLIENT_ID="" -GITHUB_APP_ID="" -INSTALLATION_ID="" - -# Source Repository Configuration (Required) -REPO_NAME="" -REPO_OWNER="" - -# Security Configuration (Required) -PEM_NAME="projects//secrets//versions/latest" -WEBHOOK_SECRET="" - -# ============================================================================= -# OPTIONAL CONFIGURATION -# ============================================================================= - -# Committer Information (Optional - defaults provided) -COMMITER_EMAIL="copier@example.com" -COMMITER_NAME="GitHub Copier App" - -# Web Server Configuration (Optional - defaults provided) -PORT="3000" -WEBSERVER_PATH="/events" - -# File Configuration (Optional - defaults provided) -# Supports both JSON and YAML formats -CONFIG_FILE="copier-config.yaml" -DEPRECATION_FILE="deprecated_examples.json" - -# Source Branch (Optional - default: main) -SRC_BRANCH="main" - -# Google Cloud Logging (Optional) -GOOGLE_CLOUD_PROJECT_ID="" -COPIER_LOG_NAME="copy-copier-log" - -# ============================================================================= -# NEW FEATURES CONFIGURATION -# ============================================================================= - -# Dry Run Mode (Optional - default: false) -# When enabled, no actual changes are made to target repositories -DRY_RUN="false" - -# Audit Logging (Optional - default: false) -# Enable MongoDB audit logging for tracking all copy operations -AUDIT_ENABLED="false" -MONGO_URI="mongodb://localhost:27017" -AUDIT_DATABASE="copier_audit" -AUDIT_COLLECTION="events" - -# Metrics Collection (Optional - default: true) -# Enable metrics collection for /metrics endpoint -METRICS_ENABLED="true" - -# ============================================================================= -# LEGACY/DEPRECATED CONFIGURATION -# These are maintained for backward compatibility with JSON configs -# ============================================================================= - -# Default Recursive Copy (Optional - default: true) -DEFAULT_RECURSIVE_COPY="true" - -# Default PR Merge (Optional - default: false) -DEFAULT_PR_MERGE="false" - -# Default Commit Message (Optional) -DEFAULT_COMMIT_MESSAGE="Automated PR with updated examples" - -# ============================================================================= -# EXAMPLE CONFIGURATIONS -# ============================================================================= - -# Example 1: Local Development with Dry Run -# DRY_RUN="true" -# AUDIT_ENABLED="false" -# METRICS_ENABLED="true" -# CONFIG_FILE="copier-config.example.yaml" - -# Example 2: Production with Audit Logging -# DRY_RUN="false" -# AUDIT_ENABLED="true" -# MONGO_URI="mongodb+srv://user:pass@cluster.mongodb.net" -# AUDIT_DATABASE="copier_audit" -# METRICS_ENABLED="true" - -# Example 3: Staging Environment -# CONFIG_FILE="copier-config.staging.yaml" -# DRY_RUN="false" -# AUDIT_ENABLED="true" -# METRICS_ENABLED="true" - diff --git a/examples-copier/configs/.env.local b/examples-copier/configs/.env.local.example similarity index 60% rename from examples-copier/configs/.env.local rename to examples-copier/configs/.env.local.example index 387e19c..4affbac 100644 --- a/examples-copier/configs/.env.local +++ b/examples-copier/configs/.env.local.example @@ -1,14 +1,25 @@ # Local Development Environment Configuration -# Copy this file to .env for local testing + +# To use this file, copy it to .env and edit with your values: +# cp configs/.env.local configs/.env +# source configs/.env +# ./examples-copier + +# Or use with make: +# make run-dry + +# Or run directly: +# COPIER_DISABLE_CLOUD_LOGGING=true DRY_RUN=true ./examples-copier # ============================================================================ # REQUIRED FOR LOCAL TESTING # ============================================================================ -# Repository Configuration -REPO_OWNER=mongodb -REPO_NAME=docs-realm -SRC_BRANCH=main +# Source Repository Configuration (Required) +REPO_NAME="" +REPO_OWNER="" +# Source Branch (Optional - default: main) +SRC_BRANCH="test" # Configuration Files CONFIG_FILE=copier-config.yaml @@ -104,19 +115,67 @@ SLACK_ICON_EMOJI=:robot_face: # Enable/disable Slack notifications (default: true if webhook URL is set) SLACK_ENABLED=false -# ============================================================================ -# NOTES -# ============================================================================ - -# To use this file: -# cp configs/.env.local configs/.env -# # Edit .env with your values -# source configs/.env -# ./examples-copier - -# Or use with make: -# make run-dry - -# Or run directly: -# COPIER_DISABLE_CLOUD_LOGGING=true DRY_RUN=true ./examples-copier +# ============================================================================= +# EXAMPLE CONFIGURATIONS +# ============================================================================= + +# Example 1: Local Development with Dry Run +# DRY_RUN="true" +# AUDIT_ENABLED="false" +# METRICS_ENABLED="true" +# CONFIG_FILE="copier-config.example.yaml" +# WEBHOOK_SECRET="test-secret-123" + +# Example 2: Local Development with MongoDB Audit Logging +# DRY_RUN="false" +# AUDIT_ENABLED="true" +# MONGO_URI="mongodb://localhost:27017" +# AUDIT_DATABASE="copier_audit" +# METRICS_ENABLED="true" + +# Example 3: Local Development with Slack Notifications +# DRY_RUN="true" +# SLACK_WEBHOOK_URL="https://hooks.slack.com/services/YOUR/WEBHOOK/URL" +# SLACK_CHANNEL="#code-examples-dev" +# SLACK_ENABLED="true" + +# ============================================================================= +# NOTES FOR LOCAL DEVELOPMENT +# ============================================================================= + +# 1. DIRECT SECRETS (Recommended for local dev): +# - Use WEBHOOK_SECRET instead of WEBHOOK_SECRET_NAME +# - Use MONGO_URI instead of MONGO_URI_SECRET_NAME +# - Use PEM_NAME pointing to Secret Manager (still needed for GitHub auth) +# +# 2. DRY RUN MODE: +# - Set DRY_RUN="true" to test without making actual commits/PRs +# - Webhooks are processed, files matched, but no changes made +# +# 3. MONGODB: +# - For local testing, use mongodb://localhost:27017 +# - Or disable audit logging: AUDIT_ENABLED="false" +# +# 4. SLACK: +# - Optional for local dev +# - Useful for testing error notifications +# +# 5. GOOGLE CLOUD: +# - You still need PEM_NAME for GitHub App authentication +# - Set up Secret Manager even for local dev +# - Or use a local PEM file (not recommended) +# +# 6. TESTING: +# - Use CONFIG_FILE="copier-config.example.yaml" for testing +# - Set WEBSERVER_PATH="/events" to match GitHub webhook +# - Use PORT="3000" or any available port + +# ============================================================================= +# SEE ALSO +# ============================================================================= + +# - env.yaml.example - Complete reference for all variables +# - env.yaml.production - Production deployment template +# - ../configs/README.md - Comparison of all env files +# - ../docs/LOCAL-TESTING.md - Local development guide diff --git a/examples-copier/configs/.env.test b/examples-copier/configs/.env.test deleted file mode 100644 index dcedb1b..0000000 --- a/examples-copier/configs/.env.test +++ /dev/null @@ -1,26 +0,0 @@ -GITHUB_APP_CLIENT_ID="Iv23licLjdtAHKK4QBuc" -INSTALLATION_ID="62138132" -GITHUB_APP_ID="1166559" - -REPO_OWNER="mongodb" -REPO_NAME="docs-code-examples" -SRC_BRANCH="main" - -COMMITER_NAME="GitHub Copier App" -COMMITER_EMAIL="bot@mongodb.com" - -PORT:"3000" -WEBSERVER_PATH="/" - -CONFIG_FILE="config.json" -DEPRECATION_FILE="deprecated_examples.json" - -COPIER_DISABLE_CLOUD_LOGGING="true" -LOG_LEVEL="debug" - -SKIP_SECRET_MANAGER="true" -GITHUB_APP_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQ...\n-----END PRIVATE KEY-----\n" - -DEFAULT_COMMIT_MESSAGE="Test commit message for code copier" -DEFAULT_RECURSIVE_COPY="false" -COPIER_COMMIT_STRATEGY="direct" \ No newline at end of file diff --git a/examples-copier/configs/README.md b/examples-copier/configs/README.md new file mode 100644 index 0000000..d6e6562 --- /dev/null +++ b/examples-copier/configs/README.md @@ -0,0 +1,272 @@ +# Environment Files Comparison + +Overview of the different environment configuration files and when to use each. + +## Files Overview + +| File | Purpose | Use Case | +|-----------------------|---------------------------------------|---------------------------------| +| `env.yaml.example` | Complete reference with all variables | First-time setup, documentation | +| `env.yaml.production` | Production-ready template | Quick deployment to production | +| `.env.example` | Local development template | Local testing and development | + +--- + +## env.yaml.example + +**Location:** `configs/env.yaml.example` + +**Purpose:** Comprehensive reference showing ALL possible environment variables + +**Contents:** +- ✅ All 30+ supported variables +- ✅ Detailed comments for each variable +- ✅ Default values shown +- ✅ Security best practices +- ✅ Deployment notes +- ✅ Local development tips + +**Use this when:** +- Setting up for the first time +- Need to understand all available options +- Want to see what features are available +- Need reference documentation + +--- + +## env.yaml.production + +**Location:** `configs/env.yaml.production` + +**Purpose:** Production-ready template with sensible defaults + +**Contents:** +- ✅ Required variables pre-filled with MongoDB values +- ✅ Secret Manager references (recommended approach) +- ✅ Essential settings only +- ✅ Audit logging enabled +- ✅ Metrics enabled +- ✅ Production-optimized + +**Use this when:** +- Deploying to production quickly +- Want a minimal, clean configuration +- Using Secret Manager (recommended) +- Don't need advanced features + +**NOT included:** +- Slack notifications (optional) +- MongoDB direct URI (use Secret Manager instead) +- Advanced options (rarely needed) + +--- + +## .env.example.new + +**Location:** `configs/.env.example.new` + +**Purpose:** Local development template (traditional .env format) + +**Contents:** +- ✅ .env format (KEY=value) +- ✅ Suitable for local testing +- ✅ Works with godotenv +- ✅ Can use direct secrets (not Secret Manager) + +**Use this when:** +- Developing locally +- Testing without Google Cloud +- Using `go run` or local server +- Don't want to set up Secret Manager + +**Format difference:** +```bash +# .env format (for local development) +GITHUB_APP_ID=123456 +REPO_OWNER=mongodb +REPO_NAME=docs-code-examples +``` + +vs + +```yaml +# env.yaml format (for App Engine deployment) +env_variables: + GITHUB_APP_ID: "123456" + REPO_OWNER: "mongodb" + REPO_NAME: "docs-code-examples" +``` + +--- + +## Usage Scenarios + +### Scenario 1: First-Time Production Deployment + +**Recommended:** `env.yaml.production` + +```bash +# Quick start +cp configs/env.yaml.production env.yaml +nano env.yaml # Update PROJECT_NUMBER and values +./scripts/grant-secret-access.sh +gcloud app deploy app.yaml # env.yaml is included via 'includes' directive +``` + +**Why:** Pre-configured with production best practices, minimal setup required. + +--- + +### Scenario 2: Need to Understand All Options + +**Recommended:** `env.yaml.example` + +```bash +# Reference all options +cat configs/env.yaml.example + +# Copy and customize +cp configs/env.yaml.example env.yaml +nano env.yaml # Enable features you need +``` + +**Why:** Shows all available features with detailed explanations. + +--- + +### Scenario 3: Local Development + +**Recommended:** `.env.example.new` + +```bash +# Local development +cp configs/.env.example.new configs/.env +nano configs/.env # Add your values + +# Run locally +go run app.go -env configs/.env +``` + +**Why:** Simpler format for local testing, no Secret Manager required. + +--- + +### Scenario 4: Custom Production Setup + +**Recommended:** Start with `env.yaml.example`, customize + +```bash +# Start with full reference +cp configs/env.yaml.example env.yaml + +# Enable features you need +nano env.yaml +# - Enable Slack notifications +# - Configure custom MongoDB settings +# - Set custom defaults + +# Deploy +gcloud app deploy app.yaml # env.yaml is included via 'includes' directive +``` + +**Why:** Need advanced features not in production template. + +--- + +## Migration Guide + +### From .env to env.yaml + +Use the conversion script: + +```bash +./scripts/convert-env-to-yaml.sh configs/.env env.yaml +``` + +Or manually convert: + +```bash +# .env format: +GITHUB_APP_ID=123456 +REPO_OWNER=mongodb + +# env.yaml format: +env_variables: + GITHUB_APP_ID: "123456" + REPO_OWNER: "mongodb" +``` + +### From env.yaml.production to env.yaml.example + +```bash +# Start with production template +cp configs/env.yaml.production env.yaml + +# Add optional features from example +# Compare files and add what you need: +diff configs/env.yaml.production configs/env.yaml.example +``` + +--- + +## Best Practices + +### ✅ DO + +- **Use `env.yaml.production` for quick production deployment** +- **Use `env.yaml.example` as reference documentation** +- **Use `.env.example.new` for local development** +- **Add `env.yaml` and `.env` to `.gitignore`** +- **Use Secret Manager for production secrets** +- **Keep comments in your env.yaml for team documentation** + +### ❌ DON'T + +- **Don't commit `env.yaml` or `.env` with actual secrets** +- **Don't use direct secrets in production (use Secret Manager)** +- **Don't mix .env and env.yaml formats** +- **Don't remove all comments (they help future you)** +- **Don't use `env.yaml.production` as-is (update values first)** + +--- + +## File Locations + +``` +examples-copier/ +├── configs/ +│ ├── env.yaml.example # ← Complete reference (all variables) +│ ├── env.yaml.production # ← Production template (essential only) +│ └── .env.example # ← Local development template +├── env.yaml # ← Your actual config (gitignored) +└── .env # ← Your local config (gitignored) +``` + +--- + +## Quick Reference + +**Need to deploy quickly?** +→ Use `env.yaml.production` + +**Need to understand all options?** +→ Read `env.yaml.example` + +**Need to develop locally?** +→ Use `.env.example.new` + +**Need advanced features?** +→ Start with `env.yaml.example`, customize + +**Need to convert formats?** +→ Use `./scripts/convert-env-to-yaml.sh` + +--- + +## See Also + +- [CONFIGURATION-GUIDE.md](../docs/CONFIGURATION-GUIDE.md) - Variable validation and reference +- [DEPLOYMENT.md](../docs/DEPLOYMENT.md) - Complete deployment guide +- [DEPLOYMENT-CHECKLIST.md](../docs/DEPLOYMENT-CHECKLIST.md) - Step-by-step checklist +- [LOCAL-TESTING.md](../docs/LOCAL-TESTING.md) - Local development guide + diff --git a/examples-copier/configs/env.yaml.example b/examples-copier/configs/env.yaml.example new file mode 100644 index 0000000..59b7584 --- /dev/null +++ b/examples-copier/configs/env.yaml.example @@ -0,0 +1,181 @@ +env_variables: + # ============================================================================= + # REQUIRED VARIABLES + # ============================================================================= + + # GitHub App Configuration + GITHUB_APP_ID: "YOUR_GITHUB_APP_ID" # Your GitHub App ID (required) + INSTALLATION_ID: "YOUR_INSTALLATION_ID" # GitHub App Installation ID (required) + REPO_OWNER: "your-org" # Source repository owner (required) + REPO_NAME: "your-repo" # Source repository name (required) + + # ============================================================================= + # SECRET MANAGER REFERENCES (RECOMMENDED - Most Secure) + # ============================================================================= + + # GitHub App Private Key - loaded from Secret Manager + # Format: projects/PROJECT_NUMBER/secrets/SECRET_NAME/versions/VERSION + GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/YOUR_PROJECT_NUMBER/secrets/CODE_COPIER_PEM/versions/latest" + + # Webhook Secret - loaded from Secret Manager (for signature verification) + WEBHOOK_SECRET_NAME: "projects/YOUR_PROJECT_NUMBER/secrets/webhook-secret/versions/latest" + + # MongoDB URI - loaded from Secret Manager (for audit logging - OPTIONAL) + MONGO_URI_SECRET_NAME: "projects/YOUR_PROJECT_NUMBER/secrets/mongo-uri/versions/latest" + + # ============================================================================= + # ALTERNATIVE: DIRECT SECRETS (NOT RECOMMENDED - Less Secure) + # ============================================================================= + # Only use these if you're NOT using Secret Manager + # WARNING: Never commit actual secrets to version control! + + # PEM_NAME: "projects/YOUR_PROJECT_NUMBER/secrets/CODE_COPIER_PEM/versions/latest" + # WEBHOOK_SECRET: "your-webhook-secret-here" + # MONGO_URI: "mongodb+srv://user:pass@cluster.mongodb.net/dbname" + + # ============================================================================= + # APPLICATION SETTINGS + # ============================================================================= + + # Web Server Configuration + # PORT: "8080" # DO NOT SET - App Engine Flexible sets this automatically + WEBSERVER_PATH: "/events" # Webhook endpoint path (default: /webhook) + + # Configuration Files + CONFIG_FILE: "copier-config.yaml" # Config file name in source repo (default: copier-config.yaml) + DEPRECATION_FILE: "deprecated_examples.json" # Deprecation tracking file (default: deprecated_examples.json) + + # Source Branch + SRC_BRANCH: "main" # Branch to copy from (default: main) + + # ============================================================================= + # COMMITTER INFORMATION + # ============================================================================= + # Used when committing deprecation file updates to source repo + + COMMITTER_NAME: "GitHub Copier App" # Git committer name (default: Copier Bot) + COMMITTER_EMAIL: "bot@example.com" # Git committer email (default: bot@example.com) + + # ============================================================================= + # GOOGLE CLOUD CONFIGURATION + # ============================================================================= + # For Google Cloud Logging integration + + GOOGLE_CLOUD_PROJECT_ID: "your-project-id" # GCP Project ID (default: github-copy-code-examples) + COPIER_LOG_NAME: "code-copier-log" # Cloud Logging log name (default: copy-copier-log) + + # ============================================================================= + # LOGGING CONFIGURATION + # ============================================================================= + # Control logging verbosity and output + + # LOG_LEVEL: "debug" # Set to "debug" for verbose logging (default: info) + # COPIER_DEBUG: "true" # Alternative way to enable debug mode (default: false) + # COPIER_DISABLE_CLOUD_LOGGING: "true" # Disable Google Cloud Logging (useful for local dev) + + # ============================================================================= + # FEATURE FLAGS + # ============================================================================= + + # Dry Run Mode - process webhooks but don't make actual commits/PRs + DRY_RUN: "false" # Enable dry-run mode (default: false) + + # Audit Logging - log all operations to MongoDB + AUDIT_ENABLED: "true" # Enable audit logging (default: false) + + # Metrics - expose /metrics endpoint + METRICS_ENABLED: "true" # Enable metrics endpoint (default: true) + + # ============================================================================= + # MONGODB AUDIT LOGGING (OPTIONAL) + # ============================================================================= + # Only needed if AUDIT_ENABLED is true + + # MongoDB Configuration + # MONGO_URI: "mongodb+srv://user:pass@cluster.mongodb.net/dbname" # Use MONGO_URI_SECRET_NAME instead + AUDIT_DATABASE: "copier_audit" # MongoDB database name (default: copier_audit) + AUDIT_COLLECTION: "events" # MongoDB collection name (default: events) + + # ============================================================================= + # SLACK NOTIFICATIONS (OPTIONAL) + # ============================================================================= + # Enable Slack notifications for errors and important events + + # SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/YOUR/WEBHOOK/URL" # Slack webhook URL (enables Slack) + # SLACK_CHANNEL: "#code-examples" # Slack channel (default: #code-examples) + # SLACK_USERNAME: "Examples Copier" # Slack bot username (default: Examples Copier) + # SLACK_ICON_EMOJI: ":robot_face:" # Slack bot icon (default: :robot_face:) + # SLACK_ENABLED: "true" # Explicitly enable/disable Slack (default: auto-enabled if URL set) + + # ============================================================================= + # DEFAULT BEHAVIORS (OPTIONAL) + # ============================================================================= + # System-wide defaults that individual config rules can override + + DEFAULT_RECURSIVE_COPY: "true" # Default recursive copy behavior (default: true) + DEFAULT_PR_MERGE: "false" # Default auto-merge PRs without review (default: false) + DEFAULT_COMMIT_MESSAGE: "Automated PR with updated examples" # Default commit message (default: shown) + + # ============================================================================= + # TESTING / DEVELOPMENT OVERRIDES (DO NOT USE IN PRODUCTION) + # ============================================================================= + + # Skip Secret Manager - for local testing/CI only + # When enabled, uses direct environment variables instead of Secret Manager + # WARNING: Never enable in production! + # SKIP_SECRET_MANAGER: "true" + + # Direct GitHub App Private Key (only when SKIP_SECRET_MANAGER=true) + # Use one of these formats: + # GITHUB_APP_PRIVATE_KEY: "-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----" + # GITHUB_APP_PRIVATE_KEY_B64: "base64-encoded-private-key" + + # Environment file path (alternative to --env-vars-file flag) + # ENV_FILE: "./configs/.env.test" + +# ============================================================================= +# DEPLOYMENT NOTES +# ============================================================================= +# +# 1. SECURITY BEST PRACTICES: +# - Use Secret Manager for all sensitive data (recommended) +# - Never commit env.yaml with actual secrets to version control +# - Add env.yaml to .gitignore +# +# 2. SECRET MANAGER SETUP: +# - Create secrets: gcloud secrets create SECRET_NAME --data-file=FILE +# - Grant access: ./scripts/grant-secret-access.sh +# - Use format: projects/PROJECT_NUMBER/secrets/SECRET_NAME/versions/latest +# +# 3. REQUIRED VARIABLES: +# - GITHUB_APP_ID +# - INSTALLATION_ID +# - REPO_OWNER +# - REPO_NAME +# - GITHUB_APP_PRIVATE_KEY_SECRET_NAME (or PEM_NAME) +# +# 4. RECOMMENDED SETUP: +# - Use Secret Manager for: GitHub private key, webhook secret, MongoDB URI +# - Enable audit logging for production +# - Enable metrics for monitoring +# - Configure Slack for error notifications +# +# 5. LOCAL DEVELOPMENT: +# - Set DRY_RUN: "true" to test without making actual changes +# - Disable audit logging if you don't have MongoDB +# - Use direct secrets (not Secret Manager) for easier local testing +# +# 6. DEPLOYMENT: +# - Copy this file: cp configs/env.yaml.example env.yaml +# - Fill in your values +# - Deploy: gcloud app deploy app.yaml --env-vars-file=env.yaml +# +# ============================================================================= +# VARIABLE REFERENCE +# ============================================================================= +# +# See CONFIG-VALIDATION-REPORT.md for complete variable documentation +# See docs/DEPLOYMENT.md for deployment guide +# See docs/DEPLOYMENT-CHECKLIST.md for step-by-step checklist +# + diff --git a/examples-copier/configs/env.yaml.production b/examples-copier/configs/env.yaml.production new file mode 100644 index 0000000..9a75ff6 --- /dev/null +++ b/examples-copier/configs/env.yaml.production @@ -0,0 +1,61 @@ +env_variables: + # ============================================================================= + # GitHub Configuration (Non-sensitive) + # ============================================================================= + GITHUB_APP_ID: "1166559" + INSTALLATION_ID: "62138132" + REPO_OWNER: "mongodb" + REPO_NAME: "docs-mongodb-internal" + SRC_BRANCH: "main" + + # ============================================================================= + # Secret Manager References (Sensitive Data - SECURE!) + # ============================================================================= + # GitHub App private key - loaded from Secret Manager + GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/1054147886816/secrets/CODE_COPIER_PEM/versions/latest" + + # Webhook secret - loaded from Secret Manager + WEBHOOK_SECRET_NAME: "projects/1054147886816/secrets/webhook-secret/versions/latest" + + # MongoDB URI - loaded from Secret Manager (for audit logging - OPTIONAL) + MONGO_URI_SECRET_NAME: "projects/1054147886816/secrets/mongo-uri/versions/latest" + + # ============================================================================= + # Application Settings (Non-sensitive) + # ============================================================================= + # PORT is automatically set by App Engine Flexible (do not override) + WEBSERVER_PATH: "/events" + CONFIG_FILE: "copier-config.yaml" + DEPRECATION_FILE: "deprecated_examples.json" + + # ============================================================================= + # Committer Information (Non-sensitive) + # ============================================================================= + COMMITTER_NAME: "GitHub Copier App" + COMMITTER_EMAIL: "bot@mongodb.com" + + # ============================================================================= + # Google Cloud Configuration (Non-sensitive) + # ============================================================================= + GOOGLE_CLOUD_PROJECT_ID: "github-copy-code-examples" + COPIER_LOG_NAME: "code-copier-log" + + # Logging Configuration (Optional - uncomment for debugging) + # LOG_LEVEL: "debug" # Enable verbose debug logging + # COPIER_DEBUG: "true" # Alternative debug flag + # COPIER_DISABLE_CLOUD_LOGGING: "true" # Disable GCP logging + + # ============================================================================= + # Feature Flags (Optional) + # ============================================================================= + AUDIT_ENABLED: "true" + METRICS_ENABLED: "true" + # DRY_RUN: "false" + + # ============================================================================= + # Default Behaviors (Optional) + # ============================================================================= + # DEFAULT_RECURSIVE_COPY: "true" + # DEFAULT_PR_MERGE: "false" + # DEFAULT_COMMIT_MESSAGE: "Automated PR with updated examples" + diff --git a/examples-copier/configs/environment.go b/examples-copier/configs/environment.go index e65069a..0693777 100644 --- a/examples-copier/configs/environment.go +++ b/examples-copier/configs/environment.go @@ -17,13 +17,15 @@ type Config struct { AppId string AppClientId string InstallationId string - CommiterName string - CommiterEmail string + CommitterName string + CommitterEmail string ConfigFile string DeprecationFile string WebserverPath string SrcBranch string PEMKeyName string + WebhookSecretName string + WebhookSecret string CopierLogName string GoogleCloudProjectId string DefaultRecursiveCopy bool @@ -31,13 +33,13 @@ type Config struct { DefaultCommitMessage string // New features - DryRun bool - AuditEnabled bool - MongoURI string - AuditDatabase string - AuditCollection string - MetricsEnabled bool - WebhookSecret string + DryRun bool + AuditEnabled bool + MongoURI string + MongoURISecretName string + AuditDatabase string + AuditCollection string + MetricsEnabled bool // Slack notifications SlackWebhookURL string @@ -55,13 +57,15 @@ const ( AppId = "GITHUB_APP_ID" AppClientId = "GITHUB_APP_CLIENT_ID" InstallationId = "INSTALLATION_ID" - CommiterName = "COMMITER_NAME" - CommiterEmail = "COMMITER_EMAIL" + CommitterName = "COMMITTER_NAME" + CommitterEmail = "COMMITTER_EMAIL" ConfigFile = "CONFIG_FILE" DeprecationFile = "DEPRECATION_FILE" WebserverPath = "WEBSERVER_PATH" SrcBranch = "SRC_BRANCH" PEMKeyName = "PEM_NAME" + WebhookSecretName = "WEBHOOK_SECRET_NAME" + WebhookSecret = "WEBHOOK_SECRET" CopierLogName = "COPIER_LOG_NAME" GoogleCloudProjectId = "GOOGLE_CLOUD_PROJECT_ID" DefaultRecursiveCopy = "DEFAULT_RECURSIVE_COPY" @@ -70,10 +74,10 @@ const ( DryRun = "DRY_RUN" AuditEnabled = "AUDIT_ENABLED" MongoURI = "MONGO_URI" + MongoURISecretName = "MONGO_URI_SECRET_NAME" AuditDatabase = "AUDIT_DATABASE" AuditCollection = "AUDIT_COLLECTION" MetricsEnabled = "METRICS_ENABLED" - WebhookSecret = "WEBHOOK_SECRET" SlackWebhookURL = "SLACK_WEBHOOK_URL" SlackChannel = "SLACK_CHANNEL" SlackUsername = "SLACK_USERNAME" @@ -85,13 +89,14 @@ const ( func NewConfig() *Config { return &Config{ Port: "8080", - CommiterName: "Copier Bot", - CommiterEmail: "bot@example.com", + CommitterName: "Copier Bot", + CommitterEmail: "bot@example.com", ConfigFile: "copier-config.yaml", DeprecationFile: "deprecated_examples.json", WebserverPath: "/webhook", SrcBranch: "main", // Default branch to copy from (NOTE: we are purposefully only allowing copying from `main` branch right now) PEMKeyName: "projects/1054147886816/secrets/CODE_COPIER_PEM/versions/latest", // default secret name for GCP Secret Manager + WebhookSecretName: "projects/1054147886816/secrets/webhook-secret/versions/latest", // default webhook secret name for GCP Secret Manager CopierLogName: "copy-copier-log", // default log name for logging to GCP GoogleCloudProjectId: "github-copy-code-examples", // default project ID for logging to GCP DefaultRecursiveCopy: true, // system-wide default for recursive copying that individual config entries can override. @@ -136,13 +141,15 @@ func LoadEnvironment(envFile string) (*Config, error) { config.AppId = os.Getenv(AppId) config.AppClientId = os.Getenv(AppClientId) config.InstallationId = os.Getenv(InstallationId) - config.CommiterName = getEnvWithDefault(CommiterName, config.CommiterName) - config.CommiterEmail = getEnvWithDefault(CommiterEmail, config.CommiterEmail) + config.CommitterName = getEnvWithDefault(CommitterName, config.CommitterName) + config.CommitterEmail = getEnvWithDefault(CommitterEmail, config.CommitterEmail) config.ConfigFile = getEnvWithDefault(ConfigFile, config.ConfigFile) config.DeprecationFile = getEnvWithDefault(DeprecationFile, config.DeprecationFile) config.WebserverPath = getEnvWithDefault(WebserverPath, config.WebserverPath) config.SrcBranch = getEnvWithDefault(SrcBranch, config.SrcBranch) config.PEMKeyName = getEnvWithDefault(PEMKeyName, config.PEMKeyName) + config.WebhookSecretName = getEnvWithDefault(WebhookSecretName, config.WebhookSecretName) + config.WebhookSecret = os.Getenv(WebhookSecret) config.DefaultRecursiveCopy = getBoolEnvWithDefault(DefaultRecursiveCopy, config.DefaultRecursiveCopy) config.DefaultPRMerge = getBoolEnvWithDefault(DefaultPRMerge, config.DefaultPRMerge) config.CopierLogName = getEnvWithDefault(CopierLogName, config.CopierLogName) @@ -153,6 +160,7 @@ func LoadEnvironment(envFile string) (*Config, error) { config.DryRun = getBoolEnvWithDefault(DryRun, false) config.AuditEnabled = getBoolEnvWithDefault(AuditEnabled, false) config.MongoURI = os.Getenv(MongoURI) + config.MongoURISecretName = os.Getenv(MongoURISecretName) config.AuditDatabase = getEnvWithDefault(AuditDatabase, "copier_audit") config.AuditCollection = getEnvWithDefault(AuditCollection, "events") config.MetricsEnabled = getBoolEnvWithDefault(MetricsEnabled, true) @@ -172,8 +180,8 @@ func LoadEnvironment(envFile string) (*Config, error) { _ = os.Setenv(AppId, config.AppId) _ = os.Setenv(AppClientId, config.AppClientId) _ = os.Setenv(InstallationId, config.InstallationId) - _ = os.Setenv(CommiterName, config.CommiterName) - _ = os.Setenv(CommiterEmail, config.CommiterEmail) + _ = os.Setenv(CommitterName, config.CommitterName) + _ = os.Setenv(CommitterEmail, config.CommitterEmail) _ = os.Setenv(ConfigFile, config.ConfigFile) _ = os.Setenv(DeprecationFile, config.DeprecationFile) _ = os.Setenv(WebserverPath, config.WebserverPath) @@ -217,7 +225,7 @@ func validateConfig(config *Config) error { requiredVars := map[string]string{ RepoName: config.RepoName, RepoOwner: config.RepoOwner, - AppId: config.AppId, + AppId: config.AppId, InstallationId: config.InstallationId, } diff --git a/examples-copier/copier-config.yaml b/examples-copier/copier-config.yaml deleted file mode 100644 index 717a5af..0000000 --- a/examples-copier/copier-config.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Examples Copier Configuration -# This is a sample configuration - customize for your needs - -source_repo: "mongodb/docs-realm" -source_branch: "main" - -copy_rules: - # Rule 1: Copy generated examples (matches actual API file paths) - - name: "Copy generated examples" - source_pattern: - type: "regex" - pattern: "^generated-examples/(?P[^/]+)/(?P.+)$" - targets: - - repo: "mongodb/target-repo" - branch: "main" - path_transform: "examples/${project}/${rest}" - commit_strategy: - type: "pull_request" - commit_message: "Update ${project} examples" - pr_title: "Update ${project} examples" - pr_body: "Automated update from ${source_repo}" - auto_merge: false - deprecation_check: - enabled: true - file: "deprecated_examples.json" - - # Rule 2: Copy all generated-examples files (fallback) - - name: "Copy all generated examples" - source_pattern: - type: "prefix" - pattern: "generated-examples/" - targets: - - repo: "mongodb/target-repo" - branch: "main" - path_transform: "docs/${path}" - commit_strategy: - type: "direct" - commit_message: "Update examples from ${source_repo}" - diff --git a/examples-copier/docs/CONFIGURATION-GUIDE.md b/examples-copier/docs/CONFIGURATION-GUIDE.md index 81d377d..510aa0a 100644 --- a/examples-copier/docs/CONFIGURATION-GUIDE.md +++ b/examples-copier/docs/CONFIGURATION-GUIDE.md @@ -885,7 +885,7 @@ Error: copy_rules[0]: name is required - [Pattern Matching Cheat Sheet](PATTERN-MATCHING-CHEATSHEET.md) - Quick reference - [Migration Guide](MIGRATION-GUIDE.md) - Migrating from legacy JSON config - [Quick Reference](../QUICK-REFERENCE.md) - Command reference -- [Deployment Guide](DEPLOYMENT-GUIDE.md) - Deploying the application +- [Deployment Guide](DEPLOYMENT.md) - Deploying the application --- diff --git a/examples-copier/docs/DEBUG-LOGGING.md b/examples-copier/docs/DEBUG-LOGGING.md new file mode 100644 index 0000000..05f3682 --- /dev/null +++ b/examples-copier/docs/DEBUG-LOGGING.md @@ -0,0 +1,376 @@ +# Debug Logging Guide + +This guide explains how to enable and use debug logging in the Examples Copier application. + +## Overview + +The Examples Copier supports configurable logging levels to help with development, troubleshooting, and debugging. By default, the application logs at the INFO level, but you can enable DEBUG logging for more verbose output. + +## Environment Variables + +### LOG_LEVEL + +**Purpose:** Set the logging level for the application + +**Values:** +- `info` (default) - Standard operational logs +- `debug` - Verbose debug logs with detailed operation information + +**Example:** +```bash +LOG_LEVEL="debug" +``` + +### COPIER_DEBUG + +**Purpose:** Alternative way to enable debug mode + +**Values:** +- `true` - Enable debug logging +- `false` (default) - Standard logging + +**Example:** +```bash +COPIER_DEBUG="true" +``` + +**Note:** Either `LOG_LEVEL="debug"` OR `COPIER_DEBUG="true"` will enable debug logging. You only need to set one. + +### COPIER_DISABLE_CLOUD_LOGGING + +**Purpose:** Disable Google Cloud Logging (useful for local development) + +**Values:** +- `true` - Disable GCP logging, only log to stdout +- `false` (default) - Enable GCP logging if configured + +**Example:** +```bash +COPIER_DISABLE_CLOUD_LOGGING="true" +``` + +**Use case:** When developing locally, you may not want logs sent to Google Cloud. This flag keeps all logs local. + +--- + +## How It Works + +### Code Implementation + +The logging system is implemented in `services/logger.go`: + +```go +// LogDebug writes debug logs only when LOG_LEVEL=debug or COPIER_DEBUG=true. +func LogDebug(message string) { + if !isDebugEnabled() { + return + } + // Mirror to GCP as info if available, plus prefix to stdout + if googleInfoLogger != nil && gcpLoggingEnabled { + googleInfoLogger.Println("[DEBUG] " + message) + } + log.Println("[DEBUG] " + message) +} + +func isDebugEnabled() bool { + if strings.EqualFold(os.Getenv("LOG_LEVEL"), "debug") { + return true + } + return strings.EqualFold(os.Getenv("COPIER_DEBUG"), "true") +} + +func isCloudLoggingDisabled() bool { + return strings.EqualFold(os.Getenv("COPIER_DISABLE_CLOUD_LOGGING"), "true") +} +``` + +### Log Levels + +The application supports the following log levels: + +| Level | Function | When to Use | Example | +|-------|----------|-------------|---------| +| **DEBUG** | `LogDebug()` | Detailed operation logs, file matching, API calls | `[DEBUG] Matched file: src/example.js` | +| **INFO** | `LogInfo()` | Standard operational logs | `[INFO] Processing webhook event` | +| **WARN** | `LogWarning()` | Warning conditions | `[WARN] File not found, skipping` | +| **ERROR** | `LogError()` | Error conditions | `[ERROR] Failed to create PR` | +| **CRITICAL** | `LogCritical()` | Critical failures | `[CRITICAL] Database connection failed` | + +--- + +## Usage Examples + +### Local Development with Debug Logging + +**Using .env file:** +```bash +# configs/.env +LOG_LEVEL="debug" +COPIER_DISABLE_CLOUD_LOGGING="true" +DRY_RUN="true" +``` + +**Using environment variables:** +```bash +export LOG_LEVEL=debug +export COPIER_DISABLE_CLOUD_LOGGING=true +export DRY_RUN=true +go run app.go +``` + +### Production with Debug Logging (Temporary) + +**env.yaml:** +```yaml +env_variables: + LOG_LEVEL: "debug" + # ... other variables +``` + +**Deploy:** +```bash +gcloud app deploy app.yaml # env.yaml is included via 'includes' directive +``` + +**Important:** Remember to disable debug logging after troubleshooting to reduce log volume and costs. + +### Local Development without Cloud Logging + +```bash +# configs/.env +COPIER_DISABLE_CLOUD_LOGGING="true" +``` + +This keeps all logs local (stdout only), which is faster and doesn't require GCP credentials. + +--- + +## What Gets Logged at DEBUG Level? + +When debug logging is enabled, you'll see additional information about: + +### 1. **File Matching Operations** +``` +[DEBUG] Checking pattern: src/**/*.js +[DEBUG] Matched file: src/examples/example1.js +[DEBUG] Excluded file: src/tests/test.js (matches exclude pattern) +``` + +### 2. **GitHub API Calls** +``` +[DEBUG] Fetching file from GitHub: src/example.js +[DEBUG] Creating PR for target repo: mongodb/docs-code-examples +[DEBUG] GitHub API response: 200 OK +``` + +### 3. **Configuration Loading** +``` +[DEBUG] Loading config file: copier-config.yaml +[DEBUG] Found 5 copy rules +[DEBUG] Rule 1: Copy src/**/*.js to examples/ +``` + +### 4. **Webhook Processing** +``` +[DEBUG] Received webhook event: pull_request +[DEBUG] PR action: closed +[DEBUG] PR merged: true +[DEBUG] Processing 3 changed files +``` + +### 5. **Pattern Matching** +``` +[DEBUG] Testing pattern: src/**/*.{js,ts} +[DEBUG] File matches: true +[DEBUG] Applying transformations: 2 +``` + +--- + +## Best Practices + +### ✅ DO + +- **Enable debug logging when troubleshooting issues** + ```bash + LOG_LEVEL="debug" + ``` + +- **Disable cloud logging for local development** + ```bash + COPIER_DISABLE_CLOUD_LOGGING="true" + ``` + +- **Use debug logging with dry run mode for testing** + ```bash + LOG_LEVEL="debug" + DRY_RUN="true" + ``` + +- **Disable debug logging in production after troubleshooting** + - High log volume can increase costs + - May expose sensitive information + +### ❌ DON'T + +- **Don't leave debug logging enabled in production long-term** + - Increases log volume and storage costs + - May impact performance + - Can expose internal implementation details + +- **Don't rely on debug logs for critical monitoring** + - Use INFO/WARN/ERROR levels for operational monitoring + - Debug logs may be disabled in production + +- **Don't log sensitive data even in debug mode** + - The code already avoids logging secrets + - Be careful when adding new debug logs + +--- + +## Troubleshooting + +### Debug Logs Not Appearing + +**Problem:** Set `LOG_LEVEL="debug"` but not seeing debug logs + +**Solutions:** + +1. **Check the variable is set correctly:** + ```bash + echo $LOG_LEVEL + # Should output: debug + ``` + +2. **Try the alternative flag:** + ```bash + COPIER_DEBUG="true" + ``` + +3. **Check case sensitivity:** + ```bash + # Both work (case-insensitive): + LOG_LEVEL="debug" + LOG_LEVEL="DEBUG" + ``` + +4. **Verify the code is calling LogDebug():** + - Not all operations have debug logs + - Check `services/logger.go` for `LogDebug()` calls + +### Logs Not Going to Google Cloud + +**Problem:** Logs appear in stdout but not in Google Cloud Logging + +**Solutions:** + +1. **Check if cloud logging is disabled:** + ```bash + # Remove or set to false: + # COPIER_DISABLE_CLOUD_LOGGING="true" + ``` + +2. **Verify GCP credentials:** + ```bash + gcloud auth application-default login + ``` + +3. **Check project ID is set:** + ```bash + GOOGLE_CLOUD_PROJECT_ID="your-project-id" + ``` + +4. **Check log name is set:** + ```bash + COPIER_LOG_NAME="code-copier-log" + ``` + +### Too Many Logs + +**Problem:** Debug logging produces too much output + +**Solutions:** + +1. **Disable debug logging:** + ```bash + # Remove or comment out: + # LOG_LEVEL="debug" + # COPIER_DEBUG="true" + ``` + +2. **Use grep to filter:** + ```bash + # Show only errors: + go run app.go 2>&1 | grep ERROR + + # Show only specific operations: + go run app.go 2>&1 | grep "pattern matching" + ``` + +3. **Redirect to file:** + ```bash + go run app.go > debug.log 2>&1 + ``` + +--- + +## Configuration Examples + +### Example 1: Local Development (Recommended) + +```bash +# configs/.env +LOG_LEVEL="debug" +COPIER_DISABLE_CLOUD_LOGGING="true" +DRY_RUN="true" +AUDIT_ENABLED="false" +METRICS_ENABLED="true" +``` + +**Why:** +- Debug logs help understand what's happening +- No cloud logging keeps it fast and local +- Dry run prevents accidental changes +- No audit logging (simpler setup) + +### Example 2: Production Troubleshooting + +```yaml +# env.yaml +env_variables: + LOG_LEVEL: "debug" + GOOGLE_CLOUD_PROJECT_ID: "your-project-id" + COPIER_LOG_NAME: "code-copier-log" + # ... other variables +``` + +**Why:** +- Temporarily enable debug for troubleshooting +- Logs go to Cloud Logging for analysis +- Remember to disable after fixing issue + +### Example 3: Local with Cloud Logging + +```bash +# configs/.env +LOG_LEVEL="debug" +GOOGLE_CLOUD_PROJECT_ID="your-project-id" +COPIER_LOG_NAME="code-copier-log-dev" +# COPIER_DISABLE_CLOUD_LOGGING not set (defaults to false) +``` + +**Why:** +- Test cloud logging integration locally +- Separate log name for dev environment +- Useful for testing logging infrastructure + +--- + +## See Also + +- [LOCAL-TESTING.md](LOCAL-TESTING.md) - Local development guide +- [TROUBLESHOOTING.md](TROUBLESHOOTING.md) - General troubleshooting +- [CONFIGURATION-GUIDE.md](CONFIGURATION-GUIDE.md) - Complete configuration reference +- [../configs/env.yaml.example](../configs/env.yaml.example) - All environment variables +- [../configs/.env.example](../configs/.env.example) - Local development template + diff --git a/examples-copier/docs/DEPLOYMENT-CHECKLIST.md b/examples-copier/docs/DEPLOYMENT-CHECKLIST.md new file mode 100644 index 0000000..fd9bb9a --- /dev/null +++ b/examples-copier/docs/DEPLOYMENT-CHECKLIST.md @@ -0,0 +1,493 @@ +# Deployment Checklist + +Quick reference checklist for deploying the GitHub Code Example Copier to Google Cloud App Engine. + +## 📋 Pre-Deployment + +### ☐ 1. Prerequisites Installed + +```bash +# Verify Go +go version # Should be 1.23+ + +# Verify gcloud +gcloud --version + +# Verify authentication +gcloud auth list +``` + +### ☐ 2. Google Cloud Project Setup + +```bash +# Set project +gcloud config set project YOUR_PROJECT_ID + +# Verify +gcloud config get-value project + +# Enable required APIs +gcloud services enable secretmanager.googleapis.com +gcloud services enable appengine.googleapis.com +``` + +### ☐ 3. Secrets in Secret Manager + +```bash +# List secrets +gcloud secrets list + +# Expected secrets: +# ✅ CODE_COPIER_PEM - GitHub App private key +# ✅ webhook-secret - Webhook signature validation +# ✅ mongo-uri - MongoDB connection (optional) +``` + +**If secrets don't exist, create them:** + +```bash +# GitHub private key +gcloud secrets create CODE_COPIER_PEM \ + --data-file=/path/to/private-key.pem \ + --replication-policy="automatic" + +# Webhook secret +WEBHOOK_SECRET=$(openssl rand -hex 32) +echo -n "$WEBHOOK_SECRET" | gcloud secrets create webhook-secret \ + --data-file=- \ + --replication-policy="automatic" +echo "Save this: $WEBHOOK_SECRET" + +# MongoDB URI (optional) +echo -n "mongodb+srv://..." | gcloud secrets create mongo-uri \ + --data-file=- \ + --replication-policy="automatic" +``` + +### ☐ 4. Grant IAM Permissions + +```bash +# Run the grant script +cd examples-copier +./scripts/grant-secret-access.sh +``` + +**Or manually:** + +```bash +PROJECT_NUMBER=$(gcloud projects describe $(gcloud config get-value project) --format="value(projectNumber)") +SERVICE_ACCOUNT="${PROJECT_NUMBER}@appspot.gserviceaccount.com" + +gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ + --member="serviceAccount:${SERVICE_ACCOUNT}" \ + --role="roles/secretmanager.secretAccessor" + +gcloud secrets add-iam-policy-binding webhook-secret \ + --member="serviceAccount:${SERVICE_ACCOUNT}" \ + --role="roles/secretmanager.secretAccessor" + +gcloud secrets add-iam-policy-binding mongo-uri \ + --member="serviceAccount:${SERVICE_ACCOUNT}" \ + --role="roles/secretmanager.secretAccessor" +``` + +**Verify:** +```bash +gcloud secrets get-iam-policy CODE_COPIER_PEM | grep @appspot +gcloud secrets get-iam-policy webhook-secret | grep @appspot +gcloud secrets get-iam-policy mongo-uri | grep @appspot +``` + +### ☐ 5. Create env.yaml + +```bash +cd examples-copier + +# Copy from template +cp configs/env.yaml.production env.yaml + +# Or convert from .env +./scripts/convert-env-to-yaml.sh configs/.env env.yaml + +# Edit with your values if needed +nano env.yaml +``` + +**Required changes in env.yaml:** +- `GITHUB_APP_ID` - Your GitHub App ID +- `INSTALLATION_ID` - Your installation ID +- `REPO_OWNER` - Source repository owner +- `REPO_NAME` - Source repository name +- `GITHUB_APP_PRIVATE_KEY_SECRET_NAME` - Update project number +- `WEBHOOK_SECRET_NAME` - Update project number +- `MONGO_URI_SECRET_NAME` - Update project number (if using audit logging) +- `GOOGLE_PROJECT_ID` - Your Google Cloud project ID + +### ☐ 6. Verify env.yaml in .gitignore + +```bash +# Check +grep "env.yaml" .gitignore + +# If not found, add it +echo "env.yaml" >> .gitignore +``` + +### ☐ 7. Verify app.yaml Configuration + +```bash +cat app.yaml +``` + +**Should contain:** +```yaml +runtime: go +runtime_config: + operating_system: "ubuntu22" + runtime_version: "1.23" +env: flex +``` + +**Should NOT contain:** +- ❌ `env_variables:` section (those go in env.yaml) + +--- + +## 🚀 Deployment + +### ☐ 8. Deploy to App Engine + +```bash +cd examples-copier + +# Deploy (env.yaml is included via 'includes' directive in app.yaml) +gcloud app deploy app.yaml +``` + +**Expected output:** +``` +Updating service [default]...done. +Setting traffic split for service [default]...done. +Deployed service [default] to [https://YOUR_APP.appspot.com] +``` + +### ☐ 9. Verify Deployment + +```bash +# Check versions +gcloud app versions list + +# Get app URL +APP_URL=$(gcloud app describe --format="value(defaultHostname)") +echo "App URL: https://${APP_URL}" +``` + +### ☐ 10. Check Logs + +```bash +# View real-time logs +gcloud app logs tail -s default +``` + +**Look for:** +- ✅ "Starting web server on port :8080" +- ✅ No errors about secrets +- ✅ No "failed to load webhook secret" +- ✅ No "failed to load MongoDB URI" + +**Should NOT see:** +- ❌ "failed to load webhook secret" +- ❌ "failed to load MongoDB URI" +- ❌ "SKIP_SECRET_MANAGER=true" + +### ☐ 11. Test Health Endpoint + +```bash +# Get app URL +APP_URL=$(gcloud app describe --format="value(defaultHostname)") + +# Test health +curl https://${APP_URL}/health +``` + +**Expected response:** +```json +{ + "status": "healthy", + "started": true, + "github": { + "status": "healthy", + "authenticated": true + }, + "queues": { + "upload_count": 0, + "deprecation_count": 0 + }, + "uptime": "1m23s" +} +``` + +--- + +## 🔗 GitHub Webhook Configuration + +### ☐ 12. Get Webhook Secret + +```bash +# Get the webhook secret value +gcloud secrets versions access latest --secret=webhook-secret +``` + +**Save this value** - you'll need it for GitHub webhook configuration. + +### ☐ 13. Configure GitHub Webhook + +1. **Go to repository settings** + - URL: `https://github.com/YOUR_ORG/YOUR_REPO/settings/hooks` + +2. **Add or edit webhook** + - **Payload URL:** `https://YOUR_APP.appspot.com/events` + - **Content type:** `application/json` + - **Secret:** (paste the value from step 12) + - **SSL verification:** Enable SSL verification + - **Events:** Select "Pull requests" + - **Active:** ✓ Checked + +3. **Save webhook** + +### ☐ 14. Test Webhook + +**Option A: Redeliver existing webhook** +1. Go to webhook settings +2. Click "Recent Deliveries" +3. Click on a delivery +4. Click "Redeliver" + +**Option B: Create test PR** +1. Create a test PR in your source repository +2. Merge it +3. Watch logs for webhook receipt + +```bash +# Watch logs +gcloud app logs tail -s default | grep webhook +``` + +--- + +## ✅ Post-Deployment Verification + +### ☐ 15. Verify Secrets Loaded + +```bash +# Check logs for secret loading +gcloud app logs read --limit=100 | grep -i "secret" +``` + +**Should NOT see:** +- ❌ "failed to load webhook secret" +- ❌ "failed to load MongoDB URI" + +### ☐ 16. Verify Webhook Signature Validation + +```bash +# Watch logs during webhook delivery +gcloud app logs tail -s default +``` + +**Look for:** +- ✅ "webhook received" +- ✅ "signature verified" +- ✅ "processing webhook" + +**Should NOT see:** +- ❌ "webhook signature verification failed" +- ❌ "invalid signature" + +### ☐ 17. Verify File Copying + +```bash +# Watch logs during PR merge +gcloud app logs tail -s default +``` + +**Look for:** +- ✅ "Config file loaded successfully" +- ✅ "file matched pattern" +- ✅ "Copied file to target repo" + +### ☐ 18. Verify Audit Logging (if enabled) + +```bash +# Connect to MongoDB +mongosh "YOUR_MONGO_URI" + +# Check for recent events +db.audit_events.find().sort({timestamp: -1}).limit(5) +``` + +### ☐ 19. Verify Metrics (if enabled) + +```bash +# Check metrics endpoint +curl https://YOUR_APP.appspot.com/metrics +``` + +**Expected response:** +```json +{ + "webhooks": { + "received": 1, + "processed": 1, + "failed": 0 + }, + "files": { + "matched": 5, + "uploaded": 5, + "failed": 0 + } +} +``` + +### ☐ 20. Security Verification + +```bash +# Verify env.yaml doesn't contain actual secrets +cat env.yaml | grep -E "BEGIN|mongodb\+srv|ghp_" +# Should return NOTHING (only Secret Manager paths) + +# Verify env.yaml is not committed +git status | grep env.yaml +# Should show: nothing to commit (or untracked) + +# Verify IAM permissions +gcloud secrets get-iam-policy CODE_COPIER_PEM | grep @appspot +gcloud secrets get-iam-policy webhook-secret | grep @appspot +# Should see the service account +``` + +--- + +## 🐛 Troubleshooting + +### Error: "failed to load webhook secret" + +**Cause:** Secret Manager access denied + +**Fix:** +```bash +./scripts/grant-secret-access.sh +``` + +### Error: "webhook signature verification failed" + +**Cause:** Secret in Secret Manager doesn't match GitHub webhook secret + +**Fix:** +```bash +# Get secret from Secret Manager +gcloud secrets versions access latest --secret=webhook-secret + +# Update GitHub webhook with this value +# OR update Secret Manager with GitHub's value +``` + +### Error: "MONGO_URI is required when audit logging is enabled" + +**Cause:** Audit logging enabled but MongoDB URI not loaded + +**Fix:** +```bash +# Option 1: Disable audit logging +# In env.yaml: AUDIT_ENABLED: "false" + +# Option 2: Ensure MONGO_URI_SECRET_NAME is set +# In env.yaml: MONGO_URI_SECRET_NAME: "projects/.../secrets/mongo-uri/versions/latest" + +# Redeploy +gcloud app deploy app.yaml +``` + +### Error: "Config file not found" + +**Cause:** `copier-config.yaml` missing from source repository + +**Fix:** +```bash +# Add copier-config.yaml to your source repository +# See documentation for config file format +``` + +--- + +## 📊 Success Criteria + +All items should be ✅: + +- ✅ Deployment completes without errors +- ✅ App Engine is running +- ✅ Health endpoint returns 200 OK +- ✅ Logs show no secret loading errors +- ✅ Webhook receives PR events +- ✅ Webhook signature validation works +- ✅ Files are copied to target repos +- ✅ Audit events logged (if enabled) +- ✅ Metrics available (if enabled) +- ✅ No secrets in config files +- ✅ env.yaml not in version control + +--- + +## 🎉 You're Done! + +Your application is deployed with: +- ✅ All secrets in Secret Manager (secure!) +- ✅ No hardcoded secrets in config files +- ✅ Easy secret rotation (just update in Secret Manager) +- ✅ Audit trail of secret access +- ✅ Fine-grained IAM permissions + +**Next steps:** +1. Monitor logs for first few PRs +2. Verify files are copied correctly +3. Set up alerts (optional) +4. Document any custom configuration + +--- + +## 📚 Quick Reference + +```bash +# Deploy +gcloud app deploy app.yaml + +# View logs +gcloud app logs tail -s default + +# Check health +curl https://YOUR_APP.appspot.com/health + +# Check metrics +curl https://YOUR_APP.appspot.com/metrics + +# List secrets +gcloud secrets list + +# Get secret value +gcloud secrets versions access latest --secret=SECRET_NAME + +# Grant access +./scripts/grant-secret-access.sh + +# Rollback +gcloud app versions list +gcloud app services set-traffic default --splits=PREVIOUS_VERSION=1 +``` + +--- + +**See also:** +- [DEPLOYMENT.md](DEPLOYMENT.md) - Complete deployment guide +- [../WEBHOOK-SECRET-MANAGER-GUIDE.md](../WEBHOOK-SECRET-MANAGER-GUIDE.md) - Secret Manager details +- [../ENV-FILES-EXPLAINED.md](../ENV-FILES-EXPLAINED.md) - Environment file explanation + diff --git a/examples-copier/docs/DEPLOYMENT-GUIDE.md b/examples-copier/docs/DEPLOYMENT-GUIDE.md deleted file mode 100644 index c880cda..0000000 --- a/examples-copier/docs/DEPLOYMENT-GUIDE.md +++ /dev/null @@ -1,369 +0,0 @@ -# Deployment Guide - -This guide walks you through deploying the examples-copier application. - -## Prerequisites - -1. **Go 1.23.4+** installed -2. **MongoDB Atlas** account (for audit logging) -3. **GitHub App** credentials -4. **Google Cloud** project (for Secret Manager and logging) - -## Step 1: Build the Application - -```bash -cd examples-copier - -# Build main application -go build -o examples-copier . - -# Build CLI validator -go build -o config-validator ./cmd/config-validator - -# Verify builds -./examples-copier -help -./config-validator -help -``` - -## Step 2: Configure Environment - -Create or update your `.env` file: - -```bash -# Copy example -cp configs/.env.example.new configs/.env - -# Edit with your values -vim configs/.env -``` - -### Required Environment Variables - -```bash -# GitHub Configuration -REPO_OWNER=your-org -REPO_NAME=your-repo -SRC_BRANCH=main -GITHUB_APP_ID=123456 -GITHUB_INSTALLATION_ID=789012 - -# Google Cloud -GCP_PROJECT_ID=your-project -PEM_KEY_NAME=projects/123/secrets/CODE_COPIER_PEM/versions/latest - -# Application Settings -PORT=8080 -WEBSERVER_PATH=/webhook -CONFIG_FILE=copier-config.yaml -DEPRECATION_FILE=deprecated_examples.json - -# New Features -DRY_RUN=false -AUDIT_ENABLED=true -METRICS_ENABLED=true -WEBHOOK_SECRET=your-webhook-secret - -# MongoDB (for audit logging) -MONGO_URI=mongodb+srv://user:pass@cluster.mongodb.net -AUDIT_DATABASE=code_copier -AUDIT_COLLECTION=audit_events -``` - -## Step 3: Create YAML Configuration - -Create `copier-config.yaml` in your repository: - -```yaml -source_repo: "your-org/source-repo" -source_branch: "main" - -copy_rules: - - name: "Copy Go examples" - source_pattern: - type: "regex" - pattern: "^examples/go/(?P[^/]+)/(?P.+)$" - targets: - - repo: "your-org/target-repo" - branch: "main" - path_transform: "docs/examples/${category}/${file}" - commit_strategy: - type: "pull_request" - commit_message: "Update ${category} examples from source" - pr_title: "Update ${category} examples" - auto_merge: false - deprecation_check: - enabled: true - file: "deprecated_examples.json" -``` - -### Validate Configuration - -```bash -# Validate config file -./config-validator validate -config copier-config.yaml -v - -# Test pattern matching -./config-validator test-pattern \ - -type regex \ - -pattern "^examples/go/(?P[^/]+)/(?P.+)$" \ - -file "examples/go/database/connect.go" - -# Test path transformation -./config-validator test-transform \ - -template "docs/examples/${category}/${file}" \ - -file "examples/go/database/connect.go" \ - -pattern "^examples/go/(?P[^/]+)/(?P.+)$" -``` - -## Step 4: Test with Dry-Run Mode - -Before deploying to production, test with dry-run mode: - -```bash -# Enable dry-run in .env -DRY_RUN=true ./examples-copier -env ./configs/.env -``` - -In dry-run mode: -- Webhooks are processed -- Files are matched and transformed -- Audit events are logged -- **NO actual commits or PRs are created** - -## Step 5: Deploy to Google Cloud App Engine - -### Update `app.yaml` - -```yaml -runtime: go123 -env: standard - -env_variables: - REPO_OWNER: "your-org" - REPO_NAME: "your-repo" - CONFIG_FILE: "copier-config.yaml" - AUDIT_ENABLED: "true" - METRICS_ENABLED: "true" - MONGO_URI: "mongodb+srv://..." - # ... other variables - -handlers: - - url: /.* - script: auto - secure: always -``` - -### Deploy - -```bash -# Deploy to App Engine -gcloud app deploy - -# View logs -gcloud app logs tail -s default - -# Check health -curl https://your-app.appspot.com/health -``` - -## Step 6: Configure GitHub Webhook - -1. Go to your repository settings -2. Navigate to **Webhooks** → **Add webhook** -3. Set **Payload URL**: `https://your-app.appspot.com/webhook` -4. Set **Content type**: `application/json` -5. Set **Secret**: (your WEBHOOK_SECRET value) -6. Select events: **Pull requests** -7. Click **Add webhook** - -## Step 7: Monitor and Verify - -### Check Health Endpoint - -```bash -curl https://your-app.appspot.com/health -``` - -Expected response: -```json -{ - "status": "healthy", - "started": true, - "github": { - "status": "healthy", - "authenticated": true - }, - "queues": { - "upload_count": 0, - "deprecation_count": 0 - }, - "uptime": "1h23m45s" -} -``` - -### Check Metrics Endpoint - -```bash -curl https://your-app.appspot.com/metrics -``` - -Expected response: -```json -{ - "webhooks": { - "received": 42, - "processed": 40, - "failed": 2, - "success_rate": 95.24 - }, - "files": { - "matched": 150, - "uploaded": 145, - "failed": 5, - "deprecated": 3 - }, - "processing_time": { - "p50": 234, - "p95": 567, - "p99": 890 - } -} -``` - -### Query Audit Logs - -Connect to MongoDB and query audit events: - -```javascript -// Recent successful copies -db.audit_events.find({ - event_type: "copy", - success: true -}).sort({timestamp: -1}).limit(10) - -// Failed operations -db.audit_events.find({ - success: false -}).sort({timestamp: -1}) - -// Statistics by rule -db.audit_events.aggregate([ - {$match: {event_type: "copy"}}, - {$group: { - _id: "$rule_name", - count: {$sum: 1}, - avg_duration: {$avg: "$duration_ms"} - }} -]) -``` - -## Step 8: Gradual Rollout - -### Phase 1: Test with One Rule - -Start with a single, simple copy rule: - -```yaml -copy_rules: - - name: "Test rule" - source_pattern: - type: "prefix" - pattern: "test/examples/" - targets: - - repo: "your-org/test-repo" - branch: "test-branch" - path_transform: "${path}" -``` - -### Phase 2: Add More Rules - -Gradually add more complex rules with regex patterns and transformations. - -### Phase 3: Enable Auto-Merge - -Once confident, enable auto-merge for specific rules: - -```yaml -commit_strategy: - type: "pull_request" - auto_merge: true -``` - -## Troubleshooting - -### Issue: Config validation fails - -```bash -# Check config syntax -./config-validator validate -config copier-config.yaml -v - -# Test specific patterns -./config-validator test-pattern -type regex -pattern "..." -file "..." -``` - -### Issue: Files not matching - -Check the audit logs for match attempts: - -```javascript -db.audit_events.find({ - source_path: "your/file/path.go" -}) -``` - -### Issue: MongoDB connection fails - -```bash -# Test connection -mongosh "mongodb+srv://user:pass@cluster.mongodb.net/code_copier" - -# Check environment variable -echo $MONGO_URI -``` - -### Issue: Webhook signature verification fails - -```bash -# Verify webhook secret matches -echo $WEBHOOK_SECRET - -# Check GitHub webhook delivery logs -# Go to Settings → Webhooks → Recent Deliveries -``` - -## Rollback Plan - -If issues arise: - -1. **Disable webhook** in GitHub repository settings -2. **Revert to previous version** using `gcloud app versions list` and `gcloud app services set-traffic` -3. **Check audit logs** to identify what was changed -4. **Fix configuration** and redeploy - -## Performance Tuning - -### Optimize Pattern Matching - -- Use **prefix** patterns for simple directory matching (fastest) -- Use **glob** patterns for wildcard matching (medium) -- Use **regex** patterns only when necessary (slowest) - -### Batch Operations - -Group multiple file changes into single commits/PRs: - -```yaml -commit_strategy: - type: "pull_request" - # All files matching this rule will be in one PR -``` - -### MongoDB Indexing - -Ensure indexes exist for common queries: - -```javascript -db.audit_events.createIndex({timestamp: -1}) -db.audit_events.createIndex({rule_name: 1, timestamp: -1}) -db.audit_events.createIndex({success: 1, timestamp: -1}) -``` diff --git a/examples-copier/docs/DEPLOYMENT.md b/examples-copier/docs/DEPLOYMENT.md new file mode 100644 index 0000000..e9d1777 --- /dev/null +++ b/examples-copier/docs/DEPLOYMENT.md @@ -0,0 +1,524 @@ +# Deployment Guide + +Complete guide for deploying the GitHub Code Example Copier to Google Cloud App Engine with Secret Manager. + +## Table of Contents + +- [Prerequisites](#prerequisites) +- [Architecture Overview](#architecture-overview) +- [Secret Manager Setup](#secret-manager-setup) +- [Configuration](#configuration) +- [Deployment](#deployment) +- [Post-Deployment](#post-deployment) +- [Monitoring](#monitoring) +- [Troubleshooting](#troubleshooting) + +## Prerequisites + +### Required Tools + +- **Go 1.23+** - For local development and testing +- **Google Cloud SDK** - For deployment +- **GitHub App** - With appropriate permissions +- **MongoDB Atlas** (optional) - For audit logging + +### Required Accounts & Access + +- Google Cloud project with billing enabled +- GitHub organization admin access (to create/configure GitHub App) +- MongoDB Atlas account (if using audit logging) + +### Install Google Cloud SDK + +```bash +# macOS +brew install --cask google-cloud-sdk + +# Verify installation +gcloud --version +``` + +### Authenticate with Google Cloud + +```bash +# Login to Google Cloud +gcloud auth login + +# Set application default credentials +gcloud auth application-default login + +# Set your project +gcloud config set project YOUR_PROJECT_ID + +# Verify +gcloud config get-value project +``` + +## Architecture Overview + +### Components + +``` +┌─────────────────────────────────────────────────────────────┐ +│ GitHub Repository │ +│ (docs-code-examples) │ +└────────────────────┬────────────────────────────────────────┘ + │ Webhook (PR merged) + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ Google Cloud App Engine │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ examples-copier Application │ │ +│ │ - Receives webhook │ │ +│ │ - Validates signature │ │ +│ │ - Loads config from source repo │ │ +│ │ - Matches files against patterns │ │ +│ │ - Copies to target repos │ │ +│ └──────────────────────────────────────────────────────┘ │ +└────────────┬────────────────────────────┬───────────────────┘ + │ │ + ↓ ↓ +┌────────────────────────┐ ┌───────────────────────────────┐ +│ Secret Manager │ │ Cloud Logging │ +│ - GitHub private key │ │ - Application logs │ +│ - Webhook secret │ │ - Webhook events │ +│ - MongoDB URI │ │ - Error tracking │ +└────────────────────────┘ └───────────────────────────────┘ + │ + ↓ +┌────────────────────────┐ +│ MongoDB Atlas │ +│ - Audit events │ +│ - Metrics │ +└────────────────────────┘ +``` + +### Environment: Flexible vs Standard + +This application uses **App Engine Flexible Environment**: + +**app.yaml:** +```yaml +runtime: go +runtime_config: + operating_system: "ubuntu22" + runtime_version: "1.23" +env: flex # ← Flexible Environment +``` + +**Key differences:** +- Environment variables in **separate file** (`env.yaml`) included via `includes` directive +- Deployment: `gcloud app deploy app.yaml` +- Better Secret Manager integration +- More flexible runtime configuration + +## Secret Manager Setup + +### Why Secret Manager? + +✅ **Security**: Secrets encrypted at rest and in transit +✅ **Audit Trail**: All access logged +✅ **Rotation**: Update secrets without redeployment +✅ **Access Control**: Fine-grained IAM permissions +✅ **No Hardcoding**: Secrets never in config files or version control + +### Enable Secret Manager API + +```bash +gcloud services enable secretmanager.googleapis.com +``` + +### Store Secrets + +#### 1. GitHub App Private Key + +```bash +# Store your GitHub App private key +gcloud secrets create CODE_COPIER_PEM \ + --data-file=/path/to/your/private-key.pem \ + --replication-policy="automatic" +``` + +#### 2. Webhook Secret + +```bash +# Generate a secure webhook secret +WEBHOOK_SECRET=$(openssl rand -hex 32) +echo "Generated: $WEBHOOK_SECRET" + +# Store in Secret Manager +echo -n "$WEBHOOK_SECRET" | gcloud secrets create webhook-secret \ + --data-file=- \ + --replication-policy="automatic" + +# Save this value - you'll need it for GitHub webhook configuration +``` + +#### 3. MongoDB URI (Optional - for audit logging) + +```bash +# Store MongoDB connection string +echo -n "mongodb+srv://user:pass@cluster.mongodb.net/dbname" | \ + gcloud secrets create mongo-uri \ + --data-file=- \ + --replication-policy="automatic" +``` + +### Grant App Engine Access + +```bash +# Get your project number +PROJECT_NUMBER=$(gcloud projects describe $(gcloud config get-value project) --format="value(projectNumber)") + +# App Engine service account +SERVICE_ACCOUNT="${PROJECT_NUMBER}@appspot.gserviceaccount.com" + +# Grant access to each secret +gcloud secrets add-iam-policy-binding CODE_COPIER_PEM \ + --member="serviceAccount:${SERVICE_ACCOUNT}" \ + --role="roles/secretmanager.secretAccessor" + +gcloud secrets add-iam-policy-binding webhook-secret \ + --member="serviceAccount:${SERVICE_ACCOUNT}" \ + --role="roles/secretmanager.secretAccessor" + +gcloud secrets add-iam-policy-binding mongo-uri \ + --member="serviceAccount:${SERVICE_ACCOUNT}" \ + --role="roles/secretmanager.secretAccessor" +``` + +**Or use the provided script:** +```bash +cd examples-copier +./scripts/grant-secret-access.sh +``` + +### Verify Secrets + +```bash +# List all secrets +gcloud secrets list + +# View secret metadata +gcloud secrets describe CODE_COPIER_PEM + +# Verify IAM permissions +gcloud secrets get-iam-policy CODE_COPIER_PEM +``` + +## Configuration + +### Create env.yaml + +The `env.yaml` file contains environment variables for App Engine deployment. + +```bash +cd examples-copier + +# Copy from production template +cp configs/env.yaml.production env.yaml + +# Or convert from .env file +./scripts/convert-env-to-yaml.sh configs/.env env.yaml + +# Edit if needed +nano env.yaml +``` + +### env.yaml Structure + +**Important Notes:** +- Do NOT set `PORT` in `env.yaml` - App Engine Flexible automatically sets this +- The application defaults to port 8080 for local development +- Secret Manager references must include `/versions/latest` or a specific version number + +```yaml +env_variables: + # ============================================================================= + # GitHub Configuration (Non-sensitive) + # ============================================================================= + GITHUB_APP_ID: "YOUR_APP_ID" + INSTALLATION_ID: "YOUR_INSTALLATION_ID" + REPO_OWNER: "your-org" + REPO_NAME: "your-repo" + SRC_BRANCH: "main" + + # ============================================================================= + # Secret Manager References (Sensitive - SECURE!) + # ============================================================================= + GITHUB_APP_PRIVATE_KEY_SECRET_NAME: "projects/PROJECT_NUMBER/secrets/CODE_COPIER_PEM/versions/latest" + WEBHOOK_SECRET_NAME: "projects/PROJECT_NUMBER/secrets/webhook-secret/versions/latest" + MONGO_URI_SECRET_NAME: "projects/PROJECT_NUMBER/secrets/mongo-uri/versions/latest" + + # ============================================================================= + # Application Settings + # ============================================================================= + # PORT: "8080" # DO NOT SET - App Engine sets this automatically + WEBSERVER_PATH: "/events" + CONFIG_FILE: "copier-config.yaml" + DEPRECATION_FILE: "deprecated_examples.json" + + # ============================================================================= + # Committer Information + # ============================================================================= + COMMITTER_NAME: "GitHub Copier App" + COMMITTER_EMAIL: "bot@example.com" + + # ============================================================================= + # Google Cloud Configuration + # ============================================================================= + GOOGLE_PROJECT_ID: "your-project-id" + GOOGLE_LOG_NAME: "code-copier-log" + + # ============================================================================= + # Feature Flags + # ============================================================================= + AUDIT_ENABLED: "true" + METRICS_ENABLED: "true" + # DRY_RUN: "false" +``` + +### Important Notes + +**✅ DO:** +- Use Secret Manager references (`*_SECRET_NAME` variables) +- Keep `env.yaml` in `.gitignore` +- Use `env.yaml.production` as template + +**❌ DON'T:** +- Put actual secrets in `env.yaml` (use `*_SECRET_NAME` instead) +- Commit `env.yaml` to version control +- Share `env.yaml` via email/chat + +### How Secrets Are Loaded + +``` +Application Startup: +1. Load env.yaml → environment variables +2. Read WEBHOOK_SECRET_NAME from env +3. Call Secret Manager API to get actual secret +4. Store in config.WebhookSecret +5. Use for webhook signature validation +``` + +**Code flow:** +```go +// app.go +config, _ := configs.LoadEnvironment(envFile) +services.LoadWebhookSecret(config) // Loads from Secret Manager +services.LoadMongoURI(config) // Loads from Secret Manager +``` + +## Deployment + +### Pre-Deployment Checklist + +- [ ] Secrets created in Secret Manager +- [ ] IAM permissions granted to App Engine +- [ ] `env.yaml` created and configured +- [ ] `env.yaml` in `.gitignore` +- [ ] `app.yaml` uses Flexible Environment + +### Deploy to App Engine + +```bash +cd examples-copier + +# Deploy (env.yaml is included via 'includes' directive in app.yaml) +gcloud app deploy app.yaml + +# Or specify project +gcloud app deploy app.yaml --project=your-project-id +``` + +### Verify Deployment + +```bash +# Check deployment status +gcloud app versions list + +# Get app URL +APP_URL=$(gcloud app describe --format="value(defaultHostname)") +echo "App URL: https://${APP_URL}" + +# View logs +gcloud app logs tail -s default +``` + +### Test Health Endpoint + +```bash +# Test health +curl https://${APP_URL}/health + +# Expected response: +# { +# "status": "healthy", +# "started": true, +# "github": { +# "status": "healthy", +# "authenticated": true +# }, +# "queues": { +# "upload_count": 0, +# "deprecation_count": 0 +# }, +# "uptime": "5m30s" +# } +``` + +## Post-Deployment + +### Configure GitHub Webhook + +1. **Navigate to repository settings** + - Go to: `https://github.com/YOUR_ORG/YOUR_REPO/settings/hooks` + +2. **Add or edit webhook** + - **Payload URL:** `https://YOUR_APP.appspot.com/events` + - **Content type:** `application/json` + - **Secret:** (the webhook secret from Secret Manager) + - **Events:** Select "Pull requests" + - **Active:** ✓ Checked + +3. **Get webhook secret from Secret Manager** + ```bash + gcloud secrets versions access latest --secret=webhook-secret + ``` + +4. **Save webhook** + +### Test Webhook + +**Option A: Merge a test PR** +```bash +# Create and merge a test PR +# Watch logs for webhook receipt +gcloud app logs tail -s default | grep webhook +``` + +**Option B: Redeliver from GitHub** +1. Go to webhook settings +2. Click "Recent Deliveries" +3. Click on a delivery +4. Click "Redeliver" +5. Watch logs + +### Verify Functionality + +```bash +# Check logs for successful processing +gcloud app logs read --limit=50 + +# Look for: +# ✅ "Starting web server on port :8080" +# ✅ "webhook received" +# ✅ "Config file loaded successfully" +# ✅ "file matched pattern" +# ✅ "Copied file to target repo" + +# Should NOT see: +# ❌ "failed to load webhook secret" +# ❌ "failed to load MongoDB URI" +# ❌ "webhook signature verification failed" +``` + +## Monitoring + +### View Logs + +```bash +# Real-time logs +gcloud app logs tail -s default + +# Recent logs +gcloud app logs read --limit=100 + +# Filter for errors +gcloud app logs read --limit=100 | grep ERROR + +# Filter for webhooks +gcloud app logs read --limit=100 | grep webhook +``` + +### Check Metrics + +```bash +# Metrics endpoint +curl https://YOUR_APP.appspot.com/metrics + +# Response includes: +# - webhooks_received +# - webhooks_processed +# - files_matched +# - files_uploaded +# - processing_time (p50, p95, p99) +``` + +### Audit Logging (if enabled) + +Query MongoDB for audit events: + +```javascript +// Connect to MongoDB +mongosh "mongodb+srv://..." + +// Recent events +db.audit_events.find().sort({timestamp: -1}).limit(10) + +// Failed operations +db.audit_events.find({success: false}) + +// Statistics by rule +db.audit_events.aggregate([ + {$match: {event_type: "copy"}}, + {$group: { + _id: "$rule_name", + count: {$sum: 1} + }} +]) +``` + +## Troubleshooting + +See [DEPLOYMENT-CHECKLIST.md](DEPLOYMENT-CHECKLIST.md) for detailed troubleshooting steps. + +### Common Issues + +| Error | Cause | Solution | +|-------|-------|----------| +| "failed to load webhook secret" | Secret Manager access denied | Run `./grant-secret-access.sh` | +| "webhook signature verification failed" | Secret mismatch | Verify secret matches GitHub webhook | +| "MONGO_URI is required" | Audit enabled but no URI | Set `MONGO_URI_SECRET_NAME` or disable audit | +| "Config file not found" | Missing copier-config.yaml | Add config file to source repo | + +### Quick Fixes + +```bash +# Grant secret access +./scripts/grant-secret-access.sh + +# View secret value +gcloud secrets versions access latest --secret=webhook-secret + +# Disable audit logging +# In env.yaml: AUDIT_ENABLED: "false" + +# Redeploy +gcloud app deploy app.yaml +``` + +## Next Steps + +1. **Monitor first few PRs** - Watch logs to ensure files are copied correctly +2. **Set up alerts** (optional) - Configure Cloud Monitoring alerts +3. **Document custom config** - Add notes about your specific setup +4. **Plan secret rotation** - Schedule regular secret updates + +--- + +**See also:** +- [DEPLOYMENT-CHECKLIST.md](DEPLOYMENT-CHECKLIST.md) - Step-by-step checklist +- [../WEBHOOK-SECRET-MANAGER-GUIDE.md](../WEBHOOK-SECRET-MANAGER-GUIDE.md) - Secret Manager details +- [../ENV-FILES-EXPLAINED.md](../ENV-FILES-EXPLAINED.md) - Environment file explanation + diff --git a/examples-copier/docs/FAQ.md b/examples-copier/docs/FAQ.md index 5074bc0..d04f378 100644 --- a/examples-copier/docs/FAQ.md +++ b/examples-copier/docs/FAQ.md @@ -176,7 +176,7 @@ Yes! See [Local Testing](LOCAL-TESTING.md) for instructions. ### How do I deploy to Google Cloud? -See [Deployment Guide](DEPLOYMENT-GUIDE.md) for step-by-step instructions. +See [Deployment Guide](DEPLOYMENT.md) for complete guide and [Deployment Checklist](DEPLOYMENT-CHECKLIST.md) for step-by-step instructions. ### Do I need MongoDB? diff --git a/examples-copier/docs/LOCAL-TESTING.md b/examples-copier/docs/LOCAL-TESTING.md index 875cb31..ac4b9d1 100644 --- a/examples-copier/docs/LOCAL-TESTING.md +++ b/examples-copier/docs/LOCAL-TESTING.md @@ -69,15 +69,15 @@ GITHUB_TOKEN=ghp_your_token_here ```bash # Terminal 1: Start the app -make run-local +make run-local-quick # You should see: # ╔════════════════════════════════════════════════════════════════╗ # ║ GitHub Code Example Copier ║ # ╠════════════════════════════════════════════════════════════════╣ # ║ Port: 8080 ║ -# ║ Webhook Path: /webhook ║ -# ║ Config File: config.json ║ +# ║ Webhook Path: /events ║ +# ║ Config File: copier-config.example.yaml ║ # ║ Dry Run: true ║ # ║ Audit Log: false ║ # ║ Metrics: true ║ @@ -87,12 +87,17 @@ make run-local ### Test with Webhook ```bash -# Terminal 2: Send test webhook -./test-webhook -payload test-payloads/example-pr-merged.json +# Terminal 2: Send test webhook (automatically fetches webhook secret) +make test-webhook-example + +# Or send webhook manually with secret +export WEBHOOK_SECRET=$(gcloud secrets versions access latest --secret=webhook-secret) +./test-webhook -payload test-payloads/example-pr-merged.json -secret "$WEBHOOK_SECRET" # Or test with real PR export GITHUB_TOKEN=ghp_... -./test-webhook -pr 456 -owner mongodb -repo docs-realm +export WEBHOOK_SECRET=$(gcloud secrets versions access latest --secret=webhook-secret) +./test-webhook -pr 456 -owner mongodb -repo docs-realm -secret "$WEBHOOK_SECRET" ``` ## What Happens in Local Mode @@ -309,17 +314,24 @@ COPIER_DISABLE_CLOUD_LOGGING=true ./examples-copier ### Error: "connection refused" when sending webhook -**Problem:** Application is not running +**Problem:** Application is not running, or you're trying to run both in the same terminal **Solution:** ```bash -# Make sure app is running in Terminal 1 -make run-local +# Terminal 1: Start the app (this blocks the terminal) +make run-local-quick -# Then send webhook in Terminal 2 -./test-webhook -payload test-payloads/example-pr-merged.json +# Terminal 2: In a NEW terminal window, send the webhook +cd examples-copier +make test-webhook-example + +# Or manually: +export WEBHOOK_SECRET=$(gcloud secrets versions access latest --secret=webhook-secret) +./test-webhook -payload test-payloads/example-pr-merged.json -secret "$WEBHOOK_SECRET" ``` +**Note:** The `make test-webhook-example` command requires the server to be running in a separate terminal. You cannot run both commands in the same terminal unless you background the server process. + ### Error: "GITHUB_TOKEN environment variable not set" **Problem:** Trying to fetch real PR without token @@ -411,19 +423,25 @@ Then you can: 3. Monitor metrics and audit logs 4. Deploy to production -See [DEPLOYMENT-GUIDE.md](DEPLOYMENT-GUIDE.md) for deployment instructions. +See [DEPLOYMENT.md](DEPLOYMENT.md) for deployment instructions. ## Quick Reference ```bash -# Start app locally -make run-local +# Terminal 1: Start app locally +make run-local-quick -# Test with example -./test-webhook -payload test-payloads/example-pr-merged.json +# Terminal 2: Test with example (auto-fetches webhook secret) +make test-webhook-example + +# Or test manually with webhook secret +export WEBHOOK_SECRET=$(gcloud secrets versions access latest --secret=webhook-secret) +./test-webhook -payload test-payloads/example-pr-merged.json -secret "$WEBHOOK_SECRET" # Test with real PR -./test-webhook -pr 456 -owner mongodb -repo docs-realm +export GITHUB_TOKEN=ghp_... +export WEBHOOK_SECRET=$(gcloud secrets versions access latest --secret=webhook-secret) +./test-webhook -pr 456 -owner mongodb -repo docs-realm -secret "$WEBHOOK_SECRET" # Check metrics curl http://localhost:8080/metrics | jq @@ -432,6 +450,6 @@ curl http://localhost:8080/metrics | jq curl http://localhost:8080/health | jq # Validate config -./config-validator validate -config config.json -v +./config-validator validate -config copier-config.yaml -v ``` diff --git a/examples-copier/docs/TROUBLESHOOTING.md b/examples-copier/docs/TROUBLESHOOTING.md index 59486a2..254f703 100644 --- a/examples-copier/docs/TROUBLESHOOTING.md +++ b/examples-copier/docs/TROUBLESHOOTING.md @@ -283,6 +283,9 @@ export COPIER_DISABLE_CLOUD_LOGGING=true export AUDIT_ENABLED=false ``` + + + ### GitHub API Rate Limit **Error:** diff --git a/examples-copier/docs/WEBHOOK-TESTING.md b/examples-copier/docs/WEBHOOK-TESTING.md index 2cf25d7..00a0d7e 100644 --- a/examples-copier/docs/WEBHOOK-TESTING.md +++ b/examples-copier/docs/WEBHOOK-TESTING.md @@ -449,5 +449,5 @@ After successful webhook testing: 5. Deploy to production 6. Set up alerts for failures -See [DEPLOYMENT-GUIDE.md](DEPLOYMENT-GUIDE.md) for deployment instructions. +See [DEPLOYMENT.md](DEPLOYMENT.md) for deployment instructions. diff --git a/examples-copier/scripts/convert-env-to-yaml.sh b/examples-copier/scripts/convert-env-to-yaml.sh new file mode 100755 index 0000000..3a41ace --- /dev/null +++ b/examples-copier/scripts/convert-env-to-yaml.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Convert .env file to env.yaml format for Google Cloud App Engine + +set -e + +# Default input/output files +INPUT_FILE="${1:-.env}" +OUTPUT_FILE="${2:-env.yaml}" + +if [ ! -f "$INPUT_FILE" ]; then + echo "Error: Input file '$INPUT_FILE' not found" + echo "Usage: $0 [input-file] [output-file]" + echo "Example: $0 .env.production env.yaml" + exit 1 +fi + +echo "Converting $INPUT_FILE to $OUTPUT_FILE..." + +# Start the YAML file +echo "env_variables:" > "$OUTPUT_FILE" + +# Read the .env file and convert to YAML +while IFS= read -r line || [ -n "$line" ]; do + # Skip empty lines and comments + if [[ -z "$line" ]] || [[ "$line" =~ ^[[:space:]]*# ]]; then + continue + fi + + # Extract key and value + if [[ "$line" =~ ^([^=]+)=(.*)$ ]]; then + key="${BASH_REMATCH[1]}" + value="${BASH_REMATCH[2]}" + + # Remove leading/trailing whitespace from key + key=$(echo "$key" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + + # Remove quotes from value if present + value=$(echo "$value" | sed 's/^["'\'']\(.*\)["'\'']$/\1/') + + # Write to YAML file with proper indentation + echo " $key: \"$value\"" >> "$OUTPUT_FILE" + fi +done < "$INPUT_FILE" + +echo "✅ Conversion complete: $OUTPUT_FILE" +echo "" +echo "⚠️ IMPORTANT: Review $OUTPUT_FILE before deploying!" +echo " - Verify all values are correct" +echo " - Check for sensitive data" +echo " - Ensure $OUTPUT_FILE is in .gitignore" + diff --git a/examples-copier/scripts/grant-secret-access.sh b/examples-copier/scripts/grant-secret-access.sh new file mode 100755 index 0000000..023832c --- /dev/null +++ b/examples-copier/scripts/grant-secret-access.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Grant App Engine access to all secrets + +set -e + +PROJECT_ID="github-copy-code-examples" +PROJECT_NUMBER="1054147886816" +SERVICE_ACCOUNT="${PROJECT_NUMBER}@appspot.gserviceaccount.com" + +echo "Granting App Engine service account access to secrets..." +echo "Service Account: ${SERVICE_ACCOUNT}" +echo "" + +# Array of secrets to grant access to +SECRETS=( + "CODE_COPIER_PEM" + "webhook-secret" + "mongo-uri" +) + +for SECRET in "${SECRETS[@]}"; do + echo "Granting access to: ${SECRET}" + gcloud secrets add-iam-policy-binding "${SECRET}" \ + --member="serviceAccount:${SERVICE_ACCOUNT}" \ + --role="roles/secretmanager.secretAccessor" \ + --project="${PROJECT_ID}" 2>&1 | grep -E "Updated|bindings" || echo " Already has access" + echo "" +done + +echo "✅ Done! Verifying permissions..." +echo "" + +for SECRET in "${SECRETS[@]}"; do + echo "Permissions for ${SECRET}:" + gcloud secrets get-iam-policy "${SECRET}" \ + --project="${PROJECT_ID}" \ + --format="table(bindings.members)" 2>&1 | grep -A 5 "serviceAccount:${SERVICE_ACCOUNT}" || echo " Not found" + echo "" +done + +echo "✅ All secrets are now accessible by App Engine!" + diff --git a/examples-copier/services/github_auth.go b/examples-copier/services/github_auth.go index 046254e..6b28189 100644 --- a/examples-copier/services/github_auth.go +++ b/examples-copier/services/github_auth.go @@ -118,6 +118,97 @@ func getPrivateKeyFromSecret() []byte { return result.Payload.Data } +// getWebhookSecretFromSecretManager retrieves the webhook secret from Google Cloud Secret Manager +func getWebhookSecretFromSecretManager(secretName string) (string, error) { + if os.Getenv("SKIP_SECRET_MANAGER") == "true" { + // For tests and local runs, use direct env var + if secret := os.Getenv(configs.WebhookSecret); secret != "" { + return secret, nil + } + return "", fmt.Errorf("SKIP_SECRET_MANAGER=true but no WEBHOOK_SECRET set") + } + + ctx := context.Background() + client, err := secretmanager.NewClient(ctx) + if err != nil { + return "", fmt.Errorf("failed to create Secret Manager client: %w", err) + } + defer client.Close() + + req := &secretmanagerpb.AccessSecretVersionRequest{ + Name: secretName, + } + result, err := client.AccessSecretVersion(ctx, req) + if err != nil { + return "", fmt.Errorf("failed to access secret version: %w", err) + } + return string(result.Payload.Data), nil +} + +// LoadWebhookSecret loads the webhook secret from Secret Manager or environment variable +func LoadWebhookSecret(config *configs.Config) error { + // If webhook secret is already set directly, use it + if config.WebhookSecret != "" { + return nil + } + + // Otherwise, load from Secret Manager + secret, err := getWebhookSecretFromSecretManager(config.WebhookSecretName) + if err != nil { + return fmt.Errorf("failed to load webhook secret: %w", err) + } + config.WebhookSecret = secret + return nil +} + +// LoadMongoURI loads the MongoDB URI from Secret Manager or environment variable +func LoadMongoURI(config *configs.Config) error { + // If MongoDB URI is already set directly, use it + if config.MongoURI != "" { + return nil + } + + // If no secret name is configured, skip (audit logging is optional) + if config.MongoURISecretName == "" { + return nil + } + + // Load from Secret Manager + uri, err := getSecretFromSecretManager(config.MongoURISecretName, "MONGO_URI") + if err != nil { + return fmt.Errorf("failed to load MongoDB URI: %w", err) + } + config.MongoURI = uri + return nil +} + +// getSecretFromSecretManager is a generic function to retrieve any secret from Secret Manager +func getSecretFromSecretManager(secretName, envVarName string) (string, error) { + if os.Getenv("SKIP_SECRET_MANAGER") == "true" { + // For tests and local runs, use direct env var + if secret := os.Getenv(envVarName); secret != "" { + return secret, nil + } + return "", fmt.Errorf("SKIP_SECRET_MANAGER=true but no %s set", envVarName) + } + + ctx := context.Background() + client, err := secretmanager.NewClient(ctx) + if err != nil { + return "", fmt.Errorf("failed to create Secret Manager client: %w", err) + } + defer client.Close() + + req := &secretmanagerpb.AccessSecretVersionRequest{ + Name: secretName, + } + result, err := client.AccessSecretVersion(ctx, req) + if err != nil { + return "", fmt.Errorf("failed to access secret version: %w", err) + } + return string(result.Payload.Data), nil +} + // getInstallationAccessToken exchanges a JWT for a GitHub App installation access token. func getInstallationAccessToken(installationId, jwtToken string, hc *http.Client) (string, error) { if installationId == "" || installationId == configs.InstallationId { diff --git a/examples-copier/services/github_write_to_source.go b/examples-copier/services/github_write_to_source.go index 54a6b8d..2f336b6 100644 --- a/examples-copier/services/github_write_to_source.go +++ b/examples-copier/services/github_write_to_source.go @@ -63,8 +63,8 @@ func uploadDeprecationFileChanges(message string, newDeprecationFileContents str Message: github.String(message), Content: []byte(newDeprecationFileContents), Branch: github.String(os.Getenv(configs.SrcBranch)), - Committer: &github.CommitAuthor{Name: github.String(os.Getenv(configs.CommiterName)), - Email: github.String(os.Getenv(configs.CommiterEmail))}, + Committer: &github.CommitAuthor{Name: github.String(os.Getenv(configs.CommitterName)), + Email: github.String(os.Getenv(configs.CommitterEmail))}, } options.SHA = targetFileContent.SHA