refactor: allow gh merge and retry values to be configurable

Author: cbullinger

examples-copier/configs/aggregation-tasks-config.yaml

@@ -0,0 +1,127 @@
+# Configuration for copying code examples from aggregation-tasks repository
+# Repository: https://github.com/cbullinger/aggregation-tasks
+# 
+# This repository contains MongoDB aggregation pipeline examples in multiple languages:
+# - java/
+# - nodejs/
+# - python/
+
+source_repo: "cbullinger/aggregation-tasks"
+source_branch: "main"
+
+copy_rules:
+  # Rule 1: Match all language directories with regex pattern
+  # This pattern extracts the language name and file path
+  - name: "aggregation-language-examples"
+    source_pattern:
+      type: "regex"
+      pattern: "^(?P<lang>java|nodejs|python)/(?P<file>.+)$"
+    targets:
+      - repo: "mongodb/docs"
+        branch: "main"
+        path_transform: "source/code-examples/aggregation/${lang}/${file}"
+        commit_strategy:
+          type: "pull_request"
+          pr_title: "Update ${lang} aggregation examples"
+          pr_body: |
+            Automated update of ${lang} aggregation pipeline examples
+
+            **Details:**
+            - Rule: ${rule_name}
+            - Source: ${source_repo}
+            - Files updated: ${file_count}
+          auto_merge: false
+        deprecation_check:
+          enabled: true
+          file: "deprecated_examples.json"
+
+  # Rule 2: Match specific language - Java only
+  - name: "java-aggregation-examples"
+    source_pattern:
+      type: "regex"
+      pattern: "^java/(?P<file>.+\\.java)$"
+    targets:
+      - repo: "mongodb/docs-java"
+        branch: "main"
+        path_transform: "source/examples/aggregation/${file}"
+        commit_strategy:
+          type: "pull_request"
+          pr_title: "Update Java aggregation examples"
+          pr_body: "Automated update of Java aggregation pipeline examples"
+          auto_merge: false
+
+  # Rule 3: Match specific language - Node.js only
+  - name: "nodejs-aggregation-examples"
+    source_pattern:
+      type: "regex"
+      pattern: "^nodejs/(?P<file>.+\\.(js|ts))$"
+    targets:
+      - repo: "mongodb/docs-node"
+        branch: "main"
+        path_transform: "source/examples/aggregation/${file}"
+        commit_strategy:
+          type: "pull_request"
+          pr_title: "Update Node.js aggregation examples"
+          pr_body: "Automated update of Node.js aggregation pipeline examples"
+          auto_merge: false
+
+  # Rule 4: Match specific language - Python only
+  - name: "python-aggregation-examples"
+    source_pattern:
+      type: "regex"
+      pattern: "^python/(?P<file>.+\\.py)$"
+    targets:
+      - repo: "mongodb/docs-python"
+        branch: "main"
+        path_transform: "source/examples/aggregation/${file}"
+        commit_strategy:
+          type: "pull_request"
+          pr_title: "Update Python aggregation examples"
+          pr_body: "Automated update of Python aggregation pipeline examples"
+          auto_merge: false
+
+  # Rule 5: Alternative pattern - match with subdirectories
+  # Use this if the repository structure has subdirectories within language folders
+  - name: "aggregation-with-subdirs"
+    source_pattern:
+      type: "regex"
+      pattern: "^(?P<lang>java|nodejs|python)/(?P<subdir>[^/]+)/(?P<file>.+)$"
+    targets:
+      - repo: "mongodb/docs"
+        branch: "main"
+        path_transform: "source/code-examples/aggregation/${lang}/${subdir}/${file}"
+        commit_strategy:
+          type: "pull_request"
+          pr_title: "Update ${lang} aggregation examples (${subdir})"
+          pr_body: |
+            Automated update of ${lang} aggregation examples
+            Category: ${subdir}
+            Files updated: ${file_count}
+          auto_merge: false
+
+  # Rule 6: Glob pattern alternative - simpler but less flexible
+  - name: "all-java-files-glob"
+    source_pattern:
+      type: "glob"
+      pattern: "java/**/*.java"
+    targets:
+      - repo: "mongodb/docs"
+        branch: "main"
+        path_transform: "source/code-examples/aggregation/java/${filename}"
+        commit_strategy:
+          type: "direct"
+          commit_message: "Update Java aggregation examples"
+
+  # Rule 7: Prefix pattern - simplest approach
+  - name: "all-languages-prefix"
+    source_pattern:
+      type: "prefix"
+      pattern: "java"
+    targets:
+      - repo: "mongodb/docs"
+        branch: "main"
+        path_transform: "source/code-examples/aggregation/${relative_path}"
+        commit_strategy:
+          type: "direct"
+          commit_message: "Update aggregation examples from ${source_repo}"
+

examples-copier/configs/env.yaml.example

@@ -111,10 +111,27 @@ env_variables:
   # DEFAULT BEHAVIORS (OPTIONAL)
   # =============================================================================
   # System-wide defaults that individual config rules can override
-  
+
   DEFAULT_RECURSIVE_COPY: "true"                   # Default recursive copy behavior (default: true)
   DEFAULT_PR_MERGE: "false"                        # Default auto-merge PRs without review (default: false)
   DEFAULT_COMMIT_MESSAGE: "Automated PR with updated examples"  # Default commit message (default: shown)
+
+  # =============================================================================
+  # GITHUB API CONFIGURATION (OPTIONAL)
+  # =============================================================================
+  # Fine-tune GitHub API retry and polling behavior
+
+  # GitHub API Retry Configuration
+  # Controls retry behavior when GitHub API calls fail due to eventual consistency
+  # GITHUB_API_MAX_RETRIES: "3"                    # Number of retry attempts (default: 3)
+  # GITHUB_API_INITIAL_RETRY_DELAY: "500"          # Initial retry delay in milliseconds (default: 500)
+  #                                                 # Uses exponential backoff: 500ms, 1s, 2s, etc.
+
+  # PR Merge Polling Configuration
+  # Controls how long to wait for GitHub to compute PR mergeability
+  # PR_MERGE_POLL_MAX_ATTEMPTS: "20"               # Max polling attempts (default: 20)
+  # PR_MERGE_POLL_INTERVAL: "500"                  # Polling interval in milliseconds (default: 500)
+  #                                                 # Total wait time = attempts × interval (default: ~10 seconds)
 
   # =============================================================================
   # TESTING / DEVELOPMENT OVERRIDES (DO NOT USE IN PRODUCTION)

examples-copier/configs/environment.go

@@ -47,6 +47,14 @@ type Config struct {
 	SlackUsername    string
 	SlackIconEmoji   string
 	SlackEnabled     bool
+
+	// GitHub API retry configuration
+	GitHubAPIMaxRetries        int
+	GitHubAPIInitialRetryDelay int // in milliseconds
+
+	// PR merge polling configuration
+	PRMergePollMaxAttempts int
+	PRMergePollInterval    int // in milliseconds
 }
 
 const (
@@ -78,11 +86,15 @@ const (
 	AuditDatabase        = "AUDIT_DATABASE"
 	AuditCollection      = "AUDIT_COLLECTION"
 	MetricsEnabled       = "METRICS_ENABLED"
-	SlackWebhookURL      = "SLACK_WEBHOOK_URL"
-	SlackChannel         = "SLACK_CHANNEL"
-	SlackUsername        = "SLACK_USERNAME"
-	SlackIconEmoji       = "SLACK_ICON_EMOJI"
-	SlackEnabled         = "SLACK_ENABLED"
+	SlackWebhookURL              = "SLACK_WEBHOOK_URL"
+	SlackChannel                 = "SLACK_CHANNEL"
+	SlackUsername                = "SLACK_USERNAME"
+	SlackIconEmoji               = "SLACK_ICON_EMOJI"
+	SlackEnabled                 = "SLACK_ENABLED"
+	GitHubAPIMaxRetries          = "GITHUB_API_MAX_RETRIES"
+	GitHubAPIInitialRetryDelay   = "GITHUB_API_INITIAL_RETRY_DELAY"
+	PRMergePollMaxAttempts       = "PR_MERGE_POLL_MAX_ATTEMPTS"
+	PRMergePollInterval          = "PR_MERGE_POLL_INTERVAL"
 )
 
 // NewConfig returns a new Config instance with default values
@@ -99,9 +111,13 @@ func NewConfig() *Config {
 		WebhookSecretName:    "projects/1054147886816/secrets/webhook-secret/versions/latest",  // default webhook secret name for GCP Secret Manager
 		CopierLogName:        "copy-copier-log",                                                // default log name for logging to GCP
 		GoogleCloudProjectId: "github-copy-code-examples",                                      // default project ID for logging to GCP
-		DefaultRecursiveCopy: true,                                                             // system-wide default for recursive copying that individual config entries can override.
-		DefaultPRMerge:       false,                                                            // system-wide default for PR merge without review that individual config entries can override.
-		DefaultCommitMessage: "Automated PR with updated examples",                             // default commit message used when per-config commit_message is absent.
+		DefaultRecursiveCopy:       true,                                                             // system-wide default for recursive copying that individual config entries can override.
+		DefaultPRMerge:             false,                                                            // system-wide default for PR merge without review that individual config entries can override.
+		DefaultCommitMessage:       "Automated PR with updated examples",                             // default commit message used when per-config commit_message is absent.
+		GitHubAPIMaxRetries:        3,                                                                // default number of retry attempts for GitHub API calls
+		GitHubAPIInitialRetryDelay: 500,                                                              // default initial retry delay in milliseconds (exponential backoff)
+		PRMergePollMaxAttempts:     20,                                                               // default max attempts to poll PR for mergeability (~10 seconds with 500ms interval)
+		PRMergePollInterval:        500,                                                              // default polling interval in milliseconds
 	}
 }
 
@@ -173,6 +189,14 @@ func LoadEnvironment(envFile string) (*Config, error) {
 	config.SlackIconEmoji = getEnvWithDefault(SlackIconEmoji, ":robot_face:")
 	config.SlackEnabled = getBoolEnvWithDefault(SlackEnabled, config.SlackWebhookURL != "")
 
+	// GitHub API retry configuration
+	config.GitHubAPIMaxRetries = getIntEnvWithDefault(GitHubAPIMaxRetries, config.GitHubAPIMaxRetries)
+	config.GitHubAPIInitialRetryDelay = getIntEnvWithDefault(GitHubAPIInitialRetryDelay, config.GitHubAPIInitialRetryDelay)
+
+	// PR merge polling configuration
+	config.PRMergePollMaxAttempts = getIntEnvWithDefault(PRMergePollMaxAttempts, config.PRMergePollMaxAttempts)
+	config.PRMergePollInterval = getIntEnvWithDefault(PRMergePollInterval, config.PRMergePollInterval)
+
 	// Export resolved values back into environment so downstream os.Getenv sees defaults
 	_ = os.Setenv(Port, config.Port)
 	_ = os.Setenv(RepoName, config.RepoName)
@@ -218,6 +242,19 @@ func getBoolEnvWithDefault(key string, defaultValue bool) bool {
 	return strings.ToLower(value) == "true"
 }
 
+// getIntEnvWithDefault returns the integer environment variable value or default if not set
+func getIntEnvWithDefault(key string, defaultValue int) int {
+	value := os.Getenv(key)
+	if value == "" {
+		return defaultValue
+	}
+	var intValue int
+	if _, err := fmt.Sscanf(value, "%d", &intValue); err != nil {
+		return defaultValue
+	}
+	return intValue
+}
+
 // validateConfig checks if all required configuration values are set
 func validateConfig(config *Config) error {
 	var missingVars []string

examples-copier/services/github_write_to_target.go

@@ -165,11 +165,26 @@ func addFilesViaPR(ctx context.Context, client *github.Client, key UploadKey,
 	LogInfo(fmt.Sprintf("PR URL: %s", pr.GetHTMLURL()))
 	if mergeWithoutReview {
 		// Poll PR for mergeability; GitHub may take a moment to compute it
-		// We poll up to ~10s with 500ms interval
+		// Get polling configuration from environment or use defaults
+		cfg := configs.NewConfig()
+		maxAttempts := cfg.PRMergePollMaxAttempts
+		if envAttempts := os.Getenv(configs.PRMergePollMaxAttempts); envAttempts != "" {
+			if parsed, err := parseIntWithDefault(envAttempts, maxAttempts); err == nil {
+				maxAttempts = parsed
+			}
+		}
+
+		pollInterval := cfg.PRMergePollInterval
+		if envInterval := os.Getenv(configs.PRMergePollInterval); envInterval != "" {
+			if parsed, err := parseIntWithDefault(envInterval, pollInterval); err == nil {
+				pollInterval = parsed
+			}
+		}
+
 		var mergeable *bool
 		var mergeableState string
 		owner, repoName := parseRepoPath(key.RepoName)
-		for i := 0; i < 20; i++ {
+		for i := 0; i < maxAttempts; i++ {
 			current, _, gerr := client.PullRequests.Get(ctx, owner, repoName, pr.GetNumber())
 			if gerr == nil && current != nil {
 				mergeable = current.Mergeable
@@ -178,7 +193,7 @@ func addFilesViaPR(ctx context.Context, client *github.Client, key UploadKey,
 					break
 				}
 			}
-			time.Sleep(500 * time.Millisecond)
+			time.Sleep(time.Duration(pollInterval) * time.Millisecond)
 		}
 		if mergeable != nil && !*mergeable || strings.EqualFold(mergeableState, "dirty") {
 			LogWarning(fmt.Sprintf("PR #%d is not mergeable (state=%s). Likely merge conflicts. Leaving PR open for manual resolution.", pr.GetNumber(), mergeableState))
@@ -269,8 +284,24 @@ func createCommitTree(ctx context.Context, client *github.Client, targetBranch U
 	// 1) Get current ref with retry logic to handle GitHub API eventual consistency
 	// When a branch is just created, it may take a moment to be visible
 	var ref *github.Reference
-	maxRetries := 3
-	retryDelay := 500 * time.Millisecond
+
+	// Get retry configuration from environment or use defaults
+	cfg := configs.NewConfig()
+	maxRetries := cfg.GitHubAPIMaxRetries
+	if envRetries := os.Getenv(configs.GitHubAPIMaxRetries); envRetries != "" {
+		if parsed, err := parseIntWithDefault(envRetries, maxRetries); err == nil {
+			maxRetries = parsed
+		}
+	}
+
+	initialRetryDelay := cfg.GitHubAPIInitialRetryDelay
+	if envDelay := os.Getenv(configs.GitHubAPIInitialRetryDelay); envDelay != "" {
+		if parsed, err := parseIntWithDefault(envDelay, initialRetryDelay); err == nil {
+			initialRetryDelay = parsed
+		}
+	}
+
+	retryDelay := time.Duration(initialRetryDelay) * time.Millisecond
 
 	for attempt := 1; attempt <= maxRetries; attempt++ {
 		ref, _, err = client.Git.GetRef(ctx, owner, repoName, targetBranch.BranchPath)
@@ -401,3 +432,12 @@ func deleteBranchIfExists(backgroundContext context.Context, client *github.Clie
 func DeleteBranchIfExistsExported(ctx context.Context, client *github.Client, repo string, ref *github.Reference) {
 	deleteBranchIfExists(ctx, client, repo, ref)
 }
+
+// parseIntWithDefault parses a string to int, returning defaultValue on error
+func parseIntWithDefault(s string, defaultValue int) (int, error) {
+	var result int
+	if _, err := fmt.Sscanf(s, "%d", &result); err != nil {
+		return defaultValue, err
+	}
+	return result, nil
+}