Merge pull request #135 from guardian/migration-script

Migration script

Author: alexduf

scripts/2025-11-18-migration-to-recipe-v3/.gitignore

@@ -0,0 +1 @@
+data

scripts/2025-11-18-migration-to-recipe-v3/.python-version

@@ -0,0 +1 @@
+3.12

scripts/2025-11-18-migration-to-recipe-v3/README.md

@@ -0,0 +1,31 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class Config:
+  capi_key: str
+  capi_url: str
+  index_url: str
+  templatiser_url: str
+  templatiser_token: str
+  integration_read_url: str
+  integration_write_url: str
+  ca_bundle_path: str | None
+
+
+def load_config(environment: str) -> Config:
+  filepath = Path.home() / '.gu' / f"feast-migration-v3-config.{environment}.json"
+  with open(filepath, 'r') as f:
+    import json
+    data = json.load(f)
+    return Config(
+      capi_key=data['capi_key'],
+      capi_url=data['capi_url'],
+      index_url=data['index_url'],
+      templatiser_url=data['templatiser_url'],
+      templatiser_token=data['templatiser_token'],
+      integration_read_url=data['integration_read_url'],
+      integration_write_url=data['integration_write_url'],
+      ca_bundle_path=data.get('ca_bundle_path'),
+    )

scripts/2025-11-18-migration-to-recipe-v3/config.py

@@ -0,0 +1,113 @@
+import dataclasses
+import os
+from csv import DictReader, DictWriter
+from dataclasses import dataclass
+from enum import Enum
+
+class Stage1ReportStatus(Enum):
+  SUCCESS = "SUCCESS"
+  ACCEPTED_BY_LLM = "ACCEPTED_BY_LLM"
+  REVIEW_NEEDED = "REVIEW_NEEDED"
+  ERROR = "ERROR"
+  INCOMPLETE = "INCOMPLETE" # the LLM could not get it to match after 5 attempts
+
+@dataclass(frozen=True)
+class Stage1Report:
+  recipe_id: str
+  capi_id: str
+  composer_id: str | None
+  filename: str
+  status: Stage1ReportStatus
+  reason: str | None
+  diff: str | None
+  expected: str | None
+  received: str | None
+  cost: str
+  revision: int
+
+  @staticmethod
+  def error(recipe_id: str, capi_id: str, reason: str) -> 'Stage1Report':
+    return Stage1Report(
+      recipe_id=recipe_id,
+      capi_id=capi_id,
+      composer_id=None,
+      filename="",
+      status=Stage1ReportStatus.ERROR,
+      reason=reason,
+      diff=None,
+      expected=None,
+      received=None,
+      cost="0",
+      revision=0,
+    )
+
+
+def stage_1_csv_filename(state_folder: str) -> str:
+  return f"{state_folder}/stage-1-results.csv"
+
+def load_stage1_csv_state(state_folder: str) -> list[Stage1Report]:
+  reports = []
+  with open(stage_1_csv_filename(state_folder), newline='') as csvfile:
+    reader = DictReader(csvfile)
+    for row in reader:
+      stage1_report = Stage1Report(
+        recipe_id=row['recipe_id'],
+        capi_id=row['capi_id'],
+        composer_id=row['composer_id'] if row['composer_id'] else None,
+        filename=row['filename'],
+        status=Stage1ReportStatus(row['status'].removeprefix("Stage1ReportStatus.")),
+        reason=row['reason'] if row['reason'] else None,
+        diff=row['diff'] if row['diff'] else None,
+        expected=row['expected'] if row['expected'] else None,
+        received=row['received'] if row['received'] else None,
+        cost=row['cost'],
+        revision=int(row['revision']),
+      )
+      reports.append(stage1_report)
+  return reports
+
+class Stage2ReportStatus(Enum):
+  SUCCESS = "success"
+  ERROR = "error"
+  CAPI_UPDATED = "capi_updated"
+
+@dataclass(frozen=True)
+class Stage2Report(Stage1Report):
+  stage2_status: Stage2ReportStatus
+  failure_reason: str | None
+
+  @staticmethod
+  def from_stage1_report(report: Stage1Report, status: Stage2ReportStatus,
+                         failure_reason: str | None) -> 'Stage2Report':
+    return Stage2Report(
+      **dataclasses.asdict(report),
+      stage2_status=status,
+      failure_reason=failure_reason,
+  )
+
+def stage_2_csv_filename(state_folder: str) -> str:
+  return f"{state_folder}/stage-2-results.csv"
+
+def append_stage2_report(state_folder: str, report: Stage2Report):
+  filename = stage_2_csv_filename(state_folder)
+  file_exists = os.path.exists(filename) and os.path.getsize(filename) > 0
+  with open(filename, 'a', newline='') as f:
+    fieldnames = [field.name for field in dataclasses.fields(Stage2Report)]
+    writer = DictWriter(f, fieldnames=fieldnames)
+    if not file_exists:
+      writer.writeheader()
+      f.flush()
+
+    writer.writerow(dataclasses.asdict(report))
+    f.flush()
+
+def load_stage2_csv_state(state_folder: str) -> list[Stage2Report]:
+  reports = []
+  filename = stage_2_csv_filename(state_folder)
+  if not os.path.exists(filename):
+    return reports
+  with open(filename, newline='') as csvfile:
+    reader = DictReader(csvfile)
+    for row in reader:
+      reports.append(Stage2Report(**row))
+  return reports

scripts/2025-11-18-migration-to-recipe-v3/csv_state.py

@@ -0,0 +1,48 @@
+import json
+import logging
+import os
+from argparse import ArgumentParser
+
+from config import load_config
+from fancy_logging import init_logger
+from services import fetch_index, fetch_flexible_article, fetch_CAPI_article
+
+logger = logging.getLogger(__name__)
+
+def main(state_folder: str, environment: str):
+  init_logger()
+  config = load_config(environment=environment)
+
+  os.makedirs(state_folder, exist_ok=True)
+
+  recipes = fetch_index(config)
+
+  capi_ids = set()
+  for recipe in recipes:
+    capi_ids.add(recipe.capi_id)
+
+  for capi_id in capi_ids:
+    try:
+      capi_fetch_response = fetch_CAPI_article(capi_id, config)
+      if capi_fetch_response is None:
+        logger.warning(f"Article {capi_id} not found in CAPI")
+        continue
+
+      # fetch the recipes from composer (flexible)
+      composer_id = capi_fetch_response["response"]["content"]["fields"].get("internalComposerCode") if capi_fetch_response is not None else None
+      flexible_article = fetch_flexible_article(composer_id, config)
+
+      for recipe in flexible_article.recipes:
+        with open(os.path.join(state_folder, f"{recipe["id"]}.json"), "w") as f:
+          f.write(json.dumps(recipe, indent=2))
+    except Exception as e:
+      logger.error(f"Error processing CAPI article {capi_id}: {e}")
+  logger.info("ALl done!")
+
+if __name__ == "__main__":
+  arg_parser = ArgumentParser(description='Stage 2 of the migration to recipe v3')
+  arg_parser.add_argument('-s', '--state-folder', type=str, required=True, help='Path to the state folder')
+  arg_parser.add_argument('-e', '--environment', type=str, default='CODE', choices=['LOCAL', 'CODE', 'PROD'], help='The environment to use (LOCAL, CODE, PROD)')
+
+  args = arg_parser.parse_args()
+  main(state_folder=args.state_folder, environment=args.environment)

scripts/2025-11-18-migration-to-recipe-v3/dump_recipe_data.py

@@ -0,0 +1,45 @@
+import logging
+from rich.logging import RichHandler
+from rich.console import Console
+
+# Shared console instance for both logging and progress
+_console = None
+
+def get_console():
+  """Get or create the shared console instance"""
+  global _console
+  if _console is None:
+    _console = Console(stderr=True)
+  return _console
+
+class ColoredFormatter(logging.Formatter):
+  # ANSI color codes
+  COLORS = {
+    'DEBUG': '\033[90m',     # Grey
+    'INFO': '\033[97m',      # White
+    'WARNING': '\033[93m',   # Yellow
+    'ERROR': '\033[91m',     # Red
+    'CRITICAL': '\033[91m',  # Red
+  }
+  RESET = '\033[0m'
+
+  def format(self, record):
+    log_color = self.COLORS.get(record.levelname, self.RESET)
+    record.levelname = f"{log_color}{record.levelname}{self.RESET}"
+    record.msg = f"{log_color}{record.msg}{self.RESET}"
+    return super().format(record)
+
+def init_logger(level=logging.INFO):
+    handler = RichHandler(
+      console=get_console(),
+      show_time=True,
+      show_path=False,
+      rich_tracebacks=True,
+      tracebacks_show_locals=True
+    )
+    logging.basicConfig(
+      level=level,
+      handlers=[handler],
+      format="%(message)s",
+      datefmt="[%Y-%m-%d %H:%M:%S]"
+    )

scripts/2025-11-18-migration-to-recipe-v3/fancy_logging.py

@@ -0,0 +1,11 @@
+[project]
+name = "2025-11-18-migration-to-recipe-v3"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "boto3>=1.41.3",
+    "requests>=2.32.5",
+    "rich>=13.9.4",
+]