make pnpm catalog the primary dependency list

.github/workflows/ci-typescript.yml

@@ -74,6 +74,13 @@ jobs:
           node-version-file: '.nvmrc'
           cache: 'pnpm'
       - run: pnpm install
+      - run: |
+          if [ -f "./handlers/${{ matrix.subproject }}/BUILDCHECK.md" ]; then
+            pnpm --filter "./buildcheck" type-check
+            pnpm --filter "./buildcheck" check-formatting
+            pnpm --filter "./buildcheck" lint
+            pnpm --filter "./buildcheck" snapshot:assert handlers/${{ matrix.subproject }}
+          fi
       - run: pnpm --filter cdk package
       - run: pnpm --filter ${{ matrix.subproject }} package
 

BUILDCHECK.md

@@ -0,0 +1,23 @@
+MANAGED FILE: to push changes see buildcheck/README.md - template: buildcheck/data/snippets/BUILDCHECK.md.ts
+# Buildcheck managed file list
+
+The files listed below are managed by buildcheck and their content is checked by the build.
+
+## HOWTO edit managed files
+1. edit the build definition in buildcheck/data/
+2. run `pnpm update-build` at the root
+
+For further details, see [buildcheck/README.md](./buildcheck/README.md)
+
+## Generated file list:
+- [handlers/alarms-handler/jest.config.js](handlers/alarms-handler/jest.config.js)
+- [handlers/alarms-handler/package.json](handlers/alarms-handler/package.json)
+- [handlers/alarms-handler/riff-raff.yaml](handlers/alarms-handler/riff-raff.yaml)
+- [handlers/alarms-handler/tsconfig.json](handlers/alarms-handler/tsconfig.json)
+- [handlers/alarms-handler/BUILDCHECK.md](handlers/alarms-handler/BUILDCHECK.md)
+- [handlers/discount-api/jest.config.js](handlers/discount-api/jest.config.js)
+- [handlers/discount-api/package.json](handlers/discount-api/package.json)
+- [handlers/discount-api/riff-raff.yaml](handlers/discount-api/riff-raff.yaml)
+- [handlers/discount-api/tsconfig.json](handlers/discount-api/tsconfig.json)
+- [handlers/discount-api/BUILDCHECK.md](handlers/discount-api/BUILDCHECK.md)
+- [BUILDCHECK.md](BUILDCHECK.md)

buildcheck/.gitignore

@@ -0,0 +1,4 @@
+node_modules/
+dist/
+*.log
+/src/dynamic/generated/

buildcheck/README.md

@@ -0,0 +1,47 @@
+# Buildcheck - build file checker
+
+## What is buildcheck?
+Buildcheck checks the content of the build files for all lambda modules and their associated cdk modules during CI.
+
+The expected content is defined programmatically in typescript, similar to how CDK defines cloudformation.
+
+If you edit the build definition, it's easy to refresh the build files using the update command provided.
+
+## Quick start: How do I...?
+### ...add a dependency
+1. open [data/build.ts](data/build.ts) 
+1. add it to the dependencies section of your lambda definition
+1. run `pnpm snapshot:update`
+### ...bump a version
+1. open [data/dependencies.ts](data/dependencies.ts)
+1. edit the version number accordingly
+1. `pnpm snapshot:update`
+### ...quickly fix buildcheck in CI
+1. run `pnpm snapshot:update` to refresh the full set of build files
+2. commit and push
+
+## Migrating an existing handler to use buildcheck
+This requires the project to be restructured, and the definition added to buildcheck and then tested and refined.
+
+1. Add your new handler to [data/build.ts](data/build.ts)
+1. Run `pnpm snapshot:assert <handler-name>` to see any differences
+1. Edit the buildcheck data (and existing files) to resolve any differences
+1. retest until you are happy
+1. Run `pnpm snapshot:assert` to ensure no other handler has been affected
+1. Run `pnpm snapshot:update` to generate and overwrite the full set of build files (this should have no effect)
+1. Push and test your changes in CODE.
+
+## Pros and cons of buildcheck
+The benefits and drawbacks of buildcheck are similar to CDK or SBT:
+- prevent inconsistencies between lambdas/modules including dependency versions and pnpm scripts
+- have a central list of recommended dependencies
+- files can be generated where necessary, or buildcheck can be pulled in as a module to reuse definitions
+- allows more fine grained modules to be manageable improving build times and organisation
+- easier to add and review new lambda boilerplate as it's guaranteed to be standard
+- all the usual DRY benefits
+
+The disadvantages are mainly around tooling:
+- automated PRs to bump dependencies will fail unless the dependencies.ts file is updated accordingly (although dependabot etc doesn't yet work with pnpm catalog either)
+- harder to add dependencies as you need to add them to build.ts and then run snapshot:update.
+- non standard, could surprise new people
+- extra level of abstraction, could slow down regular developers

buildcheck/data/build.ts

@@ -0,0 +1,45 @@
+import { pnpmCatalog } from '../src/dynamic/dependencies';
+
+/*
+This is the main build definition for all handlers.
+
+Each record defines one handler and contains anything unique compared with the
+assumed build structure.
+ */
+
+export interface HandlerDefinition {
+	name: string;
+	functionNames?: string[];
+	entryPoints?: string[];
+	dependencies?: Record<string, string>;
+	devDependencies?: Record<string, string>;
+}
+
+const alarmsHandler: HandlerDefinition = {
+	name: 'alarms-handler',
+	functionNames: ['alarms-handler-', 'alarms-handler-scheduled-'],
+	entryPoints: ['src/index.ts', 'src/indexScheduled.ts'],
+	dependencies: {
+		...pnpmCatalog['@aws-sdk/client-cloudwatch'],
+		...pnpmCatalog['@aws-sdk/credential-providers'],
+		...pnpmCatalog.zod,
+	},
+	devDependencies: {
+		...pnpmCatalog['@types/aws-lambda'],
+		...pnpmCatalog.dayjs,
+	},
+};
+
+const discountApi: HandlerDefinition = {
+	name: 'discount-api',
+	dependencies: {
+		...pnpmCatalog.dayjs,
+		...pnpmCatalog['source-map-support'],
+		...pnpmCatalog.zod,
+	},
+	devDependencies: {
+		...pnpmCatalog['@types/aws-lambda'],
+	},
+};
+
+export const build: HandlerDefinition[] = [alarmsHandler, discountApi];

buildcheck/data/handlerTemplate/jest.config.js.ts

@@ -0,0 +1,14 @@
+import { notice } from '../snippets/notices';
+
+export default `// ${notice(__filename)}
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+	preset: 'ts-jest',
+	testEnvironment: 'node',
+	runner: 'groups',
+	moduleNameMapper: {
+		'@modules/(.*)/(.*)$': '<rootDir>/../../modules/$1/src/$2',
+		'@modules/(.*)$': '<rootDir>/../../modules/$1',
+	},
+};
+`;

buildcheck/data/handlerTemplate/package.json.ts

@@ -0,0 +1,28 @@
+import { HandlerDefinition } from '../build';
+import { notice } from '../snippets/notices';
+
+export default (pkg: HandlerDefinition) => {
+	const entryPoints = pkg.entryPoints
+		? pkg.entryPoints?.join(' ')
+		: 'src/index.ts';
+	return {
+		name: `${pkg.name}`,
+		scripts: {
+			test: 'jest --group=-integration',
+			'it-test': 'jest --group=integration',
+			'type-check': 'tsc --noEmit',
+			build:
+				'esbuild --bundle --platform=node --target=node18 --outdir=target/ ' +
+				entryPoints +
+				' --sourcemap',
+			lint: 'eslint src/**/*.ts test/**/*.ts',
+			package: `pnpm type-check && pnpm lint && pnpm check-formatting && pnpm test && pnpm build && cd target && zip -qr ${pkg.name}.zip ./*.js.map ./*.js`,
+			'check-formatting': 'prettier --check **.ts',
+			'fix-formatting': 'prettier --write **.ts',
+		},
+		NOTICE1: notice(__filename),
+		NOTICE2: 'all dependencies are defined in buildcheck/data/build.ts',
+		dependencies: pkg.dependencies,
+		devDependencies: pkg.devDependencies,
+	};
+};

buildcheck/data/handlerTemplate/riff-raff.yaml.ts

@@ -0,0 +1,29 @@
+import { HandlerDefinition } from '../build';
+
+export default (pkg: HandlerDefinition) => ({
+	stacks: ['support'],
+	regions: ['eu-west-1'],
+	allowedStages: ['CODE', 'PROD'],
+	deployments: {
+		[`${pkg.name}-cloudformation`]: {
+			type: 'cloud-formation',
+			app: pkg.name,
+			parameters: {
+				templateStagePaths: {
+					CODE: `${pkg.name}-CODE.template.json`,
+					PROD: `${pkg.name}-PROD.template.json`,
+				},
+			},
+		},
+		[pkg.name]: {
+			type: 'aws-lambda',
+			parameters: {
+				fileName: `${pkg.name}.zip`,
+				bucketSsmLookup: true,
+				prefixStack: false,
+				functionNames: pkg.functionNames ?? [pkg.name + '-'],
+			},
+			dependencies: [`${pkg.name}-cloudformation`],
+		},
+	},
+});

buildcheck/data/handlerTemplate/tsconfig.json.ts

@@ -0,0 +1,3 @@
+export default {
+	extends: '../../tsconfig.json',
+};

buildcheck/data/snippets/BUILDCHECK.md.ts

@@ -0,0 +1,54 @@
+import { notice, relativePath } from './notices';
+import path from 'path';
+import { GeneratedFile } from '../../src/steps/generatedFile';
+
+// the generated file is named after this file.
+// if you rename it, clean the snapshot first, then generate afterwards
+const basename = path.basename(__filename);
+if (!basename.endsWith('.ts')) {
+	throw new Error(
+		`Expected warning file name to end with .ts, got: ${basename}`,
+	);
+}
+export const warningFileName = basename.slice(0, -3);
+
+export function generateWarningFile(
+	generatedFiles: string[],
+	pathToRoot: string,
+): GeneratedFile {
+	const buildGenHeader = `${notice(__filename)}
+# Buildcheck managed file list
+
+The files listed below are managed by buildcheck and their content is checked by the build.
+
+## HOWTO edit managed files
+1. edit the build definition in buildcheck/data/
+2. run \`pnpm update-build\` at the root
+
+For further details, see [buildcheck/README.md](${pathToRoot}/buildcheck/README.md)
+
+## Generated file list:
+`;
+
+	return {
+		relativePath: warningFileName,
+		content:
+			buildGenHeader +
+			[...generatedFiles, warningFileName]
+				.map((name) => '- [' + name + '](' + name + ')\n')
+				.join(''),
+		templatePath: relativePath(__filename),
+	};
+}
+
+export function parseGeneratedFilenames(fileLines: string[]): string[] {
+	// lines to look for are like this: "- [filename](filepath)"
+	return fileLines
+		.filter((line) => line.startsWith('- ['))
+		.map((line) => {
+			const getTheContentsOfRoundBrackets = /\(([^)]+)\)/;
+			const match = line.match(getTheContentsOfRoundBrackets);
+			return match ? match[1] : '';
+		})
+		.filter((filepath) => filepath !== '');
+}

buildcheck/data/snippets/notices.ts

@@ -0,0 +1,12 @@
+export function relativePath(filename: string) {
+	// not watertight logic, but it isn't security critical
+	const moduleName = 'buildcheck';
+	const repoRootIndex = filename.indexOf('/' + moduleName + '/');
+	return repoRootIndex !== -1
+		? filename.substring(repoRootIndex + 1)
+		: filename;
+}
+
+export const notice = (filename: string) =>
+	'MANAGED FILE: to push changes see buildcheck/README.md - template: ' +
+	relativePath(filename);

buildcheck/jest.config.js

@@ -0,0 +1,6 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+	preset: 'ts-jest',
+	testEnvironment: 'node',
+	runner: 'groups',
+};

buildcheck/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "buildcheck",
+  "version": "1.0.0",
+  "private": true,
+  "description": "Build file generator for handlers",
+  "scripts": {
+    "update-build": "pnpm snapshot:update",
+    "snapshot:update": "pnpm generate && REPO_ROOT=\"$(pwd | sed 's|/buildcheck.*||')\" && ts-node src/cli.ts --generate \"$REPO_ROOT\"",
+    "snapshot:clean": "pnpm generate && REPO_ROOT=\"$(pwd | sed 's|/buildcheck.*||')\" && ts-node src/cli.ts --clean \"$REPO_ROOT\"",
+    "snapshot:assert": "pnpm generate && REPO_ROOT=\"$(pwd | sed 's|/buildcheck.*||')\" && bash -c 'jest --testNamePattern=\"$*\"' -- \"$@\"",
+    "clean": "rm -rf src/dynamic/generated dist",
+    "generate": "bash src/dynamic/generate-template-list.sh && bash src/dynamic/generate-deps-list.sh",
+    "test": "pnpm generate && jest",
+    "type-check": "pnpm generate && tsc --noEmit",
+    "lint": "pnpm generate && eslint src/**/*.ts test/**/*.ts",
+    "check-formatting": "pnpm generate && prettier --check **.ts",
+    "fix-formatting": "pnpm generate && prettier --write **.ts"
+  },
+  "dependencies": {
+    "js-yaml": "^4.1.0"
+  },
+  "devDependencies": {
+    "@jest/globals": "^30.0.4",
+    "@types/js-yaml": "^4.0.9",
+    "@types/node": "^22.15.14",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.0.0"
+  }
+}

buildcheck/src/cli.ts

@@ -0,0 +1,31 @@
+import { generate } from './steps/generate';
+import { writeFiles, deleteRepoFiles, readLines } from './util/file-writer';
+import { parseArguments } from './util/argsParser';
+import {
+	parseGeneratedFilenames,
+	warningFileName,
+} from '../data/snippets/BUILDCHECK.md';
+
+// main entry point from pnpm
+try {
+	const { mode, repoRoot } = parseArguments(process.argv);
+
+	const previouslyGeneratedFiles = parseGeneratedFilenames(
+		readLines(repoRoot, warningFileName),
+	);
+	console.log('previouslyGeneratedFiles to delete', previouslyGeneratedFiles);
+
+	switch (mode) {
+		case 'generate':
+			deleteRepoFiles(repoRoot, previouslyGeneratedFiles);
+			const files = generate();
+			writeFiles(repoRoot, files);
+			break;
+		case 'clean':
+			deleteRepoFiles(repoRoot, previouslyGeneratedFiles);
+			break;
+	}
+} catch (error) {
+	console.error(error);
+	process.exit(1);
+}

buildcheck/src/dynamic/dependencies.ts

@@ -0,0 +1,7 @@
+import { getPnpmCatalog } from '../util/dependencyMapper';
+import { rawCatalogList } from './generated/generatedDepsList';
+
+/**
+ * catalog is populated from the pnpm workspace catalog when you run `pnpm generate`
+ */
+export const pnpmCatalog = getPnpmCatalog(rawCatalogList);

buildcheck/src/dynamic/extractCatalog.awk

@@ -0,0 +1,20 @@
+/^[[:space:]]*catalog:/ {
+    found = 1
+    next
+}
+found {
+    # Always keep comment lines, regardless of indentation or blankness
+    if ($0 ~ /^[[:space:]]*#/) {
+        sub(/^[[:space:]]+/, "", $0)
+        print
+        next
+    }
+    # If line is not indented and not blank, exit
+    if ($0 !~ /^[[:space:]]+/ && length($0) > 0) {
+        exit
+    }
+    # Trim leading and trailing whitespace
+    sub(/^[[:space:]]+/, "", $0)
+    sub(/[[:space:]]+$/, "", $0)
+    print
+}