Staging cluster deployment

.dockerignore

@@ -1,3 +1,4 @@
-target/
-node_modules/
-crates/
+target
+crates
+**/node_modules
+**/.next

.github/workflows/ci.yml

@@ -115,8 +115,6 @@ jobs:
           echo 'VITE_POSTHOG_KEY=${{ secrets.NEXT_PUBLIC_POSTHOG_KEY }}' >> .env
           echo 'VITE_POSTHOG_HOST=${{ secrets.NEXT_PUBLIC_POSTHOG_HOST }}' >> .env
           echo 'VITE_SERVER_URL=${{ secrets.NEXT_PUBLIC_WEB_URL }}' >> .env
-          echo "NEXT_PUBLIC_CAP_AWS_REGION=${{ secrets.NEXT_PUBLIC_CAP_AWS_REGION }}" >> .env
-          echo "NEXT_PUBLIC_CAP_AWS_BUCKET=${{ secrets.NEXT_PUBLIC_CAP_AWS_BUCKET }}" >> .env
 
           cat .env >> $GITHUB_ENV
 
@@ -185,8 +183,6 @@ jobs:
           echo 'VITE_POSTHOG_KEY=${{ secrets.NEXT_PUBLIC_POSTHOG_KEY }}' >> .env
           echo 'VITE_POSTHOG_HOST=${{ secrets.NEXT_PUBLIC_POSTHOG_HOST }}' >> .env
           echo 'VITE_SERVER_URL=${{ secrets.NEXT_PUBLIC_WEB_URL }}' >> .env
-          echo "NEXT_PUBLIC_CAP_AWS_REGION=${{ secrets.NEXT_PUBLIC_CAP_AWS_REGION }}" >> .env
-          echo "NEXT_PUBLIC_CAP_AWS_BUCKET=${{ secrets.NEXT_PUBLIC_CAP_AWS_BUCKET }}" >> .env
 
       - name: Copy .env to apps/desktop
         run: cp .env apps/desktop/.env

.github/workflows/docker-build-web.yml

@@ -36,8 +36,6 @@ jobs:
         run: |
           echo "WEB_URL=http://localhost:3000" > .env
           echo "NEXT_PUBLIC_DOCKER_BUILD=true" >> .env
-          echo "NEXT_PUBLIC_CAP_AWS_BUCKET=capso" >> .env
-          echo "NEXT_PUBLIC_CAP_AWS_REGION=us-east-1" >> .env
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3

.github/workflows/publish.yml

@@ -185,8 +185,6 @@ jobs:
           echo 'VITE_POSTHOG_KEY=${{ secrets.VITE_POSTHOG_KEY }}' >> .env
           echo 'VITE_POSTHOG_HOST=${{ secrets.VITE_POSTHOG_HOST }}' >> .env
           echo 'VITE_SERVER_URL=${{ secrets.NEXT_PUBLIC_WEB_URL }}' >> .env
-          echo "NEXT_PUBLIC_CAP_AWS_REGION=${{ secrets.NEXT_PUBLIC_CAP_AWS_REGION }}" >> .env
-          echo "NEXT_PUBLIC_CAP_AWS_BUCKET=${{ secrets.NEXT_PUBLIC_CAP_AWS_BUCKET }}" >> .env
           echo 'RUST_TARGET_TRIPLE=${{ matrix.settings.target }}' >> .env
 
       - name: Build app

.npmrc

@@ -1 +1,2 @@
 auto-install-peers = true
+force-legacy-deploy = true

CLAUDE.md

@@ -171,7 +171,7 @@ import { getCurrentUser } from "@cap/database/auth/session";
 export async function updateVideo(data: FormData) {
   const user = await getCurrentUser();
   if (!user?.id) throw new Error("Unauthorized");
-  
+
   // Database operations with Drizzle
   return await db().update(videos).set({ ... }).where(eq(videos.id, id));
 }
@@ -220,7 +220,6 @@ const updateMutation = useMutation({
 
 ### Build/Client (selected)
 - `NEXT_PUBLIC_WEB_URL`
-- `NEXT_PUBLIC_CAP_AWS_BUCKET`, `NEXT_PUBLIC_CAP_AWS_REGION`
 - `NEXT_PUBLIC_POSTHOG_KEY`, `NEXT_PUBLIC_POSTHOG_HOST`
 - `NEXT_PUBLIC_DOCKER_BUILD` (enables Next.js standalone output)
 
@@ -268,7 +267,7 @@ const updateMutation = useMutation({
 - **Connection errors**: Verify Docker containers are running
 - **Schema drift**: Run `pnpm --dir packages/database db:check`
 
-### Desktop App Issues  
+### Desktop App Issues
 - **IPC binding errors**: Restart dev server to regenerate `tauri.ts`
 - **Rust compile errors**: Check Cargo.toml dependencies
 - **Permission issues**: macOS/Windows may require app permissions
@@ -392,8 +391,8 @@ Minimize `useEffect` usage: compute during render, handle logic in event handler
 
 ### Media Processing Flow
 ```
-Desktop Recording → Local Files → Upload to S3 → 
-Background Processing (tasks service) → 
+Desktop Recording → Local Files → Upload to S3 →
+Background Processing (tasks service) →
 Transcription/AI Enhancement → Database Storage
 ```
 

apps/web-cluster/src/runner/index.ts

@@ -4,6 +4,7 @@ import { ClusterWorkflowEngine, RunnerAddress } from "@effect/cluster";
 import * as NodeSdk from "@effect/opentelemetry/NodeSdk";
 import {
 	FetchHttpClient,
+	Headers,
 	HttpApiBuilder,
 	HttpMiddleware,
 	HttpRouter,
@@ -24,6 +25,12 @@ import { ContainerMetadata } from "../cluster/container-metadata.ts";
 import { DatabaseLive, ShardDatabaseLive } from "../shared/database.ts";
 import { HealthServerLive } from "./health-server.ts";
 
+class RpcAuthSecret extends Effect.Service<RpcAuthSecret>()("RpcAuthSecret", {
+	effect: Effect.map(Config.string("WORKFLOWS_RPC_SECRET"), (v) => ({
+		authSecret: v,
+	})),
+}) {}
+
 const ClusterWorkflowLive = Layer.unwrapEffect(
 	Effect.gen(function* () {
 		const containerMeta = yield* ContainerMetadata;
@@ -48,6 +55,28 @@ const RpcsLive = RpcServer.layer(Workflows.RpcGroup).pipe(
 	Layer.provide(Workflows.WorkflowsLayer),
 	Layer.provide(ClusterWorkflowLive),
 	Layer.provide(RpcServer.layerProtocolHttp({ path: "/" })),
+	Layer.provide(
+		Layer.effect(
+			Workflows.SecretAuthMiddleware,
+			Effect.gen(function* () {
+				const { authSecret } = yield* RpcAuthSecret;
+
+				return Workflows.SecretAuthMiddleware.of(
+					Effect.fn(function* (options) {
+						const authHeader = Headers.get(options.headers, "authorization");
+						if (Option.isNone(authHeader) || authHeader.value !== authSecret) {
+							if (Option.isNone(authHeader))
+								yield* Effect.log("No auth header provided");
+
+							return yield* new Workflows.InvalidRpcAuth();
+						}
+
+						return yield* options.next;
+					}),
+				);
+			}),
+		),
+	),
 	Layer.provide(Workflows.RpcSerialization),
 );
 
@@ -92,6 +121,7 @@ HttpRouter.Default.serve().pipe(
 	Layer.provide(FetchHttpClient.layer),
 	Layer.provide(DatabaseLive),
 	Layer.provide(TracingLayer),
+	Layer.provide(RpcAuthSecret.Default),
 	Layer.launch,
 	NodeRuntime.runMain,
 );

apps/web/Dockerfile

@@ -12,8 +12,6 @@ RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm i --fro
 
 ARG NEXT_PUBLIC_DOCKER_BUILD=true
 ENV NEXT_PUBLIC_WEB_URL=http://localhost:3000
-ENV NEXT_PUBLIC_CAP_AWS_BUCKET=capso
-ENV NEXT_PUBLIC_CAP_AWS_REGION=us-east-1
 
 RUN pnpm run build:web
 

apps/web/lib/server.ts

@@ -16,13 +16,14 @@ import {
 import { type HttpAuthMiddleware, Video } from "@cap/web-domain";
 import {
 	FetchHttpClient,
+	Headers,
 	type HttpApi,
 	HttpApiBuilder,
 	HttpApiClient,
 	HttpMiddleware,
 	HttpServer,
 } from "@effect/platform";
-import { RpcClient } from "@effect/rpc";
+import { RpcClient, RpcMessage, RpcMiddleware } from "@effect/rpc";
 import {
 	Cause,
 	Config,
@@ -31,6 +32,7 @@ import {
 	Layer,
 	ManagedRuntime,
 	Option,
+	Redacted,
 } from "effect";
 import { cookies } from "next/headers";
 
@@ -50,11 +52,21 @@ const CookiePasswordAttachmentLive = Layer.effect(
 	}),
 );
 
+class WorkflowRpcSecret extends Effect.Service<WorkflowRpcSecret>()(
+	"WorkflowRpcSecret",
+	{
+		effect: Effect.map(
+			Config.redacted(Config.string("WORKFLOWS_RPC_SECRET")),
+			(v) => ({ authSecret: v }),
+		),
+	},
+) {}
+
 const WorkflowRpcLive = Layer.scoped(
 	Workflows.RpcClient,
 	Effect.gen(function* () {
 		const url = Option.getOrElse(
-			yield* Config.option(Config.string("REMOTE_WORKFLOW_URL")),
+			yield* Config.option(Config.string("WORKFLOWS_RPC_URL")),
 			() => "http://127.0.0.1:42169",
 		);
 
@@ -66,6 +78,22 @@ const WorkflowRpcLive = Layer.scoped(
 			),
 		);
 	}),
+).pipe(
+	Layer.provide(
+		RpcMiddleware.layerClient(Workflows.SecretAuthMiddleware, ({ request }) =>
+			Effect.gen(function* () {
+				const { authSecret } = yield* WorkflowRpcSecret;
+				return {
+					...request,
+					headers: Headers.set(
+						request.headers,
+						"authorization",
+						Redacted.value(authSecret),
+					),
+				};
+			}),
+		),
+	),
 );
 
 export const Dependencies = Layer.mergeAll(
@@ -79,7 +107,13 @@ export const Dependencies = Layer.mergeAll(
 	WorkflowRpcLive,
 	layerTracer,
 ).pipe(
-	Layer.provideMerge(Layer.mergeAll(Database.Default, FetchHttpClient.layer)),
+	Layer.provideMerge(
+		Layer.mergeAll(
+			Database.Default,
+			FetchHttpClient.layer,
+			WorkflowRpcSecret.Default,
+		),
+	),
 );
 
 // purposefully not exposed
@@ -132,6 +166,7 @@ export const apiToHandler = (
 		Layer.provide(
 			HttpApiBuilder.middleware(Effect.provide(CookiePasswordAttachmentLive)),
 		),
+		Layer.provide(layerTracer),
 		Layer.provideMerge(Dependencies),
 		HttpApiBuilder.toWebHandler,
 		(v) => (req: Request) => v.handler(req),

crates/recording/src/output_pipeline/core.rs

@@ -128,7 +128,7 @@ impl TaskPool {
                     let res = future.await;
                     match &res {
                         Ok(_) => info!("Task finished successfully"),
-                        Err(err) => error!("Task failed: {}", err),
+                        Err(err) => error!("Task failed: {:#}", err),
                     }
                     res
                 }

infra/sst-env.d.ts

@@ -5,52 +5,10 @@
 
 declare module "sst" {
 	export interface Resource {
-		AuroraDB: {
-			clusterArn: string;
-			database: string;
-			host: string;
-			password: string;
-			port: number;
-			reader: string;
-			secretArn: string;
-			type: "sst.aws.Aurora";
-			username: string;
-		};
-		CAP_AWS_ACCESS_KEY: {
+		DATABASE_URL: {
 			type: "sst.sst.Secret";
 			value: string;
 		};
-		CAP_AWS_SECRET_KEY: {
-			type: "sst.sst.Secret";
-			value: string;
-		};
-		DATABASE_URL_HTTP: {
-			type: "sst.sst.Secret";
-			value: string;
-		};
-		DATABASE_URL_MYSQL: {
-			type: "sst.sst.Secret";
-			value: string;
-		};
-		GITHUB_PAT: {
-			type: "sst.sst.Secret";
-			value: string;
-		};
-		MyApi: {
-			type: "sst.aws.ApiGatewayV2";
-			url: string;
-		};
-		Runner: {
-			service: string;
-			type: "sst.aws.Service";
-		};
-		ShardManager: {
-			service: string;
-			type: "sst.aws.Service";
-		};
-		Vpc: {
-			type: "sst.aws.Vpc";
-		};
 	}
 }
 /// <reference path="sst-env.d.ts" />

infra/sst.config.ts

@@ -1,5 +1,7 @@
 /// <reference path="./.sst/platform/config.d.ts" />
 
+import type { Input, Output } from "@pulumi/pulumi";
+
 const GITHUB_ORG = "CapSoftware";
 const GITHUB_REPO = "Cap";
 const GITHUB_APP_ID = "1196731";
@@ -30,6 +32,7 @@ export default $config({
 				aws: {},
 				planetscale: true,
 				awsx: "2.21.1",
+				random: true,
 			},
 		};
 	},
@@ -151,7 +154,11 @@ export default $config({
 		if ($app.stage === "staging" || $app.stage === "production") {
 			[
 				...vercelVariables,
-				{ key: "REMOTE_WORKFLOW_URL", value: workflowCluster.api.url },
+				{ key: "WORKFLOWS_RPC_URL", value: workflowCluster.api.url },
+				{
+					key: "WORKFLOWS_RPC_SECRET",
+					value: secrets.WORKFLOWS_RPC_SECRET.result,
+				},
 				{ key: "VERCEL_AWS_ROLE_ARN", value: vercelAwsAccessRole.arn },
 			].map(
 				(v) =>
@@ -178,6 +185,9 @@ function Secrets() {
 		CAP_AWS_ACCESS_KEY: new sst.Secret("CAP_AWS_ACCESS_KEY"),
 		CAP_AWS_SECRET_KEY: new sst.Secret("CAP_AWS_SECRET_KEY"),
 		GITHUB_PAT: new sst.Secret("GITHUB_PAT"),
+		WORKFLOWS_RPC_SECRET: new random.RandomString("WORKFLOWS_RPC_SECRET", {
+			length: 48,
+		}),
 	};
 }
 
@@ -289,11 +299,10 @@ async function WorkflowCluster(bucket: aws.s3.BucketV2, secrets: Secrets) {
 		CAP_AWS_BUCKET: bucket.bucket,
 		SHARD_DATABASE_URL: $interpolate`mysql://${db.username}:${db.password}@${db.host}:${db.port}/${db.database}`,
 		DATABASE_URL: secrets.DATABASE_URL_MYSQL.value,
-		CAP_AWS_ACCESS_KEY: secrets.CAP_AWS_ACCESS_KEY.value,
-		CAP_AWS_SECRET_KEY: secrets.CAP_AWS_SECRET_KEY.value,
 		AXIOM_API_TOKEN,
 		AXIOM_DOMAIN: "api.axiom.co",
 		AXIOM_DATASET,
+		WORKFLOWS_RPC_SECRET: secrets.WORKFLOWS_RPC_SECRET.result,
 	};
 
 	const ghcrCredentialsSecret = new aws.secretsmanager.Secret(
@@ -408,6 +417,12 @@ async function WorkflowCluster(bucket: aws.s3.BucketV2, secrets: Secrets) {
 				ghcrCredentialsTransform.taskDefinition(args);
 			},
 		},
+		permissions: [
+			{
+				actions: ["s3:*"],
+				resources: [bucket.arn, $interpolate`${bucket.arn}/*`],
+			},
+		],
 	});
 
 	const api = new sst.aws.ApiGatewayV2("MyApi", {

packages/env/build.ts

@@ -14,10 +14,6 @@ const create = () =>
 			NEXT_PUBLIC_META_PIXEL_ID: z.string().optional(),
 			NEXT_PUBLIC_GOOGLE_AW_ID: z.string().optional(),
 			NEXT_PUBLIC_WEB_URL: z.string(),
-			NEXT_PUBLIC_CAP_AWS_BUCKET: z.string(),
-			NEXT_PUBLIC_CAP_AWS_REGION: z.string(),
-			NEXT_PUBLIC_CAP_AWS_ENDPOINT: z.string().optional(),
-			NEXT_PUBLIC_CAP_AWS_BUCKET_URL: z.string().optional(),
 			NEXT_PUBLIC_DOCKER_BUILD: z.string().optional(),
 		},
 		runtimeEnv: {
@@ -28,15 +24,6 @@ const create = () =>
 			NEXT_PUBLIC_GOOGLE_AW_ID: process.env.NEXT_PUBLIC_GOOGLE_AW_ID,
 			NEXT_PUBLIC_WEB_URL:
 				process.env.WEB_URL ?? process.env.NEXT_PUBLIC_WEB_URL,
-			NEXT_PUBLIC_CAP_AWS_BUCKET:
-				process.env.CAP_AWS_BUCKET ?? process.env.NEXT_PUBLIC_CAP_AWS_BUCKET,
-			NEXT_PUBLIC_CAP_AWS_REGION:
-				process.env.CAP_AWS_REGION ?? process.env.NEXT_PUBLIC_CAP_AWS_REGION,
-			NEXT_PUBLIC_CAP_AWS_ENDPOINT:
-				process.env.CAP_AWS_ENDPOINT ??
-				process.env.NEXT_PUBLIC_CAP_AWS_ENDPOINT,
-			NEXT_PUBLIC_CAP_AWS_BUCKET_URL:
-				process.env.CAP_AWS_URL ?? process.env.NEXT_PUBLIC_CAP_AWS_BUCKET_URL,
 			NEXT_PUBLIC_DOCKER_BUILD: process.env.NEXT_PUBLIC_DOCKER_BUILD,
 		},
 	});