feat: add time retention parsing to clickhouse queries (#4389)

* feat: add time retention parsing to clickhouse queries

* add logger

* add tests

* fmt

* fmt

* fmt

* fmt

* fmt

* make rabbit happy

* make rabbit happy

* add upsert quota back

* try longer sleep

---------

Co-authored-by: Andreas Thomas <[email protected]>

Author: Flo4604

apps/docs/docs.json

@@ -268,6 +268,7 @@
                       "errors/user/bad_request/invalid_analytics_table",
                       "errors/user/bad_request/missing_required_header",
                       "errors/user/bad_request/permissions_query_syntax_error",
+                      "errors/user/bad_request/query_range_exceeds_retention",
                       "errors/user/bad_request/request_body_too_large",
                       "errors/user/bad_request/request_body_unreadable",
                       "errors/user/bad_request/request_timeout"

apps/docs/errors/user/bad_request/query_range_exceeds_retention.mdx

@@ -0,0 +1,45 @@
+---
+title: "query_range_exceeds_retention"
+description: "QueryRangeExceedsRetention indicates the query attempts to access data older than the workspace's retention period."
+---
+
+<Danger>`err:user:bad_request:query_range_exceeds_retention`</Danger>
+
+## What does this error mean?
+
+This error occurs when your analytics query attempts to access data older than your workspace's configured data retention period. By default, Unkey retains analytics data for **30 days**.
+
+**Note**: If you don't provide a time filter in your query, the system automatically adds one to scope your query to the full retention period (e.g., `time >= now() - INTERVAL 30 DAY`). This error only occurs when you explicitly specify a time range that goes beyond the retention period.
+
+## Common causes
+
+1. **Querying historical data beyond retention**: Your query's time filter includes dates older than your retention period
+   ```sql
+   -- If your retention is 30 days, this will fail if querying > 30 days ago
+   SELECT COUNT(*) FROM key_verifications_v1 WHERE time >= 1234567890000
+   ```
+
+## How to fix
+
+1. **Add a time filter within retention**: Ensure your query only accesses data within your retention period
+   ```sql
+   -- Query last 7 days (within 30-day retention)
+   SELECT COUNT(*) FROM key_verifications_v1
+   WHERE time >= now() - INTERVAL 7 DAY
+   ```
+
+2. **Adjust your query range**: If you need to analyze trends, query within your available retention window
+   ```sql
+   -- Query the full 30-day retention period
+   SELECT COUNT(*) FROM key_verifications_v1
+   WHERE time >= now() - INTERVAL 30 DAY
+   ```
+
+## Need longer retention?
+
+If your use case requires data retention beyond 30 days, please [contact our support team](https://unkey.com/support) to discuss upgrading your retention period. We can configure custom retention periods based on your needs.
+
+## Related
+
+- [Analytics documentation](/apis/features/analytics)
+- [Analytics query examples](/analytics)

go/apps/api/routes/v2_analytics_get_verifications/200_test.go

@@ -23,7 +23,7 @@ func Test200_Success(t *testing.T) {
 	h.SetupAnalytics(workspace.ID)
 	rootKey := h.CreateRootKey(workspace.ID, "api.*.read_analytics")
 
-	now := h.Clock.Now().UnixMilli()
+	now := time.Now().UnixMilli()
 
 	// Buffer some key verifications
 	for i := range 5 {
@@ -61,7 +61,7 @@ func Test200_Success(t *testing.T) {
 	}
 
 	// Wait for buffered data to be available
-	time.Sleep(2 * time.Second)
+	time.Sleep(5 * time.Second)
 
 	res := testutil.CallRoute[Request, Response](h, route, headers, req)
 	t.Logf("Status: %d, RawBody: %s", res.Status, res.RawBody)
@@ -85,7 +85,7 @@ func Test200_PermissionFiltersByApiId(t *testing.T) {
 	// Create root key with permission ONLY for api1
 	rootKey := h.CreateRootKey(workspace.ID, "api."+api1.ID+".read_analytics")
 
-	now := h.Clock.Now().UnixMilli()
+	now := time.Now().UnixMilli()
 
 	// Buffer verifications for api1
 	for i := range 3 {
@@ -166,7 +166,7 @@ func Test200_PermissionFiltersByKeySpaceId(t *testing.T) {
 	// Create root key with permission ONLY for api1
 	rootKey := h.CreateRootKey(workspace.ID, "api."+api1.ID+".read_analytics")
 
-	now := h.Clock.Now().UnixMilli()
+	now := time.Now().UnixMilli()
 
 	// Buffer verifications for api1
 	for i := range 3 {
@@ -239,3 +239,240 @@ func Test200_PermissionFiltersByKeySpaceId(t *testing.T) {
 		require.Equal(c, float64(3), count)
 	}, 30*time.Second, time.Second)
 }
+func Test200_QueryWithin30DaysRetention(t *testing.T) {
+	h := testutil.NewHarness(t)
+
+	workspace := h.CreateWorkspace()
+	api := h.CreateApi(seed.CreateApiRequest{
+		WorkspaceID: workspace.ID,
+	})
+	h.SetupAnalytics(workspace.ID)
+	rootKey := h.CreateRootKey(workspace.ID, "api.*.read_analytics")
+
+	now := time.Now().UnixMilli()
+
+	// Buffer verification from 7 days ago (within 30-day retention)
+	h.ClickHouse.BufferKeyVerification(schema.KeyVerification{
+		RequestID:   uid.New(uid.RequestPrefix),
+		Time:        now - (7 * 24 * 60 * 60 * 1000), // 7 days ago
+		WorkspaceID: workspace.ID,
+		KeySpaceID:  api.KeyAuthID.String,
+		KeyID:       uid.New(uid.KeyPrefix),
+		Region:      "us-west-1",
+		Outcome:     "VALID",
+		IdentityID:  "",
+		Tags:        []string{},
+	})
+
+	route := &Handler{
+		Logger:                     h.Logger,
+		DB:                         h.DB,
+		Keys:                       h.Keys,
+		ClickHouse:                 h.ClickHouse,
+		AnalyticsConnectionManager: h.AnalyticsConnectionManager,
+		Caches:                     h.Caches,
+	}
+	h.Register(route)
+
+	headers := http.Header{
+		"Authorization": []string{"Bearer " + rootKey},
+		"Content-Type":  []string{"application/json"},
+	}
+
+	// Query last 7 days (within 30-day retention)
+	req := Request{
+		Query: "SELECT COUNT(*) as count FROM key_verifications_v1 WHERE time >= now() - INTERVAL 7 DAY",
+	}
+
+	time.Sleep(5 * time.Second) // Wait for data
+
+	res := testutil.CallRoute[Request, Response](h, route, headers, req)
+	require.Equal(t, 200, res.Status, "Query within retention should succeed")
+	require.NotNil(t, res.Body)
+}
+
+func Test200_QueryAtExact30DayRetentionLimit(t *testing.T) {
+	h := testutil.NewHarness(t)
+
+	workspace := h.CreateWorkspace()
+	h.SetupAnalytics(workspace.ID)
+	rootKey := h.CreateRootKey(workspace.ID, "api.*.read_analytics")
+
+	route := &Handler{
+		Logger:                     h.Logger,
+		DB:                         h.DB,
+		Keys:                       h.Keys,
+		ClickHouse:                 h.ClickHouse,
+		AnalyticsConnectionManager: h.AnalyticsConnectionManager,
+		Caches:                     h.Caches,
+	}
+	h.Register(route)
+
+	headers := http.Header{
+		"Authorization": []string{"Bearer " + rootKey},
+		"Content-Type":  []string{"application/json"},
+	}
+
+	// Query exactly 30 days (at retention limit)
+	req := Request{
+		Query: "SELECT COUNT(*) as count FROM key_verifications_v1 WHERE time >= now() - INTERVAL 30 DAY",
+	}
+
+	res := testutil.CallRoute[Request, Response](h, route, headers, req)
+	require.Equal(t, 200, res.Status, "Query at retention limit should succeed")
+	require.NotNil(t, res.Body)
+}
+
+func Test200_QueryWithCustomRetention90Days(t *testing.T) {
+	h := testutil.NewHarness(t)
+
+	workspace := h.CreateWorkspace()
+	h.SetupAnalytics(workspace.ID, testutil.WithRetentionDays(90)) // 90-day retention
+	rootKey := h.CreateRootKey(workspace.ID, "api.*.read_analytics")
+
+	route := &Handler{
+		Logger:                     h.Logger,
+		DB:                         h.DB,
+		Keys:                       h.Keys,
+		ClickHouse:                 h.ClickHouse,
+		AnalyticsConnectionManager: h.AnalyticsConnectionManager,
+		Caches:                     h.Caches,
+	}
+	h.Register(route)
+
+	headers := http.Header{
+		"Authorization": []string{"Bearer " + rootKey},
+		"Content-Type":  []string{"application/json"},
+	}
+
+	// Query 60 days (within 90-day retention)
+	req := Request{
+		Query: "SELECT COUNT(*) as count FROM key_verifications_v1 WHERE time >= now() - INTERVAL 60 DAY",
+	}
+
+	res := testutil.CallRoute[Request, Response](h, route, headers, req)
+	require.Equal(t, 200, res.Status, "Query within custom retention should succeed")
+	require.NotNil(t, res.Body)
+}
+
+func Test200_RLSWorkspaceIsolation(t *testing.T) {
+	h := testutil.NewHarness(t)
+
+	// Create two separate workspaces
+	workspace1 := h.CreateWorkspace()
+	workspace2 := h.CreateWorkspace()
+
+	api1 := h.CreateApi(seed.CreateApiRequest{
+		WorkspaceID: workspace1.ID,
+	})
+	api2 := h.CreateApi(seed.CreateApiRequest{
+		WorkspaceID: workspace2.ID,
+	})
+
+	// Setup analytics for both workspaces
+	h.SetupAnalytics(workspace1.ID)
+	h.SetupAnalytics(workspace2.ID)
+
+	rootKey1 := h.CreateRootKey(workspace1.ID, "api.*.read_analytics")
+
+	// Use actual current time for analytics data since ClickHouse's now() uses real time, not mock clock
+	now := time.Now().UnixMilli()
+
+	// Buffer data for workspace 1
+	for i := range 5 {
+		h.ClickHouse.BufferKeyVerification(schema.KeyVerification{
+			RequestID:   uid.New(uid.RequestPrefix),
+			Time:        now - int64(i*1000),
+			WorkspaceID: workspace1.ID,
+			KeySpaceID:  api1.KeyAuthID.String,
+			KeyID:       uid.New(uid.KeyPrefix),
+			Region:      "us-west-1",
+			Outcome:     "VALID",
+			IdentityID:  "",
+			Tags:        []string{},
+		})
+	}
+
+	// Buffer data for workspace 2 (should NOT be accessible by workspace1's key)
+	for i := range 10 {
+		h.ClickHouse.BufferKeyVerification(schema.KeyVerification{
+			RequestID:   uid.New(uid.RequestPrefix),
+			Time:        now - int64(i*1000),
+			WorkspaceID: workspace2.ID,
+			KeySpaceID:  api2.KeyAuthID.String,
+			KeyID:       uid.New(uid.KeyPrefix),
+			Region:      "us-east-1",
+			Outcome:     "VALID",
+			IdentityID:  "",
+			Tags:        []string{},
+		})
+	}
+
+	route := &Handler{
+		Logger:                     h.Logger,
+		DB:                         h.DB,
+		Keys:                       h.Keys,
+		ClickHouse:                 h.ClickHouse,
+		AnalyticsConnectionManager: h.AnalyticsConnectionManager,
+		Caches:                     h.Caches,
+	}
+	h.Register(route)
+
+	headers := http.Header{
+		"Authorization": []string{"Bearer " + rootKey1},
+		"Content-Type":  []string{"application/json"},
+	}
+
+	// Query all verifications - should only return workspace1's data due to RLS
+	req := Request{
+		Query: "SELECT COUNT(*) as count FROM key_verifications_v1 WHERE time >= now() - INTERVAL 1 DAY",
+	}
+
+	time.Sleep(10 * time.Second) // Wait for data to be flushed to ClickHouse
+
+	res := testutil.CallRoute[Request, Response](h, route, headers, req)
+	require.Equal(t, 200, res.Status)
+	require.NotNil(t, res.Body)
+	require.Len(t, res.Body.Data, 1)
+
+	// Verify only workspace1's data is returned (5 verifications), not workspace2's (10)
+	count, ok := res.Body.Data[0]["count"]
+	require.True(t, ok)
+	require.Equal(t, float64(5), count, "RLS should filter to only workspace1's data")
+}
+
+func Test200_QueryWithoutTimeFilter_AutoAddsFilter(t *testing.T) {
+	h := testutil.NewHarness(t)
+
+	workspace := h.CreateWorkspace()
+	h.SetupAnalytics(workspace.ID)
+	rootKey := h.CreateRootKey(workspace.ID, "api.*.read_analytics")
+
+	route := &Handler{
+		Logger:                     h.Logger,
+		DB:                         h.DB,
+		Keys:                       h.Keys,
+		ClickHouse:                 h.ClickHouse,
+		AnalyticsConnectionManager: h.AnalyticsConnectionManager,
+		Caches:                     h.Caches,
+	}
+	h.Register(route)
+
+	headers := http.Header{
+		"Authorization": []string{"Bearer " + rootKey},
+		"Content-Type":  []string{"application/json"},
+	}
+
+	// Query without time filter - should auto-add time >= now() - INTERVAL 30 DAY
+	req := Request{
+		Query: "SELECT COUNT(*) as count FROM key_verifications_v1",
+	}
+
+	res := testutil.CallRoute[Request, Response](h, route, headers, req)
+	if res.Status != 200 {
+		t.Logf("Response status: %d", res.Status)
+		t.Logf("Response body: %s", res.RawBody)
+	}
+	require.Equal(t, 200, res.Status, "Query without time filter should succeed with auto-added filter")
+	require.NotNil(t, res.Body)
+}