feat(handler): update chunk resource naming to AIP-compliant nested format (#327)

**Because**

- Chunk resource names did not follow AIP nested resource naming
conventions
- gRPC integration tests were flaky due to transient errors without
retry logic
- File listing required a knowledge base, limiting flexibility for
namespace-wide queries

**This commit**

- Update chunk resource name format from
`namespaces/{ns}/files/{file}/chunks/{chunk}` to
`namespaces/{ns}/knowledge-bases/{kb}/files/{file}/chunks/{chunk}` for
AIP compliance
- Update file resource name parsing to support both flat
(`namespaces/{ns}/files/{file}`) and nested
(`namespaces/{ns}/knowledge-bases/{kb}/files/{file}`) formats
- Add validation for required knowledge base parameter in `SearchChunks`
- Allow `ListFiles` to work at namespace level without requiring a
knowledge base
- Add `grpcInvokeWithRetry` helper for automatic gRPC retry with
exponential backoff on transient errors (Unknown, DeadlineExceeded,
ResourceExhausted, Aborted, Unavailable)
- Update all 145 gRPC test invocations to use the retry helper for
improved test stability
- Update resource name validation in tests to support nested resource
formats
- Rename `collectionIds` to `knowledgeBases` in file validation

Author: pinglin

Dockerfile.dev

@@ -1,5 +1,5 @@
 ARG GOLANG_VERSION=1.25.6
-FROM golang:${GOLANG_VERSION}-bullseye
+FROM golang:${GOLANG_VERSION}-bookworm
 
 ARG SERVICE_NAME SERVICE_VERSION
 
@@ -36,7 +36,7 @@ RUN curl -fsSL https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /us
 # Also install GNU parallel for running tests in parallel
 RUN apt-get update && apt-get install -y python3 python3-pip parallel && rm -rf /var/lib/apt/lists/*
 # Install pymilvus
-RUN pip3 install pymilvus>=2.4.3
+RUN pip3 install --break-system-packages "pymilvus>=2.6"
 
 # -- set up Go
 COPY go.mod go.sum ./

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1
 	github.com/iancoleman/strcase v0.3.0
-	github.com/instill-ai/protogen-go v0.3.3-alpha.0.20260129193003-3968ab7c42a9
+	github.com/instill-ai/protogen-go v0.3.3-alpha.0.20260130101410-2df13d165253
 	github.com/instill-ai/x v0.10.1-alpha.0.20260129195415-09e1680f104d
 	github.com/knadh/koanf v1.5.0
 	github.com/mennanov/fieldmask-utils v1.1.2

go.sum

@@ -419,8 +419,8 @@ github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSAS
 github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/instill-ai/protogen-go v0.3.3-alpha.0.20260129193003-3968ab7c42a9 h1:zOKahHOtvNkvUOLsM7HC2xThbjIx3THwP4+4hWwy/GA=
-github.com/instill-ai/protogen-go v0.3.3-alpha.0.20260129193003-3968ab7c42a9/go.mod h1:bCnBosofpaUxKBuTTJM3/I3thAK37kvfBnKByjnLsl4=
+github.com/instill-ai/protogen-go v0.3.3-alpha.0.20260130101410-2df13d165253 h1:PlMigeqj4xkVoONNMja/ppDlzWs+e0pai2mmxBzpIjg=
+github.com/instill-ai/protogen-go v0.3.3-alpha.0.20260130101410-2df13d165253/go.mod h1:bCnBosofpaUxKBuTTJM3/I3thAK37kvfBnKByjnLsl4=
 github.com/instill-ai/x v0.10.1-alpha.0.20260129195415-09e1680f104d h1:FLDqMpALmq2e/j9inTGWrY4mhRFhe5/xXvjjxwQDqDg=
 github.com/instill-ai/x v0.10.1-alpha.0.20260129195415-09e1680f104d/go.mod h1:/xG0LMh43kyyK9jXGtV/JGd5p9GVcreSZ6GRGVbALEU=
 github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI=

integration-test/const.js

@@ -72,6 +72,22 @@ export const paramsHTTPWithJWT = {
   },
 };
 
+// Invalid Basic Auth credentials for testing authentication rejection
+const invalidAuthHeader = `Basic ${encoding.b64encode("invalid:wrongpassword")}`;
+export const paramsGRPCWithInvalidAuth = {
+  metadata: {
+    "Content-Type": "application/json",
+    "Authorization": invalidAuthHeader,
+  },
+};
+
+// No auth metadata for testing unauthenticated requests
+export const paramsGRPCNoAuth = {
+  metadata: {
+    "Content-Type": "application/json",
+  },
+};
+
 // Test document sample data files
 export const docSampleDoc = encoding.b64encode(
   open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/doc-sample.doc`, "b")

integration-test/grpc-kb-update.js

@@ -32,6 +32,7 @@ import { check, group } from "k6";
 
 import * as constant from "./const.js";
 import * as helper from "./helper.js";
+import { grpcInvokeWithRetry } from "./helper.js";
 
 // Use httpRetry for automatic retry on transient errors (429, 5xx)
 const http = helper.httpRetry;
@@ -59,25 +60,10 @@ export function setup() {
     console.log(`grpc-system-admin.js: Using unique test prefix: ${dbIDPrefix}`);
 
     // Authenticate with retry to handle transient failures
-    const loginResp = helper.authenticateWithRetry(
-        constant.mgmtRESTPublicHost,
-        constant.defaultUsername,
-        constant.defaultPassword
-    );
-
-    check(loginResp, {
-        "Setup: Authentication successful": (r) => r && r.status === 200,
-    });
-
-    if (!loginResp || loginResp.status !== 200) {
-        console.error("Setup: Authentication failed, cannot continue");
-        return null;
-    }
-
-    const accessToken = loginResp.json().accessToken;
+    const authHeader = helper.getBasicAuthHeader(constant.defaultUsername, constant.defaultPassword);
     const grpcMetadata = {
         "metadata": {
-            "Authorization": `Bearer ${accessToken}`,
+            "Authorization": authHeader,
         },
         "timeout": "300s",
     };
@@ -100,7 +86,7 @@ export function teardown(data) {
 
         for (const systemId of testSystemIds) {
             try {
-                client.invoke(
+                grpcInvokeWithRetry(client,
                     "artifact.v1alpha.ArtifactPrivateService/DeleteSystemAdmin",
                     { system_id: systemId },
                     data.metadata
@@ -133,7 +119,7 @@ export default function (data) {
         group("Phase 1: List all system configurations", () => {
             console.log("\n=== Phase 1: Listing all systems ===");
 
-            const listRes = client.invoke(
+            const listRes = grpcInvokeWithRetry(client,
                 "artifact.v1alpha.ArtifactPrivateService/ListSystemsAdmin",
                 {},
                 data.metadata
@@ -175,7 +161,7 @@ export default function (data) {
                 console.log("Phase 2: OpenAI system ID not found in Phase 1, skipping OpenAI tests");
             } else {
                 // Get OpenAI system using its canonical ID (sys-{hash} format)
-                const openaiRes = client.invoke(
+                const openaiRes = grpcInvokeWithRetry(client,
                     "artifact.v1alpha.ArtifactPrivateService/GetSystemAdmin",
                     { system_id: openaiSystemId },
                     data.metadata
@@ -197,7 +183,7 @@ export default function (data) {
                 console.log("Phase 2: Gemini system ID not found in Phase 1, skipping Gemini tests");
             } else {
                 // Get Gemini system using its canonical ID
-                const geminiRes = client.invoke(
+                const geminiRes = grpcInvokeWithRetry(client,
                     "artifact.v1alpha.ArtifactPrivateService/GetSystemAdmin",
                     { system_id: geminiSystemId },
                     data.metadata
@@ -221,7 +207,7 @@ export default function (data) {
         group("Phase 3: Get default system configuration", () => {
             console.log("\n=== Phase 3: Getting default system ===");
 
-            const defaultRes = client.invoke(
+            const defaultRes = grpcInvokeWithRetry(client,
                 "artifact.v1alpha.ArtifactPrivateService/GetDefaultSystemAdmin",
                 {},
                 data.metadata
@@ -249,7 +235,7 @@ export default function (data) {
             // API CHANGE: ID is now auto-generated as "sys-{hash}", use display_name instead
             const customDisplayName = `${data.dbIDPrefix}sysadmin-custom`;
 
-            const createRes = client.invoke(
+            const createRes = grpcInvokeWithRetry(client,
                 "artifact.v1alpha.ArtifactPrivateService/CreateSystemAdmin",
                 {
                     system: {
@@ -310,7 +296,7 @@ export default function (data) {
             };
             console.log(`Phase 5: Request object: ${JSON.stringify(updateRequest)}`);
 
-            const updateRes = client.invoke(
+            const updateRes = grpcInvokeWithRetry(client,
                 "artifact.v1alpha.ArtifactPrivateService/UpdateSystemAdmin",
                 updateRequest,
                 data.metadata
@@ -333,7 +319,7 @@ export default function (data) {
             // Update both config and description
             console.log("\n=== Phase 5: Updating config with field mask ===");
 
-            const updateConfigRes = client.invoke(
+            const updateConfigRes = grpcInvokeWithRetry(client,
                 "artifact.v1alpha.ArtifactPrivateService/UpdateSystemAdmin",
                 {
                     system: {
@@ -384,7 +370,7 @@ export default function (data) {
             // API CHANGE: IDs are immutable, we can only change display_name (and slug)
             const newDisplayName = `${data.dbIDPrefix}sysadmin-renamed`;
 
-            const renameRes = client.invoke(
+            const renameRes = grpcInvokeWithRetry(client,
                 "artifact.v1alpha.ArtifactPrivateService/RenameSystemAdmin",
                 {
                     system_id: customSystemId,
@@ -410,7 +396,7 @@ export default function (data) {
             }
 
             // Verify system is still accessible by the same ID (ID is immutable)
-            const getRes = client.invoke(
+            const getRes = grpcInvokeWithRetry(client,
                 "artifact.v1alpha.ArtifactPrivateService/GetSystemAdmin",
                 { system_id: customSystemId },
                 data.metadata
@@ -435,7 +421,7 @@ export default function (data) {
             }
 
             // First, get current default
-            const getCurrentDefaultRes = client.invoke(
+            const getCurrentDefaultRes = grpcInvokeWithRetry(client,
                 "artifact.v1alpha.ArtifactPrivateService/GetDefaultSystemAdmin",
                 {},
                 data.metadata
@@ -448,7 +434,7 @@ export default function (data) {
             }
 
             // Set custom system as default
-            const setDefaultRes = client.invoke(
+            const setDefaultRes = grpcInvokeWithRetry(client,
                 "artifact.v1alpha.ArtifactPrivateService/SetDefaultSystemAdmin",
                 { system_id: renamedSystemId },
                 data.metadata
@@ -461,7 +447,7 @@ export default function (data) {
             });
 
             // Verify via GetDefaultSystemAdmin
-            const verifyDefaultRes = client.invoke(
+            const verifyDefaultRes = grpcInvokeWithRetry(client,
                 "artifact.v1alpha.ArtifactPrivateService/GetDefaultSystemAdmin",
                 {},
                 data.metadata
@@ -476,7 +462,7 @@ export default function (data) {
             // Restore original default
             if (previousDefaultId && previousDefaultId !== renamedSystemId) {
                 console.log(`Phase 7: Restoring original default: ${previousDefaultId}`);
-                const restoreRes = client.invoke(
+                const restoreRes = grpcInvokeWithRetry(client,
                     "artifact.v1alpha.ArtifactPrivateService/SetDefaultSystemAdmin",
                     { system_id: previousDefaultId },
                     data.metadata
@@ -500,7 +486,7 @@ export default function (data) {
                 return;
             }
 
-            const deleteRes = client.invoke(
+            const deleteRes = grpcInvokeWithRetry(client,
                 "artifact.v1alpha.ArtifactPrivateService/DeleteSystemAdmin",
                 { system_id: renamedSystemId },
                 data.metadata
@@ -512,7 +498,7 @@ export default function (data) {
             });
 
             // Verify system is deleted
-            const getDeletedRes = client.invoke(
+            const getDeletedRes = grpcInvokeWithRetry(client,
                 "artifact.v1alpha.ArtifactPrivateService/GetSystemAdmin",
                 { system_id: renamedSystemId },
                 data.metadata
@@ -542,7 +528,7 @@ export default function (data) {
             console.log(`Phase 9: Using OpenAI system ID: ${openaiSystemId}`);
 
             // Try to delete openai system (should fail - protected)
-            const deleteOpenaiRes = client.invoke(
+            const deleteOpenaiRes = grpcInvokeWithRetry(client,
                 "artifact.v1alpha.ArtifactPrivateService/DeleteSystemAdmin",
                 { system_id: openaiSystemId },
                 data.metadata
@@ -553,7 +539,7 @@ export default function (data) {
             });
 
             // Try to rename openai system (should fail - protected)
-            const renameOpenaiRes = client.invoke(
+            const renameOpenaiRes = grpcInvokeWithRetry(client,
                 "artifact.v1alpha.ArtifactPrivateService/RenameSystemAdmin",
                 {
                     system_id: openaiSystemId,
@@ -567,7 +553,7 @@ export default function (data) {
             });
 
             // Verify openai system still exists with same ID
-            const getOpenaiRes = client.invoke(
+            const getOpenaiRes = grpcInvokeWithRetry(client,
                 "artifact.v1alpha.ArtifactPrivateService/GetSystemAdmin",
                 { system_id: openaiSystemId },
                 data.metadata