Skip to content

Commit a9cd2c4

Browse files
committed
refactor(api): update tests for AIP-compliant APIs and add private service tests (#1151)
**Because** - Integration tests were using deprecated `*UserPipeline` APIs that have been removed in favor of `*NamespacePipeline` APIs - The `LookUpPipeline` endpoint was incorrectly exposed in public service despite being marked as INTERNAL - Private service APIs (LookUpPipelineAdmin, ListPipelinesAdmin) lacked integration test coverage - Several tests were skipped due to outdated API method references - The `PipelineRun.name` field was returning empty namespace/pipeline values **This commit** - Updates all gRPC and REST integration tests to use new Namespace APIs (`CreateNamespacePipeline`, `TriggerNamespacePipeline`, etc.) - Removes public `LookUpPipeline` handler since it was removed from the public protobuf service - Adds private service integration tests (`grpc-pipeline-private.js`) with proper service-to-service authentication using `Instill-User-Uid` and `Instill-Requester-Uid` headers - Fixes `PipelineRun` to properly populate the `Pipeline` field before conversion, ensuring correct resource name generation - Updates `renamePipelineRequiredFields` to include `new_pipeline_id` per AIP-136 - Adds database query helpers (`getNamespaceUidFromId`, `getPipelineUidFromId`) for internal UID lookups - Renames JWT auth test files to Basic Auth to reflect CE edition authentication - Updates integration test protobufs to match current API definitions
1 parent 2d74899 commit a9cd2c4

40 files changed

+1244
-3325
lines changed

.github/workflows/golangci-lint.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ jobs:
4242
echo "LD_RUN_PATH=${ONNXRUNTIME_ROOT_PATH}/lib" >> $GITHUB_ENV
4343
echo "LIBRARY_PATH=${ONNXRUNTIME_ROOT_PATH}/lib" >> $GITHUB_ENV
4444
- name: golangci-lint
45-
uses: golangci/golangci-lint-action@v8
45+
uses: golangci/golangci-lint-action@v9
4646
with:
4747
version: v2.8.0
4848
args: --timeout=10m --build-tags onnx

go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ require (
4545
github.com/h2non/filetype v1.1.3
4646
github.com/iancoleman/strcase v0.3.0
4747
github.com/influxdata/influxdb-client-go/v2 v2.14.0
48-
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20260118041154-8f06ba4d527d
49-
github.com/instill-ai/x v0.10.1-alpha.0.20260118004501-5221537d0a1d
48+
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20260121105057-e8e7f1a4ac1f
49+
github.com/instill-ai/x v0.10.1-alpha.0.20260121080139-2e854df73509
5050
github.com/itchyny/gojq v0.12.17
5151
github.com/jackc/pgx/v5 v5.7.6
5252
github.com/jmoiron/sqlx v1.4.0

go.sum

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -513,10 +513,10 @@ github.com/influxdata/influxdb-client-go/v2 v2.14.0 h1:AjbBfJuq+QoaXNcrova8smSjw
513513
github.com/influxdata/influxdb-client-go/v2 v2.14.0/go.mod h1:Ahpm3QXKMJslpXl3IftVLVezreAUtBOTZssDrjZEFHI=
514514
github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf h1:7JTmneyiNEwVBOHSjoMxiWAqB992atOeepeFYegn5RU=
515515
github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
516-
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20260118041154-8f06ba4d527d h1:nQTNxSL0sakFtObb6tnZqXcB/LF1dqGFoaRfX5ywDfA=
517-
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20260118041154-8f06ba4d527d/go.mod h1:bCnBosofpaUxKBuTTJM3/I3thAK37kvfBnKByjnLsl4=
518-
github.com/instill-ai/x v0.10.1-alpha.0.20260118004501-5221537d0a1d h1:YUOzft7m56Jj+FUohdmn36d8gFnDcDo/tO2ij1zrzhw=
519-
github.com/instill-ai/x v0.10.1-alpha.0.20260118004501-5221537d0a1d/go.mod h1:NZwVnFLteR6JvV35pCYOOO75YMzTABefbrBC6YWDF/E=
516+
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20260121105057-e8e7f1a4ac1f h1:8GYg7QC7zVQlYfDak63+z6F1qc6rvTASf8O97tPNBrI=
517+
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20260121105057-e8e7f1a4ac1f/go.mod h1:bCnBosofpaUxKBuTTJM3/I3thAK37kvfBnKByjnLsl4=
518+
github.com/instill-ai/x v0.10.1-alpha.0.20260121080139-2e854df73509 h1:sVodrc+jwN/q1XOV0UJPv+DBoEDXep4IK5khxj+7hCI=
519+
github.com/instill-ai/x v0.10.1-alpha.0.20260121080139-2e854df73509/go.mod h1:YiJG0vfsXUZxDg9Fnc6wxVhpOXMpa/gR7ELaaQM+5tQ=
520520
github.com/itchyny/gojq v0.12.17 h1:8av8eGduDb5+rvEdaOO+zQUjA04MS0m3Ps8HiD+fceg=
521521
github.com/itchyny/gojq v0.12.17/go.mod h1:WBrEMkgAfAGO1LUcGOckBl5O726KPp+OlkKug0I/FEY=
522522
github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/myJ5Q=

integration-test/const.js

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,20 @@ if (__ENV.API_GATEWAY_PROTOCOL) {
1818
// API Gateway URL (localhost:8080 from host, api-gateway:8080 from container)
1919
const apiGatewayUrl = __ENV.API_GATEWAY_URL || "localhost:8080";
2020

21+
// Determine if running from host (localhost) or container
22+
export const isHostMode = apiGatewayUrl === "localhost:8080";
23+
2124
// Public hosts (via API Gateway)
2225
export const pipelinePublicHost = `${proto}://${apiGatewayUrl}`;
2326
export const mgmtPublicHost = `${proto}://${apiGatewayUrl}`;
2427
export const pipelineGRPCPublicHost = apiGatewayUrl;
2528
export const mgmtGRPCPublicHost = apiGatewayUrl;
2629

2730
// Private hosts (direct backend, for internal service calls)
28-
export const pipelinePrivateHost = `http://pipeline-backend:3081`;
29-
export const pipelineGRPCPrivateHost = `pipeline-backend:3081`;
30-
export const mgmtGRPCPrivateHost = `mgmt-backend:3084`;
31+
// Use localhost ports when running from host, container names when in container
32+
export const pipelinePrivateHost = isHostMode ? `http://localhost:3081` : `http://pipeline-backend:3081`;
33+
export const pipelineGRPCPrivateHost = isHostMode ? `localhost:3081` : `pipeline-backend:3081`;
34+
export const mgmtGRPCPrivateHost = isHostMode ? `localhost:3084` : `mgmt-backend:3084`;
3135

3236
export const dogImg = encoding.b64encode(
3337
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/dog.jpg`, "b")
@@ -60,18 +64,20 @@ export const paramsGrpc = {
6064
timeout: "300s",
6165
};
6266

63-
const randomUUID = uuidv4();
64-
export const paramsGRPCWithJwt = {
67+
// Invalid Basic Auth credentials for testing unauthorized access
68+
const invalidBasicAuth = encoding.b64encode("invalid-user:invalid-password");
69+
70+
export const paramsGRPCWithInvalidAuth = {
6571
metadata: {
6672
"Content-Type": "application/json",
67-
"Instill-User-Uid": randomUUID,
73+
"Authorization": `Basic ${invalidBasicAuth}`,
6874
},
6975
};
7076

71-
export const paramsHTTPWithJwt = {
77+
export const paramsHTTPWithInvalidAuth = {
7278
headers: {
7379
"Content-Type": "application/json",
74-
"Instill-User-Uid": randomUUID,
80+
"Authorization": `Basic ${invalidBasicAuth}`,
7581
},
7682
};
7783

@@ -116,7 +122,9 @@ if (__ENV.DB_HOST) {
116122
dbHost = __ENV.DB_HOST;
117123
}
118124

119-
export const db = sql.open(driver, `postgresql://postgres:password@${dbHost}:5432/pipeline?sslmode=disable`);
125+
// Database connections - pipeline for pipeline data, mgmt for namespace/owner data
126+
export const pipelinedb = sql.open(driver, `postgresql://postgres:password@${dbHost}:5432/pipeline?sslmode=disable`);
127+
export const mgmtDb = sql.open(driver, `postgresql://postgres:password@${dbHost}:5432/mgmt?sslmode=disable`);
120128

121129
// Since the tests rely on a pre-existing user, this prefix is used to clean
122130
// up only the resources generated by these tests.
Lines changed: 244 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,244 @@
1+
import grpc from "k6/net/grpc";
2+
import { check, group } from "k6";
3+
4+
import * as constant from "./const.js";
5+
import * as helper from "./helper.js";
6+
7+
const publicClient = new grpc.Client();
8+
const privateClient = new grpc.Client();
9+
10+
publicClient.load(["proto"], "pipeline/v1beta/pipeline_public_service.proto");
11+
privateClient.load(["proto"], "pipeline/v1beta/pipeline_private_service.proto");
12+
13+
// ============================================================================
14+
// Private Service API Tests
15+
// These tests cover admin/private service APIs (PipelinePrivateService)
16+
// that are only accessible internally (port 3081, not exposed externally)
17+
//
18+
// For service-to-service communication, we pass internal headers:
19+
// - Instill-User-Uid: The authenticated user's UUID
20+
// - Instill-Requester-Uid: The requester's UUID (namespace making the request)
21+
// ============================================================================
22+
23+
/**
24+
* Get internal service metadata with user/requester UIDs.
25+
* This simulates what the API Gateway would inject after authentication.
26+
*/
27+
function getInternalServiceMetadata() {
28+
// Get the admin user's UID from the database
29+
var userUid = helper.getNamespaceUidFromId(constant.defaultUsername);
30+
if (!userUid) {
31+
console.log(`[WARN] Could not get user UID for ${constant.defaultUsername}, using fallback`);
32+
return null;
33+
}
34+
35+
return {
36+
metadata: {
37+
"Instill-User-Uid": userUid,
38+
"Instill-Requester-Uid": userUid, // For admin user, requester is the same
39+
},
40+
};
41+
}
42+
43+
/**
44+
* Test LookUpPipelineAdmin (private service)
45+
* This endpoint is admin-only and requires the internal UID
46+
* NOTE: Private service port (3081) is not exposed outside container,
47+
* so this test only runs when executing from within the container.
48+
*/
49+
export function CheckLookUpPipelineAdmin(data) {
50+
// Skip if running from host - private service port not exposed
51+
if (constant.isHostMode) {
52+
group("Pipelines API: Look up a pipeline by UID (Admin) [SKIPPED - host mode]", () => {
53+
console.log("SKIPPED: Private service tests require running inside container");
54+
});
55+
return;
56+
}
57+
58+
group("Pipelines API: Look up a pipeline by UID (Admin)", () => {
59+
publicClient.connect(constant.pipelineGRPCPublicHost, {
60+
plaintext: true,
61+
});
62+
privateClient.connect(constant.pipelineGRPCPrivateHost, {
63+
plaintext: true,
64+
});
65+
66+
// Get internal service metadata (simulates what API Gateway injects)
67+
var internalMeta = getInternalServiceMetadata();
68+
if (!internalMeta) {
69+
console.log("[SKIP] Could not get internal service metadata, skipping LookUpPipelineAdmin");
70+
publicClient.close();
71+
privateClient.close();
72+
return;
73+
}
74+
75+
var reqBody = Object.assign(
76+
{},
77+
constant.simplePipelineWithYAMLRecipe
78+
);
79+
80+
// Create a pipeline via public API
81+
var createRes = publicClient.invoke(
82+
"pipeline.v1beta.PipelinePublicService/CreateNamespacePipeline",
83+
{
84+
parent: constant.namespace,
85+
pipeline: reqBody,
86+
},
87+
data.metadata
88+
);
89+
check(createRes, {
90+
"CreateNamespacePipeline response StatusOK": (r) => r.status === grpc.StatusOK,
91+
});
92+
93+
if (createRes.status !== grpc.StatusOK || !createRes.message || !createRes.message.pipeline) {
94+
console.log("Failed to create pipeline in CheckLookUpPipelineAdmin, skipping remaining tests");
95+
publicClient.close();
96+
privateClient.close();
97+
return;
98+
}
99+
100+
var pipelineId = createRes.message.pipeline.id;
101+
102+
// Get the internal UID from database
103+
var pipelineUid = helper.getPipelineUidFromId(pipelineId);
104+
check(pipelineUid, {
105+
"Got pipeline UID from database": (uid) => uid !== null && uid !== undefined,
106+
});
107+
108+
if (!pipelineUid) {
109+
console.log(`Failed to get pipeline UID for id=${pipelineId}, skipping LookUpAdmin test`);
110+
// Cleanup
111+
publicClient.invoke(
112+
"pipeline.v1beta.PipelinePublicService/DeleteNamespacePipeline",
113+
{ name: `${constant.namespace}/pipelines/${pipelineId}` },
114+
data.metadata
115+
);
116+
publicClient.close();
117+
privateClient.close();
118+
return;
119+
}
120+
121+
// LookUpPipelineAdmin by UID permalink (private service)
122+
// Use internal service headers for service-to-service communication
123+
var lookupRes = privateClient.invoke(
124+
"pipeline.v1beta.PipelinePrivateService/LookUpPipelineAdmin",
125+
{
126+
permalink: `pipelines/${pipelineUid}`,
127+
},
128+
internalMeta // Use internal service-to-service headers
129+
);
130+
check(lookupRes, {
131+
"LookUpPipelineAdmin response StatusOK": (r) => r.status === grpc.StatusOK,
132+
"LookUpPipelineAdmin response has pipeline": (r) =>
133+
r.message && r.message.pipeline,
134+
"LookUpPipelineAdmin response pipeline has correct id": (r) =>
135+
r.message && r.message.pipeline && r.message.pipeline.id === pipelineId,
136+
"LookUpPipelineAdmin response pipeline has name": (r) =>
137+
r.message && r.message.pipeline && r.message.pipeline.name &&
138+
r.message.pipeline.name.includes("/pipelines/"),
139+
});
140+
141+
// Delete the pipeline
142+
publicClient.invoke(
143+
"pipeline.v1beta.PipelinePublicService/DeleteNamespacePipeline",
144+
{
145+
name: `${constant.namespace}/pipelines/${pipelineId}`,
146+
},
147+
data.metadata
148+
);
149+
150+
publicClient.close();
151+
privateClient.close();
152+
});
153+
}
154+
155+
/**
156+
* Test ListPipelinesAdmin (private service)
157+
* This endpoint lists all pipelines across all namespaces (admin only)
158+
* NOTE: Private service port (3081) is not exposed outside container,
159+
* so this test only runs when executing from within the container.
160+
*/
161+
export function CheckListPipelinesAdmin(data) {
162+
// Skip if running from host - private service port not exposed
163+
if (constant.isHostMode) {
164+
group("Pipelines API: List all pipelines (Admin) [SKIPPED - host mode]", () => {
165+
console.log("SKIPPED: Private service tests require running inside container");
166+
});
167+
return;
168+
}
169+
170+
group("Pipelines API: List all pipelines (Admin)", () => {
171+
publicClient.connect(constant.pipelineGRPCPublicHost, {
172+
plaintext: true,
173+
});
174+
privateClient.connect(constant.pipelineGRPCPrivateHost, {
175+
plaintext: true,
176+
});
177+
178+
// Get internal service metadata (simulates what API Gateway injects)
179+
var internalMeta = getInternalServiceMetadata();
180+
if (!internalMeta) {
181+
console.log("[SKIP] Could not get internal service metadata, skipping ListPipelinesAdmin");
182+
publicClient.close();
183+
privateClient.close();
184+
return;
185+
}
186+
187+
var reqBody = Object.assign(
188+
{},
189+
constant.simplePipelineWithYAMLRecipe
190+
);
191+
192+
// Create a pipeline via public API
193+
var createRes = publicClient.invoke(
194+
"pipeline.v1beta.PipelinePublicService/CreateNamespacePipeline",
195+
{
196+
parent: constant.namespace,
197+
pipeline: reqBody,
198+
},
199+
data.metadata
200+
);
201+
check(createRes, {
202+
"CreateNamespacePipeline response StatusOK": (r) => r.status === grpc.StatusOK,
203+
});
204+
205+
if (createRes.status !== grpc.StatusOK || !createRes.message || !createRes.message.pipeline) {
206+
console.log("Failed to create pipeline in CheckListPipelinesAdmin, skipping remaining tests");
207+
publicClient.close();
208+
privateClient.close();
209+
return;
210+
}
211+
212+
var pipelineId = createRes.message.pipeline.id;
213+
214+
// ListPipelinesAdmin (private service)
215+
// Use internal service headers for service-to-service communication
216+
var listRes = privateClient.invoke(
217+
"pipeline.v1beta.PipelinePrivateService/ListPipelinesAdmin",
218+
{
219+
pageSize: 100,
220+
},
221+
internalMeta // Use internal service-to-service headers
222+
);
223+
check(listRes, {
224+
"ListPipelinesAdmin response StatusOK": (r) => r.status === grpc.StatusOK,
225+
"ListPipelinesAdmin response has pipelines": (r) =>
226+
r.message && r.message.pipelines,
227+
"ListPipelinesAdmin response contains created pipeline": (r) =>
228+
r.message && r.message.pipelines &&
229+
r.message.pipelines.some(p => p.id === pipelineId),
230+
});
231+
232+
// Delete the pipeline
233+
publicClient.invoke(
234+
"pipeline.v1beta.PipelinePublicService/DeleteNamespacePipeline",
235+
{
236+
name: `${constant.namespace}/pipelines/${pipelineId}`,
237+
},
238+
data.metadata
239+
);
240+
241+
publicClient.close();
242+
privateClient.close();
243+
});
244+
}

0 commit comments

Comments
 (0)