Skip to content

Commit 21d1ee6

Browse files
pinglinpinglin
authored
feat(cache,file-type): implement small file caching and enhance multimedia support (#285)
Because - Small files (< 1024 tokens) were not being cached efficiently, requiring fallback to Redis storage - Multimedia file type detection was incomplete, causing upload failures for images and other media types - File processing workflows could hang indefinitely on missing files instead of failing fast - Integration tests used undefined constants causing silent failures This commit - **Small Files Caching**: Added token-based detection to return content markdown URLs for files too small for Gemini cache, using actual token count instead of file size proxy - **Multimedia Support**: Enhanced file type detection for images (PNG, JPEG, GIF, WebP, TIFF, HEIC, HEIF, AVIF, BMP) and improved handling for audio/video files - **Workflow Reliability**: Made GetFileMetadataActivity failures fail workflows immediately with non-retryable errors for missing files - **Test Robustness**: Fixed undefined constants (sampleTxt → docSampleTxt, sampleMd → docSampleMd) across 32 test references and improved error handling - **API Consistency**: Standardized content URLs to use ViewCacheTTL and ensured derivedResourceUri always has a value Key improvements: - Small files now return accessible content URLs instead of empty cache responses - Workflows fail fast with clear error messages instead of hanging - Tests provide meaningful failure diagnostics with proper file validation
1 parent 8b12a79 commit 21d1ee6

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+1164
-392
lines changed

‎Makefile‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,13 @@ integration-test: ## Run integration tests in parallel using GNU parallel
9797
-e DB_HOST=${DB_HOST} \
9898
{} --no-usage-report" ::: \
9999
integration-test/rest.js \
100-
integration-test/rest-file-type.js \
100+
integration-test/rest-file-type.js 2>&1 | tee /tmp/artifact-integration-test.log; \
101+
bash integration-test/scripts/report-summary.sh /tmp/artifact-integration-test.log
102+
@parallel --halt now,fail=1 --tag --line-buffer \
103+
"TEST_FOLDER_ABS_PATH=${PWD} k6 run --address=\"\" \
104+
-e API_GATEWAY_PROTOCOL=${API_GATEWAY_PROTOCOL} -e API_GATEWAY_URL=${API_GATEWAY_URL} \
105+
-e DB_HOST=${DB_HOST} \
106+
{} --no-usage-report" ::: \
101107
integration-test/rest-db.js \
102108
integration-test/rest-ai-client.js \
103109
integration-test/rest-kb-e2e-file-process.js \

‎integration-test/const.js‎

Lines changed: 140 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -61,65 +61,163 @@ export const paramsHTTPWithJWT = {
6161
},
6262
};
6363

64-
// Test data files
65-
export const sampleDoc = encoding.b64encode(
66-
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/sample.doc`, "b")
64+
// Test document sample data files
65+
export const docSampleDoc = encoding.b64encode(
66+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/doc-sample.doc`, "b")
6767
);
68-
export const sampleDocx = encoding.b64encode(
69-
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/sample.docx`, "b")
68+
export const docSampleDocx = encoding.b64encode(
69+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/doc-sample.docx`, "b")
7070
);
71-
export const sampleUppercaseDoc = encoding.b64encode(
72-
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/SAMPLE-UPPERCASE-FILENAME.DOC`, "b")
71+
export const docSampleTxt = encoding.b64encode(
72+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/doc-sample.txt`, "b")
7373
);
74-
export const sampleUppercaseDocx = encoding.b64encode(
75-
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/SAMPLE-UPPERCASE-FILENAME.DOCX`, "b")
74+
export const docSampleHtml = encoding.b64encode(
75+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/doc-sample.html`, "b")
7676
);
77-
export const sampleTxt = encoding.b64encode(
78-
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/sample.txt`, "b")
77+
export const docSampleMd = encoding.b64encode(
78+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/doc-sample.md`, "b")
7979
);
80-
export const sampleHtml = encoding.b64encode(
81-
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/sample.html`, "b")
80+
export const docSampleCsv = encoding.b64encode(
81+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/doc-sample.csv`, "b")
8282
);
83-
export const sampleMd = encoding.b64encode(
84-
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/sample.md`, "b")
83+
export const docSamplePdf = encoding.b64encode(
84+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/doc-sample.pdf`, "b")
8585
);
86-
export const sampleCsv = encoding.b64encode(
87-
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/sample.csv`, "b")
86+
export const docSampleMultiPagePdf = encoding.b64encode(
87+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/doc-sample-multi-page.pdf`, "b")
8888
);
89-
export const samplePdf = encoding.b64encode(
90-
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/sample.pdf`, "b")
89+
export const docSamplePpt = encoding.b64encode(
90+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/doc-sample.ppt`, "b")
9191
);
92-
export const sampleMultiPagePdf = encoding.b64encode(
93-
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/sample-multi-page.pdf`, "b")
92+
export const docSamplePptx = encoding.b64encode(
93+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/doc-sample.pptx`, "b")
9494
);
95-
export const samplePpt = encoding.b64encode(
96-
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/sample.ppt`, "b")
95+
export const docSampleXls = encoding.b64encode(
96+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/doc-sample.xls`, "b")
9797
);
98-
export const samplePptx = encoding.b64encode(
99-
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/sample.pptx`, "b")
98+
export const docSampleXlsx = encoding.b64encode(
99+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/doc-sample.xlsx`, "b")
100100
);
101-
export const sampleXls = encoding.b64encode(
102-
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/sample.xls`, "b")
101+
102+
// Test image sample data files
103+
export const imageSamplePng = encoding.b64encode(
104+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/img-sample.png`, "b")
105+
);
106+
export const imageSampleJpeg = encoding.b64encode(
107+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/img-sample.jpeg`, "b")
108+
);
109+
export const imageSampleGif = encoding.b64encode(
110+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/img-sample.gif`, "b")
111+
);
112+
export const imageSampleWebp = encoding.b64encode(
113+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/img-sample.webp`, "b")
114+
);
115+
export const imageSampleTiff = encoding.b64encode(
116+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/img-sample.tiff`, "b")
117+
);
118+
export const imageSampleHeic = encoding.b64encode(
119+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/img-sample.heic`, "b")
120+
);
121+
export const imageSampleHeif = encoding.b64encode(
122+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/img-sample.heif`, "b")
123+
);
124+
export const imageSampleAvif = encoding.b64encode(
125+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/img-sample.avif`, "b")
126+
);
127+
export const imageSampleBmp = encoding.b64encode(
128+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/img-sample.bmp`, "b")
129+
);
130+
131+
// Test audio sample data files
132+
export const audioSampleMp3 = encoding.b64encode(
133+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/audio-sample.mp3`, "b")
134+
);
135+
export const audioSampleWav = encoding.b64encode(
136+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/audio-sample.wav`, "b")
137+
);
138+
export const audioSampleAac = encoding.b64encode(
139+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/audio-sample.aac`, "b")
140+
);
141+
export const audioSampleOgg = encoding.b64encode(
142+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/audio-sample.ogg`, "b")
143+
);
144+
export const audioSampleFlac = encoding.b64encode(
145+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/audio-sample.flac`, "b")
146+
);
147+
export const audioSampleAiff = encoding.b64encode(
148+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/audio-sample.aiff`, "b")
149+
);
150+
export const audioSampleM4a = encoding.b64encode(
151+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/audio-sample.m4a`, "b")
152+
);
153+
export const audioSampleWma = encoding.b64encode(
154+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/audio-sample.wma`, "b")
155+
);
156+
157+
// Test video sample data files
158+
export const videoSampleMp4 = encoding.b64encode(
159+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/video-sample.mp4`, "b")
160+
);
161+
export const videoSampleMkv = encoding.b64encode(
162+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/video-sample.mkv`, "b")
163+
);
164+
export const videoSampleAvi = encoding.b64encode(
165+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/video-sample.mp4`, "b")
166+
);
167+
export const videoSampleMov = encoding.b64encode(
168+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/video-sample.mkv`, "b")
169+
);
170+
export const videoSampleFlv = encoding.b64encode(
171+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/video-sample.flv`, "b")
172+
);
173+
export const videoSampleWebm = encoding.b64encode(
174+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/video-sample.webm`, "b")
175+
);
176+
export const videoSampleWmv = encoding.b64encode(
177+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/video-sample.wmv`, "b")
103178
);
104-
export const sampleXlsx = encoding.b64encode(
105-
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/sample.xlsx`, "b")
179+
export const videoSampleMpeg = encoding.b64encode(
180+
open(`${__ENV.TEST_FOLDER_ABS_PATH}/integration-test/data/video-sample.mpeg`, "b")
106181
);
107182

108183
// Mapping of sample files to their intended artifact file types
109184
export const sampleFiles = [
110-
{ originalName: "sample.txt", type: "TYPE_TEXT", content: sampleTxt },
111-
{ originalName: "sample.md", type: "TYPE_MARKDOWN", content: sampleMd },
112-
{ originalName: "sample.csv", type: "TYPE_CSV", content: sampleCsv },
113-
{ originalName: "sample.html", type: "TYPE_HTML", content: sampleHtml },
114-
{ originalName: "sample.pdf", type: "TYPE_PDF", content: samplePdf },
115-
{ originalName: "sample.ppt", type: "TYPE_PPT", content: samplePpt },
116-
{ originalName: "sample.pptx", type: "TYPE_PPTX", content: samplePptx },
117-
{ originalName: "sample.xls", type: "TYPE_XLS", content: sampleXls },
118-
{ originalName: "sample.xlsx", type: "TYPE_XLSX", content: sampleXlsx },
119-
{ originalName: "sample.doc", type: "TYPE_DOC", content: sampleDoc },
120-
{ originalName: "sample.docx", type: "TYPE_DOCX", content: sampleDocx },
121-
{ originalName: "SAMPLE-UPPERCASE-FILENAME.DOC", type: "TYPE_DOC", content: sampleUppercaseDoc },
122-
{ originalName: "SAMPLE-UPPERCASE-FILENAME.DOCX", type: "TYPE_DOCX", content: sampleUppercaseDocx },
185+
{ originalName: "doc-sample.txt", type: "TYPE_TEXT", content: docSampleTxt },
186+
{ originalName: "doc-sample.md", type: "TYPE_MARKDOWN", content: docSampleMd },
187+
{ originalName: "doc-sample.csv", type: "TYPE_CSV", content: docSampleCsv },
188+
{ originalName: "doc-sample.html", type: "TYPE_HTML", content: docSampleHtml },
189+
{ originalName: "doc-sample.pdf", type: "TYPE_PDF", content: docSamplePdf },
190+
{ originalName: "doc-sample.ppt", type: "TYPE_PPT", content: docSamplePpt },
191+
{ originalName: "doc-sample.pptx", type: "TYPE_PPTX", content: docSamplePptx },
192+
{ originalName: "doc-sample.xls", type: "TYPE_XLS", content: docSampleXls },
193+
{ originalName: "doc-sample.xlsx", type: "TYPE_XLSX", content: docSampleXlsx },
194+
{ originalName: "doc-sample.doc", type: "TYPE_DOC", content: docSampleDoc },
195+
{ originalName: "doc-sample.docx", type: "TYPE_DOCX", content: docSampleDocx },
196+
{ originalName: "img-sample.png", type: "TYPE_PNG", content: imageSamplePng },
197+
{ originalName: "img-sample.jpeg", type: "TYPE_JPEG", content: imageSampleJpeg },
198+
{ originalName: "img-sample.gif", type: "TYPE_GIF", content: imageSampleGif },
199+
{ originalName: "img-sample.webp", type: "TYPE_WEBP", content: imageSampleWebp },
200+
{ originalName: "img-sample.tiff", type: "TYPE_TIFF", content: imageSampleTiff },
201+
{ originalName: "img-sample.heic", type: "TYPE_HEIC", content: imageSampleHeic },
202+
{ originalName: "img-sample.heif", type: "TYPE_HEIF", content: imageSampleHeif },
203+
{ originalName: "img-sample.avif", type: "TYPE_AVIF", content: imageSampleAvif },
204+
{ originalName: "img-sample.bmp", type: "TYPE_BMP", content: imageSampleBmp },
205+
{ originalName: "audio-sample.mp3", type: "TYPE_MP3", content: audioSampleMp3 },
206+
{ originalName: "audio-sample.wav", type: "TYPE_WAV", content: audioSampleWav },
207+
{ originalName: "audio-sample.aac", type: "TYPE_AAC", content: audioSampleAac },
208+
{ originalName: "audio-sample.ogg", type: "TYPE_OGG", content: audioSampleOgg },
209+
{ originalName: "audio-sample.flac", type: "TYPE_FLAC", content: audioSampleFlac },
210+
{ originalName: "audio-sample.aiff", type: "TYPE_AIFF", content: audioSampleAiff },
211+
{ originalName: "audio-sample.m4a", type: "TYPE_M4A", content: audioSampleM4a },
212+
{ originalName: "audio-sample.wma", type: "TYPE_WMA", content: audioSampleWma },
213+
{ originalName: "video-sample.mp4", type: "TYPE_MP4", content: videoSampleMp4 },
214+
{ originalName: "video-sample.mkv", type: "TYPE_MKV", content: videoSampleMkv },
215+
{ originalName: "video-sample.avi", type: "TYPE_AVI", content: videoSampleAvi },
216+
{ originalName: "video-sample.mov", type: "TYPE_MOV", content: videoSampleMov },
217+
{ originalName: "video-sample.flv", type: "TYPE_FLV", content: videoSampleFlv },
218+
{ originalName: "video-sample.webm", type: "TYPE_WEBM_VIDEO", content: videoSampleWebm },
219+
{ originalName: "video-sample.wmv", type: "TYPE_WMV", content: videoSampleWmv },
220+
{ originalName: "video-sample.mpeg", type: "TYPE_MPEG", content: videoSampleMpeg },
123221
];
124222

125223
let dbHost = 'localhost';

‎integration-test/data/audio-sample.aac‎

7.84 KB
Binary file not shown.

‎integration-test/data/audio-sample.aiff‎

43.1 KB
Binary file not shown.

‎integration-test/data/audio-sample.flac‎

14.9 KB
Binary file not shown.

‎integration-test/data/audio-sample.m4a‎

8.58 KB
Binary file not shown.

‎integration-test/data/audio-sample.mp3‎

4.4 KB
Binary file not shown.

‎integration-test/data/audio-sample.ogg‎

4.58 KB
Binary file not shown.

‎integration-test/data/audio-sample.wav‎

43.1 KB
Binary file not shown.

‎integration-test/data/audio-sample.wma‎

19.3 KB
Binary file not shown.

0 commit comments

Comments
 (0)