Skip to content

Commit d931894

Browse files
authored
fix(plugin-import-export): fix exports in other non-latin scripts being broken when opened in excel (#15813)
Fixes #13929 We were not correctly marking the mime headers to our .csv files with UTF-8 encoding and as such when they were being downloaded it would affect the file's formatting. After this PR when opening Hebrew exported in CSV it will look as expected in the software. Tested manually with Microsoft Excel. A side effect of this PR is also that Excel can now correctly prompt you to adjust the delimiting every time, for our files make sure you set it to `Comma` instead of tabs to have everything properly separated by columns. <img width="583" height="447" alt="image" src="https://github.com/user-attachments/assets/322367a1-580c-4e7f-84c9-d9b9d857e5e3" />
1 parent 07f2f05 commit d931894

6 files changed

Lines changed: 217 additions & 27 deletions

File tree

packages/plugin-import-export/src/components/ExportSaveButton/index.tsx

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -99,24 +99,11 @@ export const ExportSaveButton: React.FC = () => {
9999
throw new Error(errorMsg)
100100
}
101101

102-
const fileStream = response.body
103-
const reader = fileStream?.getReader()
104-
const decoder = new TextDecoder()
105-
let result = ''
106-
107-
while (reader) {
108-
const { done, value } = await reader.read()
109-
if (done) {
110-
break
111-
}
112-
result += decoder.decode(value, { stream: true })
113-
}
114-
115-
const blob = new Blob([result], { type: 'text/plain' })
102+
const blob = await response.blob()
116103
const url = URL.createObjectURL(blob)
117104
const a = document.createElement('a')
118105
a.href = url
119-
a.download = `${data.name}.${data.format}`
106+
a.download = `${data.name}-${data.collectionSlug}.${data.format}`
120107
document.body.appendChild(a)
121108
a.click()
122109
document.body.removeChild(a)

packages/plugin-import-export/src/export/createExport.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -406,7 +406,7 @@ export const createExport = async (args: CreateExportArgs) => {
406406
return new Response(Readable.toWeb(stream) as ReadableStream, {
407407
headers: {
408408
'Content-Disposition': `attachment; filename="${name}"`,
409-
'Content-Type': isCSV ? 'text/csv' : 'application/json',
409+
'Content-Type': isCSV ? 'text/csv; charset=utf-8' : 'application/json',
410410
},
411411
})
412412
}
@@ -511,7 +511,7 @@ export const createExport = async (args: CreateExportArgs) => {
511511
req.file = {
512512
name,
513513
data: buffer,
514-
mimetype: isCSV ? 'text/csv' : 'application/json',
514+
mimetype: isCSV ? 'text/csv; charset=utf-8' : 'application/json',
515515
size: buffer.length,
516516
}
517517
} else {
@@ -522,7 +522,7 @@ export const createExport = async (args: CreateExportArgs) => {
522522
exportCollection,
523523
fileName: name,
524524
fileSize: buffer.length,
525-
mimeType: isCSV ? 'text/csv' : 'application/json',
525+
mimeType: isCSV ? 'text/csv; charset=utf-8' : 'application/json',
526526
})
527527
}
528528
try {
@@ -533,7 +533,7 @@ export const createExport = async (args: CreateExportArgs) => {
533533
file: {
534534
name,
535535
data: buffer,
536-
mimetype: isCSV ? 'text/csv' : 'application/json',
536+
mimetype: isCSV ? 'text/csv; charset=utf-8' : 'application/json',
537537
size: buffer.length,
538538
},
539539
// Override access only here so that we can be sure the export collection itself is updated as expected

test/plugin-import-export/config.ts

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { importExportPlugin } from '@payloadcms/plugin-import-export'
22
import { s3Storage } from '@payloadcms/storage-s3'
33
import { en } from '@payloadcms/translations/languages/en'
44
import { es } from '@payloadcms/translations/languages/es'
5+
import { he } from '@payloadcms/translations/languages/he'
56
import dotenv from 'dotenv'
67
import { fileURLToPath } from 'node:url'
78
import path from 'path'
@@ -54,12 +55,18 @@ export default buildConfigWithDefaults({
5455
localization: {
5556
defaultLocale: 'en',
5657
fallback: true,
57-
locales: ['en', 'es', 'de'],
58+
locales: [
59+
{ code: 'en', label: 'English' },
60+
{ code: 'es', label: 'Spanish' },
61+
{ code: 'de', label: 'German' },
62+
{ code: 'he', label: 'Hebrew', rtl: true },
63+
],
5864
},
5965
i18n: {
6066
supportedLanguages: {
6167
en,
6268
es,
69+
he,
6370
},
6471
fallbackLanguage: 'en',
6572
},

test/plugin-import-export/int.spec.ts

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -743,6 +743,9 @@ describe('@payloadcms/plugin-import-export', () => {
743743
expect(response.status).toBe(200)
744744
expect(response.headers.get('content-type')).toMatch(/application\/json/)
745745

746+
const contentDisposition = response.headers.get('content-disposition')
747+
expect(contentDisposition).toContain('-pages.json')
748+
746749
const data = await response.json()
747750

748751
expect(Array.isArray(data)).toBe(true)
@@ -2322,6 +2325,161 @@ describe('@payloadcms/plugin-import-export', () => {
23222325

23232326
await payload.delete({ collection: 'pages', id: page.id })
23242327
})
2328+
2329+
it('should preserve Hebrew characters in CSV download via streaming endpoint', async () => {
2330+
const hebrewTitle = 'Hebrew BOM Test'
2331+
const hebrewLocalized = 'בדיקה עברית'
2332+
2333+
const page = await payload.create({
2334+
collection: 'pages',
2335+
data: {
2336+
title: hebrewTitle,
2337+
localized: hebrewLocalized,
2338+
_status: 'published',
2339+
},
2340+
locale: 'he',
2341+
})
2342+
2343+
const response = await restClient.POST('/exports/download', {
2344+
body: JSON.stringify({
2345+
data: {
2346+
collectionSlug: 'pages',
2347+
fields: ['id', 'title', 'localized'],
2348+
format: 'csv',
2349+
locale: 'he',
2350+
where: { id: { equals: page.id } },
2351+
},
2352+
}),
2353+
headers: { 'Content-Type': 'application/json' },
2354+
})
2355+
2356+
expect(response.status).toBe(200)
2357+
expect(response.headers.get('content-type')).toMatch(/text\/csv/)
2358+
expect(response.headers.get('content-type')).toContain('charset=utf-8')
2359+
2360+
const contentDisposition = response.headers.get('content-disposition')
2361+
expect(contentDisposition).toContain('-pages.csv')
2362+
2363+
const buffer = Buffer.from(await response.arrayBuffer())
2364+
2365+
// Verify UTF-8 BOM is present
2366+
expect(buffer[0]).toBe(0xef)
2367+
expect(buffer[1]).toBe(0xbb)
2368+
expect(buffer[2]).toBe(0xbf)
2369+
2370+
// Verify Hebrew text is correctly encoded in the CSV body
2371+
const content = buffer.toString('utf-8')
2372+
expect(content).toContain(hebrewLocalized)
2373+
2374+
await payload.delete({ collection: 'pages', id: page.id })
2375+
})
2376+
2377+
it('should preserve Hebrew characters in job-created CSV export', async () => {
2378+
const hebrewTitle = 'Hebrew Jobs Test'
2379+
const hebrewLocalized = 'שלום עולם'
2380+
2381+
const page = await payload.create({
2382+
collection: 'pages',
2383+
data: {
2384+
title: hebrewTitle,
2385+
localized: hebrewLocalized,
2386+
_status: 'published',
2387+
},
2388+
locale: 'he',
2389+
})
2390+
2391+
let doc = await payload.create({
2392+
collection: 'exports',
2393+
user,
2394+
data: {
2395+
collectionSlug: 'pages',
2396+
fields: ['id', 'title', 'localized'],
2397+
format: 'csv',
2398+
locale: 'he',
2399+
where: { id: { equals: page.id } },
2400+
},
2401+
})
2402+
2403+
await payload.jobs.run()
2404+
2405+
doc = await payload.findByID({ collection: 'exports', id: doc.id })
2406+
2407+
// Verify filename includes collection slug and csv extension
2408+
expect(doc.filename).toContain('-pages')
2409+
expect(doc.filename).toMatch(/\.csv$/)
2410+
expect(doc.mimeType).toContain('charset=utf-8')
2411+
2412+
const csvPath = path.join(dirname, './uploads', doc.filename as string)
2413+
const buffer = fs.readFileSync(csvPath)
2414+
2415+
// Verify UTF-8 BOM
2416+
expect(buffer[0]).toBe(0xef)
2417+
expect(buffer[1]).toBe(0xbb)
2418+
expect(buffer[2]).toBe(0xbf)
2419+
2420+
// Verify Hebrew text is readable
2421+
const content = buffer.toString('utf-8')
2422+
expect(content).toContain(hebrewLocalized)
2423+
2424+
// Verify via CSV parse
2425+
const data = await readCSV(csvPath)
2426+
expect(data[0].localized).toBe(hebrewLocalized)
2427+
2428+
await payload.delete({ collection: 'pages', id: page.id })
2429+
})
2430+
2431+
it('should preserve Hebrew characters in hook-created CSV export (no jobs queue)', async () => {
2432+
const hebrewTitle = 'Hebrew Hooks Test'
2433+
const hebrewContent = 'טקסט בעברית'
2434+
2435+
const post = await payload.create({
2436+
collection: 'posts',
2437+
data: {
2438+
title: hebrewTitle,
2439+
content: richTextData,
2440+
_status: 'published',
2441+
},
2442+
})
2443+
2444+
const doc = await payload.create({
2445+
collection: 'posts-export',
2446+
user,
2447+
data: {
2448+
collectionSlug: 'posts',
2449+
fields: ['id', 'title'],
2450+
format: 'csv',
2451+
where: { id: { equals: post.id } },
2452+
},
2453+
})
2454+
2455+
const exportDoc = await payload.findByID({
2456+
collection: 'posts-export',
2457+
id: doc.id,
2458+
})
2459+
2460+
// Verify filename includes collection slug and csv extension
2461+
expect(exportDoc.filename).toContain('-posts')
2462+
expect(exportDoc.filename).toMatch(/\.csv$/)
2463+
expect(exportDoc.mimeType).toContain('charset=utf-8')
2464+
2465+
const csvPath = path.join(dirname, './uploads', exportDoc.filename as string)
2466+
const buffer = fs.readFileSync(csvPath)
2467+
2468+
// Verify UTF-8 BOM
2469+
expect(buffer[0]).toBe(0xef)
2470+
expect(buffer[1]).toBe(0xbb)
2471+
expect(buffer[2]).toBe(0xbf)
2472+
2473+
// Verify Hebrew title is correctly encoded
2474+
const content = buffer.toString('utf-8')
2475+
expect(content).toContain(hebrewTitle)
2476+
2477+
// Verify via CSV parse
2478+
const data = await readCSV(csvPath)
2479+
expect(data[0].title).toBe(hebrewTitle)
2480+
2481+
await payload.delete({ collection: 'posts', id: post.id })
2482+
})
23252483
})
23262484

23272485
describe('fields', () => {

test/plugin-import-export/payload-types.ts

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -123,10 +123,18 @@ export interface Config {
123123
db: {
124124
defaultIDType: string;
125125
};
126-
fallbackLocale: ('false' | 'none' | 'null') | false | null | ('en' | 'es' | 'de') | ('en' | 'es' | 'de')[];
126+
fallbackLocale:
127+
| ('false' | 'none' | 'null')
128+
| false
129+
| null
130+
| ('en' | 'es' | 'de' | 'he')
131+
| ('en' | 'es' | 'de' | 'he')[];
127132
globals: {};
128133
globalsSelect: {};
129-
locale: 'en' | 'es' | 'de';
134+
locale: 'en' | 'es' | 'de' | 'he';
135+
widgets: {
136+
collections: CollectionsWidget;
137+
};
130138
user: User;
131139
jobs: {
132140
tasks: {
@@ -500,7 +508,7 @@ export interface Export {
500508
page?: number | null;
501509
sort?: string | null;
502510
sortOrder?: ('asc' | 'desc') | null;
503-
locale?: ('all' | 'en' | 'es' | 'de') | null;
511+
locale?: ('all' | 'en' | 'es' | 'de' | 'he') | null;
504512
drafts?: ('yes' | 'no') | null;
505513
selectionToUse?: ('currentSelection' | 'currentFilters' | 'all') | null;
506514
fields?: string[] | null;
@@ -538,7 +546,7 @@ export interface PostsExport {
538546
page?: number | null;
539547
sort?: string | null;
540548
sortOrder?: ('asc' | 'desc') | null;
541-
locale?: ('all' | 'en' | 'es' | 'de') | null;
549+
locale?: ('all' | 'en' | 'es' | 'de' | 'he') | null;
542550
drafts?: ('yes' | 'no') | null;
543551
selectionToUse?: ('currentSelection' | 'currentFilters' | 'all') | null;
544552
fields?: string[] | null;
@@ -576,7 +584,7 @@ export interface PostsNoJobsQueueExport {
576584
page?: number | null;
577585
sort?: string | null;
578586
sortOrder?: ('asc' | 'desc') | null;
579-
locale?: ('all' | 'en' | 'es' | 'de') | null;
587+
locale?: ('all' | 'en' | 'es' | 'de' | 'he') | null;
580588
drafts?: ('yes' | 'no') | null;
581589
selectionToUse?: ('currentSelection' | 'currentFilters' | 'all') | null;
582590
fields?: string[] | null;
@@ -614,7 +622,7 @@ export interface PostsWithS3Export {
614622
page?: number | null;
615623
sort?: string | null;
616624
sortOrder?: ('asc' | 'desc') | null;
617-
locale?: ('all' | 'en' | 'es' | 'de') | null;
625+
locale?: ('all' | 'en' | 'es' | 'de' | 'he') | null;
618626
drafts?: ('yes' | 'no') | null;
619627
selectionToUse?: ('currentSelection' | 'currentFilters' | 'all') | null;
620628
fields?: string[] | null;
@@ -652,7 +660,7 @@ export interface PostsWithLimitsExport {
652660
page?: number | null;
653661
sort?: string | null;
654662
sortOrder?: ('asc' | 'desc') | null;
655-
locale?: ('all' | 'en' | 'es' | 'de') | null;
663+
locale?: ('all' | 'en' | 'es' | 'de' | 'he') | null;
656664
drafts?: ('yes' | 'no') | null;
657665
selectionToUse?: ('currentSelection' | 'currentFilters' | 'all') | null;
658666
fields?: string[] | null;
@@ -1561,6 +1569,16 @@ export interface PayloadMigrationsSelect<T extends boolean = true> {
15611569
updatedAt?: T;
15621570
createdAt?: T;
15631571
}
1572+
/**
1573+
* This interface was referenced by `Config`'s JSON-Schema
1574+
* via the `definition` "collections_widget".
1575+
*/
1576+
export interface CollectionsWidget {
1577+
data?: {
1578+
[k: string]: unknown;
1579+
};
1580+
width: 'full';
1581+
}
15641582
/**
15651583
* This interface was referenced by `Config`'s JSON-Schema
15661584
* via the `definition` "TaskCreateCollectionExport".

test/plugin-import-export/seed/index.ts

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,26 @@ export const seed = async (payload: Payload): Promise<boolean> => {
9090
},
9191
locale: 'es',
9292
})
93+
await payload.update({
94+
collection: 'pages',
95+
id: doc.id,
96+
data: {
97+
localized: `בדיקה ${i}`,
98+
},
99+
locale: 'he',
100+
})
101+
}
102+
103+
// Seed Hebrew-only pages
104+
for (let i = 0; i < 5; i++) {
105+
await payload.create({
106+
collection: 'pages',
107+
data: {
108+
title: `Hebrew ${i}`,
109+
localized: `בדיקה ${i}`,
110+
},
111+
locale: 'he',
112+
})
93113
}
94114
for (let i = 0; i < 5; i++) {
95115
await payload.create({

0 commit comments

Comments
 (0)