This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-website.git
The following commit(s) were added to refs/heads/master by this push:
new f182044ff23c [build](i18n) Isolate native compiler memory without
changing site output (#4537)
f182044ff23c is described below
commit f182044ff23c1fb0c4a8d5d3016857d8a2ddd114
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Wed Oct 28 00:24:50 2026 +0000
[build](i18n) Isolate native compiler memory without changing site output
(#4537)
The full five-language production build reached 14.36 GiB RSS and left
only about 344 MiB of available RAM, despite a 6 GiB V8 heap limit. The
peak occurs while Docusaurus runs its client and server Rspack compilers
together; closed compilers in the pinned native implementation remain
reachable.
Build the client in a disposable process, wait for its successful exit,
then compile the server and render pages in the locale process. Transfer
the exact client manifest and client MDX results with checksums and
source-stamp checks. This preserves Docusaurus 3.6.3's client-derived
MDX asset semantics and avoids a server loader waiting for an in-process
client promise. Reuse the existing content-addressed structure cache
only within a fresh, fixed-source build directory to limit repeated
parsing.
Keep atomic output assembly, all languages and active versions, strict
verification, and the existing client minifier. Record separate compiler
PIDs, phase memory and client exit ordering. Synchronize cron/manual
settings and archive compiler metrics. Sampling retains the actual
client compilation proof alongside server checks.
Validation:
- 407 Node tests: 405 passed, 2 conditional skips; 27 workflow tests
passed.
- Same-source real five-language Rspack comparison: 325 emitted files
(3.67 MB) are byte-identical between the previous parallel pipeline and
process isolation, including docs, blog excerpts/full posts and a local
image.
- Publishing fixture/browser checks, 36 native search/recovery cases and
the five-language sampling fixture passed. The sampling fixture still
rejects changed broken docs/blog modules and excludes unselected
modules.
- No local full website compilation. The next complete production run
must establish the actual memory headroom and final time budget; this PR
does not claim that measurement has already passed.
Self-review against AGENTS.md:
| Checkpoint | Conclusion |
| --- | --- |
| Goal | Addresses overlapping native compiler lifetimes; client process
exit is checked before the server starts. Full-scale resource acceptance
remains pending. |
| Focus | Pinned build adapter, transient compiler state, measurements
and focused regression fixtures. |
| Information architecture | Prose, translation ownership, active
versions, plugins and sidebars are unchanged. |
| Paths/navigation | Same-source emitted artifacts are byte-identical;
no document URL changes. |
| Frontend | Existing client minification, locale prefixes, worker
origin and theme behavior are preserved; publishing/search browser
fixtures passed. |
| Validation | Real native artifact equality, failure/manifest/cache
tests and actual compilation-proof checks cover the process boundary. |
| Other quality | No dependency upgrades or permission/secret expansion;
failed client/source/checksum validation stops before server compilation
or publication. |
Co-authored-by: morningman <[email protected]>
---
.github/scripts/publish_metrics.py | 2 +-
.github/workflows/build-check.yml | 6 +
.github/workflows/cron-deploy-website.yml | 4 +-
.github/workflows/manual-deploy-website.yml | 4 +-
scripts/build-locale.cjs | 30 ++++-
scripts/build-locales.cjs | 15 ++-
scripts/i18n/__tests__/compiler-processes.test.cjs | 64 +++++++++
scripts/i18n/ci-plan.cjs | 9 +-
scripts/i18n/compiler-processes.cjs | 148 +++++++++++++++++++++
scripts/i18n/verify-compiler-fixture.cjs | 62 +++++++++
10 files changed, 331 insertions(+), 13 deletions(-)
diff --git a/.github/scripts/publish_metrics.py
b/.github/scripts/publish_metrics.py
index ca125062ee1d..af7529129a61 100644
--- a/.github/scripts/publish_metrics.py
+++ b/.github/scripts/publish_metrics.py
@@ -67,7 +67,7 @@ def start(job):
"GITHUB_WORKFLOW_REF", "GITHUB_EVENT_NAME", "RUNNER_OS", "RUNNER_ARCH",
"ImageOS", "ImageVersion", "DEPLOY_SOURCE_SHA", "DORIS_ENABLE_JA",
"DORIS_ENABLE_KO", "DORIS_ENABLE_RU", "DOCS_VERSIONS", "NODE_OPTIONS",
- "TERSER_PARALLEL", "RAYON_NUM_THREADS", "DOCUSAURUS_SSR_CONCURRENCY")}
+ "TERSER_PARALLEL", "RAYON_NUM_THREADS", "DOCUSAURUS_SSR_CONCURRENCY",
"DORIS_COMPILER_MODE")}
write_json(REPORTS / "context.json", {
"schemaVersion": 1, "job": job, "startedAt": time.time(),
"checkoutSha": git("rev-parse", "HEAD"), "context": context,
diff --git a/.github/workflows/build-check.yml
b/.github/workflows/build-check.yml
index f8fecc3c6f85..a9873df6e599 100644
--- a/.github/workflows/build-check.yml
+++ b/.github/workflows/build-check.yml
@@ -132,6 +132,10 @@ jobs:
node --test scripts/publish/__tests__/*.test.cjs
node scripts/publish/verify-fixture.cjs
+ - name: Verify isolated compiler outputs
+ if: steps.detect.outputs.compiler_fixture == 'true'
+ run: node scripts/i18n/verify-compiler-fixture.cjs
+
- name: Verify native content manifest fixture
if: steps.detect.outputs.content_fixture == 'true'
run: |
@@ -267,6 +271,8 @@ jobs:
.i18n-build/ci-sample/verification.json
.i18n-build/ci-sampling-fixture-proof.json
.i18n-build/search-transport-verification.json
+ .i18n-build/compiler-fixture-verification.json
+ .i18n-build/compiler-metrics/*.json
.i18n-build/publish-result.json
.i18n-build/build-result.json
.i18n-build/content-fixture-verification.json
diff --git a/.github/workflows/cron-deploy-website.yml
b/.github/workflows/cron-deploy-website.yml
index 44f0cd32a993..4c348c1fcb01 100644
--- a/.github/workflows/cron-deploy-website.yml
+++ b/.github/workflows/cron-deploy-website.yml
@@ -55,7 +55,8 @@ jobs:
DORIS_ENABLE_RU: 'true'
NODE_OPTIONS: '--max-old-space-size=6144'
TERSER_PARALLEL: '2'
- RAYON_NUM_THREADS: '2'
+ RAYON_NUM_THREADS: '4'
+ DORIS_COMPILER_MODE: 'isolated'
DOCUSAURUS_SSR_CONCURRENCY: '8'
DOCUSAURUS_PERF_LOGGER: 'true'
DORIS_BUILD_METRICS: 'true'
@@ -205,6 +206,7 @@ jobs:
.i18n-build/manual-drift.json
.i18n-build/coverage.json
.i18n-build/content-manifests/*.json
+ .i18n-build/compiler-metrics/*.json
if-no-files-found: error
retention-days: 14
diff --git a/.github/workflows/manual-deploy-website.yml
b/.github/workflows/manual-deploy-website.yml
index b433d7b49e5f..859e4e88b946 100644
--- a/.github/workflows/manual-deploy-website.yml
+++ b/.github/workflows/manual-deploy-website.yml
@@ -50,7 +50,8 @@ jobs:
DORIS_ENABLE_RU: 'true'
NODE_OPTIONS: '--max-old-space-size=6144'
TERSER_PARALLEL: '2'
- RAYON_NUM_THREADS: '2'
+ RAYON_NUM_THREADS: '4'
+ DORIS_COMPILER_MODE: 'isolated'
DOCUSAURUS_SSR_CONCURRENCY: '8'
DOCUSAURUS_PERF_LOGGER: 'true'
DORIS_BUILD_METRICS: 'true'
@@ -178,6 +179,7 @@ jobs:
.i18n-build/manual-drift.json
.i18n-build/coverage.json
.i18n-build/content-manifests/*.json
+ .i18n-build/compiler-metrics/*.json
if-no-files-found: error
retention-days: 14
diff --git a/scripts/build-locale.cjs b/scripts/build-locale.cjs
index d83de9a42d11..30e9a05b88fe 100644
--- a/scripts/build-locale.cjs
+++ b/scripts/build-locale.cjs
@@ -24,10 +24,32 @@ async function main() {
directory:locale === i18n.defaultLocale ? '.' : locale,
baseUrl:localizePath({path:siteConfig.baseUrl, pathType:'url',
i18n:{...i18n,currentLocale:locale}}),
}))}, null, 2) + '\n');
- } else if (mode === 'build' || mode === 'sample') {
- const restore = mode === 'sample' ? await
require('./i18n/ci-sample.cjs').install({root:siteDir,outDir,locale:value,planFile:samplePlan})
: () => {};
- const {buildLocale} =
fromSite('@docusaurus/core/lib/commands/build/buildLocale.js');
- try {await buildLocale({siteDir, locale:value, cliOptions:{outDir}});}
finally {restore();}
+ } else if (mode === 'build' || mode === 'sample' || mode === 'client') {
+ const adapter = require('./i18n/compiler-processes.cjs');
+ const {sourceStamp} = require('./build-locales.cjs');
+ const stamp = sourceStamp(siteDir);
+ const compilerMode = process.env.DORIS_COMPILER_MODE ?? 'isolated';
+ if (!['isolated','parallel'].includes(compilerMode)) throw new
Error(`Unknown DORIS_COMPILER_MODE: ${compilerMode}`);
+ const clientResult = mode !== 'client' && compilerMode === 'isolated'
+ ? await
adapter.runClient({siteDir,outDir,locale:value,samplePlan:mode === 'sample' ?
samplePlan : null}) : null;
+ if (clientResult && JSON.stringify(clientResult.source) !==
JSON.stringify(sourceStamp(siteDir))) throw new Error('Source changed during
isolated client compilation');
+ const restore = (mode === 'sample' || mode === 'client' && samplePlan)
? await
require('./i18n/ci-sample.cjs').install({root:siteDir,outDir,locale:value,planFile:samplePlan})
: () => {};
+ let restoreCompiler = () => {};
+ let result;
+ try {
+ if (mode === 'client') result = await
adapter.buildClient({fromSite,siteDir,outDir,locale:value,samplePlan});
+ else {
+ restoreCompiler =
adapter.install({fromSite,siteDir,locale:value,clientResult});
+ const {buildLocale} =
fromSite('@docusaurus/core/lib/commands/build/buildLocale.js');
+ await buildLocale({siteDir, locale:value,
cliOptions:{outDir}});
+ }
+ } finally {try {restoreCompiler();} finally {restore();}}
+ if (JSON.stringify(stamp) !== JSON.stringify(sourceStamp(siteDir)))
throw new Error('Source changed between compiler phases');
+ if (mode === 'client')
adapter.writeClientResult(siteDir,value,{...result,source:stamp});
+ else {
+ if(mode === 'sample' &&
clientResult)adapter.mergeSamplingScope(siteDir,value,clientResult);
+
fs.rmSync(adapter.stateDirectory(siteDir,value),{recursive:true,force:true});
+ }
} else throw new Error(`Unknown locale build mode: ${mode}`);
}
main().catch(error => {console.error(error); process.exitCode = 1;});
diff --git a/scripts/build-locales.cjs b/scripts/build-locales.cjs
index 74f62979392d..e4cd004c344d 100644
--- a/scripts/build-locales.cjs
+++ b/scripts/build-locales.cjs
@@ -43,11 +43,12 @@ async function buildLocales({siteDir = process.cwd(),
outDir = 'build', locales:
if(samplePlan)report.validation={mode:'sample',publicationAllowed:false};
const reportDir = path.join(siteDir, '.i18n-build');
fs.mkdirSync(reportDir, {recursive:true});
+
fs.rmSync(path.join(reportDir,'compiler-metrics'),{recursive:true,force:true});
const reportPath = path.join(reportDir, 'build-result.json');
const execute = run ?? ((args) => new Promise((resolve, reject) => {
const child =
spawn(process.execPath,[path.join(__dirname,'build-locale.cjs'), ...args],{
cwd:siteDir,stdio:'inherit',env:{...process.env,
- ...(samplePlan &&
{DORIS_I18N_STRUCTURE_CACHE:path.join(stage,'structure-cache')})},
+ ...((samplePlan || process.env.DORIS_COMPILER_MODE !==
'parallel') &&
{DORIS_I18N_STRUCTURE_CACHE:path.join(stage,'structure-cache')})},
});
console.log(JSON.stringify({event:'locale_process_start',
mode:args[0], locale:args[0] === 'plan' ? null : args[3], pid:child.pid}));
const stop = signal => child.kill(signal);
@@ -79,7 +80,15 @@ async function buildLocales({siteDir = process.cwd(), outDir
= 'build', locales:
cleanup();
console.log(JSON.stringify({event:'locale_process_end',
mode:args[0], locale:args[0] === 'plan' ? null : args[3], pid:child.pid, code,
signal}));
if (code !== 0) reject(new Error(`Locale ${args[3]} failed
(${signal ?? code}); output was not published`));
- else resolve({pid:child.pid,...(memory && {memory})});
+ else {
+ const
compilerFile=path.join(reportDir,'compiler-metrics',args[3]+'.json');
+ if(memory && fs.existsSync(compilerFile))for(const record of
JSON.parse(fs.readFileSync(compilerFile)).records??[]){
+
memory.peakRssKiB=Math.max(memory.peakRssKiB,Math.ceil(record.peakRssBytes/1024));
+
if(record.minimumSystemAvailableKiB)memory.minimumSystemAvailableKiB=Math.min(memory.minimumSystemAvailableKiB??Infinity,record.minimumSystemAvailableKiB);
+ }
+ if(memory)memory.scope='Maximum sampled individual process RSS
across locale and isolated client; system availability includes all processes';
+ resolve({pid:child.pid,...(memory && {memory})});
+ }
});
}));
try {
@@ -153,4 +162,4 @@ if (require.main === module) {
locales:localeList === undefined ? undefined :
localeList.split(',').map(locale => locale.trim()).filter(Boolean)})
.catch(error => {console.error(error);process.exitCode = 1;});
}
-module.exports = {buildLocales, selectLocales};
+module.exports = {buildLocales, selectLocales, sourceStamp};
diff --git a/scripts/i18n/__tests__/compiler-processes.test.cjs
b/scripts/i18n/__tests__/compiler-processes.test.cjs
new file mode 100644
index 000000000000..1fffbd02ef0d
--- /dev/null
+++ b/scripts/i18n/__tests__/compiler-processes.test.cjs
@@ -0,0 +1,64 @@
+const test=require('node:test');
+const assert=require('node:assert/strict');
+const
{createServerCompile,snapshotMdx,restoreMdx,combineSamplingScopes}=require('../compiler-processes.cjs');
+const
request=()=>({configs:[{name:'client'},{name:'server'}],currentBundler:{name:'rspack'},other:'unchanged'});
+test('parent restores the isolated client manifest before compiling only the
server',async()=>{
+ const calls=[],input=request(),stats={};
+ const
compile=createServerCompile({restoreManifest:value=>{assert.equal(value,input);calls.push('restore');},compile:async
value=>{
+
assert.deepEqual(value.configs,[input.configs[1]]);assert.equal(value.other,'unchanged');calls.push('server');return
stats;
+ },measureServer:async callback=>{calls.push('measure');return
callback();}});
+ assert.equal(await
compile(input),stats);assert.deepEqual(calls,['restore','measure','server']);
+ assert.deepEqual(input.configs.map(c=>c.name),['client','server']);
+});
+test('invalid or changed manifest stops before allocating the server
compiler',async()=>{
+ let calls=0;const error=new Error('manifest changed');
+ const compile=createServerCompile({restoreManifest:()=>{throw
error;},compile:()=>calls++});
+ await
assert.rejects(compile(request()),value=>value===error);assert.equal(calls,0);
+});
+test('server failures propagate without a second client compilation',async()=>{
+ const calls=[],error=new Error('server failed');
+ const compile=createServerCompile({restoreManifest:()=>{},compile:async
value=>{calls.push(value.configs.map(c=>c.name));throw error;}});
+ await
assert.rejects(compile(request()),value=>value===error);assert.deepEqual(calls,[['server']]);
+});
+test('hash-router client output is retained without allocating a
server',async()=>{
+ const calls=[];const
compile=createServerCompile({restoreManifest:()=>calls.push('restore'),compile:()=>calls.push('unexpected')});
+ await
compile({...request(),configs:[{name:'client'}]});assert.deepEqual(calls,['restore']);
+});
+test('unknown compiler shapes fail before manifest restoration',async()=>{
+ let calls=0;const
compile=createServerCompile({restoreManifest:()=>calls++,compile:()=>calls++});
+ await
assert.rejects(compile({...request(),configs:[{name:'server'},{name:'client'}]}),/Revalidate/);assert.equal(calls,0);
+});
+test('compiler mechanism changes select native equality, publishing and search
fixtures',()=>{
+ const {policy}=require('../ci-plan.cjs');
+ for(const file of
['scripts/build-locale.cjs','scripts/build-locales.cjs','scripts/i18n/compiler-processes.cjs']){
+ const
plan=policy([file]);assert(plan.runCompilerFixture);assert(plan.runSearchFixture);assert(plan.runTranslationTests);assert(plan.runPublishFixture);assert(plan.runSamplingFixture);
+ }
+
assert(!policy(['src/theme/SearchBar/SearchBar.module.css']).runCompilerFixture);
+});
+test('MDX transfer preserves full/truncated resource keys and client asset
loader code',async()=>{
+ const full='export const assets =
require("image?emit=true");',excerpt='truncated client JSX';
+ const config={module:{rules:[{use:[{options:{crossCompilerCache:new Map([
+
['/blog/post.mdx',{promise:Promise.resolve(full)}],['/blog/post.mdx?truncated',{promise:Promise.resolve(excerpt)}],
+ ])}}]}]}};
+ const bytes=await snapshotMdx(config);
+ const options={crossCompilerCache:new
Map()},server={module:{rules:[{oneOf:[{use:{options}}]}]}};
+ restoreMdx(server,bytes);
+ assert.equal(await
options.crossCompilerCache.get('/blog/post.mdx').promise,full);
+ assert.equal(await
options.crossCompilerCache.get('/blog/post.mdx?truncated').promise,excerpt);
+ assert.throws(()=>options.crossCompilerCache.get('/missing.mdx'),/Missing
isolated/);
+});
+test('conflicting client MDX variants fail instead of silently selecting a
plugin',async()=>{
+ const entry=code=>({options:{crossCompilerCache:new
Map([['/same.mdx',{promise:Promise.resolve(code)}]])}});
+ await
assert.rejects(snapshotMdx({module:{rules:[{use:[entry('one'),entry('other')]}]}}),/Conflicting/);
+});
+test('malformed MDX snapshots fail before server compilation',()=>{
+
assert.throws(()=>restoreMdx({module:{rules:[]}},Buffer.from('{"invalid":true}')),/Invalid
isolated/);
+});
+test('sample proof preserves actual client modules and server-side checks for
the same selection',()=>{
+ const
base={sourceSha:'a'.repeat(40),locale:'ja',selected:[{pluginId:'default',version:'current',docId:'one',route:'/ja/docs/dev/one/'}]};
+ const
server={...base,anchorChecksSkipped:2,compiledMdxModules:{server:['docs/one.mdx']}},client={...base,compiledMdxModules:{client:['docs/one.mdx','blog/post.mdx']}};
+ const
merged=combineSamplingScopes(server,client);assert.equal(merged.anchorChecksSkipped,2);assert.deepEqual(merged.compiledMdxModules,{...server.compiledMdxModules,...client.compiledMdxModules});
+
assert.throws(()=>combineSamplingScopes(server,{...client,sourceSha:'b'.repeat(40)}),/scopes
differ/);
+
assert.throws(()=>combineSamplingScopes(server,{...client,selected:[]}),/scopes
differ/);
+ assert.throws(()=>combineSamplingScopes(server,{...base}),/proof is
missing/);
+});
diff --git a/scripts/i18n/ci-plan.cjs b/scripts/i18n/ci-plan.cjs
index 44a1883820d8..b02eae80433c 100644
--- a/scripts/i18n/ci-plan.cjs
+++ b/scripts/i18n/ci-plan.cjs
@@ -68,10 +68,12 @@ function policy(files, {forceFull = false, forceSample =
false} = {}) {
/verify-(content-fixture|global-language)|__tests__\/fixtures\/content-manifest/.test(file),
),
runSamplingFixture: files.some(file =>
-
/^scripts\/(build-locales?\.cjs$|i18n\/(ci-sample|ci-blog-sample|build-cache|verify-ci-sampl(?:e|ing-fixture))\.cjs$)/.test(file)),
+
/^scripts\/(build-locales?\.cjs$|i18n\/(compiler-processes|ci-sample|ci-blog-sample|build-cache|verify-ci-sampl(?:e|ing-fixture))\.cjs$)/.test(file)),
+ runCompilerFixture: mode === 'full' || files.some(file =>
+
/^scripts\/(build-locales?\.cjs|i18n\/(compiler-processes|verify-compiler-fixture)\.cjs)$/.test(file)),
runSearchFixture: mode === 'full' || files.some(file =>
-
/^(src\/theme\/(SearchBar|SearchPage)\/|src\/utils\/(fetchSearchIndex|searchMessages|searchScopes|useLanguageItems)\.[jt]sx?$|src\/hooks\/use-search-query\.ts$|docusaurus\.config\.js$|scripts\/i18n\/(search-plugin|search-worker-loader|verify-search-transport)\.cjs$)/.test(file)),
- runPublishFixture: checks.check_publish_fixture || mode === 'full',
+
/^(src\/theme\/(SearchBar|SearchPage)\/|src\/utils\/(fetchSearchIndex|searchMessages|searchScopes|useLanguageItems)\.[jt]sx?$|src\/hooks\/use-search-query\.ts$|docusaurus\.config\.js$|scripts\/build-locales?\.cjs$|scripts\/i18n\/(compiler-processes|search-plugin|search-worker-loader|verify-search-transport)\.cjs$)/.test(file)),
+ runPublishFixture: checks.check_publish_fixture || mode === 'full' ||
files.includes('scripts/i18n/compiler-processes.cjs'),
runTranslationTests: mode === 'full' || files.some(file =>
runtime.test(file) || mechanism.test(file) ||
maintenanceConfig(file) || /^scripts\/(i18n\/|build-locales?\.cjs$)/.test(file)
|| file === '.github/workflows/build-check.yml'),
@@ -186,6 +188,7 @@ if (require.main === module) {
publish_fixture: result.runPublishFixture,
sampling_fixture: result.runSamplingFixture,
search_fixture: result.runSearchFixture,
+ compiler_fixture: result.runCompilerFixture,
translation_tests: result.runTranslationTests,
docs_report: result.runDocsReport,
})
diff --git a/scripts/i18n/compiler-processes.cjs
b/scripts/i18n/compiler-processes.cjs
new file mode 100644
index 000000000000..dd5e6127e873
--- /dev/null
+++ b/scripts/i18n/compiler-processes.cjs
@@ -0,0 +1,148 @@
+// Pinned Docusaurus compiler adapter. Rspack 1.2's closed native compiler is
+// still reachable; only process exit gives this build a reliable memory bound.
+const fs = require('node:fs');
+const path = require('node:path');
+const {createHash} = require('node:crypto');
+const {createRequire} = require('node:module');
+const {spawn} = require('node:child_process');
+const digest = bytes => createHash('sha256').update(bytes).digest('hex');
+function mdxCaches(config) {
+ const caches=[];
+ function inspect(item){
+ if(!item||typeof item!=='object')return;
+ if(item.options?.crossCompilerCache instanceof
Map)caches.push(item.options);
+ for(const key of ['rules','oneOf','use']){
+ const
value=item[key];if(Array.isArray(value))value.forEach(inspect);else
inspect(value);
+ }
+ }
+ (config.module?.rules??[]).forEach(inspect);return caches;
+}
+async function snapshotMdx(config) {
+ const entries=new Map();
+ for(const options of mdxCaches(config))for(const [resource,entry]of
options.crossCompilerCache){
+ const code=await entry.promise;
+ if(typeof code!=='string')throw new Error(`Unexpected compiled MDX
value: ${resource}`);
+ if(entries.has(resource)&&entries.get(resource)!==code)throw new
Error(`Conflicting compiled MDX resources: ${resource}`);
+ entries.set(resource,code);
+ }
+ return
Buffer.from(JSON.stringify([...entries].sort(([a],[b])=>a.localeCompare(b))));
+}
+function restoreMdx(config,bytes) {
+ const entries=JSON.parse(bytes);
+
if(!Array.isArray(entries)||entries.some(row=>!Array.isArray(row)||row.length!==2||row.some(value=>typeof
value!=='string')))throw new Error('Invalid isolated MDX snapshot');
+ for(const loader of mdxCaches(config)){
+ const cache=new
Map(entries.map(([resource,code])=>[resource,{promise:Promise.resolve(code)}]));
+ cache.get=function(resource){if(!this.has(resource))throw new
Error(`Missing isolated client MDX resource: ${resource}`);return
Map.prototype.get.call(this,resource);};
+ loader.crossCompilerCache=cache;
+ }
+}
+const stateDirectory = (siteDir,locale) =>
path.join(siteDir,'.i18n-build/compiler-state',locale);
+function reportFile(siteDir,locale) {return
path.join(siteDir,'.i18n-build/compiler-metrics',locale+'.json');}
+function updateReport(siteDir,locale,update) {
+ const
file=reportFile(siteDir,locale);fs.mkdirSync(path.dirname(file),{recursive:true});
+ const
value=fs.existsSync(file)?JSON.parse(fs.readFileSync(file)):{schemaVersion:1,locale,mode:process.env.DORIS_COMPILER_MODE??'isolated',records:[]};
+
update(value);fs.writeFileSync(file,JSON.stringify(value,null,2)+'\n');return
value;
+}
+function coreModules(fromSite) {
+ const
requireCore=createRequire(fromSite.resolve('@docusaurus/core/lib/commands/build/buildLocale.js'));
+ for(const name of
['@docusaurus/core','@docusaurus/bundler','@docusaurus/mdx-loader']) {
+ const version=requireCore(name+'/package.json').version;
+ if(version!=='3.6.3')throw new Error(`Revalidate isolated compilation
for ${name} ${version}`);
+ }
+ return requireCore;
+}
+async function measure({siteDir,locale,name},compile) {
+ const startedAtMs=Date.now(),before=process.memoryUsage();let
success=false,samples=0,peakRssBytes=before.rss,minimumSystemAvailableKiB=null;
+ const
sample=()=>{samples++;peakRssBytes=Math.max(peakRssBytes,process.memoryUsage().rss);
+ try{const
available=Number(fs.readFileSync('/proc/meminfo','utf8').match(/^MemAvailable:\s+(\d+)/m)?.[1]);if(available>0)minimumSystemAvailableKiB=Math.min(minimumSystemAvailableKiB??Infinity,available);}catch{}
+ };
+ const timer=setInterval(sample,1000);timer.unref();sample();
+ try{const result=await compile();success=true;return result;}
+ finally{sample();clearInterval(timer);const
record={name,pid:process.pid,success,startedAtMs,finishedAtMs:Date.now(),sampleIntervalMs:1000,samples,peakRssBytes,minimumSystemAvailableKiB,before,afterClose:process.memoryUsage()};
+
updateReport(siteDir,locale,value=>value.records.push(record));console.log(JSON.stringify({event:'compiler_phase_end',locale,...record}));}
+}
+async function buildClient({fromSite,siteDir,outDir,locale,samplePlan}) {
+ const requireCore=coreModules(fromSite);
+ process.env.DOCUSAURUS_CURRENT_LOCALE=locale;
+ const {loadSite}=requireCore('@docusaurus/core/lib/server/site.js');
+ const
{createConfigureWebpackUtils,executePluginsConfigureWebpack}=requireCore('@docusaurus/core/lib/webpack/configure.js');
+ const
{createBuildClientConfig}=requireCore('@docusaurus/core/lib/webpack/client.js');
+ const {props}=await
loadSite({siteDir,outDir,locale,localizePath:undefined});
+ const utils=await
createConfigureWebpackUtils({siteConfig:props.siteConfig});
+ const {config:initial,clientManifestPath}=await
createBuildClientConfig({props,minify:true,faster:props.siteConfig.future.experimental_faster,configureWebpackUtils:utils,bundleAnalyzer:false});
+ const
config=executePluginsConfigureWebpack({plugins:props.plugins,config:initial,isServer:false,configureWebpackUtils:utils});
+
updateReport(siteDir,locale,value=>{value.bundler=utils.currentBundler.name;value.bundlerVersion='3.6.3';});
+ await
measure({siteDir,locale,name:'client'},()=>requireCore('@docusaurus/bundler').compile({configs:[config],currentBundler:utils.currentBundler}));
+ const
bytes=fs.readFileSync(clientManifestPath),directory=stateDirectory(siteDir,locale);fs.mkdirSync(directory,{recursive:true});
+ fs.writeFileSync(path.join(directory,'client-manifest.json'),bytes);
+ // The pinned MDX loader waits for client promises when compiling the
+ // server. Preserve their exact client JSX instead of recompiling in server
+ // mode, which would change inline image-loader behavior.
+ const mdx=await snapshotMdx(config),entryCount=JSON.parse(mdx).length;
+ fs.writeFileSync(path.join(directory,'client-mdx.json'),mdx);
+
console.log(JSON.stringify({event:'client_mdx_snapshot',locale,entries:entryCount,bytes:mdx.length}));
+ let scopeSha256;
+ if(samplePlan){const
scope=fs.readFileSync(path.join(siteDir,'.i18n-build/ci-sample/scopes',locale+'.json'));fs.writeFileSync(path.join(directory,'client-scope.json'),scope);scopeSha256=digest(scope);}
+ return
{clientManifestPath,manifestSha256:digest(bytes),mdxSha256:digest(mdx),mdxEntries:entryCount,bundler:utils.currentBundler.name,scopeSha256};
+}
+function
createServerCompile({compile,restoreManifest,measureServer=callback=>callback()})
{
+ return async options=>{
+ const names=options.configs.map(config=>config.name);
+
if(names.length>2||names[0]!=='client'||names.length===2&&names[1]!=='server')throw
new Error(`Revalidate the isolated compiler sequence: ${names.join(',')}`);
+ restoreManifest(options);
+ // The known hash-router path has no server compiler or SSG bundle.
+ if(names.length===1)return;
+ return
measureServer(()=>compile({...options,configs:[options.configs[1]]}));
+ };
+}
+function install({fromSite,siteDir,locale,clientResult}) {
+ const
requireCore=coreModules(fromSite),compiler=requireCore('@docusaurus/bundler/lib/compiler.js'),namespace=requireCore('@docusaurus/bundler');
+ const original=compiler.compile;
+ const
wrapped=clientResult?createServerCompile({compile:original,restoreManifest:options=>{
+ if(options.currentBundler.name!==clientResult.bundler)throw new
Error('Client/server bundler changed between isolated processes');
+ const generated=options.configs[0].resolve?.alias?.['@generated'];
+ if(typeof
generated!=='string'||path.resolve(generated,'client-manifest.json')!==path.resolve(clientResult.clientManifestPath))throw
new Error('Client manifest path changed between isolated processes');
+ const
bytes=fs.readFileSync(path.join(stateDirectory(siteDir,locale),'client-manifest.json'));
+ if(digest(bytes)!==clientResult.manifestSha256)throw new
Error('Isolated client manifest changed before SSG');
+
fs.mkdirSync(path.dirname(clientResult.clientManifestPath),{recursive:true});fs.writeFileSync(clientResult.clientManifestPath,bytes);
+ const
mdx=fs.readFileSync(path.join(stateDirectory(siteDir,locale),'client-mdx.json'));
+ if(digest(mdx)!==clientResult.mdxSha256)throw new Error('Isolated
client MDX changed before server compilation');
+ restoreMdx(options.configs[1]??options.configs[0],mdx);
+
},measureServer:callback=>measure({siteDir,locale,name:'server'},callback)})
+
:options=>measure({siteDir,locale,name:options.configs.map(c=>c.name).join('+')},()=>original(options));
+ compiler.compile=wrapped;
+ if(namespace.compile!==wrapped){compiler.compile=original;throw new
Error('Docusaurus compile export no longer references the pinned adapter');}
+ return()=>{compiler.compile=original;};
+}
+async function runClient({siteDir,outDir,locale,samplePlan}) {
+ const
directory=stateDirectory(siteDir,locale);fs.rmSync(directory,{recursive:true,force:true});
+
updateReport(siteDir,locale,value=>{value.mode='isolated';value.records=[];});
+ const
args=[path.resolve(__dirname,'../build-locale.cjs'),'client',siteDir,outDir,locale,...(samplePlan?[samplePlan]:[])];
+ let signal=null;
+ await new Promise((resolve,reject)=>{
+ const
child=spawn(process.execPath,args,{cwd:siteDir,stdio:'inherit',env:process.env});
+
console.log(JSON.stringify({event:'client_compiler_start',locale,pid:child.pid}));
+ const
terminate=()=>{signal='SIGTERM';child.kill(signal);},interrupt=()=>{signal='SIGINT';child.kill(signal);};
+ const
cleanup=()=>{process.removeListener('SIGTERM',terminate);process.removeListener('SIGINT',interrupt);};
+ process.once('SIGTERM',terminate);process.once('SIGINT',interrupt);
+ child.once('error',error=>{cleanup();reject(error);});
+
child.once('exit',(code,childSignal)=>{cleanup();updateReport(siteDir,locale,value=>{value.clientPid=child.pid;value.clientExitedAtMs=Date.now();value.clientExitCode=code;});
+
console.log(JSON.stringify({event:'client_compiler_exit',locale,pid:child.pid,code,signal:childSignal}));
+ if(code!==0||signal)reject(new Error(`Isolated client compiler
failed (${signal??childSignal??code})`));else resolve();});
+ });
+ return JSON.parse(fs.readFileSync(path.join(directory,'result.json')));
+}
+function writeClientResult(siteDir,locale,result)
{fs.writeFileSync(path.join(stateDirectory(siteDir,locale),'result.json'),JSON.stringify(result)+'\n');}
+function combineSamplingScopes(server,client) {
+ const
selection=scope=>JSON.stringify((scope.selected??[]).map(({pluginId,version,docId,route})=>[pluginId,version,docId,route]));
+
if(server.sourceSha!==client.sourceSha||server.locale!==client.locale||selection(server)!==selection(client))throw
new Error('Isolated sample compiler scopes differ');
+ if(!Array.isArray(client.compiledMdxModules?.client))throw new
Error('Isolated sample client compilation proof is missing');
+ return
{...server,compiledMdxModules:{...server.compiledMdxModules,client:client.compiledMdxModules.client}};
+}
+function mergeSamplingScope(siteDir,locale,clientResult) {
+ const
bytes=fs.readFileSync(path.join(stateDirectory(siteDir,locale),'client-scope.json'));
+ if(digest(bytes)!==clientResult.scopeSha256)throw new Error('Isolated
sample client proof changed');
+ const
file=path.join(siteDir,'.i18n-build/ci-sample/scopes',locale+'.json');
+
fs.writeFileSync(file,JSON.stringify(combineSamplingScopes(JSON.parse(fs.readFileSync(file)),JSON.parse(bytes)),null,2)+'\n');
+}
+module.exports={buildClient,createServerCompile,install,runClient,writeClientResult,stateDirectory,mdxCaches,snapshotMdx,restoreMdx,combineSamplingScopes,mergeSamplingScope};
diff --git a/scripts/i18n/verify-compiler-fixture.cjs
b/scripts/i18n/verify-compiler-fixture.cjs
new file mode 100644
index 000000000000..37c734d7b6bc
--- /dev/null
+++ b/scripts/i18n/verify-compiler-fixture.cjs
@@ -0,0 +1,62 @@
+const fs = require('node:fs');
+const path = require('node:path');
+const assert = require('node:assert/strict');
+const {createHash} = require('node:crypto');
+const {execFileSync,spawnSync} = require('node:child_process');
+const {createFixture} = require('./__tests__/fixtures/multi-locale.cjs');
+const root = path.resolve(__dirname,'../..');
+const fixture = path.join(root,'.i18n-build/compiler-fixture');
+function inventory(directory) {
+ const files={};
+ function walk(current){for(const entry of
fs.readdirSync(current,{withFileTypes:true})){
+ const file=path.join(current,entry.name);
+ if(entry.isDirectory())walk(file);
+ else {const
bytes=fs.readFileSync(file);files[path.relative(directory,file)]={bytes:bytes.length,sha256:createHash('sha256').update(bytes).digest('hex')};}
+ }}
+ walk(directory);return files;
+}
+async function main() {
+ fs.rmSync(fixture,{recursive:true,force:true});
+ const {write}=await createFixture(fixture);
+ write('docs/pixel.svg','<svg xmlns="http://www.w3.org/2000/svg"; width="8"
height="8"><rect width="8" height="8"/></svg>');
+ fs.appendFileSync(path.join(fixture,'docs/01-shared.md'),'\n\n');
+ write('blog/2026-01-01-first.md','---\ntitle: First post\nslug:
first\n---\n\nExcerpt.\n\n<!-- truncate -->\n\n## More\n\nFull blog body.\n');
+ write('blog/2026-01-02-second.md','---\ntitle: Second post\nslug:
second\n---\n\nSecond body.\n');
+
write('i18n/ja/docusaurus-plugin-content-blog/2026-01-01-first.md','---\ntitle:
日本語の記事\nslug: first\n---\n\n日本語。\n\n<!-- truncate -->\n\n## More\n\n記事の本文。\n');
+ fs.appendFileSync(path.join(fixture,'docusaurus.config.js'),`\nconst
original=module.exports;
+module.exports=()=>{const
config=original();config.future={experimental_faster:true};
+config.presets[0][1].blog={path:'blog',showReadingTime:false,onInlineAuthors:'ignore',onUntruncatedBlogPosts:'ignore'};
+config.plugins.push(require(${JSON.stringify(path.join(root,'config/custom-docusaurus-plugin.js'))}));return
config;};\n`);
+ write('.gitignore','node_modules\n.i18n-build/\n.docusaurus/\nbuild/\n');
+ execFileSync('git',['add','.'],{cwd:fixture});
+
execFileSync('git',['-c','user.name=Fixture','-c','[email protected]','commit','-qm','Pinned
native compiler equality fixture'],{cwd:fixture});
+ const modes={};
+ for(const mode of ['parallel','isolated']){
+ const
child=spawnSync(process.execPath,[path.join(root,'scripts/build-locales.cjs'),'--site-dir',fixture],{
+
cwd:root,stdio:'inherit',env:{...process.env,NODE_OPTIONS:'--max-old-space-size=4096',
+
DORIS_COMPILER_MODE:mode,DORIS_OPTIMIZE_PUBLISH:'false',DORIS_ASSET_ORIGIN:'',
+
TERSER_PARALLEL:'2',RAYON_NUM_THREADS:'4',DOCUSAURUS_SSR_CONCURRENCY:'4',DORIS_BUILD_METRICS:'true'}});
+ assert.equal(child.status,0,mode+' native fixture build');
+ const
build=JSON.parse(fs.readFileSync(path.join(fixture,'.i18n-build/build-result.json')));
+ assert.equal(build.locales.length,5);
+ const
metrics=build.locales.map(({locale})=>JSON.parse(fs.readFileSync(path.join(fixture,'.i18n-build/compiler-metrics',locale+'.json'))));
+ if(mode==='isolated')for(const record of metrics){
+
assert.equal(record.bundler,'rspack');assert.deepEqual(record.records.map(r=>r.name),['client','server']);
+
assert(record.records.every(r=>r.success));assert(record.records[0].finishedAtMs<=record.records[1].startedAtMs);
+
assert.notEqual(record.records[0].pid,record.records[1].pid);assert.equal(record.clientExitCode,0);
+
assert(record.clientExitedAtMs<=record.records[1].startedAtMs,'Server started
before the client process exited');
+ }
+ modes[mode]={files:inventory(path.join(fixture,'build')),metrics};
+ }
+ const paths=new
Set([...Object.keys(modes.parallel.files),...Object.keys(modes.isolated.files)]);
+ const
differences=[...paths].filter(file=>JSON.stringify(modes.parallel.files[file])!==JSON.stringify(modes.isolated.files[file]));
+ const result={passed:!differences.length,scope:'Same-source five-language
real Rspack HTML/JS/CSS output; no full website build',
+
implementationSha:execFileSync('git',['rev-parse','HEAD'],{cwd:root,encoding:'utf8'}).trim(),
+
fixtureSource:execFileSync('git',['rev-parse','HEAD'],{cwd:fixture,encoding:'utf8'}).trim(),
+
locales:5,filesCompared:paths.size,bytesCompared:Object.values(modes.isolated.files).reduce((sum,row)=>sum+row.bytes,0),
+ differences,modes};
+
fs.writeFileSync(path.join(root,'.i18n-build/compiler-fixture-verification.json'),JSON.stringify(result,null,2)+'\n');
+
console.log(JSON.stringify({passed:result.passed,filesCompared:paths.size,differences}));assert(result.passed,'Process-isolated
native compilation changed emitted artifacts');
+}
+if(require.main===module)main().catch(error=>{console.error(error);process.exitCode=1;});
+module.exports={inventory};
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
