One of the things my app needs to do is write a large JSON file to disk.
Currently, it's implemented naively:
writeStream = fs.createWriteStream(process.cwd() + "/staticRoot/album.json",
{"encoding": "utf8"});
writeStream.addListener("error", function (error) {
console.error("album.json:", error);
if (! writeError) // preserve 1st error
writeError = error;
callback(writeError); // on error, abort writing next file
});
writeStream.end(JSON.stringify(album));
This appears to work ok when the JSON file is 150k (the largest size I can
readily test). It needs to work when the JSON file is up to 500k. i'm not
sure how big the write buffer is, but I lack confidence that this is the
right way to go. The bulk of the JSON file is an array of objects, so I
could create the JSON file one array item at a time, checking the return
value from writeStream.write() in a manner analogous to the following HTML
file writing:
writeStream.addListener("drain", writeUntilBufferFull);
writeStream.write("<h1>" + title + "</h1>\n");
writeStream.write("<table>\n");
function writeUntilBufferFull() {
var longDate, urlFileName, htmlFileName, row;
console.log("writeUntilBufferFull() p=", p, " metadata:", JSON.stringify
(pictureMetadata[p]));
while (p < pictureMetadata.length) {
longDate = pictureMetadata[p].date ? pictureMetadata[p].date.
toLocaleDateString() : "";
urlFileName = encodeURIComponent(pictureMetadata[p].fileName).replace(/'/g,
'%27');
htmlFileName = pictureMetadata[p].fileName.replace(/&/g, "&").replace(
/</g, "<").replace(/>/g, ">").replace(/"/g, """);
row = " <tr><td align='right'>" + (p+1) + "</td><td align='right'>" +longDate
+
"</td><td><a class='gallery' href='pictures/" + urlAlbumName + "/" +urlFileName
+ "' title='" + longDate + "'>" + htmlFileName + "</a></td>" +
"<td><a download=\"" + htmlFileName + "\" href='full-size/" + urlAlbumName +
"/" + urlFileName + "'>full-size</a></td></tr>\n";
if (! writeStream.write(row))
break;
++p;
}
if (p < pictureMetadata.length) {
++p;
} else {
writeStream.removeListener("drain", writeUntilBufferFull);
writeStream.end();
callback(writeError);
}
}
What's the best strategy for writing large JSON files?
--
Job Board: http://jobs.nodejs.org/
Posting guidelines:
https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines
You received this message because you are subscribed to the Google
Groups "nodejs" group.
To post to this group, send email to [email protected]
To unsubscribe from this group, send email to
[email protected]
For more options, visit this group at
http://groups.google.com/group/nodejs?hl=en?hl=en
