================
@@ -132,404 +135,74 @@ Error MustacheHTMLGenerator::generateDocs(
return Err;
}
- // Track which directories we already tried to create.
- StringSet<> CreatedDirs;
- // Collect all output by file name and create the necessary directories.
- StringMap<std::vector<doc::Info *>> FileToInfos;
- for (const auto &Group : Infos) {
- llvm::TimeTraceScope TS("setup directories");
- doc::Info *Info = Group.getValue().get();
-
- SmallString<128> Path;
- sys::path::native(RootDir, Path);
- sys::path::append(Path, Info->getRelativeFilePath(""));
- if (!CreatedDirs.contains(Path)) {
- if (std::error_code EC = sys::fs::create_directories(Path))
- return createStringError(EC, "failed to create directory '%s'.",
- Path.c_str());
- CreatedDirs.insert(Path);
- }
+ {
+ llvm::TimeTraceScope TS("Generate JSON for Mustache");
+ if (auto JSONGenerator = findGeneratorByName("json")) {
+ if (Error Err = JSONGenerator.get()->generateDocs(
+ RootDir, std::move(Infos), CDCtx))
+ return Err;
+ } else
+ return JSONGenerator.takeError();
+ }
- sys::path::append(Path, Info->getFileBaseName() + ".html");
- FileToInfos[Path].push_back(Info);
+ StringMap<json::Value> JSONFileMap;
+ {
+ llvm::TimeTraceScope TS("Iterate JSON files");
+ std::error_code EC;
+ sys::fs::directory_iterator JSONIter(RootDir, EC);
+ std::vector<json::Value> JSONFiles;
+ JSONFiles.reserve(Infos.size());
+ if (EC)
+ return createStringError("Failed to create directory iterator.");
+
+ while (JSONIter != sys::fs::directory_iterator()) {
+ if (EC)
+ return createStringError(EC, "Failed to iterate directory");
+
+ auto Path = StringRef(JSONIter->path());
+ if (!Path.ends_with(".json")) {
+ JSONIter.increment(EC);
+ continue;
+ }
+
+ auto File = MemoryBuffer::getFile(Path);
+ if ((EC = File.getError()))
+ continue;
+
+ auto Parsed = json::parse((*File)->getBuffer());
+ if (!Parsed)
+ return Parsed.takeError();
+ JSONFileMap.try_emplace(Path, *Parsed);
----------------
evelez7 wrote:
> can you split the generation so that you serialize the json files and then
> ingest them for mustache separately?
Do you mean keeping them in memory instead of writing to disk? I thought about
adding that as a command line option in case someone also wants the JSON output
anyways. I've been using it to debug.
> Is there any reason to keep them around or in memory? I'd guess we'd want to
> convert these to HTML more or less serially (or in fixed size batches).
Yeah my first thought was to immediately delete an in-memory file once its
processed to cut down on memory as we move through such a large amount of
files. I'm not sure if this is better in batches for vectors. I also noticed
that we don't do batches for file output. Would that make a difference with
this volume of files?
Typing this out, a major improvement would be just linking the JSON and
Mustache generators tighter by allowing an in-memory JSON object to be
immediately passed to Mustache.
https://github.com/llvm/llvm-project/pull/149006
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
