kbobyrev updated this revision to Diff 166462.
kbobyrev marked 4 inline comments as done.
kbobyrev added a comment.
Use `llvm::Expected<SymbolSlab>`, cleanup the patch.
https://reviews.llvm.org/D52047
Files:
clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp
clang-tools-extra/clangd/index/Serialization.cpp
clang-tools-extra/clangd/index/SymbolYAML.cpp
clang-tools-extra/clangd/index/SymbolYAML.h
clang-tools-extra/clangd/index/dex/Dex.cpp
Index: clang-tools-extra/clangd/index/dex/Dex.cpp
===================================================================
--- clang-tools-extra/clangd/index/dex/Dex.cpp
+++ clang-tools-extra/clangd/index/dex/Dex.cpp
@@ -124,9 +124,6 @@
for (const auto &TokenToPostingList : TempInvertedIndex)
InvertedIndex.insert({TokenToPostingList.first,
PostingList(move(TokenToPostingList.second))});
-
- vlog("Built Dex with estimated memory usage {0} bytes.",
- estimateMemoryUsage());
}
/// Constructs iterators over tokens extracted from the query and exhausts it
@@ -238,8 +235,9 @@
Bytes += SymbolQuality.size() * sizeof(float);
Bytes += LookupTable.getMemorySize();
Bytes += InvertedIndex.getMemorySize();
- for (const auto &P : InvertedIndex)
- Bytes += P.second.bytes();
+ for (const auto &TokenAndPostingList : InvertedIndex)
+ Bytes += TokenAndPostingList.first.Data.size() +
+ TokenAndPostingList.second.bytes();
return Bytes + BackingDataSize;
}
Index: clang-tools-extra/clangd/index/SymbolYAML.h
===================================================================
--- clang-tools-extra/clangd/index/SymbolYAML.h
+++ clang-tools-extra/clangd/index/SymbolYAML.h
@@ -29,6 +29,9 @@
// Read symbols from a YAML-format string.
SymbolSlab symbolsFromYAML(llvm::StringRef YAMLContent);
+// Read symbols from YAML or RIFF file.
+llvm::Expected<SymbolSlab> symbolsFromFile(llvm::StringRef SymbolFilename);
+
// Read one symbol from a YAML-stream.
// The returned symbol is backed by Input.
Symbol SymbolFromYAML(llvm::yaml::Input &Input);
Index: clang-tools-extra/clangd/index/SymbolYAML.cpp
===================================================================
--- clang-tools-extra/clangd/index/SymbolYAML.cpp
+++ clang-tools-extra/clangd/index/SymbolYAML.cpp
@@ -9,6 +9,7 @@
#include "SymbolYAML.h"
#include "Index.h"
+#include "Logger.h"
#include "Serialization.h"
#include "Trace.h"
#include "dex/Dex.h"
@@ -172,6 +173,7 @@
namespace clangd {
SymbolSlab symbolsFromYAML(llvm::StringRef YAMLContent) {
+ trace::Span Tracer("ParseYAML");
llvm::yaml::Input Yin(YAMLContent);
std::vector<Symbol> S;
Yin >> S;
@@ -182,6 +184,24 @@
return std::move(Syms).build();
}
+llvm::Expected<SymbolSlab> symbolsFromFile(llvm::StringRef SymbolFilename) {
+ auto Buffer = llvm::MemoryBuffer::getFile(SymbolFilename);
+ if (!Buffer)
+ return llvm::make_error<llvm::StringError>(
+ ("Can't open " + SymbolFilename).str(), llvm::errc::invalid_argument);
+
+ StringRef Data = Buffer->get()->getBuffer();
+
+ if (!Data.startswith("RIFF")) { // Magic for binary index file.
+ return symbolsFromYAML(Data);
+ }
+ auto RIFF = readIndexFile(Data);
+ return RIFF ? llvm::Expected<SymbolSlab>(std::move(*RIFF->Symbols))
+ : llvm::make_error<llvm::StringError>(
+ ("Can't open " + SymbolFilename).str(),
+ llvm::errc::invalid_argument);
+}
+
Symbol SymbolFromYAML(llvm::yaml::Input &Input) {
Symbol S;
Input >> S;
@@ -206,30 +226,16 @@
llvm::ArrayRef<std::string> URISchemes,
bool UseDex) {
trace::Span OverallTracer("LoadIndex");
- auto Buffer = llvm::MemoryBuffer::getFile(SymbolFilename);
- if (!Buffer) {
- llvm::errs() << "Can't open " << SymbolFilename << "\n";
- return nullptr;
- }
- StringRef Data = Buffer->get()->getBuffer();
-
- llvm::Optional<SymbolSlab> Slab;
- if (Data.startswith("RIFF")) { // Magic for binary index file.
- trace::Span Tracer("ParseRIFF");
- if (auto RIFF = readIndexFile(Data))
- Slab = std::move(RIFF->Symbols);
- else
- llvm::errs() << "Bad RIFF: " << llvm::toString(RIFF.takeError()) << "\n";
- } else {
- trace::Span Tracer("ParseYAML");
- Slab = symbolsFromYAML(Data);
- }
-
+ auto Slab = symbolsFromFile(SymbolFilename);
if (!Slab)
return nullptr;
trace::Span Tracer("BuildIndex");
- return UseDex ? dex::Dex::build(std::move(*Slab), URISchemes)
- : MemIndex::build(std::move(*Slab), RefSlab());
+ auto Index = UseDex ? dex::Dex::build(std::move(*Slab), URISchemes)
+ : MemIndex::build(std::move(*Slab), RefSlab());
+ vlog("Loaded {0} from {1} with estimated memory usage {2}",
+ UseDex ? "Dex" : "MemIndex", SymbolFilename,
+ Index->estimateMemoryUsage());
+ return Index;
}
} // namespace clangd
Index: clang-tools-extra/clangd/index/Serialization.cpp
===================================================================
--- clang-tools-extra/clangd/index/Serialization.cpp
+++ clang-tools-extra/clangd/index/Serialization.cpp
@@ -6,9 +6,11 @@
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
+
#include "Serialization.h"
#include "Index.h"
#include "RIFF.h"
+#include "Trace.h"
#include "llvm/Support/Compression.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/Error.h"
@@ -309,6 +311,7 @@
constexpr static uint32_t Version = 3;
Expected<IndexFileIn> readIndexFile(StringRef Data) {
+ trace::Span Tracer("ParseRIFF");
auto RIFF = riff::readFile(Data);
if (!RIFF)
return RIFF.takeError();
Index: clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp
===================================================================
--- clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp
+++ clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp
@@ -33,6 +33,15 @@
return loadIndex(IndexFilename, {}, true);
}
+SymbolSlab loadSlab() {
+ auto Slab = symbolsFromFile(IndexFilename);
+ if (!Slab) {
+ llvm::errs() << llvm::toString(Slab.takeError()) << '\n';
+ exit(1);
+ }
+ return std::move(*Slab);
+}
+
// Reads JSON array of serialized FuzzyFindRequest's from user-provided file.
std::vector<FuzzyFindRequest> extractQueriesFromLogs() {
std::ifstream InputStream(RequestsFilename);
@@ -66,31 +75,65 @@
return Requests;
}
+// This is not a *real* benchmark: it shows size of built MemIndex (in bytes).
+// Same for the next "benchmark".
+// FIXME(kbobyrev): Track memory usage caused by different index parts:
+// SymbolSlab and RefSlabs, InverseIndex, PostingLists of different types for
+// Dex, etc.
+static void MemSize(benchmark::State &State) {
+ const auto Mem = buildMem();
+ for (auto _ : State)
+ // Divide size of Mem by 1000 so that it will be correctly displayed in the
+ // benchmark report (possible options for time units are ms, ns and us).
+ State.SetIterationTime(/*double Seconds=*/Mem->estimateMemoryUsage() /
+ 1000);
+ State.counters["Memory Usage (bytes)"] = Mem->estimateMemoryUsage();
+}
+BENCHMARK(MemSize)->UseManualTime()->Unit(benchmark::kMillisecond);
+
+static void DexSize(benchmark::State &State) {
+ const auto Dex = buildDex();
+ for (auto _ : State)
+ State.SetIterationTime(Dex->estimateMemoryUsage() / 1000);
+ State.counters["Memory Usage (bytes)"] = Dex->estimateMemoryUsage();
+}
+BENCHMARK(DexSize)->UseManualTime()->Unit(benchmark::kMillisecond);
+
+static void SymbolSlabLoading(benchmark::State &State) {
+ for (auto _ : State)
+ benchmark::DoNotOptimize(loadSlab());
+}
+BENCHMARK(SymbolSlabLoading)->Unit(benchmark::kMillisecond);
+
+static void DexBuild(benchmark::State &State) {
+ auto Slab = loadSlab();
+ for (auto _ : State)
+ benchmark::DoNotOptimize(dex::Dex::build(std::move(Slab), {}));
+}
+BENCHMARK(DexBuild)->Unit(benchmark::kMillisecond);
+
static void MemQueries(benchmark::State &State) {
const auto Mem = buildMem();
const auto Requests = extractQueriesFromLogs();
for (auto _ : State)
for (const auto &Request : Requests)
Mem->fuzzyFind(Request, [](const Symbol &S) {});
}
-BENCHMARK(MemQueries);
+BENCHMARK(MemQueries)->Unit(benchmark::kMillisecond);
static void DexQueries(benchmark::State &State) {
const auto Dex = buildDex();
const auto Requests = extractQueriesFromLogs();
for (auto _ : State)
for (const auto &Request : Requests)
Dex->fuzzyFind(Request, [](const Symbol &S) {});
}
-BENCHMARK(DexQueries);
+BENCHMARK(DexQueries)->Unit(benchmark::kMillisecond);
} // namespace
} // namespace clangd
} // namespace clang
-// FIXME(kbobyrev): Add index building time benchmarks.
-// FIXME(kbobyrev): Add memory consumption "benchmarks" by manually measuring
-// in-memory index size and reporting it as time.
// FIXME(kbobyrev): Create a logger wrapper to suppress debugging info printer.
int main(int argc, char *argv[]) {
if (argc < 3) {
_______________________________________________
cfe-commits mailing list
[email protected]
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
