JDevlieghere created this revision.
Herald added a subscriber: mgorny.
JDevlieghere added a subscriber: labath.
This patch is WIP and meant to showcase how the reproducer and VFS will
interact.
Repository:
rLLDB LLDB
https://reviews.llvm.org/D54617
Files:
include/lldb/Host/FileSystem.h
include/lldb/Utility/FileCollector.h
include/lldb/Utility/Reproducer.h
source/Core/Debugger.cpp
source/Host/common/FileSystem.cpp
source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
source/Utility/CMakeLists.txt
source/Utility/FileCollector.cpp
source/Utility/Reproducer.cpp
Index: source/Utility/Reproducer.cpp
===================================================================
--- source/Utility/Reproducer.cpp
+++ source/Utility/Reproducer.cpp
@@ -187,7 +187,7 @@
return llvm::Error::success();
}
-llvm::Optional<ProviderInfo> Loader::GetProviderInfo(StringRef name) {
+llvm::Optional<ProviderInfo> Loader::GetInfo(StringRef name) {
assert(m_loaded);
auto it = m_provider_info.find(name);
Index: source/Utility/FileCollector.cpp
===================================================================
--- /dev/null
+++ source/Utility/FileCollector.cpp
@@ -0,0 +1,134 @@
+//===-- FileCollector.cpp ---------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Utility/FileCollector.h"
+
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+
+using namespace lldb_private;
+using namespace llvm;
+
+static bool IsCaseSensitivePath(StringRef path) {
+ SmallString<256> tmp_dest = path, upper_dest, real_dest;
+
+ // Remove component traversals, links, etc.
+ if (!sys::fs::real_path(path, tmp_dest))
+ return true; // Current default value in vfs.yaml
+ path = tmp_dest;
+
+ // Change path to all upper case and ask for its real path, if the latter
+ // exists and is equal to path, it's not case sensitive. Default to case
+ // sensitive in the absence of real_path, since this is the YAMLVFSWriter
+ // default.
+ for (auto &C : path)
+ upper_dest.push_back(toUpper(C));
+ if (sys::fs::real_path(upper_dest, real_dest) && path.equals(real_dest))
+ return false;
+ return true;
+}
+
+FileCollector::FileCollector(const FileSpec &directory) : m_root(directory) {
+ sys::fs::create_directories(m_root.GetPath(), true);
+}
+
+bool FileCollector::GetRealPath(StringRef src_path,
+ SmallVectorImpl<char> &result) {
+ SmallString<256> real_path;
+ StringRef FileName = sys::path::filename(src_path);
+ std::string directory = sys::path::parent_path(src_path).str();
+ auto dir_with_symlink = m_symlink_map.find(directory);
+
+ // Use real_path to fix any symbolic link component present in a path.
+ // Computing the real path is expensive, cache the search through the
+ // parent path directory.
+ if (dir_with_symlink == m_symlink_map.end()) {
+ if (!sys::fs::real_path(directory, real_path))
+ return false;
+ m_symlink_map[directory] = real_path.str();
+ } else {
+ real_path = dir_with_symlink->second;
+ }
+
+ sys::path::append(real_path, FileName);
+ result.swap(real_path);
+ return true;
+}
+
+void FileCollector::AddFile(const Twine &file) {
+ std::lock_guard<std::mutex> lock(m_mutex);
+ std::string file_str = file.str();
+ if (MarkAsSeen(file_str))
+ AddFileImpl(file_str);
+}
+
+void FileCollector::AddFileImpl(StringRef src_path) {
+ std::string root = m_root.GetPath();
+
+ // We need an absolute src path to append to the root.
+ SmallString<256> absolute_src = src_path;
+ sys::fs::make_absolute(absolute_src);
+
+ // Canonicalize src to a native path to avoid mixed separator styles.
+ sys::path::native(absolute_src);
+
+ // Remove redundant leading "./" pieces and consecutive separators.
+ absolute_src = sys::path::remove_leading_dotslash(absolute_src);
+
+ // Canonicalize the source path by removing "..", "." components.
+ SmallString<256> virtual_path = absolute_src;
+ sys::path::remove_dots(virtual_path, /*remove_dot_dot=*/true);
+
+ // If a ".." component is present after a symlink component, remove_dots may
+ // lead to the wrong real destination path. Let the source be canonicalized
+ // like that but make sure we always use the real path for the destination.
+ SmallString<256> copy_from;
+ if (!GetRealPath(absolute_src, copy_from))
+ copy_from = virtual_path;
+
+ SmallString<256> dst_path = StringRef(root);
+ sys::path::append(dst_path, sys::path::relative_path(copy_from));
+
+ // Always map a canonical src path to its real path into the YAML, by doing
+ // this we map different virtual src paths to the same entry in the VFS
+ // overlay, which is a way to emulate symlink inside the VFS; this is also
+ // needed for correctness, not doing that can lead to module redefinition
+ // errors.
+ AddFileToMapping(virtual_path, dst_path);
+}
+
+std::error_code FileCollector::CopyFiles() {
+ for (auto &entry : m_vfs_writer.getMappings()) {
+ if (std::error_code ec =
+ sys::fs::create_directories(sys::path::parent_path(entry.RPath),
+ /*IgnoreExisting=*/true))
+ return ec;
+ if (std::error_code ec = sys::fs::copy_file(entry.VPath, entry.RPath))
+ return ec;
+ }
+ return {};
+}
+
+std::error_code FileCollector::WriteMapping(const FileSpec &mapping_file) {
+ std::lock_guard<std::mutex> lock(m_mutex);
+
+ const std::string root = m_root.GetPath();
+ m_vfs_writer.setCaseSensitivity(IsCaseSensitivePath(root));
+ m_vfs_writer.setUseExternalNames(false);
+
+ std::error_code ec;
+ raw_fd_ostream os(mapping_file.GetPath(), ec, sys::fs::F_Text);
+ if (ec)
+ return ec;
+
+ m_vfs_writer.write(os);
+
+ return {};
+}
Index: source/Utility/CMakeLists.txt
===================================================================
--- source/Utility/CMakeLists.txt
+++ source/Utility/CMakeLists.txt
@@ -53,6 +53,7 @@
DataEncoder.cpp
DataExtractor.cpp
Environment.cpp
+ FileCollector.cpp
FileSpec.cpp
IOObject.cpp
JSON.cpp
Index: source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
===================================================================
--- source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -3429,7 +3429,7 @@
if (!loader)
return Status("No loader provided.");
- auto provider_info = loader->GetProviderInfo("gdb-remote");
+ auto provider_info = loader->GetInfo("gdb-remote");
if (!provider_info)
return Status("No provider for gdb-remote.");
Index: source/Host/common/FileSystem.cpp
===================================================================
--- source/Host/common/FileSystem.cpp
+++ source/Host/common/FileSystem.cpp
@@ -49,10 +49,13 @@
}
void FileSystem::Initialize(IntrusiveRefCntPtr<vfs::FileSystem> fs) {
- lldbassert(!InstanceImpl() && "Already initialized.");
InstanceImpl().emplace(fs);
}
+void FileSystem::Initialize(llvm::StringRef mapping) {
+ InstanceImpl().emplace(mapping);
+}
+
void FileSystem::Terminate() {
lldbassert(InstanceImpl() && "Already terminated.");
InstanceImpl().reset();
@@ -63,6 +66,16 @@
return g_fs;
}
+FileSystem::FileSystem(llvm::StringRef mapping)
+ : m_fs(llvm::vfs::getRealFileSystem()), m_collector(nullptr) {
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> buffer =
+ m_fs->getBufferForFile(mapping);
+ if (!buffer)
+ return;
+
+ m_fs = llvm::vfs::getVFSFromYAML(std::move(buffer.get()), nullptr, "");
+}
+
sys::TimePoint<>
FileSystem::GetModificationTime(const FileSpec &file_spec) const {
return GetModificationTime(file_spec.GetPath());
@@ -231,6 +244,9 @@
std::shared_ptr<DataBufferLLVM>
FileSystem::CreateDataBuffer(const llvm::Twine &path, uint64_t size,
uint64_t offset) {
+ if (m_collector)
+ m_collector->AddFile(path);
+
const bool is_volatile = !IsLocal(path);
std::unique_ptr<llvm::WritableMemoryBuffer> buffer;
@@ -362,6 +378,9 @@
Status FileSystem::Open(File &File, const FileSpec &file_spec, uint32_t options,
uint32_t permissions) {
+ if (m_collector)
+ m_collector->AddFile(file_spec);
+
if (File.IsValid())
File.Close();
Index: source/Core/Debugger.cpp
===================================================================
--- source/Core/Debugger.cpp
+++ source/Core/Debugger.cpp
@@ -422,14 +422,47 @@
auto &r = repro::Reproducer::Instance();
if (auto e = r.SetReplay(true, FileSpec(p)))
return e;
+
+ // FIXME: The logic below should live elsewhere.
+ repro::Loader *loader = r.GetLoader();
+ if (!loader)
+ return llvm::make_error<llvm::StringError>("no loader for replay",
+ llvm::inconvertibleErrorCode());
+
+ auto provider_info = loader->GetInfo<repro::FileProvider>();
+ if (!provider_info)
+ return llvm::make_error<llvm::StringError>("no file provider for replay",
+ llvm::inconvertibleErrorCode());
+
+ if (provider_info->files.empty())
+ return llvm::make_error<llvm::StringError>("no vfs mapping for replay",
+ llvm::inconvertibleErrorCode());
+
+ FileSpec vfs_mapping(loader->GetRoot());
+ vfs_mapping.AppendPathComponent(provider_info->files.front());
+
+ FileSystem::Initialize(vfs_mapping.GetPath());
return llvm::Error::success();
}
llvm::Error Debugger::SetReproducerCapture(bool b) {
auto &r = repro::Reproducer::Instance();
auto root = HostInfo::GetReproducerTempDir();
if (auto e = r.SetCapture(b, root))
return e;
+
+ // FIXME: The logic below should live elsewhere.
+ if (!b) {
+ FileSystem::Instance().SetCollector(nullptr);
+ return llvm::Error::success();
+ }
+
+ if (repro::Generator *g = r.GetGenerator()) {
+ if (repro::FileProvider *fp = g->GetOrCreate<repro::FileProvider>()) {
+ FileSystem::Instance().SetCollector(&(fp->GetFileCollector()));
+ }
+ }
+
return llvm::Error::success();
}
Index: include/lldb/Utility/Reproducer.h
===================================================================
--- include/lldb/Utility/Reproducer.h
+++ include/lldb/Utility/Reproducer.h
@@ -10,6 +10,7 @@
#ifndef LLDB_UTILITY_REPRODUCER_H
#define LLDB_UTILITY_REPRODUCER_H
+#include "lldb/Utility/FileCollector.h"
#include "lldb/Utility/FileSpec.h"
#include "llvm/ADT/StringMap.h"
@@ -71,6 +72,29 @@
FileSpec m_root;
};
+class FileProvider : public repro::Provider {
+public:
+ static constexpr const char *NAME = "files";
+ FileProvider(const FileSpec &directory)
+ : Provider(directory),
+ m_collector(directory.CopyByAppendingPathComponent("root")) {
+ InitializeFileInfo(NAME, {"files.yaml"});
+ }
+
+ FileCollector &GetFileCollector() { return m_collector; }
+
+ void Keep() override {
+ auto mapping = GetRoot().CopyByAppendingPathComponent("files.yaml");
+ // Don't write the mapping if we can't copy the files.
+ if (auto ec = m_collector.CopyFiles())
+ return;
+ m_collector.WriteMapping(mapping);
+ }
+
+private:
+ FileCollector m_collector;
+};
+
/// The generator is responsible for the logic needed to generate a
/// reproducer. For doing so it relies on providers, who serialize data that
/// is necessary for reproducing a failure.
@@ -141,7 +165,12 @@
public:
Loader(const FileSpec &root);
- llvm::Optional<ProviderInfo> GetProviderInfo(llvm::StringRef name);
+ llvm::Optional<ProviderInfo> GetInfo(llvm::StringRef name);
+
+ template <typename T> llvm::Optional<ProviderInfo> GetInfo() {
+ return GetInfo(T::NAME);
+ }
+
llvm::Error LoadIndex();
const FileSpec &GetRoot() const { return m_root; }
Index: include/lldb/Utility/FileCollector.h
===================================================================
--- /dev/null
+++ include/lldb/Utility/FileCollector.h
@@ -0,0 +1,59 @@
+//===-- FileCollector.h -----------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_UTILITY_FILE_COLLECTOR_H
+#define LLDB_UTILITY_FILE_COLLECTOR_H
+
+#include "lldb/Utility/FileSpec.h"
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/VirtualFileSystem.h"
+
+#include <mutex>
+
+namespace lldb_private {
+
+/// Collects files into a directory and generates a mapping that can be used by
+/// the VFS.
+class FileCollector {
+public:
+ FileCollector(const FileSpec &directory);
+
+ void AddFile(const llvm::Twine &file);
+ void AddFile(const FileSpec &file) { return AddFile(file.GetPath()); }
+
+ std::error_code WriteMapping(const FileSpec &mapping_file);
+ std::error_code CopyFiles();
+
+private:
+ void AddFileImpl(llvm::StringRef src_path);
+
+ bool MarkAsSeen(llvm::StringRef path) { return m_seen.insert(path).second; }
+
+ bool GetRealPath(llvm::StringRef src_path,
+ llvm::SmallVectorImpl<char> &result);
+
+ void AddFileToMapping(llvm::StringRef virtual_path,
+ llvm::StringRef real_path) {
+ m_vfs_writer.addFileMapping(virtual_path, real_path);
+ }
+
+ std::mutex m_mutex;
+ FileSpec m_root;
+ llvm::StringSet<> m_seen;
+ llvm::vfs::YAMLVFSWriter m_vfs_writer;
+ llvm::StringMap<std::string> m_symlink_map;
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_UTILITY_FILE_COLLECTOR_H
Index: include/lldb/Host/FileSystem.h
===================================================================
--- include/lldb/Host/FileSystem.h
+++ include/lldb/Host/FileSystem.h
@@ -12,6 +12,7 @@
#include "lldb/Host/File.h"
#include "lldb/Utility/DataBufferLLVM.h"
+#include "lldb/Utility/FileCollector.h"
#include "lldb/Utility/FileSpec.h"
#include "lldb/Utility/Status.h"
@@ -31,13 +32,16 @@
static const char *DEV_NULL;
static const char *PATH_CONVERSION_ERROR;
- FileSystem() : m_fs(llvm::vfs::getRealFileSystem()) {}
- FileSystem(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs) : m_fs(fs) {}
+ FileSystem() : m_fs(llvm::vfs::getRealFileSystem()), m_collector(nullptr) {}
+ FileSystem(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs)
+ : m_fs(fs), m_collector(nullptr) {}
+ FileSystem(llvm::StringRef mapping);
static FileSystem &Instance();
static void Initialize();
static void Initialize(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs);
+ static void Initialize(llvm::StringRef mapping);
static void Terminate();
Status Symlink(const FileSpec &src, const FileSpec &dst);
@@ -151,9 +155,13 @@
std::error_code GetRealPath(const llvm::Twine &path,
llvm::SmallVectorImpl<char> &output) const;
+ void SetCollector(FileCollector *collector) { m_collector = collector; }
+
private:
static llvm::Optional<FileSystem> &InstanceImpl();
+
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> m_fs;
+ FileCollector *m_collector;
};
} // namespace lldb_private
_______________________________________________
lldb-commits mailing list
[email protected]
http://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
