splhack created this revision. Herald added subscribers: danielkiss, krytarowski. Herald added a project: All. splhack added reviewers: clayborg, labath, lanza, srhines. splhack published this revision for review. Herald added a project: LLDB. Herald added a subscriber: lldb-commits.
In Android API level 23 and above, dynamic loader is able to load .so file directly from APK, which is zip file. https://android.googlesource.com/platform/bionic/+/master/ android-changes-for-ndk-developers.md# opening-shared-libraries-directly-from-an-apk The .so file is page aligned and uncompressed, so ObjectFileELF::GetModuleSpecifications works with .so file offset and size directly from zip file without extracting it. (D152757 <https://reviews.llvm.org/D152757>) GDBRemoteCommunicationServerCommon::GetModuleInfo returns a module spec to LLDB with "zip_path!/so_path" file spec, which is passed through from Android dynamic loader, and the .so file offset and size. PlatformAndroid::DownloadModuleSlice uses 'shell dd' to download the .so file slice from the zip file with the .so file offset and size. Depends on D152494 <https://reviews.llvm.org/D152494> and D152712 <https://reviews.llvm.org/D152712> and D152757 <https://reviews.llvm.org/D152757> Repository: rG LLVM Github Monorepo https://reviews.llvm.org/D152759 Files: lldb/include/lldb/Host/android/HostInfoAndroid.h lldb/source/Host/CMakeLists.txt lldb/source/Host/android/HostInfoAndroid.cpp lldb/source/Host/android/ZipFile.cpp lldb/source/Host/android/ZipFile.h lldb/source/Plugins/Platform/Android/PlatformAndroid.cpp lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp lldb/unittests/Host/CMakeLists.txt lldb/unittests/Host/android/CMakeLists.txt lldb/unittests/Host/android/HostInfoAndroidTest.cpp lldb/unittests/Host/android/Inputs/zip-test.zip
Index: lldb/unittests/Host/android/HostInfoAndroidTest.cpp =================================================================== --- /dev/null +++ lldb/unittests/Host/android/HostInfoAndroidTest.cpp @@ -0,0 +1,67 @@ +//===-- HostInfoAndroidTest.cpp -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "lldb/Host/android/HostInfoAndroid.h" +#include "TestingSupport/SubsystemRAII.h" +#include "TestingSupport/TestUtilities.h" +#include "gtest/gtest.h" + +using namespace lldb_private; +using namespace llvm; + +namespace { +class HostInfoAndroidTest : public ::testing::Test { + SubsystemRAII<FileSystem> subsystems; +}; + +std::string TestZipPath() { + FileSpec zip_spec(GetInputFilePath("zip-test.zip")); + FileSystem::Instance().Resolve(zip_spec); + return zip_spec.GetPath(); +} +} // namespace + +TEST_F(HostInfoAndroidTest, ResolveZipPathWithNormalFile) { + const FileSpec file_spec("/system/lib64/libtest.so"); + + std::string file_path; + lldb::offset_t file_offset; + lldb::offset_t file_size; + ASSERT_TRUE(HostInfoAndroid::ResolveZipPath(file_spec, file_path, + file_offset, file_size)); + + EXPECT_EQ(file_path, file_spec.GetPath()); + EXPECT_EQ(file_offset, 0UL); + EXPECT_EQ(file_size, 0UL); +} + +TEST_F(HostInfoAndroidTest, ResolveZipPathWithZipMissing) { + const std::string zip_path = TestZipPath(); + const FileSpec file_spec(zip_path + "!/lib/arm64-v8a/libmissing.so"); + + std::string file_path; + lldb::offset_t file_offset; + lldb::offset_t file_size; + ASSERT_FALSE(HostInfoAndroid::ResolveZipPath(file_spec, file_path, + file_offset, file_size)); +} + +TEST_F(HostInfoAndroidTest, ResolveZipPathWithZipExisting) { + const std::string zip_path = TestZipPath(); + const FileSpec file_spec(zip_path + "!/lib/arm64-v8a/libzip-test.so"); + + std::string file_path; + lldb::offset_t file_offset; + lldb::offset_t file_size; + ASSERT_TRUE(HostInfoAndroid::ResolveZipPath(file_spec, file_path, + file_offset, file_size)); + + EXPECT_EQ(file_path, zip_path); + EXPECT_EQ(file_offset, 4096UL); + EXPECT_EQ(file_size, 3600UL); +} Index: lldb/unittests/Host/android/CMakeLists.txt =================================================================== --- /dev/null +++ lldb/unittests/Host/android/CMakeLists.txt @@ -0,0 +1,15 @@ +set (FILES + HostInfoAndroidTest.cpp +) + +add_lldb_unittest(HostAndroidTests + ${FILES} + LINK_LIBS + lldbHost + lldbUtilityHelpers + ) + +set(test_inputs + zip-test.zip + ) +add_unittest_inputs(HostAndroidTests "${test_inputs}") Index: lldb/unittests/Host/CMakeLists.txt =================================================================== --- lldb/unittests/Host/CMakeLists.txt +++ lldb/unittests/Host/CMakeLists.txt @@ -38,3 +38,7 @@ LLVMTestingSupport LLVMTargetParser ) + +if (CMAKE_SYSTEM_NAME MATCHES "Linux|Android") + add_subdirectory(android) +endif() Index: lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp =================================================================== --- lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp +++ lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp @@ -1326,10 +1326,33 @@ const FileSpec module_path_spec = FindModuleFile(req_module_path_spec.GetPath(), arch); - const ModuleSpec module_spec(module_path_spec, arch); + + lldb::offset_t file_offset = 0; + lldb::offset_t file_size = 0; +#ifdef __ANDROID__ + // In Android API level 23 and above, dynamic loader is able to load .so file + // directly from zip file. In that case, module_path will be + // "zip_path!/so_path". Resolve the zip file path, .so file offset and size. + // file_offset will be an non-zero for zip .so file, zero for normal file. + std::string file_path; + if (!HostInfoAndroid::ResolveZipPath(module_path_spec, file_path, + file_offset, file_size)) { + return ModuleSpec(); + } + // For zip .so file, this file_path will contain only the actual zip file + // path for the object file processing. Otherwise it is the same as + // module_path. + const FileSpec actual_module_path_spec(file_path); +#else + // It is just module_path_spec reference for other platforms. + const FileSpec &actual_module_path_spec = module_path_spec; +#endif + + const ModuleSpec module_spec(actual_module_path_spec, arch); ModuleSpecList module_specs; - if (!ObjectFile::GetModuleSpecifications(module_path_spec, 0, 0, + if (!ObjectFile::GetModuleSpecifications(actual_module_path_spec, + file_offset, file_size, module_specs)) return ModuleSpec(); @@ -1337,6 +1360,15 @@ if (!module_specs.FindMatchingModuleSpec(module_spec, matched_module_spec)) return ModuleSpec(); +#ifdef __ANDROID__ + if (file_offset > 0) { + // For zip .so file, matched_module_spec contains only the actual zip file + // path. Overwrite the matched_module_spec file spec with the original + // module_path_spec. + *matched_module_spec.GetFileSpecPtr() = module_path_spec; + } +#endif + return matched_module_spec; } Index: lldb/source/Plugins/Platform/Android/PlatformAndroid.cpp =================================================================== --- lldb/source/Plugins/Platform/Android/PlatformAndroid.cpp +++ lldb/source/Plugins/Platform/Android/PlatformAndroid.cpp @@ -238,10 +238,38 @@ const uint64_t src_offset, const uint64_t src_size, const FileSpec &dst_file_spec) { - if (src_offset != 0) - return Status("Invalid offset - %" PRIu64, src_offset); + // In Android API level 23 and above, dynamic loader is able to load .so + // file directly from APK. In that case, src_offset will be an non-zero. + if (src_offset == 0) // Use GetFile for a normal file. + return GetFile(src_file_spec, dst_file_spec); - return GetFile(src_file_spec, dst_file_spec); + std::string source_file = src_file_spec.GetPath(false); + if (source_file.find('\'') != std::string::npos) + return Status("Doesn't support single-quotes in filenames"); + + // For zip .so file, src_file_spec will be "zip_path!/so_path". + // Extract "zip_path" from the source path. + const std::string kZipSeparator = "!/"; + auto pos = source_file.find(kZipSeparator); + if (pos != std::string::npos) + source_file = source_file.substr(0, pos); + + AdbClient adb(m_device_id); + + // Use 'run-as' if necessary. + char run_as_cmd[PATH_MAX]; + if (const char *run_as = std::getenv("ANDROID_PLATFORM_RUN_AS")) + snprintf(run_as_cmd, sizeof(run_as_cmd), "run-as '%s' ", run_as); + else + run_as_cmd[0] = '\0'; + + // Use 'shell dd' to download the file slice with the offset and size. + char cmd[PATH_MAX * 2]; + snprintf(cmd, sizeof(cmd), "%sdd if='%s' iflag=skip_bytes,count_bytes " + "skip=%" PRIu64 " count=%" PRIu64 " status=none", + run_as_cmd, source_file.c_str(), src_offset, src_size); + + return adb.ShellToFile(cmd, minutes(1), dst_file_spec); } Status PlatformAndroid::DisconnectRemote() { Index: lldb/source/Host/android/ZipFile.h =================================================================== --- /dev/null +++ lldb/source/Host/android/ZipFile.h @@ -0,0 +1,32 @@ +//===-- ZipFile.h -----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef lldb_Host_android_ZipFile_h_ +#define lldb_Host_android_ZipFile_h_ + +#include "lldb/lldb-types.h" +#include <string> + +namespace lldb_private { + +/// In Android API level 23 and above, dynamic loader is able to load .so file +/// directly from APK or .zip file. This is a utility class to find .so file +/// offset and size from zip file. +/// https://android.googlesource.com/platform/bionic/+/master/ +/// android-changes-for-ndk-developers.md +/// #opening-shared-libraries-directly-from-an-apk + +class ZipFile { +public: + static bool Find(lldb::DataBufferSP zip_data, const std::string &so_path, + lldb::offset_t &file_offset, lldb::offset_t &file_size); +}; + +} // end of namespace lldb_private + +#endif // #ifndef lldb_Host_android_ZipFile_h_ Index: lldb/source/Host/android/ZipFile.cpp =================================================================== --- /dev/null +++ lldb/source/Host/android/ZipFile.cpp @@ -0,0 +1,177 @@ +//===-- ZipFile.cpp -------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "lldb/Utility/DataBuffer.h" +#include "llvm/Support/Endian.h" +#include "ZipFile.h" + +using namespace lldb_private; +using namespace llvm::support; + +namespace { + +// Zip headers. +// https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT + +// The end of central directory record. +struct EocdRecord { + static constexpr char kSignature[] = {0x50, 0x4b, 0x05, 0x06}; + char signature[sizeof(kSignature)]; + unaligned_uint16_t disks; + unaligned_uint16_t cd_start_disk; + unaligned_uint16_t cds_on_this_disk; + unaligned_uint16_t cd_records; + unaligned_uint32_t cd_size; + unaligned_uint32_t cd_offset; + unaligned_uint16_t comment_length; +}; + +// Logical find limit for the end of central directory record. +const size_t kEocdRecordFindLimit = sizeof(EocdRecord) + + std::numeric_limits<decltype(EocdRecord::comment_length)>::max(); + +// Central directory record. +struct CdRecord { + static constexpr char kSignature[] = {0x50, 0x4b, 0x01, 0x02}; + char signature[sizeof(kSignature)]; + unaligned_uint16_t version_made_by; + unaligned_uint16_t version_needed_to_extract; + unaligned_uint16_t general_purpose_bit_flag; + unaligned_uint16_t compression_method; + unaligned_uint16_t last_modification_time; + unaligned_uint16_t last_modification_date; + unaligned_uint32_t crc32; + unaligned_uint32_t compressed_size; + unaligned_uint32_t uncompressed_size; + unaligned_uint16_t file_name_length; + unaligned_uint16_t extra_field_length; + unaligned_uint16_t comment_length; + unaligned_uint16_t file_start_disk; + unaligned_uint16_t internal_file_attributes; + unaligned_uint32_t external_file_attributes; + unaligned_uint32_t local_file_header_offset; +}; +// Immediately after CdRecord, +// - file name (file_name_length) +// - extra field (extra_field_length) +// - comment (comment_length) + +// Local file header. +struct LocalFileHeader { + static constexpr char kSignature[] = {0x50, 0x4b, 0x03, 0x04}; + char signature[sizeof(kSignature)]; + unaligned_uint16_t version_needed_to_extract; + unaligned_uint16_t general_purpose_bit_flag; + unaligned_uint16_t compression_method; + unaligned_uint16_t last_modification_time; + unaligned_uint16_t last_modification_date; + unaligned_uint32_t crc32; + unaligned_uint32_t compressed_size; + unaligned_uint32_t uncompressed_size; + unaligned_uint16_t file_name_length; + unaligned_uint16_t extra_field_length; +}; +// Immediately after LocalFileHeader, +// - file name (file_name_length) +// - extra field (extra_field_length) +// - file data (should be compressed_size == uncompressed_size, page aligned) + +const EocdRecord *FindEocdRecord(lldb::DataBufferSP zip_data) { + // Find backward the end of central directory record from the end of the zip + // file to the find limit. + auto zip_data_end = zip_data->GetBytes() + zip_data->GetByteSize(); + auto find_limit = zip_data_end - kEocdRecordFindLimit; + auto p = zip_data_end - sizeof(EocdRecord); + for (; p >= zip_data->GetBytes() && p >= find_limit; p--) { + auto eocd = reinterpret_cast<const EocdRecord *>(p); + if (::memcmp(eocd->signature, EocdRecord::kSignature, + sizeof(EocdRecord::kSignature)) == 0) { + // Sanity check the values. + if (eocd->cd_records * sizeof(CdRecord) > eocd->cd_size || + zip_data->GetBytes() + eocd->cd_offset + eocd->cd_size > p) + return nullptr; + + // This is a valid record. + return eocd; + } + } + return nullptr; +} + +bool GetFile(lldb::DataBufferSP zip_data, uint32_t local_file_header_offset, + lldb::offset_t &file_offset, lldb::offset_t &file_size) { + auto local_file_header = reinterpret_cast<const LocalFileHeader *>( + zip_data->GetBytes() + local_file_header_offset); + // The signature should match. + if (::memcmp(local_file_header->signature, LocalFileHeader::kSignature, + sizeof(LocalFileHeader::kSignature)) != 0) + return false; + + auto file_data = reinterpret_cast<const uint8_t *>(local_file_header + 1) + + local_file_header->file_name_length + local_file_header->extra_field_length; + // File should be uncompressed. + if (local_file_header->compressed_size != local_file_header->uncompressed_size) + return false; + + // This file is valid. Return the file offset and size. + file_offset = file_data - zip_data->GetBytes(); + file_size = local_file_header->uncompressed_size; + return true; +} + +bool FindFile(lldb::DataBufferSP zip_data, const EocdRecord *eocd, + const std::string &path, lldb::offset_t &file_offset, + lldb::offset_t &file_size) { + // Find the file from the central directory records. + auto cd = reinterpret_cast<const CdRecord *>(zip_data->GetBytes() + + eocd->cd_offset); + size_t cd_records = eocd->cd_records; + for (size_t i = 0; i < cd_records; i++) { + // The signature should match. + if (::memcmp(cd->signature, CdRecord::kSignature, + sizeof(CdRecord::kSignature)) != 0) + return false; + + // Sanity check the file name values. + auto file_name = reinterpret_cast<const char *>(cd + 1); + size_t file_name_length = cd->file_name_length; + if (file_name + file_name_length >= reinterpret_cast<const char *>(eocd) || + file_name_length == 0) + return false; + + // Compare the file name. + if (path.size() == file_name_length && + ::strncmp(path.c_str(), file_name, file_name_length) == 0) { + // Found the file. + return GetFile(zip_data, cd->local_file_header_offset, file_offset, + file_size); + } else { + // Skip to the next central directory record. + cd = reinterpret_cast<const CdRecord *>( + reinterpret_cast<const char *>(cd) + sizeof(CdRecord) + + cd->file_name_length + cd->extra_field_length + cd->comment_length); + // Sanity check the pointer. + if (reinterpret_cast<const char *>(cd) >= + reinterpret_cast<const char *>(eocd)) + return false; + } + } + + return false; +} + +} // end anonymous namespace + +bool ZipFile::Find(lldb::DataBufferSP zip_data, const std::string &so_path, + lldb::offset_t &file_offset, lldb::offset_t &file_size) { + const EocdRecord *eocd = FindEocdRecord(zip_data); + if (!eocd) + return false; + + return FindFile(zip_data, eocd, so_path, file_offset, file_size); +} Index: lldb/source/Host/android/HostInfoAndroid.cpp =================================================================== --- lldb/source/Host/android/HostInfoAndroid.cpp +++ lldb/source/Host/android/HostInfoAndroid.cpp @@ -11,6 +11,7 @@ #include "lldb/Host/linux/HostInfoLinux.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" +#include "ZipFile.h" using namespace lldb_private; using namespace llvm; @@ -92,3 +93,42 @@ return FileSystem::Instance().Exists(file_spec); } + +bool HostInfoAndroid::ResolveZipPath(const FileSpec &file_spec, + std::string &file_path, + lldb::offset_t &file_offset, + lldb::offset_t &file_size) { + // In Android API level 23 and above, dynamic loader is able to load .so + // file directly from APK. .so file should be page aligned and uncompressed. + // In that case, the path will be "zip_path!/so_path". + // https://android.googlesource.com/platform/bionic/+/master/ + // android-changes-for-ndk-developers.md + // #opening-shared-libraries-directly-from-an-apk + const std::string kZipSeparator = "!/"; + auto path = file_spec.GetPath(); + auto pos = path.find(kZipSeparator); + if (pos == std::string::npos) { + // This path is a normal file. Offset and size should be 0. + file_path = path; + file_offset = 0; + file_size = 0; + return true; + } + + // This path is zip .so path. Try to find the .so file from the zip file. + auto zip_path = path.substr(0, pos); + auto so_path = path.substr(pos + kZipSeparator.size()); + + auto zip_file_spec = FileSpec(zip_path); + auto zip_file_size = FileSystem::Instance().GetByteSize(zip_file_spec); + auto zip_data = FileSystem::Instance().CreateDataBuffer(zip_file_spec, + zip_file_size, + 0); + if (ZipFile::Find(zip_data, so_path, file_offset, file_size)) { + // Found the .so file from the zip file and got the file offset and size. + file_path = zip_path; + return true; + } + + return false; +} Index: lldb/source/Host/CMakeLists.txt =================================================================== --- lldb/source/Host/CMakeLists.txt +++ lldb/source/Host/CMakeLists.txt @@ -106,12 +106,19 @@ linux/LibcGlue.cpp linux/Support.cpp ) + + set(ANDROID_SOURCES + android/ZipFile.cpp + android/HostInfoAndroid.cpp + ) if (CMAKE_SYSTEM_NAME MATCHES "Android") - add_host_subdirectory(android - android/HostInfoAndroid.cpp + list(APPEND ANDROID_SOURCES android/LibcGlue.cpp ) endif() + add_host_subdirectory(android + ${ANDROID_SOURCES} + ) elseif (CMAKE_SYSTEM_NAME MATCHES "FreeBSD") add_host_subdirectory(freebsd freebsd/Host.cpp Index: lldb/include/lldb/Host/android/HostInfoAndroid.h =================================================================== --- lldb/include/lldb/Host/android/HostInfoAndroid.h +++ lldb/include/lldb/Host/android/HostInfoAndroid.h @@ -20,6 +20,10 @@ static FileSpec GetDefaultShell(); static FileSpec ResolveLibraryPath(const std::string &path, const ArchSpec &arch); + static bool ResolveZipPath(const FileSpec &file_spec, + std::string &file_path, + lldb::offset_t &file_offset, + lldb::offset_t &file_size); protected: static void ComputeHostArchitectureSupport(ArchSpec &arch_32,
_______________________________________________ lldb-commits mailing list lldb-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
