From de01dd88734ba5c211618e82fe19232bffe8bfd1 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 7 Apr 2022 00:28:05 -0400 Subject: [PATCH 01/23] Enable Snappy compression support in LevelDb in cmake builds --- CMakeLists.txt | 6 + cmake/external/CMakeLists.txt | 1 + cmake/external/leveldb.cmake | 9 + cmake/external/leveldb_patch.py | 144 ++++++++++++ cmake/external/leveldb_patch_test.py | 328 +++++++++++++++++++++++++++ cmake/external/snappy.cmake | 39 ++++ 6 files changed, 527 insertions(+) create mode 100644 cmake/external/leveldb_patch.py create mode 100644 cmake/external/leveldb_patch_test.py create mode 100644 cmake/external/snappy.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index fa81ec2cee9..8ac7e437d77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -227,6 +227,12 @@ if(NOT ZLIB_FOUND) endif() +# Snappy +set(SNAPPY_BUILD_TESTS OFF CACHE BOOL "Firestore disabled") +set(SNAPPY_BUILD_BENCHMARKS OFF CACHE BOOL "Firestore disabled") +add_external_subdirectory(snappy) +firebase_ios_add_alias(Snappy::Snappy snappy) + # LevelDB set(LEVELDB_BUILD_TESTS OFF CACHE BOOL "Firestore disabled") set(LEVELDB_BUILD_BENCHMARKS OFF CACHE BOOL "Firestore disabled") diff --git a/cmake/external/CMakeLists.txt b/cmake/external/CMakeLists.txt index 2179633a849..c1de37b6d09 100644 --- a/cmake/external/CMakeLists.txt +++ b/cmake/external/CMakeLists.txt @@ -30,6 +30,7 @@ include(c-ares) include(googletest) include(GoogleUtilities) include(grpc) +include(snappy) include(leveldb) include(libfuzzer) include(nanopb) diff --git a/cmake/external/leveldb.cmake b/cmake/external/leveldb.cmake index b71a775353a..c9b691ee2c1 100644 --- a/cmake/external/leveldb.cmake +++ b/cmake/external/leveldb.cmake @@ -13,6 +13,7 @@ # limitations under the License. include(ExternalProject) +include(FindPythonInterp) if(TARGET leveldb) return() @@ -20,9 +21,16 @@ endif() set(version 1.22) +ExternalProject_Get_property(snappy SOURCE_DIR) +set(snappy_source_dir "${SOURCE_DIR}") +ExternalProject_Get_property(snappy BINARY_DIR) +set(snappy_binary_dir "${BINARY_DIR}") + ExternalProject_Add( leveldb + DEPENDS snappy + DOWNLOAD_DIR ${FIREBASE_DOWNLOAD_DIR} DOWNLOAD_NAME leveldb-${version}.tar.gz URL https://github.com/google/leveldb/archive/${version}.tar.gz @@ -34,6 +42,7 @@ ExternalProject_Add( BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + PATCH_COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/leveldb_patch.py --snappy-source-dir ${snappy_source_dir} --snappy-binary-dir ${snappy_binary_dir} HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}" ) diff --git a/cmake/external/leveldb_patch.py b/cmake/external/leveldb_patch.py new file mode 100644 index 00000000000..51a62d54ad9 --- /dev/null +++ b/cmake/external/leveldb_patch.py @@ -0,0 +1,144 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Modify the CMakeLists.txt from LevelDb to staticly link Snappy compression +support. +""" + +import argparse +import dataclasses +import os +import pathlib +from typing import Iterable, Sequence + + +def main() -> None: + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument("--snappy-source-dir", required=True) + arg_parser.add_argument("--snappy-binary-dir", required=True) + parsed_args = arg_parser.parse_args() + del arg_parser + snappy_source_dir = pathlib.Path(parsed_args.snappy_source_dir) + snappy_binary_dir = pathlib.Path(parsed_args.snappy_binary_dir) + del parsed_args + + cmakelists_txt_file = pathlib.Path("CMakeLists.txt") + with cmakelists_txt_file.open("rt", encoding="utf8") as f: + lines = tuple(f) + + patcher = CMakeListsPatcher( + lines, + os.path.abspath(__file__), + snappy_source_dir, + snappy_binary_dir, + ) + + patched_lines = tuple(patcher.patch()) + + with cmakelists_txt_file.open("wt", encoding="utf8") as f: + f.writelines(patched_lines) + + +@dataclasses.dataclass(frozen=True) +class LineComponents: + full: str + indent: str + line: str + eol: str + + +class CMakeListsPatcher: + + SNAPPY_DETECT_LINE = \ + """check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)""" + SNAPPY_INCLUDE_LINE = \ + "target_include_directories(leveldb" + SNAPPY_LINK_LINE = \ + "target_link_libraries(leveldb snappy)" + + def __init__( + self, + lines: Sequence[str], + script_name: str, + snappy_source_dir: pathlib.Path, + snappy_binary_dir: pathlib.Path) -> None: + self.i = 0 + self.lines = lines + self.script_name = script_name + self.snappy_source_dir_str = snappy_source_dir.as_posix() + self.snappy_binary_dir_str = snappy_binary_dir.as_posix() + + def patch(self) -> Iterable[str]: + while self.i < len(self.lines): + full_line = self.lines[self.i] + line = self._split_line(full_line) + self.i += 1 + + if line.line == self.SNAPPY_DETECT_LINE: + yield from self._on_snappy_detect_line(line) + elif line.line == self.SNAPPY_INCLUDE_LINE: + yield full_line + yield from self._on_leveldb_include_start() + elif line.line == self.SNAPPY_LINK_LINE: + yield from self._on_leveldb_snappy_link_line(line) + else: + yield full_line + + def _begin_mod_line(self, mod_name: str) -> str: + return f"# BEGIN: {mod_name} modification by {self.script_name}" + + def _end_mod_line(self, mod_name: str) -> str: + return f"# END: {mod_name} modification by {self.script_name}" + + def _on_snappy_detect_line(self, line: LineComponents) -> Iterable[str]: + yield self._begin_mod_line("snappy_detect_line") + line.eol + yield line.indent + "# " + line.line + line.eol + yield line.indent + """set(HAVE_SNAPPY ON CACHE BOOL "")""" + line.eol + yield self._end_mod_line("snappy_detect_line") + line.eol + + def _on_leveldb_include_start(self) -> Iterable[str]: + line1 = self._split_line(self.lines[self.i]) + line2 = self._split_line(self.lines[self.i + 1]) + begin_mod_line = self._begin_mod_line("leveldb_include_start") + + if line1.line == begin_mod_line: + return + + yield begin_mod_line + line1.eol + yield line1.indent + "PRIVATE" + line1.eol + yield line2.indent + self.snappy_source_dir_str + line2.eol + yield line2.indent + self.snappy_binary_dir_str + line2.eol + yield self._end_mod_line("leveldb_include_start") + line1.eol + + def _on_leveldb_snappy_link_line(self, line: LineComponents) -> Iterable[str]: + yield self._begin_mod_line("leveldb_snappy_link_line") + line.eol + yield line.indent + "# " + line.line + line.eol + yield line.indent + f"target_link_libraries(leveldb Snappy::Snappy)" + line.eol + yield self._end_mod_line("leveldb_snappy_link_line") + line.eol + + def _split_line(self, line: str) -> LineComponents: + line_rstripped = line.rstrip() + eol = line[len(line_rstripped):] + line_stripped = line_rstripped.strip() + indent = line_rstripped[:len(line_rstripped) - len(line_stripped)] + return LineComponents(full=line, indent=indent, line=line_stripped, eol=eol) + + +class LeveDbPatchException(Exception): + pass + + +if __name__ == "__main__": + main() diff --git a/cmake/external/leveldb_patch_test.py b/cmake/external/leveldb_patch_test.py new file mode 100644 index 00000000000..b1d62526b92 --- /dev/null +++ b/cmake/external/leveldb_patch_test.py @@ -0,0 +1,328 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import leveldb_patch +import pathlib +import unittest + + +class CMakeListsPatcherTest(unittest.TestCase): + + def setUp(self): + super().setUp() + self.sample_snappy_source_dir = pathlib.Path("a/b/snappy_source_dir") + self.sample_snappy_binary_dir = pathlib.Path("a/b/snappy_binary_dir") + + def test_snappy_detect_line_is_commented_and_replaced(self): + lines = ( + """check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)""", + ) + patcher = leveldb_patch.CMakeListsPatcher( + lines, + "MyCoolScript", + self.sample_snappy_source_dir, + self.sample_snappy_binary_dir, + ) + + patched_lines = tuple(patcher.patch()) + + self.assertSequenceEqual(patched_lines, [ + "# BEGIN: snappy_detect_line modification by MyCoolScript", + """# check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)""", + """set(HAVE_SNAPPY ON CACHE BOOL "")""", + "# END: snappy_detect_line modification by MyCoolScript", + ]) + + def test_snappy_detect_line_has_indent_and_eol_preserved(self): + lines = ( + """ check_library_exists(snappy snappy_compress "" HAVE_SNAPPY) \n""", + ) + patcher = leveldb_patch.CMakeListsPatcher( + lines, + "MyCoolScript", + self.sample_snappy_source_dir, + self.sample_snappy_binary_dir, + ) + + patched_lines = tuple(patcher.patch()) + + self.assertSequenceEqual(patched_lines, [ + "# BEGIN: snappy_detect_line modification by MyCoolScript \n", + """ # check_library_exists(snappy snappy_compress "" HAVE_SNAPPY) \n""", + """ set(HAVE_SNAPPY ON CACHE BOOL "") \n""", + "# END: snappy_detect_line modification by MyCoolScript \n", + ]) + + def test_snappy_detect_line_does_not_affect_surrounding_lines(self): + lines = ( + "aaa", + "bbb", + """check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)""", + "ccc", + "ddd", + ) + patcher = leveldb_patch.CMakeListsPatcher( + lines, + "MyCoolScript", + self.sample_snappy_source_dir, + self.sample_snappy_binary_dir, + ) + + patched_lines = tuple(patcher.patch()) + + self.assertSequenceEqual(patched_lines, [ + "aaa", + "bbb", + "# BEGIN: snappy_detect_line modification by MyCoolScript", + """# check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)""", + """set(HAVE_SNAPPY ON CACHE BOOL "")""", + "# END: snappy_detect_line modification by MyCoolScript", + "ccc", + "ddd", + ]) + + def test_snappy_include_is_amended(self): + lines = ( + "target_include_directories(leveldb", + "PUBLIC", + "path1", + "path2", + ")", + ) + patcher = leveldb_patch.CMakeListsPatcher( + lines, + script_name="MyCoolSript", + snappy_source_dir=pathlib.Path("a/b"), + snappy_binary_dir=pathlib.Path("c/d"), + ) + + patched_lines = tuple(patcher.patch()) + + self.assertSequenceEqual(patched_lines, [ + "target_include_directories(leveldb", + "# BEGIN: leveldb_include_start modification by MyCoolSript", + "PRIVATE", + "a/b", + "c/d", + "# END: leveldb_include_start modification by MyCoolSript", + "PUBLIC", + "path1", + "path2", + ")", + ]) + + def test_snappy_include_lines_adopt_indenting_and_eol_convention(self): + lines = ( + "target_include_directories(leveldb\n", + " PUBLIC \n", + " path1 \n", + " path2 \n", + ")\n", + ) + patcher = leveldb_patch.CMakeListsPatcher( + lines, + script_name="MyCoolSript", + snappy_source_dir=pathlib.Path("a/b"), + snappy_binary_dir=pathlib.Path("c/d"), + ) + + patched_lines = tuple(patcher.patch()) + + self.assertSequenceEqual(patched_lines, [ + "target_include_directories(leveldb\n", + "# BEGIN: leveldb_include_start modification by MyCoolSript \n", + " PRIVATE \n", + " a/b \n", + " c/d \n", + "# END: leveldb_include_start modification by MyCoolSript \n", + " PUBLIC \n", + " path1 \n", + " path2 \n", + ")\n", + ]) + + def test_snappy_include_line_does_not_affect_surrounding_lines(self): + lines = ( + "aaa", + "bbb", + "target_include_directories(leveldb", + "PUBLIC", + "path1", + "path2", + ")", + "ccc", + "ddd", + ) + patcher = leveldb_patch.CMakeListsPatcher( + lines, + script_name="MyCoolSript", + snappy_source_dir=pathlib.Path("a/b"), + snappy_binary_dir=pathlib.Path("c/d"), + ) + + patched_lines = tuple(patcher.patch()) + + self.assertSequenceEqual(patched_lines, [ + "aaa", + "bbb", + "target_include_directories(leveldb", + "# BEGIN: leveldb_include_start modification by MyCoolSript", + "PRIVATE", + "a/b", + "c/d", + "# END: leveldb_include_start modification by MyCoolSript", + "PUBLIC", + "path1", + "path2", + ")", + "ccc", + "ddd", + ]) + + def test_leveldb_snappy_link_line_is_commented_and_replaced(self): + lines = ( + "target_link_libraries(leveldb snappy)", + ) + patcher = leveldb_patch.CMakeListsPatcher( + lines, + script_name="MyCoolSript", + snappy_source_dir=pathlib.Path("a/b"), + snappy_binary_dir=pathlib.Path("c/d"), + ) + + patched_lines = tuple(patcher.patch()) + + self.assertSequenceEqual(patched_lines, [ + "# BEGIN: leveldb_snappy_link_line modification by MyCoolSript", + "# target_link_libraries(leveldb snappy)", + "target_link_libraries(leveldb Snappy::Snappy)", + "# END: leveldb_snappy_link_line modification by MyCoolSript", + ]) + + def test_leveldb_snappy_link_line_has_indent_and_eol_preserved(self): + lines = ( + " target_link_libraries(leveldb snappy) \n", + ) + patcher = leveldb_patch.CMakeListsPatcher( + lines, + script_name="MyCoolSript", + snappy_source_dir=pathlib.Path("a/b"), + snappy_binary_dir=pathlib.Path("c/d"), + ) + + patched_lines = tuple(patcher.patch()) + + self.assertSequenceEqual(patched_lines, [ + "# BEGIN: leveldb_snappy_link_line modification by MyCoolSript \n", + " # target_link_libraries(leveldb snappy) \n", + " target_link_libraries(leveldb Snappy::Snappy) \n", + "# END: leveldb_snappy_link_line modification by MyCoolSript \n", + ]) + + def test_leveldb_snappy_link_line_does_not_affect_surrounding_lines(self): + lines = ( + "aaa", + "bbb", + "target_link_libraries(leveldb snappy)", + "ccc", + "ddd", + ) + patcher = leveldb_patch.CMakeListsPatcher( + lines, + script_name="MyCoolSript", + snappy_source_dir=pathlib.Path("a/b"), + snappy_binary_dir=pathlib.Path("c/d"), + ) + + patched_lines = tuple(patcher.patch()) + + self.assertSequenceEqual(patched_lines, [ + "aaa", + "bbb", + "# BEGIN: leveldb_snappy_link_line modification by MyCoolSript", + "# target_link_libraries(leveldb snappy)", + "target_link_libraries(leveldb Snappy::Snappy)", + "# END: leveldb_snappy_link_line modification by MyCoolSript", + "ccc", + "ddd", + ]) + + def test_all_patches_combined(self): + lines = ( + """check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)""", + "target_include_directories(leveldb", + "PUBLIC", + "path1", + ")", + "target_link_libraries(leveldb snappy)", + ) + + patcher = leveldb_patch.CMakeListsPatcher( + lines, + script_name="MyCoolSript", + snappy_source_dir=pathlib.Path("a/b"), + snappy_binary_dir=pathlib.Path("c/d"), + ) + patched_lines = tuple(patcher.patch()) + + self.assertSequenceEqual(patched_lines, [ + "# BEGIN: snappy_detect_line modification by MyCoolSript", + """# check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)""", + """set(HAVE_SNAPPY ON CACHE BOOL "")""", + "# END: snappy_detect_line modification by MyCoolSript", + "target_include_directories(leveldb", + "# BEGIN: leveldb_include_start modification by MyCoolSript", + "PRIVATE", + "a/b", + "c/d", + "# END: leveldb_include_start modification by MyCoolSript", + "PUBLIC", + "path1", + ")", + "# BEGIN: leveldb_snappy_link_line modification by MyCoolSript", + "# target_link_libraries(leveldb snappy)", + "target_link_libraries(leveldb Snappy::Snappy)", + "# END: leveldb_snappy_link_line modification by MyCoolSript", + ]) + + def test_idempotence(self): + lines = ( + """check_library_exists(snappy snappy_compress "" HAVE_SNAPPY)\n""", + "target_include_directories(leveldb", + "PUBLIC", + "path1", + ")", + "target_link_libraries(leveldb snappy)", + ) + + patcher1 = leveldb_patch.CMakeListsPatcher( + lines, + script_name="MyCoolSript", + snappy_source_dir=pathlib.Path("a/b"), + snappy_binary_dir=pathlib.Path("c/d"), + ) + patched_lines1 = tuple(patcher1.patch()) + patcher2 = leveldb_patch.CMakeListsPatcher( + patched_lines1, + script_name="MyCoolSript", + snappy_source_dir=pathlib.Path("a/b"), + snappy_binary_dir=pathlib.Path("c/d"), + ) + patched_lines2 = tuple(patcher2.patch()) + + self.assertSequenceEqual(patched_lines1, patched_lines2) + + +if __name__ == "__main__": + unittest.main() diff --git a/cmake/external/snappy.cmake b/cmake/external/snappy.cmake new file mode 100644 index 00000000000..9f25c03d086 --- /dev/null +++ b/cmake/external/snappy.cmake @@ -0,0 +1,39 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include(ExternalProject) + +if(TARGET snappy) + return() +endif() + +set(version 1.1.9) + +ExternalProject_Add( + snappy + + DOWNLOAD_DIR ${FIREBASE_DOWNLOAD_DIR} + DOWNLOAD_NAME snappy-${version}.tar.gz + URL https://github.com/google/snappy/archive/refs/tags/${version}.tar.gz + URL_HASH SHA256=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7 + + PREFIX ${PROJECT_BINARY_DIR} + + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + + HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}" +) From db6461437f8a0dc09596ee03961935ee02987748 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 7 Apr 2022 16:01:12 -0400 Subject: [PATCH 02/23] leveldb_snappy_test.cc: added a test to verify Snappy compression support in LevelDb --- .../test/unit/local/leveldb_snappy_test.cc | 226 ++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 Firestore/core/test/unit/local/leveldb_snappy_test.cc diff --git a/Firestore/core/test/unit/local/leveldb_snappy_test.cc b/Firestore/core/test/unit/local/leveldb_snappy_test.cc new file mode 100644 index 00000000000..f15733b0114 --- /dev/null +++ b/Firestore/core/test/unit/local/leveldb_snappy_test.cc @@ -0,0 +1,226 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "Firestore/core/src/local/leveldb_util.h" +#include "Firestore/core/src/util/filesystem.h" +#include "Firestore/core/src/util/path.h" +#include "Firestore/core/test/unit/local/persistence_testing.h" + +#include "gtest/gtest.h" +#include "leveldb/db.h" + +namespace { + +using ::firebase::firestore::local::ConvertStatus; +using ::firebase::firestore::util::Filesystem; +using ::firebase::firestore::util::Path; + +// Creates a LevelDb database that uses Snappy compression for at least one of +// its blocks. Attempting to iterate over this database using a LevelDb library +// that does not have Snappy compression compiled in will return a failed status +// with reason "corruption". +Path CreateLevelDbDatabaseThatUsesSnappyCompression(); + +// This test ensures that we don't accidentally regress having added in Snappy +// compression support (https://github.com/firebase/firebase-ios-sdk/pull/9596). +TEST(LevelDbSnappy, LevelDbHasSnappySupportCompiledIn) { + Path leveldb_path = CreateLevelDbDatabaseThatUsesSnappyCompression(); + if (HasFatalFailure()) return; + + leveldb::Options options; + options.create_if_missing = false; + + std::unique_ptr db; + { + leveldb::DB* db_ptr; + leveldb::Status status = + leveldb::DB::Open(options, leveldb_path.ToUtf8String(), &db_ptr); + ASSERT_TRUE(status.ok()); + db.reset(db_ptr); + } + + // One of the assertions below will fail when LevelDb attempts to read a block + // that is compressed with Snappy and Snappy compression support is not + // compiled in. + std::unique_ptr it( + db->NewIterator(leveldb::ReadOptions())); + for (it->SeekToFirst(); it->Valid(); it->Next()) { + ASSERT_TRUE(it->status().ok()) << ConvertStatus(it->status()); + } + ASSERT_TRUE(it->status().ok()) << ConvertStatus(it->status()); +} + +const std::array LevelDbSnappyFile_000005_ldb{ + 0x84, 0x03, 0x80, 0x00, 0x42, 0x00, 0x85, 0x71, 0x75, 0x65, 0x72, 0x79, + 0x5F, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x00, 0x01, 0x8B, 0x43, 0x6F, + 0x6C, 0x41, 0x2F, 0x44, 0x6F, 0x63, 0x41, 0x2F, 0x43, 0x6F, 0x6C, 0x42, + 0x01, 0x0A, 0x68, 0x42, 0x7C, 0x66, 0x3A, 0x7C, 0x6F, 0x62, 0x3A, 0x5F, + 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x5F, 0x5F, 0x61, 0x73, 0x63, 0x00, 0x01, + 0x8C, 0x82, 0x80, 0x01, 0x07, 0x00, 0x05, 0x01, 0x08, 0x01, 0x13, 0x50, + 0x11, 0x3E, 0x01, 0x16, 0x00, 0x0A, 0x05, 0x15, 0xF0, 0x3C, 0x00, 0x08, + 0x02, 0x20, 0x05, 0x32, 0x4A, 0x12, 0x48, 0x70, 0x72, 0x6F, 0x6A, 0x65, + 0x63, 0x74, 0x73, 0x2F, 0x54, 0x65, 0x73, 0x74, 0x54, 0x65, 0x72, 0x6D, + 0x69, 0x6E, 0x61, 0x74, 0x65, 0x2F, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, + 0x73, 0x65, 0x73, 0x2F, 0x28, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6C, 0x74, + 0x29, 0x2F, 0x64, 0x6F, 0x63, 0x75, 0x6D, 0x65, 0x6E, 0x74, 0x73, 0x01, + 0x7B, 0x3E, 0x85, 0x00, 0x0C, 0x0D, 0x07, 0x50, 0x08, 0x15, 0x5A, 0x00, + 0x03, 0xFE, 0x5A, 0x00, 0x2E, 0x5A, 0x00, 0x38, 0x07, 0x12, 0x06, 0x5F, + 0x67, 0x6C, 0x6F, 0x62, 0x61, 0x6C, 0x00, 0x01, 0x80, 0x01, 0x0B, 0x11, + 0x65, 0x1C, 0x10, 0x05, 0x20, 0x01, 0x12, 0x07, 0x06, 0x09, 0x15, 0x10, + 0x00, 0x03, 0x01, 0x10, 0x04, 0x00, 0x01, 0x09, 0x10, 0x24, 0x01, 0x12, + 0x01, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6F, 0x6E, 0x01, 0x35, 0x00, 0x06, + 0x09, 0x15, 0x10, 0x37, 0x0C, 0x07, 0x01, 0x05, 0x09, 0x0B, 0x10, 0x36, + 0x0C, 0x07, 0x01, 0x04, 0x09, 0x0B, 0x10, 0x35, 0x0C, 0x07, 0x01, 0x03, + 0x09, 0x0B, 0x4C, 0x34, 0x0C, 0x07, 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x2C, 0x6E, 0xE0, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0xC0, 0xF2, 0xA1, 0xB0, 0x00, 0x09, 0x03, 0x86, 0x01, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x87, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x58, 0xC2, 0x94, 0x06, 0x8C, 0x02, 0x08, + 0x99, 0x02, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x57, 0xFB, 0x80, 0x8B, 0x24, 0x75, 0x47, 0xDB, +}; +const std::array LevelDbSnappyFile_000017_ldb{ + 0x00, 0x14, 0x50, 0x85, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x00, 0x01, + 0x8C, 0x82, 0x80, 0x01, 0x0C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + 0x02, 0x20, 0x0A, 0x32, 0x4A, 0x12, 0x48, 0x70, 0x72, 0x6F, 0x6A, 0x65, + 0x63, 0x74, 0x73, 0x2F, 0x54, 0x65, 0x73, 0x74, 0x54, 0x65, 0x72, 0x6D, + 0x69, 0x6E, 0x61, 0x74, 0x65, 0x2F, 0x64, 0x61, 0x74, 0x61, 0x62, 0x61, + 0x73, 0x65, 0x73, 0x2F, 0x28, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6C, 0x74, + 0x29, 0x2F, 0x64, 0x6F, 0x63, 0x75, 0x6D, 0x65, 0x6E, 0x74, 0x73, 0x2F, + 0x43, 0x6F, 0x6C, 0x41, 0x2F, 0x44, 0x6F, 0x63, 0x41, 0x2F, 0x43, 0x6F, + 0x6C, 0x42, 0x2F, 0x44, 0x6F, 0x63, 0x42, 0x07, 0x12, 0x06, 0x5F, 0x67, + 0x6C, 0x6F, 0x62, 0x61, 0x6C, 0x00, 0x01, 0x80, 0x01, 0x0D, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x02, 0x10, 0x0A, 0x20, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0xFE, 0xCD, 0xE0, 0x39, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0xC0, 0xF2, 0xA1, 0xB0, + 0x00, 0x09, 0x03, 0x86, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x8A, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0xE4, 0xA7, 0x7E, 0x74, 0x8F, 0x01, 0x08, 0x9C, 0x01, 0x17, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0xFB, 0x80, 0x8B, + 0x24, 0x75, 0x47, 0xDB, +}; +const std::array LevelDbSnappyFile_000085_ldb{}; +const std::array LevelDbSnappyFile_CURRENT{ + 0x4D, 0x41, 0x4E, 0x49, 0x46, 0x45, 0x53, 0x54, + 0x2D, 0x30, 0x30, 0x30, 0x30, 0x38, 0x34, 0x0A, +}; +const std::array LevelDbSnappyFile_LOG_old{ + 0x32, 0x30, 0x32, 0x32, 0x2F, 0x30, 0x34, 0x2F, 0x30, 0x34, 0x2D, 0x31, + 0x31, 0x3A, 0x33, 0x39, 0x3A, 0x34, 0x36, 0x2E, 0x32, 0x35, 0x37, 0x32, + 0x35, 0x31, 0x20, 0x30, 0x78, 0x37, 0x30, 0x30, 0x30, 0x30, 0x35, 0x33, + 0x31, 0x34, 0x30, 0x30, 0x30, 0x20, 0x52, 0x65, 0x63, 0x6F, 0x76, 0x65, + 0x72, 0x69, 0x6E, 0x67, 0x20, 0x6C, 0x6F, 0x67, 0x20, 0x23, 0x38, 0x31, + 0x0A, 0x32, 0x30, 0x32, 0x32, 0x2F, 0x30, 0x34, 0x2F, 0x30, 0x34, 0x2D, + 0x31, 0x31, 0x3A, 0x33, 0x39, 0x3A, 0x34, 0x36, 0x2E, 0x33, 0x30, 0x34, + 0x35, 0x35, 0x32, 0x20, 0x30, 0x78, 0x37, 0x30, 0x30, 0x30, 0x30, 0x35, + 0x33, 0x31, 0x34, 0x30, 0x30, 0x30, 0x20, 0x44, 0x65, 0x6C, 0x65, 0x74, + 0x65, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3D, 0x33, 0x20, 0x23, 0x38, 0x30, + 0x0A, 0x32, 0x30, 0x32, 0x32, 0x2F, 0x30, 0x34, 0x2F, 0x30, 0x34, 0x2D, + 0x31, 0x31, 0x3A, 0x33, 0x39, 0x3A, 0x34, 0x36, 0x2E, 0x33, 0x30, 0x35, + 0x30, 0x36, 0x34, 0x20, 0x30, 0x78, 0x37, 0x30, 0x30, 0x30, 0x30, 0x35, + 0x33, 0x31, 0x34, 0x30, 0x30, 0x30, 0x20, 0x44, 0x65, 0x6C, 0x65, 0x74, + 0x65, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3D, 0x30, 0x20, 0x23, 0x38, 0x31, + 0x0A, +}; +const std::array LevelDbSnappyFile_LOG{ + 0x32, 0x30, 0x32, 0x32, 0x2F, 0x30, 0x34, 0x2F, 0x30, 0x34, 0x2D, 0x31, + 0x31, 0x3A, 0x35, 0x36, 0x3A, 0x35, 0x36, 0x2E, 0x34, 0x39, 0x33, 0x31, + 0x34, 0x32, 0x20, 0x30, 0x78, 0x37, 0x30, 0x30, 0x30, 0x30, 0x61, 0x32, + 0x35, 0x34, 0x30, 0x30, 0x30, 0x20, 0x52, 0x65, 0x63, 0x6F, 0x76, 0x65, + 0x72, 0x69, 0x6E, 0x67, 0x20, 0x6C, 0x6F, 0x67, 0x20, 0x23, 0x38, 0x33, + 0x0A, 0x32, 0x30, 0x32, 0x32, 0x2F, 0x30, 0x34, 0x2F, 0x30, 0x34, 0x2D, + 0x31, 0x31, 0x3A, 0x35, 0x36, 0x3A, 0x35, 0x36, 0x2E, 0x35, 0x33, 0x34, + 0x37, 0x34, 0x35, 0x20, 0x30, 0x78, 0x37, 0x30, 0x30, 0x30, 0x30, 0x61, + 0x32, 0x35, 0x34, 0x30, 0x30, 0x30, 0x20, 0x44, 0x65, 0x6C, 0x65, 0x74, + 0x65, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3D, 0x33, 0x20, 0x23, 0x38, 0x32, + 0x0A, 0x32, 0x30, 0x32, 0x32, 0x2F, 0x30, 0x34, 0x2F, 0x30, 0x34, 0x2D, + 0x31, 0x31, 0x3A, 0x35, 0x36, 0x3A, 0x35, 0x36, 0x2E, 0x35, 0x33, 0x35, + 0x32, 0x34, 0x32, 0x20, 0x30, 0x78, 0x37, 0x30, 0x30, 0x30, 0x30, 0x61, + 0x32, 0x35, 0x34, 0x30, 0x30, 0x30, 0x20, 0x44, 0x65, 0x6C, 0x65, 0x74, + 0x65, 0x20, 0x74, 0x79, 0x70, 0x65, 0x3D, 0x30, 0x20, 0x23, 0x38, 0x33, + 0x0A, +}; +const std::array LevelDbSnappyFile_MANIFEST_000084{ + 0x45, 0x63, 0x9F, 0xDD, 0xAC, 0x00, 0x01, 0x01, 0x1A, 0x6C, 0x65, 0x76, + 0x65, 0x6C, 0x64, 0x62, 0x2E, 0x42, 0x79, 0x74, 0x65, 0x77, 0x69, 0x73, + 0x65, 0x43, 0x6F, 0x6D, 0x70, 0x61, 0x72, 0x61, 0x74, 0x6F, 0x72, 0x07, + 0x00, 0x05, 0xE5, 0x02, 0x42, 0x85, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5F, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x00, 0x01, 0x8B, 0x43, 0x6F, 0x6C, + 0x41, 0x2F, 0x44, 0x6F, 0x63, 0x41, 0x2F, 0x43, 0x6F, 0x6C, 0x42, 0x2F, + 0x44, 0x6F, 0x63, 0x42, 0x7C, 0x66, 0x3A, 0x7C, 0x6F, 0x62, 0x3A, 0x5F, + 0x5F, 0x6E, 0x61, 0x6D, 0x65, 0x5F, 0x5F, 0x61, 0x73, 0x63, 0x00, 0x01, + 0x8C, 0x82, 0x80, 0x01, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, + 0x85, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6F, 0x6E, 0x00, 0x01, 0x80, 0x01, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0x11, 0xE8, 0x01, + 0x14, 0x85, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x00, 0x01, 0x8C, 0x82, + 0x80, 0x01, 0x0C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x19, 0x85, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x5F, 0x67, 0x6C, 0x6F, 0x62, 0x61, 0x6C, + 0x00, 0x01, 0x80, 0x01, 0x0D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB1, + 0x03, 0xAC, 0xBA, 0x08, 0x00, 0x01, 0x02, 0x55, 0x09, 0x00, 0x03, 0x56, + 0x04, 0x0D, +}; + +template +void WriteFile(const Path& dir, + const std::string& file_name, + const T& data_array) { + Filesystem* fs = Filesystem::Default(); + { + auto status = fs->RecursivelyCreateDir(dir); + if (!status.ok()) { + FAIL() << "Creating directory failed: " << dir.ToUtf8String() << " (" + << status.error_message() << ")"; + } + } + + Path file = dir.AppendUtf8(file_name); + std::ofstream out_file(file.native_value(), std::ios::binary); + if (!out_file) { + FAIL() << "Unable to open file for writing: " << file.ToUtf8String(); + } + + out_file.write(reinterpret_cast(data_array.data()), + data_array.size()); + out_file.close(); + if (!out_file) { + FAIL() << "Writing to file failed: " << file.ToUtf8String(); + } +} + +Path CreateLevelDbDatabaseThatUsesSnappyCompression() { + Path leveldb_dir = ::firebase::firestore::local::LevelDbDir(); + WriteFile(leveldb_dir, "000005.ldb", LevelDbSnappyFile_000005_ldb); + WriteFile(leveldb_dir, "000017.ldb", LevelDbSnappyFile_000017_ldb); + WriteFile(leveldb_dir, "000085.ldb", LevelDbSnappyFile_000085_ldb); + WriteFile(leveldb_dir, "CURRENT", LevelDbSnappyFile_CURRENT); + WriteFile(leveldb_dir, "LOG.old", LevelDbSnappyFile_LOG_old); + WriteFile(leveldb_dir, "LOG", LevelDbSnappyFile_LOG); + WriteFile(leveldb_dir, "MANIFEST-000084", LevelDbSnappyFile_MANIFEST_000084); + return leveldb_dir; +} + +} // namespace From ffe219d8190de90cdef593195dfa8e9a167e652d Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Fri, 8 Apr 2022 00:04:28 -0400 Subject: [PATCH 03/23] project.pbxproj: add leveldb_snappy_test.cc --- .../Example/Firestore.xcodeproj/project.pbxproj | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Firestore/Example/Firestore.xcodeproj/project.pbxproj b/Firestore/Example/Firestore.xcodeproj/project.pbxproj index aaed7e5f4c4..640bfa6b994 100644 --- a/Firestore/Example/Firestore.xcodeproj/project.pbxproj +++ b/Firestore/Example/Firestore.xcodeproj/project.pbxproj @@ -45,6 +45,7 @@ 06A3926F89C847846BE4D6BE /* http.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = 618BBE9720B89AAC00B5BCE7 /* http.pb.cc */; }; 06BCEB9C65DFAA142F3D3F0B /* view_testing.cc in Sources */ = {isa = PBXBuildFile; fileRef = A5466E7809AD2871FFDE6C76 /* view_testing.cc */; }; 072D805A94E767DE4D371881 /* FSTSyncEngineTestDriver.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E02E20213FFC00B64F25 /* FSTSyncEngineTestDriver.mm */; }; + 077292C9797D97D3851F15CE /* leveldb_snappy_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = D9D94300B9C02F7069523C00 /* leveldb_snappy_test.cc */; }; 0794FACCB1C0C4881A76C28D /* value_util_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 40F9D09063A07F710811A84F /* value_util_test.cc */; }; 079E63E270F3EFCA175D2705 /* cc_compilation_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1B342370EAE3AA02393E33EB /* cc_compilation_test.cc */; }; 07A64E6C4EB700E3AF3FD496 /* document_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB6B908320322E4D00CC290A /* document_test.cc */; }; @@ -731,6 +732,7 @@ 7B8D7BAC1A075DB773230505 /* app_testing.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5467FB07203E6A44009C9584 /* app_testing.mm */; }; 7BCC5973C4F4FCC272150E31 /* FIRCollectionReferenceTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E045202154AA00B64F25 /* FIRCollectionReferenceTests.mm */; }; 7BCF050BA04537B0E7D44730 /* exponential_backoff_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = B6D1B68420E2AB1A00B35856 /* exponential_backoff_test.cc */; }; + 7C1DC1B44729381126D083AE /* leveldb_snappy_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = D9D94300B9C02F7069523C00 /* leveldb_snappy_test.cc */; }; 7C5E017689012489AAB7718D /* CodableGeoPointTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5495EB022040E90200EBA509 /* CodableGeoPointTests.swift */; }; 7C7BA1DB0B66EB899A928283 /* hashing_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 54511E8D209805F8005BD28F /* hashing_test.cc */; }; 7D40C8EB7755138F85920637 /* leveldb_target_cache_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = E76F0CDF28E5FA62D21DE648 /* leveldb_target_cache_test.cc */; }; @@ -757,6 +759,7 @@ 81AF02881A8D23D02FC202F6 /* bundle_loader_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = A853C81A6A5A51C9D0389EDA /* bundle_loader_test.cc */; }; 81B23D2D4E061074958AF12F /* target.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = 618BBE7D20B89AAC00B5BCE7 /* target.pb.cc */; }; 81D1B1D2B66BD8310AC5707F /* string_win_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 79507DF8378D3C42F5B36268 /* string_win_test.cc */; }; + 82228CD6CE4A7A9254F8E82D /* leveldb_snappy_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = D9D94300B9C02F7069523C00 /* leveldb_snappy_test.cc */; }; 822E5D5EC4955393DF26BC5C /* string_apple_benchmark.mm in Sources */ = {isa = PBXBuildFile; fileRef = 4C73C0CC6F62A90D8573F383 /* string_apple_benchmark.mm */; }; 82E3634FCF4A882948B81839 /* FIRQueryUnitTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = FF73B39D04D1760190E6B84A /* FIRQueryUnitTests.mm */; }; 8342277EB0553492B6668877 /* leveldb_opener_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 75860CD13AF47EB1EA39EC2F /* leveldb_opener_test.cc */; }; @@ -846,6 +849,7 @@ 97729B53698C0E52EB165003 /* field_filter_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = E8551D6C6FB0B1BACE9E5BAD /* field_filter_test.cc */; }; 9774A6C2AA02A12D80B34C3C /* database_id_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = AB71064B201FA60300344F18 /* database_id_test.cc */; }; 9783FAEA4CF758E8C4C2D76E /* hashing_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 54511E8D209805F8005BD28F /* hashing_test.cc */; }; + 978D9EFDC56CC2E1FA468712 /* leveldb_snappy_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = D9D94300B9C02F7069523C00 /* leveldb_snappy_test.cc */; }; 9860F493EBF43AF5AC0A88BD /* empty_credentials_provider_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 8FA60B08D59FEA0D6751E87F /* empty_credentials_provider_test.cc */; }; 98708140787A9465D883EEC9 /* leveldb_mutation_queue_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 5C7942B6244F4C416B11B86C /* leveldb_mutation_queue_test.cc */; }; 98FE82875A899A40A98AAC22 /* leveldb_opener_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 75860CD13AF47EB1EA39EC2F /* leveldb_opener_test.cc */; }; @@ -1061,6 +1065,7 @@ C4055D868A38221B332CD03D /* FSTIntegrationTestCase.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5491BC711FB44593008B3588 /* FSTIntegrationTestCase.mm */; }; C426C6E424FB2199F5C2C5BC /* document.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = 544129D821C2DDC800EFB9CC /* document.pb.cc */; }; C43A555928CB0441096F82D2 /* FIRDocumentReferenceTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E049202154AA00B64F25 /* FIRDocumentReferenceTests.mm */; }; + C4548D8C790387C8E64F0FC4 /* leveldb_snappy_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = D9D94300B9C02F7069523C00 /* leveldb_snappy_test.cc */; }; C482E724F4B10968417C3F78 /* Pods_Firestore_FuzzTests_iOS.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = B79CA87A1A01FC5329031C9B /* Pods_Firestore_FuzzTests_iOS.framework */; }; C4C7A8D11DC394EF81B7B1FA /* filesystem_testing.cc in Sources */ = {isa = PBXBuildFile; fileRef = BA02DA2FCD0001CFC6EB08DA /* filesystem_testing.cc */; }; C524026444E83EEBC1773650 /* objc_type_traits_apple_test.mm in Sources */ = {isa = PBXBuildFile; fileRef = 2A0CF41BA5AED6049B0BEB2C /* objc_type_traits_apple_test.mm */; }; @@ -1220,6 +1225,7 @@ EA38690795FBAA182A9AA63E /* FIRDatabaseTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E06C202154D500B64F25 /* FIRDatabaseTests.mm */; }; EA46611779C3EEF12822508C /* annotations.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = 618BBE9520B89AAC00B5BCE7 /* annotations.pb.cc */; }; EAA1962BFBA0EBFBA53B343F /* bundle_builder.cc in Sources */ = {isa = PBXBuildFile; fileRef = 4F5B96F3ABCD2CA901DB1CD4 /* bundle_builder.cc */; }; + EAC0914B6DCC53008483AEE3 /* leveldb_snappy_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = D9D94300B9C02F7069523C00 /* leveldb_snappy_test.cc */; }; EADD28A7859FBB9BE4D913B0 /* memory_remote_document_cache_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1CA9800A53669EFBFFB824E3 /* memory_remote_document_cache_test.cc */; }; EB04FE18E5794FEC187A09E3 /* FSTMemorySpecTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 5492E02F20213FFC00B64F25 /* FSTMemorySpecTests.mm */; }; EB264591ADDE6D93A6924A61 /* serializer_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = 61F72C5520BC48FD001A68CB /* serializer_test.cc */; }; @@ -1694,6 +1700,7 @@ D5B2593BCB52957D62F1C9D3 /* perf_spec_test.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = perf_spec_test.json; sourceTree = ""; }; D5B25E7E7D6873CBA4571841 /* FIRNumericTransformTests.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = FIRNumericTransformTests.mm; sourceTree = ""; }; D7DF4A6F740086A2D8C0E28E /* Pods_Firestore_Tests_tvOS.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Firestore_Tests_tvOS.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + D9D94300B9C02F7069523C00 /* leveldb_snappy_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; path = leveldb_snappy_test.cc; sourceTree = ""; }; DAFF0CF521E64AC30062958F /* Firestore_Example_macOS.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Firestore_Example_macOS.app; sourceTree = BUILT_PRODUCTS_DIR; }; DAFF0CF721E64AC30062958F /* AppDelegate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = AppDelegate.h; sourceTree = ""; }; DAFF0CF821E64AC30062958F /* AppDelegate.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = AppDelegate.m; sourceTree = ""; }; @@ -2067,6 +2074,7 @@ 5C7942B6244F4C416B11B86C /* leveldb_mutation_queue_test.cc */, 75860CD13AF47EB1EA39EC2F /* leveldb_opener_test.cc */, 0840319686A223CC4AD3FAB1 /* leveldb_remote_document_cache_test.cc */, + D9D94300B9C02F7069523C00 /* leveldb_snappy_test.cc */, E76F0CDF28E5FA62D21DE648 /* leveldb_target_cache_test.cc */, 88CF09277CFA45EE1273E3BA /* leveldb_transaction_test.cc */, 332485C4DCC6BA0DBB5E31B7 /* leveldb_util_test.cc */, @@ -3604,6 +3612,7 @@ 1D7919CD2A05C15803F5FE05 /* leveldb_mutation_queue_test.cc in Sources */, 23EFC681986488B033C2B318 /* leveldb_opener_test.cc in Sources */, F10A3E4E164A5458DFF7EDE6 /* leveldb_remote_document_cache_test.cc in Sources */, + 7C1DC1B44729381126D083AE /* leveldb_snappy_test.cc in Sources */, 7D40C8EB7755138F85920637 /* leveldb_target_cache_test.cc in Sources */, B46E778F9E40864B5D2B2F1C /* leveldb_transaction_test.cc in Sources */, 66FAB8EAC012A3822BD4D0C9 /* leveldb_util_test.cc in Sources */, @@ -3802,6 +3811,7 @@ 1145D70555D8CDC75183A88C /* leveldb_mutation_queue_test.cc in Sources */, 1DCA68BB2EF7A9144B35411F /* leveldb_opener_test.cc in Sources */, CD1E2F356FC71D7E74FCD26C /* leveldb_remote_document_cache_test.cc in Sources */, + 077292C9797D97D3851F15CE /* leveldb_snappy_test.cc in Sources */, 06485D6DA8F64757D72636E1 /* leveldb_target_cache_test.cc in Sources */, EC62F9E29CE3598881908FB8 /* leveldb_transaction_test.cc in Sources */, 7A3BE0ED54933C234FDE23D1 /* leveldb_util_test.cc in Sources */, @@ -4014,6 +4024,7 @@ FE701C2D739A5371BCBD62B9 /* leveldb_mutation_queue_test.cc in Sources */, 98FE82875A899A40A98AAC22 /* leveldb_opener_test.cc in Sources */, 79D86DD18BB54D2D69DC457F /* leveldb_remote_document_cache_test.cc in Sources */, + 82228CD6CE4A7A9254F8E82D /* leveldb_snappy_test.cc in Sources */, 6C388B2D0967088758FF2425 /* leveldb_target_cache_test.cc in Sources */, D4572060A0FD4D448470D329 /* leveldb_transaction_test.cc in Sources */, 3ABF84FC618016CA6E1D3C03 /* leveldb_util_test.cc in Sources */, @@ -4226,6 +4237,7 @@ A478FDD7C3F48FBFDDA7D8F5 /* leveldb_mutation_queue_test.cc in Sources */, A06FBB7367CDD496887B86F8 /* leveldb_opener_test.cc in Sources */, A27096F764227BC73526FED3 /* leveldb_remote_document_cache_test.cc in Sources */, + EAC0914B6DCC53008483AEE3 /* leveldb_snappy_test.cc in Sources */, D04CBBEDB8DC16D8C201AC49 /* leveldb_target_cache_test.cc in Sources */, 29243A4BBB2E2B1530A62C59 /* leveldb_transaction_test.cc in Sources */, 08FA4102AD14452E9587A1F2 /* leveldb_util_test.cc in Sources */, @@ -4434,6 +4446,7 @@ 98708140787A9465D883EEC9 /* leveldb_mutation_queue_test.cc in Sources */, 8342277EB0553492B6668877 /* leveldb_opener_test.cc in Sources */, 8077722A6BB175D3108CDC55 /* leveldb_remote_document_cache_test.cc in Sources */, + C4548D8C790387C8E64F0FC4 /* leveldb_snappy_test.cc in Sources */, 284A5280F868B2B4B5A1C848 /* leveldb_target_cache_test.cc in Sources */, 35DB74DFB2F174865BCCC264 /* leveldb_transaction_test.cc in Sources */, BEE0294A23AB993E5DE0E946 /* leveldb_util_test.cc in Sources */, @@ -4665,6 +4678,7 @@ 4FAD8823DC37B9CA24379E85 /* leveldb_mutation_queue_test.cc in Sources */, 4562CDD90F5FF0491F07C5DA /* leveldb_opener_test.cc in Sources */, EE6DBFB0874A50578CE97A7F /* leveldb_remote_document_cache_test.cc in Sources */, + 978D9EFDC56CC2E1FA468712 /* leveldb_snappy_test.cc in Sources */, 6380CACCF96A9B26900983DC /* leveldb_target_cache_test.cc in Sources */, DDD219222EEE13E3F9F2C703 /* leveldb_transaction_test.cc in Sources */, BC549E3F3F119D80741D8612 /* leveldb_util_test.cc in Sources */, From 9b9a19dadff59758fae325c3cdb3b22f7202cff7 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Fri, 8 Apr 2022 15:31:53 -0400 Subject: [PATCH 04/23] leveldb.cmake: use FindPython3 instead of FindPythonInterp, if it is available A Python3 interpreter is required, and some GitHub Actions will find python2 instead, causing them to fail. --- cmake/external/leveldb.cmake | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cmake/external/leveldb.cmake b/cmake/external/leveldb.cmake index c9b691ee2c1..2456cc2d651 100644 --- a/cmake/external/leveldb.cmake +++ b/cmake/external/leveldb.cmake @@ -13,7 +13,15 @@ # limitations under the License. include(ExternalProject) -include(FindPythonInterp) + +# Find a Python interpreter using the best available mechanism. +if(${CMAKE_VERSION} VERSION_LESS "3.12") + include(FindPythonInterp) + set(PATCH_PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE}") +else() + find_package(Python3 COMPONENTS Interpreter) + set(PATCH_PYTHON_EXECUTABLE "${Python3_EXECUTABLE}") +endif() if(TARGET leveldb) return() @@ -42,7 +50,7 @@ ExternalProject_Add( BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" - PATCH_COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/leveldb_patch.py --snappy-source-dir ${snappy_source_dir} --snappy-binary-dir ${snappy_binary_dir} + PATCH_COMMAND ${PATCH_PYTHON_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/leveldb_patch.py --snappy-source-dir ${snappy_source_dir} --snappy-binary-dir ${snappy_binary_dir} HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}" ) From e0ce56400a8cc1c0aa97e32580ad649e48da66c4 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Tue, 12 Apr 2022 20:53:06 -0400 Subject: [PATCH 05/23] Apply a patch to Snappy to fix the build with gcc on linux: https://github.com/google/snappy/pull/128 --- cmake/external/snappy.cmake | 1 + cmake/external/snappy.patch | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 cmake/external/snappy.patch diff --git a/cmake/external/snappy.cmake b/cmake/external/snappy.cmake index 9f25c03d086..7da24f127e4 100644 --- a/cmake/external/snappy.cmake +++ b/cmake/external/snappy.cmake @@ -34,6 +34,7 @@ ExternalProject_Add( BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" + PATCH_COMMAND patch -Np1 -i ${CMAKE_CURRENT_LIST_DIR}/snappy.patch HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}" ) diff --git a/cmake/external/snappy.patch b/cmake/external/snappy.patch new file mode 100644 index 00000000000..28bfb0837ad --- /dev/null +++ b/cmake/external/snappy.patch @@ -0,0 +1,12 @@ +diff -Naur snappy/snappy.cc snappy_patched/snappy.cc +--- snappy/snappy.cc 2022-04-12 20:44:55.000000000 -0400 ++++ snappy_patched/snappy.cc 2022-04-12 20:47:05.000000000 -0400 +@@ -1014,7 +1014,7 @@ + } + + SNAPPY_ATTRIBUTE_ALWAYS_INLINE +-size_t AdvanceToNextTag(const uint8_t** ip_p, size_t* tag) { ++inline size_t AdvanceToNextTag(const uint8_t** ip_p, size_t* tag) { + const uint8_t*& ip = *ip_p; + // This section is crucial for the throughput of the decompression loop. + // The latency of an iteration is fundamentally constrained by the From 6e0f4bc98f77c98aa41b2143545566759acd7075 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Tue, 12 Apr 2022 22:16:30 -0400 Subject: [PATCH 06/23] check_whitespace.sh: ignore snappy.patch, which has required trailing whitespace --- scripts/check_whitespace.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/check_whitespace.sh b/scripts/check_whitespace.sh index 4b2c964ec43..4feb4b87d0b 100755 --- a/scripts/check_whitespace.sh +++ b/scripts/check_whitespace.sh @@ -29,6 +29,7 @@ options=( # likely in a nanopb release after 0.3.9.7. git grep "${options[@]}" -- \ + ':(exclude)cmake/external/snappy.patch' \ ':(exclude)Crashlytics/Protogen/nanopb' \ ':(exclude)Crashlytics/ProtoSupport' \ ':(exclude)Crashlytics/UnitTests/Data' \ From 3822eba0ed5a8a60345168cd09cbaa57aed57654 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 13 Apr 2022 09:20:03 -0400 Subject: [PATCH 07/23] xcresult_logs.py: fix UnicodeEncodeError when printing logs to stdout --- scripts/xcresult_logs.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/xcresult_logs.py b/scripts/xcresult_logs.py index 5e94f667e1e..679eda3217b 100755 --- a/scripts/xcresult_logs.py +++ b/scripts/xcresult_logs.py @@ -64,7 +64,19 @@ def main(): # but also makes it harder to deal with. log_id = find_log_id(xcresult_path) log = export_log(xcresult_path, log_id) - sys.stdout.write(log) + + # Avoid UnicodeEncodeError by adapting the code example from + # https://docs.python.org/3/library/sys.html#sys.displayhook + try: + sys.stdout.write(log) + except UnicodeEncodeError: + log_encoded = log.encode(sys.stdout.encoding, errors='backslashreplace') + if hasattr(sys.stdout, 'buffer'): + sys.stdout.flush() + sys.stdout.buffer.write(log_encoded) + else: + log_decoded = log_encoded.decode(sys.stdout.encoding, errors='strict') + sys.stdout.write(log_decoded) # Most flags on the xcodebuild command-line are uninteresting, so only pull From 7db7e652af81f20d3eef4a76972fba445707f51d Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 13 Apr 2022 09:35:37 -0400 Subject: [PATCH 08/23] Only run LevelDbHasSnappySupportCompiledIn in cmake builds --- Firestore/core/test/unit/local/CMakeLists.txt | 8 ++++++++ Firestore/core/test/unit/local/leveldb_snappy_test.cc | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/Firestore/core/test/unit/local/CMakeLists.txt b/Firestore/core/test/unit/local/CMakeLists.txt index baa071d5940..57208787906 100644 --- a/Firestore/core/test/unit/local/CMakeLists.txt +++ b/Firestore/core/test/unit/local/CMakeLists.txt @@ -42,6 +42,14 @@ firebase_ios_glob( ) firebase_ios_add_test(firestore_local_test ${sources}) +# Set a preprocessor define so that tests can distinguish between having been +# built by Xcode vs. cmake. +target_compile_definitions( + firestore_local_test + PRIVATE + FIRESTORE_TESTS_CMAKE_BUILD +) + target_link_libraries( firestore_local_test PRIVATE GMock::GMock diff --git a/Firestore/core/test/unit/local/leveldb_snappy_test.cc b/Firestore/core/test/unit/local/leveldb_snappy_test.cc index f15733b0114..31a82f487e9 100644 --- a/Firestore/core/test/unit/local/leveldb_snappy_test.cc +++ b/Firestore/core/test/unit/local/leveldb_snappy_test.cc @@ -43,6 +43,12 @@ Path CreateLevelDbDatabaseThatUsesSnappyCompression(); // This test ensures that we don't accidentally regress having added in Snappy // compression support (https://github.com/firebase/firebase-ios-sdk/pull/9596). TEST(LevelDbSnappy, LevelDbHasSnappySupportCompiledIn) { + // Only run this test in cmake builds, since Snappy support is not included + // when pulling in LevelDb via CocoaPods or Swift Package Manager. + #if FIRESTORE_TESTS_CMAKE_BUILD + GTEST_SKIP() << "Snappy support is only present in cmake builds"; + #endif + Path leveldb_path = CreateLevelDbDatabaseThatUsesSnappyCompression(); if (HasFatalFailure()) return; From c4ec40ece335a8b41a4fa0b7d76c43f5c2b83afe Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 13 Apr 2022 10:31:13 -0400 Subject: [PATCH 09/23] leveldb_snappy_test.cc: format code with scripts/style.sh --- Firestore/core/test/unit/local/leveldb_snappy_test.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Firestore/core/test/unit/local/leveldb_snappy_test.cc b/Firestore/core/test/unit/local/leveldb_snappy_test.cc index 31a82f487e9..aa0386ba8f0 100644 --- a/Firestore/core/test/unit/local/leveldb_snappy_test.cc +++ b/Firestore/core/test/unit/local/leveldb_snappy_test.cc @@ -43,11 +43,11 @@ Path CreateLevelDbDatabaseThatUsesSnappyCompression(); // This test ensures that we don't accidentally regress having added in Snappy // compression support (https://github.com/firebase/firebase-ios-sdk/pull/9596). TEST(LevelDbSnappy, LevelDbHasSnappySupportCompiledIn) { - // Only run this test in cmake builds, since Snappy support is not included - // when pulling in LevelDb via CocoaPods or Swift Package Manager. - #if FIRESTORE_TESTS_CMAKE_BUILD +// Only run this test in cmake builds, since Snappy support is not included +// when pulling in LevelDb via CocoaPods or Swift Package Manager. +#if FIRESTORE_TESTS_CMAKE_BUILD GTEST_SKIP() << "Snappy support is only present in cmake builds"; - #endif +#endif Path leveldb_path = CreateLevelDbDatabaseThatUsesSnappyCompression(); if (HasFatalFailure()) return; From 73a558cd7d3693bcb9c031bc37f9bb78535c01c2 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 13 Apr 2022 10:32:33 -0400 Subject: [PATCH 10/23] leveldb_snappy_test.cc: REVERT ME: temporarily allow the test to fail on iOS to verify the fix to xcresult_logs.py in https://github.com/firebase/firebase-ios-sdk/pull/9645 --- Firestore/core/test/unit/local/leveldb_snappy_test.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Firestore/core/test/unit/local/leveldb_snappy_test.cc b/Firestore/core/test/unit/local/leveldb_snappy_test.cc index aa0386ba8f0..f15733b0114 100644 --- a/Firestore/core/test/unit/local/leveldb_snappy_test.cc +++ b/Firestore/core/test/unit/local/leveldb_snappy_test.cc @@ -43,12 +43,6 @@ Path CreateLevelDbDatabaseThatUsesSnappyCompression(); // This test ensures that we don't accidentally regress having added in Snappy // compression support (https://github.com/firebase/firebase-ios-sdk/pull/9596). TEST(LevelDbSnappy, LevelDbHasSnappySupportCompiledIn) { -// Only run this test in cmake builds, since Snappy support is not included -// when pulling in LevelDb via CocoaPods or Swift Package Manager. -#if FIRESTORE_TESTS_CMAKE_BUILD - GTEST_SKIP() << "Snappy support is only present in cmake builds"; -#endif - Path leveldb_path = CreateLevelDbDatabaseThatUsesSnappyCompression(); if (HasFatalFailure()) return; From dd113af80009f93df69cdd2e8f056d4874422718 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 13 Apr 2022 12:37:44 -0400 Subject: [PATCH 11/23] xcresult_logs.py: change the UnicodeEncodeError avoidance strategy to instead just encode to UTF-8 in a relaxed manner and write the resulting bytes directly to stdout --- scripts/xcresult_logs.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/scripts/xcresult_logs.py b/scripts/xcresult_logs.py index 679eda3217b..f21d0b62687 100755 --- a/scripts/xcresult_logs.py +++ b/scripts/xcresult_logs.py @@ -65,18 +65,11 @@ def main(): log_id = find_log_id(xcresult_path) log = export_log(xcresult_path, log_id) - # Avoid UnicodeEncodeError by adapting the code example from - # https://docs.python.org/3/library/sys.html#sys.displayhook - try: - sys.stdout.write(log) - except UnicodeEncodeError: - log_encoded = log.encode(sys.stdout.encoding, errors='backslashreplace') - if hasattr(sys.stdout, 'buffer'): - sys.stdout.flush() - sys.stdout.buffer.write(log_encoded) - else: - log_decoded = log_encoded.decode(sys.stdout.encoding, errors='strict') - sys.stdout.write(log_decoded) + # Avoid a potential UnicodeEncodeError raised by sys.stdout.write() by + # doing a relaxed encoding ourselves and writing the resulting bytes. + log_encoded = log.encode('utf8', errors='backslashreplace') + sys.stdout.flush() + sys.stdout.buffer.write(log_encoded) # Most flags on the xcodebuild command-line are uninteresting, so only pull From f45b7a781ab0b94f8cf989e07b91eccd6192f8c3 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 13 Apr 2022 14:12:30 -0400 Subject: [PATCH 12/23] xcresult_logs.py: Another fix for when sys.stdout does not have a 'buffer' attribute --- scripts/xcresult_logs.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/xcresult_logs.py b/scripts/xcresult_logs.py index f21d0b62687..744734ebfa7 100755 --- a/scripts/xcresult_logs.py +++ b/scripts/xcresult_logs.py @@ -66,10 +66,15 @@ def main(): log = export_log(xcresult_path, log_id) # Avoid a potential UnicodeEncodeError raised by sys.stdout.write() by - # doing a relaxed encoding ourselves and writing the resulting bytes. - log_encoded = log.encode('utf8', errors='backslashreplace') - sys.stdout.flush() - sys.stdout.buffer.write(log_encoded) + # doing a relaxed encoding ourselves. + if hasattr(sys.stdout, 'buffer'): + log_encoded = log.encode('utf8', errors='backslashreplace') + sys.stdout.flush() + sys.stdout.buffer.write(log_encoded) + else: + log_encoded = log.encode('ascii', errors='backslashreplace') + log_decoded = log_encoded.decode('ascii', errors='strict') + sys.stdout.write(log_decoded) # Most flags on the xcodebuild command-line are uninteresting, so only pull From 600122ea22e053af2318c3486fec486050bd810b Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 13 Apr 2022 16:06:09 -0400 Subject: [PATCH 13/23] Revert "leveldb_snappy_test.cc: REVERT ME: temporarily allow the test to fail on iOS to verify the fix to xcresult_logs.py in https://github.com/firebase/firebase-ios-sdk/pull/9645" This reverts commit 73a558cd7d3693bcb9c031bc37f9bb78535c01c2. --- Firestore/core/test/unit/local/leveldb_snappy_test.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Firestore/core/test/unit/local/leveldb_snappy_test.cc b/Firestore/core/test/unit/local/leveldb_snappy_test.cc index f15733b0114..aa0386ba8f0 100644 --- a/Firestore/core/test/unit/local/leveldb_snappy_test.cc +++ b/Firestore/core/test/unit/local/leveldb_snappy_test.cc @@ -43,6 +43,12 @@ Path CreateLevelDbDatabaseThatUsesSnappyCompression(); // This test ensures that we don't accidentally regress having added in Snappy // compression support (https://github.com/firebase/firebase-ios-sdk/pull/9596). TEST(LevelDbSnappy, LevelDbHasSnappySupportCompiledIn) { +// Only run this test in cmake builds, since Snappy support is not included +// when pulling in LevelDb via CocoaPods or Swift Package Manager. +#if FIRESTORE_TESTS_CMAKE_BUILD + GTEST_SKIP() << "Snappy support is only present in cmake builds"; +#endif + Path leveldb_path = CreateLevelDbDatabaseThatUsesSnappyCompression(); if (HasFatalFailure()) return; From 142325a5a74fbe97d49096112f3db45ca437905f Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 13 Apr 2022 18:00:53 -0400 Subject: [PATCH 14/23] leveldb_snappy_test.cc: also verify NO snappy support on iOS --- .../test/unit/local/leveldb_snappy_test.cc | 129 ++++++++++++------ 1 file changed, 85 insertions(+), 44 deletions(-) diff --git a/Firestore/core/test/unit/local/leveldb_snappy_test.cc b/Firestore/core/test/unit/local/leveldb_snappy_test.cc index aa0386ba8f0..9d8ec28e842 100644 --- a/Firestore/core/test/unit/local/leveldb_snappy_test.cc +++ b/Firestore/core/test/unit/local/leveldb_snappy_test.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -40,39 +41,106 @@ using ::firebase::firestore::util::Path; // with reason "corruption". Path CreateLevelDbDatabaseThatUsesSnappyCompression(); -// This test ensures that we don't accidentally regress having added in Snappy -// compression support (https://github.com/firebase/firebase-ios-sdk/pull/9596). -TEST(LevelDbSnappy, LevelDbHasSnappySupportCompiledIn) { -// Only run this test in cmake builds, since Snappy support is not included -// when pulling in LevelDb via CocoaPods or Swift Package Manager. +// Creates and opens a LevelDb database that contains at least one block that +// is compressed with Snappy compression, then iterates over it, invoking the +// given callback with the status at each point in the iteration. Once the +// callback is invoked with a `status` where `status.ok()` is not true, then +// iteration will stop and the callback will not be invoked again. +void IterateOverLevelDbDatabaseThatUsesSnappyCompression( + std::function); + #if FIRESTORE_TESTS_CMAKE_BUILD - GTEST_SKIP() << "Snappy support is only present in cmake builds"; -#endif - Path leveldb_path = CreateLevelDbDatabaseThatUsesSnappyCompression(); - if (HasFatalFailure()) return; +// Ensure that LevelDb is compiled with Snappy compression support. +// See https://github.com/firebase/firebase-ios-sdk/pull/9596 for details. +TEST(LevelDbSnappy, LevelDbSupportsSnappy) { + IterateOverLevelDbDatabaseThatUsesSnappyCompression( + [](const leveldb::Status& status) { + ASSERT_TRUE(status.ok()) << ConvertStatus(status); + }); +} + +#else // FIRESTORE_TESTS_CMAKE_BUILD - leveldb::Options options; - options.create_if_missing = false; +// Ensure that LevelDb is NOT compiled with Snappy compression support. +TEST(LevelDbSnappy, LevelDbDoesNotSupportSnappy) { + bool got_failed_status = false; + IterateOverLevelDbDatabaseThatUsesSnappyCompression( + [&](const leveldb::Status& status) { + if (!status.ok()) { + got_failed_status = true; + ASSERT_TRUE(status.IsCorruption()) << ConvertStatus(status); + } + }); + if (!HasFailure()) { + ASSERT_TRUE(got_failed_status) + << "Reading a Snappy-compressed LevelDb database was successful; " + "however, it should NOT have been successful " + "since Snappy support is expected to NOT be available."; + } +} + +#endif // FIRESTORE_TESTS_CMAKE_BUILD + +void IterateOverLevelDbDatabaseThatUsesSnappyCompression( + std::function callback) { std::unique_ptr db; { + Path leveldb_path = CreateLevelDbDatabaseThatUsesSnappyCompression(); + + leveldb::Options options; + options.create_if_missing = false; + leveldb::DB* db_ptr; leveldb::Status status = leveldb::DB::Open(options, leveldb_path.ToUtf8String(), &db_ptr); - ASSERT_TRUE(status.ok()); + + ASSERT_TRUE(status.ok()) + << "Opening LevelDb database " << leveldb_path.ToUtf8String() + << " failed: " << ConvertStatus(status); + db.reset(db_ptr); } - // One of the assertions below will fail when LevelDb attempts to read a block - // that is compressed with Snappy and Snappy compression support is not - // compiled in. std::unique_ptr it( db->NewIterator(leveldb::ReadOptions())); for (it->SeekToFirst(); it->Valid(); it->Next()) { - ASSERT_TRUE(it->status().ok()) << ConvertStatus(it->status()); + callback(it->status()); + if (!it->status().ok()) { + return; + } + } + + // Invoke the callback on the final status. + callback(it->status()); +} + +template +void WriteFile(const Path& dir, + const std::string& file_name, + const T& data_array) { + Filesystem* fs = Filesystem::Default(); + { + auto status = fs->RecursivelyCreateDir(dir); + if (!status.ok()) { + FAIL() << "Creating directory failed: " << dir.ToUtf8String() << " (" + << status.error_message() << ")"; + } + } + + Path file = dir.AppendUtf8(file_name); + std::ofstream out_file(file.native_value(), std::ios::binary); + if (!out_file) { + FAIL() << "Unable to open file for writing: " << file.ToUtf8String(); + } + + out_file.write(reinterpret_cast(data_array.data()), + data_array.size()); + out_file.close(); + if (!out_file) { + FAIL() << "Writing to file failed: " << file.ToUtf8String(); } - ASSERT_TRUE(it->status().ok()) << ConvertStatus(it->status()); } const std::array LevelDbSnappyFile_000005_ldb{ @@ -190,33 +258,6 @@ const std::array LevelDbSnappyFile_MANIFEST_000084{ 0x04, 0x0D, }; -template -void WriteFile(const Path& dir, - const std::string& file_name, - const T& data_array) { - Filesystem* fs = Filesystem::Default(); - { - auto status = fs->RecursivelyCreateDir(dir); - if (!status.ok()) { - FAIL() << "Creating directory failed: " << dir.ToUtf8String() << " (" - << status.error_message() << ")"; - } - } - - Path file = dir.AppendUtf8(file_name); - std::ofstream out_file(file.native_value(), std::ios::binary); - if (!out_file) { - FAIL() << "Unable to open file for writing: " << file.ToUtf8String(); - } - - out_file.write(reinterpret_cast(data_array.data()), - data_array.size()); - out_file.close(); - if (!out_file) { - FAIL() << "Writing to file failed: " << file.ToUtf8String(); - } -} - Path CreateLevelDbDatabaseThatUsesSnappyCompression() { Path leveldb_dir = ::firebase::firestore::local::LevelDbDir(); WriteFile(leveldb_dir, "000005.ldb", LevelDbSnappyFile_000005_ldb); From db22a072938e2c3504e42eb4c2f6bc8f38f4c117 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 13 Apr 2022 18:24:35 -0400 Subject: [PATCH 15/23] Rename FIRESTORE_TESTS_CMAKE_BUILD to FIREBASE_TESTS_BUILT_BY_CMAKE and move the target_compile_definitions into the firebase_ios_add_test() cmake function --- Firestore/core/test/unit/local/CMakeLists.txt | 8 -------- Firestore/core/test/unit/local/leveldb_snappy_test.cc | 6 +++--- cmake/cc_rules.cmake | 8 ++++++++ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Firestore/core/test/unit/local/CMakeLists.txt b/Firestore/core/test/unit/local/CMakeLists.txt index 57208787906..baa071d5940 100644 --- a/Firestore/core/test/unit/local/CMakeLists.txt +++ b/Firestore/core/test/unit/local/CMakeLists.txt @@ -42,14 +42,6 @@ firebase_ios_glob( ) firebase_ios_add_test(firestore_local_test ${sources}) -# Set a preprocessor define so that tests can distinguish between having been -# built by Xcode vs. cmake. -target_compile_definitions( - firestore_local_test - PRIVATE - FIRESTORE_TESTS_CMAKE_BUILD -) - target_link_libraries( firestore_local_test PRIVATE GMock::GMock diff --git a/Firestore/core/test/unit/local/leveldb_snappy_test.cc b/Firestore/core/test/unit/local/leveldb_snappy_test.cc index 9d8ec28e842..0741ae4c54e 100644 --- a/Firestore/core/test/unit/local/leveldb_snappy_test.cc +++ b/Firestore/core/test/unit/local/leveldb_snappy_test.cc @@ -49,7 +49,7 @@ Path CreateLevelDbDatabaseThatUsesSnappyCompression(); void IterateOverLevelDbDatabaseThatUsesSnappyCompression( std::function); -#if FIRESTORE_TESTS_CMAKE_BUILD +#if FIREBASE_TESTS_BUILT_BY_CMAKE // Ensure that LevelDb is compiled with Snappy compression support. // See https://github.com/firebase/firebase-ios-sdk/pull/9596 for details. @@ -60,7 +60,7 @@ TEST(LevelDbSnappy, LevelDbSupportsSnappy) { }); } -#else // FIRESTORE_TESTS_CMAKE_BUILD +#else // FIREBASE_TESTS_BUILT_BY_CMAKE // Ensure that LevelDb is NOT compiled with Snappy compression support. TEST(LevelDbSnappy, LevelDbDoesNotSupportSnappy) { @@ -81,7 +81,7 @@ TEST(LevelDbSnappy, LevelDbDoesNotSupportSnappy) { } } -#endif // FIRESTORE_TESTS_CMAKE_BUILD +#endif // FIREBASE_TESTS_BUILT_BY_CMAKE void IterateOverLevelDbDatabaseThatUsesSnappyCompression( std::function callback) { diff --git a/cmake/cc_rules.cmake b/cmake/cc_rules.cmake index 05239e326e5..89ba3518fa6 100644 --- a/cmake/cc_rules.cmake +++ b/cmake/cc_rules.cmake @@ -116,6 +116,14 @@ function(firebase_ios_add_test target) firebase_ios_set_common_target_options(${target} ${flag_OPTIONS}) + # Set a preprocessor define so that tests can distinguish between having been + # built by Xcode vs. cmake. + target_compile_definitions( + ${target} + PRIVATE + FIREBASE_TESTS_BUILT_BY_CMAKE + ) + target_link_libraries(${target} PRIVATE GTest::GTest GTest::Main) endfunction() From 0ed921ab58a7cb4b7d8fa47c7543d4eaad25bb0f Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 13 Apr 2022 18:40:07 -0400 Subject: [PATCH 16/23] leveldb_snappy_test.cc: implement LevelDbDir() ourselves --- .../core/test/unit/local/leveldb_snappy_test.cc | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Firestore/core/test/unit/local/leveldb_snappy_test.cc b/Firestore/core/test/unit/local/leveldb_snappy_test.cc index 0741ae4c54e..c888c15bad7 100644 --- a/Firestore/core/test/unit/local/leveldb_snappy_test.cc +++ b/Firestore/core/test/unit/local/leveldb_snappy_test.cc @@ -24,7 +24,6 @@ #include "Firestore/core/src/local/leveldb_util.h" #include "Firestore/core/src/util/filesystem.h" #include "Firestore/core/src/util/path.h" -#include "Firestore/core/test/unit/local/persistence_testing.h" #include "gtest/gtest.h" #include "leveldb/db.h" @@ -258,8 +257,21 @@ const std::array LevelDbSnappyFile_MANIFEST_000084{ 0x04, 0x0D, }; +Path LevelDbDir() { + Filesystem* fs = Filesystem::Default(); + Path dir = fs->TempDir().AppendUtf8("LevelDbSnappyTest"); + + // Delete the directory first to ensure isolation between runs. + auto status = fs->RecursivelyRemove(dir); + EXPECT_TRUE(status.ok()) << "Failed to clean up leveldb in directory " + << dir.ToUtf8String() << ": " << status.ToString(); + + return dir; +} + Path CreateLevelDbDatabaseThatUsesSnappyCompression() { - Path leveldb_dir = ::firebase::firestore::local::LevelDbDir(); + Path leveldb_dir = LevelDbDir(); + WriteFile(leveldb_dir, "000005.ldb", LevelDbSnappyFile_000005_ldb); WriteFile(leveldb_dir, "000017.ldb", LevelDbSnappyFile_000017_ldb); WriteFile(leveldb_dir, "000085.ldb", LevelDbSnappyFile_000085_ldb); From 47c2b31ffe55855be3a137dc0f8c258c90077e17 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 13 Apr 2022 18:40:45 -0400 Subject: [PATCH 17/23] leveldb_snappy_test.cc: fail fast if creating the LevelDb directory fails --- Firestore/core/test/unit/local/leveldb_snappy_test.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Firestore/core/test/unit/local/leveldb_snappy_test.cc b/Firestore/core/test/unit/local/leveldb_snappy_test.cc index c888c15bad7..6711feeb429 100644 --- a/Firestore/core/test/unit/local/leveldb_snappy_test.cc +++ b/Firestore/core/test/unit/local/leveldb_snappy_test.cc @@ -87,6 +87,9 @@ void IterateOverLevelDbDatabaseThatUsesSnappyCompression( std::unique_ptr db; { Path leveldb_path = CreateLevelDbDatabaseThatUsesSnappyCompression(); + if (leveldb_path.empty()) { + return; + } leveldb::Options options; options.create_if_missing = false; @@ -265,12 +268,18 @@ Path LevelDbDir() { auto status = fs->RecursivelyRemove(dir); EXPECT_TRUE(status.ok()) << "Failed to clean up leveldb in directory " << dir.ToUtf8String() << ": " << status.ToString(); + if (! status.ok()) { + return {}; + } return dir; } Path CreateLevelDbDatabaseThatUsesSnappyCompression() { Path leveldb_dir = LevelDbDir(); + if (leveldb_dir.empty()){ + return {}; + } WriteFile(leveldb_dir, "000005.ldb", LevelDbSnappyFile_000005_ldb); WriteFile(leveldb_dir, "000017.ldb", LevelDbSnappyFile_000017_ldb); @@ -279,6 +288,7 @@ Path CreateLevelDbDatabaseThatUsesSnappyCompression() { WriteFile(leveldb_dir, "LOG.old", LevelDbSnappyFile_LOG_old); WriteFile(leveldb_dir, "LOG", LevelDbSnappyFile_LOG); WriteFile(leveldb_dir, "MANIFEST-000084", LevelDbSnappyFile_MANIFEST_000084); + return leveldb_dir; } From cc3b51a49c0ef5ef9b11c436eeacc0d8ad5e6cb9 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 13 Apr 2022 18:48:56 -0400 Subject: [PATCH 18/23] leveldb_snappy_test.cc: format code with scripts/style.sh --- Firestore/core/test/unit/local/leveldb_snappy_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Firestore/core/test/unit/local/leveldb_snappy_test.cc b/Firestore/core/test/unit/local/leveldb_snappy_test.cc index 6711feeb429..0dc907b568c 100644 --- a/Firestore/core/test/unit/local/leveldb_snappy_test.cc +++ b/Firestore/core/test/unit/local/leveldb_snappy_test.cc @@ -268,7 +268,7 @@ Path LevelDbDir() { auto status = fs->RecursivelyRemove(dir); EXPECT_TRUE(status.ok()) << "Failed to clean up leveldb in directory " << dir.ToUtf8String() << ": " << status.ToString(); - if (! status.ok()) { + if (!status.ok()) { return {}; } @@ -277,7 +277,7 @@ Path LevelDbDir() { Path CreateLevelDbDatabaseThatUsesSnappyCompression() { Path leveldb_dir = LevelDbDir(); - if (leveldb_dir.empty()){ + if (leveldb_dir.empty()) { return {}; } From 7ca90fcdb0e15784eb173d195105ff9d181b4677 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Wed, 13 Apr 2022 20:49:02 -0400 Subject: [PATCH 19/23] Specify --verbose to ctest instead of --output-on-failure --- scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build.sh b/scripts/build.sh index b1d37d23134..8aa3c11aa3a 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -356,7 +356,7 @@ case "$product-$platform-$method" in echo "Building cmake build ..." ninja -k 10 all - ctest --output-on-failure + ctest --verbose ) ;; From d57360f1bf2dd7ecb85a768e8912d0045b00b88d Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Thu, 14 Apr 2022 23:49:43 -0400 Subject: [PATCH 20/23] Add FIRESTORE_INCLUDE_OBJC cmake cache variable to provide a way to avoid building the Firestore Objective-C layer. --- Firestore/CMakeLists.txt | 4 ++++ Firestore/Example/CMakeLists.txt | 4 ++++ Firestore/Source/CMakeLists.txt | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/Firestore/CMakeLists.txt b/Firestore/CMakeLists.txt index ffbe17271b1..43896cae65b 100644 --- a/Firestore/CMakeLists.txt +++ b/Firestore/CMakeLists.txt @@ -12,6 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Provide a mechanism for firebase-cpp-sdk to avoid unnecessarily building +# the Objective-C APIs and Tests, which it doesn't use. +option(FIRESTORE_INCLUDE_OBJC "Build the Firestore Objective-C layer" ON) + add_subdirectory(Protos) add_subdirectory(Source) add_subdirectory(core) diff --git a/Firestore/Example/CMakeLists.txt b/Firestore/Example/CMakeLists.txt index 6d9f3fcf9b8..dc92b503e68 100644 --- a/Firestore/Example/CMakeLists.txt +++ b/Firestore/Example/CMakeLists.txt @@ -12,6 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +if(NOT FIRESTORE_INCLUDE_OBJC) + return() +endif() + add_subdirectory(Benchmarks) add_subdirectory(App) diff --git a/Firestore/Source/CMakeLists.txt b/Firestore/Source/CMakeLists.txt index f6987ddc34e..6a3e06cd715 100644 --- a/Firestore/Source/CMakeLists.txt +++ b/Firestore/Source/CMakeLists.txt @@ -16,6 +16,10 @@ if(NOT APPLE) return() endif() +if(NOT FIRESTORE_INCLUDE_OBJC) + return() +endif() + file(GLOB headers Public/FirebaseFirestore/*.h) file(GLOB sources API/*.h API/*.m API/*.mm) From 0ed860a0360735c28a26aad9cbe899695009a236 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Fri, 15 Apr 2022 10:09:14 -0400 Subject: [PATCH 21/23] Add FIREBASE_PYTHON_EXECUTABLE cmake cache var --- CMakeLists.txt | 15 +++++++++++++++ Firestore/Protos/CMakeLists.txt | 6 ++---- Firestore/core/CMakeLists.txt | 3 +-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8ac7e437d77..2a6b005a195 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,6 +72,21 @@ option( ON ) +# Find a Python interpreter using the best available mechanism. +if(${CMAKE_VERSION} VERSION_LESS "3.12") + include(FindPythonInterp) + set(DEFAULT_FIREBASE_PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE}") +else() + find_package(Python3 COMPONENTS Interpreter) + set(DEFAULT_FIREBASE_PYTHON_EXECUTABLE "${Python3_EXECUTABLE}") +endif() + +set( + FIREBASE_PYTHON_EXECUTABLE + "${DEFAULT_FIREBASE_PYTHON_EXECUTABLE}" + CACHE FILEPATH + "The Python interpreter to use" +) list(INSERT CMAKE_MODULE_PATH 0 ${PROJECT_SOURCE_DIR}/cmake) include(compiler_setup) diff --git a/Firestore/Protos/CMakeLists.txt b/Firestore/Protos/CMakeLists.txt index 85589b35fed..69e1483fd4f 100644 --- a/Firestore/Protos/CMakeLists.txt +++ b/Firestore/Protos/CMakeLists.txt @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -include(FindPythonInterp) - # Generate output in-place. So long as the build is idempotent this helps # verify that the protoc-generated output isn't changing. set(OUTPUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}) @@ -200,7 +198,7 @@ if(FIREBASE_IOS_PROTOC_GENERATE_SOURCES) COMMENT "Generating nanopb sources" OUTPUT ${NANOPB_GENERATED_SOURCES} COMMAND - ${PYTHON_EXECUTABLE} + ${FIREBASE_PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/build_protos.py --nanopb --protoc=$ @@ -232,7 +230,7 @@ if(FIREBASE_IOS_PROTOC_GENERATE_SOURCES) COMMENT "Generating C++ protobuf sources" OUTPUT ${PROTOBUF_CPP_GENERATED_SOURCES} COMMAND - ${PYTHON_EXECUTABLE} + ${FIREBASE_PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/build_protos.py --cpp --protoc=$ diff --git a/Firestore/core/CMakeLists.txt b/Firestore/core/CMakeLists.txt index aeb96431b8e..668ab0c0757 100644 --- a/Firestore/core/CMakeLists.txt +++ b/Firestore/core/CMakeLists.txt @@ -14,7 +14,6 @@ include(CheckSymbolExists) include(CheckIncludeFiles) -include(FindPythonInterp) ## firestore_util @@ -286,7 +285,7 @@ add_custom_command( OUTPUT ${GRPC_ROOT_CERTIFICATE_SOURCES} COMMAND - ${PYTHON_EXECUTABLE} ${FIREBASE_SOURCE_DIR}/scripts/binary_to_array.py + ${FIREBASE_PYTHON_EXECUTABLE} ${FIREBASE_SOURCE_DIR}/scripts/binary_to_array.py --output_header=${OUTPUT_DIR}/grpc_root_certificates_generated.h --output_source=${OUTPUT_DIR}/grpc_root_certificates_generated.cc --cpp_namespace=firebase::firestore::remote From 953450978c49103b84093fc40630992595c5595a Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Fri, 15 Apr 2022 16:06:28 -0400 Subject: [PATCH 22/23] Dynamically create Python virtualenvs and install requirements into them --- CMakeLists.txt | 16 --- Firestore/Protos/CMakeLists.txt | 11 +- Firestore/core/CMakeLists.txt | 7 +- cmake/python_setup.cmake | 185 ++++++++++++++++++++++++++++++++ 4 files changed, 200 insertions(+), 19 deletions(-) create mode 100644 cmake/python_setup.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a6b005a195..aebceae4e1c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,22 +72,6 @@ option( ON ) -# Find a Python interpreter using the best available mechanism. -if(${CMAKE_VERSION} VERSION_LESS "3.12") - include(FindPythonInterp) - set(DEFAULT_FIREBASE_PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE}") -else() - find_package(Python3 COMPONENTS Interpreter) - set(DEFAULT_FIREBASE_PYTHON_EXECUTABLE "${Python3_EXECUTABLE}") -endif() - -set( - FIREBASE_PYTHON_EXECUTABLE - "${DEFAULT_FIREBASE_PYTHON_EXECUTABLE}" - CACHE FILEPATH - "The Python interpreter to use" -) - list(INSERT CMAKE_MODULE_PATH 0 ${PROJECT_SOURCE_DIR}/cmake) include(compiler_setup) include(sanitizer_options) diff --git a/Firestore/Protos/CMakeLists.txt b/Firestore/Protos/CMakeLists.txt index 69e1483fd4f..96da74110d5 100644 --- a/Firestore/Protos/CMakeLists.txt +++ b/Firestore/Protos/CMakeLists.txt @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +include(python_setup) +FirebaseSetupPythonInterpreter( + OUTVAR MY_PYTHON_EXECUTABLE + KEY FirestoreProtos + REQUIREMENTS six +) + # Generate output in-place. So long as the build is idempotent this helps # verify that the protoc-generated output isn't changing. set(OUTPUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}) @@ -198,7 +205,7 @@ if(FIREBASE_IOS_PROTOC_GENERATE_SOURCES) COMMENT "Generating nanopb sources" OUTPUT ${NANOPB_GENERATED_SOURCES} COMMAND - ${FIREBASE_PYTHON_EXECUTABLE} + ${MY_PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/build_protos.py --nanopb --protoc=$ @@ -230,7 +237,7 @@ if(FIREBASE_IOS_PROTOC_GENERATE_SOURCES) COMMENT "Generating C++ protobuf sources" OUTPUT ${PROTOBUF_CPP_GENERATED_SOURCES} COMMAND - ${FIREBASE_PYTHON_EXECUTABLE} + ${MY_PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/build_protos.py --cpp --protoc=$ diff --git a/Firestore/core/CMakeLists.txt b/Firestore/core/CMakeLists.txt index 668ab0c0757..a1f477cbe3b 100644 --- a/Firestore/core/CMakeLists.txt +++ b/Firestore/core/CMakeLists.txt @@ -15,6 +15,11 @@ include(CheckSymbolExists) include(CheckIncludeFiles) +include(python_setup) +FirebaseSetupPythonInterpreter( + OUTVAR MY_PYTHON_EXECUTABLE + KEY FirestoreCore +) ## firestore_util @@ -285,7 +290,7 @@ add_custom_command( OUTPUT ${GRPC_ROOT_CERTIFICATE_SOURCES} COMMAND - ${FIREBASE_PYTHON_EXECUTABLE} ${FIREBASE_SOURCE_DIR}/scripts/binary_to_array.py + ${MY_PYTHON_EXECUTABLE} ${FIREBASE_SOURCE_DIR}/scripts/binary_to_array.py --output_header=${OUTPUT_DIR}/grpc_root_certificates_generated.h --output_source=${OUTPUT_DIR}/grpc_root_certificates_generated.cc --cpp_namespace=firebase::firestore::remote diff --git a/cmake/python_setup.cmake b/cmake/python_setup.cmake new file mode 100644 index 00000000000..f8440858798 --- /dev/null +++ b/cmake/python_setup.cmake @@ -0,0 +1,185 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Sets up a Python interpreter, installing required dependencies. +# +# This function does the following: +# 1. Uses the best-available built-in cmake mechanism to find a Python +# interpreter (a.k.a. the "host" interpreter). +# 2. Creates a Python virtualenv in the cmake binary directory using the +# Python interpreter found in the previous step. +# 3. Locates the Python interpreter in the virtualenv and sets its path in +# the specified variable. +# 4. Runs pip install to install the required dependencies in the virtualenv. +# +# This function also writes "stamp files" into the virtualenv. These files +# are used to determine if the virtualenv is up-to-date from a previous cmake +# run or if it needs to be recreated from scratch. +# +# If any errors are detected (e.g. cannot install one of the given requirements) +# then a fatal error is logged, causing the cmake processing to terminate. +# +# Arguments: +# OUTVAR - The name of the variable into which to store the path of the +# Python executable from the virtualenv. +# KEY - A unique key to ensure isolation from other Python virtualenv +# environments created by this function. This value will be used in the +# path of the virtualenv on the file system and in the name of the cmake +# cache variable that stores its path. +# REQUIREMENTS - (Optional) A list of Python packages to install in the +# virtualenv using pip. +# +# Example: +# include(python_setup) +# FirebaseSetupPythonInterpreter( +# OUTVAR MY_PYTHON_EXECUTABLE +# KEY ScanStuff +# REQUIREMENTS six absl-py +# ) +# execute_process(COMMAND "${MY_PYTHON_EXECUTABLE}" scan_stuff.py) +function(FirebaseSetupPythonInterpreter) + cmake_parse_arguments( + PARSE_ARGV 0 + ARG + "" # zero-value arguments + "OUTVAR;KEY" # single-value arguments + "REQUIREMENTS" # multi-value arguments + ) + + # Validate this function's arguments. + if("${ARG_OUTVAR}" STREQUAL "") + message(FATAL_ERROR "OUTVAR must be specified to ${CMAKE_CURRENT_FUNCTION}") + elseif("${ARG_KEY}" STREQUAL "") + message(FATAL_ERROR "KEY must be specified to ${CMAKE_CURRENT_FUNCTION}") + endif() + + # Calculate the name of the cmake cache variable where to store the + # Python interpreter path. + set(CACHEVAR "FIREBASE_PYTHON_EXECUTABLE_${ARG_KEY}") + + # Find a Python interpreter using the best available mechanism. + if(${CMAKE_VERSION} VERSION_LESS "3.12") + include(FindPythonInterp) + set(DEFAULT_PYTHON_HOST_EXECUTABLE "${PYTHON_EXECUTABLE}") + else() + find_package(Python3 COMPONENTS Interpreter REQUIRED) + set(DEFAULT_PYTHON_HOST_EXECUTABLE "${Python3_EXECUTABLE}") + endif() + + # Get the Python interpreter on the host system to use. + set( + FIREBASE_PYTHON_HOST_EXECUTABLE + "${DEFAULT_PYTHON_HOST_EXECUTABLE}" + CACHE FILEPATH + "The Python interpreter on the host system to use" + ) + + # Create a virtualenv into which to install Python's dependencies. + # https://docs.python.org/3/library/venv.html + set(PYVENV_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/pyvenv/${ARG_KEY}") + set(STAMP_FILE1 "${PYVENV_DIRECTORY}/cmake_firebase_python_stamp1.txt") + set(STAMP_FILE2 "${PYVENV_DIRECTORY}/cmake_firebase_python_stamp2.txt") + + # Check if the virtualenv is already up-to-date by examining the contents of + # the stamp files. The stamp files store the path of the host Python + # interpreter and the dependencies that were installed by pip. If both of + # these files exist and contain the same Python interpreter and dependencies + # then just re-use the virtualenv; otherwise, re-create it. + if(EXISTS "${STAMP_FILE1}" AND EXISTS "${STAMP_FILE2}") + file(READ "${STAMP_FILE1}" STAMP_FILE1_CONTENTS) + file(READ "${STAMP_FILE2}" STAMP_FILE2_CONTENTS) + if( + ("${STAMP_FILE1_CONTENTS}" STREQUAL "${FIREBASE_PYTHON_HOST_EXECUTABLE}") + AND + ("${STAMP_FILE2_CONTENTS}" STREQUAL "${ARG_REQUIREMENTS}") + ) + set("${ARG_OUTVAR}" "$CACHE{${CACHEVAR}}" PARENT_SCOPE) + message(STATUS + "${CMAKE_CURRENT_FUNCTION}: Using Python interpreter: $CACHE{${CACHEVAR}}" + ) + return() + endif() + endif() + + # Create the virtualenv. + message(STATUS + "${CMAKE_CURRENT_FUNCTION}: Creating Python virtualenv in " + "${PYVENV_DIRECTORY} using ${FIREBASE_PYTHON_HOST_EXECUTABLE}" + ) + file(REMOVE_RECURSE "${PYVENV_DIRECTORY}") + execute_process( + COMMAND + "${FIREBASE_PYTHON_HOST_EXECUTABLE}" + -m + venv + "${PYVENV_DIRECTORY}" + --upgrade-deps + RESULT_VARIABLE + FIREBASE_PYVENV_CREATE_RESULT + ) + if(NOT FIREBASE_PYVENV_CREATE_RESULT EQUAL 0) + message(FATAL_ERROR + "Failed to create a Python virtualenv in ${PYVENV_DIRECTORY} " + "using ${FIREBASE_PYTHON_HOST_EXECUTABLE}") + endif() + + # Find the Python interpreter in the virtualenv. + find_program( + "${CACHEVAR}" + DOC "The Python interpreter to use for ${ARG_KEY}" + NAMES python3 python + PATHS "${PYVENV_DIRECTORY}" + PATH_SUFFIXES bin Scripts + NO_DEFAULT_PATH + ) + if(NOT ${CACHEVAR}) + message(FATAL_ERROR "Unable to find Python executable in ${PYVENV_DIRECTORY}") + else() + set(PYTHON_EXECUTABLE "$CACHE{${CACHEVAR}}") + message(STATUS + "${CMAKE_CURRENT_FUNCTION}: Found Python executable in virtualenv: " + "${PYTHON_EXECUTABLE}" + ) + endif() + + # Install the dependencies in the virtualenv, if any are requested. + if(NOT ("${ARG_REQUIREMENTS}" STREQUAL "")) + message(STATUS + "${CMAKE_CURRENT_FUNCTION}: Installing Python dependencies into " + "${PYVENV_DIRECTORY}: ${ARG_REQUIREMENTS}" + ) + execute_process( + COMMAND + "${PYTHON_EXECUTABLE}" + -m + pip + install + ${ARG_REQUIREMENTS} + RESULT_VARIABLE + PIP_INSTALL_RESULT + ) + if(NOT PIP_INSTALL_RESULT EQUAL 0) + message(FATAL_ERROR + "Failed to install Python dependencies into " + "${PYVENV_DIRECTORY}: ${ARG_REQUIREMENTS}" + ) + endif() + endif() + + # Write the stamp files. + file(WRITE "${STAMP_FILE1}" "${FIREBASE_PYTHON_HOST_EXECUTABLE}") + file(WRITE "${STAMP_FILE2}" "${ARG_REQUIREMENTS}") + + set("${ARG_OUTVAR}" "${PYTHON_EXECUTABLE}" PARENT_SCOPE) +endfunction(FirebaseSetupPythonInterpreter) From 865162a0355a788728edf6798bcca5218dd50734 Mon Sep 17 00:00:00 2001 From: Denver Coneybeare Date: Fri, 15 Apr 2022 16:59:09 -0400 Subject: [PATCH 23/23] More python_setup.cmake improvements --- cmake/external/CMakeLists.txt | 7 ++++++- cmake/external/leveldb.cmake | 15 ++++++--------- cmake/python_setup.cmake | 18 +++++++----------- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/cmake/external/CMakeLists.txt b/cmake/external/CMakeLists.txt index c1de37b6d09..91c935a9ecf 100644 --- a/cmake/external/CMakeLists.txt +++ b/cmake/external/CMakeLists.txt @@ -15,7 +15,12 @@ cmake_minimum_required(VERSION 3.5.1) project(Firebase-download C CXX) -list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}) +list( + APPEND + CMAKE_MODULE_PATH + ${CMAKE_CURRENT_LIST_DIR} + ${CMAKE_CURRENT_LIST_DIR}/.. +) set( FIREBASE_DOWNLOAD_DIR diff --git a/cmake/external/leveldb.cmake b/cmake/external/leveldb.cmake index 2456cc2d651..920bf2928a3 100644 --- a/cmake/external/leveldb.cmake +++ b/cmake/external/leveldb.cmake @@ -14,14 +14,11 @@ include(ExternalProject) -# Find a Python interpreter using the best available mechanism. -if(${CMAKE_VERSION} VERSION_LESS "3.12") - include(FindPythonInterp) - set(PATCH_PYTHON_EXECUTABLE "${PYTHON_EXECUTABLE}") -else() - find_package(Python3 COMPONENTS Interpreter) - set(PATCH_PYTHON_EXECUTABLE "${Python3_EXECUTABLE}") -endif() +include(python_setup) +FirebaseSetupPythonInterpreter( + OUTVAR MY_PYTHON_EXECUTABLE + KEY LevelDbPatch +) if(TARGET leveldb) return() @@ -50,7 +47,7 @@ ExternalProject_Add( BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" - PATCH_COMMAND ${PATCH_PYTHON_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/leveldb_patch.py --snappy-source-dir ${snappy_source_dir} --snappy-binary-dir ${snappy_binary_dir} + PATCH_COMMAND "${MY_PYTHON_EXECUTABLE}" ${CMAKE_CURRENT_LIST_DIR}/leveldb_patch.py --snappy-source-dir ${snappy_source_dir} --snappy-binary-dir ${snappy_binary_dir} HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}" ) diff --git a/cmake/python_setup.cmake b/cmake/python_setup.cmake index f8440858798..35eeb9574b0 100644 --- a/cmake/python_setup.cmake +++ b/cmake/python_setup.cmake @@ -68,6 +68,8 @@ function(FirebaseSetupPythonInterpreter) # Python interpreter path. set(CACHEVAR "FIREBASE_PYTHON_EXECUTABLE_${ARG_KEY}") + set(LOG_PREFIX "${CMAKE_CURRENT_FUNCTION}(${ARG_KEY})") + # Find a Python interpreter using the best available mechanism. if(${CMAKE_VERSION} VERSION_LESS "3.12") include(FindPythonInterp) @@ -105,17 +107,15 @@ function(FirebaseSetupPythonInterpreter) ("${STAMP_FILE2_CONTENTS}" STREQUAL "${ARG_REQUIREMENTS}") ) set("${ARG_OUTVAR}" "$CACHE{${CACHEVAR}}" PARENT_SCOPE) - message(STATUS - "${CMAKE_CURRENT_FUNCTION}: Using Python interpreter: $CACHE{${CACHEVAR}}" - ) + message(STATUS "${LOG_PREFIX}: Using Python interpreter: $CACHE{${CACHEVAR}}") return() endif() endif() # Create the virtualenv. message(STATUS - "${CMAKE_CURRENT_FUNCTION}: Creating Python virtualenv in " - "${PYVENV_DIRECTORY} using ${FIREBASE_PYTHON_HOST_EXECUTABLE}" + "${LOG_PREFIX}: Creating Python virtualenv in ${PYVENV_DIRECTORY} " + "using ${FIREBASE_PYTHON_HOST_EXECUTABLE}" ) file(REMOVE_RECURSE "${PYVENV_DIRECTORY}") execute_process( @@ -124,7 +124,6 @@ function(FirebaseSetupPythonInterpreter) -m venv "${PYVENV_DIRECTORY}" - --upgrade-deps RESULT_VARIABLE FIREBASE_PYVENV_CREATE_RESULT ) @@ -147,16 +146,13 @@ function(FirebaseSetupPythonInterpreter) message(FATAL_ERROR "Unable to find Python executable in ${PYVENV_DIRECTORY}") else() set(PYTHON_EXECUTABLE "$CACHE{${CACHEVAR}}") - message(STATUS - "${CMAKE_CURRENT_FUNCTION}: Found Python executable in virtualenv: " - "${PYTHON_EXECUTABLE}" - ) + message(STATUS "${LOG_PREFIX}: Found Python executable in virtualenv: ${PYTHON_EXECUTABLE}") endif() # Install the dependencies in the virtualenv, if any are requested. if(NOT ("${ARG_REQUIREMENTS}" STREQUAL "")) message(STATUS - "${CMAKE_CURRENT_FUNCTION}: Installing Python dependencies into " + "${LOG_PREFIX}: Installing Python dependencies into " "${PYVENV_DIRECTORY}: ${ARG_REQUIREMENTS}" ) execute_process(