Merge pull request #24 from tamatebako/maxirmx-sync-upstream

Chore: synchronize with upstream repository

.cirrus.yml

@@ -70,9 +70,7 @@ task:
     brew update
     brew install openssl@3  \
       bison flex gnu-sed bash boost double-conversion jemalloc fmt glog pkg-config \
-      libevent libsodium lz4 xz zlib
-
-# No ccache - it causes '/bin/sh: fork: Resource temporarily unavailable'  in 90% of runs
+      libevent libsodium lz4 xz zlib ccache
 
   pre_configure_script:
     echo CXX_FLAGS='-DTARGET_OS_SIMULATOR=0 -DTARGET_OS_IPHONE=0' >> $CIRRUS_ENV

CMakeLists.txt

@@ -100,20 +100,6 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
   add_compile_options(/Zc:__cplusplus /utf-8 /wd4267 /wd4244 /wd5219)
 endif()
 
-if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
-  set(CMAKE_C_LINK_LIBRARY_USING_WHOLE_ARCHIVE "-force_load <LIB_ITEM>")
-elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
-  set(CMAKE_C_LINK_LIBRARY_USING_WHOLE_ARCHIVE "/WHOLEARCHIVE:<LIBRARY>")
-elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-  set(CMAKE_C_LINK_LIBRARY_USING_WHOLE_ARCHIVE
-    "LINKER:--push-state,--whole-archive"
-    "<LINK_ITEM>"
-    "LINKER:--pop-state"
-  )
-else()
-  message(FATAL_ERROR "WHOLE_ARCHIVE is unsupported on this platform")
-endif()
-
 include(${CMAKE_SOURCE_DIR}/cmake/version.cmake)
 
 # -----------------------------------------------------------------------------------------
@@ -608,28 +594,27 @@ if("${TEBAKO_BUILD_SCOPE}" STREQUAL "ALL")
       target_link_libraries(dwarfs_main ${WINFSP})
       target_link_libraries(dwarfsuniversal delayimp.lib)
       target_link_options(dwarfsuniversal PRIVATE /DELAYLOAD:winfsp-x64.dll)
-    else()
+    else(WINFSP)
       target_compile_definitions(dwarfs_main PRIVATE FUSE_USE_VERSION=35)
       target_link_libraries(dwarfs_main PkgConfig::FUSE3)
-    endif()
+    endif(WINFSP)
     add_executable(dwarfs-bin src/dwarfs.cpp)
     target_link_libraries(dwarfs-bin dwarfs_main)
     set_target_properties(dwarfs-bin PROPERTIES OUTPUT_NAME dwarfs)
     target_link_libraries(dwarfsuniversal dwarfs_main)
     if(WINFSP)
       install(TARGETS dwarfs-bin RUNTIME DESTINATION bin)
-    else()
+    else(WINFSP)
       add_custom_command(OUTPUT  mount.dwarfs
                        COMMAND ${CMAKE_COMMAND} -E create_symlink dwarfs mount.dwarfs
                        DEPENDS dwarfs-bin)
       list(APPEND SYMLINKS mount.dwarfs)
       install(FILES ${CMAKE_CURRENT_BINARY_DIR}/mount.dwarfs DESTINATION sbin)
       install(TARGETS dwarfs-bin RUNTIME DESTINATION sbin)
-    endif()
+    endif(WINFSP)
     list(APPEND BINARY_TARGETS dwarfs-bin)
     list(APPEND MAIN_TARGETS dwarfs_main)
-  endif()
-endif()
+  endif(FUSE3_FOUND OR WINFSP)
 
 if(FUSE_FOUND AND (WITH_LEGACY_FUSE OR NOT FUSE3_FOUND))
   add_library(dwarfs2_main src/dwarfs_main.cpp)
@@ -638,19 +623,20 @@ if(FUSE_FOUND AND (WITH_LEGACY_FUSE OR NOT FUSE3_FOUND))
   target_link_libraries(dwarfs2_main PkgConfig::FUSE)
   add_executable(dwarfs2-bin src/dwarfs.cpp)
   target_link_libraries(dwarfs2-bin dwarfs2_main)
-  if(NOT FUSE3_FOUND)
-    target_link_libraries(dwarfsuniversal dwarfs2_main)
-  endif()
-  set_target_properties(dwarfs2-bin PROPERTIES OUTPUT_NAME dwarfs2)
-  add_custom_command(OUTPUT  mount.dwarfs2
-                     COMMAND ${CMAKE_COMMAND} -E create_symlink dwarfs2 mount.dwarfs2
-                     DEPENDS dwarfs2-bin)
-  list(APPEND SYMLINKS mount.dwarfs2)
-  install(TARGETS dwarfs2-bin RUNTIME DESTINATION sbin)
-  install(FILES ${CMAKE_CURRENT_BINARY_DIR}/mount.dwarfs2 DESTINATION sbin)
-  list(APPEND BINARY_TARGETS dwarfs2-bin)
-  list(APPEND MAIN_TARGETS dwarfs2_main)
-endif()
+    if(NOT FUSE3_FOUND)
+      target_link_libraries(dwarfsuniversal dwarfs2_main)
+    endif(NOT FUSE3_FOUND)
+    set_target_properties(dwarfs2-bin PROPERTIES OUTPUT_NAME dwarfs2)
+    add_custom_command(OUTPUT  mount.dwarfs2
+                       COMMAND ${CMAKE_COMMAND} -E create_symlink dwarfs2 mount.dwarfs2
+                       DEPENDS dwarfs2-bin)
+    list(APPEND SYMLINKS mount.dwarfs2)
+    install(TARGETS dwarfs2-bin RUNTIME DESTINATION sbin)
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/mount.dwarfs2 DESTINATION sbin)
+    list(APPEND BINARY_TARGETS dwarfs2-bin)
+    list(APPEND MAIN_TARGETS dwarfs2_main)
+  endif(FUSE_FOUND AND (WITH_LEGACY_FUSE OR NOT FUSE3_FOUND))
+endif("${TEBAKO_BUILD_SCOPE}" STREQUAL "ALL")
 
 add_custom_target(symlinks ALL DEPENDS ${SYMLINKS})
 

include/dwarfs/console_writer.h

@@ -35,7 +35,7 @@ namespace dwarfs {
 
 class progress;
 
-class console_writer : public logger {
+class console_writer : public stream_logger {
  public:
   using get_term_width_type = std::function<size_t()>;
 
@@ -46,25 +46,20 @@ class console_writer : public logger {
                  get_term_width_type get_term_width, level_type threshold,
                  display_mode mode = NORMAL, bool verbose = false);
 
-  void write(level_type level, const std::string& output, char const* file,
-             int line) override;
-
   void update(const progress& p, bool last);
 
  private:
+  void preamble() override;
+  void postamble() override;
+  std::string_view get_newline() const override;
   void rewind();
 
-  std::ostream& os_;
-  std::mutex mx_;
-  std::atomic<level_type> threshold_;
   std::string statebuf_;
   double frac_;
   std::atomic<size_t> counter_{0};
   progress_mode const pg_mode_;
   get_term_width_type get_term_width_;
   display_mode const mode_;
-  bool const color_;
-  bool const with_context_;
   bool const debug_progress_;
   bool writing_{false};
   speedometer<uint64_t> read_speed_;

include/dwarfs/entry.h

@@ -101,12 +101,12 @@ class entry : public entry_interface {
   virtual std::optional<uint32_t> const& inode_num() const = 0;
 
   // more methods from entry_interface
-  uint16_t get_permissions() const override;
-  void set_permissions(uint16_t perm) override;
-  uint16_t get_uid() const override;
-  void set_uid(uint16_t uid) override;
-  uint16_t get_gid() const override;
-  void set_gid(uint16_t gid) override;
+  mode_type get_permissions() const override;
+  void set_permissions(mode_type perm) override;
+  uid_type get_uid() const override;
+  void set_uid(uid_type uid) override;
+  gid_type get_gid() const override;
+  void set_gid(gid_type gid) override;
   uint64_t get_atime() const override;
   void set_atime(uint64_t atime) override;
   uint64_t get_mtime() const override;

include/dwarfs/entry_interface.h

@@ -23,12 +23,17 @@
 
 #include <string>
 
+#include "dwarfs/file_stat.h"
 #include "dwarfs/object.h"
 
 namespace dwarfs {
 
 class entry_interface : public object {
  public:
+  using uid_type = file_stat::uid_type;
+  using gid_type = file_stat::gid_type;
+  using mode_type = file_stat::mode_type;
+
   virtual std::string path_as_string() const = 0;
   virtual std::string dpath() const = 0;
   virtual std::string unix_dpath() const = 0;
@@ -37,12 +42,12 @@ class entry_interface : public object {
   virtual size_t size() const = 0;
   virtual bool is_directory() const = 0;
 
-  virtual uint16_t get_permissions() const = 0;
-  virtual void set_permissions(uint16_t perm) = 0;
-  virtual uint16_t get_uid() const = 0;
-  virtual void set_uid(uint16_t uid) = 0;
-  virtual uint16_t get_gid() const = 0;
-  virtual void set_gid(uint16_t gid) = 0;
+  virtual mode_type get_permissions() const = 0;
+  virtual void set_permissions(mode_type perm) = 0;
+  virtual uid_type get_uid() const = 0;
+  virtual void set_uid(uid_type uid) = 0;
+  virtual gid_type get_gid() const = 0;
+  virtual void set_gid(gid_type gid) = 0;
   virtual uint64_t get_atime() const = 0;
   virtual void set_atime(uint64_t atime) = 0;
   virtual uint64_t get_mtime() const = 0;

include/dwarfs/global_entry_data.h

@@ -28,21 +28,27 @@
 
 #include <folly/container/F14Map.h>
 
+#include "dwarfs/file_stat.h"
+
 namespace dwarfs {
 
 struct scanner_options;
 
 class global_entry_data {
  public:
+  using uid_type = file_stat::uid_type;
+  using gid_type = file_stat::gid_type;
+  using mode_type = file_stat::mode_type;
+
   enum class timestamp_type { ATIME, MTIME, CTIME };
 
   global_entry_data(scanner_options const& options)
       : options_(options) {}
 
-  void add_uid(uint16_t uid);
-  void add_gid(uint16_t gid);
+  void add_uid(uid_type uid);
+  void add_gid(gid_type gid);
 
-  void add_mode(uint16_t mode) { add(mode, modes_, next_mode_index_); }
+  void add_mode(mode_type mode) { add(mode, modes_, next_mode_index_); }
 
   void add_mtime(uint64_t time);
   void add_atime(uint64_t time);
@@ -56,9 +62,9 @@ class global_entry_data {
     index(symlinks_);
   }
 
-  uint16_t get_uid_index(uint16_t uid) const;
-  uint16_t get_gid_index(uint16_t gid) const;
-  uint16_t get_mode_index(uint16_t mode) const;
+  size_t get_uid_index(uid_type uid) const;
+  size_t get_gid_index(gid_type gid) const;
+  size_t get_mode_index(mode_type mode) const;
 
   uint32_t get_name_index(std::string const& name) const;
   uint32_t get_symlink_table_entry(std::string const& link) const;
@@ -67,9 +73,9 @@ class global_entry_data {
   uint64_t get_atime_offset(uint64_t time) const;
   uint64_t get_ctime_offset(uint64_t time) const;
 
-  std::vector<uint16_t> get_uids() const;
-  std::vector<uint16_t> get_gids() const;
-  std::vector<uint16_t> get_modes() const;
+  std::vector<uid_type> get_uids() const;
+  std::vector<gid_type> get_gids() const;
+  std::vector<mode_type> get_modes() const;
 
   std::vector<std::string> get_names() const;
   std::vector<std::string> get_symlinks() const;
@@ -85,23 +91,23 @@ class global_entry_data {
 
   static void index(map_type<std::string, uint32_t>& map);
 
-  void
-  add(uint16_t val, map_type<uint16_t, uint16_t>& map, uint16_t& next_index) {
+  template <typename T>
+  void add(T val, map_type<T, T>& map, T& next_index) {
     if (map.emplace(val, next_index).second) {
       ++next_index;
     }
   }
 
   uint64_t get_time_offset(uint64_t time) const;
 
-  map_type<uint16_t, uint16_t> uids_;
-  map_type<uint16_t, uint16_t> gids_;
-  map_type<uint16_t, uint16_t> modes_;
+  map_type<uid_type, uid_type> uids_;
+  map_type<gid_type, gid_type> gids_;
+  map_type<mode_type, mode_type> modes_;
   map_type<std::string, uint32_t> names_;
   map_type<std::string, uint32_t> symlinks_;
-  uint16_t next_uid_index_{0};
-  uint16_t next_gid_index_{0};
-  uint16_t next_mode_index_{0};
+  uid_type next_uid_index_{0};
+  gid_type next_gid_index_{0};
+  mode_type next_mode_index_{0};
   uint64_t timestamp_base_{std::numeric_limits<uint64_t>::max()};
   scanner_options const& options_;
 };

include/dwarfs/logger.h

@@ -88,11 +88,22 @@ class stream_logger : public logger {
   void set_threshold(level_type threshold);
   void set_with_context(bool with_context) { with_context_ = with_context; }
 
+ protected:
+  virtual void preamble();
+  virtual void postamble();
+  virtual std::string_view get_newline() const;
+
+  std::ostream& log_stream() const { return os_; }
+  std::mutex& log_mutex() const { return mx_; }
+  bool log_is_colored() const { return color_; }
+  level_type log_threshold() const { return threshold_.load(); }
+
  private:
   std::ostream& os_;
-  std::mutex mx_;
+  std::mutex mutable mx_;
   std::atomic<level_type> threshold_;
   bool const color_;
+  bool const enable_stack_trace_;
   bool with_context_;
 };
 
@@ -306,7 +317,8 @@ class log_proxy {
 };
 
 #define LOG_DETAIL_LEVEL(level, lgr, method)                                   \
-  if (lgr.is_enabled_for(::dwarfs::logger::level))                             \
+  if constexpr (std::decay_t<decltype(lgr)>::is_enabled_for(                   \
+                    ::dwarfs::logger::level))                                  \
   lgr.method(__FILE__, __LINE__)
 
 #define LOG_PROXY(policy, lgr) ::dwarfs::log_proxy<policy> log_(lgr)

include/dwarfs/metadata_types.h

@@ -33,6 +33,7 @@
 
 #include <thrift/lib/cpp2/frozen/FrozenUtil.h>
 
+#include "dwarfs/file_stat.h"
 #include "dwarfs/file_type.h"
 #include "dwarfs/string_table.h"
 
@@ -90,15 +91,19 @@ class inode_view
   friend class dir_entry_view;
 
  public:
-  uint16_t mode() const;
+  using uid_type = file_stat::uid_type;
+  using gid_type = file_stat::gid_type;
+  using mode_type = file_stat::mode_type;
+
+  mode_type mode() const;
   posix_file_type::value type() const {
     return posix_file_type::from_mode(mode());
   }
   bool is_regular_file() const { return type() == posix_file_type::regular; }
   bool is_directory() const { return type() == posix_file_type::directory; }
   bool is_symlink() const { return type() == posix_file_type::symlink; }
-  uint16_t getuid() const;
-  uint16_t getgid() const;
+  uid_type getuid() const;
+  gid_type getgid() const;
   uint32_t inode_num() const { return inode_num_; }
 
  private:

include/dwarfs/options.h

@@ -27,6 +27,7 @@
 #include <iosfwd>
 #include <optional>
 
+#include "dwarfs/file_stat.h"
 #include "dwarfs/types.h"
 
 namespace dwarfs {
@@ -97,8 +98,8 @@ struct file_order_options {
 struct scanner_options {
   file_order_options file_order;
   std::optional<std::string> file_hash_algorithm{"xxh3-128"};
-  std::optional<uint16_t> uid;
-  std::optional<uint16_t> gid;
+  std::optional<file_stat::uid_type> uid;
+  std::optional<file_stat::gid_type> gid;
   std::optional<uint64_t> timestamp;
   bool keep_all_times{false};
   bool remove_empty_dirs{false};

include/dwarfs/util.h

@@ -60,4 +60,6 @@ void shorten_path_string(std::string& path, char separator, size_t max_len);
 
 std::filesystem::path canonical_path(std::filesystem::path p);
 
+bool getenv_is_enabled(char const* var);
+
 } // namespace dwarfs