Update UIDs/GIDs/mode to 32-bit types (fixes gh #173)

include/dwarfs/entry.h

@@ -100,12 +100,12 @@ class entry : public entry_interface {
   virtual std::optional<uint32_t> const& inode_num() const = 0;
 
   // more methods from entry_interface
-  uint16_t get_permissions() const override;
-  void set_permissions(uint16_t perm) override;
-  uint16_t get_uid() const override;
-  void set_uid(uint16_t uid) override;
-  uint16_t get_gid() const override;
-  void set_gid(uint16_t gid) override;
+  mode_type get_permissions() const override;
+  void set_permissions(mode_type perm) override;
+  uid_type get_uid() const override;
+  void set_uid(uid_type uid) override;
+  gid_type get_gid() const override;
+  void set_gid(gid_type gid) override;
   uint64_t get_atime() const override;
   void set_atime(uint64_t atime) override;
   uint64_t get_mtime() const override;

include/dwarfs/entry_interface.h

@@ -23,12 +23,17 @@
 
 #include <string>
 
+#include "dwarfs/file_stat.h"
 #include "dwarfs/object.h"
 
 namespace dwarfs {
 
 class entry_interface : public object {
  public:
+  using uid_type = file_stat::uid_type;
+  using gid_type = file_stat::gid_type;
+  using mode_type = file_stat::mode_type;
+
   virtual std::string path_as_string() const = 0;
   virtual std::string dpath() const = 0;
   virtual std::string unix_dpath() const = 0;
@@ -37,12 +42,12 @@ class entry_interface : public object {
   virtual size_t size() const = 0;
   virtual bool is_directory() const = 0;
 
-  virtual uint16_t get_permissions() const = 0;
-  virtual void set_permissions(uint16_t perm) = 0;
-  virtual uint16_t get_uid() const = 0;
-  virtual void set_uid(uint16_t uid) = 0;
-  virtual uint16_t get_gid() const = 0;
-  virtual void set_gid(uint16_t gid) = 0;
+  virtual mode_type get_permissions() const = 0;
+  virtual void set_permissions(mode_type perm) = 0;
+  virtual uid_type get_uid() const = 0;
+  virtual void set_uid(uid_type uid) = 0;
+  virtual gid_type get_gid() const = 0;
+  virtual void set_gid(gid_type gid) = 0;
   virtual uint64_t get_atime() const = 0;
   virtual void set_atime(uint64_t atime) = 0;
   virtual uint64_t get_mtime() const = 0;

include/dwarfs/global_entry_data.h

@@ -28,21 +28,27 @@
 
 #include <folly/container/F14Map.h>
 
+#include "dwarfs/file_stat.h"
+
 namespace dwarfs {
 
 struct scanner_options;
 
 class global_entry_data {
  public:
+  using uid_type = file_stat::uid_type;
+  using gid_type = file_stat::gid_type;
+  using mode_type = file_stat::mode_type;
+
   enum class timestamp_type { ATIME, MTIME, CTIME };
 
   global_entry_data(scanner_options const& options)
       : options_(options) {}
 
-  void add_uid(uint16_t uid);
-  void add_gid(uint16_t gid);
+  void add_uid(uid_type uid);
+  void add_gid(gid_type gid);
 
-  void add_mode(uint16_t mode) { add(mode, modes_, next_mode_index_); }
+  void add_mode(mode_type mode) { add(mode, modes_, next_mode_index_); }
 
   void add_mtime(uint64_t time);
   void add_atime(uint64_t time);
@@ -56,9 +62,9 @@ class global_entry_data {
     index(symlinks_);
   }
 
-  uint16_t get_uid_index(uint16_t uid) const;
-  uint16_t get_gid_index(uint16_t gid) const;
-  uint16_t get_mode_index(uint16_t mode) const;
+  size_t get_uid_index(uid_type uid) const;
+  size_t get_gid_index(gid_type gid) const;
+  size_t get_mode_index(mode_type mode) const;
 
   uint32_t get_name_index(std::string const& name) const;
   uint32_t get_symlink_table_entry(std::string const& link) const;
@@ -67,9 +73,9 @@ class global_entry_data {
   uint64_t get_atime_offset(uint64_t time) const;
   uint64_t get_ctime_offset(uint64_t time) const;
 
-  std::vector<uint16_t> get_uids() const;
-  std::vector<uint16_t> get_gids() const;
-  std::vector<uint16_t> get_modes() const;
+  std::vector<uid_type> get_uids() const;
+  std::vector<gid_type> get_gids() const;
+  std::vector<mode_type> get_modes() const;
 
   std::vector<std::string> get_names() const;
   std::vector<std::string> get_symlinks() const;
@@ -85,8 +91,8 @@ class global_entry_data {
 
   static void index(map_type<std::string, uint32_t>& map);
 
-  void
-  add(uint16_t val, map_type<uint16_t, uint16_t>& map, uint16_t& next_index) {
+  template <typename T>
+  void add(T val, map_type<T, T>& map, T& next_index) {
     if (map.emplace(val, next_index).second) {
       ++next_index;
     }
@@ -94,14 +100,14 @@ class global_entry_data {
 
   uint64_t get_time_offset(uint64_t time) const;
 
-  map_type<uint16_t, uint16_t> uids_;
-  map_type<uint16_t, uint16_t> gids_;
-  map_type<uint16_t, uint16_t> modes_;
+  map_type<uid_type, uid_type> uids_;
+  map_type<gid_type, gid_type> gids_;
+  map_type<mode_type, mode_type> modes_;
   map_type<std::string, uint32_t> names_;
   map_type<std::string, uint32_t> symlinks_;
-  uint16_t next_uid_index_{0};
-  uint16_t next_gid_index_{0};
-  uint16_t next_mode_index_{0};
+  uid_type next_uid_index_{0};
+  gid_type next_gid_index_{0};
+  mode_type next_mode_index_{0};
   uint64_t timestamp_base_{std::numeric_limits<uint64_t>::max()};
   scanner_options const& options_;
 };

include/dwarfs/metadata_types.h

@@ -33,6 +33,7 @@
 
 #include <thrift/lib/cpp2/frozen/FrozenUtil.h>
 
+#include "dwarfs/file_stat.h"
 #include "dwarfs/file_type.h"
 #include "dwarfs/string_table.h"
 
@@ -90,15 +91,19 @@ class inode_view
   friend class dir_entry_view;
 
  public:
-  uint16_t mode() const;
+  using uid_type = file_stat::uid_type;
+  using gid_type = file_stat::gid_type;
+  using mode_type = file_stat::mode_type;
+
+  mode_type mode() const;
   posix_file_type::value type() const {
     return posix_file_type::from_mode(mode());
   }
   bool is_regular_file() const { return type() == posix_file_type::regular; }
   bool is_directory() const { return type() == posix_file_type::directory; }
   bool is_symlink() const { return type() == posix_file_type::symlink; }
-  uint16_t getuid() const;
-  uint16_t getgid() const;
+  uid_type getuid() const;
+  gid_type getgid() const;
   uint32_t inode_num() const { return inode_num_; }
 
  private:

include/dwarfs/options.h

@@ -27,6 +27,7 @@
 #include <iosfwd>
 #include <optional>
 
+#include "dwarfs/file_stat.h"
 #include "dwarfs/types.h"
 
 namespace dwarfs {
@@ -97,8 +98,8 @@ struct file_order_options {
 struct scanner_options {
   file_order_options file_order;
   std::optional<std::string> file_hash_algorithm{"xxh3-128"};
-  std::optional<uint16_t> uid;
-  std::optional<uint16_t> gid;
+  std::optional<file_stat::uid_type> uid;
+  std::optional<file_stat::gid_type> gid;
   std::optional<uint64_t> timestamp;
   bool keep_all_times{false};
   bool remove_empty_dirs{false};

src/dwarfs/entry.cpp

@@ -167,17 +167,17 @@ void entry::pack(thrift::metadata::inode_data& entry_v2,
 
 entry::type_t file::type() const { return E_FILE; }
 
-uint16_t entry::get_permissions() const { return stat_.permissions(); }
+auto entry::get_permissions() const -> mode_type { return stat_.permissions(); }
 
-void entry::set_permissions(uint16_t perm) { stat_.set_permissions(perm); }
+void entry::set_permissions(mode_type perm) { stat_.set_permissions(perm); }
 
-uint16_t entry::get_uid() const { return stat_.uid; }
+auto entry::get_uid() const -> uid_type { return stat_.uid; }
 
-void entry::set_uid(uint16_t uid) { stat_.uid = uid; }
+void entry::set_uid(uid_type uid) { stat_.uid = uid; }
 
-uint16_t entry::get_gid() const { return stat_.gid; }
+auto entry::get_gid() const -> gid_type { return stat_.gid; }
 
-void entry::set_gid(uint16_t gid) { stat_.gid = gid; }
+void entry::set_gid(gid_type gid) { stat_.gid = gid; }
 
 uint64_t entry::get_atime() const { return stat_.atime; }
 

src/dwarfs/global_entry_data.cpp

@@ -35,23 +35,23 @@ std::vector<T> global_entry_data::get_vector(map_type<T, U> const& map) const {
          get<0>() | as<std::vector>();
 }
 
-std::vector<uint16_t> global_entry_data::get_uids() const {
+auto global_entry_data::get_uids() const -> std::vector<uid_type> {
   return get_vector(uids_);
 }
 
-std::vector<uint16_t> global_entry_data::get_gids() const {
+auto global_entry_data::get_gids() const -> std::vector<gid_type> {
   return get_vector(gids_);
 }
 
-std::vector<uint16_t> global_entry_data::get_modes() const {
+auto global_entry_data::get_modes() const -> std::vector<mode_type> {
   return get_vector(modes_);
 }
 
-std::vector<std::string> global_entry_data::get_names() const {
+auto global_entry_data::get_names() const -> std::vector<std::string> {
   return get_vector(names_);
 }
 
-std::vector<std::string> global_entry_data::get_symlinks() const {
+auto global_entry_data::get_symlinks() const -> std::vector<std::string> {
   return get_vector(symlinks_);
 }
 
@@ -84,15 +84,15 @@ uint64_t global_entry_data::get_timestamp_base() const {
          options_.time_resolution_sec;
 }
 
-uint16_t global_entry_data::get_uid_index(uint16_t uid) const {
+size_t global_entry_data::get_uid_index(uid_type uid) const {
   return options_.uid ? *options_.uid : DWARFS_NOTHROW(uids_.at(uid));
 }
 
-uint16_t global_entry_data::get_gid_index(uint16_t gid) const {
+size_t global_entry_data::get_gid_index(gid_type gid) const {
   return options_.gid ? *options_.gid : DWARFS_NOTHROW(gids_.at(gid));
 }
 
-uint16_t global_entry_data::get_mode_index(uint16_t mode) const {
+size_t global_entry_data::get_mode_index(mode_type mode) const {
   return DWARFS_NOTHROW(modes_.at(mode));
 }
 
@@ -105,13 +105,13 @@ global_entry_data::get_symlink_table_entry(std::string const& link) const {
   return DWARFS_NOTHROW(symlinks_.at(link));
 }
 
-void global_entry_data::add_uid(uint16_t uid) {
+void global_entry_data::add_uid(uid_type uid) {
   if (!options_.uid) {
     add(uid, uids_, next_uid_index_);
   }
 }
 
-void global_entry_data::add_gid(uint16_t gid) {
+void global_entry_data::add_gid(gid_type gid) {
   if (!options_.gid) {
     add(gid, gids_, next_gid_index_);
   }

src/dwarfs/metadata_types.cpp

@@ -534,11 +534,17 @@ uint32_t global_metadata::parent_dir_entry(uint32_t ino) const {
                       : meta_->directories()[ino].parent_entry();
 }
 
-uint16_t inode_view::mode() const { return meta_->modes()[mode_index()]; }
+auto inode_view::mode() const -> mode_type {
+  return meta_->modes()[mode_index()];
+}
 
-uint16_t inode_view::getuid() const { return meta_->uids()[owner_index()]; }
+auto inode_view::getuid() const -> uid_type {
+  return meta_->uids()[owner_index()];
+}
 
-uint16_t inode_view::getgid() const { return meta_->gids()[group_index()]; }
+auto inode_view::getgid() const -> gid_type {
+  return meta_->gids()[group_index()];
+}
 
 // TODO: pretty certain some of this stuff can be simplified
 

test/dwarfs.cpp

@@ -889,3 +889,35 @@ TEST(file_scanner, input_list) {
 
   EXPECT_EQ(expected, got);
 }
+
+TEST(filesystem, uid_gid_32bit) {
+  test::test_logger lgr;
+
+  auto input = std::make_shared<test::os_access_mock>();
+
+  input->add("", {1, 040755, 1, 0, 0, 10, 42, 0, 0, 0});
+  input->add("foo16.txt", {2, 0100755, 1, 60000, 65535, 5, 42, 0, 0, 0}, "hello");
+  input->add("foo32.txt", {3, 0100755, 1, 65536, 4294967295, 5, 42, 0, 0, 0}, "world");
+
+  auto fsimage = build_dwarfs(lgr, input, "null");
+
+  auto mm = std::make_shared<test::mmap_mock>(std::move(fsimage));
+
+  filesystem_v2 fs(lgr, mm);
+
+  auto iv16 = fs.find("/foo16.txt");
+  auto iv32 = fs.find("/foo32.txt");
+
+  EXPECT_TRUE(iv16);
+  EXPECT_TRUE(iv32);
+
+  file_stat st16, st32;
+
+  EXPECT_EQ(0, fs.getattr(*iv16, &st16));
+  EXPECT_EQ(0, fs.getattr(*iv32, &st32));
+
+  EXPECT_EQ(60000, st16.uid);
+  EXPECT_EQ(65535, st16.gid);
+  EXPECT_EQ(65536, st32.uid);
+  EXPECT_EQ(4294967295, st32.gid);
+}

thrift/metadata.thrift

@@ -244,13 +244,13 @@ struct metadata {
    6: list<UInt32>     symlink_table
 
    // user ids, for lookup by `inode.owner_index`
-   7: list<UInt16>     uids
+   7: list<UInt32>     uids
 
    // group ids, for lookup by `inode.group_index`
-   8: list<UInt16>     gids
+   8: list<UInt32>     gids
 
    // inode modes, for lookup by `inode.mode_index`
-   9: list<UInt16>     modes
+   9: list<UInt32>     modes
 
    // directory entry names, for lookup by `dir_entry.name_index`
   10: list<string>     names