fuse-overlays: introduce xattr to override gid/uid/mode

introduce a new xattr "user.fuseoverlayfs.override_stat" that permit
to override the reported uid/gid/mode for lower layers.

It enables sharing storage among different users.

Since it is not possible to use "user.*" xattrs for symlinks, provide
also a privileged variant "security.fuseoverlayfs.override_stat", so
the root user can create the xattr for symlinks as well.

A script "fix-mode.py" is provided for converting an existing
layer/storage to the new model.  It is a destructive operation as
every file is converted to mode 0755, thus it is not usable anymore
with native overlay, or older versions of fuse-overlayfs.

Example with Podman:

Rootless:
Modify /.config/containers/storage.conf and add under storage.options:
additionalimagestores = ["/var/lib/shared-storage"]

Assuming an empty local storage for the user:

$ podman images
REPOSITORY                TAG     IMAGE ID      CREATED      SIZE    ReadOnly
docker.io/library/fedora  latest  a368cbcfa678  5 weeks ago  189 MB  true

and the files show the original mode and owner:

$ podman run --read-only --rm -ti docker.io/library/fedora ls -l /
lrwxrwxrwx.   1 root   root      7 Jan 28  2020 bin -> usr/bin
dr-xr-xr-x.   2 root   root      6 Jan 28  2020 boot
drwxr-xr-x.   5 root   root    360 Aug 15 13:26 dev
drwxr-xr-x.  41 root   root   4096 Jul  9 06:48 etc
drwxr-xr-x.   2 root   root      6 Jan 28  2020 home
lrwxrwxrwx.   1 root   root      7 Jan 28  2020 lib -> usr/lib
lrwxrwxrwx.   1 root   root      9 Jan 28  2020 lib64 -> usr/lib64
drwx------.   2 root   root      6 Jul  9 06:48 lost+found
drwxr-xr-x.   2 root   root      6 Jan 28  2020 media
drwxr-xr-x.   2 root   root      6 Jan 28  2020 mnt
drwxr-xr-x.   2 root   root      6 Jan 28  2020 opt
dr-xr-xr-x. 436 nobody nobody    0 Aug 15 13:26 proc
dr-xr-x---.   2 root   root    196 Jul  9 06:48 root
drwxrwxrwt.   3 root   root     80 Aug 15 13:26 run
lrwxrwxrwx.   1 root   root      8 Jan 28  2020 sbin -> usr/sbin
drwxr-xr-x.   2 root   root      6 Jan 28  2020 srv
dr-xr-xr-x.  13 nobody nobody    0 Aug  5 21:38 sys
drwxrwxrwt.   2 root   root     60 Aug 15 13:26 tmp
drwxr-xr-x.  12 root   root    144 Jul  9 06:48 usr
drwxr-xr-x.  18 root   root    235 Jul  9 06:48 var

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>

contrib/fix-mode.py

@@ -0,0 +1,51 @@
+#!/bin/python
+
+import os
+import sys
+import stat
+import errno
+
+XATTR_OVERRIDE_STAT_PRIVILEGED = "security.fuseoverlayfs.override_stat"
+XATTR_OVERRIDE_STAT = "user.fuseoverlayfs.override_stat"
+
+if os.geteuid() == 0:
+    xattr_name = XATTR_OVERRIDE_STAT_PRIVILEGED
+else:
+    xattr_name = XATTR_OVERRIDE_STAT
+
+cwd_fd = os.open(".", os.O_PATH)
+
+def fix_path(path):
+    st = os.lstat(path)
+    content = "%s:%s:%o" % (st.st_uid, st.st_gid,  stat.S_IMODE(st.st_mode))
+
+    try:
+        os.setxattr(path, xattr_name, str.encode(content), flags=os.XATTR_CREATE, follow_symlinks=False)
+    except Exception as e:
+        if e.errno == errno.EEXIST:
+            print("attr %s already present for %s: %s" % (XATTR_OVERRIDE_STAT, path, e.errno))
+            return
+        raise e
+
+    fd = os.open(path, os.O_PATH|os.O_NOFOLLOW|os.O_NONBLOCK)
+    try:
+        proc_path = "/proc/self/fd/%d" % fd
+        os.chmod(proc_path, 0o755)
+    except Exception as e:
+        if e.errno != errno.ENOTSUP:
+            raise e
+    finally:
+        os.close(fd)
+
+
+def fix_mode_directory(d):
+    for root, dirs, files in os.walk(d, topdown=False):
+        for i in dirs+files:
+            path = os.path.join(root, i)
+            fix_path(path)
+    fix_path(d)
+
+for i in sys.argv[1:]:
+    fix_mode_directory(i)
+
+        

direct.c

@@ -34,6 +34,9 @@
 
 #include "utils.h"
 
+#define XATTR_OVERRIDE_STAT "user.fuseoverlayfs.override_stat"
+#define XATTR_PRIVILEGED_OVERRIDE_STAT "security.fuseoverlayfs.override_stat"
+
 static int
 direct_file_exists (struct ovl_layer *l, const char *pathname)
 {
@@ -76,11 +79,69 @@ direct_getxattr (struct ovl_layer *l, const char *path, const char *name, char *
   return lgetxattr (full_path, name, buf, size);
 }
 
+static int
+override_mode (struct ovl_layer *l, int fd, const char *path, struct stat *st)
+{
+  int ret;
+  uid_t uid;
+  gid_t gid;
+  mode_t mode;
+  char buf[64];
+  cleanup_close int cleanup_fd = -1;
+  const char *xattr_name;
+
+  if (l->has_stat_override == 0 && l->has_privileged_stat_override == 0)
+    return 0;
+
+  xattr_name = l->has_privileged_stat_override ? XATTR_PRIVILEGED_OVERRIDE_STAT : XATTR_OVERRIDE_STAT;
+
+  if (fd >= 0)
+    {
+      ret = fgetxattr (fd, xattr_name, buf, sizeof (buf) - 1);
+      if (ret < 0)
+        return ret;
+    }
+  else
+    {
+      char full_path[PATH_MAX];
+
+      full_path[0] = '\0';
+      ret = open_fd_or_get_path (l, path, full_path, &cleanup_fd, O_RDONLY);
+      if (ret < 0)
+        return ret;
+      fd = cleanup_fd;
+
+      if (fd >= 0)
+        ret = fgetxattr (fd, xattr_name, buf, sizeof (buf) - 1);
+      else
+          ret = lgetxattr (full_path, xattr_name, buf, sizeof (buf) - 1);
+
+      if (ret < 0)
+        return ret;
+    }
+
+  buf[ret] = '\0';
+
+  ret = sscanf (buf, "%d:%d:%o", &uid, &gid, &mode);
+  if (ret != 3)
+    {
+      errno = EINVAL;
+      return -1;
+    }
+
+  st->st_uid = uid;
+  st->st_gid = gid;
+  st->st_mode = (st->st_mode & S_IFMT) | mode;
+
+  return 0;
+}
+
+
 static int
 direct_fstat (struct ovl_layer *l, int fd, const char *path, unsigned int mask, struct stat *st)
 {
-#ifdef HAVE_STATX
   int ret;
+#ifdef HAVE_STATX
   struct statx stx;
 
   ret = statx (fd, "", AT_STATX_DONT_SYNC|AT_EMPTY_PATH, mask, &stx);
@@ -88,21 +149,27 @@ direct_fstat (struct ovl_layer *l, int fd, const char *path, unsigned int mask,
   if (ret < 0 && errno == ENOSYS)
     goto fallback;
   if (ret == 0)
-    statx_to_stat (&stx, st);
+    {
+      statx_to_stat (&stx, st);
+      return override_mode (l, fd, path, st);
+    }
 
   return ret;
 #endif
 
  fallback:
-  return fstat (fd, st);
+  ret = fstat (fd, st);
+  if (ret != 0)
+    return ret;
 
+  return override_mode (l, fd, path, st);
 }
 
 static int
 direct_statat (struct ovl_layer *l, const char *path, struct stat *st, int flags, unsigned int mask)
 {
-#ifdef HAVE_STATX
   int ret;
+#ifdef HAVE_STATX
   struct statx stx;
 
   ret = statx (l->fd, path, AT_STATX_DONT_SYNC|flags, mask, &stx);
@@ -110,12 +177,19 @@ direct_statat (struct ovl_layer *l, const char *path, struct stat *st, int flags
   if (ret < 0 && errno == ENOSYS)
     goto fallback;
   if (ret == 0)
-    statx_to_stat (&stx, st);
+    {
+      statx_to_stat (&stx, st);
+      return override_mode (l, -1, path, st);
+    }
 
   return ret;
 #endif
  fallback:
-  return fstatat (l->fd, path, st, flags);
+  ret = fstatat (l->fd, path, st, flags);
+  if (ret != 0)
+    return ret;
+
+  return override_mode (l, -1, path, st);
 }
 
 static struct dirent *
@@ -164,6 +238,7 @@ direct_readlinkat (struct ovl_layer *l, const char *path, char *buf, size_t bufs
 static int
 direct_load_data_source (struct ovl_layer *l, const char *opaque, const char *path, int n_layer)
 {
+  char tmp[64];
   l->path = realpath (path, NULL);
   if (l->path == NULL)
     {
@@ -179,6 +254,11 @@ direct_load_data_source (struct ovl_layer *l, const char *opaque, const char *pa
       return l->fd;
     }
 
+  if (fgetxattr (l->fd, XATTR_PRIVILEGED_OVERRIDE_STAT, tmp, sizeof (tmp)) >= 0)
+    l->has_privileged_stat_override = 1;
+  else if (fgetxattr (l->fd, XATTR_OVERRIDE_STAT, tmp, sizeof (tmp)) >= 0)
+    l->has_stat_override = 1;
+
   return 0;
 }
 

fuse-overlayfs.h

@@ -112,6 +112,8 @@ struct ovl_layer
   bool low;
 
   void *data_source_private_data;
+  unsigned int has_stat_override : 1;
+  unsigned int has_privileged_stat_override : 1;
 };
 
 /* a data_source defines the methods for accessing a lower layer.  */