Linux内核中的overlay文件系统

一、简介

Docker 内核实现容器的功能用了linux 内核中的三个特性 Namespace、Cgroup、UnionFs，今天我们来说一下UnionFs。

linux UnionFs 实现的是overlay 文件系统

OverlayFs 文件系统分为三层，

lower 是只读层

Upper 是可读写

Merged 是 lower 和Upper 合并的目录

挂载方式可以使用mount 命令挂载：

mount -t overlay overlay -o lowerdir=lower1:lower2,upperdir=upper,workdir=work merged

二、源码分析

1.挂载overlay 设备初始化

当我们使用

mount -t overlay overlay -o lowerdir=lower1:lower2,upperdir=upper,workdir=work merged

linux 内核层，overlay 结构体声明类型

static struct file_system_type ovl_fs_type = {.owner		= THIS_MODULE,.name		= "overlay",.fs_flags	= FS_USERNS_MOUNT,.mount		= ovl_mount,.kill_sb	= kill_anon_super,
};

当我们使用overlay设备的时候，会触发结构体上挂载的mount函数指针，这个函数触发linux内核中的ovl_mount

static struct dentry *ovl_mount(struct file_system_type *fs_type, int flags,const char *dev_name, void *raw_data)
{return mount_nodev(fs_type, flags, raw_data, ovl_fill_super);
}

核心是使用ovl_fill_super，填充overlay 文件系统的超级块，申请一个ovl_fs,然后填充到

sb->s_fs_info = ofs;

详细代码:

static int ovl_fill_super(struct super_block *sb, void *data, int silent)
{struct path upperpath = { };struct dentry *root_dentry;struct ovl_entry *oe;struct ovl_fs *ofs;struct ovl_layer *layers;struct cred *cred;char *splitlower = NULL;unsigned int numlower;int err;// 如果当前用户的namespace不是超级块的ns那么返回错误 -EIOerr = -EIO;if (WARN_ON(sb->s_user_ns != current_user_ns()))goto out;// 目录操作结构体赋值sb->s_d_op = &ovl_dentry_operations;err = -ENOMEM;// 申请ovl_fs，并且对ovl_fs进行填充ofs = kzalloc(sizeof(struct ovl_fs), GFP_KERNEL);if (!ofs)goto out;err = -ENOMEM;ofs->creator_cred = cred = prepare_creds();if (!cred)goto out_err;/* Is there a reason anyone would want not to share whiteouts? */ofs->share_whiteout = true;ofs->config.index = ovl_index_def;ofs->config.uuid = true;ofs->config.nfs_export = ovl_nfs_export_def;ofs->config.xino = ovl_xino_def();ofs->config.metacopy = ovl_metacopy_def;// 装载选项err = ovl_parse_opt((char *) data, &ofs->config);if (err)goto out_err;err = -EINVAL;if (!ofs->config.lowerdir) {if (!silent)pr_err("missing 'lowerdir'\n");goto out_err;}err = -ENOMEM;splitlower = kstrdup(ofs->config.lowerdir, GFP_KERNEL);if (!splitlower)goto out_err;err = -EINVAL;numlower = ovl_split_lowerdirs(splitlower);if (numlower > OVL_MAX_STACK) {pr_err("too many lower directories, limit is %d\n",OVL_MAX_STACK);goto out_err;}err = -ENOMEM;layers = kcalloc(numlower + 1, sizeof(struct ovl_layer), GFP_KERNEL);if (!layers)goto out_err;ofs->layers = layers;/* Layer 0 is reserved for upper even if there's no upper */ofs->numlayer = 1;sb->s_stack_depth = 0;sb->s_maxbytes = MAX_LFS_FILESIZE;atomic_long_set(&ofs->last_ino, 1);/* Assume underlying fs uses 32bit inodes unless proven otherwise */if (ofs->config.xino != OVL_XINO_OFF) {ofs->xino_mode = BITS_PER_LONG - 32;if (!ofs->xino_mode) {pr_warn("xino not supported on 32bit kernel, falling back to xino=off.\n");ofs->config.xino = OVL_XINO_OFF;}}/* alloc/destroy_inode needed for setting up traps in inode cache */sb->s_op = &ovl_super_operations;if (ofs->config.upperdir) {struct super_block *upper_sb;err = -EINVAL;if (!ofs->config.workdir) {pr_err("missing 'workdir'\n");goto out_err;}err = ovl_get_upper(sb, ofs, &layers[0], &upperpath);if (err)goto out_err;upper_sb = ovl_upper_mnt(ofs)->mnt_sb;if (!ovl_should_sync(ofs)) {ofs->errseq = errseq_sample(&upper_sb->s_wb_err);if (errseq_check(&upper_sb->s_wb_err, ofs->errseq)) {err = -EIO;pr_err("Cannot mount volatile when upperdir has an unseen error. Sync upperdir fs to clear state.\n");goto out_err;}}err = ovl_get_workdir(sb, ofs, &upperpath);if (err)goto out_err;if (!ofs->workdir)sb->s_flags |= SB_RDONLY;sb->s_stack_depth = upper_sb->s_stack_depth;sb->s_time_gran = upper_sb->s_time_gran;}oe = ovl_get_lowerstack(sb, splitlower, numlower, ofs, layers);err = PTR_ERR(oe);if (IS_ERR(oe))goto out_err;/* If the upper fs is nonexistent, we mark overlayfs r/o too */if (!ovl_upper_mnt(ofs))sb->s_flags |= SB_RDONLY;if (!ofs->config.uuid && ofs->numfs > 1) {pr_warn("The uuid=off requires a single fs for lower and upper, falling back to uuid=on.\n");ofs->config.uuid = true;}if (!ovl_force_readonly(ofs) && ofs->config.index) {err = ovl_get_indexdir(sb, ofs, oe, &upperpath);if (err)goto out_free_oe;/* Force r/o mount with no index dir */if (!ofs->indexdir)sb->s_flags |= SB_RDONLY;}err = ovl_check_overlapping_layers(sb, ofs);if (err)goto out_free_oe;/* Show index=off in /proc/mounts for forced r/o mount */if (!ofs->indexdir) {ofs->config.index = false;if (ovl_upper_mnt(ofs) && ofs->config.nfs_export) {pr_warn("NFS export requires an index dir, falling back to nfs_export=off.\n");ofs->config.nfs_export = false;}}if (ofs->config.metacopy && ofs->config.nfs_export) {pr_warn("NFS export is not supported with metadata only copy up, falling back to nfs_export=off.\n");ofs->config.nfs_export = false;}if (ofs->config.nfs_export)sb->s_export_op = &ovl_export_operations;/* Never override disk quota limits or use reserved space */cap_lower(cred->cap_effective, CAP_SYS_RESOURCE);sb->s_magic = OVERLAYFS_SUPER_MAGIC;sb->s_xattr = ofs->config.userxattr ? ovl_user_xattr_handlers :ovl_trusted_xattr_handlers;sb->s_fs_info = ofs;sb->s_flags |= SB_POSIXACL;sb->s_iflags |= SB_I_SKIP_SYNC;// 把 overlay 文件系统的根目录设置到 upperDir里err = -ENOMEM;// 创建root的inode并且指向新建的inode对象root_inoderoot_dentry = ovl_get_root(sb, upperpath.dentry, oe);if (!root_dentry)goto out_free_oe;mntput(upperpath.mnt);kfree(splitlower);sb->s_root = root_dentry;return 0;out_free_oe:ovl_entry_stack_free(oe);kfree(oe);
out_err:kfree(splitlower);path_put(&upperpath);ovl_free_fs(ofs);
out:return err;
}

操作overlay 文件系统的目录操作结构体实现:

static const struct dentry_operations ovl_dentry_operations = {.d_release = ovl_dentry_release,.d_real = ovl_d_real,.d_revalidate = ovl_dentry_revalidate,.d_weak_revalidate = ovl_dentry_weak_revalidate,
};

数据结构图:

参考网址:

Linux源码剖析——OverlayFS 源码分析_linux overlay-CSDN博客

2、描述符操作结构体 

如果你做过kernel module ，读过linux设计实现.就很容易理解了

描述符操作结构体定义:

const struct file_operations ovl_dir_operations = {.read		= generic_read_dir,.open		= ovl_dir_open,.iterate	= ovl_iterate,.llseek		= ovl_dir_llseek,.fsync		= ovl_dir_fsync,.release	= ovl_dir_release,
};

当我们使用linux 系统调用打开overlay 设备文件的时候会触发操作结构体的函数，

open 函数:

static int ovl_dir_open(struct inode *inode, struct file *file)
{struct path realpath;struct file *realfile;struct ovl_dir_file *od;enum ovl_path_type type;od = kzalloc(sizeof(struct ovl_dir_file), GFP_KERNEL);if (!od)return -ENOMEM;type = ovl_path_real(file->f_path.dentry, &realpath);realfile = ovl_dir_open_realfile(file, &realpath);if (IS_ERR(realfile)) {kfree(od);return PTR_ERR(realfile);}od->realfile = realfile;od->is_real = ovl_dir_is_real(file->f_path.dentry);od->is_upper = OVL_TYPE_UPPER(type);file->private_data = od;return 0;
}

struct ovl_dir_file {bool is_real; // 是否需要合并bool is_upper; // 是否需要从upper读取struct ovl_dir_cache *cache; // 缓存目录struct list_head *cursor; // 遍历游标struct file *realfile; // 真实文件struct file *upperfile; // overlay 里 在upper目录所在位置
};

这里主要做的操作是初始化ovl_dir_file，并且把他挂载到万能指针private_data中。

读的操作是通过getdents，我们看迭代器：

static int ovl_iterate(struct file *file, struct dir_context *ctx)
{struct ovl_dir_file *od = file->private_data;struct dentry *dentry = file->f_path.dentry;struct ovl_cache_entry *p;const struct cred *old_cred;int err;old_cred = ovl_override_creds(dentry->d_sb);if (!ctx->pos)ovl_dir_reset(file);//是否需要读取真实路径if (od->is_real) {// 不需要合并直接读取真实路径/** If parent is merge, then need to adjust d_ino for '..', if* dir is impure then need to adjust d_ino for copied up* entries.*/if (ovl_xino_bits(dentry->d_sb) ||(ovl_same_fs(dentry->d_sb) &&(ovl_is_impure_dir(file) ||OVL_TYPE_MERGE(ovl_path_type(dentry->d_parent))))) {err = ovl_iterate_real(file, ctx);} else {err = iterate_dir(od->realfile, ctx);}goto out;}// 创建目录缓存if (!od->cache) {struct ovl_dir_cache *cache;cache = ovl_cache_get(dentry);err = PTR_ERR(cache);if (IS_ERR(cache))goto out;od->cache = cache;ovl_seek_cursor(od, ctx->pos);}// 直接把合并后的目录缓存，遍历返回用户层while (od->cursor != &od->cache->entries) {p = list_entry(od->cursor, struct ovl_cache_entry, l_node);if (!p->is_whiteout) {if (!p->ino) {err = ovl_cache_update_ino(&file->f_path, p);if (err)goto out;}}/* ovl_cache_update_ino() sets is_whiteout on stale entry */if (!p->is_whiteout) {if (!dir_emit(ctx, p->name, p->len, p->ino, p->type))break;}od->cursor = p->l_node.next;ctx->pos++;}err = 0;
out:revert_creds(old_cred);return err;
}