移植facebook flashcache模块支持 linux kernel 4.13以上
环境 准备在ubuntu 18.04 lts下搭建kvm+libvirt+glusterfs的虚拟化环境。 4台服务器是2块SSD，4块SATA（5400RPM）。 尝试了gluster tier，效果不好。 对于虚拟化环境下大文件的存储，tier的promote和demote的策略很难设置， 在大文件发生大量数据交换时会出现tier层"no space left"的错误，文件无法写入。 在没有更好的解决方案之前，facebook的flashcache（https://github.com/facebookarchive/flashcache )对于提速SATA盘的作用还是非常显著的。 但ubuntu18.04的apt源还没有。 https://launchpad.net/flashcache/+packages 错误 于是下载flashcache源码编译dkms模块的时候，报错： DKMS make.log for flashcache-3.1.3+git20150701 for kernel 4.15.0-36-generic (x86_64) Fri Oct 12 15:06:00 UTC 2018 make -C /lib/modules/4.15.0-36-generic/build M=/var/lib/dkms/flashcache/3.1.3+git20150701/build modules V=0 make[1]: Entering directory '/usr/src/linux-headers-4.15.0-36-generic' Makefile:975: "Cannot use CONFIG_STACK_VALIDATION=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel" CC [M] /var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_conf.o CC [M] /var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_main.o CC [M] /var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_subr.o /var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_subr.c: In function ‘flashcache_bio_endio’: /var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_subr.c:742:7: error: ‘struct bio’ has no member named ‘bi_error’; did you mean ‘bi_iter’? bio->bi_error = error; ^~~~~~~~ bi_iter scripts/Makefile.build:332: recipe for target '/var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_subr.o' failed make[2]: *** [/var/lib/dkms/flashcache/3.1.3+git20150701/build/flashcache_subr.o] Error 1 Makefile:1551: recipe for target '_module_/var/lib/dkms/flashcache/3.1.3+git20150701/build' failed make[1]: *** [_module_/var/lib/dkms/flashcache/3.1.3+git20150701/build] Error 2 make[1]: Leaving directory '/usr/src/linux-headers-4.15.0-36-generic' Makefile:35: recipe for target 'modules' failed make: *** [modules] Error 2 OS内核的版本是4.15.0-36。 原因 根据错误提示：bio->bierror ‘struct bio’ has no member named ‘bierror’; 找了4.15的内核源码看看bio的struct的定义, bio是在include/linux/blktypes.h中定义。 https://elixir.bootlin.com/linux/v4.15/source/include/linux/blktypes.h struct bio { struct bio *bi_next; /* request queue link */ struct gendisk *bi_disk; unsigned int bi_opf; /* bottom bits req flags, * top bits REQ_OP. Use * accessors. */ unsigned short bi_flags; /* status, etc and bvec pool number */ unsigned short bi_ioprio; unsigned short bi_write_hint; blk_status_t bi_status; u8 bi_partno; /* Number of segments in this BIO after * physical address coalescing is performed. */ unsigned int bi_phys_segments; /* * To keep track of the max segment size, we account for the * sizes of the first and last mergeable segments in this bio. */ unsigned int bi_seg_front_size; unsigned int bi_seg_back_size; struct bvec_iter bi_iter; atomic_t __bi_remaining; bio_end_io_t *bi_end_io; void *bi_private; #ifdef CONFIG_BLK_CGROUP /* * Optional ioc and css associated with this bio. Put on bio * release. Read comment on top of bio_associate_current(). */ struct io_context *bi_ioc; struct cgroup_subsys_state *bi_css; #ifdef CONFIG_BLK_DEV_THROTTLING_LOW void *bi_cg_private; struct blk_issue_stat bi_issue_stat; #endif #endif union { #if defined(CONFIG_BLK_DEV_INTEGRITY) struct bio_integrity_payload *bi_integrity; /* data integrity */ #endif }; unsigned short bi_vcnt; /* how many bio_vec's */ /* * Everything starting with bi_max_vecs will be preserved by bio_reset() */ unsigned short bi_max_vecs; /* max bvl_vecs we can hold */ atomic_t __bi_cnt; /* pin count */ struct bio_vec *bi_io_vec; /* the actual vec list */ struct bio_set *bi_pool; /* * We can inline a number of vecs at the end of the bio, to avoid * double allocations for a small number of bio_vecs. This member * MUST obviously be kept at the very end of the bio. */ struct bio_vec bi_inline_vecs[0]; }; 里面根本没有bierror。 再翻翻以前版本的代码，在4.12的bio定义里发现了bierror： https://elixir.bootlin.com/linux/v4.12.14/source/include/linux/blk_types.h 应该是从4.13版本内核之后，bio的结构定义发生了变化。 struct bio { struct bio *bi_next; /* request queue link */ struct block_device *bi_bdev; int bi_error; unsigned int bi_opf; /* bottom bits req flags, * top bits REQ_OP. Use * accessors. */ unsigned short bi_flags; /* status, etc and bvec pool number */ unsigned short bi_ioprio; 解决 修改flashcache_subr.c，将 bi_error改成bi_status。 重新编译，ok。 更新： flashcache_subr.c按如下修改，可以兼容以前版本的内核。 void flashcache_bio_endio(struct bio *bio, int error, struct cache_c *dmc, struct timeval *start_time) { if (unlikely(dmc->sysctl_io_latency_hist && start_time != NULL && start_time->tv_sec != 0)) flashcache_record_latency(dmc, start_time); #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,24) bio_endio(bio, bio->bi_size, error); #elif LINUX_VERSION_CODE < KERNEL_VERSION(4,3,0) bio_endio(bio, error); #elif LINUX_VERSION_CODE < KERNEL_VERSION(4,13,0) bio->bi_error = error; bio_endio(bio); #else bio->bi_status = error; bio_endio(bio); #endif }
