162306a36Sopenharmony_ci/* SPDX-License-Identifier: GPL-2.0 */ 262306a36Sopenharmony_ci#ifndef _FS_CEPH_LIBCEPH_H 362306a36Sopenharmony_ci#define _FS_CEPH_LIBCEPH_H 462306a36Sopenharmony_ci 562306a36Sopenharmony_ci#include <linux/ceph/ceph_debug.h> 662306a36Sopenharmony_ci 762306a36Sopenharmony_ci#include <asm/unaligned.h> 862306a36Sopenharmony_ci#include <linux/backing-dev.h> 962306a36Sopenharmony_ci#include <linux/completion.h> 1062306a36Sopenharmony_ci#include <linux/exportfs.h> 1162306a36Sopenharmony_ci#include <linux/bug.h> 1262306a36Sopenharmony_ci#include <linux/fs.h> 1362306a36Sopenharmony_ci#include <linux/mempool.h> 1462306a36Sopenharmony_ci#include <linux/pagemap.h> 1562306a36Sopenharmony_ci#include <linux/wait.h> 1662306a36Sopenharmony_ci#include <linux/writeback.h> 1762306a36Sopenharmony_ci#include <linux/slab.h> 1862306a36Sopenharmony_ci#include <linux/refcount.h> 1962306a36Sopenharmony_ci 2062306a36Sopenharmony_ci#include <linux/ceph/types.h> 2162306a36Sopenharmony_ci#include <linux/ceph/messenger.h> 2262306a36Sopenharmony_ci#include <linux/ceph/msgpool.h> 2362306a36Sopenharmony_ci#include <linux/ceph/mon_client.h> 2462306a36Sopenharmony_ci#include <linux/ceph/osd_client.h> 2562306a36Sopenharmony_ci#include <linux/ceph/ceph_fs.h> 2662306a36Sopenharmony_ci#include <linux/ceph/string_table.h> 2762306a36Sopenharmony_ci 2862306a36Sopenharmony_ci/* 2962306a36Sopenharmony_ci * mount options 3062306a36Sopenharmony_ci */ 3162306a36Sopenharmony_ci#define CEPH_OPT_FSID (1<<0) 3262306a36Sopenharmony_ci#define CEPH_OPT_NOSHARE (1<<1) /* don't share client with other sbs */ 3362306a36Sopenharmony_ci#define CEPH_OPT_MYIP (1<<2) /* specified my ip */ 3462306a36Sopenharmony_ci#define CEPH_OPT_NOCRC (1<<3) /* no data crc on writes (msgr1) */ 3562306a36Sopenharmony_ci#define CEPH_OPT_TCP_NODELAY (1<<4) /* TCP_NODELAY on TCP sockets */ 3662306a36Sopenharmony_ci#define CEPH_OPT_NOMSGSIGN (1<<5) /* don't sign msgs (msgr1) */ 3762306a36Sopenharmony_ci#define CEPH_OPT_ABORT_ON_FULL (1<<6) /* abort w/ ENOSPC when full */ 3862306a36Sopenharmony_ci#define CEPH_OPT_RXBOUNCE (1<<7) /* double-buffer read data */ 3962306a36Sopenharmony_ci 4062306a36Sopenharmony_ci#define CEPH_OPT_DEFAULT (CEPH_OPT_TCP_NODELAY) 4162306a36Sopenharmony_ci 4262306a36Sopenharmony_ci#define ceph_set_opt(client, opt) \ 4362306a36Sopenharmony_ci (client)->options->flags |= CEPH_OPT_##opt; 4462306a36Sopenharmony_ci#define ceph_test_opt(client, opt) \ 4562306a36Sopenharmony_ci (!!((client)->options->flags & CEPH_OPT_##opt)) 4662306a36Sopenharmony_ci 4762306a36Sopenharmony_cistruct ceph_options { 4862306a36Sopenharmony_ci int flags; 4962306a36Sopenharmony_ci struct ceph_fsid fsid; 5062306a36Sopenharmony_ci struct ceph_entity_addr my_addr; 5162306a36Sopenharmony_ci unsigned long mount_timeout; /* jiffies */ 5262306a36Sopenharmony_ci unsigned long osd_idle_ttl; /* jiffies */ 5362306a36Sopenharmony_ci unsigned long osd_keepalive_timeout; /* jiffies */ 5462306a36Sopenharmony_ci unsigned long osd_request_timeout; /* jiffies */ 5562306a36Sopenharmony_ci u32 read_from_replica; /* CEPH_OSD_FLAG_BALANCE/LOCALIZE_READS */ 5662306a36Sopenharmony_ci int con_modes[2]; /* CEPH_CON_MODE_* */ 5762306a36Sopenharmony_ci 5862306a36Sopenharmony_ci /* 5962306a36Sopenharmony_ci * any type that can't be simply compared or doesn't need 6062306a36Sopenharmony_ci * to be compared should go beyond this point, 6162306a36Sopenharmony_ci * ceph_compare_options() should be updated accordingly 6262306a36Sopenharmony_ci */ 6362306a36Sopenharmony_ci 6462306a36Sopenharmony_ci struct ceph_entity_addr *mon_addr; /* should be the first 6562306a36Sopenharmony_ci pointer type of args */ 6662306a36Sopenharmony_ci int num_mon; 6762306a36Sopenharmony_ci char *name; 6862306a36Sopenharmony_ci struct ceph_crypto_key *key; 6962306a36Sopenharmony_ci struct rb_root crush_locs; 7062306a36Sopenharmony_ci}; 7162306a36Sopenharmony_ci 7262306a36Sopenharmony_ci/* 7362306a36Sopenharmony_ci * defaults 7462306a36Sopenharmony_ci */ 7562306a36Sopenharmony_ci#define CEPH_MOUNT_TIMEOUT_DEFAULT msecs_to_jiffies(60 * 1000) 7662306a36Sopenharmony_ci#define CEPH_OSD_KEEPALIVE_DEFAULT msecs_to_jiffies(5 * 1000) 7762306a36Sopenharmony_ci#define CEPH_OSD_IDLE_TTL_DEFAULT msecs_to_jiffies(60 * 1000) 7862306a36Sopenharmony_ci#define CEPH_OSD_REQUEST_TIMEOUT_DEFAULT 0 /* no timeout */ 7962306a36Sopenharmony_ci#define CEPH_READ_FROM_REPLICA_DEFAULT 0 /* read from primary */ 8062306a36Sopenharmony_ci 8162306a36Sopenharmony_ci#define CEPH_MONC_HUNT_INTERVAL msecs_to_jiffies(3 * 1000) 8262306a36Sopenharmony_ci#define CEPH_MONC_PING_INTERVAL msecs_to_jiffies(10 * 1000) 8362306a36Sopenharmony_ci#define CEPH_MONC_PING_TIMEOUT msecs_to_jiffies(30 * 1000) 8462306a36Sopenharmony_ci#define CEPH_MONC_HUNT_BACKOFF 2 8562306a36Sopenharmony_ci#define CEPH_MONC_HUNT_MAX_MULT 10 8662306a36Sopenharmony_ci 8762306a36Sopenharmony_ci#define CEPH_MSG_MAX_CONTROL_LEN (16*1024*1024) 8862306a36Sopenharmony_ci#define CEPH_MSG_MAX_FRONT_LEN (16*1024*1024) 8962306a36Sopenharmony_ci#define CEPH_MSG_MAX_MIDDLE_LEN (16*1024*1024) 9062306a36Sopenharmony_ci 9162306a36Sopenharmony_ci/* 9262306a36Sopenharmony_ci * The largest possible rbd data object is 32M. 9362306a36Sopenharmony_ci * The largest possible rbd object map object is 64M. 9462306a36Sopenharmony_ci * 9562306a36Sopenharmony_ci * There is no limit on the size of cephfs objects, but it has to obey 9662306a36Sopenharmony_ci * rsize and wsize mount options anyway. 9762306a36Sopenharmony_ci */ 9862306a36Sopenharmony_ci#define CEPH_MSG_MAX_DATA_LEN (64*1024*1024) 9962306a36Sopenharmony_ci 10062306a36Sopenharmony_ci#define CEPH_AUTH_NAME_DEFAULT "guest" 10162306a36Sopenharmony_ci 10262306a36Sopenharmony_cistatic inline unsigned long ceph_timeout_jiffies(unsigned long timeout) 10362306a36Sopenharmony_ci{ 10462306a36Sopenharmony_ci return timeout ?: MAX_SCHEDULE_TIMEOUT; 10562306a36Sopenharmony_ci} 10662306a36Sopenharmony_ci 10762306a36Sopenharmony_cistruct ceph_mds_client; 10862306a36Sopenharmony_ci 10962306a36Sopenharmony_ci/* 11062306a36Sopenharmony_ci * per client state 11162306a36Sopenharmony_ci * 11262306a36Sopenharmony_ci * possibly shared by multiple mount points, if they are 11362306a36Sopenharmony_ci * mounting the same ceph filesystem/cluster. 11462306a36Sopenharmony_ci */ 11562306a36Sopenharmony_cistruct ceph_client { 11662306a36Sopenharmony_ci struct ceph_fsid fsid; 11762306a36Sopenharmony_ci bool have_fsid; 11862306a36Sopenharmony_ci 11962306a36Sopenharmony_ci void *private; 12062306a36Sopenharmony_ci 12162306a36Sopenharmony_ci struct ceph_options *options; 12262306a36Sopenharmony_ci 12362306a36Sopenharmony_ci struct mutex mount_mutex; /* serialize mount attempts */ 12462306a36Sopenharmony_ci wait_queue_head_t auth_wq; 12562306a36Sopenharmony_ci int auth_err; 12662306a36Sopenharmony_ci 12762306a36Sopenharmony_ci int (*extra_mon_dispatch)(struct ceph_client *, struct ceph_msg *); 12862306a36Sopenharmony_ci 12962306a36Sopenharmony_ci u64 supported_features; 13062306a36Sopenharmony_ci u64 required_features; 13162306a36Sopenharmony_ci 13262306a36Sopenharmony_ci struct ceph_messenger msgr; /* messenger instance */ 13362306a36Sopenharmony_ci struct ceph_mon_client monc; 13462306a36Sopenharmony_ci struct ceph_osd_client osdc; 13562306a36Sopenharmony_ci 13662306a36Sopenharmony_ci#ifdef CONFIG_DEBUG_FS 13762306a36Sopenharmony_ci struct dentry *debugfs_dir; 13862306a36Sopenharmony_ci struct dentry *debugfs_monmap; 13962306a36Sopenharmony_ci struct dentry *debugfs_osdmap; 14062306a36Sopenharmony_ci struct dentry *debugfs_options; 14162306a36Sopenharmony_ci#endif 14262306a36Sopenharmony_ci}; 14362306a36Sopenharmony_ci 14462306a36Sopenharmony_ci#define from_msgr(ms) container_of(ms, struct ceph_client, msgr) 14562306a36Sopenharmony_ci 14662306a36Sopenharmony_cistatic inline bool ceph_msgr2(struct ceph_client *client) 14762306a36Sopenharmony_ci{ 14862306a36Sopenharmony_ci return client->options->con_modes[0] != CEPH_CON_MODE_UNKNOWN; 14962306a36Sopenharmony_ci} 15062306a36Sopenharmony_ci 15162306a36Sopenharmony_ci/* 15262306a36Sopenharmony_ci * snapshots 15362306a36Sopenharmony_ci */ 15462306a36Sopenharmony_ci 15562306a36Sopenharmony_ci/* 15662306a36Sopenharmony_ci * A "snap context" is the set of existing snapshots when we 15762306a36Sopenharmony_ci * write data. It is used by the OSD to guide its COW behavior. 15862306a36Sopenharmony_ci * 15962306a36Sopenharmony_ci * The ceph_snap_context is refcounted, and attached to each dirty 16062306a36Sopenharmony_ci * page, indicating which context the dirty data belonged when it was 16162306a36Sopenharmony_ci * dirtied. 16262306a36Sopenharmony_ci */ 16362306a36Sopenharmony_cistruct ceph_snap_context { 16462306a36Sopenharmony_ci refcount_t nref; 16562306a36Sopenharmony_ci u64 seq; 16662306a36Sopenharmony_ci u32 num_snaps; 16762306a36Sopenharmony_ci u64 snaps[]; 16862306a36Sopenharmony_ci}; 16962306a36Sopenharmony_ci 17062306a36Sopenharmony_ciextern struct ceph_snap_context *ceph_create_snap_context(u32 snap_count, 17162306a36Sopenharmony_ci gfp_t gfp_flags); 17262306a36Sopenharmony_ciextern struct ceph_snap_context *ceph_get_snap_context( 17362306a36Sopenharmony_ci struct ceph_snap_context *sc); 17462306a36Sopenharmony_ciextern void ceph_put_snap_context(struct ceph_snap_context *sc); 17562306a36Sopenharmony_ci 17662306a36Sopenharmony_ci/* 17762306a36Sopenharmony_ci * calculate the number of pages a given length and offset map onto, 17862306a36Sopenharmony_ci * if we align the data. 17962306a36Sopenharmony_ci */ 18062306a36Sopenharmony_cistatic inline int calc_pages_for(u64 off, u64 len) 18162306a36Sopenharmony_ci{ 18262306a36Sopenharmony_ci return ((off+len+PAGE_SIZE-1) >> PAGE_SHIFT) - 18362306a36Sopenharmony_ci (off >> PAGE_SHIFT); 18462306a36Sopenharmony_ci} 18562306a36Sopenharmony_ci 18662306a36Sopenharmony_ci#define RB_BYVAL(a) (a) 18762306a36Sopenharmony_ci#define RB_BYPTR(a) (&(a)) 18862306a36Sopenharmony_ci#define RB_CMP3WAY(a, b) ((a) < (b) ? -1 : (a) > (b)) 18962306a36Sopenharmony_ci 19062306a36Sopenharmony_ci#define DEFINE_RB_INSDEL_FUNCS2(name, type, keyfld, cmpexp, keyexp, nodefld) \ 19162306a36Sopenharmony_cistatic bool __insert_##name(struct rb_root *root, type *t) \ 19262306a36Sopenharmony_ci{ \ 19362306a36Sopenharmony_ci struct rb_node **n = &root->rb_node; \ 19462306a36Sopenharmony_ci struct rb_node *parent = NULL; \ 19562306a36Sopenharmony_ci \ 19662306a36Sopenharmony_ci BUG_ON(!RB_EMPTY_NODE(&t->nodefld)); \ 19762306a36Sopenharmony_ci \ 19862306a36Sopenharmony_ci while (*n) { \ 19962306a36Sopenharmony_ci type *cur = rb_entry(*n, type, nodefld); \ 20062306a36Sopenharmony_ci int cmp; \ 20162306a36Sopenharmony_ci \ 20262306a36Sopenharmony_ci parent = *n; \ 20362306a36Sopenharmony_ci cmp = cmpexp(keyexp(t->keyfld), keyexp(cur->keyfld)); \ 20462306a36Sopenharmony_ci if (cmp < 0) \ 20562306a36Sopenharmony_ci n = &(*n)->rb_left; \ 20662306a36Sopenharmony_ci else if (cmp > 0) \ 20762306a36Sopenharmony_ci n = &(*n)->rb_right; \ 20862306a36Sopenharmony_ci else \ 20962306a36Sopenharmony_ci return false; \ 21062306a36Sopenharmony_ci } \ 21162306a36Sopenharmony_ci \ 21262306a36Sopenharmony_ci rb_link_node(&t->nodefld, parent, n); \ 21362306a36Sopenharmony_ci rb_insert_color(&t->nodefld, root); \ 21462306a36Sopenharmony_ci return true; \ 21562306a36Sopenharmony_ci} \ 21662306a36Sopenharmony_cistatic void __maybe_unused insert_##name(struct rb_root *root, type *t) \ 21762306a36Sopenharmony_ci{ \ 21862306a36Sopenharmony_ci if (!__insert_##name(root, t)) \ 21962306a36Sopenharmony_ci BUG(); \ 22062306a36Sopenharmony_ci} \ 22162306a36Sopenharmony_cistatic void erase_##name(struct rb_root *root, type *t) \ 22262306a36Sopenharmony_ci{ \ 22362306a36Sopenharmony_ci BUG_ON(RB_EMPTY_NODE(&t->nodefld)); \ 22462306a36Sopenharmony_ci rb_erase(&t->nodefld, root); \ 22562306a36Sopenharmony_ci RB_CLEAR_NODE(&t->nodefld); \ 22662306a36Sopenharmony_ci} 22762306a36Sopenharmony_ci 22862306a36Sopenharmony_ci/* 22962306a36Sopenharmony_ci * @lookup_param_type is a parameter and not constructed from (@type, 23062306a36Sopenharmony_ci * @keyfld) with typeof() because adding const is too unwieldy. 23162306a36Sopenharmony_ci */ 23262306a36Sopenharmony_ci#define DEFINE_RB_LOOKUP_FUNC2(name, type, keyfld, cmpexp, keyexp, \ 23362306a36Sopenharmony_ci lookup_param_type, nodefld) \ 23462306a36Sopenharmony_cistatic type *lookup_##name(struct rb_root *root, lookup_param_type key) \ 23562306a36Sopenharmony_ci{ \ 23662306a36Sopenharmony_ci struct rb_node *n = root->rb_node; \ 23762306a36Sopenharmony_ci \ 23862306a36Sopenharmony_ci while (n) { \ 23962306a36Sopenharmony_ci type *cur = rb_entry(n, type, nodefld); \ 24062306a36Sopenharmony_ci int cmp; \ 24162306a36Sopenharmony_ci \ 24262306a36Sopenharmony_ci cmp = cmpexp(key, keyexp(cur->keyfld)); \ 24362306a36Sopenharmony_ci if (cmp < 0) \ 24462306a36Sopenharmony_ci n = n->rb_left; \ 24562306a36Sopenharmony_ci else if (cmp > 0) \ 24662306a36Sopenharmony_ci n = n->rb_right; \ 24762306a36Sopenharmony_ci else \ 24862306a36Sopenharmony_ci return cur; \ 24962306a36Sopenharmony_ci } \ 25062306a36Sopenharmony_ci \ 25162306a36Sopenharmony_ci return NULL; \ 25262306a36Sopenharmony_ci} 25362306a36Sopenharmony_ci 25462306a36Sopenharmony_ci#define DEFINE_RB_FUNCS2(name, type, keyfld, cmpexp, keyexp, \ 25562306a36Sopenharmony_ci lookup_param_type, nodefld) \ 25662306a36Sopenharmony_ciDEFINE_RB_INSDEL_FUNCS2(name, type, keyfld, cmpexp, keyexp, nodefld) \ 25762306a36Sopenharmony_ciDEFINE_RB_LOOKUP_FUNC2(name, type, keyfld, cmpexp, keyexp, \ 25862306a36Sopenharmony_ci lookup_param_type, nodefld) 25962306a36Sopenharmony_ci 26062306a36Sopenharmony_ci/* 26162306a36Sopenharmony_ci * Shorthands for integer keys. 26262306a36Sopenharmony_ci */ 26362306a36Sopenharmony_ci#define DEFINE_RB_INSDEL_FUNCS(name, type, keyfld, nodefld) \ 26462306a36Sopenharmony_ciDEFINE_RB_INSDEL_FUNCS2(name, type, keyfld, RB_CMP3WAY, RB_BYVAL, nodefld) 26562306a36Sopenharmony_ci 26662306a36Sopenharmony_ci#define DEFINE_RB_LOOKUP_FUNC(name, type, keyfld, nodefld) \ 26762306a36Sopenharmony_ciextern type __lookup_##name##_key; \ 26862306a36Sopenharmony_ciDEFINE_RB_LOOKUP_FUNC2(name, type, keyfld, RB_CMP3WAY, RB_BYVAL, \ 26962306a36Sopenharmony_ci typeof(__lookup_##name##_key.keyfld), nodefld) 27062306a36Sopenharmony_ci 27162306a36Sopenharmony_ci#define DEFINE_RB_FUNCS(name, type, keyfld, nodefld) \ 27262306a36Sopenharmony_ciDEFINE_RB_INSDEL_FUNCS(name, type, keyfld, nodefld) \ 27362306a36Sopenharmony_ciDEFINE_RB_LOOKUP_FUNC(name, type, keyfld, nodefld) 27462306a36Sopenharmony_ci 27562306a36Sopenharmony_ciextern struct kmem_cache *ceph_inode_cachep; 27662306a36Sopenharmony_ciextern struct kmem_cache *ceph_cap_cachep; 27762306a36Sopenharmony_ciextern struct kmem_cache *ceph_cap_snap_cachep; 27862306a36Sopenharmony_ciextern struct kmem_cache *ceph_cap_flush_cachep; 27962306a36Sopenharmony_ciextern struct kmem_cache *ceph_dentry_cachep; 28062306a36Sopenharmony_ciextern struct kmem_cache *ceph_file_cachep; 28162306a36Sopenharmony_ciextern struct kmem_cache *ceph_dir_file_cachep; 28262306a36Sopenharmony_ciextern struct kmem_cache *ceph_mds_request_cachep; 28362306a36Sopenharmony_ciextern mempool_t *ceph_wb_pagevec_pool; 28462306a36Sopenharmony_ci 28562306a36Sopenharmony_ci/* ceph_common.c */ 28662306a36Sopenharmony_ciextern bool libceph_compatible(void *data); 28762306a36Sopenharmony_ci 28862306a36Sopenharmony_ciextern const char *ceph_msg_type_name(int type); 28962306a36Sopenharmony_ciextern int ceph_check_fsid(struct ceph_client *client, struct ceph_fsid *fsid); 29062306a36Sopenharmony_ciextern int ceph_parse_fsid(const char *str, struct ceph_fsid *fsid); 29162306a36Sopenharmony_ci 29262306a36Sopenharmony_cistruct fs_parameter; 29362306a36Sopenharmony_cistruct fc_log; 29462306a36Sopenharmony_cistruct ceph_options *ceph_alloc_options(void); 29562306a36Sopenharmony_ciint ceph_parse_mon_ips(const char *buf, size_t len, struct ceph_options *opt, 29662306a36Sopenharmony_ci struct fc_log *l, char delim); 29762306a36Sopenharmony_ciint ceph_parse_param(struct fs_parameter *param, struct ceph_options *opt, 29862306a36Sopenharmony_ci struct fc_log *l); 29962306a36Sopenharmony_ciint ceph_print_client_options(struct seq_file *m, struct ceph_client *client, 30062306a36Sopenharmony_ci bool show_all); 30162306a36Sopenharmony_ciextern void ceph_destroy_options(struct ceph_options *opt); 30262306a36Sopenharmony_ciextern int ceph_compare_options(struct ceph_options *new_opt, 30362306a36Sopenharmony_ci struct ceph_client *client); 30462306a36Sopenharmony_cistruct ceph_client *ceph_create_client(struct ceph_options *opt, void *private); 30562306a36Sopenharmony_cistruct ceph_entity_addr *ceph_client_addr(struct ceph_client *client); 30662306a36Sopenharmony_ciu64 ceph_client_gid(struct ceph_client *client); 30762306a36Sopenharmony_ciextern void ceph_destroy_client(struct ceph_client *client); 30862306a36Sopenharmony_ciextern void ceph_reset_client_addr(struct ceph_client *client); 30962306a36Sopenharmony_ciextern int __ceph_open_session(struct ceph_client *client, 31062306a36Sopenharmony_ci unsigned long started); 31162306a36Sopenharmony_ciextern int ceph_open_session(struct ceph_client *client); 31262306a36Sopenharmony_ciint ceph_wait_for_latest_osdmap(struct ceph_client *client, 31362306a36Sopenharmony_ci unsigned long timeout); 31462306a36Sopenharmony_ci 31562306a36Sopenharmony_ci/* pagevec.c */ 31662306a36Sopenharmony_ciextern void ceph_release_page_vector(struct page **pages, int num_pages); 31762306a36Sopenharmony_ciextern void ceph_put_page_vector(struct page **pages, int num_pages, 31862306a36Sopenharmony_ci bool dirty); 31962306a36Sopenharmony_ciextern struct page **ceph_alloc_page_vector(int num_pages, gfp_t flags); 32062306a36Sopenharmony_ciextern int ceph_copy_user_to_page_vector(struct page **pages, 32162306a36Sopenharmony_ci const void __user *data, 32262306a36Sopenharmony_ci loff_t off, size_t len); 32362306a36Sopenharmony_ciextern void ceph_copy_to_page_vector(struct page **pages, 32462306a36Sopenharmony_ci const void *data, 32562306a36Sopenharmony_ci loff_t off, size_t len); 32662306a36Sopenharmony_ciextern void ceph_copy_from_page_vector(struct page **pages, 32762306a36Sopenharmony_ci void *data, 32862306a36Sopenharmony_ci loff_t off, size_t len); 32962306a36Sopenharmony_ciextern void ceph_zero_page_vector_range(int off, int len, struct page **pages); 33062306a36Sopenharmony_ci 33162306a36Sopenharmony_ci 33262306a36Sopenharmony_ci#endif /* _FS_CEPH_SUPER_H */ 333