[pnfs] CVS: nfsv4
Marc Eshel IBM
eshel at citi.umich.edu
Tue Nov 28 13:38:32 EST 2006
CVSROOT: /cvs
Module name: nfsv4
Changes by: eshel at citi. 2006/11/28 13:38:32
Modified files:
cvs/pnfs/fs/nfsd: nfs4proc.c nfs4state.c
Log message:
Add code to manage layouts on the NFS server based on Andy's design.
It was tested for get/return layouts need more work on layout recall.
Index: fs/nfsd/nfs4proc.c
===================================================================
RCS file: /cvs/nfsv4/cvs/pnfs/fs/nfsd/nfs4proc.c,v
retrieving revision 1.20
diff -u -r1.20 nfs4proc.c
--- fs/nfsd/nfs4proc.c 8 Sep 2006 21:39:52 -0000 1.20
+++ fs/nfsd/nfs4proc.c 28 Nov 2006 18:02:53 -0000
@@ -816,30 +816,7 @@
/* Set file handle */
memcpy(&lgp->lg_fh, ¤t_fh->fh_handle, sizeof(struct knfsd_fh));
- status = nfserr_inval;
- if (sb->s_export_op->layout_get) {
- status = sb->s_export_op->layout_get(current_fh->fh_dentry->d_inode,
- (void *)lgp);
-
- dprintk("pNFS %s: status %d type %d maxcount %d \n",
- __FUNCTION__, status, lgp->lg_type, lgp->lg_mxcnt);
-
- switch (status) {
- case 0:
- break;
- case -ENOMEM:
- case -EAGAIN:
- case -EINTR:
- status = nfserr_layouttrylater;
- break;
- case -ENOENT:
- status = nfserr_badlayout;
- break;
- default:
- status = nfserr_layoutunavailable;
- }
- goto out;
- }
+ status = nfs4_pnfs_get_layout(sb, current_fh, lgp);
out:
return status;
}
@@ -939,8 +916,7 @@
goto out;
}
- status = sb->s_export_op->layout_return(current_fh->fh_dentry->d_inode,
- (void *)lrp);
+ status = nfs4_pnfs_return_layout(sb, current_fh, lrp);
out:
dprintk("pNFS %s: status %d type %d\n",
__FUNCTION__, status, lrp->lr_type);
Index: fs/nfsd/nfs4state.c
===================================================================
RCS file: /cvs/nfsv4/cvs/pnfs/fs/nfsd/nfs4state.c,v
retrieving revision 1.13
diff -u -r1.13 nfs4state.c
--- fs/nfsd/nfs4state.c 25 Jul 2006 04:35:03 -0000 1.13
+++ fs/nfsd/nfs4state.c 28 Nov 2006 18:02:54 -0000
@@ -53,6 +53,7 @@
#if defined (CONFIG_PNFS)
#include <linux/nfsd/pnfsd.h>
+#include <linux/nfs4_pnfs.h>
#endif /* CONFIG_PNFS */
#define NFSDDBG_FACILITY NFSDDBG_PROC
@@ -80,7 +81,14 @@
static char user_recovery_dirname[PATH_MAX] = "/var/lib/nfs/v4recovery";
static void nfs4_set_recdir(char *recdir);
#if defined(CONFIG_PNFS)
+/*
+ * Layout state - NFSv4.1 pNFS
+ */
+static struct list_head layout_recall_lru;
+static int num_layouts;
+static kmem_cache_t *pnfs_layout_slab = NULL;
static void release_pnfs_ds_dev_list(struct nfs4_stateid *stp);
+static void nfs4_put_layout(struct nfs4_layout *lp);
#endif /* CONFIG_PNFS */
@@ -275,6 +283,18 @@
nfs4_put_delegation(dp);
}
+/* Called under the state lock. */
+static void
+unhash_layout(struct nfs4_layout *lp)
+{
+ list_del_init(&lp->lo_perfile);
+ list_del_init(&lp->lo_perclnt);
+ spin_lock(&recall_lock);
+ list_del_init(&lp->lo_recall_lru);
+ spin_unlock(&recall_lock);
+ nfs4_put_layout(lp);
+}
+
/*
* SETCLIENTID state
*/
@@ -395,6 +415,7 @@
{
struct nfs4_stateowner *sop;
struct nfs4_delegation *dp;
+ struct nfs4_layout *lp;
struct list_head reaplist;
dprintk("NFSD: expire_client cl_count %d\n",
@@ -420,6 +441,23 @@
list_del(&clp->cl_idhash);
list_del(&clp->cl_strhash);
list_del(&clp->cl_lru);
+#if 1 //??? if pNFS
+ spin_lock(&recall_lock);
+ while (!list_empty(&clp->cl_layouts)) {
+ lp = list_entry(clp->cl_layouts.next, struct nfs4_layout, lo_perclnt);
+ dprintk("NFSD: expire client. lp %p, fp %p\n", lp,
+ lp->lo_file);
+ list_del_init(&lp->lo_perclnt);
+ list_move(&lp->lo_recall_lru, &reaplist);
+ }
+ spin_unlock(&recall_lock);
+ while (!list_empty(&reaplist)) {
+ lp = list_entry(reaplist.next, struct nfs4_layout, lo_recall_lru);
+ list_del_init(&lp->lo_recall_lru);
+ unhash_layout(lp);
+ }
+
+#endif
while (!list_empty(&clp->cl_openowners)) {
sop = list_entry(clp->cl_openowners.next, struct nfs4_stateowner, so_perclient);
release_stateowner(sop);
@@ -440,6 +478,7 @@
INIT_LIST_HEAD(&clp->cl_strhash);
INIT_LIST_HEAD(&clp->cl_openowners);
INIT_LIST_HEAD(&clp->cl_delegations);
+ INIT_LIST_HEAD(&clp->cl_layouts);
INIT_LIST_HEAD(&clp->cl_lru);
out:
return clp;
@@ -1004,6 +1043,7 @@
INIT_LIST_HEAD(&fp->fi_hash);
INIT_LIST_HEAD(&fp->fi_stateids);
INIT_LIST_HEAD(&fp->fi_delegations);
+ INIT_LIST_HEAD(&fp->fi_layouts);
list_add(&fp->fi_hash, &file_hashtbl[hashval]);
fp->fi_inode = igrab(ino);
fp->fi_id = current_fileid++;
@@ -1031,6 +1071,9 @@
nfsd4_free_slab(&file_slab);
nfsd4_free_slab(&stateid_slab);
nfsd4_free_slab(&deleg_slab);
+#if defined(CONFIG_PNFS)
+ nfsd4_free_slab(&pnfs_layout_slab);
+#endif /* CONFIG_PNFS */
}
static int
@@ -1052,6 +1095,13 @@
sizeof(struct nfs4_delegation), 0, 0, NULL, NULL);
if (deleg_slab == NULL)
goto out_nomem;
+#if defined(CONFIG_PNFS)
+ pnfs_layout_slab = kmem_cache_create("pnfs layouts",
+ sizeof(struct nfs4_layout), 0, 0, NULL, NULL);
+ if (pnfs_layout_slab == NULL)
+ goto out_nomem;
+#endif /* CONFIG_PNFS */
+
return 0;
out_nomem:
nfsd4_free_slabs();
@@ -2425,18 +2475,6 @@
*replay_owner = close->cl_stateowner;
}
nfs4_unlock_state();
-#if 0 //??? for testing, move it to the right place to do layout return
- {
- struct super_block *sb;
- struct inode *ino;
- struct layout_return lr;
-
- ino = current_fh->fh_dentry->d_inode;
- sb = ino->i_sb;
- if (sb && sb->s_export_op->layout_return)
- sb->s_export_op->layout_return(ino, &lr);
- }
-#endif
return status;
}
@@ -3347,6 +3385,109 @@
#if defined(CONFIG_PNFS)
+/* Called under the state lock. */
+static void
+pnfs_unhash_layout(struct nfs4_layout *lp)
+{
+ list_del_init(&lp->lo_perfile);
+ list_del_init(&lp->lo_perclnt);
+}
+
+static void
+free_nfs4_layout(struct kref *kref)
+{
+ struct nfs4_layout *lp;
+ struct nfs4_file *fp;
+ struct layout_return lr;
+ struct inode *ino;
+
+ lp = container_of(kref, struct nfs4_layout, lo_ref);
+ pnfs_unhash_layout(lp);
+
+ fp = lp->lo_file;
+ ino = fp->fi_inode;
+
+ dprintk("pNFS %s: lp %p fp %p ino %p\n", __FUNCTION__, lp, fp, ino);
+ lr.reclaim = 0;
+ lr.offset = lp->lo_offset;
+ lr.length = lp->lo_length;
+ lr.iomode = lp->lo_iomode;
+ lr.layout_type= lp->lo_layout_type;
+
+ if (ino->i_sb->s_export_op->layout_return)
+ ino->i_sb->s_export_op->layout_return(ino, &lr);
+
+ kmem_cache_free(pnfs_layout_slab, lp);
+ put_nfs4_file(fp);
+}
+
+static void
+nfs4_put_layout(struct nfs4_layout *lp)
+{
+ dprintk("pNFS %s: lo_ref %d fi_ref %d\n", __FUNCTION__,
+ atomic_read(&lp->lo_ref.refcount),
+ atomic_read(&lp->lo_file->fi_ref.refcount));
+ kref_put(&lp->lo_ref, free_nfs4_layout);
+}
+
+static inline void
+get_nfs4_layout(struct nfs4_layout *lp)
+{
+ kref_get(&lp->lo_ref);
+ dprintk("pNFS %s: lo_ref %d fi_ref %d\n", __FUNCTION__,
+ atomic_read(&lp->lo_ref.refcount),
+ atomic_read(&lp->lo_file->fi_ref.refcount));
+}
+
+
+static struct nfs4_layout *
+pnfs_alloc_init_layout(struct nfs4_file *fp, struct nfs4_client *clp, struct svc_fh *current_fh, struct nfsd4_pnfs_layoutget *lg)
+{
+ struct nfs4_layout *lp;
+ struct nfs4_callback *cb = &clp->cl_callback;
+ struct inode *ino = current_fh->fh_dentry->d_inode;
+
+ dprintk("NFSD alloc_init_layout\n");
+ lp = kmem_cache_alloc(pnfs_layout_slab, GFP_KERNEL);
+ if (lp == NULL)
+ return lp;
+
+ kref_init(&lp->lo_ref);
+ INIT_LIST_HEAD(&lp->lo_perfile);
+ INIT_LIST_HEAD(&lp->lo_perclnt);
+ INIT_LIST_HEAD(&lp->lo_recall_lru);
+
+ lp->lo_file = fp;
+ lp->lo_time = 0;
+ memset(&lp->lo_cb_layout, 0, sizeof(struct nfs4_cb_layout));
+ lp->lo_client = clp;
+ lp->lo_sb = ino->i_sb;
+ lp->lo_ident = cb->cb_ident;
+ lp->lo_fhlen = current_fh->fh_handle.fh_size;
+ memcpy(lp->lo_fhval, ¤t_fh->fh_handle.fh_base,
+ current_fh->fh_handle.fh_size);
+ lp->lo_layout_type = lg->lg_type;
+ lp->lo_iomode = lg->lg_iomode;
+ lp->lo_offset = lg->lg_offset;
+ lp->lo_length = lg->lg_length;
+ num_layouts++;
+ get_nfs4_file(lp->lo_file);
+ dprintk("NFSD alloc_init_layout exit\n");
+ return lp;
+}
+
+static void
+pnfs_hash_layoutget(struct nfs4_layout *lp)
+{
+ list_add(&lp->lo_perfile, &lp->lo_file->fi_layouts);
+ list_add(&lp->lo_perclnt, &lp->lo_client->cl_layouts);
+}
+
+static void
+pnfs_hash_cb_layout(struct nfs4_layout *lp)
+{
+ list_add(&lp->lo_recall_lru, &layout_recall_lru);
+}
/*
* get_state() and cb_get_state() are
*/
@@ -3378,6 +3519,141 @@
return 0;
}
+
+int nfs4_pnfs_get_layout(struct super_block *sb, struct svc_fh *current_fh,
+ struct nfsd4_pnfs_layoutget *lgp)
+{
+ int i, status = -ENOENT;
+ struct inode *ino = current_fh->fh_dentry->d_inode;
+ struct nfs4_file *fp;
+ struct nfs4_client *clp = NULL;
+ struct nfs4_layout *lp = NULL;
+ struct layout_return lr;
+
+ dprintk("NFSD: nfs4_pnfs_get_layout\n");
+
+ fp = find_file(ino);
+ if (!fp) {
+ fp = alloc_init_file(ino);
+ if (fp == NULL)
+ goto out;
+ }
+ if (sb->s_export_op->layout_get) {
+ status = sb->s_export_op->layout_get(current_fh->fh_dentry->d_inode,
+ (void *)lgp);
+
+ dprintk("pNFS %s: status %d type %d maxcount %d \n",
+ __FUNCTION__, status, lgp->lg_type, lgp->lg_mxcnt);
+
+ switch (status) {
+ case 0:
+ break;
+ case -ENOMEM:
+ case -EAGAIN:
+ case -EINTR:
+ status = nfserr_layouttrylater;
+ break;
+ case -ENOENT:
+ status = nfserr_badlayout;
+ break;
+ default:
+ status = nfserr_layoutunavailable;
+ }
+ if (status)
+ goto out;
+
+ while (!list_empty(&fp->fi_layouts)) {
+ lp = list_entry(fp->fi_layouts.next,
+ struct nfs4_layout, lo_perfile);
+ //??? if the right layout
+ break;
+ }
+ if (lp) {
+ get_nfs4_layout(lp);
+ }
+ else {
+#if 0 //???
+ //???
+ clp = find_confirmed_client((clientid_t *)&lgp->lg_clientid);
+#else
+ for (i = 0; i < CLIENT_HASH_SIZE; i++) {
+ while (!list_empty(&conf_id_hashtbl[i])) {
+ clp = list_entry(conf_id_hashtbl[i].next,
+ struct nfs4_client, cl_idhash);
+ break;
+ }
+ }
+#endif
+ dprintk("pNFS %s: clp %p \n", __FUNCTION__, clp);
+
+ if (clp) {
+ lp = pnfs_alloc_init_layout(fp, clp, current_fh, lgp);
+ }
+ if (lp) {
+ dprintk("pNFS %s: lp %p\n", __FUNCTION__, lp);
+ pnfs_hash_layoutget(lp);
+ goto out;
+ }
+ status = nfserr_layouttrylater;
+
+ if (lgp->lg_ops->layout_encode == NULL &&
+ lgp->lg_type == LAYOUT_NFSV4_FILES)
+ filelayout_free_layout(lgp->lg_layout);
+ else
+ lgp->lg_ops->layout_free(lgp->lg_layout);
+
+ lr.reclaim = 0;
+ lr.offset = lgp->lg_offset;
+ lr.length = lgp->lg_length;
+ lr.iomode = lgp->lg_iomode;
+ lr.layout_type= lgp->lg_type;
+ if (sb->s_export_op->layout_return)
+ sb->s_export_op->layout_return(ino, &lr);
+ }
+ }
+out:
+ if (fp)
+ put_nfs4_file(fp);
+
+ dprintk("pNFS %s: exit status %d \n", __FUNCTION__, status);
+ return status;
+}
+
+int nfs4_pnfs_return_layout(struct super_block *sb, struct svc_fh *current_fh,
+ struct nfsd4_pnfs_layoutreturn *lrp)
+{
+ int status = -ENOENT;
+ struct inode *ino = current_fh->fh_dentry->d_inode;
+ struct nfs4_file *fp;
+ struct nfs4_layout *lp = NULL;
+
+ dprintk("NFSD: nfs4_pnfs_return_layout\n");
+
+ fp = find_file(ino);
+ if (!fp)
+ goto out;
+
+ dprintk("pNFS %s: fp %p\n", __FUNCTION__, fp);
+
+ while (!list_empty(&fp->fi_layouts)) {
+ lp = list_entry(fp->fi_layouts.next,
+ struct nfs4_layout, lo_perfile);
+ //??? if the right layout
+ break;
+ }
+ dprintk("pNFS %s: lp %p\n", __FUNCTION__, lp);
+
+ if (lp) {
+ nfs4_put_layout(lp);
+ status = 0;
+ }
+ put_nfs4_file(fp);
+out:
+ dprintk("pNFS %s: exit status %d \n", __FUNCTION__, status);
+ return status;
+}
+
+
/*
* PNFS Metadata server export operations callback for get_state
*
@@ -3386,7 +3662,6 @@
* returns status, or pnfs_get_state* with pnfs_get_state->status set.
*
*/
-
int
nfs4_pnfs_cb_get_state(struct pnfs_get_state *arg)
{
@@ -3430,22 +3705,34 @@
daemonize("nfsv4-layout");
- /* if recall all */
+ if (lr->cbl_recall_type == RECALL_ALL) {
+ for (i = 0; i < CLIENT_HASH_SIZE; i++) {
+ list_for_each_entry(clp, &conf_str_hashtbl[i], cl_strhash) {
+ if (clp) {
+ if (!clp->cl_callback.cb_client)
+ continue;
+//??? check if this client has any layouts
+ lr->cbl_client = clp;
+ nfsd4_cb_layout(lr);
+ }
+ }
+ }
+ }
if (lr->cbl_recall_type == RECALL_FSID) {
-
for (i = 0; i < CLIENT_HASH_SIZE; i++) {
list_for_each_entry(clp, &conf_str_hashtbl[i], cl_strhash) {
if (clp) {
if (!clp->cl_callback.cb_client)
continue;
+//??? check if this client has any layouts for this FSID
lr->cbl_client = clp;
nfsd4_cb_layout(lr);
}
}
}
}
- else {
- /* recall one layout ??? */
+ if (lr->cbl_recall_type == RECALL_FILE) {
+//??? find layout for this file (given by inode)
if (lr->cbl_client)
nfsd4_cb_layout(lr);
}
More information about the pNFS
mailing list