[pnfs] [PATCH 20/38] [v1] pnfs: client layout cache: allow waiting for alloc_layout

[Benny Halevy]

we want one thread only calling alloc_layout and waiting for it.
The NFS_INO_LAYOUT_ALLOC bit in pnfs_layout_state is used
for mutual exclusion and nfs_inode.lo_waitq is used for waiting
on alloc_layout.

Signed-off-by: Benny Halevy <bhalevy at panasas.com>
---
 fs/nfs/inode.c         |    3 +++
 fs/nfs/pnfs.c          |   39 +++++++++++++++++++++++++++++++++++++--
 include/linux/nfs_fs.h |    2 ++
 3 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
index 4c47ecb..c891492 100644
--- a/fs/nfs/inode.c
+++ b/fs/nfs/inode.c
@@ -1182,6 +1182,9 @@ static inline void nfs4_init_once(struct nfs_inode *nfsi)
 	nfsi->delegation_state = 0;
 	init_rwsem(&nfsi->rwsem);
 #endif
+#ifdef CONFIG_PNFS
+	init_waitqueue_head(&nfsi->lo_waitq);
+#endif /* CONFIG_PNFS */
 }
 
 static void init_once(struct kmem_cache * cachep, void *foo)
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index b91f5d7..6757608 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -446,6 +446,14 @@ alloc_init_layout(struct inode *ino, struct layoutdriver_io_operations *io_ops)
 	return lo;
 }
 
+static int pnfs_wait_schedule(void *word)
+{
+	if (signal_pending(current))
+		return -ERESTARTSYS;
+	schedule();
+	return 0;
+}
+
 /*
  * get, possibly allocate current_layout
  */
@@ -455,13 +463,40 @@ get_alloc_layout(struct inode *ino,
 {
 	struct nfs_inode *nfsi = NFS_I(ino);
 	struct pnfs_layout_type *lo;
+	int res;
 
 	dprintk("%s Begin\n", __FUNCTION__);
 
-	if ((lo = nfsi->current_layout) == NULL) {
-		lo = nfsi->current_layout = alloc_init_layout(ino, io_ops);
+	while ((lo = nfsi->current_layout) == NULL) {
+		/* Compete against other threads on who's doing the allocation,
+		 * wait until bit is cleared if we lost this race.
+		 */
+		res = wait_on_bit_lock(
+			&nfsi->pnfs_layout_state, NFS_INO_LAYOUT_ALLOC,
+			pnfs_wait_schedule, TASK_INTERRUPTIBLE);
+		if (res) {
+			lo = ERR_PTR(res);
+			break;
+		}
+
+		/* Was current_layout already allocated while we slept?
+		 * If not, allocate it.
+		 */
+		lo = nfsi->current_layout;
+		if (!lo)
+			lo = nfsi->current_layout =
+				alloc_init_layout(ino, io_ops);
+
+		/* release the NFS_INO_LAYOUT_ALLOC bit and wake up waiters */
+		clear_bit_unlock(NFS_INO_LAYOUT_ALLOC, &nfsi->pnfs_layout_state);
+		wake_up_bit(&nfsi->pnfs_layout_state, NFS_INO_LAYOUT_ALLOC);
+
+		/* we're done here.
+		 * just check whether alloc_init_layout succeeded.
+		 */
 		if (!lo)
 			lo = ERR_PTR(-ENOMEM);
+		break;
 	}
 
 #ifdef NFS_DEBUG
diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h
index 3d6bfbf..1a093b1 100644
--- a/include/linux/nfs_fs.h
+++ b/include/linux/nfs_fs.h
@@ -178,7 +178,9 @@ struct nfs_inode {
 #if defined(CONFIG_PNFS)
 	unsigned long pnfs_layout_state;
 #define NFS_INO_LAYOUT_FAILED	0x0001	/* get layout failed, stop trying */
+#define NFS_INO_LAYOUT_ALLOC	0x0002	/* get layout failed, stop trying */
 	time_t pnfs_layout_suspend;
+	wait_queue_head_t lo_waitq;
 	struct pnfs_layout_type *current_layout;
 	/* use rpc_creds in this open_context to send LAYOUTCOMMIT to MDS */
 	struct nfs_open_context *layoutcommit_ctx;
-- 
1.5.3.3