[PATCH] support nested cb_layoturecall callbacks
Benny Halevy
bhalevy at panasas.com
Tue Jun 5 21:06:59 EDT 2007
it is possible that layoutrecall will be invoked while the state lock
is being held by the caller (e.g. on the close path). And there's no way
currently to avoid taking the state lock in the layoutrecall path.
This patch provides for detecting that the call is nested while
the state lock is taken and doing the callbacks synchronously under the
lock.
For some reason it looks like the callback path is not up
so tests doing layoutrecall fail, apprently since nfsd41_probe_callback
is never called.
Signed-off-by: Benny Halevy <bhalevy at panasas.com>
---
fs/nfsd/nfs4callback.c | 2 +
fs/nfsd/nfs4state.c | 125 +++++++++++++++++++++++++++++++++++++++++++++--
2 files changed, 121 insertions(+), 6 deletions(-)
diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c
index da20278..c6cd5ca 100644
--- a/fs/nfsd/nfs4callback.c
+++ b/fs/nfsd/nfs4callback.c
@@ -791,6 +791,8 @@ nfsd41_probe_callback(struct nfs4_client *clp)
}
cb->cb_client = clnt;
+ dprintk("nfsd: %s: clp %p cb_client %p\n", __FUNCTION__,
+ clp, clp->cl_callback.cb_client);
msg.rpc_cred = nfsd4_lookupcred(clp,0);
if (IS_ERR(msg.rpc_cred))
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 5180810..97d2ffa 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -104,6 +104,7 @@ static void release_pnfs_ds_dev_list(struct nfs4_stateid *stp);
* unconfstr_hashtbl[], uncofid_hashtbl[].
*/
static DEFINE_MUTEX(client_mutex);
+static struct thread_info *client_mutex_owner;
static kmem_cache_t *stateowner_slab = NULL;
static kmem_cache_t *file_slab = NULL;
@@ -116,14 +117,25 @@ void
nfs4_lock_state(void)
{
mutex_lock(&client_mutex);
+ client_mutex_owner = current->thread_info;
}
void
nfs4_unlock_state(void)
{
+ client_mutex_owner = NULL;
mutex_unlock(&client_mutex);
}
+static int
+nfs4_lock_state_nested(void)
+{
+ if (client_mutex_owner == current->thread_info)
+ return 0;
+ nfs4_lock_state();
+ return 1;
+}
+
static inline u32
opaque_hashval(const void *ptr, int nbytes)
{
@@ -562,6 +574,7 @@ shutdown_callback_client(struct nfs4_client *clp)
{
struct rpc_clnt *clnt = clp->cl_callback.cb_client;
+ dprintk("NFSD: %s: clp %p cb_client %p\n", __FUNCTION__, clp, clnt);
/* shutdown rpc client, ending any outstanding recall rpcs */
if (clnt) {
clp->cl_callback.cb_client = NULL;
@@ -4527,6 +4540,7 @@ cl_has_any_layout(struct nfs4_client *clp, struct nfs4_layoutrecall *dummy)
return !list_empty(&clp->cl_layouts);
}
+#if 0
/*
* Recall a layout asynchronously
* FIXME: Failures are nor reported back
@@ -4591,7 +4605,7 @@ doit:
if (unlikely(!pending->clr_client->cl_callback.cb_client)) {
printk("%s: clientid %llx has no callback path\n",
__FUNCTION__,
- (unsigned long long)clr->cb.cbl_seg.clientid);
+ (unsigned long long)pending->cb.cbl_seg.clientid);
nfs4_lock_state();
put_layoutrecall(pending);
nfs4_unlock_state();
@@ -4618,6 +4632,94 @@ doit:
return 0;
}
+#endif
+
+/*
+ * Recall a layout synchronously
+ * must be called under the state lock
+ */
+static int
+sync_layout_recall(struct nfs4_layoutrecall *clr)
+{
+ struct nfs4_layoutrecall *pending;
+ struct nfs4_client *clp = NULL;
+ unsigned int i;
+ int status;
+ struct list_head todolist;
+ static int (*__has_layout)(struct nfs4_client *,
+ struct nfs4_layoutrecall *);
+
+ BUG_ON_UNLOCKED_STATE();
+ INIT_LIST_HEAD(&todolist);
+
+ /* specific client */
+ if (clr->clr_client) {
+ list_add(&clr->clr_perclnt, &todolist);
+ clr = NULL; /* so it won't be destroyed here */
+ goto doit;
+ }
+
+ switch (clr->cb.cbl_recall_type) {
+ case RECALL_FILE:
+ __has_layout = cl_has_file_layout;
+ break;
+ case RECALL_FSID:
+ __has_layout = cl_has_fsid_layout;
+ break;
+ case RECALL_ALL:
+ __has_layout = cl_has_any_layout;
+ break;
+ }
+
+ for (i = 0; i < CLIENT_HASH_SIZE; i++)
+ list_for_each_entry(clp, &conf_str_hashtbl[i], cl_strhash)
+ if (__has_layout(clp, clr)) {
+ pending = alloc_init_layoutrecall(clr);
+ if (!pending)
+ goto doit;
+ pending->clr_client = clp;
+ list_add(&pending->clr_perclnt, &todolist);
+ }
+ /* cleanup only in the multi client, single client went into todolist */
+ put_layoutrecall(clr);
+
+doit:
+ while (!list_empty(&todolist)) {
+ pending = list_entry(todolist.next, struct nfs4_layoutrecall,
+ clr_perclnt);
+ list_del_init(&pending->clr_perclnt);
+ dprintk("%s: clp %p cb_client %p fp %p\n", __FUNCTION__,
+ pending->clr_client,
+ pending->clr_client->cl_callback.cb_client,
+ pending->clr_file);
+ if (unlikely(!pending->clr_client->cl_callback.cb_client)) {
+ printk("%s: clientid %08x/%08x has no callback path\n",
+ __FUNCTION__,
+ pending->clr_client->cl_clientid.cl_boot,
+ pending->clr_client->cl_clientid.cl_id);
+ put_layoutrecall(pending);
+ continue;
+ }
+ pending->clr_time = CURRENT_TIME;
+ hash_layoutrecall(pending);
+
+ status = nfsd4_cb_layout(pending);
+ if (status) {
+ if (status != NFSERR_NOMATCHING_LAYOUT)
+ printk("%s: clp %p cb_client %p fp %p "
+ "failed with status %d\n", __FUNCTION__,
+ pending->clr_client,
+ pending->clr_client->cl_callback.cb_client,
+ pending->clr_file,
+ status);
+ layoutrecall_done(pending);
+ }
+
+ put_layoutrecall(pending);
+ }
+
+ return 0;
+}
/*
* Spawn a thread to perform a recall layout
@@ -4625,7 +4727,7 @@ doit:
*/
int nfsd_layout_recall_cb(struct inode *inode, struct nfsd4_pnfs_cb_layout *cbl)
{
- int status;
+ int status, did_lock;
struct nfs4_layoutrecall *clr = NULL;
struct task_struct *t;
@@ -4645,7 +4747,7 @@ int nfsd_layout_recall_cb(struct inode *inode, struct nfsd4_pnfs_cb_layout *cbl)
clr->clr_client = NULL;
clr->clr_file = NULL;
- nfs4_lock_state();
+ did_lock = nfs4_lock_state_nested();
status = -ENOENT;
if (clr->cb.cbl_seg.clientid) {
clr->clr_client =
@@ -4665,17 +4767,28 @@ int nfsd_layout_recall_cb(struct inode *inode, struct nfsd4_pnfs_cb_layout *cbl)
if (cbl->cbl_recall_type == RECALL_FSID)
clr->cb.cbl_fsid = clr->clr_file->fi_fsid;
}
- nfs4_unlock_state();
+ status = sync_layout_recall(clr);
+ if (!status) {
+ if (did_lock)
+ nfs4_unlock_state();
+ return 0;
+ }
+
+#if 0
t = kthread_run(do_layout_recall, clr, "%s", "nfs4_cb_recall");
+ dprintk("NFSD %s: kthread_run done error %ld\n", __FUNCTION__,
+ IS_ERR(t) ? PTR_ERR(t) : 0L);
if (!IS_ERR(t))
return 0; /* Success!, refcounts handled by kthread */
status = PTR_ERR(t);
- nfs4_lock_state();
+ did_lock = nfs4_lock_state_nested();
+#endif
err:
put_layoutrecall(clr);
- nfs4_unlock_state();
+ if (did_lock)
+ nfs4_unlock_state();
return status;
}
--
1.5.2.86.g99b5
--------------090902040601030403090705--
More information about the pNFS
mailing list