The quotaoff operation has a rare but longstanding deadlock vector
in terms of how the operation is logged. A quotaoff start intent is
logged (synchronously) at the onset to ensure recovery can handle
the operation if interrupted before in-core changes are made. This
quotaoff intent pins the log tail while the quotaoff sequence scans
and purges dquots from all in-core inodes. While this operation
generally doesn't generate much log traffic on its own, it can be
time consuming. If unrelated, concurrent filesystem activity
consumes remaining log space before quotaoff is able to acquire log
reservation for the quotaoff end intent, the filesystem locks up
indefinitely.
quotaoff cannot allocate the end intent before the scan because the
latter can result in transaction allocation itself in certain
indirect cases (releasing an inode, for example). Further, rolling
the original transaction is difficult because the scanning work
occurs multiple layers down where caller context is lost and not
much information is available to determine how often to roll the
transaction.
To address this problem, enable automatic relogging of the quotaoff
start intent. This automatically relogs the intent whenever AIL
pushing finds the item at the tail of the log. When quotaoff
completes, wait for relogging to complete as the end intent expects
to be able to permanently remove the start intent from the log
subsystem. This ensures that the log tail is kept moving during a
particularly long quotaoff operation and avoids the log reservation
deadlock.
Note that the quotaoff reservation calculation does not need to be
updated for relog as it already (incorrectly) accounts for two
quotaoff intents.
Signed-off-by: Brian Foster <bfoster@xxxxxxxxxx>
---
fs/xfs/xfs_dquot_item.c | 26 ++++++++++++++++++++++++--
fs/xfs/xfs_qm_syscalls.c | 12 +++++++++++-
2 files changed, 35 insertions(+), 3 deletions(-)
diff --git a/fs/xfs/xfs_dquot_item.c b/fs/xfs/xfs_dquot_item.c
index 349c92d26570..86dcb6932aab 100644
--- a/fs/xfs/xfs_dquot_item.c
+++ b/fs/xfs/xfs_dquot_item.c
@@ -17,6 +17,7 @@
#include "xfs_trans_priv.h"
#include "xfs_qm.h"
#include "xfs_log.h"
+#include "xfs_log_priv.h"
static inline struct xfs_dq_logitem *DQUOT_ITEM(struct xfs_log_item *lip)
{
@@ -275,14 +276,17 @@ xfs_qm_qoff_logitem_format(
}
/*
- * There isn't much you can do to push a quotaoff item. It is simply
- * stuck waiting for the log to be flushed to disk.
+ * The quotaoff log item is stuck in the log until quotaoff completes. Either
+ * relog it to keep the tail moving or consider it locked.
*/
STATIC uint
xfs_qm_qoff_logitem_push(
struct xfs_log_item *lip,
struct list_head *buffer_list)
{
+
+ if (xfs_item_needs_relog(lip))
+ return XFS_ITEM_RELOG;
return XFS_ITEM_LOCKED;
}
@@ -314,6 +318,23 @@ xfs_qm_qoff_logitem_release(
}
}
+STATIC void
+xfs_qm_qoff_logitem_relog(
+ struct xfs_log_item *lip,
+ struct xfs_trans *tp)
+{
+ int res;
+
+ res = xfs_relog_calc_res(lip);
+
+ xfs_trans_add_item(tp, lip);
+ tp->t_ticket->t_curr_res += res;
+ tp->t_ticket->t_unit_res += res;
+ tp->t_log_res += res;
+ tp->t_flags |= XFS_TRANS_DIRTY;
+ set_bit(XFS_LI_DIRTY, &lip->li_flags);
+}
+
static const struct xfs_item_ops xfs_qm_qoffend_logitem_ops = {
.iop_size = xfs_qm_qoff_logitem_size,
.iop_format = xfs_qm_qoff_logitem_format,
@@ -327,6 +348,7 @@ static const struct xfs_item_ops xfs_qm_qoff_logitem_ops = {
.iop_format = xfs_qm_qoff_logitem_format,
.iop_push = xfs_qm_qoff_logitem_push,
.iop_release = xfs_qm_qoff_logitem_release,
+ .iop_relog = xfs_qm_qoff_logitem_relog,
};
/*
diff --git a/fs/xfs/xfs_qm_syscalls.c b/fs/xfs/xfs_qm_syscalls.c
index 7effd7a28136..5602ed2b7e8d 100644
--- a/fs/xfs/xfs_qm_syscalls.c
+++ b/fs/xfs/xfs_qm_syscalls.c
@@ -18,6 +18,7 @@
#include "xfs_quota.h"
#include "xfs_qm.h"
#include "xfs_icache.h"
+#include "xfs_trans_priv.h"
STATIC int
xfs_qm_log_quotaoff(
@@ -29,12 +30,14 @@ xfs_qm_log_quotaoff(
int error;
struct xfs_qoff_logitem *qoffi;
- error = xfs_trans_alloc(mp, &M_RES(mp)->tr_qm_quotaoff, 0, 0, 0, &tp);
+ error = xfs_trans_alloc(mp, &M_RES(mp)->tr_qm_quotaoff, 0, 0,
+ XFS_TRANS_RELOG, &tp);
if (error)
goto out;
qoffi = xfs_trans_get_qoff_item(tp, NULL, flags & XFS_ALL_QUOTA_ACCT);
xfs_trans_log_quotaoff_item(tp, qoffi);
+ xfs_trans_relog_item(tp, &qoffi->qql_item);
spin_lock(&mp->m_sb_lock);
mp->m_sb.sb_qflags = (mp->m_qflags & ~(flags)) & XFS_MOUNT_QUOTA_ALL;
@@ -71,6 +74,13 @@ xfs_qm_log_quotaoff_end(
if (error)
return error;
+ /*
+ * startqoff must be in the AIL and not the CIL when the end intent
+ * commits to ensure it is not readded to the AIL out of order. Wait on
+ * relog activity to drain to isolate startqoff to the AIL.
+ */
+ xfs_trans_relog_item_cancel(tp, &(*startqoff)->qql_item, true);
+
qoffi = xfs_trans_get_qoff_item(tp, *startqoff,
flags & XFS_ALL_QUOTA_ACCT);
xfs_trans_log_quotaoff_item(tp, qoffi);
--
2.21.3
