The log lock is currently used to protect the AIL lists and
the movements of buffers into and out of them. The lists
are self contained and no log specific items outside the
lists are accessed when starting or emptying the AIL lists.
Hence the operation of the AIL does not require the protection
of the log lock so split them out into a new AIL specific lock
to reduce the amount of traffic on the log lock. This will
also reduce the amount of serialisation that occurs when
the gfs2_logd pushes on the AIL to move it forward.
This reduces the impact of log pushing on sequential write
throughput.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
Signed-off-by: Steven Whitehouse <swhiteho@redhat.com>
BUG_ON(current->journal_info);
current->journal_info = &tr;
BUG_ON(current->journal_info);
current->journal_info = &tr;
+ spin_lock(&sdp->sd_ail_lock);
while (!list_empty(head)) {
bd = list_entry(head->next, struct gfs2_bufdata,
bd_ail_gl_list);
bh = bd->bd_bh;
gfs2_remove_from_ail(bd);
while (!list_empty(head)) {
bd = list_entry(head->next, struct gfs2_bufdata,
bd_ail_gl_list);
bh = bd->bd_bh;
gfs2_remove_from_ail(bd);
+ spin_unlock(&sdp->sd_ail_lock);
+
bd->bd_bh = NULL;
bh->b_private = NULL;
bd->bd_blkno = bh->b_blocknr;
bd->bd_bh = NULL;
bh->b_private = NULL;
bd->bd_blkno = bh->b_blocknr;
gfs2_assert_withdraw(sdp, !buffer_busy(bh));
gfs2_trans_add_revoke(sdp, bd);
gfs2_assert_withdraw(sdp, !buffer_busy(bh));
gfs2_trans_add_revoke(sdp, bd);
+ gfs2_log_unlock(sdp);
+
+ spin_lock(&sdp->sd_ail_lock);
}
gfs2_assert_withdraw(sdp, !atomic_read(&gl->gl_ail_count));
}
gfs2_assert_withdraw(sdp, !atomic_read(&gl->gl_ail_count));
+ spin_unlock(&sdp->sd_ail_lock);
gfs2_trans_end(sdp);
gfs2_log_flush(sdp, NULL);
gfs2_trans_end(sdp);
gfs2_log_flush(sdp, NULL);
unsigned int sd_log_flush_head;
u64 sd_log_flush_wrapped;
unsigned int sd_log_flush_head;
u64 sd_log_flush_wrapped;
+ spinlock_t sd_ail_lock;
struct list_head sd_ail1_list;
struct list_head sd_ail2_list;
u64 sd_ail_sync_gen;
struct list_head sd_ail1_list;
struct list_head sd_ail2_list;
u64 sd_ail_sync_gen;
*/
static void gfs2_ail1_start_one(struct gfs2_sbd *sdp, struct gfs2_ail *ai)
*/
static void gfs2_ail1_start_one(struct gfs2_sbd *sdp, struct gfs2_ail *ai)
-__releases(&sdp->sd_log_lock)
-__acquires(&sdp->sd_log_lock)
+__releases(&sdp->sd_ail_lock)
+__acquires(&sdp->sd_ail_lock)
{
struct gfs2_bufdata *bd, *s;
struct buffer_head *bh;
{
struct gfs2_bufdata *bd, *s;
struct buffer_head *bh;
list_move(&bd->bd_ail_st_list, &ai->ai_ail1_list);
get_bh(bh);
list_move(&bd->bd_ail_st_list, &ai->ai_ail1_list);
get_bh(bh);
+ spin_unlock(&sdp->sd_ail_lock);
lock_buffer(bh);
if (test_clear_buffer_dirty(bh)) {
bh->b_end_io = end_buffer_write_sync;
lock_buffer(bh);
if (test_clear_buffer_dirty(bh)) {
bh->b_end_io = end_buffer_write_sync;
unlock_buffer(bh);
brelse(bh);
}
unlock_buffer(bh);
brelse(bh);
}
+ spin_lock(&sdp->sd_ail_lock);
struct gfs2_ail *ai;
int done = 0;
struct gfs2_ail *ai;
int done = 0;
+ spin_lock(&sdp->sd_ail_lock);
head = &sdp->sd_ail1_list;
if (list_empty(head)) {
head = &sdp->sd_ail1_list;
if (list_empty(head)) {
+ spin_unlock(&sdp->sd_ail_lock);
return;
}
sync_gen = sdp->sd_ail_sync_gen++;
return;
}
sync_gen = sdp->sd_ail_sync_gen++;
if (ai->ai_sync_gen >= sync_gen)
continue;
ai->ai_sync_gen = sync_gen;
if (ai->ai_sync_gen >= sync_gen)
continue;
ai->ai_sync_gen = sync_gen;
- gfs2_ail1_start_one(sdp, ai); /* This may drop log lock */
+ gfs2_ail1_start_one(sdp, ai); /* This may drop ail lock */
+ spin_unlock(&sdp->sd_ail_lock);
}
static int gfs2_ail1_empty(struct gfs2_sbd *sdp, int flags)
}
static int gfs2_ail1_empty(struct gfs2_sbd *sdp, int flags)
struct gfs2_ail *ai, *s;
int ret;
struct gfs2_ail *ai, *s;
int ret;
+ spin_lock(&sdp->sd_ail_lock);
list_for_each_entry_safe_reverse(ai, s, &sdp->sd_ail1_list, ai_list) {
if (gfs2_ail1_empty_one(sdp, ai, flags))
list_for_each_entry_safe_reverse(ai, s, &sdp->sd_ail1_list, ai_list) {
if (gfs2_ail1_empty_one(sdp, ai, flags))
ret = list_empty(&sdp->sd_ail1_list);
ret = list_empty(&sdp->sd_ail1_list);
+ spin_unlock(&sdp->sd_ail_lock);
int wrap = (new_tail < old_tail);
int a, b, rm;
int wrap = (new_tail < old_tail);
int a, b, rm;
+ spin_lock(&sdp->sd_ail_lock);
list_for_each_entry_safe(ai, safe, &sdp->sd_ail2_list, ai_list) {
a = (old_tail <= ai->ai_first);
list_for_each_entry_safe(ai, safe, &sdp->sd_ail2_list, ai_list) {
a = (old_tail <= ai->ai_first);
+ spin_unlock(&sdp->sd_ail_lock);
struct gfs2_ail *ai;
unsigned int tail;
struct gfs2_ail *ai;
unsigned int tail;
+ spin_lock(&sdp->sd_ail_lock);
if (list_empty(&sdp->sd_ail1_list)) {
tail = sdp->sd_log_head;
if (list_empty(&sdp->sd_ail1_list)) {
tail = sdp->sd_log_head;
+ spin_unlock(&sdp->sd_ail_lock);
sdp->sd_log_commited_databuf = 0;
sdp->sd_log_commited_revoke = 0;
sdp->sd_log_commited_databuf = 0;
sdp->sd_log_commited_revoke = 0;
+ spin_lock(&sdp->sd_ail_lock);
if (!list_empty(&ai->ai_ail1_list)) {
list_add(&ai->ai_list, &sdp->sd_ail1_list);
ai = NULL;
}
if (!list_empty(&ai->ai_ail1_list)) {
list_add(&ai->ai_list, &sdp->sd_ail1_list);
ai = NULL;
}
+ spin_unlock(&sdp->sd_ail_lock);
gfs2_log_unlock(sdp);
trace_gfs2_log_flush(sdp, 0);
up_write(&sdp->sd_log_flush_lock);
gfs2_log_unlock(sdp);
trace_gfs2_log_flush(sdp, 0);
up_write(&sdp->sd_log_flush_lock);
mark_buffer_dirty(bh);
clear_buffer_pinned(bh);
mark_buffer_dirty(bh);
clear_buffer_pinned(bh);
+ spin_lock(&sdp->sd_ail_lock);
if (bd->bd_ail) {
list_del(&bd->bd_ail_st_list);
brelse(bh);
if (bd->bd_ail) {
list_del(&bd->bd_ail_st_list);
brelse(bh);
}
bd->bd_ail = ai;
list_add(&bd->bd_ail_st_list, &ai->ai_ail1_list);
}
bd->bd_ail = ai;
list_add(&bd->bd_ail_st_list, &ai->ai_ail1_list);
+ spin_unlock(&sdp->sd_ail_lock);
+
if (test_and_clear_bit(GLF_LFLUSH, &bd->bd_gl->gl_flags))
gfs2_glock_schedule_for_reclaim(bd->bd_gl);
trace_gfs2_pin(bd, 0);
if (test_and_clear_bit(GLF_LFLUSH, &bd->bd_gl->gl_flags))
gfs2_glock_schedule_for_reclaim(bd->bd_gl);
trace_gfs2_pin(bd, 0);
unlock_buffer(bh);
atomic_dec(&sdp->sd_log_pinned);
}
unlock_buffer(bh);
atomic_dec(&sdp->sd_log_pinned);
}
init_waitqueue_head(&sdp->sd_log_waitq);
init_waitqueue_head(&sdp->sd_logd_waitq);
init_waitqueue_head(&sdp->sd_log_waitq);
init_waitqueue_head(&sdp->sd_logd_waitq);
+ spin_lock_init(&sdp->sd_ail_lock);
INIT_LIST_HEAD(&sdp->sd_ail1_list);
INIT_LIST_HEAD(&sdp->sd_ail2_list);
INIT_LIST_HEAD(&sdp->sd_ail1_list);
INIT_LIST_HEAD(&sdp->sd_ail2_list);