BLOG

enum wb_reason {    
		WB_REASON_BACKGROUND,       //后台回写    
		WB_REASON_VMSCAN,           //内存压力    
		WB_REASON_SYNC,             //系统调用回写    
		WB_REASON_PERIODIC,         //定期回写    
		WB_REASON_LAPTOP_TIMER,     //Laptop模式回写    
		WB_REASON_FS_FREE_SPACE,    //当1/2的可用块变脏时开始推送delalloc，ext4使用    
		WB_REASON_FORKER_THREAD,    //紧急情况下回写    
		WB_REASON_FOREIGN_FLUSH,    //cgroup相关回写    
		WB_REASON_MAX,
};

struct wb_writeback_work {    
		long nr_pages;                      //待回写页面数量    
		struct super_block *sb;             //该writeback任务所属的
		super_block    enum writeback_sync_modes sync_mode; //同步回写模式    
		unsigned int tagged_writepages:1;   //    
		unsigned int for_kupdate:1;         //若值为1，则表示回写操作是周期性的机制；否则值为0    
		unsigned int range_cyclic:1;        //若值为0，则表示回写操作范围限制在[range_start, range_end]限定范围；若值为1，则表示内核可以对 mapping 里的页面执行多次回写操作。    
		unsigned int for_background:1;      //若值为1，表示后台回写；否则值为0    
		unsigned int for_sync:1;            /* sync(2) WB_SYNC_ALL writeback */    
		unsigned int auto_free:1;           /* free on completion */    
		enum wb_reason reason;              /* writeback触发原因 */    
		struct list_head list;              /* 待处理的工作链表 */    
		struct wb_completion *done;         /* 完成标志 */
};
/* 指定同步模式 */
enum writeback_sync_modes {    
		WB_SYNC_NONE,   /* 表示当遇到锁住的inode时，它必须等待该inode解锁，而不能跳过。 */    
		WB_SYNC_ALL,    /* 表示跳过被锁住的inode */
};

struct inode {    
		...    
		const struct inode_operations   *i_op;    
		struct super_block  *i_sb;    
		struct address_space    *i_mapping;    
		...    
union {        
		const struct file_operations    *i_fop; /* former ->i_op->default_file_ops */        
		void (*free_inode)(struct inode *);    
};    
		struct file_lock_context    *i_flctx;    
		struct address_space    i_data;    
		struct list_head    i_devices;
};

struct writeback_control {    
		long nr_to_write;       /* 要写的页数 */    
		long pages_skipped;     /* 未写的页数 */    /* 如果不为0，则为要写的页 */    
		loff_t range_start;    
		loff_t range_end;    
		enum writeback_sync_modes sync_mode;    
		unsigned for_kupdate:1;     /* 定期回写 */    
		unsigned for_background:1;  /* 后台回写 */    
		unsigned tagged_writepages:1;   /* 写入标记 */    
		unsigned for_reclaim:1;     /* 从页面分配器调用 */    
		unsigned range_cyclic:1;    /* 循环写开始位置 */    
		unsigned for_sync:1;        /* sync(2) WB_SYNC_ALL 回写 */
};

/* * Ext4 inode日志模式 */
#define EXT4_INODE_JOURNAL_DATA_MODE    0x01 
/* 日志数据模式 */
#define EXT4_INODE_ORDERED_DATA_MODE    0x02 
/* 有序数据模式 */
#define EXT4_INODE_WRITEBACK_DATA_MODE  0x04 
/* 写回数据模式 */

/*    
		I_DIRTY_SYNC        Inode为脏,但不是一定要写入    
		I_DIRTY_DATASYNC    Inode中与数据相关的更改待处理，需要被写入    
		I_DIRTY_PAGES       Inode的page脏, Inode本身可能是干净的    
		I_DIRTY_TIME        Inode的时间戳为脏，有以上标志则此标志被清除    
		I_SYNC              writeback运行标志
*/

1 I root      08:54 ?        00:00:00 [kworker/u32:1-events_unbound]
1 I root      09:51 ?        00:00:00 [kworker/u32:2-flush-259:0]
1 I root      09:57 ?        00:00:00 [kworker/u32:0-events_power_efficient]
1 I root      10:05 ?        00:00:00 [kworker/u32:3]

/* 定义 */    
struct workqueue_struct *bdi_wq;    
/* 增加work */    
queue_delayed_work(bdi_wq, &wb->dwork, timeout);

static int __init default_bdi_init(void){    
		int err;    /* 分配内存 */    
		bdi_wq = alloc_workqueue("writeback", WQ_MEM_RECLAIM | WQ_UNBOUND |                 WQ_SYSFS, 0);    
		...
}
static int wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi,           gfp_t gfp){    
		...    
		INIT_LIST_HEAD(&wb->work_list);    
		INIT_DELAYED_WORK(&wb->dwork, wb_workfn); //wb_workfn为工作队列的处理函数    
		wb->dirty_sleep = jiffies;    
		...
}

/* 
 *文件的时间戳，共有三个：ctime指inode上一次变动的时间，mtime指文件内容上一次变动的时间，atime指文件上一次打开的时间。 
 * 由atime引起inode变脏，如果有其他回写则不会进行此回写，根据dirtytime_writeback_centisecs延时唤醒,12h一次 
 */
static DECLARE_DELAYED_WORK(dirtytime_work, wakeup_dirtytime_writeback);
static void wakeup_dirtytime_writeback(struct work_struct *w){        
		list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node)            
		if (!list_empty(&wb->b_dirty_time))                
				wb_wakeup(wb);    
		schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ);
}

static void wb_wakeup(struct bdi_writeback *wb){    
		if (test_bit(WB_registered, &wb->state))        
				mod_delayed_work(bdi_wq, &wb->dwork, 0);
}
bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq, struct delayed_work *dwork, unsigned long delay){    
		/* 找到一个pending的work */    
		do {        
				ret = try_to_grab_pending(&dwork->work, true, &flags);    
		} while (unlikely(ret == -EAGAIN)); 
   
		/* 修改work的timer */    
		if (likely(ret >= 0)) {        
				__queue_delayed_work(cpu, wq, dwork, delay);        
		local_irq_restore(flags);    
		}    
		...
}

  submit_bio
  ext4_writepages
  do_writepages
  __writeback_single_inode
  writeback_sb_inodes
  wb_writeback
  wb_workfn

void wb_workfn(struct work_struct *work){    
		...    
		set_worker_desc("flush-%s", bdi_dev_name(wb->bdi)); 
   
		current->flags |= PF_SWAPWRITE;    
		if (likely(!current_is_workqueue_rescuer() ||           !test_bit(WB_registered, &wb->state))) {        
				/* 正常情况，直到work_list为空停止运行 */        
				do {            
						pages_written = wb_do_writeback(wb);        
				} while (!list_empty(&wb->work_list));    
		} else {        
				/* 紧急情况 */        
				pages_written = writeback_inodes_wb(wb, 1024, WB_REASON_FORKER_THREAD);    
		}  
  
		/* work_list非空则再次唤醒执行 */    
		if (!list_empty(&wb->work_list))       
				wb_wakeup(wb); 
		/* dirty inode在其他list中还有，设置延后唤醒，dirty_writeback_interval设置为500 */    
		else if (wb_has_dirty_io(wb) && dirty_writeback_interval)        
				wb_wakeup_delayed(wb);
    
		current->flags &= ~PF_SWAPWRITE;
}

static long wb_do_writeback(struct bdi_writeback *wb){    
		/* 如果设置了running标志则可以直接执行wb_writeback函数 */    
		set_bit(WB_writeback_running, &wb->state);
    
		while ((work = get_next_work_item(wb)) != NULL) {        
				wrote += wb_writeback(wb, work);        
				finish_writeback_work(wb, work);    
		} 
   
		/* 检查start_all标志，判断是否需要进行全部回写 */    
		wrote += wb_check_start_all(wb);
    
		/* 进行定期回写检查，检查间隔时间是否达到了5s，没有则直接返回，有则进行wb_writeback */    
		wrote += wb_check_old_data_flush(wb); 
   
		/* 进行后台检查，检查脏页数量是否超过了 dirty_background_bytes 或 dirty_background_ratio，    超过了配置的参数则需要进行回写 */    
		wrote += wb_check_background_flush(wb);    
		clear_bit(WB_writeback_running, &wb->state); 
   
		return wrote;
}

static long wb_writeback(struct bdi_writeback *wb,             
struct wb_writeback_work *work){    
		...    
		for (;;) {        
			/* 脏页已被处理 */        
		if (work->nr_pages <= 0)            
				break;   
     
		/* 后台回写和定期回写在work_list为空时可能永远运行，所以直接退出，以便这时其他的work能够被处理 */        
		if ((work->for_background || work->for_kupdate) &&            
																							!list_empty(&wb->work_list))            
				break; 
       
		/* 对于后台回写，低于dirty_background_ratio则不需要回写 */        
		if (work->for_background && !wb_over_bg_thresh(wb))            
				break;   
     
		/* 后台回写和定期回写是特殊的，需要包含所有的inode节点 */        
		if (work->for_kupdate) {            
				dirtied_before = jiffies - msecs_to_jiffies(dirty_expire_interval * 10);        
		} else if (work->for_background)            
				dirtied_before = jiffies;        

		if (list_empty(&wb->b_io))            
				queue_io(wb, work, dirtied_before);        
		
		if (work->sb)            
				progress = writeback_sb_inodes(work->sb, wb, work);        
		else            
				progress = __writeback_inodes_wb(wb, work);        

		/* 处理完毕 */        
		if (progress)            
				continue;        
		/* b_more_io list也为空则说明全部处理完毕 */        
		if (list_empty(&wb->b_more_io))            
				break;        

		/* 等待可用inode节点，否则忙碌 */        
		trace_writeback_wait(wb, work);        
		inode = wb_inode(wb->b_more_io.prev);        
		/* 重新回到睡眠状态 */        
		inode_sleep_on_writeback(inode);    
		} 
   
		return nr_pages - work->nr_pages;
}

struct writeback_control wbc = {        
		.sync_mode      = work->sync_mode,        
		.tagged_writepages  = work->tagged_writepages,        
		.for_kupdate        = work->for_kupdate,        
		.for_background     = work->for_background,        
		.for_sync       = work->for_sync,        
		.range_cyclic       = work->range_cyclic,        
		.range_start        = 0,        
		.range_end      = LLONG_MAX,    
};

int dirty_writeback_centisecs_handler(struct ctl_table *table, int write,        void *buffer, size_t *length, loff_t *ppos){    
		unsigned int old_interval = dirty_writeback_interval;    
		/* 前面通过遍历bdis和wbs的方式很麻烦，这里通过/proc文件系统进行定期唤醒 */    
		if (!ret && write && dirty_writeback_interval &&        
																		dirty_writeback_interval != old_interval)        
		wakeup_flusher_threads(WB_REASON_PERIODIC);
}

  b'submit_bio'
  b'__block_write_full_page'
  b'block_write_full_page'
  b'blkdev_writepage'
  b'__writepage'
  b'write_cache_pages'
  b'generic_writepages'
  b'blkdev_writepages'
  b'do_writepages'
  b'__filemap_fdatawrite_range'
  b'filemap_fdatawrite'
  b'fdatawrite_one_bdev'
  b'iterate_bdevs'
  b'ksys_sync'
  b'__x64_sys_sync'
  b'do_syscall_64'
  b'entry_SYSCALL_64_after_hwframe'
    --
  b'[unknown]'
    b'sync'

void ksys_sync(void){    
		int nowait = 0, wait = 1;
    
		wakeup_flusher_threads(WB_REASON_SYNC);    
		iterate_supers(sync_inodes_one_sb, NULL);    
		iterate_supers(sync_fs_one_sb, &nowait);    
		iterate_supers(sync_fs_one_sb, &wait);    
		iterate_bdevs(fdatawrite_one_bdev, NULL);    
		iterate_bdevs(fdatawait_one_bdev, NULL);    
		if (unlikely(laptop_mode))        
				laptop_sync_completion();
}

int do_writepages(struct address_space *mapping, struct writeback_control *wbc){    
		int ret;  
  
		if (wbc->nr_to_write <= 0)        
				return 0;  
  
		while (1) {        
				if (mapping->a_ops->writepages)            
						ret = mapping->a_ops->writepages(mapping, wbc);        
				else            
						ret = generic_writepages(mapping, wbc);        
				if ((ret != -ENOMEM) || (wbc->sync_mode != WB_SYNC_ALL))            
						break;
        
				cond_resched();        
				congestion_wait(BLK_RW_ASYNC, HZ/50);    
		} 
   
		return ret;
}

static long wb_check_background_flush(struct bdi_writeback *wb){    
		if (wb_over_bg_thresh(wb)) { 
				//检查当前脏页是否超过dirty_background_ratio参数        
				struct wb_writeback_work work = {            
						.nr_pages   = LONG_MAX,            
						.sync_mode  = WB_SYNC_NONE,            
						.for_background = 1,            
						.range_cyclic   = 1,            
						.reason     = WB_REASON_BACKGROUND,        
				};        
				return wb_writeback(wb, &work);    
		}    
		return 0;
}

  b'domain_dirty_limits'
  b'balance_dirty_pages_ratelimited'
  b'fault_dirty_shared_page'
  b'do_wp_page'
  b'__handle_mm_fault'
  b'handle_mm_fault'
  b'do_user_addr_fault'
  b'exc_page_fault'
  b'asm_exc_page_fault'
    --
  b'[unknown]'
  b'[unknown]'
    b'systemd-journal' [362]
    1

static void domain_dirty_limits(struct dir
ty_throttle_control *dtc){    
		unsigned long bytes = vm_dirty_bytes;    
		unsigned long bg_bytes = dirty_background_bytes;    
		/* convert ratios to per-PAGE_SIZE for higher precision */    
		unsigned long ratio = (vm_dirty_ratio * PAGE_SIZE) / 100;    
		unsigned long bg_ratio = (dirty_background_ratio * PAGE_SIZE) / 100;    
		unsigned long thresh;    
		unsigned long bg_thresh;    
		struct task_struct *tsk; 
   
		/* gdtc is !NULL iff @dtc is for memcg domain */    
		if (gdtc) {        
				unsigned long global_avail = gdtc->avail;        
				if (bytes)            
						ratio = min(DIV_ROUND_UP(bytes, global_avail),                    PAGE_SIZE);        
				if (bg_bytes)            
						bg_ratio = min(DIV_ROUND_UP(bg_bytes, global_avail),                       PAGE_SIZE);        
				bytes = bg_bytes = 0;    
		}  
  
		/* 计算ratio */    
		if (bytes)        
				thresh = DIV_ROUND_UP(bytes, PAGE_SIZE);    
		else        
				thresh = (ratio * available_memory) / PAGE_SIZE;    

		/* 计算bg_ratio */    
		if (bg_bytes)        
				bg_thresh = DIV_ROUND_UP(bg_bytes, PAGE_SIZE);    
		else        
				bg_thresh = (bg_ratio * available_memory) / PAGE_SIZE;    
		
		/* 更新数据 */    
		if (bg_thresh >= thresh)        
		bg_thresh = thresh / 2;    
		tsk = current;    
		if (rt_task(tsk)) {        
				bg_thresh += bg_thresh / 4 + global_wb_domain.dirty_limit / 32;        
				thresh += thresh / 4 + global_wb_domain.dirty_limit / 32;    
		}  
  
		dtc->thresh = thresh;    
		dtc->bg_thresh = bg_thresh;
}