LVM锁服务lvmlockd分析
LVM锁服务lvmlockd分析
LastRitter 发表于1个月前
LVM锁服务lvmlockd分析
  • 发表于 1个月前
  • 阅读 10
  • 收藏 0
  • 点赞 0
  • 评论 0

腾讯云实验室 1小时搭建人工智能应用,让技术更容易入门 免费体验 >>>   

  • lvmlockd守护进程使用LVM内部的libdaemon库实现与命令行程序的通信,为LVM命令提供分布式锁功能。
  • lvmlockd的具体实现依赖sanlock(依赖wdmd服务)和dlm(依赖CoroSync、PaceMaker和Fence设备)。
  • lvmlockd必须配合LVM的本地锁(flock)使用,是本地锁在集群环境下的扩充。
  • lvmlockd的作者也是sanlock的原作者,发起此项目的目的是替代clvmd,使得用户在dlm外有更多选择(但从他的源码结构来看,没有使用一个好的抽象接口,很难扩充第三种锁的支持,我认为这不是一个好的设计)。
  • lvmlockd可以和lvmetad共存,但clvmd却不可以(原因未知,我并没有分析出他们为什么不能共存)。
  • lvmlockd的功能没有使用LVM的锁抽象接口来实现,而是直接插入LVM源码中(我认为这不是个好的设计)。

上层接口函数

上层接口函数为LVM命令提供了与lvmlockd守护进程建立连接,并使用其提供的所服务的功能,这些函数的调用分布在LVM命令的所有需要锁的环节。

连接和断开服务

$ cat daemons/lvmlockd/lvmlockd-client.h
...

#define LVMLOCKD_SOCKET DEFAULT_RUN_DIR "/lvmlockd.socket"

/* Wrappers to open/close connection */

static inline daemon_handle lvmlockd_open(const char *sock)
{
	daemon_info lvmlockd_info = {
		.path = "lvmlockd",
		.socket = sock ?: LVMLOCKD_SOCKET,
		.protocol = "lvmlockd",
		.protocol_version = 1,
		.autostart = 0
	};

	return daemon_open(lvmlockd_info);
}

static inline void lvmlockd_close(daemon_handle h)
{
	return daemon_close(h);
}

...

锁操作

$ cat lib/locking/lvmlockd.h
...

/* vgcreate/vgremove use init/free */

int lockd_init_vg(struct cmd_context *cmd, struct volume_group *vg, const char *lock_type, int lv_lock_count);
int lockd_free_vg_before(struct cmd_context *cmd, struct volume_group *vg, int changing);
void lockd_free_vg_final(struct cmd_context *cmd, struct volume_group *vg);

/* vgrename */

int lockd_rename_vg_before(struct cmd_context *cmd, struct volume_group *vg);
int lockd_rename_vg_final(struct cmd_context *cmd, struct volume_group *vg, int success);

/* start and stop the lockspace for a vg */

int lockd_start_vg(struct cmd_context *cmd, struct volume_group *vg, int start_init);
int lockd_stop_vg(struct cmd_context *cmd, struct volume_group *vg);
int lockd_start_wait(struct cmd_context *cmd);

/* locking */

int lockd_gl_create(struct cmd_context *cmd, const char *def_mode, const char *vg_lock_type);
int lockd_gl(struct cmd_context *cmd, const char *def_mode, uint32_t flags);
int lockd_vg(struct cmd_context *cmd, const char *vg_name, const char *def_mode,
	     uint32_t flags, uint32_t *lockd_state);
int lockd_vg_update(struct volume_group *vg);

int lockd_lv_name(struct cmd_context *cmd, struct volume_group *vg,
		  const char *lv_name, struct id *lv_id,
		  const char *lock_args, const char *def_mode, uint32_t flags);
int lockd_lv(struct cmd_context *cmd, struct logical_volume *lv,
	     const char *def_mode, uint32_t flags);

/* lvcreate/lvremove use init/free */

int lockd_init_lv(struct cmd_context *cmd, struct volume_group *vg, struct logical_volume *lv,
		  struct lvcreate_params *lp);
int lockd_init_lv_args(struct cmd_context *cmd, struct volume_group *vg,
		       struct logical_volume *lv, const char *lock_type, const char **lock_args);
int lockd_free_lv(struct cmd_context *cmd, struct volume_group *vg,
		  const char *lv_name, struct id *lv_id, const char *lock_args);

const char *lockd_running_lock_type(struct cmd_context *cmd, int *found_multiple);

int handle_sanlock_lv(struct cmd_context *cmd, struct volume_group *vg);

int lockd_lv_uses_lock(struct logical_volume *lv);

...

内部数据结构

锁定范围(对象)

  • 全局
  • 卷组
  • 逻辑卷
$ cat daemons/lvmlockd/lvmlockd-internal.h
...

/* resource types */
enum {
	LD_RT_GL = 1,
	LD_RT_VG,
	LD_RT_LV,
};

...

锁定模式(状态)

  • 无效
  • 解锁
  • 空锁
  • 共享
  • 独占
...

/* lock modes, more restrictive must be larger value */
enum {
	LD_LK_IV = -1,
	LD_LK_UN = 0,
	LD_LK_NL = 1,
	LD_LK_SH = 2,
	LD_LK_EX = 3,
};

...

锁管理器类型

目前只支持DLM和sanlock。

...

/* lock manager types */
enum {
	LD_LM_NONE = 0,
	LD_LM_UNUSED = 1, /* place holder so values match lib/locking/lvmlockd.h */
	LD_LM_DLM = 2,
	LD_LM_SANLOCK = 3,
};

...

客户端管理

每个客户端连接对应一个。

...

struct list_head {
	struct list_head *next, *prev;
};

struct client {
	struct list_head list;
	pthread_mutex_t mutex;
	int pid;
	int fd;
	int pi;
	uint32_t id;
	unsigned int recv : 1;
	unsigned int dead : 1;
	unsigned int poll_ignore : 1;
	unsigned int lock_ops : 1;
	char name[MAX_NAME+1];
};

...

动作(报文)

每个客户端的请求都被填充为一个请求。

类型

...

/* operation types */
enum {
	LD_OP_HELLO = 1,
	LD_OP_QUIT,
	LD_OP_INIT,
	LD_OP_FREE,
	LD_OP_START,
	LD_OP_STOP,
	LD_OP_LOCK,
	LD_OP_UPDATE,
	LD_OP_CLOSE,
	LD_OP_ENABLE,
	LD_OP_DISABLE,
	LD_OP_START_WAIT,
	LD_OP_STOP_ALL,
	LD_OP_DUMP_INFO,
	LD_OP_DUMP_LOG,
	LD_OP_RENAME_BEFORE,
	LD_OP_RENAME_FINAL,
	LD_OP_RUNNING_LM,
	LD_OP_FIND_FREE_LOCK,
	LD_OP_KILL_VG,
	LD_OP_DROP_VG,
	LD_OP_BUSY,
};

...

其中很多操作仅供内部使用,客户端产生的动作类型有:

$ cat daemons/lvmlockd/lvmlockd-core.c
...
	case LD_OP_LOCK:
	case LD_OP_UPDATE:
	case LD_OP_ENABLE:
	case LD_OP_DISABLE:
	case LD_OP_FREE:
	case LD_OP_RENAME_BEFORE:
	case LD_OP_FIND_FREE_LOCK:
	case LD_OP_KILL_VG:
	case LD_OP_DROP_VG:
	case LD_OP_BUSY:
		rv = add_lock_action(act);
		break;
...
  • 我们最关心的是其中的LD_OP_LOCK和LD_OP_UPDATE动作,这两个锁操作会改资源的版本号。
  • 每次解除写入锁或者更新锁时就会更改资源的版本,客户端命令就会据此判断LVM的元数据是否被修改。
  • 如果客户端使用了lvmetad元数据服务,则会据此重新扫描本地磁盘中的元数据。

资源的版本号:

$ cat daemons/lvmlockd/lvmlockd-internal.h
...

/* val_blk version */
#define VAL_BLK_VERSION 0x0101

/* val_blk flags */
#define VBF_REMOVED 0x0001

struct val_blk {
	uint16_t version;
	uint16_t flags;
	uint32_t r_version;
};

...

标志

...

#define LD_AF_PERSISTENT           0x00000001
#define LD_AF_NO_CLIENT            0x00000002
#define LD_AF_UNLOCK_CANCEL        0x00000004
#define LD_AF_NEXT_VERSION         0x00000008
#define LD_AF_WAIT                 0x00000010
#define LD_AF_FORCE                0x00000020
#define LD_AF_EX_DISABLE           0x00000040
#define LD_AF_ENABLE               0x00000080
#define LD_AF_DISABLE              0x00000100
#define LD_AF_SEARCH_LS            0x00000200
#define LD_AF_WAIT_STARTING        0x00001000
#define LD_AF_DUP_GL_LS            0x00002000
#define LD_AF_ADOPT                0x00010000
#define LD_AF_WARN_GL_REMOVED	   0x00020000
#define LD_AF_LV_LOCK              0x00040000
#define LD_AF_LV_UNLOCK            0x00080000

...

结构

客户端线程使用libdaemon的ascii格式协议转换构造。

...

/*
 * Number of times to repeat a lock request after
 * a lock conflict (-EAGAIN) if unspecified in the
 * request.
 */
#define DEFAULT_MAX_RETRIES 4

struct action {
	struct list_head list;
	uint32_t client_id;
	uint32_t flags;			/* LD_AF_ */
	uint32_t version;
	uint64_t host_id;
	int8_t op;			/* operation type LD_OP_ */
	int8_t rt;			/* resource type LD_RT_ */
	int8_t mode;			/* lock mode LD_LK_ */
	int8_t lm_type;			/* lock manager: LM_DLM, LM_SANLOCK */
	int retries;
	int max_retries;
	int result;
	int lm_rv;			/* return value from lm_ function */
	char vg_uuid[64];
	char vg_name[MAX_NAME+1];
	char lv_name[MAX_NAME+1];
	char lv_uuid[MAX_NAME+1];
	char vg_args[MAX_ARGS+1];
	char lv_args[MAX_ARGS+1];
	char vg_sysid[MAX_NAME+1];
};

...

对象管理

锁定的对象被称为资源,每个资源上可以有多个锁,多个锁组成一个锁空间,各个锁空间之间的锁互相隔离。

...

struct resource {
	struct list_head list;		/* lockspace.resources */
	char name[MAX_NAME+1];		/* vg name or lv name */
	int8_t type;			/* resource type LD_RT_ */
	int8_t mode;
	unsigned int sh_count;		/* number of sh locks on locks list */
	uint32_t version;
	uint32_t last_client_id;	/* last client_id to lock or unlock resource */
	unsigned int lm_init : 1;	/* lm_data is initialized */
	unsigned int adopt : 1;		/* temp flag in remove_inactive_lvs */
	unsigned int version_zero_valid : 1;
	unsigned int use_vb : 1;
	struct list_head locks;
	struct list_head actions;
	char lv_args[MAX_ARGS+1];
	char lm_data[0];		/* lock manager specific data */
};

#define LD_LF_PERSISTENT 0x00000001

struct lock {
	struct list_head list;		/* resource.locks */
	int8_t mode;			/* lock mode LD_LK_ */
	uint32_t version;
	uint32_t flags;			/* LD_LF_ */
	uint32_t client_id; /* may be 0 for persistent or internal locks */
};

struct lockspace {
	struct list_head list;		/* lockspaces */
	char name[MAX_NAME+1];
	char vg_name[MAX_NAME+1];
	char vg_uuid[64];
	char vg_args[MAX_ARGS+1];	/* lock manager specific args */
	char vg_sysid[MAX_NAME+1];
	int8_t lm_type;			/* lock manager: LM_DLM, LM_SANLOCK */
	void *lm_data;
	uint64_t host_id;
	uint64_t free_lock_offset;	/* start search for free lock here */

	uint32_t start_client_id;	/* client_id that started the lockspace */
	pthread_t thread;		/* makes synchronous lock requests */
	pthread_cond_t cond;
	pthread_mutex_t mutex;
	unsigned int create_fail : 1;
	unsigned int create_done : 1;
	unsigned int thread_work : 1;
	unsigned int thread_stop : 1;
	unsigned int thread_done : 1;
	unsigned int sanlock_gl_enabled: 1;
	unsigned int sanlock_gl_dup: 1;
	unsigned int free_vg: 1;
	unsigned int kill_vg: 1;
	unsigned int drop_vg: 1;

	struct list_head actions;	/* new client actions */
	struct list_head resources;	/* resource/lock state for gl/vg/lv */
};

...

服务模型

  1. 主线程pool本地套接字,当有新客户端连接时,构造client结构、创建新线程client线程。
  2. 当client线程收到一个请求时转化成一个action,如果对应的lockspace不存在则创建lockspace线程,并把这个action转发给对应的lockspace线程。
  3. lockspace线程执行完任务后返回给client线程,client线程返回给客户端。
$ cat daemons/lvmlockd/lvmlockd-core.c
...

/*
 * Basic operation of lvmlockd
 *
 * lvmlockd main process runs main_loop() which uses poll().
 * poll listens for new connections from lvm commands and for
 * messages from existing connected lvm commands.
 *
 * lvm command starts and connects to lvmlockd.
 *
 * lvmlockd receives a connection request from command and adds a
 * 'struct client' to keep track of the connection to the command.
 * The client's fd is added to the set of fd's in poll().
 *
 * lvm command sends a lock request to lvmlockd.  The lock request
 * can be for the global lock, a vg lock, or an lv lock.
 *
 * lvmlockd main_loop/poll sees a message from an existing client.
 * It sets client.recv = 1, then wakes up client_thread_main.
 *
 * client_thread_main iterates through client structs (cl), looking
 * for any that need processing, finds the one with cl->recv set,
 * and calls client_recv_action(cl).
 *
 * client_recv_action(cl) reads the message/request from the client,
 * allocates a new 'struct action' (act) to represent the request,
 * sets the act with what is found in the request, then looks at
 * the specific operation in act->op (LD_OP_FOO) to decide what to
 * do with the action:
 *
 * . If the action is to start a lockspace, create a new thread
 *   to manage that lockspace: add_lockspace(act).
 *
 * . If the action is a lock request, pass the act to the thread
 *   that is managing that lockspace: add_lock_action(act).
 *
 * . Other misc actions are are passed to the worker_thread:
 *   add_work_action(act).
 *
 * Onec the client_thread has passed the action off to another
 * thread to process, it goes back to waiting for more client
 * handling work to do.
 *
 * The thread that was given the action by the client_thread
 * now processes that action according to the operation, act->op.
 * This is either a lockspace_thread (for lock ops or ops that
 * add/rem a lockspace), or the worker_thread.  See below for
 * how these ops are processed by these threads.  When the
 * given thread is done processing the action, the result is
 * set in act->result, and the act struct for the completed action
 * is passed back to the client_thread (client_results list).
 *
 * The client_thread takes completed actions (from client_results
 * list), and sends the result back to the client that sent the
 * request represented by the action.  The act struct is then freed.
 *
 * This completes the cycle of work between lvm commands (clients)
 * and lvmlockd.  In summary:
 *
 * - main process polls for new client connections and new requests
 *   from lvm commands
 * - client_thread reads requests from clients
 * - client_thread creates an action struct for each request
 * - client_thread passes the act to another thread for processing
 * - other threads pass completed act structs back to client_thread
 * - client_thread sends the act result back to the client and frees the act
 *
 *
 * Lockspace threads:
 * Each lockd VG has its own lockspace that contains locks for that VG.
 * Each 'struct lockspace' is managed by a separate lockspace_thread.
 * When the lockspace_thread is first created, the first thing it does
 * is join the lockspace in the lock manager.  This can take a long time.
 * If the join fails, the thread exits.  After the join, the thread
 * enters a loop waiting for lock actions to perform in the lockspace.
 *
 * The request to remove/leave a lockspace causes a flag to be set in
 * the lockspace struct.  When the lockspace_thread sees this flag
 * set, it leaves the lockspace, and exits.
 *
 * When the client_thread passes a new action to a lockspace_thread,
 * i.e. a new lock request, the lockspace_thread identifies which resource
 * is being locked (GL, VG, LV), and gets the 'struct resource' (r) for it.
 * r->type will be LD_RT_GL, LD_RT_VG, or LD_RT_LV.  r->name is the
 * resource name, and is fixed for GL and VG resources, but is based on
 * the LV name for LV resources.  The act is added to the resource's
 * list of actions: r->actions, i.e. outstanding lock requests on the
 * resource.
 *
 * The lockspace thread then iterates through each resource in the
 * lockspace, processing any outstanding actions on each: res_process(ls, r).
 *
 * res_process() compares the outstanding actions/requests in r->actions
 * against any existing locks on the resource in r->locks.  If the
 * action is blocked by existing locks, it's left on r->actions.  If not,
 * the action/request is passed to the lock manager.  If the result from
 * the lock manager is success, a new 'struct lock' is created for the
 * action and saved on r->locks.  The result is set in act->result and
 * the act is passed back to the client_thread to be returned to the client.
 */

...

最终处理lockspace线程处理锁空间action的函数如下:

...

/*
 * Go through queued actions, and make lock/unlock calls on the resource
 * based on the actions and the existing lock state.
 *
 * All lock operations sent to the lock manager are non-blocking.
 * This is because sanlock does not support lock queueing.
 * Eventually we could enhance this to take advantage of lock
 * queueing when available (i.e. for the dlm).
 *
 * act_close_list: list of CLOSE actions, identifying clients that have
 * closed/terminated their lvmlockd connection, and whose locks should
 * be released.  Do not remove these actions from act_close_list.
 *
 * retry_out: set to 1 if the lock manager said we should retry,
 * meaning we should call res_process() again in a short while to retry.
 */

static void res_process(struct lockspace *ls, struct resource *r,
			struct list_head *act_close_list, int *retry_out)
{
	struct action *act, *safe, *act_close;
	struct lock *lk;
	int lm_retry;
	int rv;

	/*
	 * handle version updates for ex locks
	 * (new version will be written by unlock)
	 */

	list_for_each_entry_safe(act, safe, &r->actions, list) {
		if (act->op == LD_OP_UPDATE) {
			rv = res_update(ls, r, act);
			act->result = rv;
			list_del(&act->list);
			add_client_result(act);
		}
	}

	/*
	 * handle explicit unlock actions
	 */

	list_for_each_entry_safe(act, safe, &r->actions, list) {
		if ((act->op == LD_OP_LOCK) &&
		    (act->mode == LD_LK_IV || act->mode == LD_LK_NL)) {
			act->result = -EINVAL;
			list_del(&act->list);
			add_client_result(act);
		}

		if (act->op == LD_OP_LOCK && act->mode == LD_LK_UN) {
			rv = res_unlock(ls, r, act);

			if (rv == -ENOENT && (act->flags & LD_AF_UNLOCK_CANCEL))
				rv = res_cancel(ls, r, act);

			/*
			 * possible unlock results:
			 * 0: unlock succeeded
			 * -ECANCELED: cancel succeeded
			 * -ENOENT: nothing to unlock or cancel
			 */

			act->result = rv;
			list_del(&act->list);
			add_client_result(act);
		}
	}

	/*
	 * handle implicit unlocks due to client exit,
	 * also clear any outstanding actions for the client
	 */

	list_for_each_entry(act_close, act_close_list, list) {
		res_unlock(ls, r, act_close);
		res_cancel(ls, r, act_close);
	}

	/*
	 * handle freeing a lock for an lv that has been removed
	 */

	list_for_each_entry_safe(act, safe, &r->actions, list) {
		if (act->op == LD_OP_FREE && act->rt == LD_RT_LV) {
			log_debug("S %s R %s free_lv", ls->name, r->name);
			rv = free_lv(ls, r);
			act->result = rv;
			list_del(&act->list);
			add_client_result(act);
			goto r_free;

		}
	}

	/*
	 * handle enable/disable
	 */

	list_for_each_entry_safe(act, safe, &r->actions, list) {
		if (act->op == LD_OP_ENABLE || act->op == LD_OP_DISABLE) {
			rv = res_able(ls, r, act);
			act->result = rv;
			list_del(&act->list);
			add_client_result(act);

			if (!rv && act->op == LD_OP_DISABLE) {
				log_debug("S %s R %s free disabled", ls->name, r->name);
				goto r_free;
			}
		}
	}

	/*
	 * transient requests on existing transient locks
	 */

	list_for_each_entry_safe(act, safe, &r->actions, list) {
		if (act->flags & LD_AF_PERSISTENT)
			continue;

		lk = find_lock_client(r, act->client_id);
		if (!lk)
			continue;

		if (lk->mode != act->mode) {
			/* convert below */
			/*
			act->result = -EEXIST;
			list_del(&act->list);
			add_client_result(act);
			*/
			continue;
		} else {
			/* success */
			r->last_client_id = act->client_id;
			act->result = -EALREADY;
			list_del(&act->list);
			add_client_result(act);
		}
	}

	/*
	 * persistent requests on existing persistent locks
	 *
	 * persistent locks are not owned by a client, so any
	 * existing with matching mode satisfies a request.
	 * only one persistent lock is kept on a resource.
	 * a single "unowned" persistent lock satisfies
	 * any/multiple client requests for a persistent lock.
	 */

	list_for_each_entry_safe(act, safe, &r->actions, list) {
		if (!(act->flags & LD_AF_PERSISTENT))
			continue;

		lk = find_lock_persistent(r);
		if (!lk)
			continue;

		if (lk->mode != act->mode) {
			/* convert below */
			/*
			act->result = -EEXIST;
			list_del(&act->list);
			add_client_result(act);
			*/
			continue;
		} else {
			/* success */
			r->last_client_id = act->client_id;
			act->result = -EALREADY;
			list_del(&act->list);
			add_client_result(act);
		}
	}

	/*
	 * transient requests with existing persistent locks
	 *
	 * Just grant the transient request and do not
	 * keep a record of it.  Assume that the persistent
	 * lock will not go away while the transient lock
	 * is needed.
	 *
	 * This would be used when an ex, persistent lv lock
	 * exists from activation, and then something like
	 * lvextend asks for a transient ex lock to change
	 * the lv.  The lv could not be unlocked by deactivation
	 * while the lvextend was running.
	 *
	 * The logic here for mixing T/P locks is not general
	 * support; there are a number of cases where it will
	 * not work: updating version number (lv locks have
	 * none), ex locks from multiple clients will not
	 * conflict, explicit un of the transient lock will fail.
	 */

	list_for_each_entry_safe(act, safe, &r->actions, list) {
		if (act->flags & LD_AF_PERSISTENT)
			continue;

		lk = find_lock_persistent(r);
		if (!lk)
			continue;

		if ((lk->mode == LD_LK_EX) ||
		    (lk->mode == LD_LK_SH && act->mode == LD_LK_SH)) {
			r->last_client_id = act->client_id;
			act->result = 0;
			list_del(&act->list);
			add_client_result(act);
		} else {
			/* persistent lock is sh, transient request is ex */
			/* FIXME: can we remove this case? do a convert here? */
			log_debug("res_process %s existing persistent lock new transient", r->name);
			r->last_client_id = act->client_id;
			act->result = -EEXIST;
			list_del(&act->list);
			add_client_result(act);
		}
	}

	/*
	 * persistent requests with existing transient locks
	 *
	 * If a client requests a P (persistent) lock for a T (transient)
	 * lock it already holds, we can just change T to P.  Fail if the
	 * same happens for locks from different clients.  Changing
	 * another client's lock from T to P may cause problems
	 * if that client tries to unlock or update version.
	 *
	 * I don't think this P/T combination will be used.
	 * It might be used if a command was able to take a P
	 * vg lock, in which case the T vg lock would already
	 * be held for reading.  If the T lock was sh, it would
	 * be converted to P ex.  If the T/P modes matched, the
	 * lock could just be changed from T to P.
	 */

	list_for_each_entry_safe(act, safe, &r->actions, list) {
		if (!(act->flags & LD_AF_PERSISTENT))
			continue;

		lk = find_lock_client(r, act->client_id);
		if (!lk)
			continue;

		if (lk->mode != act->mode) {
			/* FIXME: convert and change to persistent? */
			log_debug("res_process %s existing transient lock new persistent", r->name);
			r->last_client_id = act->client_id;
			act->result = -EEXIST;
			list_del(&act->list);
			add_client_result(act);
		} else {
			r->last_client_id = act->client_id;
			lk->flags |= LD_LF_PERSISTENT;
			lk->client_id = 0;
			act->result = 0;
			list_del(&act->list);
			add_client_result(act);
		}
	}

	/*
	 * convert mode of existing locks
	 */

	list_for_each_entry_safe(act, safe, &r->actions, list) {
		if (act->flags & LD_AF_PERSISTENT)
			lk = find_lock_persistent(r);
		else
			lk = find_lock_client(r, act->client_id);
		if (!lk)
			continue;

		if (lk->mode == act->mode) {
			/* should never happen, should be found above */
			log_error("convert same mode");
			continue;
		}

		/* convert fails immediately, no EAGAIN retry */
		rv = res_convert(ls, r, lk, act);
		act->result = rv;
		list_del(&act->list);
		add_client_result(act);
	}

	/*
	 * Cases above are all requests addressed by existing locks.
	 * Below handles the rest.  Transient and persistent are
	 * handled the same, except
	 * - if mode of existing lock is incompat with requested,
	 *   leave the act on r->actions
	 * - if r mode is EX, any lock action is blocked, just quit
	 *
	 * Retry a lock request that fails due to a lock conflict (-EAGAIN):
	 * if we have not exceeded max retries and lm sets lm_retry (sanlock
	 * transient conflicts from shared lock implementation), or r type
	 * is gl or vg (transient real conflicts we want to hide from command).
	 * lv lock conflicts won't be transient so don't retry them.
	 */

	if (r->mode == LD_LK_EX)
		return;

	/*
	 * r mode is SH or UN, pass lock-sh actions to lm
	 */

	list_for_each_entry_safe(act, safe, &r->actions, list) {
		/* grant in order, so break here */
		if (act->op == LD_OP_LOCK && act->mode == LD_LK_EX)
			break;

		if (act->op == LD_OP_LOCK && act->mode == LD_LK_SH) {
			lm_retry = 0;

			rv = res_lock(ls, r, act, &lm_retry);
			if ((rv == -EAGAIN) &&
			    (act->retries <= act->max_retries) &&
			    (lm_retry || (r->type != LD_RT_LV))) {
				/* leave act on list */
				log_debug("S %s R %s res_lock EAGAIN retry", ls->name, r->name);
				act->retries++;
				*retry_out = 1;
			} else {
				act->result = rv;
				list_del(&act->list);
				add_client_result(act);
			}
			if (rv == -EUNATCH)
				goto r_free;
		}
	}

	/*
	 * r mode is SH, any ex lock action is blocked, just quit
	 */

	if (r->mode == LD_LK_SH)
		return;

	/*
	 * r mode is UN, pass lock-ex action to lm
	 */

	list_for_each_entry_safe(act, safe, &r->actions, list) {
		if (act->op == LD_OP_LOCK && act->mode == LD_LK_EX) {
			lm_retry = 0;

			rv = res_lock(ls, r, act, &lm_retry);
			if ((rv == -EAGAIN) &&
			    (act->retries <= act->max_retries) &&
			    (lm_retry || (r->type != LD_RT_LV))) {
				/* leave act on list */
				log_debug("S %s R %s res_lock EAGAIN retry", ls->name, r->name);
				act->retries++;
				*retry_out = 1;
			} else {
				act->result = rv;
				list_del(&act->list);
				add_client_result(act);
			}
			if (rv == -EUNATCH)
				goto r_free;
			break;
		}
	}

	return;

r_free:
	/* For the EUNATCH case it may be possible there are queued actions? */
	list_for_each_entry_safe(act, safe, &r->actions, list) {
		log_error("S %s R %s res_process r_free cancel %s client %d",
			  ls->name, r->name, op_str(act->op), act->client_id);
		act->result = -ECANCELED;
		list_del(&act->list);
		add_client_result(act);
	}
	log_debug("S %s R %s res_process free", ls->name, r->name);
	lm_rem_resource(ls, r);
	list_del(&r->list);
	free_resource(r);
}

...
标签: LVM lvmlockd
共有 人打赏支持
粉丝 18
博文 29
码字总数 100790
×
LastRitter
如果觉得我的文章对您有用,请随意打赏。您的支持将鼓励我继续创作!
* 金额(元)
¥1 ¥5 ¥10 ¥20 其他金额
打赏人
留言
* 支付类型
微信扫码支付
打赏金额:
已支付成功
打赏金额: