Postgres-xl GTM（全局事务管理器 Globale Transaction Manager）GTM Master & Standby Failover

冷不防 2022-10-05 06:58 155阅读 0赞

![在这里插入图片描述][resize_m_lfit_w_962_pic_center]

## GTM slave 提升 ##

当GTM master宕机之后，但是GTM slave还在运行，可以使用pgxc\_ctl提升GTM slave（postgres-xc）。  
如果没有配置GTM Proxy：pgxc\_ctl没有提供提升功能，需手动处理。在gtm slave运行gtm\_ctl，编辑postgres.conf使其能够连接到gtm master，重启所有的协调节点和数据节点。  
如果配置GTM Proxy：如果GTM master挂了，事务正在运行，则事务会abort。第一步提升，failver gtm。将所有gtm\_proxy重新连接到新GTM master，reconnect gtm\_proxy all，该步骤会在gtm slave运行gtm\_ctl promote，配置被提升的gtm的gtm.conf，以使其以master角色运行。

PGXC$ failover gtm
    PGXC$ reconnect gtm_proxy all

pgxc\_Concept\_Implementation\_and\_Achievements Chapter 24 Node failover

### gtm\_ctl ###

gtm\_ctl工具处于gtm\_ctl.c中，如下为其支持的选项，用于启停（start、stop、restart）GTM server、GTM standby和GTM proxy：

// Common options
    -D GTMDATA  //location of the database storage area
    -i nodename //set gtm_proxy nodename registered on GTM (option ignored if used with GTM)
    -t SECS     //seconds to wait when using -w option
    -w          //wait until operation completes
    -W          //do not wait until operation completes
    //Options for start or restart
    -l log_file            //write (or append) server log to FILENAME
    -o OPTIONS             //command line options to pass to gtm(GTM server executable)
    -p PATH-TO-GTM/PROXY   //path to gtm/gtm_proxy executables
    -Z STARTUP-MODE        //can be \"gtm\", \"gtm_standby\" or \"gtm_proxy\
    //Options for stop or restart
    -m SHUTDOWN-MODE //smart/fast/immediate/s/f/i
    //Options for reconnect
    -t NewGTMPORT          //Port number of new GTM
    -s NewGTMHost          //Host Name of new GTM

gtm\_ctl支持如下功能：启动、停止、提升、重启、状态、重连接。

gtm_ctl start   -Z STARTUP_MODE [-w] [-t SECS] [-D DATADIR] [-l FILENAME] [-o \"OPTIONS\"]       --> do_start()->start_gtm()
    gtm_ctl stop    -Z STARTUP_MODE [-W] [-t SECS] [-D DATADIR] [-m SHUTDOWN-MODE]                   --> do_stop()函数->kill()
    gtm_ctl promote -Z STARTUP_MODE [-w] [-t SECS] [-D DATADIR]                                      --> do_promote()函数->kill((pid_t) pid, SIGUSR1)
    gtm_ctl restart -Z STARTUP_MODE [-w] [-t SECS] [-D DATADIR] [-m SHUTDOWN-MODE] [-o \"OPTIONS\"]  --> do_restart()函数
    gtm_ctl status  -Z STARTUP_MODE [-w] [-t SECS] [-D DATADIR]                                      --> do_status()函数
    gtm_ctl reconnect -Z STARTUP_MODE [-D DATADIR] -o \"OPTIONS\"]                                   --> do_reconnect()函数

<table> 
 <thead> 
  <tr> 
   <th></th> 
   <th>gtm</th> 
   <th>gtm_proxy</th> 
   <th>gtm_standby</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td>pid_file</td> 
   <td>$GTMDATA/gtm.pid</td> 
   <td>$GTMDATA/gtm_proxy.pid</td> 
   <td>$GTMDATA/gtm.pid</td> 
  </tr> 
  <tr> 
   <td>gtmopts_file</td> 
   <td>$GTMDATA/gtm.opts</td> 
   <td>$GTMDATA/gtm_proxy.opts</td> 
   <td>$GTMDATA/gtm.opts</td> 
  </tr> 
  <tr> 
   <td>conf_file</td> 
   <td>$GTMDATA/gtm.conf</td> 
   <td>$GTMDATA/gtm_proxy.conf</td> 
   <td>$GTMDATA/gtm.conf</td> 
  </tr> 
 </tbody> 
</table>

start\_gtm函数主要用于start/test/stop，如果指定了log\_file则调用RunAsDaemon函数，否则直接调用system函数执行命令。

static int start_gtm(void) { 
    	char		cmd[MAXPGPATH];
    	char		gtm_app_path[MAXPGPATH];
    	int			len;
    	/* Since there might be quotes to handle here, it is easier simply to pass everything to a shell to process them */
    	memset(gtm_app_path, 0, MAXPGPATH);
    	memset(cmd, 0, MAXPGPATH);
    	/* Build gtm binary path. We should leave one byte at the end for '\0' */
    	len = 0;
    	if (gtm_path != NULL) { 
    		strncpy(gtm_app_path, gtm_path, MAXPGPATH - len - 1);
    		len = strlen(gtm_app_path);
    		strncat(gtm_app_path, "/", MAXPGPATH - len - 1);
    		len = strlen(gtm_app_path);
    	}
    	if (strlen(gtm_app) >= (MAXPGPATH - len - 1)){ 
    		write_stderr("gtm command exceeds max size");exit(1);
    	}
    	strncat(gtm_app_path, gtm_app, MAXPGPATH - len - 1);
    
    	if (log_file != NULL)
    		len = snprintf(cmd, MAXPGPATH - 1, "\"%s\" %s%s -l %s &" ,gtm_app_path, gtmdata_opt, gtm_opts, log_file);
    	else
    		len = snprintf(cmd, MAXPGPATH - 1, "\"%s\" %s%s < \"%s\" 2>&1 &" ,gtm_app_path, gtmdata_opt, gtm_opts, DEVNULL);
    	if (len >= MAXPGPATH - 1){ 
    		write_stderr("gtm command exceeds max size");exit(1);
    	}
    	if (log_file)
    		return (RunAsDaemon(cmd));
    	else
    		return system(cmd);
    }

只有gtm\_proxy能够执行reconnect命令，用于连接到新提升的GTM Master。通过将命令行参数写入$GTMDATA/newgtm，并向进程发送SIGUSER1信号进行提升（kill((pid\_t) pid, SIGUSR1)）。

static void do_reconnect(void) { 
    	pgpid_t	pid;
    	char *reconnect_point_file_nam;
    	FILE *reconnect_point_file;
    	/* Target must be "gtm_proxy" */
    	if (strcmp(gtm_app, "gtm_proxy") != 0){ 
    		write_stderr(_("%s: only gtm_proxy can accept reconnect command\n"), progname);exit(1);
    	}
    	pid = get_pgpid();
    	if (pid == 0)/* no pid file */{ 
    		write_stderr(_("%s: PID file \"%s\" does not exist\n"), progname, pid_file);write_stderr(_("Is server running?\n"));exit(1);
    	}else if (pid < 0)/* standalone backend, not gtm */{ 
    		pid = -pid;
    		write_stderr(_("%s: cannot promote server; single-user server is running (PID: %ld)\n"), progname, pid);
    		exit(1);
    	}
    	read_gtm_opts();
    	/* Pass reconnect info to GTM-Proxy. Option arguments are written to new gtm file under -D directory. */
    	reconnect_point_file_nam = malloc(strlen(gtm_data) + 9);
    	if (reconnect_point_file_nam == NULL){ 
    		write_stderr(_("%s: No memory available.\n"), progname);exit(1);
    	}
    	snprintf(reconnect_point_file_nam, strlen(gtm_data) + 8, "%s/newgtm", gtm_data);
    	reconnect_point_file = fopen(reconnect_point_file_nam, "w");
    	if (reconnect_point_file == NULL){ 
    		write_stderr(_("%s: Cannot open reconnect point file %s\n"), progname, reconnect_point_file_nam);
    		exit(1);
    	}
    	fprintf(reconnect_point_file, "%s\n", gtm_opts);
    	fclose(reconnect_point_file);
    	free(reconnect_point_file_nam);
    	if (kill((pid_t) pid, SIGUSR1) != 0){ 
    		write_stderr(_("%s: could not send promote signal (PID: %ld): %s\n"), progname, pid, strerror(errno));
    		exit(1);
    	}
    }

GTM对SIGUSR1信号的处理：`pqsignal(SIGUSR1, GTM_SigleHandler);`

static void GTM_SigleHandler(int signal) { 
    	fprintf(stderr, "Received signal %d\n", signal);
    	switch (signal){ 
    		case SIGKILL:
    		case SIGTERM:
    		case SIGQUIT:
    		case SIGINT:
    		case SIGHUP:
    			break;
    		case SIGUSR1:
    			if (Recovery_IsStandby())
    				PromoteToActive();
    			return;
    		default:
    			fprintf(stderr, "Unknown signal %d\n", signal);
    			return;
    	}
    	/* XXX We should do a clean shutdown here. */
    	/* Rewrite Register Information (clean up unregister records) */
    	Recovery_SaveRegisterInfo();
    	/* Delete pid file before shutting down */
    	DeleteLockFile(GTM_PID_FILE);
    	PG_SETMASK(&BlockSig);
    	GTMAbortPending = true;
    	return;
    }

对于SIGUSR1信号，如果是GTM Slave，则需要调用PromoteToActive()函数，否则不做操作。最后执行关闭前清理函数。

static void PromoteToActive(void) { 
    	const char *conf_file;
    	FILE	   *fp;
    	elog(LOG, "Promote signal received. Becoming an active...");
    	/* Set starting and next client idendifier before promotion is complete */
    	GTM_SetInitialAndNextClientIdentifierAtPromote();
    	/* Do promoting things here. */
    	Recovery_StandbySetStandby(false);
    	CreateDataDirLockFile();
    	/* Update the GTM config file for the next restart.. */
    	conf_file = GetConfigOption("config_file", true);
    	elog(LOG, "Config file is %s...", conf_file);
    	if ((fp = fopen(conf_file, PG_BINARY_A)) == NULL){ 
    		ereport(FATAL,(EINVAL,errmsg("could not open GTM configuration file \"%s\": %m",conf_file)));
    	}else{ 
    		time_t		stamp_time = (time_t) time(NULL);
    		char		strfbuf[128];
    		strftime(strfbuf, sizeof(strfbuf),"%Y-%m-%d %H:%M:%S %Z",localtime(&stamp_time));
    		fprintf(fp, 
    				"#===================================================\n"
    				"# Updated due to GTM promote request\n"
    				"# %s\nstartup = ACT\n"
    				"#===================================================\n", strfbuf);
    		if (fclose(fp))
    			ereport(FATAL,(EINVAL,errmsg("could not close GTM configuration file \"%s\": %m",conf_file)));
    	}
    	GTM_SetNeedBackup();
    	GTM_WriteRestorePoint();
    	return;
    }

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2FzbWFydGtpbGxlcg_size_16_color_FFFFFF_t_70]

## GTM Standby启动 ##

GTM Standby启动流程：  
main()  
–> InitializeGTMOptions() --> InitGTMProcess() --> Parse command line options and load configuration file --> BaseInit()  
–Recovery\_IsStandby()–> Recovery\_StandbySetConnInfo(active\_addr, active\_port) –Recovery\_IsStandby()–> gtm\_standby\_start\_startup() –Recovery\_IsStandby()–> gtm\_standby\_begin\_backup() --> gtm\_standby\_restore\_next\_gxid() --> gtm\_standby\_restore\_gxid() --> gtm\_standby\_restore\_sequence()  
–> GTM\_SetNeedBackup() --> GTM\_WriteRestorePoint()  
–Recovery\_IsStandby()–> gtm\_standby\_register\_self(NodeName, GTMPortNumber, GTMDataDir) –Recovery\_IsStandby()–> gtm\_standby\_restore\_node()  
–> Establish input sockets --> Setup signal handlers  
–Recovery\_IsStandby()–> gtm\_standby\_activate\_self() --> gtm\_standby\_end\_backup() --> gtm\_standby\_finish\_startup()  
–> ServerLoop()  
GTM Standby和Master在启动过程中的交互过程：

GTM Standby连接Master：GTM Standby --gtm_standby_connectToActiveGTM 连接--> GTM Master
    GTM Standby向GTM Master发送开始备份命令：GTM Standby --set_begin_end_backup(GTM_ActiveConn, true)--> GTM Master
    GTM Standby向GTM Master发送获取next全局事务号命令：GTM Standby --get_next_gxid(GTM_ActiveConn)--> GTM Master
    GTM Standby恢复TxnInfo：GTM_RestoreTxnInfo(NULL, next_gxid, NULL, true)
    GTM Standby从GTM Master恢复gxid：GTM Standby <--gtm_standby_restore_gxid-- GTM Master
    GTM Standby从GTM Master恢复sequence：GTM Standby <--gtm_standby_restore_sequence-- GTM Master
    GTM Standby备份恢复点：GTM_WriteRestorePoint项control file中写入PointVersion、PointXid和PointSeq
    GTM Standby向GTM Master将自己注册为disconnected节点：GTM Standby --gtm_standby_register_self--> GTM Master
    GTM Standby从GTM Master上获取Node节点信息：GTM Standby <--gtm_standby_restore_node-- GTM Master
    GTM Standby将standby状态从disconnected更新为connected：GTM Standby --gtm_standby_activate_self--> GTM Master
    GTM Standby向GTM Master发送结束备份命令：GTM Standby --gtm_standby_end_backup--> GTM Master

### GTM Standby连接master ###

GTM Standby连接Master，利用Recovery\_StandbySetConnInfo(active\_addr, active\_port) 函数设置master的GTM\_ActiveAddress和GTM\_ActivePort，利用gtm\_standby\_start\_startup()建立stanby和master之间的连接（利用PQconnectGTM函数）。

## GTM Standby复制同步Master ##

gtm/recovery/replication.c

GTM Standby向GTM Master发送开始备份命令（利用gtm\_standby\_begin\_backup函数），

int gtm_standby_begin_backup(void) {  int rc = set_begin_end_backup(GTM_ActiveConn, true); return (rc ? 0 : 1); }
    int set_begin_end_backup(GTM_Conn *conn, bool begin) { 
    	GTM_Result *res = NULL;
    	time_t finish_time;
    	if (gtmpqPutMsgStart('C', true, conn)) {  goto send_failed; }
    	if(gtmpqPutInt(begin ? MSG_BEGIN_BACKUP : MSG_END_BACKUP, sizeof(GTM_MessageType), conn)) {  goto send_failed; }
    
    	if (gtmpqPutMsgEnd(conn))
    		goto send_failed;
    
    	if (gtmpqFlush(conn))
    		goto send_failed;
    
    	finish_time = time(NULL) + CLIENT_GTM_TIMEOUT;
    	if (gtmpqWaitTimed(true, false, conn, finish_time) ||
    		gtmpqReadData(conn) < 0)
    		goto receive_failed;
    
    	if ((res = GTMPQgetResult(conn)) == NULL)
    		goto receive_failed;
    
    	return res->gr_status;
    
    receive_failed:
    send_failed:
    	conn->result = makeEmptyResultIfIsNull(conn->result);
    	conn->result->gr_status = GTM_RESULT_COMM_ERROR;
    	return -1;
    }

### 节点注册信息同步 ###

### 子进程信息结构体同步 ###

### Txn/Seq同步 ###

## GTM Master Standby 切换 ##

### GTM Master检测Standby状态 ###

从GTM Standby的初始化过程，可以看出GTM Master和Standby之间只有一个连接。那么如果检查GTM Standby是否更新呢？GTM Master产生的子Threads中对连接Standby的处理，可以看到如果全局标志GTMThreads->gt\_standby\_ready为True，则说明需要连接Standby，如果子线程信息结构体中的Standby没有相应的信息（子线程创建时的线程信息结构体Standby字段会被赋值），说明目前master和standby未连接，需要重新尝试连接，并确定standby是否真连接不上。如果全局标志GTMThreads->gt\_standby\_ready为fasle，则需要将GTM Master和Standby之间断链。

for (;;) { 
            ...
    		if (GTMThreads->gt_standby_ready && thrinfo->thr_conn->standby == NULL && thrinfo->thr_status != GTM_THREAD_BACKUP){ 
    			/* Connect to GTM-Standby */
    			thrinfo->thr_conn->standby = gtm_standby_connect_to_standby();
    			if (thrinfo->thr_conn->standby == NULL)
    				GTMThreads->gt_standby_ready = false;	/* This will make other threads to disconnect from the standby, if needed.*/
    		}else if (GTMThreads->gt_standby_ready == false && thrinfo->thr_conn->standby){ 
    			/* Disconnect from GTM-Standby */
    			gtm_standby_disconnect_from_standby(thrinfo->thr_conn->standby);
    			thrinfo->thr_conn->standby = NULL;
    		}

### GTM Standby角色切换Master ###

Standby切换Master的过程如下：收到SIGUSR1信号，GTM Standby调用信号处理函数，执行PromoteToActive函数，调用GTM\_SetInitialAndNextClientIdentifierAtPromote()函数，调用Recovery\_StandbySetStandby(false)函数，创建数据目录锁，更新GTM配置文件，调用GTM\_SetNeedBackup()函数，调用GTM\_WriteRestorePoint()函数。

GTM\_SetInitialAndNextClientIdentifierAtPromote()函数会将全局线程信息GTMThreads中的gt\_starting\_client\_id更新为open transactions列表中最新的客户标识符，将gt\_next\_client\_id更新为下一个客户标识符。  
Recovery\_StandbySetStandby(false)函数**将全局standby标识设置为fasle，也就是标识自己为master**。  
GTM\_SetNeedBackup()函数将需要backup标识设置为True。  
GTM\_WriteRestorePoint()函数向log文件写入恢复点（GTM\_WriteRestorePointVersion(f)、GTM\_WriteRestorePointXid(f)、GTM\_WriteRestorePointSeq(f);）

void GTM_WriteRestorePoint(void){ 
    	FILE *f = fopen(GTMControlFile, "w");
    	if (f == NULL){ 
    		ereport(LOG, (errno,errmsg("Cannot open control file"),errhint("%s", strerror(errno))));
    		return;
    	}
    	GTM_RWLockAcquire(&gtm_bkup_lock, GTM_LOCKMODE_WRITE);
    	if (!gtm_need_bkup){ 
    		GTM_RWLockRelease(&gtm_bkup_lock);
    		fclose(f);
    		return;
    	}
    	gtm_need_bkup = FALSE;
    	GTM_RWLockRelease(&gtm_bkup_lock);
    	GTM_WriteRestorePointVersion(f);
    	GTM_WriteRestorePointXid(f);
    	GTM_WriteRestorePointSeq(f);
    	fclose(f);
    }

### GTM Proxy重连接Standby ###

GTM Master创建的子线程的信息结构体中都会包含一个成员GTM\_CON \*standby以标识Standby的连接信息。每个子线程的信息结构体中同样有唯一的客户端ID。如果这些信息能够一直从Master复制给Standby，那么就能保证在Master挂掉后，Standby能够立刻能认识出连接进来的客户端。  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2FzbWFydGtpbGxlcg_size_16_color_FFFFFF_t_70 1]

[resize_m_lfit_w_962_pic_center]: /images/20221005/78b23d9ab2034b8bb5a476f814bb5f10.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2FzbWFydGtpbGxlcg_size_16_color_FFFFFF_t_70]: /images/20221005/bb4d611fd63c42f68a191d0da887bf02.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2FzbWFydGtpbGxlcg_size_16_color_FFFFFF_t_70 1]: /images/20221005/dfc2e246260e41389f28e7f4e2d4da35.png