Oralce集群管理-19C RAC 私有网络调整为BOND1

1 尝试在线添加私有网络的新接口 是否成功。

使用oifcfg命令在线添加新的网卡接口，在还没有配置bond1的条件下
也是可以添加成功的。

[grid@orcldb1 ~]$ oifcfg getif
eno3  192.168.224.0  global  public
ens3f0  10.2.0.0  global  cluster_interconnect,asm
eno3  4509:8002:5A06:0120:0010:0000:0002:D000  global  public
[grid@orcldb1 ~]$ oifcfg setif -global bond1/10.2.0.0:cluster_interconnect,asm
[grid@orcldb1 ~]$ oifcfg getif
eno3  192.168.224.0  global  public
ens3f0  10.2.0.0  global  cluster_interconnect,asm
eno3  4509:8002:5A06:0120:0010:0000:0002:D000  global  public
bond1  10.2.0.0  global  cluster_interconnect,asm

[grid@orcldb1 ~]$ oifcfg delif -global bond1/10.2.0.0
[grid@orcldb1 ~]$ oifcfg getif
eno3  192.168.224.0  global  public
ens3f0  10.2.0.0  global  cluster_interconnect,asm
eno3  4509:8002:5A06:0120:0010:0000:0002:D000  global  public
[grid@orcldb1 ~]$ 

2 OCR备份

[root@orcldb1 bin]# ./ocrconfig -showbackup
orcldb2     2023/06/29 06:53:23     +DATA:/orcldb/OCRBACKUP/backup00.ocr.272.1140764001     479021767
orcldb2     2023/06/29 02:53:20     +DATA:/orcldb/OCRBACKUP/backup01.ocr.258.1140749597     479021767
orcldb2     2023/06/28 22:53:15     +DATA:/orcldb/OCRBACKUP/backup02.ocr.274.1140735193     479021767
orcldb2     2023/06/27 06:51:06     +DATA:/orcldb/OCRBACKUP/day.ocr.262.1140591067     479021767
orcldb2     2023/06/17 18:58:03     +DATA:/orcldb/OCRBACKUP/week.ocr.260.1139770683     479021767
orcldb2     2023/03/27 10:33:33     +DATA:/orcldb/OCRBACKUP/backup_20230327_103333.ocr.273.1132569213     479021767     
orcldb2     2023/03/27 10:22:42     +DATA:/orcldb/OCRBACKUP/backup_20230327_102242.ocr.271.1132568563     479021767     
orcldb2     2023/03/24 14:09:26     +DATA:/orcldb/OCRBACKUP/backup_20230324_140926.ocr.259.1132322967     294213661     
orcldb1     2020/08/04 15:28:17     +DATA:/orcldb/OCRBACKUP/backup_20200804_152817.ocr.269.1047569297     294213661     
orcldb1     2020/05/13 13:17:36     +DATA:/orcldb/OCRBACKUP/backup_20200513_131736.ocr.261.1040303857     2798127435     
[root@orcldb1 bin]# ./ocrconfig -manualbackup
orcldb2     2023/06/29 10:21:21     +DATA:/orcldb/OCRBACKUP/backup_20230629_102121.ocr.263.1140776483     479021767     
orcldb2     2023/03/27 10:33:33     +DATA:/orcldb/OCRBACKUP/backup_20230327_103333.ocr.273.1132569213     479021767     
orcldb2     2023/03/27 10:22:42     +DATA:/orcldb/OCRBACKUP/backup_20230327_102242.ocr.271.1132568563     479021767     

orcldb2     2023/03/24 14:09:26     +DATA:/orcldb/OCRBACKUP/backup_20230324_140926.ocr.259.1132322967     294213661     
orcldb1     2020/08/04 15:28:17     +DATA:/orcldb/OCRBACKUP/backup_20200804_152817.ocr.269.1047569297     294213661     
[root@orcldb1 bin]# 

3  查看当前网络信息

[root@orcldb1 bin]# ./oifcfg getif
eno3  192.168.224.0  global  public
ens3f0  10.2.0.0  global  cluster_interconnect,asm
eno3  4509:8002:5A06:0120:0010:0000:0002:D000  global  public
[root@orcldb1 bin]# 
oifcfg setif -global bond1/10.2.0.0:cluster_interconnect,asm
[root@orcldb2 bin]# ./oifcfg getif
eno3  192.168.224.0  global  public
ens3f0  10.2.0.0  global  cluster_interconnect,asm
eno3  4509:8002:5A06:0120:0010:0000:0002:D000  global  public
bond1  10.2.0.0  global  cluster_interconnect,asm
[root@orcldb2 bin]# 

4 停止OGG复制进程

GGSCI (orcl_DB_01) 6> info all   ---之前关闭集群总是出现 ogg无法umount的error，将dpe传输进程停止之后，报错消失。

Program     Status      Group       Lag at Chkpt  Time Since Chkpt

MANAGER     RUNNING                                           
EXTRACT     RUNNING     DPEARS_A    00:00:02      00:00:05    
EXTRACT     RUNNING     DPEARS_B    00:00:00      00:00:07    
EXTRACT     STOPPED     DPERAC_A    00:00:02      00:00:01    
EXTRACT     RUNNING     EXTARS_A    00:00:00      00:00:01    
EXTRACT     RUNNING     EXTARS_B    00:00:02      00:00:02    
EXTRACT     RUNNING     EXTRAC_A    00:00:01      00:00:07    

GGSCI (orcl_DB_01) 7> 

5 关闭节点HAS

节点1节点2关闭has，
./crsctl stop has

6 配置私有网络BOND

---配置bond1 双网卡。
slave添加如下5行

USERCTL=no
MASTER=bond1
SLAVE=yes
NM_CONTROLLED=no
HOTPLUG=no

master配置

DEVICE=bond1
IPADDR=10.2.0.120
NETMASK=255.255.255.0
#GATEWAY=10.2.0.1
ONBOOT=yes
BOOTPROTO=none
USERCTL=no

BONDING_OPTS="mode=1 miimon=100"
NM_CONTROLLED=no
HOTPLUG=no

 7 重启网络服务

systemctl restart network

[root@orcldb1 ~]# ip addr

[orcl@orcldb1 ~]$ cat /proc/net/bonding/bond1
Ethernet Channel Bonding Driver: v3.7.1 (April 27, 2011)

Bonding Mode: fault-tolerance (active-backup)
Primary Slave: None
Currently Active Slave: ens3f0
MII Status: up
MII Polling Interval (ms): 100
Up Delay (ms): 0
Down Delay (ms): 0

Slave Interface: ens3f0
MII Status: up
Speed: 1000 Mbps
Duplex: full
Link Failure Count: 0
Permanent HW addr: 34:0a:98:9c:ed:45
Slave queue ID: 0

Slave Interface: ens3f1
MII Status: down
Speed: Unknown
Duplex: Unknown
Link Failure Count: 0
Permanent HW addr: 34:0a:98:9c:ed:46
Slave queue ID: 0
[orcl@orcldb1 ~]$ 

[root@orcldb1 network-scripts]# more ifcfg-bond1
DEVICE=bond1
IPADDR=10.2.0.120
NETMASK=255.255.255.0
#GATEWAY=10.2.0.1
ONBOOT=yes
BOOTPROTO=none
USERCTL=no

BONDING_OPTS="mode=1 miimon=100"
NM_CONTROLLED=no
HOTPLUG=no
[root@orcldb1 network-scripts]# more ifcfg-ens3f0
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=none
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens3f0
UUID=cc18f2de-d837-40d8-8445-a13c5da7a8cc
DEVICE=ens3f0
ONBOOT=yes
USERCTL=no
MASTER=bond1
SLAVE=yes
NM_CONTROLLED=no
HOTPLUG=no
[root@orcldb1 network-scripts]# more ifcfg-ens3f1
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=none
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens3f1
UUID=207bf934-ff44-4e05-aea7-bc330672369b
DEVICE=ens3f1
ONBOOT=yes
USERCTL=no
MASTER=bond1
SLAVE=yes
NM_CONTROLLED=no
HOTPLUG=no
[root@orcldb1 network-scripts]# 

节点2 按照上面操作 配置bond1即可。

8 双网卡测试 ：

9  启动集群 。

10  查看GIPC日志

2023-06-29 14:54:28.170 :GIPCDCLT:221996800:  gipcdClientInterfaceMetrics: Received type(gipcdmsgtypeInterfaceMetrics), endp(000000000000028f), len(1024), buf(0x7f1ff4359488), inf(ip: 10.2.0.120:43466, mask: 255.255.255.0, subnet: 10.2.0.0, mac: , ifname: bond1) time(0), retry(0), stamp(3), send(2), recv(3)
2023-06-29 14:54:28.170 :GIPCDCLT:221996800:  gipcdClientInterfaceMetrics: enqueue interface metrics (1:orcldb1:bond1) to worklist
2023-06-29 14:54:31.231 : CLSINET:217794304:  (:CLSINE0018:)WARNING: failed to find interface available for interface definition ens3f0(:.*)?:10.2.0.0
2023-06-29 14:54:33.222 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 0]  bond1    - rank  99, node orcldb2   , avgms 2.413793 [30/29/29]
2023-06-29 14:54:33.222 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 1]  bond1    - rank  99, node orcldb1   , avgms 2.413793 [30/29/29]
2023-06-29 14:54:35.335 : CLSINET:217794304:  (:CLSINE0018:)WARNING: failed to find interface available for interface definition ens3f0(:.*)?:10.2.0.0
2023-06-29 14:54:37.178 :GIPCHTHR:210466560:  gipchaWorkerWork: workerThread heart beat, time interval since last heartBeat 30020 loopCount 31 sendCount 0 recvCount 18 postCount 0 sendCmplCount 0 recvCmplCount 0
2023-06-29 14:54:38.328 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 0]  bond1    - rank  99, node orcldb2   , avgms 1.333333 [45/45/45]
2023-06-29 14:54:38.328 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 1]  bond1    - rank  99, node orcldb1   , avgms 1.333333 [45/45/45]
2023-06-29 14:54:39.441 : CLSINET:217794304:  (:CLSINE0018:)WARNING: failed to find interface available for interface definition ens3f0(:.*)?:10.2.0.0
2023-06-29 14:54:40.380 :GIPCHTHR:208365312:  gipchaDaemonWork: DaemonThread heart beat, time interval since last heartBeat 30030 loopCount 42
发现 原来的网卡信息报错，bond1 rank正常

11 删除旧的私网信息

[grid@orcldb1 ~]$ oifcfg delif -global ens3f0/10.2.0.0
[grid@orcldb1 ~]$ oifcfg getif
eno3  192.168.224.0  global  public
eno3  4509:8002:5A06:0120:0010:0000:0002:D000  global  public
bond1  10.2.0.0  global  cluster_interconnect,asm
[grid@orcldb1 ~]$ 

再次查看gipcd 的日志
grid@orcldb1 trace]$ tail -f gipcd.trc
2023-06-29 16:03:51.324 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 0]  bond1    - rank  99, node orcldb2   , avgms 2.173913 [23/23/23]
2023-06-29 16:03:51.324 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 1]  bond1    - rank  99, node orcldb1   , avgms 2.173913 [23/23/23]
2023-06-29 16:03:56.428 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 0]  bond1    - rank  99, node orcldb2   , avgms 0.400000 [23/25/25]
2023-06-29 16:03:56.428 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 1]  bond1    - rank  99, node orcldb1   , avgms 0.400000 [23/25/25]
2023-06-29 16:03:56.428 :GIPCDMON:217794304:  gipcdMonitorPublishDiags:  key gipc_round_trip_time handle 0x7f1fec392160 writeTime 12345264 value <orcldb2:58:12339264:192us>
2023-06-29 16:04:01.524 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 0]  bond1    - rank  99, node orcldb2   , avgms 0.384615 [23/26/26]
2023-06-29 16:04:01.524 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 1]  bond1    - rank  99, node orcldb1   , avgms 0.384615 [23/26/26]
2023-06-29 16:04:05.708 :GIPCHTHR:208365312:  gipchaDaemonWork: DaemonThread heart beat, time interval since last heartBeat 30030 loopCount 42
2023-06-29 16:04:07.631 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 0]  bond1    - rank  99, node orcldb2   , avgms 0.416667 [23/24/24]
2023-06-29 16:04:07.631 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 1]  bond1    - rank  99, node orcldb1   , avgms 0.416667 [23/24/24]
2023-06-29 16:04:12.737 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 0]  bond1    - rank  99, node orcldb2   , avgms 1.739130 [23/23/23]
2023-06-29 16:04:12.737 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 1]  bond1    - rank  99, node orcldb1   , avgms 1.739130 [23/23/23]
2023-06-29 16:04:16.321 :GIPCHTHR:210466560:  gipchaWorkerWork: workerThread heart beat, time interval since last heartBeat 30040 loopCount 48 sendCount 0 recvCount 18 postCount 0 sendCmplCount 0 recvCmplCount 0
2023-06-29 16:04:17.832 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 0]  bond1    - rank  99, node orcldb2   , avgms 1.304348 [25/23/23]
2023-06-29 16:04:17.832 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 1]  bond1    - rank  99, node orcldb1   , avgms 1.304348 [25/23/23]
2023-06-29 16:04:22.937 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 0]  bond1    - rank  99, node orcldb2   , avgms 1.363636 [21/22/22]
2023-06-29 16:04:22.937 :GIPCDMON:217794304:  gipcdMonitorSaveInfMetricsNonOda: inf[ 1]  bond1    - rank  99, node orcldb1   , avgms 1.363636 [21/22/22]

报错已经终止。

12 启动OGG传输进程。