Gisida님의 Database Operator In Kubernetes study(=DOIK) 스터디 진행 중 테스트 한 내용입니다
Percona Distribution for MongoDB 오퍼레이터 - 복제 (2/3)
1. 클러스터 확인
2. 데이터 복제 확인
3. 장애상황 확인
1. 클러스터 확인
▶ 클러스터 접속 확인
# [터미널1] 클러스터 접속(ADMIN_USER)
$ kubectl exec -it myclient1 -- mongo --quiet "mongodb+srv://userAdmin:userAdmin123456@MINY-db-rs0.psmdb.svc.cluster.local/admin?replicaSet=rs0&ssl=false"
rs0:PRIMARY>
# [터미널2] 클러스터 접속(CLUSTER_USER)
$ kubectl exec -it myclient2 -- mongo --quiet "mongodb+srv://clusterAdmin:clusterAdmin123456@MINY-db-rs0.psmdb.svc.cluster.local/admin?replicaSet=rs0&ssl=false"
rs0:PRIMARY>
# [터미널3] 클러스터 접속(miny)
$ kubectl exec -it myclient3 -- mongo --quiet "mongodb+srv://miny:qwe123@MINY-db-rs0.psmdb.svc.cluster.local/admin?replicaSet=rs0&ssl=false"
rs0:PRIMARY>
▶ [터미널2] 복제 셋 정보 확인 : 구성원 상태 확인
rs0:PRIMARY> rs.status()
{
"set" : "rs0",
"date" : ISODate("2022-06-19T04:37:33.085Z"),
"myState" : 1,
"term" : NumberLong(1),
"syncSourceHost" : "",
"syncSourceId" : -1,
"heartbeatIntervalMillis" : NumberLong(2000),
"majorityVoteCount" : 2,
"writeMajorityCount" : 2,
"votingMembersCount" : 3,
"writableVotingMembersCount" : 3,
"optimes" : {
"lastCommittedOpTime" : {
"ts" : Timestamp(1655613445, 1),
"t" : NumberLong(1)
},
"lastCommittedWallTime" : ISODate("2022-06-19T04:37:25.619Z"),
"readConcernMajorityOpTime" : {
"ts" : Timestamp(1655613445, 1),
"t" : NumberLong(1)
},
"appliedOpTime" : {
"ts" : Timestamp(1655613445, 1),
"t" : NumberLong(1)
},
"durableOpTime" : {
"ts" : Timestamp(1655613445, 1),
"t" : NumberLong(1)
},
"lastAppliedWallTime" : ISODate("2022-06-19T04:37:25.619Z"),
"lastDurableWallTime" : ISODate("2022-06-19T04:37:25.619Z")
},
"lastStableRecoveryTimestamp" : Timestamp(1655613440, 1),
"electionCandidateMetrics" : {
"lastElectionReason" : "electionTimeout",
"lastElectionDate" : ISODate("2022-06-19T03:07:18.876Z"),
"electionTerm" : NumberLong(1),
"lastCommittedOpTimeAtElection" : {
"ts" : Timestamp(1655608038, 1),
"t" : NumberLong(-1)
},
"lastSeenOpTimeAtElection" : {
"ts" : Timestamp(1655608038, 1),
"t" : NumberLong(-1)
},
"numVotesNeeded" : 1,
"priorityAtElection" : 1,
"electionTimeoutMillis" : NumberLong(10000),
"newTermStartDate" : ISODate("2022-06-19T03:07:18.910Z"),
"wMajorityWriteAvailabilityDate" : ISODate("2022-06-19T03:07:18.932Z")
},
"members" : [
{
"_id" : 0,
"name" : "miny-db-rs0-0.miny-db-rs0.psmdb.svc.cluster.local:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 5490,
"optime" : {
"ts" : Timestamp(1655613445, 1),
"t" : NumberLong(1)
},
"optimeDate" : ISODate("2022-06-19T04:37:25Z"),
"lastAppliedWallTime" : ISODate("2022-06-19T04:37:25.619Z"),
"lastDurableWallTime" : ISODate("2022-06-19T04:37:25.619Z"),
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"electionTime" : Timestamp(1655608038, 2),
"electionDate" : ISODate("2022-06-19T03:07:18Z"),
"configVersion" : 93252,
"configTerm" : -1,
"self" : true,
"lastHeartbeatMessage" : ""
},
{
"_id" : 1,
"name" : "miny-db-rs0-1.miny-db-rs0.psmdb.svc.cluster.local:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 5407,
"optime" : {
"ts" : Timestamp(1655613445, 1),
"t" : NumberLong(1)
},
"optimeDurable" : {
"ts" : Timestamp(1655613445, 1),
"t" : NumberLong(1)
},
"optimeDate" : ISODate("2022-06-19T04:37:25Z"),
"optimeDurableDate" : ISODate("2022-06-19T04:37:25Z"),
"lastAppliedWallTime" : ISODate("2022-06-19T04:37:25.619Z"),
"lastDurableWallTime" : ISODate("2022-06-19T04:37:25.619Z"),
"lastHeartbeat" : ISODate("2022-06-19T04:37:31.994Z"),
"lastHeartbeatRecv" : ISODate("2022-06-19T04:37:32.936Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncSourceHost" : "miny-db-rs0-0.miny-db-rs0.psmdb.svc.cluster.local:27017",
"syncSourceId" : 0,
"infoMessage" : "",
"configVersion" : 93252,
"configTerm" : -1
},
{
"_id" : 2,
"name" : "miny-db-rs0-2.miny-db-rs0.psmdb.svc.cluster.local:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 5363,
"optime" : {
"ts" : Timestamp(1655613445, 1),
"t" : NumberLong(1)
},
"optimeDurable" : {
"ts" : Timestamp(1655613445, 1),
"t" : NumberLong(1)
},
"optimeDate" : ISODate("2022-06-19T04:37:25Z"),
"optimeDurableDate" : ISODate("2022-06-19T04:37:25Z"),
"lastAppliedWallTime" : ISODate("2022-06-19T04:37:25.619Z"),
"lastDurableWallTime" : ISODate("2022-06-19T04:37:25.619Z"),
"lastHeartbeat" : ISODate("2022-06-19T04:37:31.380Z"),
"lastHeartbeatRecv" : ISODate("2022-06-19T04:37:31.948Z"),
"pingMs" : NumberLong(1),
"lastHeartbeatMessage" : "",
"syncSourceHost" : "miny-db-rs0-1.miny-db-rs0.psmdb.svc.cluster.local:27017",
"syncSourceId" : 1,
"infoMessage" : "",
"configVersion" : 93252,
"configTerm" : -1
}
],
"ok" : 1,
"$clusterTime" : {
"clusterTime" : Timestamp(1655613445, 1),
"signature" : {
"hash" : BinData(0,"BaumUOV+vmXlBKL8ADaXcHvTDJI="),
"keyId" : NumberLong("7110782378204725253")
}
},
"operationTime" : Timestamp(1655613445, 1)
}
rs0:PRIMARY>
rs0:PRIMARY> rs.status()['members']
[
{
"_id" : 0,
"name" : "miny-db-rs0-0.miny-db-rs0.psmdb.svc.cluster.local:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 5495,
"optime" : {
"ts" : Timestamp(1655613445, 1),
"t" : NumberLong(1)
},
"optimeDate" : ISODate("2022-06-19T04:37:25Z"),
"lastAppliedWallTime" : ISODate("2022-06-19T04:37:25.619Z"),
"lastDurableWallTime" : ISODate("2022-06-19T04:37:25.619Z"),
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"electionTime" : Timestamp(1655608038, 2),
"electionDate" : ISODate("2022-06-19T03:07:18Z"),
"configVersion" : 93252,
"configTerm" : -1,
"self" : true,
"lastHeartbeatMessage" : ""
},
{
"_id" : 1,
"name" : "miny-db-rs0-1.miny-db-rs0.psmdb.svc.cluster.local:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 5412,
"optime" : {
"ts" : Timestamp(1655613445, 1),
"t" : NumberLong(1)
},
"optimeDurable" : {
"ts" : Timestamp(1655613445, 1),
"t" : NumberLong(1)
},
"optimeDate" : ISODate("2022-06-19T04:37:25Z"),
"optimeDurableDate" : ISODate("2022-06-19T04:37:25Z"),
"lastAppliedWallTime" : ISODate("2022-06-19T04:37:25.619Z"),
"lastDurableWallTime" : ISODate("2022-06-19T04:37:25.619Z"),
"lastHeartbeat" : ISODate("2022-06-19T04:37:37.997Z"),
"lastHeartbeatRecv" : ISODate("2022-06-19T04:37:36.938Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncSourceHost" : "miny-db-rs0-0.miny-db-rs0.psmdb.svc.cluster.local:27017",
"syncSourceId" : 0,
"infoMessage" : "",
"configVersion" : 93252,
"configTerm" : -1
},
{
"_id" : 2,
"name" : "miny-db-rs0-2.miny-db-rs0.psmdb.svc.cluster.local:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 5368,
"optime" : {
"ts" : Timestamp(1655613445, 1),
"t" : NumberLong(1)
},
"optimeDurable" : {
"ts" : Timestamp(1655613445, 1),
"t" : NumberLong(1)
},
"optimeDate" : ISODate("2022-06-19T04:37:25Z"),
"optimeDurableDate" : ISODate("2022-06-19T04:37:25Z"),
"lastAppliedWallTime" : ISODate("2022-06-19T04:37:25.619Z"),
"lastDurableWallTime" : ISODate("2022-06-19T04:37:25.619Z"),
"lastHeartbeat" : ISODate("2022-06-19T04:37:37.383Z"),
"lastHeartbeatRecv" : ISODate("2022-06-19T04:37:37.953Z"),
"pingMs" : NumberLong(1),
"lastHeartbeatMessage" : "",
"syncSourceHost" : "miny-db-rs0-1.miny-db-rs0.psmdb.svc.cluster.local:27017",
"syncSourceId" : 1,
"infoMessage" : "",
"configVersion" : 93252,
"configTerm" : -1
}
]
rs0:PRIMARY> db.isMaster()
{
"topologyVersion" : {
"processId" : ObjectId("62ae929bcf98a31570181f26"),
"counter" : NumberLong(8)
},
"hosts" : [
"miny-db-rs0-0.miny-db-rs0.psmdb.svc.cluster.local:27017",
"miny-db-rs0-1.miny-db-rs0.psmdb.svc.cluster.local:27017",
"miny-db-rs0-2.miny-db-rs0.psmdb.svc.cluster.local:27017"
],
"setName" : "rs0",
"setVersion" : 93252,
"ismaster" : true,
"secondary" : false,
"primary" : "miny-db-rs0-0.miny-db-rs0.psmdb.svc.cluster.local:27017",
"tags" : {
"podName" : "miny-db-rs0-0",
"serviceName" : "miny-db"
},
"me" : "miny-db-rs0-0.miny-db-rs0.psmdb.svc.cluster.local:27017",
"electionId" : ObjectId("7fffffff0000000000000001"),
"lastWrite" : {
"opTime" : {
"ts" : Timestamp(1655613529, 1),
"t" : NumberLong(1)
},
"lastWriteDate" : ISODate("2022-06-19T04:38:49Z"),
"majorityOpTime" : {
"ts" : Timestamp(1655613529, 1),
"t" : NumberLong(1)
},
"majorityWriteDate" : ISODate("2022-06-19T04:38:49Z")
},
"maxBsonObjectSize" : 16777216,
"maxMessageSizeBytes" : 48000000,
"maxWriteBatchSize" : 100000,
"localTime" : ISODate("2022-06-19T04:38:51.836Z"),
"logicalSessionTimeoutMinutes" : 30,
"connectionId" : 10064,
"minWireVersion" : 0,
"maxWireVersion" : 13,
"readOnly" : false,
"ok" : 1,
"$clusterTime" : {
"clusterTime" : Timestamp(1655613529, 1),
"signature" : {
"hash" : BinData(0,"rZb62lrLNr0/2H2TSopy0/Ffnjw="),
"keyId" : NumberLong("7110782378204725253")
}
},
"operationTime" : Timestamp(1655613529, 1)
}
▶ [터미널2] 오피로그 정보 확인 : 크기, 연산 시간
rs0:PRIMARY> rs.printReplicationInfo()
configured oplog size: 2280.877197265625MB
log length start to end: 5511secs (1.53hrs)
oplog first event time: Sun Jun 19 2022 03:07:18 GMT+0000 (UTC)
oplog last event time: Sun Jun 19 2022 04:39:09 GMT+0000 (UTC)
now: Sun Jun 19 2022 04:39:15 GMT+0000 (UTC)
▶ [터미널2] 동기화 상태 확인 : 세컨더리 구성원이 프라이머리의 어디까지 정보를 동기화했는지 확인
rs0:PRIMARY> rs.printSecondaryReplicationInfo()
source: miny-db-rs0-1.miny-db-rs0.psmdb.svc.cluster.local:27017
syncedTo: Sun Jun 19 2022 04:39:29 GMT+0000 (UTC)
0 secs (0 hrs) behind the primary
source: miny-db-rs0-2.miny-db-rs0.psmdb.svc.cluster.local:27017
syncedTo: Sun Jun 19 2022 04:39:29 GMT+0000 (UTC)
0 secs (0 hrs) behind the primary
▶ [터미널2] 복제 옵션 정보 확인
rs0:PRIMARY> rs.conf()
{
"_id" : "rs0",
"version" : 93252,
"members" : [
{
"_id" : 0,
"host" : "miny-db-rs0-0.miny-db-rs0.psmdb.svc.cluster.local:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 2,
"tags" : {
"podName" : "miny-db-rs0-0",
"serviceName" : "miny-db"
},
"secondaryDelaySecs" : NumberLong(0),
"votes" : 1
},
{
"_id" : 1,
"host" : "miny-db-rs0-1.miny-db-rs0.psmdb.svc.cluster.local:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 2,
"tags" : {
"podName" : "miny-db-rs0-1",
"serviceName" : "miny-db"
},
"secondaryDelaySecs" : NumberLong(0),
"votes" : 1
},
{
"_id" : 2,
"host" : "miny-db-rs0-2.miny-db-rs0.psmdb.svc.cluster.local:27017",
"arbiterOnly" : false,
"buildIndexes" : true,
"hidden" : false,
"priority" : 2,
"tags" : {
"podName" : "miny-db-rs0-2",
"serviceName" : "miny-db"
},
"secondaryDelaySecs" : NumberLong(0),
"votes" : 1
}
],
"protocolVersion" : NumberLong(1),
"writeConcernMajorityJournalDefault" : true,
"settings" : {
"chainingAllowed" : true,
"heartbeatIntervalMillis" : 2000,
"heartbeatTimeoutSecs" : 10,
"electionTimeoutMillis" : 10000,
"catchUpTimeoutMillis" : -1,
"catchUpTakeoverDelayMillis" : 30000,
"getLastErrorModes" : {
},
"getLastErrorDefaults" : {
"w" : 1,
"wtimeout" : 0
},
"replicaSetId" : ObjectId("62ae92e6cf98a31570181f5a")
}
}
rs0:PRIMARY> use local
switched to db local
rs0:PRIMARY> db
local
▶ [터미널2] 오피로그 사이즈 확인
rs0:PRIMARY> db.oplog.rs.stats().maxSize
NumberLong("2391673088")
▶ [터미널2] 오피로그 확인
rs0:PRIMARY> db.oplog.rs.find().limit(2).pretty()
{
"op" : "n",
"ns" : "",
"o" : {
"msg" : "initiating set"
},
"ts" : Timestamp(1655608038, 1),
"v" : NumberLong(2),
"wall" : ISODate("2022-06-19T03:07:18.810Z")
}
{
"op" : "c",
"ns" : "config.$cmd",
"ui" : UUID("6ac36f7a-ee3b-4d76-a357-c6735c6977f0"),
"o" : {
"create" : "transactions",
"idIndex" : {
"v" : 2,
"key" : {
"_id" : 1
},
"name" : "_id_"
}
},
"ts" : Timestamp(1655608038, 3),
"t" : NumberLong(1),
"v" : NumberLong(2),
"wall" : ISODate("2022-06-19T03:07:18.896Z")
}
2. 데이터 복제 확인
▶ [터미널1] 프라이머리 파드 접속(miny)
$ kubectl exec -it myclient1 -- mongo --quiet "mongodb://miny:qwe123@miny-db-rs0-0.miny-db-rs0.psmdb.svc/admin?ssl=false"
▶ [터미널1] miny디비에 콜렉션 생성 후 도큐먼트 insert
rs0:PRIMARY> use miny
rs0:PRIMARY> db.test.insertOne({ reptest: 1 })
rs0:PRIMARY> db.test.find()
rs0:PRIMARY> db.test.count()
▶ [터미널2] 세컨더리 파드1 접속(miny)
$ kubectl exec -it myclient2 -- bash -il
▶ [터미널2] 프라이머리 파드에서 생성한 콜렉션의 데이터 건수 반복조회
$ echo $'rs.secondaryOk()\nuse miny\ndb.test.count()' | mongo --quiet "mongodb://miny:qwe123@miny-db-rs0-1.miny-db-rs0.psmdb.svc/admin?ssl=false"
while true; do echo $'rs.secondaryOk()\nuse miny\ndb.test.count()' | mongo --quiet "mongodb://miny:qwe123@miny-db-rs0-1.miny-db-rs0.psmdb.svc/admin?ssl=false" | grep -v Error; date; sleep 1; done
▶ [터미널3] 세컨더리 파드2 접속(miny)
$ kubectl exec -it myclient3 -- bash -il
▶ [터미널3] 프라이머리 파드에서 생성한 콜렉션의 데이터 건수 반복조회
echo $'rs.secondaryOk()\nuse miny\ndb.test.count()' | mongo --quiet "mongodb://miny:qwe123@miny-db-rs0-2.miny-db-rs0.psmdb.svc/admin?ssl=false"
while true; do echo $'rs.secondaryOk()\nuse miny\ndb.test.count()' | mongo --quiet "mongodb://miny:qwe123@miny-db-rs0-2.miny-db-rs0.psmdb.svc/admin?ssl=false" | grep -v Error; date; sleep 1; done
▶ [터미널1] 프라이머리 파드 접속(miny) : 1000개의 도큐먼트 생성
rs0:PRIMARY> for (i=0; i<1000; i++) {db.test.insert({count: i, "created_at" : new Date()})}
rs0:PRIMARY> db.test.find({},{_id:0})
▶ [터미널2,3]에서 실시간으로 데이터 복제 확인 (건수 증가)
Sun Jun 19 04:47:36 UTC 2022
switched to db miny
763
Sun Jun 19 04:47:37 UTC 2022
switched to db miny
867
Sun Jun 19 04:47:38 UTC 2022
switched to db miny
976
Sun Jun 19 04:47:39 UTC 2022
switched to db miny
1001
3. 장애상황 확인
▶ Primary Node Delete
→ 정상상태 pod정보 확인
$ kubectl get pod -l app.kubernetes.io/instance=miny-db -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
miny-db-rs0-0 1/1 Running 0 108m 172.16.1.3 k8s-w1 <none> <none>
miny-db-rs0-1 1/1 Running 0 107m 172.16.2.3 k8s-w2 <none> <none>
miny-db-rs0-2 1/1 Running 0 106m 172.16.3.5 k8s-w3 <none> <none>
→ 강제로 rs0- 프라이머리 파드 1개 delete
$ kubectl delete pod miny-db-rs0-0
pod "miny-db-rs0-0" deleted
→ Terminating 후 바로 pod 재생성
$ kubectl get pod -l app.kubernetes.io/instance=miny-db -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
miny-db-rs0-0 1/1 Terminating 0 112m 172.16.1.3 k8s-w1 <none> <none>
miny-db-rs0-1 1/1 Running 0 111m 172.16.2.3 k8s-w2 <none> <none>
miny-db-rs0-2 1/1 Running 0 110m 172.16.3.5 k8s-w3 <none> <none>
$ get pod -l app.kubernetes.io/instance=miny-db -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
miny-db-rs0-0 0/1 Init:0/1 0 2s <none> k8s-w1 <none> <none>
miny-db-rs0-1 1/1 Running 0 111m 172.16.2.3 k8s-w2 <none> <none>
miny-db-rs0-2 1/1 Running 0 110m 172.16.3.5 k8s-w3 <none> <none>
$ kubectl get pod -l app.kubernetes.io/instance=miny-db -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
miny-db-rs0-0 0/1 Running 0 10s 172.16.1.5 k8s-w1 <none> <none>
miny-db-rs0-1 1/1 Running 0 111m 172.16.2.3 k8s-w2 <none> <none>
→ 각 파드에 접속해서 Primary파드 찾아보기 : 2번파드가 PRIMARY로 변경
#1번 파드 접속 : SECONDARY
kubectl exec -it myclient1 -- mongo --quiet "mongodb://miny:qwe123@miny-db-rs0-0.miny-db-rs0.psmdb.svc/admin?ssl=false"
rs0:SECONDARY>
#2번 파드 접속 : PRIMARY
kubectl exec -it myclient2 -- mongo --quiet "mongodb://miny:qwe123@miny-db-rs0-1.miny-db-rs0.psmdb.svc/admin?ssl=false"
rs0:PRIMARY>
#3번 파드 접속 : SECONDARY
kubectl exec -it myclient3 -- mongo --quiet "mongodb://miny:qwe123@miny-db-rs0-2.miny-db-rs0.psmdb.svc/admin?ssl=false"
rs0:SECONDARY>
▶ Primary Node Drain
→ [터미널1] 클러스터 접속(CLUSTER_USER) : 프라이머리 파드 정보 확인
$ kubectl exec -it myclient1 -- mongo --quiet "mongodb+srv://clusterAdmin:clusterAdmin123456@miny-db-rs0.psmdb.svc.cluster.local/admin?replicaSet=rs0&ssl=false"
rs0:PRIMARY> rs.status()['members']
[
{
"_id" : 0,
"name" : "miny-db-rs0-0.miny-db-rs0.psmdb.svc.cluster.local:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 391,
"optime" : {
"ts" : Timestamp(1655615088, 1),
"t" : NumberLong(2)
},
"optimeDurable" : {
"ts" : Timestamp(1655615088, 1),
"t" : NumberLong(2)
},
"optimeDate" : ISODate("2022-06-19T05:04:48Z"),
"optimeDurableDate" : ISODate("2022-06-19T05:04:48Z"),
"lastAppliedWallTime" : ISODate("2022-06-19T05:04:48.860Z"),
"lastDurableWallTime" : ISODate("2022-06-19T05:04:48.860Z"),
"lastHeartbeat" : ISODate("2022-06-19T05:04:49.596Z"),
"lastHeartbeatRecv" : ISODate("2022-06-19T05:04:49.331Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "",
"syncSourceHost" : "miny-db-rs0-1.miny-db-rs0.psmdb.svc.cluster.local:27017",
"syncSourceId" : 1,
"infoMessage" : "",
"configVersion" : 93252,
"configTerm" : -1
},
{
"_id" : 1,
"name" : "miny-db-rs0-1.miny-db-rs0.psmdb.svc.cluster.local:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 7075,
"optime" : {
"ts" : Timestamp(1655615088, 1),
"t" : NumberLong(2)
},
"optimeDate" : ISODate("2022-06-19T05:04:48Z"),
"lastAppliedWallTime" : ISODate("2022-06-19T05:04:48.860Z"),
"lastDurableWallTime" : ISODate("2022-06-19T05:04:48.860Z"),
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"electionTime" : Timestamp(1655614652, 1),
"electionDate" : ISODate("2022-06-19T04:57:32Z"),
"configVersion" : 93252,
"configTerm" : -1,
"self" : true,
"lastHeartbeatMessage" : ""
},
{
"_id" : 2,
"name" : "miny-db-rs0-2.miny-db-rs0.psmdb.svc.cluster.local:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 7002,
"optime" : {
"ts" : Timestamp(1655615088, 1),
"t" : NumberLong(2)
},
"optimeDurable" : {
"ts" : Timestamp(1655615088, 1),
"t" : NumberLong(2)
},
"optimeDate" : ISODate("2022-06-19T05:04:48Z"),
"optimeDurableDate" : ISODate("2022-06-19T05:04:48Z"),
"lastAppliedWallTime" : ISODate("2022-06-19T05:04:48.860Z"),
"lastDurableWallTime" : ISODate("2022-06-19T05:04:48.860Z"),
"lastHeartbeat" : ISODate("2022-06-19T05:04:50.272Z"),
"lastHeartbeatRecv" : ISODate("2022-06-19T05:04:50.267Z"),
"pingMs" : NumberLong(1),
"lastHeartbeatMessage" : "",
"syncSourceHost" : "miny-db-rs0-1.miny-db-rs0.psmdb.svc.cluster.local:27017",
"syncSourceId" : 1,
"infoMessage" : "",
"configVersion" : 93252,
"configTerm" : -1
}
]
→ 프라이머리 파드가 배포된 워커 노드 정보 확인
$ kubectl get pod -l app.kubernetes.io/instance=miny-db -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
miny-db-rs0-0 1/1 Running 0 7m20s 172.16.1.5 k8s-w1 <none> <none>
miny-db-rs0-1 1/1 Running 0 118m 172.16.2.3 k8s-w2 <none> <none>
miny-db-rs0-2 1/1 Running 0 118m 172.16.3.5 k8s-w3 <none> <none>
→ CoreDNS 미리 늘려놓기
$ kubectl scale deployment -n kube-system coredns --replicas=3
$ kubectl get deployment -n kube-system coredns
NAME READY UP-TO-DATE AVAILABLE AGE
coredns 3/3 3 3 160m
→ Primary가 배포 된 워커노드 drain
$ kubectl drain k8s-w2 --delete-emptydir-data --force --ignore-daemonsets && kubectl get node -w
→ Drain된 워커노드 확인
$ kubectl get node k8s-w2
NAME STATUS ROLES AGE VERSION
k8s-w2 Ready,SchedulingDisabled <none> 163m v1.23.6
$ kubectl get pod -l app.kubernetes.io/instance=miny-db -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
miny-db-rs0-0 1/1 Running 0 10m 172.16.1.5 k8s-w1 <none> <none>
miny-db-rs0-1 0/1 Pending 0 14s <none> <none> <none> <none>
miny-db-rs0-2 1/1 Running 0 121m 172.16.3.5 k8s-w3 <none> <none>
→ [터미널1] 클러스터 접속(CLUSTER_USER) : 장애 상태 확인
rs0:PRIMARY> rs.status()['members']
[
{
"_id" : 0,
"name" : "miny-db-rs0-0.miny-db-rs0.psmdb.svc.cluster.local:27017",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 668,
"optime" : {
"ts" : Timestamp(1655615334, 1),
"t" : NumberLong(3)
},
"optimeDate" : ISODate("2022-06-19T05:08:54Z"),
"lastAppliedWallTime" : ISODate("2022-06-19T05:08:54.736Z"),
"lastDurableWallTime" : ISODate("2022-06-19T05:08:54.736Z"),
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"electionTime" : Timestamp(1655615274, 1),
"electionDate" : ISODate("2022-06-19T05:07:54Z"),
"configVersion" : 93252,
"configTerm" : -1,
"self" : true,
"lastHeartbeatMessage" : ""
},
{
"_id" : 1,
"name" : "miny-db-rs0-1.miny-db-rs0.psmdb.svc.cluster.local:27017",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDurable" : {
"ts" : Timestamp(0, 0),
"t" : NumberLong(-1)
},
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"optimeDurableDate" : ISODate("1970-01-01T00:00:00Z"),
"lastAppliedWallTime" : ISODate("2022-06-19T05:08:03.315Z"),
"lastDurableWallTime" : ISODate("2022-06-19T05:08:03.315Z"),
"lastHeartbeat" : ISODate("2022-06-19T05:09:02.499Z"),
"lastHeartbeatRecv" : ISODate("2022-06-19T05:08:09.735Z"),
"pingMs" : NumberLong(0),
"lastHeartbeatMessage" : "Error connecting to miny-db-rs0-1.miny-db-rs0.psmdb.svc.cluster.local:27017 :: caused by :: Could not find address for miny-db-rs0-1.miny-db-rs0.psmdb.svc.cluster.local:27017: SocketException: Host not found (authoritative)",
"syncSourceHost" : "",
"syncSourceId" : -1,
"infoMessage" : "",
"configVersion" : 93252,
"configTerm" : -1
},
{
"_id" : 2,
"name" : "miny-db-rs0-2.miny-db-rs0.psmdb.svc.cluster.local:27017",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 664,
"optime" : {
"ts" : Timestamp(1655615334, 1),
"t" : NumberLong(3)
},
"optimeDurable" : {
"ts" : Timestamp(1655615334, 1),
"t" : NumberLong(3)
},
"optimeDate" : ISODate("2022-06-19T05:08:54Z"),
"optimeDurableDate" : ISODate("2022-06-19T05:08:54Z"),
"lastAppliedWallTime" : ISODate("2022-06-19T05:08:54.736Z"),
"lastDurableWallTime" : ISODate("2022-06-19T05:08:54.736Z"),
"lastHeartbeat" : ISODate("2022-06-19T05:09:00.848Z"),
"lastHeartbeatRecv" : ISODate("2022-06-19T05:09:02.567Z"),
"pingMs" : NumberLong(1),
"lastHeartbeatMessage" : "",
"syncSourceHost" : "miny-db-rs0-0.miny-db-rs0.psmdb.svc.cluster.local:27017",
"syncSourceId" : 0,
"infoMessage" : "",
"configVersion" : 93252,
"configTerm" : -1
}
]
rs0:PRIMARY>
→ Drain 시켰던 노드 Uncordon 설정
$ kubectl uncordon k8s-w1 ; kubectl uncordon k8s-w2 ; kubectl uncordon k8s-w3
node/k8s-w1 already uncordoned
node/k8s-w2 uncordoned
node/k8s-w3 already uncordoned
→ 파드 정상복구 확인
$ kubectl get pod -l app.kubernetes.io/instance=miny-db -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
miny-db-rs0-0 1/1 Running 0 14m 172.16.1.5 k8s-w1 <none> <none>
miny-db-rs0-1 1/1 Running 0 3m59s 172.16.2.5 k8s-w2 <none> <none>
miny-db-rs0-2 1/1 Running 0 125m 172.16.3.5 k8s-w3 <none> <none>
'K8S' 카테고리의 다른 글
| Cloud Native PostgreSQL 오퍼레이터 (0) | 2022.06.22 |
|---|---|
| Percona Distribution for MongoDB - 샤딩 (3/3) (0) | 2022.06.19 |
| Percona Distribution for MongoDB 오퍼레이터 - 기본설치 (1/3) (0) | 2022.06.19 |
| Helm을 이용해 K8S에 MariaDB Galera 설치하기 (0) | 2022.05.29 |
| AWS EC2에 쿠버네티스 테스트환경 구축하기 (0) | 2022.05.29 |
