|
|
@@ -0,0 +1,339 @@
|
|
|
+package main
|
|
|
+
|
|
|
+import (
|
|
|
+ "fmt"
|
|
|
+ "os"
|
|
|
+ "path/filepath"
|
|
|
+ "strings"
|
|
|
+ "sync/atomic"
|
|
|
+ "time"
|
|
|
+
|
|
|
+ "lc/common/protocol"
|
|
|
+ "lc/common/util"
|
|
|
+)
|
|
|
+
|
|
|
+const (
|
|
|
+ phase1Timeout = 30 * time.Second
|
|
|
+ phase2Delay = 5 * time.Minute
|
|
|
+ phase2Window = 5 * time.Minute
|
|
|
+)
|
|
|
+
|
|
|
+// HealthCheckRunner 健康检测运行器
|
|
|
+type HealthCheckRunner struct {
|
|
|
+ marker *DeployMarker
|
|
|
+ running int32
|
|
|
+ phase2Done int32
|
|
|
+}
|
|
|
+
|
|
|
+var _hcRunner *HealthCheckRunner
|
|
|
+
|
|
|
+// StartHealthCheckIfNeeded 启动时检查部署标记,如有则运行健康检测
|
|
|
+func StartHealthCheckIfNeeded() {
|
|
|
+ markerPath := filepath.Join(util.GetPath(0), "deploy_marker.json")
|
|
|
+ data, err := os.ReadFile(markerPath)
|
|
|
+ if err != nil {
|
|
|
+ // 检查是否有回滚标记
|
|
|
+ rollbackPath := filepath.Join(util.GetPath(0), "rollback_marker.json")
|
|
|
+ if rbData, err := os.ReadFile(rollbackPath); err == nil {
|
|
|
+ var marker DeployMarker
|
|
|
+ json.Unmarshal(rbData, &marker)
|
|
|
+ util.GetTagLog().Infof("sys", "检测到回滚标记,已回滚到版本:%s", marker.Version)
|
|
|
+ reportRollback(marker)
|
|
|
+ os.Remove(rollbackPath)
|
|
|
+ }
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ var marker DeployMarker
|
|
|
+ if err := json.Unmarshal(data, &marker); err != nil {
|
|
|
+ util.GetTagLog().Errorf("sys", "解析部署标记失败,err=%v", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ util.GetTagLog().Infof("sys", "检测到部署标记,开始健康检测 version=%s", marker.Version)
|
|
|
+
|
|
|
+ _hcRunner = &HealthCheckRunner{
|
|
|
+ marker: &marker,
|
|
|
+ }
|
|
|
+
|
|
|
+ go func() {
|
|
|
+ time.Sleep(3 * time.Second) // 等待 MQTT 初始化
|
|
|
+ _hcRunner.runPhase1()
|
|
|
+ }()
|
|
|
+}
|
|
|
+
|
|
|
+func (o *HealthCheckRunner) runPhase1() {
|
|
|
+ if !atomic.CompareAndSwapInt32(&o.running, 0, 1) {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ defer atomic.StoreInt32(&o.running, 0)
|
|
|
+
|
|
|
+ util.GetTagLog().Infof("sys", "HealthCheck:Phase 1 开始 version=%s", o.marker.Version)
|
|
|
+ deadline := time.Now().Add(phase1Timeout)
|
|
|
+
|
|
|
+ checks := []protocol.HealthCheck{
|
|
|
+ o.checkMQTT(deadline),
|
|
|
+ o.checkRedis(deadline),
|
|
|
+ }
|
|
|
+
|
|
|
+ allOK := true
|
|
|
+ for _, c := range checks {
|
|
|
+ if c.Status != "ok" {
|
|
|
+ allOK = false
|
|
|
+ break
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ o.reportPhase(1, allOK, checks)
|
|
|
+
|
|
|
+ if !allOK {
|
|
|
+ util.GetTagLog().Errorf("sys", "HealthCheck:Phase 1 失败,开始回滚")
|
|
|
+ o.rollback(checks)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ util.GetTagLog().Infof("sys", "HealthCheck:Phase 1 全部通过,清理部署标记")
|
|
|
+ os.Remove(filepath.Join(util.GetPath(0), "deploy_marker.json"))
|
|
|
+ DeployResultSave(o.marker.Version, 1, allOK, checks)
|
|
|
+
|
|
|
+ go func() {
|
|
|
+ time.Sleep(phase2Delay)
|
|
|
+ o.runPhase2()
|
|
|
+ }()
|
|
|
+}
|
|
|
+
|
|
|
+func (o *HealthCheckRunner) checkMQTT(deadline time.Time) protocol.HealthCheck {
|
|
|
+ start := time.Now()
|
|
|
+ check := protocol.HealthCheck{Name: "mqtt"}
|
|
|
+
|
|
|
+ for time.Now().Before(deadline) {
|
|
|
+ mgr := GetMQTTMgr()
|
|
|
+ if mgr.Cloud != nil && mgr.Cloud.IsConnected() {
|
|
|
+ check.Status = "ok"
|
|
|
+ check.Detail = "MQTT Broker 已连接"
|
|
|
+ check.DurationMs = int(time.Since(start).Milliseconds())
|
|
|
+ util.GetTagLog().Infof("sys", "HealthCheck:MQTT检测通过")
|
|
|
+ return check
|
|
|
+ }
|
|
|
+ time.Sleep(1 * time.Second)
|
|
|
+ }
|
|
|
+
|
|
|
+ check.Status = "timeout"
|
|
|
+ check.Detail = "MQTT Broker 不可达,请检查 Server/User/Password 配置、网络连通性"
|
|
|
+ check.DurationMs = int(time.Since(start).Milliseconds())
|
|
|
+ util.GetTagLog().Errorf("sys", "HealthCheck:MQTT检测失败")
|
|
|
+ return check
|
|
|
+}
|
|
|
+
|
|
|
+func (o *HealthCheckRunner) checkRedis(deadline time.Time) protocol.HealthCheck {
|
|
|
+ start := time.Now()
|
|
|
+ check := protocol.HealthCheck{Name: "redis"}
|
|
|
+
|
|
|
+ for time.Now().Before(deadline) {
|
|
|
+ if redisEdgeData != nil {
|
|
|
+ if _, err := redisEdgeData.Ping().Result(); err == nil {
|
|
|
+ check.Status = "ok"
|
|
|
+ check.Detail = "Redis PING 正常"
|
|
|
+ check.DurationMs = int(time.Since(start).Milliseconds())
|
|
|
+ util.GetTagLog().Infof("sys", "HealthCheck:Redis检测通过")
|
|
|
+ return check
|
|
|
+ }
|
|
|
+ }
|
|
|
+ time.Sleep(2 * time.Second)
|
|
|
+ }
|
|
|
+
|
|
|
+ check.Status = "timeout"
|
|
|
+ check.Detail = "Redis 连接失败,请检查 Redis Server/Password 配置"
|
|
|
+ check.DurationMs = int(time.Since(start).Milliseconds())
|
|
|
+ util.GetTagLog().Errorf("sys", "HealthCheck:Redis检测失败")
|
|
|
+ return check
|
|
|
+}
|
|
|
+
|
|
|
+func (o *HealthCheckRunner) runPhase2() {
|
|
|
+ atomic.StoreInt32(&o.phase2Done, 1)
|
|
|
+ util.GetTagLog().Infof("sys", "HealthCheck:Phase 2 开始 version=%s", o.marker.Version)
|
|
|
+
|
|
|
+ checks := []protocol.HealthCheck{
|
|
|
+ o.checkSerial(),
|
|
|
+ o.checkModbusData(),
|
|
|
+ }
|
|
|
+
|
|
|
+ allOK := true
|
|
|
+ for _, c := range checks {
|
|
|
+ if c.Status != "ok" {
|
|
|
+ allOK = false
|
|
|
+ break
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ o.reportPhase(2, allOK, checks)
|
|
|
+ DeployResultSave(o.marker.Version, 2, allOK, checks)
|
|
|
+
|
|
|
+ if !allOK {
|
|
|
+ util.GetTagLog().Warnf("sys", "HealthCheck:Phase 2 存在告警,不回滚")
|
|
|
+ } else {
|
|
|
+ util.GetTagLog().Infof("sys", "HealthCheck:Phase 2 全部通过")
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+func (o *HealthCheckRunner) checkSerial() protocol.HealthCheck {
|
|
|
+ start := time.Now()
|
|
|
+ check := protocol.HealthCheck{Name: "serial"}
|
|
|
+
|
|
|
+ sc := GetSerialMgr()
|
|
|
+ failedPorts := sc.GetFailedPorts()
|
|
|
+ if len(failedPorts) == 0 {
|
|
|
+ check.Status = "ok"
|
|
|
+ check.Detail = "所有串口打开正常"
|
|
|
+ } else {
|
|
|
+ check.Status = "fail"
|
|
|
+ parts := make([]string, len(failedPorts))
|
|
|
+ for i, p := range failedPorts {
|
|
|
+ parts[i] = fmt.Sprintf("%d", p)
|
|
|
+ }
|
|
|
+ check.Detail = "串口打开失败: " + strings.Join(parts, ", ")
|
|
|
+ }
|
|
|
+ check.DurationMs = int(time.Since(start).Milliseconds())
|
|
|
+ return check
|
|
|
+}
|
|
|
+
|
|
|
+func (o *HealthCheckRunner) checkModbusData() protocol.HealthCheck {
|
|
|
+ start := time.Now()
|
|
|
+ check := protocol.HealthCheck{Name: "modbus"}
|
|
|
+
|
|
|
+ deviceCount := 0
|
|
|
+ hasData := false
|
|
|
+ mapRtuUploadManager.Range(func(key, value interface{}) bool {
|
|
|
+ mgr, ok := value.(*RtuUploadManager)
|
|
|
+ if !ok {
|
|
|
+ return true
|
|
|
+ }
|
|
|
+ deviceCount++
|
|
|
+ mgr.DataLock.Lock()
|
|
|
+ lastDataTime := mgr.Datatime
|
|
|
+ mgr.DataLock.Unlock()
|
|
|
+ if time.Since(lastDataTime) < phase2Window {
|
|
|
+ hasData = true
|
|
|
+ return false
|
|
|
+ }
|
|
|
+ return true
|
|
|
+ })
|
|
|
+
|
|
|
+ if deviceCount == 0 {
|
|
|
+ check.Status = "ok"
|
|
|
+ check.Detail = fmt.Sprintf("无Modbus设备配置,跳过采集检测")
|
|
|
+ } else if hasData {
|
|
|
+ check.Status = "ok"
|
|
|
+ check.Detail = fmt.Sprintf("设备数据采集正常(%d台设备)", deviceCount)
|
|
|
+ } else {
|
|
|
+ check.Status = "fail"
|
|
|
+ check.Detail = fmt.Sprintf("%d台设备5分钟内无数据上报,请检查设备配置和物模型是否正确加载", deviceCount)
|
|
|
+ }
|
|
|
+ check.DurationMs = int(time.Since(start).Milliseconds())
|
|
|
+ return check
|
|
|
+}
|
|
|
+
|
|
|
+func (o *HealthCheckRunner) reportPhase(phase int, success bool, checks []protocol.HealthCheck) {
|
|
|
+ var obj protocol.Pack_HealthCheck
|
|
|
+ seq := GetNextUint64()
|
|
|
+ if str, err := obj.EnCode(appConfig.GID, appConfig.GID, seq, o.marker.Version, phase, success, checks); err == nil {
|
|
|
+ topic := GetTopic(protocol.DT_GATEWAY, appConfig.GID, protocol.TP_GW_DEPLOY_HEALTH)
|
|
|
+ GetMQTTMgr().Publish(topic, str, 0, ToCloud)
|
|
|
+ util.GetTagLog().Infof("sys", "HealthCheck:Phase %d 结果已上报 success=%v", phase, success)
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+func (o *HealthCheckRunner) rollback(checks []protocol.HealthCheck) {
|
|
|
+ cwd, _ := os.Getwd()
|
|
|
+ targetPath := filepath.Join(cwd, appname)
|
|
|
+ backupPath := targetPath + ".bak"
|
|
|
+
|
|
|
+ rbMarker := DeployMarker{
|
|
|
+ Version: o.marker.Version,
|
|
|
+ Timestamp: time.Now().Unix(),
|
|
|
+ Action: "rollback",
|
|
|
+ }
|
|
|
+ rbContent, _ := json.MarshalToString(rbMarker)
|
|
|
+ os.WriteFile(filepath.Join(util.GetPath(0), "rollback_marker.json"), []byte(rbContent), os.ModePerm)
|
|
|
+
|
|
|
+ os.Remove(filepath.Join(util.GetPath(0), "deploy_marker.json"))
|
|
|
+
|
|
|
+ DeployResultSave(o.marker.Version, 1, false, checks)
|
|
|
+
|
|
|
+ if _, err := os.Stat(backupPath); os.IsNotExist(err) {
|
|
|
+ util.GetTagLog().Errorf("sys", "HealthCheck:备份文件不存在,无法自动回滚")
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ if err := os.Rename(backupPath, targetPath); err != nil {
|
|
|
+ util.GetTagLog().Errorf("sys", "HealthCheck:回滚失败,err=%v", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ os.Chmod(targetPath, 0755)
|
|
|
+ util.GetTagLog().Infof("sys", "HealthCheck:回滚成功,退出进程")
|
|
|
+
|
|
|
+ time.Sleep(500 * time.Millisecond)
|
|
|
+ os.Exit(2)
|
|
|
+}
|
|
|
+
|
|
|
+func reportRollback(marker DeployMarker) {
|
|
|
+ var obj protocol.Pack_DeployAck
|
|
|
+ seq := GetNextUint64()
|
|
|
+ if str, err := obj.EnCode(appConfig.GID, appConfig.GID, seq, marker.Version, false,
|
|
|
+ "已自动回滚到版本 "+marker.Version); err == nil {
|
|
|
+ topic := GetTopic(protocol.DT_GATEWAY, appConfig.GID, protocol.TP_GW_DEPLOY_ACK)
|
|
|
+ GetMQTTMgr().Publish(topic, str, 0, ToCloud)
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+// ---- 部署结果持久化 ----
|
|
|
+
|
|
|
+type DeployResult struct {
|
|
|
+ Version string `json:"version"`
|
|
|
+ Timestamp int64 `json:"timestamp"`
|
|
|
+ Phase1 *DeployPhaseResult `json:"phase1,omitempty"`
|
|
|
+ Phase2 *DeployPhaseResult `json:"phase2,omitempty"`
|
|
|
+}
|
|
|
+
|
|
|
+type DeployPhaseResult struct {
|
|
|
+ Success bool `json:"success"`
|
|
|
+ Checks []protocol.HealthCheck `json:"checks"`
|
|
|
+}
|
|
|
+
|
|
|
+var _deployResult *DeployResult
|
|
|
+
|
|
|
+func DeployResultSave(version string, phase int, success bool, checks []protocol.HealthCheck) {
|
|
|
+ if _deployResult == nil || _deployResult.Version != version {
|
|
|
+ _deployResult = &DeployResult{
|
|
|
+ Version: version,
|
|
|
+ Timestamp: time.Now().Unix(),
|
|
|
+ }
|
|
|
+ }
|
|
|
+ pr := &DeployPhaseResult{Success: success, Checks: checks}
|
|
|
+ if phase == 1 {
|
|
|
+ _deployResult.Phase1 = pr
|
|
|
+ } else {
|
|
|
+ _deployResult.Phase2 = pr
|
|
|
+ }
|
|
|
+
|
|
|
+ content, _ := json.MarshalToString(_deployResult)
|
|
|
+ resultPath := filepath.Join(util.GetPath(0), "deploy_result.json")
|
|
|
+ os.WriteFile(resultPath, []byte(content), os.ModePerm)
|
|
|
+}
|
|
|
+
|
|
|
+func DeployResultGet() *DeployResult {
|
|
|
+ if _deployResult != nil {
|
|
|
+ return _deployResult
|
|
|
+ }
|
|
|
+ resultPath := filepath.Join(util.GetPath(0), "deploy_result.json")
|
|
|
+ data, err := os.ReadFile(resultPath)
|
|
|
+ if err != nil {
|
|
|
+ return nil
|
|
|
+ }
|
|
|
+ var result DeployResult
|
|
|
+ if err := json.Unmarshal(data, &result); err != nil {
|
|
|
+ return nil
|
|
|
+ }
|
|
|
+ _deployResult = &result
|
|
|
+ return &result
|
|
|
+}
|