日志和ES对接 v1

app/logapi/logController.go

@@ -0,0 +1,95 @@
+package logapi
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/gin-gonic/gin"
+	"github.com/olivere/elastic"
+	"iot_manager_service/util/common"
+	"iot_manager_service/util/es"
+	"net/http"
+	"strconv"
+	"time"
+)
+
+var LogController = new(logController)
+
+type logController struct {
+}
+
+// List 查询日志
+func (l logController) List(ctx *gin.Context) {
+	//处理分页参数,索引名
+	var indexName, logDate string
+	//var logDate date.Date
+	var page, size int
+	var err1, err2 error
+	if logDate = ctx.Query("time"); logDate == "" {
+		indexName = defaultIndex()
+	} else {
+		indexName = logDate
+	}
+	if page, err1 = strconv.Atoi(ctx.Query("current")); err1 != nil {
+		page = 1
+	}
+	if size, err2 = strconv.Atoi(ctx.Query("size")); err2 != nil {
+		size = 10
+	}
+	//查询条件
+	level := ctx.Query("level")
+	reqUri := ctx.Query("req_uri")
+	query := elastic.NewBoolQuery()
+	var q1, q2 *elastic.MatchQuery
+	if level != "" {
+		q1 = elastic.NewMatchQuery("level", level)
+		query.Must(q1)
+	}
+	if reqUri != "" {
+		q2 = elastic.NewMatchQuery("req_uri", reqUri)
+		query.Must(q2)
+	}
+	//调试-打印DSL
+	source, _ := query.Source()
+	data, _ := json.MarshalIndent(source, "", "  ")
+	fmt.Printf("source: \n%+v", string(data))
+	//执行并响应
+	result, err3 := LogService.search(indexName, page, size, query)
+	if err3 != nil {
+		ctx.JSON(http.StatusOK, common.SuccessResponse("es错误", nil))
+	} else if result != nil {
+		ctx.JSON(http.StatusOK, common.SuccessResponse(common.Success, result))
+	} else {
+		//todo 修改前端代码 data为空时前端一直处于加载页面,前端不会改
+		ctx.JSON(http.StatusOK, common.SuccessResponse(common.Succeeded, res{
+			Current: 0,
+			Pages:   0,
+			Size:    0,
+			Total:   0,
+			Records: nil,
+		}))
+	}
+}
+
+// AddDoc 存储日志 DelIndex 删除索引 在util/es下
+
+// Detail 日志详情
+func (l logController) Detail(ctx *gin.Context) {
+	fmt.Printf("okkk")
+	var id, indexName string
+	indexName = ctx.Query("indexName")
+	id = ctx.Query("id")
+
+	do, err := es.Client.Get().Index(indexName).Type("_doc").Id(id).Do(context.Background())
+	if err != nil {
+		fmt.Printf("\nerror=%v\n", err)
+	} else {
+		//响应数据给前端
+		ctx.JSON(http.StatusOK, common.SuccessResponse(common.Succeeded, do))
+	}
+}
+
+// 当天日期作为索引名
+func defaultIndex() string {
+	return time.Now().Format("2006-01-02")
+}

app/logapi/logService.go

@@ -0,0 +1,55 @@
+package logapi
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/olivere/elastic"
+	"iot_manager_service/util/es"
+)
+
+type logService struct {
+}
+
+var LogService logService
+
+func (l logService) search(indexName string,
+	page, size int,
+	query *elastic.BoolQuery) (*res, error) {
+	result, err := es.Client.Search(indexName).
+		Query(query).
+		Size(size).
+		From((page-1)*size).
+		Sort("t", false).
+		Do(context.Background())
+	//查询错误
+	if err != nil {
+		return nil, err
+	}
+	//处理结果
+	var logVOS []logVO
+	var a logVO
+	var total int64
+	if total = result.Hits.TotalHits; total > 0 {
+		fmt.Printf("Found a total of %d apilog\n", result.Hits.TotalHits)
+		// Iterate through results
+		for _, hit := range result.Hits.Hits {
+			// hit.Index contains the name of the index
+			// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
+			json.Unmarshal(*hit.Source, &a)
+			a.Id = hit.Id
+			logVOS = append(logVOS, a)
+		}
+		res := &res{
+			Current: page,
+			Size:    size,
+			Pages:   int(total)/size + 1,
+			Total:   total,
+			Records: logVOS,
+		}
+		return res, nil
+	} else {
+		// No hits
+		return nil, nil
+	}
+}

app/logapi/obj.go

@@ -0,0 +1,23 @@
+package logapi
+
+type logVO struct {
+	ClientIP    string      `json:"client_ip"`
+	LatencyTime int32       `json:"latency_time"`
+	Level       string      `json:"level"`
+	Msg         string      `json:"msg"`
+	ReqMethod   string      `json:"req_method"`
+	ReqURI      string      `json:"req_uri"`
+	Request     interface{} `json:"request"`
+	Response    interface{} `json:"response"`
+	StatusCode  int16       `json:"status_code"`
+	Time        string      `json:"time"`
+	T           int64       `json:"t"`
+	Id          string      `json:"_id"`
+}
+type res struct {
+	Current int     `json:"current"` //当前分页
+	Size    int     `json:"size"`    //每页数量
+	Pages   int     `json:"pages"`   //总页数
+	Total   int64   `json:"total"`   //总数
+	Records []logVO `json:"records"` //记录列表
+}

config/config.go

@@ -28,7 +28,7 @@ func init() {
 	})
 }
 
-//获取配置文档实例
+// 获取配置文档实例
 func Instance() *config {
 	return instance
 }
@@ -41,6 +41,7 @@ type config struct {
 	Minio       minio       `yaml:"minio"`
 	Foreign     foreign     `yaml:"foreign"`
 	MonitNotice monitNotice `yaml:"monit-notice"`
+	Es          es          `yaml:"es"`
 }
 
 type server struct {
@@ -64,9 +65,10 @@ type redis struct {
 }
 
 type logger struct {
-	Path  string `yaml:"path"`
-	Name  string `yaml:"name"`
-	Level string `yaml:"level"`
+	Switch int    `yaml:"switch"`
+	Path   string `yaml:"path"`
+	Name   string `yaml:"name"`
+	Level  string `yaml:"level"`
 }
 
 type minio struct {
@@ -101,3 +103,6 @@ type monitNotice struct {
 	Email email `yaml:"email"`
 	Sms   sms   `yaml:"sms"`
 }
+type es struct {
+	URL string `yaml:"es_url"`
+}

config/config.yaml

@@ -6,9 +6,11 @@ server:
 
 # Logger configurations.
 logger:
-  path: "./log"
+  # 1开启  0关闭
+  switch: 1
+  path:  "./log"
   level: "info"
-  name: "info"
+  name:  "info"
 
 # Database configurations.
 database:
@@ -52,4 +54,7 @@ monit-notice:
       secret_id: "AKID0tkPq1wc9TXMvEHZ5NRFo7AxVotkNltl"
       secret_key: "g7MNBk36tuhIgplSuAE6lCRVLvFTYR5U"
       sms_key: "湖南省龙驰照明"
-      warn_sms_template_id: "1533523"
+      warn_sms_template_id: "1533523"
+# switch 0-关闭 其他(1)-开启
+es:
+  es_url: "localhost:9200"

go.mod

@@ -39,17 +39,20 @@ require (
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/jonboulle/clockwork v0.3.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.15.9 // indirect
 	github.com/klauspost/cpuid/v2 v2.1.0 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/lestrrat/go-envload v0.0.0-20180220120943-6ed08b54a570 // indirect
 	github.com/lestrrat/go-strftime v0.0.0-20180220042222-ba3bf9c1d042 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/olivere/elastic v6.2.37+incompatible // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/onsi/gomega v1.20.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.1 // indirect

main.go

@@ -49,6 +49,7 @@ func main() {
 		println(gin.ReleaseMode)
 	}
 	_ = engine.Run(config.Instance().Server.Address)
+
 }
 
 func initDB() {
@@ -93,8 +94,9 @@ func initDB() {
 }
 
 // isDevEnv 是否开发环境 本地开发需要添加环境变量
-// 		[windows] CMD下执行 setx "DEV_ENV" "1"
-//		[linux] export DEV_ENV="1"
+//
+//	[windows] CMD下执行 setx "DEV_ENV" "1"
+//	[linux] export DEV_ENV="1"
 func isDevEnv() bool {
 	//return true
 	if os.Getenv("DEV_ENV") == "1" {

router/router.go

@@ -3,6 +3,7 @@ package router
 import (
 	"github.com/gin-gonic/gin"
 	device "iot_manager_service/app/device/controller"
+	"iot_manager_service/app/logapi"
 	"iot_manager_service/app/middleware"
 	multimedia "iot_manager_service/app/multimedia/controller"
 	operation "iot_manager_service/app/operation/controller"
@@ -10,6 +11,7 @@ import (
 	security "iot_manager_service/app/security/controller"
 	system "iot_manager_service/app/system/controller"
 	warn "iot_manager_service/app/warn/controller"
+	"iot_manager_service/config"
 	"iot_manager_service/util/logger"
 )
 
@@ -19,7 +21,9 @@ func InitRouter(engine *gin.Engine) {
 	// 校验权限
 	engine.Use(middleware.CheckAuth())
 	engine.Use(gin.Recovery())
-	engine.Use(logger.LogToFile())
+	if config.Instance().Logger.Switch == 1 {
+		engine.Use(logger.LogToFile())
+	}
 	engine.Use(logger.Recover)
 
 	// 操作历史记录
@@ -635,5 +639,9 @@ func InitRouter(engine *gin.Engine) {
 		//报表统计
 		captureGroup.GET("report/list", operation.Capture.ReportList)
 	}
-
+	logGroup := engine.Group("/api/blade-log/usual")
+	{
+		logGroup.GET("/list", logapi.LogController.List)
+		logGroup.GET("/detail", logapi.LogController.Detail)
+	}
 }

util/es/elastic.go

@@ -0,0 +1,48 @@
+package es
+
+import (
+	"context"
+	"fmt"
+	"github.com/olivere/elastic"
+	"iot_manager_service/config"
+	"log"
+	"os"
+	"time"
+)
+
+type es struct {
+}
+
+var Es es
+var Client *elastic.Client
+
+func (e *es) InitES() {
+	errLog := log.New(os.Stdout, "APP", log.LstdFlags)
+	url := config.Instance().Es.URL
+	client, err := elastic.NewClient(elastic.SetErrorLog(errLog))
+	elastic.SetURL(url)
+	if err != nil {
+		//todo 初始化错误处理
+		panic(err)
+	}
+	Client = client
+	//启动定时删除
+	go delTicker()
+}
+
+// 定时器
+func delTicker() {
+	ticker := time.Tick(15 * 24 * time.Hour)
+	for range ticker {
+		indexName := time.Now().Add(-15 * 24 * time.Hour).Format("2006-01-02")
+		delIndex(indexName)
+	}
+}
+
+// DelIndex 删除索引
+func delIndex(indexName string) {
+	res, err := Client.DeleteIndex(indexName).Do(context.Background())
+	if err != nil {
+		fmt.Printf("res:%v\nerror:%v\n", res, err)
+	}
+}

util/es/esWriter.go

@@ -0,0 +1,25 @@
+package es
+
+import (
+	"context"
+	"time"
+)
+
+var EWriter = &esWriter{}
+
+type esWriter struct {
+}
+
+func (e *esWriter) Write(p []byte) (int, error) {
+	str := string(p)
+	Client.Index().
+		Index(getIndexName()).
+		Type("_doc").
+		BodyJson(str).Do(context.Background())
+	return len(p), nil
+}
+
+// 当天日期作为索引名
+func getIndexName() string {
+	return time.Now().Format("2006-01-02")
+}

util/logger/lclog.go

@@ -1,34 +1,34 @@
 package logger
 
 import (
-	"github.com/druidcaesa/gotool"
-	rotatelogs "github.com/lestrrat/go-file-rotatelogs"
 	"github.com/sirupsen/logrus"
 	"iot_manager_service/config"
-	"os"
-	"path"
-	"time"
+	"iot_manager_service/util/es"
 )
 
 var Logger *logrus.Logger
 
 func InitLog() {
-	logFilePath := config.Instance().Logger.Path
-	logFileName := config.Instance().Logger.Name
+	//logFilePath := config.Instance().Logger.Path
+	//logFileName := config.Instance().Logger.Name
 
-	err := os.MkdirAll(logFilePath, os.ModeDir)
-	if err != nil {
-		gotool.Logs.ErrorLog().Println(err)
-		panic(err)
-	}
+	//err := os.MkdirAll(logFilePath, os.ModeDir)
+	//if err != nil {
+	//	gotool.Logs.ErrorLog().Println(err)
+	//	panic(err)
+	//}
 
 	// 日志文件
-	fileName := path.Join(logFilePath, logFileName)
-	writer, _ := rotatelogs.New(
-		fileName+".%Y%m%d.log",
-		rotatelogs.WithMaxAge(15*24*time.Hour),    // 文件最大保存时间
-		rotatelogs.WithRotationTime(24*time.Hour), // 日志切割时间间隔
-	)
+	//fileName := path.Join(logFilePath, logFileName)
+	//writer, _ := rotatelogs.New(
+	//	fileName+".%Y%m%d.log",
+	//	rotatelogs.WithMaxAge(15*24*time.Hour),    // 文件最大保存时间
+	//	rotatelogs.WithRotationTime(24*time.Hour), // 日志切割时间间隔
+	//)
+	if config.Instance().Logger.Switch == 0 {
+		return
+	}
+	es.Es.InitES()
 	// 实例化
 	logger := logrus.New()
 
@@ -37,6 +37,6 @@ func InitLog() {
 	})
 	// 设置日志级别
 	logger.SetLevel(logrus.DebugLevel)
-	logger.SetOutput(writer)
+	logger.SetOutput(es.EWriter)
 	Logger = logger
 }

util/logger/logger.go

@@ -10,7 +10,7 @@ import (
 	"time"
 )
 
-// LoggerToFile 日志记录到文件
+// LogToFile 日志记录到文件
 func LogToFile() gin.HandlerFunc {
 	return func(ctx *gin.Context) {
 		// 初始化bodyLogWriter
@@ -26,6 +26,11 @@ func LogToFile() gin.HandlerFunc {
 		reqMethod := ctx.Request.Method
 		// 请求路由
 		reqUri := ctx.Request.RequestURI
+		//如果是查询日志,则跳出,不记录日志
+		if strings.Contains(reqUri, "/api/blade-log/usual") {
+			ctx.Next()
+			return
+		}
 		//请求参数
 		request := getRequestBody(ctx)
 		// 请求IP
@@ -52,6 +57,7 @@ func LogToFile() gin.HandlerFunc {
 			"request":      request,
 			"response":     response,
 			"req_uri":      reqUri,
+			"t":            time.Now().UnixMilli(), //用于es中按时间排序
 		}).Info()
 	}
 }