Skip to content

Commit 5fa5cb3

Browse files
winfredLINwinfredLIN
authored
Merge pull request #2552 from actiontech/white-black-list-impl
新增黑名单,优化白名单
2 parents 9c722bb + 54c4241 commit 5fa5cb3

File tree

11 files changed

+535
-186
lines changed

11 files changed

+535
-186
lines changed

‎sqle/api/controller/v1/audit_plan.go

Lines changed: 1 addition & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,7 @@ import (
66
"encoding/csv"
77
"fmt"
88
"mime"
9-
"net"
109
"net/http"
11-
"regexp"
1210
"strconv"
1311
"strings"
1412
"time"
@@ -849,93 +847,6 @@ func GetAuditPlanReport(c echo.Context) error {
849847
})
850848
}
851849

852-
func filterSQLsByBlackList(sqls []*AuditPlanSQLReqV1, blackList []*model.BlackListAuditPlanSQL) []*AuditPlanSQLReqV1 {
853-
if len(blackList) == 0 {
854-
return sqls
855-
}
856-
filteredSQLs := []*AuditPlanSQLReqV1{}
857-
filter := ConvertToBlackFilter(blackList)
858-
for _, sql := range sqls {
859-
if filter.HasEndpointInBlackList([]string{sql.Endpoint}) || filter.IsSqlInBlackList(sql.LastReceiveText) {
860-
continue
861-
}
862-
filteredSQLs = append(filteredSQLs, sql)
863-
}
864-
return filteredSQLs
865-
}
866-
867-
func ConvertToBlackFilter(blackList []*model.BlackListAuditPlanSQL) *BlackFilter {
868-
var blackFilter BlackFilter
869-
for _, filter := range blackList {
870-
switch filter.FilterType {
871-
case model.FilterTypeSQL:
872-
blackFilter.BlackSqlList = append(blackFilter.BlackSqlList, utils.FullFuzzySearchRegexp(filter.FilterContent))
873-
case model.FilterTypeHost:
874-
blackFilter.BlackHostList = append(blackFilter.BlackHostList, utils.FullFuzzySearchRegexp(filter.FilterContent))
875-
case model.FilterTypeIP:
876-
ip := net.ParseIP(filter.FilterContent)
877-
if ip == nil {
878-
log.Logger().Errorf("wrong ip in black list,ip:%s", filter.FilterContent)
879-
continue
880-
}
881-
blackFilter.BlackIpList = append(blackFilter.BlackIpList, ip)
882-
case model.FilterTypeCIDR:
883-
_, cidr, err := net.ParseCIDR(filter.FilterContent)
884-
if err != nil {
885-
log.Logger().Errorf("wrong cidr in black list,cidr:%s,err:%v", filter.FilterContent, err)
886-
continue
887-
}
888-
blackFilter.BlackCidrList = append(blackFilter.BlackCidrList, cidr)
889-
}
890-
}
891-
return &blackFilter
892-
}
893-
894-
// 构造BlackFilter的目的是缓存黑名单中需要使用的结构体，在每个循环中复用
895-
type BlackFilter struct {
896-
BlackSqlList []*regexp.Regexp //更换正则匹配提高效率
897-
BlackIpList []net.IP
898-
BlackHostList []*regexp.Regexp
899-
BlackCidrList []*net.IPNet
900-
}
901-
902-
func (f BlackFilter) IsSqlInBlackList(checkSql string) bool {
903-
for _, blackSql := range f.BlackSqlList {
904-
if blackSql.MatchString(checkSql) {
905-
return true
906-
}
907-
}
908-
return false
909-
}
910-
911-
// 输入一组ip若其中有一个ip在黑名单中则返回true
912-
func (f BlackFilter) HasEndpointInBlackList(checkIps []string) bool {
913-
var checkNetIp net.IP
914-
for _, checkIp := range checkIps {
915-
checkNetIp = net.ParseIP(checkIp)
916-
if checkNetIp == nil {
917-
// 无法解析IP，可能是域名，需要正则匹配
918-
for _, blackHost := range f.BlackHostList {
919-
if blackHost.MatchString(checkIp) {
920-
return true
921-
}
922-
}
923-
} else {
924-
for _, blackIp := range f.BlackIpList {
925-
if blackIp.Equal(checkNetIp) {
926-
return true
927-
}
928-
}
929-
for _, blackCidr := range f.BlackCidrList {
930-
if blackCidr.Contains(checkNetIp) {
931-
return true
932-
}
933-
}
934-
}
935-
}
936-
return false
937-
}
938-
939850
type FullSyncAuditPlanSQLsReqV1 struct {
940851
SQLs []*AuditPlanSQLReqV1 `json:"audit_plan_sql_list" form:"audit_plan_sql_list" valid:"dive"`
941852
}
@@ -989,13 +900,7 @@ func FullSyncAuditPlanSQLs(c echo.Context) error {
989900

990901
l := log.NewEntry()
991902
reqSQLs := req.SQLs
992-
blackList, err := s.GetBlackListAuditPlanSQLs()
993-
if err == nil {
994-
reqSQLs = filterSQLsByBlackList(reqSQLs, blackList)
995-
} else {
996-
l.Warnf("blacklist is not used, err:%v", err)
997-
}
998-
if len(reqSQLs) == 0 {
903+
if len(req.SQLs) == 0 {
999904
return controller.JSONBaseErrorReq(c, nil)
1000905
}
1001906
sqls, err := convertToModelAuditPlanSQL(c, ap, reqSQLs)
@@ -1045,12 +950,6 @@ func PartialSyncAuditPlanSQLs(c echo.Context) error {
1045950

1046951
l := log.NewEntry()
1047952
reqSQLs := req.SQLs
1048-
blackList, err := s.GetBlackListAuditPlanSQLs()
1049-
if err == nil {
1050-
reqSQLs = filterSQLsByBlackList(reqSQLs, blackList)
1051-
} else {
1052-
l.Warnf("blacklist is not used, err:%v", err)
1053-
}
1054953
if len(reqSQLs) == 0 {
1055954
return controller.JSONBaseErrorReq(c, nil)
1056955
}

‎sqle/api/controller/v1/blacklist.go

Lines changed: 119 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,15 @@
11
package v1
22

33
import (
4+
"context"
5+
"fmt"
6+
"net/http"
47
"time"
58

69
"github.com/actiontech/sqle/sqle/api/controller"
10+
"github.com/actiontech/sqle/sqle/dms"
11+
"github.com/actiontech/sqle/sqle/errors"
12+
"github.com/actiontech/sqle/sqle/model"
713
"github.com/labstack/echo/v4"
814
)
915

@@ -25,7 +31,27 @@ type CreateBlacklistReqV1 struct {
2531
// @Success 200 {object} controller.BaseRes
2632
// @router /v1/projects/{project_name}/blacklist [post]
2733
func CreateBlacklist(c echo.Context) error {
28-
return nil
34+
req := new(CreateBlacklistReqV1)
35+
if err := controller.BindAndValidateReq(c, req); err != nil {
36+
return controller.JSONBaseErrorReq(c, err)
37+
}
38+
39+
projectUid, err := dms.GetPorjectUIDByName(context.TODO(), c.Param("project_name"), true)
40+
if err != nil {
41+
return controller.JSONBaseErrorReq(c, err)
42+
}
43+
s := model.GetStorage()
44+
err = s.Save(&model.BlackListAuditPlanSQL{
45+
ProjectId: model.ProjectUID(projectUid),
46+
FilterType: model.BlacklistFilterType(req.Type),
47+
FilterContent: req.Content,
48+
Desc: req.Desc,
49+
})
50+
if err != nil {
51+
return controller.JSONBaseErrorReq(c, err)
52+
}
53+
54+
return c.JSON(http.StatusOK, controller.NewBaseReq(nil))
2955
}
3056

3157
// DeleteBlacklist
@@ -38,7 +64,28 @@ func CreateBlacklist(c echo.Context) error {
3864
// @Success 200 {object} controller.BaseRes
3965
// @router /v1/projects/{project_name}/blacklist/{blacklist_id}/ [delete]
4066
func DeleteBlacklist(c echo.Context) error {
41-
return nil
67+
blacklistId := c.Param("blacklist_id")
68+
69+
projectUid, err := dms.GetPorjectUIDByName(context.TODO(), c.Param("project_name"))
70+
if err != nil {
71+
return controller.JSONBaseErrorReq(c, err)
72+
}
73+
74+
s := model.GetStorage()
75+
blacklist, exist, err := s.GetBlacklistByID(model.ProjectUID(projectUid), blacklistId)
76+
if err != nil {
77+
return controller.JSONBaseErrorReq(c, err)
78+
}
79+
if !exist {
80+
return controller.JSONBaseErrorReq(c, errors.New(errors.DataNotExist,
81+
fmt.Errorf("blacklist is not exist")))
82+
}
83+
84+
if err := s.Delete(blacklist); err != nil {
85+
return controller.JSONBaseErrorReq(c, err)
86+
}
87+
88+
return c.JSON(http.StatusOK, controller.NewBaseReq(nil))
4289
}
4390

4491
type UpdateBlacklistReqV1 struct {
@@ -60,7 +107,43 @@ type UpdateBlacklistReqV1 struct {
60107
// @Success 200 {object} controller.BaseRes
61108
// @router /v1/projects/{project_name}/blacklist/{blacklist_id}/ [patch]
62109
func UpdateBlacklist(c echo.Context) error {
63-
return nil
110+
req := new(UpdateBlacklistReqV1)
111+
if err := controller.BindAndValidateReq(c, req); err != nil {
112+
return controller.JSONBaseErrorReq(c, err)
113+
}
114+
115+
blacklistId := c.Param("blacklist_id")
116+
projectUid, err := dms.GetPorjectUIDByName(context.TODO(), c.Param("project_name"))
117+
if err != nil {
118+
return controller.JSONBaseErrorReq(c, err)
119+
}
120+
121+
s := model.GetStorage()
122+
blacklist, exist, err := s.GetBlacklistByID(model.ProjectUID(projectUid), blacklistId)
123+
if err != nil {
124+
return controller.JSONBaseErrorReq(c, err)
125+
}
126+
if !exist {
127+
return controller.JSONBaseErrorReq(c, errors.New(errors.DataNotExist,
128+
fmt.Errorf("blacklist is not exist")))
129+
}
130+
131+
if req.Content != nil {
132+
blacklist.FilterContent = *req.Content
133+
}
134+
if req.Type != nil {
135+
blacklist.FilterType = model.BlacklistFilterType(*req.Type)
136+
}
137+
if req.Desc != nil {
138+
blacklist.Desc = *req.Desc
139+
}
140+
141+
err = s.Save(blacklist)
142+
if err != nil {
143+
return controller.JSONBaseErrorReq(c, err)
144+
}
145+
146+
return c.JSON(http.StatusOK, controller.NewBaseReq(nil))
64147
}
65148

66149
type GetBlacklistReqV1 struct {
@@ -99,5 +182,37 @@ type BlacklistResV1 struct {
99182
// @Success 200 {object} v1.GetBlacklistResV1
100183
// @router /v1/projects/{project_name}/blacklist [get]
101184
func GetBlacklist(c echo.Context) error {
102-
return nil
185+
req := new(GetBlacklistReqV1)
186+
if err := controller.BindAndValidateReq(c, req); err != nil {
187+
return controller.JSONBaseErrorReq(c, err)
188+
}
189+
190+
projectUid, err := dms.GetPorjectUIDByName(context.TODO(), c.Param("project_name"))
191+
if err != nil {
192+
return controller.JSONBaseErrorReq(c, err)
193+
}
194+
195+
s := model.GetStorage()
196+
blacklistList, count, err := s.GetBlacklistList(model.ProjectUID(projectUid), model.BlacklistFilterType(req.FilterType), req.FuzzySearchContent, req.PageIndex, req.PageSize)
197+
if err != nil {
198+
return controller.JSONBaseErrorReq(c, err)
199+
}
200+
201+
res := make([]*BlacklistResV1, 0, len(blacklistList))
202+
for _, blacklist := range blacklistList {
203+
res = append(res, &BlacklistResV1{
204+
BlacklistID: blacklist.ID,
205+
Content: blacklist.FilterContent,
206+
Desc: blacklist.Desc,
207+
Type: string(blacklist.FilterType),
208+
MatchedCount: blacklist.MatchedCount,
209+
LastMatchTime: blacklist.LastMatchTime,
210+
})
211+
}
212+
213+
return c.JSON(http.StatusOK, &GetBlacklistResV1{
214+
BaseRes: controller.NewBaseReq(nil),
215+
Data: res,
216+
TotalNums: count,
217+
})
103218
}

‎sqle/api/controller/v1/sql_whitelist.go

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -153,10 +153,10 @@ func DeleteAuditWhitelistById(c echo.Context) error {
153153
}
154154

155155
type GetAuditWhitelistReqV1 struct {
156-
FuzzySearchValue string `json:"fuzzy_value" query:"fuzzy_value" valid:"omitempty"`
157-
FilterMatchType string `json:"filter_match_type" query:"filter_match_type" valid:"omitempty,oneof=exact_match fp_match" enums:"exact_match,fp_match"`
158-
PageIndex uint32 `json:"page_index" query:"page_index" valid:"required"`
159-
PageSize uint32 `json:"page_size" query:"page_size" valid:"required"`
156+
FuzzySearchValue *string `json:"fuzzy_value" query:"fuzzy_value" valid:"omitempty"`
157+
FilterMatchType *string `json:"filter_match_type" query:"filter_match_type" valid:"omitempty,oneof=exact_match fp_match" enums:"exact_match,fp_match"`
158+
PageIndex uint32 `json:"page_index" query:"page_index" valid:"required"`
159+
PageSize uint32 `json:"page_size" query:"page_size" valid:"required"`
160160
}
161161

162162
type GetAuditWhitelistResV1 struct {
@@ -197,17 +197,19 @@ func GetSqlWhitelist(c echo.Context) error {
197197
}
198198

199199
s := model.GetStorage()
200-
sqlWhitelist, count, err := s.GetSqlWhitelistByProjectUID(req.PageIndex, req.PageSize, model.ProjectUID(projectUid))
200+
sqlWhitelist, count, err := s.GetSqlWhitelistByProjectUID(req.PageIndex, req.PageSize, model.ProjectUID(projectUid), req.FuzzySearchValue, req.FilterMatchType)
201201
if err != nil {
202202
return controller.JSONBaseErrorReq(c, err)
203203
}
204204
whitelistRes := make([]*AuditWhitelistResV1, 0, len(sqlWhitelist))
205205
for _, v := range sqlWhitelist {
206206
whitelistRes = append(whitelistRes, &AuditWhitelistResV1{
207-
Id: v.ID,
208-
Value: v.Value,
209-
Desc: v.Desc,
210-
MatchType: v.MatchType,
207+
Id: v.ID,
208+
Value: v.Value,
209+
Desc: v.Desc,
210+
MatchType: v.MatchType,
211+
MatchedCount: uint(v.MatchedCount),
212+
LastMatchTime: v.LastMatchedTime,
211213
})
212214
}
213215
return c.JSON(http.StatusOK, &GetAuditWhitelistResV1{

0 commit comments

Comments
 (0)