Skip to content

Commit d8b3653

Browse files
taolx0taolx0
committed
feat:Support blacklist filtering.
1 parent 4522c55 commit d8b3653

File tree

6 files changed

+305
-155
lines changed

6 files changed

+305
-155
lines changed

‎sqle/api/controller/v1/audit_plan.go

Lines changed: 1 addition & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,7 @@ import (
66
"encoding/csv"
77
"fmt"
88
"mime"
9-
"net"
109
"net/http"
11-
"regexp"
1210
"strconv"
1311
"strings"
1412
"time"
@@ -849,93 +847,6 @@ func GetAuditPlanReport(c echo.Context) error {
849847
})
850848
}
851849

852-
func filterSQLsByBlackList(sqls []*AuditPlanSQLReqV1, blackList []*model.BlackListAuditPlanSQL) []*AuditPlanSQLReqV1 {
853-
if len(blackList) == 0 {
854-
return sqls
855-
}
856-
filteredSQLs := []*AuditPlanSQLReqV1{}
857-
filter := ConvertToBlackFilter(blackList)
858-
for _, sql := range sqls {
859-
if filter.HasEndpointInBlackList([]string{sql.Endpoint}) || filter.IsSqlInBlackList(sql.LastReceiveText) {
860-
continue
861-
}
862-
filteredSQLs = append(filteredSQLs, sql)
863-
}
864-
return filteredSQLs
865-
}
866-
867-
func ConvertToBlackFilter(blackList []*model.BlackListAuditPlanSQL) *BlackFilter {
868-
var blackFilter BlackFilter
869-
for _, filter := range blackList {
870-
switch filter.FilterType {
871-
case model.FilterTypeSQL:
872-
blackFilter.BlackSqlList = append(blackFilter.BlackSqlList, utils.FullFuzzySearchRegexp(filter.FilterContent))
873-
case model.FilterTypeHost:
874-
blackFilter.BlackHostList = append(blackFilter.BlackHostList, utils.FullFuzzySearchRegexp(filter.FilterContent))
875-
case model.FilterTypeIP:
876-
ip := net.ParseIP(filter.FilterContent)
877-
if ip == nil {
878-
log.Logger().Errorf("wrong ip in black list,ip:%s", filter.FilterContent)
879-
continue
880-
}
881-
blackFilter.BlackIpList = append(blackFilter.BlackIpList, ip)
882-
case model.FilterTypeCIDR:
883-
_, cidr, err := net.ParseCIDR(filter.FilterContent)
884-
if err != nil {
885-
log.Logger().Errorf("wrong cidr in black list,cidr:%s,err:%v", filter.FilterContent, err)
886-
continue
887-
}
888-
blackFilter.BlackCidrList = append(blackFilter.BlackCidrList, cidr)
889-
}
890-
}
891-
return &blackFilter
892-
}
893-
894-
// 构造BlackFilter的目的是缓存黑名单中需要使用的结构体，在每个循环中复用
895-
type BlackFilter struct {
896-
BlackSqlList []*regexp.Regexp //更换正则匹配提高效率
897-
BlackIpList []net.IP
898-
BlackHostList []*regexp.Regexp
899-
BlackCidrList []*net.IPNet
900-
}
901-
902-
func (f BlackFilter) IsSqlInBlackList(checkSql string) bool {
903-
for _, blackSql := range f.BlackSqlList {
904-
if blackSql.MatchString(checkSql) {
905-
return true
906-
}
907-
}
908-
return false
909-
}
910-
911-
// 输入一组ip若其中有一个ip在黑名单中则返回true
912-
func (f BlackFilter) HasEndpointInBlackList(checkIps []string) bool {
913-
var checkNetIp net.IP
914-
for _, checkIp := range checkIps {
915-
checkNetIp = net.ParseIP(checkIp)
916-
if checkNetIp == nil {
917-
// 无法解析IP，可能是域名，需要正则匹配
918-
for _, blackHost := range f.BlackHostList {
919-
if blackHost.MatchString(checkIp) {
920-
return true
921-
}
922-
}
923-
} else {
924-
for _, blackIp := range f.BlackIpList {
925-
if blackIp.Equal(checkNetIp) {
926-
return true
927-
}
928-
}
929-
for _, blackCidr := range f.BlackCidrList {
930-
if blackCidr.Contains(checkNetIp) {
931-
return true
932-
}
933-
}
934-
}
935-
}
936-
return false
937-
}
938-
939850
type FullSyncAuditPlanSQLsReqV1 struct {
940851
SQLs []*AuditPlanSQLReqV1 `json:"audit_plan_sql_list" form:"audit_plan_sql_list" valid:"dive"`
941852
}
@@ -989,13 +900,7 @@ func FullSyncAuditPlanSQLs(c echo.Context) error {
989900

990901
l := log.NewEntry()
991902
reqSQLs := req.SQLs
992-
blackList, err := s.GetBlackListAuditPlanSQLsByProjectID(model.ProjectUID(projectUid))
993-
if err == nil {
994-
reqSQLs = filterSQLsByBlackList(reqSQLs, blackList)
995-
} else {
996-
l.Warnf("blacklist is not used, err:%v", err)
997-
}
998-
if len(reqSQLs) == 0 {
903+
if len(req.SQLs) == 0 {
999904
return controller.JSONBaseErrorReq(c, nil)
1000905
}
1001906
sqls, err := convertToModelAuditPlanSQL(c, ap, reqSQLs)
@@ -1045,12 +950,6 @@ func PartialSyncAuditPlanSQLs(c echo.Context) error {
1045950

1046951
l := log.NewEntry()
1047952
reqSQLs := req.SQLs
1048-
blackList, err := s.GetBlackListAuditPlanSQLsByProjectID(model.ProjectUID(projectUid))
1049-
if err == nil {
1050-
reqSQLs = filterSQLsByBlackList(reqSQLs, blackList)
1051-
} else {
1052-
l.Warnf("blacklist is not used, err:%v", err)
1053-
}
1054953
if len(reqSQLs) == 0 {
1055954
return controller.JSONBaseErrorReq(c, nil)
1056955
}

‎sqle/api/controller/v1/blacklist.go

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
"context"
55
"fmt"
66
"net/http"
7-
"strings"
87
"time"
98

109
"github.com/actiontech/sqle/sqle/api/controller"
@@ -41,7 +40,6 @@ func CreateBlacklist(c echo.Context) error {
4140
if err != nil {
4241
return controller.JSONBaseErrorReq(c, err)
4342
}
44-
4543
s := model.GetStorage()
4644
err = s.Save(&model.BlackListAuditPlanSQL{
4745
ProjectId: model.ProjectUID(projectUid),
@@ -134,7 +132,7 @@ func UpdateBlacklist(c echo.Context) error {
134132
blacklist.FilterContent = *req.Content
135133
}
136134
if req.Type != nil {
137-
blacklist.FilterType = model.BlacklistFilterType(strings.ToUpper(*req.Type))
135+
blacklist.FilterType = model.BlacklistFilterType(*req.Type)
138136
}
139137
if req.Desc != nil {
140138
blacklist.Desc = *req.Desc
@@ -195,7 +193,7 @@ func GetBlacklist(c echo.Context) error {
195193
}
196194

197195
s := model.GetStorage()
198-
blacklistList, count, err := s.GetBlacklistList(model.ProjectUID(projectUid), model.BlacklistFilterType(strings.ToUpper(req.FilterType)), req.FuzzySearchContent, req.PageIndex, req.PageSize)
196+
blacklistList, count, err := s.GetBlacklistList(model.ProjectUID(projectUid), model.BlacklistFilterType(req.FilterType), req.FuzzySearchContent, req.PageIndex, req.PageSize)
199197
if err != nil {
200198
return controller.JSONBaseErrorReq(c, err)
201199
}

‎sqle/api/controller/v2/audit_plan.go

Lines changed: 9 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -312,21 +312,6 @@ type AuditPlanSQLReqV2 struct {
312312
Endpoints []string `json:"endpoints" from:"endpoints"`
313313
}
314314

315-
func filterSQLsByBlackList(sqls []*AuditPlanSQLReqV2, blackList []*model.BlackListAuditPlanSQL) []*AuditPlanSQLReqV2 {
316-
if len(blackList) == 0 {
317-
return sqls
318-
}
319-
filteredSQLs := []*AuditPlanSQLReqV2{}
320-
filter := v1.ConvertToBlackFilter(blackList)
321-
for _, sql := range sqls {
322-
if filter.HasEndpointInBlackList(sql.Endpoints) || filter.IsSqlInBlackList(sql.LastReceiveText) {
323-
continue
324-
}
325-
filteredSQLs = append(filteredSQLs, sql)
326-
}
327-
return filteredSQLs
328-
}
329-
330315
func convertToModelAuditPlanSQL(dbType string, reqSQLs []*AuditPlanSQLReqV2) ([]*auditplan.SQL, error) {
331316
var p driver.Plugin
332317
var err error
@@ -448,12 +433,6 @@ func PartialSyncAuditPlanSQLs(c echo.Context) error {
448433

449434
l := log.NewEntry()
450435
reqSQLs := req.SQLs
451-
blackList, err := s.GetBlackListAuditPlanSQLsByProjectID(model.ProjectUID(projectUid))
452-
if err == nil {
453-
reqSQLs = filterSQLsByBlackList(reqSQLs, blackList)
454-
} else {
455-
l.Warnf("blacklist is not used, err:%v", err)
456-
}
457436
if len(reqSQLs) == 0 {
458437
return controller.JSONBaseErrorReq(c, nil)
459438
}
@@ -502,12 +481,6 @@ func FullSyncAuditPlanSQLs(c echo.Context) error {
502481

503482
l := log.NewEntry()
504483
reqSQLs := req.SQLs
505-
blackList, err := s.GetBlackListAuditPlanSQLsByProjectID(model.ProjectUID(projectUid))
506-
if err == nil {
507-
reqSQLs = filterSQLsByBlackList(reqSQLs, blackList)
508-
} else {
509-
l.Warnf("blacklist is not used, err:%v", err)
510-
}
511484
if len(reqSQLs) == 0 {
512485
return controller.JSONBaseErrorReq(c, nil)
513486
}
@@ -544,11 +517,6 @@ func UploadInstanceAuditPlanSQLs(c echo.Context) error {
544517
return controller.JSONBaseErrorReq(c, err)
545518
}
546519

547-
projectUid, err := dms.GetPorjectUIDByName(c.Request().Context(), c.Param("project_name"), true)
548-
if err != nil {
549-
return controller.JSONBaseErrorReq(c, err)
550-
}
551-
552520
s := model.GetStorage()
553521

554522
ap, exist, err := s.GetActiveAuditPlanDetail(uint(apID))
@@ -560,13 +528,17 @@ func UploadInstanceAuditPlanSQLs(c echo.Context) error {
560528
}
561529

562530
l := log.NewEntry()
563-
reqSQLs := req.SQLs
564-
blackList, err := s.GetBlackListAuditPlanSQLsByProjectID(model.ProjectUID(projectUid))
565-
if err == nil {
566-
reqSQLs = filterSQLsByBlackList(reqSQLs, blackList)
531+
instance, exist, err := dms.GetInstancesById(c.Request().Context(), ap.InstanceID)
532+
if err != nil {
533+
return controller.JSONBaseErrorReq(c, err)
534+
}
535+
if exist {
536+
ap.Instance = instance
567537
} else {
568-
l.Warnf("blacklist is not used, err:%v", err)
538+
l.Errorf("instance not found, instance id: %s", ap.InstanceID)
569539
}
540+
541+
reqSQLs := req.SQLs
570542
if len(reqSQLs) == 0 {
571543
return controller.JSONBaseErrorReq(c, nil)
572544
}

‎sqle/model/audit_plan.go

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -58,20 +58,20 @@ type AuditPlanSQLV2 struct {
5858
type BlacklistFilterType string
5959

6060
const (
61-
FilterTypeSQL BlacklistFilterType = "SQL"
62-
FilterTypeFpSQL BlacklistFilterType = "FP_SQL"
63-
FilterTypeIP BlacklistFilterType = "IP"
64-
FilterTypeCIDR BlacklistFilterType = "CIDR"
65-
FilterTypeHost BlacklistFilterType = "HOST"
66-
FilterTypeInstance BlacklistFilterType = "INSTANCE"
61+
FilterTypeSQL BlacklistFilterType = "sql"
62+
FilterTypeFpSQL BlacklistFilterType = "fp_sql"
63+
FilterTypeIP BlacklistFilterType = "ip"
64+
FilterTypeCIDR BlacklistFilterType = "cidr"
65+
FilterTypeHost BlacklistFilterType = "host"
66+
FilterTypeInstance BlacklistFilterType = "instance"
6767
)
6868

6969
type BlackListAuditPlanSQL struct {
7070
Model
7171
ProjectId ProjectUID `gorm:"index; not null"`
72-
FilterContent string `json:"filter_content" gorm:"type:varchar(512);not null;"`
72+
FilterContent string `json:"filter_content" gorm:"type:varchar(3000);not null;"`
7373
Desc string `json:"desc" gorm:"type:varchar(512)"`
74-
FilterType BlacklistFilterType `json:"filter_type" gorm:"type:enum('SQL','FP_SQL','IP','CIDR','HOST','INSTANCE');default:'SQL';not null;"`
74+
FilterType BlacklistFilterType `json:"filter_type" gorm:"type:enum('sql','fp_sql','ip','cidr','host','instance');default:'SQL';not null;"`
7575
MatchedCount uint `json:"matched_count" gorm:"default:0"`
7676
LastMatchTime *time.Time `json:"last_match_time"`
7777
}
@@ -89,7 +89,7 @@ func (s *Storage) GetBlacklistByID(projectID ProjectUID, id string) (*BlackListA
8989
return bl, true, errors.New(errors.ConnectStorageError, err)
9090
}
9191

92-
func (s *Storage) GetBlackListAuditPlanSQLsByProjectID(projectID ProjectUID) ([]*BlackListAuditPlanSQL, error) {
92+
func (s *Storage) GetBlackListByProjectID(projectID ProjectUID) ([]*BlackListAuditPlanSQL, error) {
9393
var blackListAPS []*BlackListAuditPlanSQL
9494
err := s.db.Model(BlackListAuditPlanSQL{}).Where("project_id = ?", projectID).Find(&blackListAPS).Error
9595
return blackListAPS, errors.New(errors.ConnectStorageError, err)
@@ -113,6 +113,16 @@ func (s *Storage) GetBlacklistList(projectID ProjectUID, FilterType BlacklistFil
113113
return blackListAPS, uint64(count), errors.New(errors.ConnectStorageError, err)
114114
}
115115

116+
func (s *Storage) BatchUpdateBlackListCount(IdList []uint, matchedCount uint, lastMatchTime time.Time) error {
117+
m := map[string]interface{}{
118+
"matched_count": gorm.Expr("matched_count + ?", matchedCount),
119+
"last_match_time": lastMatchTime,
120+
}
121+
122+
err := s.db.Model(BlackListAuditPlanSQL{}).Where("id in (?)", IdList).Updates(m).Error
123+
return errors.New(errors.ConnectStorageError, err)
124+
}
125+
116126
func (a AuditPlanSQLV2) TableName() string {
117127
return "audit_plan_sqls_v2"
118128
}

‎sqle/model/instance_audit_plan.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,13 @@ type AuditPlanDetail struct {
5050
Instance *Instance `gorm:"-"`
5151
}
5252

53+
func (a AuditPlanDetail) GetInstanceName() string {
54+
if a.Instance == nil {
55+
return ""
56+
}
57+
return a.Instance.Name
58+
}
59+
5360
func (s *Storage) ListActiveAuditPlanDetail() ([]*AuditPlanDetail, error) {
5461
var aps []*AuditPlanDetail
5562
err := s.db.Model(AuditPlanV2{}).Joins("JOIN instance_audit_plans ON instance_audit_plans.id = audit_plans_v2.instance_audit_plan_id").
@@ -95,6 +102,7 @@ func (s *Storage) getAuditPlanDetailByID(id uint, status string) (*AuditPlanDeta
95102
if ap == nil {
96103
return nil, false, nil
97104
}
105+
98106
return ap, true, nil
99107
}
100108

0 commit comments

Comments
 (0)