From 9907425221a324f35dbad85a7227be57c0444d36 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 5 Mar 2024 16:20:53 +0530 Subject: [PATCH 001/123] wip --- WiringNilCheck.go | 56 +++++++++++++++++++++++++++++++++++++++++++++++ main.go | 2 +- test_wire_gen.go | 21 ++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 WiringNilCheck.go create mode 100644 test_wire_gen.go diff --git a/WiringNilCheck.go b/WiringNilCheck.go new file mode 100644 index 0000000000..83a357d3cc --- /dev/null +++ b/WiringNilCheck.go @@ -0,0 +1,56 @@ +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "log" + "strings" +) + +func main() { + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, "test_wire_gen.go", nil, parser.ParseComments) + if err != nil { + log.Fatalf("Error parsing file: %v", err) + } + + funcName := "InitializeAppTest" + var targetFunc *ast.FuncDecl + for _, decl := range node.Decls { + if fdecl, ok := decl.(*ast.FuncDecl); ok && fdecl.Name.Name == funcName { + targetFunc = fdecl + break + } + } + if targetFunc == nil { + log.Fatalf("Function %s not found in wire_gen.go", funcName) + } + + ast.Inspect(targetFunc.Body, func(n ast.Node) bool { + if callExpr, ok := n.(*ast.CallExpr); ok { + if ident, ok := callExpr.Fun.(*ast.Ident); ok { + if strings.HasPrefix(ident.Name, "Provider") { + fmt.Println("found call to provider function", ident.Name) + response := Provider() + if hasNilFields(response) { + fmt.Println("error, Response object has nil fields") + } + } + } + } + return true + }) +} + +func hasNilFields(response *Response) bool { + if response == nil { + return true + } + //to check for interface and add reflect support + if response.Field1 == nil || response.Field2 == nil { + return true + } + return false +} diff --git a/main.go b/main.go index 2ebf6851f6..37fbd90896 100755 --- a/main.go +++ b/main.go @@ -27,7 +27,7 @@ import ( "syscall" ) -func main() { +func main1() { app, err := InitializeApp() if err != nil { diff --git a/test_wire_gen.go b/test_wire_gen.go new file mode 100644 index 0000000000..4f43649054 --- /dev/null +++ b/test_wire_gen.go @@ -0,0 +1,21 @@ +package main + +import "fmt" + +type Response struct { + Field1 *string + Field2 *int +} + +func Provider() *Response { + // Simulate provider function returning a response object + return &Response{ + Field1: nil, + Field2: nil, + } +} + +func InitializeAppTest() { + response := Provider() + fmt.Println(response) +} From d068535feb224f5fef767334cdc33dc7b6716e27 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 7 Mar 2024 14:42:41 +0530 Subject: [PATCH 002/123] wip --- WiringNilCheck.go | 2 +- main.go | 74 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/WiringNilCheck.go b/WiringNilCheck.go index 83a357d3cc..d9bc0ac1c8 100644 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -9,7 +9,7 @@ import ( "strings" ) -func main() { +func CheckIfNilInWire() { fset := token.NewFileSet() node, err := parser.ParseFile(fset, "test_wire_gen.go", nil, parser.ParseComments) if err != nil { diff --git a/main.go b/main.go index 37fbd90896..73c4d3e9b8 100755 --- a/main.go +++ b/main.go @@ -24,15 +24,21 @@ import ( "log" "os" "os/signal" + "reflect" + "strings" "syscall" + "unsafe" ) -func main1() { +func main() { app, err := InitializeApp() if err != nil { log.Panic(err) } + nilFieldsMap := make(map[string]bool) + checkNilFields(app, nilFieldsMap) + fmt.Println(nilFieldsMap) // gracefulStop start var gracefulStop = make(chan os.Signal) signal.Notify(gracefulStop, syscall.SIGTERM) @@ -48,3 +54,69 @@ func main1() { app.Start() } + +func checkNilFields(obj interface{}, nilObjMap map[string]bool) { + val := reflect.ValueOf(obj) + if val.Kind() == reflect.Ptr { + val = val.Elem() + } + if val.Kind() != reflect.Struct { + return + } + valName := val.Type().Name() + for i := 0; i < val.NumField(); i++ { + field := val.Field(i) + fieldName := val.Type().Field(i).Name + pkgPath := val.Type().PkgPath() + if pkgPath != "main" && !strings.Contains(pkgPath, "devtron-labs/devtron") { + //package not from this repo, ignoring + continue + } + if !canFieldTypeBeNil(field) { // field can not be nil, skip + continue + } else if field.IsNil() { // check if the field is nil + mapEntry := fmt.Sprintf("%s.%s", valName, fieldName) + nilObjMap[mapEntry] = true + continue + } + if canSkipFieldStructCheck(fieldName) { + continue + } + if !isExported(fieldName) && !field.CanInterface() { + unexportedField := GetUnexportedField(field, fieldName) + checkNilFields(unexportedField, nilObjMap) + } else { + // Recurse + checkNilFields(field.Interface(), nilObjMap) + } + } +} + +func canFieldTypeBeNil(field reflect.Value) bool { + kind := field.Kind() + switch kind { + case reflect.Chan, reflect.Func, reflect.Map, reflect.Pointer, reflect.UnsafePointer, + reflect.Interface, reflect.Slice: + return true + default: //other types can not be nil + return false + } +} + +func canSkipFieldStructCheck(fieldName string) bool { + fieldName = strings.ToLower(fieldName) + for _, str := range []string{"logger", "dbconnection", "syncedenforcer"} { + if fieldName == str { + return true + } + } + return false +} + +func GetUnexportedField(field reflect.Value, fieldName string) interface{} { + return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface() +} + +func isExported(fieldName string) bool { + return strings.ToUpper(fieldName[0:1]) == fieldName[0:1] +} From ee89c85601a0b806f87113ea64e7707894f2d94a Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 21 Mar 2024 18:15:53 +0530 Subject: [PATCH 003/123] refactored nil check --- WiringNilCheck.go | 97 ++++++++++++++++++++++++++++++----------------- main.go | 72 ----------------------------------- test_wire_gen.go | 21 ---------- 3 files changed, 63 insertions(+), 127 deletions(-) delete mode 100644 test_wire_gen.go diff --git a/WiringNilCheck.go b/WiringNilCheck.go index d9bc0ac1c8..5daab1ea6b 100644 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -2,55 +2,84 @@ package main import ( "fmt" - "go/ast" - "go/parser" - "go/token" "log" + "reflect" "strings" + "unsafe" ) func CheckIfNilInWire() { - fset := token.NewFileSet() - node, err := parser.ParseFile(fset, "test_wire_gen.go", nil, parser.ParseComments) + app, err := InitializeApp() if err != nil { - log.Fatalf("Error parsing file: %v", err) + log.Panic(err) } + nilFieldsMap := make(map[string]bool) + checkNilFields(app, nilFieldsMap) + fmt.Println(nilFieldsMap) +} - funcName := "InitializeAppTest" - var targetFunc *ast.FuncDecl - for _, decl := range node.Decls { - if fdecl, ok := decl.(*ast.FuncDecl); ok && fdecl.Name.Name == funcName { - targetFunc = fdecl - break - } +func checkNilFields(obj interface{}, nilObjMap map[string]bool) { + val := reflect.ValueOf(obj) + if val.Kind() == reflect.Ptr { + val = val.Elem() } - if targetFunc == nil { - log.Fatalf("Function %s not found in wire_gen.go", funcName) + if val.Kind() != reflect.Struct { + return } - - ast.Inspect(targetFunc.Body, func(n ast.Node) bool { - if callExpr, ok := n.(*ast.CallExpr); ok { - if ident, ok := callExpr.Fun.(*ast.Ident); ok { - if strings.HasPrefix(ident.Name, "Provider") { - fmt.Println("found call to provider function", ident.Name) - response := Provider() - if hasNilFields(response) { - fmt.Println("error, Response object has nil fields") - } - } - } + valName := val.Type().Name() + for i := 0; i < val.NumField(); i++ { + field := val.Field(i) + fieldName := val.Type().Field(i).Name + pkgPath := val.Type().PkgPath() + if pkgPath != "main" && !strings.Contains(pkgPath, "devtron-labs/devtron") { + //package not from this repo, ignoring + continue } - return true - }) + if !canFieldTypeBeNil(field) { // field can not be nil, skip + continue + } else if field.IsNil() { // check if the field is nil + mapEntry := fmt.Sprintf("%s.%s", valName, fieldName) + nilObjMap[mapEntry] = true + continue + } + if canSkipFieldStructCheck(fieldName) { + continue + } + if !isExported(fieldName) && !field.CanInterface() { + unexportedField := GetUnexportedField(field, fieldName) + checkNilFields(unexportedField, nilObjMap) + } else { + // Recurse + checkNilFields(field.Interface(), nilObjMap) + } + } } -func hasNilFields(response *Response) bool { - if response == nil { +func canFieldTypeBeNil(field reflect.Value) bool { + kind := field.Kind() + switch kind { + case reflect.Chan, reflect.Func, reflect.Map, reflect.Pointer, reflect.UnsafePointer, + reflect.Interface, reflect.Slice: return true + default: //other types can not be nil + return false } - //to check for interface and add reflect support - if response.Field1 == nil || response.Field2 == nil { - return true +} + +func canSkipFieldStructCheck(fieldName string) bool { + fieldName = strings.ToLower(fieldName) + for _, str := range []string{"logger", "dbconnection", "syncedenforcer"} { + if fieldName == str { + return true + } } return false } + +func GetUnexportedField(field reflect.Value, fieldName string) interface{} { + return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface() +} + +func isExported(fieldName string) bool { + return strings.ToUpper(fieldName[0:1]) == fieldName[0:1] +} diff --git a/main.go b/main.go index 73c4d3e9b8..2ebf6851f6 100755 --- a/main.go +++ b/main.go @@ -24,10 +24,7 @@ import ( "log" "os" "os/signal" - "reflect" - "strings" "syscall" - "unsafe" ) func main() { @@ -36,9 +33,6 @@ func main() { if err != nil { log.Panic(err) } - nilFieldsMap := make(map[string]bool) - checkNilFields(app, nilFieldsMap) - fmt.Println(nilFieldsMap) // gracefulStop start var gracefulStop = make(chan os.Signal) signal.Notify(gracefulStop, syscall.SIGTERM) @@ -54,69 +48,3 @@ func main() { app.Start() } - -func checkNilFields(obj interface{}, nilObjMap map[string]bool) { - val := reflect.ValueOf(obj) - if val.Kind() == reflect.Ptr { - val = val.Elem() - } - if val.Kind() != reflect.Struct { - return - } - valName := val.Type().Name() - for i := 0; i < val.NumField(); i++ { - field := val.Field(i) - fieldName := val.Type().Field(i).Name - pkgPath := val.Type().PkgPath() - if pkgPath != "main" && !strings.Contains(pkgPath, "devtron-labs/devtron") { - //package not from this repo, ignoring - continue - } - if !canFieldTypeBeNil(field) { // field can not be nil, skip - continue - } else if field.IsNil() { // check if the field is nil - mapEntry := fmt.Sprintf("%s.%s", valName, fieldName) - nilObjMap[mapEntry] = true - continue - } - if canSkipFieldStructCheck(fieldName) { - continue - } - if !isExported(fieldName) && !field.CanInterface() { - unexportedField := GetUnexportedField(field, fieldName) - checkNilFields(unexportedField, nilObjMap) - } else { - // Recurse - checkNilFields(field.Interface(), nilObjMap) - } - } -} - -func canFieldTypeBeNil(field reflect.Value) bool { - kind := field.Kind() - switch kind { - case reflect.Chan, reflect.Func, reflect.Map, reflect.Pointer, reflect.UnsafePointer, - reflect.Interface, reflect.Slice: - return true - default: //other types can not be nil - return false - } -} - -func canSkipFieldStructCheck(fieldName string) bool { - fieldName = strings.ToLower(fieldName) - for _, str := range []string{"logger", "dbconnection", "syncedenforcer"} { - if fieldName == str { - return true - } - } - return false -} - -func GetUnexportedField(field reflect.Value, fieldName string) interface{} { - return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface() -} - -func isExported(fieldName string) bool { - return strings.ToUpper(fieldName[0:1]) == fieldName[0:1] -} diff --git a/test_wire_gen.go b/test_wire_gen.go deleted file mode 100644 index 4f43649054..0000000000 --- a/test_wire_gen.go +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import "fmt" - -type Response struct { - Field1 *string - Field2 *int -} - -func Provider() *Response { - // Simulate provider function returning a response object - return &Response{ - Field1: nil, - Field2: nil, - } -} - -func InitializeAppTest() { - response := Provider() - fmt.Println(response) -} From c68f538909da28513f6cfaa576bacfdf8a38f1aa Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 1 Apr 2024 11:25:52 +0530 Subject: [PATCH 004/123] added runtime main script for creating binary file --- Makefile | 5 ++++- WiringNilCheck.go | 2 +- runTimeMainScript.sh | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) mode change 100644 => 100755 WiringNilCheck.go create mode 100755 runTimeMainScript.sh diff --git a/Makefile b/Makefile index 1034557ae3..e69d9a9789 100755 --- a/Makefile +++ b/Makefile @@ -39,11 +39,14 @@ test-all: test-unit test-unit: go test ./pkg/pipeline -test-integration: +test-integration: make-binary-file-for-nil-check export INTEGRATION_TEST_ENV_ID=$(docker run --env TEST_BRANCH=$TEST_BRANCH --env LATEST_HASH=$LATEST_HASH --privileged -d --name dind-test -v $PWD/tests/integrationTesting/:/tmp/ docker:dind) docker exec ${INTEGRATION_TEST_ENV_ID} sh /tmp/create-test-env.sh docker exec ${INTEGRATION_TEST_ENV_ID} sh /tests/integrationTesting/run-integration-test.sh +make-binary-file-for-nil-check: + ./runTimeMainScript.sh + run: build ./devtron diff --git a/WiringNilCheck.go b/WiringNilCheck.go old mode 100644 new mode 100755 index 5daab1ea6b..70ad4d5706 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -68,7 +68,7 @@ func canFieldTypeBeNil(field reflect.Value) bool { func canSkipFieldStructCheck(fieldName string) bool { fieldName = strings.ToLower(fieldName) - for _, str := range []string{"logger", "dbconnection", "syncedenforcer"} { + for _, str := range []string{"logger", "dbconnection", "syncedenforcer", "client"} { if fieldName == str { return true } diff --git a/runTimeMainScript.sh b/runTimeMainScript.sh new file mode 100755 index 0000000000..71964fd4d6 --- /dev/null +++ b/runTimeMainScript.sh @@ -0,0 +1,16 @@ +chmod 777 WiringNilCheck.go +sed -i 's/func CheckIfNilInWire()/func main()/1' WiringNilCheck.go +sed -i 's/func main()/func tempMain()/1' main.go +sed '1i //go:build ignore' Wire.go +sed '1i //go:build ignore' main.go +sed -i 's/func InitializeApp()/func InitializeApp1()/1' Wire.go +go build -o wirenilcheckbinary *.go +sed -i 's/func InitializeApp1()/func InitializeApp()/1' Wire.go +sed '1,2d' main.go +sed '1,2d' Wire.go +sed -i 's/func tempMain()/func main()/1' main.go +sed -i 's/func main()/func CheckIfNilInWire()/1' WiringNilCheck.go +chmod 557 WiringNilCheck.go +mv wirenilcheckbinary ./tests/integrationTesting +cd ./tests/integrationTesting +ls | grep wirenilcheckbinary # checking whether binary file is created or not \ No newline at end of file From 114dc260653d2e9727f9439bbcaecf5f8782b874 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 3 Apr 2024 15:08:22 +0530 Subject: [PATCH 005/123] added handling for githubClient panic --- WiringNilCheck.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/WiringNilCheck.go b/WiringNilCheck.go index 70ad4d5706..a043c1ea45 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -42,7 +42,7 @@ func checkNilFields(obj interface{}, nilObjMap map[string]bool) { nilObjMap[mapEntry] = true continue } - if canSkipFieldStructCheck(fieldName) { + if canSkipFieldStructCheck(fieldName, valName) { continue } if !isExported(fieldName) && !field.CanInterface() { @@ -66,9 +66,13 @@ func canFieldTypeBeNil(field reflect.Value) bool { } } -func canSkipFieldStructCheck(fieldName string) bool { +func canSkipFieldStructCheck(fieldName, valName string) bool { fieldName = strings.ToLower(fieldName) - for _, str := range []string{"logger", "dbconnection", "syncedenforcer", "client"} { + valName = strings.ToLower(valName) + if valName == "githubclient" && fieldName == "client" { + return true + } + for _, str := range []string{"logger", "dbconnection", "syncedenforcer"} { if fieldName == str { return true } From 68586a4a14d743d603aaf2465abb760067d17448 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Wed, 3 Apr 2024 15:29:31 +0530 Subject: [PATCH 006/123] added some script for creating binary file --- main.go | 8 +++++++- makeBinaryFileForWireNilChecker.sh | 7 +++++++ runTimeMainScript.sh | 16 ---------------- util/GlobalConfig.go | 1 + 4 files changed, 15 insertions(+), 17 deletions(-) create mode 100755 makeBinaryFileForWireNilChecker.sh delete mode 100755 runTimeMainScript.sh diff --git a/main.go b/main.go index 2ebf6851f6..ead5c3ac2b 100755 --- a/main.go +++ b/main.go @@ -21,6 +21,7 @@ import ( "fmt" _ "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" _ "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + util2 "github.com/devtron-labs/devtron/util" "log" "os" "os/signal" @@ -28,7 +29,12 @@ import ( ) func main() { - + globalEnvVariables, err := util2.GetGlobalEnvVariables() + if globalEnvVariables.ExecuteWireNilChecker { + fmt.Println("inside execute function.....") + CheckIfNilInWire() + return + } app, err := InitializeApp() if err != nil { log.Panic(err) diff --git a/makeBinaryFileForWireNilChecker.sh b/makeBinaryFileForWireNilChecker.sh new file mode 100755 index 0000000000..b31bd6de61 --- /dev/null +++ b/makeBinaryFileForWireNilChecker.sh @@ -0,0 +1,7 @@ +echo "creating binary file..." +chmod 777 WiringNilCheck.go +go build -o wirenilcheckbinary . +chmod 557 WiringNilCheck.go +echo "binary file created" +ls | grep wirenilcheckbinary # checking whether binary file is created or not +./wirenilcheckbinary diff --git a/runTimeMainScript.sh b/runTimeMainScript.sh deleted file mode 100755 index 71964fd4d6..0000000000 --- a/runTimeMainScript.sh +++ /dev/null @@ -1,16 +0,0 @@ -chmod 777 WiringNilCheck.go -sed -i 's/func CheckIfNilInWire()/func main()/1' WiringNilCheck.go -sed -i 's/func main()/func tempMain()/1' main.go -sed '1i //go:build ignore' Wire.go -sed '1i //go:build ignore' main.go -sed -i 's/func InitializeApp()/func InitializeApp1()/1' Wire.go -go build -o wirenilcheckbinary *.go -sed -i 's/func InitializeApp1()/func InitializeApp()/1' Wire.go -sed '1,2d' main.go -sed '1,2d' Wire.go -sed -i 's/func tempMain()/func main()/1' main.go -sed -i 's/func main()/func CheckIfNilInWire()/1' WiringNilCheck.go -chmod 557 WiringNilCheck.go -mv wirenilcheckbinary ./tests/integrationTesting -cd ./tests/integrationTesting -ls | grep wirenilcheckbinary # checking whether binary file is created or not \ No newline at end of file diff --git a/util/GlobalConfig.go b/util/GlobalConfig.go index 954a0e99dd..56c558bc1d 100644 --- a/util/GlobalConfig.go +++ b/util/GlobalConfig.go @@ -10,6 +10,7 @@ type GlobalEnvVariables struct { GitOpsRepoPrefix string `env:"GITOPS_REPO_PREFIX" envDefault:""` EnableAsyncInstallDevtronChart bool `env:"ENABLE_ASYNC_INSTALL_DEVTRON_CHART" envDefault:"false"` ExposeCiMetrics bool `env:"EXPOSE_CI_METRICS" envDefault:"false"` + ExecuteWireNilChecker bool `env:"EXECUTE_WIRE_NIL_CHECKER" envDefault:"false"` } func GetGlobalEnvVariables() (*GlobalEnvVariables, error) { From 83a32d883a4c215cf884d2f92b9b9cc7e98bd419 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Wed, 3 Apr 2024 17:49:12 +0530 Subject: [PATCH 007/123] deleted binary file --- WiringNilCheck.go | 6 +++--- makeBinaryFileForWireNilChecker.sh | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/WiringNilCheck.go b/WiringNilCheck.go index a043c1ea45..97b0cb6b7d 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -46,7 +46,7 @@ func checkNilFields(obj interface{}, nilObjMap map[string]bool) { continue } if !isExported(fieldName) && !field.CanInterface() { - unexportedField := GetUnexportedField(field, fieldName) + unexportedField := GetUnexportedField(field) checkNilFields(unexportedField, nilObjMap) } else { // Recurse @@ -69,7 +69,7 @@ func canFieldTypeBeNil(field reflect.Value) bool { func canSkipFieldStructCheck(fieldName, valName string) bool { fieldName = strings.ToLower(fieldName) valName = strings.ToLower(valName) - if valName == "githubclient" && fieldName == "client" { + if valName == "githubclient" && (fieldName == "client" || fieldName == "gitopshelper") { return true } for _, str := range []string{"logger", "dbconnection", "syncedenforcer"} { @@ -80,7 +80,7 @@ func canSkipFieldStructCheck(fieldName, valName string) bool { return false } -func GetUnexportedField(field reflect.Value, fieldName string) interface{} { +func GetUnexportedField(field reflect.Value) interface{} { return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface() } diff --git a/makeBinaryFileForWireNilChecker.sh b/makeBinaryFileForWireNilChecker.sh index b31bd6de61..bde42d398c 100755 --- a/makeBinaryFileForWireNilChecker.sh +++ b/makeBinaryFileForWireNilChecker.sh @@ -1,7 +1,8 @@ echo "creating binary file..." +cp auth_model.conf ./tests/integrationTesting chmod 777 WiringNilCheck.go go build -o wirenilcheckbinary . chmod 557 WiringNilCheck.go -echo "binary file created" -ls | grep wirenilcheckbinary # checking whether binary file is created or not -./wirenilcheckbinary +mv wirenilcheckbinary ./tests/integrationTesting +cd tests/integrationTesting +echo "binary file $(ls | grep wirenilcheckbinary) is created" From a52848991ac9b9415145cb98318f1ee3fd2b2550 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Thu, 4 Apr 2024 11:15:21 +0530 Subject: [PATCH 008/123] updated go version --- ...X\266p\001\020\001\005\300p\001\300\260\001\300\260\001ad" | 0 tests/integrationTesting/run-integration-test.sh | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 "tests/integrationTesting/\001\360\303\260\001@h\267K\v@8\016@10\006\004@@@@@\020\003\020\003\b\003\004P\003P\003@P\003@\034\034\001\001\004@@X\266p\001X\266p\001\020\001\005\300p\001\300\260\001\300\260\001ad" diff --git "a/tests/integrationTesting/\001\360\303\260\001@h\267K\v@8\016@10\006\004@@@@@\020\003\020\003\b\003\004P\003P\003@P\003@\034\034\001\001\004@@X\266p\001X\266p\001\020\001\005\300p\001\300\260\001\300\260\001ad" "b/tests/integrationTesting/\001\360\303\260\001@h\267K\v@8\016@10\006\004@@@@@\020\003\020\003\b\003\004P\003P\003@P\003@\034\034\001\001\004@@X\266p\001X\266p\001\020\001\005\300p\001\300\260\001\300\260\001ad" new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integrationTesting/run-integration-test.sh b/tests/integrationTesting/run-integration-test.sh index 625033145d..70a791ca9b 100644 --- a/tests/integrationTesting/run-integration-test.sh +++ b/tests/integrationTesting/run-integration-test.sh @@ -1,5 +1,5 @@ -wget https://go.dev/dl/go1.18.10.linux-amd64.tar.gz -O go1.18.10.tar.gz -rm -rf /usr/local/go && tar -C /usr/local -xzf go1.18.10.tar.gz +wget https://go.dev/dl/go1.22.1.linux-amd64.tar.gz -O go1.22.1.tar.gz +rm -rf /usr/local/go && tar -C /usr/local -xzf go1.22.1.tar.gz export GOPATH='/usr/local/go' export PATH=$PATH:$GOPATH/bin #go test ./pkg/pipeline From 910d88ffa1c5d437a3ffa9f327cbf0b86e3eb3e6 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 5 Apr 2024 15:48:57 +0530 Subject: [PATCH 009/123] added env file support and also added other svc requirements --- Makefile | 12 +- tests/integrationTesting/create-test-env.sh | 7 +- .../devtron-nats-nodeport.yaml | 37 ++ tests/integrationTesting/devtron-secret.yaml | 14 + tests/integrationTesting/migrator.yaml | 2 +- tests/integrationTesting/nats-server.yaml | 325 ++++++++++++++++++ tests/integrationTesting/postgresql.yaml | 3 +- .../run-integration-test.sh | 2 +- wireNil.env | 41 +++ 9 files changed, 431 insertions(+), 12 deletions(-) mode change 100644 => 100755 tests/integrationTesting/create-test-env.sh create mode 100644 tests/integrationTesting/devtron-nats-nodeport.yaml create mode 100755 tests/integrationTesting/devtron-secret.yaml create mode 100644 tests/integrationTesting/nats-server.yaml mode change 100644 => 100755 tests/integrationTesting/run-integration-test.sh create mode 100644 wireNil.env diff --git a/Makefile b/Makefile index e69d9a9789..2f420f3c2b 100755 --- a/Makefile +++ b/Makefile @@ -39,14 +39,10 @@ test-all: test-unit test-unit: go test ./pkg/pipeline -test-integration: make-binary-file-for-nil-check - export INTEGRATION_TEST_ENV_ID=$(docker run --env TEST_BRANCH=$TEST_BRANCH --env LATEST_HASH=$LATEST_HASH --privileged -d --name dind-test -v $PWD/tests/integrationTesting/:/tmp/ docker:dind) - docker exec ${INTEGRATION_TEST_ENV_ID} sh /tmp/create-test-env.sh - docker exec ${INTEGRATION_TEST_ENV_ID} sh /tests/integrationTesting/run-integration-test.sh - -make-binary-file-for-nil-check: - ./runTimeMainScript.sh - +test-integration: + export INTEGRATION_TEST_ENV_ID=$(docker run --env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind) + docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" + docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" run: build ./devtron diff --git a/tests/integrationTesting/create-test-env.sh b/tests/integrationTesting/create-test-env.sh old mode 100644 new mode 100755 index df01de3ccd..4857f896e6 --- a/tests/integrationTesting/create-test-env.sh +++ b/tests/integrationTesting/create-test-env.sh @@ -18,6 +18,9 @@ kubectl create ns devtron-cd kubectl create ns devtron-ci kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/postgresql-secret.yaml kubectl -ndevtroncd apply -f $PWD/tests/integrationTesting/postgresql.yaml +kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/devtron-secret.yaml +kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/nats-server.yaml +kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/devtron-nats-nodeport.yaml yq '(select(.metadata.name == "postgresql-migrate-devtron") | .spec.template.spec.containers[0].env[0].value) = env(TEST_BRANCH)' $PWD/tests/integrationTesting/migrator.yaml -i yq '(select(.metadata.name == "postgresql-migrate-devtron") | .spec.template.spec.containers[0].env[9].value) = env(LATEST_HASH)' $PWD/tests/integrationTesting/migrator.yaml -i kubectl -ndevtroncd apply -f $PWD/tests/integrationTesting/migrator.yaml @@ -50,4 +53,6 @@ helm dependency up helm template devtron . --set installer.modules={cicd} -s templates/workflow.yaml >./argo_wf.yaml kubectl apply -f ./argo_wf.yaml while [ ! $(kubectl -n argo get deployment workflow-controller -o jsonpath="{.status.readyReplicas}") ]; do sleep 10; done -cd $PWD \ No newline at end of file +export NATS_SERVER_HOST=nats://$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address):30236 +export PG_ADDR=$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) +cd $PWD diff --git a/tests/integrationTesting/devtron-nats-nodeport.yaml b/tests/integrationTesting/devtron-nats-nodeport.yaml new file mode 100644 index 0000000000..0e08c20800 --- /dev/null +++ b/tests/integrationTesting/devtron-nats-nodeport.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Service +metadata: + name: devtron-nats-for-nodeport + namespace: devtroncd +spec: + ports: + - name: client + nodePort: 30236 + port: 4222 + protocol: TCP + targetPort: 4222 + - name: cluster + port: 6222 + protocol: TCP + targetPort: 6222 + - name: monitor + port: 8222 + protocol: TCP + targetPort: 8222 + - name: metrics + port: 7777 + protocol: TCP + targetPort: 7777 + - name: leafnodes + port: 7422 + protocol: TCP + targetPort: 7422 + - name: gateways + port: 7522 + protocol: TCP + targetPort: 7522 + selector: + app.kubernetes.io/instance: devtron-nats + app.kubernetes.io/name: nats + sessionAffinity: None + type: NodePort \ No newline at end of file diff --git a/tests/integrationTesting/devtron-secret.yaml b/tests/integrationTesting/devtron-secret.yaml new file mode 100755 index 0000000000..921fbabb14 --- /dev/null +++ b/tests/integrationTesting/devtron-secret.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +data: + PG_PASSWORD: c2hhcmVkLWRldnRyb24tcGc= +kind: Secret +metadata: + annotations: + meta.helm.sh/release-name: orchestrator-oss-shared-cd-dcd + meta.helm.sh/release-namespace: devtroncd + creationTimestamp: "2024-03-19T13:58:54Z" + labels: + app.kubernetes.io/managed-by: Helm + name: devtron-secret + namespace: devtroncd +type: Opaque diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index 33cd1d68af..5ee339c9c5 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -28,7 +28,7 @@ spec: - name: MIGRATE_TO_VERSION value: "0" - name: GIT_HASH - value: be7da471e45a501eba19eaa5f8d08dfe5601598d + value: 606a33bc5e9b15d8b1c404f0dfbd1c70b21b4063 envFrom: - secretRef: name: postgresql-migrator diff --git a/tests/integrationTesting/nats-server.yaml b/tests/integrationTesting/nats-server.yaml new file mode 100644 index 0000000000..76e5144827 --- /dev/null +++ b/tests/integrationTesting/nats-server.yaml @@ -0,0 +1,325 @@ +# Source: nats/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: devtron-nats-config + namespace: "devtroncd" + labels: + helm.sh/chart: nats-0.9.2 + + app.kubernetes.io/name: nats + app.kubernetes.io/instance: devtron-nats + app.kubernetes.io/version: "2.6.3" + app.kubernetes.io/managed-by: Helm +data: + nats.conf: | + # PID file shared with configuration reloader. + pid_file: "/var/run/nats/nats.pid" + ############### + # # + # Monitoring # + # # + ############### + http: 8222 + server_name:$POD_NAME + ################################### + # # + # NATS JetStream # + # # + ################################### + jetstream { + max_mem: 1Gi + domain: devtron-jet + } + lame_duck_duration: 120s +--- +# Source: nats/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: devtron-nats + namespace: "devtroncd" + labels: + helm.sh/chart: nats-0.9.2 + + app.kubernetes.io/name: nats + app.kubernetes.io/instance: devtron-nats + app.kubernetes.io/version: "2.6.3" + app.kubernetes.io/managed-by: Helm +spec: + selector: + + app.kubernetes.io/name: nats + app.kubernetes.io/instance: devtron-nats + clusterIP: None + ports: + - name: client + port: 4222 + - name: cluster + port: 6222 + - name: monitor + port: 8222 + - name: metrics + port: 7777 + - name: leafnodes + port: 7422 + - name: gateways + port: 7522 +--- +# Source: nats/templates/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: devtron-nats + namespace: "devtroncd" + labels: + helm.sh/chart: nats-0.9.2 + + app.kubernetes.io/name: nats + app.kubernetes.io/instance: devtron-nats + app.kubernetes.io/version: "2.6.3" + app.kubernetes.io/managed-by: Helm +spec: + selector: + matchLabels: + + app.kubernetes.io/name: nats + app.kubernetes.io/instance: devtron-nats + replicas: 1 + serviceName: devtron-nats + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "7777" + prometheus.io/scrape: "true" + labels: + + app.kubernetes.io/name: nats + app.kubernetes.io/instance: devtron-nats + spec: + # Common volumes for the containers. + volumes: + - name: config-volume + + configMap: + name: devtron-nats-config + + + # Local volume shared with the reloader. + - name: pid + emptyDir: {} + + ################# + # # + # TLS Volumes # + # # + ################# + + + + # Required to be able to HUP signal and apply config + # reload to the server without restarting the pod. + shareProcessNamespace: true + + ################# + # # + # NATS Server # + # # + ################# + terminationGracePeriodSeconds: 120 + containers: + - name: nats + image: nats:2.9.3-alpine + imagePullPolicy: IfNotPresent + resources: + {} + ports: + - containerPort: 4222 + name: client + - containerPort: 7422 + name: leafnodes + - containerPort: 7522 + name: gateways + - containerPort: 6222 + name: cluster + - containerPort: 8222 + name: monitor + - containerPort: 7777 + name: metrics + + command: + - "nats-server" + - "--config" + - "/etc/nats-config/nats.conf" + + # Required to be able to define an environment variable + # that refers to other environment variables. This env var + # is later used as part of the configuration file. + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: SERVER_NAME + value: $(POD_NAME) + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CLUSTER_ADVERTISE + value: $(POD_NAME).devtron-nats.$(POD_NAMESPACE).svc.cluster.local + volumeMounts: + - name: config-volume + mountPath: /etc/nats-config + - name: pid + mountPath: /var/run/nats + - name: data + mountPath: /tmp/nats/jetstream + + # Liveness/Readiness probes against the monitoring. + # + livenessProbe: + httpGet: + path: / + port: 8222 + initialDelaySeconds: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: 8222 + initialDelaySeconds: 10 + timeoutSeconds: 5 + + # Gracefully stop NATS Server on pod deletion or image upgrade. + # + lifecycle: + preStop: + exec: + # Using the alpine based NATS image, we add an extra sleep that is + # the same amount as the terminationGracePeriodSeconds to allow + # the NATS Server to gracefully terminate the client connections. + # + command: + - "/bin/sh" + - "-c" + - "nats-server -sl=ldm=/var/run/nats/nats.pid && /bin/sleep 120" + + ################################# + # # + # NATS Configuration Reloader # + # # + ################################# + + - name: reloader + image: quay.io/devtron/nats-server-config-reloader:0.6.2 + imagePullPolicy: IfNotPresent + resources: + null + command: + - "nats-server-config-reloader" + - "-pid" + - "/var/run/nats/nats.pid" + - "-config" + - "/etc/nats-config/nats.conf" + volumeMounts: + - name: config-volume + mountPath: /etc/nats-config + - name: pid + mountPath: /var/run/nats + + + ############################## + # # + # NATS Prometheus Exporter # + # # + ############################## + + - name: metrics + image: quay.io/devtron/prometheus-nats-exporter:0.9.0 + imagePullPolicy: IfNotPresent + resources: + {} + args: + - -connz + - -routez + - -subz + - -varz + - -jsz=all + - -prefix=nats + - -use_internal_server_id + - http://localhost:8222/ + ports: + - containerPort: 7777 + name: metrics + + + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 5Gi +--- +# Source: nats/templates/tests/test-request-reply.yaml +apiVersion: v1 +kind: Pod +metadata: + name: "devtron-nats-test-request-reply" + labels: + helm.sh/chart: nats-0.9.2 + + app.kubernetes.io/name: nats + app.kubernetes.io/instance: devtron-nats + app.kubernetes.io/version: "2.6.3" + app.kubernetes.io/managed-by: Helm + annotations: + "helm.sh/hook": test +spec: + containers: + - name: nats-box + image: quay.io/devtron/nats-box + env: + - name: NATS_HOST + value: devtron-nats + command: + - /bin/sh + - -ec + - | + nats reply -s nats://$NATS_HOST:4222 'name.>' --command "echo 1" & + - | + "&&" + - | + name=$(nats request -s nats://$NATS_HOST:4222 name.test '' 2>/dev/null) + - | + "&&" + - | + [ $name = test ] + + restartPolicy: Never +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + kind: Prometheus + app: devtron-nats + chart: nats-0.9.2 + release: monitoring + name: devtron-nats-server +spec: + endpoints: + - interval: 30s + path: /metrics + port: metrics + jobLabel: nats-server + namespaceSelector: + matchNames: + - devtroncd + selector: + matchLabels: + app.kubernetes.io/instance: devtron-nats + app.kubernetes.io/name: nats \ No newline at end of file diff --git a/tests/integrationTesting/postgresql.yaml b/tests/integrationTesting/postgresql.yaml index 1984934afb..699d71393a 100644 --- a/tests/integrationTesting/postgresql.yaml +++ b/tests/integrationTesting/postgresql.yaml @@ -67,9 +67,10 @@ metadata: chart: postgresql-8.6.4 release: "devtron" spec: - type: ClusterIP + type: NodePort ports: - name: tcp-postgresql + nodePort: 30468 port: 5432 targetPort: tcp-postgresql selector: diff --git a/tests/integrationTesting/run-integration-test.sh b/tests/integrationTesting/run-integration-test.sh old mode 100644 new mode 100755 index 70a791ca9b..2feff138d4 --- a/tests/integrationTesting/run-integration-test.sh +++ b/tests/integrationTesting/run-integration-test.sh @@ -2,7 +2,7 @@ wget https://go.dev/dl/go1.22.1.linux-amd64.tar.gz -O go1.22.1.tar.gz rm -rf /usr/local/go && tar -C /usr/local -xzf go1.22.1.tar.gz export GOPATH='/usr/local/go' export PATH=$PATH:$GOPATH/bin -#go test ./pkg/pipeline +go test ./pkg/pipeline go test -v ./pkg/pipeline -run TestExecuteWorkflow exit #to get out of container diff --git a/wireNil.env b/wireNil.env new file mode 100644 index 0000000000..5bce43e19e --- /dev/null +++ b/wireNil.env @@ -0,0 +1,41 @@ +PG_PORT=30468 +HOSTNAME=e2ecc77bfb24 +TEST_BRANCH= +SHLVL=1 +BUILD_LOG_TTL_VALUE_IN_SECS=3600 +HOME=/root +OLDPWD=/ +PG_PASSWORD=devtronpg +RUNTIME_CONFIG_LOCAL_DEV=true +ACD_PASSWORD=TJnZICwS-rmfLfl0 +PG_USER=postgres +MINIO_ACCESS_KEY=YJOczlGFxTR0k05ORsa8 +BLOB_STORAGE_S3_BUCKET_VERSIONED=true +DEVTRON_DEX_SECRET_NAMESPACE=devtroncd +MINIO_ENDPOINT=http://20.72.186.201:9000/minio/ +DIND_COMMIT=65cfcc28ab37cb75e1560e4b4738719c07c6618e +AZURE_GATEWAY_CONNECTION_INSECURE=false +CLOUD_PROVIDER=MINIO +BLOB_STORAGE_S3_ENDPOINT_INSECURE=false +TERM=xterm +ACD_URL=localhost:8000 +DEVTRON_INSTALLATION_TYPE=enterprise +DEVTRONCD_NAMESPACE=devtroncd +BLOB_STORAGE_ENABLED=true +CI_NODE_LABEL_SELECTOR=abc=dbc,bcd=def +CD_LIMIT_CI_CPU=0.5 +PG_DATABASE=orchestrator +ORCH_HOST=http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats +DOCKER_VERSION=26.0.0 +DOCKER_TLS_CERTDIR=/certs +DEVTRON_DEFAULT_NAMESPACE=devtroncd +IN_APP_LOGGING_ENABLED=false +ACD_USERNAME=admin +MINIO_SECRET_KEY=VK7auS21cIeIn5icRjY3KvogEL3SqZMoeq0XL4em +PWD=/test +SCOPED_VARIABLE_ENABLED=true +EXECUTE_WIRE_NIL_CHECKER=false +TEST_BRANCH= +LATEST_HASH= +GOPATH='/usr/local/go' +PATH=$PATH:$GOPATH/bin From 9029f505c86ae80ec9a70cdbf11a8d8c3c99b753 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 5 Apr 2024 16:01:26 +0530 Subject: [PATCH 010/123] updated makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2f420f3c2b..25bc092d5d 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test-unit: go test ./pkg/pipeline test-integration: - export INTEGRATION_TEST_ENV_ID=$(docker run --env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind) + export INTEGRATION_TEST_ENV_ID=$(docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind) docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" run: build From 76f7bbd0c454ebe0b8dde72211d1424117800f34 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 5 Apr 2024 16:19:04 +0530 Subject: [PATCH 011/123] updated makefile --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 25bc092d5d..7833d97f98 100755 --- a/Makefile +++ b/Makefile @@ -43,6 +43,7 @@ test-integration: export INTEGRATION_TEST_ENV_ID=$(docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind) docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" + docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && go run ." run: build ./devtron From 507998ac1326d3c724f27bf55306a4371376bb23 Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Fri, 5 Apr 2024 16:50:45 +0530 Subject: [PATCH 012/123] Update wireNil.env --- wireNil.env | 1 - 1 file changed, 1 deletion(-) diff --git a/wireNil.env b/wireNil.env index 5bce43e19e..35f322ccff 100644 --- a/wireNil.env +++ b/wireNil.env @@ -38,4 +38,3 @@ EXECUTE_WIRE_NIL_CHECKER=false TEST_BRANCH= LATEST_HASH= GOPATH='/usr/local/go' -PATH=$PATH:$GOPATH/bin From 933da407723a24d962e2ad2893199234366be25e Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Fri, 5 Apr 2024 20:09:02 +0530 Subject: [PATCH 013/123] Update Makefile --- Makefile | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 7833d97f98..81e851ce66 100755 --- a/Makefile +++ b/Makefile @@ -39,11 +39,18 @@ test-all: test-unit test-unit: go test ./pkg/pipeline -test-integration: - export INTEGRATION_TEST_ENV_ID=$(docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind) - docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" - docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" - docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && go run ." +.PHONY: test-integration + +test-integration: setup-test-env run-integration-tests + +setup-test-env: wireNil.env + export INTEGRATION_TEST_ENV_ID=$$(docker run --env-file=wireNil.env --privileged -d --name dind-test -v $$PWD/:/test/ docker:dind); \ + docker exec $${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" + +run-integration-tests: setup-test-env + docker exec $${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" + docker exec $${INTEGRATION_TEST_ENV_ID} sh -c "cd test && go run ." + run: build ./devtron From 2ea2791fc1a80594f59c87dd2f0727f177147092 Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Fri, 5 Apr 2024 20:28:39 +0530 Subject: [PATCH 014/123] Update Makefile --- Makefile | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 81e851ce66..d0c6faca8e 100755 --- a/Makefile +++ b/Makefile @@ -39,18 +39,11 @@ test-all: test-unit test-unit: go test ./pkg/pipeline -.PHONY: test-integration - -test-integration: setup-test-env run-integration-tests - -setup-test-env: wireNil.env - export INTEGRATION_TEST_ENV_ID=$$(docker run --env-file=wireNil.env --privileged -d --name dind-test -v $$PWD/:/test/ docker:dind); \ - docker exec $${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" - -run-integration-tests: setup-test-env - docker exec $${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" - docker exec $${INTEGRATION_TEST_ENV_ID} sh -c "cd test && go run ." - +test-integration: + docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind + docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" + docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" + docker exec dind-test sh -c "cd test && go run ." run: build ./devtron From f3f9c70747bc0b3d351c4b42f21ad5971e10d2ec Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Fri, 5 Apr 2024 20:47:41 +0530 Subject: [PATCH 015/123] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d0c6faca8e..b920733d33 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind + docker run --env-file=wireNil.env --privileged -d --name dind-test -v ./:/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c "cd test && go run ." From 2f23bae1650b2730b17693455b0160eaf7d7afe4 Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Fri, 5 Apr 2024 20:52:52 +0530 Subject: [PATCH 016/123] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b920733d33..e84a259b5a 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNil.env --privileged -d --name dind-test -v ./:/test/ docker:dind + docker run --env-file=wireNil.env --privileged -d --name dind-test -v .:/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c "cd test && go run ." From a3b599f0da372d1801284fbc52dbbaaec6b91b95 Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Fri, 5 Apr 2024 20:58:42 +0530 Subject: [PATCH 017/123] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e84a259b5a..d0c6faca8e 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNil.env --privileged -d --name dind-test -v .:/test/ docker:dind + docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c "cd test && go run ." From e6b65f70d831b55ce41f2540d8156f80c658d8fa Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Fri, 5 Apr 2024 21:02:14 +0530 Subject: [PATCH 018/123] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d0c6faca8e..16592bde18 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind + docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWDFF:/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c "cd test && go run ." From 0a06fe9d9c80fa58c50758c381cb84ed940ff8f4 Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Sat, 6 Apr 2024 23:32:45 +0530 Subject: [PATCH 019/123] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 16592bde18..beceb83261 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWDFF:/test/ docker:dind + docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD):/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c "cd test && go run ." From 1102b85c41022fd1d5d2377aee13887b5898b185 Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Sun, 7 Apr 2024 14:59:54 +0530 Subject: [PATCH 020/123] Update wireNil.env --- wireNil.env | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/wireNil.env b/wireNil.env index 35f322ccff..eae413ed10 100644 --- a/wireNil.env +++ b/wireNil.env @@ -34,7 +34,6 @@ ACD_USERNAME=admin MINIO_SECRET_KEY=VK7auS21cIeIn5icRjY3KvogEL3SqZMoeq0XL4em PWD=/test SCOPED_VARIABLE_ENABLED=true -EXECUTE_WIRE_NIL_CHECKER=false +EXECUTE_WIRE_NIL_CHECKER=true TEST_BRANCH= LATEST_HASH= -GOPATH='/usr/local/go' From 4ae2e1b247a01b1e38f9bf24fc985cab7c087997 Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Sun, 7 Apr 2024 15:00:39 +0530 Subject: [PATCH 021/123] Update run-integration-test.sh --- tests/integrationTesting/run-integration-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integrationTesting/run-integration-test.sh b/tests/integrationTesting/run-integration-test.sh index 2feff138d4..33374c6bf6 100755 --- a/tests/integrationTesting/run-integration-test.sh +++ b/tests/integrationTesting/run-integration-test.sh @@ -2,8 +2,8 @@ wget https://go.dev/dl/go1.22.1.linux-amd64.tar.gz -O go1.22.1.tar.gz rm -rf /usr/local/go && tar -C /usr/local -xzf go1.22.1.tar.gz export GOPATH='/usr/local/go' export PATH=$PATH:$GOPATH/bin -go test ./pkg/pipeline -go test -v ./pkg/pipeline -run TestExecuteWorkflow +#go test ./pkg/pipeline +#go test -v ./pkg/pipeline -run TestExecuteWorkflow exit #to get out of container From 9d0a8f36a102f065117cff645cfb6db9afe39ba4 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 8 Apr 2024 14:06:43 +0530 Subject: [PATCH 022/123] resolved merge conflicts --- Makefile | 5 ++- tests/integrationTesting/create-test-env.sh | 3 -- .../devtron-nats-nodeport.yaml | 37 ------------------- tests/integrationTesting/nats-server.yaml | 32 +++++++++------- wireNil.env | 3 ++ 5 files changed, 25 insertions(+), 55 deletions(-) delete mode 100644 tests/integrationTesting/devtron-nats-nodeport.yaml diff --git a/Makefile b/Makefile index beceb83261..65e122082b 100755 --- a/Makefile +++ b/Makefile @@ -40,10 +40,11 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD):/test/ docker:dind + docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD)/:/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" + docker exec dind-test sh -c "cd test && source ./tests/integrationTesting/export_env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" - docker exec dind-test sh -c "cd test && go run ." + docker exec dind-test sh -c 'PG_ADDR=$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$PG_ADDR:30236 sh -c "cd test && go run ."' run: build ./devtron diff --git a/tests/integrationTesting/create-test-env.sh b/tests/integrationTesting/create-test-env.sh index 4857f896e6..675fa4526d 100755 --- a/tests/integrationTesting/create-test-env.sh +++ b/tests/integrationTesting/create-test-env.sh @@ -20,7 +20,6 @@ kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/postgresql-secret.ya kubectl -ndevtroncd apply -f $PWD/tests/integrationTesting/postgresql.yaml kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/devtron-secret.yaml kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/nats-server.yaml -kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/devtron-nats-nodeport.yaml yq '(select(.metadata.name == "postgresql-migrate-devtron") | .spec.template.spec.containers[0].env[0].value) = env(TEST_BRANCH)' $PWD/tests/integrationTesting/migrator.yaml -i yq '(select(.metadata.name == "postgresql-migrate-devtron") | .spec.template.spec.containers[0].env[9].value) = env(LATEST_HASH)' $PWD/tests/integrationTesting/migrator.yaml -i kubectl -ndevtroncd apply -f $PWD/tests/integrationTesting/migrator.yaml @@ -53,6 +52,4 @@ helm dependency up helm template devtron . --set installer.modules={cicd} -s templates/workflow.yaml >./argo_wf.yaml kubectl apply -f ./argo_wf.yaml while [ ! $(kubectl -n argo get deployment workflow-controller -o jsonpath="{.status.readyReplicas}") ]; do sleep 10; done -export NATS_SERVER_HOST=nats://$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address):30236 -export PG_ADDR=$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) cd $PWD diff --git a/tests/integrationTesting/devtron-nats-nodeport.yaml b/tests/integrationTesting/devtron-nats-nodeport.yaml deleted file mode 100644 index 0e08c20800..0000000000 --- a/tests/integrationTesting/devtron-nats-nodeport.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: devtron-nats-for-nodeport - namespace: devtroncd -spec: - ports: - - name: client - nodePort: 30236 - port: 4222 - protocol: TCP - targetPort: 4222 - - name: cluster - port: 6222 - protocol: TCP - targetPort: 6222 - - name: monitor - port: 8222 - protocol: TCP - targetPort: 8222 - - name: metrics - port: 7777 - protocol: TCP - targetPort: 7777 - - name: leafnodes - port: 7422 - protocol: TCP - targetPort: 7422 - - name: gateways - port: 7522 - protocol: TCP - targetPort: 7522 - selector: - app.kubernetes.io/instance: devtron-nats - app.kubernetes.io/name: nats - sessionAffinity: None - type: NodePort \ No newline at end of file diff --git a/tests/integrationTesting/nats-server.yaml b/tests/integrationTesting/nats-server.yaml index 76e5144827..9f1b241cb7 100644 --- a/tests/integrationTesting/nats-server.yaml +++ b/tests/integrationTesting/nats-server.yaml @@ -38,33 +38,39 @@ apiVersion: v1 kind: Service metadata: name: devtron-nats - namespace: "devtroncd" - labels: - helm.sh/chart: nats-0.9.2 - - app.kubernetes.io/name: nats - app.kubernetes.io/instance: devtron-nats - app.kubernetes.io/version: "2.6.3" - app.kubernetes.io/managed-by: Helm + namespace: devtroncd spec: - selector: - - app.kubernetes.io/name: nats - app.kubernetes.io/instance: devtron-nats - clusterIP: None ports: - name: client + nodePort: 30236 port: 4222 + protocol: TCP + targetPort: 4222 - name: cluster port: 6222 + protocol: TCP + targetPort: 6222 - name: monitor port: 8222 + protocol: TCP + targetPort: 8222 - name: metrics port: 7777 + protocol: TCP + targetPort: 7777 - name: leafnodes port: 7422 + protocol: TCP + targetPort: 7422 - name: gateways port: 7522 + protocol: TCP + targetPort: 7522 + selector: + app.kubernetes.io/instance: devtron-nats + app.kubernetes.io/name: nats + sessionAffinity: None + type: NodePort --- # Source: nats/templates/statefulset.yaml apiVersion: apps/v1 diff --git a/wireNil.env b/wireNil.env index eae413ed10..f23b4d372f 100644 --- a/wireNil.env +++ b/wireNil.env @@ -37,3 +37,6 @@ SCOPED_VARIABLE_ENABLED=true EXECUTE_WIRE_NIL_CHECKER=true TEST_BRANCH= LATEST_HASH= +GOPATH=/usr/local/go +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin + From 8626eb86ab1b7eaf9b09905cdce67f2d1648388d Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 8 Apr 2024 14:17:59 +0530 Subject: [PATCH 023/123] updated makefile --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 65e122082b..71fa4f8823 100755 --- a/Makefile +++ b/Makefile @@ -42,7 +42,6 @@ test-unit: test-integration: docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD)/:/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" - docker exec dind-test sh -c "cd test && source ./tests/integrationTesting/export_env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c 'PG_ADDR=$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$PG_ADDR:30236 sh -c "cd test && go run ."' run: build From f7c71b887a1ffd1167920288732d0972a2da9aa7 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 8 Apr 2024 14:31:47 +0530 Subject: [PATCH 024/123] added --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 71fa4f8823..1dfd934e60 100755 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ test-integration: docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD)/:/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" - docker exec dind-test sh -c 'PG_ADDR=$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$PG_ADDR:30236 sh -c "cd test && go run ."' + docker exec dind-test sh -c 'PG_ADDR=$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$(PG_ADDR):30236 sh -c "cd test && go run ."' run: build ./devtron From 8abc51162ae389b0d62f042788417494c877435b Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 8 Apr 2024 14:43:40 +0530 Subject: [PATCH 025/123] added --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1dfd934e60..7d6bdb37f7 100755 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ test-integration: docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD)/:/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" - docker exec dind-test sh -c 'PG_ADDR=$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$(PG_ADDR):30236 sh -c "cd test && go run ."' + docker exec dind-test sh -c 'PG_ADDR=$$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$$PG_ADDR:30236 sh -c "cd test && go run ."' run: build ./devtron From 2585c2542623ee9b5c33a9be021256198a8909c1 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 8 Apr 2024 16:36:09 +0530 Subject: [PATCH 026/123] added --- ...\001X\266p\001\020\001\005\300p\001\300\260\001\300\260\001ad" | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 "tests/integrationTesting/\001\360\303\260\001@h\267K\v@8\016@10\006\004@@@@@\020\003\020\003\b\003\004P\003P\003@P\003@\034\034\001\001\004@@X\266p\001X\266p\001\020\001\005\300p\001\300\260\001\300\260\001ad" diff --git "a/tests/integrationTesting/\001\360\303\260\001@h\267K\v@8\016@10\006\004@@@@@\020\003\020\003\b\003\004P\003P\003@P\003@\034\034\001\001\004@@X\266p\001X\266p\001\020\001\005\300p\001\300\260\001\300\260\001ad" "b/tests/integrationTesting/\001\360\303\260\001@h\267K\v@8\016@10\006\004@@@@@\020\003\020\003\b\003\004P\003P\003@P\003@\034\034\001\001\004@@X\266p\001X\266p\001\020\001\005\300p\001\300\260\001\300\260\001ad" deleted file mode 100644 index e69de29bb2..0000000000 From b75a4a4dfbe6f5f5d979bdb53e717e1a10b6d44f Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 5 Mar 2024 16:20:53 +0530 Subject: [PATCH 027/123] wip --- WiringNilCheck.go | 56 +++++++++++++++++++++++++++++++++++++++++++++++ main.go | 2 +- test_wire_gen.go | 21 ++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 WiringNilCheck.go create mode 100644 test_wire_gen.go diff --git a/WiringNilCheck.go b/WiringNilCheck.go new file mode 100644 index 0000000000..83a357d3cc --- /dev/null +++ b/WiringNilCheck.go @@ -0,0 +1,56 @@ +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "log" + "strings" +) + +func main() { + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, "test_wire_gen.go", nil, parser.ParseComments) + if err != nil { + log.Fatalf("Error parsing file: %v", err) + } + + funcName := "InitializeAppTest" + var targetFunc *ast.FuncDecl + for _, decl := range node.Decls { + if fdecl, ok := decl.(*ast.FuncDecl); ok && fdecl.Name.Name == funcName { + targetFunc = fdecl + break + } + } + if targetFunc == nil { + log.Fatalf("Function %s not found in wire_gen.go", funcName) + } + + ast.Inspect(targetFunc.Body, func(n ast.Node) bool { + if callExpr, ok := n.(*ast.CallExpr); ok { + if ident, ok := callExpr.Fun.(*ast.Ident); ok { + if strings.HasPrefix(ident.Name, "Provider") { + fmt.Println("found call to provider function", ident.Name) + response := Provider() + if hasNilFields(response) { + fmt.Println("error, Response object has nil fields") + } + } + } + } + return true + }) +} + +func hasNilFields(response *Response) bool { + if response == nil { + return true + } + //to check for interface and add reflect support + if response.Field1 == nil || response.Field2 == nil { + return true + } + return false +} diff --git a/main.go b/main.go index 2ebf6851f6..37fbd90896 100755 --- a/main.go +++ b/main.go @@ -27,7 +27,7 @@ import ( "syscall" ) -func main() { +func main1() { app, err := InitializeApp() if err != nil { diff --git a/test_wire_gen.go b/test_wire_gen.go new file mode 100644 index 0000000000..4f43649054 --- /dev/null +++ b/test_wire_gen.go @@ -0,0 +1,21 @@ +package main + +import "fmt" + +type Response struct { + Field1 *string + Field2 *int +} + +func Provider() *Response { + // Simulate provider function returning a response object + return &Response{ + Field1: nil, + Field2: nil, + } +} + +func InitializeAppTest() { + response := Provider() + fmt.Println(response) +} From 739ac11e39200c1b80875ff810ce282aa3e01c6a Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 7 Mar 2024 14:42:41 +0530 Subject: [PATCH 028/123] wip --- WiringNilCheck.go | 2 +- main.go | 74 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/WiringNilCheck.go b/WiringNilCheck.go index 83a357d3cc..d9bc0ac1c8 100644 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -9,7 +9,7 @@ import ( "strings" ) -func main() { +func CheckIfNilInWire() { fset := token.NewFileSet() node, err := parser.ParseFile(fset, "test_wire_gen.go", nil, parser.ParseComments) if err != nil { diff --git a/main.go b/main.go index 37fbd90896..73c4d3e9b8 100755 --- a/main.go +++ b/main.go @@ -24,15 +24,21 @@ import ( "log" "os" "os/signal" + "reflect" + "strings" "syscall" + "unsafe" ) -func main1() { +func main() { app, err := InitializeApp() if err != nil { log.Panic(err) } + nilFieldsMap := make(map[string]bool) + checkNilFields(app, nilFieldsMap) + fmt.Println(nilFieldsMap) // gracefulStop start var gracefulStop = make(chan os.Signal) signal.Notify(gracefulStop, syscall.SIGTERM) @@ -48,3 +54,69 @@ func main1() { app.Start() } + +func checkNilFields(obj interface{}, nilObjMap map[string]bool) { + val := reflect.ValueOf(obj) + if val.Kind() == reflect.Ptr { + val = val.Elem() + } + if val.Kind() != reflect.Struct { + return + } + valName := val.Type().Name() + for i := 0; i < val.NumField(); i++ { + field := val.Field(i) + fieldName := val.Type().Field(i).Name + pkgPath := val.Type().PkgPath() + if pkgPath != "main" && !strings.Contains(pkgPath, "devtron-labs/devtron") { + //package not from this repo, ignoring + continue + } + if !canFieldTypeBeNil(field) { // field can not be nil, skip + continue + } else if field.IsNil() { // check if the field is nil + mapEntry := fmt.Sprintf("%s.%s", valName, fieldName) + nilObjMap[mapEntry] = true + continue + } + if canSkipFieldStructCheck(fieldName) { + continue + } + if !isExported(fieldName) && !field.CanInterface() { + unexportedField := GetUnexportedField(field, fieldName) + checkNilFields(unexportedField, nilObjMap) + } else { + // Recurse + checkNilFields(field.Interface(), nilObjMap) + } + } +} + +func canFieldTypeBeNil(field reflect.Value) bool { + kind := field.Kind() + switch kind { + case reflect.Chan, reflect.Func, reflect.Map, reflect.Pointer, reflect.UnsafePointer, + reflect.Interface, reflect.Slice: + return true + default: //other types can not be nil + return false + } +} + +func canSkipFieldStructCheck(fieldName string) bool { + fieldName = strings.ToLower(fieldName) + for _, str := range []string{"logger", "dbconnection", "syncedenforcer"} { + if fieldName == str { + return true + } + } + return false +} + +func GetUnexportedField(field reflect.Value, fieldName string) interface{} { + return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface() +} + +func isExported(fieldName string) bool { + return strings.ToUpper(fieldName[0:1]) == fieldName[0:1] +} From 81a1b4b7a2e3809dd62211ea2553ac3146bdd817 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Thu, 21 Mar 2024 18:15:53 +0530 Subject: [PATCH 029/123] refactored nil check --- WiringNilCheck.go | 97 ++++++++++++++++++++++++++++++----------------- main.go | 72 ----------------------------------- test_wire_gen.go | 21 ---------- 3 files changed, 63 insertions(+), 127 deletions(-) delete mode 100644 test_wire_gen.go diff --git a/WiringNilCheck.go b/WiringNilCheck.go index d9bc0ac1c8..5daab1ea6b 100644 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -2,55 +2,84 @@ package main import ( "fmt" - "go/ast" - "go/parser" - "go/token" "log" + "reflect" "strings" + "unsafe" ) func CheckIfNilInWire() { - fset := token.NewFileSet() - node, err := parser.ParseFile(fset, "test_wire_gen.go", nil, parser.ParseComments) + app, err := InitializeApp() if err != nil { - log.Fatalf("Error parsing file: %v", err) + log.Panic(err) } + nilFieldsMap := make(map[string]bool) + checkNilFields(app, nilFieldsMap) + fmt.Println(nilFieldsMap) +} - funcName := "InitializeAppTest" - var targetFunc *ast.FuncDecl - for _, decl := range node.Decls { - if fdecl, ok := decl.(*ast.FuncDecl); ok && fdecl.Name.Name == funcName { - targetFunc = fdecl - break - } +func checkNilFields(obj interface{}, nilObjMap map[string]bool) { + val := reflect.ValueOf(obj) + if val.Kind() == reflect.Ptr { + val = val.Elem() } - if targetFunc == nil { - log.Fatalf("Function %s not found in wire_gen.go", funcName) + if val.Kind() != reflect.Struct { + return } - - ast.Inspect(targetFunc.Body, func(n ast.Node) bool { - if callExpr, ok := n.(*ast.CallExpr); ok { - if ident, ok := callExpr.Fun.(*ast.Ident); ok { - if strings.HasPrefix(ident.Name, "Provider") { - fmt.Println("found call to provider function", ident.Name) - response := Provider() - if hasNilFields(response) { - fmt.Println("error, Response object has nil fields") - } - } - } + valName := val.Type().Name() + for i := 0; i < val.NumField(); i++ { + field := val.Field(i) + fieldName := val.Type().Field(i).Name + pkgPath := val.Type().PkgPath() + if pkgPath != "main" && !strings.Contains(pkgPath, "devtron-labs/devtron") { + //package not from this repo, ignoring + continue } - return true - }) + if !canFieldTypeBeNil(field) { // field can not be nil, skip + continue + } else if field.IsNil() { // check if the field is nil + mapEntry := fmt.Sprintf("%s.%s", valName, fieldName) + nilObjMap[mapEntry] = true + continue + } + if canSkipFieldStructCheck(fieldName) { + continue + } + if !isExported(fieldName) && !field.CanInterface() { + unexportedField := GetUnexportedField(field, fieldName) + checkNilFields(unexportedField, nilObjMap) + } else { + // Recurse + checkNilFields(field.Interface(), nilObjMap) + } + } } -func hasNilFields(response *Response) bool { - if response == nil { +func canFieldTypeBeNil(field reflect.Value) bool { + kind := field.Kind() + switch kind { + case reflect.Chan, reflect.Func, reflect.Map, reflect.Pointer, reflect.UnsafePointer, + reflect.Interface, reflect.Slice: return true + default: //other types can not be nil + return false } - //to check for interface and add reflect support - if response.Field1 == nil || response.Field2 == nil { - return true +} + +func canSkipFieldStructCheck(fieldName string) bool { + fieldName = strings.ToLower(fieldName) + for _, str := range []string{"logger", "dbconnection", "syncedenforcer"} { + if fieldName == str { + return true + } } return false } + +func GetUnexportedField(field reflect.Value, fieldName string) interface{} { + return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface() +} + +func isExported(fieldName string) bool { + return strings.ToUpper(fieldName[0:1]) == fieldName[0:1] +} diff --git a/main.go b/main.go index 73c4d3e9b8..2ebf6851f6 100755 --- a/main.go +++ b/main.go @@ -24,10 +24,7 @@ import ( "log" "os" "os/signal" - "reflect" - "strings" "syscall" - "unsafe" ) func main() { @@ -36,9 +33,6 @@ func main() { if err != nil { log.Panic(err) } - nilFieldsMap := make(map[string]bool) - checkNilFields(app, nilFieldsMap) - fmt.Println(nilFieldsMap) // gracefulStop start var gracefulStop = make(chan os.Signal) signal.Notify(gracefulStop, syscall.SIGTERM) @@ -54,69 +48,3 @@ func main() { app.Start() } - -func checkNilFields(obj interface{}, nilObjMap map[string]bool) { - val := reflect.ValueOf(obj) - if val.Kind() == reflect.Ptr { - val = val.Elem() - } - if val.Kind() != reflect.Struct { - return - } - valName := val.Type().Name() - for i := 0; i < val.NumField(); i++ { - field := val.Field(i) - fieldName := val.Type().Field(i).Name - pkgPath := val.Type().PkgPath() - if pkgPath != "main" && !strings.Contains(pkgPath, "devtron-labs/devtron") { - //package not from this repo, ignoring - continue - } - if !canFieldTypeBeNil(field) { // field can not be nil, skip - continue - } else if field.IsNil() { // check if the field is nil - mapEntry := fmt.Sprintf("%s.%s", valName, fieldName) - nilObjMap[mapEntry] = true - continue - } - if canSkipFieldStructCheck(fieldName) { - continue - } - if !isExported(fieldName) && !field.CanInterface() { - unexportedField := GetUnexportedField(field, fieldName) - checkNilFields(unexportedField, nilObjMap) - } else { - // Recurse - checkNilFields(field.Interface(), nilObjMap) - } - } -} - -func canFieldTypeBeNil(field reflect.Value) bool { - kind := field.Kind() - switch kind { - case reflect.Chan, reflect.Func, reflect.Map, reflect.Pointer, reflect.UnsafePointer, - reflect.Interface, reflect.Slice: - return true - default: //other types can not be nil - return false - } -} - -func canSkipFieldStructCheck(fieldName string) bool { - fieldName = strings.ToLower(fieldName) - for _, str := range []string{"logger", "dbconnection", "syncedenforcer"} { - if fieldName == str { - return true - } - } - return false -} - -func GetUnexportedField(field reflect.Value, fieldName string) interface{} { - return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface() -} - -func isExported(fieldName string) bool { - return strings.ToUpper(fieldName[0:1]) == fieldName[0:1] -} diff --git a/test_wire_gen.go b/test_wire_gen.go deleted file mode 100644 index 4f43649054..0000000000 --- a/test_wire_gen.go +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import "fmt" - -type Response struct { - Field1 *string - Field2 *int -} - -func Provider() *Response { - // Simulate provider function returning a response object - return &Response{ - Field1: nil, - Field2: nil, - } -} - -func InitializeAppTest() { - response := Provider() - fmt.Println(response) -} From f04af277a6150743606ca70f21df4456e5559539 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 1 Apr 2024 11:25:52 +0530 Subject: [PATCH 030/123] added runtime main script for creating binary file --- Makefile | 5 ++++- WiringNilCheck.go | 2 +- runTimeMainScript.sh | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) mode change 100644 => 100755 WiringNilCheck.go create mode 100755 runTimeMainScript.sh diff --git a/Makefile b/Makefile index 1034557ae3..e69d9a9789 100755 --- a/Makefile +++ b/Makefile @@ -39,11 +39,14 @@ test-all: test-unit test-unit: go test ./pkg/pipeline -test-integration: +test-integration: make-binary-file-for-nil-check export INTEGRATION_TEST_ENV_ID=$(docker run --env TEST_BRANCH=$TEST_BRANCH --env LATEST_HASH=$LATEST_HASH --privileged -d --name dind-test -v $PWD/tests/integrationTesting/:/tmp/ docker:dind) docker exec ${INTEGRATION_TEST_ENV_ID} sh /tmp/create-test-env.sh docker exec ${INTEGRATION_TEST_ENV_ID} sh /tests/integrationTesting/run-integration-test.sh +make-binary-file-for-nil-check: + ./runTimeMainScript.sh + run: build ./devtron diff --git a/WiringNilCheck.go b/WiringNilCheck.go old mode 100644 new mode 100755 index 5daab1ea6b..70ad4d5706 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -68,7 +68,7 @@ func canFieldTypeBeNil(field reflect.Value) bool { func canSkipFieldStructCheck(fieldName string) bool { fieldName = strings.ToLower(fieldName) - for _, str := range []string{"logger", "dbconnection", "syncedenforcer"} { + for _, str := range []string{"logger", "dbconnection", "syncedenforcer", "client"} { if fieldName == str { return true } diff --git a/runTimeMainScript.sh b/runTimeMainScript.sh new file mode 100755 index 0000000000..71964fd4d6 --- /dev/null +++ b/runTimeMainScript.sh @@ -0,0 +1,16 @@ +chmod 777 WiringNilCheck.go +sed -i 's/func CheckIfNilInWire()/func main()/1' WiringNilCheck.go +sed -i 's/func main()/func tempMain()/1' main.go +sed '1i //go:build ignore' Wire.go +sed '1i //go:build ignore' main.go +sed -i 's/func InitializeApp()/func InitializeApp1()/1' Wire.go +go build -o wirenilcheckbinary *.go +sed -i 's/func InitializeApp1()/func InitializeApp()/1' Wire.go +sed '1,2d' main.go +sed '1,2d' Wire.go +sed -i 's/func tempMain()/func main()/1' main.go +sed -i 's/func main()/func CheckIfNilInWire()/1' WiringNilCheck.go +chmod 557 WiringNilCheck.go +mv wirenilcheckbinary ./tests/integrationTesting +cd ./tests/integrationTesting +ls | grep wirenilcheckbinary # checking whether binary file is created or not \ No newline at end of file From 167c0789e640e89bb837698fdcad8a4c5fe4f7fa Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 3 Apr 2024 15:08:22 +0530 Subject: [PATCH 031/123] added handling for githubClient panic --- WiringNilCheck.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/WiringNilCheck.go b/WiringNilCheck.go index 70ad4d5706..a043c1ea45 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -42,7 +42,7 @@ func checkNilFields(obj interface{}, nilObjMap map[string]bool) { nilObjMap[mapEntry] = true continue } - if canSkipFieldStructCheck(fieldName) { + if canSkipFieldStructCheck(fieldName, valName) { continue } if !isExported(fieldName) && !field.CanInterface() { @@ -66,9 +66,13 @@ func canFieldTypeBeNil(field reflect.Value) bool { } } -func canSkipFieldStructCheck(fieldName string) bool { +func canSkipFieldStructCheck(fieldName, valName string) bool { fieldName = strings.ToLower(fieldName) - for _, str := range []string{"logger", "dbconnection", "syncedenforcer", "client"} { + valName = strings.ToLower(valName) + if valName == "githubclient" && fieldName == "client" { + return true + } + for _, str := range []string{"logger", "dbconnection", "syncedenforcer"} { if fieldName == str { return true } From a701b854ed5d9d4d14fbb2c71238b962627ef788 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Wed, 3 Apr 2024 15:29:31 +0530 Subject: [PATCH 032/123] added some script for creating binary file --- main.go | 8 +++++++- makeBinaryFileForWireNilChecker.sh | 7 +++++++ runTimeMainScript.sh | 16 ---------------- util/GlobalConfig.go | 1 + 4 files changed, 15 insertions(+), 17 deletions(-) create mode 100755 makeBinaryFileForWireNilChecker.sh delete mode 100755 runTimeMainScript.sh diff --git a/main.go b/main.go index 2ebf6851f6..ead5c3ac2b 100755 --- a/main.go +++ b/main.go @@ -21,6 +21,7 @@ import ( "fmt" _ "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" _ "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + util2 "github.com/devtron-labs/devtron/util" "log" "os" "os/signal" @@ -28,7 +29,12 @@ import ( ) func main() { - + globalEnvVariables, err := util2.GetGlobalEnvVariables() + if globalEnvVariables.ExecuteWireNilChecker { + fmt.Println("inside execute function.....") + CheckIfNilInWire() + return + } app, err := InitializeApp() if err != nil { log.Panic(err) diff --git a/makeBinaryFileForWireNilChecker.sh b/makeBinaryFileForWireNilChecker.sh new file mode 100755 index 0000000000..b31bd6de61 --- /dev/null +++ b/makeBinaryFileForWireNilChecker.sh @@ -0,0 +1,7 @@ +echo "creating binary file..." +chmod 777 WiringNilCheck.go +go build -o wirenilcheckbinary . +chmod 557 WiringNilCheck.go +echo "binary file created" +ls | grep wirenilcheckbinary # checking whether binary file is created or not +./wirenilcheckbinary diff --git a/runTimeMainScript.sh b/runTimeMainScript.sh deleted file mode 100755 index 71964fd4d6..0000000000 --- a/runTimeMainScript.sh +++ /dev/null @@ -1,16 +0,0 @@ -chmod 777 WiringNilCheck.go -sed -i 's/func CheckIfNilInWire()/func main()/1' WiringNilCheck.go -sed -i 's/func main()/func tempMain()/1' main.go -sed '1i //go:build ignore' Wire.go -sed '1i //go:build ignore' main.go -sed -i 's/func InitializeApp()/func InitializeApp1()/1' Wire.go -go build -o wirenilcheckbinary *.go -sed -i 's/func InitializeApp1()/func InitializeApp()/1' Wire.go -sed '1,2d' main.go -sed '1,2d' Wire.go -sed -i 's/func tempMain()/func main()/1' main.go -sed -i 's/func main()/func CheckIfNilInWire()/1' WiringNilCheck.go -chmod 557 WiringNilCheck.go -mv wirenilcheckbinary ./tests/integrationTesting -cd ./tests/integrationTesting -ls | grep wirenilcheckbinary # checking whether binary file is created or not \ No newline at end of file diff --git a/util/GlobalConfig.go b/util/GlobalConfig.go index 5728015bef..d82f16e2db 100644 --- a/util/GlobalConfig.go +++ b/util/GlobalConfig.go @@ -19,6 +19,7 @@ type GlobalEnvVariables struct { GitOpsRepoPrefix string `env:"GITOPS_REPO_PREFIX" envDefault:""` EnableAsyncInstallDevtronChart bool `env:"ENABLE_ASYNC_INSTALL_DEVTRON_CHART" envDefault:"false"` ExposeCiMetrics bool `env:"EXPOSE_CI_METRICS" envDefault:"false"` + ExecuteWireNilChecker bool `env:"EXECUTE_WIRE_NIL_CHECKER" envDefault:"false"` } type DevtronSecretConfig struct { From bf90098ecbc4dd0323fd1c79afc25c8a75a8cc24 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Wed, 3 Apr 2024 17:49:12 +0530 Subject: [PATCH 033/123] deleted binary file --- WiringNilCheck.go | 6 +++--- makeBinaryFileForWireNilChecker.sh | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/WiringNilCheck.go b/WiringNilCheck.go index a043c1ea45..97b0cb6b7d 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -46,7 +46,7 @@ func checkNilFields(obj interface{}, nilObjMap map[string]bool) { continue } if !isExported(fieldName) && !field.CanInterface() { - unexportedField := GetUnexportedField(field, fieldName) + unexportedField := GetUnexportedField(field) checkNilFields(unexportedField, nilObjMap) } else { // Recurse @@ -69,7 +69,7 @@ func canFieldTypeBeNil(field reflect.Value) bool { func canSkipFieldStructCheck(fieldName, valName string) bool { fieldName = strings.ToLower(fieldName) valName = strings.ToLower(valName) - if valName == "githubclient" && fieldName == "client" { + if valName == "githubclient" && (fieldName == "client" || fieldName == "gitopshelper") { return true } for _, str := range []string{"logger", "dbconnection", "syncedenforcer"} { @@ -80,7 +80,7 @@ func canSkipFieldStructCheck(fieldName, valName string) bool { return false } -func GetUnexportedField(field reflect.Value, fieldName string) interface{} { +func GetUnexportedField(field reflect.Value) interface{} { return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface() } diff --git a/makeBinaryFileForWireNilChecker.sh b/makeBinaryFileForWireNilChecker.sh index b31bd6de61..bde42d398c 100755 --- a/makeBinaryFileForWireNilChecker.sh +++ b/makeBinaryFileForWireNilChecker.sh @@ -1,7 +1,8 @@ echo "creating binary file..." +cp auth_model.conf ./tests/integrationTesting chmod 777 WiringNilCheck.go go build -o wirenilcheckbinary . chmod 557 WiringNilCheck.go -echo "binary file created" -ls | grep wirenilcheckbinary # checking whether binary file is created or not -./wirenilcheckbinary +mv wirenilcheckbinary ./tests/integrationTesting +cd tests/integrationTesting +echo "binary file $(ls | grep wirenilcheckbinary) is created" From 67f4c3b65f3a73e7fb0075e35b0eb78c62e4b972 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Thu, 4 Apr 2024 11:15:21 +0530 Subject: [PATCH 034/123] updated go version --- ...X\266p\001\020\001\005\300p\001\300\260\001\300\260\001ad" | 0 tests/integrationTesting/run-integration-test.sh | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 "tests/integrationTesting/\001\360\303\260\001@h\267K\v@8\016@10\006\004@@@@@\020\003\020\003\b\003\004P\003P\003@P\003@\034\034\001\001\004@@X\266p\001X\266p\001\020\001\005\300p\001\300\260\001\300\260\001ad" diff --git "a/tests/integrationTesting/\001\360\303\260\001@h\267K\v@8\016@10\006\004@@@@@\020\003\020\003\b\003\004P\003P\003@P\003@\034\034\001\001\004@@X\266p\001X\266p\001\020\001\005\300p\001\300\260\001\300\260\001ad" "b/tests/integrationTesting/\001\360\303\260\001@h\267K\v@8\016@10\006\004@@@@@\020\003\020\003\b\003\004P\003P\003@P\003@\034\034\001\001\004@@X\266p\001X\266p\001\020\001\005\300p\001\300\260\001\300\260\001ad" new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integrationTesting/run-integration-test.sh b/tests/integrationTesting/run-integration-test.sh index 625033145d..70a791ca9b 100644 --- a/tests/integrationTesting/run-integration-test.sh +++ b/tests/integrationTesting/run-integration-test.sh @@ -1,5 +1,5 @@ -wget https://go.dev/dl/go1.18.10.linux-amd64.tar.gz -O go1.18.10.tar.gz -rm -rf /usr/local/go && tar -C /usr/local -xzf go1.18.10.tar.gz +wget https://go.dev/dl/go1.22.1.linux-amd64.tar.gz -O go1.22.1.tar.gz +rm -rf /usr/local/go && tar -C /usr/local -xzf go1.22.1.tar.gz export GOPATH='/usr/local/go' export PATH=$PATH:$GOPATH/bin #go test ./pkg/pipeline From 04dafc1e8e5334e84dae763f8e5f3d1236d3f018 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 5 Apr 2024 15:48:57 +0530 Subject: [PATCH 035/123] added env file support and also added other svc requirements --- Makefile | 12 +- tests/integrationTesting/create-test-env.sh | 7 +- .../devtron-nats-nodeport.yaml | 37 ++ tests/integrationTesting/devtron-secret.yaml | 14 + tests/integrationTesting/migrator.yaml | 2 +- tests/integrationTesting/nats-server.yaml | 325 ++++++++++++++++++ tests/integrationTesting/postgresql.yaml | 3 +- .../run-integration-test.sh | 2 +- wireNil.env | 41 +++ 9 files changed, 431 insertions(+), 12 deletions(-) mode change 100644 => 100755 tests/integrationTesting/create-test-env.sh create mode 100644 tests/integrationTesting/devtron-nats-nodeport.yaml create mode 100755 tests/integrationTesting/devtron-secret.yaml create mode 100644 tests/integrationTesting/nats-server.yaml mode change 100644 => 100755 tests/integrationTesting/run-integration-test.sh create mode 100644 wireNil.env diff --git a/Makefile b/Makefile index e69d9a9789..2f420f3c2b 100755 --- a/Makefile +++ b/Makefile @@ -39,14 +39,10 @@ test-all: test-unit test-unit: go test ./pkg/pipeline -test-integration: make-binary-file-for-nil-check - export INTEGRATION_TEST_ENV_ID=$(docker run --env TEST_BRANCH=$TEST_BRANCH --env LATEST_HASH=$LATEST_HASH --privileged -d --name dind-test -v $PWD/tests/integrationTesting/:/tmp/ docker:dind) - docker exec ${INTEGRATION_TEST_ENV_ID} sh /tmp/create-test-env.sh - docker exec ${INTEGRATION_TEST_ENV_ID} sh /tests/integrationTesting/run-integration-test.sh - -make-binary-file-for-nil-check: - ./runTimeMainScript.sh - +test-integration: + export INTEGRATION_TEST_ENV_ID=$(docker run --env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind) + docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" + docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" run: build ./devtron diff --git a/tests/integrationTesting/create-test-env.sh b/tests/integrationTesting/create-test-env.sh old mode 100644 new mode 100755 index df01de3ccd..4857f896e6 --- a/tests/integrationTesting/create-test-env.sh +++ b/tests/integrationTesting/create-test-env.sh @@ -18,6 +18,9 @@ kubectl create ns devtron-cd kubectl create ns devtron-ci kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/postgresql-secret.yaml kubectl -ndevtroncd apply -f $PWD/tests/integrationTesting/postgresql.yaml +kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/devtron-secret.yaml +kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/nats-server.yaml +kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/devtron-nats-nodeport.yaml yq '(select(.metadata.name == "postgresql-migrate-devtron") | .spec.template.spec.containers[0].env[0].value) = env(TEST_BRANCH)' $PWD/tests/integrationTesting/migrator.yaml -i yq '(select(.metadata.name == "postgresql-migrate-devtron") | .spec.template.spec.containers[0].env[9].value) = env(LATEST_HASH)' $PWD/tests/integrationTesting/migrator.yaml -i kubectl -ndevtroncd apply -f $PWD/tests/integrationTesting/migrator.yaml @@ -50,4 +53,6 @@ helm dependency up helm template devtron . --set installer.modules={cicd} -s templates/workflow.yaml >./argo_wf.yaml kubectl apply -f ./argo_wf.yaml while [ ! $(kubectl -n argo get deployment workflow-controller -o jsonpath="{.status.readyReplicas}") ]; do sleep 10; done -cd $PWD \ No newline at end of file +export NATS_SERVER_HOST=nats://$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address):30236 +export PG_ADDR=$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) +cd $PWD diff --git a/tests/integrationTesting/devtron-nats-nodeport.yaml b/tests/integrationTesting/devtron-nats-nodeport.yaml new file mode 100644 index 0000000000..0e08c20800 --- /dev/null +++ b/tests/integrationTesting/devtron-nats-nodeport.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Service +metadata: + name: devtron-nats-for-nodeport + namespace: devtroncd +spec: + ports: + - name: client + nodePort: 30236 + port: 4222 + protocol: TCP + targetPort: 4222 + - name: cluster + port: 6222 + protocol: TCP + targetPort: 6222 + - name: monitor + port: 8222 + protocol: TCP + targetPort: 8222 + - name: metrics + port: 7777 + protocol: TCP + targetPort: 7777 + - name: leafnodes + port: 7422 + protocol: TCP + targetPort: 7422 + - name: gateways + port: 7522 + protocol: TCP + targetPort: 7522 + selector: + app.kubernetes.io/instance: devtron-nats + app.kubernetes.io/name: nats + sessionAffinity: None + type: NodePort \ No newline at end of file diff --git a/tests/integrationTesting/devtron-secret.yaml b/tests/integrationTesting/devtron-secret.yaml new file mode 100755 index 0000000000..921fbabb14 --- /dev/null +++ b/tests/integrationTesting/devtron-secret.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +data: + PG_PASSWORD: c2hhcmVkLWRldnRyb24tcGc= +kind: Secret +metadata: + annotations: + meta.helm.sh/release-name: orchestrator-oss-shared-cd-dcd + meta.helm.sh/release-namespace: devtroncd + creationTimestamp: "2024-03-19T13:58:54Z" + labels: + app.kubernetes.io/managed-by: Helm + name: devtron-secret + namespace: devtroncd +type: Opaque diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index 33cd1d68af..5ee339c9c5 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -28,7 +28,7 @@ spec: - name: MIGRATE_TO_VERSION value: "0" - name: GIT_HASH - value: be7da471e45a501eba19eaa5f8d08dfe5601598d + value: 606a33bc5e9b15d8b1c404f0dfbd1c70b21b4063 envFrom: - secretRef: name: postgresql-migrator diff --git a/tests/integrationTesting/nats-server.yaml b/tests/integrationTesting/nats-server.yaml new file mode 100644 index 0000000000..76e5144827 --- /dev/null +++ b/tests/integrationTesting/nats-server.yaml @@ -0,0 +1,325 @@ +# Source: nats/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: devtron-nats-config + namespace: "devtroncd" + labels: + helm.sh/chart: nats-0.9.2 + + app.kubernetes.io/name: nats + app.kubernetes.io/instance: devtron-nats + app.kubernetes.io/version: "2.6.3" + app.kubernetes.io/managed-by: Helm +data: + nats.conf: | + # PID file shared with configuration reloader. + pid_file: "/var/run/nats/nats.pid" + ############### + # # + # Monitoring # + # # + ############### + http: 8222 + server_name:$POD_NAME + ################################### + # # + # NATS JetStream # + # # + ################################### + jetstream { + max_mem: 1Gi + domain: devtron-jet + } + lame_duck_duration: 120s +--- +# Source: nats/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: devtron-nats + namespace: "devtroncd" + labels: + helm.sh/chart: nats-0.9.2 + + app.kubernetes.io/name: nats + app.kubernetes.io/instance: devtron-nats + app.kubernetes.io/version: "2.6.3" + app.kubernetes.io/managed-by: Helm +spec: + selector: + + app.kubernetes.io/name: nats + app.kubernetes.io/instance: devtron-nats + clusterIP: None + ports: + - name: client + port: 4222 + - name: cluster + port: 6222 + - name: monitor + port: 8222 + - name: metrics + port: 7777 + - name: leafnodes + port: 7422 + - name: gateways + port: 7522 +--- +# Source: nats/templates/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: devtron-nats + namespace: "devtroncd" + labels: + helm.sh/chart: nats-0.9.2 + + app.kubernetes.io/name: nats + app.kubernetes.io/instance: devtron-nats + app.kubernetes.io/version: "2.6.3" + app.kubernetes.io/managed-by: Helm +spec: + selector: + matchLabels: + + app.kubernetes.io/name: nats + app.kubernetes.io/instance: devtron-nats + replicas: 1 + serviceName: devtron-nats + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "7777" + prometheus.io/scrape: "true" + labels: + + app.kubernetes.io/name: nats + app.kubernetes.io/instance: devtron-nats + spec: + # Common volumes for the containers. + volumes: + - name: config-volume + + configMap: + name: devtron-nats-config + + + # Local volume shared with the reloader. + - name: pid + emptyDir: {} + + ################# + # # + # TLS Volumes # + # # + ################# + + + + # Required to be able to HUP signal and apply config + # reload to the server without restarting the pod. + shareProcessNamespace: true + + ################# + # # + # NATS Server # + # # + ################# + terminationGracePeriodSeconds: 120 + containers: + - name: nats + image: nats:2.9.3-alpine + imagePullPolicy: IfNotPresent + resources: + {} + ports: + - containerPort: 4222 + name: client + - containerPort: 7422 + name: leafnodes + - containerPort: 7522 + name: gateways + - containerPort: 6222 + name: cluster + - containerPort: 8222 + name: monitor + - containerPort: 7777 + name: metrics + + command: + - "nats-server" + - "--config" + - "/etc/nats-config/nats.conf" + + # Required to be able to define an environment variable + # that refers to other environment variables. This env var + # is later used as part of the configuration file. + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: SERVER_NAME + value: $(POD_NAME) + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CLUSTER_ADVERTISE + value: $(POD_NAME).devtron-nats.$(POD_NAMESPACE).svc.cluster.local + volumeMounts: + - name: config-volume + mountPath: /etc/nats-config + - name: pid + mountPath: /var/run/nats + - name: data + mountPath: /tmp/nats/jetstream + + # Liveness/Readiness probes against the monitoring. + # + livenessProbe: + httpGet: + path: / + port: 8222 + initialDelaySeconds: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: 8222 + initialDelaySeconds: 10 + timeoutSeconds: 5 + + # Gracefully stop NATS Server on pod deletion or image upgrade. + # + lifecycle: + preStop: + exec: + # Using the alpine based NATS image, we add an extra sleep that is + # the same amount as the terminationGracePeriodSeconds to allow + # the NATS Server to gracefully terminate the client connections. + # + command: + - "/bin/sh" + - "-c" + - "nats-server -sl=ldm=/var/run/nats/nats.pid && /bin/sleep 120" + + ################################# + # # + # NATS Configuration Reloader # + # # + ################################# + + - name: reloader + image: quay.io/devtron/nats-server-config-reloader:0.6.2 + imagePullPolicy: IfNotPresent + resources: + null + command: + - "nats-server-config-reloader" + - "-pid" + - "/var/run/nats/nats.pid" + - "-config" + - "/etc/nats-config/nats.conf" + volumeMounts: + - name: config-volume + mountPath: /etc/nats-config + - name: pid + mountPath: /var/run/nats + + + ############################## + # # + # NATS Prometheus Exporter # + # # + ############################## + + - name: metrics + image: quay.io/devtron/prometheus-nats-exporter:0.9.0 + imagePullPolicy: IfNotPresent + resources: + {} + args: + - -connz + - -routez + - -subz + - -varz + - -jsz=all + - -prefix=nats + - -use_internal_server_id + - http://localhost:8222/ + ports: + - containerPort: 7777 + name: metrics + + + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 5Gi +--- +# Source: nats/templates/tests/test-request-reply.yaml +apiVersion: v1 +kind: Pod +metadata: + name: "devtron-nats-test-request-reply" + labels: + helm.sh/chart: nats-0.9.2 + + app.kubernetes.io/name: nats + app.kubernetes.io/instance: devtron-nats + app.kubernetes.io/version: "2.6.3" + app.kubernetes.io/managed-by: Helm + annotations: + "helm.sh/hook": test +spec: + containers: + - name: nats-box + image: quay.io/devtron/nats-box + env: + - name: NATS_HOST + value: devtron-nats + command: + - /bin/sh + - -ec + - | + nats reply -s nats://$NATS_HOST:4222 'name.>' --command "echo 1" & + - | + "&&" + - | + name=$(nats request -s nats://$NATS_HOST:4222 name.test '' 2>/dev/null) + - | + "&&" + - | + [ $name = test ] + + restartPolicy: Never +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + kind: Prometheus + app: devtron-nats + chart: nats-0.9.2 + release: monitoring + name: devtron-nats-server +spec: + endpoints: + - interval: 30s + path: /metrics + port: metrics + jobLabel: nats-server + namespaceSelector: + matchNames: + - devtroncd + selector: + matchLabels: + app.kubernetes.io/instance: devtron-nats + app.kubernetes.io/name: nats \ No newline at end of file diff --git a/tests/integrationTesting/postgresql.yaml b/tests/integrationTesting/postgresql.yaml index 1984934afb..699d71393a 100644 --- a/tests/integrationTesting/postgresql.yaml +++ b/tests/integrationTesting/postgresql.yaml @@ -67,9 +67,10 @@ metadata: chart: postgresql-8.6.4 release: "devtron" spec: - type: ClusterIP + type: NodePort ports: - name: tcp-postgresql + nodePort: 30468 port: 5432 targetPort: tcp-postgresql selector: diff --git a/tests/integrationTesting/run-integration-test.sh b/tests/integrationTesting/run-integration-test.sh old mode 100644 new mode 100755 index 70a791ca9b..2feff138d4 --- a/tests/integrationTesting/run-integration-test.sh +++ b/tests/integrationTesting/run-integration-test.sh @@ -2,7 +2,7 @@ wget https://go.dev/dl/go1.22.1.linux-amd64.tar.gz -O go1.22.1.tar.gz rm -rf /usr/local/go && tar -C /usr/local -xzf go1.22.1.tar.gz export GOPATH='/usr/local/go' export PATH=$PATH:$GOPATH/bin -#go test ./pkg/pipeline +go test ./pkg/pipeline go test -v ./pkg/pipeline -run TestExecuteWorkflow exit #to get out of container diff --git a/wireNil.env b/wireNil.env new file mode 100644 index 0000000000..5bce43e19e --- /dev/null +++ b/wireNil.env @@ -0,0 +1,41 @@ +PG_PORT=30468 +HOSTNAME=e2ecc77bfb24 +TEST_BRANCH= +SHLVL=1 +BUILD_LOG_TTL_VALUE_IN_SECS=3600 +HOME=/root +OLDPWD=/ +PG_PASSWORD=devtronpg +RUNTIME_CONFIG_LOCAL_DEV=true +ACD_PASSWORD=TJnZICwS-rmfLfl0 +PG_USER=postgres +MINIO_ACCESS_KEY=YJOczlGFxTR0k05ORsa8 +BLOB_STORAGE_S3_BUCKET_VERSIONED=true +DEVTRON_DEX_SECRET_NAMESPACE=devtroncd +MINIO_ENDPOINT=http://20.72.186.201:9000/minio/ +DIND_COMMIT=65cfcc28ab37cb75e1560e4b4738719c07c6618e +AZURE_GATEWAY_CONNECTION_INSECURE=false +CLOUD_PROVIDER=MINIO +BLOB_STORAGE_S3_ENDPOINT_INSECURE=false +TERM=xterm +ACD_URL=localhost:8000 +DEVTRON_INSTALLATION_TYPE=enterprise +DEVTRONCD_NAMESPACE=devtroncd +BLOB_STORAGE_ENABLED=true +CI_NODE_LABEL_SELECTOR=abc=dbc,bcd=def +CD_LIMIT_CI_CPU=0.5 +PG_DATABASE=orchestrator +ORCH_HOST=http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats +DOCKER_VERSION=26.0.0 +DOCKER_TLS_CERTDIR=/certs +DEVTRON_DEFAULT_NAMESPACE=devtroncd +IN_APP_LOGGING_ENABLED=false +ACD_USERNAME=admin +MINIO_SECRET_KEY=VK7auS21cIeIn5icRjY3KvogEL3SqZMoeq0XL4em +PWD=/test +SCOPED_VARIABLE_ENABLED=true +EXECUTE_WIRE_NIL_CHECKER=false +TEST_BRANCH= +LATEST_HASH= +GOPATH='/usr/local/go' +PATH=$PATH:$GOPATH/bin From 9958f7036d8cad6b89b9ba2323176676cad33739 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 5 Apr 2024 16:01:26 +0530 Subject: [PATCH 036/123] updated makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2f420f3c2b..25bc092d5d 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test-unit: go test ./pkg/pipeline test-integration: - export INTEGRATION_TEST_ENV_ID=$(docker run --env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind) + export INTEGRATION_TEST_ENV_ID=$(docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind) docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" run: build From e660b6a48559b2a96fcecde8c07247e5ba978376 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 5 Apr 2024 16:19:04 +0530 Subject: [PATCH 037/123] updated makefile --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 25bc092d5d..7833d97f98 100755 --- a/Makefile +++ b/Makefile @@ -43,6 +43,7 @@ test-integration: export INTEGRATION_TEST_ENV_ID=$(docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind) docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" + docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && go run ." run: build ./devtron From 2f1a154fffbdec5dde3c511ef35ff458c914f82c Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Fri, 5 Apr 2024 16:50:45 +0530 Subject: [PATCH 038/123] Update wireNil.env --- wireNil.env | 1 - 1 file changed, 1 deletion(-) diff --git a/wireNil.env b/wireNil.env index 5bce43e19e..35f322ccff 100644 --- a/wireNil.env +++ b/wireNil.env @@ -38,4 +38,3 @@ EXECUTE_WIRE_NIL_CHECKER=false TEST_BRANCH= LATEST_HASH= GOPATH='/usr/local/go' -PATH=$PATH:$GOPATH/bin From 36feae0059d2f9f03c3ec2f9c3352eaeb001fbb9 Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Fri, 5 Apr 2024 20:09:02 +0530 Subject: [PATCH 039/123] Update Makefile --- Makefile | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 7833d97f98..81e851ce66 100755 --- a/Makefile +++ b/Makefile @@ -39,11 +39,18 @@ test-all: test-unit test-unit: go test ./pkg/pipeline -test-integration: - export INTEGRATION_TEST_ENV_ID=$(docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind) - docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" - docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" - docker exec ${INTEGRATION_TEST_ENV_ID} sh -c "cd test && go run ." +.PHONY: test-integration + +test-integration: setup-test-env run-integration-tests + +setup-test-env: wireNil.env + export INTEGRATION_TEST_ENV_ID=$$(docker run --env-file=wireNil.env --privileged -d --name dind-test -v $$PWD/:/test/ docker:dind); \ + docker exec $${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" + +run-integration-tests: setup-test-env + docker exec $${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" + docker exec $${INTEGRATION_TEST_ENV_ID} sh -c "cd test && go run ." + run: build ./devtron From 54886a2d7cebc1e3daa5c3c7cbd937509f6ae913 Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Fri, 5 Apr 2024 20:28:39 +0530 Subject: [PATCH 040/123] Update Makefile --- Makefile | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 81e851ce66..d0c6faca8e 100755 --- a/Makefile +++ b/Makefile @@ -39,18 +39,11 @@ test-all: test-unit test-unit: go test ./pkg/pipeline -.PHONY: test-integration - -test-integration: setup-test-env run-integration-tests - -setup-test-env: wireNil.env - export INTEGRATION_TEST_ENV_ID=$$(docker run --env-file=wireNil.env --privileged -d --name dind-test -v $$PWD/:/test/ docker:dind); \ - docker exec $${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" - -run-integration-tests: setup-test-env - docker exec $${INTEGRATION_TEST_ENV_ID} sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" - docker exec $${INTEGRATION_TEST_ENV_ID} sh -c "cd test && go run ." - +test-integration: + docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind + docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" + docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" + docker exec dind-test sh -c "cd test && go run ." run: build ./devtron From a1fc7df748f137e6312a2a450270bc02bb336745 Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Fri, 5 Apr 2024 20:47:41 +0530 Subject: [PATCH 041/123] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d0c6faca8e..b920733d33 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind + docker run --env-file=wireNil.env --privileged -d --name dind-test -v ./:/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c "cd test && go run ." From 45da527d411fe58d42d82d2568090cb6687f3539 Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Fri, 5 Apr 2024 20:52:52 +0530 Subject: [PATCH 042/123] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b920733d33..e84a259b5a 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNil.env --privileged -d --name dind-test -v ./:/test/ docker:dind + docker run --env-file=wireNil.env --privileged -d --name dind-test -v .:/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c "cd test && go run ." From b9fa1d8a252efec92ae0fe4c8744a1c0d15ae5df Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Fri, 5 Apr 2024 20:58:42 +0530 Subject: [PATCH 043/123] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e84a259b5a..d0c6faca8e 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNil.env --privileged -d --name dind-test -v .:/test/ docker:dind + docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c "cd test && go run ." From 96165f99975bdada15b6fb4d278320bbd06030f6 Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Fri, 5 Apr 2024 21:02:14 +0530 Subject: [PATCH 044/123] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d0c6faca8e..16592bde18 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWD/:/test/ docker:dind + docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWDFF:/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c "cd test && go run ." From 56a48847819b281b60f7e7c94862bea3a9f97b5f Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Sat, 6 Apr 2024 23:32:45 +0530 Subject: [PATCH 045/123] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 16592bde18..beceb83261 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNil.env --privileged -d --name dind-test -v $PWDFF:/test/ docker:dind + docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD):/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c "cd test && go run ." From 1df34c699a2a398b1423281c3d7d5e8385d061c0 Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Sun, 7 Apr 2024 14:59:54 +0530 Subject: [PATCH 046/123] Update wireNil.env --- wireNil.env | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/wireNil.env b/wireNil.env index 35f322ccff..eae413ed10 100644 --- a/wireNil.env +++ b/wireNil.env @@ -34,7 +34,6 @@ ACD_USERNAME=admin MINIO_SECRET_KEY=VK7auS21cIeIn5icRjY3KvogEL3SqZMoeq0XL4em PWD=/test SCOPED_VARIABLE_ENABLED=true -EXECUTE_WIRE_NIL_CHECKER=false +EXECUTE_WIRE_NIL_CHECKER=true TEST_BRANCH= LATEST_HASH= -GOPATH='/usr/local/go' From c62b34dfe681919934a8c122a155265830a786cb Mon Sep 17 00:00:00 2001 From: Laeeqdev <155711991+Laeeqdev@users.noreply.github.com> Date: Sun, 7 Apr 2024 15:00:39 +0530 Subject: [PATCH 047/123] Update run-integration-test.sh --- tests/integrationTesting/run-integration-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integrationTesting/run-integration-test.sh b/tests/integrationTesting/run-integration-test.sh index 2feff138d4..33374c6bf6 100755 --- a/tests/integrationTesting/run-integration-test.sh +++ b/tests/integrationTesting/run-integration-test.sh @@ -2,8 +2,8 @@ wget https://go.dev/dl/go1.22.1.linux-amd64.tar.gz -O go1.22.1.tar.gz rm -rf /usr/local/go && tar -C /usr/local -xzf go1.22.1.tar.gz export GOPATH='/usr/local/go' export PATH=$PATH:$GOPATH/bin -go test ./pkg/pipeline -go test -v ./pkg/pipeline -run TestExecuteWorkflow +#go test ./pkg/pipeline +#go test -v ./pkg/pipeline -run TestExecuteWorkflow exit #to get out of container From 2c59a5445d0da8933af0e164887f4a7e2c4d2503 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 8 Apr 2024 14:06:43 +0530 Subject: [PATCH 048/123] resolved merge conflicts --- Makefile | 5 ++- tests/integrationTesting/create-test-env.sh | 3 -- .../devtron-nats-nodeport.yaml | 37 ------------------- tests/integrationTesting/nats-server.yaml | 32 +++++++++------- wireNil.env | 3 ++ 5 files changed, 25 insertions(+), 55 deletions(-) delete mode 100644 tests/integrationTesting/devtron-nats-nodeport.yaml diff --git a/Makefile b/Makefile index beceb83261..65e122082b 100755 --- a/Makefile +++ b/Makefile @@ -40,10 +40,11 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD):/test/ docker:dind + docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD)/:/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" + docker exec dind-test sh -c "cd test && source ./tests/integrationTesting/export_env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" - docker exec dind-test sh -c "cd test && go run ." + docker exec dind-test sh -c 'PG_ADDR=$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$PG_ADDR:30236 sh -c "cd test && go run ."' run: build ./devtron diff --git a/tests/integrationTesting/create-test-env.sh b/tests/integrationTesting/create-test-env.sh index 4857f896e6..675fa4526d 100755 --- a/tests/integrationTesting/create-test-env.sh +++ b/tests/integrationTesting/create-test-env.sh @@ -20,7 +20,6 @@ kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/postgresql-secret.ya kubectl -ndevtroncd apply -f $PWD/tests/integrationTesting/postgresql.yaml kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/devtron-secret.yaml kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/nats-server.yaml -kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/devtron-nats-nodeport.yaml yq '(select(.metadata.name == "postgresql-migrate-devtron") | .spec.template.spec.containers[0].env[0].value) = env(TEST_BRANCH)' $PWD/tests/integrationTesting/migrator.yaml -i yq '(select(.metadata.name == "postgresql-migrate-devtron") | .spec.template.spec.containers[0].env[9].value) = env(LATEST_HASH)' $PWD/tests/integrationTesting/migrator.yaml -i kubectl -ndevtroncd apply -f $PWD/tests/integrationTesting/migrator.yaml @@ -53,6 +52,4 @@ helm dependency up helm template devtron . --set installer.modules={cicd} -s templates/workflow.yaml >./argo_wf.yaml kubectl apply -f ./argo_wf.yaml while [ ! $(kubectl -n argo get deployment workflow-controller -o jsonpath="{.status.readyReplicas}") ]; do sleep 10; done -export NATS_SERVER_HOST=nats://$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address):30236 -export PG_ADDR=$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) cd $PWD diff --git a/tests/integrationTesting/devtron-nats-nodeport.yaml b/tests/integrationTesting/devtron-nats-nodeport.yaml deleted file mode 100644 index 0e08c20800..0000000000 --- a/tests/integrationTesting/devtron-nats-nodeport.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: devtron-nats-for-nodeport - namespace: devtroncd -spec: - ports: - - name: client - nodePort: 30236 - port: 4222 - protocol: TCP - targetPort: 4222 - - name: cluster - port: 6222 - protocol: TCP - targetPort: 6222 - - name: monitor - port: 8222 - protocol: TCP - targetPort: 8222 - - name: metrics - port: 7777 - protocol: TCP - targetPort: 7777 - - name: leafnodes - port: 7422 - protocol: TCP - targetPort: 7422 - - name: gateways - port: 7522 - protocol: TCP - targetPort: 7522 - selector: - app.kubernetes.io/instance: devtron-nats - app.kubernetes.io/name: nats - sessionAffinity: None - type: NodePort \ No newline at end of file diff --git a/tests/integrationTesting/nats-server.yaml b/tests/integrationTesting/nats-server.yaml index 76e5144827..9f1b241cb7 100644 --- a/tests/integrationTesting/nats-server.yaml +++ b/tests/integrationTesting/nats-server.yaml @@ -38,33 +38,39 @@ apiVersion: v1 kind: Service metadata: name: devtron-nats - namespace: "devtroncd" - labels: - helm.sh/chart: nats-0.9.2 - - app.kubernetes.io/name: nats - app.kubernetes.io/instance: devtron-nats - app.kubernetes.io/version: "2.6.3" - app.kubernetes.io/managed-by: Helm + namespace: devtroncd spec: - selector: - - app.kubernetes.io/name: nats - app.kubernetes.io/instance: devtron-nats - clusterIP: None ports: - name: client + nodePort: 30236 port: 4222 + protocol: TCP + targetPort: 4222 - name: cluster port: 6222 + protocol: TCP + targetPort: 6222 - name: monitor port: 8222 + protocol: TCP + targetPort: 8222 - name: metrics port: 7777 + protocol: TCP + targetPort: 7777 - name: leafnodes port: 7422 + protocol: TCP + targetPort: 7422 - name: gateways port: 7522 + protocol: TCP + targetPort: 7522 + selector: + app.kubernetes.io/instance: devtron-nats + app.kubernetes.io/name: nats + sessionAffinity: None + type: NodePort --- # Source: nats/templates/statefulset.yaml apiVersion: apps/v1 diff --git a/wireNil.env b/wireNil.env index eae413ed10..f23b4d372f 100644 --- a/wireNil.env +++ b/wireNil.env @@ -37,3 +37,6 @@ SCOPED_VARIABLE_ENABLED=true EXECUTE_WIRE_NIL_CHECKER=true TEST_BRANCH= LATEST_HASH= +GOPATH=/usr/local/go +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin + From 1f1f3c59ecea828e733ef5f9a60bcdaf426e5a95 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 8 Apr 2024 14:17:59 +0530 Subject: [PATCH 049/123] updated makefile --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 65e122082b..71fa4f8823 100755 --- a/Makefile +++ b/Makefile @@ -42,7 +42,6 @@ test-unit: test-integration: docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD)/:/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" - docker exec dind-test sh -c "cd test && source ./tests/integrationTesting/export_env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c 'PG_ADDR=$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$PG_ADDR:30236 sh -c "cd test && go run ."' run: build From 9f51058f013d02e63463a2539ef8db93f559dbcf Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 8 Apr 2024 14:31:47 +0530 Subject: [PATCH 050/123] added --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 71fa4f8823..1dfd934e60 100755 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ test-integration: docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD)/:/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" - docker exec dind-test sh -c 'PG_ADDR=$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$PG_ADDR:30236 sh -c "cd test && go run ."' + docker exec dind-test sh -c 'PG_ADDR=$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$(PG_ADDR):30236 sh -c "cd test && go run ."' run: build ./devtron From 122aa50d9d4da736988d6c208f746f0ef963f426 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 8 Apr 2024 14:43:40 +0530 Subject: [PATCH 051/123] added --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1dfd934e60..7d6bdb37f7 100755 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ test-integration: docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD)/:/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" - docker exec dind-test sh -c 'PG_ADDR=$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$(PG_ADDR):30236 sh -c "cd test && go run ."' + docker exec dind-test sh -c 'PG_ADDR=$$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$$PG_ADDR:30236 sh -c "cd test && go run ."' run: build ./devtron From a148abdeb95718e387f0a97faeae1006d2eb2c48 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 8 Apr 2024 16:36:09 +0530 Subject: [PATCH 052/123] added --- ...\001X\266p\001\020\001\005\300p\001\300\260\001\300\260\001ad" | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 "tests/integrationTesting/\001\360\303\260\001@h\267K\v@8\016@10\006\004@@@@@\020\003\020\003\b\003\004P\003P\003@P\003@\034\034\001\001\004@@X\266p\001X\266p\001\020\001\005\300p\001\300\260\001\300\260\001ad" diff --git "a/tests/integrationTesting/\001\360\303\260\001@h\267K\v@8\016@10\006\004@@@@@\020\003\020\003\b\003\004P\003P\003@P\003@\034\034\001\001\004@@X\266p\001X\266p\001\020\001\005\300p\001\300\260\001\300\260\001ad" "b/tests/integrationTesting/\001\360\303\260\001@h\267K\v@8\016@10\006\004@@@@@\020\003\020\003\b\003\004P\003P\003@P\003@\034\034\001\001\004@@X\266p\001X\266p\001\020\001\005\300p\001\300\260\001\300\260\001ad" deleted file mode 100644 index e69de29bb2..0000000000 From ba21a44e26536629aff83410085dba05df9d726c Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 8 Apr 2024 17:58:09 +0530 Subject: [PATCH 053/123] added readonly option for docker mount --- Makefile | 2 +- WiringNilCheck.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 7d6bdb37f7..c80249ac7a 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD)/:/test/ docker:dind + docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD)/:/test/:ro docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c 'PG_ADDR=$$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$$PG_ADDR:30236 sh -c "cd test && go run ."' diff --git a/WiringNilCheck.go b/WiringNilCheck.go index 97b0cb6b7d..72f8eed13f 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -15,7 +15,7 @@ func CheckIfNilInWire() { } nilFieldsMap := make(map[string]bool) checkNilFields(app, nilFieldsMap) - fmt.Println(nilFieldsMap) + fmt.Println("NIL Fields present in impls are: ", nilFieldsMap) } func checkNilFields(obj interface{}, nilObjMap map[string]bool) { From bf87497791f3165245987af5924f6129faf0b314 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 8 Apr 2024 18:21:22 +0530 Subject: [PATCH 054/123] reverted ro change --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c80249ac7a..7d6bdb37f7 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD)/:/test/:ro docker:dind + docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD)/:/test/ docker:dind docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c 'PG_ADDR=$$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$$PG_ADDR:30236 sh -c "cd test && go run ."' From 13b2981c12595eb8fbf2d3bf2b0653d5ee16f8b5 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 9 Apr 2024 11:26:54 +0530 Subject: [PATCH 055/123] updated makefile --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7d6bdb37f7..0ed1fcd066 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,8 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD)/:/test/ docker:dind + docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro docker:dind + docker exec dind-test sh -c "mkdir test && cp -r wirenil/* test/" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c 'PG_ADDR=$$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$$PG_ADDR:30236 sh -c "cd test && go run ."' From fadfcc0528de4d18f83cd62901aa0c0e223876bc Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Tue, 9 Apr 2024 13:13:44 +0530 Subject: [PATCH 056/123] run go tidy and mod commands --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a9fadaaac6..644dc53a9c 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/devtron-labs/protos v0.0.3-0.20240326053929-48e42d9d4534 github.com/evanphx/json-patch v5.6.0+incompatible github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 + github.com/go-errors/errors v1.4.2 github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-git/v5 v5.11.0 github.com/go-pg/pg v6.15.1+incompatible @@ -137,7 +138,6 @@ require ( github.com/fatih/camelcase v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fvbommel/sortorder v1.0.1 // indirect - github.com/go-errors/errors v1.4.2 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect From 89bf4a55b4571a5fd936c3751adba292ad64ba1c Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 9 Apr 2024 15:15:30 +0530 Subject: [PATCH 057/123] update main file --- main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index acfe5f1ff3..6e33bafb2e 100755 --- a/main.go +++ b/main.go @@ -30,8 +30,11 @@ import ( func main() { globalEnvVariables, err := util2.GetEnvironmentVariables() + if err != nil { + fmt.Println("error while getting env variables reason:", err) + return + } if globalEnvVariables.GlobalEnvVariables.ExecuteWireNilChecker { - fmt.Println("inside execute function.....") CheckIfNilInWire() return } From 432130fd0c04822d71712dc026e1c892adf47acf Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 9 Apr 2024 15:32:49 +0530 Subject: [PATCH 058/123] deleted binary file --- makeBinaryFileForWireNilChecker.sh | 8 -------- 1 file changed, 8 deletions(-) delete mode 100755 makeBinaryFileForWireNilChecker.sh diff --git a/makeBinaryFileForWireNilChecker.sh b/makeBinaryFileForWireNilChecker.sh deleted file mode 100755 index bde42d398c..0000000000 --- a/makeBinaryFileForWireNilChecker.sh +++ /dev/null @@ -1,8 +0,0 @@ -echo "creating binary file..." -cp auth_model.conf ./tests/integrationTesting -chmod 777 WiringNilCheck.go -go build -o wirenilcheckbinary . -chmod 557 WiringNilCheck.go -mv wirenilcheckbinary ./tests/integrationTesting -cd tests/integrationTesting -echo "binary file $(ls | grep wirenilcheckbinary) is created" From 61e46134530cb3ede152dc37f1903d8d8d619ad6 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 9 Apr 2024 17:42:52 +0530 Subject: [PATCH 059/123] added fileds and value in skip function --- Makefile | 2 +- WiringNilCheck.go | 20 +++++++++++++++++++- main.go | 2 +- util/GlobalConfig.go | 21 +++++---------------- wireNil.env => wireNilChecker.env | 0 5 files changed, 26 insertions(+), 19 deletions(-) rename wireNil.env => wireNilChecker.env (100%) diff --git a/Makefile b/Makefile index 0ed1fcd066..02343ad02d 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNil.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro docker:dind + docker run --env-file=wireNilChecker.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro docker:dind docker exec dind-test sh -c "mkdir test && cp -r wirenil/* test/" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" diff --git a/WiringNilCheck.go b/WiringNilCheck.go index 72f8eed13f..c2955a6de5 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -69,9 +69,27 @@ func canFieldTypeBeNil(field reflect.Value) bool { func canSkipFieldStructCheck(fieldName, valName string) bool { fieldName = strings.ToLower(fieldName) valName = strings.ToLower(valName) - if valName == "githubclient" && (fieldName == "client" || fieldName == "gitopshelper") { + if valName == "cicdconfig" { return true } + fieldAndValName := map[string][]string{ + "app": {"enforcerv2", "server"}, + "githubclient": {"client", "gitopshelper"}, + "gitfactory": {"client"}, + "argocdconnectionmanagerimpl": {"argocdsettings"}, + "enforcerimpl": {"cache", "enforcerv2"}, + "helmappclientimpl": {"applicationserviceclient"}, + "modulecronserviceimpl": {"cron"}, + "oteltracingserviceimpl": {"traceprovider"}, + "terminalaccessrepositoryimpl": {"templatescache"}, + } + if _, ok := fieldAndValName[valName]; ok { + for _, ignoreFieldName := range fieldAndValName[valName] { + if ignoreFieldName == fieldName { + return true + } + } + } for _, str := range []string{"logger", "dbconnection", "syncedenforcer"} { if fieldName == str { return true diff --git a/main.go b/main.go index 6e33bafb2e..c5061fe4f9 100755 --- a/main.go +++ b/main.go @@ -31,7 +31,7 @@ import ( func main() { globalEnvVariables, err := util2.GetEnvironmentVariables() if err != nil { - fmt.Println("error while getting env variables reason:", err) + log.Println("error while getting env variables reason:", err) return } if globalEnvVariables.GlobalEnvVariables.ExecuteWireNilChecker { diff --git a/util/GlobalConfig.go b/util/GlobalConfig.go index b88b8b1bc1..d82f16e2db 100644 --- a/util/GlobalConfig.go +++ b/util/GlobalConfig.go @@ -1,9 +1,7 @@ package util import ( - "fmt" "github.com/caarlos0/env" - "github.com/go-errors/errors" ) type EnvironmentVariables struct { @@ -24,6 +22,11 @@ type GlobalEnvVariables struct { ExecuteWireNilChecker bool `env:"EXECUTE_WIRE_NIL_CHECKER" envDefault:"false"` } +type DevtronSecretConfig struct { + DevtronSecretName string `env:"DEVTRON_SECRET_NAME" envDefault:"devtron-secret"` + DevtronDexSecretNamespace string `env:"DEVTRON_DEX_SECRET_NAMESPACE" envDefault:"devtroncd"` +} + func GetEnvironmentVariables() (*EnvironmentVariables, error) { cfg := &EnvironmentVariables{ GlobalEnvVariables: &GlobalEnvVariables{}, @@ -36,17 +39,3 @@ func GetEnvironmentVariables() (*EnvironmentVariables, error) { } return cfg, err } - -type DevtronSecretConfig struct { - DevtronSecretName string `env:"DEVTRON_SECRET_NAME" envDefault:"devtron-secret"` - DevtronDexSecretNamespace string `env:"DEVTRON_DEX_SECRET_NAMESPACE" envDefault:"devtroncd"` -} - -func GetDevtronSecretName() (*DevtronSecretConfig, error) { - secretConfig := &DevtronSecretConfig{} - err := env.Parse(secretConfig) - if err != nil { - return nil, errors.New(fmt.Sprintf("could not get devtron secret name from environment : %v", err)) - } - return secretConfig, err -} diff --git a/wireNil.env b/wireNilChecker.env similarity index 100% rename from wireNil.env rename to wireNilChecker.env From ec5478493b3fb83b6b050251f576529b21775727 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 9 Apr 2024 18:06:27 +0530 Subject: [PATCH 060/123] map length added --- WiringNilCheck.go | 1 + 1 file changed, 1 insertion(+) diff --git a/WiringNilCheck.go b/WiringNilCheck.go index c2955a6de5..8b8e4c8188 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -16,6 +16,7 @@ func CheckIfNilInWire() { nilFieldsMap := make(map[string]bool) checkNilFields(app, nilFieldsMap) fmt.Println("NIL Fields present in impls are: ", nilFieldsMap) + fmt.Println(len(nilFieldsMap)) } func checkNilFields(obj interface{}, nilObjMap map[string]bool) { From 3df9b004dcfae61565fda7e21156f7d28426503c Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 9 Apr 2024 18:09:17 +0530 Subject: [PATCH 061/123] map length added --- WiringNilCheck.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WiringNilCheck.go b/WiringNilCheck.go index 8b8e4c8188..18e3ab7051 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -16,7 +16,7 @@ func CheckIfNilInWire() { nilFieldsMap := make(map[string]bool) checkNilFields(app, nilFieldsMap) fmt.Println("NIL Fields present in impls are: ", nilFieldsMap) - fmt.Println(len(nilFieldsMap)) + fmt.Println("output=", len(nilFieldsMap)) } func checkNilFields(obj interface{}, nilObjMap map[string]bool) { From 673bb13cd5b179aa3813512d86519884ad97bda7 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 9 Apr 2024 18:42:17 +0530 Subject: [PATCH 062/123] create a file output.env --- Makefile | 2 +- WiringNilCheck.go | 12 ++++++++++++ output.env | 0 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100755 output.env diff --git a/Makefile b/Makefile index 02343ad02d..6a60d1d906 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNilChecker.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro docker:dind + docker run --env-file=wireNilChecker.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro -v $(PWD)/temp/:/tmp docker:dind docker exec dind-test sh -c "mkdir test && cp -r wirenil/* test/" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" diff --git a/WiringNilCheck.go b/WiringNilCheck.go index 18e3ab7051..5ff171d456 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -3,6 +3,7 @@ package main import ( "fmt" "log" + "os" "reflect" "strings" "unsafe" @@ -17,6 +18,17 @@ func CheckIfNilInWire() { checkNilFields(app, nilFieldsMap) fmt.Println("NIL Fields present in impls are: ", nilFieldsMap) fmt.Println("output=", len(nilFieldsMap)) + file, errs := os.Create("output.env") + if errs != nil { + log.Println("Failed to create file:", errs) + return + } + defer file.Close() + _, errs = file.WriteString(fmt.Sprintf("OUTPUT=%d", len(nilFieldsMap))) + if errs != nil { + log.Println("Failed to write to file:", errs) + return + } } func checkNilFields(obj interface{}, nilObjMap map[string]bool) { diff --git a/output.env b/output.env new file mode 100755 index 0000000000..e69de29bb2 From a60d74a3f7e8f58f82ebc1a8deb1e0b8b777f171 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 9 Apr 2024 18:44:25 +0530 Subject: [PATCH 063/123] changes in makefile --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 6a60d1d906..1ea5e85ffb 100755 --- a/Makefile +++ b/Makefile @@ -45,6 +45,7 @@ test-integration: docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c 'PG_ADDR=$$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$$PG_ADDR:30236 sh -c "cd test && go run ."' + source ./temp/output.env run: build ./devtron From 66e5bd8d8f9187ac3ade9dc29f9703ce299acf23 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 9 Apr 2024 18:55:55 +0530 Subject: [PATCH 064/123] changes in makefile --- Makefile | 2 +- WiringNilCheck.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 1ea5e85ffb..fe3a2d6270 100755 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test-unit: go test ./pkg/pipeline test-integration: - docker run --env-file=wireNilChecker.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro -v $(PWD)/temp/:/tmp docker:dind + docker run --env-file=wireNilChecker.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro -v $(PWD)/temp/:/tempfile docker:dind docker exec dind-test sh -c "mkdir test && cp -r wirenil/* test/" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" diff --git a/WiringNilCheck.go b/WiringNilCheck.go index 5ff171d456..9aff607c0c 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -18,7 +18,7 @@ func CheckIfNilInWire() { checkNilFields(app, nilFieldsMap) fmt.Println("NIL Fields present in impls are: ", nilFieldsMap) fmt.Println("output=", len(nilFieldsMap)) - file, errs := os.Create("output.env") + file, errs := os.Create("/tempfile/output.env") if errs != nil { log.Println("Failed to create file:", errs) return From 5058f462e3d4804224d0da6d99c6cacb713fcb7d Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 9 Apr 2024 19:45:43 +0530 Subject: [PATCH 065/123] again updated makefile --- Makefile | 2 ++ temp/output.env | 0 2 files changed, 2 insertions(+) create mode 100755 temp/output.env diff --git a/Makefile b/Makefile index fe3a2d6270..540f642a59 100755 --- a/Makefile +++ b/Makefile @@ -45,6 +45,8 @@ test-integration: docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c 'PG_ADDR=$$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$$PG_ADDR:30236 sh -c "cd test && go run ."' + docker exec dind-test sh -c "cp ./test/output.env ./tempfile" + chmod 777 ./temp/output.env source ./temp/output.env run: build ./devtron diff --git a/temp/output.env b/temp/output.env new file mode 100755 index 0000000000..e69de29bb2 From 6fe5df0e989520411ea3a54dde6227e07db9a1cd Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Wed, 10 Apr 2024 00:36:03 +0530 Subject: [PATCH 066/123] updated makefile --- Makefile | 2 -- temp/output.env | 0 2 files changed, 2 deletions(-) delete mode 100755 temp/output.env diff --git a/Makefile b/Makefile index 540f642a59..4705afb892 100755 --- a/Makefile +++ b/Makefile @@ -46,8 +46,6 @@ test-integration: docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c 'PG_ADDR=$$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$$PG_ADDR:30236 sh -c "cd test && go run ."' docker exec dind-test sh -c "cp ./test/output.env ./tempfile" - chmod 777 ./temp/output.env - source ./temp/output.env run: build ./devtron diff --git a/temp/output.env b/temp/output.env deleted file mode 100755 index e69de29bb2..0000000000 From 73f42b002a5d9e28372a535c19996fc45188940c Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Wed, 10 Apr 2024 00:48:19 +0530 Subject: [PATCH 067/123] updated makefile --- WiringNilCheck.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WiringNilCheck.go b/WiringNilCheck.go index 9aff607c0c..a0682a1596 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -18,7 +18,7 @@ func CheckIfNilInWire() { checkNilFields(app, nilFieldsMap) fmt.Println("NIL Fields present in impls are: ", nilFieldsMap) fmt.Println("output=", len(nilFieldsMap)) - file, errs := os.Create("/tempfile/output.env") + file, errs := os.Create("/test/output.env") if errs != nil { log.Println("Failed to create file:", errs) return From 7fe8b28e9615e973eb58ecd6407ad95a1bb22873 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Wed, 10 Apr 2024 12:01:44 +0530 Subject: [PATCH 068/123] added skipUnnecessaryFiledsForCheck function --- WiringNilCheck.go | 50 +++++++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/WiringNilCheck.go b/WiringNilCheck.go index a0682a1596..0d7916bf27 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -17,16 +17,9 @@ func CheckIfNilInWire() { nilFieldsMap := make(map[string]bool) checkNilFields(app, nilFieldsMap) fmt.Println("NIL Fields present in impls are: ", nilFieldsMap) - fmt.Println("output=", len(nilFieldsMap)) - file, errs := os.Create("/test/output.env") - if errs != nil { - log.Println("Failed to create file:", errs) - return - } - defer file.Close() - _, errs = file.WriteString(fmt.Sprintf("OUTPUT=%d", len(nilFieldsMap))) - if errs != nil { - log.Println("Failed to write to file:", errs) + //writing length of nilFieldsMap in a file (eg. output.env) so that we can export this data in pre ci pipeline bash and can failed the pre ci pipeline if nil nilFieldsMap's length is greater than zero + err = writeResultToFile(len(nilFieldsMap)) + if err != nil { return } } @@ -48,6 +41,9 @@ func checkNilFields(obj interface{}, nilObjMap map[string]bool) { //package not from this repo, ignoring continue } + if skipUnnecessaryFiledsForCheck(fieldName, valName) { // skip unnecessary fileds and values + continue + } if !canFieldTypeBeNil(field) { // field can not be nil, skip continue } else if field.IsNil() { // check if the field is nil @@ -82,12 +78,23 @@ func canFieldTypeBeNil(field reflect.Value) bool { func canSkipFieldStructCheck(fieldName, valName string) bool { fieldName = strings.ToLower(fieldName) valName = strings.ToLower(valName) + if valName == "githubclient" && (fieldName == "client" || fieldName == "gitopshelper") { + return true + } + for _, str := range []string{"logger", "dbconnection", "syncedenforcer"} { + if fieldName == str { + return true + } + } + return false +} + +func skipUnnecessaryFiledsForCheck(fieldName, valName string) bool { if valName == "cicdconfig" { return true } fieldAndValName := map[string][]string{ "app": {"enforcerv2", "server"}, - "githubclient": {"client", "gitopshelper"}, "gitfactory": {"client"}, "argocdconnectionmanagerimpl": {"argocdsettings"}, "enforcerimpl": {"cache", "enforcerv2"}, @@ -103,14 +110,8 @@ func canSkipFieldStructCheck(fieldName, valName string) bool { } } } - for _, str := range []string{"logger", "dbconnection", "syncedenforcer"} { - if fieldName == str { - return true - } - } return false } - func GetUnexportedField(field reflect.Value) interface{} { return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface() } @@ -118,3 +119,18 @@ func GetUnexportedField(field reflect.Value) interface{} { func isExported(fieldName string) bool { return strings.ToUpper(fieldName[0:1]) == fieldName[0:1] } + +func writeResultToFile(data int) error { + file, err := os.Create("/test/output.env") + if err != nil { + log.Println("Failed to create file:", err) + return err + } + defer file.Close() + _, err = file.WriteString(fmt.Sprintf("OUTPUT=%d", data)) + if err != nil { + log.Println("Failed to write to file:", err) + return err + } + return nil +} From dbbdeca16ab58d167ef350d75e2cfd9981c6ac5b Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Wed, 10 Apr 2024 12:17:43 +0530 Subject: [PATCH 069/123] have done some changes --- Makefile | 2 +- WiringNilCheck.go | 1 + tests/integrationTesting/nats-server.yaml | 25 +---------------------- 3 files changed, 3 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index 4705afb892..36bd2a8770 100755 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ test-integration: docker exec dind-test sh -c "mkdir test && cp -r wirenil/* test/" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" - docker exec dind-test sh -c 'PG_ADDR=$$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) NATS_SERVER_HOST=nats://$$PG_ADDR:30236 sh -c "cd test && go run ."' + docker exec dind-test sh -c 'NODE_IP_ADDRESS=$$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) PG_ADDR=$$NODE_IP_ADDRESS NATS_SERVER_HOST=nats://$$NODE_IP_ADDRESS:30236 sh -c "cd test && go run ."' docker exec dind-test sh -c "cp ./test/output.env ./tempfile" run: build ./devtron diff --git a/WiringNilCheck.go b/WiringNilCheck.go index 0d7916bf27..6ecbd568f5 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -90,6 +90,7 @@ func canSkipFieldStructCheck(fieldName, valName string) bool { } func skipUnnecessaryFiledsForCheck(fieldName, valName string) bool { + fmt.Printf("hello my name is laeeq valname is %s fieldname is %s\n", valName, fieldName) if valName == "cicdconfig" { return true } diff --git a/tests/integrationTesting/nats-server.yaml b/tests/integrationTesting/nats-server.yaml index 9f1b241cb7..0330a18685 100644 --- a/tests/integrationTesting/nats-server.yaml +++ b/tests/integrationTesting/nats-server.yaml @@ -305,27 +305,4 @@ spec: - | [ $name = test ] - restartPolicy: Never ---- -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - kind: Prometheus - app: devtron-nats - chart: nats-0.9.2 - release: monitoring - name: devtron-nats-server -spec: - endpoints: - - interval: 30s - path: /metrics - port: metrics - jobLabel: nats-server - namespaceSelector: - matchNames: - - devtroncd - selector: - matchLabels: - app.kubernetes.io/instance: devtron-nats - app.kubernetes.io/name: nats \ No newline at end of file + restartPolicy: Never \ No newline at end of file From dc3d117f6e8e8a27e80d9facf064a711dc81852c Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Wed, 10 Apr 2024 12:26:25 +0530 Subject: [PATCH 070/123] have done some changes --- WiringNilCheck.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WiringNilCheck.go b/WiringNilCheck.go index 6ecbd568f5..5d102b991b 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -90,7 +90,6 @@ func canSkipFieldStructCheck(fieldName, valName string) bool { } func skipUnnecessaryFiledsForCheck(fieldName, valName string) bool { - fmt.Printf("hello my name is laeeq valname is %s fieldname is %s\n", valName, fieldName) if valName == "cicdconfig" { return true } @@ -107,6 +106,7 @@ func skipUnnecessaryFiledsForCheck(fieldName, valName string) bool { if _, ok := fieldAndValName[valName]; ok { for _, ignoreFieldName := range fieldAndValName[valName] { if ignoreFieldName == fieldName { + fmt.Printf("hello my name is laeeq valname is %s fieldname is %s\n", valName, fieldName) return true } } From 4617f25e66ceffa2e70ebbb7cfc9a9b695e70fe9 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Wed, 10 Apr 2024 12:35:18 +0530 Subject: [PATCH 071/123] have done some changes --- WiringNilCheck.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/WiringNilCheck.go b/WiringNilCheck.go index 5d102b991b..5d61f633fd 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -90,6 +90,8 @@ func canSkipFieldStructCheck(fieldName, valName string) bool { } func skipUnnecessaryFiledsForCheck(fieldName, valName string) bool { + fieldName = strings.ToLower(fieldName) + valName = strings.ToLower(valName) if valName == "cicdconfig" { return true } From beaedd40033ab1b89f526ac6882c8adc406ea2e2 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Wed, 10 Apr 2024 13:05:52 +0530 Subject: [PATCH 072/123] have done some changes --- WiringNilCheck.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/WiringNilCheck.go b/WiringNilCheck.go index 5d61f633fd..025c86894d 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -96,19 +96,22 @@ func skipUnnecessaryFiledsForCheck(fieldName, valName string) bool { return true } fieldAndValName := map[string][]string{ - "app": {"enforcerv2", "server"}, - "gitfactory": {"client"}, - "argocdconnectionmanagerimpl": {"argocdsettings"}, - "enforcerimpl": {"cache", "enforcerv2"}, - "helmappclientimpl": {"applicationserviceclient"}, - "modulecronserviceimpl": {"cron"}, - "oteltracingserviceimpl": {"traceprovider"}, - "terminalaccessrepositoryimpl": {"templatescache"}, + "app": {"enforcerv2", "server"}, + "gitfactory": {"client"}, + "argocdconnectionmanagerimpl": {"argocdsettings"}, + "enforcerimpl": {"cache", "enforcerv2"}, + "helmappclientimpl": {"applicationserviceclient"}, + "modulecronserviceimpl": {"cron"}, + "oteltracingserviceimpl": {"traceprovider"}, + "terminalaccessrepositoryimpl": {"templatescache"}, //this line + "appstoredeploymentresthandlerimpl": {"helmappresthandler"}, + "clusterserviceimpl": {"clusterrbacserviceimpl"}, + "enforcerutilhelmimpl": {"installedapprepository", "apprepository", "teamrepository"}, + "scopedvariableserviceimpl": {"devtronresourcesearchablekeyservice"}, } if _, ok := fieldAndValName[valName]; ok { for _, ignoreFieldName := range fieldAndValName[valName] { if ignoreFieldName == fieldName { - fmt.Printf("hello my name is laeeq valname is %s fieldname is %s\n", valName, fieldName) return true } } From fbb8d368d2423d72b4542c5ac12e7f370f9e4066 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Wed, 10 Apr 2024 13:33:22 +0530 Subject: [PATCH 073/123] removed some data --- WiringNilCheck.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/WiringNilCheck.go b/WiringNilCheck.go index 025c86894d..5cbcbd7143 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -96,18 +96,14 @@ func skipUnnecessaryFiledsForCheck(fieldName, valName string) bool { return true } fieldAndValName := map[string][]string{ - "app": {"enforcerv2", "server"}, - "gitfactory": {"client"}, - "argocdconnectionmanagerimpl": {"argocdsettings"}, - "enforcerimpl": {"cache", "enforcerv2"}, - "helmappclientimpl": {"applicationserviceclient"}, - "modulecronserviceimpl": {"cron"}, - "oteltracingserviceimpl": {"traceprovider"}, - "terminalaccessrepositoryimpl": {"templatescache"}, //this line - "appstoredeploymentresthandlerimpl": {"helmappresthandler"}, - "clusterserviceimpl": {"clusterrbacserviceimpl"}, - "enforcerutilhelmimpl": {"installedapprepository", "apprepository", "teamrepository"}, - "scopedvariableserviceimpl": {"devtronresourcesearchablekeyservice"}, + "app": {"enforcerv2", "server"}, + "gitfactory": {"client"}, + "argocdconnectionmanagerimpl": {"argocdsettings"}, + "enforcerimpl": {"cache", "enforcerv2"}, + "helmappclientimpl": {"applicationserviceclient"}, + "modulecronserviceimpl": {"cron"}, + "oteltracingserviceimpl": {"traceprovider"}, + "terminalaccessrepositoryimpl": {"templatescache"}, } if _, ok := fieldAndValName[valName]; ok { for _, ignoreFieldName := range fieldAndValName[valName] { From 3c9fef914204479bd110f165738748804548e38f Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 12 Apr 2024 10:53:30 +0530 Subject: [PATCH 074/123] updated comment --- WiringNilCheck.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WiringNilCheck.go b/WiringNilCheck.go index 5cbcbd7143..65f3e11a9e 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -17,7 +17,7 @@ func CheckIfNilInWire() { nilFieldsMap := make(map[string]bool) checkNilFields(app, nilFieldsMap) fmt.Println("NIL Fields present in impls are: ", nilFieldsMap) - //writing length of nilFieldsMap in a file (eg. output.env) so that we can export this data in pre ci pipeline bash and can failed the pre ci pipeline if nil nilFieldsMap's length is greater than zero + //Writes the length of nilFieldsMap to a file (e.g., output.env) so that we can export this file's data in a pre-CI pipeline bash script and fail the pre-CI pipeline if the length of nilFieldsMap is greater than zero. err = writeResultToFile(len(nilFieldsMap)) if err != nil { return From 356ccd06ecff31bdc9e6682fba441f47330528c9 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 12 Apr 2024 11:54:31 +0530 Subject: [PATCH 075/123] PR code review comments resolved --- Makefile | 8 +++++--- WiringNilCheck.go | 4 ++-- output.env | 0 3 files changed, 7 insertions(+), 5 deletions(-) delete mode 100755 output.env diff --git a/Makefile b/Makefile index 36bd2a8770..1fbfc04a25 100755 --- a/Makefile +++ b/Makefile @@ -39,20 +39,22 @@ test-all: test-unit test-unit: go test ./pkg/pipeline -test-integration: +test-integration:make-temp-dir docker run --env-file=wireNilChecker.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro -v $(PWD)/temp/:/tempfile docker:dind docker exec dind-test sh -c "mkdir test && cp -r wirenil/* test/" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" + docker exec dind-test sh -c "cd test && touch output.env" docker exec dind-test sh -c 'NODE_IP_ADDRESS=$$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) PG_ADDR=$$NODE_IP_ADDRESS NATS_SERVER_HOST=nats://$$NODE_IP_ADDRESS:30236 sh -c "cd test && go run ."' docker exec dind-test sh -c "cp ./test/output.env ./tempfile" + rm -r temp run: build ./devtron - .PHONY: build docker-build-image: build docker build -t devtron:$(TAG) . - +make-temp-dir: + mkdir -p temp .PHONY: build, all, wire, clean, run, set-docker-build-env, docker-build-push, devtron, docker-build-push: docker-build-image docker tag devtron:${TAG} ${REGISTRY}/devtron:${TAG} diff --git a/WiringNilCheck.go b/WiringNilCheck.go index 65f3e11a9e..1f2b5907cc 100755 --- a/WiringNilCheck.go +++ b/WiringNilCheck.go @@ -41,7 +41,7 @@ func checkNilFields(obj interface{}, nilObjMap map[string]bool) { //package not from this repo, ignoring continue } - if skipUnnecessaryFiledsForCheck(fieldName, valName) { // skip unnecessary fileds and values + if skipUnnecessaryFieldsForCheck(fieldName, valName) { // skip unnecessary fileds and values continue } if !canFieldTypeBeNil(field) { // field can not be nil, skip @@ -89,7 +89,7 @@ func canSkipFieldStructCheck(fieldName, valName string) bool { return false } -func skipUnnecessaryFiledsForCheck(fieldName, valName string) bool { +func skipUnnecessaryFieldsForCheck(fieldName, valName string) bool { fieldName = strings.ToLower(fieldName) valName = strings.ToLower(valName) if valName == "cicdconfig" { diff --git a/output.env b/output.env deleted file mode 100755 index e69de29bb2..0000000000 From f7713ef027f49ec412ab5e05de687dcfa8fcf3bf Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 12 Apr 2024 12:02:44 +0530 Subject: [PATCH 076/123] final commit --- Makefile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 1fbfc04a25..719e58777a 100755 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ test-all: test-unit test-unit: go test ./pkg/pipeline -test-integration:make-temp-dir +test-integration: docker run --env-file=wireNilChecker.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro -v $(PWD)/temp/:/tempfile docker:dind docker exec dind-test sh -c "mkdir test && cp -r wirenil/* test/" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" @@ -47,14 +47,11 @@ test-integration:make-temp-dir docker exec dind-test sh -c "cd test && touch output.env" docker exec dind-test sh -c 'NODE_IP_ADDRESS=$$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) PG_ADDR=$$NODE_IP_ADDRESS NATS_SERVER_HOST=nats://$$NODE_IP_ADDRESS:30236 sh -c "cd test && go run ."' docker exec dind-test sh -c "cp ./test/output.env ./tempfile" - rm -r temp run: build ./devtron .PHONY: build docker-build-image: build docker build -t devtron:$(TAG) . -make-temp-dir: - mkdir -p temp .PHONY: build, all, wire, clean, run, set-docker-build-env, docker-build-push, devtron, docker-build-push: docker-build-image docker tag devtron:${TAG} ${REGISTRY}/devtron:${TAG} From 57e58036223759fddd185a2c54db9341715bccf1 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 12 Apr 2024 12:06:44 +0530 Subject: [PATCH 077/123] updated nats-server.yaml file --- tests/integrationTesting/nats-server.yaml | 39 +---------------------- 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/tests/integrationTesting/nats-server.yaml b/tests/integrationTesting/nats-server.yaml index 0330a18685..dcf123eb26 100644 --- a/tests/integrationTesting/nats-server.yaml +++ b/tests/integrationTesting/nats-server.yaml @@ -268,41 +268,4 @@ spec: accessModes: [ "ReadWriteOnce" ] resources: requests: - storage: 5Gi ---- -# Source: nats/templates/tests/test-request-reply.yaml -apiVersion: v1 -kind: Pod -metadata: - name: "devtron-nats-test-request-reply" - labels: - helm.sh/chart: nats-0.9.2 - - app.kubernetes.io/name: nats - app.kubernetes.io/instance: devtron-nats - app.kubernetes.io/version: "2.6.3" - app.kubernetes.io/managed-by: Helm - annotations: - "helm.sh/hook": test -spec: - containers: - - name: nats-box - image: quay.io/devtron/nats-box - env: - - name: NATS_HOST - value: devtron-nats - command: - - /bin/sh - - -ec - - | - nats reply -s nats://$NATS_HOST:4222 'name.>' --command "echo 1" & - - | - "&&" - - | - name=$(nats request -s nats://$NATS_HOST:4222 name.test '' 2>/dev/null) - - | - "&&" - - | - [ $name = test ] - - restartPolicy: Never \ No newline at end of file + storage: 5Gi \ No newline at end of file From b83a3d9e4d66eaf8ab52cf82a99945cf8484e2aa Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 10 May 2024 18:51:31 +0530 Subject: [PATCH 078/123] updated nats image --- Makefile | 2 -- setenv.sh | 5 +++++ tests/integrationTesting/nats-server.yaml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 setenv.sh diff --git a/Makefile b/Makefile index b1fcac6827..ee73af770e 100755 --- a/Makefile +++ b/Makefile @@ -45,8 +45,6 @@ test-integration: docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" docker exec dind-test sh -c "cd test && touch output.env" - docker exec dind-test sh -c 'NODE_IP_ADDRESS=$$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) PG_ADDR=$$NODE_IP_ADDRESS NATS_SERVER_HOST=nats://$$NODE_IP_ADDRESS:30236 sh -c "cd test && go run ."' - docker exec dind-test sh -c "cp ./test/output.env ./tempfile" run: build ./devtron .PHONY: build diff --git a/setenv.sh b/setenv.sh new file mode 100644 index 0000000000..2b30a8bacf --- /dev/null +++ b/setenv.sh @@ -0,0 +1,5 @@ +export NODE_IP_ADDRESS=$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) +export PG_ADDR=$NODE_IP_ADDRESS +export NATS_SERVER_HOST=nats://$NODE_IP_ADDRESS:30236 +cd test +go run . \ No newline at end of file diff --git a/tests/integrationTesting/nats-server.yaml b/tests/integrationTesting/nats-server.yaml index dcf123eb26..662e428ac8 100644 --- a/tests/integrationTesting/nats-server.yaml +++ b/tests/integrationTesting/nats-server.yaml @@ -136,7 +136,7 @@ spec: terminationGracePeriodSeconds: 120 containers: - name: nats - image: nats:2.9.3-alpine + image: quay.io/devtron/nats:2.9.3-alpine imagePullPolicy: IfNotPresent resources: {} From ab0686ce914f9a959ba61a2defd4009af13df4ea Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 10 May 2024 19:25:12 +0530 Subject: [PATCH 079/123] refactored code --- Makefile | 5 +---- .../integrationTesting/execute_script_inside_docker.sh | 10 ++++++++-- 2 files changed, 9 insertions(+), 6 deletions(-) rename setenv.sh => tests/integrationTesting/execute_script_inside_docker.sh (52%) mode change 100644 => 100755 diff --git a/Makefile b/Makefile index ee73af770e..1e93e88a98 100755 --- a/Makefile +++ b/Makefile @@ -41,10 +41,7 @@ test-unit: test-integration: docker run --env-file=wireNilChecker.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro -v $(PWD)/temp/:/tempfile docker:dind - docker exec dind-test sh -c "mkdir test && cp -r wirenil/* test/" - docker exec dind-test sh -c "cd test && ./tests/integrationTesting/create-test-env.sh" - docker exec dind-test sh -c "cd test && ./tests/integrationTesting/run-integration-test.sh" - docker exec dind-test sh -c "cd test && touch output.env" + docker exec dind-test sh -c "./tests/integrationTesting/execute_script_inside_docker.sh" run: build ./devtron .PHONY: build diff --git a/setenv.sh b/tests/integrationTesting/execute_script_inside_docker.sh old mode 100644 new mode 100755 similarity index 52% rename from setenv.sh rename to tests/integrationTesting/execute_script_inside_docker.sh index 2b30a8bacf..8d74eedcb6 --- a/setenv.sh +++ b/tests/integrationTesting/execute_script_inside_docker.sh @@ -1,5 +1,11 @@ +mkdir test +cp -r wirenil/* test/ +cd test +./tests/integrationTesting/create-test-env.sh +./tests/integrationTesting/run-integration-test.sh +touch output.env export NODE_IP_ADDRESS=$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) export PG_ADDR=$NODE_IP_ADDRESS export NATS_SERVER_HOST=nats://$NODE_IP_ADDRESS:30236 -cd test -go run . \ No newline at end of file +go run . +cp ./test/output.env ./tempfile \ No newline at end of file From 48de3c02ecab9de1137ff47e4a43443b23a9bc10 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 10 May 2024 19:29:00 +0530 Subject: [PATCH 080/123] resolved error --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1e93e88a98..5a3e09d5c9 100755 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ test-unit: test-integration: docker run --env-file=wireNilChecker.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro -v $(PWD)/temp/:/tempfile docker:dind - docker exec dind-test sh -c "./tests/integrationTesting/execute_script_inside_docker.sh" + docker exec dind-test sh -c "./test/tests/integrationTesting/execute_script_inside_docker.sh" run: build ./devtron .PHONY: build From 35be5f037f4921ae56099a4530c1376e0743e9b8 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 10 May 2024 19:36:17 +0530 Subject: [PATCH 081/123] resolved error --- Makefile | 2 +- tests/integrationTesting/execute_script_inside_docker.sh | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5a3e09d5c9..b9de56e76d 100755 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ test-unit: test-integration: docker run --env-file=wireNilChecker.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro -v $(PWD)/temp/:/tempfile docker:dind - docker exec dind-test sh -c "./test/tests/integrationTesting/execute_script_inside_docker.sh" + docker exec dind-test sh -c "mkdir test && cp -r wirenil/* test/./test/tests/integrationTesting/execute_script_inside_docker.sh" run: build ./devtron .PHONY: build diff --git a/tests/integrationTesting/execute_script_inside_docker.sh b/tests/integrationTesting/execute_script_inside_docker.sh index 8d74eedcb6..e64fd1be27 100755 --- a/tests/integrationTesting/execute_script_inside_docker.sh +++ b/tests/integrationTesting/execute_script_inside_docker.sh @@ -1,5 +1,3 @@ -mkdir test -cp -r wirenil/* test/ cd test ./tests/integrationTesting/create-test-env.sh ./tests/integrationTesting/run-integration-test.sh From c6053691008210de86ecc99047891dfc0c4360cd Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 10 May 2024 19:39:45 +0530 Subject: [PATCH 082/123] resolved error --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b9de56e76d..04025f766f 100755 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ test-unit: test-integration: docker run --env-file=wireNilChecker.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro -v $(PWD)/temp/:/tempfile docker:dind - docker exec dind-test sh -c "mkdir test && cp -r wirenil/* test/./test/tests/integrationTesting/execute_script_inside_docker.sh" + docker exec dind-test sh -c "mkdir test && cp -r wirenil/* test/ && ./test/tests/integrationTesting/execute_script_inside_docker.sh" run: build ./devtron .PHONY: build From 0b04b3a515938a3ed0681756864f568d7a577f06 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 10 May 2024 19:44:56 +0530 Subject: [PATCH 083/123] resolved error --- tests/integrationTesting/execute_script_inside_docker.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integrationTesting/execute_script_inside_docker.sh b/tests/integrationTesting/execute_script_inside_docker.sh index e64fd1be27..6a314749fd 100755 --- a/tests/integrationTesting/execute_script_inside_docker.sh +++ b/tests/integrationTesting/execute_script_inside_docker.sh @@ -5,5 +5,6 @@ touch output.env export NODE_IP_ADDRESS=$(kubectl get node --no-headers -o custom-columns=INTERNAL-IP:status.addresses[0].address) export PG_ADDR=$NODE_IP_ADDRESS export NATS_SERVER_HOST=nats://$NODE_IP_ADDRESS:30236 +go mod tidy go run . cp ./test/output.env ./tempfile \ No newline at end of file From aa2248f102ff98b23746b42b68c6c82b5397f2f5 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 10 May 2024 19:54:33 +0530 Subject: [PATCH 084/123] resolved error --- tests/integrationTesting/execute_script_inside_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrationTesting/execute_script_inside_docker.sh b/tests/integrationTesting/execute_script_inside_docker.sh index 6a314749fd..1292d8bad4 100755 --- a/tests/integrationTesting/execute_script_inside_docker.sh +++ b/tests/integrationTesting/execute_script_inside_docker.sh @@ -7,4 +7,4 @@ export PG_ADDR=$NODE_IP_ADDRESS export NATS_SERVER_HOST=nats://$NODE_IP_ADDRESS:30236 go mod tidy go run . -cp ./test/output.env ./tempfile \ No newline at end of file +cp ./output.env ./tempfile \ No newline at end of file From aa444725a49eb78480ce41cc60f6197810d26449 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 10 May 2024 20:29:54 +0530 Subject: [PATCH 085/123] resolved error --- tests/integrationTesting/execute_script_inside_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrationTesting/execute_script_inside_docker.sh b/tests/integrationTesting/execute_script_inside_docker.sh index 1292d8bad4..0313de76c7 100755 --- a/tests/integrationTesting/execute_script_inside_docker.sh +++ b/tests/integrationTesting/execute_script_inside_docker.sh @@ -7,4 +7,4 @@ export PG_ADDR=$NODE_IP_ADDRESS export NATS_SERVER_HOST=nats://$NODE_IP_ADDRESS:30236 go mod tidy go run . -cp ./output.env ./tempfile \ No newline at end of file +cp output.env ../tempfile \ No newline at end of file From bc925d97d5c4a9c56f428b39b1f89ad7a1016dcd Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 10 May 2024 20:47:49 +0530 Subject: [PATCH 086/123] uncommented code --- tests/integrationTesting/run-integration-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrationTesting/run-integration-test.sh b/tests/integrationTesting/run-integration-test.sh index 33374c6bf6..70a791ca9b 100755 --- a/tests/integrationTesting/run-integration-test.sh +++ b/tests/integrationTesting/run-integration-test.sh @@ -3,7 +3,7 @@ rm -rf /usr/local/go && tar -C /usr/local -xzf go1.22.1.tar.gz export GOPATH='/usr/local/go' export PATH=$PATH:$GOPATH/bin #go test ./pkg/pipeline -#go test -v ./pkg/pipeline -run TestExecuteWorkflow +go test -v ./pkg/pipeline -run TestExecuteWorkflow exit #to get out of container From 218e23d7f1516b6d947b9ea0119acaecf445683b Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 10 May 2024 22:57:34 +0530 Subject: [PATCH 087/123] renamed filename --- Makefile | 2 +- ...ide_docker.sh => exportingReqEnvsExecutingWireNilChecker.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/integrationTesting/{execute_script_inside_docker.sh => exportingReqEnvsExecutingWireNilChecker.sh} (100%) diff --git a/Makefile b/Makefile index 04025f766f..df772d7531 100755 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ test-unit: test-integration: docker run --env-file=wireNilChecker.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro -v $(PWD)/temp/:/tempfile docker:dind - docker exec dind-test sh -c "mkdir test && cp -r wirenil/* test/ && ./test/tests/integrationTesting/execute_script_inside_docker.sh" + docker exec dind-test sh -c "mkdir test && cp -r wirenil/* test/ && ./test/tests/integrationTesting/exportingReqEnvsExecutingWireNilChecker.sh" run: build ./devtron .PHONY: build diff --git a/tests/integrationTesting/execute_script_inside_docker.sh b/tests/integrationTesting/exportingReqEnvsExecutingWireNilChecker.sh similarity index 100% rename from tests/integrationTesting/execute_script_inside_docker.sh rename to tests/integrationTesting/exportingReqEnvsExecutingWireNilChecker.sh From e741c2fc833acae8940176cd1ac675a9c8d188ef Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Fri, 10 May 2024 23:02:21 +0530 Subject: [PATCH 088/123] renamed filename --- Makefile | 2 +- ...tingWireNilChecker.sh => exportEnvsExecuteWireNilChecker.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/integrationTesting/{exportingReqEnvsExecutingWireNilChecker.sh => exportEnvsExecuteWireNilChecker.sh} (100%) diff --git a/Makefile b/Makefile index df772d7531..8bcd505a2d 100755 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ test-unit: test-integration: docker run --env-file=wireNilChecker.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro -v $(PWD)/temp/:/tempfile docker:dind - docker exec dind-test sh -c "mkdir test && cp -r wirenil/* test/ && ./test/tests/integrationTesting/exportingReqEnvsExecutingWireNilChecker.sh" + docker exec dind-test sh -c "mkdir test && cp -r wirenil/* test/ && ./test/tests/integrationTesting/exportEnvsExecuteWireNilChecker.sh" run: build ./devtron .PHONY: build diff --git a/tests/integrationTesting/exportingReqEnvsExecutingWireNilChecker.sh b/tests/integrationTesting/exportEnvsExecuteWireNilChecker.sh similarity index 100% rename from tests/integrationTesting/exportingReqEnvsExecutingWireNilChecker.sh rename to tests/integrationTesting/exportEnvsExecuteWireNilChecker.sh From 748b87ee80ca4ca6fe21561d300c1e472ff53ec3 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 13 May 2024 13:31:37 +0530 Subject: [PATCH 089/123] updated migartion yaml file --- tests/integrationTesting/migrator.yaml | 65 ++++++++++++++++++++------ 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index 5ee339c9c5..b04584076c 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -1,40 +1,77 @@ apiVersion: batch/v1 kind: Job metadata: - name: postgresql-migrate-devtron + name: postgresql-migrate-orchestrator + namespace: devtroncd spec: + activeDeadlineSeconds: 1500 + backoffLimit: 20 + completions: 1 + parallelism: 1 + selector: + suspend: false template: + metadata: + labels: + job-name: postgresql-migrate-orchestrator spec: containers: - - name: postgresql-migrate-devtron - image: quay.io/devtron/migrator:71748de9-149-11112 + - command: + - /bin/sh + - -c + - if [ $(MIGRATE_TO_VERSION) -eq "0" ]; then migrate -path $(SCRIPT_LOCATION) -database postgres://$(DB_USER_NAME):$(DB_PASSWORD)@$(DB_HOST):$(DB_PORT)/$(DB_NAME)?sslmode=disable up; else echo $(MIGRATE_TO_VERSION); migrate -path $(SCRIPT_LOCATION) -database postgres://$(DB_USER_NAME):$(DB_PASSWORD)@$(DB_HOST):$(DB_PORT)/$(DB_NAME)?sslmode=disable goto $(MIGRATE_TO_VERSION); fi env: - - name: GIT_BRANCH - value: main - name: SCRIPT_LOCATION - value: scripts/sql/ - - name: GIT_REPO_URL - value: https://github.com/devtron-labs/devtron.git + value: /shared/sql/ - name: DB_TYPE value: postgres - name: DB_USER_NAME value: postgres - name: DB_HOST - value: postgresql-postgresql + value: prod-cd-postgres-14.devtroncd - name: DB_PORT value: "5432" - name: DB_NAME value: orchestrator - name: MIGRATE_TO_VERSION value: "0" - - name: GIT_HASH - value: 606a33bc5e9b15d8b1c404f0dfbd1c70b21b4063 envFrom: - secretRef: - name: postgresql-migrator + name: postgres-creds-3 + image: migrate/migrate:v4.16.1 + imagePullPolicy: IfNotPresent + name: postgresql-migrate-devtron + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /shared + name: shared-volume + dnsPolicy: ClusterFirst + imagePullSecrets: + - name: devtron-image-pull + initContainers: + - command: + - /bin/sh + - -c + - cp -r scripts/sql /shared/ + image: devtroninc.azurecr.io/devtron:bb92488a-4-22580 + imagePullPolicy: IfNotPresent + name: init-devtron + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /shared + name: shared-volume restartPolicy: OnFailure - backoffLimit: 20 - activeDeadlineSeconds: 1500 + schedulerName: default-scheduler + securityContext: {} + terminationGracePeriodSeconds: 30 + volumes: + - emptyDir: {} + name: shared-volume + ttlSecondsAfterFinished: 600 --- apiVersion: batch/v1 kind: Job From df70207fd98e23d6f2edb8c4fe79d971af7e5c80 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 13 May 2024 14:23:51 +0530 Subject: [PATCH 090/123] git hash comment resolved --- tests/integrationTesting/create-test-env.sh | 1 + tests/integrationTesting/migrator.yaml | 65 +++++---------------- wireNilChecker.env | 1 + 3 files changed, 16 insertions(+), 51 deletions(-) diff --git a/tests/integrationTesting/create-test-env.sh b/tests/integrationTesting/create-test-env.sh index 675fa4526d..472ce01fc6 100755 --- a/tests/integrationTesting/create-test-env.sh +++ b/tests/integrationTesting/create-test-env.sh @@ -16,6 +16,7 @@ install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl kubectl create ns devtroncd kubectl create ns devtron-cd kubectl create ns devtron-ci +kubectl -n devtroncd create cm git-hash-cm --from-literal=GIT_HASH=$GIT_HASH kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/postgresql-secret.yaml kubectl -ndevtroncd apply -f $PWD/tests/integrationTesting/postgresql.yaml kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/devtron-secret.yaml diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index b04584076c..a25f4672f6 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -1,34 +1,26 @@ apiVersion: batch/v1 kind: Job metadata: - name: postgresql-migrate-orchestrator - namespace: devtroncd + name: postgresql-migrate-devtron spec: - activeDeadlineSeconds: 1500 - backoffLimit: 20 - completions: 1 - parallelism: 1 - selector: - suspend: false template: - metadata: - labels: - job-name: postgresql-migrate-orchestrator spec: containers: - - command: - - /bin/sh - - -c - - if [ $(MIGRATE_TO_VERSION) -eq "0" ]; then migrate -path $(SCRIPT_LOCATION) -database postgres://$(DB_USER_NAME):$(DB_PASSWORD)@$(DB_HOST):$(DB_PORT)/$(DB_NAME)?sslmode=disable up; else echo $(MIGRATE_TO_VERSION); migrate -path $(SCRIPT_LOCATION) -database postgres://$(DB_USER_NAME):$(DB_PASSWORD)@$(DB_HOST):$(DB_PORT)/$(DB_NAME)?sslmode=disable goto $(MIGRATE_TO_VERSION); fi + - name: postgresql-migrate-devtron + image: quay.io/devtron/migrator:71748de9-149-11112 env: + - name: GIT_BRANCH + value: main - name: SCRIPT_LOCATION - value: /shared/sql/ + value: scripts/sql/ + - name: GIT_REPO_URL + value: https://github.com/devtron-labs/devtron.git - name: DB_TYPE value: postgres - name: DB_USER_NAME value: postgres - name: DB_HOST - value: prod-cd-postgres-14.devtroncd + value: postgresql-postgresql - name: DB_PORT value: "5432" - name: DB_NAME @@ -37,41 +29,12 @@ spec: value: "0" envFrom: - secretRef: - name: postgres-creds-3 - image: migrate/migrate:v4.16.1 - imagePullPolicy: IfNotPresent - name: postgresql-migrate-devtron - resources: {} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /shared - name: shared-volume - dnsPolicy: ClusterFirst - imagePullSecrets: - - name: devtron-image-pull - initContainers: - - command: - - /bin/sh - - -c - - cp -r scripts/sql /shared/ - image: devtroninc.azurecr.io/devtron:bb92488a-4-22580 - imagePullPolicy: IfNotPresent - name: init-devtron - resources: {} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /shared - name: shared-volume + name: postgresql-migrator + - configMapRef: + name: git-hash-cm restartPolicy: OnFailure - schedulerName: default-scheduler - securityContext: {} - terminationGracePeriodSeconds: 30 - volumes: - - emptyDir: {} - name: shared-volume - ttlSecondsAfterFinished: 600 + backoffLimit: 20 + activeDeadlineSeconds: 1500 --- apiVersion: batch/v1 kind: Job diff --git a/wireNilChecker.env b/wireNilChecker.env index f23b4d372f..a209f9a6c8 100644 --- a/wireNilChecker.env +++ b/wireNilChecker.env @@ -39,4 +39,5 @@ TEST_BRANCH= LATEST_HASH= GOPATH=/usr/local/go PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin +GIT_HASH=d6e81951979389b3b6f1db4ffb56fbb87b650834 From 79db9a7a3ae1f46da7fabde4d772f008a30a6989 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 13 May 2024 15:47:55 +0530 Subject: [PATCH 091/123] git hash comment resolved --- Makefile | 7 ++++++- wireNilChecker.env | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8bcd505a2d..04c27c9390 100755 --- a/Makefile +++ b/Makefile @@ -41,7 +41,12 @@ test-unit: test-integration: docker run --env-file=wireNilChecker.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro -v $(PWD)/temp/:/tempfile docker:dind - docker exec dind-test sh -c "mkdir test && cp -r wirenil/* test/ && ./test/tests/integrationTesting/exportEnvsExecuteWireNilChecker.sh" + GIT_HASH=$(GIT_HASH) + @echo "GIT_HASH is $(GIT_HASH)" + ifndef GIT_HASH + GIT_HASH := $(shell git log --pretty=format:'%h' -n 1) + @echo "My GIT_HASH is $(GIT_HASH)" + docker exec dind-test sh -c "export GIT_HASH=$$GIT_HASH && mkdir test && cp -r wirenil/* test/ && ./test/tests/integrationTesting/exportEnvsExecuteWireNilChecker.sh" run: build ./devtron .PHONY: build diff --git a/wireNilChecker.env b/wireNilChecker.env index a209f9a6c8..f23b4d372f 100644 --- a/wireNilChecker.env +++ b/wireNilChecker.env @@ -39,5 +39,4 @@ TEST_BRANCH= LATEST_HASH= GOPATH=/usr/local/go PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin -GIT_HASH=d6e81951979389b3b6f1db4ffb56fbb87b650834 From 5c0d76338931614ca5ea9bdb6c7f0c7abfcd138e Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 13 May 2024 15:54:53 +0530 Subject: [PATCH 092/123] updated makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 04c27c9390..76d0ecab13 100755 --- a/Makefile +++ b/Makefile @@ -44,8 +44,8 @@ test-integration: GIT_HASH=$(GIT_HASH) @echo "GIT_HASH is $(GIT_HASH)" ifndef GIT_HASH - GIT_HASH := $(shell git log --pretty=format:'%h' -n 1) - @echo "My GIT_HASH is $(GIT_HASH)" + GIT_HASH = $(shell git log --pretty=format:'%h' -n 1) + endif docker exec dind-test sh -c "export GIT_HASH=$$GIT_HASH && mkdir test && cp -r wirenil/* test/ && ./test/tests/integrationTesting/exportEnvsExecuteWireNilChecker.sh" run: build ./devtron From 90e1edb7ccaf68c1e7cd487d485427a42b9a9334 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 13 May 2024 15:57:49 +0530 Subject: [PATCH 093/123] updated makefile --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 76d0ecab13..1bfb41fe0b 100755 --- a/Makefile +++ b/Makefile @@ -45,7 +45,6 @@ test-integration: @echo "GIT_HASH is $(GIT_HASH)" ifndef GIT_HASH GIT_HASH = $(shell git log --pretty=format:'%h' -n 1) - endif docker exec dind-test sh -c "export GIT_HASH=$$GIT_HASH && mkdir test && cp -r wirenil/* test/ && ./test/tests/integrationTesting/exportEnvsExecuteWireNilChecker.sh" run: build ./devtron From 2611223b9d06dd51ff3bf911996b028b7017ce00 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 13 May 2024 16:03:48 +0530 Subject: [PATCH 094/123] updated makefile --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 1bfb41fe0b..76d0ecab13 100755 --- a/Makefile +++ b/Makefile @@ -45,6 +45,7 @@ test-integration: @echo "GIT_HASH is $(GIT_HASH)" ifndef GIT_HASH GIT_HASH = $(shell git log --pretty=format:'%h' -n 1) + endif docker exec dind-test sh -c "export GIT_HASH=$$GIT_HASH && mkdir test && cp -r wirenil/* test/ && ./test/tests/integrationTesting/exportEnvsExecuteWireNilChecker.sh" run: build ./devtron From 0d66f8ad162fead48feeafd0ad081fdd6a6998d2 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 13 May 2024 17:14:05 +0530 Subject: [PATCH 095/123] move docker exec cmd into other folder --- Makefile | 7 +------ tests/integrationTesting/exportGitHash.sh | 7 +++++++ wireNilChecker.env | 1 + 3 files changed, 9 insertions(+), 6 deletions(-) create mode 100755 tests/integrationTesting/exportGitHash.sh diff --git a/Makefile b/Makefile index 76d0ecab13..b1c9a007c2 100755 --- a/Makefile +++ b/Makefile @@ -41,12 +41,7 @@ test-unit: test-integration: docker run --env-file=wireNilChecker.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro -v $(PWD)/temp/:/tempfile docker:dind - GIT_HASH=$(GIT_HASH) - @echo "GIT_HASH is $(GIT_HASH)" - ifndef GIT_HASH - GIT_HASH = $(shell git log --pretty=format:'%h' -n 1) - endif - docker exec dind-test sh -c "export GIT_HASH=$$GIT_HASH && mkdir test && cp -r wirenil/* test/ && ./test/tests/integrationTesting/exportEnvsExecuteWireNilChecker.sh" + ./tests/integrationTesting/exportGitHash.sh run: build ./devtron .PHONY: build diff --git a/tests/integrationTesting/exportGitHash.sh b/tests/integrationTesting/exportGitHash.sh new file mode 100755 index 0000000000..1de27195a9 --- /dev/null +++ b/tests/integrationTesting/exportGitHash.sh @@ -0,0 +1,7 @@ +GITHASH=$GIT_HASH +if [ -z "$GITHASH" ]; then + GIT_HASH=$(git log --pretty=format:'%h' -n 1); + echo "laeeq" $GITHASH +fi; +echo "GIT_HASH is" $GITHASH +docker exec dind-test sh -c "export GIT_HASH=$GITHASH && mkdir test && cp -r wirenil/* test/ && ./test/tests/integrationTesting/exportEnvsExecuteWireNilChecker.sh" \ No newline at end of file diff --git a/wireNilChecker.env b/wireNilChecker.env index f23b4d372f..63b0f7487a 100644 --- a/wireNilChecker.env +++ b/wireNilChecker.env @@ -40,3 +40,4 @@ LATEST_HASH= GOPATH=/usr/local/go PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin + From 87e54589443002859768a2f3b22f27881a9c9b48 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 13 May 2024 17:52:00 +0530 Subject: [PATCH 096/123] move docker exec cmd into other folder --- tests/integrationTesting/exportGitHash.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integrationTesting/exportGitHash.sh b/tests/integrationTesting/exportGitHash.sh index 1de27195a9..c0bde7ebe5 100755 --- a/tests/integrationTesting/exportGitHash.sh +++ b/tests/integrationTesting/exportGitHash.sh @@ -1,6 +1,9 @@ GITHASH=$GIT_HASH if [ -z "$GITHASH" ]; then - GIT_HASH=$(git log --pretty=format:'%h' -n 1); + echo "-------laeeeeeeeeeeeeq-------------" + git log + echo "------laeeqenccddddd----------------" + GIT_HASH=$$(git log --pretty=format:'%h' -n 1); echo "laeeq" $GITHASH fi; echo "GIT_HASH is" $GITHASH From 6a5041e36f0b946b9032f53aab22efef4d8051c4 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 13 May 2024 17:56:34 +0530 Subject: [PATCH 097/123] move docker exec cmd into other folder --- tests/integrationTesting/exportGitHash.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integrationTesting/exportGitHash.sh b/tests/integrationTesting/exportGitHash.sh index c0bde7ebe5..04e9c8911d 100755 --- a/tests/integrationTesting/exportGitHash.sh +++ b/tests/integrationTesting/exportGitHash.sh @@ -3,7 +3,6 @@ if [ -z "$GITHASH" ]; then echo "-------laeeeeeeeeeeeeq-------------" git log echo "------laeeqenccddddd----------------" - GIT_HASH=$$(git log --pretty=format:'%h' -n 1); echo "laeeq" $GITHASH fi; echo "GIT_HASH is" $GITHASH From fac18ab378189bbb4a2fdcef1a4f800d24c9a344 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 13 May 2024 18:15:45 +0530 Subject: [PATCH 098/123] move docker exec cmd into other folder --- tests/integrationTesting/exportGitHash.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integrationTesting/exportGitHash.sh b/tests/integrationTesting/exportGitHash.sh index 04e9c8911d..5ecdf360e5 100755 --- a/tests/integrationTesting/exportGitHash.sh +++ b/tests/integrationTesting/exportGitHash.sh @@ -1,9 +1,8 @@ GITHASH=$GIT_HASH if [ -z "$GITHASH" ]; then echo "-------laeeeeeeeeeeeeq-------------" - git log + git log --pretty=format:'%h' -n 1 echo "------laeeqenccddddd----------------" - echo "laeeq" $GITHASH fi; echo "GIT_HASH is" $GITHASH docker exec dind-test sh -c "export GIT_HASH=$GITHASH && mkdir test && cp -r wirenil/* test/ && ./test/tests/integrationTesting/exportEnvsExecuteWireNilChecker.sh" \ No newline at end of file From a16d771e8da689f0a386708c8ae2e06a10205155 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 13 May 2024 18:24:38 +0530 Subject: [PATCH 099/123] move docker exec cmd into other folder --- tests/integrationTesting/exportGitHash.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integrationTesting/exportGitHash.sh b/tests/integrationTesting/exportGitHash.sh index 5ecdf360e5..4e878d0ef1 100755 --- a/tests/integrationTesting/exportGitHash.sh +++ b/tests/integrationTesting/exportGitHash.sh @@ -1,7 +1,8 @@ +GITHASH=123 GITHASH=$GIT_HASH if [ -z "$GITHASH" ]; then echo "-------laeeeeeeeeeeeeq-------------" - git log --pretty=format:'%h' -n 1 + GITHASH=$(git log --pretty=format:'%h' -n 1) echo "------laeeqenccddddd----------------" fi; echo "GIT_HASH is" $GITHASH From b60242c4ae59125cb2cf147f915272d50aecef71 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 13 May 2024 18:29:53 +0530 Subject: [PATCH 100/123] move docker exec cmd into other folder --- tests/integrationTesting/exportGitHash.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrationTesting/exportGitHash.sh b/tests/integrationTesting/exportGitHash.sh index 4e878d0ef1..46e6b381f4 100755 --- a/tests/integrationTesting/exportGitHash.sh +++ b/tests/integrationTesting/exportGitHash.sh @@ -2,7 +2,7 @@ GITHASH=123 GITHASH=$GIT_HASH if [ -z "$GITHASH" ]; then echo "-------laeeeeeeeeeeeeq-------------" - GITHASH=$(git log --pretty=format:'%h' -n 1) + export GITHASH=$(git log --pretty=format:'%h' -n 1) echo "------laeeqenccddddd----------------" fi; echo "GIT_HASH is" $GITHASH From 882270d800cfa38cc78d13b5416f48991ab156d1 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 13 May 2024 18:33:02 +0530 Subject: [PATCH 101/123] create a new file --- tests/integrationTesting/exportGitHash.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/integrationTesting/exportGitHash.sh b/tests/integrationTesting/exportGitHash.sh index 46e6b381f4..677ba83553 100755 --- a/tests/integrationTesting/exportGitHash.sh +++ b/tests/integrationTesting/exportGitHash.sh @@ -1,9 +1,5 @@ -GITHASH=123 GITHASH=$GIT_HASH if [ -z "$GITHASH" ]; then - echo "-------laeeeeeeeeeeeeq-------------" export GITHASH=$(git log --pretty=format:'%h' -n 1) - echo "------laeeqenccddddd----------------" fi; -echo "GIT_HASH is" $GITHASH docker exec dind-test sh -c "export GIT_HASH=$GITHASH && mkdir test && cp -r wirenil/* test/ && ./test/tests/integrationTesting/exportEnvsExecuteWireNilChecker.sh" \ No newline at end of file From 4d1b91e50b2ba3b8f41e2fc763ed3f4d171fbd50 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Mon, 13 May 2024 18:54:39 +0530 Subject: [PATCH 102/123] updated code and renamed file name --- Makefile | 2 +- .../{exportGitHash.sh => exportGitHashRunDockerExec.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/integrationTesting/{exportGitHash.sh => exportGitHashRunDockerExec.sh} (100%) diff --git a/Makefile b/Makefile index b1c9a007c2..ad6420edae 100755 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ test-unit: test-integration: docker run --env-file=wireNilChecker.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro -v $(PWD)/temp/:/tempfile docker:dind - ./tests/integrationTesting/exportGitHash.sh + ./tests/integrationTesting/exportGitHashRunDockerExec.sh run: build ./devtron .PHONY: build diff --git a/tests/integrationTesting/exportGitHash.sh b/tests/integrationTesting/exportGitHashRunDockerExec.sh similarity index 100% rename from tests/integrationTesting/exportGitHash.sh rename to tests/integrationTesting/exportGitHashRunDockerExec.sh From 5cc19315496154fcf612a6dbe8d8a7aec0335c27 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 14 May 2024 20:16:04 +0530 Subject: [PATCH 103/123] increased migrator deadline --- tests/integrationTesting/migrator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index a25f4672f6..8d9f466e77 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -34,7 +34,7 @@ spec: name: git-hash-cm restartPolicy: OnFailure backoffLimit: 20 - activeDeadlineSeconds: 1500 + activeDeadlineSeconds: 1500000 --- apiVersion: batch/v1 kind: Job From caa3acd18dee6944b230979c7f5bef29a100e1b7 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 14 May 2024 20:28:44 +0530 Subject: [PATCH 104/123] added sleep --- tests/integrationTesting/migrator.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index 8d9f466e77..fbbaff00cd 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -8,6 +8,7 @@ spec: containers: - name: postgresql-migrate-devtron image: quay.io/devtron/migrator:71748de9-149-11112 + command: ['sh', '-c', 'echo "Hello, Kubernetes!" && sleep 36000'] env: - name: GIT_BRANCH value: main @@ -34,7 +35,7 @@ spec: name: git-hash-cm restartPolicy: OnFailure backoffLimit: 20 - activeDeadlineSeconds: 1500000 + activeDeadlineSeconds: 150 --- apiVersion: batch/v1 kind: Job From 7410b32f19c12c16c364f195b7551d9a9f01d46f Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Wed, 15 May 2024 00:06:44 +0530 Subject: [PATCH 105/123] added sleep --- tests/integrationTesting/migrator.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index fbbaff00cd..d756e17e5a 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -8,7 +8,8 @@ spec: containers: - name: postgresql-migrate-devtron image: quay.io/devtron/migrator:71748de9-149-11112 - command: ['sh', '-c', 'echo "Hello, Kubernetes!" && sleep 36000'] + command: [ "/bin/sh", "-c" ] + args: [ "sleep 3600" ] env: - name: GIT_BRANCH value: main From 61cf41aa997ce6edf59a834fc9dbd655946afe2c Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Thu, 16 May 2024 12:54:28 +0530 Subject: [PATCH 106/123] testing migrator --- .../exportGitHashRunDockerExec.sh | 4 ---- tests/integrationTesting/migrator.yaml | 17 ++++++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/integrationTesting/exportGitHashRunDockerExec.sh b/tests/integrationTesting/exportGitHashRunDockerExec.sh index 677ba83553..fc918613e0 100755 --- a/tests/integrationTesting/exportGitHashRunDockerExec.sh +++ b/tests/integrationTesting/exportGitHashRunDockerExec.sh @@ -1,5 +1 @@ -GITHASH=$GIT_HASH -if [ -z "$GITHASH" ]; then - export GITHASH=$(git log --pretty=format:'%h' -n 1) -fi; docker exec dind-test sh -c "export GIT_HASH=$GITHASH && mkdir test && cp -r wirenil/* test/ && ./test/tests/integrationTesting/exportEnvsExecuteWireNilChecker.sh" \ No newline at end of file diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index d756e17e5a..aadb8f3fe2 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -7,16 +7,13 @@ spec: spec: containers: - name: postgresql-migrate-devtron - image: quay.io/devtron/migrator:71748de9-149-11112 - command: [ "/bin/sh", "-c" ] - args: [ "sleep 3600" ] + image: laeeqa222/abcd:5f5ebe0d-43-225 + volumeMounts: + - mountPath: /tmp/app/scripts/sql/ + name: sql-scripts-volume env: - - name: GIT_BRANCH - value: main - name: SCRIPT_LOCATION value: scripts/sql/ - - name: GIT_REPO_URL - value: https://github.com/devtron-labs/devtron.git - name: DB_TYPE value: postgres - name: DB_USER_NAME @@ -29,12 +26,18 @@ spec: value: orchestrator - name: MIGRATE_TO_VERSION value: "0" + - name: SCRIPT_MOUNTED + value: true envFrom: - secretRef: name: postgresql-migrator - configMapRef: name: git-hash-cm restartPolicy: OnFailure + volumes: + - name: sql-scripts-volume + hostPath: + path: /test/scripts/sql/ backoffLimit: 20 activeDeadlineSeconds: 150 --- From cf9dacae2ab6f0964f156920327b235a141a4a7b Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Thu, 16 May 2024 13:01:46 +0530 Subject: [PATCH 107/123] upadted env value --- tests/integrationTesting/migrator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index aadb8f3fe2..c04a32c1d4 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -27,7 +27,7 @@ spec: - name: MIGRATE_TO_VERSION value: "0" - name: SCRIPT_MOUNTED - value: true + value: "true" envFrom: - secretRef: name: postgresql-migrator From d2790b44903cd691ac35b6f9482dc2881e9a2d87 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Thu, 16 May 2024 13:39:42 +0530 Subject: [PATCH 108/123] upadted env value --- tests/integrationTesting/migrator.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index c04a32c1d4..e2abfb764b 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -1,17 +1,21 @@ apiVersion: batch/v1 kind: Job metadata: - name: postgresql-migrate-devtron + name: laeeq spec: template: spec: containers: - name: postgresql-migrate-devtron - image: laeeqa222/abcd:5f5ebe0d-43-225 + image: laeeqa222/abcd:5f5ebe0d-43-225////quay.io/devtron/migrator:71748de9-149-11112 volumeMounts: - mountPath: /tmp/app/scripts/sql/ name: sql-scripts-volume env: + - name: GIT_HASH + value: be7da471e45a501eba19eaa5f8d08dfe5601598d + - name: GIT_REPO_URL + value: https://github.com/devtron-labs/devtron.git - name: SCRIPT_LOCATION value: scripts/sql/ - name: DB_TYPE @@ -31,8 +35,6 @@ spec: envFrom: - secretRef: name: postgresql-migrator - - configMapRef: - name: git-hash-cm restartPolicy: OnFailure volumes: - name: sql-scripts-volume From 16e3bca9110ae463b8babf16a17515fd13ab37cb Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Thu, 16 May 2024 13:40:19 +0530 Subject: [PATCH 109/123] upadted img --- tests/integrationTesting/migrator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index e2abfb764b..9cc0bd40e9 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -7,7 +7,7 @@ spec: spec: containers: - name: postgresql-migrate-devtron - image: laeeqa222/abcd:5f5ebe0d-43-225////quay.io/devtron/migrator:71748de9-149-11112 + image: laeeqa222/abcd:5f5ebe0d-43-225 volumeMounts: - mountPath: /tmp/app/scripts/sql/ name: sql-scripts-volume From 5e2afc1e1d7c34feff4c537d216b125bff679a17 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Thu, 16 May 2024 13:51:45 +0530 Subject: [PATCH 110/123] upadted img --- tests/integrationTesting/migrator.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index 9cc0bd40e9..cb32fe7c7c 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -1,7 +1,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: laeeq + name: postgresql-migrate-devtron spec: template: spec: @@ -41,7 +41,7 @@ spec: hostPath: path: /test/scripts/sql/ backoffLimit: 20 - activeDeadlineSeconds: 150 + activeDeadlineSeconds: 150000 --- apiVersion: batch/v1 kind: Job From 828426076c05f210286c43492d96d51d4ee34e19 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Thu, 16 May 2024 13:59:23 +0530 Subject: [PATCH 111/123] upadted img --- tests/integrationTesting/migrator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index cb32fe7c7c..590c9fa439 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -39,7 +39,7 @@ spec: volumes: - name: sql-scripts-volume hostPath: - path: /test/scripts/sql/ + path: //test/scripts/sql/ backoffLimit: 20 activeDeadlineSeconds: 150000 --- From ff761221222a10215e1a2d8bcc0aefff97bff626 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Thu, 16 May 2024 14:11:28 +0530 Subject: [PATCH 112/123] upadted img --- tests/integrationTesting/migrator.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index 590c9fa439..d0a356f850 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -11,6 +11,8 @@ spec: volumeMounts: - mountPath: /tmp/app/scripts/sql/ name: sql-scripts-volume + command: [ "/bin/sh", "-c" ] + args: [ "sleep 3600" ] env: - name: GIT_HASH value: be7da471e45a501eba19eaa5f8d08dfe5601598d @@ -39,7 +41,7 @@ spec: volumes: - name: sql-scripts-volume hostPath: - path: //test/scripts/sql/ + path: /test/scripts/sql/ backoffLimit: 20 activeDeadlineSeconds: 150000 --- From 21ec99693c9b24d48a5d714bc86d42f7c25f0bef Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Thu, 16 May 2024 14:18:27 +0530 Subject: [PATCH 113/123] upadted img --- tests/integrationTesting/migrator.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index d0a356f850..b66d9dfc25 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -9,7 +9,7 @@ spec: - name: postgresql-migrate-devtron image: laeeqa222/abcd:5f5ebe0d-43-225 volumeMounts: - - mountPath: /tmp/app/scripts/sql/ + - mountPath: /laeeq/app/scripts/sql/ name: sql-scripts-volume command: [ "/bin/sh", "-c" ] args: [ "sleep 3600" ] From 910aa6db48d4d533fe7b41c9bc2e9de0df72ac98 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Thu, 16 May 2024 14:27:34 +0530 Subject: [PATCH 114/123] upadted img --- tests/integrationTesting/migrator.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index b66d9dfc25..e5be639f87 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -42,6 +42,7 @@ spec: - name: sql-scripts-volume hostPath: path: /test/scripts/sql/ + type: DirectoryOrCreate backoffLimit: 20 activeDeadlineSeconds: 150000 --- From a89ef63212dc81e8d2c30ed063fe7fe45120139f Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Thu, 16 May 2024 14:37:31 +0530 Subject: [PATCH 115/123] upadted img --- tests/integrationTesting/migrator.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index e5be639f87..c2c9590218 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -9,7 +9,7 @@ spec: - name: postgresql-migrate-devtron image: laeeqa222/abcd:5f5ebe0d-43-225 volumeMounts: - - mountPath: /laeeq/app/scripts/sql/ + - mountPath: /laeeq/app/scripts/sql name: sql-scripts-volume command: [ "/bin/sh", "-c" ] args: [ "sleep 3600" ] @@ -41,7 +41,7 @@ spec: volumes: - name: sql-scripts-volume hostPath: - path: /test/scripts/sql/ + path: /test/scripts/sql type: DirectoryOrCreate backoffLimit: 20 activeDeadlineSeconds: 150000 From b2a88b7c7afdb3c31444a660025a7203448cacad Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 21 May 2024 13:44:13 +0530 Subject: [PATCH 116/123] add volume mounted functionality --- Makefile | 2 +- tests/integrationTesting/create-test-env.sh | 1 + .../exportGitHashRunDockerExec.sh | 1 - tests/integrationTesting/migrator.yaml | 14 +- .../Microsoft/go-winio/internal/fs/fs.go | 202 ++++++++++++++++++ 5 files changed, 208 insertions(+), 12 deletions(-) delete mode 100755 tests/integrationTesting/exportGitHashRunDockerExec.sh create mode 100644 vendor/github.com/Microsoft/go-winio/internal/fs/fs.go diff --git a/Makefile b/Makefile index ad6420edae..8bcd505a2d 100755 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ test-unit: test-integration: docker run --env-file=wireNilChecker.env --privileged -d --name dind-test -v $(PWD)/:/wirenil/:ro -v $(PWD)/temp/:/tempfile docker:dind - ./tests/integrationTesting/exportGitHashRunDockerExec.sh + docker exec dind-test sh -c "mkdir test && cp -r wirenil/* test/ && ./test/tests/integrationTesting/exportEnvsExecuteWireNilChecker.sh" run: build ./devtron .PHONY: build diff --git a/tests/integrationTesting/create-test-env.sh b/tests/integrationTesting/create-test-env.sh index 472ce01fc6..50f15dccbf 100755 --- a/tests/integrationTesting/create-test-env.sh +++ b/tests/integrationTesting/create-test-env.sh @@ -21,6 +21,7 @@ kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/postgresql-secret.ya kubectl -ndevtroncd apply -f $PWD/tests/integrationTesting/postgresql.yaml kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/devtron-secret.yaml kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/nats-server.yaml +docker cp $PWD/scripts/sql/ k3d-it-cluster-server-0:./tmp/scripts yq '(select(.metadata.name == "postgresql-migrate-devtron") | .spec.template.spec.containers[0].env[0].value) = env(TEST_BRANCH)' $PWD/tests/integrationTesting/migrator.yaml -i yq '(select(.metadata.name == "postgresql-migrate-devtron") | .spec.template.spec.containers[0].env[9].value) = env(LATEST_HASH)' $PWD/tests/integrationTesting/migrator.yaml -i kubectl -ndevtroncd apply -f $PWD/tests/integrationTesting/migrator.yaml diff --git a/tests/integrationTesting/exportGitHashRunDockerExec.sh b/tests/integrationTesting/exportGitHashRunDockerExec.sh deleted file mode 100755 index fc918613e0..0000000000 --- a/tests/integrationTesting/exportGitHashRunDockerExec.sh +++ /dev/null @@ -1 +0,0 @@ -docker exec dind-test sh -c "export GIT_HASH=$GITHASH && mkdir test && cp -r wirenil/* test/ && ./test/tests/integrationTesting/exportEnvsExecuteWireNilChecker.sh" \ No newline at end of file diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index c2c9590218..a621a7446f 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -7,17 +7,11 @@ spec: spec: containers: - name: postgresql-migrate-devtron - image: laeeqa222/abcd:5f5ebe0d-43-225 + image: laeeqa222/abcd:96531e29-12-4216 volumeMounts: - - mountPath: /laeeq/app/scripts/sql + - mountPath: /tmp/app/ name: sql-scripts-volume - command: [ "/bin/sh", "-c" ] - args: [ "sleep 3600" ] env: - - name: GIT_HASH - value: be7da471e45a501eba19eaa5f8d08dfe5601598d - - name: GIT_REPO_URL - value: https://github.com/devtron-labs/devtron.git - name: SCRIPT_LOCATION value: scripts/sql/ - name: DB_TYPE @@ -41,10 +35,10 @@ spec: volumes: - name: sql-scripts-volume hostPath: - path: /test/scripts/sql + path: /tmp/scripts/ type: DirectoryOrCreate backoffLimit: 20 - activeDeadlineSeconds: 150000 + activeDeadlineSeconds: 1500 --- apiVersion: batch/v1 kind: Job diff --git a/vendor/github.com/Microsoft/go-winio/internal/fs/fs.go b/vendor/github.com/Microsoft/go-winio/internal/fs/fs.go new file mode 100644 index 0000000000..509b3ec641 --- /dev/null +++ b/vendor/github.com/Microsoft/go-winio/internal/fs/fs.go @@ -0,0 +1,202 @@ +//go:build windows + +package fs + +import ( + "golang.org/x/sys/windows" + + "github.com/Microsoft/go-winio/internal/stringbuffer" +) + +//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go fs.go + +// https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew +//sys CreateFile(name string, access AccessMask, mode FileShareMode, sa *syscall.SecurityAttributes, createmode FileCreationDisposition, attrs FileFlagOrAttribute, templatefile windows.Handle) (handle windows.Handle, err error) [failretval==windows.InvalidHandle] = CreateFileW + +const NullHandle windows.Handle = 0 + +// AccessMask defines standard, specific, and generic rights. +// +// Bitmask: +// 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 +// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 +// +---------------+---------------+-------------------------------+ +// |G|G|G|G|Resvd|A| StandardRights| SpecificRights | +// |R|W|E|A| |S| | | +// +-+-------------+---------------+-------------------------------+ +// +// GR Generic Read +// GW Generic Write +// GE Generic Exectue +// GA Generic All +// Resvd Reserved +// AS Access Security System +// +// https://learn.microsoft.com/en-us/windows/win32/secauthz/access-mask +// +// https://learn.microsoft.com/en-us/windows/win32/secauthz/generic-access-rights +// +// https://learn.microsoft.com/en-us/windows/win32/fileio/file-access-rights-constants +type AccessMask = windows.ACCESS_MASK + +//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API. +const ( + // Not actually any. + // + // For CreateFile: "query certain metadata such as file, directory, or device attributes without accessing that file or device" + // https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew#parameters + FILE_ANY_ACCESS AccessMask = 0 + + // Specific Object Access + // from ntioapi.h + + FILE_READ_DATA AccessMask = (0x0001) // file & pipe + FILE_LIST_DIRECTORY AccessMask = (0x0001) // directory + + FILE_WRITE_DATA AccessMask = (0x0002) // file & pipe + FILE_ADD_FILE AccessMask = (0x0002) // directory + + FILE_APPEND_DATA AccessMask = (0x0004) // file + FILE_ADD_SUBDIRECTORY AccessMask = (0x0004) // directory + FILE_CREATE_PIPE_INSTANCE AccessMask = (0x0004) // named pipe + + FILE_READ_EA AccessMask = (0x0008) // file & directory + FILE_READ_PROPERTIES AccessMask = FILE_READ_EA + + FILE_WRITE_EA AccessMask = (0x0010) // file & directory + FILE_WRITE_PROPERTIES AccessMask = FILE_WRITE_EA + + FILE_EXECUTE AccessMask = (0x0020) // file + FILE_TRAVERSE AccessMask = (0x0020) // directory + + FILE_DELETE_CHILD AccessMask = (0x0040) // directory + + FILE_READ_ATTRIBUTES AccessMask = (0x0080) // all + + FILE_WRITE_ATTRIBUTES AccessMask = (0x0100) // all + + FILE_ALL_ACCESS AccessMask = (STANDARD_RIGHTS_REQUIRED | SYNCHRONIZE | 0x1FF) + FILE_GENERIC_READ AccessMask = (STANDARD_RIGHTS_READ | FILE_READ_DATA | FILE_READ_ATTRIBUTES | FILE_READ_EA | SYNCHRONIZE) + FILE_GENERIC_WRITE AccessMask = (STANDARD_RIGHTS_WRITE | FILE_WRITE_DATA | FILE_WRITE_ATTRIBUTES | FILE_WRITE_EA | FILE_APPEND_DATA | SYNCHRONIZE) + FILE_GENERIC_EXECUTE AccessMask = (STANDARD_RIGHTS_EXECUTE | FILE_READ_ATTRIBUTES | FILE_EXECUTE | SYNCHRONIZE) + + SPECIFIC_RIGHTS_ALL AccessMask = 0x0000FFFF + + // Standard Access + // from ntseapi.h + + DELETE AccessMask = 0x0001_0000 + READ_CONTROL AccessMask = 0x0002_0000 + WRITE_DAC AccessMask = 0x0004_0000 + WRITE_OWNER AccessMask = 0x0008_0000 + SYNCHRONIZE AccessMask = 0x0010_0000 + + STANDARD_RIGHTS_REQUIRED AccessMask = 0x000F_0000 + + STANDARD_RIGHTS_READ AccessMask = READ_CONTROL + STANDARD_RIGHTS_WRITE AccessMask = READ_CONTROL + STANDARD_RIGHTS_EXECUTE AccessMask = READ_CONTROL + + STANDARD_RIGHTS_ALL AccessMask = 0x001F_0000 +) + +type FileShareMode uint32 + +//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API. +const ( + FILE_SHARE_NONE FileShareMode = 0x00 + FILE_SHARE_READ FileShareMode = 0x01 + FILE_SHARE_WRITE FileShareMode = 0x02 + FILE_SHARE_DELETE FileShareMode = 0x04 + FILE_SHARE_VALID_FLAGS FileShareMode = 0x07 +) + +type FileCreationDisposition uint32 + +//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API. +const ( + // from winbase.h + + CREATE_NEW FileCreationDisposition = 0x01 + CREATE_ALWAYS FileCreationDisposition = 0x02 + OPEN_EXISTING FileCreationDisposition = 0x03 + OPEN_ALWAYS FileCreationDisposition = 0x04 + TRUNCATE_EXISTING FileCreationDisposition = 0x05 +) + +// CreateFile and co. take flags or attributes together as one parameter. +// Define alias until we can use generics to allow both + +// https://learn.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants +type FileFlagOrAttribute uint32 + +//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API. +const ( // from winnt.h + FILE_FLAG_WRITE_THROUGH FileFlagOrAttribute = 0x8000_0000 + FILE_FLAG_OVERLAPPED FileFlagOrAttribute = 0x4000_0000 + FILE_FLAG_NO_BUFFERING FileFlagOrAttribute = 0x2000_0000 + FILE_FLAG_RANDOM_ACCESS FileFlagOrAttribute = 0x1000_0000 + FILE_FLAG_SEQUENTIAL_SCAN FileFlagOrAttribute = 0x0800_0000 + FILE_FLAG_DELETE_ON_CLOSE FileFlagOrAttribute = 0x0400_0000 + FILE_FLAG_BACKUP_SEMANTICS FileFlagOrAttribute = 0x0200_0000 + FILE_FLAG_POSIX_SEMANTICS FileFlagOrAttribute = 0x0100_0000 + FILE_FLAG_OPEN_REPARSE_POINT FileFlagOrAttribute = 0x0020_0000 + FILE_FLAG_OPEN_NO_RECALL FileFlagOrAttribute = 0x0010_0000 + FILE_FLAG_FIRST_PIPE_INSTANCE FileFlagOrAttribute = 0x0008_0000 +) + +type FileSQSFlag = FileFlagOrAttribute + +//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API. +const ( // from winbase.h + SECURITY_ANONYMOUS FileSQSFlag = FileSQSFlag(SecurityAnonymous << 16) + SECURITY_IDENTIFICATION FileSQSFlag = FileSQSFlag(SecurityIdentification << 16) + SECURITY_IMPERSONATION FileSQSFlag = FileSQSFlag(SecurityImpersonation << 16) + SECURITY_DELEGATION FileSQSFlag = FileSQSFlag(SecurityDelegation << 16) + + SECURITY_SQOS_PRESENT FileSQSFlag = 0x00100000 + SECURITY_VALID_SQOS_FLAGS FileSQSFlag = 0x001F0000 +) + +// GetFinalPathNameByHandle flags +// +// https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfinalpathnamebyhandlew#parameters +type GetFinalPathFlag uint32 + +//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API. +const ( + GetFinalPathDefaultFlag GetFinalPathFlag = 0x0 + + FILE_NAME_NORMALIZED GetFinalPathFlag = 0x0 + FILE_NAME_OPENED GetFinalPathFlag = 0x8 + + VOLUME_NAME_DOS GetFinalPathFlag = 0x0 + VOLUME_NAME_GUID GetFinalPathFlag = 0x1 + VOLUME_NAME_NT GetFinalPathFlag = 0x2 + VOLUME_NAME_NONE GetFinalPathFlag = 0x4 +) + +// getFinalPathNameByHandle facilitates calling the Windows API GetFinalPathNameByHandle +// with the given handle and flags. It transparently takes care of creating a buffer of the +// correct size for the call. +// +// https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfinalpathnamebyhandlew +func GetFinalPathNameByHandle(h windows.Handle, flags GetFinalPathFlag) (string, error) { + b := stringbuffer.NewWString() + //TODO: can loop infinitely if Win32 keeps returning the same (or a larger) n? + for { + n, err := windows.GetFinalPathNameByHandle(h, b.Pointer(), b.Cap(), uint32(flags)) + if err != nil { + return "", err + } + // If the buffer wasn't large enough, n will be the total size needed (including null terminator). + // Resize and try again. + if n > b.Cap() { + b.ResizeTo(n) + continue + } + // If the buffer is large enough, n will be the size not including the null terminator. + // Convert to a Go string and return. + return b.String(), nil + } +} From 78984442545c6c650293bd2a7df8d02363b0f7fe Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 21 May 2024 13:49:57 +0530 Subject: [PATCH 117/123] remove sql dir --- tests/integrationTesting/migrator.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index a621a7446f..47594e8eac 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -12,8 +12,6 @@ spec: - mountPath: /tmp/app/ name: sql-scripts-volume env: - - name: SCRIPT_LOCATION - value: scripts/sql/ - name: DB_TYPE value: postgres - name: DB_USER_NAME From 79bfe0dd9932944ab0041c5b0c826a180b46d4ea Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 21 May 2024 16:00:49 +0530 Subject: [PATCH 118/123] removed enforceUtil field --- tests/integrationTesting/migrator.yaml | 8 ++++---- util/rbac/EnforcerUtil.go | 7 +------ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index 47594e8eac..f79b25b508 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -7,7 +7,7 @@ spec: spec: containers: - name: postgresql-migrate-devtron - image: laeeqa222/abcd:96531e29-12-4216 + image: quay.io/devtron/test:e026843e-866-11925 volumeMounts: - mountPath: /tmp/app/ name: sql-scripts-volume @@ -47,7 +47,7 @@ spec: spec: containers: - name: postgresql-migrate-casbin - image: quay.io/devtron/migrator:71748de9-149-11112 + image: quay.io/devtron/test:e026843e-866-11925 env: - name: SCRIPT_LOCATION value: scripts/casbin/ @@ -85,7 +85,7 @@ spec: spec: containers: - name: postgresql-migrate-gitsensor - image: quay.io/devtron/migrator:71748de9-149-11112 + image: quay.io/devtron/test:e026843e-866-11925 env: - name: SCRIPT_LOCATION value: scripts/sql/ @@ -123,7 +123,7 @@ spec: spec: containers: - name: postgresql-migrate-lens - image: quay.io/devtron/migrator:71748de9-149-11112 + image: quay.io/devtron/test:e026843e-866-11925 env: - name: SCRIPT_LOCATION value: scripts/sql/ diff --git a/util/rbac/EnforcerUtil.go b/util/rbac/EnforcerUtil.go index b926e77b74..0eef86135f 100644 --- a/util/rbac/EnforcerUtil.go +++ b/util/rbac/EnforcerUtil.go @@ -88,7 +88,6 @@ type EnforcerUtilImpl struct { ciPipelineRepository pipelineConfig.CiPipelineRepository clusterRepository repository.ClusterRepository enforcer casbin.Enforcer - *EnforcerUtilHelmImpl } func NewEnforcerUtilImpl(logger *zap.SugaredLogger, teamRepository team.TeamRepository, @@ -103,11 +102,7 @@ func NewEnforcerUtilImpl(logger *zap.SugaredLogger, teamRepository team.TeamRepo pipelineRepository: pipelineRepository, ciPipelineRepository: ciPipelineRepository, clusterRepository: clusterRepository, - EnforcerUtilHelmImpl: &EnforcerUtilHelmImpl{ - logger: logger, - clusterRepository: clusterRepository, - }, - enforcer: enforcer, + enforcer: enforcer, } } From 8c676aa24c114d965f8bb3c7a8912587fc6e846a Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 21 May 2024 16:01:40 +0530 Subject: [PATCH 119/123] removed enforceUtil field --- tests/integrationTesting/create-test-env.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integrationTesting/create-test-env.sh b/tests/integrationTesting/create-test-env.sh index 50f15dccbf..133740caae 100755 --- a/tests/integrationTesting/create-test-env.sh +++ b/tests/integrationTesting/create-test-env.sh @@ -21,6 +21,7 @@ kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/postgresql-secret.ya kubectl -ndevtroncd apply -f $PWD/tests/integrationTesting/postgresql.yaml kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/devtron-secret.yaml kubectl -n devtroncd apply -f $PWD/tests/integrationTesting/nats-server.yaml +# we are copying sql scripts into node container and this conainer's name is fixed docker cp $PWD/scripts/sql/ k3d-it-cluster-server-0:./tmp/scripts yq '(select(.metadata.name == "postgresql-migrate-devtron") | .spec.template.spec.containers[0].env[0].value) = env(TEST_BRANCH)' $PWD/tests/integrationTesting/migrator.yaml -i yq '(select(.metadata.name == "postgresql-migrate-devtron") | .spec.template.spec.containers[0].env[9].value) = env(LATEST_HASH)' $PWD/tests/integrationTesting/migrator.yaml -i From 3ac29ef81f92245f892ea3605c91f67c058cf36e Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 21 May 2024 16:45:52 +0530 Subject: [PATCH 120/123] updated migrator image --- tests/integrationTesting/migrator.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integrationTesting/migrator.yaml b/tests/integrationTesting/migrator.yaml index f79b25b508..a34663f404 100644 --- a/tests/integrationTesting/migrator.yaml +++ b/tests/integrationTesting/migrator.yaml @@ -7,7 +7,7 @@ spec: spec: containers: - name: postgresql-migrate-devtron - image: quay.io/devtron/test:e026843e-866-11925 + image: quay.io/devtron/migrator:e026843e-866-11925 volumeMounts: - mountPath: /tmp/app/ name: sql-scripts-volume @@ -47,7 +47,7 @@ spec: spec: containers: - name: postgresql-migrate-casbin - image: quay.io/devtron/test:e026843e-866-11925 + image: quay.io/devtron/migrator:e026843e-866-11925 env: - name: SCRIPT_LOCATION value: scripts/casbin/ @@ -85,7 +85,7 @@ spec: spec: containers: - name: postgresql-migrate-gitsensor - image: quay.io/devtron/test:e026843e-866-11925 + image: quay.io/devtron/migrator:e026843e-866-11925 env: - name: SCRIPT_LOCATION value: scripts/sql/ @@ -123,7 +123,7 @@ spec: spec: containers: - name: postgresql-migrate-lens - image: quay.io/devtron/test:e026843e-866-11925 + image: quay.io/devtron/migrator:e026843e-866-11925 env: - name: SCRIPT_LOCATION value: scripts/sql/ From caddec036889c0eefc3c846a549278bf40587720 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 21 May 2024 18:33:15 +0530 Subject: [PATCH 121/123] added error in mod file --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index 50d62e68db..4570029193 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/evanphx/json-patch v5.6.0+incompatible github.com/gammazero/workerpool v1.1.3 github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 - github.com/go-errors/errors v1.4.2 github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-git/v5 v5.11.0 github.com/go-pg/pg v6.15.1+incompatible From ed89cc686eef339af824bd82161be9ab218d3509 Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 21 May 2024 19:28:49 +0530 Subject: [PATCH 122/123] added PROXY_SERVICE_CONFIG env --- wireNilChecker.env | 1 + 1 file changed, 1 insertion(+) diff --git a/wireNilChecker.env b/wireNilChecker.env index 63b0f7487a..5ead1a4787 100644 --- a/wireNilChecker.env +++ b/wireNilChecker.env @@ -39,5 +39,6 @@ TEST_BRANCH= LATEST_HASH= GOPATH=/usr/local/go PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin +PROXY_SERVICE_CONFIG={} From 1ecc6cccbd837bfe404da0b422857afd3e15b93f Mon Sep 17 00:00:00 2001 From: Laeeqdev Date: Tue, 21 May 2024 19:50:13 +0530 Subject: [PATCH 123/123] removed rbac fields --- pkg/cluster/ClusterService.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/cluster/ClusterService.go b/pkg/cluster/ClusterService.go index 45e561a59e..e2ea1e1271 100644 --- a/pkg/cluster/ClusterService.go +++ b/pkg/cluster/ClusterService.go @@ -197,7 +197,6 @@ type ClusterServiceImpl struct { userAuthRepository repository3.UserAuthRepository userRepository repository3.UserRepository roleGroupRepository repository3.RoleGroupRepository - *ClusterRbacServiceImpl } func NewClusterServiceImpl(repository repository.ClusterRepository, logger *zap.SugaredLogger, @@ -212,9 +211,6 @@ func NewClusterServiceImpl(repository repository.ClusterRepository, logger *zap. userAuthRepository: userAuthRepository, userRepository: userRepository, roleGroupRepository: roleGroupRepository, - ClusterRbacServiceImpl: &ClusterRbacServiceImpl{ - logger: logger, - }, } go clusterService.buildInformer() return clusterService