Skip to content

Commit 58f27e7

Browse files
ffforestffforest
authored
fix: issues with resource, run logs, dev-portal-enabled and enable better panic handling (#1387)
1 parent 2b68e63 commit 58f27e7

File tree

7 files changed

+79
-60
lines changed

7 files changed

+79
-60
lines changed

‎pkg/domain/constant/run.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ const (
1818
RunStatusSucceeded RunStatus = "Succeeded"
1919
RunStatusCancelled RunStatus = "Cancelled"
2020
RunStatusQueued RunStatus = "Queued"
21+
RunResultFailed string = "{\"result\":\"Operation Failed\"}"
22+
RunResultCancelled string = "{\"result\":\"Operation Cancelled\"}"
2123
)
2224

2325
// ParseRunType parses a string into a RunType.

‎pkg/server/handler/stack/execute_async.go

Lines changed: 38 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -77,47 +77,50 @@ func (h *Handler) PreviewStackAsync() http.HandlerFunc {
7777
logger.Info("Async preview in progress")
7878
var previewChanges any
7979
newCtx, cancel := CopyToNewContextWithTimeout(ctx, constant.RunTimeOut)
80-
defer cancel() // make sure the context is canceled to free resources
81-
defer handleCrash(newCtx) // recover from possible panic
80+
defer cancel() // make sure the context is canceled to free resources
81+
defer handleCrash(newCtx, h.setRunToFailed, runEntity.ID) // recover from possible panic
8282

8383
// update status of the run when exiting the async run
8484
defer func() {
8585
select {
8686
case <-newCtx.Done():
87-
logger.Info("preview execution timed out", "stackID", params.StackID, "time", time.Now(), "timeout", newCtx.Err())
87+
logutil.LogToAll(logger, runLogger, "info", "preview execution timed out", "stackID", params.StackID, "time", time.Now(), "timeout", newCtx.Err())
8888
h.setRunToCancelled(newCtx, runEntity.ID)
8989
default:
9090
if err != nil {
91-
logger.Info("preview failed for stack", "stackID", params.StackID, "time", time.Now())
91+
logutil.LogToAll(logger, runLogger, "error", "preview failed for stack", "stackID", params.StackID, "time", time.Now())
9292
h.setRunToFailed(newCtx, runEntity.ID)
9393
} else {
94-
logger.Info("preview completed for stack", "stackID", params.StackID, "time", time.Now())
94+
logutil.LogToAll(logger, runLogger, "info", "preview completed for stack", "stackID", params.StackID, "time", time.Now())
9595
if pc, ok := previewChanges.(*models.Changes); ok {
9696
h.setRunToSuccess(newCtx, runEntity.ID, pc)
9797
} else {
98-
logger.Error("Error casting preview changes to models.Changes", "error", "casting error")
98+
logutil.LogToAll(logger, runLogger, "error", "Error casting preview changes to models.Changes", "error", "casting error")
9999
h.setRunToFailed(newCtx, runEntity.ID)
100100
}
101101
}
102102
}
103103
}()
104104

105+
defer handleCrash(newCtx, h.setRunToFailed, runEntity.ID) // recover from possible panic
106+
105107
// Call preview stack
106-
changes, err := h.stackManager.PreviewStack(newCtx, params, requestPayload.ImportedResources)
108+
var changes *models.Changes
109+
changes, err = h.stackManager.PreviewStack(newCtx, params, requestPayload.ImportedResources)
107110
if err != nil {
108-
logger.Error("Error previewing stack", "error", err)
111+
logutil.LogToAll(logger, runLogger, "error", "Error previewing stack", "error", err)
109112
return
110113
}
111114

112115
previewChanges, err = stackmanager.ProcessChanges(newCtx, w, changes, params.Format, params.ExecuteParams.Detail)
113116
if err != nil {
114-
logger.Error("Error processing preview changes", "error", err)
117+
logutil.LogToAll(logger, runLogger, "error", "Error processing preview changes", "error", err)
115118
return
116119
}
117120
})
118121
defer func() {
119122
if inBufferZone {
120-
logger.Info("The task is in the buffer zone, waiting for an available worker")
123+
logutil.LogToAll(logger, runLogger, "info", "The task is in the buffer zone, waiting for an available worker")
121124
h.setRunToQueued(ctx, runEntity.ID)
122125
}
123126
}()
@@ -182,42 +185,44 @@ func (h *Handler) ApplyStackAsync() http.HandlerFunc {
182185
// defer safe.HandleCrash(aciLoggingRecoverHandler(h.aciClient, &req, log))
183186
logger.Info("Async apply in progress")
184187
newCtx, cancel := CopyToNewContextWithTimeout(ctx, constant.RunTimeOut)
185-
defer cancel() // make sure the context is canceled to free resources
186-
defer handleCrash(newCtx) // recover from possible panic
188+
defer cancel() // make sure the context is canceled to free resources
189+
defer handleCrash(newCtx, h.setRunToFailed, runEntity.ID) // recover from possible panic
187190

188191
// update status of the run when exiting the async run
189192
defer func() {
190193
select {
191194
case <-newCtx.Done():
192-
logger.Info("apply execution timed out", "stackID", params.StackID, "time", time.Now(), "timeout", newCtx.Err())
195+
logutil.LogToAll(logger, runLogger, "info", "apply execution timed out", "stackID", params.StackID, "time", time.Now(), "timeout", newCtx.Err())
193196
h.setRunToCancelled(newCtx, runEntity.ID)
194197
default:
195198
if err != nil {
196-
logger.Info("apply failed for stack", "stackID", params.StackID, "time", time.Now())
199+
logutil.LogToAll(logger, runLogger, "error", "apply failed for stack", "stackID", params.StackID, "time", time.Now())
197200
h.setRunToFailed(newCtx, runEntity.ID)
198201
} else {
199-
logger.Info("apply completed for stack", "stackID", params.StackID, "time", time.Now())
202+
logutil.LogToAll(logger, runLogger, "info", "apply completed for stack", "stackID", params.StackID, "time", time.Now())
200203
h.setRunToSuccess(newCtx, runEntity.ID, "apply completed")
201204
}
202205
}
203206
}()
204207

208+
defer handleCrash(newCtx, h.setRunToFailed, runEntity.ID) // recover from possible panic
209+
205210
// call apply stack
206211
err = h.stackManager.ApplyStack(newCtx, params, requestPayload.ImportedResources)
207212
if err != nil {
208213
if err == stackmanager.ErrDryrunDestroy {
209214
render.Render(w, r, handler.SuccessResponse(ctx, "Dry-run mode enabled, the above resources will be applied if dryrun is set to false"))
210215
return
211216
} else {
212-
logger.Error("Error applying stack", "error", err)
217+
logutil.LogToAll(logger, runLogger, "error", "Error applying stack", "error", err)
213218
return
214219
}
215220
}
216221
})
217222

218223
defer func() {
219224
if inBufferZone {
220-
logger.Info("The task is in the buffer zone, waiting for an available worker")
225+
logutil.LogToAll(logger, runLogger, "info", "The task is in the buffer zone, waiting for an available worker")
221226
h.setRunToQueued(ctx, runEntity.ID)
222227
}
223228
}()
@@ -279,26 +284,26 @@ func (h *Handler) GenerateStackAsync() http.HandlerFunc {
279284
// defer safe.HandleCrash(aciLoggingRecoverHandler(h.aciClient, &req, log))
280285
logger.Info("Async generate in progress")
281286
newCtx, cancel := CopyToNewContextWithTimeout(ctx, constant.RunTimeOut)
282-
defer cancel() // make sure the context is canceled to free resources
283-
defer handleCrash(newCtx) // recover from possible panic
287+
defer cancel() // make sure the context is canceled to free resources
288+
defer handleCrash(newCtx, h.setRunToFailed, runEntity.ID) // recover from possible panic
284289

285290
var sp *apiv1.Spec
286291
// update status of the run when exiting the async run
287292
defer func() {
288293
select {
289294
case <-newCtx.Done():
290-
logger.Info("generate execution timed out", "stackID", params.StackID, "time", time.Now(), "timeout", newCtx.Err())
295+
logutil.LogToAll(logger, runLogger, "info", "generate execution timed out", "stackID", params.StackID, "time", time.Now(), "timeout", newCtx.Err())
291296
h.setRunToCancelled(newCtx, runEntity.ID)
292297
default:
293298
if err != nil {
294-
logger.Info("generate failed for stack", "stackID", params.StackID, "time", time.Now())
299+
logutil.LogToAll(logger, runLogger, "error", "generate failed for stack", "stackID", params.StackID, "time", time.Now())
295300
h.setRunToFailed(newCtx, runEntity.ID)
296301
} else {
297-
logger.Info("generate completed for stack", "stackID", params.StackID, "time", time.Now())
302+
logutil.LogToAll(logger, runLogger, "info", "generate completed for stack", "stackID", params.StackID, "time", time.Now())
298303
if yaml, err := yamlv2.Marshal(sp); err == nil {
299304
h.setRunToSuccess(newCtx, runEntity.ID, string(yaml))
300305
} else {
301-
logger.Error("Error marshalling generated spec", "error", err)
306+
logutil.LogToAll(logger, runLogger, "error", "Error marshalling generated spec", "error", err)
302307
h.setRunToFailed(newCtx, runEntity.ID)
303308
}
304309
}
@@ -308,14 +313,14 @@ func (h *Handler) GenerateStackAsync() http.HandlerFunc {
308313
// Call generate stack
309314
_, sp, err = h.stackManager.GenerateSpec(newCtx, params)
310315
if err != nil {
311-
logger.Error("Error generating stack", "error", err)
316+
logutil.LogToAll(logger, runLogger, "error", "Error generating stack", "error", err)
312317
return
313318
}
314319
})
315320

316321
defer func() {
317322
if inBufferZone {
318-
logger.Info("The task is in the buffer zone, waiting for an available worker")
323+
logutil.LogToAll(logger, runLogger, "info", "The task is in the buffer zone, waiting for an available worker")
319324
h.setRunToQueued(ctx, runEntity.ID)
320325
}
321326
}()
@@ -376,21 +381,21 @@ func (h *Handler) DestroyStackAsync() http.HandlerFunc {
376381
inBufferZone := h.workerPool.Do(func() {
377382
logger.Info("Async destroy in progress")
378383
newCtx, cancel := CopyToNewContextWithTimeout(ctx, constant.RunTimeOut)
379-
defer cancel() // make sure the context is canceled to free resources
380-
defer handleCrash(newCtx) // recover from possible panic
384+
defer cancel() // make sure the context is canceled to free resources
385+
defer handleCrash(newCtx, h.setRunToFailed, runEntity.ID) // recover from possible panic
381386

382387
// update status of the run when exiting the async run
383388
defer func() {
384389
select {
385390
case <-newCtx.Done():
386-
logger.Info("destroy execution timed out", "stackID", params.StackID, "time", time.Now(), "timeout", newCtx.Err())
391+
logutil.LogToAll(logger, runLogger, "info", "destroy execution timed out", "stackID", params.StackID, "time", time.Now(), "timeout", newCtx.Err())
387392
h.setRunToCancelled(newCtx, runEntity.ID)
388393
default:
389394
if err != nil {
390-
logger.Info("destroy failed for stack", "stackID", params.StackID, "time", time.Now())
395+
logutil.LogToAll(logger, runLogger, "error", "destroy failed for stack", "stackID", params.StackID, "time", time.Now())
391396
h.setRunToFailed(newCtx, runEntity.ID)
392397
} else {
393-
logger.Info("destroy completed for stack", "stackID", params.StackID, "time", time.Now())
398+
logutil.LogToAll(logger, runLogger, "info", "destroy completed for stack", "stackID", params.StackID, "time", time.Now())
394399
h.setRunToSuccess(newCtx, runEntity.ID, "destroy completed")
395400
}
396401
}
@@ -400,18 +405,18 @@ func (h *Handler) DestroyStackAsync() http.HandlerFunc {
400405
err = h.stackManager.DestroyStack(newCtx, params, w)
401406
if err != nil {
402407
if err == stackmanager.ErrDryrunDestroy {
403-
logger.Info("Dry-run mode enabled, the above resources will be destroyed if dryrun is set to false")
408+
logutil.LogToAll(logger, runLogger, "info", "Dry-run mode enabled, the above resources will be destroyed if dryrun is set to false")
404409
return
405410
} else {
406-
logger.Error("Error destroying stack", "error", err)
411+
logutil.LogToAll(logger, runLogger, "error", "Error destroying stack", "error", err)
407412
return
408413
}
409414
}
410415
})
411416

412417
defer func() {
413418
if inBufferZone {
414-
logger.Info("The task is in the buffer zone, waiting for an available worker")
419+
logutil.LogToAll(logger, runLogger, "info", "The task is in the buffer zone, waiting for an available worker")
415420
h.setRunToQueued(ctx, runEntity.ID)
416421
}
417422
}()

‎pkg/server/handler/stack/utils.go

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"net/http"
88
"runtime"
99
"strconv"
10+
"strings"
1011
"time"
1112

1213
"github.com/go-chi/chi/v5"
@@ -28,7 +29,6 @@ func (h *Handler) setRunToSuccess(ctx context.Context, runID uint, result any) {
2829
logger.Error("Error marshalling preview changes", "error", err)
2930
return
3031
}
31-
logger.Info("Result", "result", string(resultBytes))
3232
// Update the Run object in database to include the preview result
3333
updateRunResultPayload := request.UpdateRunResultRequest{
3434
Result: string(resultBytes),
@@ -45,10 +45,11 @@ func (h *Handler) setRunToSuccess(ctx context.Context, runID uint, result any) {
4545
func (h *Handler) setRunToFailed(ctx context.Context, runID uint) {
4646
logger := logutil.GetLogger(ctx)
4747
runLogs := logutil.GetRunLoggerBuffer(ctx)
48+
logsNew := strings.ReplaceAll(runLogs.String(), "\n", "\n\n")
4849
updateRunResultPayload := request.UpdateRunResultRequest{
49-
Result: "",
50+
Result: constant.RunResultFailed,
5051
Status: string(constant.RunStatusFailed),
51-
Logs: runLogs.String(),
52+
Logs: logsNew,
5253
}
5354
_, err := h.stackManager.UpdateRunResultAndStatusByID(ctx, runID, updateRunResultPayload)
5455
if err != nil {
@@ -60,7 +61,7 @@ func (h *Handler) setRunToCancelled(ctx context.Context, runID uint) {
6061
logger := logutil.GetLogger(ctx)
6162
runLogs := logutil.GetRunLoggerBuffer(ctx)
6263
updateRunResultPayload := request.UpdateRunResultRequest{
63-
Result: "",
64+
Result: constant.RunResultCancelled,
6465
Status: string(constant.RunStatusCancelled),
6566
Logs: runLogs.String(),
6667
}
@@ -188,7 +189,9 @@ func logStackTrace(runLogger *httplog.Logger) {
188189
runLogger.Error(string(buf[:stackSize]))
189190
}
190191

191-
func handleCrash(ctx context.Context) {
192+
type SetRunToFailedFunc func(context.Context, uint)
193+
194+
func handleCrash(ctx context.Context, statusHandlingFunc SetRunToFailedFunc, runID uint) {
192195
if r := recover(); r != nil {
193196
logger := logutil.GetLogger(ctx)
194197
runLogger := logutil.GetRunLogger(ctx)
@@ -197,6 +200,7 @@ func handleCrash(ctx context.Context) {
197200
runLogger.Error("Panic recovered", "error", r)
198201
logStackTrace(runLogger)
199202
}
203+
statusHandlingFunc(ctx, runID)
200204
}
201205

202206
func updateRunRequestPayload(requestPayload *request.CreateRunRequest, params *stackmanager.StackRequestParams, runType constant.RunType) {

‎pkg/server/manager/stack/resource.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ func (m *StackManager) ReconcileResources(ctx context.Context, stackID uint, rel
7272
var resourceToBeDeleted []*entity.Resource
7373

7474
for _, resource := range currentResources.Resources {
75-
if !isInRelease(release, resource.KusionResourceID) {
75+
if !isInRelease(release, resource.KusionResourceID, resource.Stack) {
7676
resource.LastAppliedTimestamp = release.ModifiedTime
7777
resource.Status = constant.StatusResourceDestroyed
7878
resourceToBeDeleted = append(resourceToBeDeleted, resource)

‎pkg/server/manager/stack/types.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ var (
2929
ErrInvalidWatchTimeout = errors.New("watchTimeout should be a number")
3030
ErrWorkspaceEmpty = errors.New("workspace should not be empty in query")
3131
ErrRunRequestBodyEmpty = errors.New("run request body should not be empty")
32+
ErrRunCrashed = errors.New("run crashed")
3233
)
3334

3435
type StackManager struct {

‎pkg/server/manager/stack/util.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -500,7 +500,12 @@ func getReleasePath(namespace, source, projectPath, workspace string) string {
500500
return fmt.Sprintf("%s/%s/%s/%s", namespace, source, projectPath, workspace)
501501
}
502502

503-
func isInRelease(release *v1.Release, id string) bool {
503+
func isInRelease(release *v1.Release, id string, resourceStack *entity.Stack) bool {
504+
if resourceStack != nil {
505+
if release.Stack != resourceStack.Name || release.Project != resourceStack.Project.Name {
506+
return false
507+
}
508+
}
504509
for _, resource := range release.State.Resources {
505510
if resource.ID == id {
506511
return true

‎pkg/server/route/route.go

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -98,29 +98,31 @@ func NewCoreRoute(config *server.Config) (*chi.Mux, error) {
9898
})
9999
})
100100

101-
// Serve static files from embedded filesystem
102-
buildFS, _ := fs.Sub(ui.Embedded, "build")
103-
fileServer := http.FileServer(http.FS(buildFS))
101+
if config.DevPortalEnabled {
102+
// Serve static files from embedded filesystem
103+
buildFS, _ := fs.Sub(ui.Embedded, "build")
104+
fileServer := http.FileServer(http.FS(buildFS))
104105

105-
// Handle static file requests under /public path
106-
router.Get("/public/*", func(w http.ResponseWriter, r *http.Request) {
107-
r.URL.Path = strings.TrimPrefix(r.URL.Path, "/public")
108-
fileServer.ServeHTTP(w, r)
109-
})
106+
// Handle static file requests under /public path
107+
router.Get("/public/*", func(w http.ResponseWriter, r *http.Request) {
108+
r.URL.Path = strings.TrimPrefix(r.URL.Path, "/public")
109+
fileServer.ServeHTTP(w, r)
110+
})
110111

111-
// Handle all other paths by serving index.html for SPA routing
112-
router.Get("/*", func(w http.ResponseWriter, r *http.Request) {
113-
indexFile, err := buildFS.Open("index.html")
114-
if err != nil {
115-
logger.Warn("Failed to open index.html from embedded filesystem", "error", err.Error())
116-
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
117-
return
118-
}
119-
defer indexFile.Close()
112+
// Handle all other paths by serving index.html for SPA routing
113+
router.Get("/*", func(w http.ResponseWriter, r *http.Request) {
114+
indexFile, err := buildFS.Open("index.html")
115+
if err != nil {
116+
logger.Warn("Failed to open index.html from embedded filesystem", "error", err.Error())
117+
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
118+
return
119+
}
120+
defer indexFile.Close()
120121

121-
w.Header().Set("Content-Type", "text/html; charset=utf-8")
122-
io.Copy(w, indexFile)
123-
})
122+
w.Header().Set("Content-Type", "text/html; charset=utf-8")
123+
io.Copy(w, indexFile)
124+
})
125+
}
124126

125127
logger.Info(fmt.Sprintf("Listening on :%d", config.Port))
126128
http.ListenAndServe(fmt.Sprintf(":%d", config.Port), router)

0 commit comments

Comments
 (0)