add: pagination integrated with frontend

Author: upsaurav12

backend/cmd/pagination.go

@@ -150,9 +150,15 @@ func GetContextKeyAndKContext(w http.ResponseWriter,
 	return ctx, span, contextKey, kContext, nil
 }
 
+func shouldSkipCache(path string) bool {
+	return strings.Contains(path, "version") ||
+		strings.Contains(path, "selfsubjectrulesreview") ||
+		strings.Contains(path, "selfsubjectaccessreviews")
+}
+
 // CacheMiddleWare is Middleware for Caching purpose. It involves generating key for a request,
 // authorizing user , store resource data in cache and returns data if key is present.
-func CacheMiddleWare(c *HeadlampConfig) mux.MiddlewareFunc {
+func CacheMiddleWare(c *HeadlampConfig) mux.MiddlewareFunc { //no-lint:gocognit
 	return func(next http.Handler) http.Handler {
 		if !c.CacheEnabled {
 			return next
@@ -163,6 +169,11 @@ func CacheMiddleWare(c *HeadlampConfig) mux.MiddlewareFunc {
 				return
 			}
 
+			if shouldSkipCache(r.URL.Path) {
+				next.ServeHTTP(w, r)
+				return
+			}
+
 			ctx, span, contextKey, kContext, err := GetContextKeyAndKContext(w, r, c)
 			if err != nil {
 				return
@@ -198,7 +209,7 @@ func CacheMiddleWare(c *HeadlampConfig) mux.MiddlewareFunc {
 			}
 
 			if served {
-				c.telemetryHandler.RecordEvent(span, "Served from cache")
+				c.telemetryHandler.RecordEvent(span, "served from cache")
 				return
 			}
 

backend/cmd/server.go

@@ -28,7 +28,11 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	"time"
 
 	"github.com/kubernetes-sigs/headlamp/backend/pkg/cache"
@@ -38,9 +42,20 @@ import (
 // CachedResponseData stores information such as StatusCode, Headers, and Body.
 // It helps cache responses efficiently and serve them from the cache.
 type CachedResponseData struct {
-	StatusCode int         `json:"statusCode"`
-	Headers    http.Header `json:"headers"`
-	Body       string      `json:"body"`
+	StatusCode int              `json:"statusCode"`
+	Headers    http.Header      `json:"headers"`
+	Body       ResourceResponse `json:"body"`
+}
+
+type Item struct {
+	Metadata metav1.ObjectMeta `json:"metadata"`
+}
+
+type ResourceResponse struct {
+	Kind     string          `json:"kind"`
+	Version  string          `json:"apiVersion"`
+	Metadata metav1.ListMeta `json:"metadata"`
+	Items    []Item          `json:"items"`
 }
 
 // GetResponseBody decompresses a gzip-encoded response body and returns it as a string.
@@ -172,6 +187,32 @@ func FilterHeaderForCache(responseHeaders http.Header, encoding string) http.Hea
 	return cacheHeader
 }
 
+func sliceTheResponse(page int, pageSize int, sizeOfResponse int) (int, int) {
+	start := (page - 1) * pageSize
+	if start >= sizeOfResponse {
+		return sizeOfResponse, sizeOfResponse // slice empty if page out of range
+	}
+	end := start + pageSize
+	if end > sizeOfResponse {
+		end = sizeOfResponse
+	}
+	return start, end
+}
+
+func ReturnResponseToClient(rcw *ResponseCapture, v any, w http.ResponseWriter) error {
+	var err error
+	if rcw.Header().Get("Content-Encoding") == "gzip" {
+		w.Header().Set("Content-Encoding", "gzip")
+		gz := gzip.NewWriter(w)
+		defer gz.Close()
+		err = json.NewEncoder(gz).Encode(v)
+	} else {
+		err = json.NewEncoder(w).Encode(v)
+	}
+
+	return err
+}
+
 // LoadFromCache checks if a cached resource exists and the user has permission to view it.
 // If found, it writes the cached data to the ResponseWriter and returns (true, nil).
 // If not found or on error, it returns (false, error).
@@ -186,14 +227,30 @@ func LoadFromCache(k8scache cache.Cache[string], isAllowed bool,
 		}
 
 		SetHeader(cachedData, w)
-		_, writeErr := w.Write([]byte(cachedData.Body))
+		w.Header().Set("Content-Type", "application/json")
+
+		// // 🔹 Always paginate, even for first request
+		pageNo := r.URL.Query().Get("p")
+		limit := 10
 
-		if writeErr != nil {
-			return false, writeErr
+		page, _ := strconv.Atoi(pageNo)
+
+		if page > 1 {
+			start, end := sliceTheResponse(page, limit, len(cachedData.Body.Items))
+			cachedData.Body.Items = cachedData.Body.Items[start:end]
+		}
+
+		bodyBytes, err := json.Marshal(cachedData.Body)
+		if err != nil {
+			return false, err
 		}
 
-		logger.Log(logger.LevelInfo, nil, nil, "serving from the cache with key "+key)
+		if _, err := w.Write(bodyBytes); err != nil {
+			// fmt.Println("since : ", time.Since(startNow))
+			return false, err
+		}
 
+		logger.Log(logger.LevelInfo, nil, nil, "serving from cache with key "+key)
 		return true, nil
 	}
 
@@ -219,11 +276,18 @@ func StoreK8sResponseInCache(k8scache cache.Cache[string],
 	}
 
 	headersToCache := FilterHeaderForCache(capturedHeaders, encoding)
+
+	var resourceBody ResourceResponse
+	if err := json.Unmarshal([]byte(dcmpBody), &resourceBody); err != nil {
+		logger.Log(logger.LevelError, nil, err, "failed to unmarshal resource response body")
+		return err
+	}
+
 	if !strings.Contains(url.Path, "selfsubjectrulesreviews") {
 		cachedData := CachedResponseData{
 			StatusCode: rcw.StatusCode,
 			Headers:    headersToCache,
-			Body:       dcmpBody,
+			Body:       resourceBody,
 		}
 
 		jsonBytes, err := json.Marshal(cachedData)
@@ -239,6 +303,5 @@ func StoreK8sResponseInCache(k8scache cache.Cache[string],
 			logger.Log(logger.LevelInfo, nil, nil, "k8s resource was stored with the key "+key)
 		}
 	}
-
 	return nil
 }

backend/pkg/k8cache/cacheStore.go

@@ -81,6 +81,8 @@ export function ResourceSearch({
   const [isFocused, setIsFocused] = useState(false);
   const theme = useTheme();
   const monaco = useMonaco();
+
+  // console.log("resource: ", resources , "selectedClusters: ", selectedClusters , "maxItemsPerResource: ", maxItemsPerResource , "refreshIntervalMs: ", refetchIntervalMs)
   const {
     items: allItems,
     errors,
@@ -104,6 +106,8 @@ export function ResourceSearch({
 
     searchWithQuery(allItems, deferredQuery, interruptRef).then(result => {
       if (interruptRef.current === false) {
+
+        console.log("results: ", result.results , "query : ", deferredQuery)
         setResults({ items: result.results, query: deferredQuery });
       }
     });