Skip to content

Commit f4cdabf

Browse files
leonzzleonzz
authored
skip certain memory metrics when not available (#880)
* skip certain memory metrics when not available Change-Id: I8b4310fc9e77a32b6c255e5de54790e0070fa7a4 * add test case for incomplete container memory stats Change-Id: I5e19f5ba07ba297188cb72b655ee5d632657c4cf
1 parent 7b268d4 commit f4cdabf

File tree

2 files changed

+125
-29
lines changed

2 files changed

+125
-29
lines changed

‎kubelet-to-gcm/monitor/kubelet/translate.go‎

Lines changed: 25 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -525,34 +525,28 @@ func translateMemory(memory *stats.MemoryStats, tsFactory *timeSeriesFactory, st
525525

526526
// Only send page fault metric if start time is before current time. Right after container is started, kubelet can return start time == end time.
527527
if pageFaultsMD != nil && memory.Time.Time.After(startTime) {
528-
if memory.MajorPageFaults == nil {
529-
return nil, fmt.Errorf("MajorPageFaults missing in MemoryStats %v", memory)
528+
if memory.MajorPageFaults != nil {
529+
// Major page faults.
530+
majorPFPoint := tsFactory.newPoint(&v3.TypedValue{
531+
Int64Value: monitor.Int64Ptr(int64(*memory.MajorPageFaults)),
532+
ForceSendFields: []string{"Int64Value"},
533+
}, startTime, memory.Time.Time, pageFaultsMD.MetricKind)
534+
timeSeries = append(timeSeries, tsFactory.newTimeSeries(majorPageFaultLabels, pageFaultsMD, majorPFPoint))
530535
}
531-
if memory.PageFaults == nil {
532-
return nil, fmt.Errorf("PageFaults missing in MemoryStats %v", memory)
536+
if memory.PageFaults != nil {
537+
// Minor page faults.
538+
minorPFPoint := tsFactory.newPoint(&v3.TypedValue{
539+
Int64Value: monitor.Int64Ptr(int64(*memory.PageFaults - *memory.MajorPageFaults)),
540+
ForceSendFields: []string{"Int64Value"},
541+
}, startTime, memory.Time.Time, pageFaultsMD.MetricKind)
542+
timeSeries = append(timeSeries, tsFactory.newTimeSeries(minorPageFaultLabels, pageFaultsMD, minorPFPoint))
533543
}
534-
// Major page faults.
535-
majorPFPoint := tsFactory.newPoint(&v3.TypedValue{
536-
Int64Value: monitor.Int64Ptr(int64(*memory.MajorPageFaults)),
537-
ForceSendFields: []string{"Int64Value"},
538-
}, startTime, memory.Time.Time, pageFaultsMD.MetricKind)
539-
timeSeries = append(timeSeries, tsFactory.newTimeSeries(majorPageFaultLabels, pageFaultsMD, majorPFPoint))
540-
// Minor page faults.
541-
minorPFPoint := tsFactory.newPoint(&v3.TypedValue{
542-
Int64Value: monitor.Int64Ptr(int64(*memory.PageFaults - *memory.MajorPageFaults)),
543-
ForceSendFields: []string{"Int64Value"},
544-
}, startTime, memory.Time.Time, pageFaultsMD.MetricKind)
545-
timeSeries = append(timeSeries, tsFactory.newTimeSeries(minorPageFaultLabels, pageFaultsMD, minorPFPoint))
546544
}
547545

548546
if memUsedMD != nil {
549547
if memory.WorkingSetBytes == nil {
550548
return nil, fmt.Errorf("WorkingSetBytes information missing in MemoryStats %v", memory)
551549
}
552-
if memory.UsageBytes == nil {
553-
return nil, fmt.Errorf("UsageBytes information missing in MemoryStats %v", memory)
554-
}
555-
556550
// Non-evictable memory.
557551
nonEvictMemPoint := tsFactory.newPoint(&v3.TypedValue{
558552
Int64Value: monitor.Int64Ptr(int64(*memory.WorkingSetBytes)),
@@ -564,16 +558,18 @@ func translateMemory(memory *stats.MemoryStats, tsFactory *timeSeriesFactory, st
564558
}
565559
timeSeries = append(timeSeries, tsFactory.newTimeSeries(labels, memUsedMD, nonEvictMemPoint))
566560

567-
// Evictable memory.
568-
evictMemPoint := tsFactory.newPoint(&v3.TypedValue{
569-
Int64Value: monitor.Int64Ptr(int64(*memory.UsageBytes - *memory.WorkingSetBytes)),
570-
ForceSendFields: []string{"Int64Value"},
571-
}, startTime, memory.Time.Time, memUsedMD.MetricKind)
572-
labels = map[string]string{"memory_type": "evictable"}
573-
if component != "" {
574-
labels["component"] = component
561+
if memory.UsageBytes != nil {
562+
// Evictable memory.
563+
evictMemPoint := tsFactory.newPoint(&v3.TypedValue{
564+
Int64Value: monitor.Int64Ptr(int64(*memory.UsageBytes - *memory.WorkingSetBytes)),
565+
ForceSendFields: []string{"Int64Value"},
566+
}, startTime, memory.Time.Time, memUsedMD.MetricKind)
567+
labels = map[string]string{"memory_type": "evictable"}
568+
if component != "" {
569+
labels["component"] = component
570+
}
571+
timeSeries = append(timeSeries, tsFactory.newTimeSeries(labels, memUsedMD, evictMemPoint))
575572
}
576-
timeSeries = append(timeSeries, tsFactory.newTimeSeries(labels, memUsedMD, evictMemPoint))
577573
}
578574

579575
if memTotalMD != nil {

‎kubelet-to-gcm/monitor/kubelet/translate_test.go‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,96 @@ const (
154154
}
155155
]
156156
}`
157+
158+
incompleteContainerMemStatJSON = `{
159+
"podRef": {
160+
"name": "auditproxy-gke-12345678-1248-332a-vm",
161+
"namespace": "kube-system",
162+
"uid": "43780f1ce6e2171bc9e70cae3118ab6b"
163+
},
164+
"startTime": "2025-04-06T11:28:17Z",
165+
"containers": [
166+
{
167+
"name": "auditproxy",
168+
"startTime": "2025-04-23T15:17:45Z",
169+
"cpu": {
170+
"time": "2025-04-29T15:24:11Z",
171+
"usageNanoCores": 12352231,
172+
"usageCoreNanoSeconds": 4106274919000
173+
},
174+
"memory": {
175+
"time": "2025-04-29T15:24:11Z",
176+
"workingSetBytes": 76451840
177+
},
178+
"rootfs": {
179+
"time": "2025-04-29T15:24:04Z",
180+
"availableBytes": 4546961408,
181+
"capacityBytes": 16656896000,
182+
"usedBytes": 57344,
183+
"inodesFree": 963008,
184+
"inodes": 1036320,
185+
"inodesUsed": 18
186+
},
187+
"logs": {
188+
"time": "2025-04-29T15:24:11Z",
189+
"availableBytes": 4546961408,
190+
"capacityBytes": 16656896000,
191+
"usedBytes": 44961792,
192+
"inodesFree": 963008,
193+
"inodes": 1036320,
194+
"inodesUsed": 9
195+
},
196+
"swap": {
197+
"time": "2025-04-29T15:24:11Z",
198+
"swapAvailableBytes": 0,
199+
"swapUsageBytes": 0
200+
}
201+
}
202+
],
203+
"cpu": {
204+
"time": "2025-04-29T15:23:59Z",
205+
"usageNanoCores": 7444062,
206+
"usageCoreNanoSeconds": 15900193844000
207+
},
208+
"memory": {
209+
"time": "2025-04-29T15:23:59Z",
210+
"usageBytes": 91631616,
211+
"workingSetBytes": 88707072,
212+
"rssBytes": 70098944,
213+
"pageFaults": 63466572,
214+
"majorPageFaults": 2461
215+
},
216+
"network": {
217+
"time": "2025-04-29T15:24:07Z",
218+
"name": "eth0",
219+
"rxBytes": 208635268392,
220+
"rxErrors": 0,
221+
"txBytes": 939343805472,
222+
"txErrors": 0,
223+
"interfaces": [
224+
{
225+
"name": "eth0",
226+
"rxBytes": 208635268392,
227+
"rxErrors": 0,
228+
"txBytes": 939343805472,
229+
"txErrors": 0
230+
}
231+
]
232+
},
233+
"ephemeral-storage": {
234+
"time": "2025-04-29T15:24:11Z",
235+
"availableBytes": 4546961408,
236+
"capacityBytes": 16656896000,
237+
"usedBytes": 45023232,
238+
"inodesFree": 963008,
239+
"inodes": 1036320,
240+
"inodesUsed": 28
241+
},
242+
"swap": {
243+
"time": "2025-04-29T15:23:59Z",
244+
"swapUsageBytes": 0
245+
}
246+
}`
157247
)
158248

159249
// TestTranslator
@@ -226,6 +316,10 @@ func TestTranslateContainers(t *testing.T) {
226316
badTimestampOnCumulativeMetricsContrainer.CPU.Time = badTimestampOnCumulativeMetricsContrainer.StartTime
227317
legacyTsPerContainer := 11
228318
tsPerContainer := 8
319+
inCompleteContainerMemPodStat := &stats.PodStats{}
320+
if err := json.Unmarshal([]byte(incompleteContainerMemStatJSON), inCompleteContainerMemPodStat); err != nil {
321+
t.Errorf("Failed to unmarshal incompleteContainerMemStatJSON, err: %v", err)
322+
}
229323
testCases := []struct {
230324
name string
231325
ExpectedLegacyTSCount int
@@ -320,6 +414,12 @@ func TestTranslateContainers(t *testing.T) {
320414
),
321415
},
322416
},
417+
{
418+
name: "inCompleteContainerMemPodStat missing memory usageBytes pageFaults, and majorPageFaults",
419+
ExpectedLegacyTSCount: 7,
420+
ExpectedTSCount: 4,
421+
pods: []stats.PodStats{*inCompleteContainerMemPodStat},
422+
},
323423
}
324424

325425
for _, tc := range testCases {

0 commit comments

Comments
 (0)