Prevent invalid timestamps in cpu usage metrics

Right after container is started, kubelet returns time == start_time for
that metric, which causes Monitoring API to return 400 error.

Author: x13n

kubelet-to-gcm/Makefile

@@ -15,7 +15,7 @@
 OUT_DIR = build
 PACKAGE = github.com/GoogleCloudPlatform/k8s-stackdriver/kubelet-to-gcm
 PREFIX = staging-k8s.gcr.io
-TAG = 1.3.5
+TAG = 1.3.6
 
 # Rules for building the real image for deployment to gcr.io
 

kubelet-to-gcm/monitor/kubelet/translate.go

@@ -435,6 +435,10 @@ func translateCPU(cpu *stats.CPUStats, tsFactory *timeSeriesFactory, startTime t
 	if cpu.UsageCoreNanoSeconds == nil {
 		return nil, fmt.Errorf("UsageCoreNanoSeconds missing from CPUStats %v", cpu)
 	}
+	// Only send cpu usage metric if start time is before current time. Right after container is started, kubelet can return start time == end time.
+	if !cpu.Time.Time.After(startTime) {
+		return nil, nil
+	}
 
 	// Total CPU utilization for all time. Convert from nanosec to sec.
 	cpuTotalPoint := tsFactory.newPoint(&v3.TypedValue{
@@ -519,7 +523,7 @@ func translateMemory(memory *stats.MemoryStats, tsFactory *timeSeriesFactory, st
 		return nil, fmt.Errorf("Memory information missing.")
 	}
 
-	// Only send page fault metric if start time is before current time. Right after container is started, kubelet can return start time == end time. This doesn't seem to happen with other metrics.
+	// Only send page fault metric if start time is before current time. Right after container is started, kubelet can return start time == end time.
 	if pageFaultsMD != nil && memory.Time.Time.After(startTime) {
 		if memory.MajorPageFaults == nil {
 			return nil, fmt.Errorf("MajorPageFaults missing in MemoryStats %v", memory)

kubelet-to-gcm/monitor/kubelet/translate_test.go

@@ -221,8 +221,9 @@ func TestTranslateContainers(t *testing.T) {
 	noLogStatsContainer.Logs = nil
 	noRootfsStatsContainer := *getContainerStats(false)
 	noRootfsStatsContainer.Rootfs = nil
-	badTimestampOnPageFaultContrainer := *getContainerStats(false)
-	badTimestampOnPageFaultContrainer.Memory.Time = badTimestampOnPageFaultContrainer.StartTime
+	badTimestampOnCumulativeMetricsContrainer := *getContainerStats(false)
+	badTimestampOnCumulativeMetricsContrainer.Memory.Time = badTimestampOnCumulativeMetricsContrainer.StartTime
+	badTimestampOnCumulativeMetricsContrainer.CPU.Time = badTimestampOnCumulativeMetricsContrainer.StartTime
 	legacyTsPerContainer := 11
 	tsPerContainer := 8
 	testCases := []struct {
@@ -310,12 +311,12 @@ func TestTranslateContainers(t *testing.T) {
 			},
 		},
 		{
-			name:                  "bad timestamp for page_fault_count",
-			ExpectedLegacyTSCount: legacyTsPerContainer - 2,
-			ExpectedTSCount:       tsPerContainer - 2,
+			name:                  "bad timestamp for cumulative metrics",
+			ExpectedLegacyTSCount: legacyTsPerContainer - 3,
+			ExpectedTSCount:       tsPerContainer - 3,
 			pods: []stats.PodStats{
 				getPodStats(
-					badTimestampOnPageFaultContrainer,
+					badTimestampOnCumulativeMetricsContrainer,
 				),
 			},
 		},
@@ -364,6 +365,9 @@ func getContainerStats(skipUsageNanoCores bool) *stats.ContainerStats {
 	if v.Memory.Time.Time.Before(v.StartTime.Time) {
 		v.Memory.Time, v.StartTime = v.StartTime, v.Memory.Time
 	}
+	if v.CPU.Time.Time.Before(v.StartTime.Time) {
+		v.CPU.Time, v.StartTime = v.StartTime, v.CPU.Time
+	}
 	return v
 }