Skip to content

Commit 240fb23

Browse files
sutaakarMarianMacik
sutaakar
authored and
MarianMacik
committed
[KOGITO-950] Store test logs into local filesystem (apache#165)
Signed-off-by: Karel Suta <ksuta@redhat.com>
1 parent a4980ec commit 240fb23

File tree

3 files changed

+212
-5
lines changed

3 files changed

+212
-5
lines changed

‎bddframework/smoke/framework/kubernetes.go

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ func InitKubeClient() error {
3939
mux.Lock()
4040
defer mux.Unlock()
4141
if kubeClient == nil {
42-
newClient, err := client.NewClientBuilder().WithDiscoveryClient().WithBuildClient().Build()
42+
newClient, err := client.NewClientBuilder().WithDiscoveryClient().WithBuildClient().WithKubernetesExtensionClient().Build()
4343
if err != nil {
4444
return fmt.Errorf("Error initializing kube client: %v", err)
4545
}
@@ -81,7 +81,7 @@ func IsNamespace(namespace string) (bool, error) {
8181
// WaitForPods waits for pods with specific label to be available and running
8282
func WaitForPods(namespace, labelName, labelValue string, numberOfPods, timeoutInMin int) error {
8383
return WaitFor(namespace, fmt.Sprintf("Pods with label name '%s' and value '%s' available and running", labelName, labelValue), time.Duration(timeoutInMin)*time.Minute, func() (bool, error) {
84-
pods, err := GetPods(namespace, map[string]string{labelName: labelValue})
84+
pods, err := GetPodsWithLabels(namespace, map[string]string{labelName: labelValue})
8585
if err != nil || (len(pods.Items) != numberOfPods) {
8686
return false, err
8787
}
@@ -90,8 +90,17 @@ func WaitForPods(namespace, labelName, labelValue string, numberOfPods, timeoutI
9090
})
9191
}
9292

93-
// GetPods retrieves pods based on label name and value
94-
func GetPods(namespace string, labels map[string]string) (*corev1.PodList, error) {
93+
// GetPods retrieves all pods in namespace
94+
func GetPods(namespace string) (*corev1.PodList, error) {
95+
pods := &corev1.PodList{}
96+
if err := kubernetes.ResourceC(kubeClient).ListWithNamespace(namespace, pods); err != nil {
97+
return nil, err
98+
}
99+
return pods, nil
100+
}
101+
102+
// GetPodsWithLabels retrieves pods based on label name and value
103+
func GetPodsWithLabels(namespace string, labels map[string]string) (*corev1.PodList, error) {
95104
pods := &corev1.PodList{}
96105
if err := kubernetes.ResourceC(kubeClient).ListWithNamespaceAndLabel(namespace, pods, labels); err != nil {
97106
return nil, err
@@ -102,13 +111,18 @@ func GetPods(namespace string, labels map[string]string) (*corev1.PodList, error
102111
// CheckPodsAreRunning returns true if all pods are running
103112
func CheckPodsAreRunning(pods *corev1.PodList) bool {
104113
for _, pod := range pods.Items {
105-
if pod.Status.Phase != corev1.PodRunning {
114+
if !IsPodRunning(&pod) {
106115
return false
107116
}
108117
}
109118
return true
110119
}
111120

121+
// IsPodRunning returns true if pod is running
122+
func IsPodRunning(pod *corev1.Pod) bool {
123+
return pod.Status.Phase == corev1.PodRunning
124+
}
125+
112126
// WaitForStatefulSetRunning waits for a stateful set to be running, with a specific number of pod
113127
func WaitForStatefulSetRunning(namespace, ssName string, podNb int, timeoutInMin int) error {
114128
return WaitFor(namespace, fmt.Sprintf("StatefulSet %s running", ssName), time.Duration(timeoutInMin)*time.Minute, func() (bool, error) {

‎bddframework/smoke/framework/logging.go

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
// Copyright 2020 Red Hat, Inc. and/or its affiliates
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package framework
16+
17+
import (
18+
"errors"
19+
"fmt"
20+
"os"
21+
"time"
22+
23+
"github.com/kiegroup/kogito-cloud-operator/pkg/client/kubernetes"
24+
25+
"io/ioutil"
26+
)
27+
28+
const (
29+
logFolder = "logs"
30+
logSuffix = ".log"
31+
)
32+
33+
var (
34+
monitoredNamespaces = make(map[string]monitoredNamespace)
35+
)
36+
37+
// StartPodLogCollector monitors a namespace and stores logs of all pods running in the namespace
38+
func StartPodLogCollector(namespace string) error {
39+
if isNamespaceMonitored(namespace) {
40+
return errors.New("namespace is already monitored")
41+
}
42+
43+
if err := createLogFolder(namespace); err != nil {
44+
return fmt.Errorf("Error while creating log folder: %v", err)
45+
}
46+
47+
monitoredNamespace := monitoredNamespace{
48+
pods: make(map[string]monitoredPod),
49+
stopMonitoring: make(chan bool),
50+
}
51+
monitoredNamespaces[namespace] = monitoredNamespace
52+
53+
scanningPeriod := time.NewTicker(5 * time.Second)
54+
defer scanningPeriod.Stop()
55+
for {
56+
select {
57+
case <-monitoredNamespace.stopMonitoring:
58+
return nil
59+
case <-scanningPeriod.C:
60+
if pods, err := GetPods(namespace); err != nil {
61+
GetLogger(namespace).Errorf("Error while getting pods in namespace '%s': %v", namespace, err)
62+
} else {
63+
for _, pod := range pods.Items {
64+
if !isPodMonitored(namespace, pod.Name) && IsPodRunning(&pod) {
65+
initMonitoredPod(namespace, pod.Name)
66+
for _, container := range pod.Spec.Containers {
67+
initMonitoredContainer(namespace, pod.Name, container.Name)
68+
go storeContainerLogWithFollow(namespace, pod.Name, container.Name)
69+
}
70+
}
71+
}
72+
}
73+
}
74+
}
75+
}
76+
77+
func isNamespaceMonitored(namespace string) bool {
78+
_, exists := monitoredNamespaces[namespace]
79+
return exists
80+
}
81+
82+
func createLogFolder(namespace string) error {
83+
return os.MkdirAll(logFolder+"/"+namespace, os.ModePerm)
84+
}
85+
86+
func isPodMonitored(namespace, podName string) bool {
87+
_, exists := monitoredNamespaces[namespace].pods[podName]
88+
return exists
89+
}
90+
91+
func initMonitoredPod(namespace, podName string) {
92+
monitoredPod := monitoredPod{
93+
containers: make(map[string]monitoredContainer),
94+
}
95+
monitoredNamespaces[namespace].pods[podName] = monitoredPod
96+
}
97+
98+
func initMonitoredContainer(namespace, podName, containerName string) {
99+
monitoredContainer := monitoredContainer{loggingFinished: false}
100+
monitoredNamespaces[namespace].pods[podName].containers[containerName] = monitoredContainer
101+
}
102+
103+
func storeContainerLogWithFollow(namespace, podName, containerName string) {
104+
log, err := getContainerLogWithFollow(namespace, podName, containerName)
105+
if err != nil {
106+
GetLogger(namespace).Errorf("Error while retrieving log of pod '%s': %v", podName, err)
107+
}
108+
109+
if isContainerLoggingFinished(namespace, podName, containerName) {
110+
GetLogger(namespace).Debugf("Logging of container '%s' of pod '%s' already finished, retrieved log will be ignored.", containerName, podName)
111+
} else {
112+
markContainerLoggingAsFinished(namespace, podName, containerName)
113+
if err := writeLogIntoTheFile(namespace, podName, containerName, log); err != nil {
114+
GetLogger(namespace).Errorf("Error while writing log into the file: %v", err)
115+
}
116+
}
117+
}
118+
119+
// Log is returned once container is terminated
120+
func getContainerLogWithFollow(namespace, podName, containerName string) (string, error) {
121+
return kubernetes.PodC(kubeClient).GetLogsWithFollow(namespace, podName, containerName)
122+
}
123+
124+
func isContainerLoggingFinished(namespace, podName, containerName string) bool {
125+
monitoredContainer := monitoredNamespaces[namespace].pods[podName].containers[containerName]
126+
return monitoredContainer.loggingFinished
127+
}
128+
129+
func markContainerLoggingAsFinished(namespace, podName, containerName string) {
130+
monitoredContainer := monitoredNamespaces[namespace].pods[podName].containers[containerName]
131+
monitoredContainer.loggingFinished = true
132+
}
133+
134+
func writeLogIntoTheFile(namespace, podName, containerName, log string) error {
135+
return ioutil.WriteFile(logFolder+"/"+namespace+"/"+podName+"-"+containerName+logSuffix, []byte(log), 0644)
136+
}
137+
138+
// StopPodLogCollector waits until all logs are stored on disc
139+
func StopPodLogCollector(namespace string) error {
140+
if !isNamespaceMonitored(namespace) {
141+
return errors.New("namespace is not monitored")
142+
}
143+
stopNamespaceMonitoring(namespace)
144+
storeUnfinishedContainersLog(namespace)
145+
return nil
146+
}
147+
148+
func stopNamespaceMonitoring(namespace string) {
149+
monitoredNamespaces[namespace].stopMonitoring <- true
150+
close(monitoredNamespaces[namespace].stopMonitoring)
151+
}
152+
153+
// Write log of all containers of pods in namespace which didn't store their log yet
154+
func storeUnfinishedContainersLog(namespace string) {
155+
for podName, pod := range monitoredNamespaces[namespace].pods {
156+
for containerName, container := range pod.containers {
157+
if !container.loggingFinished {
158+
storeContainerLog(namespace, podName, containerName)
159+
}
160+
}
161+
}
162+
}
163+
164+
// Write container log into filesystem
165+
func storeContainerLog(namespace string, podName, containerName string) {
166+
log, err := getContainerLog(namespace, podName, containerName)
167+
if err != nil {
168+
GetLogger(namespace).Errorf("Error while retrieving log of container '%s' in pod '%s': %v", containerName, podName, err)
169+
}
170+
171+
if err := writeLogIntoTheFile(namespace, podName, containerName, log); err != nil {
172+
GetLogger(namespace).Errorf("Error while writing log into the file: %v", err)
173+
}
174+
}
175+
176+
func getContainerLog(namespace, podName, containerName string) (string, error) {
177+
return kubernetes.PodC(kubeClient).GetLogs(namespace, podName, containerName)
178+
}
179+
180+
type monitoredNamespace struct {
181+
pods map[string]monitoredPod
182+
stopMonitoring chan bool
183+
}
184+
185+
type monitoredPod struct {
186+
containers map[string]monitoredContainer
187+
}
188+
189+
type monitoredContainer struct {
190+
loggingFinished bool
191+
}

‎bddframework/smoke/steps/data.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ func (data *Data) BeforeScenario(s interface{}) {
5050
data.Namespace = framework.GenerateNamespaceName()
5151

5252
framework.GetLogger(data.Namespace).Info(fmt.Sprintf("Scenario %s", getScenarioName(s)))
53+
go framework.StartPodLogCollector(data.Namespace)
5354
}
5455

5556
// BeforeStep configure the data before a scenario is launched
@@ -59,6 +60,7 @@ func (data *Data) BeforeStep(s *gherkin.Step) {
5960

6061
// AfterScenario executes some actions on data after a scenario is finished
6162
func (data *Data) AfterScenario(s interface{}, err error) {
63+
framework.StopPodLogCollector(data.Namespace)
6264
endTime := time.Now()
6365
duration := endTime.Sub(data.StartTime)
6466
framework.GetLogger(data.Namespace).Infof("Scenario '%s'. Duration = %s", getScenarioName(s), duration.String())

0 commit comments

Comments
 (0)