Skip to content

Commit 6a879ab

Browse files
gaius-qigaius-qi
authored
test(e2e/v2): add test cases for dfcache (#3833)
Signed-off-by: Gaius <gaius.qi@gmail.com>
1 parent 8c684ee commit 6a879ab

File tree

2 files changed

+336
-0
lines changed

2 files changed

+336
-0
lines changed

‎test/e2e/v2/dfcache_test.go‎

Lines changed: 308 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,308 @@
1+
/*
2+
* Copyright 2025 The Dragonfly Authors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package e2e
18+
19+
import (
20+
"fmt"
21+
22+
. "github.com/onsi/ginkgo/v2" //nolint
23+
. "github.com/onsi/gomega" //nolint
24+
25+
"d7y.io/dragonfly/v2/test/e2e/v2/util"
26+
)
27+
28+
var _ = Describe("Import and Export Using Dfcache", func() {
29+
Context("1MiB file", func() {
30+
var (
31+
testFile *util.File
32+
err error
33+
)
34+
35+
BeforeEach(func() {
36+
testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB)
37+
Expect(err).NotTo(HaveOccurred())
38+
Expect(testFile).NotTo(BeNil())
39+
})
40+
41+
AfterEach(func() {
42+
err = util.GetFileServer().DeleteFile(testFile.GetInfo())
43+
Expect(err).NotTo(HaveOccurred())
44+
})
45+
46+
It("import and export should be ok", Label("dfcache", "import and export"), func() {
47+
var clientPod *util.PodExec
48+
clientPod, err = util.ClientExec()
49+
fmt.Println(err)
50+
Expect(err).NotTo(HaveOccurred())
51+
52+
_, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput()
53+
Expect(err).NotTo(HaveOccurred())
54+
55+
sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID())
56+
Expect(err).NotTo(HaveOccurred())
57+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
58+
59+
sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath())
60+
Expect(err).NotTo(HaveOccurred())
61+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
62+
63+
importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput()
64+
fmt.Println(string(importOut), err)
65+
Expect(err).NotTo(HaveOccurred())
66+
67+
seedClientPod, err := util.SeedClientExec(0)
68+
fmt.Println(err)
69+
Expect(err).NotTo(HaveOccurred())
70+
71+
exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --output %s", testFile.GetSha256(), testFile.GetOutputPath())).CombinedOutput()
72+
fmt.Println(string(exportOut), err)
73+
Expect(err).NotTo(HaveOccurred())
74+
75+
sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, testFile.GetSha256())
76+
Expect(err).NotTo(HaveOccurred())
77+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
78+
79+
sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath())
80+
Expect(err).NotTo(HaveOccurred())
81+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
82+
})
83+
})
84+
85+
Context("10MiB file", func() {
86+
var (
87+
testFile *util.File
88+
err error
89+
)
90+
91+
BeforeEach(func() {
92+
testFile, err = util.GetFileServer().GenerateFile(util.FileSize10MiB)
93+
Expect(err).NotTo(HaveOccurred())
94+
Expect(testFile).NotTo(BeNil())
95+
})
96+
97+
AfterEach(func() {
98+
err = util.GetFileServer().DeleteFile(testFile.GetInfo())
99+
Expect(err).NotTo(HaveOccurred())
100+
})
101+
102+
It("import and export should be ok", Label("dfcache", "import and export"), func() {
103+
var clientPod *util.PodExec
104+
clientPod, err = util.ClientExec()
105+
fmt.Println(err)
106+
Expect(err).NotTo(HaveOccurred())
107+
108+
_, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput()
109+
Expect(err).NotTo(HaveOccurred())
110+
111+
sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID())
112+
Expect(err).NotTo(HaveOccurred())
113+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
114+
115+
sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath())
116+
Expect(err).NotTo(HaveOccurred())
117+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
118+
119+
importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput()
120+
fmt.Println(string(importOut), err)
121+
Expect(err).NotTo(HaveOccurred())
122+
123+
seedClientPod, err := util.SeedClientExec(0)
124+
fmt.Println(err)
125+
Expect(err).NotTo(HaveOccurred())
126+
127+
exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --output %s", testFile.GetSha256(), testFile.GetOutputPath())).CombinedOutput()
128+
fmt.Println(string(exportOut), err)
129+
Expect(err).NotTo(HaveOccurred())
130+
131+
sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, testFile.GetSha256())
132+
Expect(err).NotTo(HaveOccurred())
133+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
134+
135+
sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath())
136+
Expect(err).NotTo(HaveOccurred())
137+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
138+
})
139+
})
140+
141+
Context("100MiB file", func() {
142+
var (
143+
testFile *util.File
144+
err error
145+
)
146+
147+
BeforeEach(func() {
148+
testFile, err = util.GetFileServer().GenerateFile(util.FileSize100MiB)
149+
Expect(err).NotTo(HaveOccurred())
150+
Expect(testFile).NotTo(BeNil())
151+
})
152+
153+
AfterEach(func() {
154+
err = util.GetFileServer().DeleteFile(testFile.GetInfo())
155+
Expect(err).NotTo(HaveOccurred())
156+
})
157+
158+
It("import and export should be ok", Label("dfcache", "import and export"), func() {
159+
var clientPod *util.PodExec
160+
clientPod, err = util.ClientExec()
161+
fmt.Println(err)
162+
Expect(err).NotTo(HaveOccurred())
163+
164+
_, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput()
165+
Expect(err).NotTo(HaveOccurred())
166+
167+
sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID())
168+
Expect(err).NotTo(HaveOccurred())
169+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
170+
171+
sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath())
172+
Expect(err).NotTo(HaveOccurred())
173+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
174+
175+
importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput()
176+
fmt.Println(string(importOut), err)
177+
Expect(err).NotTo(HaveOccurred())
178+
179+
seedClientPod, err := util.SeedClientExec(0)
180+
fmt.Println(err)
181+
Expect(err).NotTo(HaveOccurred())
182+
183+
exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --output %s", testFile.GetSha256(), testFile.GetOutputPath())).CombinedOutput()
184+
fmt.Println(string(exportOut), err)
185+
Expect(err).NotTo(HaveOccurred())
186+
187+
sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, testFile.GetSha256())
188+
Expect(err).NotTo(HaveOccurred())
189+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
190+
191+
sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath())
192+
Expect(err).NotTo(HaveOccurred())
193+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
194+
})
195+
})
196+
197+
Context("1MiB file and set export transfer-from-dfdaemon", func() {
198+
var (
199+
testFile *util.File
200+
err error
201+
)
202+
203+
BeforeEach(func() {
204+
testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB)
205+
Expect(err).NotTo(HaveOccurred())
206+
Expect(testFile).NotTo(BeNil())
207+
})
208+
209+
AfterEach(func() {
210+
err = util.GetFileServer().DeleteFile(testFile.GetInfo())
211+
Expect(err).NotTo(HaveOccurred())
212+
})
213+
214+
It("import and export should be ok", Label("dfcache", "import and export", "export transfer-from-dfdaemon"), func() {
215+
var clientPod *util.PodExec
216+
clientPod, err = util.ClientExec()
217+
fmt.Println(err)
218+
Expect(err).NotTo(HaveOccurred())
219+
220+
_, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput()
221+
Expect(err).NotTo(HaveOccurred())
222+
223+
sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID())
224+
Expect(err).NotTo(HaveOccurred())
225+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
226+
227+
sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath())
228+
Expect(err).NotTo(HaveOccurred())
229+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
230+
231+
importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput()
232+
fmt.Println(string(importOut), err)
233+
Expect(err).NotTo(HaveOccurred())
234+
235+
seedClientPod, err := util.SeedClientExec(0)
236+
fmt.Println(err)
237+
Expect(err).NotTo(HaveOccurred())
238+
239+
exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --transfer-from-dfdaemon --output %s", testFile.GetSha256(), testFile.GetOutputPath())).CombinedOutput()
240+
fmt.Println(string(exportOut), err)
241+
Expect(err).NotTo(HaveOccurred())
242+
243+
sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, testFile.GetSha256())
244+
Expect(err).NotTo(HaveOccurred())
245+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
246+
247+
sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath())
248+
Expect(err).NotTo(HaveOccurred())
249+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
250+
})
251+
})
252+
253+
Context("10MiB file and set export transfer-from-dfdaemon", func() {
254+
var (
255+
testFile *util.File
256+
err error
257+
)
258+
259+
BeforeEach(func() {
260+
testFile, err = util.GetFileServer().GenerateFile(util.FileSize10MiB)
261+
Expect(err).NotTo(HaveOccurred())
262+
Expect(testFile).NotTo(BeNil())
263+
})
264+
265+
AfterEach(func() {
266+
err = util.GetFileServer().DeleteFile(testFile.GetInfo())
267+
Expect(err).NotTo(HaveOccurred())
268+
})
269+
270+
It("import and export should be ok", Label("dfcache", "import and export", "export transfer-from-dfdaemon"), func() {
271+
var clientPod *util.PodExec
272+
clientPod, err = util.ClientExec()
273+
fmt.Println(err)
274+
Expect(err).NotTo(HaveOccurred())
275+
276+
_, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput()
277+
Expect(err).NotTo(HaveOccurred())
278+
279+
sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID())
280+
Expect(err).NotTo(HaveOccurred())
281+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
282+
283+
sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath())
284+
Expect(err).NotTo(HaveOccurred())
285+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
286+
287+
importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput()
288+
fmt.Println(string(importOut), err)
289+
Expect(err).NotTo(HaveOccurred())
290+
291+
seedClientPod, err := util.SeedClientExec(0)
292+
fmt.Println(err)
293+
Expect(err).NotTo(HaveOccurred())
294+
295+
exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --transfer-from-dfdaemon --output %s", testFile.GetSha256(), testFile.GetOutputPath())).CombinedOutput()
296+
fmt.Println(string(exportOut), err)
297+
Expect(err).NotTo(HaveOccurred())
298+
299+
sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, testFile.GetSha256())
300+
Expect(err).NotTo(HaveOccurred())
301+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
302+
303+
sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath())
304+
Expect(err).NotTo(HaveOccurred())
305+
Expect(testFile.GetSha256()).To(Equal(sha256sum))
306+
})
307+
})
308+
})

‎test/e2e/v2/util/task.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,34 @@ func CalculateSha256ByTaskID(pods []*PodExec, taskID string) (string, error) {
7373
return sha256sum, nil
7474
}
7575

76+
func CalculateSha256ByPersistentCacheTaskID(pods []*PodExec, taskID string) (string, error) {
77+
var sha256sum string
78+
for _, pod := range pods {
79+
contentPath := fmt.Sprintf("%s/persistent-cache-tasks/%s/%s", clientContentDir, taskID[:3], taskID)
80+
if _, err := pod.Command("ls", contentPath).CombinedOutput(); err != nil {
81+
// If the path does not exist, skip this client.
82+
fmt.Printf("path %s does not exist: %s\n", contentPath, err.Error())
83+
continue
84+
}
85+
86+
// Calculate sha256sum of the task content.
87+
out, err := pod.Command("sh", "-c", fmt.Sprintf("sha256sum %s", contentPath)).CombinedOutput()
88+
if err != nil {
89+
return "", fmt.Errorf("calculate sha256sum of %s failed: %s", contentPath, err.Error())
90+
}
91+
92+
fmt.Println("sha256sum: " + string(out))
93+
sha256sum = strings.Split(string(out), " ")[0]
94+
break
95+
}
96+
97+
if sha256sum == "" {
98+
return "", errors.New("can not found sha256sum")
99+
}
100+
101+
return sha256sum, nil
102+
}
103+
76104
func CalculateSha256ByOutput(pods []*PodExec, output string) (string, error) {
77105
var sha256sum string
78106
for _, pod := range pods {

0 commit comments

Comments
 (0)