Skip to content

Commit 4cb2ecc

Browse files
authored
Fix loading of results collection with JLD2 (#236)
* Fix loading of results collection with JLD2 * Update Tests for results collection * Update Tests to verify BSON and JLD2 * Optical corrections * Remove indention * Bump version
1 parent 946400b commit 4cb2ecc

File tree

4 files changed

+89
-75
lines changed

4 files changed

+89
-75
lines changed

Project.toml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name = "DrWatson"
22
uuid = "634d3b9d-ee7a-5ddf-bec9-22491ea816e1"
33
repo = "https://github.com/JuliaDynamics/DrWatson.jl.git"
4-
version = "2.0.0"
4+
version = "2.0.1"
55

66
[deps]
77
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
@@ -22,10 +22,11 @@ julia = "1.0"
2222
BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0"
2323
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
2424
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
25+
FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
2526
JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
2627
Parameters = "d96e819e-fc66-5662-9728-84c9c7592b0a"
2728
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
2829
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
2930

3031
[targets]
31-
test = ["Test", "BSON", "Parameters", "DataFrames", "JLD2", "Statistics", "Dates"]
32+
test = ["Test", "BSON", "FileIO", "Parameters", "DataFrames", "JLD2", "Statistics", "Dates"]

src/result_collection.jl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ function collect_results!(filename, folder;
8383
df = DataFrames.DataFrame()
8484
else
8585
verbose && @info "Loading existing result collection..."
86-
df = wload(filename)[:df]
86+
df = wload(filename)["df"]
8787
end
8888
@info "Scanning folder $folder for result files."
8989

@@ -116,7 +116,7 @@ function collect_results!(filename, folder;
116116
n += 1
117117
end
118118
verbose && @info "Added $n entries."
119-
!newfile && wsave(filename, Dict(:df => df))
119+
!newfile && wsave(filename, Dict("df" => df))
120120
return df
121121
end
122122

test/savefiles_tests.jl

Lines changed: 64 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using DrWatson, Test
2+
using BSON, JLD2
23
cd(@__DIR__)
34
T = 1000
45
N = 50 # spatial extent
@@ -14,60 +15,62 @@ function f(simulation)
1415
return @strdict a b simulation
1516
end
1617

17-
18+
@testset "Tagsafe ($ending)" for ending ["bson", "jld2"]
1819
################################################################################
1920
# tagsave #
2021
################################################################################
21-
t = f(simulation)
22-
tagsave(savename(simulation, "bson"), t, gitpath=findproject())
23-
file = load(savename(simulation, "bson"))
24-
@test "gitcommit" keys(file)
25-
@test file["gitcommit"] |> typeof == String
26-
rm(savename(simulation, "bson"))
27-
28-
t = f(simulation)
29-
@tagsave(savename(simulation, "bson"), t, safe=false, gitpath=findproject())
30-
file = load(savename(simulation, "bson"))
31-
@test "gitcommit" keys(file)
32-
@test file["gitcommit"] |> typeof == String
33-
@test "script" keys(file)
34-
@test file["script"] |> typeof == String
35-
@test file["script"] == joinpath("test", "savefiles_tests.jl#29")
36-
37-
t = f(simulation)
38-
@tagsave(savename(simulation, "bson"), t, safe=true, gitpath=findproject())
39-
sn = savename(simulation, "bson")[1:end-5]*"_#1"*".bson"
40-
@test isfile(sn)
41-
rm(sn)
42-
43-
t = f(simulation)
44-
tagsave(savename(simulation, "bson"), t, safe=true, gitpath=findproject())
45-
sn = savename(simulation, "bson")[1:end-5]*"_#1"*".bson"
46-
@test isfile(sn)
47-
rm(sn)
48-
49-
t = f(simulation)
50-
t["gitcommit"] = ""
51-
@test @tagsave(savename(simulation, "bson"), t, safe=true, gitpath=findproject())["gitcommit"] == ""
52-
@test isfile(sn)
53-
rm(sn)
54-
@test @tagsave(savename(simulation, "bson"), t, safe=true, force=true, gitpath=findproject())["gitcommit"] != ""
55-
@test isfile(sn)
56-
rm(sn)
57-
58-
rm(savename(simulation, "bson"))
59-
@test !isfile(savename(simulation, "bson"))
60-
61-
ex = @macroexpand @tagsave("testname.bson", (@dict a b c ), storepatch=false; safe=true)
62-
ex2 = @macroexpand @tagsave("testname.bson", @dict a b c; storepatch=false, safe=true)
63-
@test ex.args[1:end-1] == ex2.args[1:end-1]
64-
65-
# Remove leftover
22+
t = f(simulation)
23+
tagsave(savename(simulation, ending), t, gitpath=findproject())
24+
file = load(savename(simulation, ending))
25+
@test "gitcommit" keys(file)
26+
@test file["gitcommit"] |> typeof == String
27+
rm(savename(simulation, ending))
28+
29+
t = f(simulation)
30+
@tagsave(savename(simulation, ending), t, safe=false, gitpath=findproject())
31+
file = load(savename(simulation, ending))
32+
@test "gitcommit" keys(file)
33+
@test file["gitcommit"] |> typeof == String
34+
@test "script" keys(file)
35+
@test file["script"] |> typeof == String
36+
@test file["script"] == joinpath("test", "savefiles_tests.jl#30")
37+
38+
t = f(simulation)
39+
@tagsave(savename(simulation, ending), t, safe=true, gitpath=findproject())
40+
sn = savename(simulation, ending)[1:end-5]*"_#1"*"."*ending
41+
@test isfile(sn)
42+
rm(sn)
43+
44+
t = f(simulation)
45+
tagsave(savename(simulation, ending), t, safe=true, gitpath=findproject())
46+
sn = savename(simulation, ending)[1:end-5]*"_#1"*"."*ending
47+
@test isfile(sn)
48+
rm(sn)
49+
50+
t = f(simulation)
51+
t["gitcommit"] = ""
52+
@test @tagsave(savename(simulation, ending), t, safe=true, gitpath=findproject())["gitcommit"] == ""
53+
@test isfile(sn)
54+
rm(sn)
55+
@test @tagsave(savename(simulation,ending), t, safe=true, force=true, gitpath=findproject())["gitcommit"] != ""
56+
@test isfile(sn)
57+
rm(sn)
58+
59+
rm(savename(simulation, ending))
60+
@test !isfile(savename(simulation, ending))
61+
62+
ex = @macroexpand @tagsave("testname."*ending, (@dict a b c ), storepatch=false; safe=true)
63+
ex2 = @macroexpand @tagsave("testname."*ending, @dict a b c; storepatch=false, safe=true)
64+
@test ex.args[1:end-1] == ex2.args[1:end-1]
65+
66+
# Remove leftover
67+
end
6668

6769
################################################################################
6870
# produce or load #
6971
################################################################################
70-
for ending ("bson", "jld2")
72+
73+
@testset "Produce or Load ($ending)" for ending ["bson", "jld2"]
7174
@test !isfile(savename(simulation, ending))
7275
sim, path = produce_or_load(simulation, f; suffix = ending)
7376
@test isfile(savename(simulation, ending))
@@ -113,15 +116,18 @@ rm(savename(simulation, "jld2"))
113116
################################################################################
114117
# Backup files before saving #
115118
################################################################################
116-
filepath = "test.#backup.jld2"
117-
data = [Dict( "a" => i, "b" => rand(rand(1:10))) for i = 1:3]
118-
for i = 1:3
119-
safesave(filepath, data[i])
120-
@test data[i] == load(filepath)
119+
120+
@testset "Backup ($ending)" for ending ["bson", "jld2"]
121+
filepath = "test.#backup."*ending
122+
data = [Dict( "a" => i, "b" => rand(rand(1:10))) for i = 1:3]
123+
for i = 1:3
124+
safesave(filepath, data[i])
125+
@test data[i] == load(filepath)
126+
end
127+
@test data[2] == load("test.#backup_#1."*ending)
128+
@test data[1] == load("test.#backup_#2."*ending)
129+
@test data[3] == load("test.#backup."*ending)
130+
rm("test.#backup."*ending)
131+
rm("test.#backup_#1."*ending)
132+
rm("test.#backup_#2."*ending)
121133
end
122-
@test data[2] == load("test.#backup_#1.jld2")
123-
@test data[1] == load("test.#backup_#2.jld2")
124-
@test data[3] == load("test.#backup.jld2")
125-
rm("test.#backup.jld2")
126-
rm("test.#backup_#1.jld2")
127-
rm("test.#backup_#2.jld2")

test/update_results_tests.jl

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
using DrWatson, Test
2-
using BSON, DataFrames, JLD2
2+
using BSON, DataFrames, FileIO, JLD2
3+
4+
5+
@testset "Collect Results ($ending)" for ending ["bson", "jld2"]
6+
37
###############################################################################
48
# Setup Folder structure #
59
###############################################################################
@@ -17,31 +21,32 @@ mkdir(datadir("results"))
1721
cd(datadir("results"))
1822

1923
d = Dict("a" => 1, "b" => "2", "c" => rand(10))
20-
DrWatson.wsave(savename(d)*".bson", d)
24+
DrWatson.wsave(savename(d)*"."*ending, d)
2125

2226
d = Dict("a" => 3, "b" => "4", "c" => rand(10), "d" => Float64)
23-
DrWatson.wsave(savename(d)*".bson", d)
27+
DrWatson.wsave(savename(d)*"."*ending, d)
2428

2529
d = Dict("a" => 3, "c" => rand(10), "d" => Float64)
26-
DrWatson.wsave(savename(d)*".jld2", d)
30+
DrWatson.wsave(savename(d)*"."*ending, d)
2731

2832
mkdir("subfolder")
2933
cd("subfolder")
3034

3135
d = Dict("a" => 4., "b" => "twenty" , "d" => Int)
32-
DrWatson.wsave(savename(d)*".bson", d)
36+
DrWatson.wsave(savename(d)*"."*ending, d)
3337

3438
###############################################################################
3539
# Collect Data Into DataFrame #
3640
###############################################################################
3741
using Statistics
3842
special_list = [ :lv_mean => data -> mean(data["c"]),
39-
:lv_var => data -> var(data["c"])]
43+
:lv_var => data -> var(data["c"])]
4044

4145
black_list = ["c"]
4246

4347
folder = datadir("results")
44-
defaultname = joinpath(dirname(folder), "results_$(basename(folder)).bson")
48+
49+
defaultname = joinpath(dirname(folder), "results_$(basename(folder))."*ending)
4550
isfile(defaultname) && rm(defaultname)
4651
cres = collect_results!(defaultname, folder;
4752
subfolders = true, special_list=special_list, black_list = black_list)
@@ -53,7 +58,7 @@ end
5358
@test "c" names(cres)
5459
@test all(startswith.(cres[!,"path"], projectdir()))
5560

56-
relpathname = joinpath(dirname(folder), "results_relpath_$(basename(folder)).bson")
61+
relpathname = joinpath(dirname(folder), "results_relpath_$(basename(folder))."*ending)
5762
cres_relpath = collect_results!(relpathname, folder;
5863
subfolders = true, special_list=special_list, black_list = black_list,
5964
rpath = projectdir())
@@ -82,13 +87,13 @@ cres2 = collect_results!(defaultname, folder;
8287
###############################################################################
8388

8489
special_list2 = [ :lv_mean => data -> mean(data["c"]),
85-
data -> :lv_var => var(data["c"]),
86-
data -> [:lv_mean2 => mean(data["c"]),
87-
:lv_var2 => var(data["c"])]]
90+
data -> :lv_var => var(data["c"]),
91+
data -> [:lv_mean2 => mean(data["c"]),
92+
:lv_var2 => var(data["c"])]]
8893
black_list = ["c"]
8994

9095
folder = datadir("results")
91-
defaultname2 = joinpath(dirname(folder), "results_betterspeciallist.bson")
96+
defaultname2 = joinpath(dirname(folder), "results_betterspeciallist."*ending)
9297
isfile(defaultname2) && rm(defaultname2)
9398
cres10 = collect_results!(defaultname2, folder;
9499
subfolders = true, special_list=special_list2, black_list = black_list)
@@ -102,7 +107,7 @@ end
102107
# Load and analyze DataFrame #
103108
###############################################################################
104109

105-
df = BSON.load(defaultname)[:df]
110+
df = load(defaultname)["df"]
106111
@test size(df) == size(cres2)
107112
@test sort(names(df)) == sort(names(cres2))
108113

@@ -146,3 +151,5 @@ include(joinpath("testdir", "testinclude.jl"))
146151

147152
cd(@__DIR__)
148153
rm("testdir", recursive=true)
154+
155+
end

0 commit comments

Comments
 (0)