QOL improvements on the what should be attempted to be tagged (#260)

* add function `istaggable`

* incorporate `istaggable` into `produce_or_load`.

* add docs about it

* add tests, up version, cangelog

Author: Datseris

CHANGELOG.md

@@ -1,3 +1,5 @@
+# 2.0.4
+* `produce_or_load` now will not attempt to `tagsave` for inappropriate file formats, like `.csv`.
 # 2.0.3
 * Added a kwarg `equals` to `savename` to extend applicability.
 # 2.0

Project.toml

@@ -1,7 +1,7 @@
 name = "DrWatson"
 uuid = "634d3b9d-ee7a-5ddf-bec9-22491ea816e1"
 repo = "https://github.com/JuliaDynamics/DrWatson.jl.git"
-version = "2.0.3"
+version = "2.0.4"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
@@ -22,6 +22,7 @@ julia = "1.0"
 
 [extras]
 BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0"
+CSVFiles = "5d742f6a-9f54-50ce-8119-2520741973ca"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
@@ -31,4 +32,7 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test", "BSON", "FileIO", "Parameters", "DataFrames", "JLD2", "Statistics", "Dates"]
+test = [
+    "Test", "BSON", "FileIO", "Parameters", "DataFrames",
+    "JLD2", "Statistics", "Dates", "CSVFiles"
+]

docs/src/save.md

@@ -56,8 +56,11 @@ In addition, it attempts to minimize computing energy spent on getting a result.
 
 ```@docs
 produce_or_load
+istaggable
 ```
-See [Stopping "Did I run this?"](@ref) for an example usage of `produce_or_load`.
+
+See [Stopping "Did I run this?"](@ref) for an example usage of `produce_or_load`. While `produce_or_load` will try to by default tag your data if possible, you can also use it with other formats. An example is when your simulation function `f` returns a `DataFrame` and the file suffix is `"csv"`. In this case tagging will not happen, but `produce_or_load` will work as expected.
+
 
 ## Converting a struct to a dictionary
 [`savename`](@ref) gives great support for getting a name out of any Julia composite type. To save something though, one needs a dictionary. So the following function can be conveniently used to directly save a struct using any saving function:

src/DrWatson.jl

@@ -27,8 +27,8 @@ _wsave(filename, data...; kwargs...) = FileIO.save(filename, data...; kwargs...)
     wsave(filename, data...; kwargs...)
 
 Save `data` at `filename` by first creating the appropriate paths.
-Default fallback is `FileIO.save`. Extend `wsave` for your type
-by extending `DrWatson._wsave(filename, data...; kwargs...)`.
+Default fallback is `FileIO.save`. Extend this for your types
+by extending `DrWatson._wsave(filename, data::YourType...; kwargs...)`.
 """
 function wsave(filename, data...; kwargs...)
     mkpath(dirname(filename))

src/saving_files.jl

@@ -11,17 +11,19 @@ that produces your data. Then save `file` as `s` and then return `file, s`.
 The function `f` must return a dictionary,
 the macros [`@dict`](@ref) and [`@strdict`](@ref) can help with that.
 
-You can use [do-block](https://docs.julialang.org/en/v1/manual/functions/#Do-Block-Syntax-for-Function-Arguments) instead of defining a function to pass in. For example,
+You can use [do-block]
+(https://docs.julialang.org/en/v1/manual/functions/#Do-Block-Syntax-for-Function-Arguments)
+instead of defining a function to pass in. For example,
 ```julia
 produce_or_load([path="",] c) do c
     # simulation wiht config `c` runs here
 end
 ```
 
 ## Keywords
-* `tag = true` : Save the file using [`tagsave`](@ref).
+* `suffix = "jld2", prefix = default_prefix(c)` : Used in [`savename`](@ref).
+* `tag::Bool = istaggable(suffix)` : Save the file using [`tagsave`](@ref) if `true`.
 * `gitpath, storepatch` : Given to [`tagsave`](@ref) if `tag` is `true`.
-* `suffix = "jld2", prefix = default_prefix(c)` : Used in `savename`.
 * `force = false` : If `true` then don't check if file `s` exists and produce
   it and save it anyway.
 * `loadfile = true` : If `false`, this function does not actually load the
@@ -30,15 +32,13 @@ end
   exist it is still produced and saved.
 * `verbose = true` : print info about the process, if the file doesn't exist.
 * `kwargs...` : All other keywords are propagated to `savename`.
-
-See also [`savename`](@ref).
 """
 produce_or_load(c, f; kwargs...) = produce_or_load("", c, f; kwargs...)
 produce_or_load(f::Function, c; kwargs...) = produce_or_load(c, f; kwargs...)
 produce_or_load(f::Function, path, c; kwargs...) = produce_or_load(path, c, f; kwargs...)
 function produce_or_load(path, c, f::Function;
-    tag::Bool = true, gitpath = projectdir(), loadfile = true,
     suffix = "jld2", prefix = default_prefix(c),
+    tag::Bool = istaggable(suffix), gitpath = projectdir(), loadfile = true,
     force = false, verbose = true, storepatch = true, kwargs...)
 
     s = joinpath(path, savename(prefix, c, suffix; kwargs...))

src/saving_tools.jl

@@ -1,6 +1,11 @@
 export gitdescribe, current_commit, tag!, @tag!
 export dict_list, dict_list_count
+export struct2dict, struct2ntuple
+export istaggable
 
+########################################################################################
+# Obtaining Git information
+########################################################################################
 """
     gitdescribe(gitpath = projectdir()) -> gitstr
 
@@ -134,12 +139,11 @@ function gitpatch(path = projectdir(); try_submodule_diff=true)
         end
     end
     return nothing
-    # tree = LibGit2.GitTree(repo, "HEAD^{tree}")
-    # diff = LibGit2.diff_tree(repo, tree)
-    # now there is no way to generate the patch with LibGit2.jl.
-    # Instead use commands:
 end
 
+########################################################################################
+# Tagging
+########################################################################################
 """
     tag!(d::Dict; gitpath = projectdir(), storepatch = true, force = false) -> d
 Tag `d` by adding an extra field `gitcommit` which will have as value
@@ -201,7 +205,7 @@ function tag!(d::Dict{K,T}; gitpath = projectdir(), storepatch = true, force = f
             d[patchname] = patch
         end
     end
-    if source != nothing && !force
+    if source !== nothing && !force
         if haskey(d, scriptname)
             @warn "The dictionary already has a key named `script`. We won't "*
             "overwrite it with the script name."
@@ -249,7 +253,23 @@ macro tag!(d,args...)
     return :(tag!($(esc(d)),$(esc.(convert_to_kw.(args))...),source=$s))
 end
 
-export struct2dict
+########################################################################################
+# Tagging Utilities
+########################################################################################
+const TAGGABLE_FILE_ENDINGS = ("bson", "jld", "jld2")
+"""
+    istaggable(file::AbstractStrig) → bool
+Return `true` if the file save format (file ending) is "taggable", i.e. allows adding
+additional data fields as strings. Currently endings that can do this are:
+```
+$(TAGGABLE_FILE_ENDINGS)
+```
+
+    istaggable(x) = x isa AbstractDictionary
+"""
+istaggable(file::AbstractString) = any(endswith(file, e) for e ∈ TAGGABLE_FILE_ENDINGS)
+istaggable(x) = x isa AbstractDict
+
 
 """
     struct2dict(s) -> d
@@ -263,8 +283,6 @@ function struct2dict(s)
     Dict(x => getfield(s, x) for x in fieldnames(typeof(s)))
 end
 
-export struct2ntuple
-
 """
     struct2ntuple(s) -> n
 Convert a Julia composite type `s` to a NamedTuple `n`.

test/savefiles_tests.jl

@@ -15,10 +15,10 @@ function f(simulation)
     return @strdict a b simulation
 end
 
-@testset "Tagsafe ($ending)" for ending ∈ ["bson", "jld2"]
 ################################################################################
 #                                 tagsave                                      #
 ################################################################################
+@testset "Tagsafe ($ending)" for ending ∈ ["bson", "jld2"]
     t = f(simulation)
     tagsave(savename(simulation, ending), t, gitpath=findproject())
     file = load(savename(simulation, ending))
@@ -113,6 +113,21 @@ end
 rm(savename(simulation, "jld2"))
 @test !isfile(savename(simulation, "jld2"))
 
+using DataFrames
+@testset "produce_or_load with dataframe" begin
+    function makedf(c)
+        a = c[:a]
+        return DataFrame(A = a, B = rand(3))
+    end
+    c = (a = 0.5,)
+    _, spath = produce_or_load(c, makedf; suffix = "csv")
+    @test isfile(spath)
+    df = DataFrame(load(spath))
+    @test df.A == [0.5, 0.5, 0.5]
+    rm(spath)
+    @test !isfile(spath)
+end
+
 ################################################################################
 #                          Backup files before saving                          #
 ################################################################################

test/stools_tests.jl

@@ -262,3 +262,8 @@ end
 rm(tmpdir, force = true, recursive = true)
 @test !isdir(tmpdir)
 
+## is taggable 
+@test DrWatson.istaggable("test.jld2")
+@test !DrWatson.istaggable("test.csv")
+@test !DrWatson.istaggable(0.5)
+@test DrWatson.istaggable(Dict(:a => 0.5))