use environment variables for tagging (#290)

* use environment variables for tagging

Hope I did everything right!

* fix tests

* changelog

Author: Datseris

CHANGELOG.md

@@ -1,3 +1,5 @@
+# 2.7.0
+* DrWatson-related `ENV`ironment variables are now available to globally set the default values for e.g. story git patches, tagging, or safe-saving in various functions like `tagsave` or `produce_or_load`.
 # 2.6.0
 * Use `JLD2`'s jldopen in `collect_results!` to speed up loading of metadata.
 # 2.5.0

Project.toml

@@ -1,7 +1,7 @@
 name = "DrWatson"
 uuid = "634d3b9d-ee7a-5ddf-bec9-22491ea816e1"
 repo = "https://github.com/JuliaDynamics/DrWatson.jl.git"
-version = "2.6.2"
+version = "2.7.0"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"

src/naming.jl

@@ -91,11 +91,9 @@ function savename(prefix::String, c, suffix::String;
                   sort = true, equals = "=")
 
     if any(sep in prefix for sep in ['/', '\\'])
-        @warn """
-            Path separators in `savename` prefixes may break reproducibility on other OS.
-            The recommended way is using the `*dir()` methods or `joinpath` with
-            `savename` (e.g. `datadir("path", "to", "folder", savename("prefix", data))`).
-        """
+        @warn "Path separators in `savename` prefixes may break reproducibility on other OS. "*
+            "The recommended way is using the `*dir()` methods or `joinpath` with "*
+            "`savename` (e.g. `datadir(\"path\", \"to\", \"folder\", savename(\"prefix\", data))`)."
     end
     sigdigits = digits === nothing ? sigdigits : nothing
     val2string = val_to_string === nothing ? (val -> valtostring(val, digits, sigdigits)) : val_to_string

src/saving_files.jl

@@ -24,7 +24,8 @@ end
 
 ## Keywords
 * `suffix = "jld2", prefix = default_prefix(config)` : Used in [`savename`](@ref).
-* `tag::Bool = istaggable(suffix)` : Save the file using [`tagsave`](@ref) if `true`.
+* `tag::Bool = get(ENV, "DRWATSON_TAG", istaggable(suffix))` : Save the file 
+  using [`tagsave`](@ref) if `true` (which is the default).
 * `gitpath, storepatch` : Given to [`tagsave`](@ref) if `tag` is `true`.
 * `force = false` : If `true` then don't check if file `s` exists and produce
   it and save it anyway.
@@ -41,9 +42,13 @@ produce_or_load(c, f; kwargs...) = produce_or_load("", c, f; kwargs...)
 produce_or_load(f::Function, c; kwargs...) = produce_or_load(c, f; kwargs...)
 produce_or_load(f::Function, path, c; kwargs...) = produce_or_load(path, c, f; kwargs...)
 function produce_or_load(path, c, f::Function;
-    suffix = "jld2", prefix = default_prefix(c),
-    tag::Bool = istaggable(suffix), gitpath = projectdir(), loadfile = true,
-    force = false, verbose = true, storepatch = true, wsave_kwargs = Dict(), kwargs...)
+        suffix = "jld2", prefix = default_prefix(c),
+        tag::Bool = get(ENV, "DRWATSON_TAG", istaggable(suffix)), 
+        gitpath = projectdir(), loadfile = true,
+        storepatch::Bool = get(ENV, "DRWATSON_STOREPATCH", true), 
+        force = false, verbose = true, wsave_kwargs = Dict(), 
+        kwargs...
+    )
 
     s = joinpath(path, savename(prefix, c, suffix; kwargs...))
 
@@ -118,21 +123,29 @@ end
 #                             tag saving                                       #
 ################################################################################
 """
-    tagsave(file::String, d::AbstractDict; safe = false, gitpath = projectdir(), storepatch = true, force = false, kwargs...)
+    tagsave(file::String, d::AbstractDict; kwargs...)
 First [`tag!`](@ref) dictionary `d` and then save `d` in `file`.
-If `safe = true` save the file using [`safesave`](@ref).
 
 "Tagging" means that when saving the dictionary, an extra field
 `:gitcommit` is added to establish reproducibility of results using
-Git. If the Git repository is dirty, one more field `:gitpatch` is
-added that stores the difference string.  If a dictionary already
-contains a key `:gitcommit`, it is not overwritten, unless,
+Git. If the Git repository is dirty and `storepatch=true`, one more field `:gitpatch` is
+added that stores the difference string. If a dictionary already
+contains a key `:gitcommit`, it is not overwritten, unless
 `force=true`. For more details, see [`tag!`](@ref).
 
+Keywords `gitpath, storepatch, force,` are propagated to [`tag!`](@ref).
 Any additional keyword arguments are propagated to `wsave`, to e.g.
 enable compression.
+
+The keyword `safe = get(ENV, "DRWATSON_SAFESAVE", false)` decides whether
+to save the file using [`safesave`](@ref).
 """
-function tagsave(file, d; safe::Bool = false, gitpath = projectdir(), storepatch = true, force = false, source = nothing, kwargs...)
+function tagsave(file, d; 
+        gitpath = projectdir(), 
+        safe::Bool = get(ENV, "DRWATSON_SAFESAVE", false), 
+        storepatch::Bool = get(ENV, "DRWATSON_STOREPATCH", true), 
+        force = false, source = nothing, kwargs...
+    )
     d2 = tag!(d, gitpath=gitpath, storepatch=storepatch, force=force, source=source)
     if safe
         safesave(file, copy(d2); kwargs...)
@@ -200,7 +213,7 @@ function increment_backup_num(filepath)
     path, filename = splitdir(filepath)
     fname, suffix = splitext(filename)
     m = match(r"^(.*)_#([0-9]+)$", fname)
-    if m == nothing
+    if m === nothing
         return joinpath(path, "$(fname)_#1$(suffix)")
     end
     newnum = string(parse(Int, m.captures[2]) +1)
@@ -228,7 +241,7 @@ Save each entry in `dicts` into a unique temporary file in the directory `tmp`.
 Then return the list of file names (relative to `tmp`) that were used
 for saving each dictionary. Each dictionary can then be loaded back by calling
 
-    FileIO.load(nth_tmpfilename, "params")
+    wload(nth_tmpfilename, "params")
 
 `tmp` defaults to `projectdir("_research", "tmp")`.
 

src/saving_tools.jl

@@ -141,7 +141,7 @@ function gitpatch(path = projectdir(); try_submodule_diff=true)
             # Remove the submodule option as it is not supported by older git versions.
             return gitpatch(path; try_submodule_diff = false)
         else
-            @warn "`gitpatch` failed with error $(result.err) $(result.exception) and , returning `nothing` instead."
+            @warn "`gitpatch` failed with error $(result.err) $(result.exception). Returning `nothing` instead."
         end
     catch er
         if isa(er,LibGit2.GitError) && er.code == LibGit2.Error.ENOTFOUND
@@ -160,7 +160,7 @@ end
 # Tagging
 ########################################################################################
 """
-    tag!(d::AbstractDict; gitpath = projectdir(), storepatch = true, force = false) -> d
+    tag!(d::AbstractDict; kwargs...) -> d
 Tag `d` by adding an extra field `gitcommit` which will have as value
 the [`gitdescribe`](@ref) of the repository at `gitpath` (by default
 the project's gitpath). Do nothing if a key `gitcommit` already exists
@@ -178,6 +178,12 @@ To restore a repository to the state of a particular model-run do:
 1. checkout the relevant commit with `git checkout xyz` where xyz is the value stored
 2. apply the patch `git apply patch`, where the string stored in the `gitpatch` field needs to be written to the file `patch`.
 
+## Keywords
+* `gitpath = projectdir()`
+* `force = false`
+* `storepatch = get(ENV, "DRWATSON_STOREPATCH", true)`: Whether to collect and store the 
+  output of [`gitpatch`](@ref) as well.
+
 ## Examples
 ```julia
 julia> d = Dict(:x => 3, :y => 4)
@@ -192,7 +198,10 @@ Dict{Symbol,Any} with 3 entries:
   :x => 3
 ```
 """
-function tag!(d::AbstractDict{K,T}; gitpath = projectdir(), storepatch = true, force = false, source = nothing) where {K,T}
+function tag!(d::AbstractDict{K,T}; 
+        gitpath = projectdir(), force = false, source = nothing,
+        storepatch::Bool = get(ENV, "DRWATSON_STOREPATCH", true), 
+    ) where {K,T}
     @assert (K <: Union{Symbol,String}) "We only know how to tag dictionaries that have keys that are strings or symbols"
     c = gitdescribe(gitpath)
     c === nothing && return d # gitpath is not a git repo
@@ -210,7 +219,7 @@ function tag!(d::AbstractDict{K,T}; gitpath = projectdir(), storepatch = true, f
         # Only include patch info if `storepatch` is true and if we can get the info.
         if storepatch
             patch = gitpatch(gitpath)
-            if (patch != nothing) && (patch != "")
+            if (patch !== nothing) && (patch != "")
                 d[patchname] = patch
             end
         end
@@ -330,10 +339,13 @@ additional data fields as strings. Currently endings that can do this are:
 ```
 $(TAGGABLE_FILE_ENDINGS)
 ```
-
-    istaggable(x) = x isa AbstractDictionary
 """
 istaggable(file::AbstractString) = any(endswith(file, e) for e ∈ TAGGABLE_FILE_ENDINGS)
+
+"""
+    istaggable(x) = x isa AbstractDict
+For non-string input the function just checks if input is dictionary.
+"""
 istaggable(x) = x isa AbstractDict