Allow for OrderedDicts to be passed to dict_list and other similar functions (#283)

* Allow for OrderedDicts to be passed to dict_list and other similar functions. Anywhere where functions required the input to be a ::Dict{K,T} were changed to ::AbstractDict{K,T} to allow for OrderedDicts.
The only place this was not changed was in the project_setup where it gets the pkg["name"] because that returns a Dict.

Added an optional kwarg `order` argument to struct2dict to return an OrderedDict (default is false), where order refers to insertion order.
If a set of OrderedDicts is used with dict_list an OrderedDict should be returned.

checktagtype! will also return a regular Dict.

OrderedDict from DataStructures is also exported from DrWatson.

* Removed `DataStructures` dependency.

struct2dict now allows the user to specify the type of dictionary to be returned. If one is not specified, then a `Dict` is returned.

* Added tests for example usage of OrderedDicts to `stool_tests.jl`

`tostringdict` and `tosymboldict` also allow for a specific type of `dict` to be returned.

Still have the `DataStructures` dependency. Unsure why and not sure how to fix. Have checked the files and the only place DataStructures is used is in
`stool_tests.jl`

* `DataStructures` dependency is actually removed this time.

* Fixed minor typos and comments in `naming.jl` and `saving_tools.jl`.

Author: rs7q5

Project.toml

@@ -23,17 +23,15 @@ julia = "1.0"
 [extras]
 BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0"
 CSVFiles = "5d742f6a-9f54-50ce-8119-2520741973ca"
+CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
 JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
 Parameters = "d96e819e-fc66-5662-9728-84c9c7592b0a"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
 
 [targets]
-test = [
-    "Test", "BSON", "FileIO", "Parameters", "DataFrames",
-    "JLD2", "Statistics", "Dates", "CSVFiles", "CodecZlib"
-]
+test = ["Test", "BSON", "FileIO", "Parameters", "DataFrames", "JLD2", "Statistics", "Dates", "CSVFiles", "CodecZlib", "DataStructures"]

src/DrWatson.jl

@@ -1,7 +1,6 @@
 "The perfect sidekick to your scientific inquiries"
 module DrWatson
 import Pkg, LibGit2
-
 const PATH_SEPARATOR = joinpath("_", "_")[2]
 
 # Misc functions for kw-macros

src/dict_list.jl

@@ -1,5 +1,5 @@
 """
-    dict_list(c::Dict)
+    dict_list(c::AbstractDict)
 Expand the dictionary `c` into a vector of dictionaries.
 Each entry has a unique combination from the product of the `Vector`
 values of the dictionary while the non-`Vector` values are kept constant
@@ -47,7 +47,7 @@ julia> dict_list(c)
  Dict(:a=>2,:b=>4,:run=>"tri",:e=>[3, 5],:model=>"linear")
 ```
 """
-function dict_list(c::Dict)
+function dict_list(c::AbstractDict)
     if contains_partially_restricted(c)
         # The method for generating the restricted parameter set is as follows:
         # 1. Remove any nested parameter restrictions (#209)
@@ -94,7 +94,7 @@ function is_solution_subset_of_existing(trial, trial_solutions)
     return false
 end
 
-function _dict_list(c::Dict)
+function _dict_list(c::AbstractDict)
     iterable_fields = filter(k -> typeof(c[k]) <: Vector, keys(c))
     non_iterables = setdiff(keys(c), iterable_fields)
 
@@ -152,7 +152,7 @@ function DependentParameter(value::DependentParameter, condition::Function)
     DependentParameter(value.value, new_condition)
 end
 
-contains_partially_restricted(d::Dict) = any(contains_partially_restricted,values(d))
+contains_partially_restricted(d::AbstractDict) = any(contains_partially_restricted,values(d))
 contains_partially_restricted(d::Vector) = any(contains_partially_restricted,d)
 contains_partially_restricted(::DependentParameter) = true
 contains_partially_restricted(::Any) = false
@@ -170,7 +170,7 @@ In a case like this:
 
 Broadcasting is obviously not wanted as `:b` should retain it's type of `Vector{Int}`.
 """
-function unexpand_restricted(c::Dict{T}) where T
+function unexpand_restricted(c::AbstractDict{T}) where T
     _c = Dict{T,Any}() # There are hardly any cases where this will not be any.
     for k in keys(c)
         if c[k] isa AbstractVector && any(el->eltype(el) <: DependentParameter, c[k])

src/naming.jl

@@ -60,7 +60,7 @@ See also [`parse_savename`](@ref) and [`@savename`](@ref).
   called with its default arguments (so customization here is possible only
   by rolling your own container type). Containers leading to empty `savename`
   are skipped.
-* `equals = "="` : Connector between name and value. Can be useful to modify for 
+* `equals = "="` : Connector between name and value. Can be useful to modify for
   adding space `" = "`.
 
 ## Examples
@@ -271,7 +271,7 @@ end
 
 """
     esc_dict_expr_from_vars(vars)
-Transform a `Tuple` of `Symbol` and assignments (`a=b`) 
+Transform a `Tuple` of `Symbol` and assignments (`a=b`)
 into a dictionary where each `Symbol` in `vars`
 defines a key-value pair. The value is obtained by evaluating the `Symbol` in
 the macro calling environment.
@@ -356,24 +356,26 @@ ntuple2dict(nt::NamedTuple) = Dict(k => nt[k] for k in keys(nt))
 Convert a dictionary (with `Symbol` or `String` as key type) to
 a `NamedTuple`.
 """
-function dict2ntuple(dict::Dict{String, T}) where T
+function dict2ntuple(dict::AbstractDict{String, T}) where T
     NamedTuple{Tuple(Symbol.(keys(dict)))}(values(dict))
 end
-function dict2ntuple(dict::Dict{Symbol, T}) where T
+function dict2ntuple(dict::AbstractDict{Symbol, T}) where T
     NamedTuple{Tuple(keys(dict))}(values(dict))
 end
 
 """
     tostringdict(d)
 Change a dictionary with key type `Symbol` to have key type `String`.
 """
-tostringdict(d) = Dict(zip(String.(keys(d)), values(d)))
+tostringdict(::Type{DT},d) where {DT<:AbstractDict} = DT(zip(String.(keys(d)), values(d)))
+tostringdict(d) = tostringdict(Dict,d)
 
 """
     tosymboldict(d)
 Change a dictionary with key type `String` to have key type `Symbol`.
 """
-tosymboldict(d) = Dict(zip(Symbol.(keys(d)), values(d)))
+tosymboldict(::Type{DT},d) where {DT<:AbstractDict} = DT(zip(Symbol.(keys(d)), values(d)))
+tosymboldict(d) = tosymboldict(Dict,d)
 
 """
     parse_savename(filename::AbstractString; kwargs...)

src/saving_files.jl

@@ -10,7 +10,7 @@ with the global path that it is saved at (`s`).
 If the file does not exist then call `file = f(config)`, with `f` your function
 that produces your data. Then save the `file` as `s` and then return `file, s`.
 
-The function `f` should return a dictionary if the data are saved in the default 
+The function `f` should return a dictionary if the data are saved in the default
 format of JLD2.jl., the macro [`@strdict`](@ref) can help with that.
 
 You can use a [do-block]
@@ -118,7 +118,7 @@ end
 #                             tag saving                                       #
 ################################################################################
 """
-    tagsave(file::String, d::Dict; safe = false, gitpath = projectdir(), storepatch = true, force = false, kwargs...)
+    tagsave(file::String, d::AbstractDict; safe = false, gitpath = projectdir(), storepatch = true, force = false, kwargs...)
 First [`tag!`](@ref) dictionary `d` and then save `d` in `file`.
 If `safe = true` save the file using [`safesave`](@ref).
 
@@ -144,7 +144,7 @@ end
 
 
 """
-    @tagsave(file::String, d::Dict; kwargs...)
+    @tagsave(file::String, d::AbstractDict; kwargs...)
 Same as [`tagsave`](@ref) but one more field `:script` is added that records
 the local path of the script and line number that called `@tagsave`, see [`@tag!`](@ref).
 """

src/saving_tools.jl

@@ -96,7 +96,7 @@ end
 """
     read_stdout_stderr(cmd::Cmd)
 
-Run `cmd` synchronously and capture stdout, stdin and a possible error exception. 
+Run `cmd` synchronously and capture stdout, stdin and a possible error exception.
 Return a `NamedTuple` with the fields `exception`, `out` and `err`.
 """
 function read_stdout_stderr(cmd::Cmd)
@@ -121,7 +121,7 @@ compared to its last commit; i.e. what `git diff HEAD` produces.
 The `gitpath` needs to point to a directory within a git repository,
 otherwise `nothing` is returned.
 
-Be aware that `gitpatch` needs a working installation of Git, that 
+Be aware that `gitpatch` needs a working installation of Git, that
 can be found in the current PATH.
 """
 function gitpatch(path = projectdir(); try_submodule_diff=true)
@@ -160,7 +160,7 @@ end
 # Tagging
 ########################################################################################
 """
-    tag!(d::Dict; gitpath = projectdir(), storepatch = true, force = false) -> d
+    tag!(d::AbstractDict; gitpath = projectdir(), storepatch = true, force = false) -> d
 Tag `d` by adding an extra field `gitcommit` which will have as value
 the [`gitdescribe`](@ref) of the repository at `gitpath` (by default
 the project's gitpath). Do nothing if a key `gitcommit` already exists
@@ -192,7 +192,7 @@ Dict{Symbol,Any} with 3 entries:
   :x => 3
 ```
 """
-function tag!(d::Dict{K,T}; gitpath = projectdir(), storepatch = true, force = false, source = nothing) where {K,T}
+function tag!(d::AbstractDict{K,T}; gitpath = projectdir(), storepatch = true, force = false, source = nothing) where {K,T}
     @assert (K <: Union{Symbol,String}) "We only know how to tag dictionaries that have keys that are strings or symbols"
     c = gitdescribe(gitpath)
     c === nothing && return d # gitpath is not a git repo
@@ -205,7 +205,7 @@ function tag!(d::Dict{K,T}; gitpath = projectdir(), storepatch = true, force = f
         @warn "The dictionary already has a key named `gitcommit`. We won't "*
         "add any Git information."
     else
-        d = checktagtype!(d) 
+        d = checktagtype!(d)
         d[commitname] = c
         # Only include patch info if `storepatch` is true and if we can get the info.
         if storepatch
@@ -225,37 +225,46 @@ function tag!(d::Dict{K,T}; gitpath = projectdir(), storepatch = true, force = f
 end
 
 """
-    keyname(d::Dict{K,T}, key) where {K<:Union{Symbol,String},T}
+    keyname(d::AbstractDict{K,T}, key) where {K<:Union{Symbol,String},T}
 
 Check the key type of `d` and convert `key` to the appropriate type.
 """
-function keyname(d::Dict{K,T}, key) where {K<:Union{Symbol,String},T}
+function keyname(d::AbstractDict{K,T}, key) where {K<:Union{Symbol,String},T}
     if K == Symbol
         return Symbol(key)
     end
     return String(key)
 end
 
 """
-    checktagtype!(d::Dict{K,T}) where {K<:Union{Symbol,String},T}
+    checktagtype!(d::AbstractDict{K,T}) where {K<:Union{Symbol,String},T}
 
 Check if the value type of `d` allows `String` and promote it to do so if not.
 """
-function checktagtype!(d::Dict{K,T}) where {K<:Union{Symbol,String},T}
+function checktagtype!(d::AbstractDict{K,T}) where {K<:Union{Symbol,String},T}
+    DT = get_rawtype(typeof(d)) #concrete type of dictionary
     if !(String <: T)
-        d = Dict{K, promote_type(T, String)}(d)
+        d = DT{K, promote_type(T, String)}(d)
     end
     d
 end
 
 """
-    scripttag!(d::Dict{K,T}, source::LineNumberNode; gitpath = projectdir(), force = false) where {K<:Union{Symbol,String},T}
+    get_rawtype(D::DataType) = getproperty(parentmodule(D), nameof(D))
+
+Return Concrete DataType from an `AbstractDict` `D`. Found online at:
+https://discourse.julialang.org/t/retrieve-the-type-of-abstractdict-without-parameters-from-a-concrete-dictionary-type/67567/3
+"""
+get_rawtype(D::DataType) = getproperty(parentmodule(D), nameof(D))
+
+"""
+    scripttag!(d::AbstractDict{K,T}, source::LineNumberNode; gitpath = projectdir(), force = false) where {K<:Union{Symbol,String},T}
 
 Include a `script` field in `d`, containing the source file and line number in
 `source`. Do nothing if the field is already present unless `force = true`. Uses
 `gitpath` to make the source file path relative.
 """
-function scripttag!(d::Dict{K,T}, source; gitpath = projectdir(), force = false) where {K,T}
+function scripttag!(d::AbstractDict{K,T}, source; gitpath = projectdir(), force = false) where {K,T}
     # We want this functionality to be separate from `tag!` to allow
     # inclusion of this information without the git tagging
     # functionality.
@@ -329,16 +338,18 @@ istaggable(x) = x isa AbstractDict
 
 
 """
-    struct2dict(s) -> d
+    struct2dict([type = Dict,] s) -> d
 Convert a Julia composite type `s` to a dictionary `d` with key type `Symbol`
-that maps each field of `s` to its value. This can be useful in e.g. saving:
+that maps each field of `s` to its value. Simply passing `s` will return a regular dictionary.
+This can be useful in e.g. saving:
 ```
 tagsave(savename(s), struct2dict(s))
 ```
 """
-function struct2dict(s)
-    Dict(x => getfield(s, x) for x in fieldnames(typeof(s)))
+function struct2dict(::Type{DT},s) where {DT<:AbstractDict}
+        DT(x => getfield(s, x) for x in fieldnames(typeof(s)))
 end
+struct2dict(s) = struct2dict(Dict,s)
 
 """
     struct2ntuple(s) -> n

test/stools_tests.jl

@@ -1,4 +1,6 @@
 using DrWatson, Test
+using DataStructures
+using JLD2
 
 # Test commit function
 com = gitdescribe(@__DIR__)
@@ -239,8 +241,8 @@ p = Dict(
 
 # Testing nested @onlyif calls
 @test Set(dict_list(Dict(
-                   :a=>[1,2], 
-                   :b => [3,4], 
+                   :a=>[1,2],
+                   :b => [3,4],
                    :c => @onlyif( :a == 2, [5, @onlyif(:b == 4, 6)])
                   ))) == Set([Dict(:a => 1,:b => 3),
                               Dict(:a => 2,:b => 3,:c => 5),
@@ -270,9 +272,59 @@ for r in ret
 end
 rm(tmpdir, force = true, recursive = true)
 @test !isdir(tmpdir)
-
-## is taggable 
+## is taggable
 @test DrWatson.istaggable("test.jld2")
 @test !DrWatson.istaggable("test.csv")
 @test !DrWatson.istaggable(0.5)
 @test DrWatson.istaggable(Dict(:a => 0.5))
+
+## Testing OrderedDict usage
+@testset "OrderedDict Tests" begin
+    cd(@__DIR__)
+    struct TestStruct
+        z::Float64
+        y::Int
+        x::String
+    end
+
+    struct TestStruct2 #this structure allows for the if statement to be run in checktagtype!, (will promote the valuetype to Any)
+        z::Int64
+        y::Int64
+        x::Int64
+    end
+
+    #test struct2dict
+    t = TestStruct(2.0,1,"3") #this tests the case where struct2dict will by default not work
+    d1 = struct2dict(t)
+    d2 = struct2dict(OrderedDict,t)
+    @test !all(collect(fieldnames(typeof(t))).==keys(d1)) #the example struct given does not have the keys in the same order when converted to a dict
+    @test all(collect(fieldnames(typeof(t))).==keys(d2)) #OrderedDict should have the key in the same order as the struct
+
+    #test struct2dict
+    t2 = TestStruct2(1,3,4)
+    d3 = struct2dict(t2)
+    d4 = struct2dict(OrderedDict,t2)
+    @test isa(d3,Dict)
+    @test isa(d4,OrderedDict)
+
+    #test tostringdict and tosymboldict
+    d10 = tostringdict(OrderedDict,d4)
+    @test isa(d10,OrderedDict)
+    d11 = tosymboldict(OrderedDict,d10)
+    @test isa(d11,OrderedDict)
+
+    #test checktagtype!
+    @test isa(DrWatson.checktagtype!(d3),Dict)
+    @test isa(DrWatson.checktagtype!(d11),OrderedDict)
+
+    #check tagsave
+    sn = savename(d10,"jld2")
+    tagsave(sn,d10,gitpath=findproject())
+
+    file = load(sn)
+    display(file)
+    @test "gitcommit" in keys(file)
+    @test file["gitcommit"] |>typeof ==String
+    rm(sn)
+
+end