Add filtering file names when collecting results (#312)

* Added filtering functionality

* Add filtering tests

* Tests passed

* Add changelog entry

* Increment version in Project.toml

* Add example, assert rgx expr

* Fix versioning, bump patch version

* Fix silly spelling mistake

Author: junglegobs

CHANGELOG.md

@@ -1,3 +1,5 @@
+# 2.8.0
+* Add filtering of `collect_results` using `rinclude` and `rexclude` keyword arguments.
 # 2.7.2
 * By default `storepatch` keywords are `false`. This means that `gitpatch` is NOT stored by default. This is a BUGFIX, because there is an unknown problem of non-halting when storing the patch.
 # 2.7.0

Project.toml

@@ -1,7 +1,7 @@
 name = "DrWatson"
 uuid = "634d3b9d-ee7a-5ddf-bec9-22491ea816e1"
 repo = "https://github.com/JuliaDynamics/DrWatson.jl.git"
-version = "2.7.6"
+version = "2.8.0"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"

docs/src/real_world.md

@@ -433,7 +433,22 @@ As `@onlyif` is meant to be used with [`dict_list`](@ref), it supports the vecto
 This is achieved by automatically broadcasting every `@onlyif` call over `Vector` arguments, which allows chaining those calls to combine conditions.
 So in terms of the result, `@onlyif( :a == 2, [5, @onlyif(:b == 4, 6)])` is equivalent to `[@onlyif( :a == 2, 5), @onlyif(:a == 2 && :b == 4, 6)]`.
 
-## Advanced Usage of collect_results
+## Filtering by name with collect_results
+
+Using [`collect_results`](@ref) on a folder with many (e.g. 1,000) files in it can be noticeably slow. To speed this up, you can use the `rinclude` and `rexclude` keyword arguments, both of which are vectors of [Regex expressions](https://docs.julialang.org/en/v1/manual/strings/#man-regex-literals). The results returned will have a filename which matches **any** of the Regex expressions in `rinclude` and does not match **any** of the Regex expressions in `rexclude`.
+
+```julia
+df = collect_results(datadir("results"); rinclude=[r"a=1"])
+# Only include results whose filename contains "a=1"
+
+df = collect_results(datadir("results"); rexclude=[r"a=3"])
+# Exclude any results whose filename contains "a=3"
+
+df = collect_results(datadir("results"); rinclude=[r"a=1", r"b=5"], rexclude=[r"a=3"])
+# Only include results whose filename contains "a=1" OR "b=5" and exclude any which contain "a=3"
+```
+
+## Advanced usage of collect_results
 At some point in your work you may want to run a single function
 that returns multiple fields that you want to include in your
 results `DataFrame`.

src/result_collection.jl

@@ -41,6 +41,8 @@ See also [`collect_results`](@ref).
 * `verbose = true` : Print (using `@info`) information about the process.
 * `update = false` : Update data from modified files and remove entries for deleted
   files.
+* `rinclude = [r\"\"]` : Only include files whose name matches any of these Regex expressions. Default value includes all files.
+* `rexclude = [r\"^\\b\$\"]` : Exclude any files whose name matches any of these Regex expressions. Default value does not exclude any files.
 * `white_list` : List of keys to use from result file. By default
   uses all keys from all loaded result-files.
 * `black_list = [:gitcommit, :gitpatch, :script]`: List of keys not to include from result-file.
@@ -84,8 +86,12 @@ function collect_results!(filename, folder;
     verbose = true,
     update = false,
     newfile = false, # keyword only for defining collect_results without !
+    rinclude = [r""],
+    rexclude = [r"^\b$"],
     kwargs...)
 
+    @assert all(eltype(r) <: Regex for r in (rinclude, rexclude)) "Elements of `rinclude` and `rexclude` must be Regex expressions."
+
     if newfile || !isfile(filename)
         !newfile && verbose && @info "Starting a new result collection..."
         df = DataFrames.DataFrame()
@@ -116,6 +122,19 @@ function collect_results!(filename, folder;
     else
         allfiles = joinpath.(Ref(folder), readdir(folder))
     end
+    
+    if (rinclude == [r""] && rexclude == [r"^\b$"]) == false
+        idx_filt = Int[]
+        for i in eachindex(allfiles)
+            file = allfiles[i]
+            include_bool = any(match(rgx, file) !== nothing for rgx in rinclude)
+            exclude_bool = any(match(rgx, file) !== nothing for rgx in rexclude)
+            if include_bool == false || exclude_bool == true
+                push!(idx_filt, i)
+            end 
+        end
+        deleteat!(allfiles, idx_filt)
+    end
 
     n = 0 # new entries added
     u = 0 # entries updated

test/update_results_tests.jl

@@ -64,6 +64,21 @@ cres_relpath = collect_results!(relpathname, folder;
     rpath = projectdir())
 @info all(startswith.(cres[!,"path"], "data"))
 
+###############################################################################
+#                           Include or exclude files                          #
+###############################################################################
+
+@test_throws AssertionError collect_results(datadir("results"); rinclude=["a=1"])
+
+df = collect_results(datadir("results"); rinclude=[r"a=1", r"b=3"])
+@test all(row -> row["a"] == 1 || row["b"] == "2", eachrow(df))
+
+df = collect_results(datadir("results"); rexclude=[r"a=3"])
+@test all(df[:,"a"] .!== 3)
+
+df = collect_results(datadir("results"); rinclude=[r"a=3"], rexclude=[r"a=3"])
+@test isempty(df)
+
 ###############################################################################
 #                           Add another file in a sub sub folder              #
 ###############################################################################