Merge pull request #924 from JuliaRobotics/master

v0.18.9-rc1

Author: dehann

Project.toml

@@ -1,6 +1,6 @@
 name = "DistributedFactorGraphs"
 uuid = "b5cc3c7e-6572-11e9-2517-99fb8daf2f04"
-version = "0.18.8"
+version = "0.18.9"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
@@ -17,6 +17,7 @@ ManifoldsBase = "3362f125-f0bb-47a3-aa74-596ffd7ef2fb"
 Neo4j = "d2adbeaf-5838-5367-8a2f-e46d570981db"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
 RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
@@ -40,6 +41,7 @@ ManifoldsBase = "0.11, 0.12, 0.13"
 Neo4j = "2"
 OrderedCollections = "1.4"
 Pkg = "1.4, 1.5"
+ProgressMeter = "1"
 RecursiveArrayTools = "2"
 Reexport = "1"
 Requires = "1"

src/DataBlobs/services/AbstractDataEntries.jl

@@ -41,11 +41,25 @@ function getDataEntry(var::AbstractDFGVariable, blobId::UUID)
             return v
         end
     end
-    error("No dataEntry with blobId $(blobId) found in variable $(getLabel(var))")
+    throw(
+        KeyError("No dataEntry with blobId $(blobId) found in variable $(getLabel(var))")
+    )
 end
+function getDataEntry(var::AbstractDFGVariable, key::Regex)
+    for (k,v) in var.dataDict
+        if occursin(key, string(v.label))
+            return v
+        end
+    end
+    throw(
+        KeyError("No dataEntry with label matching regex $(key) found in variable $(getLabel(var))")
+    )
+end
+getDataEntry(var::AbstractDFGVariable, key::AbstractString) = getDataEntry(var,Regex(key))
+
 
-getDataEntry(dfg::AbstractDFG, label::Symbol, key::UUID) = getDataEntry(getVariable(dfg, label), key)
-getDataEntry(dfg::AbstractDFG, label::Symbol, key::Symbol) = getDataEntry(getVariable(dfg, label), key)
+getDataEntry(dfg::AbstractDFG, label::Symbol, key::Union{Symbol, UUID, <:AbstractString, Regex}) = getDataEntry(getVariable(dfg, label), key)
+# getDataEntry(dfg::AbstractDFG, label::Symbol, key::Symbol) = getDataEntry(getVariable(dfg, label), key)
 
 
 """
@@ -196,8 +210,11 @@ function mergeDataEntries!(
     des = _makevec(des_)
     # don't add data entries that already exist 
     dde = listDataEntries(dst, dlbl)
-    uids = (s->s.id).(dde)
-    filter!(s -> !(s.id in uids), des)
+        # HACK, verb list should just return vector of Symbol. NCE36
+        _getid(s) = s
+        _getid(s::AbstractDataEntry) = s.id
+    uids = _getid.(dde) # (s->s.id).(dde)
+    filter!(s -> !(_getid(s) in uids), des)
     # add any data entries not already in the destination variable, by uuid
     addDataEntry!.(dst, dlbl, des)
 end
@@ -212,9 +229,67 @@ function mergeDataEntries!(
     des = listDataEntries(src, slbl)
     # don't add data entries that already exist 
     dde = listDataEntries(dst, dlbl)
-    uids = (s->s.id).(dde)
-    filter!(s -> !(s.id in uids), des)
+        # HACK, verb list should just return vector of Symbol. NCE36
+        _getid(s) = s
+        _getid(s::AbstractDataEntry) = s.id    
+    uids = _getid.(dde) # (s->s.id).(dde)
+    filter!(s -> !(_getid(s) in uids), des)
     if 0  < length(des)
         union(((s->mergeDataEntries!(dst, dlbl, src, slbl, s.id)).(des))...)
     end
-end
+end
+
+function mergeDataEntries!(
+    dest::AbstractDFG,
+    src::AbstractDFG,
+    w...;
+    varList::AbstractVector = listVariables(dest) |> sortDFG
+)
+    @showprogress 1 "merging data entries" for vl in varList
+        mergeDataEntries!(dest, vl, src, vl, w...)
+    end
+    varList
+end
+
+"""
+    $SIGNATURES
+
+If the blob label `datalabel` already exists, then this function will return the name `datalabel_1`.
+If the blob label `datalabel_1` already exists, then this function will return the name `datalabel_2`.
+"""
+function incrDataLabelSuffix(
+    dfg::AbstractDFG,
+    vla,
+    bllb::S; 
+    datalabel=Ref("")
+) where {S <: Union{Symbol, <:AbstractString}}
+    count = 1
+    hasund = false
+    len = 0
+    try
+        de,_ = getData(dfg, Symbol(vla), bllb)
+        bllb = string(bllb)
+        # bllb *= bllb[end] != '_' ? "_" : ""
+        datalabel[] = string(de.label)
+        dlb = match(r"\d*", reverse(datalabel[]))
+        # slightly complicated search if blob name already has an underscore number suffix, e.g. `_4`
+        count, hasund, len = if occursin(Regex(dlb.match*"_"), reverse(datalabel[]))
+            parse(Int, dlb.match |> reverse)+1, true, length(dlb.match)
+        else
+            1, datalabel[][end] == '_', 0
+        end
+    catch err
+        # append latest count
+        if !(err isa KeyError)
+        throw(err)
+        end
+    end
+    # the piece from old label without the suffix count number
+    bllb = datalabel[][1:(end-len)]
+    if !hasund || bllb[end] != '_'
+        bllb *= "_"
+    end
+    bllb *= string(count)
+
+    S(bllb)
+end

src/DataBlobs/services/BlobStores.jl

@@ -5,7 +5,6 @@
 function getDataBlob(dfg::AbstractDFG, entry::BlobStoreEntry)
     # cannot use entry.blobstore because the blob can be in any one of the blobstores
     stores = getBlobStores(dfg)
-    blob = UInt8[]
     for (k,store) in stores
         try
             blob = getDataBlob(store, entry)
@@ -18,7 +17,7 @@ function getDataBlob(dfg::AbstractDFG, entry::BlobStoreEntry)
     end
     throw(
         KeyError(
-            "could not find $(entry.label), uuid $(entry.id)) in any of the listed blobstores:\n $((s->getKey(s)).(stores))"
+            "could not find $(entry.label), uuid $(entry.id)) in any of the listed blobstores:\n $([s->getKey(s) for (s,v) in stores]))"
         )
     )
 end
@@ -85,7 +84,7 @@ function getData(
     dfg::AbstractDFG, 
     blobstore::AbstractBlobStore, 
     label::Symbol, 
-    key::Union{Symbol,UUID}; 
+    key::Union{Symbol,UUID, <:AbstractString, Regex}; 
     hashfunction = sha256,
     checkhash::Bool=true
 )

src/DataBlobs/services/DataEntryBlob.jl

@@ -114,7 +114,7 @@ end
 function getData(
     dfg::AbstractDFG, 
     vlabel::Symbol, 
-    key::Union{Symbol,UUID}; 
+    key::Union{Symbol,UUID, <:AbstractString, Regex}; 
     hashfunction = sha256,
     checkhash::Bool=true
 )

src/DistributedFactorGraphs.jl

@@ -30,6 +30,7 @@ using SparseArrays
 using UUIDs
 using Pkg
 using TensorCast
+using ProgressMeter
 
 # used for @defVariable
 import ManifoldsBase
@@ -192,12 +193,14 @@ export getSolvedCount, isSolved, setSolvedCount!, isInitialized, isMarginalized,
 export getNeighborhood, getNeighbors, _getDuplicatedEmptyDFG
 export findFactorsBetweenNaive
 export copyGraph!, deepcopyGraph, deepcopyGraph!, buildSubgraph, mergeGraph!
-# Big Data
+
+# Entry Blob Data
 ##------------------------------------------------------------------------------
 export addDataEntry!, getDataEntry, updateDataEntry!, deleteDataEntry!, getDataEntries, listDataEntries, hasDataEntry, hasDataEntry
 export listDataEntrySequence
 # convenience wrappers
 export addDataEntry!, mergeDataEntries!
+export incrDataLabelSuffix
 # aliases
 export addData!
 

src/services/AbstractDFG.jl

@@ -950,7 +950,7 @@ function copyGraph!(destDFG::AbstractDFG,
                     copyGraphMetadata::Bool=false,
                     overwriteDest::Bool=false,
                     deepcopyNodes::Bool=false,
-                    verbose::Bool = true)
+                    verbose::Bool = false)
     # Split into variables and factors
     sourceVariables = map(vId->getVariable(sourceDFG, vId), variableLabels)
     sourceFactors = map(fId->getFactor(sourceDFG, fId), factorLabels)

src/services/Serialization.jl

@@ -25,7 +25,7 @@ end
 function _versionCheck(props::Dict{String, Any})
     if haskey(props, "_version")
         if props["_version"] != _getDFGVersion()
-            @warn "This data was serialized using DFG $(props["_version"]) but you have $(_getDFGVersion()) installed, there may be deserialization issues."
+            @warn "This data was serialized using DFG $(props["_version"]) but you have $(_getDFGVersion()) installed, there may be deserialization issues." maxlog=10
         end
     else
         @warn "There isn't a version tag in this data so it's older than v0.10, there may be deserialization issues."
@@ -442,9 +442,13 @@ end
 
 
 # Returns `::DFGFactor`
-function unpackFactor(dfg::G, packedProps::Dict{String, Any}) where G <: AbstractDFG
+function unpackFactor(
+    dfg::G, 
+    packedProps::Dict{String, Any};
+    skipVersionCheck::Bool=false
+) where G <: AbstractDFG
     # Version checking.
-    _versionCheck(packedProps)
+    !skipVersionCheck && _versionCheck(packedProps)
 
     label = packedProps["label"]
     # Make sure that the timestamp is correctly formatted with subseconds

test/consol_DataEntryBlobTests.jl

@@ -92,8 +92,18 @@ ds = FolderStore{Vector{UInt8}}(:filestore, "/tmp/dfgFolderStore")
 addBlobStore!(dfg, ds)
 
 ade,adb = addData!(dfg, :filestore, :x1, :random, dataset1)
+_,_     = addData!(dfg, :filestore, :x1, :another_1, dataset1)
+_,_ = getData(dfg, :x1, "random")
+_,_ = getData(dfg, :x1, r"rando")
 gde,gdb = getData(dfg, :x1, :random)
+
+@test incrDataLabelSuffix(dfg,:x1,:random) == :random_1
+@test incrDataLabelSuffix(dfg,:x1,:another_1) == :another_2
+# @test incrDataLabelSuffix(dfg,:x1,:another) == :another_2 # TODO exand support for Regex likely search on labels
+# @test incrDataLabelSuffix(dfg,:x1,"random") == "random_1" # TODO expand support for label::String
+
 dde,ddb = deleteData!(dfg, :x1, :random)
+_,_     = deleteData!(dfg, :x1, :another_1)
 
 @test ade == gde == dde
 @test adb == gdb == ddb

test/testBlocks.jl

@@ -1437,6 +1437,10 @@ function ProducingDotFiles(testDFGAPI,
 
     addVariable!(dotdfg, v1)
     addVariable!(dotdfg, v2)
+    # FIXME, fix deprecation
+    # ┌ Warning: addFactor!(dfg, variables, factor) is deprecated, use addFactor!(dfg, factor)
+    # │   caller = ProducingDotFiles(testDFGAPI::Type{GraphsDFG}, v1::Nothing, v2::Nothing, f1::Nothing; VARTYPE::Type{DFGVariable}, FACTYPE::Type{DFGFactor}) at testBlocks.jl:1440
+    # └ @ Main ~/.julia/dev/DistributedFactorGraphs/test/testBlocks.jl:1440
     addFactor!(dotdfg, [v1, v2], f1)
     #NOTE hardcoded toDot will have different results so test LightGraphs seperately
     if testDFGAPI <: LightDFG || testDFGAPI <: GraphsDFG || testDFGAPI <: Neo4jDFG
@@ -1490,7 +1494,8 @@ function CopyFunctionsTest(testDFGAPI; kwargs...)
     @test issetequal(lsf(dcdfg_part), flbls)
 
     # deepcopy subgraph ignoring orphans
-    @test_logs (:warn, r"orphan") dcdfg_part = deepcopyGraph(LightDFG, dfg, vlbls, union(flbls, [:x1x2f1]))
+    # @test_logs (:warn, r"orphan") # orphan warning has been suppressed given overwhelming printouts
+    dcdfg_part = deepcopyGraph(LightDFG, dfg, vlbls, union(flbls, [:x1x2f1]))
     @test issetequal(ls(dcdfg_part), vlbls)
     @test issetequal(lsf(dcdfg_part), flbls)