From 19b6210d8083677608c211031b862202949f385b Mon Sep 17 00:00:00 2001 From: Johannes Terblanche Date: Wed, 9 Jul 2025 11:05:27 +0200 Subject: [PATCH 1/5] More DFGv1 checked and Blobstore updates --- src/DataBlobs/entities/BlobStores.jl | 30 +++++++++++ src/DataBlobs/services/BlobEntry.jl | 4 +- src/DataBlobs/services/BlobStores.jl | 74 +++++++++++++--------------- src/DistributedFactorGraphs.jl | 19 ++++++- src/GraphsDFG/GraphsDFG.jl | 3 +- src/GraphsDFG/services/GraphsDFG.jl | 25 +++------- src/errors.jl | 38 ++++++++++++++ src/services/AbstractDFG.jl | 63 +++++++++++++++++------ test/testBlocks.jl | 4 ++ 9 files changed, 183 insertions(+), 77 deletions(-) diff --git a/src/DataBlobs/entities/BlobStores.jl b/src/DataBlobs/entities/BlobStores.jl index 43622e9b..5d20c358 100644 --- a/src/DataBlobs/entities/BlobStores.jl +++ b/src/DataBlobs/entities/BlobStores.jl @@ -1 +1,31 @@ +""" + AbstractBlobstore{T} + +Abstract supertype for all blobstore implementations. + +# Usage + +Subtypes of `AbstractBlobstore{T}` must implement the required interface for blob storage and retrieval, such as: + +- `add!(store, blobId, blob)`: Add a new blob to the store. +- `get(store, blobId)`: Retrieve a blob by its ID. +- `list(store)`: List all blob IDs in the store. + +The parameter `T` represents the type of blobs stored (e.g., `Vector{UInt8}` or a custom `Blob` type). + +See concrete implementations for details. + +Design Notes +- `blobId` is not considered unique across blobstores with different labels only within a single blobstore. +- We cannot guarantee that `blobId` is unique across different blobstores with the same label and this is up to the end user. +- Within a single blobstore `addBlob!` will fail if there is a UUID collision. +- TODO: We should consider using uuid7 for `blobId`s (requires jl v1.12). +- `Blobstrores`are identified by a `label::Symbol`, which allows for multiple blobstores to coexist in the same system. + +TODO: If we want to make the `blobId`=>Blob pair immutable: +- We can use the tombstone pattern to mark a blob as deleted. See FolderStore in PR#TODO. + +Design goal: all `Blobstore`s with the same `label` can contain the same `blobId`=>`Blob` pair and the blobs should be identical since they are immutable. + +""" abstract type AbstractBlobstore{T} end diff --git a/src/DataBlobs/services/BlobEntry.jl b/src/DataBlobs/services/BlobEntry.jl index 28f717e2..6874cb47 100644 --- a/src/DataBlobs/services/BlobEntry.jl +++ b/src/DataBlobs/services/BlobEntry.jl @@ -168,7 +168,7 @@ function addBlobentry!(var::VariableDFG, entry::Blobentry) return entry end -function addBlobentry!(dfg::AbstractDFG, vLbl::Symbol, entry::Blobentry;) +function addBlobentry!(dfg::AbstractDFG, vLbl::Symbol, entry::Blobentry) return addBlobentry!(getVariable(dfg, vLbl), entry) end @@ -238,7 +238,7 @@ end """ $SIGNATURES -Does a blob entry (element) exist with `blobLabel`. +Does a blob entry exist with `blobLabel`. """ hasBlobentry(var::AbstractDFGVariable, blobLabel::Symbol) = haskey(var.dataDict, blobLabel) diff --git a/src/DataBlobs/services/BlobStores.jl b/src/DataBlobs/services/BlobStores.jl index 436cd645..42eee96f 100644 --- a/src/DataBlobs/services/BlobStores.jl +++ b/src/DataBlobs/services/BlobStores.jl @@ -13,7 +13,7 @@ $(METHODLIST) function getBlob end """ -Adds a blob to the blob store or dfg with the given entry. +Adds a blob to the blob store or dfg with the blobId. Related [`addBlobentry!`](@ref) @@ -81,34 +81,11 @@ end ##============================================================================== ## AbstractBlobstore derived CRUD for Blob ##============================================================================== -#TODO looking in all the blobstores does not make sense since since there is a chance that the blobId is not unique across blobstores. -# using the cached blobstore is the right way to go here. +#TODO maybe we should generalize and move the cached blobstore to DFG. function getBlob(dfg::AbstractDFG, entry::Blobentry) - stores = getBlobstores(dfg) - storekeys = collect(keys(stores)) - # first check the saved blobstore and then fall back to the rest - fidx = findfirst(==(entry.blobstore), storekeys) - if !isnothing(fidx) - skey = storekeys[fidx] - popat!(storekeys, fidx) - pushfirst!(storekeys, skey) - end - for k in storekeys - store = stores[k] - try - blob = getBlob(store, entry) - return blob - catch err - if !(err isa KeyError) - throw(err) - end - end - end - throw( - KeyError( - "could not find $(entry.label), uuid $(entry.blobId) in any of the listed blobstores:\n $([s->getLabel(s) for (s,v) in stores]))", - ), - ) + storeLabel = entry.blobstore + store = getBlobstore(dfg, storeLabel) + return getBlob(store, entry.blobId) end function getBlob(store::AbstractBlobstore, entry::Blobentry) @@ -204,19 +181,18 @@ function getBlob(store::FolderStore{T}, blobId::UUID) where {T} return read(f) end else - throw(KeyError("Could not find file '$(blobfilename)'.")) + throw(IdNotFoundError("Blob", blobId)) end end function addBlob!(store::FolderStore{T}, blobId::UUID, data::T) where {T} blobfilename = joinpath(store.folder, string(store.label), string(blobId)) if isfile(blobfilename) - throw(KeyError("Key '$blobId' blob already exists.")) + throw(IdExistsError("Blob", blobId)) else open(blobfilename, "w") do f return write(f, data) end - # return data return blobId end end @@ -235,11 +211,13 @@ end function deleteBlob!(store::FolderStore{T}, blobId::UUID) where {T} blobfilename = joinpath(store.folder, string(store.label), string(blobId)) + if !isfile(blobfilename) + throw(IdNotFoundError("Blob", blobId)) + end rm(blobfilename) return 1 end -#hasBlob or existsBlob? function hasBlob(store::FolderStore, blobId::UUID) blobfilename = joinpath(store.folder, string(store.label), string(blobId)) return isfile(blobfilename) @@ -265,12 +243,15 @@ function InMemoryBlobstore(storeKey::Symbol = :default_inmemory_store) end function getBlob(store::InMemoryBlobstore, blobId::UUID) + if !haskey(store.blobs, blobId) + throw(IdNotFoundError("Blob", blobId)) + end return store.blobs[blobId] end function addBlob!(store::InMemoryBlobstore{T}, blobId::UUID, data::T) where {T} if haskey(store.blobs, blobId) - error("Key '$blobId' blob already exists.") + throw(IdExistsError("Blob", blobId)) end store.blobs[blobId] = data return blobId @@ -284,6 +265,9 @@ function updateBlob!(store::InMemoryBlobstore{T}, blobId::UUID, data::T) where { end function deleteBlob!(store::InMemoryBlobstore, blobId::UUID) + if !haskey(store.blobs, blobId) + throw(IdNotFoundError("Blob", blobId)) + end pop!(store.blobs, blobId) return 1 end @@ -319,25 +303,28 @@ end function getBlob(store::LinkStore, blobId::UUID) fname = get(store.cache, blobId, nothing) + if isnothing(fname) + throw(IdNotFoundError("Blob", blobId)) + end return read(fname) end function addBlob!(store::LinkStore, entry::Blobentry, linkfile::String) - return addBlob!(store, entry.blobId, nothing, linkfile::String) + return addBlob!(store, entry.blobId, linkfile) end -function addBlob!(store::LinkStore, blobId::UUID, blob::Any, linkfile::String) +function addBlob!(store::LinkStore, blobId::UUID, linkfile::String) if haskey(store.cache, blobId) - error("blobId $blobId already exists in the store") + throw(IdExistsError("Blob", blobId)) end push!(store.cache, blobId => linkfile) open(store.csvfile, "a") do f return println(f, blobId, ",", linkfile) end - return getBlob(store, blobId) + return blobId end -function deleteBlob!(store::LinkStore, args...) +function deleteBlob!(store::LinkStore) return error("deleteDataBlob(::LinkStore) not supported") end @@ -404,12 +391,15 @@ Tables.rows(store::RowBlobstore) = values(store.blobs) ## function getBlob(store::RowBlobstore, blobId::UUID) + if !haskey(store.blobs, blobId) + throw(IdNotFoundError("Blob", blobId)) + end return getfield(store.blobs[blobId], :blob) end function addBlob!(store::RowBlobstore{T}, blobId::UUID, blob::T) where {T} if haskey(store.blobs, blobId) - error("Key '$blobId' blob already exists.") + throw(IdExistsError("Blob", blobId)) end store.blobs[blobId] = RowBlob(blobId, blob) return blobId @@ -423,7 +413,10 @@ function updateBlob!(store::RowBlobstore{T}, blobId::UUID, blob::T) where {T} end function deleteBlob!(store::RowBlobstore, blobId::UUID) - getfield(pop!(store.blobs, blobId), :blob) + if !haskey(store.blobs, blobId) + throw(IdNotFoundError("Blob", blobId)) + end + pop!(store.blobs, blobId) return 1 end @@ -469,3 +462,4 @@ if false Tables.rowtable(sstore) end ## +## diff --git a/src/DistributedFactorGraphs.jl b/src/DistributedFactorGraphs.jl index f23c9770..e8d947fc 100644 --- a/src/DistributedFactorGraphs.jl +++ b/src/DistributedFactorGraphs.jl @@ -60,11 +60,28 @@ using InteractiveUtils: subtypes # v1 name, signiture, return, and error checked export getFactor, getBlobentry, getGraphBlobentry +export addVariable!, addFactor!, addBlobentry! + +export deleteVariable! + # v1 name, signiture, and return # v1 name only export getVariable, getBlob, addBlob! +export hasVariable, hasFactor + +# getBlob TODO do we want all of them easy portable vs convenience? +# getBlob(::AbstractBlobstore, ::UUID) +# getBlob(::AbstractBlobstore, ::Blobentry) +# getBlob(::AbstractDFG, ::Blobentry) + +# TODO get,add,delete|Blob still needs immutability discussion. but errors checked, tests needs updating though. + +# TODO not yet implemented in DFG +# addAgentBlobentry! +# addGraphBlobentry! + ## const DFG = DistributedFactorGraphs export DFG @@ -121,9 +138,7 @@ export InMemoryDFGTypes, LocalDFG # AbstractDFG Interface export exists, - addVariable!, addVariables!, - addFactor!, addFactors!, mergeVariable!, mergeFactor!, diff --git a/src/GraphsDFG/GraphsDFG.jl b/src/GraphsDFG/GraphsDFG.jl index 222930f3..0898a5f7 100644 --- a/src/GraphsDFG/GraphsDFG.jl +++ b/src/GraphsDFG/GraphsDFG.jl @@ -24,7 +24,8 @@ import ...DistributedFactorGraphs: getAddHistory, addFactor!, getSolverParams, - exists, + hasVariable, + hasFactor, isVariable, isFactor, mergeVariable!, diff --git a/src/GraphsDFG/services/GraphsDFG.jl b/src/GraphsDFG/services/GraphsDFG.jl index 11afe3f3..5a4952a0 100644 --- a/src/GraphsDFG/services/GraphsDFG.jl +++ b/src/GraphsDFG/services/GraphsDFG.jl @@ -6,24 +6,14 @@ function getDFGMetadata(fg::GraphsDFG) return metaprops end -function exists( - dfg::GraphsDFG{P, V, F}, - node::V, -) where {P <: AbstractParams, V <: AbstractDFGVariable, F <: AbstractDFGFactor} - return haskey(dfg.g.variables, node.label) +function hasVariable(dfg::GraphsDFG, label::Symbol) + return haskey(dfg.g.variables, label) end -function exists( - dfg::GraphsDFG{P, V, F}, - node::F, -) where {P <: AbstractParams, V <: AbstractDFGVariable, F <: AbstractDFGFactor} - return haskey(dfg.g.factors, node.label) +function hasFactor(dfg::GraphsDFG, label::Symbol) + return haskey(dfg.g.factors, label) end -exists(dfg::GraphsDFG, nId::Symbol) = haskey(dfg.g.labels, nId) - -exists(dfg::GraphsDFG, node::DFGNode) = exists(dfg, node.label) - function isVariable( dfg::GraphsDFG{P, V, F}, sym::Symbol, @@ -109,7 +99,7 @@ function addFactor!( # @assert FactorGraphs.addFactor!(dfg.g, getVariableOrder(factor), factor) variableLabels = Symbol[factor._variableOrderSymbols...] for vlabel in variableLabels - !exists(dfg, vlabel) && throw(LabelNotFoundError("Variable", vlabel)) + !hasVariable(dfg, vlabel) && throw(LabelNotFoundError("Variable", vlabel)) end @assert FactorGraphs.addFactor!(dfg.g, variableLabels, factor) return factor @@ -290,7 +280,7 @@ function listNeighbors(dfg::GraphsDFG, node::DFGNode; solvable::Int = 0) end function listNeighbors(dfg::GraphsDFG, label::Symbol; solvable::Int = 0) - if !exists(dfg, label) + if !(hasVariable(dfg, label) || hasFactor(dfg, label)) throw(LabelNotFoundError(label)) end @@ -468,7 +458,8 @@ function findShortestPathDijkstra( dfg end - if !exists(dfg_, from) || !exists(dfg_, to) + if !(hasVariable(dfg_, from) || hasFactor(dfg_, from)) || + !(hasVariable(dfg_, to) || hasFactor(dfg_, to)) # assume filters excluded either `to` or `from` and hence no shortest path return Symbol[] end diff --git a/src/errors.jl b/src/errors.jl index 4b1d7fe0..a8993693 100644 --- a/src/errors.jl +++ b/src/errors.jl @@ -43,6 +43,44 @@ function Base.showerror(io::IO, ex::LabelExistsError) ) end +""" + IdNotFoundError(Id, available) + +Error thrown when a requested Id is not found. +""" +struct IdNotFoundError <: Exception + name::String + Id::UUID + available::Vector{UUID} +end + +IdNotFoundError(name::String, Id::UUID) = IdNotFoundError(name, Id, UUID[]) +IdNotFoundError(Id::UUID) = IdNotFoundError("Node", Id, UUID[]) + +function Base.showerror(io::IO, ex::IdNotFoundError) + print(io, "IdNotFoundError: ", ex.name, " Id '", ex.Id, "' not found.") + if !isempty(ex.available) + println(io, " Available Ids:") + show(io, ex.available) + end +end + +""" + IdExistsError(Id) + +Error thrown when attempting to add an Id that already exists in the collection. +""" +struct IdExistsError <: Exception + name::String + Id::UUID +end + +IdExistsError(Id::UUID) = IdExistsError("Node", Id) + +function Base.showerror(io::IO, ex::IdExistsError) + return print(io, "IdExistsError: ", ex.name, " Id '", ex.Id, "' already exists.") +end + """ SerializationError(msg) diff --git a/src/services/AbstractDFG.jl b/src/services/AbstractDFG.jl index 9c37b606..8ae8f9f6 100644 --- a/src/services/AbstractDFG.jl +++ b/src/services/AbstractDFG.jl @@ -247,7 +247,17 @@ function listModelBlobentries end # AbstractBlobstore should have label or overwrite getLabel getBlobstores(dfg::AbstractDFG) = dfg.blobStores -getBlobstore(dfg::AbstractDFG, key::Symbol) = dfg.blobStores[key] + +function getBlobstore(dfg::AbstractDFG, storeLabel::Symbol) + store = get(dfg.blobStores, storeLabel, nothing) + if isnothing(store) + throw( + LabelNotFoundError("Blobstore", storeLabel, collect(keys(getBlobstores(dfg)))), + ) + end + return store +end + function addBlobstore!(dfg::AbstractDFG, bs::AbstractBlobstore) return push!(dfg.blobStores, getLabel(bs) => bs) end @@ -267,26 +277,28 @@ listBlobstores(dfg::AbstractDFG) = collect(keys(dfg.blobStores)) ##------------------------------------------------------------------------------ ## Variable And Factor CRUD ##------------------------------------------------------------------------------ + """ $(SIGNATURES) -True if the variable or factor exists in the graph. +True if the variable exists in the graph. """ -function exists(dfg::AbstractDFG, node::DFGNode) - return error("exists not implemented for $(typeof(dfg))") +function hasVariable(dfg::AbstractDFG, label::Symbol) + return error("hasVariable not implemented for $(typeof(dfg))") end -function exists(dfg::AbstractDFG, label::Symbol) - return error("exists not implemented for $(typeof(dfg))") +""" + $(SIGNATURES) +True if the factor exists in the graph. +""" +function hasFactor(dfg::AbstractDFG, label::Symbol) + return error("hasFactor not implemented for $(typeof(dfg))") end """ $(SIGNATURES) Add a VariableCompute to a DFG. """ -function addVariable!( - dfg::G, - variable::V, -) where {G <: AbstractDFG, V <: AbstractDFGVariable} +function addVariable!(dfg::AbstractDFG, ::AbstractDFGVariable) return error("addVariable! not implemented for $(typeof(dfg))") end @@ -294,7 +306,7 @@ end $(SIGNATURES) Add a Vector{VariableCompute} to a DFG. """ -function addVariables!(dfg::AbstractDFG, variables::Vector{<:AbstractDFGVariable}) +function addVariables!(dfg::AbstractDFG, ::Vector{<:AbstractDFGVariable}) return asyncmap(variables) do v return addVariable!(dfg, v) end @@ -304,7 +316,7 @@ end $(SIGNATURES) Add a FactorCompute to a DFG. """ -function addFactor!(dfg::AbstractDFG, factor::F) where {F <: AbstractDFGFactor} +function addFactor!(dfg::AbstractDFG, ::AbstractDFGFactor) return error("addFactor! not implemented for $(typeof(dfg))(dfg, factor)") end @@ -513,6 +525,8 @@ Get a VariableCompute with a specific solver key. In memory types still return a reference, other types returns a variable with only solveKey. """ function getVariable(dfg::AbstractDFG, label::Symbol, solveKey::Symbol) + # TODO maybe change solveKey param to stateLabelFilter + # function getVariable(dfg::AbstractDFG, label::Symbol; stateLabelFilter::Union{Nothing, ...} = nothing) var = getVariable(dfg, label) if isa(var, VariableCompute) && !haskey(var.solverDataDict, solveKey) @@ -1038,6 +1052,25 @@ function findVariableNearTimestamp( ) end +##============================================================================== +## exists - alias for hasVariable || hasFactor +##============================================================================== +# exists alone is ambiguous and only for variables and factors where there rest of the nouns use has, +# TODO therefore, keep as internal or deprecate? +# additionally - variables and factors can possibly have the same label in other drivers such as NvaSDK + +""" + $(SIGNATURES) +True if a variable or factor with `label` exists in the graph. +""" +function exists(dfg::AbstractDFG, label::Symbol) + return hasVariable(dfg, label) || hasFactor(dfg, label) +end + +function exists(dfg::AbstractDFG, node::DFGNode) + return exists(dfg, node.label) +end + ##============================================================================== ## Copy Functions ##============================================================================== @@ -1074,7 +1107,7 @@ function copyGraph!( @showprogress desc = "copy variables" enabled = showprogress for variable in sourceVariables variableCopy = deepcopyNodes ? deepcopy(variable) : variable - if !exists(destDFG, variable) + if !hasVariable(destDFG, variable.label) addVariable!(destDFG, variableCopy) elseif overwriteDest mergeVariable!(destDFG, variableCopy) @@ -1089,14 +1122,14 @@ function copyGraph!( # Find the labels and associated variables in our new subgraph factVariableIds = Symbol[] for variable in sourceFactorVariableIds - if exists(destDFG, variable) + if hasVariable(destDFG, variable) push!(factVariableIds, variable) end end # Only if we have all of them should we add it (otherwise strange things may happen on evaluation) if length(factVariableIds) == length(sourceFactorVariableIds) factorCopy = deepcopyNodes ? deepcopy(factor) : factor - if !exists(destDFG, factor) + if !hasFactor(destDFG, factor.label) addFactor!(destDFG, factorCopy) elseif overwriteDest mergeFactor!(destDFG, factorCopy) diff --git a/test/testBlocks.jl b/test/testBlocks.jl index ba484c80..ef262e4a 100644 --- a/test/testBlocks.jl +++ b/test/testBlocks.jl @@ -562,8 +562,12 @@ function VariablesandFactorsCRUD_SET!(fg, v1, v2, v3, f0, f1, f2) @test_throws LabelNotFoundError getFactor(fg, :a) # Existence + @test hasVariable(fg, :a) + @test !hasVariable(fg, :c) @test exists(fg, :a) @test !exists(fg, :c) + @test hasFactor(fg, :abf1) + @test !hasFactor(fg, :bcf1) @test exists(fg, :abf1) @test !exists(fg, :bcf1) From 55b181f60be5dc553a74d853a68acd40d2d18d2d Mon Sep 17 00:00:00 2001 From: Johannes Terblanche Date: Wed, 9 Jul 2025 13:38:46 +0200 Subject: [PATCH 2/5] _version -> VersionNumber --- src/DataBlobs/entities/BlobEntry.jl | 4 +-- .../services/HelpersDataWrapEntryBlob.jl | 4 +-- src/FileDFG/FileDFG.jl | 3 --- src/FileDFG/entities/FileDFG.jl | 5 ---- src/entities/DFGFactor.jl | 4 +-- src/entities/DFGVariable.jl | 8 +++--- src/services/AbstractDFG.jl | 2 +- src/services/Serialization.jl | 27 ++++--------------- test/fileDFGTests.jl | 2 +- test/testBlocks.jl | 13 ++++++++- 10 files changed, 29 insertions(+), 43 deletions(-) delete mode 100644 src/FileDFG/entities/FileDFG.jl diff --git a/src/DataBlobs/entities/BlobEntry.jl b/src/DataBlobs/entities/BlobEntry.jl index ce9712cc..c07c00bd 100644 --- a/src/DataBlobs/entities/BlobEntry.jl +++ b/src/DataBlobs/entities/BlobEntry.jl @@ -41,8 +41,8 @@ Base.@kwdef struct Blobentry createdTimestamp::Union{ZonedDateTime, Nothing} = nothing """ Use carefully, but necessary to support advanced usage such as time synchronization over Blob data. """ lastUpdatedTimestamp::Union{ZonedDateTime, Nothing} = nothing - """ Type version of this Blobentry. TBD.jl consider upgrading to `::VersionNumber`. """ - _version::String = string(_getDFGVersion()) + """ Type version of this Blobentry.""" + _version::VersionNumber = _getDFGVersion() end StructTypes.StructType(::Type{Blobentry}) = StructTypes.UnorderedStruct() diff --git a/src/DataBlobs/services/HelpersDataWrapEntryBlob.jl b/src/DataBlobs/services/HelpersDataWrapEntryBlob.jl index acfc67f5..06e822ff 100644 --- a/src/DataBlobs/services/HelpersDataWrapEntryBlob.jl +++ b/src/DataBlobs/services/HelpersDataWrapEntryBlob.jl @@ -57,7 +57,7 @@ function Blobentry( timestamp::ZonedDateTime = entry.timestamp, createdTimestamp = entry.createdTimestamp, lastUpdatedTimestamp = entry.lastUpdatedTimestamp, - _version::String = entry._version, + _version = entry._version, ) return Blobentry(; id, @@ -267,7 +267,7 @@ function updateData!( blobstore = getLabel(blobstore), hash = string(bytes2hex(hashfunction(blob))), origin = buildSourceString(dfg, label), - _version = string(_getDFGVersion()), + _version = _getDFGVersion(), ) mergeBlobentry!(dfg, label, newEntry) updateBlob!(blobstore, newEntry, blob) diff --git a/src/FileDFG/FileDFG.jl b/src/FileDFG/FileDFG.jl index 1e79bb9a..54d0cb05 100644 --- a/src/FileDFG/FileDFG.jl +++ b/src/FileDFG/FileDFG.jl @@ -1,6 +1,3 @@ -# Entities -include("entities/FileDFG.jl") - # Services include("services/FileDFG.jl") diff --git a/src/FileDFG/entities/FileDFG.jl b/src/FileDFG/entities/FileDFG.jl deleted file mode 100644 index 34abd9a4..00000000 --- a/src/FileDFG/entities/FileDFG.jl +++ /dev/null @@ -1,5 +0,0 @@ - -# mutable struct FileDFG -# folderName::String -# FileDFG(folderName::String)::FileDFG = new(foldername) -# end diff --git a/src/entities/DFGFactor.jl b/src/entities/DFGFactor.jl index cfd2da48..e71e1e1f 100644 --- a/src/entities/DFGFactor.jl +++ b/src/entities/DFGFactor.jl @@ -64,7 +64,7 @@ Base.@kwdef struct FactorDFG <: AbstractDFGFactor solvable::Int data::Union{Nothing, String} = nothing #TODO v0.27 deprecate data completely, left as a bridge to old serialization structure metadata::String - _version::String = string(_getDFGVersion()) + _version::VersionNumber = _getDFGVersion() state::FactorState observJSON::String # serialized observation # blobEntries::Vector{Blobentry}#TODO should factor have blob entries? @@ -91,7 +91,7 @@ function FactorDFG( solvable::Int, data::Union{Nothing, String}, metadata::String, - _version::String, + _version, state::Union{Nothing, FactorState} = nothing, observJSON::Union{Nothing, String} = nothing, ) diff --git a/src/entities/DFGVariable.jl b/src/entities/DFGVariable.jl index 37405a60..bb2fb7de 100644 --- a/src/entities/DFGVariable.jl +++ b/src/entities/DFGVariable.jl @@ -122,7 +122,7 @@ Base.@kwdef mutable struct PackedVariableState solvedCount::Int solveKey::Symbol covar::Vector{Float64} - _version::String = string(_getDFGVersion()) + _version::VersionNumber = _getDFGVersion() end # maybe add # createdTimestamp::DateTime#! @@ -158,7 +158,7 @@ Base.@kwdef struct MeanMaxPPE <: AbstractPointParametricEst max::Vector{Float64} mean::Vector{Float64} _type::String = "MeanMaxPPE" - _version::String = string(_getDFGVersion()) + _version::VersionNumber = _getDFGVersion() createdTimestamp::Union{ZonedDateTime, Nothing} = nothing lastUpdatedTimestamp::Union{ZonedDateTime, Nothing} = nothing end @@ -185,7 +185,7 @@ function MeanMaxPPE( max, mean, "MeanMaxPPE", - string(_getDFGVersion()), + _getDFGVersion(), now(tz"UTC"), now(tz"UTC"), ) @@ -223,7 +223,7 @@ Base.@kwdef struct VariableDFG <: AbstractDFGVariable ppes::Vector{MeanMaxPPE} = MeanMaxPPE[] blobEntries::Vector{Blobentry} = Blobentry[] variableType::String - _version::String = string(_getDFGVersion()) + _version::VersionNumber = _getDFGVersion() metadata::String = "e30=" solvable::Int = 1 solverData::Vector{PackedVariableState} = PackedVariableState[] diff --git a/src/services/AbstractDFG.jl b/src/services/AbstractDFG.jl index 8ae8f9f6..9dbfe441 100644 --- a/src/services/AbstractDFG.jl +++ b/src/services/AbstractDFG.jl @@ -306,7 +306,7 @@ end $(SIGNATURES) Add a Vector{VariableCompute} to a DFG. """ -function addVariables!(dfg::AbstractDFG, ::Vector{<:AbstractDFGVariable}) +function addVariables!(dfg::AbstractDFG, variables::Vector{<:AbstractDFGVariable}) return asyncmap(variables) do v return addVariable!(dfg, v) end diff --git a/src/services/Serialization.jl b/src/services/Serialization.jl index 4716b4ba..acb90cd6 100644 --- a/src/services/Serialization.jl +++ b/src/services/Serialization.jl @@ -1,28 +1,11 @@ ## Version checking #NOTE fixed really bad function but kept similar as fallback #TODO upgrade to use pkgversion(m::Module) function _getDFGVersion() - if VERSION >= v"1.9" - return pkgversion(DistributedFactorGraphs) - end - #TODO when we drop jl<1.9 remove the rest here - pkgorigin = get(Base.pkgorigins, Base.PkgId(DistributedFactorGraphs), nothing) - if !isnothing(pkgorigin) && !isnothing(pkgorigin.version) - return pkgorigin.version - end - dep = - get(Pkg.dependencies(), Base.UUID("b5cc3c7e-6572-11e9-2517-99fb8daf2f04"), nothing) - if !isnothing(dep) - return dep.version - else - # This is arguably slower, but needed for Travis. - return Pkg.TOML.parse( - read(joinpath(dirname(pathof(@__MODULE__)), "..", "Project.toml"), String), - )["version"] |> VersionNumber - end + return pkgversion(DistributedFactorGraphs) end function _versionCheck(node::Union{<:VariableDFG, <:FactorDFG}) - if VersionNumber(node._version).minor < _getDFGVersion().minor + if node._version.minor < _getDFGVersion().minor @warn "This data was serialized using DFG $(node._version) but you have $(_getDFGVersion()) installed, there may be deserialization issues." maxlog = 10 end @@ -159,7 +142,7 @@ function packVariableState(d::VariableState{T}) where {T <: VariableStateType} d.solvedCount, d.solveKey, isempty(d.covar) ? Float64[] : vec(d.covar[1]), - string(_getDFGVersion()), + _getDFGVersion(), ) end @@ -234,7 +217,7 @@ function packVariable( solvable = v.solvable, variableType = stringVariableType(DFG.getVariableType(v)), blobEntries = collect(values(v.dataDict)), - _version = string(DFG._getDFGVersion()), + _version = _getDFGVersion(), ) end @@ -312,7 +295,7 @@ function packFactor(f::FactorCompute) solvable = getSolvable(f), metadata = base64encode(JSON3.write(f.smallData)), # Pack the node data - _version = string(_getDFGVersion()), + _version = _getDFGVersion(), state = f.state, observJSON = JSON3.write(packObservation(f)), ) diff --git a/test/fileDFGTests.jl b/test/fileDFGTests.jl index bbac060e..946f333e 100644 --- a/test/fileDFGTests.jl +++ b/test/fileDFGTests.jl @@ -161,7 +161,7 @@ using UUIDs end ## -# file = filter(f -> endswith(f, ".tar.gz"), readdir(joinpath(@__DIR__, "data")))[2] +# file = filter(f -> endswith(f, ".tar.gz"), readdir(joinpath(@__DIR__, "data")))[1] # loadFile = joinpath(@__DIR__, "data", file) # fg = loadDFG(loadFile) ## diff --git a/test/testBlocks.jl b/test/testBlocks.jl index ef262e4a..9c104666 100644 --- a/test/testBlocks.jl +++ b/test/testBlocks.jl @@ -1089,6 +1089,7 @@ function blobsStoresTestBlock!(fg) @test listBlobstores(fg) == [fs.label] # Getting @test getBlobstore(fg, fs.label) == fs + @test_throws LabelNotFoundError getBlobstore(fg, :notfound) # Deleting @test deleteBlobstore!(fg, fs.label) == 1 # Updating @@ -1100,8 +1101,18 @@ function blobsStoresTestBlock!(fg) # Add it back addBlobstore!(fg, fs) - # Data functions + # Blob testData = rand(UInt8, 50) + blobId = addBlob!(fs, testData) + @test blobId isa UUID + @test_throws DFG.IdExistsError addBlob!(fs, blobId, testData) + @test getBlob(fs, blobId) == testData + @test_throws DFG.IdNotFoundError getBlob(fs, uuid4()) + @test_throws DFG.IdNotFoundError deleteBlob!(fs, uuid4()) + @test deleteBlob!(fs, blobId) == 1 + @test_throws DFG.IdNotFoundError getBlob(fs, blobId) + + # Data functions # Adding newData = addData!(fg, fs.label, :a, :testing, testData) # convenience wrapper over addBlob! # Listing From 7f9481d404eddb0eba79633c0e3952cd2c377121 Mon Sep 17 00:00:00 2001 From: Johannes Terblanche Date: Wed, 9 Jul 2025 15:41:06 +0200 Subject: [PATCH 3/5] more cleanup and unexporting functions --- src/DataBlobs/entities/BlobEntry.jl | 2 - src/Deprecated.jl | 98 +++++++++++++++++++++++++++++ src/DistributedFactorGraphs.jl | 6 +- src/GraphsDFG/services/GraphsDFG.jl | 45 ------------- src/entities/DFGFactor.jl | 13 ---- src/entities/DFGVariable.jl | 15 +++-- src/services/AbstractDFG.jl | 89 ++++---------------------- src/services/DFGVariable.jl | 32 ---------- test/iifInterfaceTests.jl | 4 +- test/runtests.jl | 2 +- test/testBlocks.jl | 55 ++++------------ 11 files changed, 137 insertions(+), 224 deletions(-) diff --git a/src/DataBlobs/entities/BlobEntry.jl b/src/DataBlobs/entities/BlobEntry.jl index c07c00bd..94c999b8 100644 --- a/src/DataBlobs/entities/BlobEntry.jl +++ b/src/DataBlobs/entities/BlobEntry.jl @@ -48,5 +48,3 @@ end StructTypes.StructType(::Type{Blobentry}) = StructTypes.UnorderedStruct() StructTypes.idproperty(::Type{Blobentry}) = :id StructTypes.omitempties(::Type{Blobentry}) = (:id,) - -_fixtimezone(cts::NamedTuple) = ZonedDateTime(cts.utc_datetime * "+00") diff --git a/src/Deprecated.jl b/src/Deprecated.jl index 799a805b..63bc9369 100644 --- a/src/Deprecated.jl +++ b/src/Deprecated.jl @@ -51,6 +51,104 @@ function mergeGraphVariableData!(args...) ) end +#NOTE List types funcction do not fit verb noun and will be deprecated. +# should return types + +""" + $SIGNATURES + +Return `Vector{Symbol}` of all unique variable types in factor graph. +""" +function lsTypes(dfg::AbstractDFG) + vars = getVariables(dfg) + alltypes = Set{Symbol}() + for v in vars + varType = Symbol(typeof(getVariableType(v))) + push!(alltypes, varType) + end + return collect(alltypes) +end + +""" + $SIGNATURES + +Return `::Dict{Symbol, Vector{Symbol}}` of all unique variable types with labels in a factor graph. +""" +function lsTypesDict(dfg::AbstractDFG) + vars = getVariables(dfg) + alltypes = Dict{Symbol, Vector{Symbol}}() + for v in vars + varType = Symbol(typeof(getVariableType(v))) + d = get!(alltypes, varType, Symbol[]) + push!(d, v.label) + end + return alltypes +end + +""" + $SIGNATURES + +Return `Vector{Symbol}` of all unique factor types in factor graph. +""" +function lsfTypes(dfg::AbstractDFG) + facs = getFactors(dfg) + alltypes = Set{Symbol}() + for f in facs + facType = typeof(getFactorType(f)) |> nameof + push!(alltypes, facType) + end + return collect(alltypes) +end + +""" + $SIGNATURES + +Return `::Dict{Symbol, Vector{Symbol}}` of all unique factors types with labels in a factor graph. +""" +function lsfTypesDict(dfg::AbstractDFG) + facs = getFactors(dfg) + alltypes = Dict{Symbol, Vector{Symbol}}() + for f in facs + facType = typeof(getFactorType(f)) |> nameof + d = get!(alltypes, facType, Symbol[]) + push!(d, f.label) + end + return alltypes +end + +# solvekey is deprecated and sync!/copyto! is the better verb. +#TODO replace with syncVariableStates! or similar +""" + $SIGNATURES +Duplicate a `solveKey`` into a destination from a source. + +Notes +- Can copy between graphs, or to different solveKeys within one graph. +""" +function cloneSolveKey!( + dest_dfg::AbstractDFG, + dest::Symbol, + src_dfg::AbstractDFG, + src::Symbol; + solvable::Int = 0, + labels = intersect(ls(dest_dfg; solvable = solvable), ls(src_dfg; solvable = solvable)), + verbose::Bool = false, +) + # + for x in labels + sd = deepcopy(getVariableState(getVariable(src_dfg, x), src)) + copytoVariableState!(dest_dfg, x, dest, sd) + end + + return nothing +end + +function cloneSolveKey!(dfg::AbstractDFG, dest::Symbol, src::Symbol; kw...) + # + @assert dest != src "Must copy to a different solveKey within the same graph, $dest." + return cloneSolveKey!(dfg, dest, dfg, src; kw...) +end + ## ================================================================================ ## Deprecated in v0.27 ##================================================================================= diff --git a/src/DistributedFactorGraphs.jl b/src/DistributedFactorGraphs.jl index e8d947fc..a846b8ff 100644 --- a/src/DistributedFactorGraphs.jl +++ b/src/DistributedFactorGraphs.jl @@ -218,8 +218,7 @@ export getVariableStates, addVariableState!, mergeVariableState!, deleteVariableState!, - listVariableStates, - cloneSolveKey! + listVariableStates # PPE ##------------------------------------------------------------------------------ @@ -245,7 +244,7 @@ export packVariableState, unpackVariableState export getSolvedCount, isSolved, setSolvedCount!, isInitialized, isMarginalized, setMarginalized! -export getNeighborhood, listNeighbors, _getDuplicatedEmptyDFG +export listNeighborhood, listNeighbors export findFactorsBetweenNaive export copyGraph!, deepcopyGraph, deepcopyGraph!, buildSubgraph, mergeGraph! @@ -298,7 +297,6 @@ export isValidLabel ## List export ls, lsf, ls2 -export lsTypes, lsfTypes, lsTypesDict, lsfTypesDict export lsWho export isPrior, lsfPriors export hasTags, hasTagsNeighbors diff --git a/src/GraphsDFG/services/GraphsDFG.jl b/src/GraphsDFG/services/GraphsDFG.jl index 5a4952a0..62dc58af 100644 --- a/src/GraphsDFG/services/GraphsDFG.jl +++ b/src/GraphsDFG/services/GraphsDFG.jl @@ -1,11 +1,3 @@ - -function getDFGMetadata(fg::GraphsDFG) - metafields = Set(fieldnames(GraphsDFG)) - setdiff!(metafields, [:g, :solverParams]) - metaprops = NamedTuple(k => getproperty(fg, k) for k in metafields) - return metaprops -end - function hasVariable(dfg::GraphsDFG, label::Symbol) return haskey(dfg.g.variables, label) end @@ -51,43 +43,6 @@ function addVariable!( return addVariable!(dfg, VD(variable)) end -#moved to abstract -# function addFactor!(dfg::GraphsDFG{<:AbstractParams, V, F}, variables::Vector{<:V}, factor::F)::F where {V <: AbstractDFGVariable, F <: AbstractDFGFactor} -# -# #TODO should this be an error -# if haskey(dfg.g.factors, factor.label) -# error("Factor '$(factor.label)' already exists in the factor graph") -# end -# # for v in variables -# # if !(v.label in keys(dfg.g.metaindex[:label])) -# # error("Variable '$(v.label)' not found in graph when creating Factor '$(factor.label)'") -# # end -# # end -# -# variableLabels = map(v->v.label, variables) -# -# resize!(factor._variableOrderSymbols, length(variableLabels)) -# factor._variableOrderSymbols .= variableLabels -# # factor._variableOrderSymbols = copy(variableLabels) -# -# @assert FactorGraphs.addFactor!(dfg.g, variableLabels, factor) -# return factor -# end -# -# function addFactor!(dfg::GraphsDFG{<:AbstractParams, <:AbstractDFGVariable, F}, variableLabels::Vector{Symbol}, factor::F)::F where F <: AbstractDFGFactor -# #TODO should this be an error -# if haskey(dfg.g.factors, factor.label) -# error("Factor '$(factor.label)' already exists in the factor graph") -# end -# -# resize!(factor._variableOrderSymbols, length(variableLabels)) -# factor._variableOrderSymbols .= variableLabels -# -# @assert FactorGraphs.addFactor!(dfg.g, variableLabels, factor) -# -# return factor -# end - function addFactor!( dfg::GraphsDFG{<:AbstractParams, <:AbstractDFGVariable, F}, factor::F, diff --git a/src/entities/DFGFactor.jl b/src/entities/DFGFactor.jl index e71e1e1f..618c3808 100644 --- a/src/entities/DFGFactor.jl +++ b/src/entities/DFGFactor.jl @@ -359,15 +359,6 @@ end ##------------------------------------------------------------------------------ ## Constructors -#NOTE I feel like a want to force a variableOrderSymbols -function FactorSkeleton( - id::Union{UUID, Nothing}, - label::Symbol, - variableOrderSymbols::Vector{Symbol} = Symbol[], -) - @warn "FactorSkeleton(id::Union{UUID, Nothing}...) is deprecated, use FactorSkeleton(label, variableOrderSymbols) instead" - return FactorSkeleton(id, label, Set{Symbol}(), variableOrderSymbols) -end function FactorSkeleton( label::Symbol, variableOrderSymbols::Vector{Symbol}; @@ -377,10 +368,6 @@ function FactorSkeleton( return FactorSkeleton(id, label, tags, variableOrderSymbols) end -StructTypes.StructType(::Type{FactorSkeleton}) = StructTypes.OrderedStruct() -StructTypes.idproperty(::Type{FactorSkeleton}) = :id -StructTypes.omitempties(::Type{FactorSkeleton}) = (:id,) - ##============================================================================== ## Define factor levels ##============================================================================== diff --git a/src/entities/DFGVariable.jl b/src/entities/DFGVariable.jl index bb2fb7de..89972ee7 100644 --- a/src/entities/DFGVariable.jl +++ b/src/entities/DFGVariable.jl @@ -89,6 +89,13 @@ function VariableState(variableType::VariableStateType; kwargs...) return VariableState{typeof(variableType)}(; kwargs...) end + +function VariableState(state::VariableState; kwargs...) + return VariableState{typeof(getVariableType(state))}(; + (key => deepcopy(getproperty(state, key)) for key in fieldnames(VariableState))..., + kwargs..., + ) +end ##============================================================================== ## PackedVariableState.jl ##============================================================================== @@ -452,10 +459,6 @@ function VariableSkeleton( return VariableSkeleton(id, label, tags) end -StructTypes.StructType(::Type{VariableSkeleton}) = StructTypes.UnorderedStruct() -StructTypes.idproperty(::Type{VariableSkeleton}) = :id -StructTypes.omitempties(::Type{VariableSkeleton}) = (:id,) - ##============================================================================== # Define variable levels ##============================================================================== @@ -473,7 +476,7 @@ function VariableSummary(v::VariableCompute) v.id, v.label, v.timestamp, - deepcopy(v.tags), + copy(v.tags), deepcopy(v.ppeDict), Symbol(typeof(getVariableType(v))), v.dataDict, @@ -481,5 +484,5 @@ function VariableSummary(v::VariableCompute) end function VariableSkeleton(v::VariableDataLevel1) - return VariableSkeleton(v.id, v.label, deepcopy(v.tags)) + return VariableSkeleton(v.id, v.label, copy(v.tags)) end diff --git a/src/services/AbstractDFG.jl b/src/services/AbstractDFG.jl index 9dbfe441..a3bb3377 100644 --- a/src/services/AbstractDFG.jl +++ b/src/services/AbstractDFG.jl @@ -793,8 +793,8 @@ end Helper to return neighbors at distance 2 around a given node. """ function ls2(dfg::AbstractDFG, label::Symbol) - l2 = getNeighborhood(dfg, label, 2) - l1 = getNeighborhood(dfg, label, 1) + l2 = listNeighborhood(dfg, label, 2) + l1 = listNeighborhood(dfg, label, 1) return setdiff(l2, l1) end ls2(dfg::AbstractDFG, v::AbstractDFGVariable) = ls(dfg, getLabel(v)) @@ -846,71 +846,6 @@ function lsWho(dfg::AbstractDFG, type::Symbol) return labels end -## list types -##----------- - -""" - $SIGNATURES - -Return `Vector{Symbol}` of all unique variable types in factor graph. -""" -function lsTypes(dfg::AbstractDFG) - vars = getVariables(dfg) - alltypes = Set{Symbol}() - for v in vars - varType = typeof(getVariableType(v)) |> nameof - push!(alltypes, varType) - end - return collect(alltypes) -end - -""" - $SIGNATURES - -Return `::Dict{Symbol, Vector{Symbol}}` of all unique variable types with labels in a factor graph. -""" -function lsTypesDict(dfg::AbstractDFG) - vars = getVariables(dfg) - alltypes = Dict{Symbol, Vector{Symbol}}() - for v in vars - varType = typeof(getVariableType(v)) |> nameof - d = get!(alltypes, varType, Symbol[]) - push!(d, v.label) - end - return alltypes -end - -""" - $SIGNATURES - -Return `Vector{Symbol}` of all unique factor types in factor graph. -""" -function lsfTypes(dfg::AbstractDFG) - facs = getFactors(dfg) - alltypes = Set{Symbol}() - for f in facs - facType = typeof(getFactorType(f)) |> nameof - push!(alltypes, facType) - end - return collect(alltypes) -end - -""" - $SIGNATURES - -Return `::Dict{Symbol, Vector{Symbol}}` of all unique factors types with labels in a factor graph. -""" -function lsfTypesDict(dfg::AbstractDFG) - facs = getFactors(dfg) - alltypes = Dict{Symbol, Vector{Symbol}}() - for f in facs - facType = typeof(getFactorType(f)) |> nameof - d = get!(alltypes, facType, Symbol[]) - push!(d, f.label) - end - return alltypes -end - ##------------------------------------------------------------------------------ ## tags ##------------------------------------------------------------------------------ @@ -1086,7 +1021,7 @@ Related: - [`deepcopyGraph`](@ref) - [`deepcopyGraph!`](@ref) - [`buildSubgraph`](@ref) -- [`getNeighborhood`](@ref) +- [`listNeighborhood`](@ref) - [`mergeGraph!`](@ref) """ function copyGraph!( @@ -1155,7 +1090,7 @@ see [`copyGraph!`](@ref) for more detail. Related: - [`deepcopyGraph`](@ref) - [`buildSubgraph`](@ref) -- [`getNeighborhood`](@ref) +- [`listNeighborhood`](@ref) - [`mergeGraph!`](@ref) """ function deepcopyGraph!( @@ -1182,7 +1117,7 @@ see [`copyGraph!`](@ref) for more detail. Related: - [`deepcopyGraph!`](@ref) - [`buildSubgraph`](@ref) -- [`getNeighborhood`](@ref) +- [`listNeighborhood`](@ref) - [`mergeGraph!`](@ref) """ function deepcopyGraph( @@ -1323,7 +1258,7 @@ Related: - [`deepcopyGraph`](@ref) - [`mergeGraph!`](@ref) """ -function getNeighborhood(dfg::AbstractDFG, label::Symbol, distance::Int) +function listNeighborhood(dfg::AbstractDFG, label::Symbol, distance::Int) neighborList = Set{Symbol}([label]) curList = Set{Symbol}([label]) @@ -1341,7 +1276,7 @@ function getNeighborhood(dfg::AbstractDFG, label::Symbol, distance::Int) return collect(neighborList) end -function getNeighborhood( +function listNeighborhood( dfg::AbstractDFG, variableFactorLabels::Vector{Symbol}, distance::Int; @@ -1351,7 +1286,7 @@ function getNeighborhood( neighbors = Set{Symbol}() if distance > 0 for l in variableFactorLabels - union!(neighbors, getNeighborhood(dfg, l, distance)) + union!(neighbors, listNeighborhood(dfg, l, distance)) end end @@ -1368,7 +1303,7 @@ Build a deep subgraph copy from the DFG given a list of variables and factors an Note: Orphaned factors (where the subgraph does not contain all the related variables) are not returned. Related: - [`copyGraph!`](@ref) -- [`getNeighborhood`](@ref) +- [`listNeighborhood`](@ref) - [`deepcopyGraph`](@ref) - [`mergeGraph!`](@ref) Dev Notes @@ -1387,7 +1322,7 @@ function buildSubgraph( !isnothing(sessionId) && @warn "sessionId is deprecated, use graphLabel instead" #build up the neighborhood from variableFactorLabels - allvarfacs = getNeighborhood(dfg, variableFactorLabels, distance; solvable = solvable) + allvarfacs = listNeighborhood(dfg, variableFactorLabels, distance; solvable = solvable) variableLabels = intersect(allvarfacs, listVariables(dfg)) factorLabels = intersect(allvarfacs, listFactors(dfg)) @@ -1414,7 +1349,7 @@ Notes: Related: - [`copyGraph!`](@ref) - [`buildSubgraph`](@ref) -- [`getNeighborhood`](@ref) +- [`listNeighborhood`](@ref) - [`deepcopyGraph`](@ref) """ function mergeGraph!( @@ -1428,7 +1363,7 @@ function mergeGraph!( ) # find neighbors at distance to add - allvarfacs = getNeighborhood( + allvarfacs = listNeighborhood( sourceDFG, union(variableLabels, factorLabels), distance; diff --git a/src/services/DFGVariable.jl b/src/services/DFGVariable.jl index 9ca42c2e..09976948 100644 --- a/src/services/DFGVariable.jl +++ b/src/services/DFGVariable.jl @@ -747,38 +747,6 @@ function copytoVariableState!( return mergeVariableState!(dfg, variableLabel, newstate) end -""" - $SIGNATURES -Duplicate a `solveKey`` into a destination from a source. - -Notes -- Can copy between graphs, or to different solveKeys within one graph. -""" -function cloneSolveKey!( - dest_dfg::AbstractDFG, - dest::Symbol, - src_dfg::AbstractDFG, - src::Symbol; - solvable::Int = 0, - labels = intersect(ls(dest_dfg; solvable = solvable), ls(src_dfg; solvable = solvable)), - verbose::Bool = false, -) - # - for x in labels - sd = deepcopy(getVariableState(getVariable(src_dfg, x), src)) - sd.solveKey = dest - updateVariableSolverData!(dest_dfg, x, sd, true, Symbol[]; warn_if_absent = verbose) - end - - return nothing -end - -function cloneSolveKey!(dfg::AbstractDFG, dest::Symbol, src::Symbol; kw...) - # - @assert dest != src "Must copy to a different solveKey within the same graph, $dest." - return cloneSolveKey!(dfg, dest, dfg, src; kw...) -end - # """ diff --git a/test/iifInterfaceTests.jl b/test/iifInterfaceTests.jl index 5d9c6aca..19289f68 100644 --- a/test/iifInterfaceTests.jl +++ b/test/iifInterfaceTests.jl @@ -144,7 +144,7 @@ end @test !isPrior(dfg, :abf1) # f1 is not a prior @test lsfPriors(dfg) == [] - @test lsfTypes(dfg) == [:LinearRelative] + @test DFG.lsfTypes(dfg) == [:LinearRelative] @test ls(dfg, LinearRelative) == [:abf1] @test lsf(dfg, LinearRelative) == [:abf1] @@ -152,7 +152,7 @@ end @test getVariableType(v1) isa Position{1} @test getVariableType(dfg, :a) isa Position{1} - @test lsTypes(dfg) == [:Position] + @test DFG.lsTypes(dfg) == [Symbol("Position{1}")] @test issetequal(ls(dfg, Position{1}), [:a, :b]) @test issetequal(lsWho(dfg, :Position), [:a, :b]) diff --git a/test/runtests.jl b/test/runtests.jl index cdd7951b..2cc07373 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -151,7 +151,7 @@ struct NotImplementedDFG{T} <: AbstractDFG{T} end @test_throws ErrorException listNeighbors(dfg, v1) @test_throws ErrorException listNeighbors(dfg, :a) - @test_throws ErrorException _getDuplicatedEmptyDFG(dfg) + @test_throws ErrorException DFG._getDuplicatedEmptyDFG(dfg) @test_throws ErrorException isVariable(dfg, :a) @test_throws ErrorException isFactor(dfg, :a) diff --git a/test/testBlocks.jl b/test/testBlocks.jl index 9c104666..62bc3cc3 100644 --- a/test/testBlocks.jl +++ b/test/testBlocks.jl @@ -8,24 +8,12 @@ using DistributedFactorGraphs: LabelExistsError, LabelNotFoundError import Base: convert # import DistributedFactorGraphs: getData, addData!, updateData!, deleteData! -# Test VariableStateType Types -# struct TestVariableType1 <: VariableStateType -# dims::Int -# manifolds::Tuple{Symbol} -# TestVariableType1() = new(1,(:Euclid,)) -# end - Base.convert(::Type{<:Tuple}, ::typeof(Euclidean(1))) = (:Euclid,) Base.convert(::Type{<:Tuple}, ::typeof(Euclidean(2))) = (:Euclid, :Euclid) @defVariable TestVariableType1 Euclidean(1) [0.0;] -@defVariable TestVariableType2 Euclidean(2) [0; 0.0] - -# struct TestVariableType2 <: VariableStateType -# dims::Int -# manifolds::Tuple{Symbol, Symbol} -# TestVariableType2() = new(2,(:Euclid,:Circular,)) -# end +DFG.@defVarstateTypeN TestVariableType{N} Euclidean(N) zeros(N) +const TestVariableType2 = TestVariableType{2} struct TestFunctorInferenceType1 <: AbstractRelative end struct TestFunctorInferenceType2 <: AbstractRelative end @@ -796,26 +784,10 @@ function VSDTestBlock!(fg, v1) @test mergeVariableState!(fg, :a, vnd) == 1 # Bulk copy update x0 - @test updateVariableSolverData!(fg, [v1], :default) == nothing + @test DFG.copytoVariableState!(fg, v1.label, :default, getVariableState(fg, v1.label, :default)) == 1 altVnd = vnd |> deepcopy keepVnd = getVariableState(getVariable(fg, :a), :parametric) |> deepcopy - altVnd.infoPerCoord .= [-99.0;] - retVnd = updateVariableSolverData!(fg, :a, altVnd, false, [:infoPerCoord;]) - @test retVnd == altVnd - - altVnd.bw = -ones(1, 1) - retVnd = updateVariableSolverData!(fg, :a, altVnd, false, [:bw;]) - @test retVnd == altVnd - - altVnd.infoPerCoord[1] = -98.0 - @test retVnd != altVnd - - # restore without copy - @test updateVariableSolverData!(fg, :a, keepVnd, false, [:infoPerCoord; :bw]) == vnd - @test getVariableState(getVariable(fg, :a), :parametric).infoPerCoord[1] != - altVnd.infoPerCoord[1] - @test getVariableState(getVariable(fg, :a), :parametric).bw != altVnd.bw # Delete parametric from v1 @test deleteVariableState!(fg, :a, :parametric) == 1 @@ -840,11 +812,9 @@ function VSDTestBlock!(fg, v1) # Delete it @test deleteVariableState!(fg, :a, :parametric) == 1 # Update add it - updateVariableSolverData!(fg, :a, vnd) + mergeVariableState!(fg, :a, vnd) # Update update it - updateVariableSolverData!(fg, :a, vnd) - # Bulk copy update x0 - updateVariableSolverData!(fg, [v1], :default) + mergeVariableState!(fg, :a, vnd) # Delete parametric from v1 deleteVariableState!(fg, :a, :parametric) @@ -1105,6 +1075,7 @@ function blobsStoresTestBlock!(fg) testData = rand(UInt8, 50) blobId = addBlob!(fs, testData) @test blobId isa UUID + @test hasBlob(fs, blobId) @test_throws DFG.IdExistsError addBlob!(fs, blobId, testData) @test getBlob(fs, blobId) == testData @test_throws DFG.IdNotFoundError getBlob(fs, uuid4()) @@ -1164,10 +1135,10 @@ function testGroup!(fg, v1, v2, f0, f1) @test isPrior(fg, :af1) # if f1 is prior @test lsfPriors(fg) == [:af1] - @test issetequal([:TestFunctorInferenceType1, :TestAbstractPrior], lsfTypes(fg)) + @test issetequal([:TestFunctorInferenceType1, :TestAbstractPrior], DFG.lsfTypes(fg)) - facTypesDict = lsfTypesDict(fg) - @test issetequal(collect(keys(facTypesDict)), lsfTypes(fg)) + facTypesDict = DFG.lsfTypesDict(fg) + @test issetequal(collect(keys(facTypesDict)), DFG.lsfTypes(fg)) @test issetequal(facTypesDict[:TestFunctorInferenceType1], [:abf1]) @test issetequal(facTypesDict[:TestAbstractPrior], [:af1]) @@ -1182,10 +1153,10 @@ function testGroup!(fg, v1, v2, f0, f1) @test ls2(fg, :a) == [:b] - @test issetequal([:TestVariableType1, :TestVariableType2], lsTypes(fg)) + @test issetequal([:TestVariableType1, :TestVariableType2], DFG.lsTypes(fg)) - varTypesDict = lsTypesDict(fg) - @test issetequal(collect(keys(varTypesDict)), lsTypes(fg)) + varTypesDict = DFG.lsTypesDict(fg) + @test issetequal(collect(keys(varTypesDict)), DFG.lsTypes(fg)) @test issetequal(varTypesDict[:TestVariableType1], [:a]) @test issetequal(varTypesDict[:TestVariableType2], [:b]) @@ -1428,7 +1399,7 @@ function connectivityTestGraph( ), ) - foreach(v -> addVariable!(dfg, v), vars) + addVariables!(dfg, vars) if FACTYPE == FactorCompute #change ready and solveInProgress for x7,x8 for improved tests on x7x8f1 From d73d92a377cac219ec2c81c5e530778ed2fdf222 Mon Sep 17 00:00:00 2001 From: Johannes Terblanche Date: Wed, 9 Jul 2025 15:41:45 +0200 Subject: [PATCH 4/5] format --- src/DistributedFactorGraphs.jl | 5 +---- src/entities/DFGVariable.jl | 1 - 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/DistributedFactorGraphs.jl b/src/DistributedFactorGraphs.jl index a846b8ff..cd16cee2 100644 --- a/src/DistributedFactorGraphs.jl +++ b/src/DistributedFactorGraphs.jl @@ -215,10 +215,7 @@ export getMetadata, # CRUD & SET export getVariableStates, - addVariableState!, - mergeVariableState!, - deleteVariableState!, - listVariableStates + addVariableState!, mergeVariableState!, deleteVariableState!, listVariableStates # PPE ##------------------------------------------------------------------------------ diff --git a/src/entities/DFGVariable.jl b/src/entities/DFGVariable.jl index 89972ee7..ebc1247d 100644 --- a/src/entities/DFGVariable.jl +++ b/src/entities/DFGVariable.jl @@ -89,7 +89,6 @@ function VariableState(variableType::VariableStateType; kwargs...) return VariableState{typeof(variableType)}(; kwargs...) end - function VariableState(state::VariableState; kwargs...) return VariableState{typeof(getVariableType(state))}(; (key => deepcopy(getproperty(state, key)) for key in fieldnames(VariableState))..., From 0ffe309350eb47620686dfe03bbed410297478b0 Mon Sep 17 00:00:00 2001 From: Johannes Terblanche Date: Wed, 9 Jul 2025 16:25:07 +0200 Subject: [PATCH 5/5] fix docs --- src/Deprecated.jl | 52 +++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/Deprecated.jl b/src/Deprecated.jl index 63bc9369..28901498 100644 --- a/src/Deprecated.jl +++ b/src/Deprecated.jl @@ -21,10 +21,10 @@ const InferenceType = AbstractPackedFactorObservation const PackedSamplableBelief = PackedBelief export setSolverData! -""" - $SIGNATURES -Set solver data structure stored in a variable. -""" +# """ +# $SIGNATURES +# Set solver data structure stored in a variable. +# """ function setSolverData!(v::VariableCompute, data::VariableState, key::Symbol = :default) Base.depwarn( "setSolverData!(v::VariableCompute, data::VariableState, key::Symbol = :default) is deprecated, use mergeVariableState! instead.", @@ -54,11 +54,11 @@ end #NOTE List types funcction do not fit verb noun and will be deprecated. # should return types -""" - $SIGNATURES +# """ +# $SIGNATURES -Return `Vector{Symbol}` of all unique variable types in factor graph. -""" +# Return `Vector{Symbol}` of all unique variable types in factor graph. +# """ function lsTypes(dfg::AbstractDFG) vars = getVariables(dfg) alltypes = Set{Symbol}() @@ -69,11 +69,11 @@ function lsTypes(dfg::AbstractDFG) return collect(alltypes) end -""" - $SIGNATURES +# """ +# $SIGNATURES -Return `::Dict{Symbol, Vector{Symbol}}` of all unique variable types with labels in a factor graph. -""" +# Return `::Dict{Symbol, Vector{Symbol}}` of all unique variable types with labels in a factor graph. +# """ function lsTypesDict(dfg::AbstractDFG) vars = getVariables(dfg) alltypes = Dict{Symbol, Vector{Symbol}}() @@ -85,11 +85,11 @@ function lsTypesDict(dfg::AbstractDFG) return alltypes end -""" - $SIGNATURES +# """ +# $SIGNATURES -Return `Vector{Symbol}` of all unique factor types in factor graph. -""" +# Return `Vector{Symbol}` of all unique factor types in factor graph. +# """ function lsfTypes(dfg::AbstractDFG) facs = getFactors(dfg) alltypes = Set{Symbol}() @@ -100,11 +100,11 @@ function lsfTypes(dfg::AbstractDFG) return collect(alltypes) end -""" - $SIGNATURES +# """ +# $SIGNATURES -Return `::Dict{Symbol, Vector{Symbol}}` of all unique factors types with labels in a factor graph. -""" +# Return `::Dict{Symbol, Vector{Symbol}}` of all unique factors types with labels in a factor graph. +# """ function lsfTypesDict(dfg::AbstractDFG) facs = getFactors(dfg) alltypes = Dict{Symbol, Vector{Symbol}}() @@ -118,13 +118,13 @@ end # solvekey is deprecated and sync!/copyto! is the better verb. #TODO replace with syncVariableStates! or similar -""" - $SIGNATURES -Duplicate a `solveKey`` into a destination from a source. +# """ +# $SIGNATURES +# Duplicate a `solveKey`` into a destination from a source. -Notes -- Can copy between graphs, or to different solveKeys within one graph. -""" +# Notes +# - Can copy between graphs, or to different solveKeys within one graph. +# """ function cloneSolveKey!( dest_dfg::AbstractDFG, dest::Symbol,