diff --git a/Project.toml b/Project.toml index e5959e06..6397e746 100644 --- a/Project.toml +++ b/Project.toml @@ -23,6 +23,7 @@ RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" +StructUtils = "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" TensorCast = "02d47bb6-7ce6-556a-be16-bb1710789e2b" @@ -64,6 +65,7 @@ SHA = "0.7, 1" SparseArrays = "1.11" StaticArrays = "1" Statistics = "1.11" +StructUtils = "2.6.0" Tables = "1.11.1" Tar = "1.9" TensorCast = "0.3.3, 0.4" diff --git a/docs/src/GraphData.md b/docs/src/GraphData.md index 3b710564..0ae6b3ee 100644 --- a/docs/src/GraphData.md +++ b/docs/src/GraphData.md @@ -44,7 +44,17 @@ Labels are the principle identifier of a variable or factor. #### Timestamps -Each variable or factor can have a timestamp associated with it. +Each variable or factor can have a timestamp associated with it representing when the physical event/observation occurred. + +**Time Standard**: Timestamps use UTC-based time (Coordinated Universal Time) via the `TimeDateZone` type, not TAI (International Atomic Time). The key difference is that UTC includes leap seconds to keep synchronized with Earth's rotation, while TAI is a continuous monotonic time scale without leap seconds. + +**Timezone Support**: While timestamps default to UTC (`tz"UTC"`), `TimeDateZone` supports any timezone (e.g., `tz"America/New_York"`, `tz"Europe/London"`). The timezone offset is preserved when stored and retrieved. + +**Event Time vs Database Time**: The timestamp represents the physical event time (when a sensor measurement was taken, when a robot was at a pose, etc.), not when the variable was added to the database. Database metadata timestamps may be tracked separately by specific backend implementations. + +**Temporal Uncertainty**: This single timestamp value is metadata and does not represent temporal uncertainty in non-parametric beliefs. For problems where time itself is a state variable requiring inference (e.g., using `SGal3` which includes temporal components), include time as part of your state type rather than relying on this metadata field. + +Related functions: - [`getTimestamp`](@ref) diff --git a/src/Common.jl b/src/Common.jl index 8fe4b071..2f469e64 100644 --- a/src/Common.jl +++ b/src/Common.jl @@ -187,7 +187,19 @@ function calcDeltatime(from::TimeDateZone, to::TimeDateZone) end calcDeltatime(from_node, to_node) = calcDeltatime(from_node.timestamp, to_node.timestamp) -function tdz_now(zone = tz"UTC") #TODO or default to slower localzone()? +Timestamp(args...) = TimeDateZone(args...) +function Timestamp(epoch::Val{:unix}, t::Nanosecond, zone = tz"UTC") + return TimeDateZone(TimeDate(1970) + t, zone) +end +function Timestamp(epoch::Val{:unix}, t::Float64, zone = tz"UTC") + return Timestamp(epoch, Nanosecond(t * 10^9), zone) +end +Timestamp(t::Float64, zone = tz"UTC") = Timestamp(Val(:unix), t, zone) +function Timestamp(epoch::Val{:rata}, t::Float64, zone = tz"UTC") + return TimeDateZone(convert(DateTime, Millisecond(t*10^3)), zone) +end + +function now_tdz(zone = tz"UTC") t = time() - return TimeDateZone(TimeDate(1970) + Nanosecond(t * 10^9), zone) + return Timestamp(t, zone) end diff --git a/src/DataBlobs/entities/BlobEntry.jl b/src/DataBlobs/entities/BlobEntry.jl index 14a6e8b1..9aaf2dd7 100644 --- a/src/DataBlobs/entities/BlobEntry.jl +++ b/src/DataBlobs/entities/BlobEntry.jl @@ -44,7 +44,7 @@ StructUtils.@kwarg struct Blobentry """ Storage for a couple of bytes directly in the graph. Use with caution and keep it small and simple.""" metadata::JSONText = JSONText("{}") """ When the Blob itself was first created. Serialized as an ISO 8601 string.""" - timestamp::TimeDateZone = tdz_now() + timestamp::TimeDateZone = now_tdz() """ Type version of this Blobentry.""" version::VersionNumber = DFG.version(Blobentry) end diff --git a/src/Deprecated.jl b/src/Deprecated.jl index 54f92e4d..7219f82f 100644 --- a/src/Deprecated.jl +++ b/src/Deprecated.jl @@ -317,10 +317,6 @@ function getFactorState(args...) ) end -function updateBlob!(args...) - return error("updateBlob! is obsolete as blobid=>Blob pairs are immutable.") -end - function setTags!(node, tags::Union{Vector{Symbol}, Set{Symbol}}) Base.depwarn("setTags! is deprecated, use mergeTags! or addTags! instead.", :setTags!) node.tags !== tags && empty!(node.tags) @@ -343,7 +339,6 @@ const DFGFactorSummary = FactorSummary const DFGFactor = FactorCompute const PackedFactor = FactorDFG const Factor = FactorDFG -const SmallDataTypes = MetadataTypes const AbstractPrior = PriorObservation const AbstractRelative = RelativeObservation const AbstractParams = AbstractDFGParams @@ -462,14 +457,6 @@ function getDFGInfo(dfg::AbstractDFG) ) end -# """ -# $TYPEDSIGNATURES -# List all the solvekeys used amongst all variables in the distributed factor graph object. - -# Related - -# [`listSolveKeys`](@ref), [`refStates`](@ref), [`listVariables`](@ref) -# """ function listSolveKeys( variable::VariableCompute, filterSolveKeys::Union{Regex, Nothing} = nothing, @@ -522,15 +509,7 @@ const listSupersolves = listSolveKeys #TODO mergeBlobentries! does not fit with merge definition, should probably be updated to copyto or sync. # leaving here until it is done. - -# """ -# $SIGNATURES - -# Add a blob entry into the destination variable which already exists -# in a source variable. - -# See also: [`addBlobentry!`](@ref), [`getBlobentry`](@ref), [`listBlobentries`](@ref), [`getBlob`](@ref) -# """ +# Add a blob entry into the destination variable which already exists in a source variable. function mergeBlobentries!( dst::AbstractDFG, dlbl::Symbol, @@ -583,15 +562,6 @@ function mergeBlobentries!( return varList end -# """ -# $(SIGNATURES) - -# Get all blob entries matching a Regex pattern over variables - -# Notes -# - Use `dropEmpties=true` to not include empty lists in result. -# - Use keyword `varList` for which variables to search through. -# """ function getBlobentriesVariables( dfg::AbstractDFG, bLblPattern::Regex; @@ -621,13 +591,9 @@ function getBlobentries(dfg::AbstractDFG, label::Symbol, regex::Regex) return entries = getBlobentries(dfg, label; labelFilter = contains(regex)) end -function getBlobentries( - dfg::AbstractDFG, - label::Symbol, - skey::Union{Symbol, <:AbstractString}, -) +function getBlobentries(dfg::AbstractDFG, label::Symbol, skey::AbstractString) Base.depwarn( - "getBlobentries(dfg, label, ::Union{Symbol, <:AbstractString}) is deprecated, use getBlobentries(dfg, label; labelFilter=contains(regex)) instead.", + "getBlobentries(dfg, label, regex::AbstractString) is deprecated, use getBlobentries(dfg, label; labelFilter=contains(regex)) instead.", :getBlobentries, ) return getBlobentries(dfg, label, Regex(string(skey))) @@ -674,666 +640,8 @@ end setMetadata!(args...) = error("setMetadata is obsolete, use Bloblets instead.") -function updateData!( - dfg::AbstractDFG, - label::Symbol, - entry::Blobentry, - blob::Vector{UInt8}; - hashfunction = sha256, - checkhash::Bool = true, -) - @warn "updateData! is obsolete." - checkhash && assertHash(entry, blob; hashfunction) - # order of ops with unknown new blobId not tested - mergeBlobentry!(dfg, label, entry) - db = updateBlob!(dfg, de, blob) - return 2 -end - -function updateData!( - dfg::AbstractDFG, - blobstore::AbstractBlobstore, - label::Symbol, - entry::Blobentry, - blob::Vector{UInt8}; - hashfunction = sha256, -) - @warn "updateData! is obsolete." - # Recalculate the hash - NOTE Assuming that this is going to be a Blobentry. TBD. - # order of operations with unknown new blobId not tested - newEntry = Blobentry( - entry; # and kwargs to override new values - blobstore = getLabel(blobstore), - hash = string(bytes2hex(hashfunction(blob))), - origin = buildSourceString(dfg, label), - _version = _getDFGVersion(), - ) - mergeBlobentry!(dfg, label, newEntry) - updateBlob!(blobstore, newEntry, blob) - return 2 -end - -function updateBlob!(store::RowBlobstore{T}, blobId::UUID, blob::T) where {T} - @warn "updateBlob! is obsolete." - if haskey(store.blobs, blobId) - @warn "Key '$blobId' doesn't exist." - end - return store.blobs[blobId] = RowBlob(blobId, blob) -end - -function getData( - dfg::AbstractDFG, - vlabel::Symbol, - key::Union{Symbol, UUID, <:AbstractString, Regex}; - hashfunction = sha256, - checkhash::Bool = true, - getlast::Bool = true, -) - Base.depwarn("getData is deprecated, use loadBlob_Variable instead.", :getData) - _getblobentr(g, v, k) = getBlobentries(g, v, k) - _getblobentr(g, v, k::UUID) = [getfirstBlobentry(g, v, k);] - de_ = _getblobentr(dfg, vlabel, key) - lbls = (s -> s.label).(de_) - idx = sortperm(lbls; rev = getlast) - _first(s) = s - _first(s::AbstractVector) = 0 < length(s) ? s[1] : nothing - de = _first(de_[idx]) - if isnothing(de) - @error "Could not find in $vlabel the key $key" - return nothing - end - db = getBlob(dfg, de) - - checkhash && assertHash(de, db; hashfunction = hashfunction) - return de => db -end - -# This is the normal one -function getData( - dfg::AbstractDFG, - blobstore::AbstractBlobstore, - var_label::Symbol, - entry_label::Symbol; - hashfunction = sha256, - checkhash::Bool = true, - getlast::Bool = true, -) - Base.depwarn("getData is deprecated, use loadBlob_Variable instead.", :getData) - de = getBlobentry(dfg, var_label, entry_label) - db = getBlob(blobstore, de) - checkhash && assertHash(de, db; hashfunction) - return de => db -end - -#FIXME Should `addData!`` not return entry=>blob pair? -function addData!( - dfg::AbstractDFG, - label::Symbol, - entry::Blobentry, - blob::Vector{UInt8}; - hashfunction = sha256, - checkhash::Bool = false, -) - Base.depwarn("addData! is obsolete, use saveBlob_Variable! instead.", :addData!) - checkhash && assertHash(entry, blob; hashfunction) - blobId = addBlob!(dfg, entry, blob) |> UUID - newEntry = Blobentry(entry; blobId) #, size=length(blob)) - return addBlobentry!(dfg, label, newEntry) -end - -function addData!( - dfg::AbstractDFG, - blobstore::AbstractBlobstore, - label::Symbol, - entry::Blobentry, - blob::Vector{UInt8}; - hashfunction = sha256, - checkhash::Bool = false, -) - Base.depwarn("addData! is obsolete, use saveBlob_Variable! instead.", :addData!) - checkhash && assertHash(entry, blob; hashfunction) - blobId = addBlob!(blobstore, entry, blob) |> UUID - newEntry = Blobentry(entry; blobId) #, size=length(blob)) - return addBlobentry!(dfg, label, newEntry) -end - -function addData!( - dfg::AbstractDFG, - blobstorekey::Symbol, - vLbl::Symbol, - bLbl::Symbol, - blob::Vector{UInt8}, - timestamp = now(localzone()); - kwargs..., -) - Base.depwarn("addData! is obsolete, use saveBlob_Variable! instead.", :addData!) - return addData!( - dfg, - getBlobstore(dfg, blobstorekey), - vLbl, - bLbl, - blob, - timestamp; - kwargs..., - ) -end - -function addData!( - dfg::AbstractDFG, - blobstore::AbstractBlobstore, - vLbl::Symbol, - bLbl::Symbol, - blob::Vector{UInt8}, - timestamp = now(localzone()); - description = "", - metadata = "", - mimeType::String = "application/octet-stream", - id::Union{UUID, Nothing} = nothing, - blobId::UUID = uuid4(), - hashfunction = sha256, -) - Base.depwarn("addData! is obsolete, use saveBlob_Variable! instead.", :addData!) - # - entry = Blobentry(; - id, - blobId, - label = bLbl, - blobstore = getLabel(blobstore), - hash = string(bytes2hex(hashfunction(blob))), - origin = buildSourceString(dfg, vLbl), - description, - mimeType, - metadata, - timestamp, - ) - - return addData!(dfg, blobstore, vLbl, entry, blob; hashfunction) -end - -function addData!( - dfg::AbstractDFG, - blobstore::AbstractBlobstore{T}, - vLbl::Symbol, - blobLabel::Symbol, - blob::T, - timestamp = now(localzone()); - description = "", - metadata = "", - mimeType::String = "application/octet-stream", - origin = buildSourceString(dfg, vLbl), - # hashfunction = sha256, -) where {T} - Base.depwarn("addData! is obsolete, use saveBlob_Variable! instead.", :addData!) - # - # checkhash && assertHash(entry, blob; hashfunction) - blobId = addBlob!(blobstore, blob) - - entry = Blobentry(; - blobId, - label = blobLabel, - blobstore = getLabel(blobstore), - # hash = string(bytes2hex(hashfunction(blob))), - hash = "", - origin, - description, - mimeType, - metadata, - timestamp, - ) - addBlobentry!(dfg, vLbl, entry) - return entry => blob -end - -function deleteData!(dfg::AbstractDFG, vLbl::Symbol, bLbl::Symbol) - Base.depwarn( - "deleteData! is deprecated, use deleteBlob_Variable! instead.", - :deleteData!, - ) - de = getBlobentry(dfg, vLbl, bLbl) - deleteBlobentry!(dfg, vLbl, bLbl) - deleteBlob!(dfg, de) - return 2 -end - -function deleteData!( - dfg::AbstractDFG, - blobstore::AbstractBlobstore, - vLbl::Symbol, - entry::Blobentry, -) - Base.depwarn( - "deleteData! is deprecated, use deleteBlob_Variable! instead.", - :deleteData!, - ) - return deleteData!(dfg, blobstore, vLbl, entry.label) -end - -function deleteData!( - dfg::AbstractDFG, - blobstore::AbstractBlobstore, - vLbl::Symbol, - bLbl::Symbol, -) - Base.depwarn( - "deleteData! is deprecated, use deleteBlob_Variable! instead.", - :deleteData!, - ) - de = getBlobentry(dfg, vLbl, bLbl) - deleteBlobentry!(dfg, vLbl, bLbl) - deleteBlob!(blobstore, de) - return 2 -end - -## ================================================================================ -## Deprecated in v0.27 -##================================================================================= - -# const AbstractFactor = AbstractObservation -# const AbstractPackedFactor = AbstractPackedObservation -# const FactorOperationalMemory = FactorCache -# const VariableNodeData = State - -# @deprecate getNeighborhood(args...; kwargs...) listNeighborhood(args...; kwargs...) -# @deprecate addBlob!(store::AbstractBlobstore, blobId::UUID, data, ::String) addBlob!( -# store, -# blobId, -# data, -# ) -# @deprecate addBlob!(store::AbstractBlobstore{T}, data::T, ::String) where {T} addBlob!( -# store, -# uuid4(), -# data, -# ) - -# @deprecate updateVariable!(args...) mergeVariable!(args...) -# @deprecate updateFactor!(args...) mergeFactor!(args...) - -# @deprecate updateBlobEntry!(args...) mergeBlobentry!(args...) -# @deprecate updateGraphBlobEntry!(args...) mergeGraphBlobentry!(args...) -# @deprecate updateAgentBlobEntry!(args...) mergeAgentBlobentry!(args...) - -# @deprecate getBlobStore(args...) getBlobstore(args...) -# @deprecate addBlobStore!(args...) addBlobstore!(args...) -# @deprecate updateBlobStore!(args...) updateBlobstore!(args...) -# @deprecate deleteBlobStore!(args...) deleteBlobstore!(args...) -# @deprecate emptyBlobStore!(args...) emptyBlobstore!(args...) -# @deprecate listBlobStores(args...) listBlobstores(args...) - -# @deprecate BlobEntry(args...; kwargs...) Blobentry(args...; kwargs...) -# @deprecate getGraphBlobEntry(args...; kwargs...) getGraphBlobentry(args...; kwargs...) -# @deprecate getGraphBlobEntries(args...; kwargs...) getGraphBlobentries(args...; kwargs...) -# @deprecate addGraphBlobEntry!(args...; kwargs...) addGraphBlobentry!(args...; kwargs...) -# @deprecate addGraphBlobEntries!(args...; kwargs...) addGraphBlobentries!(args...; kwargs...) -# @deprecate mergeGraphBlobEntry!(args...; kwargs...) mergeGraphBlobentry!(args...; kwargs...) -# @deprecate deleteGraphBlobEntry!(args...; kwargs...) deleteGraphBlobentry!( -# args...; -# kwargs..., -# ) -# @deprecate getAgentBlobEntry(args...; kwargs...) getAgentBlobentry(args...; kwargs...) -# @deprecate getAgentBlobEntries(args...; kwargs...) getAgentBlobentries(args...; kwargs...) -# @deprecate addAgentBlobEntry!(args...; kwargs...) addAgentBlobentry!(args...; kwargs...) -# @deprecate addAgentBlobEntries!(args...; kwargs...) addAgentBlobentries!(args...; kwargs...) -# @deprecate mergeAgentBlobEntry!(args...; kwargs...) mergeAgentBlobentry!(args...; kwargs...) -# @deprecate deleteAgentBlobEntry!(args...; kwargs...) deleteAgentBlobentry!( -# args...; -# kwargs..., -# ) -# @deprecate listGraphBlobEntries(args...; kwargs...) listGraphBlobentries(args...; kwargs...) -# @deprecate listAgentBlobEntries(args...; kwargs...) listAgentBlobentries(args...; kwargs...) -# @deprecate hasBlobEntry(args...; kwargs...) hasBlobentry(args...; kwargs...) -# @deprecate getBlobEntry(args...; kwargs...) getBlobentry(args...; kwargs...) -# @deprecate getBlobEntryFirst(args...; kwargs...) getfirstBlobentry(args...; kwargs...) -# @deprecate getBlobentry(var::AbstractGraphVariable, blobId::UUID) getfirstBlobentry( -# var::AbstractGraphVariable, -# blobId::UUID, -# ) -# @deprecate addBlobEntry!(args...; kwargs...) addBlobentry!(args...; kwargs...) -# @deprecate addBlobEntries!(args...; kwargs...) addBlobentries!(args...; kwargs...) -# @deprecate mergeBlobEntry!(args...; kwargs...) mergeBlobentry!(args...; kwargs...) -# @deprecate deleteBlobEntry!(args...; kwargs...) deleteBlobentry!(args...; kwargs...) -# @deprecate listBlobEntrySequence(args...; kwargs...) listBlobentrySequence( -# args...; -# kwargs..., -# ) -# @deprecate mergeBlobEntries!(args...; kwargs...) mergeBlobentries!(args...; kwargs...) - -# @deprecate getVariableSolverData(args...; kwargs...) getState(args...; kwargs...) -# @deprecate addVariableSolverData!(args...; kwargs...) addState!(args...; kwargs...) -# @deprecate deleteVariableSolverData!(args...; kwargs...) deleteState!(args...; kwargs...) -# @deprecate listVariableSolverData(args...; kwargs...) listStates(args...; kwargs...) -# @deprecate getVariableSolverDataAll(args...; kwargs...) getStates(args...; kwargs...) - -# @deprecate getSolverData(v::VariableCompute, solveKey::Symbol = :default) getState( -# v, -# solveKey, -# ) false - -# @deprecate packVariableNodeData(args...; kwargs...) packState(args...; kwargs...) -# @deprecate unpackVariableNodeData(args...; kwargs...) unpackState(args...; kwargs...) - -# #TODO possibly completely deprecated or not exported until update verb is standardized -# function updateVariableSolverData!( -# dfg::AbstractDFG, -# variablekey::Symbol, -# vnd::State, -# useCopy::Bool = false, -# fields::Vector{Symbol} = Symbol[]; -# warn_if_absent::Bool = true, -# ) -# Base.depwarn( -# "updateVariableSolverData! is deprecated, use mergeState! or copytoState! instead", -# :updateVariableSolverData!, -# ) -# #This is basically just setSolverData -# var = getVariable(dfg, variablekey) -# warn_if_absent && -# !haskey(var.solverDataDict, vnd.solveKey) && -# @warn "State '$(vnd.solveKey)' does not exist, adding" - -# # for InMemoryDFGTypes do memory copy or repointing, for cloud this would be an different kind of update. -# usevnd = vnd # useCopy ? deepcopy(vnd) : vnd -# # should just one, or many pointers be updated? -# useExisting = -# haskey(var.solverDataDict, vnd.solveKey) && -# isa(var.solverDataDict[vnd.solveKey], State) && -# length(fields) != 0 -# # @error useExisting vnd.solveKey -# if useExisting -# # change multiple pointers inside the VND var.solverDataDict[solvekey] -# for field in fields -# destField = getfield(var.solverDataDict[vnd.solveKey], field) -# srcField = getfield(usevnd, field) -# if isa(destField, Array) && size(destField) == size(srcField) -# # use broadcast (in-place operation) -# destField .= srcField -# else -# # change pointer of destination VND object member -# setfield!(var.solverDataDict[vnd.solveKey], field, srcField) -# end -# end -# else -# # change a single pointer in var.solverDataDict -# var.solverDataDict[vnd.solveKey] = usevnd -# end - -# return var.solverDataDict[vnd.solveKey] -# end - -# function updateVariableSolverData!( -# dfg::AbstractDFG, -# variablekey::Symbol, -# vnd::State, -# solveKey::Symbol, -# useCopy::Bool = false, -# fields::Vector{Symbol} = Symbol[]; -# warn_if_absent::Bool = true, -# ) -# # TODO not very clean -# if vnd.solveKey != solveKey -# Base.depwarn( -# "updateVariableSolverData with solveKey is deprecated use copytoState! instead.", -# :updateVariableSolverData!, -# ) -# usevnd = useCopy ? deepcopy(vnd) : vnd -# usevnd.solveKey = solveKey -# return updateVariableSolverData!( -# dfg, -# variablekey, -# usevnd, -# useCopy, -# fields; -# warn_if_absent = warn_if_absent, -# ) -# else -# return updateVariableSolverData!( -# dfg, -# variablekey, -# vnd, -# useCopy, -# fields; -# warn_if_absent = warn_if_absent, -# ) -# end -# end - -# function updateVariableSolverData!( -# dfg::AbstractDFG, -# sourceVariable::VariableCompute, -# solveKey::Symbol = :default, -# useCopy::Bool = false, -# fields::Vector{Symbol} = Symbol[]; -# warn_if_absent::Bool = true, -# ) -# # -# vnd = getSolverData(sourceVariable, solveKey) -# # toshow = listSolveKeys(sourceVariable) |> collect -# # @info "update DFGVar solveKey" solveKey vnd.solveKey -# # @show toshow -# @assert solveKey == vnd.solveKey "State's solveKey=:$(vnd.solveKey) does not match requested :$solveKey" -# return updateVariableSolverData!( -# dfg, -# sourceVariable.label, -# vnd, -# useCopy, -# fields; -# warn_if_absent = warn_if_absent, -# ) -# end - -# function updateVariableSolverData!( -# dfg::AbstractDFG, -# sourceVariables::Vector{<:VariableCompute}, -# solveKey::Symbol = :default, -# useCopy::Bool = false, -# fields::Vector{Symbol} = Symbol[]; -# warn_if_absent::Bool = true, -# ) -# #I think cloud would do this in bulk for speed -# for var in sourceVariables -# updateVariableSolverData!( -# dfg, -# var.label, -# getSolverData(var, solveKey), -# useCopy, -# fields; -# warn_if_absent = warn_if_absent, -# ) -# end -# end - -# ## factor refactor deprecations -# Base.@kwdef mutable struct GenericFunctionNodeData{ -# T <: Union{<:AbstractPackedObservation, <:AbstractObservation, <:FactorCache}, -# } -# eliminated::Bool = false -# potentialused::Bool = false -# edgeIDs::Vector{Int} = Int[] -# fnc::T -# multihypo::Vector{Float64} = Float64[] # TODO re-evaluate after refactoring w #477 -# certainhypo::Vector{Int} = Int[] -# nullhypo::Float64 = 0.0 -# solveInProgress::Int = 0 -# inflation::Float64 = 0.0 -# end - -# function FactorCompute( -# label::Symbol, -# timestamp::Union{DateTime, ZonedDateTime}, -# nstime::Nanosecond, -# tags::Set{Symbol}, -# solverData::GenericFunctionNodeData, -# solvable::Int, -# variableOrder::Union{Vector{Symbol}, Tuple}; -# observation = getFactorType(solverData), -# state::FactorState = FactorState(), -# solvercache::Base.RefValue{<:FactorCache} = Ref{FactorCache}(), -# id::Union{UUID, Nothing} = nothing, -# smallData::Dict{Symbol, MetadataTypes} = Dict{Symbol, MetadataTypes}(), -# ) -# error( -# "This constructor is deprecated, use FactorCompute(label, variableOrder, solverData; ...) instead", -# ) -# return FactorCompute( -# id, -# label, -# tags, -# Tuple(variableOrder), -# timestamp, -# nstime, -# Ref(solverData), -# Ref(solvable), -# smallData, -# observation, -# state, -# solvercache, -# ) -# end - -# function getSolverData(f::FactorCompute) -# return error( -# "getSolverData(f::FactorCompute) is obsolete, use getFactorState, getObservation, or getCache instead", -# ) -# end - -# function setSolverData!(f::FactorCompute, data::GenericFunctionNodeData) -# return error( -# "setSolverData!(f::FactorCompute, data::GenericFunctionNodeData) is obsolete, use setState!, or setCache! instead", -# ) -# end - -# @deprecate unpackFactor(dfg::AbstractDFG, factor::FactorDFG; skipVersionCheck::Bool = false) unpackFactor( -# factor; -# skipVersionCheck, -# ) false - -# @deprecate rebuildFactorMetadata!(args...; kwargs...) rebuildFactorCache!( -# args...; -# kwargs..., -# ) - -# function reconstFactorData end - -# function decodePackedType( -# dfg::AbstractDFG, -# varOrder::AbstractVector{Symbol}, -# ::Type{T}, -# packeddata::GenericFunctionNodeData{PT}, -# ) where {T <: FactorCache, PT} -# error("decodePackedType is obsolete") -# # -# # TODO, to solve IIF 1424 -# # variables = map(lb->getVariable(dfg, lb), varOrder) - -# # Also look at parentmodule -# usrtyp = convertStructType(PT) -# fulltype = DFG.FunctionNodeData{T{usrtyp}} -# factordata = reconstFactorData(dfg, varOrder, fulltype, packeddata) -# return factordata -# end - -# function _packSolverData(f::FactorCompute, fnctype::AbstractObservation) -# # -# error("_packSolverData is deprecated, use seperate packing of observation #TODO") -# packtype = convertPackedType(fnctype) -# try -# packed = convert(PackedFunctionNodeData{packtype}, getSolverData(f)) #TODO getSolverData -# packedJson = packed -# return packedJson -# catch ex -# io = IOBuffer() -# showerror(io, ex, catch_backtrace()) -# err = String(take!(io)) -# msg = "Error while packing '$(f.label)' as '$fnctype', please check the unpacking/packing converters for this factor - \r\n$err" -# error(msg) -# end -# end - -# const PackedFunctionNodeData{T} = -# GenericFunctionNodeData{T} where {T <: AbstractPackedObservation} -# function PackedFunctionNodeData(args...; kw...) -# error("PackedFunctionNodeData is obsolete") -# return PackedFunctionNodeData{typeof(args[4])}(args...; kw...) -# end - -# const FunctionNodeData{T} = -# GenericFunctionNodeData{T} where {T <: Union{<:AbstractObservation, <:FactorCache}} -# FunctionNodeData(args...; kw...) = FunctionNodeData{typeof(args[4])}(args...; kw...) - -# # this is the GenericFunctionNodeData for packed types -# #TODO deprecate FactorData in favor of FactorState (with no more distinction between packed and compute) -# const FactorData = PackedFunctionNodeData{AbstractPackedObservation} - -# function FactorCompute( -# label::Symbol, -# variableOrder::Union{Vector{Symbol}, Tuple}, -# solverData::GenericFunctionNodeData; -# tags::Set{Symbol} = Set{Symbol}(), -# timestamp::Union{DateTime, ZonedDateTime} = now(localzone()), -# solvable::Int = 1, -# nstime::Nanosecond = Nanosecond(0), -# id::Union{UUID, Nothing} = nothing, -# smallData::Dict{Symbol, MetadataTypes} = Dict{Symbol, MetadataTypes}(), -# ) -# Base.depwarn( -# "`FactorCompute` constructor with `GenericFunctionNodeData` is deprecated. observation, state, and solvercache should be provided explicitly.", -# :FactorCompute, -# ) -# observation = getFactorType(solverData) -# state = FactorState( -# solverData.eliminated, -# solverData.potentialused, -# solverData.multihypo, -# solverData.certainhypo, -# solverData.nullhypo, -# solverData.solveInProgress, -# solverData.inflation, -# ) - -# if solverData.fnc isa FactorCache -# solvercache = solverData.fnc -# else -# solvercache = nothing -# end - -# return FactorCompute( -# label, -# Tuple(variableOrder), -# observation, -# state, -# solvercache; -# id, -# timestamp, -# nstime, -# tags, -# smallData, -# solvable, -# ) -# end - -# # Deprecated check usefull? # packedFnc = fncStringToData(factor.fnctype, factor.data) -# # Deprecated check usefull? # decodeType = getFactorOperationalMemoryType(dfg) -# # Deprecated check usefull? # fullFactorData = decodePackedType(dfg, factor.variableorder, decodeType, packedFnc) -# function fncStringToData(args...; kwargs...) -# @warn "fncStringToData is obsolete, called with" args kwargs -# return error("fncStringToData is obsolete.") -# end - -# #TODO make sure getFactorOperationalMemoryType is obsolete -# function getFactorOperationalMemoryType(dummy) -# return error( -# "Please extend your workspace with function getFactorOperationalMemoryType(<:AbstractParams) for your usecase, e.g. IncrementalInference uses `CommonConvWrapper <: FactorCache`", -# ) -# end -# function getFactorOperationalMemoryType(dfg::AbstractDFG) -# return getFactorOperationalMemoryType(getSolverParams(dfg)) -# end - -# function typeModuleName(variableType::StateType) -# Base.depwarn("typeModuleName is obsolete", :typeModuleName) -# io = IOBuffer() -# ioc = IOContext(io, :module => DistributedFactorGraphs) -# show(ioc, typeof(variableType)) -# return String(take!(io)) -# end - -# typeModuleName(varT::Type{<:StateType}) = typeModuleName(varT()) +updateData!(args...; kwargs...) = error("updateData! is obsolete.") +updateBlob!(args...; kwargs...) = error("updateBlob! is obsolete.") +getData(args...; kwargs...) = error("getData is obsolete, use loadBlob_Variable") +addData!(args...; kwargs...) = error("addData! is obsolete, use saveBlob_Variable!") +deleteData!(args...; kwargs...) = error("deleteData! is obsolete, use deleteBlob_Variable!") diff --git a/src/DistributedFactorGraphs.jl b/src/DistributedFactorGraphs.jl index 8afc9ef5..9931a966 100644 --- a/src/DistributedFactorGraphs.jl +++ b/src/DistributedFactorGraphs.jl @@ -52,6 +52,10 @@ using StaticArrays using InteractiveUtils: subtypes +using StructUtils: @kwarg, @tags +public @tags +public @kwarg + ##============================================================================== # Exports ##============================================================================== @@ -582,7 +586,7 @@ const unstable_functions::Vector{Symbol} = [ # :listPPEs, # :mergePPEs!, Symbol("@defVariable"), - :SmallDataTypes, + # :SmallDataTypes, :NoSolverParams, :AbstractParams, # Deprecated in v0.27 diff --git a/src/entities/DFGFactor.jl b/src/entities/DFGFactor.jl index 3c01cb38..ce523429 100644 --- a/src/entities/DFGFactor.jl +++ b/src/entities/DFGFactor.jl @@ -43,7 +43,7 @@ end # *not available without reconstruction # Packed Factor constructor -function assembleFactorName(xisyms::Union{Vector{String}, Vector{Symbol}}) +function assembleFactorName(xisyms) return Symbol(xisyms..., "_f", randstring(4)) end @@ -68,7 +68,7 @@ StructUtils.@kwarg struct FactorDFG{T <: AbstractObservation, N} <: AbstractGrap variableorder::NTuple{N, Symbol} & (choosetype = x->NTuple{length(x), Symbol},) # NOTE v0.29 renamed from _variableOrderSymbols """Variable timestamp. Accessors: [`getTimestamp`](@ref)""" - timestamp::TimeDateZone = tdz_now() # NOTE v0.29 changed from ZonedDateTime + timestamp::TimeDateZone = now_tdz() # NOTE v0.29 changed from ZonedDateTime # TODO # """(Optional) Steady (monotonic) time in nanoseconds `Nanosecond` (`Int64``)""" # nstime::Nanosecond #NOTE v0.29 REMOVED as not used, add when needed, or now as steadytime. @@ -103,7 +103,7 @@ function FactorDFG( variableorder::Union{<:Tuple, Vector{Symbol}}, observation::AbstractObservation; label::Symbol = assembleFactorName(variableorder), - timestamp::Union{TimeDateZone, ZonedDateTime} = tdz_now(), + timestamp::Union{TimeDateZone, ZonedDateTime} = now_tdz(), tags::Union{Set{Symbol}, Vector{Symbol}} = Set{Symbol}([:FACTOR]), bloblets::Bloblets = Bloblets(), multihypo::Vector{Float64} = Float64[], @@ -160,7 +160,7 @@ function FactorDFG( state::Recipestate = Recipestate(), cache = nothing; tags::Set{Symbol} = Set{Symbol}([:FACTOR]), - timestamp::Union{DateTime, ZonedDateTime, TimeDateZone} = tdz_now(), + timestamp::Union{DateTime, ZonedDateTime, TimeDateZone} = now_tdz(), solvable::Int = 1, bloblets::Bloblets = Bloblets(), blobentries::Blobentries = Blobentries(), @@ -235,7 +235,7 @@ end function FactorSummary( label::Symbol, variableorder::Union{Vector{Symbol}, Tuple}; - timestamp::TimeDateZone = tdz_now(), + timestamp::TimeDateZone = now_tdz(), tags::Set{Symbol} = Set{Symbol}(), ) return FactorSummary(label, tags, Tuple(variableorder), timestamp) diff --git a/src/entities/DFGVariable.jl b/src/entities/DFGVariable.jl index ac0f8280..25165fad 100644 --- a/src/entities/DFGVariable.jl +++ b/src/entities/DFGVariable.jl @@ -128,6 +128,15 @@ end ##------------------------------------------------------------------------------ # The Variable information packed in a way that accomdates multi-lang using json. +variable_timestamp_note = """ +!!! note + This single timestamp does not represent the temporal uncertainty of non-parametric beliefs. + A single timestamp value cannot capture the distribution of temporal + information across all particles/points in a belief. For problems where time is a state variable + requiring inference (e.g., `SGal3` which includes temporal components), include time as part of + your state type rather than relying on this metadata field. +""" + """ $(TYPEDEF) Complete variable structure for a DistributedFactorGraph variable. @@ -140,12 +149,12 @@ $(TYPEDFIELDS) """Variable label, e.g. :x1. Accessor: [`getLabel`](@ref)""" label::Symbol - """Variable timestamp. + """Variable event timestamp (UTC-based) with timezone support. + $variable_timestamp_note Accessors: [`getTimestamp`](@ref)""" - timestamp::TimeDateZone = tdz_now() #NOTE changed to TimeDateZone in v0.29 + timestamp::TimeDateZone = now_tdz() #NOTE changed to TimeDateZone in v0.29 # """Nanoseconds since a user-understood epoch (i.e unix epoch, robot boot time, etc.)""" - # steadytime::Union{Nothing, Nanosecond} = nothing #NOTE changed to TimeDateZone in v0.29 - #nstime::String = "0" #NOTE different uses, as 0-999_999 nanosecond part of timestamp now in timestamp, as steady timestamp now in steadytime + # nstime::String = "0" #NOTE deprecated field in v0.29 """Variable tags, e.g [:POSE, :VARIABLE, and :LANDMARK]. Accessors: [`listTags`](@ref), [`mergeTags!`](@ref), and [`deleteTags!`](@ref)""" tags::Set{Symbol} = Set{Symbol}() @@ -174,7 +183,7 @@ function StructUtils.fielddefaults( ::Type{VariableDFG{T, P, N}}, ) where {T, P, N} return ( - timestamp = tdz_now(), + timestamp = now_tdz(), tags = Set{Symbol}(), # states = OrderedDict{Symbol, State{T, P, N}}(), bloblets = Bloblets(), @@ -213,9 +222,8 @@ function VariableDFG( label::Symbol, statetype::Union{T, Type{T}}; tags::Union{Set{Symbol}, Vector{Symbol}} = Set{Symbol}(), - timestamp::Union{TimeDateZone, ZonedDateTime} = tdz_now(), + timestamp::Union{TimeDateZone, ZonedDateTime} = now_tdz(), solvable::Union{Int, Base.RefValue{Int}} = Ref{Int}(1), - # steadytime::Union{Nothing, Nanosecond} = nothing, nanosecondtime = nothing, smalldata = nothing, kwargs..., @@ -285,7 +293,8 @@ $(TYPEDFIELDS) """Variable label, e.g. :x1. Accessor: [`getLabel`](@ref)""" label::Symbol - """Variable timestamp. + """Variable event timestamp. + $variable_timestamp_note Accessors: [`getTimestamp`](@ref)""" timestamp::TimeDateZone """Variable tags, e.g [:POSE, :VARIABLE, and :LANDMARK]. diff --git a/src/serialization/PackedSerialization.jl b/src/serialization/PackedSerialization.jl index 8bbbebcc..2c300af6 100644 --- a/src/serialization/PackedSerialization.jl +++ b/src/serialization/PackedSerialization.jl @@ -84,3 +84,38 @@ end # push!(md, kwargs...) # return md # end + +""" + @packed + +Macro annotation for DFG serialization metadata on struct fields. +Expands to `(lower = DFG.Packed, choosetype = DFG.resolvePackedType)` for use with +StructTypes.jl field annotations. + +Used to mark belief fields in factor types for serialization through +the DFG packing system. The `lower` function converts the field to a `Packed` wrapper +during serialization, and `choosetype` resolves the correct type during deserialization. + +# Usage +Use with the `&` operator in `@kwarg` or `@tags` struct definitions: + +```julia +@kwarg struct Pose2Point2Range{T} <: AbstractRelativeObservation + Z::T & DFG.@packed + partial::Tuple{Int, Int} = (1, 2) +end +``` + +This is equivalent to writing: +```julia +@kwarg struct Pose2Point2Range{T} <: AbstractRelativeObservation + Z::T & (lower = DFG.Packed, choosetype = DFG.resolvePackedType) + partial::Tuple{Int, Int} = (1, 2) +end +``` + +See also: [`Packed`](@ref), [`pack`](@ref), [`unpack`](@ref), [`resolvePackedType`](@ref) +""" +macro packed() + return esc(:(lower = DFG.Packed, choosetype = DFG.resolvePackedType)) +end diff --git a/src/services/AbstractDFG.jl b/src/services/AbstractDFG.jl index 45bc3c39..b92c460a 100644 --- a/src/services/AbstractDFG.jl +++ b/src/services/AbstractDFG.jl @@ -359,17 +359,17 @@ function listNeighbors end """ $(SIGNATURES) Get a VariableDFG with a specific solver key. -In memory types still return a reference, other types returns a variable with only solveKey. +In memory types still return a reference, other types returns a variable with only stateLabel. """ -function getVariable(dfg::AbstractDFG, label::Symbol, solveKey::Symbol) - # TODO maybe change solveKey param to stateLabelFilter +function getVariable(dfg::AbstractDFG, label::Symbol, stateLabel::Symbol) + # TODO maybe change stateLabel param to stateLabelFilter # function getVariable(dfg::AbstractDFG, label::Symbol; stateLabelFilter::Union{Nothing, ...} = nothing) var = getVariable(dfg, label) - if isa(var, VariableDFG) && !haskey(var.states, solveKey) - throw(LabelNotFoundError("VariableNode", solveKey)) + if isa(var, VariableDFG) && !haskey(var.states, stateLabel) + throw(LabelNotFoundError("VariableNode", stateLabel)) elseif !isa(var, VariableDFG) - @warn "getVariable(dfg, label, solveKey) only supported for type VariableDFG." + @warn "getVariable(dfg, label, stateLabel) only supported for type VariableDFG." end return var @@ -559,7 +559,12 @@ function deepcopyGraph( graphLabel::Symbol = Symbol(getGraphLabel(sourceDFG), "_cp_$(string(uuid4())[1:6])"), kwargs..., ) where {T <: AbstractDFG} - destDFG = T(; graph = sourceDFG.graph, agent = sourceDFG.agent, graphLabel) + destDFG = T(; + solverParams = getSolverParams(sourceDFG), + graph = sourceDFG.graph, + agent = sourceDFG.agent, + graphLabel, + ) copyGraph!( destDFG, sourceDFG, diff --git a/src/services/CustomPrinting.jl b/src/services/CustomPrinting.jl index 691520d8..202c08f9 100644 --- a/src/services/CustomPrinting.jl +++ b/src/services/CustomPrinting.jl @@ -11,7 +11,6 @@ function printVariable( compact::Bool = true, limit::Bool = true, skipfields::Vector{Symbol} = Symbol[], - solveKeys::Vector{Symbol} = Symbol[], ) ioc = IOContext(io, :limit => limit, :compact => compact)