diff --git a/Project.toml b/Project.toml
index 661aaf6e..10734ae0 100644
--- a/Project.toml
+++ b/Project.toml
@@ -7,6 +7,7 @@ version = "0.14.12"
 ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
 ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0"
 Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
+ConstantRNGs = "aa9b60e7-6b1c-4c29-a6e5-e43521412437"
 ConstructionBase = "187b0558-2788-49d3-abe0-74a17ed4e7c9"
 DensityInterface = "b429d917-457f-4dbc-8f4c-0cc954292b1d"
 FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b"
@@ -33,6 +34,7 @@ Tricks = "410a4b4d-49e4-4fbc-ab6d-cb71b17b3775"
 ChainRulesCore = "1"
 ChangesOfVariables = "0.1.3"
 Compat = "3.35, 4"
+ConstantRNGs = "0.1.1"
 ConstructionBase = "1.3"
 DensityInterface = "0.4"
 FillArrays = "0.12, 0.13, 1"
diff --git a/src/MeasureBase.jl b/src/MeasureBase.jl
index 5918c250..ccc808d3 100644
--- a/src/MeasureBase.jl
+++ b/src/MeasureBase.jl
@@ -22,6 +22,7 @@ using DensityInterface
 
 using InverseFunctions
 using ChangesOfVariables
+using ConstantRNGs
 
 import Base.iterate
 import ConstructionBase
@@ -147,7 +148,6 @@ include("standard/stdnormal.jl")
 include("combinators/half.jl")
 
 include("rand.jl")
-include("fixedrng.jl")
 
 include("density.jl")
 include("density-core.jl")
diff --git a/src/fixedrng.jl b/src/fixedrng.jl
deleted file mode 100644
index 232b0891..00000000
--- a/src/fixedrng.jl
+++ /dev/null
@@ -1,19 +0,0 @@
-export FixedRNG
-struct FixedRNG <: AbstractRNG end
-
-Base.rand(::FixedRNG) = one(Float64) / 2
-Random.randn(::FixedRNG) = zero(Float64)
-Random.randexp(::FixedRNG) = one(Float64)
-
-Base.rand(::FixedRNG, ::Type{T}) where {T<:Real} = one(T) / 2
-Random.randn(::FixedRNG, ::Type{T}) where {T<:Real} = zero(T)
-Random.randexp(::FixedRNG, ::Type{T}) where {T<:Real} = one(T)
-
-# We need concrete type parameters to avoid amiguity for these cases
-for T in [Float16, Float32, Float64]
-    @eval begin
-        Base.rand(::FixedRNG, ::Type{$T}) = one($T) / 2
-        Random.randn(::FixedRNG, ::Type{$T}) = zero($T)
-        Random.randexp(::FixedRNG, ::Type{$T}) = one($T)
-    end
-end
diff --git a/src/utils.jl b/src/utils.jl
index d0f85481..4b841c29 100644
--- a/src/utils.jl
+++ b/src/utils.jl
@@ -13,7 +13,7 @@ export testvalue
 
 @inline testvalue(μ) = testvalue(Float64, μ)
 
-@inline testvalue(::Type{T}, μ) where {T} = rand(FixedRNG(), T, μ)
+@inline testvalue(::Type{T}, μ) where {T} = rand(ConstantRNG(), T, μ)
 
 testvalue(::Type{T}) where {T} = zero(T)