[ENH] MI and CMI simulation module and relevant estimators from the forest

.gitignore

@@ -11,6 +11,8 @@ coverage
 commit.txt
 sktree/_lib/sklearn/
 
+*.png
+
 # Sphinx documentation
 docs/_build/
 docs/generated/

.spin/cmds.py

@@ -111,6 +111,7 @@ def setup_submodule(forcesubmodule=False):
                 commit_fpath,
             ],
         )
+        print(commit_fpath)
         with open(commit_fpath, "w") as f:
             f.write(current_hash)
 

README.md

@@ -101,6 +101,10 @@ You can also do the same thing using Meson/Ninja itself. Run the following to bu
     python -c "from sktree import tree"
     python -c "import sklearn; print(sklearn.__version__);"
 
+Alternatively, you can use editable installs
+
+    pip install --no-build-isolation --editable .
+
 References
 ==========
 [1]: [`Li, Adam, et al. "Manifold Oblique Random Forests: Towards Closing the Gap on Convolutional Deep Networks." arXiv preprint arXiv:1909.11799 (2019)`](https://arxiv.org/abs/1909.11799)

docs/api.rst

@@ -65,3 +65,59 @@ The trees that comprise those forests are also available as standalone classes.
 
    tree.UnsupervisedDecisionTree
    tree.UnsupervisedObliqueDecisionTree
+
+
+Distance Metrics
+----------------
+Trees inherently produce a "distance-like" metric. We provide an API for
+extracting pairwise distances from the trees that include a correction
+that turns the "tree-distance" into a proper distance metric.
+
+.. currentmodule:: sktree.tree
+.. autosummary::
+   :toctree: generated/
+
+   compute_forest_similarity_matrix
+
+In addition to providing a distance metric based on leaves, tree-models
+provide a natural way to compute neighbors based on the splits. We provide
+an API for extracting the nearest neighbors from a tree-model. This provides
+an API-like interface similar to :class:`~sklearn.neighbors.NearestNeighbors`.
+
+.. currentmodule:: sktree
+.. autosummary::
+   :toctree: generated/
+
+   NearestNeighborsMetaEstimator
+
+
+Experimental Functionality
+--------------------------
+We also include experimental functionality that is works in progress.
+
+.. currentmodule:: sktree.experimental
+.. autosummary::
+   :toctree: generated/
+
+   mutual_info_ksg
+
+We also include functions that help simulate and evaluate mutual information (MI)
+and conditional mutual information (CMI) estimators. Specifically, functions that
+help simulate multivariate gaussian data and compute the analytical solutions
+for the entropy, MI and CMI of the Gaussian distributions.
+
+.. currentmodule:: sktree.experimental.simulate
+.. autosummary::
+   :toctree: generated/
+
+   simulate_multivariate_gaussian
+   simulate_helix
+   simulate_sphere
+
+.. currentmodule:: sktree.experimental.mutual_info
+.. autosummary::
+   :toctree: generated/
+
+   mi_gaussian
+   cmi_gaussian
+   entropy_gaussian

docs/conf.py

@@ -162,6 +162,7 @@
     "PatchObliqueDecisionTreeClassifier": "sktree.tree.PatchObliqueDecisionTreeClassifier",
     "ObliqueDecisionTreeRegressor": "sktree.tree.ObliqueDecisionTreeRegressor",
     "PatchObliqueDecisionTreeRegressor": "sktree.tree.PatchObliqueDecisionTreeRegressor",
+    "UnsupervisedObliqueRandomForest": "sktree.ensemble.UnsupervisedObliqueRandomForest",
     "DecisionTreeClassifier": "sklearn.tree.DecisionTreeClassifier",
     "DecisionTreeRegressor": "sklearn.tree.DecisionTreeRegressor",
     "pipeline.Pipeline": "sklearn.pipeline.Pipeline",
@@ -204,6 +205,19 @@
     "_type_",
     "MetadataRequest",
     "~utils.metadata_routing.MetadataRequest",
+    "quantiles",
+    "n_quantiles",
+    "metric",
+    "n_queries",
+    "BaseForest",
+    "BaseDecisionTree",
+    "n_indexed",
+    "n_queries",
+    "n_features_x",
+    "n_features_y",
+    "n_features_z",
+    "n_neighbors",
+    "one",
 }
 
 # validation
@@ -354,6 +368,8 @@ def replace_sklearn_fork_with_sklearn(app, what, name, obj, options, lines):
     # Use regular expressions to replace 'sklearn_fork' with 'sklearn'
     content = re.sub(r"`pipeline.Pipeline", r"`~sklearn.pipeline.Pipeline", content)
     content = re.sub(r"`~utils.metadata_routing.MetadataRequest", r"``MetadataRequest``", content)
+    content = re.sub(r"`np.quantile", r"`numpy.quantile", content)
+    content = re.sub(r"`~np.quantile", r"`numpy.quantile", content)
 
     # Convert the modified string back to a list of lines
     lines[:] = content.split("\n")

docs/references.bib

@@ -54,4 +54,29 @@ @article{TomitaSPORF2020
   number  = {104},
   pages   = {1--39},
   url     = {http://jmlr.org/papers/v21/18-664.html}
+}
+
+@article{Darbellay1999Entropy,
+  title={Estimation of the Information by an Adaptive Partitioning of the Observation Space},
+  author={Georges A. Darbellay and Igor Vajda},
+  journal={IEEE Trans. Inf. Theory},
+  year={1999},
+  volume={45},
+  pages={1315-1321}
+}
+
+@article{Kraskov_2004,
+  title    = {Estimating mutual information},
+  volume   = {69},
+  url      = {https://link.aps.org/doi/10.1103/PhysRevE.69.066138},
+  doi      = {10.1103/PhysRevE.69.066138},
+  number   = {6},
+  urldate  = {2023-01-27},
+  journal  = {Physical Review E},
+  author   = {Kraskov, Alexander and Stögbauer, Harald and Grassberger, Peter},
+  month    = jun,
+  year     = {2004},
+  note     = {Publisher: American Physical Society},
+  pages    = {066138},
+  file     = {APS Snapshot:/Users/adam2392/Zotero/storage/GRW23BYU/PhysRevE.69.html:text/html;Full Text PDF:/Users/adam2392/Zotero/storage/NJT9QCVA/Kraskov et al. - 2004 - Estimating mutual information.pdf:application/pdf}
 }

docs/whats_new/v0.1.rst

@@ -36,6 +36,8 @@ Changelog
 - |Feature| A general-kernel MORF is now implemented where users can pass in a kernel library, by `Adam Li`_ (:pr:`70`)
 - |Feature| Implementation of ObliqueDecisionTreeRegressor, PatchObliqueDecisionTreeRegressor, ObliqueRandomForestRegressor, PatchObliqueRandomForestRegressor, by `SUKI-O`_ (:pr:`72`)
 - |Feature| Implementation of HonestTreeClassifier, HonestForestClassifier, by `Sambit Panda`_, `Adam Li`_, `Ronan Perry`_ and `Haoyin Xu`_ (:pr:`57`)
+- |Feature| Implementation of (conditional) mutual information estimation via unsupervised tree models and added NearestNeighborsMetaEstimator by `Adam Li`_ (:pr:`83`)
+
 
 Code and Documentation Contributors
 -----------------------------------