Skip to content

Commit

Permalink
Merge branch 'develop' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
federicoVisi committed Mar 25, 2021
2 parents 66901f4 + d807eee commit 9f97972
Show file tree
Hide file tree
Showing 28 changed files with 9,543 additions and 10,762 deletions.
1,433 changes: 1,433 additions & 0 deletions Examples/GIMLeT_example_MiM_1ANN.maxpat

Large diffs are not rendered by default.

Large diffs are not rendered by default.

183 changes: 183 additions & 0 deletions Examples/GIMLeT_temporal_example_settings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
{
"pattrstorage" : {
"name" : "GIMLeT_temporal_example_settings",
"slots" : {
"1" : {
"id" : 1,
"data" : {
"Sound Synthesis::GIMLeT.grain::alphaMax" : [ 4.0 ],
"Sound Synthesis::GIMLeT.grain::alphaMin" : [ 4.0 ],
"Sound Synthesis::GIMLeT.grain::density" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain::gainMax" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain::gainMin[1]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain::gswitch2" : [ 0 ],
"Sound Synthesis::GIMLeT.grain::gswitch2[10]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain::gswitch2[11]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain::gswitch2[12]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain::gswitch2[13]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain::gswitch2[14]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain::gswitch2[1]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain::gswitch2[2]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain::gswitch2[3]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain::gswitch2[4]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain::gswitch2[5]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain::gswitch2[6]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain::gswitch2[7]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain::gswitch2[8]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain::gswitch2[9]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain::lengthMax" : [ 150.0 ],
"Sound Synthesis::GIMLeT.grain::lengthMin" : [ 150.0 ],
"Sound Synthesis::GIMLeT.grain::live.gain~" : [ -6.0 ],
"Sound Synthesis::GIMLeT.grain::live.text" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain::live.text[10]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain::live.text[11]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain::live.text[12]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain::live.text[1]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain::live.text[2]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain::live.text[3]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain::live.text[4]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain::live.text[5]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain::live.text[6]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain::live.text[7]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain::live.text[8]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain::live.text[9]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain::number" : [ 0.5 ],
"Sound Synthesis::GIMLeT.grain::number[1]" : [ 50.0 ],
"Sound Synthesis::GIMLeT.grain::number[2]" : [ 2001 ],
"Sound Synthesis::GIMLeT.grain::number[3]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain::number[4]" : [ 1001 ],
"Sound Synthesis::GIMLeT.grain::offsetMax" : [ 2.0 ],
"Sound Synthesis::GIMLeT.grain::offsetMin" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain::panMax" : [ -0.28571429848671 ],
"Sound Synthesis::GIMLeT.grain::panMin" : [ 0.28571429848671 ],
"Sound Synthesis::GIMLeT.grain::pitchMax" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain::pitchMin" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain::textbutton" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[2]::alphaMax" : [ 4.0 ],
"Sound Synthesis::GIMLeT.grain[2]::alphaMin" : [ 4.0 ],
"Sound Synthesis::GIMLeT.grain[2]::density" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[2]::gainMax" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[2]::gainMin[1]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[2]::gswitch2" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[2]::gswitch2[10]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[2]::gswitch2[11]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain[2]::gswitch2[12]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain[2]::gswitch2[13]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain[2]::gswitch2[14]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[2]::gswitch2[1]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[2]::gswitch2[2]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[2]::gswitch2[3]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain[2]::gswitch2[4]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain[2]::gswitch2[5]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain[2]::gswitch2[6]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain[2]::gswitch2[7]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[2]::gswitch2[8]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[2]::gswitch2[9]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[2]::lengthMax" : [ 150.0 ],
"Sound Synthesis::GIMLeT.grain[2]::lengthMin" : [ 150.0 ],
"Sound Synthesis::GIMLeT.grain[2]::live.gain~" : [ -6.0 ],
"Sound Synthesis::GIMLeT.grain[2]::live.text" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain[2]::live.text[10]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[2]::live.text[11]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[2]::live.text[12]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain[2]::live.text[1]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain[2]::live.text[2]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[2]::live.text[3]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[2]::live.text[4]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain[2]::live.text[5]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain[2]::live.text[6]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain[2]::live.text[7]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain[2]::live.text[8]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[2]::live.text[9]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[2]::number" : [ 0.5 ],
"Sound Synthesis::GIMLeT.grain[2]::number[1]" : [ 50.0 ],
"Sound Synthesis::GIMLeT.grain[2]::number[2]" : [ 2002 ],
"Sound Synthesis::GIMLeT.grain[2]::number[3]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[2]::number[4]" : [ 1001 ],
"Sound Synthesis::GIMLeT.grain[2]::offsetMax" : [ 2.0 ],
"Sound Synthesis::GIMLeT.grain[2]::offsetMin" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[2]::panMax" : [ -0.28571429848671 ],
"Sound Synthesis::GIMLeT.grain[2]::panMin" : [ 0.28571429848671 ],
"Sound Synthesis::GIMLeT.grain[2]::pitchMax" : [ 3.465133858267725 ],
"Sound Synthesis::GIMLeT.grain[2]::pitchMin" : [ 1.953511811023631 ],
"Sound Synthesis::GIMLeT.grain[2]::textbutton" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[1]::alphaMax" : [ 4.0 ],
"Sound Synthesis::GIMLeT.grain[1]::alphaMin" : [ 4.0 ],
"Sound Synthesis::GIMLeT.grain[1]::density" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[1]::gainMax" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[1]::gainMin[1]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[1]::gswitch2" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[1]::gswitch2[10]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[1]::gswitch2[11]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain[1]::gswitch2[12]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain[1]::gswitch2[13]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain[1]::gswitch2[14]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[1]::gswitch2[1]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[1]::gswitch2[2]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain[1]::gswitch2[3]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain[1]::gswitch2[4]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain[1]::gswitch2[5]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain[1]::gswitch2[6]" : [ 1 ],
"Sound Synthesis::GIMLeT.grain[1]::gswitch2[7]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[1]::gswitch2[8]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[1]::gswitch2[9]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[1]::lengthMax" : [ 150.0 ],
"Sound Synthesis::GIMLeT.grain[1]::lengthMin" : [ 150.0 ],
"Sound Synthesis::GIMLeT.grain[1]::live.gain~" : [ -6.0 ],
"Sound Synthesis::GIMLeT.grain[1]::live.text" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain[1]::live.text[10]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[1]::live.text[11]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain[1]::live.text[12]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain[1]::live.text[1]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain[1]::live.text[2]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[1]::live.text[3]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[1]::live.text[4]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain[1]::live.text[5]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain[1]::live.text[6]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain[1]::live.text[7]" : [ 1.0 ],
"Sound Synthesis::GIMLeT.grain[1]::live.text[8]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[1]::live.text[9]" : [ 0.0 ],
"Sound Synthesis::GIMLeT.grain[1]::number" : [ 0.5 ],
"Sound Synthesis::GIMLeT.grain[1]::number[1]" : [ 50.0 ],
"Sound Synthesis::GIMLeT.grain[1]::number[2]" : [ 2003 ],
"Sound Synthesis::GIMLeT.grain[1]::number[3]" : [ 0 ],
"Sound Synthesis::GIMLeT.grain[1]::number[4]" : [ 1001 ],
"Sound Synthesis::GIMLeT.grain[1]::offsetMax" : [ 18.66656494140625 ],
"Sound Synthesis::GIMLeT.grain[1]::offsetMin" : [ 16.66656494140625 ],
"Sound Synthesis::GIMLeT.grain[1]::panMax" : [ 0.371119499206543 ],
"Sound Synthesis::GIMLeT.grain[1]::panMin" : [ -0.371119499206543 ],
"Sound Synthesis::GIMLeT.grain[1]::pitchMax" : [ 1.944763779527558 ],
"Sound Synthesis::GIMLeT.grain[1]::pitchMin" : [ 1.440889763779527 ],
"Sound Synthesis::GIMLeT.grain[1]::textbutton" : [ 0 ],
"Gesture Folllowing::GIMLeT.ml.temporal::gswitch2" : [ 0 ],
"Gesture Folllowing::GIMLeT.ml.temporal::gswitch2[1]" : [ 0 ],
"Gesture Folllowing::GIMLeT.ml.temporal::live.text" : [ 0.0 ],
"Gesture Folllowing::GIMLeT.ml.temporal::live.text[1]" : [ 0.0 ],
"Gesture Folllowing::GIMLeT.ml.temporal::live.text[2]" : [ 0.0 ],
"Gesture Folllowing::GIMLeT.ml.temporal::live.text[3]" : [ 0.0 ],
"Gesture Folllowing::GIMLeT.ml.temporal::live.text[4]" : [ 0.0 ],
"Gesture Folllowing::GIMLeT.ml.temporal::live.text[5]" : [ 0.0 ],
"Gesture Folllowing::GIMLeT.ml.temporal::live.text[6]" : [ 1.0 ],
"Gesture Folllowing::GIMLeT.ml.temporal::live.text[7]" : [ 1.0 ],
"Gesture Folllowing::GIMLeT.ml.temporal::live.text[8]" : [ 0.0 ],
"Gesture Folllowing::GIMLeT.ml.temporal::live.toggle[2]" : [ 0.0 ],
"Gesture Folllowing::GIMLeT.ml.temporal::multislider[39]" : [ 0.0 ],
"Gesture Folllowing::GIMLeT.ml.temporal::multislider[40]" : [ 0.0 ],
"Gesture Folllowing::GIMLeT.ml.temporal::multislider[41]" : [ 0.0 ],
"Gesture Folllowing::GIMLeT.ml.temporal::multislider[42]" : [ 0.210171490907669, 0.959999978542328 ],
"Gesture Folllowing::GIMLeT.ml.temporal::number" : [ 1 ],
"Gesture Folllowing::GIMLeT.ml.temporal::number[1]" : [ 2002 ],
"Gesture Folllowing::GIMLeT.ml.temporal::number[3]" : [ 1001 ],
"Gesture Folllowing::GIMLeT.ml.temporal::number[4]" : [ 2001 ],
"Gesture Folllowing::GIMLeT.ml.temporal::number[5]" : [ 2003 ],
"Gesture Folllowing::GIMLeT.ml.temporal::number[6]" : [ 2004 ],
"Gesture Folllowing::GIMLeT.ml.temporal::number[7]" : [ 8000 ]
}

}

}

}

}
Binary file added Media/bass.wav
Binary file not shown.
Binary file modified Media/cello.wav
Binary file not shown.
Binary file added Media/cymbal.wav
Binary file not shown.
Binary file added Media/electric_piano.wav
Binary file not shown.
33 changes: 23 additions & 10 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,21 @@

A set of Max patches for gesture analysis, interactive machine learning, and gesture-sound interaction design. GIMLeT features a modular design that allows to easily share meaningfully structured data between several gesture tracking devices, machine learning, and sound synthesis modules.

## Video Tutorial
https://youtu.be/Dace1sHy1IM
## Video Tutorials
1. Installation and linear regression with artifical neural networks: https://youtu.be/Dace1sHy1IM
2. Gesture following with PoseNet and GVF: https://youtu.be/GoNqiCvVgoY

## Installation
Download the .zip file, open it, copy the GIMLeT folder in your Max "Packages" folder.
### Launching Example Patches
### Install the required packages
1. Download the modosc package .zip file: https://github.com/motiondescriptors/modosc/archive/main.zip
2. Open the .zip file and copy the `modosc` folder in your `/Max 8/Packages` folder.
3. Download the GIMLeT package .zip file: https://github.com/federicoVisi/GIMLeT/archive/main.zip
4. Open the .zip file and copy the `GIMLeT` folder in your `/Max 8/Packages` folder.

### Launch the example patches
Launch Max, click on Extras->"GIMLeT examples" on the menu bar, choose an example.
### Installing the TouchOSC layout

### Install the TouchOSC layout
- Install TouchOSC on your smarphone (iOS or Android, you'll find it on the app store)
- Sync the /TouchOSC/GIMLeT_TouchOSC_remote.touchosc using this guide: https://hexler.net/docs/touchosc-editor-sync
- Connect TouchOSC to your computer followng this guide: https://hexler.net/docs/touchosc-configuration-connections-osc
Expand All @@ -19,10 +26,7 @@ Launch Max, click on Extras->"GIMLeT examples" on the menu bar, choose an exampl
NOTE: the required objects from these libraries are included in the package in order to make distribution easier.

- o.dot : OSC-centred multipardigm dynamic programming in Max
https://github.com/CNMAT/CNMAT-odot

- modosc : real-time motion feature extraction
https://github.com/motiondescriptors/modosc
https://github.com/CNMAT/CNMAT-odot

- rapidmax : Max external for interactive machine learning
https://github.com/samparkewolfe/RapidMax (Mac)
Expand All @@ -36,14 +40,23 @@ NOTE: the required objects from these libraries are included in the package in o

- HfMT Optitrack OSC bridge
https://github.com/HfMT-ZM4/motion-tracking


## Literature
### Book chapter with an overview of interactive machine learning of musical gesture
Visi, F. G., & Tanaka, A. (2021). Interactive Machine Learning of Musical Gesture. In E. R. Miranda (Ed.), Handbook of Artificial Intelligence for Music: Foundations, Advanced Approaches, and Developments for Creativity. Springer, forthcoming. Preprint: http://arxiv.org/abs/2011.13487

### Paper on the Gesture Variation Follower algorithm
Caramiaux, B., Montecchio, N., Tanaka, A., & Bevilacqua, F. (2014). Adaptive Gesture Recognition with Variation Estimation for Interactive Systems. ACM Transactions on Interactive Intelligent Systems, 4(4), 1–34. https://doi.org/10.1145/2643204

## Acknowledgements

KiSS: Kinetics in Sound and Space – HfMT Hamburg, Germany.

`gimlet.mangle` is based on a synth design by Atau Tanaka.
The data recorder in `gimlet.ml.ann` is based on a design by Michael Zbyszyński.

GEMM))) Gesture Embodiment and Machines in Music – Piteå School of Music – Luleå University of Technology, Sweden.

## Contact

mail[at]federicovisi[dot]com
Expand Down
21 changes: 11 additions & 10 deletions extras/GIMLeT examples.maxpat
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@
"appversion" : {
"major" : 8,
"minor" : 1,
"revision" : 3,
"revision" : 10,
"architecture" : "x64",
"modernui" : 1
}
,
"classnamespace" : "box",
"rect" : [ 542.0, 194.0, 697.0, 690.0 ],
"rect" : [ 508.0, 171.0, 697.0, 690.0 ],
"bglocked" : 0,
"openinpresentation" : 0,
"default_fontsize" : 12.0,
Expand All @@ -37,6 +37,7 @@
"tags" : "",
"style" : "",
"subpatcher_template" : "",
"assistshowspatchername" : 0,
"boxes" : [ {
"box" : {
"id" : "obj-25",
Expand All @@ -45,7 +46,7 @@
"numinlets" : 1,
"numoutlets" : 0,
"patching_rect" : [ 14.0, 541.080795057526075, 678.0, 47.0 ],
"text" : "Motion tracking: TouchOSC accelerometers\nMachine learning: gesture variation follower\nSound synthesis: 4 grain synths"
"text" : "Motion tracking: PoseNet using Node for Max\nMachine learning: Gesture Variation Follower\nSound synthesis: 3 grain synths"
}

}
Expand Down Expand Up @@ -81,8 +82,8 @@
"numinlets" : 2,
"numoutlets" : 1,
"outlettype" : [ "" ],
"patching_rect" : [ 14.0, 514.080795057526075, 250.0, 25.0 ],
"text" : "GIMLeT_example_temporalAnalysis"
"patching_rect" : [ 14.0, 514.080795057526075, 316.0, 25.0 ],
"text" : "GIMLeT_example_PoseNet_temporalAnalysis"
}

}
Expand Down Expand Up @@ -341,7 +342,7 @@
],
"dependency_cache" : [ {
"name" : "Gimlet_logo_solo.svg",
"bootpath" : "~/Dropbox/_FV_git/GitHub/GIMLeT/media",
"bootpath" : "~/Documents/Max 8/Packages/GIMLeT/media",
"patcherrelativepath" : "../media",
"type" : "svg",
"implicit" : 1
Expand All @@ -351,8 +352,8 @@
"styles" : [ {
"name" : "filtergraphBronze",
"default" : {
"color" : [ 0.010881, 0.909804, 0.896768, 1.0 ],
"bgcolor" : [ 0.285714, 0.256629, 0.217237, 1.0 ]
"bgcolor" : [ 0.285714, 0.256629, 0.217237, 1.0 ],
"color" : [ 0.010881, 0.909804, 0.896768, 1.0 ]
}
,
"parentstyle" : "",
Expand Down Expand Up @@ -415,8 +416,8 @@
, {
"name" : "newobjYellow-1",
"default" : {
"fontsize" : [ 12.059008 ],
"accentcolor" : [ 0.82517, 0.78181, 0.059545, 1.0 ]
"accentcolor" : [ 0.82517, 0.78181, 0.059545, 1.0 ],
"fontsize" : [ 12.059008 ]
}
,
"parentstyle" : "",
Expand Down
4 changes: 2 additions & 2 deletions package-info.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"author" : "Federico Ghelli Visi",
"description" : "A set of Max patches for gesture analysis, interactive machine learning, and gesture-sound interaction design.",
"homepatcher" : "GIMLeT examples.maxpat",
"max_version_min" : "7.1", "max_version_max" : "none",
"max_version_min" : "8.1", "max_version_max" : "none",
"name" : "GIMLeT",
"os" : {
"macintosh" : {
Expand All @@ -18,6 +18,6 @@

},
"tags" : ["interaction design", "machine learning"],
"version" : "0.2",
"version" : "2.0",
"website" : "http://www.federicovisi.com"
}
Loading

0 comments on commit 9f97972

Please sign in to comment.