Dev/point process decoder design

Bonsai.ML.sln

@@ -1,4 +1,4 @@
-
+
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.0.31903.59
@@ -30,6 +30,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.LinearDynamicalSy
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.HiddenMarkovModels.Design", "src\Bonsai.ML.HiddenMarkovModels.Design\Bonsai.ML.HiddenMarkovModels.Design.csproj", "{FC395DDC-62A4-4E14-A198-272AB05B33C7}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.Torch", "src\Bonsai.ML.Torch\Bonsai.ML.Torch.csproj", "{06FCC9AF-CE38-44BB-92B3-0D451BE88537}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.PointProcessDecoder", "src\Bonsai.ML.PointProcessDecoder\Bonsai.ML.PointProcessDecoder.csproj", "{AD32C680-1E8C-4340-81B1-DA19C9104516}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bonsai.ML.PointProcessDecoder.Design", "src\Bonsai.ML.PointProcessDecoder.Design\Bonsai.ML.PointProcessDecoder.Design.csproj", "{91C3E252-9457-43AB-A21A-6064E2404BAA}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -60,18 +66,30 @@ Global
 		{39A4414F-52B1-42D7-82FA-E65DAD885264}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{39A4414F-52B1-42D7-82FA-E65DAD885264}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{39A4414F-52B1-42D7-82FA-E65DAD885264}.Release|Any CPU.Build.0 = Release|Any CPU
-        {A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-        {A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13}.Debug|Any CPU.Build.0 = Debug|Any CPU
-        {A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13}.Release|Any CPU.ActiveCfg = Release|Any CPU
-        {A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13}.Release|Any CPU.Build.0 = Release|Any CPU
-        {17DF50BE-F481-4904-A4C8-5DF9725B2CA1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-        {17DF50BE-F481-4904-A4C8-5DF9725B2CA1}.Debug|Any CPU.Build.0 = Debug|Any CPU
-        {17DF50BE-F481-4904-A4C8-5DF9725B2CA1}.Release|Any CPU.ActiveCfg = Release|Any CPU
-        {17DF50BE-F481-4904-A4C8-5DF9725B2CA1}.Release|Any CPU.Build.0 = Release|Any CPU
-        {FC395DDC-62A4-4E14-A198-272AB05B33C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-        {FC395DDC-62A4-4E14-A198-272AB05B33C7}.Debug|Any CPU.Build.0 = Debug|Any CPU
-        {FC395DDC-62A4-4E14-A198-272AB05B33C7}.Release|Any CPU.ActiveCfg = Release|Any CPU
-        {FC395DDC-62A4-4E14-A198-272AB05B33C7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13}.Release|Any CPU.Build.0 = Release|Any CPU
+		{17DF50BE-F481-4904-A4C8-5DF9725B2CA1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{17DF50BE-F481-4904-A4C8-5DF9725B2CA1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{17DF50BE-F481-4904-A4C8-5DF9725B2CA1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{17DF50BE-F481-4904-A4C8-5DF9725B2CA1}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FC395DDC-62A4-4E14-A198-272AB05B33C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FC395DDC-62A4-4E14-A198-272AB05B33C7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FC395DDC-62A4-4E14-A198-272AB05B33C7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FC395DDC-62A4-4E14-A198-272AB05B33C7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{06FCC9AF-CE38-44BB-92B3-0D451BE88537}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{06FCC9AF-CE38-44BB-92B3-0D451BE88537}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{06FCC9AF-CE38-44BB-92B3-0D451BE88537}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{06FCC9AF-CE38-44BB-92B3-0D451BE88537}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AD32C680-1E8C-4340-81B1-DA19C9104516}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AD32C680-1E8C-4340-81B1-DA19C9104516}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AD32C680-1E8C-4340-81B1-DA19C9104516}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AD32C680-1E8C-4340-81B1-DA19C9104516}.Release|Any CPU.Build.0 = Release|Any CPU
+		{91C3E252-9457-43AB-A21A-6064E2404BAA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{91C3E252-9457-43AB-A21A-6064E2404BAA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{91C3E252-9457-43AB-A21A-6064E2404BAA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{91C3E252-9457-43AB-A21A-6064E2404BAA}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -84,8 +102,11 @@ Global
 		{81DB65B3-EA65-4947-8CF1-0E777324C082} = {461FE3E2-21C4-47F9-8405-DF72326AAB2B}
 		{BAD0A733-8EFB-4EAF-9648-9851656AF7FF} = {12312384-8828-4786-AE19-EFCEDF968290}
 		{39A4414F-52B1-42D7-82FA-E65DAD885264} = {12312384-8828-4786-AE19-EFCEDF968290}
-        {A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13} = {12312384-8828-4786-AE19-EFCEDF968290}
-        {17DF50BE-F481-4904-A4C8-5DF9725B2CA1} = {12312384-8828-4786-AE19-EFCEDF968290}
+		{A135C7DB-EA50-4FC6-A6CB-6A5A5CC5FA13} = {12312384-8828-4786-AE19-EFCEDF968290}
+		{17DF50BE-F481-4904-A4C8-5DF9725B2CA1} = {12312384-8828-4786-AE19-EFCEDF968290}
+		{06FCC9AF-CE38-44BB-92B3-0D451BE88537} = {12312384-8828-4786-AE19-EFCEDF968290}
+		{AD32C680-1E8C-4340-81B1-DA19C9104516} = {12312384-8828-4786-AE19-EFCEDF968290}
+		{91C3E252-9457-43AB-A21A-6064E2404BAA} = {12312384-8828-4786-AE19-EFCEDF968290}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {B6468F13-97CD-45E0-9E1E-C122D7F1E09F}

README.md

@@ -42,6 +42,12 @@ Facilitates inference using Hidden Markov Models (HMMs). It interfaces with the
 ### Bonsai.ML.HiddenMarkovModels.Design
 Visualizers and editor features for the HiddenMarkovModels package.
 
+### Bonsai.ML.PointProcessDecoder
+Interfaces with the [PointProcessDecoder](https://github.com/ncguilbeault/PointProcessDecoder) package which can be used for decoding neural activity and other point processes.
+
+### Bonsai.ML.Torch
+Interfaces with the [TorchSharp](https://github.com/dotnet/TorchSharp) package, a C# wrapper around the torch library. Provides tooling for manipulating tensors, performing linear algebra, training and inference with deep neural networks, and more. 
+
 > [!NOTE]
 > Bonsai.ML packages can be installed through Bonsai's integrated package manager and are generally ready for immediate use. However, some packages may require additional installation steps. Refer to the specific package section for detailed installation guides and documentation.
 

docs/articles/PointProcessDecoder/ppd-getting-started.md

@@ -0,0 +1,27 @@
+# Getting Started
+
+The `Bonsai.ML.PointProcessDecoder` package provides a Bonsai interface to the [PointProcessDecoder](https://github.com/ncguilbeault/PointProcessDecoder) package used for decoding neural activity (point processes), and relies on the `Bonsai.ML.Torch` package for tensor operations.
+
+## Installation
+
+The package can be installed by going to the bonsai package manager and installing the `Bonsai.ML.PointProcessDecoder`. Additional installation steps are required for installing the CPU or GPU version of the `Bonsai.ML.Torch` package. See the [Torch installation guide](../Torch/torch-overview.md) for more information.
+
+## Package Overview
+
+The `PointProcessDecoder` package is a C# implementation of a Bayesian state space point process decoder inspired by the [replay_trajectory_classification repository](https://github.com/Eden-Kramer-Lab/replay_trajectory_classification) from the Eden-Kramer Lab. It can decode latent state observations from spike-train data or clusterless mark data based on point processes using Bayesian state space models.
+
+For more detailed information and documentation about the model, please see the [PointProcessDecoder repo](https://github.com/ncguilbeault/PointProcessDecoder).
+
+## Bonsai Implementation
+
+The following workflow showcases the core functionality of the `Bonsai.ML.PointProcessDecoder` package.
+
+:::workflow
+![Point Process Decoder Implementation](~/workflows/PointProcessDecoder.bonsai)
+:::
+
+The `CreatePointProcessModel` node is used to define a model and configure it's parameters. For details on model configuration, please see the [PointProcessDecoder documentation](https://github.com/ncguilbeault/PointProcessDecoder). Crucially, the user must specify the `Name` property in the `Model Parameters` section, as this is what allows you to reference the specific model in the `Encode` and `Decode` nodes, the two main methods that the model will use.
+
+During encoding, the user passes in a tuple of `Observation`s and `SpikeCounts`. Observations are variables that the user measures (for example, the animal's position), represented as a (M, N) tensor, where M is the number of samples in a batch and N is the dimensionality of the observations. Spike counts are the data you will use for decoding. For instance, spike counts might be sorted spike data or clusterless marks. If the data are sorted spiking units, then the `SpikeCounts` tensor will be an (M, U) tensor, where U is the number of sorted units. If the data are clusterles marks, then the `SpikeCounts` tensor will be an (M, F, C) tensor, where F is the number of features computed for each mark (for instance, the maximum spike amplitude across electrodes), and C is the number of independant recording channels (for example, individual tetrodes). Internally, the model fits the data as a point process, which will be used during decoding.
+
+Decoding is the process of taking just the `SpikeCounts` and inferring what is the latent `Observation`. To do this, the model uses a bayesian state space model to predict a posterior distribution over the latent observation space using the information contained in the spike counts data. The output of the `Decode` node will provide you with an (M x D*) tensor, with D* representing a discrete latent state space determined by the parameters defined in the model.

docs/articles/Torch/torch-getting-started.md

@@ -0,0 +1,13 @@
+# Getting Started
+
+The aim of the `Bonsai.ML.Torch` package is to integrate the [TorchSharp](https://github.com/dotnet/TorchSharp) package, a C# wrapper around the powerful libtorch library, into Bonsai. In the current version, the package primarily provides tooling and functionality for users to interact with and manipulate `Tensor`s, the core data type of libtorch which underlies many of the advanced torch operations. Additionally, the package provides some capabilities for defining neural network architectures, running forward inference, and learning via back propogation.
+
+## Tensor Operations
+The package provides several ways to work with tensors. Users can initialize tensors, (`Ones`, `Zeros`, etc.), create tensors from .NET data types, (`ToTensor`), and define custom tensors using Python-like syntax (`CreateTensor`). Tensors can be converted back to .NET types using the `ToArray` node (for flattening tensors into a single array) or the `ToNDArray` node (for preserving multidimensional array shapes). Furthermore, the `Tensor` data types contains many extension methods which can be used via scripting, such as using `ExpressionTransform` (for example, it.sum() to sum a tensor, or it.T to transpose), and works with overloaded operators, for example, `Zip` -> `Multiply`. Thus, `ExpressionTransform` can also be used to access individual elements of a tensor, using the syntax `it.data<T>.ReadCpuT(0)` where `T` is a primitive .NET data type.
+
+
+## Running on the GPU
+Users must be explicit about running tensors on the GPU. First, the `InitializeDeviceType` node must run with a CUDA-compatible GPU. Afterwards, tensors are moved to the GPU using the `ToDevice` node. Converting tensors back to .NET data types requires moving the tensor back to the CPU before converting.
+
+## Neural Networks
+The package provides initial support for working with torch `Module`s, the conventional object for deep neural networks. The `LoadModuleFromArchitecture` node allows users to select from a list of common architectures, and can optionally load in pretrained weights from disk. Additionally, the package supports loading `TorchScript` modules with the `LoadScriptModule` node, which enables users to use torch modules saved in the `.pt` file format. Users can then use the `Forward` node to run inference and the `Backward` node to run back propogation.   

docs/articles/Torch/torch-overview.md

@@ -0,0 +1,27 @@
+# Bonsai.ML.Torch Overview
+
+The Torch package provides a Bonsai interface to interact with the [TorchSharp](https://github.com/dotnet/TorchSharp) package, a C# implementation of the torch library.
+
+## General Guide
+
+The Bonsai.ML.Torch package can be installed through the Bonsai Package Manager and depends on the TorchSharp library. Additionally, running the package requires installing the specific torch DLLs needed for your desired application. The steps for installing are outlined below.
+
+### Running on the CPU 
+For running the package using the CPU, the `TorchSharp-cpu` library can be installed though the `nuget` package source.
+
+### Running on the GPU
+To run torch on the GPU, you first need to ensure that you have a CUDA compatible device installed on your system. 
+
+Next, you must follow the [CUDA installation guide for Windows](https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html) or the [guide for Linux](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html). Make sure to install the correct `CUDA v12.1` version [found here](https://developer.nvidia.com/cuda-12-1-0-download-archive). Ensure that you have the correct CUDA version (v12.1) installed, as `TorchSharp` currently only supports this version.
+
+Next, you need to install the `cuDNN v9` library following the [guide for Windows](https://docs.nvidia.com/deeplearning/cudnn/latest/installation/windows.html) or the [guide for Linux](https://docs.nvidia.com/deeplearning/cudnn/latest/installation/linux.html). Again, you need to ensure you have the correct version installed (v9). You should consult [nvidia's support matrix](https://docs.nvidia.com/deeplearning/cudnn/latest/reference/support-matrix.html) to ensure the versions of CUDA and cuDNN you installed are compatible with your specific OS, graphics driver, and hardware.
+
+Once complete, you need to install the cuda-compatible torch libraries and place them into the correct location. You can download the libraries from [the pytorch website](https://pytorch.org/get-started/locally/) with the following options selected:
+
+- PyTorch Build: Stable (2.5.1)
+- OS: [Your OS]
+- Package: LibTorch
+- Language: C++/Java
+- Compute Platform: CUDA 12.1
+
+Finally, extract the zip folder and copy the contents of the `lib` folder into the `Extensions` folder of your bonsai installation directory.

docs/articles/toc.yml

@@ -13,4 +13,12 @@
 - name: Overview
   href: HiddenMarkovModels/hmm-overview.md
 - name: Getting Started
-  href: HiddenMarkovModels/hmm-getting-started.md
+  href: HiddenMarkovModels/hmm-getting-started.md
+- name: PointProcessDecoder
+- name: Getting Started
+  href: PointProcessDecoder/ppd-getting-started.md
+- name: Torch
+- name: Overview
+  href: Torch/torch-overview.md
+- name: Getting Started
+  href: Torch/torch-getting-started.md

docs/workflows/PointProcessDecoder.bonsai

@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="utf-8"?>
+<WorkflowBuilder Version="2.8.5"
+                 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                 xmlns:p1="clr-namespace:Bonsai.ML.PointProcessDecoder;assembly=Bonsai.ML.PointProcessDecoder"
+                 xmlns:rx="clr-namespace:Bonsai.Reactive;assembly=Bonsai.Core"
+                 xmlns="https://bonsai-rx.org/2018/workflow">
+  <Workflow>
+    <Nodes>
+      <Expression xsi:type="Combinator">
+        <Combinator xsi:type="p1:CreatePointProcessModel">
+          <p1:Name>PointProcessModel</p1:Name>
+          <p1:EstimationMethod>KernelDensity</p1:EstimationMethod>
+          <p1:TransitionsType>RandomWalk</p1:TransitionsType>
+          <p1:EncoderType>SortedSpikeEncoder</p1:EncoderType>
+          <p1:DecoderType>StateSpaceDecoder</p1:DecoderType>
+          <p1:StateSpaceType>DiscreteUniformStateSpace</p1:StateSpaceType>
+          <p1:LikelihoodType>Poisson</p1:LikelihoodType>
+          <p1:ScalarType xsi:nil="true" />
+          <p1:StateSpaceDimensions>2</p1:StateSpaceDimensions>
+          <p1:MinStateSpace>
+            <p1:double>0</p1:double>
+            <p1:double>0</p1:double>
+          </p1:MinStateSpace>
+          <p1:MaxStateSpace>
+            <p1:double>120</p1:double>
+            <p1:double>120</p1:double>
+          </p1:MaxStateSpace>
+          <p1:StepsStateSpace>
+            <p1:long>50</p1:long>
+            <p1:long>50</p1:long>
+          </p1:StepsStateSpace>
+          <p1:ObservationBandwidth>
+            <p1:double>5</p1:double>
+            <p1:double>5</p1:double>
+          </p1:ObservationBandwidth>
+          <p1:NUnits>104</p1:NUnits>
+          <p1:MarkDimensions xsi:nil="true" />
+          <p1:MarkChannels xsi:nil="true" />
+          <p1:DistanceThreshold>1.5</p1:DistanceThreshold>
+          <p1:SigmaRandomWalk xsi:nil="true" />
+        </Combinator>
+      </Expression>
+      <Expression xsi:type="SubscribeSubject">
+        <Name>Observation</Name>
+      </Expression>
+      <Expression xsi:type="SubscribeSubject">
+        <Name>SpikeCounts</Name>
+      </Expression>
+      <Expression xsi:type="Combinator">
+        <Combinator xsi:type="rx:Zip" />
+      </Expression>
+      <Expression xsi:type="Combinator">
+        <Combinator xsi:type="p1:Encode">
+          <p1:Model>PointProcessModel</p1:Model>
+        </Combinator>
+      </Expression>
+      <Expression xsi:type="SubscribeSubject">
+        <Name>SortedSpikeCounts</Name>
+      </Expression>
+      <Expression xsi:type="Combinator">
+        <Combinator xsi:type="p1:Decode">
+          <p1:Model>PointProcessModel</p1:Model>
+        </Combinator>
+      </Expression>
+    </Nodes>
+    <Edges>
+      <Edge From="1" To="3" Label="Source1" />
+      <Edge From="2" To="3" Label="Source2" />
+      <Edge From="3" To="4" Label="Source1" />
+      <Edge From="5" To="6" Label="Source1" />
+    </Edges>
+  </Workflow>
+</WorkflowBuilder>