VER: Release 0.3.2

.github/workflows/release.yaml

@@ -245,7 +245,7 @@ jobs:
           name: wheels
           path: dist
 
-  publish:
+  publish-py:
     runs-on: ubuntu-latest
     needs:
       [
@@ -279,16 +279,41 @@ jobs:
           path: "*"
           upload_url: ${{ needs.tag-release.outputs.upload_url }}
 
-      - name: Publish dbn to crates.io
-        id: publish-dbn-crate
-        uses: katyo/publish-crates@v2
+  publish-rs:
+    runs-on: ubuntu-latest
+    needs:
+      [
+        tag-release,
+        macos-release,
+        windows-release,
+        linux-release,
+        linux-musl-release,
+      ]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Cargo setup
+      - name: Set up Cargo cache
+        uses: actions/cache@v3
         with:
-          path: './rust/dbn'
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}
+
+      - name: Publish dbn-macros to crates.io
+        run: cargo publish --token ${CARGO_REGISTRY_TOKEN} --package dbn-macros
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+
+      - name: Publish dbn to crates.io
+        run: cargo publish --token ${CARGO_REGISTRY_TOKEN} --package dbn
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
 
       - name: Publish dbn-cli to crates.io
-        id: publish-dbn-cli-crate
-        uses: katyo/publish-crates@v2
-        with:
-          path: './rust/dbn-cli'
-          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+        run: cargo publish --token ${CARGO_REGISTRY_TOKEN} --package dbn-cli
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.3.2 - 2023-03-01
+- Added records and `Metadata` as exports of `databento_dbn` Python package
+- Improved how `Metadata` appears in Python and added `__repr__`
+- Fixed bug where `dbn` CLI tool didn't truncate existing files
+
 ## 0.3.1 - 2023-02-27
 - Added improved Python bindings for decoding DBN
 - Fixed bug with `encode_metadata` Python function

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 members = [
   "rust/dbn-cli",
+  "rust/dbn-macros",
   "rust/dbn",
   "python",
 ]

python/Cargo.toml

@@ -1,11 +1,13 @@
 [package]
 name = "databento-dbn"
 authors = ["Databento <support@databento.com>"]
-version = "0.3.1"
+version = "0.3.2"
 edition = "2021"
 description = "Python library written in Rust for working with Databento Binary Encoding (DBN)"
 license = "Apache-2.0"
 repository = "https://github.com/databento/dbn"
+# This crate should only be published as a Python package
+publish = false
 
 [lib]
 name = "databento_dbn" # Python modules can't contain dashes

python/src/lib.rs

@@ -1,30 +1,47 @@
 //! Python bindings for the [`dbn`] crate using [`pyo3`].
 use std::io::{self, Write};
 
-use pyo3::{prelude::*, wrap_pyfunction};
+use pyo3::{prelude::*, wrap_pyfunction, PyClass};
 
 use dbn::{
     decode::dbn::{MetadataDecoder, RecordDecoder},
     enums::rtype,
     python::to_val_err,
     record::{
-        ErrorMsg, InstrumentDefMsg, MboMsg, Mbp10Msg, Mbp1Msg, OhlcvMsg, SymbolMappingMsg, TradeMsg,
+        BidAskPair, ErrorMsg, InstrumentDefMsg, MboMsg, Mbp10Msg, Mbp1Msg, OhlcvMsg, RecordHeader,
+        SymbolMappingMsg, TradeMsg,
     },
 };
 
 /// A Python module wrapping dbn functions
 #[pymodule] // The name of the function must match `lib.name` in `Cargo.toml`
 fn databento_dbn(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
+    fn checked_add_class<T: PyClass>(m: &PyModule) -> PyResult<()> {
+        // ensure a module was specified, otherwise it defaults to builtins
+        assert_eq!(T::MODULE.unwrap(), "databento_dbn");
+        m.add_class::<T>()
+    }
     // all functions exposed to Python need to be added here
     m.add_wrapped(wrap_pyfunction!(dbn::python::decode_metadata))?;
     m.add_wrapped(wrap_pyfunction!(dbn::python::encode_metadata))?;
     m.add_wrapped(wrap_pyfunction!(dbn::python::update_encoded_metadata))?;
     m.add_wrapped(wrap_pyfunction!(dbn::python::write_dbn_file))?;
-    m.add_class::<DbnDecoder>()?;
+    checked_add_class::<DbnDecoder>(m)?;
+    checked_add_class::<dbn::Metadata>(m)?;
+    checked_add_class::<RecordHeader>(m)?;
+    checked_add_class::<MboMsg>(m)?;
+    checked_add_class::<BidAskPair>(m)?;
+    checked_add_class::<TradeMsg>(m)?;
+    checked_add_class::<Mbp1Msg>(m)?;
+    checked_add_class::<Mbp10Msg>(m)?;
+    checked_add_class::<OhlcvMsg>(m)?;
+    checked_add_class::<InstrumentDefMsg>(m)?;
+    checked_add_class::<ErrorMsg>(m)?;
+    checked_add_class::<SymbolMappingMsg>(m)?;
     Ok(())
 }
 
-#[pyclass]
+#[pyclass(module = "databento_dbn")]
 struct DbnDecoder {
     buffer: io::Cursor<Vec<u8>>,
     has_decoded_metadata: bool,

rust/dbn-cli/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "dbn-cli"
 authors = ["Databento <support@databento.com>"]
-version = "0.3.1"
+version = "0.3.2"
 edition = "2021"
 description = "Command-line utility for converting Databento Binary Encoding (DBN) files to text-based formats"
 default-run = "dbn"
@@ -17,7 +17,7 @@ path = "src/main.rs"
 
 [dependencies]
 # Databento common DBN library
-dbn = { path = "../dbn", version = "0.3.1" }
+dbn = { path = "../dbn", version = "=0.3.2" }
 
 # Error handling
 anyhow = "1.0.68"

rust/dbn-cli/src/lib.rs

@@ -172,6 +172,7 @@ pub fn output_from_args(args: &Args) -> anyhow::Result<Box<dyn io::Write>> {
 fn open_output_file(path: &PathBuf, force: bool) -> anyhow::Result<File> {
     let mut options = File::options();
     options.write(true);
+    options.truncate(true);
     if force {
         options.create(true);
     } else if path.exists() {

rust/dbn-cli/tests/integration_tests.rs

@@ -1,5 +1,7 @@
-use std::fs;
-use std::io::Read;
+use std::{
+    fs,
+    io::{Read, Write},
+};
 
 use assert_cmd::Command;
 use predicates::str::{contains, ends_with, is_empty, starts_with};
@@ -189,6 +191,29 @@ fn force_overwrite() {
     output_file.as_file().read_to_string(&mut contents).unwrap();
 }
 
+#[test]
+fn force_truncates_file() {
+    let mut output_file = NamedTempFile::new().unwrap();
+    // Fill file with 10KB of bytes
+    for _ in 0..(10 * 1024) {
+        output_file.write_all(b"\0").unwrap();
+    }
+    let before_size = output_file.path().metadata().unwrap().len();
+    cmd()
+        .args(&[
+            &format!("{TEST_DATA_PATH}/test_data.ohlcv-1d.dbn.zst"),
+            "--output",
+            &output_file.path().to_str().unwrap(),
+            "-C", // CSV
+            "--force",
+        ])
+        .assert()
+        .success()
+        .stdout(is_empty());
+    let after_size = output_file.path().metadata().unwrap().len();
+    assert!(after_size < before_size);
+}
+
 #[test]
 fn cant_specify_json_and_csv() {
     cmd()

rust/dbn-macros/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "dbn-macros"
+authors = ["Databento <support@databento.com>"]
+version = "0.3.2"
+edition = "2021"
+description = "Proc macros for dbn crate"
+license = "Apache-2.0"
+repository = "https://github.com/databento/dbn"
+
+[lib]
+proc-macro = true
+
+[dependencies]

rust/dbn-macros/src/lib.rs

@@ -0,0 +1,10 @@
+use proc_macro::TokenStream;
+
+/// Dummy derive macro to get around `cfg_attr` incompatibility of several
+/// of pyo3's attribute macros. See <https://github.com/PyO3/pyo3/issues/780>.
+///
+/// `MockPyo3` is an invented trait.
+#[proc_macro_derive(MockPyo3, attributes(pyo3))]
+pub fn derive_mock_pyo3(_item: TokenStream) -> TokenStream {
+    TokenStream::new()
+}

rust/dbn/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "dbn"
 authors = ["Databento <support@databento.com>"]
-version = "0.3.1"
+version = "0.3.2"
 edition = "2021"
 description = "Library for working with Databento Binary Encoding (DBN)"
 license = "Apache-2.0"
@@ -18,6 +18,8 @@ python = ["pyo3"]
 trivial_copy = []
 
 [dependencies]
+dbn-macros = { version = "=0.3.2", path = "../dbn-macros" }
+
 # error handling
 anyhow = "1.0.68"
 # async (de)compression

rust/dbn/src/lib.rs

@@ -1,5 +1,10 @@
 //! A crate for reading DBN and legacy DBZ files and converting them to other
 //! [`Encoding`](enums::Encoding)s.
+
+// #[cfg_attr(not(feature = "python"), macro_use)]
+// #[cfg(not(feature = "python"))]
+// extern crate dbn_macros;
+
 #[deny(missing_docs)]
 #[deny(rustdoc::broken_intra_doc_links)]
 #[deny(clippy::missing_errors_doc)]

rust/dbn/src/metadata.rs

@@ -6,35 +6,53 @@ use serde::Serialize;
 
 use crate::enums::{rtype, SType, Schema};
 
+// Dummy derive macro to get around `cfg_attr` incompatibility of several
+// of pyo3's attribute macros. See https://github.com/PyO3/pyo3/issues/780
+#[cfg(not(feature = "python"))]
+pub use dbn_macros::MockPyo3;
+
 /// Information about the data contained in a DBN file or stream. DBN requires the
 /// Metadata to be included at the start of the encoded data.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
-#[cfg_attr(feature = "python", pyo3::pyclass(get_all))]
+#[cfg_attr(not(feature = "python"), derive(MockPyo3))] // bring `pyo3` attribute into scope
+#[cfg_attr(feature = "python", pyo3::pyclass(module = "databento_dbn"))]
 pub struct Metadata {
     /// The DBN schema version number. Newly-encoded DBN files will use [`crate::DBN_VERSION`].
+    #[pyo3(get)]
     pub version: u8,
     /// The dataset code.
+    #[pyo3(get)]
     pub dataset: String,
     /// The data record schema. Specifies which record type is stored in the Zstd-compressed DBN file.
+    #[pyo3(get)]
     pub schema: Schema,
     /// The UNIX nanosecond timestamp of the query start, or the first record if the file was split.
+    #[pyo3(get)]
     pub start: u64,
     /// The UNIX nanosecond timestamp of the query end, or the last record if the file was split.
+    #[pyo3(get)]
     pub end: Option<NonZeroU64>,
-    #[serde(serialize_with = "serialize_as_raw")]
     /// The optional maximum number of records for the query.
+    #[pyo3(get)]
+    #[serde(serialize_with = "serialize_as_raw")]
     pub limit: Option<NonZeroU64>,
     /// The total number of data records.
+    #[pyo3(get)]
     pub record_count: Option<u64>,
     /// The input symbology type to map from.
+    #[pyo3(get)]
     pub stype_in: SType,
     /// The output symbology type to map to.
+    #[pyo3(get)]
     pub stype_out: SType,
     /// The original query input symbols from the request.
+    #[pyo3(get)]
     pub symbols: Vec<String>,
     /// Symbols that did not resolve for _at least one day_ in the query time range.
+    #[pyo3(get)]
     pub partial: Vec<String>,
     /// Symbols that did not resolve for _any_ day in the query time range.
+    #[pyo3(get)]
     pub not_found: Vec<String>,
     /// Symbol mappings containing a native symbol and its mapping intervals.
     pub mappings: Vec<SymbolMapping>,