Skip to content

v1.1.0 - scrubcsv --output-stats-to-file #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
seamusabshere merged 4 commits into from
Oct 14, 2025
Merged

v1.1.0 - scrubcsv --output-stats-to-file #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a68cf12
v1.1.0 - scrubcsv --output-stats-to-file
seamusabshere Oct 14, 2025
ebae7b8
udpates
seamusabshere Oct 14, 2025
0acac45
stuff
seamusabshere Oct 14, 2025
d67260b
fix cross
seamusabshere Oct 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
747 changes: 479 additions & 268 deletions Cargo.lock
View file
Open in desktop

Large diffs are not rendered by default.

8 changes: 4 additions & 4 deletions build-release
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ VERSION="$RELEASE_VERSION"

# Install 'cargo deny'.
echo "Installing cargo-deny"
cargo_deny_version=0.12.1
cargo_deny_version=0.18.5
cargo_deny_basename=cargo-deny-$cargo_deny_version-$HOST
curl -fLO https://github.com/EmbarkStudios/cargo-deny/releases/download/$cargo_deny_version/$cargo_deny_basename.tar.gz
tar xf $cargo_deny_basename.tar.gz
Expand All @@ -40,9 +40,9 @@ cargo test --package "$PKG_NAME"
# Install cross if needed.
if [ "$CARGO_COMMAND" = "cross" ]; then
echo "Installing cross for cross-compilation"
cross_version=v0.2.1
cross_basename=cross-$cross_version-$HOST
curl -fLO https://github.com/rust-embedded/cross/releases/download/$cross_version/$cross_basename.tar.gz
cross_version=v0.2.5
cross_basename=cross-$HOST
curl -fLO https://github.com/cross-rs/cross/releases/download/$cross_version/$cross_basename.tar.gz
tar xf $cross_basename.tar.gz
mv cross /usr/local/bin/
rm -rf $cross_basename.tar.gz
Expand Down
6 changes: 6 additions & 0 deletions catcsv/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [1.0.1] - 2025-10-14

### Changed

- Updated `env_logger` from 0.9 to 0.11

## [1.0.0] - 2022-05-25

### Added
Expand Down
4 changes: 2 additions & 2 deletions catcsv/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "catcsv"
version = "1.0.0"
version = "1.0.1"
authors = ["Eric Kidd <git@randomhacks.net>"]
edition = "2018"

Expand All @@ -16,7 +16,7 @@ cli_test_dir = "0.1.2"
[dependencies]
csv = "1"
docopt = "1"
env_logger = "0.9.0"
env_logger = "0.11"
error-chain = "0.12.4"
log = "0.4.14"
serde = "1.0.123"
Expand Down
9 changes: 5 additions & 4 deletions catcsv/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ use walkdir::WalkDir;
use crate::errors::*;

/// A module to hold `Error`, etc., types generated by `error-chain`.
#[allow(unexpected_cfgs)]
mod errors {
use error_chain::error_chain;
use std::io;
Expand Down Expand Up @@ -111,15 +112,15 @@ fn run() -> Result<()> {
// Check the filename to see if we can handle this file type.
if filename.ends_with(".csv") {
debug!("Processing as *.csv");
let mut file = File::open(path).chain_err(&mkerr)?;
let mut file = File::open(path).chain_err(mkerr)?;
output_csv(&mut file, &mut first_headers, &mut out)
.chain_err(&mkerr)?;
.chain_err(mkerr)?;
} else if filename.ends_with(".csv.sz") {
debug!("Processing as *.csv.sz");
let file = File::open(path).chain_err(&mkerr)?;
let file = File::open(path).chain_err(mkerr)?;
let mut decompressed = snap::read::FrameDecoder::new(file);
output_csv(&mut decompressed, &mut first_headers, &mut out)
.chain_err(&mkerr)?;
.chain_err(mkerr)?;
} else {
let msg =
format!("{} does not appear to be a CSV file", path.display());
Expand Down
14 changes: 4 additions & 10 deletions deny.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,24 +3,18 @@
# These policies can be enforced using `cargo deny check`.

[licenses]
# Don't allow code with an unclear license.
unlicensed = "deny"

# Don't allow "copylefted" licenses unless they're listed below.
copyleft = "deny"
# In cargo-deny 0.16.0+, all licenses are denied by default unless explicitly
# allowed. This means unlicensed code, copyleft licenses, and AGPL are
# automatically denied since they're not in the allow list.

# Allow common non-restrictive licenses.
allow = ["MIT", "Apache-2.0", "BSD-3-Clause", "CC0-1.0"]
allow = ["MIT", "Apache-2.0", "BSD-3-Clause", "CC0-1.0", "Unicode-3.0"]

# Also fine to allow. ISC is used for various DNS and crypto things, and it's a
# minimally restrictive open source license.
#
# "BSD-2-Clause", "ISC", "OpenSSL", "Zlib"

# Many organizations ban AGPL-licensed code
# https://opensource.google/docs/using/agpl-policy/
deny = ["AGPL-3.0"]

[bans]
# Do we want to know about multiple versions of the same dependency?
multiple-versions = "allow"
Expand Down
9 changes: 9 additions & 0 deletions fixed2csv/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [1.0.1] - 2025-10-14

### Changed

- Updated `env_logger` from 0.9 to 0.11
- Updated `humansize` from 1 to 2
- Migrated from `structopt` to `clap` 4 derive
- Replaced unmaintained `failure` crate with `anyhow`

## [1.0.0] - 2022-05-25

### Added
Expand Down
10 changes: 5 additions & 5 deletions fixed2csv/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "fixed2csv"
version = "1.0.0"
version = "1.0.1"
authors = ["Eric Kidd <git@randomhacks.net>"]
edition = "2018"

Expand All @@ -11,10 +11,10 @@ repository = "https://github.com/faradayio/csv-tools"
homepage = "https://github.com/faradayio/csv-tools/tree/main/fixed2csv"

[dependencies]
anyhow = "1"
clap = { version = "4", features = ["derive"] }
csv = "1.0.2"
env_logger = "0.9.0"
failure = "0.1.3"
humansize = "1"
env_logger = "0.11"
humansize = "2"
humantime = "2"
log = "0.4.6"
structopt = "0.3.21"
22 changes: 9 additions & 13 deletions fixed2csv/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
use failure::Error;
use humansize::{file_size_opts, FileSize};
use anyhow::Error;
use clap::Parser;
use humansize::{format_size, BINARY};
use humantime::format_duration;
use log::debug;
use std::{
cmp::min,
io::{prelude::*, stdin, stdout, BufReader},
time::{Duration, SystemTime},
};
use structopt::StructOpt;

#[derive(Debug, StructOpt)]
/// Convert fixed-width fields on stdin to CSV data on stdout.
#[derive(Debug, Parser)]
#[command(about, version)]
struct Opt {
/// Print summary statistics.
#[structopt(short = "v", long = "verbose")]
#[arg(short = 'v', long = "verbose")]
verbose: bool,

/// One of more field widths, as separate command-line arguments.
Expand All @@ -22,7 +22,7 @@ struct Opt {

/// Our main entry point.
fn main() -> Result<(), Error> {
let opt = Opt::from_args();
let opt = Opt::parse();
debug!("Options: {:?}", opt);

// Keep track of how much time this takes.
Expand All @@ -42,13 +42,9 @@ fn main() -> Result<(), Error> {
};
eprintln!(
"Processed {} in {}, {}/s",
total
.file_size(file_size_opts::BINARY)
.expect("size can never be negative"),
format_size(total, BINARY),
format_duration(simple_elapsed),
(total as u64 / elapsed.as_secs())
.file_size(file_size_opts::BINARY)
.expect("size can never be negative"),
format_size(total as u64 / elapsed.as_secs(), BINARY),
);
}

Expand Down
6 changes: 6 additions & 0 deletions geochunk/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [1.0.1] - 2025-10-14

### Changed

- Updated `env_logger` from 0.9 to 0.11

## [1.0.0] - 2022-05-25

### Added
Expand Down
4 changes: 2 additions & 2 deletions geochunk/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "geochunk"
version = "1.0.0"
version = "1.0.1"
authors = ["Eric Kidd <git@randomhacks.net>"]
edition = "2018"

Expand All @@ -16,7 +16,7 @@ cli_test_dir = "0.1.2"
[dependencies]
csv = "1"
docopt = "1.1"
env_logger = "0.9.0"
env_logger = "0.11"
error-chain = "0.12.4"
lazy_static = "1.4"
log = "0.4.14"
Expand Down
1 change: 1 addition & 0 deletions geochunk/src/errors.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
//! A module to hold `Error`, etc., types generated by `error-chain`.
#![allow(unexpected_cfgs)]

use std::io;

Expand Down
8 changes: 8 additions & 0 deletions hashcsv/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [1.0.2] - 2025-10-14

### Changed

- Updated `env_logger` from 0.9 to 0.11
- Migrated from `structopt` to `clap` 4 derive
- Updated `clap` from 2.33 to 4

## [1.0.1] - 2022-05-25

### Added
Expand Down
7 changes: 3 additions & 4 deletions hashcsv/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "hashcsv"
version = "1.0.1"
version = "1.0.2"
authors = ["Eric Kidd <git@randomhacks.net>"]
edition = "2018"
description = "Append an `id` column to each row of a CSV file, containing a UUID v5 hash of the row"
Expand All @@ -12,13 +12,12 @@ homepage = "https://github.com/faradayio/csv-tools/blob/main/hashcsv/README.md"

[dependencies]
anyhow = "1.0.38"
clap = { version = "2.33.3", features = ["wrap_help"] }
clap = { version = "4", features = ["derive", "wrap_help"] }
csv = "1.1.5"
env_logger = "0.9.0"
env_logger = "0.11"
log = "0.4.14"
regex = "1.4.3"
serde = "1.0.123"
structopt = "0.3.21"
time = "0.3.9"
uuid = { version = "1.0.0", features = ["v5"] }

Expand Down
15 changes: 8 additions & 7 deletions hashcsv/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use anyhow::{Context, Result};
use clap::Parser;
use csv::ByteRecord;
use log::debug;
use std::{
Expand All @@ -7,7 +8,6 @@ use std::{
path::PathBuf,
process,
};
use structopt::StructOpt;
use uuid::Uuid;

/// Use reasonably large input and output buffers. In other CSV tools, this
Expand All @@ -16,16 +16,17 @@ use uuid::Uuid;
const BUFFER_SIZE: usize = 256 * 1024;

/// Command-line options.
#[derive(Debug, StructOpt)]
#[structopt(
about = "Add an `id` column to a CSV file based on a hash of the other columns"
#[derive(Debug, Parser)]
#[command(
about = "Add an `id` column to a CSV file based on a hash of the other columns",
version
)]
struct Opt {
/// Input file (uses stdin if omitted).
input: Option<PathBuf>,

/// The column name for the new, hash-based ID column.
#[structopt(long = "id-column-name", short = "c", default_value = "id")]
#[arg(long = "id-column-name", short = 'c', default_value = "id")]
id_column_name: String,
}

Expand All @@ -34,8 +35,8 @@ fn main() {
// Set up logging.
env_logger::init();

// Parse our command-line arguments using `docopt`.
let opt: Opt = Opt::from_args();
// Parse our command-line arguments.
let opt: Opt = Opt::parse();
debug!("Options: {:#?}", opt);

if let Err(err) = run(&opt) {
Expand Down
13 changes: 13 additions & 0 deletions scrubcsv/CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [1.1.0] - 2025-10-14

### Added

- `--output-stats-to-file <PATH>` option to write processing statistics to a JSON file. The output includes rows, bad_rows, elapsed_seconds, bytes_processed, and bytes_per_second.

### Changed

- Migrated from deprecated `structopt` to `clap` v4.5 with derive macros.
- Updated `env_logger` from 0.9 to 0.11.
- Updated `humansize` from 1.0 to 2.1.
- Updated all other dependencies to their latest compatible versions.

## [1.0.0] - 2022-05-25

### Added
Expand Down
16 changes: 8 additions & 8 deletions scrubcsv/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[package]
name = "scrubcsv"
version = "1.0.0"
authors = ["Eric Kidd <git@randomhacks.net>"]
version = "1.1.0"
authors = ["Eric Kidd <git@randomhacks.net>", "Seamus Abshere <seamus@abshere.net>"]
edition = "2018"

description = "Remove bad lines from large CSV files and normalize the rest"
Expand All @@ -11,16 +11,16 @@ repository = "https://github.com/faradayio/csv-tools"
homepage = "https://github.com/faradayio/csv-tools/tree/main/scrubcsv"

[dependencies]
clap = { version = "2.33.0", features = ["wrap_help"] }
clap = { version = "4.5", features = ["derive", "wrap_help"] }
csv = "1"
env_logger = "0.9.0"
humansize = "1.0.1"
lazy_static = "1.2.0"
libc = "0.2.18"
env_logger = "0.11"
humansize = "2.1"
lazy_static = "1.5"
libc = "0.2"
log = "0.4"
regex = "1"
serde = "1.0"
structopt = "0.3.3"
serde_json = "1.0"

[dev-dependencies]
cli_test_dir = "0.1.1"
4 changes: 2 additions & 2 deletions scrubcsv/src/errors.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use std::{error, fmt, result};

/// Our error type. We used a boxed dynamic error because we don't care much
/// about the details and we're only going to print it for the user anyways.
pub type Error = Box<dyn error::Error + 'static>;
pub type Error = Box<dyn error::Error + Send + Sync + 'static>;

/// Our custom `Result` type. Defaults the `E` parameter to our error type.
pub type Result<T, E = Error> = result::Result<T, E>;
Expand Down Expand Up @@ -51,7 +51,7 @@ pub trait ResultExt<T, E>: Sized {
F: FnOnce(&E) -> C;
}

impl<T, E: error::Error + 'static> ResultExt<T, E> for Result<T, E> {
impl<T, E: error::Error + Send + Sync + 'static> ResultExt<T, E> for Result<T, E> {
fn with_context<C, F>(self, build_context: F) -> Result<T>
where
C: Into<String>,
Expand Down
Loading
Loading