Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion lockfile/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ anyhow = "1.0.44"
ignore = "0.4.20"
lockfile_generator = { path = "../lockfile_generator", optional = true }
log = "0.4.6"
nom = "7.1.1"
nom = "8.0.0"
nom-language = "0.1.0"
phylum_types = { git = "https://github.com/phylum-dev/phylum-types", branch = "development" }
purl = "0.1.1"
quick-xml = { version = "0.37.1", features = [
Expand Down
2 changes: 1 addition & 1 deletion lockfile/src/golang.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ use anyhow::{anyhow, Context};
use lockfile_generator::go::Go as GoGenerator;
#[cfg(feature = "generator")]
use lockfile_generator::Generator;
use nom::error::convert_error;
use nom::Finish;
use nom_language::error::convert_error;

use crate::parsers::{go_mod, go_sum};
use crate::{Package, Parse};
Expand Down
2 changes: 1 addition & 1 deletion lockfile/src/java.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ use lockfile_generator::gradle::Gradle as GradleGenerator;
use lockfile_generator::maven::Maven as MavenGenerator;
#[cfg(feature = "generator")]
use lockfile_generator::Generator;
use nom::error::convert_error;
use nom::Finish;
use nom_language::error::convert_error;
use phylum_types::ecosystems::maven::{Dependency, Plugin, Project};
use phylum_types::types::package::PackageType;
use serde::Deserialize;
Expand Down
2 changes: 1 addition & 1 deletion lockfile/src/javascript.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ use lockfile_generator::yarn::Yarn as YarnGenerator;
#[cfg(feature = "generator")]
use lockfile_generator::Generator;
use log::debug;
use nom::error::convert_error;
use nom::Finish;
use nom_language::error::convert_error;
use phylum_types::types::package::PackageType;
use serde::Deserialize;
use serde_json::Value as JsonValue;
Expand Down
31 changes: 16 additions & 15 deletions lockfile/src/parsers/gem.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@ use nom::branch::alt;
use nom::bytes::complete::{tag, take_until};
use nom::character::complete::{line_ending, not_line_ending, satisfy, space0};
use nom::combinator::{opt, recognize};
use nom::error::{VerboseError, VerboseErrorKind};
use nom::multi::{many1, many_till};
use nom::sequence::{delimited, tuple};
use nom::Err as NomErr;
use nom::sequence::delimited;
use nom::{Err as NomErr, Parser};
use nom_language::error::{VerboseError, VerboseErrorKind};
use phylum_types::types::package::PackageType;

use crate::parsers::{take_till_blank_line, IResult};
Expand Down Expand Up @@ -36,7 +36,8 @@ impl<'a> Section<'a> {
let (new_input, consumed) = recognize(many_till(
take_till_line_end,
alt((tag("GEM"), tag("GIT"), tag("PATH"), tag("BUNDLED WITH"))),
))(input)?;
))
.parse(input)?;

// Check for type of section head.
let section_type = if consumed.ends_with("GEM") {
Expand Down Expand Up @@ -193,10 +194,8 @@ fn revision(input: &str) -> IResult<&str, &str> {
}

fn specs(input: &str) -> IResult<&str, &str> {
recognize(many_till(
take_till_line_end,
recognize(tuple((space0, tag("specs:"), opt(line_ending)))),
))(input)
recognize(many_till(take_till_line_end, recognize((space0, tag("specs:"), opt(line_ending)))))
.parse(input)
}

fn package(input: &str) -> Result<Option<SpecsPackage>, NomErr<VerboseError<&str>>> {
Expand All @@ -219,8 +218,8 @@ fn package(input: &str) -> Result<Option<SpecsPackage>, NomErr<VerboseError<&str
}

fn package_name(input: &str) -> IResult<&str, &str> {
let (input, _) = recognize(space0)(input)?;
recognize(alt((take_until(" "), not_line_ending)))(input)
let (input, _) = recognize(space0).parse(input)?;
recognize(alt((take_until(" "), not_line_ending))).parse(input)
}

/// Parser allowing for loose `(>= 1.2.0, < 2.0, != 1.2.3)` and strict
Expand All @@ -238,27 +237,29 @@ fn loose_package_version(input: &str) -> IResult<&str, &str> {
c.is_ascii_alphanumeric() || LOOSE_VERSION_CHARS.contains(&c)
}))),
tag(")"),
)(input)
)
.parse(input)
}

/// Parser allowing only strict `1.2.3.alpha.1` versions.
fn strict_package_version(input: &str) -> IResult<&str, &str> {
let (input, _) = space0(input)?;
recognize(many1(satisfy(|c: char| {
c.is_ascii_alphanumeric() || STRICT_VERSION_CHARS.contains(&c)
})))(input)
})))
.parse(input)
}

/// Get the value for a key in a ` key: value` line.
fn key<'a>(input: &'a str, key: &str) -> IResult<&'a str, &'a str> {
let (input, _key) = recognize(tuple((space0, tag(key), tag(": "))))(input)?;
let (input, _key) = recognize((space0, tag(key), tag(": "))).parse(input)?;
take_till_line_end(input)
}

/// Take everything until a line end, swallowing the line end character
/// completely.
fn take_till_line_end(input: &str) -> IResult<&str, &str> {
let (input, consumed) = recognize(alt((take_until("\n"), take_until("\r\n"))))(input)?;
let (input, _) = alt((tag("\n"), tag("\r\n")))(input)?;
let (input, consumed) = recognize(alt((take_until("\n"), take_until("\r\n")))).parse(input)?;
let (input, _) = alt((tag("\n"), tag("\r\n"))).parse(input)?;
Ok((input, consumed))
}
49 changes: 24 additions & 25 deletions lockfile/src/parsers/go_mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ use nom::bytes::complete::{tag, take_till, take_till1, take_while};
use nom::character::complete::{char, multispace0, space0, space1};
use nom::combinator::{map, opt};
use nom::multi::many0;
use nom::sequence::{delimited, preceded, tuple};
use nom::IResult;
use nom::sequence::{delimited, preceded};
use nom::{IResult, Parser};

use crate::golang::GoDeps;
use crate::{Package, PackageType, PackageVersion};
Expand Down Expand Up @@ -68,7 +68,7 @@ impl From<ModuleReplacement> for Package {
}

pub fn parse(input: &str) -> IResult<&str, GoDeps> {
let (_, directives) = many0(directive)(input)?;
let (_, directives) = many0(directive).parse(input)?;

let mut required: Vec<Module> = Vec::new();
let mut excluded: Vec<Module> = Vec::new();
Expand Down Expand Up @@ -126,28 +126,26 @@ pub fn parse(input: &str) -> IResult<&str, GoDeps> {

fn directive(input: &str) -> IResult<&str, Directive<'_>> {
let (input, _) = take_while(|c: char| c == '\n')(input)?;
alt((module_directive, go_directive, require_directive, replace_directive, exclude_directive))(
input.trim(),
)
alt((module_directive, go_directive, require_directive, replace_directive, exclude_directive))
.parse(input.trim())
}

fn module_directive(input: &str) -> IResult<&str, Directive<'_>> {
let (input, module_name) =
preceded(tuple((tag("module"), space1)), take_till(|c| c == '\n'))(input)?;
preceded((tag("module"), space1), take_till(|c| c == '\n')).parse(input)?;
Ok((input, Directive::Module(module_name)))
}

fn go_directive(input: &str) -> IResult<&str, Directive<'_>> {
let (input, go_version) =
preceded(tuple((tag("go"), space1)), take_till(|c| c == '\n'))(input)?;
preceded((tag("go"), space1), take_till(|c| c == '\n')).parse(input)?;
Ok((input, Directive::Go(go_version.trim())))
}

fn require_directive(input: &str) -> IResult<&str, Directive<'_>> {
let (input, deps) = preceded(
tuple((tag("require"), space1)),
alt((module_block, map(require_spec, |r| vec![r]))),
)(input)?;
let (input, deps) =
preceded((tag("require"), space1), alt((module_block, map(require_spec, |r| vec![r]))))
.parse(input)?;
Ok((input, Directive::Require(deps)))
}

Expand All @@ -158,7 +156,8 @@ fn require_spec(input: &str) -> IResult<&str, Module> {
let (input, _) = space0(input)?;

// Check if there is a comment starting with "//".
let (input, comments) = opt(preceded(tag("//"), take_till1(|c: char| c == '\n')))(input)?;
let (input, comments) =
opt(preceded(tag("//"), take_till1(|c: char| c == '\n'))).parse(input)?;

// Determine if the comment indicates the module is indirect.
let indirect = comments.is_some_and(|s: &str| s.trim().eq("indirect"));
Expand All @@ -168,10 +167,9 @@ fn require_spec(input: &str) -> IResult<&str, Module> {
}

fn replace_directive(input: &str) -> IResult<&str, Directive<'_>> {
preceded(tuple((tag("replace"), space1)), alt((replace_block, map(replace_spec, |r| vec![r]))))(
input,
)
.map(|(next_input, reps)| (next_input, Directive::Replace(reps)))
preceded((tag("replace"), space1), alt((replace_block, map(replace_spec, |r| vec![r]))))
.parse(input)
.map(|(next_input, reps)| (next_input, Directive::Replace(reps)))
}

fn replace_spec(input: &str) -> IResult<&str, ModuleReplacement> {
Expand All @@ -187,13 +185,14 @@ fn replace_spec(input: &str) -> IResult<&str, ModuleReplacement> {
};

// Consume "=>" with surrounding spaces.
let (input, _) = tuple((space1, tag("=>"), space1))(input)?;
let (input, _) = (space1, tag("=>"), space1).parse(input)?;

// Parse the destination path and optional version.
let (input, (dest_path, dest_version)) = tuple((
let (input, (dest_path, dest_version)) = (
take_till1(|c: char| c.is_whitespace()),
opt(preceded(space1, take_till1(|c: char| c.is_whitespace()))),
))(input)?;
)
.parse(input)?;

let replacement = if let Some(version) = dest_version {
Replacement::Module(Module {
Expand All @@ -213,10 +212,9 @@ fn replace_spec(input: &str) -> IResult<&str, ModuleReplacement> {
}

fn exclude_directive(input: &str) -> IResult<&str, Directive<'_>> {
preceded(tuple((tag("exclude"), space1)), alt((module_block, map(require_spec, |r| vec![r]))))(
input,
)
.map(|(next_input, deps)| (next_input, Directive::Exclude(deps)))
preceded((tag("exclude"), space1), alt((module_block, map(require_spec, |r| vec![r]))))
.parse(input)
.map(|(next_input, deps)| (next_input, Directive::Exclude(deps)))
}

fn parse_block<T, F>(input: &str, line_parser: F) -> IResult<&str, Vec<T>>
Expand All @@ -227,7 +225,8 @@ where
char('('),
many0(preceded(multispace0, line_parser)),
preceded(multispace0, char(')')),
)(input)
)
.parse(input)
}

fn module_block(input: &str) -> IResult<&str, Vec<Module>> {
Expand Down
14 changes: 8 additions & 6 deletions lockfile/src/parsers/go_sum.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,15 @@ use nom::bytes::complete::{tag, take_until};
use nom::character::complete::{alphanumeric1, line_ending, space0, space1};
use nom::combinator::{opt, recognize};
use nom::multi::{many0, many1};
use nom::sequence::{preceded, tuple};
use nom::sequence::preceded;
use nom::Parser;
use phylum_types::types::package::PackageType;

use crate::parsers::IResult;
use crate::{Package, PackageVersion};

pub fn parse(input: &str) -> IResult<&str, Vec<Package>> {
let (input, pkgs) = many0(package)(input)?;
let (input, pkgs) = many0(package).parse(input)?;

let pkgs = pkgs
.into_iter()
Expand Down Expand Up @@ -42,19 +43,20 @@ fn package_name(input: &str) -> IResult<&str, &str> {
let (input, _) = space0(input)?;

// The package name will be everything up until a space.
recognize(take_until(" "))(input)
recognize(take_until(" ")).parse(input)
}

fn package_version(input: &str) -> IResult<&str, &str> {
// Take away any leading whitespace.
let (input, _) = space0(input)?;

// Accept all of `v[a-zA-Z0-9.+-]+` with an optional "/go.mod" suffix.
let (input, version) = recognize(tuple((
let (input, version) = recognize((
tag("v"),
many1(alt((alphanumeric1, tag("."), tag("-"), tag("+")))),
opt(tag("/go.mod")),
)))(input)?;
))
.parse(input)?;

// Expect at least one whitespace after version.
let (input, _) = space1(input)?;
Expand All @@ -70,7 +72,7 @@ fn package_hash(input: &str) -> IResult<&str, &str> {
let base64_parser = recognize(many1(alt((alphanumeric1, tag("+"), tag("/"), tag("=")))));

// Parse base64 hash with `h1:` prefix.
let (input, hash) = preceded(tag("h1:"), base64_parser)(input)?;
let (input, hash) = preceded(tag("h1:"), base64_parser).parse(input)?;

// Expect EOL.
let (input, _) = line_ending(input)?;
Expand Down
5 changes: 3 additions & 2 deletions lockfile/src/parsers/gradle_dep.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
use nom::branch::alt;
use nom::bytes::complete::{tag, take_till};
use nom::combinator::eof;
use nom::error::VerboseError;
use nom::Parser;
use nom_language::error::VerboseError;
use phylum_types::types::package::PackageType;

use crate::parsers::IResult;
Expand Down Expand Up @@ -33,7 +34,7 @@ fn package(input: &str) -> Result<Package, nom::Err<VerboseError<&str>>> {
let (input, _) = tag(":")(input)?;

let (input, version) = not_space_until(input, '=')?;
let _ = alt((tag("="), eof))(input)?;
let _ = alt((tag("="), eof)).parse(input)?;

Ok(Package {
name: format!("{group_id}:{artifact_id}"),
Expand Down
13 changes: 7 additions & 6 deletions lockfile/src/parsers/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@ use nom::branch::alt;
use nom::bytes::complete::{tag, take_until};
use nom::character::complete::{line_ending, not_line_ending, space0};
use nom::combinator::{eof, opt, recognize, rest};
use nom::error::{context, VerboseError};
use nom::error::context;
use nom::multi::many_till;
use nom::sequence::{delimited, terminated, tuple};
use nom::AsChar;
use nom::sequence::{delimited, terminated};
use nom::{AsChar, Parser};
use nom_language::error::VerboseError;

pub mod gem;
pub mod go_mod;
Expand All @@ -17,12 +18,12 @@ pub mod yarn;

/// Consume everything until the next `\n` or `\r\n`.
fn take_till_line_end(input: &str) -> IResult<&str, &str> {
recognize(terminated(not_line_ending, line_ending))(input)
recognize(terminated(not_line_ending, line_ending)).parse(input)
}

/// Consume everything until the next `\n\n` or `\r\n\r\n`.
fn take_till_blank_line(input: &str) -> IResult<&str, &str> {
recognize(alt((take_until("\n\n"), take_until("\r\n\r\n"))))(input)
recognize(alt((take_until("\n\n"), take_until("\r\n\r\n")))).parse(input)
}

/// Consume the next line.
Expand All @@ -32,7 +33,7 @@ fn take_till_blank_line(input: &str) -> IResult<&str, &str> {
fn take_continued_line(mut input: &str) -> IResult<&str, ()> {
loop {
// Get everything up to the next NL or EOF.
let (new_input, line) = recognize(alt((take_till_line_end, rest)))(input)?;
let (new_input, line) = recognize(alt((take_till_line_end, rest))).parse(input)?;
input = new_input;

// Stop consuming lines once there are no continuations.
Expand Down
Loading