From ce170f6d234dcd63596b4267910c5642b0396ec0 Mon Sep 17 00:00:00 2001 From: YoungWan Kwon <87213416+000wan@users.noreply.github.com> Date: Fri, 29 Sep 2023 02:27:25 +0900 Subject: [PATCH 1/8] Refactor `clmul` using union --- clmul/src/backend.rs | 99 ++++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 54 deletions(-) diff --git a/clmul/src/backend.rs b/clmul/src/backend.rs index d75ff379..748b1ed0 100644 --- a/clmul/src/backend.rs +++ b/clmul/src/backend.rs @@ -56,6 +56,16 @@ cfg_if! { } } +pub struct Clmul { + inner: Inner, +} + +union Inner { + intrinsics: intrinsics::Clmul, + soft: soft::Clmul, +} + +/* cfg_if! { if #[cfg(any(all(target_arch = "aarch64", feature = "armv8"), any(target_arch = "x86_64", target_arch = "x86")))]{ #[derive(Clone, Copy, Debug)] @@ -76,6 +86,7 @@ cfg_if! { } } } +*/ // #[derive(Clone, Copy)] // pub struct Clmul { @@ -88,28 +99,24 @@ impl Clmul { cfg_if! { if #[cfg(feature = "force-soft")] { Self { - intrinsics: None, - soft: Some(soft::Clmul::new(h)), + inner: Inner { soft: soft::Clmul::new(h) }, } } else if #[cfg(any(all(target_arch = "aarch64", feature = "armv8"), any(target_arch = "x86_64", target_arch = "x86")))]{ if mul_intrinsics::get() { Self { - intrinsics: Some(intrinsics::Clmul::new(h)), - soft: None, + inner: Inner { intrinsics: intrinsics::Clmul::new(h) }, } } else { // supported arch was found but intrinsics are not available Self { - intrinsics: None, - soft: Some(soft::Clmul::new(h)), + inner: Inner { soft: soft::Clmul::new(h) }, } } } else { // "force-soft" feature was not enabled but neither was // supported arch found. Falling back to soft backend. Self { - intrinsics: None, - soft: Some(soft::Clmul::new(h)), + inner: Inner { soft: soft::Clmul::new(h) }, } } } @@ -117,42 +124,32 @@ impl Clmul { /// Performs carryless multiplication pub fn clmul(self, x: Self) -> (Self, Self) { - match self.intrinsics { - Some(s_intr) => match x.intrinsics { - Some(x_intr) => { + unsafe { + match (self.inner, x.inner) { + (Inner { intrinsics: s_intr }, Inner { intrinsics: x_intr }) => { let (r0, r1) = s_intr.clmul(x_intr); ( Self { - intrinsics: Some(r0), - soft: None, + inner: Inner { intrinsics: r0 }, }, Self { - intrinsics: Some(r1), - soft: None, + inner: Inner { intrinsics: r1 }, }, ) } - None => unreachable!(), - }, - None => match self.soft { - Some(s_soft) => match x.soft { - Some(x_soft) => { - let (r0, r1) = s_soft.clmul(x_soft); - ( - Self { - intrinsics: None, - soft: Some(r0), - }, - Self { - intrinsics: None, - soft: Some(r1), - }, - ) - } - None => unreachable!(), - }, - None => unreachable!(), - }, + (Inner { soft: s_soft }, Inner { soft: x_soft }) => { + let (r0, r1) = s_soft.clmul(x_soft); + ( + Self { + inner: Inner { soft: r0 }, + }, + Self { + inner: Inner { soft: r1 }, + }, + ) + } + _ => unreachable!(), + } } } @@ -161,26 +158,20 @@ impl Clmul { /// to clmul() where we create new objects containing the result. /// The high bits will be placed in `self`, the low bits - in `x`. pub fn clmul_reuse(&mut self, x: &mut Self) { - match self.intrinsics { - Some(s_intr) => match x.intrinsics { - Some(x_intr) => { + unsafe { + match (self.inner, x.inner) { + (Inner { intrinsics: s_intr }, Inner { intrinsics: x_intr }) => { let (r0, r1) = s_intr.clmul(x_intr); - self.intrinsics = Some(r0); - x.intrinsics = Some(r1); + self.inner.intrinsics = r0; + x.inner.intrinsics = r1; } - None => unreachable!(), - }, - None => match self.soft { - Some(s_soft) => match x.soft { - Some(x_soft) => { - let (r0, r1) = s_soft.clmul(x_soft); - self.soft = Some(r0); - x.soft = Some(r1); - } - None => unreachable!(), - }, - None => unreachable!(), - }, + (Inner { soft: s_soft }, Inner { soft: x_soft }) => { + let (r0, r1) = s_soft.clmul(x_soft); + self.inner.soft = r0; + x.inner.soft = r1; + } + _ => unreachable!(), + } } } From a3fd4aed2bf040245d9d0315e440a7ce3ebe0086 Mon Sep 17 00:00:00 2001 From: YoungWan Kwon <87213416+000wan@users.noreply.github.com> Date: Fri, 29 Sep 2023 03:32:55 +0900 Subject: [PATCH 2/8] Refactor all clmul methods --- clmul/src/backend.rs | 169 +++++++++++++++++++++++-------------------- 1 file changed, 92 insertions(+), 77 deletions(-) diff --git a/clmul/src/backend.rs b/clmul/src/backend.rs index 748b1ed0..4a804f62 100644 --- a/clmul/src/backend.rs +++ b/clmul/src/backend.rs @@ -58,6 +58,7 @@ cfg_if! { pub struct Clmul { inner: Inner, + is_intr: bool, } union Inner { @@ -100,16 +101,19 @@ impl Clmul { if #[cfg(feature = "force-soft")] { Self { inner: Inner { soft: soft::Clmul::new(h) }, + is_intr: false, } } else if #[cfg(any(all(target_arch = "aarch64", feature = "armv8"), any(target_arch = "x86_64", target_arch = "x86")))]{ if mul_intrinsics::get() { Self { inner: Inner { intrinsics: intrinsics::Clmul::new(h) }, + is_intr: true, } } else { // supported arch was found but intrinsics are not available Self { inner: Inner { soft: soft::Clmul::new(h) }, + is_intr: false, } } } else { @@ -117,6 +121,7 @@ impl Clmul { // supported arch found. Falling back to soft backend. Self { inner: Inner { soft: soft::Clmul::new(h) }, + is_intr: false, } } } @@ -125,26 +130,36 @@ impl Clmul { /// Performs carryless multiplication pub fn clmul(self, x: Self) -> (Self, Self) { unsafe { - match (self.inner, x.inner) { - (Inner { intrinsics: s_intr }, Inner { intrinsics: x_intr }) => { + match (self.is_intr, x.is_intr) { + (true, true) => { + let s_intr = self.inner.intrinsics; + let x_intr = x.inner.intrinsics; + let (r0, r1) = s_intr.clmul(x_intr); ( Self { inner: Inner { intrinsics: r0 }, + is_intr: true, }, Self { inner: Inner { intrinsics: r1 }, + is_intr: true, }, ) } - (Inner { soft: s_soft }, Inner { soft: x_soft }) => { + (false, false) => { + let s_soft = self.inner.soft; + let x_soft = x.inner.soft; + let (r0, r1) = s_soft.clmul(x_soft); ( Self { inner: Inner { soft: r0 }, + is_intr: false, }, Self { inner: Inner { soft: r1 }, + is_intr: false, }, ) } @@ -159,13 +174,19 @@ impl Clmul { /// The high bits will be placed in `self`, the low bits - in `x`. pub fn clmul_reuse(&mut self, x: &mut Self) { unsafe { - match (self.inner, x.inner) { - (Inner { intrinsics: s_intr }, Inner { intrinsics: x_intr }) => { + match (self.is_intr, x.is_intr) { + (true, true) => { + let s_intr = self.inner.intrinsics; + let x_intr = x.inner.intrinsics; + let (r0, r1) = s_intr.clmul(x_intr); self.inner.intrinsics = r0; x.inner.intrinsics = r1; } - (Inner { soft: s_soft }, Inner { soft: x_soft }) => { + (false, false) => { + let s_soft = self.inner.soft; + let x_soft = x.inner.soft; + let (r0, r1) = s_soft.clmul(x_soft); self.inner.soft = r0; x.inner.soft = r1; @@ -178,36 +199,36 @@ impl Clmul { /// Reduces the polynomial represented in bits modulo the GCM polynomial x^128 + x^7 + x^2 + x + 1. /// x and y are resp. upper and lower bits of the polynomial. pub fn reduce_gcm(x: Self, y: Self) -> Self { - match x.intrinsics { - Some(x_intr) => match y.intrinsics { - Some(y_intr) => { + unsafe { + match (x.is_intr, y.is_intr) { + (true, true) => { + let x_intr = x.inner.intrinsics; + let y_intr = y.inner.intrinsics; + cfg_if! { if #[cfg(any(all(target_arch = "aarch64", feature = "armv8"), any(target_arch = "x86_64", target_arch = "x86")))]{ let r = intrinsics::Clmul::reduce_gcm(x_intr, y_intr); - }else{ + } else { let r = soft::Clmul::reduce_gcm(x_intr, y_intr); } } Self { - intrinsics: Some(r), - soft: None, + inner: Inner { intrinsics: r }, + is_intr: true, } } - None => unreachable!(), - }, - None => match x.soft { - Some(x_soft) => match y.soft { - Some(y_soft) => { - let r = soft::Clmul::reduce_gcm(x_soft, y_soft); - Self { - intrinsics: None, - soft: Some(r), - } + (false, false) => { + let x_soft = x.inner.soft; + let y_soft = y.inner.soft; + + let r = soft::Clmul::reduce_gcm(x_soft, y_soft); + Self { + inner: Inner { soft: r }, + is_intr: false, } - None => unreachable!(), - }, - None => unreachable!(), - }, + } + _ => unreachable!(), + } } } } @@ -215,12 +236,12 @@ impl Clmul { impl From for [u8; 16] { #[inline] fn from(m: Clmul) -> [u8; 16] { - match m.intrinsics { - Some(intr) => intr.into(), - None => match m.soft { - Some(soft) => soft.into(), - None => unreachable!(), - }, + unsafe { + if m.is_intr { + m.inner.intrinsics.into() + } else { + m.inner.soft.into() + } } } } @@ -230,24 +251,26 @@ impl BitXor for Clmul { #[inline] fn bitxor(self, other: Self) -> Self::Output { - match self.intrinsics { - Some(a) => match other.intrinsics { - Some(b) => Self { - intrinsics: Some(a ^ b), - soft: None, - }, - None => unreachable!(), - }, - None => match self.soft { - Some(a) => match other.soft { - Some(b) => Self { - intrinsics: None, - soft: Some(a ^ b), - }, - None => unreachable!(), - }, - None => unreachable!(), - }, + unsafe { + match (self.is_intr, other.is_intr) { + (true, true) => { + let a = self.inner.intrinsics; + let b = other.inner.intrinsics; + Self { + inner: Inner { intrinsics: a ^ b }, + is_intr: true, + } + } + (false, false) => { + let a = self.inner.soft; + let b = other.inner.soft; + Self { + inner: Inner { soft: a ^ b }, + is_intr: false, + } + } + _ => unreachable!(), + } } } } @@ -255,40 +278,32 @@ impl BitXor for Clmul { impl BitXorAssign for Clmul { #[inline] fn bitxor_assign(&mut self, other: Self) { - match self.intrinsics { - Some(a) => match other.intrinsics { - Some(b) => { - self.intrinsics = Some(a ^ b); + unsafe { + match (self.is_intr, other.is_intr) { + (true, true) => { + let a = self.inner.intrinsics; + let b = other.inner.intrinsics; + self.inner.intrinsics = a ^ b; } - None => unreachable!(), - }, - None => match self.soft { - Some(a) => match other.soft { - Some(b) => { - self.soft = Some(a ^ b); - } - None => unreachable!(), - }, - None => unreachable!(), - }, + (false, false) => { + let a = self.inner.soft; + let b = other.inner.soft; + self.inner.soft = a ^ b; + } + _ => unreachable!(), + } } } } impl PartialEq for Clmul { fn eq(&self, other: &Self) -> bool { - match self.intrinsics { - Some(a) => match other.intrinsics { - Some(b) => a == b, - None => unreachable!(), - }, - None => match self.soft { - Some(a) => match other.soft { - Some(b) => a == b, - None => unreachable!(), - }, - None => unreachable!(), - }, + unsafe { + match (self.is_intr, other.is_intr) { + (true, true) => self.inner.intrinsics == other.inner.intrinsics, + (false, false) => self.inner.soft == other.inner.soft, + _ => unreachable!(), + } } } } From 946f0fc4e7c2c3254dfbe0028690517a63de8835 Mon Sep 17 00:00:00 2001 From: YoungWan Kwon <87213416+000wan@users.noreply.github.com> Date: Fri, 29 Sep 2023 03:46:48 +0900 Subject: [PATCH 3/8] Add debug trait for Clmul --- clmul/src/backend.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/clmul/src/backend.rs b/clmul/src/backend.rs index 4a804f62..9732786c 100644 --- a/clmul/src/backend.rs +++ b/clmul/src/backend.rs @@ -56,16 +56,29 @@ cfg_if! { } } +#[derive(Clone, Copy)] pub struct Clmul { inner: Inner, is_intr: bool, } +#[derive(Clone, Copy)] union Inner { intrinsics: intrinsics::Clmul, soft: soft::Clmul, } +impl core::fmt::Debug for Clmul { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + unsafe { + match self.is_intr { + true => self.inner.intrinsics.fmt(f), + false => self.inner.soft.fmt(f), + } + } + } +} + /* cfg_if! { if #[cfg(any(all(target_arch = "aarch64", feature = "armv8"), any(target_arch = "x86_64", target_arch = "x86")))]{ From dac8b49084b0086fdb793951755cff347d4baa78 Mon Sep 17 00:00:00 2001 From: YoungWan Kwon <87213416+000wan@users.noreply.github.com> Date: Fri, 29 Sep 2023 05:37:05 +0900 Subject: [PATCH 4/8] Fix clippy error --- ot/mpz-ot/src/kos/receiver.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ot/mpz-ot/src/kos/receiver.rs b/ot/mpz-ot/src/kos/receiver.rs index 3856e145..8fe4b11d 100644 --- a/ot/mpz-ot/src/kos/receiver.rs +++ b/ot/mpz-ot/src/kos/receiver.rs @@ -182,7 +182,7 @@ where let actual_delta = <[u8; 16]>::from_lsb0_iter(choices).into(); if expected_delta != actual_delta { - return Err(ReceiverVerifyError::InconsistentDelta).map_err(ReceiverError::from)?; + return Err(ReceiverError::from(ReceiverVerifyError::InconsistentDelta)); } self.state = State::Verify(receiver.start_verification(actual_delta)?); From 0bf31e6e0ec558d6333bce7c093da922f89b0267 Mon Sep 17 00:00:00 2001 From: YoungWan Kwon <87213416+000wan@users.noreply.github.com> Date: Sat, 30 Sep 2023 02:10:03 +0900 Subject: [PATCH 5/8] Use static variable to check intrinsics --- clmul/Cargo.toml | 1 + clmul/src/backend.rs | 246 +++++++++++++++++++------------------------ 2 files changed, 108 insertions(+), 139 deletions(-) diff --git a/clmul/Cargo.toml b/clmul/Cargo.toml index 8f56c8b2..d7b8c6c4 100644 --- a/clmul/Cargo.toml +++ b/clmul/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] cfg-if.workspace = true bytemuck = {workspace = true, features = ["derive"]} +lazy_static = "1.4.0" [target.'cfg(any(target_arch = "aarch64", target_arch = "x86_64", target_arch = "x86"))'.dependencies] cpufeatures.workspace = true diff --git a/clmul/src/backend.rs b/clmul/src/backend.rs index 9732786c..4c00ccc3 100644 --- a/clmul/src/backend.rs +++ b/clmul/src/backend.rs @@ -56,10 +56,26 @@ cfg_if! { } } +use lazy_static::lazy_static; +lazy_static! { + static ref IS_INTR: bool = { + cfg_if! { + if #[cfg(feature = "force-soft")] { + false + } else if #[cfg(any(all(target_arch = "aarch64", feature = "armv8"), any(target_arch = "x86_64", target_arch = "x86")))]{ + mul_intrinsics::get() + } else { + // "force-soft" feature was not enabled but neither was + // supported arch found. Falling back to soft backend. + false + } + } + }; +} + #[derive(Clone, Copy)] pub struct Clmul { inner: Inner, - is_intr: bool, } #[derive(Clone, Copy)] @@ -71,9 +87,10 @@ union Inner { impl core::fmt::Debug for Clmul { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { unsafe { - match self.is_intr { - true => self.inner.intrinsics.fmt(f), - false => self.inner.soft.fmt(f), + if *IS_INTR { + self.inner.intrinsics.fmt(f) + } else { + self.inner.soft.fmt(f) } } } @@ -110,32 +127,17 @@ cfg_if! { impl Clmul { pub fn new(h: &[u8; 16]) -> Self { - cfg_if! { - if #[cfg(feature = "force-soft")] { - Self { - inner: Inner { soft: soft::Clmul::new(h) }, - is_intr: false, - } - } else if #[cfg(any(all(target_arch = "aarch64", feature = "armv8"), any(target_arch = "x86_64", target_arch = "x86")))]{ - if mul_intrinsics::get() { - Self { - inner: Inner { intrinsics: intrinsics::Clmul::new(h) }, - is_intr: true, - } - } else { - // supported arch was found but intrinsics are not available - Self { - inner: Inner { soft: soft::Clmul::new(h) }, - is_intr: false, - } - } - } else { - // "force-soft" feature was not enabled but neither was - // supported arch found. Falling back to soft backend. - Self { - inner: Inner { soft: soft::Clmul::new(h) }, - is_intr: false, - } + if *IS_INTR { + Self { + inner: Inner { + intrinsics: intrinsics::Clmul::new(h), + }, + } + } else { + Self { + inner: Inner { + soft: soft::Clmul::new(h), + }, } } } @@ -143,40 +145,32 @@ impl Clmul { /// Performs carryless multiplication pub fn clmul(self, x: Self) -> (Self, Self) { unsafe { - match (self.is_intr, x.is_intr) { - (true, true) => { - let s_intr = self.inner.intrinsics; - let x_intr = x.inner.intrinsics; - - let (r0, r1) = s_intr.clmul(x_intr); - ( - Self { - inner: Inner { intrinsics: r0 }, - is_intr: true, - }, - Self { - inner: Inner { intrinsics: r1 }, - is_intr: true, - }, - ) - } - (false, false) => { - let s_soft = self.inner.soft; - let x_soft = x.inner.soft; - - let (r0, r1) = s_soft.clmul(x_soft); - ( - Self { - inner: Inner { soft: r0 }, - is_intr: false, - }, - Self { - inner: Inner { soft: r1 }, - is_intr: false, - }, - ) - } - _ => unreachable!(), + if *IS_INTR { + let s_intr = self.inner.intrinsics; + let x_intr = x.inner.intrinsics; + + let (r0, r1) = s_intr.clmul(x_intr); + ( + Self { + inner: Inner { intrinsics: r0 }, + }, + Self { + inner: Inner { intrinsics: r1 }, + }, + ) + } else { + let s_soft = self.inner.soft; + let x_soft = x.inner.soft; + + let (r0, r1) = s_soft.clmul(x_soft); + ( + Self { + inner: Inner { soft: r0 }, + }, + Self { + inner: Inner { soft: r1 }, + }, + ) } } } @@ -187,24 +181,20 @@ impl Clmul { /// The high bits will be placed in `self`, the low bits - in `x`. pub fn clmul_reuse(&mut self, x: &mut Self) { unsafe { - match (self.is_intr, x.is_intr) { - (true, true) => { - let s_intr = self.inner.intrinsics; - let x_intr = x.inner.intrinsics; - - let (r0, r1) = s_intr.clmul(x_intr); - self.inner.intrinsics = r0; - x.inner.intrinsics = r1; - } - (false, false) => { - let s_soft = self.inner.soft; - let x_soft = x.inner.soft; + if *IS_INTR { + let s_intr = self.inner.intrinsics; + let x_intr = x.inner.intrinsics; - let (r0, r1) = s_soft.clmul(x_soft); - self.inner.soft = r0; - x.inner.soft = r1; - } - _ => unreachable!(), + let (r0, r1) = s_intr.clmul(x_intr); + self.inner.intrinsics = r0; + x.inner.intrinsics = r1; + } else { + let s_soft = self.inner.soft; + let x_soft = x.inner.soft; + + let (r0, r1) = s_soft.clmul(x_soft); + self.inner.soft = r0; + x.inner.soft = r1; } } } @@ -213,34 +203,22 @@ impl Clmul { /// x and y are resp. upper and lower bits of the polynomial. pub fn reduce_gcm(x: Self, y: Self) -> Self { unsafe { - match (x.is_intr, y.is_intr) { - (true, true) => { - let x_intr = x.inner.intrinsics; - let y_intr = y.inner.intrinsics; - - cfg_if! { - if #[cfg(any(all(target_arch = "aarch64", feature = "armv8"), any(target_arch = "x86_64", target_arch = "x86")))]{ - let r = intrinsics::Clmul::reduce_gcm(x_intr, y_intr); - } else { - let r = soft::Clmul::reduce_gcm(x_intr, y_intr); - } - } - Self { - inner: Inner { intrinsics: r }, - is_intr: true, - } + if *IS_INTR { + let x_intr = x.inner.intrinsics; + let y_intr = y.inner.intrinsics; + + let r = intrinsics::Clmul::reduce_gcm(x_intr, y_intr); + Self { + inner: Inner { intrinsics: r }, } - (false, false) => { - let x_soft = x.inner.soft; - let y_soft = y.inner.soft; + } else { + let x_soft = x.inner.soft; + let y_soft = y.inner.soft; - let r = soft::Clmul::reduce_gcm(x_soft, y_soft); - Self { - inner: Inner { soft: r }, - is_intr: false, - } + let r = soft::Clmul::reduce_gcm(x_soft, y_soft); + Self { + inner: Inner { soft: r }, } - _ => unreachable!(), } } } @@ -250,7 +228,7 @@ impl From for [u8; 16] { #[inline] fn from(m: Clmul) -> [u8; 16] { unsafe { - if m.is_intr { + if *IS_INTR { m.inner.intrinsics.into() } else { m.inner.soft.into() @@ -265,24 +243,18 @@ impl BitXor for Clmul { #[inline] fn bitxor(self, other: Self) -> Self::Output { unsafe { - match (self.is_intr, other.is_intr) { - (true, true) => { - let a = self.inner.intrinsics; - let b = other.inner.intrinsics; - Self { - inner: Inner { intrinsics: a ^ b }, - is_intr: true, - } + if *IS_INTR { + let a = self.inner.intrinsics; + let b = other.inner.intrinsics; + Self { + inner: Inner { intrinsics: a ^ b }, } - (false, false) => { - let a = self.inner.soft; - let b = other.inner.soft; - Self { - inner: Inner { soft: a ^ b }, - is_intr: false, - } + } else { + let a = self.inner.soft; + let b = other.inner.soft; + Self { + inner: Inner { soft: a ^ b }, } - _ => unreachable!(), } } } @@ -292,18 +264,14 @@ impl BitXorAssign for Clmul { #[inline] fn bitxor_assign(&mut self, other: Self) { unsafe { - match (self.is_intr, other.is_intr) { - (true, true) => { - let a = self.inner.intrinsics; - let b = other.inner.intrinsics; - self.inner.intrinsics = a ^ b; - } - (false, false) => { - let a = self.inner.soft; - let b = other.inner.soft; - self.inner.soft = a ^ b; - } - _ => unreachable!(), + if *IS_INTR { + let a = self.inner.intrinsics; + let b = other.inner.intrinsics; + self.inner.intrinsics = a ^ b; + } else { + let a = self.inner.soft; + let b = other.inner.soft; + self.inner.soft = a ^ b; } } } @@ -312,10 +280,10 @@ impl BitXorAssign for Clmul { impl PartialEq for Clmul { fn eq(&self, other: &Self) -> bool { unsafe { - match (self.is_intr, other.is_intr) { - (true, true) => self.inner.intrinsics == other.inner.intrinsics, - (false, false) => self.inner.soft == other.inner.soft, - _ => unreachable!(), + if *IS_INTR { + self.inner.intrinsics == other.inner.intrinsics + } else { + self.inner.soft == other.inner.soft } } } From 68b82ad7f4f9fe518dbd846c963dd30b70327e86 Mon Sep 17 00:00:00 2001 From: YoungWan Kwon <87213416+000wan@users.noreply.github.com> Date: Sat, 30 Sep 2023 02:34:05 +0900 Subject: [PATCH 6/8] Remove previous codes --- clmul/src/backend.rs | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/clmul/src/backend.rs b/clmul/src/backend.rs index 4c00ccc3..c92e1f1d 100644 --- a/clmul/src/backend.rs +++ b/clmul/src/backend.rs @@ -74,6 +74,7 @@ lazy_static! { } #[derive(Clone, Copy)] +/// Carryless multiplication pub struct Clmul { inner: Inner, } @@ -96,35 +97,6 @@ impl core::fmt::Debug for Clmul { } } -/* -cfg_if! { - if #[cfg(any(all(target_arch = "aarch64", feature = "armv8"), any(target_arch = "x86_64", target_arch = "x86")))]{ - #[derive(Clone, Copy, Debug)] - /// Carryless multiplication - pub struct Clmul { - intrinsics: Option, - soft: Option, - } - } else { - #[derive(Clone, Copy, Debug)] - /// Carryless multiplication - pub struct Clmul { - // intrinsics will never be used on a non-supported arch but Rust - // won't allow to declare it with a None type, so we need to - // provide some type - intrinsics: Option, - soft: Option, - } - } -} -*/ - -// #[derive(Clone, Copy)] -// pub struct Clmul { -// intrinsics: Option, -// soft: Option, -// } - impl Clmul { pub fn new(h: &[u8; 16]) -> Self { if *IS_INTR { From c0c846d6cb47e7b8645e84ce16f1eca8fd162261 Mon Sep 17 00:00:00 2001 From: YoungWan Kwon <87213416+000wan@users.noreply.github.com> Date: Sun, 1 Oct 2023 01:51:14 +0900 Subject: [PATCH 7/8] Use `InitToken` for Clmul --- clmul/Cargo.toml | 1 - clmul/src/backend.rs | 104 +++++++++++++++++++------------------------ 2 files changed, 46 insertions(+), 59 deletions(-) diff --git a/clmul/Cargo.toml b/clmul/Cargo.toml index d7b8c6c4..8f56c8b2 100644 --- a/clmul/Cargo.toml +++ b/clmul/Cargo.toml @@ -7,7 +7,6 @@ edition = "2021" [dependencies] cfg-if.workspace = true bytemuck = {workspace = true, features = ["derive"]} -lazy_static = "1.4.0" [target.'cfg(any(target_arch = "aarch64", target_arch = "x86_64", target_arch = "x86"))'.dependencies] cpufeatures.workspace = true diff --git a/clmul/src/backend.rs b/clmul/src/backend.rs index c92e1f1d..69e21deb 100644 --- a/clmul/src/backend.rs +++ b/clmul/src/backend.rs @@ -56,27 +56,11 @@ cfg_if! { } } -use lazy_static::lazy_static; -lazy_static! { - static ref IS_INTR: bool = { - cfg_if! { - if #[cfg(feature = "force-soft")] { - false - } else if #[cfg(any(all(target_arch = "aarch64", feature = "armv8"), any(target_arch = "x86_64", target_arch = "x86")))]{ - mul_intrinsics::get() - } else { - // "force-soft" feature was not enabled but neither was - // supported arch found. Falling back to soft backend. - false - } - } - }; -} - #[derive(Clone, Copy)] /// Carryless multiplication pub struct Clmul { inner: Inner, + token: mul_intrinsics::InitToken, } #[derive(Clone, Copy)] @@ -88,7 +72,7 @@ union Inner { impl core::fmt::Debug for Clmul { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { unsafe { - if *IS_INTR { + if self.token.get() { self.inner.intrinsics.fmt(f) } else { self.inner.soft.fmt(f) @@ -99,51 +83,52 @@ impl core::fmt::Debug for Clmul { impl Clmul { pub fn new(h: &[u8; 16]) -> Self { - if *IS_INTR { - Self { - inner: Inner { - intrinsics: intrinsics::Clmul::new(h), - }, + let (token, has_intrinsics) = mul_intrinsics::init_get(); + + let inner = if cfg!(feature = "force-soft") { + Inner { + soft: soft::Clmul::new(h), + } + } else if has_intrinsics { + Inner { + intrinsics: intrinsics::Clmul::new(h), } } else { - Self { - inner: Inner { - soft: soft::Clmul::new(h), - }, + Inner { + soft: soft::Clmul::new(h), } - } + }; + + Self { inner, token } } /// Performs carryless multiplication pub fn clmul(self, x: Self) -> (Self, Self) { unsafe { - if *IS_INTR { + let (in0, in1) = if self.token.get() { let s_intr = self.inner.intrinsics; let x_intr = x.inner.intrinsics; let (r0, r1) = s_intr.clmul(x_intr); - ( - Self { - inner: Inner { intrinsics: r0 }, - }, - Self { - inner: Inner { intrinsics: r1 }, - }, - ) + (Inner { intrinsics: r0 }, Inner { intrinsics: r1 }) } else { let s_soft = self.inner.soft; let x_soft = x.inner.soft; let (r0, r1) = s_soft.clmul(x_soft); - ( - Self { - inner: Inner { soft: r0 }, - }, - Self { - inner: Inner { soft: r1 }, - }, - ) - } + (Inner { soft: r0 }, Inner { soft: r1 }) + }; + + ( + Self { + inner: in0, + token: self.token, + }, + Self { + inner: in1, + token: x.token, + }, + ) } } @@ -153,7 +138,7 @@ impl Clmul { /// The high bits will be placed in `self`, the low bits - in `x`. pub fn clmul_reuse(&mut self, x: &mut Self) { unsafe { - if *IS_INTR { + if self.token.get() { let s_intr = self.inner.intrinsics; let x_intr = x.inner.intrinsics; @@ -175,13 +160,14 @@ impl Clmul { /// x and y are resp. upper and lower bits of the polynomial. pub fn reduce_gcm(x: Self, y: Self) -> Self { unsafe { - if *IS_INTR { + if x.token.get() { let x_intr = x.inner.intrinsics; let y_intr = y.inner.intrinsics; let r = intrinsics::Clmul::reduce_gcm(x_intr, y_intr); Self { inner: Inner { intrinsics: r }, + token: x.token, } } else { let x_soft = x.inner.soft; @@ -190,6 +176,7 @@ impl Clmul { let r = soft::Clmul::reduce_gcm(x_soft, y_soft); Self { inner: Inner { soft: r }, + token: x.token, } } } @@ -200,7 +187,7 @@ impl From for [u8; 16] { #[inline] fn from(m: Clmul) -> [u8; 16] { unsafe { - if *IS_INTR { + if m.token.get() { m.inner.intrinsics.into() } else { m.inner.soft.into() @@ -215,18 +202,19 @@ impl BitXor for Clmul { #[inline] fn bitxor(self, other: Self) -> Self::Output { unsafe { - if *IS_INTR { + let inner = if self.token.get() { let a = self.inner.intrinsics; let b = other.inner.intrinsics; - Self { - inner: Inner { intrinsics: a ^ b }, - } + Inner { intrinsics: a ^ b } } else { let a = self.inner.soft; let b = other.inner.soft; - Self { - inner: Inner { soft: a ^ b }, - } + Inner { soft: a ^ b } + }; + + Self { + inner, + token: self.token, } } } @@ -236,7 +224,7 @@ impl BitXorAssign for Clmul { #[inline] fn bitxor_assign(&mut self, other: Self) { unsafe { - if *IS_INTR { + if self.token.get() { let a = self.inner.intrinsics; let b = other.inner.intrinsics; self.inner.intrinsics = a ^ b; @@ -252,7 +240,7 @@ impl BitXorAssign for Clmul { impl PartialEq for Clmul { fn eq(&self, other: &Self) -> bool { unsafe { - if *IS_INTR { + if self.token.get() { self.inner.intrinsics == other.inner.intrinsics } else { self.inner.soft == other.inner.soft From 570d04288329bf8d7eddc484b841d97ee33504d4 Mon Sep 17 00:00:00 2001 From: YoungWan Kwon <87213416+000wan@users.noreply.github.com> Date: Mon, 2 Oct 2023 10:00:13 +0900 Subject: [PATCH 8/8] Enable force-soft feature --- clmul/src/backend.rs | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/clmul/src/backend.rs b/clmul/src/backend.rs index 69e21deb..b704049a 100644 --- a/clmul/src/backend.rs +++ b/clmul/src/backend.rs @@ -69,10 +69,17 @@ union Inner { soft: soft::Clmul, } +impl mul_intrinsics::InitToken { + #[inline(always)] + fn get_intr(&self) -> bool { + !cfg!(feature = "force-soft") && self.get() + } +} + impl core::fmt::Debug for Clmul { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { unsafe { - if self.token.get() { + if self.token.get_intr() { self.inner.intrinsics.fmt(f) } else { self.inner.soft.fmt(f) @@ -103,9 +110,10 @@ impl Clmul { } /// Performs carryless multiplication + #[inline] pub fn clmul(self, x: Self) -> (Self, Self) { unsafe { - let (in0, in1) = if self.token.get() { + let (in0, in1) = if self.token.get_intr() { let s_intr = self.inner.intrinsics; let x_intr = x.inner.intrinsics; @@ -136,9 +144,10 @@ impl Clmul { /// operands to return the result. This gives a ~6x speed up compared /// to clmul() where we create new objects containing the result. /// The high bits will be placed in `self`, the low bits - in `x`. + #[inline] pub fn clmul_reuse(&mut self, x: &mut Self) { unsafe { - if self.token.get() { + if self.token.get_intr() { let s_intr = self.inner.intrinsics; let x_intr = x.inner.intrinsics; @@ -158,9 +167,10 @@ impl Clmul { /// Reduces the polynomial represented in bits modulo the GCM polynomial x^128 + x^7 + x^2 + x + 1. /// x and y are resp. upper and lower bits of the polynomial. + #[inline] pub fn reduce_gcm(x: Self, y: Self) -> Self { unsafe { - if x.token.get() { + if x.token.get_intr() { let x_intr = x.inner.intrinsics; let y_intr = y.inner.intrinsics; @@ -187,7 +197,7 @@ impl From for [u8; 16] { #[inline] fn from(m: Clmul) -> [u8; 16] { unsafe { - if m.token.get() { + if m.token.get_intr() { m.inner.intrinsics.into() } else { m.inner.soft.into() @@ -202,7 +212,7 @@ impl BitXor for Clmul { #[inline] fn bitxor(self, other: Self) -> Self::Output { unsafe { - let inner = if self.token.get() { + let inner = if self.token.get_intr() { let a = self.inner.intrinsics; let b = other.inner.intrinsics; Inner { intrinsics: a ^ b } @@ -224,7 +234,7 @@ impl BitXorAssign for Clmul { #[inline] fn bitxor_assign(&mut self, other: Self) { unsafe { - if self.token.get() { + if self.token.get_intr() { let a = self.inner.intrinsics; let b = other.inner.intrinsics; self.inner.intrinsics = a ^ b; @@ -238,9 +248,10 @@ impl BitXorAssign for Clmul { } impl PartialEq for Clmul { + #[inline] fn eq(&self, other: &Self) -> bool { unsafe { - if self.token.get() { + if self.token.get_intr() { self.inner.intrinsics == other.inner.intrinsics } else { self.inner.soft == other.inner.soft