update canton to 20241121.14638.ve6ce9764 (#20347)

* update canton to 20241121.14638.ve6ce9764 tell-slack: canton * Removed java8 dependency from community/base --------- Co-authored-by: Azure Pipelines Daml Build <support@digitalasset.com> Co-authored-by: Joao Sa <joao.sousa@digitalasset.com>
digital-asset · Nov 22, 2024 · 4b8a0ac · 4b8a0ac
1 parent c054a1c
commit 4b8a0ac
Show file tree

Hide file tree

Showing 235 changed files with 7,737 additions and 2,850 deletions.
diff --git a/sdk/canton/BUILD.bazel b/sdk/canton/BUILD.bazel
@@ -567,7 +567,6 @@ scala_library(
         "@maven//:org_bouncycastle_bcprov_jdk15on",
         "@maven//:org_postgresql_postgresql",
         "@maven//:org_scala_lang_modules_scala_collection_contrib_2_13",
-        "@maven//:org_scala_lang_modules_scala_java8_compat_2_13",
         "@maven//:org_scalaz_scalaz_core_2_13",
         "@maven//:org_slf4j_slf4j_api",
         "@maven//:org_typelevel_cats_core_2_13",

diff --git a/.../src/main/protobuf/com/digitalasset/canton/admin/participant/v30/inspection_service.proto b/.../src/main/protobuf/com/digitalasset/canton/admin/participant/v30/inspection_service.proto
@@ -85,7 +85,14 @@ message CommitmentContractMeta {
 // The participant needs to be authorized to see the requested contracts, for example using JWT authorization
 message InspectCommitmentContracts {
   message Request {
+    // the contracts whose state and payload we want to retrieve
     repeated bytes cids = 1;
+    // the domain on which we expect the contracts to be active
+    string expected_domain_id = 2;
+    // timestamp at which we retrieve the state of the contracts on the expected domain
+    google.protobuf.Timestamp timestamp = 3;
+    // whether to retrieve the payload of the contracts
+    bool download_payload = 4;
   }
 
   message Response {
@@ -96,8 +103,59 @@ message InspectCommitmentContracts {
 
 message CommitmentContract {
   option (scalapb.message).companion_extends = "com.digitalasset.canton.version.AlphaProtoVersion";
-  Contract serialized_contract = 1;
-  bytes creating_tx_id = 2;
+  bytes cid = 1;
+  bool active_on_expected_domain = 2;
+  Contract serialized_contract = 3;
+  repeated ContractState.DomainState states = 4;
+}
+
+message ContractState {
+
+  message DomainState {
+    option (scalapb.message).companion_extends = "com.digitalasset.canton.version.AlphaProtoVersion";
+    string domain_id = 1;
+    // The Canton ACS has a few more states, e.g., "purged". However, we do not include them because they will likely
+    // not be accessible over the ledger API when we switch to that.
+    oneof state {
+      ContractState.Created created = 2;
+      ContractState.Archived archived = 3;
+      ContractState.Unassigned unassigned = 4;
+      ContractState.Assigned assigned = 5;
+      ContractState.Unknown unknown = 6;
+    }
+  }
+
+  message Created {
+    option (scalapb.message).companion_extends = "com.digitalasset.canton.version.AlphaProtoVersion";
+  }
+
+  message Assigned {
+    option (scalapb.message).companion_extends = "com.digitalasset.canton.version.AlphaProtoVersion";
+    int64 reassignment_counter_target = 1;
+    ReassignmentId reassignment_id = 2;
+  }
+
+  message Archived {
+    option (scalapb.message).companion_extends = "com.digitalasset.canton.version.AlphaProtoVersion";
+  }
+
+  message Unassigned {
+    option (scalapb.message).companion_extends = "com.digitalasset.canton.version.AlphaProtoVersion";
+    string target_domain_id = 1;
+    int64 reassignment_counter_src = 2;
+    ReassignmentId reassignment_id = 3;
+  }
+
+  // the contract is unknown to the domain, but may have existed in the past and have been pruned meanwhile
+  message Unknown {
+    option (scalapb.message).companion_extends = "com.digitalasset.canton.version.AlphaProtoVersion";
+  }
+
+  message ReassignmentId {
+    option (scalapb.message).companion_extends = "com.digitalasset.canton.version.AlphaProtoVersion";
+    string source_domain_id = 1;
+    google.protobuf.Timestamp unassign_timestamp = 2;
+  }
 }
 
 

diff --git a/.../src/main/scala/com/digitalasset/canton/admin/api/client/commands/LedgerApiCommands.scala b/.../src/main/scala/com/digitalasset/canton/admin/api/client/commands/LedgerApiCommands.scala
@@ -1223,7 +1223,7 @@ object LedgerApiCommands {
           )
         case DeduplicationPeriod.DeduplicationOffset(offset) =>
           Commands.DeduplicationPeriod.DeduplicationOffset(
-            offset.toLong
+            offset.fold(0L)(_.unwrap)
           )
       },
       minLedgerTimeAbs = minLedgerTimeAbs.map(ProtoConverter.InstantConverter.toProtoPrimitive),
@@ -1390,6 +1390,7 @@ object LedgerApiCommands {
         domainId: Option[DomainId],
         applicationId: String,
         packageIdSelectionPreference: Seq[LfPackageId],
+        verboseHashing: Boolean,
     ) extends BaseCommand[
           PrepareSubmissionRequest,
           PrepareSubmissionResponse,
@@ -1411,6 +1412,7 @@ object LedgerApiCommands {
             disclosedContracts = disclosedContracts,
             domainId = domainId.map(_.toProtoPrimitive).getOrElse(""),
             packageIdSelectionPreference = packageIdSelectionPreference,
+            verboseHashing = verboseHashing,
           )
         )
 
@@ -1467,7 +1469,7 @@ object LedgerApiCommands {
           )
         case DeduplicationPeriod.DeduplicationOffset(offset) =>
           ExecuteSubmissionRequest.DeduplicationPeriod.DeduplicationOffset(
-            offset.toLong
+            offset.fold(0L)(_.unwrap)
           )
       }
 

diff --git a/...in/scala/com/digitalasset/canton/admin/api/client/commands/ParticipantAdminCommands.scala b/...in/scala/com/digitalasset/canton/admin/api/client/commands/ParticipantAdminCommands.scala
@@ -1052,14 +1052,20 @@ object ParticipantAdminCommands {
     final case class CommitmentContracts(
         observer: StreamObserver[v30.InspectCommitmentContracts.Response],
         contracts: Seq[LfContractId],
+        expectedDomain: DomainId,
+        timestamp: CantonTimestamp,
+        downloadPayload: Boolean,
     ) extends Base[
           v30.InspectCommitmentContracts.Request,
           CancellableContext,
           CancellableContext,
         ] {
       override protected def createRequest() = Right(
         v30.InspectCommitmentContracts.Request(
-          contracts.map(_.toBytes.toByteString)
+          contracts.map(_.toBytes.toByteString),
+          expectedDomain.toProtoPrimitive,
+          Some(timestamp.toProtoTimestamp),
+          downloadPayload,
         )
       )
 

diff --git a/...ton/community/app-base/src/main/scala/com/digitalasset/canton/console/ConsoleMacros.scala b/...ton/community/app-base/src/main/scala/com/digitalasset/canton/console/ConsoleMacros.scala
@@ -1031,12 +1031,13 @@ trait ConsoleMacros extends NamedLogging with NoTracing {
         binaryOutputFile: Option[String] = None,
         readableOutputFile: Option[String] = None,
     ): Unit = {
+
       // TODO(#9557) 0. If integrityChecks is true, check that, at the given mismatch timestamp, the target
       //  participant's own commitment and received counterCommitment indeed mismatch
       //  We read these commitment from the target participant's store using R5 endpoints
 
       // TODO(#9557) 1. Downloading the shared contract metadata from counter-participant:
-      //  counterParticipant.commitments.open_commitment(...)
+      // val counterParticipantCmtMetadata = counterParticipant.commitments.open_commitment(...)
 
       // TODO(#9557) 2. If integrityChecks is true, check that the contract metadata sent matches the counter commitment
       //  by uploading the contract metadata to the target participant.
@@ -1045,17 +1046,32 @@ trait ConsoleMacros extends NamedLogging with NoTracing {
       //  hierarchical commitments. In this case, we can perform the check as the last step, after we retrieve the
       //  contract payloads from the counter-participant.
 
-      // TODO(#9557) 3. Identify mismatching contracts by checking the counterParticipant's contracts metadata
+      // TODO(#9557) 3. Download the shared contract metadata from target participant
+      // val targetParticipantCmtMetadata = targetParticipant.commitments.open_commitment()
+
+      // TODO(#9557) 4. Identify mismatching contracts by checking the counterParticipant's contracts metadata
       //  against the ACS contracts of the target participant:
-      //  targetParticipant.commitments.active_contracts_mismatches(...)
+      // CommitmentContractMetadata.compare(targetParticipantCmtMetadata, counterParticipantCmtMetadata)
+
       // TODO(#20583) Investigate fetching the ACS snapshot via LAPI without the contract payload. LAPI has longer lived data
       //  and allows for party filtering.
 
-      // TODO(#9557) 4. Request contract payloads from the counterParticipant for shared contracts that cause mismatches
+      // TODO(#9557) 5. Identify mismatch reasons from the target participant for shared contracts that cause mismatches
+      // targetParticipant.commitments.inspect_commitment_contract()
+
+      // TODO(#9557) 6. Request mismatch reasons contract payloads from the counterParticipant for shared contracts that cause mismatches
+      // counterParticipant.commitments.inspect_commitment_contract()
+
+      // TODO(#9557) 7. Compile the data in 4 and 5 into mismatch reasons and write them to the binary output file:
+      // counterParticipant.commitments.inspect_commitment_contract()
+
+      // mismatch reason lives only in console macros
+
+      // TODO(#9557) 8. Request contract payloads from the counterParticipant for shared contracts that cause mismatches
       //   and write them to the binary output file:
       //   counterParticipant.commitments.download_contract_reconciliation_payloads(...)
 
-      // TODO(#9557) 5. Write user-readable data in the readable output file regarding mismatching contracts ids
+      // TODO(#9557) 9. Write user-readable data in the readable output file regarding mismatching contracts ids
     }
   }
 }

diff --git a/...ase/src/main/scala/com/digitalasset/canton/console/commands/LedgerApiAdministration.scala b/...ase/src/main/scala/com/digitalasset/canton/console/commands/LedgerApiAdministration.scala
@@ -451,6 +451,7 @@ trait BaseLedgerApiAdministration extends NoTracing {
           disclosedContracts: Seq[DisclosedContract] = Seq.empty,
           applicationId: String = applicationId,
           userPackageSelectionPreference: Seq[LfPackageId] = Seq.empty,
+          verboseHashing: Boolean = false,
       ): PrepareResponseProto =
         consoleEnvironment.run {
           ledgerApiCommand(
@@ -464,6 +465,7 @@ trait BaseLedgerApiAdministration extends NoTracing {
               domainId,
               applicationId,
               userPackageSelectionPreference,
+              verboseHashing,
             )
           )
         }
@@ -1911,15 +1913,13 @@ trait BaseLedgerApiAdministration extends NoTracing {
             actAs: Seq[PartyId],
             commands: Seq[javab.data.Command],
             domainId: Option[DomainId] = None,
-            workflowId: String = "",
             commandId: String = "",
-            deduplicationPeriod: Option[DeduplicationPeriod] = None,
-            submissionId: String = "",
             minLedgerTimeAbs: Option[Instant] = None,
             readAs: Seq[PartyId] = Seq.empty,
             disclosedContracts: Seq[javab.data.DisclosedContract] = Seq.empty,
             applicationId: String = applicationId,
             userPackageSelectionPreference: Seq[LfPackageId] = Seq.empty,
+            verboseHashing: Boolean = false,
         ): PrepareResponseProto =
           consoleEnvironment.run {
             ledgerApiCommand(
@@ -1933,6 +1933,7 @@ trait BaseLedgerApiAdministration extends NoTracing {
                 domainId,
                 applicationId,
                 userPackageSelectionPreference,
+                verboseHashing,
               )
             )
           }

diff --git a/...e/src/main/scala/com/digitalasset/canton/console/commands/ParticipantAdministration.scala b/...e/src/main/scala/com/digitalasset/canton/console/commands/ParticipantAdministration.scala
@@ -3,7 +3,6 @@
 
 package com.digitalasset.canton.console.commands
 
-import better.files.File
 import cats.syntax.either.*
 import cats.syntax.option.*
 import cats.syntax.traverse.*
@@ -73,7 +72,7 @@ import com.digitalasset.canton.console.{
 import com.digitalasset.canton.crypto.SyncCryptoApiProvider
 import com.digitalasset.canton.data.{CantonTimestamp, CantonTimestampSecond}
 import com.digitalasset.canton.discard.Implicits.DiscardOps
-import com.digitalasset.canton.grpc.{ByteStringStreamObserver, FileStreamObserver}
+import com.digitalasset.canton.grpc.ByteStringStreamObserver
 import com.digitalasset.canton.logging.{NamedLoggerFactory, NamedLogging, TracedLogger}
 import com.digitalasset.canton.participant.ParticipantNode
 import com.digitalasset.canton.participant.admin.ResourceLimits
@@ -83,7 +82,10 @@ import com.digitalasset.canton.participant.pruning.AcsCommitmentProcessor.{
   ReceivedCmtState,
   SentCmtState,
 }
-import com.digitalasset.canton.participant.pruning.{CommitmentContractMetadata, MismatchReason}
+import com.digitalasset.canton.participant.pruning.{
+  CommitmentContractMetadata,
+  CommitmentInspectContract,
+}
 import com.digitalasset.canton.protocol.messages.{
   AcsCommitment,
   CommitmentPeriod,
@@ -726,7 +728,7 @@ class LocalCommitmentsAdministrationGroup(
 }
 
 class CommitmentsAdministrationGroup(
-    runner: AdminCommandRunner,
+    runner: AdminCommandRunner with BaseInspection[ParticipantNode],
     val consoleEnvironment: ConsoleEnvironment,
     val loggerFactory: NamedLoggerFactory,
 ) extends FeatureFlagFilter
@@ -797,76 +799,68 @@ class CommitmentsAdministrationGroup(
       counterContractsMetadata
     }
 
-  // TODO(#9557) R2. We'll either reuse the existing export ACS mechanism or, if that doesn't work, we'll introduce a
-  //  new gRPC download endpoint, and filter by parties hosted on the counter-participant. The output will be contracts
-  //  for parties hosted by both the counter-participant and the local participant, as the local participant doesn't have
-  //  access to contracts of parties that it doesn't host.
   @Help.Summary(
-    "From a given set of contract ids and reassignment counters, identify the contracts that are either not active, or have" +
-      "a different reassignment counter, or are not shared with the given counter-participant.",
+    "Download states of contracts and contract payloads necessary for commitment inspection and reconciliation",
     FeatureFlag.Preview,
   )
   @Help.Description(
-    """ Returns the contract ids and the mismatch reason.
-      | Returns an error if the participant cannot anymore retrieve the data for the given contracts.
+    """ Returns the contract states (created, assigned, unassigned, archived, unknown) of the given contracts on
+      | all domains the participant knows from the beginning of time until the present time on each domain.
+      | The command returns best-effort the contract changes available. Specifically, it does not fail if the ACS
+      | and/or reassignment state has been pruned during the time interval, or if parts of the time interval
+      | are ahead of the clean ACS state.
+      | Optionally returns the contract payload if requested and available.
       | The arguments are:
-      | - counterParticipantContracts: The contract ids and reassignment counters that we check against our ACS
-      | - expectedDomain: The domain that the counterParticipant believes the given contracts reside on
-      | - timestamp: The timestamp when the given contracts are active on the counter-participant
-      | - counterParticipant: The counter participant with whom the contracts should be shared
+      | - contracts: The contract ids whose state and payload we want to fetch
+      | - timestamp: The timestamp when some counter-participants reported the given contracts as active on the
+      | expected domain.
+      | - expectedDomain: The domain that the contracts are expected to be active on
+      | - downloadPayload: If true, the payload of the contracts is also downloaded
       | - timeout: Time limit for the grpc call to complete
       """.stripMargin
   )
-  def active_contracts_mismatches(
-      counterParticipantContracts: Seq[CommitmentContractMetadata],
-      expectedDomain: DomainId,
+  def inspect_commitment_contracts(
+      contracts: Seq[LfContractId],
       timestamp: CantonTimestamp,
-      counterParticipant: ParticipantId,
+      expectedDomain: DomainId,
+      downloadPayload: Boolean = false,
       timeout: NonNegativeDuration = timeouts.unbounded,
-  ): Map[LfContractId, MismatchReason] =
-    check(FeatureFlag.Preview) {
-      Map.empty[LfContractId, MismatchReason]
-    }
+  ): Seq[CommitmentInspectContract] = {
 
-  @Help.Summary(
-    "Download the contract payloads from the counter participant necessary for reconciliation",
-    FeatureFlag.Preview,
-  )
-  @Help.Description(
-    """ Returns the contract ids and the mismatch reason.
-      | Returns an error if the participant cannot retrieve the data for the given commitment anymore.
-      | The arguments are:
-      | - contracts: The contract ids whose payload we want to download from the counterParticipant
-      | - timeout: Time limit for the grpc call to complete
-      | - binaryOutputFile: The file where to write the payload and mismatch information for the given mismatching
-      """.stripMargin
-  )
-  def download_contract_reconciliation_payloads(
-      contracts: Seq[LfContractId],
-      timeout: NonNegativeDuration = timeouts.unbounded,
-      binaryOutputFile: String = CommitmentsAdministrationGroup.ExportMismatchDefaultBinaryFile,
-  ): Unit = check(FeatureFlag.Preview) {
-    val file = File(binaryOutputFile)
-    consoleEnvironment.run {
+    val contractsData = consoleEnvironment.run {
       val responseObserver =
-        new FileStreamObserver[InspectCommitmentContracts.Response](file, _.chunk)
+        new ByteStringStreamObserver[InspectCommitmentContracts.Response](_.chunk)
 
       def call: ConsoleCommandResult[Context.CancellableContext] =
         adminCommand(
           ParticipantAdminCommands.Inspection.CommitmentContracts(
             responseObserver,
             contracts,
+            expectedDomain,
+            timestamp,
+            downloadPayload,
           )
         )
 
       processResult(
         call,
-        responseObserver.result,
+        responseObserver.resultBytes,
         timeout,
-        request = "Downloading contract from counter-participant that cause mismatch",
-        cleanupOnError = () => file.delete(),
+        "Retrieving the shared contract metadata",
       )
     }
+
+    val parsedContractsData =
+      GrpcStreamingUtils
+        .parseDelimitedFromTrusted[CommitmentInspectContract](
+          contractsData.newInput(),
+          CommitmentInspectContract,
+        )
+        .valueOr(msg => throw DeserializationException(msg))
+    logger.debug(
+      s"Requested data for ${contracts.size}, retrieved data for ${parsedContractsData.size} contracts at time $timestamp on any domain"
+    )
+    parsedContractsData
   }
 
   @Help.Summary(
@@ -1355,10 +1349,6 @@ class CommitmentsAdministrationGroup(
   private implicit val ec: ExecutionContext = consoleEnvironment.environment.executionContext
 }
 
-object CommitmentsAdministrationGroup {
-  private val ExportMismatchDefaultBinaryFile = "canton-acs-mismatch-export.gz"
-}
-
 class ParticipantReplicationAdministrationGroup(
     runner: AdminCommandRunner,
     consoleEnvironment: ConsoleEnvironment,