From c5712225078d4a757eef7e25ffbf26fdff2d56c5 Mon Sep 17 00:00:00 2001 From: Wes Biggs Date: Mon, 1 Jul 2024 14:38:47 -0500 Subject: [PATCH 01/11] DIP-273 Content Addressing * Update the definition of DSNP Content Hash to include DSNP CIDs * Change ProfileResource to use bytes, not string, for cid field, in line with Announcement usage * Update Activity Content Hash extension to include CID option * Update various example hashes to use CIDv1 * Update to pre-1.3.0 versioning and sync prerelease changelogs --- .spellcheckerdict.txt | 4 +++- pages/ActivityContent/Associated/Attachments.md | 6 ++++-- pages/ActivityContent/Associated/Hash.md | 15 ++++----------- pages/ActivityContent/Overview.md | 7 ++++--- pages/ActivityContent/Types/Profile.md | 12 ++++++------ pages/DSNP/Identifiers.md | 9 +++++---- pages/DSNP/Overview.md | 9 ++++++--- pages/DSNP/Types/ProfileResource.md | 4 ++-- 8 files changed, 34 insertions(+), 32 deletions(-) diff --git a/.spellcheckerdict.txt b/.spellcheckerdict.txt index ec34aeab..98f5c699 100644 --- a/.spellcheckerdict.txt +++ b/.spellcheckerdict.txt @@ -9,6 +9,7 @@ CC0 Changelog changeType (cid|CID)s? +CIDv1 [Cc]odec('s)? contentHash cryptographic @@ -19,7 +20,7 @@ CtxSharedSecretBob Curve25519 decrypt(ed)? Delegator -Deserialize +[Dd]eserialize(d)? Diffie-Hellman discoverability [Dd]iscoverable @@ -71,6 +72,7 @@ PNG Polkadot Poly1305 pre-configured +Prerelease PRId([ABs])? ProfileResource [Pp]ublicKey diff --git a/pages/ActivityContent/Associated/Attachments.md b/pages/ActivityContent/Associated/Attachments.md index 5f2d76aa..ccd8d72e 100644 --- a/pages/ActivityContent/Associated/Attachments.md +++ b/pages/ActivityContent/Associated/Attachments.md @@ -44,7 +44,8 @@ "href": "https://upload.wikimedia.org/wikipedia/commons/d/d9/Wilhelm_Scream.ogg", "mediaType": "audio/ogg", "hash": [ - "QmQrGdv6Ky5sJhaVdw27y4aod5pdfihDkBTxiBkRaSGJJ7" + "bafkreibwacl4s4k7kwv4yjvlopys62nxgddgb67qdh6ghsvgqw72tnsylm", + "QmQrGdv6Ky5sJhaVdw27y4aod5pdfihDkBTxiBkRaSGJJ7" ] } ], @@ -106,7 +107,7 @@ "height": 228, "mediaType": "image/jpg", "hash": [ - "2Drjgb5yoVWTpubcWmDBLJqkxrFkZamekzJoYLSWwM2ezpFkab" + "bafkreieuibebwm2ih2yv6rhbknv6m5kdu5nlyi2dz5r6vl7yee2sqwuvrq" ] } ] @@ -169,6 +170,7 @@ "height": 2250, "mediaType": "video/webm", "hash": [ + "bafybeiberftlan32pi5ef45kb363kc3ouleui636fmbhvjdi4fgwqif5ou", "2Drjgb4a8eC4XheBKCBcbAcaVdEWcKjMbCSZ2L2c9CQs4x98jf" ] } diff --git a/pages/ActivityContent/Associated/Hash.md b/pages/ActivityContent/Associated/Hash.md index 69c6886f..39f21b5c 100644 --- a/pages/ActivityContent/Associated/Hash.md +++ b/pages/ActivityContent/Associated/Hash.md @@ -4,24 +4,17 @@ Activity objects linking to external content such as audio, image or video files must include a `"hash"` field for users to validate linked content. The value of this `"hash"` field must be an array of strings, each representing a hash output using a specific algorithm. -AT LEAST ONE hash in the array MUST be one of the [supported algorithms](#supported-algorithms), although others may also be used. - -Hashes MUST be encoded using the [multihash](https://github.com/multiformats/multihash) specification, and serialized as a [multibase](https://github.com/multiformats/multibase) string. - -### Supported Algorithms - -| Algorithm | Multihash Name | Leading bytes (as [varint](https://github.com/multiformats/unsigned-varint)) | Reference | DSNP Version Added | -| --- | --- | --- | --- | --- | -| SHA-256 | `sha2-256` | `0x1220` | [RFC 6234](https://tools.ietf.org/html/rfc6234) | 1.2.0 | -| BLAKE2b | `blake2b-256` | `0xa0e40220` | [RFC 7693](https://tools.ietf.org/html/rfc7693) | 1.2.0 | +Each item in the array MUST be a valid [multihash](https://github.com/multiformats/multihash) value, serialized as a [multibase](https://github.com/multiformats/multibase) string using either the `base32` or `base58btc` encodings. + At least one of the deserialized multihash values must be a valid [DSNP Content Hash](../../DSNP/Identifiers.md#dsnp-content-hash). ### Example -This example gives SHA-256 and BLAKE2b hashes for the [PDF version of the DSNP whitepaper](https://github.com/LibertyDSNP/papers/raw/main/whitepaper/dsnp_whitepaper.pdf). +This example gives CIDv1 (serialized as `base32`), SHA-256 (serialized as `base58btc`), and BLAKE2b (serialized as `base58btc`) hashes for the [PDF version of the DSNP whitepaper](https://github.com/LibertyDSNP/papers/raw/main/whitepaper/dsnp_whitepaper.pdf). ```json { "hash": [ + "bafybeida7z24mig7j3oagjru7s2gw6xbfkh7fryvah6ho2ar77xb7aleom", "QmQNHNfHnbgJJ6nK4UPx2VtTUCafAKCbqZJ6ZRYUGjoeFj", "2DrjgbGgSsXRhTiBWckoVwBFC6H4qiBWWNumSsRwdUt82YnTdN" ] diff --git a/pages/ActivityContent/Overview.md b/pages/ActivityContent/Overview.md index 0b9d73f2..564f3472 100644 --- a/pages/ActivityContent/Overview.md +++ b/pages/ActivityContent/Overview.md @@ -1,5 +1,5 @@ # Activity Content Specification -__Version 1.2.0__ +__Version pre-1.3.0__ Content references shared via the DSNP consist of URLs pointing to documents containing Activity Streams JSON objects. For the purposes of the DSNP, restrictions are placed on the [Activity Streams 2.0](https://www.w3.org/TR/activitystreams-core/) specification. @@ -45,9 +45,10 @@ URLs in DSNP-compatible Activity Content MUST use one of the following URL schem | [LibertyDSNP/activity-content-java](https://github.com/LibertyDSNP/activity-content-java) | Java/Kotlin | | [LibertyDSNP/activity-content-swift](https://github.com/LibertyDSNP/activity-content-swift) | Swift | - ## Prerelease Changelog ----> + +- DIP-273 Content Addressing ## Releases diff --git a/pages/ActivityContent/Types/Profile.md b/pages/ActivityContent/Types/Profile.md index ad2c3dc8..3cd5d55f 100644 --- a/pages/ActivityContent/Types/Profile.md +++ b/pages/ActivityContent/Types/Profile.md @@ -27,22 +27,22 @@ Profiles are used to provide additional user information display. "icon": [ { "type": "Link", - "href": "https://placekitten.com/256/256", + "href": "https://upload.wikimedia.org/wikipedia/commons/thumb/1/1d/1-month-old_kittens_32.jpg/256px-1-month-old_kittens_32.jpg", "mediaType": "image/jpeg", "width": "256", - "height": "256", + "height": "171", "hash": [ - "QmVmUqGYtHcVgpTFR64bHNcLGGFEeWxmUP6pV2C2RbWpKT" + "bafkreiddyiraqlkgoybzrydxqs5vikejphecjoizfk5rlghd7tohxahbfa" ] }, { "type": "Link", - "href": "https://placekitten.com/64/64", + "href": "https://upload.wikimedia.org/wikipedia/commons/thumb/1/1d/1-month-old_kittens_32.jpg/64px-1-month-old_kittens_32.jpg", "mediaType": "image/jpeg", "width": "64", - "height": "64", + "height": "43", "hash": [ - "QmcAh1rov5GcddekCffGeRnaSyiji6ATmfGWpxXYJHgJZx" + "bafkreid7m3fxtdxokdb4vmvpjuu7ngo72dpjsgpttfh7bfh2gdhq6ydbjm" ] } ], diff --git a/pages/DSNP/Identifiers.md b/pages/DSNP/Identifiers.md index 2020e237..9f225b3a 100644 --- a/pages/DSNP/Identifiers.md +++ b/pages/DSNP/Identifiers.md @@ -12,7 +12,7 @@ Graph connections are formed through the DSNP User Id. ## DSNP Content Hash - Variable length byte array (fixed length for a given hashing algorithm) -- MUST be a valid [multihash](https://github.com/multiformats/multihash) encoding of the hash output for the bytes of the content, generated with a [Supported Hashing Algorithm](Announcements.md#supported-hashing-algorithms) +- MUST be a valid [multihash](https://github.com/multiformats/multihash) encoding of the hash or content address output for the bytes of the content, generated with a [Supported Hashing Algorithm](Announcements.md#supported-hashing-algorithms) ### Supported Hashing Algorithms @@ -20,6 +20,7 @@ Graph connections are formed through the DSNP User Id. | --- | --- | --- | --- | --- | | SHA-256 | `sha2-256` | `0x1220` | [RFC 6234](https://tools.ietf.org/html/rfc6234) | 1.2.0 | | BLAKE2b | `blake2b-256` | `0xa0e40220` | [RFC 7693](https://tools.ietf.org/html/rfc7693) | 1.2.0 | +| CIDv1 | `cidv1` | `0x01` | See [DSNP CID](#dsnp-cid) | 1.3.0 | ## DSNP Protocol Scheme @@ -60,7 +61,7 @@ dsnp://78187493520/QmQNHNfHnbgJJ6nK4UPx2VtTUCafAKCbqZJ6ZRYUGjoeFj ## DSNP CID -A DSNP CID is a valid [Content IDentifier](https://github.com/multiformats/cid) generated using the following parameters. +A DSNP CID is a valid binary [Content IDentifier](https://github.com/multiformats/cid) generated using the following parameters. ### Supported CID Parameters @@ -69,9 +70,9 @@ In order for DSNP applications to interoperate, the required functionality is li - CID version: MUST be version 1, in order to distinguish CIDs from simple multihash values in situations where either may be used - Hash algorithm: MUST be `sha2-256` or `blake2b-256` -- Encoding: MUST be `base58btc` or `base32` - Codec: MUST be `dag-pb` for data 256*1024 bytes or longer; `raw` for data less than 256*1024 bytes - Chunking: Non-leaf nodes MUST be 256*1024 bytes The rationale for these options is to allow consuming applications to attempt to generate a matching CID from a byte stream for validation purposes without the need to reprocess the stream. -These options are intentionally aligned to interoperate with the default output of the [Kubo](https://github.com/ipfs/kubo) IPFS command line utility when invoked as `ipfs add --cid-version=1 ...`. + +These options are intentionally aligned to interoperate with the default output of the [Kubo](https://github.com/ipfs/kubo) IPFS command line utility when invoked as `ipfs add --cid-version=1 ...` (note that the output will additionally be serialized using a multibase prefix). diff --git a/pages/DSNP/Overview.md b/pages/DSNP/Overview.md index 8b3ee813..9c72a340 100644 --- a/pages/DSNP/Overview.md +++ b/pages/DSNP/Overview.md @@ -1,5 +1,5 @@ # DSNP Specification -__Version 1.2.0__ +__Version pre-1.3.0__ DSNP (Decentralized Social Networking Protocol) is a social networking protocol designed to run on a blockchain. It specifies a set of social primitives along with requirements for interoperability. @@ -29,9 +29,12 @@ Compliant DSNP system specifications MUST document how each of the required DSNP A compliant specification MUST specify a mapping from its system-specific state change data (for example, the events emitted by a blockchain) to the DSNP State Change Records that data represents. - ## Prerelease Changelog ----> + +- DIP-263 User Data for Public Keys +- DIP-267 User Data for Profile Resources +- DIP-273 Content Addressing ## Releases diff --git a/pages/DSNP/Types/ProfileResource.md b/pages/DSNP/Types/ProfileResource.md index b8419653..df384fa5 100644 --- a/pages/DSNP/Types/ProfileResource.md +++ b/pages/DSNP/Types/ProfileResource.md @@ -20,7 +20,7 @@ ProfileResource object serialization MUST conform to the following [Avro](https: }, { "name": "cid", - "type": "string", + "type": "bytes", "doc": "Content IDentifier of resource" }, { @@ -48,7 +48,7 @@ All other document type values are reserved for future expansion. ### cid -- MUST be a valid [DSNP CID](../Identifiers.md#dsnp-cid] +- MUST be a valid (binary) [DSNP CID](../Identifiers.md#dsnp-cid) ### length From 7da7a6d35cbc7dcaa1da77b10c1570d6d1187887 Mon Sep 17 00:00:00 2001 From: Wes Biggs Date: Wed, 10 Jul 2024 22:17:47 -0500 Subject: [PATCH 02/11] Revert to multihash only, but with blake3 in place of blake2b-256 as an option --- .../ActivityContent/Associated/Attachments.md | 11 ++++---- pages/ActivityContent/Associated/Hash.md | 9 +++---- pages/ActivityContent/Types/Profile.md | 4 +-- pages/DSNP/Identifiers.md | 25 +++---------------- pages/DSNP/Types/ProfileResource.md | 12 ++++----- 5 files changed, 20 insertions(+), 41 deletions(-) diff --git a/pages/ActivityContent/Associated/Attachments.md b/pages/ActivityContent/Associated/Attachments.md index ccd8d72e..39118ca9 100644 --- a/pages/ActivityContent/Associated/Attachments.md +++ b/pages/ActivityContent/Associated/Attachments.md @@ -44,8 +44,7 @@ "href": "https://upload.wikimedia.org/wikipedia/commons/d/d9/Wilhelm_Scream.ogg", "mediaType": "audio/ogg", "hash": [ - "bafkreibwacl4s4k7kwv4yjvlopys62nxgddgb67qdh6ghsvgqw72tnsylm", - "QmQrGdv6Ky5sJhaVdw27y4aod5pdfihDkBTxiBkRaSGJJ7" + "bdyqbcji3okmzxobvaqgduz5prixmumyndzopyufultmslndi4pdebii" ] } ], @@ -74,7 +73,7 @@ | `type` | [Activity Vocabulary 2.0](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-type) | YES | Identifies the type of the object | MUST be set to `Link` | | `href` | [Activity Vocabulary 2.0](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-href) | YES | The URL for the given image | MUST be a [Supported URL Schema](../Overview.md#supported-url-schema) | | `mediaType` | [Activity Vocabulary 2.0](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-mediatype) | YES | MIME type of `href` content | | -| `hash` | [DSNP 1.0](Hash.md) | YES | Array of hashes for linked content validation | MUST include at least one [supported hash](Hash.md#supported-algorithms) | +| `hash` | [DSNP 1.0](Hash.md) | YES | Array of hashes for linked content validation | MUST include at least one [supported hash](../../DSNP/Identifiers.md#supported-hashing-algorithms) | | `height` | [Activity Vocabulary 2.0](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-height) | no | A hint as to the rendering height in device-independent pixels | | | `width` | [Activity Vocabulary 2.0](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-width) | no | A hint as to the rendering width in device-independent pixels | | @@ -107,7 +106,8 @@ "height": 228, "mediaType": "image/jpg", "hash": [ - "bafkreieuibebwm2ih2yv6rhbknv6m5kdu5nlyi2dz5r6vl7yee2sqwuvrq" + "bciqjiqcidmzuqpvrl5cocu3l4z2uhj22xqruht3d5kx7qijvfbnjlda", + "bdyqbjzt5drgji5w7xhsddsynusgx2vdmakcsrr4sfin5fyfkwlpup6q" ] } ] @@ -170,8 +170,7 @@ "height": 2250, "mediaType": "video/webm", "hash": [ - "bafybeiberftlan32pi5ef45kb363kc3ouleui636fmbhvjdi4fgwqif5ou", - "2Drjgb4a8eC4XheBKCBcbAcaVdEWcKjMbCSZ2L2c9CQs4x98jf" + "bdyqed7dnok3batd7tr64trqmovfxam5tqsgxkiv2op5765pq43swtui" ] } ] diff --git a/pages/ActivityContent/Associated/Hash.md b/pages/ActivityContent/Associated/Hash.md index 39f21b5c..97c32890 100644 --- a/pages/ActivityContent/Associated/Hash.md +++ b/pages/ActivityContent/Associated/Hash.md @@ -4,19 +4,18 @@ Activity objects linking to external content such as audio, image or video files must include a `"hash"` field for users to validate linked content. The value of this `"hash"` field must be an array of strings, each representing a hash output using a specific algorithm. -Each item in the array MUST be a valid [multihash](https://github.com/multiformats/multihash) value, serialized as a [multibase](https://github.com/multiformats/multibase) string using either the `base32` or `base58btc` encodings. +Each item in the array MUST be a valid [multihash](https://github.com/multiformats/multihash) value, serialized as a [multibase](https://github.com/multiformats/multibase) string using the `base32` encoding. At least one of the deserialized multihash values must be a valid [DSNP Content Hash](../../DSNP/Identifiers.md#dsnp-content-hash). ### Example -This example gives CIDv1 (serialized as `base32`), SHA-256 (serialized as `base58btc`), and BLAKE2b (serialized as `base58btc`) hashes for the [PDF version of the DSNP whitepaper](https://github.com/LibertyDSNP/papers/raw/main/whitepaper/dsnp_whitepaper.pdf). +This example gives SHA-256 and BLAKE3 hashes for the [PDF version of the DSNP whitepaper](https://github.com/LibertyDSNP/papers/raw/main/whitepaper/dsnp_whitepaper.pdf). ```json { "hash": [ - "bafybeida7z24mig7j3oagjru7s2gw6xbfkh7fryvah6ho2ar77xb7aleom", - "QmQNHNfHnbgJJ6nK4UPx2VtTUCafAKCbqZJ6ZRYUGjoeFj", - "2DrjgbGgSsXRhTiBWckoVwBFC6H4qiBWWNumSsRwdUt82YnTdN" + "bciqgb7tvyyqn6tw4amtdj7funn5ockup6ldrkap4o5ubd77od6awi4y", + "bdyqdua4t4pxgy37mdmjyqv3dejp5betyqsznimpneyujsur23yubzna" ] } ``` diff --git a/pages/ActivityContent/Types/Profile.md b/pages/ActivityContent/Types/Profile.md index 3cd5d55f..e9a5225f 100644 --- a/pages/ActivityContent/Types/Profile.md +++ b/pages/ActivityContent/Types/Profile.md @@ -32,7 +32,7 @@ Profiles are used to provide additional user information display. "width": "256", "height": "171", "hash": [ - "bafkreiddyiraqlkgoybzrydxqs5vikejphecjoizfk5rlghd7tohxahbfa" + "bdyqphnphmjdoumkxqbsuspribxvlsx2hx6525u3fh2dkr5bxnqritzi" ] }, { @@ -42,7 +42,7 @@ Profiles are used to provide additional user information display. "width": "64", "height": "43", "hash": [ - "bafkreid7m3fxtdxokdb4vmvpjuu7ngo72dpjsgpttfh7bfh2gdhq6ydbjm" + "bdyqktr7p5hc27bx4ernmngs6tj7uyukfb4atrtq44mdmx4yntuh2s5y" ] } ], diff --git a/pages/DSNP/Identifiers.md b/pages/DSNP/Identifiers.md index 9f225b3a..6fb76c93 100644 --- a/pages/DSNP/Identifiers.md +++ b/pages/DSNP/Identifiers.md @@ -16,11 +16,10 @@ Graph connections are formed through the DSNP User Id. ### Supported Hashing Algorithms -| Algorithm | Multihash Name | Leading bytes (as [varint](https://github.com/multiformats/unsigned-varint)) | Reference | DSNP Version Added | +| Algorithm | Multihash Name | Leading bytes | Reference | DSNP Version Added | | --- | --- | --- | --- | --- | -| SHA-256 | `sha2-256` | `0x1220` | [RFC 6234](https://tools.ietf.org/html/rfc6234) | 1.2.0 | -| BLAKE2b | `blake2b-256` | `0xa0e40220` | [RFC 7693](https://tools.ietf.org/html/rfc7693) | 1.2.0 | -| CIDv1 | `cidv1` | `0x01` | See [DSNP CID](#dsnp-cid) | 1.3.0 | +| SHA-256 | `sha2-256` | `0x1220` | [RFC 6234](https://tools.ietf.org/html/rfc6234) | 1.2 | +| BLAKE3 | `blake3` | `0x1e20` | [blake3.io](https://blake3.io) | 1.3 | ## DSNP Protocol Scheme @@ -58,21 +57,3 @@ dsnp://78187493520/QmQNHNfHnbgJJ6nK4UPx2VtTUCafAKCbqZJ6ZRYUGjoeFj | Scheme | `dsnp://` | | User Id | `78187493520` | | Content Hash | `QmQNHNfHnbgJJ6nK4UPx2VtTUCafAKCbqZJ6ZRYUGjoeFj` | - -## DSNP CID - -A DSNP CID is a valid binary [Content IDentifier](https://github.com/multiformats/cid) generated using the following parameters. - -### Supported CID Parameters - -The CID specification allows CIDs to be generated with a wide and ever-growing range of possible hashing algorithms, string encodings, and block sizes. -In order for DSNP applications to interoperate, the required functionality is limited as follows: - -- CID version: MUST be version 1, in order to distinguish CIDs from simple multihash values in situations where either may be used -- Hash algorithm: MUST be `sha2-256` or `blake2b-256` -- Codec: MUST be `dag-pb` for data 256*1024 bytes or longer; `raw` for data less than 256*1024 bytes -- Chunking: Non-leaf nodes MUST be 256*1024 bytes - -The rationale for these options is to allow consuming applications to attempt to generate a matching CID from a byte stream for validation purposes without the need to reprocess the stream. - -These options are intentionally aligned to interoperate with the default output of the [Kubo](https://github.com/ipfs/kubo) IPFS command line utility when invoked as `ipfs add --cid-version=1 ...` (note that the output will additionally be serialized using a multibase prefix). diff --git a/pages/DSNP/Types/ProfileResource.md b/pages/DSNP/Types/ProfileResource.md index df384fa5..eca6169d 100644 --- a/pages/DSNP/Types/ProfileResource.md +++ b/pages/DSNP/Types/ProfileResource.md @@ -19,14 +19,14 @@ ProfileResource object serialization MUST conform to the following [Avro](https: "doc": "Type of resource" }, { - "name": "cid", + "name": "contentHash", "type": "bytes", - "doc": "Content IDentifier of resource" + "doc": "Multihash digest of resource content" }, { "name": "length", "type": "int", - "doc": "Length of document in bytes" + "doc": "Length of resource in bytes" } ] } @@ -46,11 +46,11 @@ ProfileResource object serialization MUST conform to the following [Avro](https: All other document type values are reserved for future expansion. -### cid +### contentHash -- MUST be a valid (binary) [DSNP CID](../Identifiers.md#dsnp-cid) +- MUST be a valid [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) ### length - MUST be a positive integer -- MUST match the length in bytes of the document +- MUST match the length in bytes of the resource From 1329f7a84df6aaf1ef485bc05bc608ea7452235e Mon Sep 17 00:00:00 2001 From: Wes Biggs Date: Fri, 12 Jul 2024 22:22:22 -0500 Subject: [PATCH 03/11] Improve description of usage of content hash vs URL --- .spellcheckerdict.txt | 3 +-- pages/DSNP/Announcements.md | 8 ++++++-- pages/DSNP/Types/ProfileResource.md | 2 ++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.spellcheckerdict.txt b/.spellcheckerdict.txt index 98f5c699..ed46a146 100644 --- a/.spellcheckerdict.txt +++ b/.spellcheckerdict.txt @@ -2,14 +2,13 @@ Alexa announcementType Avro Base58 -BLAKE2b +BLAKE3 Brötli CalVer CC0 Changelog changeType (cid|CID)s? -CIDv1 [Cc]odec('s)? contentHash cryptographic diff --git a/pages/DSNP/Announcements.md b/pages/DSNP/Announcements.md index 8ba5fb8c..a45157d5 100644 --- a/pages/DSNP/Announcements.md +++ b/pages/DSNP/Announcements.md @@ -34,8 +34,12 @@ Implementations MUST provide a way to validate that the [identifier](Identifiers ## External Content URLs and Hashes Where Announcements refer to external documents (such as Activity Content documents), these are referenced by both a URL and a [DSNP Content Hash](Identifiers.md#dsnp-content-hash). -The content hash MUST be generated by applying a [Supported Hashing Algorithm](Identifiers.md#supported-hashing-algorithms) to the full, unaltered contents retrieved from the URL. -When readers retrieve content referenced in an Announcement, they can validate the authenticity of the content by regenerating the hash output and comparing it with the content hash recorded in the Announcement. +The content hash MUST be generated by applying a [Supported Hashing Algorithm](Identifiers.md#supported-hashing-algorithms) to the full, unaltered contents of the document. + +The URL associated with a content hash should be construed as a hint to initially locate a document matching the content hash, but is in no way meant to be the only way to locate the indicated document. +Over time, a URL may cease to reference the specified document, or might have its contents altered; therefore, the content hash should be considered the authoritative value and the URL only one of many possible ways of locating a document. +For example, services may cache documents or retrieve them from a content-addressed file system by applying the content hash (or a value derived from the content hash, such as a CID). +When readers retrieve content referenced in an Announcement, they can validate the authenticity of the content, regardless of where it is hosted, by regenerating the hash output and comparing it with the content hash recorded in the Announcement. ## Duplicate Handling diff --git a/pages/DSNP/Types/ProfileResource.md b/pages/DSNP/Types/ProfileResource.md index eca6169d..ff8c10fa 100644 --- a/pages/DSNP/Types/ProfileResource.md +++ b/pages/DSNP/Types/ProfileResource.md @@ -50,6 +50,8 @@ All other document type values are reserved for future expansion. - MUST be a valid [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) +A compliant DSNP system MUST specify how to use the `contentHash` field to retrieve the target document. + ### length - MUST be a positive integer From 09b606b620232a582bdaec66c7a7db8649d2c7e5 Mon Sep 17 00:00:00 2001 From: Wes Biggs Date: Fri, 2 Aug 2024 13:02:51 -0500 Subject: [PATCH 04/11] Update ProfileResource --- .spellcheckerdict.txt | 1 + pages/DSNP/Types/ProfileResource.md | 31 ++++++++++------------------- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/.spellcheckerdict.txt b/.spellcheckerdict.txt index ed46a146..14fe2750 100644 --- a/.spellcheckerdict.txt +++ b/.spellcheckerdict.txt @@ -10,6 +10,7 @@ Changelog changeType (cid|CID)s? [Cc]odec('s)? +contentAddress contentHash cryptographic cryptographically diff --git a/pages/DSNP/Types/ProfileResource.md b/pages/DSNP/Types/ProfileResource.md index ff8c10fa..c3732350 100644 --- a/pages/DSNP/Types/ProfileResource.md +++ b/pages/DSNP/Types/ProfileResource.md @@ -19,14 +19,9 @@ ProfileResource object serialization MUST conform to the following [Avro](https: "doc": "Type of resource" }, { - "name": "contentHash", - "type": "bytes", - "doc": "Multihash digest of resource content" - }, - { - "name": "length", - "type": "int", - "doc": "Length of resource in bytes" + "name": "contentAddress", + "type": "string", + "doc": "Content address for the resource" } ] } @@ -40,19 +35,15 @@ ProfileResource object serialization MUST conform to the following [Avro](https: #### Supported Profile-Linked Resource Types -| Value | Description | Specification | Content Type | DSNP Version Added | -| --- | --- | --- | --- | -| 1 | Activity Content Profile with DSNP extensions | [DSNP Profile](../../ActivityContent/Types/Profile.md) | `application/json` | 1.3 | - -All other document type values are reserved for future expansion. - -### contentHash +| Value | Description | Specification | Content Type | Maximum File Size | Content Address Type | DSNP Version Added | +| --- | --- | --- | --- | --- | --- | --- | +| 1 | Activity Content Profile with DSNP extensions | [DSNP Profile](../../ActivityContent/Types/Profile.md) | `application/json` | 256 Kb | Supported [IPFS CID](https://docs.ipfs.tech/concepts/content-addressing/)1 | 1.3 | -- MUST be a valid [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) +All other resource type values are reserved for future expansion. -A compliant DSNP system MUST specify how to use the `contentHash` field to retrieve the target document. +1Supported IPFS CIDs must be CID version 1, using either `sha2-256` or `blake3` hashes with the `raw` codec and the `base32` serialization. -### length +### contentAddress -- MUST be a positive integer -- MUST match the length in bytes of the resource +- MUST be a valid content address for the specified `type` (see table above) +- MUST contain sufficient information for an application to perform content integrity validation, for example by comparing the `contentAddress` field to the address derived by recalculating the content address from the bytes of the retrieved resource From 47aad295faba3d9ca53d715a2186cb8b994ed3e1 Mon Sep 17 00:00:00 2001 From: Wes Biggs Date: Fri, 2 Aug 2024 13:19:30 -0500 Subject: [PATCH 05/11] Fix content URI definition; edit versioning on start page --- pages/DSNP/Identifiers.md | 5 +++-- pages/index.md | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pages/DSNP/Identifiers.md b/pages/DSNP/Identifiers.md index 6fb76c93..8b817694 100644 --- a/pages/DSNP/Identifiers.md +++ b/pages/DSNP/Identifiers.md @@ -46,14 +46,15 @@ The DSNP Content URI consists of three parts: the scheme, the user id, and the c It is used to uniquely identify an Announcement from a given user with content. Any [Announcement Types](Announcements.md#announcement-types) with a `fromId` and `contentHash` have a DSNP Content URI. +When encoding a DSNP Content URI, the `contentHash` field MUST be serialized as [hexadecimal](./Serializations.md#hexadecimal). ### Example ``` -dsnp://78187493520/QmQNHNfHnbgJJ6nK4UPx2VtTUCafAKCbqZJ6ZRYUGjoeFj +dsnp://78187493520/0x1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef ``` | part | value | | ---- | ----- | | Scheme | `dsnp://` | | User Id | `78187493520` | -| Content Hash | `QmQNHNfHnbgJJ6nK4UPx2VtTUCafAKCbqZJ6ZRYUGjoeFj` | +| Content Hash | `0x1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef` | diff --git a/pages/index.md b/pages/index.md index f12377fd..16fac045 100644 --- a/pages/index.md +++ b/pages/index.md @@ -15,8 +15,8 @@ There are three core specifications currently. | Name | Version | Description | | --- | --- | --- | -| [DSNP](DSNP/Overview.md) | 1.2.0 | The system-agnostic DSNP specification | -| [Activity Content](ActivityContent/Overview.md) | 1.2.0 | A specification for DSNP-referenced content (subset of W3C Activity Streams) | +| [DSNP](DSNP/Overview.md) | pre-1.3.0 | The system-agnostic DSNP specification | +| [Activity Content](ActivityContent/Overview.md) | pre-1.3.0 | A specification for DSNP-referenced content (subset of W3C Activity Streams) | | [DSNP Over Frequency](Frequency/Overview.md) | - | DSNP Over Frequency specification | Each specification is divided into several modules. From 51c0218ef809d119b688c2225617820520c7035f Mon Sep 17 00:00:00 2001 From: Wes Biggs Date: Wed, 7 Aug 2024 14:42:09 -0500 Subject: [PATCH 06/11] "DSNP Content Hash" now refers to the base32-encoded multihash of supported types only. Announcements have been updated to use strings for contentHash and targetContentHash fields. --- pages/ActivityContent/Associated/Hash.md | 5 ++--- pages/DSNP/BatchPublications.md | 14 ++++++------- pages/DSNP/Identifiers.md | 25 +++++++++++++++++++----- pages/DSNP/Types/Broadcast.md | 4 ++-- pages/DSNP/Types/Reply.md | 4 ++-- pages/DSNP/Types/Tombstone.md | 4 ++-- pages/DSNP/Types/Update.md | 8 ++++---- 7 files changed, 39 insertions(+), 25 deletions(-) diff --git a/pages/ActivityContent/Associated/Hash.md b/pages/ActivityContent/Associated/Hash.md index 97c32890..0d06ec82 100644 --- a/pages/ActivityContent/Associated/Hash.md +++ b/pages/ActivityContent/Associated/Hash.md @@ -3,9 +3,8 @@ *NOT* part of the Activity Streams 2.0 Vocabulary. Activity objects linking to external content such as audio, image or video files must include a `"hash"` field for users to validate linked content. -The value of this `"hash"` field must be an array of strings, each representing a hash output using a specific algorithm. -Each item in the array MUST be a valid [multihash](https://github.com/multiformats/multihash) value, serialized as a [multibase](https://github.com/multiformats/multibase) string using the `base32` encoding. - At least one of the deserialized multihash values must be a valid [DSNP Content Hash](../../DSNP/Identifiers.md#dsnp-content-hash). +The value of this `"hash"` field must be an array of strings. +Each item in the array MUST be a valid [DSNP Content Hash](../../DSNP/Identifiers.md#dsnp-content-hash) for the content associated with the hash. ### Example diff --git a/pages/DSNP/BatchPublications.md b/pages/DSNP/BatchPublications.md index 3a3608ef..0d6c8d77 100644 --- a/pages/DSNP/BatchPublications.md +++ b/pages/DSNP/BatchPublications.md @@ -43,13 +43,13 @@ See also [Announcement Types](Announcements.md). #### Columns with Bloom Filters -| Column | Parquet Type | -| ------ | ---- | -| contentHash | `BYTE_ARRAY` | -| emoji | `BYTE_ARRAY` | -| fromId | `BYTE_ARRAY` | -| inReplyTo | `BYTE_ARRAY` | -| objectId | `BYTE_ARRAY` | +| Column | Primitive Type | Logical Type | Converted Type (deprecated) | +| ------ | ---- | ---- | +| contentHash | `BYTE_ARRAY` | `STRING` | `UTF8` | +| emoji | `BYTE_ARRAY` | `STRING` | `UTF8` | +| fromId | `INT64` | `INT(64, false)` | `UINT_64` | +| inReplyTo | `BYTE_ARRAY` | `STRING` | `UTF8` | +| targetContentHash | `BYTE_ARRAY` | `STRING` | `UTF8` | ## Non-Normative diff --git a/pages/DSNP/Identifiers.md b/pages/DSNP/Identifiers.md index 8b817694..3a69b2e5 100644 --- a/pages/DSNP/Identifiers.md +++ b/pages/DSNP/Identifiers.md @@ -11,8 +11,23 @@ Graph connections are formed through the DSNP User Id. ## DSNP Content Hash -- Variable length byte array (fixed length for a given hashing algorithm) -- MUST be a valid [multihash](https://github.com/multiformats/multihash) encoding of the hash or content address output for the bytes of the content, generated with a [Supported Hashing Algorithm](Announcements.md#supported-hashing-algorithms) +- MUST be a multibase string using the `base32` encoding +- MUST represent a valid [multihash](https://github.com/multiformats/multihash) encoding of the hashing algorithm output for the bytes of the content +- MUST use a [Supported Hashing Algorithm](Announcements.md#supported-hashing-algorithms) + +### Serialization Steps + +1. Apply the Supported Hashing Algorithm to create a digest of the content. +2. Prepend the leading bytes from the table below indicating the hashing algorithm in the multicodec table and the length of the hash output. +3. Encode to UTF-8 using the `base32` alphabet. +4. Prepend the `'b'` character indicating `base32` encoding. + +### Example + +1. Applying the BLAKE3 algorithm to the [DSNP Whitepaper](https://dsnp.org/dsnp_whitepaper.pdf) yields the following 32 bytes: `0x3a0393e3ee6c6fec1b13885763225fd0927884b2d431ed262899523ade281cb4`. +2. Prepending the multihash indicator (`0x1e` for `blake3`) and hash length (`0x20` for 32 bytes) gives `0x1e203a0393e3ee6c6fec1b13885763225fd0927884b2d431ed262899523ade281cb4` +3. Encoding this to the `base32` alphabet outputs the string `dyqdua4t4pxgy37mdmjyqv3dejp5betyqsznimpneyujsur23yubzna`. +4. Prepending the `b` character to indicate `base32` gives us the final DSNP Content Hash of `bdyqdua4t4pxgy37mdmjyqv3dejp5betyqsznimpneyujsur23yubzna`. ### Supported Hashing Algorithms @@ -46,15 +61,15 @@ The DSNP Content URI consists of three parts: the scheme, the user id, and the c It is used to uniquely identify an Announcement from a given user with content. Any [Announcement Types](Announcements.md#announcement-types) with a `fromId` and `contentHash` have a DSNP Content URI. -When encoding a DSNP Content URI, the `contentHash` field MUST be serialized as [hexadecimal](./Serializations.md#hexadecimal). +When encoding a DSNP Content URI, the `contentHash` field MUST be serialized exactly as it appears in the Announcement (that is, as a base32 multihash string). ### Example ``` -dsnp://78187493520/0x1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef +dsnp://78187493520/bdyqdua4t4pxgy37mdmjyqv3dejp5betyqsznimpneyujsur23yubzna ``` | part | value | | ---- | ----- | | Scheme | `dsnp://` | | User Id | `78187493520` | -| Content Hash | `0x1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcdef` | +| Content Hash | `bdyqdua4t4pxgy37mdmjyqv3dejp5betyqsznimpneyujsur23yubzna` | diff --git a/pages/DSNP/Types/Broadcast.md b/pages/DSNP/Types/Broadcast.md index 2a5d19c1..c9ef4cf4 100644 --- a/pages/DSNP/Types/Broadcast.md +++ b/pages/DSNP/Types/Broadcast.md @@ -7,7 +7,7 @@ A Broadcast Announcement is a way to send a public message to everyone. | Field | Description | Data Type | Serialization | Parquet Type | Bloom Filter | | ----- | ----------- | --------- | ------------- | ------------ | ------------ | | announcementType | Announcement Type Enum (`2`) | enum | [decimal](../Serializations.md#decimal) | `INT32` | no | -| contentHash | multihash-encoded hash of content stored at URL | variable length byte array | [hexadecimal](../Serializations.md#hexadecimal) | `BYTE_ARRAY` | YES +| contentHash | [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) of content | UTF-8 | multihash as base32 string | `UTF8` | YES | fromId | id of the user creating the Announcement | 64-bit unsigned integer | [decimal](../Serializations.md#decimal) | `UINT_64` | YES | url | content URL | UTF-8 | [UTF-8](https://datatracker.ietf.org/doc/html/rfc3629) | `UTF8` | no @@ -19,7 +19,7 @@ A Broadcast Announcement is a way to send a public message to everyone. ### contentHash -- MUST be a [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) +- MUST be a valid [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) ### fromId diff --git a/pages/DSNP/Types/Reply.md b/pages/DSNP/Types/Reply.md index 739da11f..69900bf6 100644 --- a/pages/DSNP/Types/Reply.md +++ b/pages/DSNP/Types/Reply.md @@ -8,7 +8,7 @@ but includes an `inReplyTo` field for noting it as a reply to a given [DSNP Cont | Field | Description | Data Type | Serialization | Parquet Type | Bloom Filter | | ----- | ----------- | --------- | ------------- | ------------ | ------------ | | announcementType | Announcement Type Enum (`3`) | enum | [decimal](../Serializations.md#decimal) | `INT32` | no | -| contentHash | multihash-encoded hash of content stored at URL | variable length byte array | [hexadecimal](../Serializations.md#hexadecimal) | `BYTE_ARRAY` | YES +| contentHash | [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) of content | UTF-8 | multihash as base32 string | `UTF8` | YES | fromId | id of the user creating the Announcement | 64-bit unsigned integer | [decimal](../Serializations.md#decimal) | `UINT_64` | YES | inReplyTo | Target [DSNP Content URI](../Identifiers.md#dsnp-content-uri) | UTF-8 | [UTF-8](https://datatracker.ietf.org/doc/html/rfc3629) | `UTF8` | YES | url | content URL | UTF-8 | [UTF-8](https://datatracker.ietf.org/doc/html/rfc3629) | `UTF8` | no @@ -21,7 +21,7 @@ but includes an `inReplyTo` field for noting it as a reply to a given [DSNP Cont ### contentHash -- MUST be a [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) +- MUST be a valid [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) ### fromId diff --git a/pages/DSNP/Types/Tombstone.md b/pages/DSNP/Types/Tombstone.md index de8187f9..503c0f9a 100644 --- a/pages/DSNP/Types/Tombstone.md +++ b/pages/DSNP/Types/Tombstone.md @@ -10,7 +10,7 @@ It is NOT possible to revert a tombstone. | announcementType | Announcement Type Enum (`0`) | enum | [decimal](../Serializations.md#decimal) | `INT32` | no | | fromId | Id of the user creating the Announcement and the Tombstoned Announcement | 64-bit unsigned integer | [decimal](../Serializations.md#decimal) | `UINT_64` | YES | targetAnnouncementType | target tombstoned Announcement type | enum | [decimal](../Serializations.md#decimal) | `INT32` | no | -| targetContentHash | target `contentHash` of the original Announcement to tombstone | variable length byte array | [hexadecimal](../Serializations.md#hexadecimal) | `BYTE_ARRAY` | YES +| targetContentHash | target `contentHash` of the original Announcement to tombstone | multihash as base32 string | `UTF8` | YES ## Field Requirements @@ -37,4 +37,4 @@ It is NOT possible to revert a tombstone. ### targetContentHash -- MUST match a `contentHash` of previous Announcement with the same `fromId` as the Tombstone Announcement +- MUST be the `contentHash` of a previous Announcement of an Allowed Announcement Type with the same `fromId` as the Tombstone Announcement diff --git a/pages/DSNP/Types/Update.md b/pages/DSNP/Types/Update.md index 1cfac528..fc00bc82 100644 --- a/pages/DSNP/Types/Update.md +++ b/pages/DSNP/Types/Update.md @@ -10,10 +10,10 @@ Updates should be ignored. | ----- | ----------- | --------- | ------------- | ------------ | ------------ | | announcementType | Announcement Type Enum (`6`) | enum | [decimal](../Serializations.md#decimal) | `INT32` | no | | fromId | id of the user creating the announcement | 64 bit unsigned integer | [decimal](../Serializations.md#decimal) | `UINT_64` | YES -| contentHash | multihash-encoded hash of content stored at URL | variable length byte array | [hexadecimal](../Serializations.md#hexadecimal) | `BYTE_ARRAY` | YES +| contentHash | [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) of content | UTF-8 | multihash as base32 string | `UTF8` | YES | url | updated content URL | UTF-8 | [UTF-8](https://datatracker.ietf.org/doc/html/rfc3629) | `UTF8` | no | targetAnnouncementType | target updated Announcement type | enum | [decimal](../Serializations.md#decimal) | `INT32` | no | -| targetContentHash | multihash-encoded hash of target content | variable length byte array | [hexadecimal](../Serializations.md#hexadecimal) | `BYTE_ARRAY` | YES +| targetContentHash | target `contentHash` of the original Announcement to update | multihash as base32 string | `UTF8` | YES ## Field Requirements @@ -27,7 +27,7 @@ Updates should be ignored. ### contentHash -- MUST be a [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) +- MUST be a valid [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) ### url @@ -49,4 +49,4 @@ Updates should be ignored. ### targetContentHash -- MUST be the `contentHash` of an allowed Announcement type with the same `fromId` as the Update Announcement +- MUST be the `contentHash` of a previous Announcement of an Allowed Announcement Type with the same `fromId` as the Update Announcement From 9a43e283fcf90326a6cdf5a9b396cfe0438b31be Mon Sep 17 00:00:00 2001 From: Wes Biggs Date: Wed, 7 Aug 2024 14:47:19 -0500 Subject: [PATCH 07/11] Lint --- .spellcheckerdict.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.spellcheckerdict.txt b/.spellcheckerdict.txt index 14fe2750..36e938a1 100644 --- a/.spellcheckerdict.txt +++ b/.spellcheckerdict.txt @@ -1,6 +1,7 @@ Alexa announcementType Avro +base32 Base58 BLAKE3 Brötli @@ -72,6 +73,7 @@ PNG Polkadot Poly1305 pre-configured +Prepending Prerelease PRId([ABs])? ProfileResource @@ -112,6 +114,6 @@ W3C WebM WebP websocket -whitepaper +[Ww]hitepaper X25519 XSalsa20 From 25435002ea731ca9e006b04c2cb3874b1bfa14ca Mon Sep 17 00:00:00 2001 From: Wes Biggs Date: Wed, 7 Aug 2024 15:41:26 -0500 Subject: [PATCH 08/11] Break out base32 multibase as a Serialization --- .spellcheckerdict.txt | 2 +- pages/DSNP/Identifiers.md | 8 +++----- pages/DSNP/Serializations.md | 21 +++++++++++---------- pages/DSNP/Types/Broadcast.md | 2 +- pages/DSNP/Types/Reply.md | 2 +- pages/DSNP/Types/Update.md | 4 ++-- 6 files changed, 19 insertions(+), 20 deletions(-) diff --git a/.spellcheckerdict.txt b/.spellcheckerdict.txt index 36e938a1..9e00df04 100644 --- a/.spellcheckerdict.txt +++ b/.spellcheckerdict.txt @@ -58,7 +58,7 @@ mdBook MDX MP[34] MSA -multibase +[Mm]ultibase multicodec [Mm]ultihash multihash-encoded diff --git a/pages/DSNP/Identifiers.md b/pages/DSNP/Identifiers.md index 3a69b2e5..02ea457a 100644 --- a/pages/DSNP/Identifiers.md +++ b/pages/DSNP/Identifiers.md @@ -19,15 +19,13 @@ Graph connections are formed through the DSNP User Id. 1. Apply the Supported Hashing Algorithm to create a digest of the content. 2. Prepend the leading bytes from the table below indicating the hashing algorithm in the multicodec table and the length of the hash output. -3. Encode to UTF-8 using the `base32` alphabet. -4. Prepend the `'b'` character indicating `base32` encoding. +3. Serialize as a [base32 multibase](./Serializations.md#base32-multibase) string. ### Example 1. Applying the BLAKE3 algorithm to the [DSNP Whitepaper](https://dsnp.org/dsnp_whitepaper.pdf) yields the following 32 bytes: `0x3a0393e3ee6c6fec1b13885763225fd0927884b2d431ed262899523ade281cb4`. -2. Prepending the multihash indicator (`0x1e` for `blake3`) and hash length (`0x20` for 32 bytes) gives `0x1e203a0393e3ee6c6fec1b13885763225fd0927884b2d431ed262899523ade281cb4` -3. Encoding this to the `base32` alphabet outputs the string `dyqdua4t4pxgy37mdmjyqv3dejp5betyqsznimpneyujsur23yubzna`. -4. Prepending the `b` character to indicate `base32` gives us the final DSNP Content Hash of `bdyqdua4t4pxgy37mdmjyqv3dejp5betyqsznimpneyujsur23yubzna`. +2. Prepending the multihash indicator (`0x1e` for `blake3`) and hash length (`0x20` for 32 bytes) gives `0x1e203a0393e3ee6c6fec1b13885763225fd0927884b2d431ed262899523ade281cb4`. +3. Serializing as a base32 multibase string gives us the final DSNP Content Hash of `bdyqdua4t4pxgy37mdmjyqv3dejp5betyqsznimpneyujsur23yubzna`. ### Supported Hashing Algorithms diff --git a/pages/DSNP/Serializations.md b/pages/DSNP/Serializations.md index c2a81cc1..088e3916 100644 --- a/pages/DSNP/Serializations.md +++ b/pages/DSNP/Serializations.md @@ -18,20 +18,21 @@ Strings are used to avoid issues with different implementations of numbers. | 291 | Must be a string | `"291"` | | `291n` | `BigInt(291)` serialization appends an `n` | `"291"` | -## hexadecimal +## base32 multibase Used to represent bytes. +A base32 multibase string is self-identifying and always begins with the `b` character. +The [Multibase Table](https://github.com/multiformats/multibase?tab=readme-ov-file#multibase-table) describes this encoding as "RFC4648 case-insensitive - no padding". -- MUST use 0-9 and a-f representation +- MUST use [RFC4648](https://datatracker.ietf.org/doc/html/rfc4648) §6 alphabet `abcdefghijklmnopqrstuvwxyz234567` - MUST be lowercase -- MUST be prefixed with a `0x` +- MUST be prefixed with `b` - MUST NOT have spaces or separators -- MUST have two characters per byte in addition to the `0x` characters +- MUST NOT end with or contain padding characters (`=`) -| Bytes | Invalid | Valid | +| Invalid | Why | Valid | | --- | --- | --- | -| 2 | `0x123` | `0x0123` | -| 2 | `123h` | `0x0123` | -| 2 | `0x0ABC` | `0x0abc` | -| 8 | `0xabc` | `0x0000000000000abc` | -| 32 | `0x3e34c4325f4461b9355027b314f3eb56d31af549f7da7bd9ef1ce951651e` | `0x00003e34c4325f4461b9355027b314f3eb56d31af549f7da7bd9ef1ce951651e` | +| `BDYQDUA4T` | Must user lowercase | `bdyqdua4t` | +| `dyqdua4t` | Missing `b` prefix | `bdyqdua4t` | +| `b3og3k0sj` | Wrong alphabet (`base32hex` was used) | `bdyqdua4t` | +| `bdyqdua4t=` | Must not have padding characters | `bdyqdua4t` | diff --git a/pages/DSNP/Types/Broadcast.md b/pages/DSNP/Types/Broadcast.md index c9ef4cf4..b58ff9dd 100644 --- a/pages/DSNP/Types/Broadcast.md +++ b/pages/DSNP/Types/Broadcast.md @@ -7,7 +7,7 @@ A Broadcast Announcement is a way to send a public message to everyone. | Field | Description | Data Type | Serialization | Parquet Type | Bloom Filter | | ----- | ----------- | --------- | ------------- | ------------ | ------------ | | announcementType | Announcement Type Enum (`2`) | enum | [decimal](../Serializations.md#decimal) | `INT32` | no | -| contentHash | [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) of content | UTF-8 | multihash as base32 string | `UTF8` | YES +| contentHash | [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) of content | UTF-8 | [base32 multibase](../Serializations.md#base32-multibase) | `UTF8` | YES | fromId | id of the user creating the Announcement | 64-bit unsigned integer | [decimal](../Serializations.md#decimal) | `UINT_64` | YES | url | content URL | UTF-8 | [UTF-8](https://datatracker.ietf.org/doc/html/rfc3629) | `UTF8` | no diff --git a/pages/DSNP/Types/Reply.md b/pages/DSNP/Types/Reply.md index 69900bf6..b0aa44ba 100644 --- a/pages/DSNP/Types/Reply.md +++ b/pages/DSNP/Types/Reply.md @@ -8,7 +8,7 @@ but includes an `inReplyTo` field for noting it as a reply to a given [DSNP Cont | Field | Description | Data Type | Serialization | Parquet Type | Bloom Filter | | ----- | ----------- | --------- | ------------- | ------------ | ------------ | | announcementType | Announcement Type Enum (`3`) | enum | [decimal](../Serializations.md#decimal) | `INT32` | no | -| contentHash | [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) of content | UTF-8 | multihash as base32 string | `UTF8` | YES +| contentHash | [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) of content | UTF-8 | [base32 multibase](../Serializations.md#base32-multibase) | `UTF8` | YES | fromId | id of the user creating the Announcement | 64-bit unsigned integer | [decimal](../Serializations.md#decimal) | `UINT_64` | YES | inReplyTo | Target [DSNP Content URI](../Identifiers.md#dsnp-content-uri) | UTF-8 | [UTF-8](https://datatracker.ietf.org/doc/html/rfc3629) | `UTF8` | YES | url | content URL | UTF-8 | [UTF-8](https://datatracker.ietf.org/doc/html/rfc3629) | `UTF8` | no diff --git a/pages/DSNP/Types/Update.md b/pages/DSNP/Types/Update.md index fc00bc82..cbbd7215 100644 --- a/pages/DSNP/Types/Update.md +++ b/pages/DSNP/Types/Update.md @@ -10,10 +10,10 @@ Updates should be ignored. | ----- | ----------- | --------- | ------------- | ------------ | ------------ | | announcementType | Announcement Type Enum (`6`) | enum | [decimal](../Serializations.md#decimal) | `INT32` | no | | fromId | id of the user creating the announcement | 64 bit unsigned integer | [decimal](../Serializations.md#decimal) | `UINT_64` | YES -| contentHash | [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) of content | UTF-8 | multihash as base32 string | `UTF8` | YES +| contentHash | [DSNP Content Hash](../Identifiers.md#dsnp-content-hash) of content | UTF-8 | [base32 multibase](../Serializations.md#base32-multibase) | `UTF8` | YES | url | updated content URL | UTF-8 | [UTF-8](https://datatracker.ietf.org/doc/html/rfc3629) | `UTF8` | no | targetAnnouncementType | target updated Announcement type | enum | [decimal](../Serializations.md#decimal) | `INT32` | no | -| targetContentHash | target `contentHash` of the original Announcement to update | multihash as base32 string | `UTF8` | YES +| targetContentHash | target `contentHash` of the original Announcement to update | UTF-8 | [base32 multibase](../Serializations.md#base32-multibase) | `UTF8` | YES ## Field Requirements From eda2857592c6461533f81dc56dd3a9114bef06ef Mon Sep 17 00:00:00 2001 From: Wes Biggs Date: Wed, 7 Aug 2024 15:51:48 -0500 Subject: [PATCH 09/11] Formatting issues --- pages/DSNP/BatchPublications.md | 2 +- pages/DSNP/Types/Tombstone.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pages/DSNP/BatchPublications.md b/pages/DSNP/BatchPublications.md index 0d6c8d77..69d461bb 100644 --- a/pages/DSNP/BatchPublications.md +++ b/pages/DSNP/BatchPublications.md @@ -44,7 +44,7 @@ See also [Announcement Types](Announcements.md). #### Columns with Bloom Filters | Column | Primitive Type | Logical Type | Converted Type (deprecated) | -| ------ | ---- | ---- | +| ------ | ---- | ---- | --- | | contentHash | `BYTE_ARRAY` | `STRING` | `UTF8` | | emoji | `BYTE_ARRAY` | `STRING` | `UTF8` | | fromId | `INT64` | `INT(64, false)` | `UINT_64` | diff --git a/pages/DSNP/Types/Tombstone.md b/pages/DSNP/Types/Tombstone.md index 503c0f9a..d09cd0dc 100644 --- a/pages/DSNP/Types/Tombstone.md +++ b/pages/DSNP/Types/Tombstone.md @@ -10,7 +10,7 @@ It is NOT possible to revert a tombstone. | announcementType | Announcement Type Enum (`0`) | enum | [decimal](../Serializations.md#decimal) | `INT32` | no | | fromId | Id of the user creating the Announcement and the Tombstoned Announcement | 64-bit unsigned integer | [decimal](../Serializations.md#decimal) | `UINT_64` | YES | targetAnnouncementType | target tombstoned Announcement type | enum | [decimal](../Serializations.md#decimal) | `INT32` | no | -| targetContentHash | target `contentHash` of the original Announcement to tombstone | multihash as base32 string | `UTF8` | YES +| targetContentHash | target `contentHash` of the original Announcement to tombstone | UTF-8 | [base32 multibase](../Serializations.md#base32-multibase) | `UTF8` | YES ## Field Requirements From f3ba6e3c1d1595f7b0de5e36b0f98df606d59cca Mon Sep 17 00:00:00 2001 From: Wes Biggs Date: Wed, 7 Aug 2024 15:54:00 -0500 Subject: [PATCH 10/11] Tweaking decimal examples that bothered me --- pages/DSNP/Serializations.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pages/DSNP/Serializations.md b/pages/DSNP/Serializations.md index 088e3916..b92e3d30 100644 --- a/pages/DSNP/Serializations.md +++ b/pages/DSNP/Serializations.md @@ -14,9 +14,9 @@ Strings are used to avoid issues with different implementations of numbers. | Invalid | Why | Valid | | --- | --- | --- | -| `0x123` | Must be decimal | `"291"` | -| 291 | Must be a string | `"291"` | -| `291n` | `BigInt(291)` serialization appends an `n` | `"291"` | +| `"0x123"` | Must be decimal | `"291"` | +| `291` | Must be a string | `"291"` | +| `"291n"` | `BigInt(291)` serialization appends an `n` | `"291"` | ## base32 multibase From a0804183b981f53f8be1315c5bd96ab62857c0c2 Mon Sep 17 00:00:00 2001 From: Wes Biggs Date: Mon, 12 Aug 2024 11:59:22 -0500 Subject: [PATCH 11/11] Fix hashes for dsnp_whitepaper.pdf --- pages/ActivityContent/Associated/Hash.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pages/ActivityContent/Associated/Hash.md b/pages/ActivityContent/Associated/Hash.md index 0d06ec82..d87336f6 100644 --- a/pages/ActivityContent/Associated/Hash.md +++ b/pages/ActivityContent/Associated/Hash.md @@ -13,8 +13,8 @@ This example gives SHA-256 and BLAKE3 hashes for the [PDF version of the DSNP wh ```json { "hash": [ - "bciqgb7tvyyqn6tw4amtdj7funn5ockup6ldrkap4o5ubd77od6awi4y", - "bdyqdua4t4pxgy37mdmjyqv3dejp5betyqsznimpneyujsur23yubzna" + "bciqdnu347gcfmxzbkhgoubiobphm6readngitfywktdtbdocgogop2q", + "bdyqhwoxp2mc6oyaqpqyd2fvaxralslk32ggazv6nxpp342iec6652tq" ] } ```