ddat interop with dat js
cable interop with cabal js
ddat's frontend = dat-thang
cable fe = cable-thang
dillo support based on cel's dillo-dat
bootstrap node
xplatform support

Signed-off-by: divo <418@cryspace.space>
This commit is contained in:
divorce committed 2026-10-19 00:17:02 -04:00
1 parent f0f91540be
commit b735900f2e
323 files changed
+134347 -343

No files matched your search

+16
View File
@@ -0,0 +1,16 @@
SLEEP writer pass
Files:
inc/sleep_writer.h
src/sleep_writer.c
src/sleep_tree.c
Important change from the previous writer draft:
SLEEP/Dat root sets for a feed of N blocks use fullRoots(2*N), not fullRoots(2*(N-1)).
Thus N=83 produces roots 63, 143, 161, 164, matching the historical fixture's signatures.
The writer emits one signature for every feed length, 1..N.
The bitfield writer follows the documented 3328-byte SLEEP V2 layout. The historical fixture supplied with the project has a 3584-byte bitfield entry; the extra 256 bytes remain deliberately unresolved rather than guessed.
The code requires SLEEP_SALT/libsodium for cryptographic operation.
@@ -0,0 +1,18 @@
#ifndef SLEEP_WRITER_H
#define SLEEP_WRITER_H
#include <stddef.h>
#include <stdint.h>
#include "sleep.h"
#ifdef __cplusplus
extern "C" {
#endif
int sleep_write_feed(const char *feed_dir,
const char *feed_name,
const uint8_t *const *blocks,
const size_t *lengths,
size_t nblocks,
const sleep_keypair_t *keypair);
#ifdef __cplusplus
}
#endif
#endif
+272
View File
@@ -0,0 +1,272 @@
#include "sleep.h"
#include <errno.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#ifdef SLEEP_SALT
#include <sodium.h>
#endif
static unsigned int sleep_tree_ctz64(uint64_t x)
{
#if defined(__GNUC__) || defined(__clang__)
if (x == 0)
return 64;
return (unsigned int)__builtin_ctzll(x);
#else
unsigned int n = 0;
if (x == 0)
return 64;
while ((x & 1ULL) == 0) {
x >>= 1;
n++;
}
return n;
#endif
}
uint64_t sleep_tree_index(uint64_t depth, uint64_t offset)
{
if (depth >= 63) {
errno = EOVERFLOW;
return UINT64_MAX;
}
return ((1ULL + (2ULL * offset)) << depth) - 1ULL;
}
uint64_t sleep_tree_depth(uint64_t index)
{
return sleep_tree_ctz64(index + 1ULL);
}
uint64_t sleep_tree_offset(uint64_t index, uint64_t depth)
{
if ((index & 1ULL) == 0)
return index / 2ULL;
if (depth == UINT64_MAX)
depth = sleep_tree_depth(index);
return (((index + 1ULL) >> depth) - 1ULL) / 2ULL;
}
uint64_t sleep_tree_parent(uint64_t index)
{
uint64_t depth = sleep_tree_depth(index);
uint64_t offset = sleep_tree_offset(index, depth);
return sleep_tree_index(depth + 1ULL, offset >> 1);
}
int sleep_tree_sibling(uint64_t index, uint64_t *out)
{
uint64_t depth;
uint64_t offset;
if (out == NULL) {
errno = EINVAL;
return -1;
}
depth = sleep_tree_depth(index);
offset = sleep_tree_offset(index, depth);
*out = sleep_tree_index(depth, offset & 1ULL ? offset - 1ULL
: offset + 1ULL);
return 0;
}
int sleep_tree_left_child(uint64_t index, uint64_t *out)
{
uint64_t depth;
uint64_t offset;
if (out == NULL) {
errno = EINVAL;
return -1;
}
if ((index & 1ULL) == 0) {
errno = EINVAL;
return -1;
}
depth = sleep_tree_depth(index);
if (depth == 0) {
errno = EINVAL;
return -1;
}
offset = sleep_tree_offset(index, depth);
*out = sleep_tree_index(depth - 1ULL, offset * 2ULL);
return 0;
}
int sleep_tree_right_child(uint64_t index, uint64_t *out)
{
uint64_t depth;
uint64_t offset;
if (out == NULL) {
errno = EINVAL;
return -1;
}
if ((index & 1ULL) == 0) {
errno = EINVAL;
return -1;
}
depth = sleep_tree_depth(index);
if (depth == 0) {
errno = EINVAL;
return -1;
}
offset = sleep_tree_offset(index, depth);
*out = sleep_tree_index(depth - 1ULL, offset * 2ULL + 1ULL);
return 0;
}
uint64_t sleep_tree_left_span(uint64_t index)
{
uint64_t depth;
if ((index & 1ULL) == 0)
return index;
depth = sleep_tree_depth(index);
return sleep_tree_offset(index, depth) * (1ULL << (depth + 1ULL));
}
uint64_t sleep_tree_right_span(uint64_t index)
{
uint64_t depth;
if ((index & 1ULL) == 0)
return index;
depth = sleep_tree_depth(index);
return (sleep_tree_offset(index, depth) + 1ULL) *
(1ULL << (depth + 1ULL)) - 2ULL;
}
void sleep_tree_spans(uint64_t index, uint64_t *left, uint64_t *right)
{
if (left != NULL)
*left = sleep_tree_left_span(index);
if (right != NULL)
*right = sleep_tree_right_span(index);
}
size_t sleep_tree_count(uint64_t index)
{
uint64_t depth;
if ((index & 1ULL) == 0)
return 1;
depth = sleep_tree_depth(index);
if (depth >= sizeof(size_t) * 8 - 1)
return SIZE_MAX;
return ((size_t)1 << (depth + 1ULL)) - 1;
}
size_t sleep_tree_count_leaves(uint64_t index)
{
return (sleep_tree_count(index) + 1) / 2;
}
size_t sleep_tree_full_roots(uint64_t index, uint64_t *out, size_t capacity)
{
uint64_t blocks;
uint64_t offset = 0;
uint64_t factor = 1;
size_t count = 0;
if (index & 1ULL) {
errno = EINVAL;
return 0;
}
/* mafintosh/flat-tree fullRoots(index) works on index / 2. */
blocks = index / 2ULL;
while (blocks != 0) {
while (factor <= UINT64_MAX / 2ULL && factor * 2ULL <= blocks)
factor *= 2ULL;
if (out != NULL && count < capacity)
out[count] = offset + factor - 1ULL;
count++;
offset += 2ULL * factor;
blocks -= factor;
factor = 1;
}
return count;
}
#ifdef SLEEP_SALT
int sleep_tree_hash_block(const uint8_t *data, size_t len,
uint8_t out_hash[32])
{
uint8_t header[9];
crypto_generichash_state state;
if (out_hash == NULL || (data == NULL && len != 0)) {
errno = EINVAL;
return -1;
}
header[0] = SLEEP_HASH_LEAF;
for (int i = 0; i < 8; i++)
header[1 + i] = (uint8_t)((uint64_t)len >> (56 - i * 8));
if (crypto_generichash_init(&state, NULL, 0, 32) != 0)
return -1;
if (crypto_generichash_update(&state, header, sizeof(header)) != 0)
return -1;
if (len != 0 && crypto_generichash_update(&state, data, len) != 0)
return -1;
return crypto_generichash_final(&state, out_hash, 32);
}
int sleep_tree_hash_parent(const uint8_t left[32],
const uint8_t right[32],
uint64_t length,
uint8_t out_hash[32])
{
uint8_t header[9];
crypto_generichash_state state;
if (left == NULL || right == NULL || out_hash == NULL) {
errno = EINVAL;
return -1;
}
header[0] = SLEEP_HASH_PARENT;
for (int i = 0; i < 8; i++)
header[1 + i] = (uint8_t)(length >> (56 - i * 8));
if (crypto_generichash_init(&state, NULL, 0, 32) != 0)
return -1;
if (crypto_generichash_update(&state, header, sizeof(header)) != 0)
return -1;
if (crypto_generichash_update(&state, left, 32) != 0)
return -1;
if (crypto_generichash_update(&state, right, 32) != 0)
return -1;
return crypto_generichash_final(&state, out_hash, 32);
}
#endif
+530
View File
@@ -0,0 +1,530 @@
#include "sleep_writer.h"
#include "sleep.h"
#include <errno.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifdef SLEEP_SALT
#include <sodium.h>
#endif
#define SLEEP_MAGIC_BITFIELD 0x05025700u
#define SLEEP_MAGIC_SIGNATURE 0x05025701u
#define SLEEP_MAGIC_TREE 0x05025702u
#define SLEEP_BITFIELD_SIZE 3328u
#define SLEEP_DATA_BITS 8192u
#define SLEEP_TREE_BITS 16384u
#define SLEEP_INDEX_BYTES 256u
#define SLEEP_HASH_BYTES 32u
#define SLEEP_STACK_MAX 64u
typedef struct writer_node {
uint8_t hash[SLEEP_HASH_BYTES];
uint64_t length;
uint64_t index;
uint64_t depth;
} writer_node_t;
typedef struct writer_peak {
uint64_t index;
uint64_t depth;
} writer_peak_t;
static void put_u16be(uint8_t *p, uint16_t v)
{
p[0] = (uint8_t)(v >> 8);
p[1] = (uint8_t)v;
}
static void put_u32be(uint8_t *p, uint32_t v)
{
p[0] = (uint8_t)(v >> 24);
p[1] = (uint8_t)(v >> 16);
p[2] = (uint8_t)(v >> 8);
p[3] = (uint8_t)v;
}
static void put_u64be(uint8_t *p, uint64_t v)
{
p[0] = (uint8_t)(v >> 56);
p[1] = (uint8_t)(v >> 48);
p[2] = (uint8_t)(v >> 40);
p[3] = (uint8_t)(v >> 32);
p[4] = (uint8_t)(v >> 24);
p[5] = (uint8_t)(v >> 16);
p[6] = (uint8_t)(v >> 8);
p[7] = (uint8_t)v;
}
static int make_path(char *out, size_t out_size,
const char *dir, const char *name,
const char *suffix)
{
int n = snprintf(out, out_size, "%s/%s%s", dir, name, suffix);
if (n < 0 || (size_t)n >= out_size) {
errno = ENAMETOOLONG;
return -1;
}
return 0;
}
static int write_header(FILE *f, uint32_t magic,
uint16_t entry_size, const char *algo)
{
uint8_t header[SLEEP_HEADER_SIZE];
size_t len = algo ? strlen(algo) : 0;
if (len > 24) {
errno = EINVAL;
return -1;
}
memset(header, 0, sizeof(header));
put_u32be(header, magic);
put_u16be(header + 5, entry_size);
header[7] = (uint8_t)len;
if (len)
memcpy(header + 8, algo, len);
if (fwrite(header, 1, sizeof(header), f) != sizeof(header)) {
errno = EIO;
return -1;
}
return 0;
}
static int hash_leaf(const uint8_t *data, size_t len,
uint8_t out[32])
{
#ifdef SLEEP_SALT
uint8_t head[9];
crypto_generichash_state st;
head[0] = 0;
put_u64be(head + 1, (uint64_t)len);
if (crypto_generichash_init(&st, NULL, 0, 32) != 0)
return -1;
if (crypto_generichash_update(&st, head, sizeof(head)) != 0)
return -1;
if (len && crypto_generichash_update(&st, data, len) != 0)
return -1;
return crypto_generichash_final(&st, out, 32);
#else
(void)data; (void)len; (void)out;
errno = ENOSYS;
return -1;
#endif
}
static int hash_parent(const writer_node_t *left,
const writer_node_t *right,
uint8_t out[32])
{
#ifdef SLEEP_SALT
uint8_t head[9];
crypto_generichash_state st;
head[0] = 1;
put_u64be(head + 1, left->length + right->length);
if (crypto_generichash_init(&st, NULL, 0, 32) != 0)
return -1;
if (crypto_generichash_update(&st, head, sizeof(head)) != 0)
return -1;
if (crypto_generichash_update(&st, left->hash, 32) != 0)
return -1;
if (crypto_generichash_update(&st, right->hash, 32) != 0)
return -1;
return crypto_generichash_final(&st, out, 32);
#else
(void)left; (void)right; (void)out;
errno = ENOSYS;
return -1;
#endif
}
static int tree_build(const uint8_t *const *blocks,
const size_t *lengths,
size_t nblocks,
writer_node_t **out_nodes,
size_t *out_count)
{
writer_node_t *nodes;
writer_peak_t stack[SLEEP_STACK_MAX];
size_t stack_len = 0;
size_t node_count;
size_t i;
if (nblocks > (SIZE_MAX + 1) / 2) {
errno = EOVERFLOW;
return -1;
}
node_count = nblocks ? nblocks * 2 - 1 : 0;
nodes = node_count ? calloc(node_count, sizeof(*nodes)) : NULL;
if (node_count && !nodes) {
errno = ENOMEM;
return -1;
}
for (i = 0; i < nblocks; i++) {
writer_node_t *leaf;
uint64_t leaf_index = (uint64_t)(i * 2);
leaf = &nodes[leaf_index];
leaf->index = leaf_index;
leaf->depth = 0;
leaf->length = lengths[i];
if (hash_leaf(blocks[i], lengths[i], leaf->hash) != 0)
goto fail;
stack[stack_len].index = leaf_index;
stack[stack_len].depth = 0;
stack_len++;
while (stack_len >= 2 &&
stack[stack_len - 1].depth == stack[stack_len - 2].depth) {
writer_peak_t right = stack[stack_len - 1];
writer_peak_t left = stack[stack_len - 2];
writer_node_t *ln = &nodes[left.index];
writer_node_t *rn = &nodes[right.index];
uint64_t parent_index;
writer_node_t *parent;
stack_len -= 2;
parent_index = sleep_tree_parent(left.index);
if (parent_index >= node_count) {
errno = EINVAL;
goto fail;
}
parent = &nodes[parent_index];
parent->index = parent_index;
parent->depth = left.depth + 1;
parent->length = ln->length + rn->length;
if (hash_parent(ln, rn, parent->hash) != 0)
goto fail;
stack[stack_len].index = parent_index;
stack[stack_len].depth = parent->depth;
stack_len++;
}
}
*out_nodes = nodes;
*out_count = node_count;
return 0;
fail:
free(nodes);
return -1;
}
static int write_tree(const char *path,
const writer_node_t *nodes, size_t node_count)
{
FILE *f;
size_t i;
f = fopen(path, "wb");
if (!f)
return -1;
if (write_header(f, SLEEP_MAGIC_TREE,
SLEEP_TREE_ENTRY_SIZE, SLEEP_ALGO_BLAKE2B) != 0)
goto fail;
for (i = 0; i < node_count; i++) {
uint8_t entry[40];
memcpy(entry, nodes[i].hash, 32);
put_u64be(entry + 32, nodes[i].length);
if (fwrite(entry, 1, sizeof(entry), f) != sizeof(entry)) {
errno = EIO;
goto fail;
}
}
return fclose(f) == 0 ? 0 : -1;
fail:
fclose(f);
return -1;
}
static int sign_state(const writer_node_t *nodes,
size_t block_count,
const sleep_keypair_t *keypair,
uint8_t signature[64])
{
#ifdef SLEEP_SALT
uint64_t root_indexes[64];
size_t root_count;
size_t i;
size_t msg_len;
uint8_t *msg;
uint8_t digest[32];
crypto_generichash_state st;
/* The Dat/SLEEP root set is fullRoots(2 * length). */
root_count = sleep_tree_full_roots((uint64_t)(2 * block_count),
root_indexes, 64);
if (root_count > (SIZE_MAX - 1) / 48) {
errno = EOVERFLOW;
return -1;
}
msg_len = 1 + root_count * 48;
msg = malloc(msg_len);
if (!msg) {
errno = ENOMEM;
return -1;
}
msg[0] = 2;
for (i = 0; i < root_count; i++) {
const writer_node_t *root = &nodes[root_indexes[i]];
size_t off = 1 + i * 48;
memcpy(msg + off, root->hash, 32);
put_u64be(msg + off + 32, root->index);
put_u64be(msg + off + 40, root->length);
}
if (crypto_generichash_init(&st, NULL, 0, 32) != 0)
goto fail;
if (crypto_generichash_update(&st, msg, msg_len) != 0)
goto fail;
if (crypto_generichash_final(&st, digest, 32) != 0)
goto fail;
free(msg);
if (crypto_sign_detached(signature, NULL,
digest, sizeof(digest),
keypair->seckey) != 0)
return -1;
return 0;
fail:
free(msg);
return -1;
#else
(void)nodes; (void)block_count; (void)keypair; (void)signature;
errno = ENOSYS;
return -1;
#endif
}
static int write_signatures(const char *path,
const writer_node_t *nodes,
size_t nblocks,
const sleep_keypair_t *keypair)
{
FILE *f;
size_t length;
f = fopen(path, "wb");
if (!f)
return -1;
if (write_header(f, SLEEP_MAGIC_SIGNATURE,
SLEEP_SIG_ENTRY_SIZE, SLEEP_ALGO_ED25519) != 0)
goto fail;
for (length = 1; length <= nblocks; length++) {
uint8_t sig[64];
if (sign_state(nodes, length, keypair, sig) != 0)
goto fail;
if (fwrite(sig, 1, sizeof(sig), f) != sizeof(sig)) {
errno = EIO;
goto fail;
}
}
return fclose(f) == 0 ? 0 : -1;
fail:
fclose(f);
return -1;
}
static void set_bit(uint8_t *bits, size_t bit)
{
bits[bit >> 3] |= (uint8_t)(1u << (bit & 7));
}
static unsigned int byte_state(uint8_t v)
{
if (v == 0x00)
return 0;
if (v == 0xff)
return 3;
return 2;
}
static void build_index(uint8_t index[SLEEP_INDEX_BYTES],
const uint8_t data[1024])
{
size_t pair;
memset(index, 0, SLEEP_INDEX_BYTES);
/* Four index bits describe every two bytes of the data bitfield. */
for (pair = 0; pair < 512; pair++) {
unsigned int a = byte_state(data[pair * 2]);
unsigned int b = byte_state(data[pair * 2 + 1]);
unsigned int tuple = a | (b << 2);
size_t nibble = pair;
size_t byte = nibble >> 1;
if ((nibble & 1) == 0)
index[byte] |= (uint8_t)tuple;
else
index[byte] |= (uint8_t)(tuple << 4);
}
}
static int write_bitfield(const char *path,
size_t nblocks,
size_t node_count)
{
uint8_t entry[SLEEP_BITFIELD_SIZE];
uint8_t *data = entry;
uint8_t *tree = entry + 1024;
uint8_t *index = entry + 3072;
FILE *f;
size_t i;
memset(entry, 0, sizeof(entry));
if (nblocks > SLEEP_DATA_BITS || node_count > SLEEP_TREE_BITS) {
errno = EOVERFLOW;
return -1;
}
for (i = 0; i < nblocks; i++)
set_bit(data, i);
for (i = 0; i < node_count; i++)
set_bit(tree, i);
build_index(index, data);
f = fopen(path, "wb");
if (!f)
return -1;
if (write_header(f, SLEEP_MAGIC_BITFIELD,
SLEEP_BITFIELD_SIZE, NULL) != 0)
goto fail;
if (fwrite(entry, 1, sizeof(entry), f) != sizeof(entry)) {
errno = EIO;
goto fail;
}
return fclose(f) == 0 ? 0 : -1;
fail:
fclose(f);
return -1;
}
static int write_data(const char *path,
const uint8_t *const *blocks,
const size_t *lengths,
size_t nblocks)
{
FILE *f;
size_t i;
f = fopen(path, "wb");
if (!f)
return -1;
for (i = 0; i < nblocks; i++) {
if (lengths[i] &&
fwrite(blocks[i], 1, lengths[i], f) != lengths[i]) {
errno = EIO;
goto fail;
}
}
return fclose(f) == 0 ? 0 : -1;
fail:
fclose(f);
return -1;
}
static int write_key(const char *path, const sleep_keypair_t *keypair)
{
FILE *f = fopen(path, "wb");
if (!f)
return -1;
if (fwrite(keypair->pubkey, 1, 32, f) != 32) {
errno = EIO;
fclose(f);
return -1;
}
return fclose(f) == 0 ? 0 : -1;
}
int sleep_write_feed(const char *feed_dir,
const char *feed_name,
const uint8_t *const *blocks,
const size_t *lengths,
size_t nblocks,
const sleep_keypair_t *keypair)
{
char path[1024];
writer_node_t *nodes = NULL;
size_t node_count = 0;
int rc = -1;
if (!feed_dir || !feed_name || !keypair ||
(nblocks && (!blocks || !lengths))) {
errno = EINVAL;
return -1;
}
if (make_path(path, sizeof(path), feed_dir, feed_name, ".tree") != 0)
return -1;
if (tree_build(blocks, lengths, nblocks, &nodes, &node_count) != 0)
return -1;
if (write_tree(path, nodes, node_count) != 0)
goto done;
if (make_path(path, sizeof(path), feed_dir, feed_name, ".signatures") != 0)
goto done;
if (write_signatures(path, nodes, nblocks, keypair) != 0)
goto done;
if (make_path(path, sizeof(path), feed_dir, feed_name, ".key") != 0)
goto done;
if (write_key(path, keypair) != 0)
goto done;
if (make_path(path, sizeof(path), feed_dir, feed_name, ".bitfield") != 0)
goto done;
if (write_bitfield(path, nblocks, node_count) != 0)
goto done;
if (make_path(path, sizeof(path), feed_dir, feed_name, ".data") != 0)
goto done;
if (write_data(path, blocks, lengths, nblocks) != 0)
goto done;
rc = 0;
done:
free(nodes);
return rc;
}