feat: integrate Helia as IPFS client replacing Kubo

.github/workflows/tests.yaml

@@ -12,7 +12,7 @@ jobs:
 
     steps:
       - name: Install wget2 for cloning
-        run: sudo apt-get install -y wget2
+        run: sudo apt-get update && sudo apt-get install -y wget2
       - uses: actions/checkout@v3
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3

ansible/roles/distributed_press/defaults/main.yml

@@ -5,16 +5,14 @@ distributed_press_user: "press"
 distributed_press_home: "/home/{{distributed_press_user}}"
 distributed_press_service_name: "distributed.press"
 distributed_press_data: "{{distributed_press_home}}/.local/share/distributed-press-nodejs"
-distributed_press_site_storage: "{{distributed_press_data}}/sites/sites"
+distributed_press_site_storage: "{{distributed_press_data}}/sites"
 
 # 3932 is DWEB on a phone number pad
 distributed_press_port: "3932"
-distributed_press_host: "localhost"
-
-distributed_press_ipfs_provider: "builtin"
+distributed_press_host: "0.0.0.0"
 
 distributed_press_git_repo: "https://github.com/hyphacoop/api.distributed.press.git"
-distributed_press_git_branch: "v3.0.0"
+distributed_press_git_branch: "v3.1.0"
 distributed_press_source: "{{distributed_press_home}}/api.distributed.press"
 
 distributed_press_domain: "example.com"

ansible/roles/distributed_press/tasks/main.yml

@@ -30,8 +30,9 @@
     pkg:
       - git
       - ufw
-    state: latest
-    update_cache: true
+    state: present
+    update_cache: yes
+    cache_valid_time: 3600 
 
 - name: Add distributed.press systemd service
   template:
@@ -66,11 +67,14 @@
 - name: "Enable DNS traffic through firewall"
   shell: "ufw allow in 53/udp"
 
-- name: "Enable libp2p udp traffic through firewall"
-  shell: "ufw allow in 7976/udp"
-
 - name: "Enable libp2p tcp traffic through firewall"
-  shell: "ufw allow in 7976/tcp"
+  shell: "ufw allow in 4001/tcp"
+
+- name: "Enable libp2p WebSocket TCP traffic through firewall"
+  shell: "ufw allow in 4002/tcp"
+
+- name: "Enable libp2p WebRTC UDP traffic through firewall (50000-60000)"
+  shell: "ufw allow 50000:60000/udp"
 
 - name: "Check if JWT keys got generated already"
   stat:
@@ -153,12 +157,14 @@
     enabled: true
     name: "{{social_inbox_service_name}}"
 
-- name: "Add cron job to restart api.distributed.press service every few days"
+- name: "Add cron job to restart api.distributed.press service every day"
   cron:
     name: "Restart api.distributed.press service"
     hour: "0"
     minute: "0"
-    weekday: "6" # Saturday
+    day: "*"
+    month: "*"
+    weekday: "*"
     job: "/bin/systemctl restart {{ distributed_press_service_name }}"
   become: yes
   tags:
@@ -170,8 +176,8 @@
       - certbot
       - nginx
       - python3-certbot-nginx
-    state: latest
-    update_cache: true
+    state: present
+    update_cache: no
 
 - name: "Enable NGINX firewall"
   shell: "ufw allow 'Nginx Full'"

ansible/roles/distributed_press/templates/distributed.press.service.j2

@@ -5,7 +5,7 @@ After=network-online.target
 # Environment="DEBUG=*"
 WorkingDirectory={{distributed_press_source}}
 User={{distributed_press_user}}
-ExecStart=npx ts-node-esm index.ts --port {{distributed_press_port}} --host {{distributed_press_host}} --domain {{distributed_press_domain}} --ipfs-provider {{distributed_press_ipfs_provider}}
+ExecStart=npx ts-node-esm index.ts --port {{distributed_press_port}} --host {{distributed_press_host}} --domain {{distributed_press_domain}}
 Restart=always
 RestartSec=3
 LimitNOFILE=4096

api/index.ts

@@ -42,6 +42,7 @@ export type APIConfig = Partial<{
   useMemoryBackedDB: boolean
   useSigIntHandler: boolean
   useWebringDirectoryListing: boolean
+  useWebRTC: boolean
 }> & ServerI
 
 async function apiBuilder (cfg: APIConfig): Promise<FastifyTypebox> {
@@ -55,7 +56,7 @@ async function apiBuilder (cfg: APIConfig): Promise<FastifyTypebox> {
   const protocols = new ConcreteProtocolManager({
     ipfs: {
       path: path.join(protocolStoragePath, 'ipfs'),
-      provider: cfg.ipfsProvider
+      useWebRTC: cfg.useWebRTC ?? true
     },
     hyper: {
       path: path.join(protocolStoragePath, 'hyper')

config/index.ts

@@ -5,7 +5,6 @@ import { AdminStore } from './admin.js'
 import { PublisherStore } from './publisher.js'
 import { RevocationStore } from './revocations.js'
 import { SiteConfigStore } from './sites.js'
-import path from 'path'
 import { ProtocolManager } from '../protocols/index.js'
 
 export interface StoreI {
@@ -28,9 +27,7 @@ export default class Store implements StoreI {
     this.db = db
 
     const basePath = cfg.storage
-    const siteStoragePath = path.join(basePath, 'sites')
-
-    this.fs = new SiteFileSystem(siteStoragePath)
+    this.fs = new SiteFileSystem(basePath)
     this.admin = new AdminStore(this.db.sublevel('admin', { valueEncoding: 'json' }))
     this.publisher = new PublisherStore(this.db.sublevel('publisher', { valueEncoding: 'json' }))
     this.sites = new SiteConfigStore(this.db.sublevel('sites', { valueEncoding: 'json' }), protocols)

fixtures/site/index.html

@@ -1,2 +1,2 @@
 <title>Hello Distributed Press!</title>
-<h1>Hello World!</h1>
+<h1>Hello Helia!!</h1>

fixtures/spawnServer.ts

@@ -5,7 +5,6 @@ import fs from 'fs'
 import { generateKeyPair } from '../authorization/jwt.js'
 import apiBuilder, { FastifyTypebox } from '../api/index.js'
 import { nanoid } from 'nanoid'
-import { BUILTIN } from '../protocols/ipfs.js'
 import getPort from 'get-port'
 
 const paths = envPaths('distributed-press')
@@ -22,7 +21,6 @@ export async function spawnTestServer (): Promise<FastifyTypebox> {
     dnsport: await getPort(),
     host: 'localhost',
     domain: 'example.com',
-    storage: storagePath,
-    ipfsProvider: BUILTIN
+    storage: storagePath
   })
 }

fs/index.ts

@@ -30,14 +30,30 @@ export class SiteFileSystem {
   /// the target directory
   async extract (tarballPath: string, siteId: string): Promise<string> {
     const sitePath = this.getPath(siteId)
-    await pipeline(
-      fs.createReadStream(tarballPath),
-      gunzip(),
-      tar.extract(sitePath, {
-        readable: true,
-        writable: true
-      })
-    )
-    return sitePath
+    console.log(`[fs] Extracting tarball from ${tarballPath} to ${sitePath}`)
+
+    try {
+      await pipeline(
+        fs.createReadStream(tarballPath),
+        gunzip(),
+        tar.extract(sitePath, {
+          readable: true,
+          writable: true,
+          map: (header) => {
+            console.log(`[fs] Extracting file: ${header.name}`)
+            return header
+          }
+        })
+      )
+
+      // Verify extraction by listing directory contents
+      const files = await fs.promises.readdir(sitePath)
+      console.log(`[fs] Extraction complete. Files in ${sitePath}:`, files)
+
+      return sitePath
+    } catch (err) {
+      console.error(`[fs] Error extracting tarball: ${err instanceof Error ? err.message : String(err)}`)
+      throw err
+    }
   }
 }

index.ts

@@ -1,8 +1,25 @@
+// Polyfill CustomEvent for Node.js environment
+// Shim Web Crypto API to global scope for browser-compatible libraries
+import { webcrypto } from 'node:crypto'
+
 import apiBuilder from './api/index.js'
 import yargs from 'yargs'
 import { hideBin } from 'yargs/helpers'
 import envPaths from 'env-paths'
-import { IPFSProvider, BUILTIN } from './protocols/ipfs.js'
+
+if (typeof CustomEvent === 'undefined') {
+  class CustomEvent<T = any> extends Event {
+    detail: T
+    constructor (type: string, options?: CustomEventInit<T>) {
+      super(type, options)
+      this.detail = options?.detail as T
+    }
+  }
+  (globalThis as any).CustomEvent = CustomEvent
+}
+if (typeof (globalThis as any).crypto === 'undefined') {
+  (globalThis as any).crypto = webcrypto
+}
 const paths = envPaths('distributed-press')
 
 const argv = yargs(hideBin(process.argv)).options({
@@ -11,7 +28,7 @@ const argv = yargs(hideBin(process.argv)).options({
   host: { type: 'string' },
   domain: { type: 'string' },
   data: { type: 'string' },
-  ipfsProvider: { type: 'string' }
+  useWebRTC: { type: 'boolean', default: undefined }
 }).parseSync()
 
 export interface ServerI {
@@ -20,16 +37,16 @@ export interface ServerI {
   host: string
   domain: string
   storage: string
-  ipfsProvider: IPFSProvider
+  useWebRTC?: boolean
 }
 
 const cfg: ServerI = {
   port: Number(argv.port ?? process.env.PORT ?? '8080'),
   dnsport: Number(argv.dnsport ?? process.env.DNSPORT ?? '53'),
-  host: argv.host ?? process.env.HOST ?? 'localhost',
+  host: argv.host ?? process.env.HOST ?? '0.0.0.0',
   domain: argv.domain ?? process.env.DOMAIN ?? 'localhost',
   storage: argv.data ?? paths.data,
-  ipfsProvider: (argv.ipfsProvider as IPFSProvider) ?? BUILTIN
+  useWebRTC: argv.useWebRTC ?? (process.env.USE_WEBRTC?.toLowerCase() === 'false' ? false : process.env.CI !== 'true')
 }
 
 const server = await apiBuilder({