diff --git a/connect/src/client/hb-encode.js b/connect/src/client/hb-encode.js index b89c41fab..36d3a7111 100644 --- a/connect/src/client/hb-encode.js +++ b/connect/src/client/hb-encode.js @@ -1,25 +1,32 @@ import base64url from 'base64url' -import { Buffer } from 'buffer/index.js' +import { Buffer as BufferShim } from 'buffer/index.js' /** * polyfill in Browser */ -if (!globalThis.Buffer) globalThis.Buffer = Buffer +if (!globalThis.Buffer) globalThis.Buffer = BufferShim /** * ****** * HyperBEAM Http Encoding * - * TODO: bundle into a package with - * - * - export encode() - * - export encodeDataItem() to convert object - * or ans104 to http message - * - exported signers for both node and browser environments - * (currently located in wallet.js modules) + * TODO: bundle into a separate package * ****** */ +const MAX_HEADER_LENGTH = 4096 + +async function hasNewline (value) { + if (typeof value === 'string') return value.includes('\n') + if (value instanceof Blob) { + value = await value.text() + return value.includes('\n') + } + if (isBytes(value)) return Buffer.from(value).includes('\n') + + return false +} + /** * @param {ArrayBuffer} data */ @@ -27,159 +34,307 @@ async function sha256 (data) { return crypto.subtle.digest('SHA-256', data) } -function partition (pred, arr) { - return arr.reduce((acc, cur) => { - acc[pred(cur) ? 0 : 1].push(cur) - return acc - }, - [[], []]) -} - function isBytes (value) { return value instanceof ArrayBuffer || ArrayBuffer.isView(value) } -function hbEncodeValue (key, value) { - const typeK = `converge-type-${key}` +function isPojo (value) { + return !isBytes(value) && + !Array.isArray(value) && + !(value instanceof Blob) && + typeof value === 'object' && + value !== null +} + +function hbEncodeValue (value) { + if (isBytes(value)) { + if (value.byteLength === 0) return hbEncodeValue('') + return [undefined, value] + } if (typeof value === 'string') { - if (value.length === 0) return { [typeK]: 'empty-binary' } - return { [key]: value } + if (value.length === 0) return [undefined, 'empty-binary'] + return [undefined, value] } - if (Array.isArray(value) && value.length === 0) { - return { [typeK]: 'empty-list' } + if (Array.isArray(value)) { + if (value.length === 0) return ['empty-list', undefined] + const encoded = value.reduce( + (acc, cur) => { + let [type, curEncoded] = hbEncodeValue(cur) + if (!type) type = 'binary' + acc.push(`(ao-type-${type}) ${curEncoded}`) + return acc + }, + [] + ) + return ['list', encoded.join(',')] } if (typeof value === 'number') { - if (!Number.isInteger(value)) return { [typeK]: 'float', [key]: `${value}` } - return { [typeK]: 'integer', [key]: `${value}` } + if (!Number.isInteger(value)) return ['float', `${value}`] + return ['integer', `${value}`] } if (typeof value === 'symbol') { - return { [typeK]: 'atom', [key]: value.description } + return ['atom', value.description] } throw new Error(`Cannot encode value: ${value.toString()}`) } -function hbEncode (obj, parent = '') { - return Object.entries(obj).reduce((acc, [key, value]) => { - const flatK = (parent ? `${parent}/${key}` : key) - .toLowerCase() +export function hbEncodeLift (obj, parent = '', top = {}) { + const [flattened, types] = Object.entries({ ...obj }) + .reduce((acc, [key, value]) => { + const flatK = (parent ? `${parent}/${key}` : key) + .toLowerCase() - // skip nullish values - if (value == null) return acc + // skip nullish values + if (value == null) return acc - // binary data - if (isBytes(value)) { - if (value.byteLength === 0) { - return Object.assign(acc, hbEncodeValue(flatK, '')) + // list of objects + if (Array.isArray(value) && value.some(isPojo)) { + /** + * Convert the list of maps into an object + * where keys are indices and values are the maps + * + * This will match the isPojo check below, + * which will handle the recursive lifting that we want. + */ + value = value.reduce((indexedObj, v, idx) => + Object.assign(indexedObj, { [idx]: v }), {}) } - return Object.assign(acc, { [flatK]: value }) - } - // first/{idx}/name flatten array - if (Array.isArray(value)) { - if (value.length === 0) { - return Object.assign(acc, hbEncodeValue(flatK, value)) + // first/second lift object + if (isPojo(value)) { + /** + * Encode the pojo on top, but then continuing iterating + * through the current object level + */ + hbEncodeLift(value, flatK, top) + return acc } - value.forEach((v, i) => - Object.assign(acc, hbEncode(v, `${flatK}/${i}`)) - ) + + // leaf encode value + const [type, encoded] = hbEncodeValue(value) + if (encoded) { + /** + * This value is too large to be potentially encoded + * in a multipart header, so we instead need to "lift" it + * as a top level field on result, to be encoded as its own part + * + * So use flatK to preserve the nesting hierarchy + * While ensure it will be encoded as its own part + */ + if (Buffer.from(encoded).byteLength > MAX_HEADER_LENGTH) { + top[flatK] = encoded + /** + * Encode at the current level as a normal field + */ + } else acc[0][key] = encoded + } + if (type) acc[1][key] = type return acc - } + }, [{}, {}]) - // first/second flatten object - if (typeof value === 'object' && value !== null) { - return Object.assign(acc, hbEncode(value, flatK)) - } + if (Object.keys(flattened).length === 0) return top + + /** + * Add the ao-types key for this specific object, + * as a structured dictionary + */ + if (Object.keys(types).length > 0) { + const aoTypes = Object.entries(types) + .map(([key, value]) => `${key.toLowerCase()}=${value}`) + .join(',') - // leaf encode value - Object.assign(acc, hbEncodeValue(flatK, value)) - return acc - }, {}) + /** + * The ao-types header was too large to encoded as a header + * so lift to the top, to be encoded as its own part + */ + if (Buffer.from(aoTypes).byteLength > MAX_HEADER_LENGTH) { + const flatK = (parent ? `${parent}/ao-types` : 'ao-types') + top[flatK] = aoTypes + } else flattened['ao-types'] = aoTypes + } + + if (parent) top[parent] = flattened + // Merge non-pojo values at top level + else Object.assign(top, flattened) + return top } -async function boundaryFrom (bodyParts = []) { - const base = new Blob( - bodyParts.flatMap((p, i, arr) => - i < arr.length - 1 ? [p, '\r\n'] : [p]) - ) +function encodePart (name, { headers, body }) { + const parts = Object + .entries(Object.fromEntries(headers)) + .reduce((acc, [name, value]) => { + acc.push(`${name}: `, value, '\r\n') + return acc + }, [`content-disposition: form-data;name="${name}"\r\n`]) + + if (body) parts.push('\r\n', body) - const hash = await sha256(await base.arrayBuffer()) - return base64url.encode(Buffer.from(hash)) + return new Blob(parts) } /** - * Encode the object as HyperBEAM HTTP multipart - * message. Nested objects are flattened to a single - * depth multipart + * Encoded the object as a HyperBEAM HTTP Multipart Message + * - Nested object are "lifted" to the top level, while preserving + * the hierarchy using "/", to be encoded as a part in the multipart body + * + * - Adds "ao-types" field on each nested object, that defines types + * for each nested field, encoded as a structured dictionary header on the part. + * + * - Conditionally "lifts" fields that too large to be encoded as headers, + * to the top level, to be encoded as a separate part, while preserving + * the hierarchy using "/" */ export async function encode (obj = {}) { if (Object.keys(obj) === 0) return - const flattened = hbEncode(obj) + const flattened = hbEncodeLift(obj) + /** * Some values may be encoded into headers, * while others may be encoded into the body */ - const [bodyKeys, headerKeys] = partition( - (key) => { - if (key.includes('/')) return true - const bytes = Buffer.from(flattened[key]) + const bodyKeys = [] + const headerKeys = [] + await Promise.all( + Object.keys(flattened).map(async (key) => { + const value = flattened[key] + /** + * Sub maps are always encoded as subparts + * in the body. + * + * Since hbEncodeLift already lifts + * objects to the top level, there should only ever + * be 1 recursive call here. + */ + if (isPojo(value)) { + // Empty object or nil + const subPart = await encode(value) + if (!subPart) return + + bodyKeys.push(key) + flattened[key] = encodePart(key, subPart) + return + } + /** - * Anything larger than 4k goes into - * the body + * There are special cases that will force a field to be + * encoded into the body: + * + * - The field includes any whitespace + * - The field includes '/' + * - The fields size exceeds the max header length + * + * In all cases, the field is forced to be encoded into the body + * as a sub-part, where the part has a single Content-Disposition + * header denoting the field, and the body of the sub-part + * being the field value itself. + * + * (These special cases happen to cover a multitude of issues that + * could cause a Data Item's 'data' to not be encodable as a header, + * but extends that coverage to any sort of field) */ - return bytes.byteLength > 4096 - }, - Object.keys(flattened).sort() + if ( + await hasNewline(value) || + key.includes('/') || + Buffer.from(value).byteLength > MAX_HEADER_LENGTH + ) { + bodyKeys.push(key) + flattened[key] = new Blob([ + `content-disposition: form-data;name="${key}"\r\n\r\n`, + value + ]) + return + } + + headerKeys.push(key) + flattened[key] = value + }) ) const h = new Headers() headerKeys.forEach((key) => h.append(key, flattened[key])) + + // If there is a data field that didn't otherwise get encoded into a multipart body, + // and there are no other body parts, then we need to encode the data as an + // `inline-body-key`. While doing so, we remove the `data` header that would + // otherwise be duplicated. + if (h.has('data')) { + bodyKeys.push('data') + } /** * Add headers that indicates and orders body-keys * for the purpose of determinstically reconstructing * content-digest on the server + * + * TODO: remove dead code. Apparently, this is only needed + * on the HB side, but keeping the commented code here + * just in case we need it client side. */ // const bk = hbEncodeValue('body-keys', bodyKeys) // Object.keys(bk).forEach((key) => h.append(key, bk[key])) - let body + let body, finalContent if (bodyKeys.length) { - const bodyParts = await Promise.all( - bodyKeys.map((name) => new Blob([ - `content-disposition: form-data;name="${name}"\r\n\r\n`, - flattened[name] - ]).arrayBuffer()) - ) + if (bodyKeys.length === 1) { + // If there is only one element, promote it to be the full body and set the + // `inline-body-key` such that the server knows its name. + body = new Blob([obj.data]) + h.append('inline-body-key', bodyKeys[0]) + h.delete(bodyKeys[0]) + } else { + // This is a multipart body, so we generate and insert the boundary + // appropriately. + const bodyParts = await Promise.all( + bodyKeys.map((name) => { + return flattened[name].arrayBuffer() + }) + ) - const boundary = await boundaryFrom(bodyParts) + /** + * Generate a deterministic boundary, from the parts + * to use for the multipart body boundary + */ + const base = new Blob( + bodyParts.flatMap((p, i, arr) => + i < arr.length - 1 ? [p, '\r\n'] : [p]) + ) + const hash = await sha256(await base.arrayBuffer()) + const boundary = base64url.encode(Buffer.from(hash)) - /** - * Segment each part with the multipart boundary - */ - const blobParts = bodyParts - .flatMap((p) => [`--${boundary}\r\n`, p, '\r\n']) + /** + * Segment each part with the multipart boundary + */ + const blobParts = bodyParts + .flatMap((p) => [`--${boundary}\r\n`, p, '\r\n']) - /** - * Add the terminating boundary - */ - blobParts.push(`--${boundary}--`) + /** + * Add the terminating boundary + */ + blobParts.push(`--${boundary}--`) - body = new Blob(blobParts) + h.set('Content-Type', `multipart/form-data; boundary="${boundary}"`) + body = new Blob(blobParts) + } /** * calculate the content-digest */ - const contentDigest = await sha256(await body.arrayBuffer()) + finalContent = await body.arrayBuffer() + const contentDigest = await sha256(finalContent) const base64 = base64url.toBase64(base64url.encode(contentDigest)) - h.set('Content-Type', `multipart/form-data; boundary="${boundary}"`) h.append('Content-Digest', `sha-256=:${base64}:`) } + // console.log('Encoded headers:') + // console.log(h) + // console.log('Encoded body:') + // console.log(finalContent) + return { headers: h, body } } diff --git a/connect/src/client/hb.js b/connect/src/client/hb.js index 34e395a3d..c73d3bffb 100644 --- a/connect/src/client/hb.js +++ b/connect/src/client/hb.js @@ -1,9 +1,12 @@ import { Rejected, fromPromise, of } from 'hyper-async' +import { omit, keys } from 'ramda' import base64url from 'base64url' import { joinUrl } from '../lib/utils.js' import { encode } from './hb-encode.js' -import { toHttpSigner } from './signer.js' +import { toHttpSigner, toDataItemSigner } from './signer.js' + +const reqFormatCache = {} /** * Map data item members to corresponding HB HTTP message @@ -14,7 +17,7 @@ export async function encodeDataItem ({ processId, data, tags, anchor }) { if (processId) obj.target = processId if (anchor) obj.anchor = anchor - if (tags) tags.forEach(t => { obj[t.name] = t.value }) + if (tags) tags.forEach(t => { obj[t.name.toLowerCase()] = t.value }) /** * Always ensure the variant is mainnet for hyperbeam * TODO: change default variant to be this eventually @@ -53,59 +56,89 @@ export function httpSigName (address) { return `http-sig-${hexString}` } -export function requestWith ({ fetch, logger: _logger, HB_URL, signer }) { +export function requestWith (args) { + const { fetch, logger: _logger, HB_URL, signer } = args + let signingFormat = args.signingFormat const logger = _logger.child('request') - return (fields) => { + return async function (fields) { const { path, method, ...restFields } = fields - return of({ path, method, fields: restFields }) - .chain(fromPromise(({ path, method, fields }) => - encode(fields).then(({ headers, body }) => ({ + signingFormat = fields.signingFormat + if (!signingFormat) { + signingFormat = reqFormatCache[fields.path] ?? 'HTTP' + } + + try { + let fetchRequest = { } + // console.log('SIGNING FORMAT: ', signingFormat, '. REQUEST: ', fields) + if (signingFormat === 'ANS-104') { + const ans104Request = toANS104Request(restFields) + // console.log('ANS-104 REQUEST PRE-SIGNING: ', ans104Request) + const signedRequest = await toDataItemSigner(signer)(ans104Request.item) + // console.log('SIGNED ANS-104 ITEM: ', signedRequest) + fetchRequest = { + body: signedRequest.raw, + url: joinUrl({ url: HB_URL, path }), path, method, - headers, - body - })) - )) - .chain(fromPromise(async ({ path, method, headers, body }) => - toHttpSigner(signer)(toSigBaseArgs({ + headers: ans104Request.headers + } + } else { + // Step 2: Create and execute the signing request + const req = await encode(restFields) + const signingArgs = toSigBaseArgs({ url: joinUrl({ url: HB_URL, path }), method, - headers - // this does not work with hyperbeam - // includePath: true - })).then((req) => ({ ...req, body })) - )) - .map(logger.tap('Sending HTTP signed message to HB: %o')) - .chain((request) => of(request) - .chain(fromPromise(({ url, method, headers, body }) => { - return fetch(url, { method, headers, body, redirect: 'follow' }) - .then(async res => { - if (res.status < 300) { - const contentType = res.headers.get('content-type') - - if (contentType && contentType.includes('multipart/form-data')) { - return res - } else if (contentType && contentType.includes('application/json')) { - const body = await res.json() - return { - headers: res.headers, - body - } - } else { - const body = await res.text() - return { - headers: res.headers, - body - } - } - } - return res - }) - } - )) - ).toPromise() + headers: req.headers + }) + + const signedRequest = await toHttpSigner(signer)(signingArgs) + fetchRequest = { ...signedRequest, body: req.body, path, method } + } + + // Log the request + logger.tap('Sending signed message to HB: %o')(fetchRequest) + + // Step 4: Send the request + // console.log('SENDING REQUEST: ', fetchRequest) + + const res = await fetch(fetchRequest.url, { + method: fetchRequest.method, + headers: fetchRequest.headers, + body: fetchRequest.body, + redirect: 'follow' + }) + + // console.log('PUSH FORMAT: ', signingFormat, '. RESPONSE:', res) + const body = await res.text() + // Step 5: Handle specific status codes + if (res.status === 422 && signingFormat === 'HTTP') { + // Try again with different signing format + reqFormatCache[fields.path] = 'ANS-104' + return requestWith({ ...args, signingFormat: 'ANS-104' })(fields) + } + + if (res.status >= 400) { + console.log('ERROR RESPONSE: ', res) + process.exit(1) + throw new Error(`${res.status}: ${await res.text()}`) + } + + if (res.status >= 300) { + return res + } + + // Step 6: Return the response + return { + headers: res.headers, + body + } + } catch (error) { + // Handle errors appropriately + console.error('Request failed:', error) + throw error + } } } @@ -217,10 +250,10 @@ export function loadResultWith ({ fetch, logger: _logger, HB_URL, signer }) { return of(args) .chain(fromPromise(async ({ id, processId }) => { const { headers, body } = await encodeDataItem({ processId }) - headers.append('slot+integer', id) + headers.append('slot', id) headers.append('accept', 'application/json') return toHttpSigner(signer)(toSigBaseArgs({ - url: `${HB_URL}/${processId}/compute&slot+integer=${id}/results/json`, + url: `${HB_URL}/${processId}~process@1.0/compute&slot=${id}/results/json`, method: 'POST', headers })).then((req) => ({ ...req, body })) @@ -246,6 +279,48 @@ export function loadResultWith ({ fetch, logger: _logger, HB_URL, signer }) { } } +export function toANS104Request (fields) { + // console.log('TO ANS 104 REQUEST: ', fields) + const dataItem = { + target: fields.target, + anchor: fields.anchor ?? '', + tags: keys( + omit( + [ + 'Target', + 'target', + 'Anchor', + 'anchor', + 'Data', + 'data', + 'data-protocol', + 'Data-Protocol', + 'variant', + 'Variant', + 'dryrun', + 'Dryrun', + 'Type', + 'type', + 'path', + 'method' + ], + fields + ) + ) + .map(function (key) { + return { name: key, value: fields[key] } + }, fields) + .concat([ + { name: 'Data-Protocol', value: 'ao' }, + { name: 'Type', value: fields.Type ?? 'Message' }, + { name: 'Variant', value: fields.Variant ?? 'ao.N.1' } + ]), + data: fields?.data || '' + } + // console.log('ANS104 REQUEST: ', dataItem) + return { headers: { 'Content-Type': 'application/ans104', 'codec-device': 'ans104@1.0' }, item: dataItem } +} + export class InsufficientFunds extends Error { name = 'InsufficientFunds' } diff --git a/connect/src/lib/request/index.js b/connect/src/lib/request/index.js index bc6c942ae..7fba6d48e 100644 --- a/connect/src/lib/request/index.js +++ b/connect/src/lib/request/index.js @@ -184,6 +184,7 @@ if mode == 'process' then request should create a pure httpsig from fields const transformToMap = (mode) => (result) => { const map = {} if (mode === 'relay@1.0') { + // console.log('Mainnet (M1) result', result) if (typeof result === 'string') { return result } @@ -217,6 +218,7 @@ if mode == 'process' then request should create a pure httpsig from fields } return map } else { + // console.log('Mainnet (M2) result', result) const res = result let body = '' res.headers.forEach((v, k) => { @@ -228,38 +230,16 @@ if mode == 'process' then request should create a pure httpsig from fields if (typeof res.body === 'string') { try { body = JSON.parse(res.body) - - if (body.Output && body.Output.data) { - map.Output = { - text: () => Promise.resolve(body.Output.data) - } - } - if (body.Messages) { - map.Messages = body.Messages.map((m) => { - const miniMap = {} - m.Tags.forEach((t) => { - miniMap[t.name] = { - text: () => Promise.resolve(t.value) - } - }) - miniMap.Data = { - text: () => Promise.resolve(m.Data), - json: () => Promise.resolve(JSON.parse(m.Data)), - binary: () => Promise.resolve(Buffer.from(m.Data)) - } - miniMap.Target = { - text: () => Promise.resolve(m.Target) - } - miniMap.Anchor = { - text: () => Promise.resolve(m.Anchor) - } - return miniMap - }) + if (typeof body === 'object') { + return { ...map, ...body } } + return { ...map, body } } catch (e) { - map.body = { text: () => Promise.resolve(body) } + // console.log('Mainnet (M2) error', e) + map.body = body } } + // console.log('Mainnet (M2) default reply', map) return map } } @@ -279,8 +259,9 @@ if mode == 'process' then request should create a pure httpsig from fields .map((res) => { logger( - 'Received response from message sent to path "%s"', - fields?.path ?? '/' + 'Received response from message sent to path "%s" with res %O', + fields?.path ?? '/', + res ) return res })