diff --git a/src/api.ts b/src/api.ts new file mode 100644 index 0000000..c1f2a9f --- /dev/null +++ b/src/api.ts @@ -0,0 +1,18 @@ +// @ts-check + +import { ApiPromise, WsProvider } from '@polkadot/api'; +import { registerJoystreamTypes } from '@joystream/types'; + +export default async function create_api () { + // Initialise the provider to connect to the local node + const provider = new WsProvider('ws://127.0.0.1:9944'); + + // register types before creating the api + registerJoystreamTypes(); + + // Create the API and wait until ready + let api = await ApiPromise.create({ provider }); + await api.isReady; + + return api; +} \ No newline at end of file diff --git a/src/export_forum.ts b/src/export_forum.ts new file mode 100644 index 0000000..748a88f --- /dev/null +++ b/src/export_forum.ts @@ -0,0 +1,162 @@ +import create_api from './api'; +import { ApiPromise } from '@polkadot/api'; +import { PostId, ThreadId, BlockAndTime } from '@joystream/types/common' +import { Post, CategoryId, Category, + Thread, OptionModerationAction, VecPostTextChange, + OptionChildPositionInParentCategory, ModerationAction +} from '@joystream/types/forum'; +import { Codec, CodecArg } from '@polkadot/types/types'; +import { Text, bool as Bool, u32, Option, u64 } from '@polkadot/types'; + +// Note: Codec.toHex() re-encodes the value, based on how the type +// was registered. It does NOT produce the same value read from storage +// unless it was correctly defined with exact match. +// Also toJSON() behaves similarly., and special case for types that are registered Vec vs Text +// `Vec` produces a json array of numbers (byte array), `Text` produces a json string + +main() + +async function main () { + const api = await create_api(); + + const categories = await get_all_categories(api); + const posts = await get_all_posts(api); + const threads = await get_all_threads(api); + + let forum_data = { + categories: categories.map(category => category.toHex()), + posts: posts.map(post => post.toHex()), + threads: threads.map(thread => thread.toHex()), + }; + + console.log(JSON.stringify(forum_data)); + + api.disconnect(); +} + +// Fetches a value from map directly from storage and through the query api. +// It ensures the value actually exists in the map +async function get_forum_checked_storage(api: ApiPromise, map: string, id: CodecArg) : Promise { + const key = api.query.forum[map].key(id); + const raw_value = await api.rpc.state.getStorage(key) as unknown as Option; + + if (raw_value.isNone) { + console.error(`Error: value does not exits: ${map} key: ${id}`); + process.exit(-1); + } else { + return (await api.query.forum[map](id) as T) + } +} + +async function get_all_posts(api: ApiPromise) { + let first = 1; + let next = (await api.query.forum.nextPostId() as PostId).toNumber(); + + let posts = []; + + for (let id = first; id < next; id++ ) { + let post = await get_forum_checked_storage(api, 'postById', id) as Post; + + // Transformation to a value that makes sense in a new chain. + post = new Post({ + id: post.id, + thread_id: post.thread_id, + nr_in_thread: post.nr_in_thread, + current_text: new Text(post.current_text), + moderation: moderationActionAtBlockOne(post.moderation), + // No reason to preserve change history + text_change_history: new VecPostTextChange(), + author_id: post.author_id, + created_at: new BlockAndTime({ + // old block number on a new chain doesn't make any sense + block: new u32(1), + time: new u64(post.created_at.momentDate.valueOf()) + }), + }); + + posts.push(post) + } + + return posts; +} + +async function get_all_categories(api: ApiPromise) { + let first = 1; + let next = (await api.query.forum.nextCategoryId() as CategoryId).toNumber(); + + let categories = []; + + for (let id = first; id < next; id++ ) { + let category = await get_forum_checked_storage(api, 'categoryById', id) as Category; + + category = new Category({ + id: new CategoryId(category.id), + title: new Text(category.title), + description: new Text(category.description), + created_at: new BlockAndTime({ + // old block number on a new chain doesn't make any sense + block: new u32(1), + time: new u64(category.created_at.momentDate.valueOf()) + }), + deleted: new Bool(category.deleted), + archived: new Bool(category.archived), + num_direct_subcategories: new u32(category.num_direct_subcategories), + num_direct_unmoderated_threads: new u32(category.num_direct_unmoderated_threads), + num_direct_moderated_threads: new u32(category.num_direct_moderated_threads), + position_in_parent_category: new OptionChildPositionInParentCategory(category.position_in_parent_category), + moderator_id: category.moderator_id, + }); + + categories.push(category) + } + + return categories; +} + +async function get_all_threads(api: ApiPromise) { + let first = 1; + let next = (await api.query.forum.nextThreadId() as ThreadId).toNumber(); + + let threads = []; + + for (let id = first; id < next; id++ ) { + let thread = await get_forum_checked_storage(api, 'threadById', id) as Thread; + + thread = new Thread({ + id: new ThreadId(thread.id), + title: new Text(thread.title), + category_id: new CategoryId(thread.category_id), + nr_in_category: new u32(thread.nr_in_category), + moderation: moderationActionAtBlockOne(thread.moderation), + num_unmoderated_posts: new u32(thread.num_unmoderated_posts), + num_moderated_posts: new u32(thread.num_moderated_posts), + created_at: new BlockAndTime({ + // old block number on a new chain doesn't make any sense + block: new u32(1), + time: new u64(thread.created_at.momentDate.valueOf()) + }), + author_id: thread.author_id, + }); + + threads.push(thread); + } + + return threads; +} + +function moderationActionAtBlockOne( + action: ModerationAction | undefined) : OptionModerationAction { + + if(!action) { + return new OptionModerationAction(); + } else { + return new OptionModerationAction(new ModerationAction({ + moderated_at: new BlockAndTime({ + block: new u32(1), + time: new u64(action.moderated_at.momentDate.valueOf()), + }), + moderator_id: action.moderator_id, + rationale: new Text(action.rationale) + })); + } +} \ No newline at end of file diff --git a/src/export_members.ts b/src/export_members.ts new file mode 100644 index 0000000..5c835f5 --- /dev/null +++ b/src/export_members.ts @@ -0,0 +1,50 @@ +import create_api from './api' +import { ApiPromise } from '@polkadot/api' +import { MemberId, Profile } from '@joystream/types/members' +import { Option, u32, u64 } from '@polkadot/types/' +import { BlockAndTime } from '@joystream/types/common' + +import { GenesisMember } from './genesis_member' + +main() + +async function main () { + const api = await create_api(); + + const members = await get_all_members(api); + + console.log(JSON.stringify(members)); + + api.disconnect(); +} + +async function get_all_members(api: ApiPromise) : Promise { + const first = 0 + const next = (await api.query.members.membersCreated() as MemberId).toNumber(); + + let members = []; + + for (let id = first; id < next; id++ ) { + const profile = await api.query.members.memberProfile(id) as Option; + + if (profile.isSome) { + const p = profile.unwrap() as Profile; + members.push(new GenesisMember({ + member_id: new MemberId(id), + root_account: p.root_account, + controller_account: p.controller_account, + handle: p.handle, + avatar_uri: p.avatar_uri, + about: p.about, + registered_at_time: fixedTimestamp(p.registered_at_time) + })); + } + } + + return members; +} + +function fixedTimestamp(time: u64) { + const blockAndTime = new BlockAndTime({ block: new u32(1), time }) + return new u64(blockAndTime.momentDate.valueOf()) +} \ No newline at end of file diff --git a/src/export_versioned_store.ts b/src/export_versioned_store.ts new file mode 100644 index 0000000..785ffcf --- /dev/null +++ b/src/export_versioned_store.ts @@ -0,0 +1,206 @@ +import create_api from './api' +import { ApiPromise } from '@polkadot/api' +import { Codec, CodecArg } from '@polkadot/types/types' +import { Option, Tuple, u64, u32 } from '@polkadot/types' +import { SingleLinkedMapEntry } from './linkedMap' +import { Credential, BlockAndTime } from '@joystream/types/common' +import { ClassId, EntityId, Class, Entity } from '@joystream/types/versioned-store' +import { ClassPermissionsType } from '@joystream/types/versioned-store/permissions' +import { DataObject, ContentId } from '@joystream/types/media' +import { Channel, ChannelId } from '@joystream/types/content-working-group' +import { assert } from '@polkadot/util' + +// Nicaea schemas +const VIDEO_CLASS_ID = 7 +const MEDIA_OBJECT_CLASS_ID = 1 +const VIDEO_SCHEMA_MEDIA_OBJECT_IN_CLASS_INDEX = 7 +const MEDIA_OBJECT_SCHEMA_CONTENT_ID_IN_CLASS_INDEX = 0 + +// Entity Ids of 'Video' entities we don't wish to export +const EXCLUDED_VIDEOS = [773, 769, 765, 761, 757, 753, 751] + +type ExportedEntities = Array<{entity: Entity, maintainer: Credential | undefined}> + +main() + +async function main () { + const api = await create_api(); + + const classes = await get_all_classes(api) + + const exportedEntities = await get_all_entities(api) + + const dataDirectory = await get_data_directory_from_entities( + api, + exportedEntities.map(({ entity }) => entity) + ) + + const channels = await get_channels(api) + + const versioned_store_data = { + classes: classes.map(classAndPermissions => ({ + class: classAndPermissions.class.toHex(), + permissions: classAndPermissions.permissions.toHex() + })), + entities: exportedEntities.map( + ({ entity, maintainer }) => ({ + entity: entity.toHex(), + maintainer: maintainer ? maintainer.toHex() : null + }) + ), + data_objects: dataDirectory.map(content => ({ + content_id: content.content_id.toHex(), + data_object: content.data_object.toHex() + })), + channels: channels.map(channel => ({id: channel.id, channel: channel.channel.toHex()})) + } + + api.disconnect() + + console.log(JSON.stringify(versioned_store_data)) +} + +// Fetches a value from map directly from storage and through the query api. +// It ensures the value actually exists in the map. +async function get_checked_map_value(api: ApiPromise, module: string, map: string, key: CodecArg) : Promise { + const storageKey = api.query[module][map].key(key); + const raw_value = await api.rpc.state.getStorage(storageKey) as unknown as Option; + + if (raw_value.isNone) { + console.error(`Error: value does not exits: ${map} key: ${key}`); + process.exit(-1); + } else { + return (await api.query[module][map](key) as T) + } +} + +async function get_all_classes(api: ApiPromise) { + let first = 1; + let next = (await api.query.versionedStore.nextClassId() as ClassId).toNumber(); + + let values = []; + + for (let id = first; id < next; id++ ) { + const clazz = await get_checked_map_value(api, 'versionedStore', 'classById', id) as Class; + const permissions = new SingleLinkedMapEntry( + ClassPermissionsType, + ClassId, + await api.query.versionedStorePermissions.classPermissionsByClassId(id) + ) + + // encoding as hex assuming in import the parity codec will be identical + // if we have issues will need to serialize as json instead + values.push({ + class: clazz, + permissions: permissions.value, + }) + } + + return values +} + +async function get_all_entities(api: ApiPromise) : Promise { + const first = 1 + const next = (await api.query.versionedStore.nextEntityId() as EntityId).toNumber() + const entities: ExportedEntities = [] + + for (let id = first; id < next; id++ ) { + let entity = await get_checked_map_value(api, 'versionedStore', 'entityById', id) as Entity + + const maintainerStorageKey = api.query.versionedStorePermissions.entityMaintainerByEntityId.key(id) + const maintainerEntry = await api.rpc.state.getStorage(maintainerStorageKey) as unknown as Option + const maintainer = maintainerEntry.isSome ? new SingleLinkedMapEntry( + Credential, + EntityId, + maintainerEntry.unwrap() + ).value : undefined + + if (entity.class_id.eq(VIDEO_CLASS_ID) && EXCLUDED_VIDEOS.includes(id)) { + continue + } + + entities.push({ + entity, + maintainer, + }) + } + + return entities +} + +async function get_data_directory_from_entities(api: ApiPromise, entities: Entity[]) { + const videoEntities = entities.filter(entity => + entity.class_id.eq(VIDEO_CLASS_ID) + && !EXCLUDED_VIDEOS.includes(entity.id.toNumber()) + ) + + const mediaObjectEntityIds = videoEntities.filter(entity => + entity.entity_values.length + ).map(entity => { + const property = entity.entity_values[VIDEO_SCHEMA_MEDIA_OBJECT_IN_CLASS_INDEX] + assert(property && property.in_class_index.eq(VIDEO_SCHEMA_MEDIA_OBJECT_IN_CLASS_INDEX), 'Unexpected Video Schema') + return property.value.value + }) + + const contentIds = mediaObjectEntityIds.map((entityId) => { + const entity = entities.find((entity) => entity.id.eq(entityId)) + // Runtime protects against this invalid state..just sanity check + if (!entity) { + throw new Error('Referenced Entity Not Found') + } + + if(!entity.class_id.eq(MEDIA_OBJECT_CLASS_ID)) { + throw new Error('Referenced Entity Is Not a Media Object Entity!') + } + + const property = entity.entity_values[MEDIA_OBJECT_SCHEMA_CONTENT_ID_IN_CLASS_INDEX] + assert(property && property.in_class_index.eq(MEDIA_OBJECT_SCHEMA_CONTENT_ID_IN_CLASS_INDEX), 'Unexpected Media Object Schema') + const contentId = property.value.value.toString() + return ContentId.decode(contentId) + }) + + const dataDirectory = [] + + for (let i = 0; i < contentIds.length; i++) { + const content_id = contentIds[i] + const data_object = await api.query.dataDirectory.dataObjectByContentId(content_id) as Option + + if (data_object.isNone) { + console.log('Warning: Entity references a non existent contentId') + continue + } + + const obj = data_object.unwrap() + + dataDirectory.push({ + content_id, + data_object: new DataObject({ + owner: obj.owner, + added_at: new BlockAndTime({ + block: new u32(1), + time: obj.added_at.time, + }), + type_id: obj.type_id, + size: obj.size_in_bytes, + liaison: new u64(0), + liaison_judgement: obj.liaison_judgement, + ipfs_content_id: obj.ipfs_content_id + }) + }) + } + + return dataDirectory +} + +async function get_channels(api: ApiPromise) { + const firstChannelId = 1 + const nextChannelId = (await api.query.contentWorkingGroup.nextChannelId()) as ChannelId + const channels = [] + + for (let id = firstChannelId; nextChannelId.gtn(id); id++) { + const channel = (await api.query.contentWorkingGroup.channelById(id)) as Channel + channels.push({id, channel}) + } + + return channels +} \ No newline at end of file diff --git a/src/genesis_member.ts b/src/genesis_member.ts new file mode 100644 index 0000000..0f2f4d3 --- /dev/null +++ b/src/genesis_member.ts @@ -0,0 +1,60 @@ +import { u64, Text } from '@polkadot/types' +import { Moment } from '@polkadot/types/interfaces' +import { JoyStruct } from '@joystream/types/common' +import AccountId from '@polkadot/types/primitive/Generic/AccountId' +import { MemberId } from '@joystream/types/members' + +export type IGenesisMember = { + member_id: MemberId + root_account: AccountId + controller_account: AccountId + handle: Text + avatar_uri: Text + about: Text + registered_at_time: Moment +} + +export class GenesisMember extends JoyStruct { + constructor(value?: IGenesisMember) { + super( + { + member_id: MemberId, + root_account: AccountId, + controller_account: AccountId, + handle: Text, + avatar_uri: Text, + about: Text, + registered_at_time: u64, + }, + value + ) + } + + get member_id(): MemberId { + return this.get('member_id') as MemberId + } + + get handle(): Text { + return this.get('handle') as Text + } + + get avatar_uri(): Text { + return this.get('avatar_uri') as Text + } + + get about(): Text { + return this.get('about') as Text + } + + get registered_at_time(): u64 { + return this.get('registered_at_time') as u64 + } + + get root_account(): AccountId { + return this.get('root_account') as AccountId + } + + get controller_account(): AccountId { + return this.get('controller_account') as AccountId + } +} \ No newline at end of file diff --git a/src/linkedMap.ts b/src/linkedMap.ts new file mode 100644 index 0000000..07ed7c0 --- /dev/null +++ b/src/linkedMap.ts @@ -0,0 +1,20 @@ +import { Tuple } from '@polkadot/types'; +import { Codec, Constructor } from '@polkadot/types/types'; +import Linkage from '@polkadot/types/codec/Linkage'; + +export class SingleLinkedMapEntry extends Tuple { + constructor (ValueType: Constructor, KeyType: Constructor, value?: any) { + super({ + value: ValueType, + linkage: Linkage.withKey(KeyType) + }, value); + } + + get value (): T { + return this[0] as unknown as T; + } + + get linkage (): Linkage { + return this[1] as unknown as Linkage; + } +} \ No newline at end of file