Merge pull request #2758 from Lezek123/giza-qn-bootstrap

Giza: Query node bootstrap scripts

query-node/manifest.yml

@@ -284,7 +284,7 @@ mappings:
     #- extrinsic: Sudo.batchCall
     #  handler: handleSudoCall(DatabaseManager,SubstrateEvent)
   preBlockHooks:
-    - handler: loadGenesisData
+    - handler: bootstrapData
       filter:
         height: "[0,0]" # will be executed only at genesis
   postBlockHooks:

query-node/mappings/bootstrap-data/data/workingGroups.json

@@ -0,0 +1,8 @@
+{
+  "STORAGE": {
+    "workers": []
+  },
+  "GATEWAY": {
+    "workers": []
+  }
+}

query-node/mappings/bootstrap-data/index.ts

@@ -0,0 +1,10 @@
+import { MemberJson, StorageSystemJson, WorkingGroupsJson } from './types'
+import storageSystemJson from './data/storageSystem.json'
+import membersJson from './data/members.json'
+import workingGroupsJson from './data/workingGroups.json'
+
+const storageSystemData: StorageSystemJson = storageSystemJson
+const membersData: MemberJson[] = membersJson
+const workingGroupsData: WorkingGroupsJson = workingGroupsJson
+
+export { storageSystemData, membersData, workingGroupsData }

query-node/mappings/bootstrap-data/scripts/api.ts

@@ -0,0 +1,19 @@
+import { ApiPromise, WsProvider } from '@polkadot/api'
+import types from '@joystream/sumer-types/augment/all/defs.json'
+
+export default async function createApi(): Promise<ApiPromise> {
+  // Get URL to websocket endpoint from environment or connect to local node by default
+  const WS_URL = process.env.WS_PROVIDER_ENDPOINT_URI || 'ws://127.0.0.1:9944'
+
+  // explicitely state what RPC we are connecting to
+  console.error('Connecting to RPC at: ' + WS_URL)
+
+  // Initialise the provider
+  const provider = new WsProvider(WS_URL)
+
+  // Create the API and wait until ready
+  const api = await ApiPromise.create({ provider, types })
+  await api.isReadyOrError
+
+  return api
+}

query-node/mappings/bootstrap-data/scripts/fetchMembersData.ts

@@ -0,0 +1,51 @@
+import createApi from './api'
+import { ApiPromise } from '@polkadot/api'
+import { MemberId, Membership } from '@joystream/sumer-types/augment/all'
+import { BlockHash } from '@polkadot/types/interfaces'
+import { MemberJson } from '../types'
+import fs from 'fs'
+import path from 'path'
+
+async function main() {
+  const api = await createApi()
+  const blockNumner = parseInt(process.env.AT_BLOCK_NUMBER || '')
+  const hash = process.env.AT_BLOCK_NUMBER ? await api.rpc.chain.getBlockHash(blockNumner) : undefined
+
+  if (!hash) {
+    console.warn('No AT_BLOCK_NUMBER was specified! Exporting from current block...')
+  }
+
+  const members = await getAllMembers(api, hash)
+
+  fs.writeFileSync(path.resolve(__dirname, '../data/members.json'), JSON.stringify(members, undefined, 4))
+  const lastMemberId = Math.max(...members.map((m) => parseInt(m.memberId)))
+  console.log(`${members.length} members exported & saved! Last member id: ${lastMemberId}`)
+
+  await api.disconnect()
+}
+
+async function getAllMembers(api: ApiPromise, hash?: BlockHash): Promise<MemberJson[]> {
+  const memberStorageEntries = hash
+    ? await api.query.members.membershipById.entriesAt(hash)
+    : await api.query.members.membershipById.entries()
+  const memberEntries: [MemberId, Membership][] = memberStorageEntries.map(([{ args: [memberId] }, member]) => [
+    memberId,
+    member,
+  ])
+  const members: MemberJson[] = memberEntries.map(([memberId, member]) => ({
+    memberId: memberId.toString(),
+    rootAccount: member.root_account.toString(),
+    controllerAccount: member.controller_account.toString(),
+    handle: member.handle.toString(),
+    avatarUri: member.avatar_uri.toString(),
+    about: member.about.toString(),
+    registeredAtTime: member.registered_at_time.toNumber(),
+    registeredAtBlock: member.registered_at_block.toNumber(),
+  }))
+
+  return members
+}
+
+main()
+  .then(() => process.exit())
+  .catch(console.error)

query-node/mappings/bootstrap-data/scripts/fetchWorkingGroupsData.ts

@@ -0,0 +1,64 @@
+import createApi from './api'
+import { BlockHash } from '@polkadot/types/interfaces'
+import { ApiPromise } from '@polkadot/api'
+import { WorkerJson, WorkingGroupJson, WorkingGroupsJson } from '../types'
+import fs from 'fs'
+import path from 'path'
+
+export enum WorkingGroups {
+  Storage = 'storageWorkingGroup',
+  Gateway = 'gatewayWorkingGroup',
+}
+
+// export flow
+async function main() {
+  // prepare api connection
+  const api = await createApi()
+
+  const blockNumner = parseInt(process.env.AT_BLOCK_NUMBER || '')
+  const hash = process.env.AT_BLOCK_NUMBER ? await api.rpc.chain.getBlockHash(blockNumner) : undefined
+  const now = new Date()
+
+  // get results for all relevant groups
+  const workingGroups: WorkingGroupsJson = {
+    STORAGE: await getWorkingGroupData(api, WorkingGroups.Storage, hash, now),
+    GATEWAY: await getWorkingGroupData(api, WorkingGroups.Gateway, hash, now),
+  }
+
+  // output results
+  fs.writeFileSync(path.resolve(__dirname, '../data/workingGroups.json'), JSON.stringify(workingGroups, undefined, 4))
+  console.log(`${workingGroups.GATEWAY?.workers.length || 0} GATEWAY workers exported & saved!`)
+  console.log(`${workingGroups.STORAGE?.workers.length || 0} STORAGE workers exported & saved!`)
+
+  // disconnect api
+  api.disconnect()
+}
+
+// retrieves all active workers in working group
+async function getWorkingGroupData(
+  api: ApiPromise,
+  group: WorkingGroups,
+  hash: BlockHash | undefined,
+  now: Date
+): Promise<WorkingGroupJson> {
+  // get working group entries
+  const entries = await (hash ? api.query[group].workerById.entriesAt(hash) : api.query[group].workerById.entries())
+
+  const workers: WorkerJson[] = []
+  entries.forEach(([storageKey]) => {
+    // prepare workerId
+    const workerId = storageKey.args[0]
+    // add record
+    workers.push({
+      workerId: workerId.toString(),
+      // set time of running this script as createdAt
+      createdAt: now.getTime(),
+    })
+  })
+
+  return { workers }
+}
+
+main()
+  .then(() => process.exit())
+  .catch(console.error)

query-node/mappings/bootstrap-data/types.ts

@@ -0,0 +1,35 @@
+export type MemberJson = {
+  memberId: string
+  rootAccount: string
+  controllerAccount: string
+  handle: string
+  about?: string
+  avatarUri?: string
+  registeredAtTime: number
+  registeredAtBlock: number
+}
+
+export type StorageSystemJson = {
+  id: string
+  blacklist: string[]
+  storageBucketsPerBagLimit: number
+  distributionBucketsPerBagLimit: number
+  uploadingBlocked: boolean
+  dataObjectFeePerMb: number | string
+  storageBucketMaxObjectsCountLimit: number | string
+  storageBucketMaxObjectsSizeLimit: number | string
+}
+
+export type WorkerJson = {
+  workerId: string
+  metadata?: string
+  createdAt: number
+}
+
+export type WorkingGroupJson = {
+  workers: WorkerJson[]
+}
+
+export type WorkingGroupsJson = {
+  [group in 'GATEWAY' | 'STORAGE']?: WorkingGroupJson
+}

query-node/mappings/bootstrap.ts

@@ -0,0 +1,58 @@
+import { StoreContext } from '@joystream/hydra-common'
+import BN from 'bn.js'
+import { Membership, MembershipEntryMethod, StorageSystemParameters, Worker, WorkerType } from 'query-node/dist/model'
+import { workerEntityId } from './workingGroup'
+import { storageSystemData, membersData, workingGroupsData } from './bootstrap-data'
+
+export async function bootstrapData({ store }: StoreContext): Promise<void> {
+  // Storage system
+  await store.save<StorageSystemParameters>(
+    new StorageSystemParameters({
+      ...storageSystemData,
+      storageBucketMaxObjectsCountLimit: new BN(storageSystemData.storageBucketMaxObjectsCountLimit),
+      storageBucketMaxObjectsSizeLimit: new BN(storageSystemData.storageBucketMaxObjectsSizeLimit),
+      dataObjectFeePerMb: new BN(storageSystemData.dataObjectFeePerMb),
+    })
+  )
+
+  // Members
+  const members = membersData.map(
+    (m) =>
+      new Membership({
+        // main data
+        id: m.memberId,
+        rootAccount: m.rootAccount,
+        controllerAccount: m.controllerAccount,
+        handle: m.handle,
+        about: m.about,
+        avatarUri: m.avatarUri,
+        createdInBlock: m.registeredAtBlock,
+        entry: m.registeredAtBlock === 1 ? MembershipEntryMethod.GENESIS : MembershipEntryMethod.PAID,
+        // fill in auto-generated fields
+        createdAt: new Date(m.registeredAtTime),
+        updatedAt: new Date(m.registeredAtTime),
+      })
+  )
+  await Promise.all(members.map((m) => store.save<Membership>(m)))
+
+  // Workers
+  let workers: Worker[] = []
+  ;(['GATEWAY', 'STORAGE'] as const).map((group) => {
+    const workersJson = workingGroupsData[group]?.workers || []
+    workers = workers.concat(
+      workersJson.map(
+        (w) =>
+          new Worker({
+            id: workerEntityId(WorkerType[group], w.workerId),
+            workerId: w.workerId,
+            isActive: true,
+            type: WorkerType[group],
+            createdAt: new Date(w.createdAt),
+            updatedAt: new Date(w.createdAt),
+            metadata: w.metadata,
+          })
+      )
+    )
+  })
+  await Promise.all(workers.map((w) => store.save<Worker>(w)))
+}

query-node/mappings/genesis-data/index.ts

@@ -1,8 +0,0 @@
-import { MemberJson, StorageSystemJson } from './types'
-import storageSystemJson from './storageSystem.json'
-import membersJson from './members.json'
-
-const storageSystem: StorageSystemJson = storageSystemJson
-const members: MemberJson[] = membersJson
-
-export { storageSystem, members }

query-node/mappings/genesis-data/types.ts

@@ -1,20 +0,0 @@
-export type MemberJson = {
-  member_id: string
-  root_account: string
-  controller_account: string
-  handle: string
-  about?: string
-  avatar_uri?: string
-  registered_at_time: number
-}
-
-export type StorageSystemJson = {
-  id: string
-  blacklist: string[]
-  storageBucketsPerBagLimit: number
-  distributionBucketsPerBagLimit: number
-  uploadingBlocked: boolean
-  dataObjectFeePerMb: number | string
-  storageBucketMaxObjectsCountLimit: number | string
-  storageBucketMaxObjectsSizeLimit: number | string
-}

query-node/mappings/genesis.ts

@@ -1,35 +0,0 @@
-import { StoreContext } from '@joystream/hydra-common'
-import BN from 'bn.js'
-import { Membership, MembershipEntryMethod, StorageSystemParameters } from 'query-node/dist/model'
-import { storageSystem, members } from './genesis-data'
-
-export async function loadGenesisData({ store }: StoreContext): Promise<void> {
-  // Storage system
-  await store.save<StorageSystemParameters>(
-    new StorageSystemParameters({
-      ...storageSystem,
-      storageBucketMaxObjectsCountLimit: new BN(storageSystem.storageBucketMaxObjectsCountLimit),
-      storageBucketMaxObjectsSizeLimit: new BN(storageSystem.storageBucketMaxObjectsSizeLimit),
-      dataObjectFeePerMb: new BN(storageSystem.dataObjectFeePerMb),
-    })
-  )
-  // Members
-  for (const m of members) {
-    // create new membership
-    const member = new Membership({
-      // main data
-      id: m.member_id,
-      rootAccount: m.root_account,
-      controllerAccount: m.controller_account,
-      handle: m.handle,
-      about: m.about,
-      avatarUri: m.avatar_uri,
-      createdInBlock: 0,
-      entry: MembershipEntryMethod.GENESIS,
-      // fill in auto-generated fields
-      createdAt: new Date(m.registered_at_time),
-      updatedAt: new Date(m.registered_at_time),
-    })
-    await store.save<Membership>(member)
-  }
-}

query-node/mappings/index.ts

@@ -2,4 +2,4 @@ export * from './membership'
 export * from './workingGroup'
 export * from './content'
 export * from './storage'
-export * from './genesis'
+export * from './bootstrap'

query-node/mappings/package.json

@@ -10,13 +10,17 @@
     "clean": "rm -rf lib",
     "lint": "eslint . --quiet --ext .ts",
     "checks": "prettier ./ --check && yarn lint",
-    "format": "prettier ./ --write "
+    "format": "prettier ./ --write ",
+    "bootstrap-data:fetch:members": "yarn ts-node ./bootstrap-data/scripts/fetchMembersData.ts",
+    "bootstrap-data:fetch:workingGroups": "yarn ts-node ./bootstrap-data/scripts/fetchWorkingGroupsData.ts",
+    "bootstrap-data:fetch": "yarn bootstrap-data:fetch:members && yarn bootstrap-data:fetch:workingGroups"
   },
   "dependencies": {
     "@polkadot/types": "5.9.1",
     "@joystream/hydra-common": "3.1.0-alpha.1",
     "@joystream/hydra-db-utils": "3.1.0-alpha.1",
     "@joystream/metadata-protobuf": "^1.0.0",
+    "@joystream/sumer-types": "npm:@joystream/types@^0.16.0",
     "@joystream/types": "^0.17.0",
     "@joystream/warthog": "2.35.0"
   },

query-node/mappings/tsconfig.json

@@ -15,7 +15,8 @@
     "resolveJsonModule": true,
     "baseUrl": ".",
     "paths": {
-      "@polkadot/types/augment": ["../../types/augment/augment-types.ts"]
+      "@polkadot/types/augment": ["../../types/augment/augment-types.ts"],
+      "@polkadot/api/augment": ["../../types/augment/augment-api.ts"]
     }
   },
   "include": ["./**/*"]

query-node/mappings/workingGroup.ts

@@ -4,6 +4,10 @@ import { Worker, WorkerType } from 'query-node/dist/model'
 import { StorageWorkingGroup } from './generated/types'
 import { WorkerId } from '@joystream/types/augment'
 
+export function workerEntityId(type: WorkerType, workerId: string | WorkerId): string {
+  return `${type}-${workerId.toString()}`
+}
+
 export async function workingGroup_OpeningFilled({ event, store }: EventContext & StoreContext): Promise<void> {
   const workerType = getWorkerType(event)
   if (!workerType) {
@@ -115,7 +119,7 @@ async function createWorker(
 ): Promise<void> {
   // create entity
   const newWorker = new Worker({
-    id: `${workerType}-${workerId.toString()}`,
+    id: workerEntityId(workerType, workerId),
     workerId: workerId.toString(),
     type: workerType,
     isActive: true,

yarn.lock

@@ -2127,7 +2127,7 @@
     yaml "^1.10.0"
     yaml-validator "^3.0.0"
 
-"@joystream/types@^0.16.1":
+"@joystream/sumer-types@npm:@joystream/types@^0.16.0", "@joystream/types@^0.16.1":
   version "0.16.1"
   resolved "https://registry.yarnpkg.com/@joystream/types/-/types-0.16.1.tgz#40f5014a9b64928ccea634a1f0f5d2b0392de293"
   integrity sha512-Jz8M6F4oRKH4WtEn8kpZvSMi0mVbfGSmjt38CcEu2946TYmCwlC3Ad1RFH8Wlcylqz/fMLL+pe0z1Dvo6dfzJA==