Colossus & storage dev-init script

cli/content-test.sh

@@ -8,7 +8,8 @@ echo "{}" > ~/tmp/empty.json
 
 export AUTO_CONFIRM=true
 
-yarn workspace api-scripts initialize-content-lead
+# Init content lead
+GROUP=contentDirectoryWorkingGroup yarn workspace api-scripts initialize-lead
 # Test create/update/remove category
 yarn joystream-cli content:createVideoCategory -i ./examples/content/CreateCategory.json
 yarn joystream-cli content:createVideoCategory -i ./examples/content/CreateCategory.json

start.sh

@@ -25,10 +25,10 @@ docker-compose up -d joystream-node
 
 ## Storage Infrastructure
 # Configure a dev storage node and start storage node
-DEBUG=joystream:storage-cli:dev yarn storage-cli dev-init
+yarn workspace api-scripts storage-dev-init
 docker-compose up -d colossus
 # Create a new content directory lead
-yarn workspace api-scripts initialize-content-lead
+GROUP=contentDirectoryWorkingGroup yarn workspace api-scripts initialize-lead
 
 ## Query Node Infrastructure
 # Initialize a new database for the query node infrastructure

storage-node/packages/runtime-api/identities.js

@@ -127,12 +127,20 @@ class IdentitiesApi {
     return memberIds.length > 0 // true if at least one member id exists for the acccount
   }
 
+  async getMembersEntries() {
+    const memberEntries = await this.base.api.query.members.membershipById.entries()
+    memberEntries.map(([storageKey, member]) => [storageKey.args[0], member])
+  }
+
   /*
    * Return all the member IDs of an account by the root account id
    */
   async memberIdsOf(accountId) {
     const decoded = this.keyring.decodeAddress(accountId)
-    return this.base.api.query.members.memberIdsByRootAccountId(decoded)
+    // FIXME: Very slow, but probably the only way to retrieve this data now
+    return this.getMembersEntries()
+      .filter(([, member]) => member.root_account.eq(decoded))
+      .map(([id]) => id)
   }
 
   /*
@@ -140,16 +148,17 @@ class IdentitiesApi {
    */
   async memberIdsOfController(accountId) {
     const decoded = this.keyring.decodeAddress(accountId)
-    return this.base.api.query.members.memberIdsByControllerAccountId(decoded)
+    // FIXME: Very slow, but probably the only way to retrieve this data now
+    return this.getMembersEntries()
+      .filter(([, member]) => member.controller_account.eq(decoded))
+      .map(([id]) => id)
   }
 
   /*
    * Return the first member ID of an account, or undefined if not a member root account.
    */
   async firstMemberIdOf(accountId) {
-    const decoded = this.keyring.decodeAddress(accountId)
-    const ids = await this.base.api.query.members.memberIdsByRootAccountId(decoded)
-    return ids[0]
+    return this.memberIdsOf(accountId)[0]
   }
 
   /*
@@ -194,13 +203,12 @@ class IdentitiesApi {
     const tx = this.base.api.tx.members.buyMembership({
       root_account: accountId,
       controller_account: accountId,
-      name: userInfo.handle,
       handle: userInfo.handle,
     })
 
     return this.base.signAndSendThenGetEventResult(accountId, tx, {
       module: 'members',
-      event: 'MemberRegistered',
+      event: 'MembershipBought',
       type: 'MemberId',
       index: 0,
     })

utils/api-scripts/dev-init-storage.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -e
+
+SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+cd $SCRIPT_PATH
+
+export GROUP="storageWorkingGroup"
+export WORKER_ROLE_URI="//Colossus"
+export INITIAL_WORKER_BALANCE_TOP_UP="10000"
+
+yarn initialize-lead
+yarn initialize-worker

utils/api-scripts/package.json

@@ -7,7 +7,9 @@
     "status": "ts-node src/status",
     "script": "ts-node src/script",
     "tsnode-strict": "node -r ts-node/register --unhandled-rejections=strict",
-    "initialize-content-lead": "ts-node src/initialize-content-lead"
+    "initialize-lead": "ts-node src/initialize-lead",
+    "initialize-worker": "ts-node src/initialize-worker",
+    "storage-dev-init": "./dev-init-storage.sh"
   },
   "dependencies": {
     "@joystream/types": "^0.17.0",

utils/api-scripts/src/initialize-content-lead.ts → utils/api-scripts/src/initialize-lead.ts

@@ -6,6 +6,17 @@ import { SubmittableExtrinsic } from '@polkadot/api/types'
 import { ExtrinsicsHelper, getAlicePair, getKeyFromSuri } from './helpers/extrinsics'
 import BN from 'bn.js'
 
+const workingGroupModules = [
+  'storageWorkingGroup',
+  'contentDirectoryWorkingGroup',
+  'forumWorkingGroup',
+  'membershipWorkingGroup',
+  'operationsWorkingGroup',
+  'gatewayWorkingGroup',
+] as const
+
+type WorkingGroupModuleName = typeof workingGroupModules[number]
+
 const MIN_APPLICATION_STAKE = new BN(2000)
 const STAKING_ACCOUNT_CANDIDATE_STAKE = new BN(200)
 
@@ -16,10 +27,16 @@ async function main() {
   const provider = new WsProvider(WS_URI)
   const api = await ApiPromise.create({ provider, types })
 
+  const Group = process.env.GROUP || 'contentDirectoryWorkingGroup'
   const LeadKeyPair = process.env.LEAD_URI ? getKeyFromSuri(process.env.LEAD_URI) : getAlicePair()
   const SudoKeyPair = process.env.SUDO_URI ? getKeyFromSuri(process.env.SUDO_URI) : getAlicePair()
   const StakeKeyPair = LeadKeyPair.derive(`//stake${Date.now()}`)
 
+  if (!workingGroupModules.includes(Group as WorkingGroupModuleName)) {
+    throw new Error(`Invalid working group: ${Group}`)
+  }
+  const groupModule = Group as WorkingGroupModuleName
+
   const txHelper = new ExtrinsicsHelper(api)
 
   const sudo = (tx: SubmittableExtrinsic<'promise'>) => api.tx.sudo.sudo(tx)
@@ -53,17 +70,17 @@ async function main() {
   }
 
   // Create a new lead opening
-  if ((await api.query.contentDirectoryWorkingGroup.currentLead()).isSome) {
-    console.log('Curators lead already exists, aborting...')
+  if ((await api.query[groupModule].currentLead()).isSome) {
+    console.log(`${groupModule} lead already exists, aborting...`)
   } else {
-    console.log(`Making member id: ${memberId} the content lead.`)
-    // Create curator lead opening
-    console.log('Creating curator lead opening...')
+    console.log(`Making member id: ${memberId} the ${groupModule} lead.`)
+    // Create lead opening
+    console.log(`Creating ${groupModule} lead opening...`)
     const [openingRes] = await txHelper.sendAndCheck(
       SudoKeyPair,
       [
         sudo(
-          api.tx.contentDirectoryWorkingGroup.addOpening(
+          api.tx[groupModule].addOpening(
             '',
             'Leader',
             {
@@ -74,9 +91,9 @@ async function main() {
           )
         ),
       ],
-      'Failed to create Content Curators Lead opening!'
+      `Failed to create ${groupModule} lead opening!`
     )
-    const openingId = openingRes.findRecord('contentDirectoryWorkingGroup', 'OpeningAdded')!.event.data[0] as OpeningId
+    const openingId = openingRes.findRecord(groupModule, 'OpeningAdded')!.event.data[0] as OpeningId
 
     // Set up stake account
     const addCandidateTx = api.tx.members.addStakingAccountCandidate(memberId)
@@ -98,11 +115,11 @@ async function main() {
     console.log((await api.query.system.account(StakeKeyPair.address)).toHuman())
 
     // Apply to lead opening
-    console.log('Applying to curator lead opening...')
+    console.log(`Applying to ${groupModule} lead opening...`)
     const [applicationRes] = await txHelper.sendAndCheck(
       LeadKeyPair,
       [
-        api.tx.contentDirectoryWorkingGroup.applyOnOpening({
+        api.tx[groupModule].applyOnOpening({
           member_id: memberId,
           role_account_id: LeadKeyPair.address,
           opening_id: openingId,
@@ -115,21 +132,13 @@ async function main() {
       'Failed to apply on lead opening!'
     )
 
-    const applicationId = applicationRes.findRecord('contentDirectoryWorkingGroup', 'AppliedOnOpening')!.event
-      .data[1] as ApplicationId
+    const applicationId = applicationRes.findRecord(groupModule, 'AppliedOnOpening')!.event.data[1] as ApplicationId
 
     // Fill opening
     console.log('Filling the opening...')
     await txHelper.sendAndCheck(
       LeadKeyPair,
-      [
-        sudo(
-          api.tx.contentDirectoryWorkingGroup.fillOpening(
-            openingId,
-            new (JoyBTreeSet(ApplicationId))(registry, [applicationId])
-          )
-        ),
-      ],
+      [sudo(api.tx[groupModule].fillOpening(openingId, new (JoyBTreeSet(ApplicationId))(registry, [applicationId])))],
       'Failed to fill the opening'
     )
   }

utils/api-scripts/src/initialize-worker.ts

@@ -0,0 +1,163 @@
+import { registry, types } from '@joystream/types'
+import { JoyBTreeSet, MemberId } from '@joystream/types/common'
+import { ApplicationId, OpeningId } from '@joystream/types/working-group'
+import { ApiPromise, WsProvider } from '@polkadot/api'
+import { ExtrinsicsHelper, getAlicePair, getKeyFromSuri } from './helpers/extrinsics'
+import BN from 'bn.js'
+
+const workingGroupModules = [
+  'storageWorkingGroup',
+  'contentDirectoryWorkingGroup',
+  'forumWorkingGroup',
+  'membershipWorkingGroup',
+  'operationsWorkingGroup',
+  'gatewayWorkingGroup',
+] as const
+
+type WorkingGroupModuleName = typeof workingGroupModules[number]
+
+const MIN_APPLICATION_STAKE = new BN(2000)
+const STAKING_ACCOUNT_CANDIDATE_STAKE = new BN(200)
+const OPENING_STAKE = new BN(2000)
+
+async function main() {
+  // Init api
+  const WS_URI = process.env.WS_URI || 'ws://127.0.0.1:9944'
+  console.log(`Initializing the api (${WS_URI})...`)
+  const provider = new WsProvider(WS_URI)
+  const api = await ApiPromise.create({ provider, types })
+
+  // Input data
+  const Group = process.env.GROUP || 'contentDirectoryWorkingGroup'
+  const LeadRoleKeyPair = process.env.LEAD_URI ? getKeyFromSuri(process.env.LEAD_URI) : getAlicePair()
+  const WorkerMemberKeyPair = process.env.WORKER_MEMBER_URI
+    ? getKeyFromSuri(process.env.WORKER_MEMBER_URI)
+    : getAlicePair()
+  const WorkerRoleKeyPair = process.env.WORKER_ROLE_URI ? getKeyFromSuri(process.env.WORKER_ROLE_URI) : getAlicePair()
+  const StakeKeyPair = WorkerRoleKeyPair.derive(`//stake${Date.now()}`)
+  const InitialWorkerBalanceTopUp = parseInt(process.env.INITIAL_WORKER_BALANCE_TOP_UP || '0') // In order to dev-initialize the storage worker
+
+  // Check if group exists
+  if (!workingGroupModules.includes(Group as WorkingGroupModuleName)) {
+    throw new Error(`Invalid working group: ${Group}`)
+  }
+  const groupModule = Group as WorkingGroupModuleName
+
+  const txHelper = new ExtrinsicsHelper(api)
+
+  // Check if current lead exists
+  const currentLead = await api.query[groupModule].currentLead()
+  if (!currentLead.isSome) {
+    throw new Error(`${groupModule} lead not set!`)
+  }
+
+  // Check if lead keypair is valid
+  const leadWorker = await api.query[groupModule].workerById(currentLead.unwrap())
+  if (!leadWorker.role_account_id.eq(LeadRoleKeyPair.address)) {
+    throw new Error(`${groupModule} lead keypair invalid!`)
+  }
+
+  // Check if worker member exists
+  const memberEntries = await api.query.members.membershipById.entries()
+  const matchingMemberEntry = memberEntries.find(([, member]) =>
+    member.controller_account.eq(WorkerMemberKeyPair.address)
+  )
+  const memberId: MemberId | undefined = matchingMemberEntry?.[0].args[0] as MemberId | undefined
+  if (!memberId) {
+    throw new Error('Make sure WORKER_MEMBER_URI is for a member!')
+  }
+
+  // Check if worker already exists
+  const workerEntries = await api.query[groupModule].workerById.entries()
+  const matchingWorkerEntry = workerEntries.find(([, worker]) => worker.role_account_id.eq(WorkerRoleKeyPair.address))
+  if (matchingWorkerEntry) {
+    throw new Error(`Worker with role key ${WorkerRoleKeyPair.address} already exists`)
+  }
+
+  // Send opening stake to lead's stake account
+  console.log(`Topping up lead's staking account with OPENING_STAKE...`)
+  await txHelper.sendAndCheck(
+    LeadRoleKeyPair,
+    [api.tx.balances.transferKeepAlive(leadWorker.staking_account_id, OPENING_STAKE)],
+    'Lead stake top-up failed'
+  )
+
+  // Create a new opening
+  console.log(`Creating ${groupModule} worker opening...`)
+  const [openingRes] = await txHelper.sendAndCheck(
+    LeadRoleKeyPair,
+    [
+      api.tx[groupModule].addOpening(
+        '',
+        'Regular',
+        {
+          stake_amount: MIN_APPLICATION_STAKE,
+          leaving_unstaking_period: 99999,
+        },
+        null
+      ),
+    ],
+    `Failed to create ${groupModule} worker opening!`
+  )
+  const openingId = openingRes.findRecord(groupModule, 'OpeningAdded')!.event.data[0] as OpeningId
+
+  // Setting up stake account
+  const addCandidateTx = api.tx.members.addStakingAccountCandidate(memberId)
+  const addCandidateFee = (await addCandidateTx.paymentInfo(StakeKeyPair.address)).partialFee
+  const stakingAccountBalance = MIN_APPLICATION_STAKE.add(STAKING_ACCOUNT_CANDIDATE_STAKE).add(addCandidateFee)
+  console.log('Setting up staking account...')
+  await txHelper.sendAndCheck(
+    WorkerMemberKeyPair,
+    [api.tx.balances.transfer(StakeKeyPair.address, stakingAccountBalance)],
+    `Failed to send funds to staing account (${stakingAccountBalance})`
+  )
+  await txHelper.sendAndCheck(StakeKeyPair, [addCandidateTx], 'Failed to add staking candidate')
+  await txHelper.sendAndCheck(
+    WorkerMemberKeyPair,
+    [api.tx.members.confirmStakingAccount(memberId, StakeKeyPair.address)],
+    'Failed to confirm staking account'
+  )
+
+  console.log((await api.query.system.account(StakeKeyPair.address)).toHuman())
+
+  // Applying to worker opening
+  console.log(`Applying to ${groupModule} worker opening...`)
+  const [applicationRes] = await txHelper.sendAndCheck(
+    WorkerMemberKeyPair,
+    [
+      api.tx[groupModule].applyOnOpening({
+        member_id: memberId,
+        role_account_id: WorkerRoleKeyPair.address,
+        opening_id: openingId,
+        stake_parameters: {
+          stake: MIN_APPLICATION_STAKE,
+          staking_account_id: StakeKeyPair.address,
+        },
+      }),
+    ],
+    'Failed to apply on worker opening!'
+  )
+
+  const applicationId = applicationRes.findRecord(groupModule, 'AppliedOnOpening')!.event.data[1] as ApplicationId
+
+  // Filling the opening
+  console.log('Filling the opening...')
+  await txHelper.sendAndCheck(
+    LeadRoleKeyPair,
+    [api.tx[groupModule].fillOpening(openingId, new (JoyBTreeSet(ApplicationId))(registry, [applicationId]))],
+    'Failed to fill the opening'
+  )
+
+  if (InitialWorkerBalanceTopUp) {
+    console.log(`Topping up worker balance (${InitialWorkerBalanceTopUp})`)
+    await txHelper.sendAndCheck(
+      WorkerMemberKeyPair,
+      [api.tx.balances.transferKeepAlive(WorkerRoleKeyPair.address, InitialWorkerBalanceTopUp)],
+      'Worker initial balance top-up failed'
+    )
+  }
+}
+
+main()
+  .then(() => process.exit())
+  .catch((e) => console.error(e))