tests: deterministic key derivation for testing accounts

tests/network-tests/.env

@@ -56,3 +56,5 @@ STAKE_DECREMENT = 3
 MINT_CAPACITY_INCREMENT = 1000
 # Storage node address to download content from
 STORAGE_NODE_URL = http://localhost:3001/asset/v0
+# Mini-secret or mnemonic used in SURI for deterministic key derivation
+SURI_MINI_SECRET = ""

tests/network-tests/src/Api.ts

@@ -42,13 +42,22 @@ export enum WorkingGroups {
 export class ApiFactory {
   private readonly api: ApiPromise
   private readonly keyring: Keyring
+  // number used as part of soft key derivation path
+  private keyId = 0
+  // mapping from account address to key id.
+  // To be able to re-derive keypair externally when mini-secret is known.
+  readonly addressesToKeyId: Map<string, number> = new Map()
+  // mini secret used in SURI key derivation path
+  private readonly miniSecret: string
+
   // source of funds for all new accounts
   private readonly treasuryAccount: string
 
   public static async create(
     provider: WsProvider,
     treasuryAccountUri: string,
-    sudoAccountUri: string
+    sudoAccountUri: string,
+    miniSecret: string
   ): Promise<ApiFactory> {
     const debug = extendDebug('api-factory')
     let connectAttempts = 0
@@ -65,7 +74,7 @@ export class ApiFactory {
         // Give it a few seconds to be ready.
         await Utils.wait(5000)
 
-        return new ApiFactory(api, treasuryAccountUri, sudoAccountUri)
+        return new ApiFactory(api, treasuryAccountUri, sudoAccountUri, miniSecret)
       } catch (err) {
         if (connectAttempts === 3) {
           throw new Error('Unable to connect to chain')
@@ -75,32 +84,43 @@ export class ApiFactory {
     }
   }
 
-  constructor(api: ApiPromise, treasuryAccountUri: string, sudoAccountUri: string) {
+  constructor(api: ApiPromise, treasuryAccountUri: string, sudoAccountUri: string, miniSecret: string) {
     this.api = api
     this.keyring = new Keyring({ type: 'sr25519' })
     this.treasuryAccount = this.keyring.addFromUri(treasuryAccountUri).address
     this.keyring.addFromUri(sudoAccountUri)
+    this.miniSecret = miniSecret
+    this.addressesToKeyId = new Map()
+    this.keyId = 0
   }
 
   public getApi(label: string): Api {
-    return new Api(this.api, this.treasuryAccount, this.keyring, label)
+    return new Api(this, this.api, this.treasuryAccount, this.keyring, label)
   }
 
-  // public close(): void {
-  //   this.api.disconnect()
-  // }
+  public createKeyPairs(n: number): { key: KeyringPair; id: number }[] {
+    const keys: { key: KeyringPair; id: number }[] = []
+    for (let i = 0; i < n; i++) {
+      const id = this.keyId++
+      const uri = `${this.miniSecret}//testing/${id}`
+      const key = this.keyring.addFromUri(uri)
+      keys.push({ key, id })
+      this.addressesToKeyId.set(key.address, id)
+    }
+    return keys
+  }
 }
 
 export class Api {
+  private readonly factory: ApiFactory
   private readonly api: ApiPromise
   private readonly sender: Sender
-  private readonly keyring: Keyring
   // source of funds for all new accounts
   private readonly treasuryAccount: string
 
-  constructor(api: ApiPromise, treasuryAccount: string, keyring: Keyring, label: string) {
+  constructor(factory: ApiFactory, api: ApiPromise, treasuryAccount: string, keyring: Keyring, label: string) {
+    this.factory = factory
     this.api = api
-    this.keyring = keyring
     this.treasuryAccount = treasuryAccount
     this.sender = new Sender(api, keyring, label)
   }
@@ -113,12 +133,12 @@ export class Api {
     this.sender.setLogLevel(LogLevel.Verbose)
   }
 
-  public createKeyPairs(n: number): KeyringPair[] {
-    const nKeyPairs: KeyringPair[] = []
-    for (let i = 0; i < n; i++) {
-      nKeyPairs.push(this.keyring.addFromUri(i + uuid().substring(0, 8)))
-    }
-    return nKeyPairs
+  public createKeyPairs(n: number): { key: KeyringPair; id: number }[] {
+    return this.factory.createKeyPairs(n)
+  }
+
+  public getAccountToKeyIdMappings() {
+    return this.factory.addressesToKeyId
   }
 
   // Well known WorkingGroup enum defined in runtime

tests/network-tests/src/Scenario.ts

@@ -24,11 +24,12 @@ export async function scenario(scene: (props: ScenarioProps) => Promise<void>):
   // Connect api to the chain
   const nodeUrl: string = env.NODE_URL || 'ws://127.0.0.1:9944'
   const provider = new WsProvider(nodeUrl)
-
+  const miniSecret = process.env.SURI_MINI_SECRET || ''
   const apiFactory = await ApiFactory.create(
     provider,
     env.TREASURY_ACCOUNT_URI || '//Alice',
-    env.SUDO_ACCOUNT_URI || '//Alice'
+    env.SUDO_ACCOUNT_URI || '//Alice',
+    miniSecret
   )
 
   const queryNodeUrl: string = env.QUERY_NODE_URL || 'http://127.0.0.1:8081/graphql'

tests/network-tests/src/fixtures/membershipModule.ts

@@ -33,6 +33,8 @@ export class BuyMembershipHappyCaseFixture extends BaseFixture {
 
     this.api.treasuryTransferBalanceToAccounts(this.accounts, membershipTransactionFee.add(new BN(membershipFee)))
 
+    // Note: Member alias is dervied from the account so if it is not unique the member registration
+    // will fail with HandleAlreadyRegistered error
     this.memberIds = (
       await Promise.all(
         this.accounts.map((account) =>
@@ -46,6 +48,7 @@ export class BuyMembershipHappyCaseFixture extends BaseFixture {
     this.debug(`Registered ${this.memberIds.length} new members`)
 
     assert.equal(this.memberIds.length, this.accounts.length)
+    // log the member id and corresponding key id
   }
 }
 

tests/network-tests/src/fixtures/proposalsModule.ts

@@ -551,7 +551,7 @@ export class SpendingProposalFixture extends BaseFixture {
 
     await this.api.sudoSetCouncilMintCapacity(this.mintCapacity)
 
-    const fundingRecipient = this.api.createKeyPairs(1)[0].address
+    const fundingRecipient = this.api.createKeyPairs(1)[0].key.address
 
     // Proposal creation
     const result = await this.api.proposeSpending(

tests/network-tests/src/fixtures/workingGroupModule.ts

@@ -485,7 +485,7 @@ export class UpdateRewardAccountFixture extends BaseFixture {
     this.api.treasuryTransferBalance(workerRoleAccount, updateRewardAccountFee)
 
     // Update reward account
-    const createdAccount: KeyringPair = this.api.createKeyPairs(1)[0]
+    const createdAccount: KeyringPair = this.api.createKeyPairs(1)[0].key
     await this.api.updateRewardAccount(workerRoleAccount, this.workerId, createdAccount.address, this.module)
     const newRewardAccount: string = await this.api.getWorkerRewardAccount(this.workerId, this.module)
     assert(
@@ -514,7 +514,7 @@ export class UpdateRoleAccountFixture extends BaseFixture {
     this.api.treasuryTransferBalance(workerRoleAccount, updateRoleAccountFee)
 
     // Update role account
-    const createdAccount: KeyringPair = this.api.createKeyPairs(1)[0]
+    const createdAccount: KeyringPair = this.api.createKeyPairs(1)[0].key
     await this.api.updateRoleAccount(workerRoleAccount, this.workerId, createdAccount.address, this.module)
     const newRoleAccount: string = (await this.api.getWorkerById(this.workerId, this.module)).role_account_id.toString()
     assert(

tests/network-tests/src/flows/council/setup.ts

@@ -23,8 +23,8 @@ export default async function councilSetup({ api, env, lock }: FlowProps): Promi
   debug('Electing new council')
 
   const numberOfApplicants = (await api.getCouncilSize()).toNumber() * 2
-  const applicants = api.createKeyPairs(numberOfApplicants).map((key) => key.address)
-  const voters = api.createKeyPairs(5).map((key) => key.address)
+  const applicants = api.createKeyPairs(numberOfApplicants).map(({ key }) => key.address)
+  const voters = api.createKeyPairs(5).map(({ key }) => key.address)
 
   const paidTerms: PaidTermId = api.createPaidTermId(new BN(+env.MEMBERSHIP_PAID_TERMS!))
   const K: number = +env.COUNCIL_ELECTION_K!

tests/network-tests/src/flows/membership/creatingMemberships.ts

@@ -15,8 +15,8 @@ export default async function membershipCreation({ api, env }: FlowProps): Promi
 
   const N: number = +env.MEMBERSHIP_CREATION_N!
   assert(N > 0)
-  const nAccounts = api.createKeyPairs(N).map((key) => key.address)
-  const aAccount = api.createKeyPairs(1)[0].address
+  const nAccounts = api.createKeyPairs(N).map(({ key }) => key.address)
+  const aAccount = api.createKeyPairs(1)[0].key.address
   const paidTerms: PaidTermId = api.createPaidTermId(new BN(+env.MEMBERSHIP_PAID_TERMS!))
 
   // Assert membership can be bought if sufficient funds are available

tests/network-tests/src/flows/proposals/manageLeaderRole.ts

@@ -36,7 +36,7 @@ async function manageLeaderRole(api: Api, env: NodeJS.ProcessEnv, group: Working
   debug('Started')
   await lock(Resource.Proposals)
 
-  const leaderAccount = api.createKeyPairs(1)[0].address
+  const leaderAccount = api.createKeyPairs(1)[0].key.address
 
   const paidTerms: PaidTermId = api.createPaidTermId(new BN(+env.MEMBERSHIP_PAID_TERMS!))
   const applicationStake: BN = new BN(env.WORKING_GROUP_APPLICATION_STAKE!)

tests/network-tests/src/flows/proposals/updateRuntime.ts

@@ -28,7 +28,7 @@ export default async function updateRuntime({ api, env, lock }: FlowProps): Prom
   // Some tests after runtime update
   const createMembershipsFixture = new BuyMembershipHappyCaseFixture(
     api,
-    api.createKeyPairs(1).map((key) => key.address),
+    api.createKeyPairs(1).map(({ key }) => key.address),
     paidTerms
   )
   await new FixtureRunner(createMembershipsFixture).run()

tests/network-tests/src/flows/workingGroup/leaderSetup.ts

@@ -25,7 +25,7 @@ async function leaderSetup(api: Api, env: NodeJS.ProcessEnv, group: WorkingGroup
   const existingLead = await api.getGroupLead(group)
   assert.equal(existingLead, undefined, 'Lead is already set')
 
-  const leadKeyPair = api.createKeyPairs(1)[0]
+  const leadKeyPair = api.createKeyPairs(1)[0].key
   const paidTerms: PaidTermId = api.createPaidTermId(new BN(+env.MEMBERSHIP_PAID_TERMS!))
   const applicationStake: BN = new BN(env.WORKING_GROUP_APPLICATION_STAKE!)
   const roleStake: BN = new BN(env.WORKING_GROUP_ROLE_STAKE!)

tests/network-tests/src/flows/workingGroup/manageWorkerAsLead.ts

@@ -41,7 +41,7 @@ async function manageWorkerAsLead(api: Api, env: NodeJS.ProcessEnv, group: Worki
   const lead = await api.getGroupLead(group)
   assert(lead)
 
-  const applicants = api.createKeyPairs(5).map((key) => key.address)
+  const applicants = api.createKeyPairs(5).map(({ key }) => key.address)
   const memberSetFixture = new BuyMembershipHappyCaseFixture(api, applicants, paidTerms)
   await new FixtureRunner(memberSetFixture).run()
 

tests/network-tests/src/flows/workingGroup/manageWorkerAsWorker.ts

@@ -41,7 +41,7 @@ async function manageWorkerAsWorker(api: Api, env: NodeJS.ProcessEnv, group: Wor
   const lead = await api.getGroupLead(group)
   assert(lead)
 
-  const newMembers = api.createKeyPairs(1).map((key) => key.address)
+  const newMembers = api.createKeyPairs(1).map(({ key }) => key.address)
 
   const memberSetFixture = new BuyMembershipHappyCaseFixture(api, newMembers, paidTerms)
   // Recreating set of members

tests/network-tests/src/flows/workingGroup/workerPayout.ts

@@ -48,7 +48,7 @@ async function workerPayouts(api: Api, env: NodeJS.ProcessEnv, group: WorkingGro
   const lead = await api.getGroupLead(group)
   assert(lead)
 
-  const newMembers = api.createKeyPairs(5).map((key) => key.address)
+  const newMembers = api.createKeyPairs(5).map(({ key }) => key.address)
 
   const memberSetFixture = new BuyMembershipHappyCaseFixture(api, newMembers, paidTerms)
   // Recreating set of members

tests/network-tests/src/scenarios/setup-new-chain.ts

@@ -0,0 +1,30 @@
+import councilSetup from '../flows/council/setup'
+import leaderSetup from '../flows/workingGroup/leaderSetup'
+import mockContentFlow from '../sumer/mockContentFlow'
+
+import { scenario } from '../Scenario'
+
+scenario(async ({ job }) => {
+  const council = job('Create Council', councilSetup)
+
+  const leads = job('Setup WorkingGroup Leads', [leaderSetup.storage, leaderSetup.content])
+
+  // After content lead is created create some mock content in content directory
+  // Without uploading actual media
+  const mockContent = job('Create Mock Content', mockContentFlow).after(leads)
+
+  // Dump the account key ids that where generated in scenario so they can be re-derived at a later time
+  job('Dump accounts', async ({ api }) => {
+    const mappings = api.getAccountToKeyIdMappings()
+    console.log(mappings)
+
+    // TODO: get each account we are interested in knowing the keyid for..
+    // const api = apiFactory.getApi('get interesting accounts')
+    // Member accounts of council, lead, workers, and worker role accounts.
+    // let accounts = api.getInterestingAccounts()
+    // console.log(Api.addressesToKeyId.get(account))
+  })
+    .after(leads)
+    .after(council)
+    .after(mockContent)
+})

tests/network-tests/src/sumer/mockContentFlow.ts

@@ -0,0 +1,18 @@
+// import { assert } from 'chai'
+// import { ContentId } from '@joystream/types/storage'
+// import { registry } from '@joystream/types'
+
+import { FlowProps } from '../Flow'
+// import { Utils } from '../utils'
+import { extendDebug } from '../Debugger'
+
+export default async function mockContent({ api }: FlowProps): Promise<void> {
+  const debug = extendDebug('flow:createMockContent')
+  debug('Started')
+  // TODO: implement and use new fixtures:
+  // create categories with lead
+  // pick N member accounts
+  // create one channel per member
+  // upload V videos per channel (same contentid)
+  debug('Done')
+}

tests/network-tests/test-setup-new-chain.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+set -e
+
+SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+cd $SCRIPT_PATH
+
+# Location that will be mounted as the /data volume in containers
+# This is where the initial members and balances files and generated chainspec files will be located.
+DATA_PATH=${DATA_PATH:=~/tmp}
+mkdir -p ${DATA_PATH}
+
+# Initial account balance for sudo account
+# Will be the source of funds for all new accounts that are created in the tests.
+SUDO_INITIAL_BALANCE=${SUDO_INITIAL_BALANCE:=100000000}
+export SUDO_ACCOUNT_URI=${SUDO_ACCOUNT_URI:="//Alice"}
+SUDO_ACCOUNT=$(subkey inspect ${SUDO_ACCOUNT_URI} --output-type json | jq .ss58Address -r)
+export TREASURY_ACCOUNT_URI=${SUDO_ACCOUNT_URI}
+
+# The docker image tag to use for joystream/node
+RUNTIME=${RUNTIME:=latest}
+
+echo "{
+  \"balances\":[
+    [\"$SUDO_ACCOUNT\", $SUDO_INITIAL_BALANCE]
+  ]
+}" > ${DATA_PATH}/initial-balances.json
+
+# Make sudo a member as well - do we really need this ?
+# echo "
+#   [{
+#     \"member_id\":0,
+#     \"root_account\":\"$SUDO_ACCOUNT\",
+#     \"controller_account\":\"$SUDO_ACCOUNT\",
+#     \"handle\":\"sudosudo\",
+#     \"avatar_uri\":\"https://sudo.com/avatar.png\",
+#     \"about\":\"Sudo\",
+#     \"registered_at_time\":0
+#   }]
+# " > ${DATA_PATH}/initial-members.json
+
+echo "creating chainspec file"
+# Create a chain spec file
+docker run --rm -v ${DATA_PATH}:/data --entrypoint ./chain-spec-builder joystream/node:${RUNTIME} \
+  new \
+  --authority-seeds Alice \
+  --sudo-account ${SUDO_ACCOUNT} \
+  --deployment dev \
+  --chain-spec-path /data/chain-spec.json \
+  --initial-balances-path /data/initial-balances.json
+  # --initial-members-path /data/initial-members.json
+
+echo "converting chainspec to raw format"
+# Convert the chain spec file to a raw chainspec file
+docker run --rm -v ${DATA_PATH}:/data joystream/node:${RUNTIME} build-spec \
+  --raw --disable-default-bootnode \
+  --chain /data/chain-spec.json > ~/tmp/chain-spec-raw.json
+
+NETWORK_ARG=
+if [ "$ATTACH_TO_NETWORK" != "" ]; then
+  NETWORK_ARG="--network ${ATTACH_TO_NETWORK}"
+fi
+
+echo "starting joystream-node container"
+# Start a chain with generated chain spec
+# Add "-l ws=trace,ws::handler=info" to get websocket trace logs
+CONTAINER_ID=`docker run -d -v ${DATA_PATH}:/data -p 9944:9944 ${NETWORK_ARG} joystream/node:${RUNTIME} \
+  --validator --alice --unsafe-ws-external --rpc-cors=all -l runtime \
+  --chain /data/chain-spec-raw.json`
+
+function cleanup() {
+    docker logs ${CONTAINER_ID} --tail 15
+    docker stop ${CONTAINER_ID}
+    docker rm ${CONTAINER_ID}
+}
+
+trap cleanup EXIT
+
+# Display runtime version
+yarn workspace api-scripts tsnode-strict src/status.ts | grep Runtime
+
+# Init chain state
+echo 'executing scenario'
+./run-test-scenario.sh setup-new-chain