Merge branch 'tests-setup-new-chain' into giza-setup-new-chain-script

tests/network-tests/.env

@@ -56,3 +56,8 @@ STAKE_DECREMENT = 3
 MINT_CAPACITY_INCREMENT = 1000
 # Storage node address to download content from
 STORAGE_NODE_URL = http://localhost:3001/asset/v0
+# Mini-secret or mnemonic used in SURI for deterministic key derivation
+SURI_MINI_SECRET = ""
+# The starting key id to use when running a scenario. This will allow scenario
+# to be able to use all accounts generated in a prior scenario run against the same chain
+START_KEY_ID = 0

tests/network-tests/.gitignore

@@ -0,0 +1,2 @@
+output.json
+

tests/network-tests/src/Api.ts

@@ -29,9 +29,13 @@ import {
   OpeningId,
 } from '@joystream/types/hiring'
 import { FillOpeningParameters, ProposalId } from '@joystream/types/proposals'
-import { v4 as uuid } from 'uuid'
+// import { v4 as uuid } from 'uuid'
 import { extendDebug } from './Debugger'
 import { InvertedPromise } from './InvertedPromise'
+import { VideoId } from '@joystream/types/content'
+import { ChannelId } from '@joystream/types/common'
+import { ChannelCategoryMetadata, VideoCategoryMetadata } from '@joystream/metadata-protobuf'
+import { metadataToBytes } from '../../../cli/lib/helpers/serialization'
 
 export enum WorkingGroups {
   StorageWorkingGroup = 'storageWorkingGroup',
@@ -43,16 +47,29 @@ export enum WorkingGroups {
   OperationsWorkingGroupGamma = 'operationsWorkingGroupGamma',
 }
 
+type AnyMetadata = {
+  serializeBinary(): Uint8Array
+}
+
 export class ApiFactory {
   private readonly api: ApiPromise
   private readonly keyring: Keyring
+  // number used as part of key derivation path
+  private keyId = 0
+  // mapping from account address to key id.
+  // To be able to re-derive keypair externally when mini-secret is known.
+  readonly addressesToKeyId: Map<string, number> = new Map()
+  // mini secret used in SURI key derivation path
+  private readonly miniSecret: string
+
   // source of funds for all new accounts
   private readonly treasuryAccount: string
 
   public static async create(
     provider: WsProvider,
     treasuryAccountUri: string,
-    sudoAccountUri: string
+    sudoAccountUri: string,
+    miniSecret: string
   ): Promise<ApiFactory> {
     const debug = extendDebug('api-factory')
     let connectAttempts = 0
@@ -69,7 +86,7 @@ export class ApiFactory {
         // Give it a few seconds to be ready.
         await Utils.wait(5000)
 
-        return new ApiFactory(api, treasuryAccountUri, sudoAccountUri)
+        return new ApiFactory(api, treasuryAccountUri, sudoAccountUri, miniSecret)
       } catch (err) {
         if (connectAttempts === 3) {
           throw new Error('Unable to connect to chain')
@@ -79,32 +96,56 @@ export class ApiFactory {
     }
   }
 
-  constructor(api: ApiPromise, treasuryAccountUri: string, sudoAccountUri: string) {
+  constructor(api: ApiPromise, treasuryAccountUri: string, sudoAccountUri: string, miniSecret: string) {
     this.api = api
     this.keyring = new Keyring({ type: 'sr25519' })
     this.treasuryAccount = this.keyring.addFromUri(treasuryAccountUri).address
     this.keyring.addFromUri(sudoAccountUri)
+    this.miniSecret = miniSecret
+    this.addressesToKeyId = new Map()
+    this.keyId = 0
   }
 
   public getApi(label: string): Api {
-    return new Api(this.api, this.treasuryAccount, this.keyring, label)
+    return new Api(this, this.api, this.treasuryAccount, this.keyring, label)
+  }
+
+  public createKeyPairs(n: number): { key: KeyringPair; id: number }[] {
+    const keys: { key: KeyringPair; id: number }[] = []
+    for (let i = 0; i < n; i++) {
+      const id = this.keyId++
+      const uri = `${this.miniSecret}//testing//${id}`
+      const key = this.keyring.addFromUri(uri)
+      keys.push({ key, id })
+      this.addressesToKeyId.set(key.address, id)
+    }
+    return keys
+  }
+
+  public keyGenInfo(): { start: number; final: number } {
+    const start = 0
+    const final = this.keyId
+    return {
+      start,
+      final,
+    }
   }
 
-  // public close(): void {
-  //   this.api.disconnect()
-  // }
+  public getAllGeneratedAccounts(): { [k: string]: number } {
+    return Object.fromEntries(this.addressesToKeyId)
+  }
 }
 
 export class Api {
+  private readonly factory: ApiFactory
   private readonly api: ApiPromise
   private readonly sender: Sender
-  private readonly keyring: Keyring
   // source of funds for all new accounts
   private readonly treasuryAccount: string
 
-  constructor(api: ApiPromise, treasuryAccount: string, keyring: Keyring, label: string) {
+  constructor(factory: ApiFactory, api: ApiPromise, treasuryAccount: string, keyring: Keyring, label: string) {
+    this.factory = factory
     this.api = api
-    this.keyring = keyring
     this.treasuryAccount = treasuryAccount
     this.sender = new Sender(api, keyring, label)
   }
@@ -117,12 +158,16 @@ export class Api {
     this.sender.setLogLevel(LogLevel.Verbose)
   }
 
-  public createKeyPairs(n: number): KeyringPair[] {
-    const nKeyPairs: KeyringPair[] = []
-    for (let i = 0; i < n; i++) {
-      nKeyPairs.push(this.keyring.addFromUri(i + uuid().substring(0, 8)))
-    }
-    return nKeyPairs
+  public createKeyPairs(n: number): { key: KeyringPair; id: number }[] {
+    return this.factory.createKeyPairs(n)
+  }
+
+  public keyGenInfo(): { start: number; final: number } {
+    return this.factory.keyGenInfo()
+  }
+
+  public getAllgeneratedAccounts(): { [k: string]: number } {
+    return this.factory.getAllGeneratedAccounts()
   }
 
   // Well known WorkingGroup enum defined in runtime
@@ -1710,4 +1755,101 @@ export class Api {
   public getMaxWorkersCount(module: WorkingGroups): BN {
     return this.api.createType('u32', this.api.consts[module].maxWorkerNumberLimit)
   }
+
+  async getMemberControllerAccount(memberId: number): Promise<string | undefined> {
+    return (await this.api.query.members.membershipById(memberId))?.controller_account.toString()
+  }
+
+  async createMockChannel(memberId: number, memberControllerAccount?: string): Promise<ChannelId | null> {
+    memberControllerAccount = memberControllerAccount || (await this.getMemberControllerAccount(memberId))
+
+    if (!memberControllerAccount) {
+      throw new Error('invalid member id')
+    }
+
+    // Create a channel without any assets
+    const tx = this.api.tx.content.createChannel(
+      { Member: memberId },
+      {
+        assets: [],
+        meta: null,
+        reward_account: null,
+      }
+    )
+
+    const result = await this.sender.signAndSend(tx, memberControllerAccount)
+
+    const record = this.findEventRecord(result.events, 'content', 'ChannelCreated')
+    if (record) {
+      return record.event.data[1] as ChannelId
+    }
+
+    return null
+  }
+
+  async createMockVideo(
+    memberId: number,
+    channelId: number,
+    memberControllerAccount?: string
+  ): Promise<VideoId | null> {
+    memberControllerAccount = memberControllerAccount || (await this.getMemberControllerAccount(memberId))
+
+    if (!memberControllerAccount) {
+      throw new Error('invalid member id')
+    }
+
+    // Create a video without any assets
+    const tx = this.api.tx.content.createVideo({ Member: memberId }, channelId, {
+      assets: [],
+      meta: null,
+    })
+
+    const result = await this.sender.signAndSend(tx, memberControllerAccount)
+
+    const record = this.findEventRecord(result.events, 'content', 'VideoCreated')
+    if (record) {
+      return record.event.data[2] as VideoId
+    }
+
+    return null
+  }
+
+  async createChannelCategoryAsLead(name: string): Promise<ISubmittableResult> {
+    const lead = await this.getGroupLead(WorkingGroups.ContentWorkingGroup)
+
+    if (!lead) {
+      throw new Error('No Content Lead asigned, cannot create channel category')
+    }
+
+    const account = lead?.role_account_id
+    const meta = new ChannelCategoryMetadata({
+      name,
+    })
+
+    return this.sender.signAndSend(
+      this.api.tx.content.createChannelCategory(
+        { Lead: null },
+        { meta: metadataToBytes(ChannelCategoryMetadata, meta) }
+      ),
+      account?.toString()
+    )
+  }
+
+  async createVideoCategoryAsLead(name: string): Promise<ISubmittableResult> {
+    const lead = await this.getGroupLead(WorkingGroups.ContentWorkingGroup)
+
+    if (!lead) {
+      throw new Error('No Content Lead asigned, cannot create channel category')
+    }
+
+    const account = lead?.role_account_id
+    const meta = new VideoCategoryMetadata({
+      name,
+    })
+
+    return this.sender.signAndSend(
+      this.api.tx.content.createVideoCategory({ Lead: null }, { meta: metadataToBytes(VideoCategoryMetadata, meta) }),
+      account?.toString()
+    )
+  }
 }

tests/network-tests/src/Scenario.ts

@@ -9,6 +9,7 @@ import { Job } from './Job'
 import { JobManager } from './JobManager'
 import { ResourceManager } from './Resources'
 import fetch from 'cross-fetch'
+import fs from 'fs'
 
 export type ScenarioProps = {
   env: NodeJS.ProcessEnv
@@ -24,13 +25,22 @@ export async function scenario(scene: (props: ScenarioProps) => Promise<void>):
   // Connect api to the chain
   const nodeUrl: string = env.NODE_URL || 'ws://127.0.0.1:9944'
   const provider = new WsProvider(nodeUrl)
-
+  const miniSecret = env.SURI_MINI_SECRET || ''
   const apiFactory = await ApiFactory.create(
     provider,
     env.TREASURY_ACCOUNT_URI || '//Alice',
-    env.SUDO_ACCOUNT_URI || '//Alice'
+    env.SUDO_ACCOUNT_URI || '//Alice',
+    miniSecret
   )
 
+  const api = apiFactory.getApi('Key Generation')
+
+  // Generate all key ids before START_KEY_ID
+  const startKeyId = parseInt(env.START_KEY_ID || '0')
+  if (startKeyId) {
+    api.createKeyPairs(startKeyId)
+  }
+
   const queryNodeUrl: string = env.QUERY_NODE_URL || 'http://127.0.0.1:8081/graphql'
 
   const queryNodeProvider = new ApolloClient({
@@ -49,18 +59,34 @@ export async function scenario(scene: (props: ScenarioProps) => Promise<void>):
 
   const resources = new ResourceManager()
 
+  let exitCode = 0
+
   try {
     await jobs.run(resources)
   } catch (err) {
     console.error(err)
-    process.exit(-1)
+    exitCode = -1
   }
 
+  // account to key ids
+  const accounts = api.getAllgeneratedAccounts()
+
+  // first and last key id used to generate keys in this scenario
+  const keyIds = api.keyGenInfo()
+
+  const output = {
+    accounts,
+    keyIds,
+    miniSecret,
+  }
+
+  fs.writeFileSync('output.json', JSON.stringify(output, undefined, 2))
+
   // Note: disconnecting and then reconnecting to the chain in the same process
   // doesn't seem to work!
   // Disconnecting is causing error to be thrown:
   // RPC-CORE: getStorage(key: StorageKey, at?: BlockHash): StorageData:: disconnected from ws://127.0.0.1:9944: 1000:: Normal connection closure
   // Are there subsciptions somewhere?
   // apiFactory.close()
-  process.exit()
+  process.exit(exitCode)
 }

tests/network-tests/src/fixtures/membershipModule.ts

@@ -33,6 +33,8 @@ export class BuyMembershipHappyCaseFixture extends BaseFixture {
 
     this.api.treasuryTransferBalanceToAccounts(this.accounts, membershipTransactionFee.add(new BN(membershipFee)))
 
+    // Note: Member alias is dervied from the account so if it is not unique the member registration
+    // will fail with HandleAlreadyRegistered error
     this.memberIds = (
       await Promise.all(
         this.accounts.map((account) =>
@@ -46,6 +48,7 @@ export class BuyMembershipHappyCaseFixture extends BaseFixture {
     this.debug(`Registered ${this.memberIds.length} new members`)
 
     assert.equal(this.memberIds.length, this.accounts.length)
+    // log the member id and corresponding key id
   }
 }
 

tests/network-tests/src/fixtures/proposalsModule.ts

@@ -551,7 +551,7 @@ export class SpendingProposalFixture extends BaseFixture {
 
     await this.api.sudoSetCouncilMintCapacity(this.mintCapacity)
 
-    const fundingRecipient = this.api.createKeyPairs(1)[0].address
+    const fundingRecipient = this.api.createKeyPairs(1)[0].key.address
 
     // Proposal creation
     const result = await this.api.proposeSpending(

tests/network-tests/src/fixtures/workingGroupModule.ts

@@ -485,7 +485,7 @@ export class UpdateRewardAccountFixture extends BaseFixture {
     this.api.treasuryTransferBalance(workerRoleAccount, updateRewardAccountFee)
 
     // Update reward account
-    const createdAccount: KeyringPair = this.api.createKeyPairs(1)[0]
+    const createdAccount: KeyringPair = this.api.createKeyPairs(1)[0].key
     await this.api.updateRewardAccount(workerRoleAccount, this.workerId, createdAccount.address, this.module)
     const newRewardAccount: string = await this.api.getWorkerRewardAccount(this.workerId, this.module)
     assert(
@@ -514,7 +514,7 @@ export class UpdateRoleAccountFixture extends BaseFixture {
     this.api.treasuryTransferBalance(workerRoleAccount, updateRoleAccountFee)
 
     // Update role account
-    const createdAccount: KeyringPair = this.api.createKeyPairs(1)[0]
+    const createdAccount: KeyringPair = this.api.createKeyPairs(1)[0].key
     await this.api.updateRoleAccount(workerRoleAccount, this.workerId, createdAccount.address, this.module)
     const newRoleAccount: string = (await this.api.getWorkerById(this.workerId, this.module)).role_account_id.toString()
     assert(

tests/network-tests/src/flows/council/setup.ts

@@ -23,8 +23,8 @@ export default async function councilSetup({ api, env, lock }: FlowProps): Promi
   debug('Electing new council')
 
   const numberOfApplicants = (await api.getCouncilSize()).toNumber() * 2
-  const applicants = api.createKeyPairs(numberOfApplicants).map((key) => key.address)
-  const voters = api.createKeyPairs(5).map((key) => key.address)
+  const applicants = api.createKeyPairs(numberOfApplicants).map(({ key }) => key.address)
+  const voters = api.createKeyPairs(5).map(({ key }) => key.address)
 
   const paidTerms: PaidTermId = api.createPaidTermId(new BN(+env.MEMBERSHIP_PAID_TERMS!))
   const K: number = +env.COUNCIL_ELECTION_K!

tests/network-tests/src/flows/membership/creatingMemberships.ts

@@ -15,8 +15,8 @@ export default async function membershipCreation({ api, env }: FlowProps): Promi
 
   const N: number = +env.MEMBERSHIP_CREATION_N!
   assert(N > 0)
-  const nAccounts = api.createKeyPairs(N).map((key) => key.address)
-  const aAccount = api.createKeyPairs(1)[0].address
+  const nAccounts = api.createKeyPairs(N).map(({ key }) => key.address)
+  const aAccount = api.createKeyPairs(1)[0].key.address
   const paidTerms: PaidTermId = api.createPaidTermId(new BN(+env.MEMBERSHIP_PAID_TERMS!))
 
   // Assert membership can be bought if sufficient funds are available

tests/network-tests/src/flows/proposals/manageLeaderRole.ts

@@ -36,7 +36,7 @@ async function manageLeaderRole(api: Api, env: NodeJS.ProcessEnv, group: Working
   debug('Started')
   await lock(Resource.Proposals)
 
-  const leaderAccount = api.createKeyPairs(1)[0].address
+  const leaderAccount = api.createKeyPairs(1)[0].key.address
 
   const paidTerms: PaidTermId = api.createPaidTermId(new BN(+env.MEMBERSHIP_PAID_TERMS!))
   const applicationStake: BN = new BN(env.WORKING_GROUP_APPLICATION_STAKE!)

tests/network-tests/src/flows/proposals/updateRuntime.ts

@@ -28,7 +28,7 @@ export default async function updateRuntime({ api, env, lock }: FlowProps): Prom
   // Some tests after runtime update
   const createMembershipsFixture = new BuyMembershipHappyCaseFixture(
     api,
-    api.createKeyPairs(1).map((key) => key.address),
+    api.createKeyPairs(1).map(({ key }) => key.address),
     paidTerms
   )
   await new FixtureRunner(createMembershipsFixture).run()

tests/network-tests/src/flows/workingGroup/leaderSetup.ts

@@ -25,7 +25,7 @@ async function leaderSetup(api: Api, env: NodeJS.ProcessEnv, group: WorkingGroup
   const existingLead = await api.getGroupLead(group)
   assert.equal(existingLead, undefined, 'Lead is already set')
 
-  const leadKeyPair = api.createKeyPairs(1)[0]
+  const leadKeyPair = api.createKeyPairs(1)[0].key
   const paidTerms: PaidTermId = api.createPaidTermId(new BN(+env.MEMBERSHIP_PAID_TERMS!))
   const applicationStake: BN = new BN(env.WORKING_GROUP_APPLICATION_STAKE!)
   const roleStake: BN = new BN(env.WORKING_GROUP_ROLE_STAKE!)

tests/network-tests/src/flows/workingGroup/manageWorkerAsLead.ts

@@ -41,7 +41,7 @@ async function manageWorkerAsLead(api: Api, env: NodeJS.ProcessEnv, group: Worki
   const lead = await api.getGroupLead(group)
   assert(lead)
 
-  const applicants = api.createKeyPairs(5).map((key) => key.address)
+  const applicants = api.createKeyPairs(5).map(({ key }) => key.address)
   const memberSetFixture = new BuyMembershipHappyCaseFixture(api, applicants, paidTerms)
   await new FixtureRunner(memberSetFixture).run()
 

tests/network-tests/src/flows/workingGroup/manageWorkerAsWorker.ts

@@ -41,7 +41,7 @@ async function manageWorkerAsWorker(api: Api, env: NodeJS.ProcessEnv, group: Wor
   const lead = await api.getGroupLead(group)
   assert(lead)
 
-  const newMembers = api.createKeyPairs(1).map((key) => key.address)
+  const newMembers = api.createKeyPairs(1).map(({ key }) => key.address)
 
   const memberSetFixture = new BuyMembershipHappyCaseFixture(api, newMembers, paidTerms)
   // Recreating set of members

tests/network-tests/src/flows/workingGroup/workerPayout.ts

@@ -48,7 +48,7 @@ async function workerPayouts(api: Api, env: NodeJS.ProcessEnv, group: WorkingGro
   const lead = await api.getGroupLead(group)
   assert(lead)
 
-  const newMembers = api.createKeyPairs(5).map((key) => key.address)
+  const newMembers = api.createKeyPairs(5).map(({ key }) => key.address)
 
   const memberSetFixture = new BuyMembershipHappyCaseFixture(api, newMembers, paidTerms)
   // Recreating set of members

tests/network-tests/src/giza/createCategories.ts

@@ -0,0 +1,27 @@
+import { BaseFixture } from '../Fixture'
+
+export class CreateMockCategories extends BaseFixture {
+  public async execute(): Promise<void> {
+    const categories = [
+      'Film & Animation',
+      'Autos & Vehicles',
+      'Music',
+      'Pets & Animals',
+      'Sports',
+      'Travel & Events',
+      'Gaming',
+      'People & Blogs',
+      'Comedy',
+      'Entertainment',
+      'News & Politics',
+      'Howto & Style',
+      'Education',
+      'Science & Technology',
+      'Nonprofits & Activism',
+    ]
+
+    await Promise.all(categories.map((name) => this.api.createChannelCategoryAsLead(name)))
+
+    await Promise.all(categories.map((name) => this.api.createVideoCategoryAsLead(name)))
+  }
+}

tests/network-tests/src/giza/createChannelsAsMemberFixture.ts

@@ -0,0 +1,36 @@
+import { BaseFixture } from '../Fixture'
+import { Api } from '../Api'
+// import { MemberId } from '@joystream/types/members'
+import { ChannelId } from '@joystream/types/common'
+import { assert } from 'chai'
+
+export class CreateChannelsAsMemberFixture extends BaseFixture {
+  // Member that will be channel owner
+  private memberId: number
+  private numChannels: number
+  private createdChannels: (ChannelId | null)[] = []
+
+  constructor(api: Api, memberId: number, numChannels: number) {
+    super(api)
+    this.memberId = memberId
+    this.numChannels = numChannels
+  }
+
+  public getCreatedChannels(): (ChannelId | null)[] {
+    return this.createdChannels.slice()
+  }
+
+  public async execute(): Promise<void> {
+    const account = await this.api.getMemberControllerAccount(this.memberId)
+
+    const channels = []
+    for (let i = 0; i < this.numChannels; i++) {
+      channels.push(this.api.createMockChannel(this.memberId, account))
+    }
+
+    const channelIds = await Promise.all(channels)
+    this.createdChannels = channelIds.filter((id) => id !== null)
+
+    assert.equal(this.createdChannels.length, this.numChannels)
+  }
+}

tests/network-tests/src/giza/createVideosAsMemberFixture.ts

@@ -0,0 +1,32 @@
+import { BaseFixture } from '../Fixture'
+import { Api } from '../Api'
+// import { MemberId } from '@joystream/types/members'
+// import { ChannelId } from '@joystream/types/common'
+import { assert } from 'chai'
+
+export class CreateVideosAsMemberFixture extends BaseFixture {
+  // Member that will be channel owner
+  private memberId: number
+  private numVideos: number
+  private channelId: number
+
+  constructor(api: Api, memberId: number, channelId: number, numVideos: number) {
+    super(api)
+    this.memberId = memberId
+    this.numVideos = numVideos
+    this.channelId = channelId
+  }
+
+  public async execute(): Promise<void> {
+    const account = await this.api.getMemberControllerAccount(this.memberId)
+
+    const videos = []
+    for (let i = 0; i < this.numVideos; i++) {
+      videos.push(this.api.createMockVideo(this.memberId, this.channelId, account))
+    }
+
+    const videoIds = await Promise.all(videos)
+    const created = videoIds.filter((id) => id !== null).length
+    assert.equal(created, this.numVideos)
+  }
+}

tests/network-tests/src/giza/mockContentFlow.ts

@@ -0,0 +1,60 @@
+// import { assert } from 'chai'
+// import { registry } from '@joystream/types'
+import { CreateChannelsAsMemberFixture } from './createChannelsAsMemberFixture'
+import { CreateVideosAsMemberFixture } from './createVideosAsMemberFixture'
+import { BuyMembershipHappyCaseFixture } from '../fixtures/membershipModule'
+import { CreateMockCategories } from './createCategories'
+
+import { FlowProps } from '../Flow'
+import { FixtureRunner } from '../Fixture'
+import { extendDebug } from '../Debugger'
+import BN from 'bn.js'
+
+export default async function mockContent({ api }: FlowProps): Promise<void> {
+  const debug = extendDebug('flow:createMockContent')
+  debug('Started')
+
+  // create categories with lead
+  const createCategories = new CreateMockCategories(api)
+  debug('Creating Categories')
+  await new FixtureRunner(createCategories).run()
+
+  const memberAccount = api.createKeyPairs(1)[0].key.address
+  const createMember: BuyMembershipHappyCaseFixture = new BuyMembershipHappyCaseFixture(
+    api,
+    [memberAccount],
+    api.createPaidTermId(new BN(0))
+  )
+  await new FixtureRunner(createMember).run()
+
+  const memberId = createMember.getCreatedMembers()[0].toNumber()
+
+  // If we are too "aggressive" seeing
+  // 'ExtrinsicStatus:: 1010: Invalid Transaction: Transaction is outdated' errors
+  const numberOfChannelsPerRound = 100
+  const numberOfRoundsChannel = 5
+  const numberOfVideosPerRound = 100
+  const numberOfRoundsVideo = 100
+
+  const channelIds: number[] = []
+
+  // create mock channels
+  debug('Creating Channels')
+  for (let n = 0; n < numberOfRoundsChannel; n++) {
+    const createChannels = new CreateChannelsAsMemberFixture(api, memberId, numberOfChannelsPerRound)
+    await new FixtureRunner(createChannels).run()
+    createChannels.getCreatedChannels().forEach((id) => channelIds.push(id!.toNumber()))
+  }
+
+  // Create all videos in same channel
+  const channelId = channelIds[0]
+
+  // create mock videos
+  for (let n = 0; n < numberOfRoundsVideo; n++) {
+    debug('Creating Videos round', n)
+    const createVideos = new CreateVideosAsMemberFixture(api, memberId, channelId, numberOfVideosPerRound)
+    await new FixtureRunner(createVideos).run()
+  }
+
+  debug('Done')
+}

tests/network-tests/src/scenarios/setup-new-chain.ts

@@ -0,0 +1,14 @@
+import councilSetup from '../flows/council/setup'
+import leaderSetup from '../flows/workingGroup/leaderSetup'
+import mockContentFlow from '../giza/mockContentFlow'
+
+import { scenario } from '../Scenario'
+
+scenario(async ({ job }) => {
+  const council = job('Create Council', councilSetup)
+
+  const leads = job('Setup WorkingGroup Leads', [leaderSetup.storage, leaderSetup.content])
+
+  // Create some mock content in content directory - without assets or any real metadata
+  const mockContent = job('Create Mock Content', mockContentFlow).after(leads)
+})

tests/network-tests/src/sender.ts

@@ -17,7 +17,7 @@ export enum LogLevel {
 
 export class Sender {
   private readonly api: ApiPromise
-  private static readonly asyncLock: AsyncLock = new AsyncLock()
+  private static readonly asyncLock: AsyncLock = new AsyncLock({ maxPending: 2048 })
   private readonly keyring: Keyring
   private readonly debug: Debugger.Debugger
   private logs: LogLevel = LogLevel.None

tests/network-tests/test-setup-new-chain.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+set -e
+
+SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+cd $SCRIPT_PATH
+
+# Location that will be mounted as the /data volume in containers
+# This is where the initial members and balances files and generated chainspec files will be located.
+DATA_PATH=${DATA_PATH:=~/tmp}
+mkdir -p ${DATA_PATH}
+
+# Initial account balance for sudo account
+# Will be the source of funds for all new accounts that are created in the tests.
+SUDO_INITIAL_BALANCE=${SUDO_INITIAL_BALANCE:=100000000}
+export SUDO_ACCOUNT_URI=${SUDO_ACCOUNT_URI:="//Alice"}
+SUDO_ACCOUNT=$(subkey inspect ${SUDO_ACCOUNT_URI} --output-type json | jq .ss58Address -r)
+export TREASURY_ACCOUNT_URI=${SUDO_ACCOUNT_URI}
+
+# The docker image tag to use for joystream/node
+RUNTIME=${RUNTIME:=latest}
+
+echo "{
+  \"balances\":[
+    [\"$SUDO_ACCOUNT\", $SUDO_INITIAL_BALANCE]
+  ]
+}" > ${DATA_PATH}/initial-balances.json
+
+# Make sudo a member as well - do we really need this ?
+# echo "
+#   [{
+#     \"member_id\":0,
+#     \"root_account\":\"$SUDO_ACCOUNT\",
+#     \"controller_account\":\"$SUDO_ACCOUNT\",
+#     \"handle\":\"sudosudo\",
+#     \"avatar_uri\":\"https://sudo.com/avatar.png\",
+#     \"about\":\"Sudo\",
+#     \"registered_at_time\":0
+#   }]
+# " > ${DATA_PATH}/initial-members.json
+
+echo "creating chainspec file"
+# Create a chain spec file
+docker run --rm -v ${DATA_PATH}:/data --entrypoint ./chain-spec-builder joystream/node:${RUNTIME} \
+  new \
+  --authority-seeds Alice \
+  --sudo-account ${SUDO_ACCOUNT} \
+  --deployment dev \
+  --chain-spec-path /data/chain-spec.json \
+  --initial-balances-path /data/initial-balances.json
+  # --initial-members-path /data/initial-members.json
+
+echo "converting chainspec to raw format"
+# Convert the chain spec file to a raw chainspec file
+docker run --rm -v ${DATA_PATH}:/data joystream/node:${RUNTIME} build-spec \
+  --raw --disable-default-bootnode \
+  --chain /data/chain-spec.json > ~/tmp/chain-spec-raw.json
+
+NETWORK_ARG=
+if [ "$ATTACH_TO_NETWORK" != "" ]; then
+  NETWORK_ARG="--network ${ATTACH_TO_NETWORK}"
+fi
+
+echo "starting joystream-node container"
+# Start a chain with generated chain spec
+# Add "-l ws=trace,ws::handler=info" to get websocket trace logs
+CONTAINER_ID=`docker run -d -v ${DATA_PATH}:/data -p 9944:9944 ${NETWORK_ARG} --name joystream-node joystream/node:${RUNTIME} \
+  --validator --alice --unsafe-ws-external --rpc-cors=all -l runtime \
+  --chain /data/chain-spec-raw.json`
+
+function cleanup() {
+    docker logs ${CONTAINER_ID} --tail 15
+    docker stop ${CONTAINER_ID}
+    docker rm ${CONTAINER_ID}
+}
+
+trap cleanup EXIT
+
+# Display runtime version
+yarn workspace api-scripts tsnode-strict src/status.ts | grep Runtime
+
+# Init chain state
+echo 'executing scenario'
+./run-test-scenario.sh setup-new-chain