Merge branch 'query_node_active_video_counters_giza_staging' into vnft_schema_mappings_second

cli/src/Api.ts

@@ -1,5 +1,5 @@
 import BN from 'bn.js'
-import { createType, types } from '@joystream/types/'
+import { createType, types } from '@joystream/types'
 import { ApiPromise, WsProvider } from '@polkadot/api'
 import { AugmentedQuery, SubmittableExtrinsic } from '@polkadot/api/types'
 import { formatBalance } from '@polkadot/util'
@@ -208,7 +208,7 @@ export default class Api {
     return this._api.query[module]
   }
 
-  protected async membershipById(memberId: MemberId): Promise<Membership | null> {
+  async membershipById(memberId: MemberId): Promise<Membership | null> {
     const profile = await this._api.query.members.membershipById(memberId)
 
     // Can't just use profile.isEmpty because profile.suspended is Bool (which isEmpty method always returns false)
@@ -484,11 +484,6 @@ export default class Api {
     }
   }
 
-  async getMemberIdsByControllerAccount(address: string): Promise<MemberId[]> {
-    const ids = await this._api.query.members.memberIdsByControllerAccountId(address)
-    return ids.toArray()
-  }
-
   async workerExitRationaleConstraint(group: WorkingGroups): Promise<InputValidationLengthConstraint> {
     return await this.workingGroupApiQuery(group).workerExitRationaleText()
   }

cli/src/base/AccountsCommandBase.ts

@@ -355,7 +355,7 @@ export default abstract class AccountsCommandBase extends ApiCommandBase {
     return this.selectedMember
   }
 
-  async getKnownMembers(allowedIds?: MemberId[]): Promise<ISelectedMember[]> {
+  private async getKnownMembers(allowedIds?: MemberId[]): Promise<ISelectedMember[]> {
     const membersEntries = allowedIds
       ? await this.getApi().memberEntriesByIds(allowedIds)
       : await this.getApi().allMemberEntries()
@@ -382,6 +382,20 @@ export default abstract class AccountsCommandBase extends ApiCommandBase {
     return availableMemberships[memberIndex]
   }
 
+  private async initSelectedMember(): Promise<void> {
+    const memberIdString = this.getPreservedState().selectedMemberId
+
+    const memberId = this.createType('MemberId', memberIdString)
+    const member = await this.getApi().membershipById(memberId)
+
+    // ensure selected member exists
+    if (!member) {
+      return
+    }
+
+    this.selectedMember = [memberId, member]
+  }
+
   async init(): Promise<void> {
     await super.init()
     try {
@@ -391,8 +405,6 @@ export default abstract class AccountsCommandBase extends ApiCommandBase {
     }
     await this.initKeyring()
 
-    const availableMemberships = await this.getKnownMembers()
-    const memberId = this.getPreservedState().selectedMemberId
-    this.selectedMember = availableMemberships.find((item) => item[0].toString() === memberId) || undefined
+    await this.initSelectedMember()
   }
 }

cli/src/base/ApiCommandBase.ts

@@ -89,6 +89,9 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
       if (this.requiresQueryNode && !queryNodeUri) {
         this.warn('Query node endpoint uri is required in order to run this command!')
         queryNodeUri = await this.promptForQueryNodeUri(true)
+      } else if (queryNodeUri === undefined) {
+        this.warn("You haven't provided a Joystream query node uri for the CLI to connect to yet!")
+        queryNodeUri = await this.promptForQueryNodeUri()
       }
 
       const { metadataCache } = this.getPreservedState()

cli/src/commands/account/chooseMember.ts

@@ -3,38 +3,52 @@ import chalk from 'chalk'
 import { flags } from '@oclif/command'
 import ExitCodes from '../../ExitCodes'
 
-export default class AccountChoose extends AccountsCommandBase {
-  static description = 'Choose default account to use in the CLI'
+export default class AccountChooseMember extends AccountsCommandBase {
+  static description = 'Choose default member to use in the CLI'
   static flags = {
-    address: flags.string({
-      description: 'Select account by address (if available)',
-      char: 'a',
+    memberId: flags.string({
+      description: 'Select member (if available)',
+      char: 'm',
       required: false,
     }),
   }
 
   async run() {
-    const { address } = this.parse(AccountChoose).flags
+    const { memberId } = this.parse(AccountChooseMember).flags
 
-    const memberData = address
-      ? await this.selectKnownMember(address)
+    const memberData = memberId
+      ? await this.selectKnownMember(memberId)
       : await this.getRequiredMemberContext(undefined, false)
 
     await this.setSelectedMember(memberData)
 
-    this.log(chalk.greenBright(`\nAccount switched to ${chalk.magentaBright(address)} (MemberId: ${memberData[0]})!`))
+    this.log(
+      chalk.greenBright(
+        `\nMember to id ${chalk.magentaBright(
+          memberData[0]
+        )} (account: ${memberData[1].controller_account.toString()})!`
+      )
+    )
   }
 
-  async selectKnownMember(address: string): Promise<ISelectedMember> {
-    const knownMembersData = await this.getKnownMembers()
-    const memberData = knownMembersData.find(([, member]) => member.controller_account.toString() === address)
+  async selectKnownMember(memberIdString: string): Promise<ISelectedMember> {
+    const memberId = this.createType('MemberId', memberIdString)
+    const member = await this.getApi().membershipById(memberId)
 
-    if (!memberData) {
-      this.error(`Selected account address not found among known members!`, {
+    if (!member) {
+      this.error(`Selected member id not found among known members!`, {
         exit: ExitCodes.AccessDenied,
       })
     }
 
-    return memberData
+    const selectedMember = [memberId, member] as ISelectedMember
+
+    if (!this.isKeyAvailable(member.controller_account)) {
+      this.error(`Selected member's account is not imported to CLI!`, {
+        exit: ExitCodes.AccessDenied,
+      })
+    }
+
+    return selectedMember
   }
 }

query-node/mappings/content/utils.ts

@@ -527,7 +527,7 @@ export async function unsetAssetRelations(store: DatabaseManager, dataObject: St
     await store.save<Video>(video)
 
     // update video active counters
-    await updateVideoActiveCounters(store, wasFullyActive as IVideoActiveStatus, undefined)
+    await updateVideoActiveCounters(store, wasFullyActive, undefined)
 
     // emit log event
     logger.info('Content has been disconnected from Video', {
@@ -549,19 +549,8 @@ export interface IVideoActiveStatus {
 }
 
 export function getVideoActiveStatus(video: Video): IVideoActiveStatus {
-  const productionEnv = () => {
-    const isFullyActive =
-      !!video.isPublic && !video.isCensored && !!video.thumbnailPhoto?.isAccepted && !!video.media?.isAccepted
-
-    return isFullyActive
-  }
-  const testEnv = () => {
-    const isFullyActive = !!video.isPublic && !video.isCensored
-
-    return isFullyActive
-  }
-
-  const isFullyActive = process.env.QN_TEST_ENV ? testEnv() : productionEnv()
+  const isFullyActive =
+    !!video.isPublic && !video.isCensored && !!video.thumbnailPhoto?.isAccepted && !!video.media?.isAccepted
 
   const videoCategory = video.category
   const channel = video.channel
@@ -581,18 +570,13 @@ export async function updateVideoActiveCounters(
   initialActiveStatus: IVideoActiveStatus | null | undefined,
   activeStatus: IVideoActiveStatus | null | undefined
 ): Promise<void> {
-  // definition of generic type for Hydra DatabaseManager's methods
-  type EntityType<T> = {
-    new (...args: any[]): T
-  }
-
   async function updateSingleEntity<Entity extends VideoCategory | Channel>(
     entity: Entity,
     counterChange: number
   ): Promise<void> {
     entity.activeVideosCounter += counterChange
 
-    await store.save<EntityType<Entity>>(entity)
+    await store.save(entity)
   }
 
   async function reflectUpdate<Entity extends VideoCategory | Channel>(
@@ -624,16 +608,16 @@ export async function updateVideoActiveCounters(
 
     // didEntityChange === true
 
-    if (oldEntity) {
-      // if video was fully active before, prepare to decrease counter; increase counter otherwise
-      const counterChange = initFullyActive ? -1 : 1
+    if (oldEntity && initFullyActive) {
+      // if video was fully active before, prepare to decrease counter
+      const counterChange = -1
 
       await updateSingleEntity(oldEntity, counterChange)
     }
 
-    if (newEntity) {
-      // if video is fully active now, prepare to increase counter; decrease counter otherwise
-      const counterChange = nowFullyActive ? 1 : -1
+    if (newEntity && nowFullyActive) {
+      // if video is fully active now, prepare to increase counter
+      const counterChange = 1
 
       await updateSingleEntity(newEntity, counterChange)
     }

query-node/mappings/storage/index.ts

@@ -273,9 +273,7 @@ export async function storage_PendingDataObjectsAccepted({ event, store }: Event
     performance issues. In that case, a unit test for this mapping will be required.
   */
   // load relevant videos one by one and update related active-video-counters
-  await initialActiveStates.reduce(async (accPromise, initialActiveState) => {
-    await accPromise
-
+  for (const initialActiveState of initialActiveStates) {
     // load refreshed version of videos and related entities (channel, channel category, category)
 
     const video = (await store.get(Video, {
@@ -284,7 +282,7 @@ export async function storage_PendingDataObjectsAccepted({ event, store }: Event
     })) as Video
 
     await updateVideoActiveCounters(store, initialActiveState, getVideoActiveStatus(video))
-  }, Promise.resolve())
+  }
 }
 
 export async function storage_DataObjectsMoved({ event, store }: EventContext & StoreContext): Promise<void> {
@@ -311,19 +309,16 @@ export async function storage_DataObjectsDeleted({ event, store }: EventContext
   await Promise.all(
     dataObjects.map(async (dataObject) => {
       // remember if video is fully active before update
-      const initialVideoActiveStatusThumbnail = dataObject.videoThumbnail
-        ? getVideoActiveStatus(dataObject.videoThumbnail)
-        : null
-      const initialVideoActiveStatusMedia = dataObject.videoMedia ? getVideoActiveStatus(dataObject.videoMedia) : null
+      const initialVideoActiveStatus =
+        (dataObject.videoThumbnail && getVideoActiveStatus(dataObject.videoThumbnail)) ||
+        (dataObject.videoMedia && getVideoActiveStatus(dataObject.videoMedia)) ||
+        null
 
       await unsetAssetRelations(store, dataObject)
 
       // update video active counters
-      if (initialVideoActiveStatusThumbnail) {
-        await updateVideoActiveCounters(store, initialVideoActiveStatusThumbnail, undefined)
-      }
-      if (initialVideoActiveStatusMedia) {
-        await updateVideoActiveCounters(store, initialVideoActiveStatusMedia, undefined)
+      if (initialVideoActiveStatus) {
+        await updateVideoActiveCounters(store, initialVideoActiveStatus, undefined)
       }
     })
   )

tests/network-tests/.gitignore

@@ -1,5 +1,2 @@
 output.json
 data/
-src/__CliApi_tempfile.json
-src/__CliApi_tempfile__rejectedContent.json
-src/__CliApi_appdata

tests/network-tests/src/Api.ts

@@ -30,7 +30,6 @@ import {
   OpeningId,
 } from '@joystream/types/hiring'
 import { FillOpeningParameters, ProposalId } from '@joystream/types/proposals'
-// import { v4 as uuid } from 'uuid'
 import { extendDebug } from './Debugger'
 import { InvertedPromise } from './InvertedPromise'
 import { VideoId, VideoCategoryId } from '@joystream/types/content'
@@ -39,7 +38,6 @@ import { ChannelCategoryMetadata, VideoCategoryMetadata } from '@joystream/metad
 import { metadataToBytes } from '../../../cli/lib/helpers/serialization'
 import { assert } from 'chai'
 import { WorkingGroups } from './WorkingGroups'
-import { v4 as uuid } from 'uuid'
 
 const workingGroupNameByGroup: { [key in WorkingGroups]: string } = {
   'distributionWorkingGroup': 'Distribution',
@@ -133,25 +131,25 @@ export class ApiFactory {
     const keys: { key: KeyringPair; id: number }[] = []
     for (let i = 0; i < n; i++) {
       const id = this.keyId++
-      const key = this.createCustomKeyPair(`${id}`)
+      const key = this.createKeyPair(`${id}`)
       keys.push({ key, id })
       this.addressesToKeyId.set(key.address, id)
     }
     return keys
   }
 
-  private createKeyPair(suriPath: string, isCustom = false): KeyringPair {
+  private createKeyPair(suriPath: string, isCustom = false, isFinalPath = false): KeyringPair {
     if (isCustom) {
       this.customKeys.push(suriPath)
     }
-    const uri = `${this.miniSecret}//testing//${suriPath}/${uuid().substring(0, 8)}`
+    const uri = isFinalPath ? suriPath : `${this.miniSecret}//testing//${suriPath}`
     const pair = this.keyring.addFromUri(uri)
     this.addressesToSuri.set(pair.address, uri)
     return pair
   }
 
-  public createCustomKeyPair(customPath: string): KeyringPair {
-    return this.createKeyPair(customPath, true)
+  public createCustomKeyPair(customPath: string, isFinalPath: boolean): KeyringPair {
+    return this.createKeyPair(customPath, true, isFinalPath)
   }
 
   public keyGenInfo(): KeyGenInfo {
@@ -245,8 +243,8 @@ export class Api {
     return this.factory.createKeyPairs(n)
   }
 
-  public createCustomKeyPair(path: string): KeyringPair {
-    return this.factory.createCustomKeyPair(path)
+  public createCustomKeyPair(path: string, finalPath = false): KeyringPair {
+    return this.factory.createCustomKeyPair(path, finalPath)
   }
 
   public keyGenInfo(): KeyGenInfo {
@@ -945,14 +943,6 @@ export class Api {
     return (events.sort((a, b) => new BN(a.index).cmp(new BN(b.index))) as unknown) as EventType<S, M>[]
   }
 
-  public findStorageBucketCreated(events: EventRecord[]): DataObjectId | undefined {
-    const record = this.findEvent(events, 'storage', 'StorageBucketCreated')
-
-    if (record) {
-      return (record.data[0] as unknown) as DataObjectId
-    }
-  }
-
   // Subscribe to system events, resolves to an InvertedPromise or rejects if subscription fails.
   // The inverted promise wraps a promise which resolves when the Proposal with id specified
   // is executed.
@@ -1970,16 +1960,12 @@ export class Api {
     channelId: string,
     addStorageBuckets: StorageBucketId[]
   ) {
-    const bagId = { Dynamic: { Channel: channelId } }
-    const encodedStorageBucketIds = new BTreeSet<StorageBucketId>(
-      this.api.registry,
-      'StorageBucketId',
-      addStorageBuckets.map((item) => item.toString())
-    )
-    const noBucketsToRemove = new BTreeSet<StorageBucketId>(this.api.registry, 'StorageBucketId', [])
-
     return this.sender.signAndSend(
-      this.api.tx.storage.updateStorageBucketsForBag(bagId, encodedStorageBucketIds, noBucketsToRemove),
+      this.api.tx.storage.updateStorageBucketsForBag(
+        this.api.createType('BagId', { Dynamic: { Channel: channelId } }),
+        this.api.createType('BTreeSet<StorageBucketId>', [addStorageBuckets.map((item) => item.toString())]),
+        this.api.createType('BTreeSet<StorageBucketId>', [])
+      ),
       accountFrom
     )
   }

tests/network-tests/src/CliApi.ts

@@ -1,244 +0,0 @@
-import * as path from 'path'
-import { spawnSync } from 'child_process'
-import * as fs from 'fs'
-import { KeyringPair } from '@polkadot/keyring/types'
-import { v4 as uuid } from 'uuid'
-
-export interface ICreatedVideoData {
-  videoId: number
-  assetContentIds: string[]
-}
-
-/**
-  Adapter for calling CLI commands from integration tests.
-*/
-export class CliApi {
-  private tmpFilePath: string // filepath for temporary file needed to transfer data from and to cli
-  readonly cliExamplesFolderPath: string
-
-  public constructor() {
-    this.tmpFilePath = path.join(__dirname, '/__CliApi_tempfile.json')
-    this.cliExamplesFolderPath = path.dirname(require.resolve('@joystream/cli/package.json')) + '/examples/content'
-  }
-
-  /**
-    Runs CLI command with specified arguments.
-  */
-  private runCommand(
-    parameters: string[],
-    env: Record<string, string> = {}
-  ): { error: boolean; stdout: string; stderr: string } {
-    // use sync spawn if that works without issues
-    const output = spawnSync('yarn', ['joystream-cli', ...parameters], {
-      env: {
-        ...env,
-        PATH: process.env.PATH,
-        APPDATA: path.join(__dirname, '/__CliApi_appdata/'),
-      },
-    })
-
-    return {
-      error: !!output.error,
-      stdout: (output.stdout || '').toString(),
-      stderr: (output.stderr || '').toString(),
-    }
-  }
-
-  /**
-    Saves data to temporary file that can be passed to CLI as data input.
-  */
-  private saveToTempFile(content: string) {
-    try {
-      fs.writeFileSync(this.tmpFilePath, content + '\n')
-    } catch (e) {
-      throw new Error(`Can't write to temporary file "${this.tmpFilePath}"`)
-    }
-  }
-
-  /**
-    Parses `id` of newly created content entity from CLI's stdout.
-  */
-  private parseCreatedIdFromStdout(stdout: string): number {
-    return parseInt((stdout.match(/with id (\d+) successfully created/) as RegExpMatchArray)[1])
-  }
-
-  /**
-    Checks if CLI's stderr contains warning about no storage provider available.
-  */
-  private containsWarningNoStorage(text: string): boolean {
-    return !!text.match(/^\s*\S\s*Warning: No storage provider is currently available!/m)
-  }
-
-  /**
-    Checks if CLI's stderr contains warning about no password used when importing account.
-  */
-  private containsWarningEmptyPassword(text: string): boolean {
-    return !!text.match(/^\s*\S\s*Warning: Using empty password is not recommended!/)
-  }
-
-  /**
-    Setups API URI for the CLI.
-  */
-  async setApiUri(uri = 'ws://localhost:9944') {
-    const { stderr } = this.runCommand(['api:setUri', uri])
-
-    if (stderr) {
-      throw new Error(`Unexpected CLI failure on setting API URI: "${stderr}"`)
-    }
-  }
-
-  /**
-    Setups QN URI for the CLI.
-  */
-  async setQueryNodeUri(uri = 'http://localhost:8081/graphql') {
-    const { stderr } = this.runCommand(['api:setQueryNodeEndpoint', uri])
-
-    if (stderr) {
-      throw new Error(`Unexpected CLI failure on setting QN URI: "${stderr}"`)
-    }
-  }
-
-  /**
-    Imports an account from Polkadot's keyring keypair to CLI.
-  */
-  async importAccount(keyringPair: KeyringPair, password = ''): Promise<void> {
-    const importableAccount = JSON.stringify(keyringPair.toJson(password))
-    this.saveToTempFile(importableAccount)
-
-    const { stderr } = this.runCommand([
-      'account:import',
-      '--name',
-      (keyringPair.meta.name as string) || uuid().substring(0, 8),
-      '--password',
-      password,
-      '--backupFilePath',
-      this.tmpFilePath,
-    ])
-
-    if (stderr && !this.containsWarningEmptyPassword(stderr)) {
-      throw new Error(`Unexpected CLI failure on importing account: "${stderr}"`)
-    }
-  }
-
-  async chooseMemberAccount(accountAddress: string) {
-    const { stderr } = this.runCommand(['account:chooseMember', '--address', accountAddress])
-
-    if (stderr) {
-      throw new Error(`Unexpected CLI failure on choosing account: "${stderr}"`)
-    }
-  }
-
-  /**
-    Creates a new channel.
-  */
-  async createChannel(channel: unknown): Promise<number> {
-    this.saveToTempFile(JSON.stringify(channel))
-
-    const { stdout, stderr } = this.runCommand(
-      ['content:createChannel', '--input', this.tmpFilePath, '--context', 'Member'],
-      { AUTO_CONFIRM: 'true' }
-    )
-
-    if (stderr && !this.containsWarningNoStorage(stderr)) {
-      // ignore warnings
-      throw new Error(`Unexpected CLI failure on creating channel: "${stderr}"`)
-    }
-
-    return this.parseCreatedIdFromStdout(stdout)
-  }
-
-  /**
-    Creates a new channel category.
-  */
-  async createChannelCategory(channelCategory: unknown): Promise<number> {
-    this.saveToTempFile(JSON.stringify(channelCategory))
-
-    const { stdout, stderr } = this.runCommand(
-      ['content:createChannelCategory', '--input', this.tmpFilePath, '--context', 'Lead'],
-      { AUTO_CONFIRM: 'true' }
-    )
-
-    if (stderr) {
-      throw new Error(`Unexpected CLI failure on creating channel category: "${stderr}"`)
-    }
-
-    return this.parseCreatedIdFromStdout(stdout)
-  }
-
-  /**
-    Creates a new video.
-  */
-  async createVideo(channelId: number, video: unknown): Promise<ICreatedVideoData> {
-    this.saveToTempFile(JSON.stringify(video))
-
-    const { stdout, stderr } = this.runCommand(
-      ['content:createVideo', '--input', this.tmpFilePath, '--channelId', channelId.toString()],
-      { AUTO_CONFIRM: 'true' }
-    )
-
-    if (stderr && !this.containsWarningNoStorage(stderr)) {
-      // ignore warnings
-      throw new Error(`Unexpected CLI failure on creating video: "${stderr}"`)
-    }
-
-    const videoId = this.parseCreatedIdFromStdout(stdout)
-    const assetContentIds = Array.from(stdout.matchAll(/ objectId: '([a-z0-9]+)'/g)).map((item) => item[1])
-
-    return {
-      videoId,
-      assetContentIds,
-    }
-  }
-
-  /**
-    Creates a new video category.
-  */
-  async createVideoCategory(videoCategory: unknown): Promise<number> {
-    this.saveToTempFile(JSON.stringify(videoCategory))
-
-    const { stdout, stderr } = this.runCommand(
-      ['content:createVideoCategory', '--input', this.tmpFilePath, '--context', 'Lead'],
-      { AUTO_CONFIRM: 'true' }
-    )
-
-    if (stderr) {
-      throw new Error(`Unexpected CLI failure on creating video category: "${stderr}"`)
-    }
-
-    return this.parseCreatedIdFromStdout(stdout)
-  }
-
-  /**
-    Updates an existing video.
-  */
-  async updateVideo(videoId: number, video: unknown): Promise<void> {
-    this.saveToTempFile(JSON.stringify(video))
-
-    const { stdout, stderr } = this.runCommand(
-      ['content:updateVideo', '--input', this.tmpFilePath, videoId.toString()],
-      { AUTO_CONFIRM: 'true' }
-    )
-
-    if (stderr && !this.containsWarningNoStorage(stderr)) {
-      // ignore warnings
-      throw new Error(`Unexpected CLI failure on creating video category: "${stderr}"`)
-    }
-  }
-
-  /**
-    Updates a channel.
-  */
-  async updateChannel(channelId: number, channel: unknown): Promise<void> {
-    this.saveToTempFile(JSON.stringify(channel))
-
-    const { stdout, stderr } = this.runCommand(
-      ['content:updateChannel', '--input', this.tmpFilePath, channelId.toString()],
-      { AUTO_CONFIRM: 'true' }
-    )
-
-    if (stderr && !this.containsWarningNoStorage(stderr)) {
-      // ignore warnings
-      throw new Error(`Unexpected CLI failure on creating video category: "${stderr}"`)
-    }
-  }
-}

tests/network-tests/src/Fixture.ts

@@ -3,7 +3,6 @@ import { assert } from 'chai'
 import { ISubmittableResult } from '@polkadot/types/types/'
 import { DispatchResult } from '@polkadot/types/interfaces/system'
 import { QueryNodeApi } from './QueryNodeApi'
-import { CliApi } from './CliApi'
 
 export abstract class BaseFixture {
   protected readonly api: Api
@@ -87,12 +86,10 @@ export abstract class BaseFixture {
 
 export abstract class BaseQueryNodeFixture extends BaseFixture {
   protected readonly query: QueryNodeApi
-  protected readonly cli: CliApi
 
-  constructor(api: Api, query: QueryNodeApi, cli: CliApi) {
+  constructor(api: Api, query: QueryNodeApi) {
     super(api)
     this.query = query
-    this.cli = cli
   }
 }
 

tests/network-tests/src/Flow.ts

@@ -1,13 +1,11 @@
 import { Api } from './Api'
 import { QueryNodeApi } from './QueryNodeApi'
 import { ResourceLocker } from './Resources'
-import { CliApi } from './CliApi'
 
 export type FlowProps = {
   api: Api
   env: NodeJS.ProcessEnv
   query: QueryNodeApi
-  cli: CliApi
   lock: ResourceLocker
 }
 export type Flow = (args: FlowProps) => Promise<void>

tests/network-tests/src/Job.ts

@@ -5,13 +5,11 @@ import { QueryNodeApi } from './QueryNodeApi'
 import { Flow } from './Flow'
 import { InvertedPromise } from './InvertedPromise'
 import { ResourceManager } from './Resources'
-import { CliApi } from './CliApi'
 
 export type JobProps = {
   apiFactory: ApiFactory
   env: NodeJS.ProcessEnv
   query: QueryNodeApi
-  cli: CliApi
 }
 
 export enum JobOutcome {
@@ -106,7 +104,6 @@ export class Job {
             api: jobProps.apiFactory.getApi(`${this.label}:${flow.name}-${index}`),
             env: jobProps.env,
             query: jobProps.query,
-            cli: jobProps.cli,
             lock: locker.lock,
           })
         } catch (err) {

tests/network-tests/src/JobManager.ts

@@ -4,31 +4,18 @@ import { Job, JobOutcome, JobProps } from './Job'
 import { ApiFactory } from './Api'
 import { QueryNodeApi } from './QueryNodeApi'
 import { ResourceManager } from './Resources'
-import { CliApi } from './CliApi'
 
 export class JobManager extends EventEmitter {
   private _jobs: Job[] = []
   private readonly _apiFactory: ApiFactory
   private readonly _env: NodeJS.ProcessEnv
   private readonly _query: QueryNodeApi
-  private readonly _cli: CliApi
 
-  constructor({
-    apiFactory,
-    env,
-    query,
-    cli,
-  }: {
-    apiFactory: ApiFactory
-    env: NodeJS.ProcessEnv
-    query: QueryNodeApi
-    cli: CliApi
-  }) {
+  constructor({ apiFactory, env, query }: { apiFactory: ApiFactory; env: NodeJS.ProcessEnv; query: QueryNodeApi }) {
     super()
     this._apiFactory = apiFactory
     this._env = env
     this._query = query
-    this._cli = cli
   }
 
   public createJob(label: string, flows: Flow[] | Flow): Job {
@@ -45,7 +32,6 @@ export class JobManager extends EventEmitter {
       env: this._env,
       query: this._query,
       apiFactory: this._apiFactory,
-      cli: this._cli,
     }
   }
 

tests/network-tests/src/QueryNodeApi.ts

@@ -1,3 +1,6 @@
+import { BLOCKTIME } from './consts'
+import { gql, ApolloClient, ApolloQueryResult, DocumentNode, NormalizedCacheObject } from '@apollo/client/core'
+import { extendDebug, Debugger } from './Debugger'
 import {
   StorageDataObjectFieldsFragment,
   GetDataObjectsByIdsQuery,
@@ -10,9 +13,6 @@ import {
 } from './graphql/generated/queries'
 import { Maybe } from './graphql/generated/schema'
 import { OperationDefinitionNode } from 'graphql'
-import { gql, ApolloClient, ApolloQueryResult, DocumentNode, NormalizedCacheObject } from '@apollo/client'
-import { BLOCKTIME } from './consts'
-import { extendDebug, Debugger } from './Debugger'
 import { Utils } from './utils'
 
 export class QueryNodeApi {

tests/network-tests/src/Scenario.ts

@@ -10,7 +10,6 @@ import { JobManager } from './JobManager'
 import { ResourceManager } from './Resources'
 import fetch from 'cross-fetch'
 import fs from 'fs'
-import { CliApi } from './CliApi'
 
 export type ScenarioProps = {
   env: NodeJS.ProcessEnv
@@ -43,7 +42,7 @@ function writeOutput(api: Api, miniSecret: string) {
   fs.writeFileSync(OUTPUT_FILE_PATH, JSON.stringify(output, undefined, 2))
 }
 
-export async function scenario(scene: (props: ScenarioProps) => Promise<void>): Promise<void> {
+export async function scenario(label: string, scene: (props: ScenarioProps) => Promise<void>): Promise<void> {
   // Load env variables
   config()
   const env = process.env
@@ -86,11 +85,11 @@ export async function scenario(scene: (props: ScenarioProps) => Promise<void>):
 
   const query = new QueryNodeApi(queryNodeProvider)
 
-  const cli = new CliApi()
-
   const debug = extendDebug('scenario')
 
-  const jobs = new JobManager({ apiFactory, query, env, cli })
+  debug(label)
+
+  const jobs = new JobManager({ apiFactory, query, env })
 
   await scene({ env, debug, job: jobs.createJob.bind(jobs) })
 

tests/network-tests/src/cli/base.ts

@@ -1,9 +1,14 @@
 import path from 'path'
-import { execFile } from 'child_process'
+import { execFile, ChildProcess, PromiseWithChild, ExecFileException, ExecException } from 'child_process'
 import { promisify } from 'util'
 import { Sender } from '../sender'
 
-export type CommandResult = { stdout: string; stderr: string; out: string }
+export type CommandResult = {
+  exitCode: number
+  stdout: string
+  stderr: string
+  out: string
+}
 
 export abstract class CLI {
   protected env: Record<string, string>
@@ -38,17 +43,53 @@ export abstract class CLI {
     return nextArg
   }
 
-  async run(command: string, customArgs: string[] = [], lockKeys: string[] = []): Promise<CommandResult> {
+  async run(
+    command: string,
+    customArgs: string[] = [],
+    lockKeys: string[] = [],
+    requireSuccess = true
+  ): Promise<CommandResult> {
+    const defaultError = 1
+
     const pExecFile = promisify(execFile)
     const { env } = this
-    const { stdout, stderr } = await Sender.asyncLock.acquire(
+    const { stdout, stderr, exitCode } = await Sender.asyncLock.acquire(
       lockKeys.map((k) => `nonce-${k}`),
-      () =>
-        pExecFile(this.binPath, [command, ...this.getArgs(customArgs)], {
-          env,
-          cwd: this.rootPath,
-        })
+
+      async () => {
+        try {
+          // execute command and wait for std outputs (or error)
+          const execOutputs = await pExecFile(this.binPath, [command, ...this.getArgs(customArgs)], {
+            env,
+            cwd: this.rootPath,
+          })
+
+          // return outputs and exit code
+          return {
+            ...execOutputs,
+            exitCode: 0,
+          }
+        } catch (error: unknown) {
+          const errorTyped = error as ExecFileException & { stdout: string; stderr: string }
+          // escape if command's success is required
+          if (requireSuccess) {
+            throw error
+          }
+
+          return {
+            exitCode: errorTyped.code || defaultError,
+            stdout: errorTyped.stdout || '',
+            stderr: errorTyped.stderr || '',
+          }
+        }
+      }
     )
-    return { stdout, stderr, out: stdout.trim() }
+
+    return {
+      exitCode,
+      stdout,
+      stderr,
+      out: stdout.trim(),
+    }
   }
 }

tests/network-tests/src/cli/joystream.ts

@@ -3,9 +3,15 @@ import path from 'path'
 import { CLI, CommandResult } from './base'
 import { TmpFileManager } from './utils'
 import { ChannelInputParameters } from '@joystream/cli/src/Types'
+import { MemberId } from '@joystream/types/members'
 
 const CLI_ROOT_PATH = path.resolve(__dirname, '../../../../cli')
 
+export interface ICreatedVideoData {
+  videoId: number
+  assetContentIds: string[]
+}
+
 export class JoystreamCLI extends CLI {
   protected keys: string[] = []
   protected tmpFileManager: TmpFileManager
@@ -14,16 +20,24 @@ export class JoystreamCLI extends CLI {
     const defaultEnv = {
       HOME: tmpFileManager.tmpDataDir,
     }
+
     super(CLI_ROOT_PATH, defaultEnv)
     this.tmpFileManager = tmpFileManager
   }
 
+  /**
+    Inits all required connections, etc.
+  */
   async init(): Promise<void> {
     await this.run('api:setUri', [process.env.NODE_URL || 'ws://127.0.0.1:9944'])
     await this.run('api:setQueryNodeEndpoint', [process.env.QUERY_NODE_URL || 'http://127.0.0.1:8081/graphql'])
   }
 
-  async importKey(pair: KeyringPair): Promise<void> {
+  /**
+    Imports accounts key to CLI.
+  */
+  async importAccount(pair: KeyringPair): Promise<void> {
+    const password = ''
     const jsonFile = this.tmpFileManager.jsonFile(pair.toJson())
     await this.run('account:import', [
       '--backupFilePath',
@@ -31,17 +45,171 @@ export class JoystreamCLI extends CLI {
       '--name',
       `Account${this.keys.length}`,
       '--password',
-      '',
+      password,
     ])
     this.keys.push(pair.address)
   }
 
-  async run(command: string, customArgs: string[] = [], keyLocks?: string[]): Promise<CommandResult> {
-    return super.run(command, customArgs, keyLocks || this.keys)
+  /**
+    Runs Joystream CLI command.
+  */
+  async run(
+    command: string,
+    customArgs: string[] = [],
+    keyLocks?: string[],
+    requireSuccess = true
+  ): Promise<CommandResult> {
+    return super.run(command, customArgs, keyLocks || this.keys, requireSuccess)
+  }
+
+  /**
+    Getter for temporary-file manager.
+  */
+  public getTmpFileManager(): TmpFileManager {
+    return this.tmpFileManager
+  }
+
+  /**
+    Parses `id` of newly created content entity from CLI's stdout.
+  */
+  private parseCreatedIdFromStdout(stdout: string): number {
+    return parseInt((stdout.match(/with id (\d+) successfully created/) as RegExpMatchArray)[1])
   }
 
-  async createChannel(inputData: ChannelInputParameters, args: string[]): Promise<CommandResult> {
-    const jsonFile = this.tmpFileManager.jsonFile(inputData)
-    return this.run('content:createChannel', ['--input', jsonFile, ...args])
+  /**
+    Checks if CLI's stderr contains warning about no storage provider available.
+  */
+  private containsWarningNoStorage(stderr: string): boolean {
+    return !!stderr.match(/^\s*\S\s*Warning: No storage provider is currently available!/m)
+  }
+
+  /**
+    Checks if CLI's stderr contains warning about no password used when importing account.
+  */
+  private containsWarningEmptyPassword(text: string): boolean {
+    return !!text.match(/^\s*\S\s*Warning: Using empty password is not recommended!/)
+  }
+
+  /**
+    Selects active member for CLI commands.
+  */
+  async chooseMemberAccount(memberId: MemberId) {
+    const { stderr } = await this.run('account:chooseMember', ['--memberId', memberId.toString()])
+
+    if (stderr) {
+      throw new Error(`Unexpected CLI failure on choosing account: "${stderr}"`)
+    }
+  }
+
+  /**
+    Creates a new channel.
+  */
+  async createChannel(channel: unknown): Promise<number> {
+    const jsonFile = this.tmpFileManager.jsonFile(channel)
+
+    const { stdout, stderr, exitCode } = await this.run('content:createChannel', [
+      '--input',
+      jsonFile,
+      '--context',
+      'Member',
+    ])
+
+    if (exitCode && !this.containsWarningNoStorage(stderr)) {
+      // ignore warnings
+      throw new Error(`Unexpected CLI failure on creating channel: "${stderr}"`)
+    }
+
+    return this.parseCreatedIdFromStdout(stdout)
+  }
+
+  /**
+    Creates a new channel category.
+  */
+  async createChannelCategory(channelCategory: unknown): Promise<number> {
+    const jsonFile = this.tmpFileManager.jsonFile(channelCategory)
+
+    const { stdout, stderr } = await this.run('content:createChannelCategory', [
+      '--input',
+      jsonFile,
+      '--context',
+      'Lead',
+    ])
+
+    if (stderr) {
+      throw new Error(`Unexpected CLI failure on creating channel category: "${stderr}"`)
+    }
+
+    return this.parseCreatedIdFromStdout(stdout)
+  }
+
+  /**
+    Creates a new video.
+  */
+  async createVideo(channelId: number, video: unknown, canOmitUpload = true): Promise<ICreatedVideoData> {
+    const jsonFile = this.tmpFileManager.jsonFile(video)
+
+    const { stdout, stderr, exitCode } = await this.run(
+      'content:createVideo',
+      ['--input', jsonFile, '--channelId', channelId.toString()],
+      undefined,
+      !canOmitUpload
+    )
+
+    // prevent error from CLI that create
+    if (canOmitUpload && exitCode && !this.containsWarningNoStorage(stderr)) {
+      // ignore warnings
+      throw new Error(`Unexpected CLI failure on creating video: "${stderr}"`)
+    }
+
+    const videoId = this.parseCreatedIdFromStdout(stdout)
+    const assetContentIds = Array.from(stdout.matchAll(/ objectId: '([a-z0-9]+)'/g)).map((item) => item[1])
+
+    return {
+      videoId,
+      assetContentIds,
+    }
+  }
+
+  /**
+    Creates a new video category.
+  */
+  async createVideoCategory(videoCategory: unknown): Promise<number> {
+    const jsonFile = this.tmpFileManager.jsonFile(videoCategory)
+
+    const { stdout, stderr } = await this.run('content:createVideoCategory', ['--input', jsonFile, '--context', 'Lead'])
+
+    if (stderr) {
+      throw new Error(`Unexpected CLI failure on creating video category: "${stderr}"`)
+    }
+
+    return this.parseCreatedIdFromStdout(stdout)
+  }
+
+  /**
+    Updates an existing video.
+  */
+  async updateVideo(videoId: number, video: unknown): Promise<void> {
+    const jsonFile = this.tmpFileManager.jsonFile(video)
+
+    const { stdout, stderr } = await this.run('content:updateVideo', ['--input', jsonFile, videoId.toString()])
+
+    if (stderr && !this.containsWarningNoStorage(stderr)) {
+      // ignore warnings
+      throw new Error(`Unexpected CLI failure on creating video category: "${stderr}"`)
+    }
+  }
+
+  /**
+    Updates a channel.
+  */
+  async updateChannel(channelId: number, channel: unknown): Promise<void> {
+    const jsonFile = this.tmpFileManager.jsonFile(channel)
+
+    const { stdout, stderr } = await this.run('content:updateChannel', ['--input', jsonFile, channelId.toString()])
+
+    if (stderr && !this.containsWarningNoStorage(stderr)) {
+      // ignore warnings
+      throw new Error(`Unexpected CLI failure on creating video category: "${stderr}"`)
+    }
   }
 }

tests/network-tests/src/consts.ts

@@ -1,2 +1,2 @@
 // Test chain blocktime
-export const BLOCKTIME = 1000
+export const BLOCKTIME = 6000

tests/network-tests/src/fixtures/content/activeVideoCounters.ts

@@ -7,13 +7,11 @@ import { BuyMembershipHappyCaseFixture } from '../membershipModule'
 import { KeyringPair } from '@polkadot/keyring/types'
 import { Bytes } from '@polkadot/types'
 import { QueryNodeApi } from '../../QueryNodeApi'
-import { CliApi, ICreatedVideoData } from '../../CliApi'
 import { PaidTermId, MemberId } from '@joystream/types/members'
 import { Debugger, extendDebug } from '../../Debugger'
 import BN from 'bn.js'
-import { addWorkerToGroup } from './addWorkerToGroup'
 import { Worker, WorkerId } from '@joystream/types/working-group'
-import { DataObjectId } from '@joystream/types/storage'
+
 import {
   getMemberDefaults,
   getChannelCategoryDefaults,
@@ -21,179 +19,84 @@ import {
   getVideoDefaults,
   getVideoCategoryDefaults,
 } from './contentTemplates'
-
-interface IMember {
-  keyringPair: KeyringPair
-  account: string
-  memberId: MemberId
-}
-
-// QN connection paramaters
-const qnConnection = {
-  numberOfRepeats: 20, // QN can take some time to catch up with node - repeat until then
-  repeatDelay: 3000, // delay between failed QN requests
-}
-
-// settings
-const contentDirectoryWorkingGroupId = 1 // TODO: retrieve group id programmatically
-const sufficientTopupAmount = new BN(1000000) // some very big number to cover fees of all transactions
+import { JoystreamCLI, ICreatedVideoData } from '../../cli/joystream'
+import * as path from 'path'
 
 /**
   Fixture that test Joystream content can be created, is reflected in query node,
   and channel and categories counts their active videos properly.
+
+  Assuming all videos start in channel, video category, and channel category respectively
+  `channelIds[0]`, `channelCategoryIds[0]`, and `videoCategoryIds[0]`.
 */
 export class ActiveVideoCountersFixture extends BaseQueryNodeFixture {
-  private paidTerms: PaidTermId
   private debug: Debugger.Debugger
-  private env: NodeJS.ProcessEnv
-
-  constructor(api: Api, query: QueryNodeApi, cli: CliApi, env: NodeJS.ProcessEnv, paidTerms: PaidTermId) {
-    super(api, query, cli)
-    this.paidTerms = paidTerms
-    this.env = env
+  private cli: JoystreamCLI
+  private channelIds: number[]
+  private videosData: ICreatedVideoData[]
+  private channelCategoryIds: number[]
+  private videoCategoryIds: number[]
+
+  constructor(
+    api: Api,
+    query: QueryNodeApi,
+    cli: JoystreamCLI,
+    channelIds: number[],
+    videosData: ICreatedVideoData[],
+    channelCategoryIds: number[],
+    videoCategoryIds: number[]
+  ) {
+    super(api, query)
+    this.cli = cli
+    this.channelIds = channelIds
+    this.videosData = videosData
+    this.channelCategoryIds = channelCategoryIds
+    this.videoCategoryIds = videoCategoryIds
     this.debug = extendDebug('fixture:ActiveVideoCountersFixture')
   }
 
-  // this could be used by other modules in some shared fixture or whatnot; membership creation is common to many flows
-  private async createMembers(numberOfMembers: number): Promise<IMember[]> {
-    const keyringPairs = (await this.api.createKeyPairs(numberOfMembers)).map((kp) => kp.key)
-    const accounts = keyringPairs.map((item) => item.address)
-    const buyMembershipsFixture = new BuyMembershipHappyCaseFixture(this.api, accounts, this.paidTerms)
-
-    await new FixtureRunner(buyMembershipsFixture).run()
-
-    const memberIds = buyMembershipsFixture.getCreatedMembers()
-
-    return keyringPairs.map((item, index) => ({
-      keyringPair: item,
-      account: accounts[index],
-      memberId: memberIds[index],
-    }))
-  }
-
-  /*
-    Topup a bunch of accounts by specified amount.
-  */
-  private async topupAddresses(accounts: string[], amount: BN) {
-    await this.api.treasuryTransferBalanceToAccounts(accounts, amount)
-  }
-
   /*
     Execute this Fixture.
   */
   public async execute(): Promise<void> {
-    const videoCount = 2
-    const videoCategoryCount = 2
-    const channelCount = 2
-    const channelCategoryCount = 2
-
-    // prepare accounts for group leads, storage worker, and content author
-
-    this.debug('Loading working group leaders')
-    const { contentLeader, storageLeader } = await this.retrieveWorkingGroupLeaders()
-
-    // prepare memberships
-    this.debug('Creating members')
-    const members = await this.createMembers(1)
-
-    const authorMemberIndex = 0
-    const author = members[authorMemberIndex]
-    author.keyringPair.setMeta({
-      ...author.keyringPair.meta,
-      ...getMemberDefaults(authorMemberIndex),
-    })
-    const storageGroupWorker = author
-
-    this.debug('Top-uping accounts')
-    await this.topupAddresses(
-      [
-        ...members.map((item) => item.keyringPair.address),
-        contentLeader.role_account_id.toString(),
-        storageLeader.role_account_id.toString(),
-      ],
-      sufficientTopupAmount
-    )
-
-    // switch to lead and create category structure as lead
-
-    this.debug(`Choosing content working group lead's account`)
-    // this expects lead account to be already imported into CLI
-    await this.cli.chooseMemberAccount(contentLeader.role_account_id.toString())
-
-    this.debug('Creating channel categories')
-    const channelCategoryIds = await this.createChannelCategories(channelCategoryCount)
-
-    this.debug('Creating video categories')
-    const videoCategoryIds = await this.createVideoCategories(videoCategoryCount)
-
-    // switch to authors account
-
-    this.debug(`Importing author's account`)
-    await this.cli.importAccount(author.keyringPair)
-    await this.cli.chooseMemberAccount(author.keyringPair.address)
-
-    // create content entities
-
-    this.debug('Creating channels')
-    const channelIds = await this.createChannels(channelCount, channelCategoryIds[0], author.account)
-
-    this.debug('Creating videos')
-    const videosData = await this.createVideos(videoCount, channelIds[0], videoCategoryIds[0])
-
-    // add `storageGroupWorker` to storage group, storage bucket and accept all storage content
-    const { workerId: storageGroupWorkerId, storageBucketId } = await this.prepareAssetStorage(
-      storageLeader,
-      storageGroupWorker
-    )
-
-    this.debug('Adding storage bag to bucket')
-    await this.api.updateStorageBucketsForBag(storageLeader.role_account_id.toString(), channelIds[0].toString(), [
-      storageBucketId,
-    ])
-
-    this.debug('Accepting content to storage bag')
-    const allAssetIds = videosData.map((item) => item.assetContentIds).flat()
-    await this.api.acceptPendingDataObjects(
-      storageGroupWorker.keyringPair.address,
-      storageGroupWorkerId,
-      storageBucketId,
-      channelIds[0].toString(),
-      allAssetIds
-    )
+    const videoCount = this.videosData.length
+    const videoCategoryCount = this.videoCategoryIds.length
+    const channelCount = this.channelIds.length
+    const channelCategoryCount = this.channelCategoryIds.length
 
     // check channel and categories con are counted as active
 
     this.debug('Checking channels active video counters')
-    await this.assertCounterMatch('channels', channelIds[0], videoCount)
+    await this.assertCounterMatch('channels', this.channelIds[0], videoCount)
 
     this.debug('Checking channel categories active video counters')
-    await this.assertCounterMatch('channelCategories', channelCategoryIds[0], videoCount)
+    await this.assertCounterMatch('channelCategories', this.channelCategoryIds[0], videoCount)
 
     this.debug('Checking video categories active video counters')
-    await this.assertCounterMatch('videoCategories', videoCategoryIds[0], videoCount)
+    await this.assertCounterMatch('videoCategories', this.videoCategoryIds[0], videoCount)
 
     // move channel to different channel category and video to different videoCategory
 
-    const oneMovedVideoCount = 1
+    const oneMovedItemCount = 1
     this.debug('Move channel to different channel category')
-    await this.cli.updateChannel(channelIds[0], {
-      category: channelCategoryIds[1], // move from category 1 to category 2
+    await this.cli.updateChannel(this.channelIds[0], {
+      category: this.channelCategoryIds[1], // move from category 1 to category 2
     })
 
     this.debug('Move video to different video category')
-    await this.cli.updateVideo(videosData[0].videoId, {
-      category: videoCategoryIds[1], // move from category 1 to category 2
+    await this.cli.updateVideo(this.videosData[0].videoId, {
+      category: this.videoCategoryIds[1], // move from category 1 to category 2
     })
 
     // check counters of channel category and video category with newly moved in video/channel
 
     this.debug('Checking channel categories active video counters (2)')
-    await this.assertCounterMatch('channelCategories', channelCategoryIds[1], videoCount)
+    await this.assertCounterMatch('channelCategories', this.channelCategoryIds[1], videoCount)
 
     this.debug('Checking video categories active video counters (2)')
-    await this.assertCounterMatch('videoCategories', videoCategoryIds[1], oneMovedVideoCount)
+    await this.assertCounterMatch('videoCategories', this.videoCategoryIds[1], oneMovedItemCount)
 
-    /** Giza doesn't support changing channels - uncoment this on later releases where it's \\\
+    /** Giza doesn't support changing channels - uncomment this on later releases where it's supported
 
     // move one video to another channel
 
@@ -205,8 +108,8 @@ export class ActiveVideoCountersFixture extends BaseQueryNodeFixture {
     // check counter of channel with newly moved video
 
     this.debug('Checking channels active video counters (2)')
-    await this.assertCounterMatch('channels', channelIds[0], videoCount - oneMovedVideoCount)
-    await this.assertCounterMatch('channels', channelIds[1], oneMovedVideoCount)
+    await this.assertCounterMatch('channels', channelIds[0], videoCount - oneMovedItemCount)
+    await this.assertCounterMatch('channels', channelIds[1], oneMovedItemCount)
 
     // end
     */
@@ -214,51 +117,6 @@ export class ActiveVideoCountersFixture extends BaseQueryNodeFixture {
     this.debug('Done')
   }
 
-  /**
-    Prepares storage requisites.
-  */
-  private async prepareAssetStorage(storageLeader: Worker, storageGroupWorker: IMember) {
-    const noLimit = 10000000 // virtually no limit
-    const bucketSettings = {
-      sizeLimit: noLimit,
-      objectsLimit: noLimit,
-    }
-    const storageBucketsPerBag = 10 // something in boundaries of StorageBucketsPerBagValueConstraint (see runtime)
-
-    this.debug('Setting up storage buckets per bag limit')
-    await this.api.updateStorageBucketsPerBagLimit(storageLeader.role_account_id.toString(), storageBucketsPerBag)
-
-    this.debug('Setting up storage buckets voucher limits')
-    await this.api.updateStorageBucketsVoucherMaxLimits(
-      storageLeader.role_account_id.toString(),
-      bucketSettings.sizeLimit,
-      bucketSettings.objectsLimit
-    )
-
-    this.debug('Adding worker to content directory group')
-    const workerId = await addWorkerToGroup(
-      this.api,
-      this.env,
-      WorkingGroups.Storage,
-      storageGroupWorker.keyringPair.address
-    )
-
-    this.debug('Creating storage bucket')
-    const createBucketResult = await this.api.createStorageBucket(
-      storageLeader.role_account_id.toString(),
-      bucketSettings.sizeLimit,
-      bucketSettings.objectsLimit,
-      workerId
-    )
-
-    const storageBucketId = this.api.findEvent(createBucketResult.events, 'storage', 'StorageBucketCreated')?.data[0] as DataObjectId
-
-    this.debug('Accepting storage bucket invitation')
-    await this.api.acceptStorageBucketInvitation(storageGroupWorker.keyringPair.address, workerId, storageBucketId)
-
-    return { workerId, storageBucketId }
-  }
-
   /**
     Asserts a channel, or a video/channel categories have their active videos counter set properly
     in Query node.
@@ -268,11 +126,12 @@ export class ActiveVideoCountersFixture extends BaseQueryNodeFixture {
     entityId: number,
     expectedCount: number
   ) {
-    const qnConnectionNumberOfRepeats = 10
-
-    const getterName = `get${entityName[0].toUpperCase()}${entityName.slice(1)}`
+    const getterName = `get${entityName[0].toUpperCase()}${entityName.slice(1)}` as
+      | 'getChannels'
+      | 'getChannelCategories'
+      | 'getVideoCategories'
     await this.query.tryQueryWithTimeout(
-      () => (this.query as any)[getterName](),
+      () => this.query[getterName](),
       (tmpEntity) => {
         const entities = (tmpEntity as any).data[entityName]
         assert(entities.length > 0) // some entities were loaded
@@ -281,98 +140,7 @@ export class ActiveVideoCountersFixture extends BaseQueryNodeFixture {
 
         // all videos created in this fixture should be active and belong to first entity
         assert(entity.activeVideosCounter === expectedCount)
-      },
-      qnConnection.repeatDelay,
-      qnConnection.numberOfRepeats
-    )
-  }
-
-  /**
-    Retrieves information about accounts of group leads for content and storage working groups.
-  */
-  private async retrieveWorkingGroupLeaders(): Promise<{ contentLeader: Worker; storageLeader: Worker }> {
-    const retrieveGroupLeader = async (group: WorkingGroups) => {
-      const leader = await this.api.getGroupLead(group)
-      if (!leader) {
-        throw new Error(`Working group leader for "${group}" is missing!`)
       }
-      return leader
-    }
-
-    return {
-      contentLeader: await retrieveGroupLeader(WorkingGroups.Content),
-      storageLeader: await retrieveGroupLeader(WorkingGroups.Storage),
-    }
-  }
-
-  /**
-    Creates a new video.
-
-    Note: Assets have to be accepted later on for videos to be counted as active.
-  */
-  private async createVideos(count: number, channelId: number, videoCategoryId: number): Promise<ICreatedVideoData[]> {
-    const createVideo = async (index: number) => {
-      return await this.cli.createVideo(channelId, {
-        ...getVideoDefaults(index, this.cli.cliExamplesFolderPath),
-        category: videoCategoryId,
-      })
-    }
-    const newVideosData = (await this.createCommonEntities(count, createVideo)) as ICreatedVideoData[]
-
-    return newVideosData
-  }
-
-  /**
-    Creates a new video category. Can only be executed as content group leader.
-  */
-  private async createVideoCategories(count: number): Promise<number[]> {
-    const createdIds = (await this.createCommonEntities(count, (index) =>
-      this.cli.createVideoCategory({
-        ...getVideoCategoryDefaults(index),
-      })
-    )) as number[]
-
-    return createdIds
-  }
-
-  /**
-    Creates a new channel.
-  */
-  private async createChannels(count: number, channelCategoryId: number, authorAddress: string): Promise<number[]> {
-    const createdIds = (await this.createCommonEntities(count, (index) =>
-      this.cli.createChannel({
-        ...getChannelDefaults(index, authorAddress),
-        category: channelCategoryId,
-      })
-    )) as number[]
-
-    return createdIds
-  }
-
-  /**
-    Creates a new channel category. Can only be executed as content group leader.
-  */
-  private async createChannelCategories(count: number): Promise<number[]> {
-    const createdIds = (await this.createCommonEntities(count, (index) =>
-      this.cli.createChannelCategory({
-        ...getChannelCategoryDefaults(index),
-      })
-    )) as number[]
-
-    return createdIds
-  }
-
-  /**
-    Creates a bunch of content entities.
-  */
-  private async createCommonEntities<T>(count: number, createPromise: (index: number) => Promise<T>): Promise<T[]> {
-    const createdIds = await Array.from(Array(count).keys()).reduce(async (accPromise, index: number) => {
-      const acc = await accPromise
-      const createdId = await createPromise(index)
-
-      return [...acc, createdId]
-    }, Promise.resolve([]) as Promise<T[]>)
-
-    return createdIds
+    )
   }
 }

tests/network-tests/src/fixtures/content/addWorkerToGroup.ts

@@ -1,77 +0,0 @@
-// this file could be used by more fixtures - adding worker to group is quite common
-
-import {
-  AddWorkerOpeningFixture,
-  ApplyForOpeningFixture,
-  BeginApplicationReviewFixture,
-  FillOpeningFixture,
-  IncreaseStakeFixture,
-  UpdateRewardAccountFixture,
-} from '../../fixtures/workingGroupModule'
-import { FixtureRunner } from '../../Fixture'
-import { OpeningId } from '@joystream/types/hiring'
-import { Api } from '../../Api'
-import { WorkingGroups } from '../../WorkingGroups'
-import BN from 'bn.js'
-import { WorkerId } from '@joystream/types/working-group'
-
-export async function addWorkerToGroup(
-  api: Api,
-  env: NodeJS.ProcessEnv,
-  group: WorkingGroups,
-  applicant: string
-): Promise<WorkerId> {
-  const applicationStake: BN = new BN(env.WORKING_GROUP_APPLICATION_STAKE!)
-  const roleStake: BN = new BN(env.WORKING_GROUP_ROLE_STAKE!)
-  const firstRewardInterval: BN = new BN(env.LONG_REWARD_INTERVAL!)
-  const rewardInterval: BN = new BN(env.LONG_REWARD_INTERVAL!)
-  const payoutAmount: BN = new BN(env.PAYOUT_AMOUNT!)
-  const unstakingPeriod: BN = new BN(env.STORAGE_WORKING_GROUP_UNSTAKING_PERIOD!)
-  const openingActivationDelay: BN = new BN(0)
-  const paidTerms = api.createPaidTermId(new BN(+env.MEMBERSHIP_PAID_TERMS!))
-
-  const addWorkerOpeningFixture = new AddWorkerOpeningFixture(
-    api,
-    applicationStake,
-    roleStake,
-    openingActivationDelay,
-    unstakingPeriod,
-    group
-  )
-  // Add worker opening
-  await new FixtureRunner(addWorkerOpeningFixture).run()
-
-  // First apply for worker opening
-  const applyForWorkerOpeningFixture = new ApplyForOpeningFixture(
-    api,
-    [applicant],
-    applicationStake,
-    roleStake,
-    addWorkerOpeningFixture.getCreatedOpeningId() as OpeningId,
-    group
-  )
-  await new FixtureRunner(applyForWorkerOpeningFixture).run()
-  const applicationIdToHire = applyForWorkerOpeningFixture.getApplicationIds()[0]
-
-  // Begin application review
-  const beginApplicationReviewFixture = new BeginApplicationReviewFixture(
-    api,
-    addWorkerOpeningFixture.getCreatedOpeningId() as OpeningId,
-    group
-  )
-  await new FixtureRunner(beginApplicationReviewFixture).run()
-
-  // Fill worker opening
-  const fillOpeningFixture = new FillOpeningFixture(
-    api,
-    [applicationIdToHire],
-    addWorkerOpeningFixture.getCreatedOpeningId() as OpeningId,
-    firstRewardInterval,
-    rewardInterval,
-    payoutAmount,
-    group
-  )
-  await new FixtureRunner(fillOpeningFixture).run()
-
-  return fillOpeningFixture.getWorkerIds()[0]
-}

tests/network-tests/src/fixtures/content/createChannelsAndVideos.ts

@@ -0,0 +1,222 @@
+import { BaseQueryNodeFixture, FixtureRunner } from '../../Fixture'
+import { Debugger, extendDebug } from '../../Debugger'
+import { JoystreamCLI, ICreatedVideoData } from '../../cli/joystream'
+import { PaidTermId, MemberId } from '@joystream/types/members'
+import { QueryNodeApi } from '../../QueryNodeApi'
+import { Api } from '../../Api'
+import * as path from 'path'
+import { getMemberDefaults, getVideoDefaults, getChannelDefaults } from './contentTemplates'
+import { KeyringPair } from '@polkadot/keyring/types'
+import { BuyMembershipHappyCaseFixture } from '../membershipModule'
+import BN from 'bn.js'
+import { DataObjectId, StorageBucketId } from '@joystream/types/storage'
+import { Worker, WorkerId } from '@joystream/types/working-group'
+import { createType } from '@joystream/types'
+import { singleBucketConfig } from '../../flows/storagev2/initStorage'
+
+interface IMember {
+  keyringPair: KeyringPair
+  account: string
+  memberId: MemberId
+}
+
+// settings
+const sufficientTopupAmount = new BN(1000000) // some very big number to cover fees of all transactions
+
+const cliExamplesFolderPath = path.dirname(require.resolve('@joystream/cli/package.json')) + '/examples/content'
+
+export class CreateChannelsAndVideosFixture extends BaseQueryNodeFixture {
+  private paidTerms: PaidTermId
+  private debug: Debugger.Debugger
+  private cli: JoystreamCLI
+  private channelCount: number
+  private videoCount: number
+  private channelCategoryId: number
+  private videoCategoryId: number
+  private createdItems: {
+    channelIds: number[]
+    videosData: ICreatedVideoData[]
+  }
+
+  constructor(
+    api: Api,
+    query: QueryNodeApi,
+    cli: JoystreamCLI,
+    paidTerms: PaidTermId,
+    channelCount: number,
+    videoCount: number,
+    channelCategoryId: number,
+    videoCategoryId: number
+  ) {
+    super(api, query)
+    this.paidTerms = paidTerms
+    this.cli = cli
+    this.channelCount = channelCount
+    this.videoCount = videoCount
+    this.channelCategoryId = channelCategoryId
+    this.videoCategoryId = videoCategoryId
+    this.debug = extendDebug('fixture:CreateChannelsAndVideosFixture')
+
+    this.createdItems = {
+      channelIds: [],
+      videosData: [],
+    }
+  }
+
+  public getCreatedItems() {
+    return this.createdItems
+  }
+
+  /*
+    Execute this Fixture.
+  */
+  public async execute(): Promise<void> {
+    this.debug('Creating members')
+    const author = await this.prepareAuthor()
+
+    this.debug('Creating channels')
+    this.createdItems.channelIds = await this.createChannels(this.channelCount, this.channelCategoryId, author.account)
+
+    this.debug('Creating videos')
+    this.createdItems.videosData = await this.createVideos(
+      this.videoCount,
+      this.createdItems.channelIds[0],
+      this.videoCategoryId
+    )
+
+    const { storageBucketId, storageGroupWorkerId, storageGroupWorkerAccount } = await this.retrieveBucket()
+
+    this.debug('Accepting content to storage bag')
+    const allAssetIds = this.createdItems.videosData.map((item) => item.assetContentIds).flat()
+    await this.api.acceptPendingDataObjects(
+      // storageGroupWorker.keyringPair.address,
+      storageGroupWorkerAccount,
+      storageGroupWorkerId,
+      storageBucketId,
+      this.createdItems.channelIds[0].toString(),
+      allAssetIds
+    )
+  }
+
+  /**
+    Prepares author for the content to be created.
+  */
+  private async prepareAuthor(): Promise<IMember> {
+    // prepare memberships
+    const members = await this.createMembers(1)
+
+    const authorMemberIndex = 0
+    const author = members[authorMemberIndex]
+    author.keyringPair.setMeta({
+      ...author.keyringPair.meta,
+      ...getMemberDefaults(authorMemberIndex),
+    })
+
+    this.debug('Top-uping accounts')
+    await this.api.treasuryTransferBalanceToAccounts([author.keyringPair.address], sufficientTopupAmount)
+
+    return author
+  }
+
+  /**
+    Creates new accounts and registers memberships for them.
+  */
+  private async createMembers(numberOfMembers: number): Promise<IMember[]> {
+    const keyringPairs = (await this.api.createKeyPairs(numberOfMembers)).map((kp) => kp.key)
+    const accounts = keyringPairs.map((item) => item.address)
+    const buyMembershipsFixture = new BuyMembershipHappyCaseFixture(this.api, accounts, this.paidTerms)
+
+    await new FixtureRunner(buyMembershipsFixture).run()
+
+    const memberIds = buyMembershipsFixture.getCreatedMembers()
+
+    return keyringPairs.map((item, index) => ({
+      keyringPair: item,
+      account: accounts[index],
+      memberId: memberIds[index],
+    }))
+  }
+
+  /**
+    Retrieves storage bucket info.
+  */
+  private async retrieveBucket(): Promise<{
+    storageBucketId: StorageBucketId
+    storageGroupWorkerId: WorkerId
+    storageGroupWorkerAccount: string
+  }> {
+    // read existing storage buckets from runtime
+    const bucketEntries = await this.api.query.storage.storageBucketById.entries()
+
+    // create WorkerId object
+    const storageGroupWorkerId = createType('WorkerId', singleBucketConfig.buckets[0].operatorId)
+
+    // find some bucket created by worker
+    const bucketTuple = bucketEntries.find(
+      ([, /* storageBucketId */ storageBucket]) =>
+        (storageBucket.operator_status as any).isStorageWorker &&
+        (storageBucket.operator_status as any).asStorageWorker[0].toString() === storageGroupWorkerId.toString()
+    )
+    if (!bucketTuple) {
+      throw new Error('Storage bucket not initialized')
+    }
+
+    // retrieve worker address
+    const storageGroupWorkerAccount = this.api.createCustomKeyPair(singleBucketConfig.buckets[0].transactorUri, true)
+      .address
+
+    // create StorageBucketId object
+    const storageBucketId = createType('StorageBucketId', bucketTuple[0].args[0])
+
+    return {
+      storageBucketId,
+      storageGroupWorkerId,
+      storageGroupWorkerAccount,
+    }
+  }
+
+  /**
+    Creates a new channel.
+  */
+  private async createChannels(count: number, channelCategoryId: number, authorAddress: string): Promise<number[]> {
+    const createdIds = (await this.createCommonEntities(count, (index) =>
+      this.cli.createChannel({
+        ...getChannelDefaults(index, authorAddress),
+        category: channelCategoryId,
+      })
+    )) as number[]
+
+    return createdIds
+  }
+
+  /**
+    Creates a new video.
+
+    Note: Assets have to be accepted later on for videos to be counted as active.
+  */
+  private async createVideos(count: number, channelId: number, videoCategoryId: number): Promise<ICreatedVideoData[]> {
+    const createVideo = async (index: number) => {
+      return await this.cli.createVideo(channelId, {
+        ...getVideoDefaults(index, cliExamplesFolderPath),
+        category: videoCategoryId,
+      })
+    }
+    const newVideosData = (await this.createCommonEntities(count, createVideo)) as ICreatedVideoData[]
+
+    return newVideosData
+  }
+
+  /**
+    Creates a bunch of content entities.
+  */
+  private async createCommonEntities<T>(count: number, createPromise: (index: number) => Promise<T>): Promise<T[]> {
+    const createdIds = await Array.from(Array(count).keys()).reduce(async (accPromise, index: number) => {
+      const acc = await accPromise
+      const createdId = await createPromise(index)
+
+      return [...acc, createdId]
+    }, Promise.resolve([]) as Promise<T[]>)
+
+    return createdIds
+  }
+}

tests/network-tests/src/fixtures/content/createContentStructure.ts

@@ -0,0 +1,125 @@
+import { BaseQueryNodeFixture, FixtureRunner } from '../../Fixture'
+import { JoystreamCLI } from '../../cli/joystream'
+import { QueryNodeApi } from '../../QueryNodeApi'
+import { PaidTermId, MemberId } from '@joystream/types/members'
+import { Debugger, extendDebug } from '../../Debugger'
+import { Api } from '../../Api'
+import { WorkingGroups } from '../../WorkingGroups'
+import { Worker, WorkerId } from '@joystream/types/working-group'
+import { getVideoCategoryDefaults, getChannelCategoryDefaults } from './contentTemplates'
+import BN from 'bn.js'
+
+export class CreateContentStructureFixture extends BaseQueryNodeFixture {
+  private debug: Debugger.Debugger
+  private cli: JoystreamCLI
+  private channelCategoryCount: number
+  private videoCategoryCount: number
+  private createdItems: {
+    channelCategoryIds: number[]
+    videoCategoryIds: number[]
+  }
+
+  constructor(
+    api: Api,
+    query: QueryNodeApi,
+    cli: JoystreamCLI,
+    channelCategoryCount: number,
+    videoCategoryCount: number
+  ) {
+    super(api, query)
+    this.cli = cli
+    this.channelCategoryCount = channelCategoryCount
+    this.videoCategoryCount = videoCategoryCount
+    this.debug = extendDebug('fixture:CreateContentStructureFixture')
+
+    this.createdItems = {
+      channelCategoryIds: [],
+      videoCategoryIds: [],
+    }
+  }
+
+  public getCreatedItems() {
+    return this.createdItems
+  }
+
+  /*
+    Execute this Fixture.
+  */
+  public async execute(): Promise<void> {
+    // prepare accounts for working group leads
+
+    this.debug('Loading working group leaders')
+    const { contentLeader, storageLeader } = await this.retrieveWorkingGroupLeaders()
+
+    // switch to lead and create category structure as lead
+
+    this.debug(`Choosing content working group lead's account`)
+    const contentLeaderKeyPair = this.api.getKeypair(contentLeader.role_account_id.toString())
+    await this.cli.importAccount(contentLeaderKeyPair)
+    await this.cli.chooseMemberAccount(contentLeader.member_id)
+
+    this.debug('Creating channel categories')
+    this.createdItems.channelCategoryIds = await this.createChannelCategories(this.channelCategoryCount)
+
+    this.debug('Creating video categories')
+    this.createdItems.videoCategoryIds = await this.createVideoCategories(this.videoCategoryCount)
+  }
+
+  /**
+    Retrieves information about accounts of group leads for content and storage working groups.
+  */
+  private async retrieveWorkingGroupLeaders(): Promise<{ contentLeader: Worker; storageLeader: Worker }> {
+    const retrieveGroupLeader = async (group: WorkingGroups) => {
+      const leader = await this.api.getGroupLead(group)
+      if (!leader) {
+        throw new Error(`Working group leader for "${group}" is missing!`)
+      }
+      return leader
+    }
+
+    return {
+      contentLeader: await retrieveGroupLeader(WorkingGroups.Content),
+      storageLeader: await retrieveGroupLeader(WorkingGroups.Storage),
+    }
+  }
+
+  /**
+    Creates a new channel category. Can only be executed as content group leader.
+  */
+  private async createChannelCategories(count: number): Promise<number[]> {
+    const createdIds = (await this.createCommonEntities(count, (index) =>
+      this.cli.createChannelCategory({
+        ...getChannelCategoryDefaults(index),
+      })
+    )) as number[]
+
+    return createdIds
+  }
+
+  /**
+    Creates a new video category. Can only be executed as content group leader.
+  */
+  private async createVideoCategories(count: number): Promise<number[]> {
+    const createdIds = (await this.createCommonEntities(count, (index) =>
+      this.cli.createVideoCategory({
+        ...getVideoCategoryDefaults(index),
+      })
+    )) as number[]
+
+    return createdIds
+  }
+
+  /**
+    Creates a bunch of content entities.
+  */
+  private async createCommonEntities<T>(count: number, createPromise: (index: number) => Promise<T>): Promise<T[]> {
+    const createdIds = await Array.from(Array(count).keys()).reduce(async (accPromise, index: number) => {
+      const acc = await accPromise
+      const createdId = await createPromise(index)
+
+      return [...acc, createdId]
+    }, Promise.resolve([]) as Promise<T[]>)
+
+    return createdIds
+  }
+}

tests/network-tests/src/fixtures/content/index.ts

@@ -1 +1,3 @@
 export * from './activeVideoCounters'
+export * from './createChannelsAndVideos'
+export * from './createContentStructure'

tests/network-tests/src/flows/clis/createChannel.ts

@@ -8,6 +8,7 @@ import { TmpFileManager } from '../../cli/utils'
 import { assert } from 'chai'
 import { Utils } from '../../utils'
 import { statSync } from 'fs'
+import { createJoystreamCli } from '../utils'
 
 export default async function createChannel({ api, env, query }: FlowProps): Promise<void> {
   const debug = extendDebug('flow:createChannel')
@@ -18,22 +19,23 @@ export default async function createChannel({ api, env, query }: FlowProps): Pro
   const paidTermId = api.createPaidTermId(new BN(+(env.MEMBERSHIP_PAID_TERMS || 0)))
   const buyMembershipFixture = new BuyMembershipHappyCaseFixture(api, [channelOwnerKeypair.key.address], paidTermId)
   await new FixtureRunner(buyMembershipFixture).run()
+  const memberId = buyMembershipFixture.getCreatedMembers()[0]
 
   // Send some funds to pay the deletion_prize
   const channelOwnerBalance = api.consts.storage.dataObjectDeletionPrize.muln(2)
   await api.treasuryTransferBalance(channelOwnerKeypair.key.address, channelOwnerBalance)
 
-  // Create Joystream CLI
-  const tmpFileManager = new TmpFileManager()
-  const joystreamCli = new JoystreamCLI(tmpFileManager)
+  // Create and init Joystream CLI
+  const joystreamCli = await createJoystreamCli()
 
-  // Init CLI, import & select channel owner key
+  // Import & select channel owner key
   await joystreamCli.init()
-  await joystreamCli.importKey(channelOwnerKeypair.key)
+  await joystreamCli.importAccount(channelOwnerKeypair.key)
+  await joystreamCli.chooseMemberAccount(memberId)
 
   // Create channel
-  const avatarPhotoPath = tmpFileManager.randomImgFile(300, 300)
-  const coverPhotoPath = tmpFileManager.randomImgFile(1920, 500)
+  const avatarPhotoPath = joystreamCli.getTmpFileManager().randomImgFile(300, 300)
+  const coverPhotoPath = joystreamCli.getTmpFileManager().randomImgFile(1920, 500)
   const channelInput = {
     title: 'Test channel',
     avatarPhotoPath,
@@ -43,16 +45,11 @@ export default async function createChannel({ api, env, query }: FlowProps): Pro
     language: 'en',
     rewardAccount: channelOwnerKeypair.key.address,
   }
-  const { out: createChannelOut } = await joystreamCli.createChannel(channelInput, ['--context', 'Member'])
 
-  const channelIdMatch = /Channel with id ([0-9]+) successfully created/.exec(createChannelOut)
-  if (!channelIdMatch) {
-    throw new Error(`No channel id found in output:\n${createChannelOut}`)
-  }
-  const [, channelId] = channelIdMatch
+  const channelId = await joystreamCli.createChannel(channelInput)
 
   await query.tryQueryWithTimeout(
-    () => query.channelById(channelId),
+    () => query.channelById(channelId.toString()),
     (channel) => {
       Utils.assert(channel, 'Channel not found')
       assert.equal(channel.title, channelInput.title)

tests/network-tests/src/flows/content/activeVideoCounters.ts

@@ -1,18 +1,67 @@
 import { FlowProps } from '../../Flow'
 import { extendDebug } from '../../Debugger'
 import { FixtureRunner } from '../../Fixture'
-import { ActiveVideoCountersFixture } from '../../fixtures/content'
+import {
+  ActiveVideoCountersFixture,
+  CreateChannelsAndVideosFixture,
+  CreateContentStructureFixture,
+} from '../../fixtures/content'
 import { PaidTermId } from '@joystream/types/members'
 import BN from 'bn.js'
+import { createJoystreamCli } from '../utils'
 
-export default async function activeVideoCounters({ api, query, cli, env }: FlowProps): Promise<void> {
+export default async function activeVideoCounters({ api, query, env }: FlowProps): Promise<void> {
   const debug = extendDebug('flow:active-video-counters')
   debug('Started')
   api.enableDebugTxLogs()
 
+  // create Joystream CLI
+  const joystreamCli = await createJoystreamCli()
+
+  // settings
+  const videoCount = 2
+  const videoCategoryCount = 2
+  const channelCount = 2
+  const channelCategoryCount = 2
+
   const paidTerms: PaidTermId = api.createPaidTermId(new BN(+env.MEMBERSHIP_PAID_TERMS!))
 
-  const activeVideoCountersFixture = new ActiveVideoCountersFixture(api, query, cli, env, paidTerms)
+  // flow itself
+
+  const createContentStructureFixture = new CreateContentStructureFixture(
+    api,
+    query,
+    joystreamCli,
+    videoCategoryCount,
+    channelCategoryCount
+  )
+  await new FixtureRunner(createContentStructureFixture).run()
+
+  const { channelCategoryIds, videoCategoryIds } = createContentStructureFixture.getCreatedItems()
+
+  const createChannelsAndVideos = new CreateChannelsAndVideosFixture(
+    api,
+    query,
+    joystreamCli,
+    paidTerms,
+    videoCount,
+    channelCount,
+    channelCategoryIds[0],
+    videoCategoryIds[0]
+  )
+  await new FixtureRunner(createChannelsAndVideos).run()
+
+  const { channelIds, videosData } = createChannelsAndVideos.getCreatedItems()
+
+  const activeVideoCountersFixture = new ActiveVideoCountersFixture(
+    api,
+    query,
+    joystreamCli,
+    channelCategoryIds,
+    videosData,
+    channelIds,
+    videoCategoryIds
+  )
   await new FixtureRunner(activeVideoCountersFixture).run()
 
   debug('Done')

tests/network-tests/src/flows/storagev2/initStorage.ts

@@ -15,6 +15,7 @@ type StorageBucketConfig = {
   objectsLimit: number
   operatorId: number
   transactorKey: string
+  transactorUri: string
 }
 
 type InitStorageConfig = {
@@ -48,6 +49,7 @@ export const singleBucketConfig: InitStorageConfig = {
       storageLimit: new BN(1_000_000_000_000),
       objectsLimit: 1000000000,
       transactorKey: process.env.COLOSSUS_1_TRANSACTOR_KEY || '5DkE5YD8m5Yzno6EH2RTBnH268TDnnibZMEMjxwYemU4XevU', // //Colossus1
+      transactorUri: process.env.COLOSSUS_1_TRANSACTOR_URI || '//Colossus1',
     },
   ],
 }
@@ -65,6 +67,7 @@ export const doubleBucketConfig: InitStorageConfig = {
       storageLimit: new BN(1_000_000_000_000),
       objectsLimit: 1000000000,
       transactorKey: process.env.COLOSSUS_1_TRANSACTOR_KEY || '5DkE5YD8m5Yzno6EH2RTBnH268TDnnibZMEMjxwYemU4XevU', // //Colossus1
+      transactorUri: process.env.COLOSSUS_1_TRANSACTOR_URI || '//Colossus1',
     },
     {
       metadata: { endpoint: process.env.STORAGE_2_URL || 'http://localhost:3335' },
@@ -73,12 +76,13 @@ export const doubleBucketConfig: InitStorageConfig = {
       storageLimit: new BN(1_000_000_000_000),
       objectsLimit: 1000000000,
       transactorKey: process.env.COLOSSUS_2_TRANSACTOR_KEY || '5FbzYmQ3HogiEEDSXPYJe58yCcmSh3vsZLodTdBB6YuLDAj7', // //Colossus2
+      transactorUri: process.env.COLOSSUS_2_TRANSACTOR_URI || '//Colossus2',
     },
   ],
 }
 
 export default function createFlow({ buckets, dynamicBagPolicy }: InitStorageConfig) {
-  return async function initDistribution({ api }: FlowProps): Promise<void> {
+  return async function initStorage({ api }: FlowProps): Promise<void> {
     const debug = extendDebug('flow:initStorage')
     debug('Started')
 

tests/network-tests/src/flows/utils.ts

@@ -0,0 +1,15 @@
+import { JoystreamCLI } from '../cli/joystream'
+import { TmpFileManager } from '../cli/utils'
+import { v4 as uuid } from 'uuid'
+
+export async function createJoystreamCli(): Promise<JoystreamCLI> {
+  const tmpFileManager = new TmpFileManager()
+
+  // create Joystream CLI
+  const joystreamCli = new JoystreamCLI(tmpFileManager)
+
+  // init CLI
+  await joystreamCli.init()
+
+  return joystreamCli
+}

tests/network-tests/src/flows/workingGroup/leaderSetup.ts

@@ -8,11 +8,11 @@ import { assert } from 'chai'
 // import { KeyringPair } from '@polkadot/keyring/types'
 import { FixtureRunner } from '../../Fixture'
 import { extendDebug } from '../../Debugger'
-import { CliApi } from '../../CliApi'
+import { createJoystreamCli } from '../utils'
 
 export default function (group: WorkingGroups, canSkip = false) {
-  return async function ({ api, env, cli }: FlowProps): Promise<void> {
-    return leaderSetup(api, env, cli, group, canSkip)
+  return async function ({ api, env }: FlowProps): Promise<void> {
+    return leaderSetup(api, env, group, canSkip)
   }
 }
 
@@ -20,7 +20,6 @@ export default function (group: WorkingGroups, canSkip = false) {
 async function leaderSetup(
   api: Api,
   env: NodeJS.ProcessEnv,
-  cli: CliApi,
   group: WorkingGroups,
   skipIfAlreadySet = false
 ): Promise<void> {
@@ -63,15 +62,6 @@ async function leaderSetup(
   assert.notEqual(hiredLead, undefined, `${group} group Lead was not hired!`)
   assert(hiredLead!.role_account_id.eq(leadKeyPair.address))
 
-  // setup (ensure) CLI connection to node and query node
-
-  await cli.setApiUri()
-
-  await cli.setQueryNodeUri()
-
-  debug(`Importing leader's account to CLI`)
-  await cli.importAccount(leadKeyPair)
-
   debug('Done')
 
   // Who ever needs it will need to get it from the Api layer

tests/network-tests/src/scenarios/combined.ts

@@ -10,7 +10,7 @@ import createChannel from '../flows/clis/createChannel'
 import { scenario } from '../Scenario'
 import { WorkingGroups } from '../WorkingGroups'
 
-scenario(async ({ job }) => {
+scenario('Combined', async ({ job }) => {
   // These tests assume:
   // - storage setup (including hired lead)
   // - existing council
@@ -37,8 +37,10 @@ scenario(async ({ job }) => {
     manageWorkerAsWorker.distribution,
   ]).requires(leadSetupJob)
 
-  const createChannelJob = job('create channel via CLI', createChannel)
-  job('init storage and distribution buckets via CLI', [initDistributionBucket, initStorageBucket]).after(
-    createChannelJob
-  )
+  const initBucketsJob = job('init storage and distribution buckets via CLI', [
+    initDistributionBucket,
+    initStorageBucket,
+  ]).requires(leadSetupJob)
+
+  const createChannelJob = job('create channel via CLI', createChannel).requires(initBucketsJob)
 })

tests/network-tests/src/scenarios/content-directory.ts

@@ -1,13 +1,17 @@
 import leaderSetup from '../flows/workingGroup/leaderSetup'
 import activeVideoCounters from '../flows/content/activeVideoCounters'
+import initStorageBucket from '../flows/clis/initStorageBucket'
+import initStorage, { singleBucketConfig as storageConfig } from '../flows/storagev2/initStorage'
 import { WorkingGroups } from '../WorkingGroups'
 import { scenario } from '../Scenario'
 
-scenario(async ({ job }) => {
-  const setupContentLeaderSetup = job('setup content lead', leaderSetup(WorkingGroups.Content))
-  const setupStorageLeaderSetup = job('setup storage lead', leaderSetup(WorkingGroups.Storage))
+scenario('Content directory', async ({ job }) => {
+  const leadSetupJob = job('setup working group leads', [
+    leaderSetup(WorkingGroups.Content, true),
+    leaderSetup(WorkingGroups.Storage, true),
+  ])
 
-  job('check active video counters', activeVideoCounters)
-    .requires(setupContentLeaderSetup)
-    .requires(setupStorageLeaderSetup)
+  const initStorageJob = job('initialize storage system', initStorage(storageConfig)).requires(leadSetupJob)
+
+  job('check active video counters', activeVideoCounters).requires(initStorageJob)
 })

tests/network-tests/src/scenarios/giza-issue-reproduction-setup.ts

@@ -6,7 +6,7 @@ import initStorage, { doubleBucketConfig as storageConfig } from '../flows/stora
 import { WorkingGroups } from '../WorkingGroups'
 import { scenario } from '../Scenario'
 
-scenario(async ({ job }) => {
+scenario('Giza issue reproduction setup', async ({ job }) => {
   job('Make Alice a member', makeAliceMember)
 
   const leads = job('Set Storage Lead', leaderSetup(WorkingGroups.Storage))

tests/network-tests/src/scenarios/init-storage-and-distribution.ts

@@ -5,7 +5,7 @@ import { scenario } from '../Scenario'
 import { WorkingGroups } from '../WorkingGroups'
 import updateAccountsFlow from '../misc/updateAllWorkerRoleAccountsFlow'
 
-scenario(async ({ job }) => {
+scenario('Init storage and distribution', async ({ job }) => {
   const setupLead = job('setup leads', [
     leaderSetup(WorkingGroups.Distribution, true),
     leaderSetup(WorkingGroups.Storage, true),

tests/network-tests/src/scenarios/post-migration.ts

@@ -1,6 +1,6 @@
 import postMigrationAssertions from '../misc/postMigrationAssertionsFlow'
 import { scenario } from '../Scenario'
 
-scenario(async ({ job }) => {
+scenario('Post migration', async ({ job }) => {
   job('Verify post-migration chain state', postMigrationAssertions)
 })

tests/network-tests/src/scenarios/proposals.ts

@@ -8,7 +8,7 @@ import validatorCountProposal from '../flows/proposals/validatorCountProposal'
 import wgMintCapacityProposal from '../flows/proposals/workingGroupMintCapacityProposal'
 import { scenario } from '../Scenario'
 
-scenario(async ({ job }) => {
+scenario('Proposals', async ({ job }) => {
   job('creating members', creatingMemberships)
 
   const councilJob = job('council setup', councilSetup)

tests/network-tests/src/scenarios/setup-new-chain.ts

@@ -7,7 +7,7 @@ import initDistribution, { singleBucketConfig as defaultDistributionConfig } fro
 import { AllWorkingGroups } from '../WorkingGroups'
 import { scenario } from '../Scenario'
 
-scenario(async ({ job }) => {
+scenario('Setup new chain', async ({ job }) => {
   const COUNCIL_SIZE = 1
   job('Create Council', assignCouncil(COUNCIL_SIZE))
 

tests/network-tests/src/scenarios/tests/resource-locks-1.ts

@@ -6,6 +6,6 @@ async function flow1({ lock }: FlowProps) {
   await lock(Resource.Council)
 }
 
-scenario(async ({ job }) => {
+scenario('Resource locks 1', async ({ job }) => {
   job('test', [flow1, flow1])
 })

tests/network-tests/src/scenarios/tests/resource-locks-2.ts

@@ -6,7 +6,7 @@ async function flow({ lock }: FlowProps) {
   await lock(Resource.Proposals)
 }
 
-scenario(async ({ job }) => {
+scenario('Resource locks 2', async ({ job }) => {
   // Runtime is configured for MaxActiveProposalLimit = 5
   // So we should ensure we don't exceed that number of active proposals
   // which limits the number of concurrent tests that create proposals