integration-tests: async lock tx queue not per account

storage-node/packages/runtime-api/index.js

@@ -124,8 +124,8 @@ class RuntimeApi {
     return this.workers.isRoleAccountOfStorageProvider(this.storageProviderId, this.identities.key.address)
   }
 
-  executeWithAccountLock(accountId, func) {
-    return this.asyncLock.acquire(`${accountId}`, func)
+  executeWithAccountLock(func) {
+    return this.asyncLock.acquire('tx-queue', func)
   }
 
   static matchingEvents(subscribed = [], events = []) {
@@ -207,7 +207,7 @@ class RuntimeApi {
     }
 
     // synchronize access to nonce
-    await this.executeWithAccountLock(accountId, async () => {
+    await this.executeWithAccountLock(async () => {
       const nonce = await this.api.rpc.system.accountNextIndex(accountId)
       const signed = tx.sign(fromKey, { nonce })
       const txhash = signed.hash

tests/network-tests/src/Api.ts

@@ -16,7 +16,7 @@ import { ElectionStake, Seat } from '@joystream/types/council'
 import { AccountInfo, Balance, BalanceOf, BlockNumber, Event, EventRecord } from '@polkadot/types/interfaces'
 import BN from 'bn.js'
 import { SubmittableExtrinsic } from '@polkadot/api/types'
-import { Sender } from './sender'
+import { Sender, LogLevel } from './sender'
 import { Utils } from './utils'
 import { Stake, StakedState, StakeId } from '@joystream/types/stake'
 import { RewardRelationship, RewardRelationshipId } from '@joystream/types/recurring-rewards'
@@ -35,9 +35,7 @@ import { VideoEntity } from '@joystream/cd-schemas/types/entities/VideoEntity'
 import { initializeContentDir, InputParser } from '@joystream/cd-schemas'
 import { OperationType } from '@joystream/types/content-directory'
 import { ContentId, DataObject } from '@joystream/types/media'
-
 import Debugger from 'debug'
-const debug = Debugger('api')
 
 export enum WorkingGroups {
   StorageWorkingGroup = 'storageWorkingGroup',
@@ -55,6 +53,7 @@ export class ApiFactory {
     treasuryAccountUri: string,
     sudoAccountUri: string
   ): Promise<ApiFactory> {
+    const debug = Debugger('api-factory')
     let connectAttempts = 0
     while (true) {
       connectAttempts++
@@ -109,8 +108,12 @@ export class Api {
     this.sender = new Sender(api, keyring, label)
   }
 
-  public enableTxLogs(): void {
-    this.sender.enableLogs()
+  public enableDebugTxLogs(): void {
+    this.sender.setLogLevel(LogLevel.Debug)
+  }
+
+  public enableVerboseTxLogs(): void {
+    this.sender.setLogLevel(LogLevel.Debug)
   }
 
   public createKeyPairs(n: number): KeyringPair[] {

tests/network-tests/src/flows/membership/creatingMemberships.ts

@@ -12,7 +12,7 @@ import { assert } from 'chai'
 export default async function membershipCreation({ api, env }: FlowProps): Promise<void> {
   const debug = Debugger('flow:memberships')
   debug('Started')
-  api.enableTxLogs()
+  api.enableDebugTxLogs()
 
   const N: number = +env.MEMBERSHIP_CREATION_N!
   assert(N > 0)

tests/network-tests/src/flows/workingGroup/atLeastValueBug.ts

@@ -10,7 +10,7 @@ import { FixtureRunner } from '../../Fixture'
 export default async function zeroAtLeastValueBug({ api, env }: FlowProps): Promise<void> {
   const debug = Debugger('flow:atLeastValueBug')
   debug('Started')
-  api.enableTxLogs()
+  api.enableDebugTxLogs()
 
   const applicationStake: BN = new BN(env.WORKING_GROUP_APPLICATION_STAKE!)
   const roleStake: BN = new BN(env.WORKING_GROUP_ROLE_STAKE!)

tests/network-tests/src/sender.ts

@@ -9,18 +9,24 @@ import Debugger from 'debug'
 import AsyncLock from 'async-lock'
 import { assert } from 'chai'
 
+export enum LogLevel {
+  None,
+  Debug,
+  Verbose,
+}
+
 export class Sender {
   private readonly api: ApiPromise
   private static readonly asyncLock: AsyncLock = new AsyncLock()
   private readonly keyring: Keyring
   private readonly debug: Debugger.Debugger
-  private logFailedTransactions = false
-  private static _count = 0
+  private logs: LogLevel = LogLevel.None
+  private static instance = 0
 
   constructor(api: ApiPromise, keyring: Keyring, label: string) {
     this.api = api
     this.keyring = keyring
-    this.debug = Debugger(`Sender:${Sender._count++}:${label}`)
+    this.debug = Debugger(`Sender:${Sender.instance++}:${label}`)
   }
 
   // Synchronize all sending of transactions into mempool, so we can always safely read
@@ -29,12 +35,8 @@ export class Sender {
   // The promise resolves on tx finalization (For both Dispatch success and failure)
   // The promise is rejected if transaction is rejected by node.
 
-  public enableLogs(): void {
-    this.logFailedTransactions = true
-  }
-
-  public disableLogs(): void {
-    this.logFailedTransactions = false
+  public setLogLevel(level: LogLevel): void {
+    this.logs = level
   }
 
   public async signAndSend(
@@ -70,7 +72,7 @@ export class Sender {
       const failed = result.findRecord('system', 'ExtrinsicFailed')
 
       // Log failed transactions
-      if (this.logFailedTransactions) {
+      if (this.logs === LogLevel.Debug || this.logs === LogLevel.Verbose) {
         if (failed) {
           const record = failed as EventRecord
           assert(record)
@@ -108,16 +110,24 @@ export class Sender {
       if (success || failed) finalized(result)
     }
 
-    await Sender.asyncLock.acquire(`${senderKeyPair.address}`, async () => {
+    // We used to do this: Sender.asyncLock.acquire(`${senderKeyPair.address}` ...
+    // Instead use a single lock for all calls, to force all transactions to be submitted in same order
+    // of call to signAndSend. Otherwise it raises chance of race conditions.
+    // It happens in rare cases and has lead some tests to fail occasionally in the past
+    await Sender.asyncLock.acquire('tx-queue', async () => {
       const nonce = await this.api.rpc.system.accountNextIndex(senderKeyPair.address)
       const signedTx = tx.sign(senderKeyPair, { nonce })
       sentTx = signedTx.toHuman()
       const { method, section } = signedTx.method
       try {
         await signedTx.send(handleEvents)
-        this.debug('Submitted tx:', `${section}.${method}`)
+        if (this.logs === LogLevel.Verbose) {
+          this.debug('Submitted tx:', `${section}.${method}`)
+        }
       } catch (err) {
-        this.debug('Submitting tx failed:', sentTx)
+        if (this.logs === LogLevel.Debug) {
+          this.debug('Submitting tx failed:', sentTx, err)
+        }
         throw err
       }
     })