Merge branch 'antioch' of github.com:Joystream/joystream into antioch-params

.dockerignore

@@ -5,3 +5,5 @@ target/
 query-node/generated
 query-node/**/dist
 query-node/lib
+cli/
+tests/

.github/workflows/content-directory-schemas.yml

@@ -17,4 +17,7 @@ jobs:
     - name: validate
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
+        yarn workspace @joystream/cd-schemas generate:all
+        yarn workspace @joystream/cd-schemas build
         yarn workspace @joystream/cd-schemas checks --quiet

.github/workflows/joystream-cli.yml

@@ -17,6 +17,9 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
+        yarn workspace @joystream/cd-schemas generate:all
+        yarn workspace @joystream/cd-schemas build
         yarn workspace @joystream/cli checks --quiet
     - name: yarn pack test
       run: |
@@ -39,6 +42,9 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace @joystream/types build
+        yarn workspace @joystream/cd-schemas generate:all
+        yarn workspace @joystream/cd-schemas build        
         yarn workspace @joystream/cli checks --quiet
     - name: yarn pack test
       run: |

.github/workflows/joystream-types.yml

@@ -17,6 +17,7 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
         yarn workspace @joystream/types checks --quiet
     - name: npm pack test
       run: |
@@ -40,6 +41,7 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace @joystream/types build
         yarn workspace @joystream/types checks --quiet
     - name: npm pack test
       run: |

.github/workflows/network-tests.yml

@@ -17,6 +17,9 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
+        yarn workspace @joystream/cd-schemas generate:all
+        yarn workspace @joystream/cd-schemas build
         yarn workspace network-tests checks --quiet
 
   network_build_osx:
@@ -34,4 +37,7 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace @joystream/types build
+        yarn workspace @joystream/cd-schemas generate:all
+        yarn workspace @joystream/cd-schemas build
         yarn workspace network-tests checks --quiet

.github/workflows/pioneer.yml

@@ -17,6 +17,7 @@ jobs:
     - name: build
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
         yarn workspace pioneer build
 
   pioneer_build_osx:
@@ -34,6 +35,7 @@ jobs:
     - name: build
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace @joystream/types build
         yarn workspace pioneer build
 
   pioneer_lint_ubuntu:
@@ -51,6 +53,7 @@ jobs:
     - name: lint
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
         yarn workspace pioneer lint --quiet
 
   pioneer_lint_osx:
@@ -68,4 +71,5 @@ jobs:
     - name: lint
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace @joystream/types build
         yarn workspace pioneer lint --quiet

.github/workflows/run-network-tests.yml

@@ -79,6 +79,7 @@ jobs:
           path: joystream-node-docker-image.tar.gz
 
   basic_runtime_with_upgrade:
+    if: ${{ false }}  # Antioch will be a new chain
     name: Integration Tests (Runtime Upgrade)
     needs: build_images
     runs-on: ubuntu-latest
@@ -96,7 +97,9 @@ jobs:
           docker load --input joystream-node-docker-image.tar.gz
           docker images
       - name: Install packages and dependencies
-        run: yarn install --frozen-lockfile
+        run: |
+          yarn install --frozen-lockfile
+          yarn build:packages
       - name: Ensure tests are runnable
         run: yarn workspace network-tests build
       - name: Execute network tests
@@ -120,7 +123,9 @@ jobs:
           docker load --input joystream-node-docker-image.tar.gz
           docker images
       - name: Install packages and dependencies
-        run: yarn install --frozen-lockfile
+        run: |
+          yarn install --frozen-lockfile
+          yarn build:packages
       - name: Ensure tests are runnable
         run: yarn workspace network-tests build
       - name: Execute network tests
@@ -144,7 +149,11 @@ jobs:
           docker load --input joystream-node-docker-image.tar.gz
           docker images
       - name: Install packages and dependencies
-        run: yarn install --frozen-lockfile
+        run: |
+          yarn install --frozen-lockfile
+          yarn workspace @joystream/types build
+          yarn workspace @joystream/cd-schemas generate:all
+          yarn workspace @joystream/cd-schemas build
       - name: Ensure tests are runnable
         run: yarn workspace @joystream/cd-schemas checks --quiet
       - name: Start chain
@@ -170,9 +179,12 @@ jobs:
           docker load --input joystream-node-docker-image.tar.gz
           docker images
       - name: Install packages and dependencies
-        run: yarn install --frozen-lockfile
-      - name: Ensure query-node builds
-        run: yarn workspace query-node-root build
+        run: |
+          yarn install --frozen-lockfile
+          yarn workspace @joystream/types build
+          yarn workspace @joystream/cd-schemas generate:all
+          yarn workspace @joystream/cd-schemas build
+          yarn workspace query-node-root build
       - name: Ensure tests are runnable
         run: yarn workspace network-tests build
       # Bring up hydra query-node development instance, then run content directory
@@ -198,7 +210,9 @@ jobs:
           docker load --input joystream-node-docker-image.tar.gz
           docker images
       - name: Install packages and dependencies
-        run: yarn install --frozen-lockfile
+        run: |
+          yarn install --frozen-lockfile
+          yarn workspace @joystream/types build
       - name: Build storage node
         run: yarn workspace storage-node build
       - name: Start Services

.github/workflows/storage-node.yml

@@ -17,7 +17,9 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
         yarn workspace storage-node checks --quiet
+        yarn workspace storage-node build
 
   storage_node_build_osx:
     name: MacOS Checks
@@ -34,4 +36,6 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace @joystream/types build
         yarn workspace storage-node checks --quiet
+        yarn workspace storage-node build

README.md

@@ -25,12 +25,11 @@ After cloning the repo run the following initialization scripts:
 # Install rust toolchain
 ./setup.sh
 
-# Install npm package dependencies
-# Also good habit to run this when switching between branches
-yarn install
+# Install npm package dependencies, build packages and docker images
+yarn build
 
-# run some tests
-yarn cargo-checks
+# start a local development network
+yarn start
 ```
 
 ## Software
@@ -75,6 +74,7 @@ The HEAD of the master branch should always be used for the correct version of t
 ```sh
 git checkout master
 yarn install
+yarn build:packages
 yarn workspace pioneer start
 ```
 
@@ -89,7 +89,7 @@ You can also run your our own joystream-node:
 
 ```sh
 git checkout master
-WASM_BUILD_TOOLCHAIN=nightly-2020-05-23 cargo build --release
+WASM_BUILD_TOOLCHAIN=nightly-2021-02-20 cargo build --release
 ./target/release/joystream-node -- --pruning archive --chain testnets/joy-testnet-4.json
 ```
 

apps.Dockerfile

@@ -2,13 +2,31 @@ FROM node:12 as builder
 
 WORKDIR /joystream
 COPY . /joystream
+RUN  rm -fr /joystream/pioneer
 
 # Do not set NODE_ENV=production until after running yarn install
 # to ensure dev dependencies are installed.
-RUN yarn install --frozen-lockfile
+RUN yarn --forzen-lockfile
 
-RUN yarn workspace pioneer build
-RUN yarn workspace storage-node build
+RUN yarn workspace @joystream/types build
 RUN yarn workspace query-node-root build
+RUN yarn workspace storage-node build
+
+# Second stage to reduce image size, enable it when
+# all packages have correctly identified what is a devDependency and what is not.
+# It will reduce the image size by about 500MB (down from 2.2GB to 1.7GB)
+
+# # Remove files that are not needed after build.
+# # We will re-fetch only dependencies needed for running the apps.
+# RUN rm -fr node_modules/
+# RUN rm -fr .git/
+
+# FROM node:12
+# WORKDIR /joystream
+# COPY --from=builder /joystream/ /joystream/
+
+# # Skip installing devDependencies, since we have already built the packages.
+# ENV NODE_ENV=production
+# RUN yarn install --forzen-lockfile --production
 
 ENTRYPOINT [ "yarn" ]

build-docker-images.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -e
+
+if ! command -v docker-compose &> /dev/null
+then
+  echo "docker-compose not found. Skipping docker image builds."
+  exit 0
+fi
+
+# Build or fetch cached joystream/node docker image
+if [[ "$SKIP_JOYSTREAM_NODE" = 1 || "$SKIP_JOYSTREAM_NODE" = "true" ]]; then
+  echo "Skipping build of joystream/node docker image."
+else
+  # Fetch a cached joystream/node image if one is found matching code shasum instead of building
+  CODE_HASH=`scripts/runtime-code-shasum.sh`
+  IMAGE=joystream/node:${CODE_HASH}
+  echo "Trying to fetch cached ${IMAGE} image"
+  docker pull ${IMAGE} || :
+
+  if ! docker inspect ${IMAGE} > /dev/null;
+  then
+    echo "Fetch failed, building image locally"
+    docker-compose build joystream-node
+  else
+    echo "Tagging cached image as 'latest'"
+    docker image tag ${IMAGE} joystream/node:latest
+  fi
+fi
+
+# Build joystream/apps docker image
+echo "Building 'joystream/apps' docker image..."
+docker-compose build colossus
+
+# Build the pioneer docker image
+echo "Building pioneer docker image"
+docker-compose build pioneer

build-npm-packages.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+
+yarn
+yarn workspace @joystream/types build
+yarn workspace @joystream/cd-schemas generate:all
+yarn workspace @joystream/cd-schemas build
+yarn workspace query-node-root build
+yarn workspace @joystream/cli build
+yarn workspace storage-node build
+yarn workspace pioneer build

build.sh

@@ -1,55 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-yarn
-yarn workspace @joystream/types build
-yarn workspace @joystream/cd-schemas generate:all
-yarn workspace @joystream/cd-schemas build
-yarn workspace @joystream/cli build
-yarn workspace query-node-root build
-yarn workspace storage-node build
-# Not strictly needed during development, we run "yarn workspace pioneer start" to start
-# a dev instance, but will show highlight build issues
-yarn workspace pioneer build
-
-if ! command -v docker-compose &> /dev/null
-then
-  echo "docker-compose not found, skipping docker build!"
-else
-  # Build joystream/apps docker image
-  docker-compose build pioneer
-
-  # Optionally build joystream/node docker image
-  # TODO: Try to fetch a cached joystream/node image
-  # if one is found matching code shasum instead of building
-  while true
-  do
-    read -p "Rebuild joystream/node docker image? (y/N): " answer2
-
-    case $answer2 in
-    [yY]* ) docker-compose build joystream-node
-            break;;
-
-    [nN]* ) break;;
-
-    * )     break;;
-    esac
-  done
-fi
-
-# Build cargo crates: native binaries joystream/node, wasm runtime, and chainspec builder.
-while true
-do
-  read -p "Compile joystream node native binary? (y/N): " answer1
-
-  case $answer1 in
-   [yY]* ) yarn cargo-checks
-           yarn cargo-build
-           break;;
-
-   [nN]* ) break;;
-
-   * )     break;;
-  esac
-done

docker-compose.yml

@@ -101,11 +101,8 @@ services:
     command: ["workspace", "query-node-root", "processor:start"]
 
   indexer:
-    image: joystream/apps
+    image: joystream/hydra-indexer:2.0.1-beta-v3.9
     restart: unless-stopped
-    build: 
-      context: .
-      dockerfile: apps.Dockerfile
     env_file:
       # relative to working directory where docker-compose was run from
       - .env
@@ -113,13 +110,19 @@ services:
       - TYPEORM_HOST=db
       - TYPEORM_DATABASE=${INDEXER_DB_NAME}
       - INDEXER_WORKERS=5
-      - PROCESSOR_POLL_INTERVAL=1000 # refresh every second 
+      - PROCESSOR_POLL_INTERVAL=1000 # refresh every second
       - REDIS_URI=redis://redis:6379/0
       - DEBUG=index-builder:*
       - WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944
-    depends_on: 
+      - TYPES_JSON=types.json
+      - DB_NAME=${INDEXER_DB_NAME}
+      - DB_HOST=db
+    depends_on:
       - db
-    command: ["workspace", "query-node-root", "indexer:start"] 
+    volumes:
+      - ./types/augment/all/defs.json:/home/hydra/packages/hydra-indexer/types.json
+    command: >
+      sh -c 'yarn db:bootstrap && yarn start:prod'
 
   indexer-api-gateway:
     image: joystream/hydra-indexer-gateway:latest
@@ -151,10 +154,9 @@ services:
       - "127.0.0.1:6379:6379"
 
   pioneer:
-    image: joystream/apps
+    image: joystream/pioneer
     build:
       context: .
-      dockerfile: apps.Dockerfile
+      dockerfile: pioneer.Dockerfile
     ports:
-      - "127.0.0.1:3000:3000"
-    command: workspace pioneer start
+      - "127.0.0.1:3000:80"

joystream-node.Dockerfile

@@ -3,7 +3,7 @@ RUN rustup component add rustfmt clippy
 RUN rustup install nightly-2021-02-20 --force
 RUN rustup target add wasm32-unknown-unknown --toolchain nightly-2021-02-20
 RUN apt-get update && \
-  apt-get install -y curl git gcc xz-utils sudo pkg-config unzip clang libc6-dev
+  apt-get install -y curl git gcc xz-utils sudo pkg-config unzip clang llvm libc6-dev
 
 FROM rustup AS builder
 LABEL description="Compiles all workspace artifacts"

node/README.md

@@ -26,7 +26,7 @@ cd joystream/
 Compile the node and runtime:
 
 ```bash
-WASM_BUILD_TOOLCHAIN=nightly-2020-05-23 cargo build --release
+WASM_BUILD_TOOLCHAIN=nightly-2021-02-20 cargo build --release
 ```
 
 This produces the binary in `./target/release/joystream-node`
@@ -79,7 +79,7 @@ If you are building a tagged release from `master` branch and want to install th
 This will install the executable `joystream-node` to your `~/.cargo/bin` folder, which you would normally have in your `$PATH` environment.
 
 ```bash
-WASM_BUILD_TOOLCHAIN=nightly-2020-05-23 cargo install joystream-node --path node/ --locked
+WASM_BUILD_TOOLCHAIN=nightly-2021-02-20 cargo install joystream-node --path node/ --locked
 ```
 
 Now you can run and connect to the testnet:

package.json

@@ -4,8 +4,10 @@
   "version": "1.0.0",
   "license": "GPL-3.0-only",
   "scripts": {
-    "postinstall": "yarn workspace @joystream/types build && yarn workspace @joystream/cd-schemas generate:all && yarn workspace @joystream/cd-schemas build && yarn workspace @joystream/cli build",
-    "build": "./build.sh",
+    "build": "./build-npm-packages.sh && ./build-docker-images.sh",
+    "build:packages": "./build-npm-packages.sh",
+    "build:docker": "./build-docker-images.sh",
+    "setup": "./setup.sh",
     "start": "./start.sh",
     "cargo-checks": "devops/git-hooks/pre-commit && devops/git-hooks/pre-push",
     "cargo-build": "scripts/cargo-build.sh"

pioneer.Dockerfile

@@ -0,0 +1,18 @@
+FROM node:12 as builder
+
+WORKDIR /joystream
+COPY . /joystream
+
+RUN yarn
+RUN NODE_ENV=production yarn workspace @joystream/types build
+RUN NODE_ENV=production yarn workspace pioneer build
+
+FROM ubuntu:18.04
+
+RUN apt-get update && apt-get -y install nginx
+
+COPY --from=builder /joystream/pioneer/packages/apps/build /var/www/html
+
+EXPOSE 80
+
+CMD ["nginx", "-g", "daemon off;"]

pioneer/README.md

@@ -37,7 +37,7 @@ To start off, this repo (along with others in the [@polkadot](https://github.com
 
 To get started -
 
-1. Clone the repo locally, via `git clone https://github.com/joystream/apps <optional local path>`
+1. Clone the repo locally, via `git clone https://github.com/joystream/joystream <optional local path>`
 2. Ensure that you have a recent LTS version of Node.js, for development purposes [Node >=10.13.0](https://nodejs.org/en/) is recommended.
 3. Ensure that you have a recent version of Yarn, for development purposes [Yarn >=1.10.1](https://yarnpkg.com/docs/install) is required.
 4. Install the dependencies by running `yarn`

pioneer/packages/apps/public/locales/en/joy-media.json

@@ -0,0 +1 @@
+{}

scripts/raspberry-cross-build.sh

@@ -9,7 +9,7 @@
 export WORKSPACE_ROOT=`cargo metadata --offline --no-deps --format-version 1 | jq .workspace_root -r`
 
 docker run \
-    -e WASM_BUILD_TOOLCHAIN=nightly-2020-05-23 \
+    -e WASM_BUILD_TOOLCHAIN=nightly-2021-02-20 \
     --volume ${WORKSPACE_ROOT}/:/home/cross/project \
     --volume ${HOME}/.cargo/registry:/home/cross/.cargo/registry \
     joystream/rust-raspberry \

scripts/run-dev-chain.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-export WASM_BUILD_TOOLCHAIN=nightly-2020-05-23
+export WASM_BUILD_TOOLCHAIN=nightly-2021-02-20
 
 # Build release binary
 cargo build --release

scripts/runtime-code-shasum.sh

@@ -13,7 +13,7 @@ if [[ "$OSTYPE" == "darwin"* ]]; then
 fi
 
 # sort/owner/group/mtime arguments only work with gnu version of tar!
-${TAR} -c --sort=name --owner=root:0 --group=root:0 --mtime='UTC 2020-01-01' \
+${TAR} -c --sort=name --owner=root:0 --group=root:0 --mode 644 --mtime='UTC 2020-01-01' \
     Cargo.lock \
     Cargo.toml \
     runtime \

setup.sh

@@ -5,7 +5,7 @@ set -e
 if [[ "$OSTYPE" == "linux-gnu" ]]; then
     # code build tools
     sudo apt-get update
-    sudo apt-get install -y coreutils clang jq curl gcc xz-utils sudo pkg-config unzip clang libc6-dev-i386 make libssl-dev python
+    sudo apt-get install -y coreutils clang llvm jq curl gcc xz-utils sudo pkg-config unzip libc6-dev make libssl-dev python
     # docker
     sudo apt-get install -y docker.io docker-compose containerd runc
 elif [[ "$OSTYPE" == "darwin"* ]]; then
@@ -15,7 +15,7 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then
     fi
     # install additional packages
     brew update
-    brew install b2sum gnu-tar jq curl
+    brew install coreutils gnu-tar jq curl
     echo "It is recommended to setup Docker desktop from: https://www.docker.com/products/docker-desktop"
 fi
 
@@ -25,24 +25,22 @@ curl https://getsubstrate.io -sSf | bash -s -- --fast
 
 source ~/.cargo/env
 
-rustup component add rustfmt clippy
-
 rustup install nightly-2021-02-20
 rustup target add wasm32-unknown-unknown --toolchain nightly-2021-02-20
 
 rustup install 1.51.0
 rustup default 1.51.0
 
+rustup component add rustfmt clippy
+
 # Volta nodejs, npm, yarn tools manager
 curl https://get.volta.sh | bash
 
 # source env variables added by Volta
-source source ~/.bash_profile || ~/.profile || source ~/.bashrc || :
+source ~/.bash_profile || source ~/.profile || source ~/.bashrc || :
 
 volta install node@12
 volta install yarn
 volta install npx
 
-echo "Starting new terminal/shell session to make newly installed tools available."
-
-exec bash -l
+echo "You may need to open a new terminal/shell session to make newly installed tools available."

start.sh

@@ -38,7 +38,10 @@ yarn workspace query-node-root db:migrate
 docker-compose up -d graphql-server
 docker-compose up -d processor
 
-echo "press Ctrl+C to shutdown"
+docker-compose up -d pioneer
 
-# Start a dev instance of pioneer and wait for exit
-docker-compose up pioneer
+echo "use Ctrl+C to shutdown the development network."
+
+while true; do 
+  read
+done

storage-node/packages/runtime-api/index.js

@@ -65,7 +65,8 @@ class RuntimeApi {
       }
 
       try {
-        this.api = await ApiPromise.create({ provider, types: types })
+        this.api = new ApiPromise({ provider, types })
+        await this.api.isReadyOrError
         break
       } catch (err) {
         debug('Connecting to node failed, will retry..')
@@ -73,8 +74,6 @@ class RuntimeApi {
       await sleep(5000)
     }
 
-    await this.api.isReady
-
     this.asyncLock = new AsyncLock()
 
     // The storage provider id to use

tests/network-tests/run-test-node.sh

@@ -0,0 +1,59 @@
+# Location that will be mounted as the /data volume in containers
+# This is how we access the initial members and balances files from
+# the containers and where generated chainspec files will be located.
+DATA_PATH="test-data"
+
+# Initial account balance for Alice
+# Alice is the source of funds for all new accounts that are created in the tests.
+ALICE_INITIAL_BALANCE=100000000
+
+rm -Rf ${DATA_PATH}
+mkdir -p ${DATA_PATH}
+
+echo "{
+  \"balances\":[
+    [\"5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY\", ${ALICE_INITIAL_BALANCE}]
+  ]
+}" > ${DATA_PATH}/initial-balances.json
+
+# Make Alice a member
+echo '
+  [{
+    "member_id": 0,
+    "root_account": "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY",
+    "controller_account": "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY",
+    "handle":"alice_with_a_long_handle",
+    "avatar_uri":"https://alice.com/avatar.png",
+    "about":"Alice",
+    "name": "Alice",
+    "registered_at_time": 0
+  },
+  {
+    "member_id": 1,
+    "root_account": "5FUeDYFzvvizNhhHyidsuchG7jnToKj7zfimbWBpWKzT9Fqe",
+    "controller_account": "5FUeDYFzvvizNhhHyidsuchG7jnToKj7zfimbWBpWKzT9Fqe",
+    "handle":"bob_with_a_long_handle",
+    "avatar_uri":"https://bob.com/avatar.png",
+    "about":"Bob",
+    "name": "Bob",
+    "registered_at_time": 0
+  }
+]
+' > ${DATA_PATH}/initial-members.json
+
+function cleanup() {
+    rm -Rf ${DATA_PATH}/alice
+}
+
+trap cleanup EXIT
+
+# Create a chain spec file
+./target/release/chain-spec-builder new -a Alice \
+  --chain-spec-path ${DATA_PATH}/chain-spec.json \
+  --initial-balances-path ${DATA_PATH}/initial-balances.json \
+  --initial-members-path ${DATA_PATH}/initial-members.json \
+  --deployment dev \
+  --sudo-account 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY
+
+./target/release/joystream-node --base-path ${DATA_PATH}/alice \
+  --validator --chain ${DATA_PATH}/chain-spec.json --alice --unsafe-ws-external --rpc-cors all

tests/network-tests/run-test-scenario.sh

@@ -10,4 +10,4 @@ SCENARIO=$1
 SCENARIO=${SCENARIO:=full}
 
 # Execute the tests
-time DEBUG=* yarn workspace network-tests node-ts-strict src/scenarios/${SCENARIO}.ts
+time DEBUG=integration-tests* yarn workspace network-tests node-ts-strict src/scenarios/${SCENARIO}.ts

tests/network-tests/run-tests.sh

@@ -95,7 +95,4 @@ fi
 # Display runtime version
 yarn workspace api-scripts tsnode-strict src/status.ts | grep Runtime
 
-echo "Waiting for chain to startup..."
-sleep 10
-
 ./run-test-scenario.sh $1

tests/network-tests/src/Api.ts

@@ -13,7 +13,7 @@ import {
   Opening as WorkingGroupOpening,
 } from '@joystream/types/working-group'
 import { ElectionStake, Seat } from '@joystream/types/council'
-import { AccountInfo, Balance, BalanceOf, BlockNumber, Event, EventRecord } from '@polkadot/types/interfaces'
+import { AccountInfo, Balance, BalanceOf, BlockNumber, EventRecord } from '@polkadot/types/interfaces'
 import BN from 'bn.js'
 import { SubmittableExtrinsic } from '@polkadot/api/types'
 import { Sender, LogLevel } from './sender'
@@ -35,7 +35,8 @@ import { VideoEntity } from '@joystream/cd-schemas/types/entities/VideoEntity'
 import { initializeContentDir, InputParser } from '@joystream/cd-schemas'
 import { OperationType } from '@joystream/types/content-directory'
 import { ContentId, DataObject } from '@joystream/types/media'
-import Debugger from 'debug'
+import { extendDebug } from './Debugger'
+import { InvertedPromise } from './InvertedPromise'
 
 export enum WorkingGroups {
   StorageWorkingGroup = 'storageWorkingGroup',
@@ -53,16 +54,16 @@ export class ApiFactory {
     treasuryAccountUri: string,
     sudoAccountUri: string
   ): Promise<ApiFactory> {
-    const debug = Debugger('api-factory')
+    const debug = extendDebug('api-factory')
     let connectAttempts = 0
     while (true) {
       connectAttempts++
       debug(`Connecting to chain, attempt ${connectAttempts}..`)
       try {
-        const api = await ApiPromise.create({ provider, types })
+        const api = new ApiPromise({ provider, types })
 
         // Wait for api to be connected and ready
-        await api.isReady
+        await api.isReadyOrError
 
         // If a node was just started up it might take a few seconds to start producing blocks
         // Give it a few seconds to be ready.
@@ -849,32 +850,36 @@ export class Api {
     }
   }
 
-  // Resolves to true when proposal finalized and executed successfully
-  // Resolved to false when proposal finalized and execution fails
-  public waitForProposalToFinalize(id: ProposalId): Promise<[boolean, EventRecord[]]> {
-    return new Promise(async (resolve) => {
-      const unsubscribe = await this.api.query.system.events<Vec<EventRecord>>((events) => {
-        events.forEach((record) => {
-          if (
-            record.event.method &&
-            record.event.method.toString() === 'ProposalStatusUpdated' &&
-            record.event.data[0].eq(id) &&
-            record.event.data[1].toString().includes('Executed')
-          ) {
-            unsubscribe()
-            resolve([true, events])
-          } else if (
-            record.event.method &&
-            record.event.method.toString() === 'ProposalStatusUpdated' &&
-            record.event.data[0].eq(id) &&
-            record.event.data[1].toString().includes('ExecutionFailed')
-          ) {
-            unsubscribe()
-            resolve([false, events])
-          }
-        })
+  // Subscribe to system events, resolves to an InvertedPromise or rejects if subscription fails.
+  // The inverted promise wraps a promise which resolves when the Proposal with id specified
+  // is executed.
+  // - On successful execution the wrapped promise resolves to `[true, events]`
+  // - On failed execution the wrapper promise resolves to `[false, events]`
+  public async subscribeToProposalExecutionResult(id: ProposalId): Promise<InvertedPromise<[boolean, EventRecord[]]>> {
+    const invertedPromise = new InvertedPromise<[boolean, EventRecord[]]>()
+    const unsubscribe = await this.api.query.system.events<Vec<EventRecord>>((events) => {
+      events.forEach((record) => {
+        if (
+          record.event.method &&
+          record.event.method.toString() === 'ProposalStatusUpdated' &&
+          record.event.data[0].eq(id) &&
+          record.event.data[1].toString().includes('executed')
+        ) {
+          unsubscribe()
+          invertedPromise.resolve([true, events])
+        } else if (
+          record.event.method &&
+          record.event.method.toString() === 'ProposalStatusUpdated' &&
+          record.event.data[0].eq(id) &&
+          record.event.data[1].toString().includes('executionFailed')
+        ) {
+          unsubscribe()
+          invertedPromise.resolve([false, events])
+        }
       })
     })
+
+    return invertedPromise
   }
 
   public findOpeningFilledEvent(
@@ -887,21 +892,6 @@ export class Api {
     }
   }
 
-  // Looks for the first occurance of an expected event, and resolves.
-  // Use this when the event we are expecting is not particular to a specific extrinsic
-  public waitForSystemEvent(eventName: string): Promise<Event> {
-    return new Promise(async (resolve) => {
-      const unsubscribe = await this.api.query.system.events<Vec<EventRecord>>((events) => {
-        events.forEach((record) => {
-          if (record.event.method && record.event.method.toString() === eventName) {
-            unsubscribe()
-            resolve(record.event)
-          }
-        })
-      })
-    })
-  }
-
   public findApplicationReviewBeganEvent(
     events: EventRecord[],
     workingGroup: WorkingGroups

tests/network-tests/src/Debugger.ts

@@ -0,0 +1,9 @@
+import Debugger from 'debug'
+
+// Global debugger
+const debug = Debugger('integration-tests')
+
+// Extend function bound to the global debugger
+const extendDebug = debug.extend.bind(debug)
+
+export { Debugger, debug, extendDebug }

tests/network-tests/src/Job.ts

@@ -1,4 +1,4 @@
-import Debugger from 'debug'
+import { Debugger, extendDebug } from './Debugger'
 import { EventEmitter } from 'events'
 import { ApiFactory } from './Api'
 import { QueryNodeApi } from './QueryNodeApi'
@@ -30,7 +30,7 @@ export class Job {
     this._flows = flows
     this._outcome = new InvertedPromise<JobOutcome>()
     this._manager.on('run', this.run.bind(this))
-    this.debug = Debugger(`job:${this._label}`)
+    this.debug = extendDebug(`job:${this._label}`)
   }
 
   // Depend on another job to complete successfully

tests/network-tests/src/Resources.ts

@@ -1,8 +1,8 @@
 import { assert } from 'chai'
 import { Utils } from './utils'
-import Debugger from 'debug'
+import { extendDebug } from './Debugger'
 
-const debug = Debugger('resources')
+const debug = extendDebug('resources')
 
 type NamedLocks = Record<Resource, Lock>
 export type ResourceLocker = (resource: Resource, timeout?: number) => Promise<() => void>
@@ -70,10 +70,11 @@ export class ResourceManager {
     return {
       [Resource.Council]: this.add(Resource.Council),
       // We assume that a flow will only have one active proposal at a time
-      // Runtime is configured for MaxActiveProposalLimit = 5
+      // Runtime is configured for MaxActiveProposalLimit = 20
       // So we should ensure we don't exceed that number of active proposals
       // which limits the number of concurrent tests that create proposals
-      [Resource.Proposals]: this.add(Resource.Proposals, 5),
+      // TODO: Get the value from api.consts.proposalsEngine.maxActiveProposalLimit
+      [Resource.Proposals]: this.add(Resource.Proposals, 20),
     }
   }
 

tests/network-tests/src/Scenario.ts

@@ -3,7 +3,7 @@ import { ApiFactory } from './Api'
 import { QueryNodeApi } from './QueryNodeApi'
 import { config } from 'dotenv'
 import { ApolloClient, InMemoryCache, HttpLink } from '@apollo/client'
-import Debugger from 'debug'
+import { Debugger, extendDebug } from './Debugger'
 import { Flow } from './Flow'
 import { Job } from './Job'
 import { JobManager } from './JobManager'
@@ -41,7 +41,7 @@ export async function scenario(scene: (props: ScenarioProps) => Promise<void>):
 
   const query = new QueryNodeApi(queryNodeProvider)
 
-  const debug = Debugger('scenario')
+  const debug = extendDebug('scenario')
 
   const jobs = new JobManager({ apiFactory, query, env })
 

tests/network-tests/src/fixtures/membershipModule.ts

@@ -3,7 +3,7 @@ import BN from 'bn.js'
 import { assert } from 'chai'
 import { BaseFixture } from '../Fixture'
 import { PaidTermId, MemberId } from '@joystream/types/members'
-import Debugger from 'debug'
+import { Debugger, extendDebug } from '../Debugger'
 
 export class BuyMembershipHappyCaseFixture extends BaseFixture {
   private accounts: string[]
@@ -15,7 +15,7 @@ export class BuyMembershipHappyCaseFixture extends BaseFixture {
     super(api)
     this.accounts = accounts
     this.paidTerms = paidTerms
-    this.debug = Debugger('fixture:BuyMembershipHappyCaseFixture')
+    this.debug = extendDebug('fixture:BuyMembershipHappyCaseFixture')
   }
 
   public getCreatedMembers(): MemberId[] {

tests/network-tests/src/fixtures/proposalsModule.ts

@@ -471,11 +471,13 @@ export class ElectionParametersProposalFixture extends BaseFixture {
       proposedMinCouncilStake,
       proposedMinVotingStake
     )
+
     const proposalNumber = this.api.findProposalCreatedEvent(proposalCreationResult.events) as ProposalId
     assert.notEqual(proposalNumber, undefined)
 
     const approveProposalFixture = new VoteForProposalFixture(this.api, proposalNumber)
     await approveProposalFixture.execute()
+
     assert(approveProposalFixture.proposalExecuted)
 
     // Assertions
@@ -733,10 +735,10 @@ export class VoteForProposalFixture extends BaseFixture {
     this.api.treasuryTransferBalanceToAccounts(councilAccounts, proposalVoteFee)
 
     // Approving the proposal
-    const onProposalFinalized = this.api.waitForProposalToFinalize(this.proposalNumber)
+    const proposalExecutionResult = await this.api.subscribeToProposalExecutionResult(this.proposalNumber)
     const approvals = await this.api.batchApproveProposal(this.proposalNumber)
     approvals.map((result) => this.expectDispatchSuccess(result, 'Proposal Approval Vote Expected To Be Successful'))
-    const proposalOutcome = await onProposalFinalized
+    const proposalOutcome = await proposalExecutionResult.promise
     this._proposalExecuted = proposalOutcome[0]
     this._events = proposalOutcome[1]
   }

tests/network-tests/src/flows/contentDirectory/contentDirectoryInitialization.ts

@@ -1,6 +1,6 @@
 import { FlowProps } from '../../Flow'
-import Debugger from 'debug'
-const debug = Debugger('initializeContentDirectory')
+import { extendDebug } from '../../Debugger'
+const debug = extendDebug('initializeContentDirectory')
 
 export default async function initializeContentDirectory({ api }: FlowProps): Promise<void> {
   debug('Started')

tests/network-tests/src/flows/contentDirectory/creatingChannel.ts

@@ -5,7 +5,7 @@ import { CreateChannelFixture } from '../../fixtures/contentDirectoryModule'
 import { ChannelEntity } from '@joystream/cd-schemas/types/entities/ChannelEntity'
 import { assert } from 'chai'
 import { FixtureRunner } from '../../Fixture'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 
 export function createSimpleChannelFixture(api: Api): CreateChannelFixture {
   const channelEntity: ChannelEntity = {
@@ -31,7 +31,7 @@ function assertChannelMatchQueriedResult(queriedChannel: any, channel: ChannelEn
 }
 
 export default async function channelCreation({ api, query }: FlowProps): Promise<void> {
-  const debug = Debugger('flow:creatingChannel')
+  const debug = extendDebug('flow:creatingChannel')
   debug('Started')
 
   const createChannelHappyCaseFixture = createSimpleChannelFixture(api)

tests/network-tests/src/flows/contentDirectory/creatingVideo.ts

@@ -5,7 +5,7 @@ import { VideoEntity } from '@joystream/cd-schemas/types/entities/VideoEntity'
 import { assert } from 'chai'
 import { Utils } from '../../utils'
 import { FixtureRunner } from '../../Fixture'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 
 export function createVideoReferencingChannelFixture(api: Api, handle: string): CreateVideoFixture {
   const videoEntity: VideoEntity = {
@@ -56,7 +56,7 @@ function assertVideoMatchQueriedResult(queriedVideo: any, video: VideoEntity) {
 }
 
 export default async function createVideo({ api, query }: FlowProps): Promise<void> {
-  const debug = Debugger('flow:creatingVideo')
+  const debug = extendDebug('flow:creatingVideo')
   debug('Started')
 
   const channelTitle = 'New channel example'

tests/network-tests/src/flows/contentDirectory/updatingChannel.ts

@@ -5,7 +5,7 @@ import { ChannelEntity } from '@joystream/cd-schemas/types/entities/ChannelEntit
 import { assert } from 'chai'
 import { Utils } from '../../utils'
 import { FixtureRunner } from '../../Fixture'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 
 export function createUpdateChannelHandleFixture(api: Api, handle: string, description: string): UpdateChannelFixture {
   // Create partial channel entity, only containing the fields we wish to update
@@ -19,7 +19,7 @@ export function createUpdateChannelHandleFixture(api: Api, handle: string, descr
 }
 
 export default async function updateChannel({ api, query }: FlowProps): Promise<void> {
-  const debug = Debugger('flow:updateChannel')
+  const debug = extendDebug('flow:updateChannel')
   debug('Started')
 
   const handle = 'New channel example'

tests/network-tests/src/flows/council/setup.ts

@@ -3,13 +3,13 @@ import { PaidTermId } from '@joystream/types/members'
 import { FlowProps } from '../../Flow'
 import { ElectCouncilFixture } from '../../fixtures/councilElectionModule'
 import { BuyMembershipHappyCaseFixture } from '../../fixtures/membershipModule'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 import { FixtureRunner } from '../../Fixture'
 import { Resource } from '../../Resources'
 
 export default async function councilSetup({ api, env, lock }: FlowProps): Promise<void> {
   const label = 'councilSetup'
-  const debug = Debugger(`flow:${label}`)
+  const debug = extendDebug(`flow:${label}`)
 
   debug('Started')
 

tests/network-tests/src/flows/membership/creatingMemberships.ts

@@ -5,12 +5,12 @@ import {
 } from '../../fixtures/membershipModule'
 import { PaidTermId } from '@joystream/types/members'
 import BN from 'bn.js'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 import { FixtureRunner } from '../../Fixture'
 import { assert } from 'chai'
 
 export default async function membershipCreation({ api, env }: FlowProps): Promise<void> {
-  const debug = Debugger('flow:memberships')
+  const debug = extendDebug('flow:memberships')
   debug('Started')
   api.enableDebugTxLogs()
 

tests/network-tests/src/flows/proposals/electionParametersProposal.ts

@@ -2,12 +2,12 @@ import { FlowProps } from '../../Flow'
 import { ElectionParametersProposalFixture } from '../../fixtures/proposalsModule'
 import { assert } from 'chai'
 import { FixtureRunner } from '../../Fixture'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 import { Resource } from '../../Resources'
 
 // Election parameters proposal scenario
 export default async function electionParametersProposal({ api, lock }: FlowProps): Promise<void> {
-  const debug = Debugger('flow:electionParametersProposal')
+  const debug = extendDebug('flow:electionParametersProposal')
   debug('Started')
   await lock(Resource.Proposals)
 
@@ -18,6 +18,7 @@ export default async function electionParametersProposal({ api, lock }: FlowProp
   const proposer = council[0].member.toString()
 
   const electionParametersProposalFixture = new ElectionParametersProposalFixture(api, proposer)
+
   await new FixtureRunner(electionParametersProposalFixture).run()
 
   debug('Done')

tests/network-tests/src/flows/proposals/manageLeaderRole.ts

@@ -19,7 +19,7 @@ import { ProposalId } from '@joystream/types/proposals'
 import { WorkerId } from '@joystream/types/working-group'
 import { assert } from 'chai'
 import { FixtureRunner } from '../../Fixture'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 import { Resource, ResourceLocker } from '../../Resources'
 
 export default {
@@ -32,7 +32,7 @@ export default {
 }
 
 async function manageLeaderRole(api: Api, env: NodeJS.ProcessEnv, group: WorkingGroups, lock: ResourceLocker) {
-  const debug = Debugger(`flow:managerLeaderRole:${group}`)
+  const debug = extendDebug(`flow:managerLeaderRole:${group}`)
   debug('Started')
   await lock(Resource.Proposals)
 

tests/network-tests/src/flows/proposals/spendingProposal.ts

@@ -3,11 +3,11 @@ import { FlowProps } from '../../Flow'
 import { SpendingProposalFixture } from '../../fixtures/proposalsModule'
 import { assert } from 'chai'
 import { FixtureRunner } from '../../Fixture'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 import { Resource } from '../../Resources'
 
 export default async function spendingProposal({ api, env, lock }: FlowProps): Promise<void> {
-  const debug = Debugger('flow:spendingProposals')
+  const debug = extendDebug('flow:spendingProposals')
   debug('Started')
   await lock(Resource.Proposals)
 

tests/network-tests/src/flows/proposals/textProposal.ts

@@ -2,11 +2,11 @@ import { FlowProps } from '../../Flow'
 import { TextProposalFixture } from '../../fixtures/proposalsModule'
 import { assert } from 'chai'
 import { FixtureRunner } from '../../Fixture'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 import { Resource } from '../../Resources'
 
 export default async function textProposal({ api, lock }: FlowProps): Promise<void> {
-  const debug = Debugger('flow:textProposal')
+  const debug = extendDebug('flow:textProposal')
   debug('Started')
   await lock(Resource.Proposals)
 

tests/network-tests/src/flows/proposals/updateRuntime.ts

@@ -5,11 +5,11 @@ import { UpdateRuntimeFixture } from '../../fixtures/proposalsModule'
 import { PaidTermId } from '@joystream/types/members'
 import { assert } from 'chai'
 import { FixtureRunner } from '../../Fixture'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 import { Resource } from '../../Resources'
 
 export default async function updateRuntime({ api, env, lock }: FlowProps): Promise<void> {
-  const debug = Debugger('flow:updateRuntime')
+  const debug = extendDebug('flow:updateRuntime')
   debug('Started')
   await lock(Resource.Proposals)
 

tests/network-tests/src/flows/proposals/validatorCountProposal.ts

@@ -3,11 +3,11 @@ import { FlowProps } from '../../Flow'
 import { ValidatorCountProposalFixture } from '../../fixtures/proposalsModule'
 import { assert } from 'chai'
 import { FixtureRunner } from '../../Fixture'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 import { Resource } from '../../Resources'
 
 export default async function validatorCount({ api, lock }: FlowProps): Promise<void> {
-  const debug = Debugger('flow:validatorCountProposal')
+  const debug = extendDebug('flow:validatorCountProposal')
   debug('Started')
   await lock(Resource.Proposals)
 

tests/network-tests/src/flows/proposals/workingGroupMintCapacityProposal.ts

@@ -5,7 +5,7 @@ import { VoteForProposalFixture, WorkingGroupMintCapacityProposalFixture } from
 import { ProposalId } from '@joystream/types/proposals'
 import { assert } from 'chai'
 import { FixtureRunner } from '../../Fixture'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 import { Resource, ResourceLocker } from '../../Resources'
 
 export default {
@@ -19,7 +19,7 @@ export default {
 }
 
 async function workingGroupMintCapactiy(api: Api, env: NodeJS.ProcessEnv, group: WorkingGroups, lock: ResourceLocker) {
-  const debug = Debugger(`flow:workingGroupMintCapacityProposal:${group}`)
+  const debug = extendDebug(`flow:workingGroupMintCapacityProposal:${group}`)
   debug('Started')
   await lock(Resource.Proposals)
 

tests/network-tests/src/flows/storageNode/getContentFromStorageNode.ts

@@ -5,10 +5,10 @@ import { registry } from '@joystream/types'
 
 import { FlowProps } from '../../Flow'
 import { Utils } from '../../utils'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 
 export default async function getContentFromStorageNode({ api, query }: FlowProps): Promise<void> {
-  const debug = Debugger('flow:getContentFromStorageNode')
+  const debug = extendDebug('flow:getContentFromStorageNode')
   debug('Started')
 
   const videoTitle = 'Storage node test'

tests/network-tests/src/flows/workingGroup/atLeastValueBug.ts

@@ -3,12 +3,12 @@ import { FlowProps } from '../../Flow'
 import { AddWorkerOpeningFixture } from '../../fixtures/workingGroupModule'
 import BN from 'bn.js'
 import { assert } from 'chai'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 import { FixtureRunner } from '../../Fixture'
 
 // Zero at least value bug scenario
 export default async function zeroAtLeastValueBug({ api, env }: FlowProps): Promise<void> {
-  const debug = Debugger('flow:atLeastValueBug')
+  const debug = extendDebug('flow:atLeastValueBug')
   debug('Started')
   api.enableDebugTxLogs()
 

tests/network-tests/src/flows/workingGroup/leaderSetup.ts

@@ -6,7 +6,7 @@ import { SudoHireLeadFixture } from '../../fixtures/sudoHireLead'
 import { assert } from 'chai'
 // import { KeyringPair } from '@polkadot/keyring/types'
 import { FixtureRunner } from '../../Fixture'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 
 export default {
   storage: async function ({ api, env }: FlowProps): Promise<void> {
@@ -19,7 +19,7 @@ export default {
 
 // Worker application happy case scenario
 async function leaderSetup(api: Api, env: NodeJS.ProcessEnv, group: WorkingGroups): Promise<void> {
-  const debug = Debugger(`flow:leaderSetup:${group}`)
+  const debug = extendDebug(`flow:leaderSetup:${group}`)
   debug('Started')
 
   const existingLead = await api.getGroupLead(group)

tests/network-tests/src/flows/workingGroup/manageWorkerAsLead.ts

@@ -13,7 +13,7 @@ import { BuyMembershipHappyCaseFixture } from '../../fixtures/membershipModule'
 import BN from 'bn.js'
 import { OpeningId } from '@joystream/types/hiring'
 import { assert } from 'chai'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 import { FixtureRunner } from '../../Fixture'
 
 export default {
@@ -26,7 +26,7 @@ export default {
 }
 
 async function manageWorkerAsLead(api: Api, env: NodeJS.ProcessEnv, group: WorkingGroups): Promise<void> {
-  const debug = Debugger(`flow:manageWorkerAsLead:${group}`)
+  const debug = extendDebug(`flow:manageWorkerAsLead:${group}`)
   debug('Started')
 
   const applicationStake: BN = new BN(env.WORKING_GROUP_APPLICATION_STAKE!)

tests/network-tests/src/flows/workingGroup/manageWorkerAsWorker.ts

@@ -13,7 +13,7 @@ import { OpeningId } from '@joystream/types/hiring'
 import { BuyMembershipHappyCaseFixture } from '../../fixtures/membershipModule'
 import { assert } from 'chai'
 import { FixtureRunner } from '../../Fixture'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 
 export default {
   storage: async function ({ api, env }: FlowProps): Promise<void> {
@@ -26,7 +26,7 @@ export default {
 
 // Manage worker as worker
 async function manageWorkerAsWorker(api: Api, env: NodeJS.ProcessEnv, group: WorkingGroups) {
-  const debug = Debugger(`flow:manageWorkerAsWorker:${group}`)
+  const debug = extendDebug(`flow:manageWorkerAsWorker:${group}`)
   debug('Started')
 
   const applicationStake: BN = new BN(env.WORKING_GROUP_APPLICATION_STAKE!)

tests/network-tests/src/flows/workingGroup/workerPayout.ts

@@ -15,7 +15,7 @@ import { ProposalId } from '@joystream/types/proposals'
 import { BuyMembershipHappyCaseFixture } from '../../fixtures/membershipModule'
 import { assert } from 'chai'
 import { FixtureRunner } from '../../Fixture'
-import Debugger from 'debug'
+import { extendDebug } from '../../Debugger'
 import { Resource, ResourceLocker } from '../../Resources'
 
 export default {
@@ -28,7 +28,7 @@ export default {
 }
 
 async function workerPayouts(api: Api, env: NodeJS.ProcessEnv, group: WorkingGroups, lock: ResourceLocker) {
-  const debug = Debugger(`flow:workerPayout:${group}`)
+  const debug = extendDebug(`flow:workerPayout:${group}`)
   debug('Started')
   await lock(Resource.Proposals)
 

tests/network-tests/src/sender.ts

@@ -5,7 +5,7 @@ import { AccountId, EventRecord } from '@polkadot/types/interfaces'
 import { DispatchError, DispatchResult } from '@polkadot/types/interfaces/system'
 import { TypeRegistry } from '@polkadot/types'
 import { KeyringPair } from '@polkadot/keyring/types'
-import Debugger from 'debug'
+import { Debugger, extendDebug } from './Debugger'
 import AsyncLock from 'async-lock'
 import { assert } from 'chai'
 
@@ -26,7 +26,7 @@ export class Sender {
   constructor(api: ApiPromise, keyring: Keyring, label: string) {
     this.api = api
     this.keyring = keyring
-    this.debug = Debugger(`Sender:${Sender.instance++}:${label}`)
+    this.debug = extendDebug(`sender:${Sender.instance++}:${label}`)
   }
 
   // Synchronize all sending of transactions into mempool, so we can always safely read

types/README.md

@@ -28,8 +28,6 @@ async function main() {
   // Create the API and wait until ready
   const api = await ApiPromise.create({ provider, types })
 
-  await api.isReady
-
   // Retrieve the chain & node information information via rpc calls
   const [chain, nodeName, nodeVersion] = await Promise.all([
     api.rpc.system.chain(),

utils/api-scripts/README.md

@@ -22,8 +22,6 @@ async function main() {
   // Create the API and wait until ready
   const api = await ApiPromise.create({ provider, types })
 
-  await api.isReady
-
   // Retrieve the chain & node information information via rpc calls
   const [chain, nodeName, nodeVersion] = await Promise.all([
     api.rpc.system.chain(),

utils/api-scripts/src/dev-set-runtime-code.ts

@@ -1,4 +1,4 @@
-import { ApiPromise } from '@polkadot/api'
+import { ApiPromise, WsProvider } from '@polkadot/api'
 import { types } from '@joystream/types'
 import { Keyring } from '@polkadot/keyring'
 import { ISubmittableResult } from '@polkadot/types/types/'
@@ -7,9 +7,6 @@ import { TypeRegistry } from '@polkadot/types'
 import fs from 'fs'
 import { compactAddLength } from '@polkadot/util'
 
-// Patched WsProvider with larger fragment size for messages
-import WsProvider from './patched-ws-provider'
-
 function onApiDisconnected() {
   process.exit(2)
 }
@@ -35,8 +32,8 @@ async function main() {
   let retry = 6
   while (true) {
     try {
-      api = await ApiPromise.create({ provider, types })
-      await api.isReady
+      api = new ApiPromise({ provider, types })
+      await api.isReadyOrError
       break
     } catch (err) {
       // failed to connect to node

utils/api-scripts/src/patched-ws-provider.ts

@@ -1,460 +0,0 @@
-// Copyright 2017-2020 @polkadot/rpc-provider authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-/* eslint-disable camelcase */
-
-// WsProvider implementation copied from @polkadot/api v1.26.1
-// It wasn't possible to extend or even monkey-patch it because of the
-// use of ECMAScript Private Fields.
-// The only modification is construction of the `w3cwebsocket` instance
-// in the connect() method to increase the fragment size to 256K from the
-// default of 16K
-
-import {
-  JsonRpcResponse,
-  ProviderInterface,
-  ProviderInterfaceCallback,
-  ProviderInterfaceEmitted,
-  ProviderInterfaceEmitCb,
-} from '@polkadot/rpc-provider/types'
-
-import EventEmitter from 'eventemitter3'
-import { assert, isNull, isUndefined, isChildClass, logger } from '@polkadot/util'
-
-import Coder from '@polkadot/rpc-provider/coder'
-import defaults from '@polkadot/rpc-provider/defaults'
-import getWSClass from '@polkadot/rpc-provider/ws/getWSClass'
-
-interface SubscriptionHandler {
-  callback: ProviderInterfaceCallback
-  type: string
-}
-
-interface WsStateAwaiting {
-  callback: ProviderInterfaceCallback
-  method: string
-  params: any[]
-  subscription?: SubscriptionHandler
-}
-
-interface WsStateSubscription extends SubscriptionHandler {
-  method: string
-  params: any[]
-}
-
-interface WSProviderInterface extends ProviderInterface {
-  connect(): void
-}
-
-const ALIASSES: { [index: string]: string } = {
-  chain_finalisedHead: 'chain_finalizedHead',
-  chain_subscribeFinalisedHeads: 'chain_subscribeFinalizedHeads',
-  chain_unsubscribeFinalisedHeads: 'chain_unsubscribeFinalizedHeads',
-}
-
-const l = logger('api-ws')
-
-/**
- * # @polkadot/rpc-provider/ws
- *
- * @name WsProvider
- *
- * @description The WebSocket Provider allows sending requests using WebSocket to a WebSocket RPC server TCP port. Unlike the [[HttpProvider]], it does support subscriptions and allows listening to events such as new blocks or balance changes.
- *
- * @example
- * <BR>
- *
- * ```javascript
- * import Api from '@polkadot/api/promise';
- * import WsProvider from '@polkadot/rpc-provider/ws';
- *
- * const provider = new WsProvider('ws://127.0.0.1:9944');
- * const api = new Api(provider);
- * ```
- *
- * @see [[HttpProvider]]
- */
-export default class WsProvider implements WSProviderInterface {
-  readonly coder: Coder
-
-  readonly endpoints: string[]
-
-  readonly headers: Record<string, string>
-
-  readonly eventemitter: EventEmitter
-
-  readonly handlers: Record<string, WsStateAwaiting> = {}
-
-  readonly queued: Record<string, string> = {}
-
-  readonly waitingForId: Record<string, JsonRpcResponse> = {}
-
-  private autoConnectMs: number
-
-  private endpointIndex: number
-
-  private _isConnected = false
-
-  private subscriptions: Record<string, WsStateSubscription> = {}
-
-  private websocket: WebSocket | null
-
-  /**
-   * @param {string | string[]}  endpoint    The endpoint url. Usually `ws://ip:9944` or `wss://ip:9944`, may provide an array of endpoint strings.
-   * @param {boolean} autoConnect Whether to connect automatically or not.
-   */
-  constructor(
-    endpoint: string | string[] = defaults.WS_URL,
-    autoConnectMs: number | false = 1000,
-    headers: Record<string, string> = {}
-  ) {
-    const endpoints = Array.isArray(endpoint) ? endpoint : [endpoint]
-
-    assert(endpoints.length !== 0, 'WsProvider requires at least one Endpoint')
-
-    endpoints.forEach((endpoint) => {
-      assert(/^(wss|ws):\/\//.test(endpoint), `Endpoint should start with 'ws://', received '${endpoint}'`)
-    })
-
-    this.eventemitter = new EventEmitter()
-    this.autoConnectMs = autoConnectMs || 0
-    this.coder = new Coder()
-    this.endpointIndex = -1
-    this.endpoints = endpoints
-    this.headers = headers
-    this.websocket = null
-
-    if (autoConnectMs > 0) {
-      // eslint-disable-next-line @typescript-eslint/no-floating-promises
-      this.connect()
-    }
-  }
-
-  /**
-   * @summary `true` when this provider supports subscriptions
-   */
-  public get hasSubscriptions(): boolean {
-    return true
-  }
-
-  /**
-   * @description Returns a clone of the object
-   */
-  public clone(): WsProvider {
-    return new WsProvider(this.endpoints)
-  }
-
-  /**
-   * @summary Manually connect
-   * @description The [[WsProvider]] connects automatically by default, however if you decided otherwise, you may
-   * connect manually using this method.
-   */
-  public async connect(): Promise<void> {
-    try {
-      this.endpointIndex = (this.endpointIndex + 1) % this.endpoints.length
-
-      const WS = await getWSClass()
-
-      this.websocket =
-        typeof WebSocket !== 'undefined' && isChildClass(WebSocket, WS)
-          ? new WS(this.endpoints[this.endpointIndex])
-          : // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-            // @ts-ignore - WS may be an instance of w3cwebsocket, which supports headers
-            new WS(this.endpoints[this.endpointIndex], undefined, undefined, this.headers, undefined, {
-              // default: true
-              fragmentOutgoingMessages: true,
-              // default: 16K
-              fragmentationThreshold: 256 * 1024,
-            })
-      this.websocket.onclose = this.onSocketClose
-      this.websocket.onerror = this.onSocketError
-      this.websocket.onmessage = this.onSocketMessage
-      this.websocket.onopen = this.onSocketOpen
-    } catch (error) {
-      l.error(error)
-    }
-  }
-
-  /**
-   * @description Manually disconnect from the connection, clearing autoconnect logic
-   */
-  public disconnect(): void {
-    if (isNull(this.websocket)) {
-      throw new Error('Cannot disconnect on a non-open websocket')
-    }
-
-    // switch off autoConnect, we are in manual mode now
-    this.autoConnectMs = 0
-
-    // 1000 - Normal closure; the connection successfully completed
-    this.websocket.close(1000)
-    this.websocket = null
-  }
-
-  /**
-   * @summary Whether the node is connected or not.
-   * @return {boolean} true if connected
-   */
-  public isConnected(): boolean {
-    return this._isConnected
-  }
-
-  /**
-   * @summary Listens on events after having subscribed using the [[subscribe]] function.
-   * @param  {ProviderInterfaceEmitted} type Event
-   * @param  {ProviderInterfaceEmitCb}  sub  Callback
-   * @return unsubscribe function
-   */
-  public on(type: ProviderInterfaceEmitted, sub: ProviderInterfaceEmitCb): () => void {
-    this.eventemitter.on(type, sub)
-
-    return (): void => {
-      this.eventemitter.removeListener(type, sub)
-    }
-  }
-
-  /**
-   * @summary Send JSON data using WebSockets to configured HTTP Endpoint or queue.
-   * @param method The RPC methods to execute
-   * @param params Encoded paramaters as appliucable for the method
-   * @param subscription Subscription details (internally used)
-   */
-  public send(method: string, params: any[], subscription?: SubscriptionHandler): Promise<any> {
-    return new Promise((resolve, reject): void => {
-      try {
-        const json = this.coder.encodeJson(method, params)
-        const id = this.coder.getId()
-
-        const callback = (error?: Error | null, result?: any): void => {
-          error ? reject(error) : resolve(result)
-        }
-
-        l.debug((): string[] => ['calling', method, json])
-
-        this.handlers[id] = {
-          callback,
-          method,
-          params,
-          subscription,
-        }
-
-        if (this.isConnected() && !isNull(this.websocket)) {
-          this.websocket.send(json)
-        } else {
-          this.queued[id] = json
-        }
-      } catch (error) {
-        reject(error)
-      }
-    })
-  }
-
-  /**
-   * @name subscribe
-   * @summary Allows subscribing to a specific event.
-   * @param  {string}                     type     Subscription type
-   * @param  {string}                     method   Subscription method
-   * @param  {any[]}                 params   Parameters
-   * @param  {ProviderInterfaceCallback} callback Callback
-   * @return {Promise<number>}                     Promise resolving to the dd of the subscription you can use with [[unsubscribe]].
-   *
-   * @example
-   * <BR>
-   *
-   * ```javascript
-   * const provider = new WsProvider('ws://127.0.0.1:9944');
-   * const rpc = new Rpc(provider);
-   *
-   * rpc.state.subscribeStorage([[storage.system.account, <Address>]], (_, values) => {
-   *   console.log(values)
-   * }).then((subscriptionId) => {
-   *   console.log('balance changes subscription id: ', subscriptionId)
-   * })
-   * ```
-   */
-  public async subscribe(
-    type: string,
-    method: string,
-    params: any[],
-    callback: ProviderInterfaceCallback
-  ): Promise<number | string> {
-    const id = (await this.send(method, params, { callback, type })) as Promise<number | string>
-
-    return id
-  }
-
-  /**
-   * @summary Allows unsubscribing to subscriptions made with [[subscribe]].
-   */
-  public async unsubscribe(type: string, method: string, id: number | string): Promise<boolean> {
-    const subscription = `${type}::${id}`
-
-    // FIXME This now could happen with re-subscriptions. The issue is that with a re-sub
-    // the assigned id now does not match what the API user originally received. It has
-    // a slight complication in solving - since we cannot rely on the send id, but rather
-    // need to find the actual subscription id to map it
-    if (isUndefined(this.subscriptions[subscription])) {
-      l.debug((): string => `Unable to find active subscription=${subscription}`)
-
-      return false
-    }
-
-    delete this.subscriptions[subscription]
-
-    const result = (await this.send(method, [id])) as Promise<boolean>
-
-    return result
-  }
-
-  emit = (type: ProviderInterfaceEmitted, ...args: any[]): void => {
-    this.eventemitter.emit(type, ...args)
-  }
-
-  onSocketClose = (event: CloseEvent): void => {
-    if (this.autoConnectMs > 0) {
-      l.error(`disconnected from ${this.endpoints[this.endpointIndex]} code: '${event.code}' reason: '${event.reason}'`)
-    }
-
-    this._isConnected = false
-    this.emit('disconnected')
-
-    if (this.autoConnectMs > 0) {
-      setTimeout((): void => {
-        // eslint-disable-next-line @typescript-eslint/no-floating-promises
-        this.connect()
-      }, this.autoConnectMs)
-    }
-  }
-
-  onSocketError = (error: Event): void => {
-    l.debug((): any => ['socket error', error])
-    this.emit('error', error)
-  }
-
-  onSocketMessage = (message: MessageEvent): void => {
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-return
-    l.debug(() => ['received', message.data])
-
-    const response = JSON.parse(message.data as string) as JsonRpcResponse
-
-    return isUndefined(response.method) ? this.onSocketMessageResult(response) : this.onSocketMessageSubscribe(response)
-  }
-
-  onSocketMessageResult = (response: JsonRpcResponse): void => {
-    const handler = this.handlers[response.id]
-
-    if (!handler) {
-      l.debug((): string => `Unable to find handler for id=${response.id}`)
-
-      return
-    }
-
-    try {
-      const { method, params, subscription } = handler
-      const result = this.coder.decodeResponse(response) as string
-
-      // first send the result - in case of subs, we may have an update
-      // immediately if we have some queued results already
-      handler.callback(null, result)
-
-      if (subscription) {
-        const subId = `${subscription.type}::${result}`
-
-        this.subscriptions[subId] = {
-          ...subscription,
-          method,
-          params,
-        }
-
-        // if we have a result waiting for this subscription already
-        if (this.waitingForId[subId]) {
-          this.onSocketMessageSubscribe(this.waitingForId[subId])
-        }
-      }
-    } catch (error) {
-      handler.callback(error, undefined)
-    }
-
-    delete this.handlers[response.id]
-  }
-
-  onSocketMessageSubscribe = (response: JsonRpcResponse): void => {
-    const method = ALIASSES[response.method as string] || response.method || 'invalid'
-    const subId = `${method}::${response.params.subscription}`
-    const handler = this.subscriptions[subId]
-
-    if (!handler) {
-      // store the JSON, we could have out-of-order subid coming in
-      this.waitingForId[subId] = response
-
-      l.debug((): string => `Unable to find handler for subscription=${subId}`)
-
-      return
-    }
-
-    // housekeeping
-    delete this.waitingForId[subId]
-
-    try {
-      const result = this.coder.decodeResponse(response)
-
-      handler.callback(null, result)
-    } catch (error) {
-      handler.callback(error, undefined)
-    }
-  }
-
-  onSocketOpen = (): boolean => {
-    assert(!isNull(this.websocket), 'WebSocket cannot be null in onOpen')
-
-    l.debug((): any[] => ['connected to', this.endpoints[this.endpointIndex]])
-
-    this._isConnected = true
-
-    this.emit('connected')
-    this.sendQueue()
-    this.resubscribe()
-
-    return true
-  }
-
-  resubscribe = (): void => {
-    const subscriptions = this.subscriptions
-
-    this.subscriptions = {}
-
-    // eslint-disable-next-line @typescript-eslint/no-misused-promises
-    Object.keys(subscriptions).forEach(
-      async (id): Promise<void> => {
-        const { callback, method, params, type } = subscriptions[id]
-
-        // only re-create subscriptions which are not in author (only area where
-        // transactions are created, i.e. submissions such as 'author_submitAndWatchExtrinsic'
-        // are not included (and will not be re-broadcast)
-        if (type.startsWith('author_')) {
-          return
-        }
-
-        try {
-          await this.subscribe(type, method, params, callback)
-        } catch (error) {
-          l.error(error)
-        }
-      }
-    )
-  }
-
-  sendQueue = (): void => {
-    Object.keys(this.queued).forEach((id): void => {
-      try {
-        // we have done the websocket check in onSocketOpen, if an issue, will catch it
-        ;(this.websocket as WebSocket).send(this.queued[id])
-
-        delete this.queued[id]
-      } catch (error) {
-        l.error(error)
-      }
-    })
-  }
-}

utils/api-scripts/src/script.ts

@@ -22,7 +22,7 @@ async function main() {
 
   const provider = new WsProvider('ws://127.0.0.1:9944')
 
-  const api = await ApiPromise.create({ provider, types: joyTypes })
+  const api = new ApiPromise({ provider, types: joyTypes })
 
   await api.isReady
 

utils/api-scripts/src/status.ts

@@ -15,11 +15,12 @@ async function main() {
   let retry = 6
   while (true) {
     try {
-      api = await ApiPromise.create({ provider, types })
-      await api.isReady
+      api = new ApiPromise({ provider, types })
+      await api.isReadyOrError
       break
     } catch (err) {
       // failed to connect to node
+      console.error('Caught Error', err)
     }
 
     if (retry-- === 0) {

yarn.lock

@@ -4969,6 +4969,11 @@
   resolved "https://registry.yarnpkg.com/@types/faker/-/faker-4.1.12.tgz#065d37343677df1aa757c622650bd14666c42602"
   integrity sha512-0MEyzJrLLs1WaOCx9ULK6FzdCSj2EuxdSP9kvuxxdBEGujZYUOZ4vkPXdgu3dhyg/pOdn7VCatelYX7k0YShlA==
 
+"@types/figlet@^1.2.0":
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/@types/figlet/-/figlet-1.2.1.tgz#ae3ba62ae70fc2aa10ac76b42541a99680b080a1"
+  integrity sha512-uw+Xmam7sMFHYyVGq5CGH/6Lu0T792bf4T6+jPFtPkWnJ9YVjLKqOLzF7DHuSo/LEHtLWleUO7Zi7EKkg6rSIg==
+
 "@types/file-saver@^2.0.1":
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/@types/file-saver/-/file-saver-2.0.1.tgz#e18eb8b069e442f7b956d313f4fadd3ef887354e"
@@ -8204,7 +8209,7 @@ bluebird@^3.1.1, bluebird@^3.3.5, bluebird@^3.5.1, bluebird@^3.5.3, bluebird@^3.
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
   integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
 
-bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.1.1, bn.js@^4.11.8, bn.js@^4.11.9, bn.js@^4.4.0, bn.js@^5.1.2, bn.js@^5.1.3:
+bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.1.1, bn.js@^4.11.8, bn.js@^4.11.9, bn.js@^4.4.0, bn.js@^5.1.1, bn.js@^5.1.2, bn.js@^5.1.3:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-5.1.2.tgz#c9686902d3c9a27729f43ab10f9d79c2004da7b0"
   integrity sha512-40rZaf3bUNKTVYu9sIeeEGOg7g14Yvnj9kH7b50EiwX0Q7A6umbvfI5tvHaOERH0XigqKkfLkFQxzb4e6CIXnA==
@@ -9406,18 +9411,6 @@ cli-cursor@^3.1.0:
   dependencies:
     restore-cursor "^3.1.0"
 
-cli-highlight@^2.1.10:
-  version "2.1.10"
-  resolved "https://registry.yarnpkg.com/cli-highlight/-/cli-highlight-2.1.10.tgz#26a087da9209dce4fcb8cf5427dc97cd96ac173a"
-  integrity sha512-CcPFD3JwdQ2oSzy+AMG6j3LRTkNjM82kzcSKzoVw6cLanDCJNlsLjeqVTOTfOfucnWv5F0rmBemVf1m9JiIasw==
-  dependencies:
-    chalk "^4.0.0"
-    highlight.js "^10.0.0"
-    mz "^2.4.0"
-    parse5 "^5.1.1"
-    parse5-htmlparser2-tree-adapter "^6.0.0"
-    yargs "^16.0.0"
-
 cli-highlight@^2.1.4:
   version "2.1.4"
   resolved "https://registry.yarnpkg.com/cli-highlight/-/cli-highlight-2.1.4.tgz#098cb642cf17f42adc1c1145e07f960ec4d7522b"
@@ -10879,6 +10872,16 @@ date-fns@^2.0.1:
   resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.15.0.tgz#424de6b3778e4e69d3ff27046ec136af58ae5d5f"
   integrity sha512-ZCPzAMJZn3rNUvvQIMlXhDr4A+Ar07eLeGsGREoWU19a3Pqf5oYa+ccd+B3F6XVtQY6HANMFdOQ8A+ipFnvJdQ==
 
+date-format@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/date-format/-/date-format-2.1.0.tgz#31d5b5ea211cf5fd764cd38baf9d033df7e125cf"
+  integrity sha512-bYQuGLeFxhkxNOF3rcMtiZxvCBAquGzZm6oWA1oZ0g2THUzivaRhv8uOhdr19LmoobSOLoIAxeUK2RdbM8IFTA==
+
+date-format@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/date-format/-/date-format-3.0.0.tgz#eb8780365c7d2b1511078fb491e6479780f3ad95"
+  integrity sha512-eyTcpKOcamdhWJXj56DpQMo1ylSQpcGtGKXcU0Tb97+K56/CF5amAqqqNj0+KvA0iw2ynxtHWFsPDSClCxe48w==
+
 date-utils@*:
   version "1.2.21"
   resolved "https://registry.yarnpkg.com/date-utils/-/date-utils-1.2.21.tgz#61fb16cdc1274b3c9acaaffe9fc69df8720a2b64"
@@ -13322,7 +13325,7 @@ figgy-pudding@^3.4.1, figgy-pudding@^3.5.1:
   resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.1.tgz#862470112901c727a0e495a80744bd5baa1d6790"
   integrity sha512-vNKxJHTEKNThjfrdJwHc7brvM6eVevuO5nTj6ez8ZQ1qbXTvGthucRF7S4vf2cr71QVnT70V34v0S1DyQsti0w==
 
-figlet@^1.1.1:
+figlet@^1.1.1, figlet@^1.4.0:
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/figlet/-/figlet-1.5.0.tgz#2db4d00a584e5155a96080632db919213c3e003c"
   integrity sha512-ZQJM4aifMpz6H19AW1VqvZ7l4pOE9p7i/3LyxgO2kp+PO/VcDYNqIHEMtkccqIhTXMKci4kjueJr/iCQEaT/Ww==
@@ -13638,6 +13641,11 @@ flatted@^2.0.0:
   resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.1.tgz#69e57caa8f0eacbc281d2e2cb458d46fdb449e08"
   integrity sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==
 
+flatted@^2.0.1:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.2.tgz#4575b21e2bcee7434aa9be662f4b7b5f9c2b5138"
+  integrity sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==
+
 flatten@^1.0.2:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/flatten/-/flatten-1.0.3.tgz#c1283ac9f27b368abc1e36d1ff7b04501a30356b"
@@ -19715,6 +19723,17 @@ log-update@^2.3.0:
     cli-cursor "^2.0.0"
     wrap-ansi "^3.0.1"
 
+log4js@^6.2.1:
+  version "6.3.0"
+  resolved "https://registry.yarnpkg.com/log4js/-/log4js-6.3.0.tgz#10dfafbb434351a3e30277a00b9879446f715bcb"
+  integrity sha512-Mc8jNuSFImQUIateBFwdOQcmC6Q5maU0VVvdC2R6XMb66/VnT+7WS4D/0EeNMZu1YODmJe5NIn2XftCzEocUgw==
+  dependencies:
+    date-format "^3.0.0"
+    debug "^4.1.1"
+    flatted "^2.0.1"
+    rfdc "^1.1.4"
+    streamroller "^2.2.4"
+
 loglevel@^1.6.7:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.7.0.tgz#728166855a740d59d38db01cf46f042caa041bb0"
@@ -22563,13 +22582,6 @@ parse5-htmlparser2-tree-adapter@^5.1.1:
   dependencies:
     parse5 "^5.1.1"
 
-parse5-htmlparser2-tree-adapter@^6.0.0:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-6.0.1.tgz#2cdf9ad823321140370d4dbf5d3e92c7c8ddc6e6"
-  integrity sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==
-  dependencies:
-    parse5 "^6.0.1"
-
 parse5@4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/parse5/-/parse5-4.0.0.tgz#6d78656e3da8d78b4ec0b906f7c08ef1dfe3f608"
@@ -22580,11 +22592,6 @@ parse5@5.1.1, parse5@^5.0.0, parse5@^5.1.1:
   resolved "https://registry.yarnpkg.com/parse5/-/parse5-5.1.1.tgz#f68e4e5ba1852ac2cadc00f4555fff6c2abb6178"
   integrity sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==
 
-parse5@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/parse5/-/parse5-6.0.1.tgz#e1a1c085c569b3dc08321184f19a39cc27f7c30b"
-  integrity sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==
-
 parseurl@^1.3.2, parseurl@^1.3.3, parseurl@~1.3.2, parseurl@~1.3.3:
   version "1.3.3"
   resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4"
@@ -22711,6 +22718,14 @@ path-type@^4.0.0:
   resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b"
   integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==
 
+path@^0.12.7:
+  version "0.12.7"
+  resolved "https://registry.yarnpkg.com/path/-/path-0.12.7.tgz#d4dc2a506c4ce2197eb481ebfcd5b36c0140b10f"
+  integrity sha1-1NwqUGxM4hl+tIHr/NWzbAFAsQ8=
+  dependencies:
+    process "^0.11.1"
+    util "^0.10.3"
+
 pathval@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/pathval/-/pathval-1.1.0.tgz#b942e6d4bde653005ef6b71361def8727d0645e0"
@@ -22774,25 +22789,11 @@ pg-connection-string@^2.4.0:
   resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.4.0.tgz#c979922eb47832999a204da5dbe1ebf2341b6a10"
   integrity sha512-3iBXuv7XKvxeMrIgym7njT+HlZkwZqqGX4Bu9cci8xHZNT+Um1gWKqCsAzcC0d95rcKMU5WBg6YRUcHyV0HZKQ==
 
-pg-format@^1.0.4:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/pg-format/-/pg-format-1.0.4.tgz#27734236c2ad3f4e5064915a59334e20040a828e"
-  integrity sha1-J3NCNsKtP05QZJFaWTNOIAQKgo4=
-
 pg-int8@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c"
   integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==
 
-pg-listen@^1.7.0:
-  version "1.7.0"
-  resolved "https://registry.yarnpkg.com/pg-listen/-/pg-listen-1.7.0.tgz#5a5c68a1cabf88d2b78ed9cf133667f597d3b860"
-  integrity sha512-MKDwKLm4ryhy7iq1yw1K1MvUzBdTkaT16HZToddX9QaT8XSdt3Kins5mYH6DLECGFzFWG09VdXvWOIYogjXrsg==
-  dependencies:
-    debug "^4.1.1"
-    pg-format "^1.0.4"
-    typed-emitter "^0.1.0"
-
 pg-packet-stream@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/pg-packet-stream/-/pg-packet-stream-1.1.0.tgz#e45c3ae678b901a2873af1e17b92d787962ef914"
@@ -22821,6 +22822,11 @@ pg-protocol@^1.3.0:
   resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.3.0.tgz#3c8fb7ca34dbbfcc42776ce34ac5f537d6e34770"
   integrity sha512-64/bYByMrhWULUaCd+6/72c9PMWhiVFs3EVxl9Ct6a3v/U8+rKgqP2w+kKg/BIGgMJyB+Bk/eNivT32Al+Jghw==
 
+pg-protocol@^1.4.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.4.0.tgz#43a71a92f6fe3ac559952555aa3335c8cb4908be"
+  integrity sha512-El+aXWcwG/8wuFICMQjM5ZSAm6OWiJicFdNYo+VY3QP+8vI4SvLIWVe51PppTzMhikUJR+PsyIFKqfdXPz/yxA==
+
 pg-types@1.*:
   version "1.13.0"
   resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.13.0.tgz#75f490b8a8abf75f1386ef5ec4455ecf6b345c63"
@@ -22871,6 +22877,19 @@ pg@^7.12.1:
     pgpass "1.x"
     semver "4.3.2"
 
+pg@^8.0.3:
+  version "8.5.1"
+  resolved "https://registry.yarnpkg.com/pg/-/pg-8.5.1.tgz#34dcb15f6db4a29c702bf5031ef2e1e25a06a120"
+  integrity sha512-9wm3yX9lCfjvA98ybCyw2pADUivyNWT/yIP4ZcDVpMN0og70BUWYEGXPCTAQdGTAqnytfRADb7NERrY1qxhIqw==
+  dependencies:
+    buffer-writer "2.0.0"
+    packet-reader "1.0.0"
+    pg-connection-string "^2.4.0"
+    pg-pool "^3.2.2"
+    pg-protocol "^1.4.0"
+    pg-types "^2.1.0"
+    pgpass "1.x"
+
 pg@^8.3.3:
   version "8.4.2"
   resolved "https://registry.yarnpkg.com/pg/-/pg-8.4.2.tgz#2aa58166a23391e91d56a7ea57c6d99931c0642a"
@@ -23719,7 +23738,7 @@ process-nextick-args@~1.0.6:
   resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3"
   integrity sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=
 
-process@^0.11.10:
+process@^0.11.1, process@^0.11.10:
   version "0.11.10"
   resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182"
   integrity sha1-czIwDoQBYb2j5podHZGn1LwW8YI=
@@ -25606,6 +25625,11 @@ reusify@^1.0.0:
   resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76"
   integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==
 
+rfdc@^1.1.4:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/rfdc/-/rfdc-1.3.0.tgz#d0b7c441ab2720d05dc4cf26e01c89631d9da08b"
+  integrity sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==
+
 rgb-regex@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/rgb-regex/-/rgb-regex-1.0.1.tgz#c0e0d6882df0e23be254a475e8edd41915feaeb1"
@@ -26926,6 +26950,15 @@ stream-to-pull-stream@^1.7.2:
     looper "^3.0.0"
     pull-stream "^3.2.3"
 
+streamroller@^2.2.4:
+  version "2.2.4"
+  resolved "https://registry.yarnpkg.com/streamroller/-/streamroller-2.2.4.tgz#c198ced42db94086a6193608187ce80a5f2b0e53"
+  integrity sha512-OG79qm3AujAM9ImoqgWEY1xG4HX+Lw+yY6qZj9R1K2mhF5bEmQ849wvrb+4vt4jLMLzwXttJlQbOdPOQVRv7DQ==
+  dependencies:
+    date-format "^2.1.0"
+    debug "^4.1.1"
+    fs-extra "^8.1.0"
+
 streamsearch@0.1.2:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a"
@@ -28232,7 +28265,7 @@ ts-log@^2.1.4:
   resolved "https://registry.yarnpkg.com/ts-log/-/ts-log-2.1.4.tgz#063c5ad1cbab5d49d258d18015963489fb6fb59a"
   integrity sha512-P1EJSoyV+N3bR/IWFeAqXzKPZwHpnLY6j7j58mAvewHRipo+BQM2Y1f9Y9BjEQznKwgqqZm7H8iuixmssU7tYQ==
 
-ts-node-dev@^1.0.0-pre.60:
+ts-node-dev@^1.0.0-pre.40, ts-node-dev@^1.0.0-pre.60:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/ts-node-dev/-/ts-node-dev-1.1.6.tgz#ee2113718cb5a92c1c8f4229123ad6afbeba01f8"
   integrity sha512-RTUi7mHMNQospArGz07KiraQcdgUVNXKsgO2HAi7FoiyPMdTDqdniB6K1dqyaIxT7c9v/VpSbfBZPS6uVpaFLQ==
@@ -28472,11 +28505,6 @@ type@^2.0.0:
   resolved "https://registry.yarnpkg.com/type/-/type-2.0.0.tgz#5f16ff6ef2eb44f260494dae271033b29c09a9c3"
   integrity sha512-KBt58xCHry4Cejnc2ISQAF7QY+ORngsWfxezO68+12hKV6lQY8P/psIkcbjeHWn7MqcgciWJyCCevFMJdIXpow==
 
-typed-emitter@^0.1.0:
-  version "0.1.0"
-  resolved "https://registry.yarnpkg.com/typed-emitter/-/typed-emitter-0.1.0.tgz#ca532f100ccbf850e3a73b8ebf43d43e4f1f3849"
-  integrity sha512-Tfay0l6gJMP5rkil8CzGbLthukn+9BN/VXWcABVFPjOoelJ+koW8BuPZYk+h/L+lEeIp1fSzVRiWRPIjKVjPdg==
-
 typed-styles@^0.0.7:
   version "0.0.7"
   resolved "https://registry.yarnpkg.com/typed-styles/-/typed-styles-0.0.7.tgz#93392a008794c4595119ff62dde6809dbc40a3d9"
@@ -28581,28 +28609,6 @@ typeorm@^0.2.25, typeorm@^0.2.26:
     yargonaut "^1.1.2"
     yargs "^16.0.3"
 
-typeorm@^0.2.31:
-  version "0.2.31"
-  resolved "https://registry.yarnpkg.com/typeorm/-/typeorm-0.2.31.tgz#82b8a1b233224f81c738f53b0380386ccf360917"
-  integrity sha512-dVvCEVHH48DG0QPXAKfo0l6ecQrl3A8ucGP4Yw4myz4YEDMProebTQo8as83uyES+nrwCbu3qdkL4ncC2+qcMA==
-  dependencies:
-    "@sqltools/formatter" "1.2.2"
-    app-root-path "^3.0.0"
-    buffer "^5.5.0"
-    chalk "^4.1.0"
-    cli-highlight "^2.1.10"
-    debug "^4.1.1"
-    dotenv "^8.2.0"
-    glob "^7.1.6"
-    js-yaml "^3.14.0"
-    mkdirp "^1.0.4"
-    reflect-metadata "^0.1.13"
-    sha.js "^2.4.11"
-    tslib "^1.13.0"
-    xml2js "^0.4.23"
-    yargonaut "^1.1.2"
-    yargs "^16.0.3"
-
 typescript-formatter@^7.2.2:
   version "7.2.2"
   resolved "https://registry.yarnpkg.com/typescript-formatter/-/typescript-formatter-7.2.2.tgz#a147181839b7bb09c2377b072f20f6336547c00a"
@@ -28611,7 +28617,7 @@ typescript-formatter@^7.2.2:
     commandpost "^1.0.0"
     editorconfig "^0.15.0"
 
-typescript@^3.0.3, typescript@^3.7.2, typescript@^3.7.5, typescript@^3.8.3, typescript@^3.9.5, typescript@^3.9.6, typescript@^3.9.7:
+typescript@3.5.2, typescript@^3.0.3, typescript@^3.7.2, typescript@^3.7.5, typescript@^3.8.3, typescript@^3.9.5, typescript@^3.9.6, typescript@^3.9.7:
   version "3.9.7"
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.7.tgz#98d600a5ebdc38f40cb277522f12dc800e9e25fa"
   integrity sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==
@@ -29651,69 +29657,6 @@ warning@^4.0.2, warning@^4.0.3:
     typeorm-typedi-extensions "^0.2.3"
     typescript "^3.9.7"
 
-"warthog@https://github.com/metmirr/warthog/releases/download/v2.23.0/warthog-v2.23.0.tgz":
-  version "2.23.0"
-  resolved "https://github.com/metmirr/warthog/releases/download/v2.23.0/warthog-v2.23.0.tgz#4582fc35554580e0af0f43a9b3725aad2eb808c6"
-  dependencies:
-    "@types/app-root-path" "^1.2.4"
-    "@types/bn.js" "^4.11.6"
-    "@types/caller" "^1.0.0"
-    "@types/cosmiconfig" "^6.0.0"
-    "@types/debug" "^4.1.5"
-    "@types/dotenv" "^8.2.0"
-    "@types/express" "^4.17.2"
-    "@types/graphql" "^14.5.0"
-    "@types/graphql-fields" "^1.3.2"
-    "@types/graphql-iso-date" "^3.3.3"
-    "@types/graphql-type-json" "^0.3.2"
-    "@types/isomorphic-fetch" "^0.0.35"
-    "@types/lodash" "^4.14.148"
-    "@types/mkdirp" "^0.5.2"
-    "@types/node" "^12.12.8"
-    "@types/node-emoji" "^1.8.1"
-    "@types/open" "^6.2.1"
-    "@types/pg" "^7.11.2"
-    "@types/prettier" "^1.18.3"
-    "@types/shortid" "^0.0.29"
-    "@types/ws" "^6.0.3"
-    apollo-link-error "^1.1.12"
-    apollo-link-http "^1.5.16"
-    apollo-server "^2.9.9"
-    apollo-server-express "^2.9.9"
-    app-root-path "^3.0.0"
-    caller "^1.0.1"
-    class-transformer "^0.2.3"
-    class-validator "^0.11.0"
-    cosmiconfig "^6.0.0"
-    cross-fetch "^3.0.4"
-    dataloader "^1.4.0"
-    debug "^4.1.1"
-    execa "^4.0.3"
-    express "^4.17.1"
-    gluegun "^4.1.0"
-    graphql "^14.5.8"
-    graphql-binding "^2.5.2"
-    graphql-fields "^2.0.3"
-    graphql-import-node "^0.0.4"
-    graphql-iso-date "^3.6.1"
-    graphql-scalars "^1.2.6"
-    graphql-tools "^4.0.6"
-    graphql-type-json "^0.3.0"
-    lodash "^4.17.15"
-    mkdirp "^0.5.1"
-    node-emoji "^1.10.0"
-    open "^7.0.0"
-    pg "^7.12.1"
-    pgtools "^0.3.0"
-    prettier "^1.19.1"
-    reflect-metadata "^0.1.13"
-    shortid "^2.2.15"
-    type-graphql "^0.17.5"
-    typedi "^0.8.0"
-    typeorm "^0.2.25"
-    typeorm-typedi-extensions "^0.2.3"
-    typescript "^3.9.7"
-
 watchpack-chokidar2@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/watchpack-chokidar2/-/watchpack-chokidar2-2.0.0.tgz#9948a1866cbbd6cb824dea13a7ed691f6c8ddff0"
@@ -30638,30 +30581,30 @@ yargs@^15.0.0, yargs@^15.0.1, yargs@^15.3.1, yargs@^15.4.1:
     y18n "^4.0.0"
     yargs-parser "^18.1.2"
 
-yargs@^16.0.0, yargs@^16.2.0:
-  version "16.2.0"
-  resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.2.0.tgz#1c82bf0f6b6a66eafce7ef30e376f49a12477f66"
-  integrity sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==
+yargs@^16.0.3:
+  version "16.1.0"
+  resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.1.0.tgz#fc333fe4791660eace5a894b39d42f851cd48f2a"
+  integrity sha512-upWFJOmDdHN0syLuESuvXDmrRcWd1QafJolHskzaw79uZa7/x53gxQKiR07W59GWY1tFhhU/Th9DrtSfpS782g==
   dependencies:
     cliui "^7.0.2"
     escalade "^3.1.1"
     get-caller-file "^2.0.5"
     require-directory "^2.1.1"
     string-width "^4.2.0"
-    y18n "^5.0.5"
+    y18n "^5.0.2"
     yargs-parser "^20.2.2"
 
-yargs@^16.0.3:
-  version "16.1.0"
-  resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.1.0.tgz#fc333fe4791660eace5a894b39d42f851cd48f2a"
-  integrity sha512-upWFJOmDdHN0syLuESuvXDmrRcWd1QafJolHskzaw79uZa7/x53gxQKiR07W59GWY1tFhhU/Th9DrtSfpS782g==
+yargs@^16.2.0:
+  version "16.2.0"
+  resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.2.0.tgz#1c82bf0f6b6a66eafce7ef30e376f49a12477f66"
+  integrity sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==
   dependencies:
     cliui "^7.0.2"
     escalade "^3.1.1"
     get-caller-file "^2.0.5"
     require-directory "^2.1.1"
     string-width "^4.2.0"
-    y18n "^5.0.2"
+    y18n "^5.0.5"
     yargs-parser "^20.2.2"
 
 yargs@^2.1.1: