Merge pull request #7 from Joystream/iznik

Iznik

.github/workflows/joystream-cli.yml

@@ -18,12 +18,11 @@ jobs:
       run: |
         yarn install --frozen-lockfile
         yarn workspace @joystream/cli checks
-    - name: npm pack test
+    - name: yarn pack test
       run: |
-        cd cli
-        npm pack | tail -1 | xargs tar xzf
-        cd package && npm link
-        joystream-cli help
+        yarn workspace @joystream/cli pack --filename cli-pack-test.tgz
+        tar zxvf ./cli/cli-pack-test.tgz -C cli
+        cd ./cli/package && yarn link
 
   cli_build_osx:
     name: MacOS Checks
@@ -41,9 +40,8 @@ jobs:
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
         yarn workspace @joystream/cli checks
-    - name: npm pack test
+    - name: yarn pack test
       run: |
-        cd cli
-        npm pack | tail -1 | xargs tar xzf
-        cd package && npm link
-        joystream-cli help
+        yarn workspace @joystream/cli pack --filename cli-pack-test.tgz
+        tar zxvf ./cli/cli-pack-test.tgz -C cli
+        cd ./cli/package && yarn link

.github/workflows/run-network-tests.yml

@@ -0,0 +1,25 @@
+name: run-network-tests
+on:
+  pull_request:
+    types: [labeled]
+  workflow_dispatch:
+
+jobs:
+  run_ansible_tests:
+    if: github.event.label.name == 'run-network-tests' || github.event.action == null
+    name: run network tests using ansible
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [12.x]
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: install toolchain
+        run: curl https://getsubstrate.io -sSf | bash -s -- --fast
+      - name: ansible build and tests
+        run: |
+          cd ./devops/ansible
+          ansible-playbook -i hosts build-and-run-tests-single-node-playbook.yml --become -v

.travis.yml

@@ -7,15 +7,9 @@ language: rust
 # sometimes break the build. When cache is enabled do not use the produced WASM build.
 # This also means the binary should not be used to produce the final chainspec file (because the same
 # one is embedded in the binary)
-cache: cargo
+# cache: cargo
 
-rust:
-  - stable
-
-matrix:
-  include:
-    - os: linux
-      env: TARGET=x86_64-unknown-linux-gnu
+rust: stable
 
 # Skip Rust build in a pull request if no rust project files were modified
 before_install:
@@ -30,7 +24,7 @@ before_install:
     fi
 
 install:
-  - rustup install nightly-2020-05-23
+  - rustup install nightly-2020-05-23 --force
   - rustup target add wasm32-unknown-unknown --toolchain nightly-2020-05-23
   # travis installs rust using rustup with the "minimal" profile so these tools are not installed by default
   - rustup component add rustfmt
@@ -40,8 +34,10 @@ before_script:
   - cargo fmt --all -- --check
 
 script:
-  # we set release as build type for all steps to benefit from already compiled packages in prior steps
-  - BUILD_DUMMY_WASM_BINARY=1 cargo clippy --release --target=${TARGET} -- -D warnings
-  - BUILD_DUMMY_WASM_BINARY=1 cargo test --release --verbose --all --target=${TARGET}
-  - TRIGGER_WASM_BUILD=1 WASM_BUILD_TOOLCHAIN=nightly-2020-05-23 cargo build --release --target=${TARGET} -p joystream-node
-  - ls -l ./target/${TARGET}/release/wbuild/joystream-node-runtime/
+  - export WASM_BUILD_TOOLCHAIN=nightly-2020-05-23
+  - BUILD_DUMMY_WASM_BINARY=1 cargo clippy --release --all -- -D warnings
+  - travis_wait 75 cargo test --release --verbose --all -- --ignored
+  - cargo build --release
+  - ls -l ./target/release/wbuild/joystream-node-runtime/
+  - ./target/release/joystream-node --version
+  - ./target/release/chain-spec-builder --version

README.md

@@ -93,8 +93,8 @@ You can also run your our own joystream-node:
 
 ```sh
 git checkout master
-cargo build --release
-cargo run --release -- --pruning archive --chain testnets/rome.json
+WASM_BUILD_TOOLCHAIN=nightly-2020-05-23 cargo build --release
+./target/release/joystream-node -- --pruning archive --chain testnets/rome.json
 ```
 
 Wait for the node to sync to the latest block, then change pioneer settings "remote node" option to "Local Node", or follow the link below:
@@ -123,6 +123,12 @@ During a rebase/merge you may want to skip all hooks, you can use `HUSKY_SKIP_HO
 HUSKY_SKIP_HOOKS=1 git rebase ...
 ```
 
+## RLS Extension in VScode or Atom Editors
+
+If you use RLS extension in your IDE, start your editor with the `BUILD_DUMMY_WASM_BINARY=1` environment set to workaround a build issue that occurs in the IDE only.
+
+`BUILD_DUMMY_WASM_BINARY=1 code ./joystream`
+
 ## Authors
 
 See the list of [contributors](https://github.com/Joystream/joystream/graphs/contributors) who participated in this project.

cli/README.md

@@ -108,7 +108,7 @@ OPTIONS
   --showSpecial  Whether to show special (DEV chain) accounts
 ```
 
-_See code: [src/commands/account/choose.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/account/choose.ts)_
+_See code: [src/commands/account/choose.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/account/choose.ts)_
 
 ## `joystream-cli account:create NAME`
 
@@ -122,7 +122,7 @@ ARGUMENTS
   NAME  Account name
 ```
 
-_See code: [src/commands/account/create.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/account/create.ts)_
+_See code: [src/commands/account/create.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/account/create.ts)_
 
 ## `joystream-cli account:current`
 
@@ -137,7 +137,7 @@ ALIASES
   $ joystream-cli account:default
 ```
 
-_See code: [src/commands/account/current.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/account/current.ts)_
+_See code: [src/commands/account/current.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/account/current.ts)_
 
 ## `joystream-cli account:export PATH`
 
@@ -154,7 +154,7 @@ OPTIONS
   -a, --all  If provided, exports all existing accounts into "exported_accounts" folder inside given path
 ```
 
-_See code: [src/commands/account/export.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/account/export.ts)_
+_See code: [src/commands/account/export.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/account/export.ts)_
 
 ## `joystream-cli account:forget`
 
@@ -165,7 +165,7 @@ USAGE
   $ joystream-cli account:forget
 ```
 
-_See code: [src/commands/account/forget.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/account/forget.ts)_
+_See code: [src/commands/account/forget.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/account/forget.ts)_
 
 ## `joystream-cli account:import BACKUPFILEPATH`
 
@@ -179,7 +179,7 @@ ARGUMENTS
   BACKUPFILEPATH  Path to account backup JSON file
 ```
 
-_See code: [src/commands/account/import.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/account/import.ts)_
+_See code: [src/commands/account/import.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/account/import.ts)_
 
 ## `joystream-cli account:transferTokens RECIPIENT AMOUNT`
 
@@ -194,7 +194,7 @@ ARGUMENTS
   AMOUNT     Amount of tokens to transfer
 ```
 
-_See code: [src/commands/account/transferTokens.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/account/transferTokens.ts)_
+_See code: [src/commands/account/transferTokens.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/account/transferTokens.ts)_
 
 ## `joystream-cli api:getUri`
 
@@ -205,7 +205,7 @@ USAGE
   $ joystream-cli api:getUri
 ```
 
-_See code: [src/commands/api/getUri.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/api/getUri.ts)_
+_See code: [src/commands/api/getUri.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/api/getUri.ts)_
 
 ## `joystream-cli api:inspect`
 
@@ -221,15 +221,15 @@ OPTIONS
       If no "--method" flag is provided then all methods in that module will be listed along with the descriptions.
 
   -a, --callArgs=callArgs
-      Specifies the arguments to use when calling a method. Multiple arguments can be separated with a comma, ie.
+      Specifies the arguments to use when calling a method. Multiple arguments can be separated with a comma, ie. 
       "-a=arg1,arg2".
       You can omit this flag even if the method requires some aguments.
       In that case you will be promted to provide value for each required argument.
-      Ommiting this flag is recommended when input parameters are of more complex types (and it's hard to specify them as
+      Ommiting this flag is recommended when input parameters are of more complex types (and it's hard to specify them as 
       just simple comma-separated strings)
 
   -e, --exec
-      Provide this flag if you want to execute the actual call, instead of displaying the method description (which is
+      Provide this flag if you want to execute the actual call, instead of displaying the method description (which is 
       default)
 
   -m, --method=method
@@ -244,12 +244,12 @@ EXAMPLES
   $ api:inspect
   $ api:inspect -t=query
   $ api:inspect -t=query -M=members
-  $ api:inspect -t=query -M=members -m=memberProfile
-  $ api:inspect -t=query -M=members -m=memberProfile -e
-  $ api:inspect -t=query -M=members -m=memberProfile -e -a=1
+  $ api:inspect -t=query -M=members -m=membershipById
+  $ api:inspect -t=query -M=members -m=membershipById -e
+  $ api:inspect -t=query -M=members -m=membershipById -e -a=1
 ```
 
-_See code: [src/commands/api/inspect.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/api/inspect.ts)_
+_See code: [src/commands/api/inspect.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/api/inspect.ts)_
 
 ## `joystream-cli api:setUri [URI]`
 
@@ -263,7 +263,7 @@ ARGUMENTS
   URI  Uri of the node api WS provider (if skipped, a prompt will be displayed)
 ```
 
-_See code: [src/commands/api/setUri.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/api/setUri.ts)_
+_See code: [src/commands/api/setUri.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/api/setUri.ts)_
 
 ## `joystream-cli autocomplete [SHELL]`
 
@@ -297,7 +297,7 @@ USAGE
   $ joystream-cli council:info
 ```
 
-_See code: [src/commands/council/info.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/council/info.ts)_
+_See code: [src/commands/council/info.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/council/info.ts)_
 
 ## `joystream-cli help [COMMAND]`
 
@@ -333,7 +333,7 @@ OPTIONS
                      Available values are: storageProviders.
 ```
 
-_See code: [src/commands/working-groups/application.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/working-groups/application.ts)_
+_See code: [src/commands/working-groups/application.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/application.ts)_
 
 ## `joystream-cli working-groups:createOpening`
 
@@ -359,7 +359,7 @@ OPTIONS
   -s, --skipPrompts          Whether to skip all prompts when adding from draft (will use all default values)
 ```
 
-_See code: [src/commands/working-groups/createOpening.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/working-groups/createOpening.ts)_
+_See code: [src/commands/working-groups/createOpening.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/createOpening.ts)_
 
 ## `joystream-cli working-groups:decreaseWorkerStake WORKERID`
 
@@ -378,7 +378,7 @@ OPTIONS
                      Available values are: storageProviders.
 ```
 
-_See code: [src/commands/working-groups/decreaseWorkerStake.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/working-groups/decreaseWorkerStake.ts)_
+_See code: [src/commands/working-groups/decreaseWorkerStake.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/decreaseWorkerStake.ts)_
 
 ## `joystream-cli working-groups:evictWorker WORKERID`
 
@@ -397,7 +397,7 @@ OPTIONS
                      Available values are: storageProviders.
 ```
 
-_See code: [src/commands/working-groups/evictWorker.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/working-groups/evictWorker.ts)_
+_See code: [src/commands/working-groups/evictWorker.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/evictWorker.ts)_
 
 ## `joystream-cli working-groups:fillOpening WGOPENINGID`
 
@@ -416,7 +416,7 @@ OPTIONS
                      Available values are: storageProviders.
 ```
 
-_See code: [src/commands/working-groups/fillOpening.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/working-groups/fillOpening.ts)_
+_See code: [src/commands/working-groups/fillOpening.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/fillOpening.ts)_
 
 ## `joystream-cli working-groups:increaseStake`
 
@@ -432,7 +432,7 @@ OPTIONS
                      Available values are: storageProviders.
 ```
 
-_See code: [src/commands/working-groups/increaseStake.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/working-groups/increaseStake.ts)_
+_See code: [src/commands/working-groups/increaseStake.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/increaseStake.ts)_
 
 ## `joystream-cli working-groups:leaveRole`
 
@@ -448,7 +448,7 @@ OPTIONS
                      Available values are: storageProviders.
 ```
 
-_See code: [src/commands/working-groups/leaveRole.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/working-groups/leaveRole.ts)_
+_See code: [src/commands/working-groups/leaveRole.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/leaveRole.ts)_
 
 ## `joystream-cli working-groups:opening WGOPENINGID`
 
@@ -467,7 +467,7 @@ OPTIONS
                      Available values are: storageProviders.
 ```
 
-_See code: [src/commands/working-groups/opening.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/working-groups/opening.ts)_
+_See code: [src/commands/working-groups/opening.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/opening.ts)_
 
 ## `joystream-cli working-groups:openings`
 
@@ -483,7 +483,7 @@ OPTIONS
                      Available values are: storageProviders.
 ```
 
-_See code: [src/commands/working-groups/openings.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/working-groups/openings.ts)_
+_See code: [src/commands/working-groups/openings.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/openings.ts)_
 
 ## `joystream-cli working-groups:overview`
 
@@ -499,7 +499,7 @@ OPTIONS
                      Available values are: storageProviders.
 ```
 
-_See code: [src/commands/working-groups/overview.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/working-groups/overview.ts)_
+_See code: [src/commands/working-groups/overview.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/overview.ts)_
 
 ## `joystream-cli working-groups:slashWorker WORKERID`
 
@@ -518,7 +518,7 @@ OPTIONS
                      Available values are: storageProviders.
 ```
 
-_See code: [src/commands/working-groups/slashWorker.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/working-groups/slashWorker.ts)_
+_See code: [src/commands/working-groups/slashWorker.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/slashWorker.ts)_
 
 ## `joystream-cli working-groups:startAcceptingApplications WGOPENINGID`
 
@@ -537,7 +537,7 @@ OPTIONS
                      Available values are: storageProviders.
 ```
 
-_See code: [src/commands/working-groups/startAcceptingApplications.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/working-groups/startAcceptingApplications.ts)_
+_See code: [src/commands/working-groups/startAcceptingApplications.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/startAcceptingApplications.ts)_
 
 ## `joystream-cli working-groups:startReviewPeriod WGOPENINGID`
 
@@ -556,7 +556,7 @@ OPTIONS
                      Available values are: storageProviders.
 ```
 
-_See code: [src/commands/working-groups/startReviewPeriod.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/working-groups/startReviewPeriod.ts)_
+_See code: [src/commands/working-groups/startReviewPeriod.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/startReviewPeriod.ts)_
 
 ## `joystream-cli working-groups:terminateApplication WGAPPLICATIONID`
 
@@ -575,7 +575,7 @@ OPTIONS
                      Available values are: storageProviders.
 ```
 
-_See code: [src/commands/working-groups/terminateApplication.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/working-groups/terminateApplication.ts)_
+_See code: [src/commands/working-groups/terminateApplication.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/terminateApplication.ts)_
 
 ## `joystream-cli working-groups:updateRewardAccount [ACCOUNTADDRESS]`
 
@@ -594,7 +594,7 @@ OPTIONS
                      Available values are: storageProviders.
 ```
 
-_See code: [src/commands/working-groups/updateRewardAccount.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/working-groups/updateRewardAccount.ts)_
+_See code: [src/commands/working-groups/updateRewardAccount.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/updateRewardAccount.ts)_
 
 ## `joystream-cli working-groups:updateRoleAccount [ACCOUNTADDRESS]`
 
@@ -613,7 +613,7 @@ OPTIONS
                      Available values are: storageProviders.
 ```
 
-_See code: [src/commands/working-groups/updateRoleAccount.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/working-groups/updateRoleAccount.ts)_
+_See code: [src/commands/working-groups/updateRoleAccount.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/updateRoleAccount.ts)_
 
 ## `joystream-cli working-groups:updateWorkerReward WORKERID`
 
@@ -632,5 +632,5 @@ OPTIONS
                      Available values are: storageProviders.
 ```
 
-_See code: [src/commands/working-groups/updateWorkerReward.ts](https://github.com/Joystream/substrate-runtime-joystream/blob/master/cli/src/commands/working-groups/updateWorkerReward.ts)_
+_See code: [src/commands/working-groups/updateWorkerReward.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/updateWorkerReward.ts)_
 <!-- commandsstop -->

cli/package.json

@@ -1,21 +1,21 @@
 {
   "name": "@joystream/cli",
   "description": "Command Line Interface for Joystream community and governance activities",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "author": "Leszek Wiesner",
   "bin": {
     "joystream-cli": "./bin/run"
   },
   "bugs": "https://github.com/Joystream/joystream/issues",
   "dependencies": {
-    "@joystream/types": "^0.12.0",
+    "@joystream/types": "^0.13.0",
     "@oclif/command": "^1.5.19",
     "@oclif/config": "^1.14.0",
     "@oclif/plugin-autocomplete": "^0.2.0",
     "@oclif/plugin-help": "^2.2.3",
     "@oclif/plugin-not-found": "^1.2.4",
     "@oclif/plugin-warn-if-update-available": "^1.7.0",
-    "@polkadot/api": "^0.96.1",
+    "@polkadot/api": "1.26.1",
     "@types/inquirer": "^6.5.0",
     "@types/proper-lockfile": "^4.1.1",
     "@types/slug": "^0.9.1",
@@ -35,7 +35,7 @@
     "@types/mocha": "^5.2.7",
     "@types/node": "^10.17.18",
     "chai": "^4.2.0",
-    "eslint": "^5.16.0",
+    "eslint": "^7.6.0",
     "eslint-config-oclif": "^3.1.0",
     "eslint-config-oclif-typescript": "^0.1.0",
     "globby": "^10.0.2",

cli/src/Api.ts

@@ -1,13 +1,12 @@
 import BN from 'bn.js'
-import { registerJoystreamTypes } from '@joystream/types/'
+import { types } from '@joystream/types/'
 import { ApiPromise, WsProvider } from '@polkadot/api'
-import { QueryableStorageMultiArg } from '@polkadot/api/types'
+import { QueryableStorageMultiArg, SubmittableExtrinsic, QueryableStorageEntry } from '@polkadot/api/types'
 import { formatBalance } from '@polkadot/util'
-import { Hash, Balance, Moment, BlockNumber } from '@polkadot/types/interfaces'
+import { Balance, Moment, BlockNumber } from '@polkadot/types/interfaces'
 import { KeyringPair } from '@polkadot/keyring/types'
-import { Codec } from '@polkadot/types/types'
-import { Option, Vec } from '@polkadot/types'
-import { u32 } from '@polkadot/types/primitive'
+import { Codec, CodecArg } from '@polkadot/types/types'
+import { Option, Vec, UInt } from '@polkadot/types'
 import {
   AccountSummary,
   CouncilInfoObj,
@@ -24,7 +23,7 @@ import {
   UnstakingPeriods,
   StakingPolicyUnstakingPeriodKey,
 } from './Types'
-import { DerivedFees, DerivedBalances } from '@polkadot/api-derive/types'
+import { DeriveBalancesAll } from '@polkadot/api-derive/types'
 import { CLIError } from '@oclif/errors'
 import ExitCodes from './ExitCodes'
 import {
@@ -43,15 +42,14 @@ import {
   OpeningId,
   StakingPolicy,
 } from '@joystream/types/hiring'
-import { MemberId, Profile } from '@joystream/types/members'
+import { MemberId, Membership } from '@joystream/types/members'
 import { RewardRelationship, RewardRelationshipId } from '@joystream/types/recurring-rewards'
 import { Stake, StakeId } from '@joystream/types/stake'
-import { LinkageResult } from '@polkadot/types/codec/Linkage'
 
 import { InputValidationLengthConstraint } from '@joystream/types/common'
 
 export const DEFAULT_API_URI = 'ws://localhost:9944/'
-const DEFAULT_DECIMALS = new u32(12)
+const DEFAULT_DECIMALS = new BN(12)
 
 // Mapping of working group to api module
 export const apiModuleByGroup: { [key in WorkingGroups]: string } = {
@@ -72,8 +70,7 @@ export default class Api {
 
   private static async initApi(apiUri: string = DEFAULT_API_URI): Promise<ApiPromise> {
     const wsProvider: WsProvider = new WsProvider(apiUri)
-    registerJoystreamTypes()
-    const api = await ApiPromise.create({ provider: wsProvider })
+    const api = await ApiPromise.create({ provider: wsProvider, types })
 
     // Initializing some api params based on pioneer/packages/react-api/Api.tsx
     const [properties] = await Promise.all([api.rpc.system.properties()])
@@ -95,23 +92,27 @@ export default class Api {
     return new Api(originalApi)
   }
 
-  private async queryMultiOnce(queries: Parameters<typeof ApiPromise.prototype.queryMulti>[0]): Promise<Codec[]> {
-    let results: Codec[] = []
-
-    const unsub = await this._api.queryMulti(queries, (res) => {
-      results = res
+  private queryMultiOnce(queries: Parameters<typeof ApiPromise.prototype.queryMulti>[0]): Promise<Codec[]> {
+    return new Promise((resolve, reject) => {
+      let unsub: () => void
+      this._api
+        .queryMulti(queries, (res) => {
+          // unsub should already be set at this point
+          if (!unsub) {
+            reject(new CLIError('API queryMulti issue - unsub method not set!', { exit: ExitCodes.ApiError }))
+          }
+          unsub()
+          resolve(res)
+        })
+        .then((unsubscribe) => (unsub = unsubscribe))
+        .catch((e) => reject(e))
     })
-    unsub()
-
-    if (!results.length || results.length !== queries.length) {
-      throw new CLIError('API querying issue', { exit: ExitCodes.ApiError })
-    }
-
-    return results
   }
 
-  async getAccountsBalancesInfo(accountAddresses: string[]): Promise<DerivedBalances[]> {
-    const accountsBalances: DerivedBalances[] = await this._api.derive.balances.votingBalances(accountAddresses)
+  async getAccountsBalancesInfo(accountAddresses: string[]): Promise<DeriveBalancesAll[]> {
+    const accountsBalances: DeriveBalancesAll[] = await Promise.all(
+      accountAddresses.map((addr) => this._api.derive.balances.all(addr))
+    )
 
     return accountsBalances
   }
@@ -119,7 +120,7 @@ export default class Api {
   // Get on-chain data related to given account.
   // For now it's just account balances
   async getAccountSummary(accountAddresses: string): Promise<AccountSummary> {
-    const balances: DerivedBalances = (await this.getAccountsBalancesInfo([accountAddresses]))[0]
+    const balances: DeriveBalancesAll = (await this.getAccountsBalancesInfo([accountAddresses]))[0]
     // TODO: Some more information can be fetched here in the future
 
     return { balances }
@@ -146,34 +147,29 @@ export default class Api {
     return createCouncilInfoObj(...results)
   }
 
-  // TODO: This formula is probably not too good, so some better implementation will be required in the future
-  async estimateFee(account: KeyringPair, recipientAddr: string, amount: BN): Promise<BN> {
-    const transfer = this._api.tx.balances.transfer(recipientAddr, amount)
-    const signature = account.sign(transfer.toU8a())
-    const transactionByteSize: BN = new BN(transfer.encodedLength + signature.length)
-
-    const fees: DerivedFees = await this._api.derive.balances.fees()
-
-    const estimatedFee = fees.transactionBaseFee.add(fees.transactionByteFee.mul(transactionByteSize))
-
-    return estimatedFee
+  async estimateFee(account: KeyringPair, tx: SubmittableExtrinsic<'promise'>): Promise<Balance> {
+    const paymentInfo = await tx.paymentInfo(account)
+    return paymentInfo.partialFee
   }
 
-  async transfer(account: KeyringPair, recipientAddr: string, amount: BN): Promise<Hash> {
-    const txHash = await this._api.tx.balances.transfer(recipientAddr, amount).signAndSend(account)
-    return txHash
+  createTransferTx(recipient: string, amount: BN) {
+    return this._api.tx.balances.transfer(recipient, amount)
   }
 
   // Working groups
-  // TODO: This is a lot of repeated logic from "/pioneer/joy-roles/src/transport.substrate.ts"
-  // (although simplified a little bit)
-  // Hopefully this will be refactored to "joystream-js" soon
-  protected singleLinkageResult<T extends Codec>(result: LinkageResult) {
-    return result[0] as T
-  }
+  // TODO: This is a lot of repeated logic from "/pioneer/joy-utils/transport"
+  // It will be refactored to "joystream-js" soon
+  async entriesByIds<IDType extends UInt, ValueType extends Codec>(
+    apiMethod: QueryableStorageEntry<'promise'>,
+    firstKey?: CodecArg // First key in case of double maps
+  ): Promise<[IDType, ValueType][]> {
+    const entries: [IDType, ValueType][] = (await apiMethod.entries<ValueType>(firstKey)).map(([storageKey, value]) => [
+      // If double-map (first key is provided), we map entries by second key
+      storageKey.args[firstKey !== undefined ? 1 : 0] as IDType,
+      value,
+    ])
 
-  protected multiLinkageResult<K extends Codec, V extends Codec>(result: LinkageResult): [Vec<K>, Vec<V>] {
-    return [result[0] as Vec<K>, result[1] as Vec<V>]
+    return entries.sort((a, b) => a[0].toNumber() - b[0].toNumber())
   }
 
   protected async blockHash(height: number): Promise<string> {
@@ -193,10 +189,11 @@ export default class Api {
     return this._api.query[module]
   }
 
-  protected async memberProfileById(memberId: MemberId): Promise<Profile | null> {
-    const profile = (await this._api.query.members.memberProfile(memberId)) as Option<Profile>
+  protected async membershipById(memberId: MemberId): Promise<Membership | null> {
+    const profile = (await this._api.query.members.membershipById(memberId)) as Membership
 
-    return profile.unwrapOr(null)
+    // Can't just use profile.isEmpty because profile.suspended is Bool (which isEmpty method always returns false)
+    return profile.handle.isEmpty ? null : profile
   }
 
   async groupLead(group: WorkingGroups): Promise<GroupMember | null> {
@@ -213,7 +210,7 @@ export default class Api {
   }
 
   protected async stakeValue(stakeId: StakeId): Promise<Balance> {
-    const stake = this.singleLinkageResult<Stake>((await this._api.query.stake.stakes(stakeId)) as LinkageResult)
+    const stake = await this._api.query.stake.stakes<Stake>(stakeId)
     return stake.value
   }
 
@@ -222,8 +219,8 @@ export default class Api {
   }
 
   protected async workerReward(relationshipId: RewardRelationshipId): Promise<Reward> {
-    const rewardRelationship = this.singleLinkageResult<RewardRelationship>(
-      (await this._api.query.recurringRewards.rewardRelationships(relationshipId)) as LinkageResult
+    const rewardRelationship = await this._api.query.recurringRewards.rewardRelationships<RewardRelationship>(
+      relationshipId
     )
 
     return {
@@ -238,7 +235,7 @@ export default class Api {
     const roleAccount = worker.role_account_id
     const memberId = worker.member_id
 
-    const profile = await this.memberProfileById(memberId)
+    const profile = await this.membershipById(memberId)
 
     if (!profile) {
       throw new Error(`Group member profile not found! (member id: ${memberId.toNumber()})`)
@@ -265,18 +262,16 @@ export default class Api {
   }
 
   async workerByWorkerId(group: WorkingGroups, workerId: number): Promise<Worker> {
-    const nextId = (await this.workingGroupApiQuery(group).nextWorkerId()) as WorkerId
+    const nextId = await this.workingGroupApiQuery(group).nextWorkerId<WorkerId>()
 
     // This is chain specfic, but if next id is still 0, it means no workers have been added yet
     if (workerId < 0 || workerId >= nextId.toNumber()) {
       throw new CLIError('Invalid worker id!')
     }
 
-    const worker = this.singleLinkageResult<Worker>(
-      (await this.workingGroupApiQuery(group).workerById(workerId)) as LinkageResult
-    )
+    const worker = await this.workingGroupApiQuery(group).workerById<Worker>(workerId)
 
-    if (!worker.is_active) {
+    if (worker.isEmpty) {
       throw new CLIError('This worker is not active anymore')
     }
 
@@ -285,67 +280,51 @@ export default class Api {
 
   async groupMember(group: WorkingGroups, workerId: number) {
     const worker = await this.workerByWorkerId(group, workerId)
-    return await this.parseGroupMember(new WorkerId(workerId), worker)
+    return await this.parseGroupMember(this._api.createType('WorkerId', workerId), worker)
   }
 
   async groupMembers(group: WorkingGroups): Promise<GroupMember[]> {
-    const nextId = (await this.workingGroupApiQuery(group).nextWorkerId()) as WorkerId
-
-    // This is chain specfic, but if next id is still 0, it means no workers have been added yet
-    if (nextId.eq(0)) {
-      return []
-    }
+    const workerEntries = await this.entriesByIds<WorkerId, Worker>(this.workingGroupApiQuery(group).workerById)
 
-    const [workerIds, workers] = this.multiLinkageResult<WorkerId, Worker>(
-      (await this.workingGroupApiQuery(group).workerById()) as LinkageResult
+    const groupMembers: GroupMember[] = await Promise.all(
+      workerEntries.map(([id, worker]) => this.parseGroupMember(id, worker))
     )
 
-    const groupMembers: GroupMember[] = []
-    for (const [index, worker] of Object.entries(workers.toArray())) {
-      const workerId = workerIds[parseInt(index)]
-      if (worker.is_active) {
-        groupMembers.push(await this.parseGroupMember(workerId, worker))
-      }
-    }
-
-    return groupMembers.reverse()
+    return groupMembers.reverse() // Sort by newest
   }
 
   async openingsByGroup(group: WorkingGroups): Promise<GroupOpening[]> {
-    const openings: GroupOpening[] = []
-    const nextId = (await this.workingGroupApiQuery(group).nextOpeningId()) as OpeningId
+    let openings: GroupOpening[] = []
+    const nextId = await this.workingGroupApiQuery(group).nextOpeningId<OpeningId>()
 
     // This is chain specfic, but if next id is still 0, it means no openings have been added yet
     if (!nextId.eq(0)) {
-      const highestId = nextId.toNumber() - 1
-      for (let i = highestId; i >= 0; i--) {
-        openings.push(await this.groupOpening(group, i))
-      }
+      const ids = Array.from(Array(nextId.toNumber()).keys()).reverse() // Sort by newest
+      openings = await Promise.all(ids.map((id) => this.groupOpening(group, id)))
     }
 
     return openings
   }
 
   protected async hiringOpeningById(id: number | OpeningId): Promise<Opening> {
-    const result = (await this._api.query.hiring.openingById(id)) as LinkageResult
-    return this.singleLinkageResult<Opening>(result)
+    const result = await this._api.query.hiring.openingById<Opening>(id)
+    return result
   }
 
   protected async hiringApplicationById(id: number | ApplicationId): Promise<Application> {
-    const result = (await this._api.query.hiring.applicationById(id)) as LinkageResult
-    return this.singleLinkageResult<Application>(result)
+    const result = await this._api.query.hiring.applicationById<Application>(id)
+    return result
   }
 
   async wgApplicationById(group: WorkingGroups, wgApplicationId: number): Promise<WGApplication> {
-    const nextAppId = (await this.workingGroupApiQuery(group).nextApplicationId()) as ApplicationId
+    const nextAppId = await this.workingGroupApiQuery(group).nextApplicationId<ApplicationId>()
 
     if (wgApplicationId < 0 || wgApplicationId >= nextAppId.toNumber()) {
       throw new CLIError('Invalid working group application ID!')
     }
 
-    return this.singleLinkageResult<WGApplication>(
-      (await this.workingGroupApiQuery(group).applicationById(wgApplicationId)) as LinkageResult
-    )
+    const result = await this.workingGroupApiQuery(group).applicationById<WGApplication>(wgApplicationId)
+    return result
   }
 
   protected async parseApplication(wgApplicationId: number, wgApplication: WGApplication): Promise<GroupApplication> {
@@ -358,7 +337,7 @@ export default class Api {
       wgApplicationId,
       applicationId: appId.toNumber(),
       wgOpeningId: wgApplication.opening_id.toNumber(),
-      member: await this.memberProfileById(wgApplication.member_id),
+      member: await this.membershipById(wgApplication.member_id),
       roleAccout: wgApplication.role_account_id,
       stakes: {
         application: appStakingId.isSome ? (await this.stakeValue(appStakingId.unwrap())).toNumber() : 0,
@@ -375,18 +354,15 @@ export default class Api {
   }
 
   protected async groupOpeningApplications(group: WorkingGroups, wgOpeningId: number): Promise<GroupApplication[]> {
-    const applications: GroupApplication[] = []
-
-    const nextAppId = (await this.workingGroupApiQuery(group).nextApplicationId()) as ApplicationId
-    for (let i = 0; i < nextAppId.toNumber(); i++) {
-      const wgApplication = await this.wgApplicationById(group, i)
-      if (wgApplication.opening_id.toNumber() !== wgOpeningId) {
-        continue
-      }
-      applications.push(await this.parseApplication(i, wgApplication))
-    }
+    const wgApplicationEntries = await this.entriesByIds<ApplicationId, WGApplication>(
+      this.workingGroupApiQuery(group).applicationById
+    )
 
-    return applications
+    return Promise.all(
+      wgApplicationEntries
+        .filter(([, /* id */ wgApplication]) => wgApplication.opening_id.eqn(wgOpeningId))
+        .map(([id, wgApplication]) => this.parseApplication(id.toNumber(), wgApplication))
+    )
   }
 
   async groupOpening(group: WorkingGroups, wgOpeningId: number): Promise<GroupOpening> {
@@ -396,9 +372,7 @@ export default class Api {
       throw new CLIError('Invalid working group opening ID!')
     }
 
-    const groupOpening = this.singleLinkageResult<WGOpening>(
-      (await this.workingGroupApiQuery(group).openingById(wgOpeningId)) as LinkageResult
-    )
+    const groupOpening = await this.workingGroupApiQuery(group).openingById<WGOpening>(wgOpeningId)
 
     const openingId = groupOpening.hiring_opening_id.toNumber()
     const opening = await this.hiringOpeningById(openingId)
@@ -416,19 +390,19 @@ export default class Api {
       sp.isSome ? unstakingPeriod(sp.unwrap()[key]) : 0
 
     const unstakingPeriods: Partial<UnstakingPeriods> = {
-      ['review_period_expired_application_stake_unstaking_period_length']: spUnstakingPeriod(
+      'review_period_expired_application_stake_unstaking_period_length': spUnstakingPeriod(
         applSP,
         'review_period_expired_unstaking_period_length'
       ),
-      ['crowded_out_application_stake_unstaking_period_length']: spUnstakingPeriod(
+      'crowded_out_application_stake_unstaking_period_length': spUnstakingPeriod(
         applSP,
         'crowded_out_unstaking_period_length'
       ),
-      ['review_period_expired_role_stake_unstaking_period_length']: spUnstakingPeriod(
+      'review_period_expired_role_stake_unstaking_period_length': spUnstakingPeriod(
         roleSP,
         'review_period_expired_unstaking_period_length'
       ),
-      ['crowded_out_role_stake_unstaking_period_length']: spUnstakingPeriod(
+      'crowded_out_role_stake_unstaking_period_length': spUnstakingPeriod(
         roleSP,
         'crowded_out_unstaking_period_length'
       ),
@@ -492,11 +466,11 @@ export default class Api {
   }
 
   async getMemberIdsByControllerAccount(address: string): Promise<MemberId[]> {
-    const ids = (await this._api.query.members.memberIdsByControllerAccountId(address)) as Vec<MemberId>
+    const ids = await this._api.query.members.memberIdsByControllerAccountId<Vec<MemberId>>(address)
     return ids.toArray()
   }
 
   async workerExitRationaleConstraint(group: WorkingGroups): Promise<InputValidationLengthConstraint> {
-    return (await this.workingGroupApiQuery(group).workerExitRationaleText()) as InputValidationLengthConstraint
+    return await this.workingGroupApiQuery(group).workerExitRationaleText<InputValidationLengthConstraint>()
   }
 }

cli/src/Types.ts

@@ -5,10 +5,10 @@ import { Constructor, Codec } from '@polkadot/types/types'
 import { Struct, Vec } from '@polkadot/types/codec'
 import { u32 } from '@polkadot/types/primitive'
 import { BlockNumber, Balance, AccountId } from '@polkadot/types/interfaces'
-import { DerivedBalances } from '@polkadot/api-derive/types'
+import { DeriveBalancesAll } from '@polkadot/api-derive/types'
 import { KeyringPair } from '@polkadot/keyring/types'
 import { WorkerId, OpeningType } from '@joystream/types/working-group'
-import { Profile, MemberId } from '@joystream/types/members'
+import { Membership, MemberId } from '@joystream/types/members'
 import {
   GenericJoyStreamRoleSchema,
   JobSpecifics,
@@ -25,6 +25,7 @@ import {
 import ajv from 'ajv'
 import { Opening, StakingPolicy, ApplicationStageKeys } from '@joystream/types/hiring'
 import { Validator } from 'inquirer'
+import { JoyStructCustom } from '@joystream/types/common'
 
 // KeyringPair type extended with mandatory "meta.name"
 // It's used for accounts/keys management within CLI.
@@ -37,7 +38,7 @@ export type NamedKeyringPair = KeyringPair & {
 
 // Summary of the account information fetched from the api for "account:current" purposes (currently just balances)
 export type AccountSummary = {
-  balances: DerivedBalances
+  balances: DeriveBalancesAll
 }
 
 // This function allows us to easily transform the tuple into the object
@@ -103,7 +104,7 @@ export type GroupMember = {
   workerId: WorkerId
   memberId: MemberId
   roleAccount: AccountId
-  profile: Profile
+  profile: Membership
   stake?: Balance
   reward?: Reward
 }
@@ -112,7 +113,7 @@ export type GroupApplication = {
   wgApplicationId: number
   applicationId: number
   wgOpeningId: number
-  member: Profile | null
+  member: Membership | null
   roleAccout: AccountId
   stakes: {
     application: number
@@ -186,187 +187,174 @@ export type GroupOpening = {
 // Note those types are not part of the runtime etc., we just use them to simplify prompting for values
 // (since there exists functionality that handles that for substrate types like: Struct, Vec etc.)
 interface WithJSONable<T> {
-  toJSON: () => T
+  toJSONObj: () => T
 }
-export class HRTJobSpecificsStruct extends Struct implements WithJSONable<JobSpecifics> {
-  constructor(value?: JobSpecifics) {
-    super(
-      {
-        title: 'Text',
-        description: 'Text',
-      },
-      value
-    )
-  }
+export class HRTJobSpecificsStruct
+  extends JoyStructCustom({
+    title: Text,
+    description: Text,
+  })
+  implements WithJSONable<JobSpecifics> {
   get title(): string {
-    return (this.get('title') as Text).toString()
+    return this.getField('title').toString()
   }
+
   get description(): string {
-    return (this.get('description') as Text).toString()
+    return this.getField('description').toString()
   }
-  toJSON(): JobSpecifics {
+
+  toJSONObj(): JobSpecifics {
     const { title, description } = this
     return { title, description }
   }
 }
-export class HRTEntryInMembershipModukeStruct extends Struct implements WithJSONable<EntryInMembershipModuke> {
-  constructor(value?: EntryInMembershipModuke) {
-    super(
-      {
-        handle: 'Text',
-      },
-      value
-    )
-  }
+export class HRTEntryInMembershipModukeStruct
+  extends JoyStructCustom({
+    handle: Text,
+  })
+  implements WithJSONable<EntryInMembershipModuke> {
   get handle(): string {
-    return (this.get('handle') as Text).toString()
+    return this.getField('handle').toString()
   }
-  toJSON(): EntryInMembershipModuke {
+
+  toJSONObj(): EntryInMembershipModuke {
     const { handle } = this
     return { handle }
   }
 }
-export class HRTCreatorDetailsStruct extends Struct implements WithJSONable<CreatorDetails> {
-  constructor(value?: CreatorDetails) {
-    super(
-      {
-        membership: HRTEntryInMembershipModukeStruct,
-      },
-      value
-    )
-  }
+export class HRTCreatorDetailsStruct
+  extends JoyStructCustom({
+    membership: HRTEntryInMembershipModukeStruct,
+  })
+  implements WithJSONable<CreatorDetails> {
   get membership(): EntryInMembershipModuke {
-    return (this.get('membership') as HRTEntryInMembershipModukeStruct).toJSON()
+    return this.getField('membership').toJSONObj()
   }
-  toJSON(): CreatorDetails {
+
+  toJSONObj(): CreatorDetails {
     const { membership } = this
     return { membership }
   }
 }
-export class HRTHiringProcessStruct extends Struct implements WithJSONable<HiringProcess> {
-  constructor(value?: HiringProcess) {
-    super(
-      {
-        details: 'Vec<Text>',
-      },
-      value
-    )
-  }
+export class HRTHiringProcessStruct
+  extends JoyStructCustom({
+    details: Vec.with(Text),
+  })
+  implements WithJSONable<HiringProcess> {
   get details(): AdditionalRolehiringProcessDetails {
-    return (this.get('details') as Vec<Text>).toArray().map((v) => v.toString())
+    return this.getField('details')
+      .toArray()
+      .map((v) => v.toString())
   }
-  toJSON(): HiringProcess {
+
+  toJSONObj(): HiringProcess {
     const { details } = this
     return { details }
   }
 }
-export class HRTQuestionFieldStruct extends Struct implements WithJSONable<QuestionField> {
-  constructor(value?: QuestionField) {
-    super(
-      {
-        title: 'Text',
-        type: 'Text',
-      },
-      value
-    )
-  }
+export class HRTQuestionFieldStruct
+  extends JoyStructCustom({
+    title: Text,
+    type: Text,
+  })
+  implements WithJSONable<QuestionField> {
   get title(): string {
-    return (this.get('title') as Text).toString()
+    return this.getField('title').toString()
   }
+
   get type(): string {
-    return (this.get('type') as Text).toString()
+    return this.getField('type').toString()
   }
-  toJSON(): QuestionField {
+
+  toJSONObj(): QuestionField {
     const { title, type } = this
     return { title, type }
   }
 }
 class HRTQuestionsFieldsVec extends Vec.with(HRTQuestionFieldStruct) implements WithJSONable<QuestionsFields> {
-  toJSON(): QuestionsFields {
-    return this.toArray().map((v) => v.toJSON())
+  toJSONObj(): QuestionsFields {
+    return this.toArray().map((v) => v.toJSONObj())
   }
 }
-export class HRTQuestionSectionStruct extends Struct implements WithJSONable<QuestionSection> {
-  constructor(value?: QuestionSection) {
-    super(
-      {
-        title: 'Text',
-        questions: HRTQuestionsFieldsVec,
-      },
-      value
-    )
-  }
+export class HRTQuestionSectionStruct
+  extends JoyStructCustom({
+    title: Text,
+    questions: HRTQuestionsFieldsVec,
+  })
+  implements WithJSONable<QuestionSection> {
   get title(): string {
-    return (this.get('title') as Text).toString()
+    return this.getField('title').toString()
   }
+
   get questions(): QuestionsFields {
-    return (this.get('questions') as HRTQuestionsFieldsVec).toJSON()
+    return this.getField('questions').toJSONObj()
   }
-  toJSON(): QuestionSection {
+
+  toJSONObj(): QuestionSection {
     const { title, questions } = this
     return { title, questions }
   }
 }
 export class HRTQuestionSectionsVec extends Vec.with(HRTQuestionSectionStruct)
   implements WithJSONable<QuestionSections> {
-  toJSON(): QuestionSections {
-    return this.toArray().map((v) => v.toJSON())
+  toJSONObj(): QuestionSections {
+    return this.toArray().map((v) => v.toJSONObj())
   }
 }
-export class HRTApplicationDetailsStruct extends Struct implements WithJSONable<ApplicationDetails> {
-  constructor(value?: ApplicationDetails) {
-    super(
-      {
-        sections: HRTQuestionSectionsVec,
-      },
-      value
-    )
-  }
+export class HRTApplicationDetailsStruct
+  extends JoyStructCustom({
+    sections: HRTQuestionSectionsVec,
+  })
+  implements WithJSONable<ApplicationDetails> {
   get sections(): QuestionSections {
-    return (this.get('sections') as HRTQuestionSectionsVec).toJSON()
+    return this.getField('sections').toJSONObj()
   }
-  toJSON(): ApplicationDetails {
+
+  toJSONObj(): ApplicationDetails {
     const { sections } = this
     return { sections }
   }
 }
-export class HRTStruct extends Struct implements WithJSONable<GenericJoyStreamRoleSchema> {
-  constructor(value?: GenericJoyStreamRoleSchema) {
-    super(
-      {
-        version: 'u32',
-        headline: 'Text',
-        job: HRTJobSpecificsStruct,
-        application: HRTApplicationDetailsStruct,
-        reward: 'Text',
-        creator: HRTCreatorDetailsStruct,
-        process: HRTHiringProcessStruct,
-      },
-      value
-    )
-  }
+export class HRTStruct
+  extends JoyStructCustom({
+    version: u32,
+    headline: Text,
+    job: HRTJobSpecificsStruct,
+    application: HRTApplicationDetailsStruct,
+    reward: Text,
+    creator: HRTCreatorDetailsStruct,
+    process: HRTHiringProcessStruct,
+  })
+  implements WithJSONable<GenericJoyStreamRoleSchema> {
   get version(): number {
-    return (this.get('version') as u32).toNumber()
+    return this.getField('version').toNumber()
   }
+
   get headline(): string {
-    return (this.get('headline') as Text).toString()
+    return this.getField('headline').toString()
   }
+
   get job(): JobSpecifics {
-    return (this.get('job') as HRTJobSpecificsStruct).toJSON()
+    return this.getField('job').toJSONObj()
   }
+
   get application(): ApplicationDetails {
-    return (this.get('application') as HRTApplicationDetailsStruct).toJSON()
+    return this.getField('application').toJSONObj()
   }
+
   get reward(): string {
-    return (this.get('reward') as Text).toString()
+    return this.getField('reward').toString()
   }
+
   get creator(): CreatorDetails {
-    return (this.get('creator') as HRTCreatorDetailsStruct).toJSON()
+    return this.getField('creator').toJSONObj()
   }
+
   get process(): HiringProcess {
-    return (this.get('process') as HRTHiringProcessStruct).toJSON()
+    return this.getField('process').toJSONObj()
   }
-  toJSON(): GenericJoyStreamRoleSchema {
+
+  toJSONObj(): GenericJoyStreamRoleSchema {
     const { version, headline, job, application, reward, creator, process } = this
     return { version, headline, job, application, reward, creator, process }
   }

cli/src/base/AccountsCommandBase.ts

@@ -8,7 +8,7 @@ import ApiCommandBase from './ApiCommandBase'
 import { Keyring } from '@polkadot/api'
 import { formatBalance } from '@polkadot/util'
 import { NamedKeyringPair } from '../Types'
-import { DerivedBalances } from '@polkadot/api-derive/types'
+import { DeriveBalancesAll } from '@polkadot/api-derive/types'
 import { toFixedLength } from '../helpers/display'
 
 const ACCOUNTS_DIRNAME = 'accounts'
@@ -54,7 +54,9 @@ export default abstract class AccountsCommandBase extends ApiCommandBase {
     const keyring = new Keyring({ type: 'sr25519' })
     keyring.addFromUri('//Alice', { name: 'Alice' })
     keyring.addFromUri('//Bob', { name: 'Bob' })
-    keyring.getPairs().forEach((pair) => this.saveAccount({ ...pair, meta: { name: pair.meta.name } }, '', true))
+    keyring
+      .getPairs()
+      .forEach((pair) => this.saveAccount({ ...pair, meta: { name: pair.meta.name as string } }, '', true))
   }
 
   fetchAccountFromJsonFile(jsonBackupFilePath: string): NamedKeyringPair {
@@ -186,7 +188,7 @@ export default abstract class AccountsCommandBase extends ApiCommandBase {
     message = 'Select an account',
     showBalances = true
   ): Promise<NamedKeyringPair> {
-    let balances: DerivedBalances[]
+    let balances: DeriveBalancesAll[]
     if (showBalances) {
       balances = await this.getApi().getAccountsBalancesInfo(accounts.map((acc) => acc.address))
     }

cli/src/base/ApiCommandBase.ts

@@ -2,13 +2,14 @@ import ExitCodes from '../ExitCodes'
 import { CLIError } from '@oclif/errors'
 import StateAwareCommandBase from './StateAwareCommandBase'
 import Api from '../Api'
-import { getTypeDef, createType, Option, Tuple, Bytes } from '@polkadot/types'
-import { Codec, TypeDef, TypeDefInfo, Constructor } from '@polkadot/types/types'
+import { getTypeDef, Option, Tuple, Bytes } from '@polkadot/types'
+import { Registry, Codec, CodecArg, TypeDef, TypeDefInfo, Constructor } from '@polkadot/types/types'
+
 import { Vec, Struct, Enum } from '@polkadot/types/codec'
 import { ApiPromise, WsProvider } from '@polkadot/api'
 import { KeyringPair } from '@polkadot/keyring/types'
 import chalk from 'chalk'
-import { SubmittableResultImpl } from '@polkadot/api/types'
+import { InterfaceTypes } from '@polkadot/types/types/registry'
 import ajv from 'ajv'
 import { ApiMethodArg, ApiMethodNamedArgs, ApiParamsOptions, ApiParamOptions } from '../Types'
 import { createParamOptions } from '../helpers/promptOptions'
@@ -32,6 +33,14 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
     return this.getApi().getOriginalApi()
   }
 
+  getTypesRegistry(): Registry {
+    return this.getOriginalApi().registry
+  }
+
+  createType<K extends keyof InterfaceTypes>(typeName: K, value?: unknown): InterfaceTypes[K] {
+    return this.getOriginalApi().createType(typeName, value)
+  }
+
   async init() {
     await super.init()
     let apiUri: string = this.getPreservedState().apiUri
@@ -81,6 +90,7 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
 
   isApiUriValid(uri: string) {
     try {
+      // eslint-disable-next-line no-new
       new WsProvider(uri)
     } catch (e) {
       return false
@@ -90,8 +100,8 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
 
   // This is needed to correctly handle some structs, enums etc.
   // Where the main typeDef doesn't provide enough information
-  protected getRawTypeDef(type: string) {
-    const instance = createType(type as any)
+  protected getRawTypeDef(type: keyof InterfaceTypes) {
+    const instance = this.createType(type)
     return getTypeDef(instance.toRawType())
   }
 
@@ -120,7 +130,7 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
   async promptForSimple(typeDef: TypeDef, paramOptions?: ApiParamOptions): Promise<Codec> {
     // If no default provided - get default value resulting from providing empty string
     const defaultValueString =
-      paramOptions?.value?.default?.toString() || createType(typeDef.type as any, '').toString()
+      paramOptions?.value?.default?.toString() || this.createType(typeDef.type as any, '').toString()
     const providedValue = await this.simplePrompt({
       message: `Provide value for ${this.paramName(typeDef)}`,
       type: 'input',
@@ -129,7 +139,7 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
       default: (defaultValueString === '0x' ? '' : defaultValueString) || undefined,
       validate: paramOptions?.validator,
     })
-    return createType(typeDef.type as any, providedValue)
+    return this.createType(typeDef.type as any, providedValue)
   }
 
   // Prompt for Option<Codec> value
@@ -149,10 +159,10 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
         createParamOptions(subtype.name, defaultValue?.unwrapOr(undefined))
       )
       this.closeIndentGroup()
-      return new Option(subtype.type as any, value)
+      return this.createType(`Option<${subtype.type}>` as any, value)
     }
 
-    return new Option(subtype.type as any, null)
+    return this.createType(`Option<${subtype.type}>` as any, null)
   }
 
   // Prompt for Tuple
@@ -173,7 +183,7 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
     }
     this.closeIndentGroup()
 
-    return new Tuple(subtypes.map((subtype) => subtype.type) as any, result)
+    return new Tuple(this.getTypesRegistry(), subtypes.map((subtype) => subtype.type) as any, result)
   }
 
   // Prompt for Struct
@@ -182,7 +192,7 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
 
     this.openIndentGroup()
     const structType = typeDef.type
-    const rawTypeDef = this.getRawTypeDef(structType)
+    const rawTypeDef = this.getRawTypeDef(structType as keyof InterfaceTypes)
     // We assume struct typeDef always has array of typeDefs inside ".sub"
     const structSubtypes = rawTypeDef.sub as TypeDef[]
     const structDefault = paramOptions?.value?.default as Struct | undefined
@@ -200,7 +210,7 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
     }
     this.closeIndentGroup()
 
-    return createType(structType as any, structValues)
+    return this.createType(structType as any, structValues)
   }
 
   // Prompt for Vec
@@ -228,12 +238,12 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
     } while (addAnother)
     this.closeIndentGroup()
 
-    return new Vec(subtype.type as any, entries)
+    return this.createType(`Vec<${subtype.type}>` as any, entries)
   }
 
   // Prompt for Enum
   async promptForEnum(typeDef: TypeDef, paramOptions?: ApiParamOptions): Promise<Enum> {
-    const enumType = typeDef.type
+    const enumType = typeDef.type as keyof InterfaceTypes
     const rawTypeDef = this.getRawTypeDef(enumType)
     // We assume enum always has array on TypeDefs inside ".sub"
     const enumSubtypes = rawTypeDef.sub as TypeDef[]
@@ -253,12 +263,12 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
 
     if (enumSubtype.type !== 'Null') {
       const subtypeOptions = createParamOptions(enumSubtype.name, defaultValue?.value)
-      return createType(enumType as any, {
+      return this.createType(enumType as any, {
         [enumSubtype.name!]: await this.promptForParam(enumSubtype.type, subtypeOptions),
       })
     }
 
-    return createType(enumType as any, enumSubtype.name)
+    return this.createType(enumType as any, enumSubtype.name)
   }
 
   // Prompt for param based on "paramType" string (ie. Option<MemeberId>)
@@ -268,7 +278,7 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
     paramOptions?: ApiParamOptions // TODO: This is not fully implemented for all types yet
   ): Promise<ApiMethodArg> {
     const typeDef = getTypeDef(paramType)
-    const rawTypeDef = this.getRawTypeDef(paramType)
+    const rawTypeDef = this.getRawTypeDef(paramType as keyof InterfaceTypes)
 
     if (paramOptions?.forcedName) {
       typeDef.name = paramOptions.forcedName
@@ -309,18 +319,23 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
     defaultValue?: Bytes,
     schemaValidator?: ajv.ValidateFunction
   ) {
-    const rawType = new jsonStruct().toRawType()
+    const JsonStructObject = jsonStruct
+    const rawType = new JsonStructObject(this.getTypesRegistry()).toRawType()
     const typeDef = getTypeDef(rawType)
 
     const defaultStruct =
-      defaultValue && new jsonStruct(JSON.parse(Buffer.from(defaultValue.toHex().replace('0x', ''), 'hex').toString()))
+      defaultValue &&
+      new JsonStructObject(
+        this.getTypesRegistry(),
+        JSON.parse(Buffer.from(defaultValue.toHex().replace('0x', ''), 'hex').toString())
+      )
 
     if (argName) {
       typeDef.name = argName
     }
 
-    let isValid = true,
-      jsonText: string
+    let isValid = true
+    let jsonText: string
     do {
       const structVal = await this.promptForStruct(typeDef, createParamOptions(typeDef.name, defaultStruct))
       jsonText = JSON.stringify(structVal.toJSON())
@@ -338,7 +353,7 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
       }
     } while (!isValid)
 
-    return new Bytes('0x' + Buffer.from(jsonText, 'ascii').toString('hex'))
+    return this.createType('Bytes', '0x' + Buffer.from(jsonText, 'ascii').toString('hex'))
   }
 
   async promptForExtrinsicParams(
@@ -364,18 +379,18 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
     return values
   }
 
-  sendExtrinsic(account: KeyringPair, module: string, method: string, params: Codec[]) {
+  sendExtrinsic(account: KeyringPair, module: string, method: string, params: CodecArg[]) {
     return new Promise((resolve, reject) => {
       const extrinsicMethod = this.getOriginalApi().tx[module][method]
       let unsubscribe: () => void
       extrinsicMethod(...params)
-        .signAndSend(account, {}, (result: SubmittableResultImpl) => {
+        .signAndSend(account, {}, (result) => {
           // Implementation loosely based on /pioneer/packages/react-signer/src/Modal.tsx
           if (!result || !result.status) {
             return
           }
 
-          if (result.status.isFinalized) {
+          if (result.status.isInBlock) {
             unsubscribe()
             result.events
               .filter(({ event: { section } }): boolean => section === 'system')
@@ -401,7 +416,7 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
     account: KeyringPair,
     module: string,
     method: string,
-    params: Codec[],
+    params: CodecArg[],
     warnOnly = false // If specified - only warning will be displayed (instead of error beeing thrown)
   ) {
     try {
@@ -449,7 +464,7 @@ export default abstract class ApiCommandBase extends StateAwareCommandBase {
       const argName = arg.name.toString()
       const argType = arg.type.toString()
       try {
-        parsedArgs.push({ name: argName, value: createType(argType as any, draftJSONObj[parseInt(index)]) })
+        parsedArgs.push({ name: argName, value: this.createType(argType as any, draftJSONObj[parseInt(index)]) })
       } catch (e) {
         throw new CLIError(`Couldn't parse ${argName} value from draft at ${draftFilePath}!`, {
           exit: ExitCodes.InvalidFile,

cli/src/base/WorkingGroupsCommandBase.ts

@@ -110,13 +110,13 @@ export default abstract class WorkingGroupsCommandBase extends AccountsCommandBa
       })),
     })
 
-    return acceptedApplications
+    return acceptedApplications.sort() // Sort just in case, since runtime expects them to be sorted
   }
 
   async promptForNewOpeningDraftName() {
-    let draftName = '',
-      fileExists = false,
-      overrideConfirmed = false
+    let draftName = ''
+    let fileExists = false
+    let overrideConfirmed = false
 
     do {
       draftName = await this.simplePrompt({

cli/src/commands/account/current.ts

@@ -1,6 +1,5 @@
 import AccountsCommandBase from '../../base/AccountsCommandBase'
 import { AccountSummary, NameValueObj, NamedKeyringPair } from '../../Types'
-import { DerivedBalances } from '@polkadot/api-derive/types'
 import { displayHeader, displayNameValueTable } from '../../helpers/display'
 import { formatBalance } from '@polkadot/util'
 import moment from 'moment'
@@ -15,7 +14,7 @@ export default class AccountCurrent extends AccountsCommandBase {
 
     displayHeader('Account information')
     const creationDate: string = currentAccount.meta.whenCreated
-      ? moment(currentAccount.meta.whenCreated).format('YYYY-MM-DD HH:mm:ss')
+      ? moment(currentAccount.meta.whenCreated as string | number).format('YYYY-MM-DD HH:mm:ss')
       : '?'
     const accountRows: NameValueObj[] = [
       { name: 'Account name:', value: currentAccount.meta.name },
@@ -25,7 +24,7 @@ export default class AccountCurrent extends AccountsCommandBase {
     displayNameValueTable(accountRows)
 
     displayHeader('Balances')
-    const balances: DerivedBalances = summary.balances
+    const balances = summary.balances
     const balancesRows: NameValueObj[] = [
       { name: 'Total balance:', value: formatBalance(balances.votingBalance) },
       { name: 'Transferable balance:', value: formatBalance(balances.availableBalance) },

cli/src/commands/account/transferTokens.ts

@@ -6,7 +6,6 @@ import { formatBalance } from '@polkadot/util'
 import { Hash } from '@polkadot/types/interfaces'
 import { NamedKeyringPair } from '../../Types'
 import { checkBalance, validateAddress } from '../../helpers/validation'
-import { DerivedBalances } from '@polkadot/api-derive/types'
 
 type AccountTransferArgs = {
   recipient: string
@@ -36,15 +35,16 @@ export default class AccountTransferTokens extends AccountsCommandBase {
 
     // Initial validation
     validateAddress(args.recipient, 'Invalid recipient address')
-    const accBalances: DerivedBalances = (await this.getApi().getAccountsBalancesInfo([selectedAccount.address]))[0]
+    const accBalances = (await this.getApi().getAccountsBalancesInfo([selectedAccount.address]))[0]
     checkBalance(accBalances, amountBN)
 
     await this.requestAccountDecoding(selectedAccount)
 
     this.log(chalk.white('Estimating fee...'))
+    const tx = await this.getApi().createTransferTx(args.recipient, amountBN)
     let estimatedFee: BN
     try {
-      estimatedFee = await this.getApi().estimateFee(selectedAccount, args.recipient, amountBN)
+      estimatedFee = await this.getApi().estimateFee(selectedAccount, tx)
     } catch (e) {
       this.error('Could not estimate the fee.', { exit: ExitCodes.UnexpectedException })
     }
@@ -57,7 +57,7 @@ export default class AccountTransferTokens extends AccountsCommandBase {
     await this.requireConfirmation('Do you confirm the transfer?')
 
     try {
-      const txHash: Hash = await this.getApi().transfer(selectedAccount, args.recipient, amountBN)
+      const txHash: Hash = await tx.signAndSend(selectedAccount)
       this.log(chalk.greenBright('Transaction succesfully sent!'))
       this.log(chalk.white('Hash:', txHash.toString()))
     } catch (e) {

cli/src/commands/api/inspect.ts

@@ -2,9 +2,8 @@ import { flags } from '@oclif/command'
 import { CLIError } from '@oclif/errors'
 import { displayNameValueTable } from '../../helpers/display'
 import { ApiPromise } from '@polkadot/api'
-import { Option } from '@polkadot/types'
 import { Codec } from '@polkadot/types/types'
-import { ConstantCodec } from '@polkadot/api-metadata/consts/types'
+import { ConstantCodec } from '@polkadot/metadata/Decorated/consts/types'
 import ExitCodes from '../../ExitCodes'
 import chalk from 'chalk'
 import { NameValueObj, ApiMethodArg } from '../../Types'
@@ -35,9 +34,9 @@ export default class ApiInspect extends ApiCommandBase {
     '$ api:inspect',
     '$ api:inspect -t=query',
     '$ api:inspect -t=query -M=members',
-    '$ api:inspect -t=query -M=members -m=memberProfile',
-    '$ api:inspect -t=query -M=members -m=memberProfile -e',
-    '$ api:inspect -t=query -M=members -m=memberProfile -e -a=1',
+    '$ api:inspect -t=query -M=members -m=membershipById',
+    '$ api:inspect -t=query -M=members -m=membershipById -e',
+    '$ api:inspect -t=query -M=members -m=membershipById -e -a=1',
   ]
 
   static flags = {
@@ -99,7 +98,7 @@ export default class ApiInspect extends ApiCommandBase {
       return [type.asDoubleMap.key1.toString(), type.asDoubleMap.key2.toString()]
     }
     if (type.isMap) {
-      return type.asMap.linked.isTrue ? [`Option<${type.asMap.key.toString()}>`] : [type.asMap.key.toString()]
+      return [type.asMap.key.toString()]
     }
     return []
   }
@@ -110,14 +109,17 @@ export default class ApiInspect extends ApiCommandBase {
       const {
         meta: { type, modifier },
       } = method.creator
+      let typeName = type.toString()
       if (type.isDoubleMap) {
-        return type.asDoubleMap.value.toString()
+        typeName = type.asDoubleMap.value.toString()
       }
-      if (modifier.isOptional) {
-        return `Option<${type.toString()}>`
+      if (type.isMap) {
+        typeName = type.asMap.value.toString()
       }
+
+      return modifier.isOptional ? `Option<${typeName}>` : typeName
     }
-    // Fallback for "query" and default for "consts"
+    // Fallback for "consts"
     return this.getMethodMeta(apiType, apiModule, apiMethod).type.toString()
   }
 
@@ -127,7 +129,7 @@ export default class ApiInspect extends ApiCommandBase {
     api: ApiPromise,
     flags: ApiInspectFlags
   ): { apiType: ApiType | undefined; apiModule: string | undefined; apiMethod: string | undefined } {
-    let apiType: ApiType | undefined = undefined
+    let apiType: ApiType | undefined
     const { module: apiModule, method: apiMethod } = flags
 
     if (flags.type !== undefined) {
@@ -155,12 +157,7 @@ export default class ApiInspect extends ApiCommandBase {
     for (const [key, paramType] of Object.entries(paramTypes)) {
       this.log(chalk.bold.white(`Parameter no. ${parseInt(key) + 1} (${paramType}):`))
       const paramValue = await this.promptForParam(paramType)
-      if (paramValue instanceof Option && paramValue.isSome) {
-        result.push(paramValue.unwrap())
-      } else if (!(paramValue instanceof Option)) {
-        result.push(paramValue)
-      }
-      // In case of empty option we MUST NOT add anything to the array (otherwise it causes some error)
+      result.push(paramValue)
     }
 
     return result

cli/src/commands/working-groups/application.ts

@@ -11,6 +11,7 @@ export default class WorkingGroupsApplication extends WorkingGroupsCommandBase {
       description: 'Working Group Application ID',
     },
   ]
+
   static flags = {
     ...WorkingGroupsCommandBase.flags,
   }

cli/src/commands/working-groups/createOpening.ts

@@ -53,8 +53,8 @@ export default class WorkingGroupsCreateOpening extends WorkingGroupsCommandBase
       const module = apiModuleByGroup[this.group]
       const method = 'addOpening'
 
-      let saveDraft = false,
-        params: ApiMethodArg[]
+      let saveDraft = false
+      let params: ApiMethodArg[]
       if (flags.createDraftOnly) {
         params = await this.promptForExtrinsicParams(module, method, promptOptions)
         saveDraft = true

cli/src/commands/working-groups/decreaseWorkerStake.ts

@@ -1,6 +1,5 @@
 import WorkingGroupsCommandBase from '../../base/WorkingGroupsCommandBase'
 import { apiModuleByGroup } from '../../Api'
-import { WorkerId } from '@joystream/types/working-group'
 import { Balance } from '@polkadot/types/interfaces'
 import { formatBalance } from '@polkadot/util'
 import { minMaxInt } from '../../validators/common'
@@ -11,6 +10,7 @@ export default class WorkingGroupsDecreaseWorkerStake extends WorkingGroupsComma
   static description =
     'Decreases given worker stake by an amount that will be returned to the worker role account. ' +
     'Requires lead access.'
+
   static args = [
     {
       name: 'workerId',
@@ -18,6 +18,7 @@ export default class WorkingGroupsDecreaseWorkerStake extends WorkingGroupsComma
       description: 'Worker ID',
     },
   ]
+
   static flags = {
     ...WorkingGroupsCommandBase.flags,
   }
@@ -41,10 +42,7 @@ export default class WorkingGroupsDecreaseWorkerStake extends WorkingGroupsComma
 
     await this.requestAccountDecoding(account)
 
-    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'decreaseStake', [
-      new WorkerId(workerId),
-      balance,
-    ])
+    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'decreaseStake', [workerId, balance])
 
     this.log(
       chalk.green(

cli/src/commands/working-groups/evictWorker.ts

@@ -1,7 +1,5 @@
 import WorkingGroupsCommandBase from '../../base/WorkingGroupsCommandBase'
 import { apiModuleByGroup } from '../../Api'
-import { WorkerId } from '@joystream/types/working-group'
-import { bool } from '@polkadot/types/primitive'
 import { formatBalance } from '@polkadot/util'
 import chalk from 'chalk'
 import { createParamOptions } from '../../helpers/promptOptions'
@@ -15,6 +13,7 @@ export default class WorkingGroupsEvictWorker extends WorkingGroupsCommandBase {
       description: 'Worker ID',
     },
   ]
+
   static flags = {
     ...WorkingGroupsCommandBase.flags,
   }
@@ -43,9 +42,9 @@ export default class WorkingGroupsEvictWorker extends WorkingGroupsCommandBase {
     await this.requestAccountDecoding(account)
 
     await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'terminateRole', [
-      new WorkerId(workerId),
+      workerId,
       rationale,
-      new bool(shouldSlash),
+      shouldSlash,
     ])
 
     this.log(chalk.green(`Worker ${chalk.white(workerId)} has been evicted!`))

cli/src/commands/working-groups/fillOpening.ts

@@ -1,8 +1,6 @@
 import WorkingGroupsCommandBase from '../../base/WorkingGroupsCommandBase'
 import { OpeningStatus } from '../../Types'
 import { apiModuleByGroup } from '../../Api'
-import { OpeningId } from '@joystream/types/hiring'
-import { ApplicationIdSet, RewardPolicy } from '@joystream/types/working-group'
 import chalk from 'chalk'
 import { createParamOptions } from '../../helpers/promptOptions'
 
@@ -15,6 +13,7 @@ export default class WorkingGroupsFillOpening extends WorkingGroupsCommandBase {
       description: 'Working Group Opening ID',
     },
   ]
+
   static flags = {
     ...WorkingGroupsCommandBase.flags,
   }
@@ -30,16 +29,13 @@ export default class WorkingGroupsFillOpening extends WorkingGroupsCommandBase {
     const opening = await this.getOpeningForLeadAction(openingId, OpeningStatus.InReview)
 
     const applicationIds = await this.promptForApplicationsToAccept(opening)
-    const rewardPolicyOpt = await this.promptForParam(
-      `Option<${RewardPolicy.name}>`,
-      createParamOptions('RewardPolicy')
-    )
+    const rewardPolicyOpt = await this.promptForParam(`Option<RewardPolicy>`, createParamOptions('RewardPolicy'))
 
     await this.requestAccountDecoding(account)
 
     await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'fillOpening', [
-      new OpeningId(openingId),
-      new ApplicationIdSet(applicationIds),
+      openingId,
+      applicationIds,
       rewardPolicyOpt,
     ])
 

cli/src/commands/working-groups/opening.ts

@@ -15,6 +15,7 @@ export default class WorkingGroupsOpening extends WorkingGroupsCommandBase {
       description: 'Working Group Opening ID',
     },
   ]
+
   static flags = {
     ...WorkingGroupsCommandBase.flags,
   }

cli/src/commands/working-groups/slashWorker.ts

@@ -1,6 +1,5 @@
 import WorkingGroupsCommandBase from '../../base/WorkingGroupsCommandBase'
 import { apiModuleByGroup } from '../../Api'
-import { WorkerId } from '@joystream/types/working-group'
 import { Balance } from '@polkadot/types/interfaces'
 import { formatBalance } from '@polkadot/util'
 import { minMaxInt } from '../../validators/common'
@@ -16,6 +15,7 @@ export default class WorkingGroupsSlashWorker extends WorkingGroupsCommandBase {
       description: 'Worker ID',
     },
   ]
+
   static flags = {
     ...WorkingGroupsCommandBase.flags,
   }
@@ -39,10 +39,7 @@ export default class WorkingGroupsSlashWorker extends WorkingGroupsCommandBase {
 
     await this.requestAccountDecoding(account)
 
-    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'slashStake', [
-      new WorkerId(workerId),
-      balance,
-    ])
+    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'slashStake', [workerId, balance])
 
     this.log(
       chalk.green(

cli/src/commands/working-groups/startAcceptingApplications.ts

@@ -1,7 +1,6 @@
 import WorkingGroupsCommandBase from '../../base/WorkingGroupsCommandBase'
 import { OpeningStatus } from '../../Types'
 import { apiModuleByGroup } from '../../Api'
-import { OpeningId } from '@joystream/types/hiring'
 import chalk from 'chalk'
 
 export default class WorkingGroupsStartAcceptingApplications extends WorkingGroupsCommandBase {
@@ -13,6 +12,7 @@ export default class WorkingGroupsStartAcceptingApplications extends WorkingGrou
       description: 'Working Group Opening ID',
     },
   ]
+
   static flags = {
     ...WorkingGroupsCommandBase.flags,
   }
@@ -29,9 +29,7 @@ export default class WorkingGroupsStartAcceptingApplications extends WorkingGrou
 
     await this.requestAccountDecoding(account)
 
-    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'acceptApplications', [
-      new OpeningId(openingId),
-    ])
+    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'acceptApplications', [openingId])
 
     this.log(
       chalk.green(`Opening ${chalk.white(openingId)} status changed to: ${chalk.white('Accepting Applications')}`)

cli/src/commands/working-groups/startReviewPeriod.ts

@@ -1,7 +1,6 @@
 import WorkingGroupsCommandBase from '../../base/WorkingGroupsCommandBase'
 import { OpeningStatus } from '../../Types'
 import { apiModuleByGroup } from '../../Api'
-import { OpeningId } from '@joystream/types/hiring'
 import chalk from 'chalk'
 
 export default class WorkingGroupsStartReviewPeriod extends WorkingGroupsCommandBase {
@@ -13,6 +12,7 @@ export default class WorkingGroupsStartReviewPeriod extends WorkingGroupsCommand
       description: 'Working Group Opening ID',
     },
   ]
+
   static flags = {
     ...WorkingGroupsCommandBase.flags,
   }
@@ -29,9 +29,7 @@ export default class WorkingGroupsStartReviewPeriod extends WorkingGroupsCommand
 
     await this.requestAccountDecoding(account)
 
-    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'beginApplicantReview', [
-      new OpeningId(openingId),
-    ])
+    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'beginApplicantReview', [openingId])
 
     this.log(chalk.green(`Opening ${chalk.white(openingId)} status changed to: ${chalk.white('In Review')}`))
   }

cli/src/commands/working-groups/terminateApplication.ts

@@ -1,6 +1,6 @@
 import WorkingGroupsCommandBase from '../../base/WorkingGroupsCommandBase'
 import { apiModuleByGroup } from '../../Api'
-import { ApplicationStageKeys, ApplicationId } from '@joystream/types/hiring'
+import { ApplicationStageKeys } from '@joystream/types/hiring'
 import chalk from 'chalk'
 
 export default class WorkingGroupsTerminateApplication extends WorkingGroupsCommandBase {
@@ -12,6 +12,7 @@ export default class WorkingGroupsTerminateApplication extends WorkingGroupsComm
       description: 'Working Group Application ID',
     },
   ]
+
   static flags = {
     ...WorkingGroupsCommandBase.flags,
   }
@@ -29,9 +30,7 @@ export default class WorkingGroupsTerminateApplication extends WorkingGroupsComm
 
     await this.requestAccountDecoding(account)
 
-    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'terminateApplication', [
-      new ApplicationId(applicationId),
-    ])
+    await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'terminateApplication', [applicationId])
 
     this.log(chalk.green(`Application ${chalk.white(applicationId)} has been succesfully terminated!`))
   }

cli/src/commands/working-groups/updateRewardAccount.ts

@@ -1,7 +1,6 @@
 import WorkingGroupsCommandBase from '../../base/WorkingGroupsCommandBase'
 import { apiModuleByGroup } from '../../Api'
 import { validateAddress } from '../../helpers/validation'
-import { GenericAccountId } from '@polkadot/types'
 import chalk from 'chalk'
 import ExitCodes from '../../ExitCodes'
 
@@ -14,6 +13,7 @@ export default class WorkingGroupsUpdateRewardAccount extends WorkingGroupsComma
       description: 'New reward account address (if omitted, one of the existing CLI accounts can be selected)',
     },
   ]
+
   static flags = {
     ...WorkingGroupsCommandBase.flags,
   }
@@ -40,7 +40,7 @@ export default class WorkingGroupsUpdateRewardAccount extends WorkingGroupsComma
 
     await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'updateRewardAccount', [
       worker.workerId,
-      new GenericAccountId(newRewardAccount),
+      newRewardAccount,
     ])
 
     this.log(chalk.green(`Succesfully updated the reward account to: ${chalk.white(newRewardAccount)})`))

cli/src/commands/working-groups/updateRoleAccount.ts

@@ -1,7 +1,6 @@
 import WorkingGroupsCommandBase from '../../base/WorkingGroupsCommandBase'
 import { apiModuleByGroup } from '../../Api'
 import { validateAddress } from '../../helpers/validation'
-import { GenericAccountId } from '@polkadot/types'
 import chalk from 'chalk'
 
 export default class WorkingGroupsUpdateRoleAccount extends WorkingGroupsCommandBase {
@@ -13,6 +12,7 @@ export default class WorkingGroupsUpdateRoleAccount extends WorkingGroupsCommand
       description: 'New role account address (if omitted, one of the existing CLI accounts can be selected)',
     },
   ]
+
   static flags = {
     ...WorkingGroupsCommandBase.flags,
   }
@@ -34,7 +34,7 @@ export default class WorkingGroupsUpdateRoleAccount extends WorkingGroupsCommand
 
     await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'updateRoleAccount', [
       worker.workerId,
-      new GenericAccountId(newRoleAccount),
+      newRoleAccount,
     ])
 
     this.log(chalk.green(`Succesfully updated the role account to: ${chalk.white(newRoleAccount)})`))

cli/src/commands/working-groups/updateWorkerReward.ts

@@ -1,6 +1,5 @@
 import WorkingGroupsCommandBase from '../../base/WorkingGroupsCommandBase'
 import { apiModuleByGroup } from '../../Api'
-import { WorkerId } from '@joystream/types/working-group'
 import { formatBalance } from '@polkadot/util'
 import chalk from 'chalk'
 import { Reward } from '../../Types'
@@ -17,6 +16,7 @@ export default class WorkingGroupsUpdateWorkerReward extends WorkingGroupsComman
       description: 'Worker ID',
     },
   ]
+
   static flags = {
     ...WorkingGroupsCommandBase.flags,
   }
@@ -56,7 +56,7 @@ export default class WorkingGroupsUpdateWorkerReward extends WorkingGroupsComman
     await this.requestAccountDecoding(account)
 
     await this.sendAndFollowExtrinsic(account, apiModuleByGroup[this.group], 'updateRewardAmount', [
-      new WorkerId(workerId),
+      workerId,
       newRewardValue,
     ])
 

cli/src/helpers/validation.ts

@@ -1,7 +1,7 @@
 import BN from 'bn.js'
 import ExitCodes from '../ExitCodes'
 import { decodeAddress } from '@polkadot/util-crypto'
-import { DerivedBalances } from '@polkadot/api-derive/types'
+import { DeriveBalancesAll } from '@polkadot/api-derive/types'
 import { CLIError } from '@oclif/errors'
 
 export function validateAddress(address: string, errorMessage = 'Invalid address'): void {
@@ -12,7 +12,7 @@ export function validateAddress(address: string, errorMessage = 'Invalid address
   }
 }
 
-export function checkBalance(accBalances: DerivedBalances, requiredBalance: BN): void {
+export function checkBalance(accBalances: DeriveBalancesAll, requiredBalance: BN): void {
   if (requiredBalance.gt(accBalances.availableBalance)) {
     throw new CLIError('Not enough balance available', { exit: ExitCodes.InvalidInput })
   }

cli/src/promptOptions/addWorkerOpening.ts

@@ -1,32 +1,32 @@
 import { ApiParamsOptions, ApiParamOptions, HRTStruct } from '../Types'
-import {
-  OpeningType,
-  SlashingTerms,
-  UnslashableTerms,
-  OpeningType_Worker as OpeningTypeWorker,
-  WorkingGroupOpeningPolicyCommitment,
-} from '@joystream/types/working-group'
+import { OpeningType, WorkingGroupOpeningPolicyCommitment } from '@joystream/types/working-group'
+import { SlashingTerms } from '@joystream/types/common'
 import { Bytes } from '@polkadot/types'
 import { schemaValidator } from '@joystream/types/hiring'
+import { createType } from '@joystream/types'
 
 class OpeningPolicyCommitmentOptions implements ApiParamsOptions {
   [paramName: string]: ApiParamOptions
   public role_slashing_terms: ApiParamOptions<SlashingTerms> = {
     value: {
-      default: SlashingTerms.create('Unslashable', new UnslashableTerms()),
+      default: createType('SlashingTerms', { Unslashable: null }),
       locked: true,
     },
   }
+
   // Rename fields containing "curator" (solivg minor UI issue related to flat namespace)
   public terminate_curator_application_stake_unstaking_period: ApiParamOptions = {
     forcedName: 'terminate_application_stake_unstaking_period',
   }
+
   public terminate_curator_role_stake_unstaking_period: ApiParamOptions = {
     forcedName: 'terminate_role_stake_unstaking_period',
   }
+
   public exit_curator_role_application_stake_unstaking_period: ApiParamOptions = {
     forcedName: 'exit_role_application_stake_unstaking_period',
   }
+
   public exit_curator_role_stake_unstaking_period: ApiParamOptions = {
     forcedName: 'exit_role_stake_unstaking_period',
   }
@@ -37,10 +37,11 @@ class AddWrokerOpeningOptions implements ApiParamsOptions {
   // Lock value for opening_type
   public opening_type: ApiParamOptions<OpeningType> = {
     value: {
-      default: OpeningType.create('Worker', new OpeningTypeWorker()),
+      default: createType('OpeningType', { Worker: null }),
       locked: true,
     },
   }
+
   // Json schema for human_readable_text
   public human_readable_text: ApiParamOptions<Bytes> = {
     jsonSchema: {
@@ -48,6 +49,7 @@ class AddWrokerOpeningOptions implements ApiParamsOptions {
       struct: HRTStruct,
     },
   }
+
   // Lock value for role_slashing_terms
   public commitment: ApiParamOptions<WorkingGroupOpeningPolicyCommitment> = {
     nestedOptions: new OpeningPolicyCommitmentOptions(),

cli/tsconfig.json

@@ -9,7 +9,11 @@
     "target": "es2017",
     "esModuleInterop": true,
     "types" : [ "node" ],
-    "noUnusedLocals": true
+    "noUnusedLocals": true,
+    "baseUrl": ".",
+    "paths": {
+      "@polkadot/types/augment": ["../types/augment-codec/augment-types.ts"],
+    }
   },
   "include": [
     "src/**/*"

devops/ansible/build-and-run-tests-exported-chainspec-playbook.yml

@@ -0,0 +1,22 @@
+- hosts: 127.0.0.1
+  user: root
+  become: yes
+  become_method: sudo
+
+  tasks:
+
+    - name: install dependencies
+      include_role:
+        name: install_dependencies
+
+    - name: alter block creation time
+      include_role:
+        name: alter_block_creation_time
+
+    - name: build node
+      include_role:
+        name: build_docker_image
+
+    - name: run tests
+      include_role:
+        name: run_tests_exported_chainspec

devops/ansible/build-and-run-tests-single-node-playbook.yml

@@ -4,14 +4,19 @@
   become_method: sudo
 
   tasks:
+
     - name: install dependencies
       include_role:
         name: install_dependencies
 
+    - name: alter block creation time
+      include_role:
+        name: alter_block_creation_time
+
     - name: build node
       include_role:
         name: build_docker_image
-        
+
     - name: run tests
       include_role:
         name: run_tests_single_node

devops/ansible/build-and-run-tests-two-nodes-playbook.yml

@@ -4,14 +4,19 @@
   become_method: sudo
 
   tasks:
+
     - name: install dependencies
       include_role:
         name: install_dependencies
 
+    - name: alter block creation time
+      include_role:
+        name: alter_block_creation_time
+
     - name: build node
       include_role:
         name: build_docker_image
-        
+
     - name: run tests
       include_role:
         name: run_tests_two_nodes

devops/ansible/build-image-playbook.yml

@@ -4,10 +4,15 @@
   become_method: sudo
 
   tasks:
+
     - name: install dependencies
       include_role:
         name: install_dependencies
 
+    - name: alter block creation time
+      include_role:
+        name: alter_block_creation_time
+
     - name: build node
       include_role:
-        name: build_docker_image
+        name: build_docker_image

devops/ansible/docker-compose.yml

@@ -3,7 +3,7 @@ services:
   node_alice:
     image: joystream/node-testing
     container_name: alice
-    entrypoint: ./node --chain=chainspec.json --alice --validator --ws-external --rpc-cors=all
+    entrypoint: ./node --dev --alice --validator --unsafe-ws-external --rpc-cors=all
     ports:
       - "30333:30333"
       - "9933:9933"
@@ -15,7 +15,7 @@ services:
   node_bob:
     image: joystream/node-testing
     container_name: bob
-    entrypoint: ./node --chain=chainspec.json --bob --ws-external --rpc-cors=all
+    entrypoint: ./node --dev --bob --validator --unsafe-ws-external --rpc-cors=all
     ports:
       - "30335:30333"
       - "9935:9933"

devops/ansible/roles/alter_block_creation_time/tasks/main.yml

@@ -0,0 +1,4 @@
+- name: alter block creation time
+  shell: ./scripts/alter-block-creation-time.sh
+  args:
+    chdir: ../../

devops/ansible/roles/build_docker_image/tasks/main.yml

@@ -1,4 +1,4 @@
 - name: create testing node docker image
-  shell: ./scripts/build-joystream-testing-node-docker-image.sh
+  shell: ./scripts/build-joystream-node-docker-image.sh
   args:
     chdir: ../../

devops/ansible/roles/install_dependencies/tasks/main.yml

@@ -1,7 +1,13 @@
-- name: install pip on Debian
+- name: install pip and npm on Debian
   block:
+    - name: create temporary folder
+      file:
+        path: ../../.tmp
+        state: directory
     - name: install pip using apt
       apt: name=python-pip state=present
+    - name: install npm using apt
+      apt: name=npm state=present
   when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu'
 
 - name: install pip on Mac
@@ -11,15 +17,15 @@
         path: ../../.tmp
         state: directory
     - name: get pip installer using curl
-      get_url: 
+      get_url:
         url: https://bootstrap.pypa.io/get-pip.py
         dest: ../../.tmp/get-pip.py
     - name: install pip
       shell: python ../../.tmp/get-pip.py
   when: ansible_distribution == 'MacOSX'
-  always: 
+  always:
     - name: remove pip installer script
-      file: 
+      file:
         path: ../../.tmp/get-pip.py
         state: absent
 
@@ -31,5 +37,10 @@
     name: yarn
     global: yes
 
+- name: Install pyrsistent
+  pip:
+    name: pyrsistent==0.16.0
+
 - name: Install docker compose
-  pip: name=docker-compose
+  pip:
+    name: docker-compose==1.26.2

devops/ansible/roles/run_tests_exported_chainspec/tasks/main.yml

@@ -0,0 +1,38 @@
+- name: run network
+  block:
+    - name: yarn install for joystream types
+      shell: yarn workspace @joystream/types install
+
+    - name: yarn build for joystream types
+      shell: yarn workspace @joystream/types build
+
+    - name: yarn install for network tests
+      shell: yarn workspace joystream-testing install
+
+    - name: run docker container
+      docker_container:
+        name: "joystream-node"
+        image: "joystream/node"
+        ports:
+          - "9944:9944"
+        mounts:
+          - target: /testnet-state
+            source: "{{ playbook_dir }}/../../testnets/nicaea-exported-state"
+            type: bind
+            read_only: yes
+        entrypoint: ./node --chain ../testnet-state/raw_chain_spec.json --alice --validator --unsafe-ws-external --rpc-cors=all
+        state: started
+
+    - name: execute network tests
+      shell: yarn test >> ../../.tmp/tests.log
+      args:
+        chdir: ../../tests/network-tests/
+
+  always:
+    - name: display tests log
+      shell: cat ../../.tmp/tests.log
+
+    - name: stop docker container
+      docker_container:
+        name: "joystream-node-testing"
+        state: absent

devops/ansible/roles/run_tests_single_node/tasks/main.yml

@@ -1,21 +1,32 @@
 - name: run network
   block:
+    - name: yarn install for joystream types
+      shell: yarn workspace @joystream/types install
+
+    - name: yarn build for joystream types
+      shell: yarn workspace @joystream/types build
+
+    - name: yarn install for network tests
+      shell: yarn workspace joystream-testing install
 
     - name: run docker container
       docker_container:
-        name: "joystream-node-testing"
-        image: "joystream/node-testing"
+        name: "joystream-node"
+        image: "joystream/node"
         ports:
           - "9944:9944"
-        entrypoint: ./node --chain=chainspec.json --alice --validator --ws-external --rpc-cors=all
+        entrypoint: ./node --dev --alice --validator --unsafe-ws-external --rpc-cors=all
         state: started
 
     - name: execute network tests
-      shell: yarn debug >> ../../.tmp/tests.log
+      shell: yarn test >> ../../.tmp/tests.log
       args:
         chdir: ../../tests/network-tests/
-        
+
   always:
+    - name: display tests log
+      shell: cat ../../.tmp/tests.log
+
     - name: stop docker container
       docker_container:
         name: "joystream-node-testing"

devops/dockerfiles/ansible-node/Dockerfile

@@ -1,37 +0,0 @@
-FROM joystream/rust-builder AS builder
-LABEL description="Compiles all workspace artifacts"
-WORKDIR /joystream
-COPY . /joystream
-
-# Build joystream-node and its dependencies - runtime
-RUN cargo build --release -p joystream-node
-RUN /joystream/scripts/create-test-chainspec.sh
-
-FROM debian:stretch
-LABEL description="Joystream node"
-WORKDIR /joystream
-COPY --from=builder /joystream/target/release/joystream-node /joystream/node
-COPY --from=builder /joystream/target/release/wbuild/joystream-node-runtime/joystream_node_runtime.compact.wasm /joystream/runtime.compact.wasm
-COPY --from=builder /joystream/.tmp/chainspec.json /joystream/chainspec.json
-
-# confirm it works
-RUN /joystream/node --version
-
-# https://manpages.debian.org/stretch/coreutils/b2sum.1.en.html
-# RUN apt-get install coreutils
-# print the blake2 256 hash of the wasm blob
-RUN b2sum -l 256 /joystream/runtime.compact.wasm
-# print the blake2 512 hash of the wasm blob
-RUN b2sum -l 512 /joystream/runtime.compact.wasm
-
-EXPOSE 30333 9933 9944
-
-# Use these volumes to persits chain state and keystore, eg.:
-# --base-path /data
-# optionally separate keystore (otherwise it will be stored in the base path)
-# --keystore-path /keystore
-# if base-path isn't specified, chain state is stored inside container in ~/.local/share/joystream-node/
-# which is not ideal
-VOLUME ["/data", "/keystore"]
-
-ENTRYPOINT ["/joystream/node"]

devops/dockerfiles/node-and-runtime/Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /joystream
 COPY . /joystream
 
 # Build joystream-node and its dependencies - runtime
-RUN cargo build --release -p joystream-node
+RUN WASM_BUILD_TOOLCHAIN=nightly-2020-05-23 cargo build --release -p joystream-node
 
 FROM debian:stretch
 LABEL description="Joystream node"

devops/dockerfiles/rust-builder/Dockerfile

@@ -1,8 +1,8 @@
-FROM liuchong/rustup:1.43.0 AS builder
+FROM liuchong/rustup:1.46.0 AS builder
 LABEL description="Rust and WASM build environment for joystream and substrate"
 
 WORKDIR /setup
 COPY setup.sh /setup
 ENV TERM=xterm
 
-RUN ./setup.sh
+RUN ./setup.sh

devops/eslint-config/index.js

@@ -43,7 +43,7 @@ module.exports = {
     // should prefer using 'debug' package at least to allow control of
     // output verbosity if logging to console.
     'no-console': 'off',
-    '@typescript-eslint/camelcase': 'off',
+    'camelcase': 'off',
     '@typescript-eslint/class-name-casing': 'off',
     "@typescript-eslint/naming-convention": [
       "error",

devops/eslint-config/package.json

@@ -17,14 +17,15 @@
   },
   "homepage": "https://github.com/joystream/joystream#readme",
   "peerDependencies": {
-    "eslint": ">= 5"
+    "eslint": "^7.6.0"
   },
   "dependencies": {
-    "@typescript-eslint/parser": "^2.34.0",
+    "@typescript-eslint/eslint-plugin": "3.8.0",
+    "@typescript-eslint/parser": "3.8.0",
     "eslint-config-prettier": "^6.11.0",
     "eslint-plugin-prettier": "^3.1.3",
-    "eslint-plugin-react": "^7.16.0",
-    "eslint-plugin-react-hooks": "^2.3.0",
+    "eslint-plugin-react": "^7.20.5",
+    "eslint-plugin-react-hooks": "^4.0.8",
     "eslint-config-standard": "^14.1.1",
     "eslint-plugin-standard": "^4.0.1",
     "eslint-plugin-promise": "^4.2.1",

devops/git-hooks/pre-push

@@ -1,10 +1,10 @@
 #!/bin/sh
 set -e
 
-echo '+cargo test --release --all'
-BUILD_DUMMY_WASM_BINARY=1 cargo test --all
-
-echo '+cargo clippy --release --all -- -D warnings'
-BUILD_DUMMY_WASM_BINARY=1 cargo clippy --all -- -D warnings
+export WASM_BUILD_TOOLCHAIN=nightly-2020-05-23
 
+echo '+cargo clippy --all -- -D warnings'
+BUILD_DUMMY_WASM_BINARY=1 cargo clippy --release --all -- -D warnings
 
+echo '+cargo test --all'
+cargo test --release --all

node/Cargo.toml

@@ -1,9 +1,9 @@
 [package]
-authors = ['Joystream']
+authors = ['Joystream contributors']
 build = 'build.rs'
 edition = '2018'
 name = 'joystream-node'
-version = '2.7.0'
+version = '3.1.0'
 default-run = "joystream-node"
 
 [[bin]]
@@ -14,143 +14,99 @@ path = 'bin/main.rs'
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-hex-literal = '0.2.1'
-derive_more = '0.14.0'
-exit-future = '0.1.4'
-futures = '0.1.29'
-log = '0.4.8'
-parking_lot = '0.9.0'
-tokio = '0.1.22'
-jsonrpc-core = '13.2.0'
-rand = '0.7.2'
-structopt = '=0.3.5'
+# third-party dependencies
+serde = { version = "1.0.102", features = ["derive"] }
+futures = { version = "0.3.1", features = ["compat"] }
+jsonrpc-core = "14.2.0"
+structopt = { version = "0.3.8", optional = true}
 serde_json = '1.0'
-serde = '1.0'
-hex = '0.4'
-# https://users.rust-lang.org/t/failure-derive-compilation-error/39062
-# quote = '<=1.0.2'
-
-[dependencies.node-runtime]
-package = 'joystream-node-runtime'
-path = '../runtime'
-
-[dependencies.substrate-basic-authorship]
-git = 'https://github.com/paritytech/substrate.git'
-package = 'substrate-basic-authorship'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.babe]
-git = 'https://github.com/paritytech/substrate.git'
-package = 'substrate-consensus-babe'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.babe-primitives]
-git = 'https://github.com/paritytech/substrate.git'
-package = 'substrate-consensus-babe-primitives'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.codec]
-package = 'parity-scale-codec'
-version = '1.0.0'
-
-[dependencies.ctrlc]
-features = ['termination']
-version = '3.0'
-
-[dependencies.inherents]
-git = 'https://github.com/paritytech/substrate.git'
-package = 'substrate-inherents'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.network]
-git = 'https://github.com/paritytech/substrate.git'
-package = 'substrate-network'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.primitives]
-git = 'https://github.com/paritytech/substrate.git'
-package = 'substrate-primitives'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.sr-io]
-git = 'https://github.com/paritytech/substrate.git'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.substrate-cli]
-git = 'https://github.com/paritytech/substrate.git'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.substrate-client]
-git = 'https://github.com/paritytech/substrate.git'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.substrate-executor]
-git = 'https://github.com/paritytech/substrate.git'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.substrate-service]
-git = 'https://github.com/paritytech/substrate.git'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.transaction-pool]
-git = 'https://github.com/paritytech/substrate.git'
-package = 'substrate-transaction-pool'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.substrate-telemetry]
-git = 'https://github.com/paritytech/substrate.git'
-package = 'substrate-telemetry'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.grandpa]
-git = 'https://github.com/paritytech/substrate.git'
-package = 'substrate-finality-grandpa'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.grandpa-primitives]
-git = 'https://github.com/paritytech/substrate.git'
-package = 'substrate-finality-grandpa-primitives'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.im-online]
-default_features = false
-git = 'https://github.com/paritytech/substrate.git'
-package = 'srml-im-online'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.substrate-rpc]
-default_features = false
-git = 'https://github.com/paritytech/substrate.git'
-package = 'substrate-rpc'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.authority-discovery]
-default_features = false
-git = 'https://github.com/paritytech/substrate.git'
-package = 'substrate-authority-discovery'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.client-db]
-default_features = false
-git = 'https://github.com/paritytech/substrate.git'
-package = 'substrate-client-db'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.runtime-primitives]
-default_features = false
-git = 'https://github.com/paritytech/substrate.git'
-package = 'sr-primitives'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.offchain]
-default_features = false
-git = 'https://github.com/paritytech/substrate.git'
-package = 'substrate-offchain'
-rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
-
-[dependencies.libp2p]
-version = '0.13.2'
-default-features = false
+codec = { package = "parity-scale-codec", version = "1.3.1" }
+hex = { package = "hex", version = "0.4.2" }
+
+# primitives
+sp-authority-discovery = { package = 'sp-authority-discovery', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sp-consensus-babe = { package = 'sp-consensus-babe', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sp-finality-grandpa = { package = 'sp-finality-grandpa', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sp-core = { package = 'sp-core', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sp-runtime = { package = 'sp-runtime', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sp-finality-tracker = { package = 'sp-finality-tracker', default-features = false, git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sp-inherents = { package = 'sp-inherents', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sp-consensus = { package = 'sp-consensus', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sp-transaction-pool = { package = 'sp-transaction-pool', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sp-api = { package = 'sp-api', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sp-blockchain = { package = 'sp-blockchain', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sp-block-builder = { package = 'sp-block-builder', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+
+# client dependencies
+sc-client-api = { package = 'sc-client-api', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sc-chain-spec = { package = 'sc-chain-spec', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sc-consensus = { package = 'sc-consensus', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sc-transaction-pool = { package = 'sc-transaction-pool', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sc-network = { package = 'sc-network', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sc-consensus-babe = { package = 'sc-consensus-babe', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4'}
+sc-finality-grandpa = { package = 'sc-finality-grandpa', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sc-basic-authorship = { package = 'sc-basic-authorship', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sc-service = { package = 'sc-service', default-features = false, git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sc-authority-discovery = { package = 'sc-authority-discovery', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sc-consensus-epochs = { package = 'sc-consensus-epochs', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sc-keystore = { package = 'sc-keystore', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sc-consensus-babe-rpc = { package = 'sc-consensus-babe-rpc', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sc-finality-grandpa-rpc = { package = 'sc-finality-grandpa-rpc', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sc-rpc-api = { package = 'sc-rpc-api', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sc-executor = { package = 'sc-executor', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+
+# frame dependencies
+pallet-im-online = { package = 'pallet-im-online', default-features = false, git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+pallet-transaction-payment-rpc = { package = 'pallet-transaction-payment-rpc', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+substrate-frame-rpc-system = { package = 'substrate-frame-rpc-system', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+frame-benchmarking = { package = 'frame-benchmarking', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+
+# node-specific dependencies
+node-runtime = { package= "joystream-node-runtime", path = "../runtime" }
+
+# CLI-specific dependencies
+sc-cli = { package = 'sc-cli', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4', optional = true }
+frame-benchmarking-cli = { package = 'frame-benchmarking-cli', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4', optional = true }
+node-inspect = { package = 'node-inspect', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4', optional = true }
+
+# WASM-specific dependencies
+wasm-bindgen = { version = "0.2.57", optional = true }
+wasm-bindgen-futures = { version = "0.4.7", optional = true }
+browser-utils = { package = 'substrate-browser-utils', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4', optional = true}
+
+[dev-dependencies]
+tempfile = "3.1.0"
+sp-timestamp = { package = 'sp-timestamp', default-features = false, git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sp-keyring = { package = 'sp-keyring', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+sc-consensus-babe = { git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4', features = ["test-helpers"]}
+sc-service-test = { git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+frame-system = { package = 'frame-system', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+pallet-transaction-payment = { package = 'pallet-transaction-payment', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
+pallet-grandpa = { package = 'pallet-grandpa', git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4' }
 
 [build-dependencies]
-vergen = '3'
+structopt = { version = "0.3.8", optional = true }
+node-inspect = { git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4', optional = true}
+sc-cli = { git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4', optional = true}
+frame-benchmarking-cli = { git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4', optional = true }
+substrate-build-script-utils = { git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4', optional = true }
+
+[features]
+default = [ "cli" ]
+browser = [
+	"browser-utils",
+	"wasm-bindgen",
+	"wasm-bindgen-futures",
+]
+cli = [
+	"node-inspect",
+	"sc-cli",
+	"frame-benchmarking-cli",
+	"sc-service/db",
+	"structopt",
+	"substrate-build-script-utils",
+]
+runtime-benchmarks = [
+	"node-runtime/runtime-benchmarks",
+	"frame-benchmarking-cli",
+]

node/README.md

@@ -26,7 +26,7 @@ cd joystream/
 Compile the node and runtime:
 
 ```bash
-cargo build --release
+WASM_BUILD_TOOLCHAIN=nightly-2020-05-23 cargo build --release
 ```
 
 This produces the binary in `./target/release/joystream-node`
@@ -34,7 +34,7 @@ This produces the binary in `./target/release/joystream-node`
 ### Running local development chain
 
 ```bash
-cargo run --release -- --dev
+./target/release/joystream-node --dev
 ```
 
 If you repeatedly need to restart a new chain,
@@ -49,7 +49,7 @@ this script will build and run a fresh new local development chain (purging exis
 Use the `--chain` argument, and specify the path to the genesis `chain.json` file for that public network. The JSON "chain spec" files for Joystream public networks can be found in [../testnets/](../testnets/).
 
 ```bash
-cargo run --release -- --chain testnets/rome.json
+./target/release/joystream-node --chain testnets/rome.json
 ```
 
 ### Tests and code quality

node/bin/main.rs

@@ -14,27 +14,10 @@
 // You should have received a copy of the GNU General Public License
 // along with Joystream node.  If not, see <http://www.gnu.org/licenses/>.
 
-//! Substrate Node Template CLI library.
+//! Joystream Node.
 
 #![warn(missing_docs)]
-#![warn(unused_extern_crates)]
 
-use joystream_node::cli;
-pub use substrate_cli::{error, IntoExit, VersionInfo};
-
-fn main() {
-    let version = VersionInfo {
-        name: "Joystream Node",
-        commit: env!("VERGEN_SHA_SHORT"),
-        version: env!("CARGO_PKG_VERSION"),
-        executable_name: "joystream-node",
-        author: "Joystream",
-        description: "Joystream substrate node",
-        support_url: "https://www.joystream.org/",
-    };
-
-    if let Err(e) = cli::run(::std::env::args(), cli::Exit, version) {
-        eprintln!("Fatal error: {}\n\n{:?}", e, e);
-        std::process::exit(1)
-    }
+fn main() -> sc_cli::Result<()> {
+    joystream_node::command::run()
 }

node/build.rs

@@ -1,27 +1,72 @@
-use std::{env, path::PathBuf};
+// This file is part of Substrate.
 
-use vergen::{generate_cargo_keys, ConstantsFlags};
+// Copyright (C) 2017-2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
 
-const ERROR_MSG: &str = "Failed to generate metadata files";
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 fn main() {
-    generate_cargo_keys(ConstantsFlags::SHA_SHORT).expect(ERROR_MSG);
-
-    let mut manifest_dir = PathBuf::from(
-        env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is always set by cargo."),
-    );
-
-    while manifest_dir.parent().is_some() {
-        if manifest_dir.join(".git/HEAD").exists() {
-            println!(
-                "cargo:rerun-if-changed={}",
-                manifest_dir.join(".git/HEAD").display()
-            );
-            return;
-        }
+    #[cfg(feature = "cli")]
+    cli::main();
+}
+
+#[cfg(feature = "cli")]
+mod cli {
+    include!("src/cli.rs");
+
+    use sc_cli::structopt::clap::Shell;
+    use std::{env, fs, path::Path};
+    use substrate_build_script_utils::{generate_cargo_keys, rerun_if_git_head_changed};
+
+    pub fn main() {
+        build_shell_completion();
+        generate_cargo_keys();
 
-        manifest_dir.pop();
+        rerun_if_git_head_changed();
     }
 
-    println!("cargo:warning=Could not find `.git/HEAD` from manifest dir!");
+    /// Build shell completion scripts for all known shells
+    /// Full list in https://github.com/kbknapp/clap-rs/blob/e9d0562a1dc5dfe731ed7c767e6cee0af08f0cf9/src/app/parser.rs#L123
+    fn build_shell_completion() {
+        for shell in &[
+            Shell::Bash,
+            Shell::Fish,
+            Shell::Zsh,
+            Shell::Elvish,
+            Shell::PowerShell,
+        ] {
+            build_completion(shell);
+        }
+    }
+
+    /// Build the shell auto-completion for a given Shell
+    fn build_completion(shell: &Shell) {
+        let outdir = match env::var_os("OUT_DIR") {
+            None => return,
+            Some(dir) => dir,
+        };
+        let path = Path::new(&outdir)
+            .parent()
+            .unwrap()
+            .parent()
+            .unwrap()
+            .parent()
+            .unwrap()
+            .join("completion-scripts");
+
+        fs::create_dir(&path).ok();
+
+        Cli::clap().gen_completions("joystream-node", *shell, &path);
+    }
 }

node/res/forum_data_empty.json

@@ -1 +0,0 @@
-{ "categories":[], "posts":[], "threads":[] }

node/src/chain_spec.rs

@@ -1,369 +0,0 @@
-// Copyright 2019 Joystream Contributors
-// This file is part of Joystream node.
-
-// Joystream node is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Joystream node is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Joystream node.  If not, see <http://www.gnu.org/licenses/>.
-
-// Clippy linter warning.
-// Disable it because we use such syntax for a code readability.
-// Example:  voting_period: 1 * DAY
-#![allow(clippy::identity_op)]
-
-use node_runtime::{
-    versioned_store::InputValidationLengthConstraint as VsInputValidation,
-    AuthorityDiscoveryConfig, BabeConfig, Balance, BalancesConfig, ContentWorkingGroupConfig,
-    CouncilConfig, CouncilElectionConfig, DataObjectStorageRegistryConfig,
-    DataObjectTypeRegistryConfig, ElectionParameters, GrandpaConfig, ImOnlineConfig, IndicesConfig,
-    MembersConfig, MigrationConfig, Perbill, ProposalsCodexConfig, SessionConfig, SessionKeys,
-    Signature, StakerStatus, StakingConfig, StorageWorkingGroupConfig, SudoConfig, SystemConfig,
-    VersionedStoreConfig, DAYS, WASM_BINARY,
-};
-pub use node_runtime::{AccountId, GenesisConfig};
-use primitives::{sr25519, Pair, Public};
-use runtime_primitives::traits::{IdentifyAccount, Verify};
-
-use babe_primitives::AuthorityId as BabeId;
-use grandpa_primitives::AuthorityId as GrandpaId;
-use im_online::sr25519::AuthorityId as ImOnlineId;
-use serde_json as json;
-
-type AccountPublic = <Signature as Verify>::Signer;
-
-/// Specialized `ChainSpec`. This is a specialization of the general Substrate ChainSpec type.
-pub type ChainSpec = substrate_service::ChainSpec<GenesisConfig>;
-
-use node_runtime::common::constraints::InputValidationLengthConstraint;
-
-/// The chain specification option. This is expected to come in from the CLI and
-/// is little more than one of a number of alternatives which can easily be converted
-/// from a string (`--chain=...`) into a `ChainSpec`.
-#[derive(Clone, Debug)]
-pub enum Alternative {
-    /// Whatever the current runtime is, with just Alice as an auth.
-    Development,
-    /// Whatever the current runtime is, with simple Alice/Bob auths.
-    LocalTestnet,
-}
-
-/// Helper function to generate a crypto pair from seed
-pub fn get_from_seed<TPublic: Public>(seed: &str) -> <TPublic::Pair as Pair>::Public {
-    TPublic::Pair::from_string(&format!("//{}", seed), None)
-        .expect("static values are valid; qed")
-        .public()
-}
-
-/// Helper function to generate an account ID from seed
-pub fn get_account_id_from_seed<TPublic: Public>(seed: &str) -> AccountId
-where
-    AccountPublic: From<<TPublic::Pair as Pair>::Public>,
-{
-    AccountPublic::from(get_from_seed::<TPublic>(seed)).into_account()
-}
-
-/// Helper function to generate stash, controller and session key from seed
-pub fn get_authority_keys_from_seed(
-    seed: &str,
-) -> (AccountId, AccountId, GrandpaId, BabeId, ImOnlineId) {
-    (
-        get_account_id_from_seed::<sr25519::Public>(&format!("{}//stash", seed)),
-        get_account_id_from_seed::<sr25519::Public>(seed),
-        get_from_seed::<GrandpaId>(seed),
-        get_from_seed::<BabeId>(seed),
-        get_from_seed::<ImOnlineId>(seed),
-    )
-}
-
-fn session_keys(grandpa: GrandpaId, babe: BabeId, im_online: ImOnlineId) -> SessionKeys {
-    SessionKeys {
-        grandpa,
-        babe,
-        im_online,
-    }
-}
-
-impl Alternative {
-    /// Get an actual chain config from one of the alternatives.
-    pub(crate) fn load(self) -> Result<ChainSpec, String> {
-        Ok(match self {
-            Alternative::Development => ChainSpec::from_genesis(
-                "Development",
-                "dev",
-                || {
-                    testnet_genesis(
-                        vec![get_authority_keys_from_seed("Alice")],
-                        get_account_id_from_seed::<sr25519::Public>("Alice"),
-                        vec![
-                            get_account_id_from_seed::<sr25519::Public>("Alice"),
-                            get_account_id_from_seed::<sr25519::Public>("Bob"),
-                            get_account_id_from_seed::<sr25519::Public>("Alice//stash"),
-                            get_account_id_from_seed::<sr25519::Public>("Bob//stash"),
-                        ],
-                        crate::proposals_config::development(),
-                    )
-                },
-                vec![],
-                None,
-                None,
-                Some(chain_spec_properties()),
-                None,
-            ),
-            Alternative::LocalTestnet => ChainSpec::from_genesis(
-                "Local Testnet",
-                "local_testnet",
-                || {
-                    testnet_genesis(
-                        vec![
-                            get_authority_keys_from_seed("Alice"),
-                            get_authority_keys_from_seed("Bob"),
-                        ],
-                        get_account_id_from_seed::<sr25519::Public>("Alice"),
-                        vec![
-                            get_account_id_from_seed::<sr25519::Public>("Alice"),
-                            get_account_id_from_seed::<sr25519::Public>("Bob"),
-                            get_account_id_from_seed::<sr25519::Public>("Charlie"),
-                            get_account_id_from_seed::<sr25519::Public>("Dave"),
-                            get_account_id_from_seed::<sr25519::Public>("Eve"),
-                            get_account_id_from_seed::<sr25519::Public>("Ferdie"),
-                            get_account_id_from_seed::<sr25519::Public>("Alice//stash"),
-                            get_account_id_from_seed::<sr25519::Public>("Bob//stash"),
-                            get_account_id_from_seed::<sr25519::Public>("Charlie//stash"),
-                            get_account_id_from_seed::<sr25519::Public>("Dave//stash"),
-                            get_account_id_from_seed::<sr25519::Public>("Eve//stash"),
-                            get_account_id_from_seed::<sr25519::Public>("Ferdie//stash"),
-                        ],
-                        crate::proposals_config::development(),
-                    )
-                },
-                vec![],
-                None,
-                None,
-                Some(chain_spec_properties()),
-                None,
-            ),
-        })
-    }
-
-    pub(crate) fn from(s: &str) -> Option<Self> {
-        match s {
-            "dev" => Some(Alternative::Development),
-            "local" => Some(Alternative::LocalTestnet),
-            _ => None,
-        }
-    }
-}
-
-fn new_vs_validation(min: u16, max_min_diff: u16) -> VsInputValidation {
-    VsInputValidation { min, max_min_diff }
-}
-
-pub fn chain_spec_properties() -> json::map::Map<String, json::Value> {
-    let mut properties: json::map::Map<String, json::Value> = json::map::Map::new();
-    properties.insert(
-        String::from("tokenDecimals"),
-        json::Value::Number(json::Number::from(0)),
-    );
-    properties.insert(
-        String::from("tokenSymbol"),
-        json::Value::String(String::from("JOY")),
-    );
-    properties
-}
-
-pub fn testnet_genesis(
-    initial_authorities: Vec<(AccountId, AccountId, GrandpaId, BabeId, ImOnlineId)>,
-    root_key: AccountId,
-    endowed_accounts: Vec<AccountId>,
-    cpcp: node_runtime::ProposalsConfigParameters,
-) -> GenesisConfig {
-    const CENTS: Balance = 1;
-    const DOLLARS: Balance = 100 * CENTS;
-    const STASH: Balance = 20 * DOLLARS;
-    const ENDOWMENT: Balance = 100_000 * DOLLARS;
-
-    let default_text_constraint = node_runtime::working_group::default_text_constraint();
-
-    GenesisConfig {
-        system: Some(SystemConfig {
-            code: WASM_BINARY.to_vec(),
-            changes_trie_config: Default::default(),
-        }),
-        balances: Some(BalancesConfig {
-            balances: endowed_accounts
-                .iter()
-                .cloned()
-                .map(|k| (k, ENDOWMENT))
-                .chain(initial_authorities.iter().map(|x| (x.0.clone(), STASH)))
-                .collect(),
-            vesting: vec![],
-        }),
-        indices: Some(IndicesConfig { ids: vec![] }),
-        session: Some(SessionConfig {
-            keys: initial_authorities
-                .iter()
-                .map(|x| {
-                    (
-                        x.0.clone(),
-                        session_keys(x.2.clone(), x.3.clone(), x.4.clone()),
-                    )
-                })
-                .collect::<Vec<_>>(),
-        }),
-        staking: Some(StakingConfig {
-            current_era: 0,
-            validator_count: 20,
-            minimum_validator_count: 1,
-            stakers: initial_authorities
-                .iter()
-                .map(|x| (x.0.clone(), x.1.clone(), STASH, StakerStatus::Validator))
-                .collect(),
-            invulnerables: initial_authorities.iter().map(|x| x.0.clone()).collect(),
-            slash_reward_fraction: Perbill::from_percent(10),
-            ..Default::default()
-        }),
-        sudo: Some(SudoConfig { key: root_key }),
-        babe: Some(BabeConfig {
-            authorities: vec![],
-        }),
-        im_online: Some(ImOnlineConfig { keys: vec![] }),
-        authority_discovery: Some(AuthorityDiscoveryConfig { keys: vec![] }),
-        grandpa: Some(GrandpaConfig {
-            authorities: vec![],
-        }),
-        council: Some(CouncilConfig {
-            active_council: vec![],
-            term_ends_at: 1,
-        }),
-        election: Some(CouncilElectionConfig {
-            auto_start: true,
-            election_parameters: ElectionParameters {
-                announcing_period: 3 * DAYS,
-                voting_period: 1 * DAYS,
-                revealing_period: 1 * DAYS,
-                council_size: 12,
-                candidacy_limit: 25,
-                min_council_stake: 10 * DOLLARS,
-                new_term_duration: 14 * DAYS,
-                min_voting_stake: 1 * DOLLARS,
-            },
-        }),
-        members: Some(MembersConfig {
-            default_paid_membership_fee: 100u128,
-            members: vec![],
-        }),
-        forum: Some(crate::forum_config::from_serialized::create(
-            endowed_accounts[0].clone(),
-        )),
-        data_object_type_registry: Some(DataObjectTypeRegistryConfig {
-            first_data_object_type_id: 1,
-        }),
-        data_object_storage_registry: Some(DataObjectStorageRegistryConfig {
-            first_relationship_id: 1,
-        }),
-        working_group_Instance2: Some(StorageWorkingGroupConfig {
-            phantom: Default::default(),
-            storage_working_group_mint_capacity: 0,
-            opening_human_readable_text_constraint: default_text_constraint,
-            worker_application_human_readable_text_constraint: default_text_constraint,
-            worker_exit_rationale_text_constraint: default_text_constraint,
-        }),
-        versioned_store: Some(VersionedStoreConfig {
-            class_by_id: vec![],
-            entity_by_id: vec![],
-            next_class_id: 1,
-            next_entity_id: 1,
-            property_name_constraint: new_vs_validation(1, 99),
-            property_description_constraint: new_vs_validation(1, 999),
-            class_name_constraint: new_vs_validation(1, 99),
-            class_description_constraint: new_vs_validation(1, 999),
-        }),
-        content_wg: Some(ContentWorkingGroupConfig {
-            mint_capacity: 100_000,
-            curator_opening_by_id: vec![],
-            next_curator_opening_id: 0,
-            curator_application_by_id: vec![],
-            next_curator_application_id: 0,
-            channel_by_id: vec![],
-            next_channel_id: 1,
-            channel_id_by_handle: vec![],
-            curator_by_id: vec![],
-            next_curator_id: 0,
-            principal_by_id: vec![],
-            next_principal_id: 0,
-            channel_creation_enabled: true, // there is no extrinsic to change it so enabling at genesis
-            unstaker_by_stake_id: vec![],
-            channel_handle_constraint: InputValidationLengthConstraint::new(5, 20),
-            channel_description_constraint: InputValidationLengthConstraint::new(1, 1024),
-            opening_human_readable_text: InputValidationLengthConstraint::new(1, 2048),
-            curator_application_human_readable_text: InputValidationLengthConstraint::new(1, 2048),
-            curator_exit_rationale_text: InputValidationLengthConstraint::new(1, 2048),
-            channel_avatar_constraint: InputValidationLengthConstraint::new(5, 1024),
-            channel_banner_constraint: InputValidationLengthConstraint::new(5, 1024),
-            channel_title_constraint: InputValidationLengthConstraint::new(5, 1024),
-        }),
-        migration: Some(MigrationConfig {}),
-        proposals_codex: Some(ProposalsCodexConfig {
-            set_validator_count_proposal_voting_period: cpcp
-                .set_validator_count_proposal_voting_period,
-            set_validator_count_proposal_grace_period: cpcp
-                .set_validator_count_proposal_grace_period,
-            runtime_upgrade_proposal_voting_period: cpcp.runtime_upgrade_proposal_voting_period,
-            runtime_upgrade_proposal_grace_period: cpcp.runtime_upgrade_proposal_grace_period,
-            text_proposal_voting_period: cpcp.text_proposal_voting_period,
-            text_proposal_grace_period: cpcp.text_proposal_grace_period,
-            set_election_parameters_proposal_voting_period: cpcp
-                .set_election_parameters_proposal_voting_period,
-            set_election_parameters_proposal_grace_period: cpcp
-                .set_election_parameters_proposal_grace_period,
-            set_content_working_group_mint_capacity_proposal_voting_period: cpcp
-                .set_content_working_group_mint_capacity_proposal_voting_period,
-            set_content_working_group_mint_capacity_proposal_grace_period: cpcp
-                .set_content_working_group_mint_capacity_proposal_grace_period,
-            set_lead_proposal_voting_period: cpcp.set_lead_proposal_voting_period,
-            set_lead_proposal_grace_period: cpcp.set_lead_proposal_grace_period,
-            spending_proposal_voting_period: cpcp.spending_proposal_voting_period,
-            spending_proposal_grace_period: cpcp.spending_proposal_grace_period,
-            add_working_group_opening_proposal_voting_period: cpcp
-                .add_working_group_opening_proposal_voting_period,
-            add_working_group_opening_proposal_grace_period: cpcp
-                .add_working_group_opening_proposal_grace_period,
-            begin_review_working_group_leader_applications_proposal_voting_period: cpcp
-                .begin_review_working_group_leader_applications_proposal_voting_period,
-            begin_review_working_group_leader_applications_proposal_grace_period: cpcp
-                .begin_review_working_group_leader_applications_proposal_grace_period,
-            fill_working_group_leader_opening_proposal_voting_period: cpcp
-                .fill_working_group_leader_opening_proposal_voting_period,
-            fill_working_group_leader_opening_proposal_grace_period: cpcp
-                .fill_working_group_leader_opening_proposal_grace_period,
-            set_working_group_mint_capacity_proposal_voting_period: cpcp
-                .set_content_working_group_mint_capacity_proposal_voting_period,
-            set_working_group_mint_capacity_proposal_grace_period: cpcp
-                .set_content_working_group_mint_capacity_proposal_grace_period,
-            decrease_working_group_leader_stake_proposal_voting_period: cpcp
-                .decrease_working_group_leader_stake_proposal_voting_period,
-            decrease_working_group_leader_stake_proposal_grace_period: cpcp
-                .decrease_working_group_leader_stake_proposal_grace_period,
-            slash_working_group_leader_stake_proposal_voting_period: cpcp
-                .slash_working_group_leader_stake_proposal_voting_period,
-            slash_working_group_leader_stake_proposal_grace_period: cpcp
-                .slash_working_group_leader_stake_proposal_grace_period,
-            set_working_group_leader_reward_proposal_voting_period: cpcp
-                .set_working_group_leader_reward_proposal_voting_period,
-            set_working_group_leader_reward_proposal_grace_period: cpcp
-                .set_working_group_leader_reward_proposal_grace_period,
-            terminate_working_group_leader_role_proposal_voting_period: cpcp
-                .terminate_working_group_leader_role_proposal_voting_period,
-            terminate_working_group_leader_role_proposal_grace_period: cpcp
-                .terminate_working_group_leader_role_proposal_grace_period,
-        }),
-    }
-}

node/src/chain_spec/content_config.rs

@@ -0,0 +1,391 @@
+use codec::Decode;
+use node_runtime::common::constraints::InputValidationLengthConstraint;
+use node_runtime::{
+    content_wg::{Channel, ChannelId, Principal, PrincipalId},
+    data_directory::DataObject,
+    primitives::{AccountId, BlockNumber, Credential},
+    versioned_store::{Class, ClassId, Entity, EntityId},
+    versioned_store_permissions::ClassPermissions,
+    ContentId, ContentWorkingGroupConfig, DataDirectoryConfig, Runtime, VersionedStoreConfig,
+    VersionedStorePermissionsConfig,
+};
+use serde::Deserialize;
+use std::{fs, path::Path};
+
+// Because of the way that the @joystream/types were implemented the getters for
+// the string types return a `string` not the `Text` type so when we are serializing
+// them to json we get a string rather than an array of bytes, so deserializing them
+// is failing. So we are relying on parity codec encoding instead..
+#[derive(Decode)]
+struct ClassAndPermissions {
+    class: Class,
+    permissions: ClassPermissions<ClassId, Credential, u16, BlockNumber>,
+}
+
+#[derive(Decode)]
+struct EntityAndMaintainer {
+    entity: Entity,
+    maintainer: Option<Credential>,
+}
+
+#[derive(Decode)]
+struct DataObjectAndContentId {
+    content_id: ContentId,
+    data_object: DataObject<Runtime>,
+}
+
+#[derive(Decode)]
+struct ContentData {
+    /// classes and their associted permissions
+    classes: Vec<ClassAndPermissions>,
+    /// entities and their associated maintainer
+    entities: Vec<EntityAndMaintainer>,
+    /// DataObject(s) and ContentId
+    data_objects: Vec<DataObjectAndContentId>,
+    /// Media Channels
+    channels: Vec<ChannelAndId>,
+}
+
+#[derive(Deserialize)]
+struct EncodedClassAndPermissions {
+    /// hex encoded Class
+    class: String,
+    /// hex encoded ClassPermissions<ClassId, Credential, u16, BlockNumber>,
+    permissions: String,
+}
+
+impl EncodedClassAndPermissions {
+    fn decode(&self) -> ClassAndPermissions {
+        // hex string must not include '0x' prefix!
+        let encoded_class =
+            hex::decode(&self.class[2..].as_bytes()).expect("failed to parse class hex string");
+        let encoded_permissions = hex::decode(&self.permissions[2..].as_bytes())
+            .expect("failed to parse class permissions hex string");
+        ClassAndPermissions {
+            class: Decode::decode(&mut encoded_class.as_slice()).unwrap(),
+            permissions: Decode::decode(&mut encoded_permissions.as_slice()).unwrap(),
+        }
+    }
+}
+
+#[derive(Deserialize)]
+struct EncodedEntityAndMaintainer {
+    /// hex encoded Entity
+    entity: String,
+    /// hex encoded Option<Credential>
+    maintainer: Option<String>,
+}
+
+impl EncodedEntityAndMaintainer {
+    fn decode(&self) -> EntityAndMaintainer {
+        // hex string must not include '0x' prefix!
+        let encoded_entity =
+            hex::decode(&self.entity[2..].as_bytes()).expect("failed to parse entity hex string");
+        let encoded_maintainer = self.maintainer.as_ref().map(|maintainer| {
+            hex::decode(&maintainer[2..].as_bytes()).expect("failed to parse maintainer hex string")
+        });
+        EntityAndMaintainer {
+            entity: Decode::decode(&mut encoded_entity.as_slice()).unwrap(),
+            maintainer: encoded_maintainer
+                .map(|maintainer| Decode::decode(&mut maintainer.as_slice()).unwrap()),
+        }
+    }
+}
+
+#[derive(Deserialize)]
+struct EncodedDataObjectAndContentId {
+    /// hex encoded ContentId
+    content_id: String,
+    /// hex encoded DataObject<Runtime>
+    data_object: String,
+}
+
+impl EncodedDataObjectAndContentId {
+    fn decode(&self) -> DataObjectAndContentId {
+        // hex string must not include '0x' prefix!
+        let encoded_content_id = hex::decode(&self.content_id[2..].as_bytes())
+            .expect("failed to parse content_id hex string");
+        let encoded_data_object = hex::decode(&self.data_object[2..].as_bytes())
+            .expect("failed to parse data_object hex string");
+        DataObjectAndContentId {
+            content_id: Decode::decode(&mut encoded_content_id.as_slice()).unwrap(),
+            data_object: Decode::decode(&mut encoded_data_object.as_slice()).unwrap(),
+        }
+    }
+}
+
+#[derive(Decode)]
+struct ChannelAndId {
+    id: ChannelId<Runtime>,
+    channel: Channel<u64, AccountId, BlockNumber, PrincipalId<Runtime>>,
+}
+
+#[derive(Deserialize)]
+struct EncodedChannelAndId {
+    /// ChannelId number
+    id: u64,
+    /// hex encoded Channel
+    channel: String,
+}
+
+impl EncodedChannelAndId {
+    fn decode(&self) -> ChannelAndId {
+        let id = self.id;
+        let encoded_channel =
+            hex::decode(&self.channel[2..].as_bytes()).expect("failed to parse channel hex string");
+        ChannelAndId {
+            id: id as ChannelId<Runtime>,
+            channel: Decode::decode(&mut encoded_channel.as_slice()).unwrap(),
+        }
+    }
+}
+
+#[derive(Deserialize)]
+struct EncodedContentData {
+    /// classes and their associted permissions
+    classes: Vec<EncodedClassAndPermissions>,
+    /// entities and their associated maintainer
+    entities: Vec<EncodedEntityAndMaintainer>,
+    /// DataObject(s) and ContentId
+    data_objects: Vec<EncodedDataObjectAndContentId>,
+    /// Media Channels
+    channels: Vec<EncodedChannelAndId>,
+}
+
+fn parse_content_data(data_file: &Path) -> EncodedContentData {
+    let data = fs::read_to_string(data_file).expect("Failed reading file");
+    serde_json::from_str(&data).expect("failed parsing content data")
+}
+
+impl EncodedContentData {
+    pub fn decode(&self) -> ContentData {
+        ContentData {
+            classes: self
+                .classes
+                .iter()
+                .map(|class_and_perm| class_and_perm.decode())
+                .collect(),
+            entities: self
+                .entities
+                .iter()
+                .map(|entities_and_maintainer| entities_and_maintainer.decode())
+                .collect(),
+            data_objects: self
+                .data_objects
+                .iter()
+                .map(|data_objects| data_objects.decode())
+                .collect(),
+            channels: self
+                .channels
+                .iter()
+                .map(|channel_and_id| channel_and_id.decode())
+                .collect(),
+        }
+    }
+}
+
+/// Generates a VersionedStoreConfig genesis config
+/// with pre-populated classes and entities parsed from a json file serialized
+/// as a ContentData struct.
+pub fn versioned_store_config_from_json(data_file: &Path) -> VersionedStoreConfig {
+    let content = parse_content_data(data_file).decode();
+    let base_config = empty_versioned_store_config();
+    let first_id = 1;
+
+    let next_class_id: ClassId = content
+        .classes
+        .last()
+        .map_or(first_id, |class_and_perm| class_and_perm.class.id + 1);
+    assert_eq!(next_class_id, (content.classes.len() + 1) as ClassId);
+
+    let next_entity_id: EntityId = content
+        .entities
+        .last()
+        .map_or(first_id, |entity_and_maintainer| {
+            entity_and_maintainer.entity.id + 1
+        });
+
+    VersionedStoreConfig {
+        class_by_id: content
+            .classes
+            .into_iter()
+            .map(|class_and_permissions| {
+                (class_and_permissions.class.id, class_and_permissions.class)
+            })
+            .collect(),
+        entity_by_id: content
+            .entities
+            .into_iter()
+            .map(|entity_and_maintainer| {
+                (
+                    entity_and_maintainer.entity.id,
+                    entity_and_maintainer.entity,
+                )
+            })
+            .collect(),
+        next_class_id,
+        next_entity_id,
+        ..base_config
+    }
+}
+
+/// Generates basic empty VersionedStoreConfig genesis config
+pub fn empty_versioned_store_config() -> VersionedStoreConfig {
+    VersionedStoreConfig {
+        class_by_id: vec![],
+        entity_by_id: vec![],
+        next_class_id: 1,
+        next_entity_id: 1,
+        property_name_constraint: InputValidationLengthConstraint::new(1, 99),
+        property_description_constraint: InputValidationLengthConstraint::new(1, 999),
+        class_name_constraint: InputValidationLengthConstraint::new(1, 99),
+        class_description_constraint: InputValidationLengthConstraint::new(1, 999),
+    }
+}
+
+/// Generates a basic empty VersionedStorePermissionsConfig genesis config
+pub fn empty_versioned_store_permissions_config() -> VersionedStorePermissionsConfig {
+    VersionedStorePermissionsConfig {
+        class_permissions_by_class_id: vec![],
+        entity_maintainer_by_entity_id: vec![],
+    }
+}
+
+/// Generates a `VersionedStorePermissionsConfig` genesis config
+/// pre-populated with permissions and entity maintainers parsed from
+/// a json file serialized as a `ContentData` struct.
+pub fn versioned_store_permissions_config_from_json(
+    data_file: &Path,
+) -> VersionedStorePermissionsConfig {
+    let content = parse_content_data(data_file).decode();
+
+    VersionedStorePermissionsConfig {
+        class_permissions_by_class_id: content
+            .classes
+            .into_iter()
+            .map(|class_and_perm| (class_and_perm.class.id, class_and_perm.permissions))
+            .collect(),
+        entity_maintainer_by_entity_id: content
+            .entities
+            .into_iter()
+            .filter_map(|entity_and_maintainer| {
+                entity_and_maintainer
+                    .maintainer
+                    .map(|maintainer| (entity_and_maintainer.entity.id, maintainer))
+            })
+            .collect(),
+    }
+}
+
+/// Generates a basic empty `DataDirectoryConfig` genesis config
+pub fn empty_data_directory_config() -> DataDirectoryConfig {
+    DataDirectoryConfig {
+        data_object_by_content_id: vec![],
+        known_content_ids: vec![],
+    }
+}
+
+/// Generates a `DataDirectoryConfig` genesis config
+/// pre-populated with data objects and known content ids parsed from
+/// a json file serialized as a `ContentData` struct
+pub fn data_directory_config_from_json(data_file: &Path) -> DataDirectoryConfig {
+    let content = parse_content_data(data_file).decode();
+
+    DataDirectoryConfig {
+        data_object_by_content_id: content
+            .data_objects
+            .iter()
+            .map(|object| (object.content_id, object.data_object.clone()))
+            .collect(),
+        known_content_ids: content
+            .data_objects
+            .into_iter()
+            .map(|object| object.content_id)
+            .collect(),
+    }
+}
+
+/// Generates a basic `ContentWorkingGroupConfig` genesis config without any active curators
+/// curator lead or channels.
+pub fn empty_content_working_group_config() -> ContentWorkingGroupConfig {
+    ContentWorkingGroupConfig {
+        mint_capacity: 0,
+        curator_opening_by_id: vec![],
+        next_curator_opening_id: 0,
+        curator_application_by_id: vec![],
+        next_curator_application_id: 0,
+        channel_by_id: vec![],
+        next_channel_id: 1,
+        channel_id_by_handle: vec![],
+        curator_by_id: vec![],
+        next_curator_id: 0,
+        principal_by_id: vec![],
+        next_principal_id: 0,
+        channel_creation_enabled: true, // there is no extrinsic to change it so enabling at genesis
+        unstaker_by_stake_id: vec![],
+        channel_handle_constraint: InputValidationLengthConstraint::new(5, 20),
+        channel_description_constraint: InputValidationLengthConstraint::new(1, 1024),
+        opening_human_readable_text: InputValidationLengthConstraint::new(1, 2048),
+        curator_application_human_readable_text: InputValidationLengthConstraint::new(1, 2048),
+        curator_exit_rationale_text: InputValidationLengthConstraint::new(1, 2048),
+        channel_avatar_constraint: InputValidationLengthConstraint::new(5, 1024),
+        channel_banner_constraint: InputValidationLengthConstraint::new(5, 1024),
+        channel_title_constraint: InputValidationLengthConstraint::new(5, 1024),
+    }
+}
+
+/// Generates a `ContentWorkingGroupConfig` genesis config
+/// pre-populated with channels and corresponding princial channel owners
+/// parsed from a json file serialized as a `ContentData` struct
+pub fn content_working_group_config_from_json(data_file: &Path) -> ContentWorkingGroupConfig {
+    let content = parse_content_data(data_file).decode();
+    let first_channel_id = 1;
+    let first_principal_id = 0;
+
+    let next_channel_id: ChannelId<Runtime> = content
+        .channels
+        .last()
+        .map_or(first_channel_id, |channel_and_id| channel_and_id.id + 1);
+    assert_eq!(
+        next_channel_id,
+        (content.channels.len() + 1) as ChannelId<Runtime>
+    );
+
+    let base_config = empty_content_working_group_config();
+
+    ContentWorkingGroupConfig {
+        channel_by_id: content
+            .channels
+            .iter()
+            .enumerate()
+            .map(|(ix, channel_and_id)| {
+                (
+                    channel_and_id.id,
+                    Channel {
+                        principal_id: first_principal_id + ix as PrincipalId<Runtime>,
+                        ..channel_and_id.channel.clone()
+                    },
+                )
+            })
+            .collect(),
+        next_channel_id,
+        channel_id_by_handle: content
+            .channels
+            .iter()
+            .map(|channel_and_id| (channel_and_id.channel.handle.clone(), channel_and_id.id))
+            .collect(),
+        principal_by_id: content
+            .channels
+            .iter()
+            .enumerate()
+            .map(|(ix, channel_and_id)| {
+                (
+                    first_principal_id + ix as PrincipalId<Runtime>,
+                    Principal::ChannelOwner(channel_and_id.id),
+                )
+            })
+            .collect(),
+        next_principal_id: first_principal_id + content.channels.len() as PrincipalId<Runtime>,
+        ..base_config
+    }
+}

node/src/chain_spec/forum_config.rs

@@ -0,0 +1,149 @@
+use codec::Decode;
+use node_runtime::{
+    common::constraints::InputValidationLengthConstraint,
+    forum::{Category, CategoryId, Post, Thread},
+    AccountId, BlockNumber, ForumConfig, Moment, PostId, ThreadId,
+};
+use serde::Deserialize;
+use std::{fs, path::Path};
+
+fn new_validation(min: u16, max_min_diff: u16) -> InputValidationLengthConstraint {
+    InputValidationLengthConstraint { min, max_min_diff }
+}
+
+#[derive(Decode)]
+struct ForumData {
+    categories: Vec<Category<BlockNumber, Moment, AccountId>>,
+    posts: Vec<Post<BlockNumber, Moment, AccountId, ThreadId, PostId>>,
+    threads: Vec<Thread<BlockNumber, Moment, AccountId, ThreadId>>,
+}
+
+#[derive(Deserialize)]
+struct EncodedForumData {
+    /// hex encoded categories
+    categories: Vec<String>,
+    /// hex encoded posts
+    posts: Vec<String>,
+    /// hex encoded threads
+    threads: Vec<String>,
+}
+
+impl EncodedForumData {
+    fn decode(&self) -> ForumData {
+        ForumData {
+            categories: self
+                .categories
+                .iter()
+                .map(|category| {
+                    let encoded_category = hex::decode(&category[2..].as_bytes())
+                        .expect("failed to parse category hex string");
+                    Decode::decode(&mut encoded_category.as_slice()).unwrap()
+                })
+                .collect(),
+            posts: self
+                .posts
+                .iter()
+                .map(|post| {
+                    let encoded_post = hex::decode(&post[2..].as_bytes())
+                        .expect("failed to parse post hex string");
+                    Decode::decode(&mut encoded_post.as_slice()).unwrap()
+                })
+                .collect(),
+            threads: self
+                .threads
+                .iter()
+                .map(|thread| {
+                    let encoded_thread = hex::decode(&thread[2..].as_bytes())
+                        .expect("failed to parse thread hex string");
+                    Decode::decode(&mut encoded_thread.as_slice()).unwrap()
+                })
+                .collect(),
+        }
+    }
+}
+
+fn parse_forum_json(data_file: &Path) -> EncodedForumData {
+    let data = fs::read_to_string(data_file).expect("Failed reading file");
+    serde_json::from_str(&data).expect("failed parsing members data")
+}
+
+/// Generates a `ForumConfig` geneis config pre-populated with
+/// categories, threads and posts parsed
+/// from a json file serialized as `EncodedForumData`
+pub fn from_json(forum_sudo: AccountId, data_file: &Path) -> ForumConfig {
+    let forum_data = parse_forum_json(data_file);
+    create(forum_sudo, forum_data)
+}
+
+/// Generates a basic empty `ForumConfig` geneis config
+pub fn empty(forum_sudo: AccountId) -> ForumConfig {
+    let forum_data = EncodedForumData {
+        categories: vec![],
+        threads: vec![],
+        posts: vec![],
+    };
+    create(forum_sudo, forum_data)
+}
+
+fn create(forum_sudo: AccountId, forum_data: EncodedForumData) -> ForumConfig {
+    let first_id = 1;
+    let forum_data = forum_data.decode();
+
+    let next_category_id: CategoryId = forum_data
+        .categories
+        .last()
+        .map_or(first_id, |category| category.id + 1);
+
+    assert_eq!(
+        next_category_id,
+        (forum_data.categories.len() + 1) as CategoryId
+    );
+
+    let next_thread_id: ThreadId = forum_data
+        .threads
+        .last()
+        .map_or(first_id, |thread| thread.id + 1);
+
+    assert_eq!(next_thread_id, (forum_data.threads.len() + 1) as ThreadId);
+
+    let next_post_id: PostId = forum_data.posts.last().map_or(first_id, |post| post.id + 1);
+
+    assert_eq!(next_post_id, (forum_data.posts.len() + 1) as PostId);
+
+    ForumConfig {
+        category_by_id: forum_data
+            .categories
+            .into_iter()
+            .map(|encoded_category| {
+                let category = encoded_category;
+                (category.id, category)
+            })
+            .collect(),
+        thread_by_id: forum_data
+            .threads
+            .into_iter()
+            .map(|encoded_thread| {
+                let thread = encoded_thread;
+                (thread.id, thread)
+            })
+            .collect(),
+        post_by_id: forum_data
+            .posts
+            .into_iter()
+            .map(|encoded_post| {
+                let post = encoded_post;
+                (post.id, post)
+            })
+            .collect(),
+        next_category_id,
+        next_thread_id,
+        next_post_id,
+        forum_sudo,
+        category_title_constraint: new_validation(10, 90),
+        category_description_constraint: new_validation(10, 490),
+        thread_title_constraint: new_validation(10, 90),
+        post_text_constraint: new_validation(10, 2990),
+        thread_moderation_rationale_constraint: new_validation(10, 290),
+        post_moderation_rationale_constraint: new_validation(10, 290),
+    }
+}

node/src/chain_spec/initial_balances.rs

@@ -0,0 +1,18 @@
+use node_runtime::{AccountId, Balance};
+use serde::Deserialize;
+use std::{fs, path::Path};
+
+#[derive(Deserialize)]
+struct SerializedInitialBalances {
+    balances: Vec<(AccountId, Balance)>,
+}
+
+fn parse_json(data_file: &Path) -> SerializedInitialBalances {
+    let data = fs::read_to_string(data_file).expect("Failed reading file");
+    serde_json::from_str(&data).expect("failed parsing balances data")
+}
+
+/// Deserializes initial balances from json file
+pub fn from_json(data_file: &Path) -> Vec<(AccountId, Balance)> {
+    parse_json(data_file).balances
+}

node/src/chain_spec/initial_members.rs

@@ -0,0 +1,13 @@
+use node_runtime::{membership, AccountId, Moment};
+use std::{fs, path::Path};
+
+/// Generates a Vec of genesis members parsed from a json file
+pub fn from_json(data_file: &Path) -> Vec<membership::genesis::Member<u64, AccountId, Moment>> {
+    let data = fs::read_to_string(data_file).expect("Failed reading file");
+    serde_json::from_str(&data).expect("failed parsing members data")
+}
+
+/// Generates an empty Vec of genesis members
+pub fn none() -> Vec<membership::genesis::Member<u64, AccountId, Moment>> {
+    vec![]
+}

node/src/chain_spec/mod.rs

@@ -0,0 +1,466 @@
+// Copyright 2019 Joystream Contributors
+// This file is part of Joystream node.
+
+// Joystream node is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Joystream node is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Joystream node.  If not, see <http://www.gnu.org/licenses/>.
+
+// Clippy linter warning.
+// Disable it because we use such syntax for a code readability.
+// Example:  voting_period: 1 * DAY
+#![allow(clippy::identity_op)]
+
+use pallet_im_online::sr25519::AuthorityId as ImOnlineId;
+use serde_json as json;
+use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId;
+use sp_consensus_babe::AuthorityId as BabeId;
+use sp_core::{sr25519, Pair, Public};
+use sp_finality_grandpa::AuthorityId as GrandpaId;
+use sp_runtime::traits::{IdentifyAccount, Verify};
+use sp_runtime::Perbill;
+
+use node_runtime::{
+    membership, AuthorityDiscoveryConfig, BabeConfig, Balance, BalancesConfig,
+    ContentWorkingGroupConfig, CouncilConfig, CouncilElectionConfig, DataDirectoryConfig,
+    DataObjectStorageRegistryConfig, DataObjectTypeRegistryConfig, ElectionParameters, ForumConfig,
+    GrandpaConfig, ImOnlineConfig, MembersConfig, Moment, ProposalsCodexConfig, SessionConfig,
+    SessionKeys, Signature, StakerStatus, StakingConfig, StorageWorkingGroupConfig, SudoConfig,
+    SystemConfig, VersionedStoreConfig, VersionedStorePermissionsConfig, DAYS, WASM_BINARY,
+};
+
+// Exported to be used by chain-spec-builder
+pub use node_runtime::{AccountId, GenesisConfig};
+
+pub mod content_config;
+pub mod forum_config;
+pub mod initial_balances;
+pub mod initial_members;
+pub mod proposals_config;
+
+type AccountPublic = <Signature as Verify>::Signer;
+
+/// Specialized `ChainSpec`. This is a specialization of the general Substrate ChainSpec type.
+pub type ChainSpec = sc_service::GenericChainSpec<GenesisConfig>;
+
+use sc_chain_spec::ChainType;
+
+/// The chain specification option. This is expected to come in from the CLI and
+/// is little more than one of a number of alternatives which can easily be converted
+/// from a string (`--chain=...`) into a `ChainSpec`.
+#[derive(Clone, Debug)]
+pub enum Alternative {
+    /// Whatever the current runtime is, with just Alice as an auth.
+    Development,
+    /// Whatever the current runtime is, with simple Alice/Bob auths.
+    LocalTestnet,
+}
+
+/// Helper function to generate a crypto pair from seed
+pub fn get_from_seed<TPublic: Public>(seed: &str) -> <TPublic::Pair as Pair>::Public {
+    TPublic::Pair::from_string(&format!("//{}", seed), None)
+        .expect("static values are valid; qed")
+        .public()
+}
+
+/// Helper function to generate an account ID from seed
+pub fn get_account_id_from_seed<TPublic: Public>(seed: &str) -> AccountId
+where
+    AccountPublic: From<<TPublic::Pair as Pair>::Public>,
+{
+    AccountPublic::from(get_from_seed::<TPublic>(seed)).into_account()
+}
+
+/// Helper function to generate stash, controller and session key from seed
+pub fn get_authority_keys_from_seed(
+    seed: &str,
+) -> (
+    AccountId,
+    AccountId,
+    GrandpaId,
+    BabeId,
+    ImOnlineId,
+    AuthorityDiscoveryId,
+) {
+    (
+        get_account_id_from_seed::<sr25519::Public>(&format!("{}//stash", seed)),
+        get_account_id_from_seed::<sr25519::Public>(seed),
+        get_from_seed::<GrandpaId>(seed),
+        get_from_seed::<BabeId>(seed),
+        get_from_seed::<ImOnlineId>(seed),
+        get_from_seed::<AuthorityDiscoveryId>(seed),
+    )
+}
+
+fn session_keys(
+    grandpa: GrandpaId,
+    babe: BabeId,
+    im_online: ImOnlineId,
+    authority_discovery: AuthorityDiscoveryId,
+) -> SessionKeys {
+    SessionKeys {
+        grandpa,
+        babe,
+        im_online,
+        authority_discovery,
+    }
+}
+
+impl Alternative {
+    /// Get an actual chain config from one of the alternatives.
+    pub(crate) fn load(self) -> Result<ChainSpec, String> {
+        Ok(match self {
+            Alternative::Development => ChainSpec::from_genesis(
+                "Development",
+                "dev",
+                ChainType::Development,
+                || {
+                    testnet_genesis(
+                        vec![get_authority_keys_from_seed("Alice")],
+                        get_account_id_from_seed::<sr25519::Public>("Alice"),
+                        vec![
+                            get_account_id_from_seed::<sr25519::Public>("Alice"),
+                            get_account_id_from_seed::<sr25519::Public>("Bob"),
+                            get_account_id_from_seed::<sr25519::Public>("Alice//stash"),
+                            get_account_id_from_seed::<sr25519::Public>("Bob//stash"),
+                        ],
+                        proposals_config::development(),
+                        initial_members::none(),
+                        forum_config::empty(get_account_id_from_seed::<sr25519::Public>("Alice")),
+                        content_config::empty_versioned_store_config(),
+                        content_config::empty_versioned_store_permissions_config(),
+                        content_config::empty_data_directory_config(),
+                        content_config::empty_content_working_group_config(),
+                        vec![],
+                    )
+                },
+                Vec::new(),
+                None,
+                None,
+                Some(chain_spec_properties()),
+                None,
+            ),
+            Alternative::LocalTestnet => ChainSpec::from_genesis(
+                "Local Testnet",
+                "local_testnet",
+                ChainType::Local,
+                || {
+                    testnet_genesis(
+                        vec![
+                            get_authority_keys_from_seed("Alice"),
+                            get_authority_keys_from_seed("Bob"),
+                        ],
+                        get_account_id_from_seed::<sr25519::Public>("Alice"),
+                        vec![
+                            get_account_id_from_seed::<sr25519::Public>("Alice"),
+                            get_account_id_from_seed::<sr25519::Public>("Bob"),
+                            get_account_id_from_seed::<sr25519::Public>("Charlie"),
+                            get_account_id_from_seed::<sr25519::Public>("Dave"),
+                            get_account_id_from_seed::<sr25519::Public>("Eve"),
+                            get_account_id_from_seed::<sr25519::Public>("Ferdie"),
+                            get_account_id_from_seed::<sr25519::Public>("Alice//stash"),
+                            get_account_id_from_seed::<sr25519::Public>("Bob//stash"),
+                            get_account_id_from_seed::<sr25519::Public>("Charlie//stash"),
+                            get_account_id_from_seed::<sr25519::Public>("Dave//stash"),
+                            get_account_id_from_seed::<sr25519::Public>("Eve//stash"),
+                            get_account_id_from_seed::<sr25519::Public>("Ferdie//stash"),
+                        ],
+                        proposals_config::development(),
+                        initial_members::none(),
+                        forum_config::empty(get_account_id_from_seed::<sr25519::Public>("Alice")),
+                        content_config::empty_versioned_store_config(),
+                        content_config::empty_versioned_store_permissions_config(),
+                        content_config::empty_data_directory_config(),
+                        content_config::empty_content_working_group_config(),
+                        vec![],
+                    )
+                },
+                Vec::new(),
+                None,
+                None,
+                Some(chain_spec_properties()),
+                None,
+            ),
+        })
+    }
+}
+
+pub fn chain_spec_properties() -> json::map::Map<String, json::Value> {
+    let mut properties: json::map::Map<String, json::Value> = json::map::Map::new();
+    properties.insert(
+        String::from("tokenDecimals"),
+        json::Value::Number(json::Number::from(0)),
+    );
+    properties.insert(
+        String::from("tokenSymbol"),
+        json::Value::String(String::from("JOY")),
+    );
+    properties
+}
+// This method should be refactored after Alexandria to reduce number of arguments
+// as more args will likely be needed
+#[allow(clippy::too_many_arguments)]
+pub fn testnet_genesis(
+    initial_authorities: Vec<(
+        AccountId,
+        AccountId,
+        GrandpaId,
+        BabeId,
+        ImOnlineId,
+        AuthorityDiscoveryId,
+    )>,
+    root_key: AccountId,
+    endowed_accounts: Vec<AccountId>,
+    cpcp: node_runtime::ProposalsConfigParameters,
+    members: Vec<membership::genesis::Member<u64, AccountId, Moment>>,
+    forum_config: ForumConfig,
+    versioned_store_config: VersionedStoreConfig,
+    versioned_store_permissions_config: VersionedStorePermissionsConfig,
+    data_directory_config: DataDirectoryConfig,
+    content_working_group_config: ContentWorkingGroupConfig,
+    initial_balances: Vec<(AccountId, Balance)>,
+) -> GenesisConfig {
+    const STASH: Balance = 5_000;
+    const ENDOWMENT: Balance = 100_000_000;
+
+    let default_text_constraint = node_runtime::working_group::default_text_constraint();
+
+    GenesisConfig {
+        system: Some(SystemConfig {
+            code: WASM_BINARY.to_vec(),
+            changes_trie_config: Default::default(),
+        }),
+        pallet_balances: Some(BalancesConfig {
+            balances: endowed_accounts
+                .iter()
+                .cloned()
+                .map(|k| (k, ENDOWMENT))
+                .chain(initial_authorities.iter().map(|x| (x.0.clone(), STASH)))
+                .chain(
+                    initial_balances
+                        .iter()
+                        .map(|(account, balance)| (account.clone(), *balance)),
+                )
+                .collect(),
+        }),
+        pallet_staking: Some(StakingConfig {
+            validator_count: 20,
+            minimum_validator_count: initial_authorities.len() as u32,
+            stakers: initial_authorities
+                .iter()
+                .map(|x| (x.0.clone(), x.1.clone(), STASH, StakerStatus::Validator))
+                .collect(),
+            invulnerables: initial_authorities.iter().map(|x| x.0.clone()).collect(),
+            slash_reward_fraction: Perbill::from_percent(10),
+            history_depth: 336,
+            ..Default::default()
+        }),
+        pallet_sudo: Some(SudoConfig { key: root_key }),
+        pallet_babe: Some(BabeConfig {
+            authorities: vec![],
+        }),
+        pallet_im_online: Some(ImOnlineConfig { keys: vec![] }),
+        pallet_authority_discovery: Some(AuthorityDiscoveryConfig { keys: vec![] }),
+        pallet_grandpa: Some(GrandpaConfig {
+            authorities: vec![],
+        }),
+        pallet_session: Some(SessionConfig {
+            keys: initial_authorities
+                .iter()
+                .map(|x| {
+                    (
+                        x.0.clone(),
+                        x.0.clone(),
+                        session_keys(x.2.clone(), x.3.clone(), x.4.clone(), x.5.clone()),
+                    )
+                })
+                .collect::<Vec<_>>(),
+        }),
+        council: Some(CouncilConfig {
+            active_council: vec![],
+            term_ends_at: 1,
+        }),
+        election: Some(CouncilElectionConfig {
+            auto_start: true,
+            election_parameters: ElectionParameters {
+                announcing_period: 2 * DAYS,
+                voting_period: 1 * DAYS,
+                revealing_period: 1 * DAYS,
+                council_size: 6,
+                candidacy_limit: 25,
+                min_council_stake: 1_000,
+                new_term_duration: 10 * DAYS,
+                min_voting_stake: 100,
+            },
+        }),
+        membership: Some(MembersConfig {
+            default_paid_membership_fee: 100u128,
+            members,
+        }),
+        forum: Some(forum_config),
+        data_directory: Some(data_directory_config),
+        data_object_type_registry: Some(DataObjectTypeRegistryConfig {
+            first_data_object_type_id: 1,
+        }),
+        data_object_storage_registry: Some(DataObjectStorageRegistryConfig {
+            first_relationship_id: 1,
+        }),
+        working_group_Instance2: Some(StorageWorkingGroupConfig {
+            phantom: Default::default(),
+            storage_working_group_mint_capacity: 0,
+            opening_human_readable_text_constraint: default_text_constraint,
+            worker_application_human_readable_text_constraint: default_text_constraint,
+            worker_exit_rationale_text_constraint: default_text_constraint,
+        }),
+        versioned_store: Some(versioned_store_config),
+        versioned_store_permissions: Some(versioned_store_permissions_config),
+        content_wg: Some(content_working_group_config),
+        proposals_codex: Some(ProposalsCodexConfig {
+            set_validator_count_proposal_voting_period: cpcp
+                .set_validator_count_proposal_voting_period,
+            set_validator_count_proposal_grace_period: cpcp
+                .set_validator_count_proposal_grace_period,
+            runtime_upgrade_proposal_voting_period: cpcp.runtime_upgrade_proposal_voting_period,
+            runtime_upgrade_proposal_grace_period: cpcp.runtime_upgrade_proposal_grace_period,
+            text_proposal_voting_period: cpcp.text_proposal_voting_period,
+            text_proposal_grace_period: cpcp.text_proposal_grace_period,
+            set_election_parameters_proposal_voting_period: cpcp
+                .set_election_parameters_proposal_voting_period,
+            set_election_parameters_proposal_grace_period: cpcp
+                .set_election_parameters_proposal_grace_period,
+            set_content_working_group_mint_capacity_proposal_voting_period: cpcp
+                .set_content_working_group_mint_capacity_proposal_voting_period,
+            set_content_working_group_mint_capacity_proposal_grace_period: cpcp
+                .set_content_working_group_mint_capacity_proposal_grace_period,
+            set_lead_proposal_voting_period: cpcp.set_lead_proposal_voting_period,
+            set_lead_proposal_grace_period: cpcp.set_lead_proposal_grace_period,
+            spending_proposal_voting_period: cpcp.spending_proposal_voting_period,
+            spending_proposal_grace_period: cpcp.spending_proposal_grace_period,
+            add_working_group_opening_proposal_voting_period: cpcp
+                .add_working_group_opening_proposal_voting_period,
+            add_working_group_opening_proposal_grace_period: cpcp
+                .add_working_group_opening_proposal_grace_period,
+            begin_review_working_group_leader_applications_proposal_voting_period: cpcp
+                .begin_review_working_group_leader_applications_proposal_voting_period,
+            begin_review_working_group_leader_applications_proposal_grace_period: cpcp
+                .begin_review_working_group_leader_applications_proposal_grace_period,
+            fill_working_group_leader_opening_proposal_voting_period: cpcp
+                .fill_working_group_leader_opening_proposal_voting_period,
+            fill_working_group_leader_opening_proposal_grace_period: cpcp
+                .fill_working_group_leader_opening_proposal_grace_period,
+            set_working_group_mint_capacity_proposal_voting_period: cpcp
+                .set_content_working_group_mint_capacity_proposal_voting_period,
+            set_working_group_mint_capacity_proposal_grace_period: cpcp
+                .set_content_working_group_mint_capacity_proposal_grace_period,
+            decrease_working_group_leader_stake_proposal_voting_period: cpcp
+                .decrease_working_group_leader_stake_proposal_voting_period,
+            decrease_working_group_leader_stake_proposal_grace_period: cpcp
+                .decrease_working_group_leader_stake_proposal_grace_period,
+            slash_working_group_leader_stake_proposal_voting_period: cpcp
+                .slash_working_group_leader_stake_proposal_voting_period,
+            slash_working_group_leader_stake_proposal_grace_period: cpcp
+                .slash_working_group_leader_stake_proposal_grace_period,
+            set_working_group_leader_reward_proposal_voting_period: cpcp
+                .set_working_group_leader_reward_proposal_voting_period,
+            set_working_group_leader_reward_proposal_grace_period: cpcp
+                .set_working_group_leader_reward_proposal_grace_period,
+            terminate_working_group_leader_role_proposal_voting_period: cpcp
+                .terminate_working_group_leader_role_proposal_voting_period,
+            terminate_working_group_leader_role_proposal_grace_period: cpcp
+                .terminate_working_group_leader_role_proposal_grace_period,
+        }),
+    }
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use super::*;
+    use crate::service::{new_full, new_light};
+    use sc_service_test;
+
+    fn local_testnet_genesis_instant_single() -> GenesisConfig {
+        testnet_genesis(
+            vec![get_authority_keys_from_seed("Alice")],
+            get_account_id_from_seed::<sr25519::Public>("Alice"),
+            vec![get_authority_keys_from_seed("Alice").0],
+            proposals_config::development(),
+            initial_members::none(),
+            forum_config::empty(get_account_id_from_seed::<sr25519::Public>("Alice")),
+            content_config::empty_versioned_store_config(),
+            content_config::empty_versioned_store_permissions_config(),
+            content_config::empty_data_directory_config(),
+            content_config::empty_content_working_group_config(),
+            vec![],
+        )
+    }
+
+    /// Local testnet config (single validator - Alice)
+    pub fn integration_test_config_with_single_authority() -> ChainSpec {
+        ChainSpec::from_genesis(
+            "Integration Test",
+            "test",
+            ChainType::Development,
+            local_testnet_genesis_instant_single,
+            vec![],
+            None,
+            None,
+            None,
+            Default::default(),
+        )
+    }
+
+    fn local_testnet_genesis() -> GenesisConfig {
+        testnet_genesis(
+            vec![
+                get_authority_keys_from_seed("Alice"),
+                get_authority_keys_from_seed("Bob"),
+            ],
+            get_account_id_from_seed::<sr25519::Public>("Alice"),
+            vec![
+                get_authority_keys_from_seed("Alice").0,
+                get_authority_keys_from_seed("Bob").0,
+            ],
+            proposals_config::development(),
+            initial_members::none(),
+            forum_config::empty(get_account_id_from_seed::<sr25519::Public>("Alice")),
+            content_config::empty_versioned_store_config(),
+            content_config::empty_versioned_store_permissions_config(),
+            content_config::empty_data_directory_config(),
+            content_config::empty_content_working_group_config(),
+            vec![],
+        )
+    }
+
+    /// Local testnet config (multivalidator Alice + Bob)
+    pub fn integration_test_config_with_two_authorities() -> ChainSpec {
+        ChainSpec::from_genesis(
+            "Integration Test",
+            "test",
+            ChainType::Development,
+            local_testnet_genesis,
+            vec![],
+            None,
+            None,
+            None,
+            Default::default(),
+        )
+    }
+
+    #[test]
+    #[ignore]
+    fn test_connectivity() {
+        sc_service_test::connectivity(
+            integration_test_config_with_two_authorities(),
+            |config| new_full(config),
+            |config| new_light(config),
+        );
+    }
+}

node/src/chain_spec/proposals_config.rs

@@ -0,0 +1,17 @@
+use node_runtime::ProposalsConfigParameters;
+
+/// Development chain config. 0 grace period for all proposals, ie.
+/// proposals executed immediatly. Short voting period.
+pub fn development() -> ProposalsConfigParameters {
+    ProposalsConfigParameters::with_grace_and_voting_periods(0, 200)
+}
+
+/// Staging chain config. Shorter grace periods and voting periods than default.
+pub fn staging() -> ProposalsConfigParameters {
+    ProposalsConfigParameters::with_grace_and_voting_periods(20, 30)
+}
+
+/// The default configuration as defined in the runtime module
+pub fn production() -> ProposalsConfigParameters {
+    ProposalsConfigParameters::default()
+}

node/src/cli.rs

@@ -1,128 +1,48 @@
-use crate::chain_spec;
-use crate::new_full_start;
-use crate::service;
-use futures::{future, sync::oneshot, Future};
-use log::info;
-use std::cell::RefCell;
-pub use substrate_cli::{error, IntoExit, VersionInfo};
-use substrate_cli::{informant, parse_and_prepare, NoCustom, ParseAndPrepare};
-use substrate_service::{AbstractService, Configuration, Roles as ServiceRoles};
-use tokio::runtime::Runtime;
-
-/// Parse command line arguments into service configuration.
-pub fn run<I, T, E>(args: I, exit: E, version: VersionInfo) -> error::Result<()>
-where
-    I: IntoIterator<Item = T>,
-    T: Into<std::ffi::OsString> + Clone,
-    E: IntoExit,
-{
-    type Config<T> = Configuration<(), T>;
-    match parse_and_prepare::<NoCustom, NoCustom, _>(&version, "joystream-node", args) {
-        ParseAndPrepare::Run(cmd) => cmd.run(
-            load_spec,
-            exit,
-            |exit, _cli_args, _custom_args, config: Config<_>| {
-                info!("{}", version.name);
-                info!("  version {}", config.full_version());
-                info!("  by {}, 2019", version.author);
-                info!("Chain specification: {}", config.chain_spec.name());
-                info!("Node name: {}", config.name);
-                info!("Roles: {:?}", config.roles);
-                let runtime = Runtime::new().map_err(|e| format!("{:?}", e))?;
-                match config.roles {
-                    ServiceRoles::LIGHT => run_until_exit(
-                        runtime,
-                        service::new_light(config).map_err(|e| format!("{:?}", e))?,
-                        exit,
-                    ),
-                    _ => run_until_exit(
-                        runtime,
-                        service::new_full(config).map_err(|e| format!("{:?}", e))?,
-                        exit,
-                    ),
-                }
-                .map_err(|e| format!("{:?}", e))
-            },
-        ),
-        ParseAndPrepare::BuildSpec(cmd) => cmd.run::<NoCustom, _, _, _>(load_spec),
-        ParseAndPrepare::ExportBlocks(cmd) => cmd.run_with_builder(
-            |config: Config<_>| Ok(new_full_start!(config).0),
-            load_spec,
-            exit,
-        ),
-        ParseAndPrepare::ImportBlocks(cmd) => cmd.run_with_builder(
-            |config: Config<_>| Ok(new_full_start!(config).0),
-            load_spec,
-            exit,
-        ),
-        ParseAndPrepare::PurgeChain(cmd) => cmd.run(load_spec),
-        ParseAndPrepare::RevertChain(cmd) => {
-            cmd.run_with_builder(|config: Config<_>| Ok(new_full_start!(config).0), load_spec)
-        }
-        ParseAndPrepare::CustomCommand(_) => Ok(()),
-    }?;
-
-    Ok(())
-}
-
-fn load_spec(id: &str) -> Result<Option<chain_spec::ChainSpec>, String> {
-    Ok(match chain_spec::Alternative::from(id) {
-        Some(spec) => Some(spec.load()?),
-        None => None,
-    })
-}
-
-fn run_until_exit<T, E>(mut runtime: Runtime, service: T, e: E) -> error::Result<()>
-where
-    T: AbstractService,
-    E: IntoExit,
-{
-    let (exit_send, exit) = exit_future::signal();
-
-    let informant = informant::build(&service);
-    runtime.executor().spawn(exit.until(informant).map(|_| ()));
-
-    // we eagerly drop the service so that the internal exit future is fired,
-    // but we need to keep holding a reference to the global telemetry guard
-    let _telemetry = service.telemetry();
-
-    let service_res = {
-        let exit = e
-            .into_exit()
-            .map_err(|_| error::Error::Other("Exit future failed.".into()));
-        let service = service.map_err(error::Error::Service);
-        let select = service.select(exit).map(|_| ()).map_err(|(err, _)| err);
-        runtime.block_on(select)
-    };
-
-    exit_send.fire();
-
-    // TODO [andre]: timeout this future #1318
-    let _ = runtime.shutdown_on_idle().wait();
-
-    service_res
+// Copyright 2019 Joystream Contributors
+// This file is part of Joystream node.
+
+// Joystream node is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Joystream node is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Joystream node.  If not, see <http://www.gnu.org/licenses/>.
+
+use sc_cli::RunCmd;
+use structopt::StructOpt;
+
+/// An overarching CLI command definition.
+#[derive(Debug, StructOpt)]
+pub struct Cli {
+    /// Possible subcommand with parameters.
+    #[structopt(subcommand)]
+    pub subcommand: Option<Subcommand>,
+    #[allow(missing_docs)]
+    #[structopt(flatten)]
+    pub run: RunCmd,
 }
 
-// handles ctrl-c
-pub struct Exit;
-impl IntoExit for Exit {
-    type Exit = future::MapErr<oneshot::Receiver<()>, fn(oneshot::Canceled) -> ()>;
-    fn into_exit(self) -> Self::Exit {
-        // can't use signal directly here because CtrlC takes only `Fn`.
-        let (exit_send, exit) = oneshot::channel();
-
-        let exit_send_cell = RefCell::new(Some(exit_send));
-        ctrlc::set_handler(move || {
-            let exit_send = exit_send_cell
-                .try_borrow_mut()
-                .expect("signal handler not reentrant; qed")
-                .take();
-            if let Some(exit_send) = exit_send {
-                exit_send.send(()).expect("Error sending exit notification");
-            }
-        })
-        .expect("Error setting Ctrl-C handler");
-
-        exit.map_err(drop)
-    }
+/// Possible subcommands of the main binary.
+#[derive(Debug, StructOpt)]
+pub enum Subcommand {
+    /// A set of base subcommands handled by `sc_cli`.
+    #[structopt(flatten)]
+    Base(sc_cli::Subcommand),
+
+    /// The custom inspect subcommmand for decoding blocks and extrinsics.
+    #[structopt(
+        name = "inspect",
+        about = "Decode given block or extrinsic using current native runtime."
+    )]
+    Inspect(node_inspect::cli::InspectCmd),
+
+    /// The custom benchmark subcommmand benchmarking runtime pallets.
+    #[structopt(name = "benchmark", about = "Benchmark runtime pallets.")]
+    Benchmark(frame_benchmarking_cli::BenchmarkCmd),
 }

node/src/command.rs

@@ -0,0 +1,100 @@
+// Copyright 2019 Joystream Contributors
+// This file is part of Joystream node.
+
+// Joystream node is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Joystream node is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Joystream node.  If not, see <http://www.gnu.org/licenses/>.
+
+use crate::cli::{Cli, Subcommand};
+use crate::node_executor;
+use crate::node_rpc;
+use crate::{chain_spec, service};
+
+use node_executor::Executor;
+use node_runtime::{opaque::Block, RuntimeApi};
+use sc_cli::{Result, SubstrateCli};
+use sc_finality_grandpa::{self as grandpa};
+
+impl SubstrateCli for Cli {
+    fn impl_name() -> &'static str {
+        "Joystream Node"
+    }
+
+    fn support_url() -> &'static str {
+        "https://www.joystream.org/"
+    }
+
+    fn copyright_start_year() -> i32 {
+        2019
+    }
+
+    fn executable_name() -> &'static str {
+        "joystream-node"
+    }
+
+    fn impl_version() -> &'static str {
+        env!("SUBSTRATE_CLI_IMPL_VERSION")
+    }
+
+    fn description() -> &'static str {
+        env!("CARGO_PKG_DESCRIPTION")
+    }
+
+    fn author() -> &'static str {
+        env!("CARGO_PKG_AUTHORS")
+    }
+
+    fn load_spec(&self, id: &str) -> std::result::Result<Box<dyn sc_service::ChainSpec>, String> {
+        Ok(match id {
+            "dev" => Box::new(chain_spec::Alternative::Development.load()?),
+            "local" => Box::new(chain_spec::Alternative::LocalTestnet.load()?),
+            path => Box::new(chain_spec::ChainSpec::from_json_file(
+                std::path::PathBuf::from(path),
+            )?),
+        })
+    }
+}
+
+/// Parse command line arguments into service configuration.
+pub fn run() -> Result<()> {
+    let cli = Cli::from_args();
+
+    match &cli.subcommand {
+        None => {
+            let runner = cli.create_runner(&cli.run)?;
+            runner.run_node(service::new_light, service::new_full, node_runtime::VERSION)
+        }
+        Some(Subcommand::Inspect(cmd)) => {
+            let runner = cli.create_runner(cmd)?;
+
+            runner.sync_run(|config| cmd.run::<Block, RuntimeApi, Executor>(config))
+        }
+        Some(Subcommand::Benchmark(cmd)) => {
+            if cfg!(feature = "runtime-benchmarks") {
+                let runner = cli.create_runner(cmd)?;
+
+                runner.sync_run(|config| cmd.run::<Block, Executor>(config))
+            } else {
+                println!(
+                    "Benchmarking wasn't enabled when building the node. \
+				You can enable it with `--features runtime-benchmarks`."
+                );
+                Ok(())
+            }
+        }
+        Some(Subcommand::Base(subcommand)) => {
+            let runner = cli.create_runner(subcommand)?;
+
+            runner.run_subcommand(subcommand, |config| Ok(new_full_start!(config).0))
+        }
+    }
+}

node/src/forum_config/from_encoded.rs

@@ -1,90 +0,0 @@
-// This module is not used but included as sample code
-// and highlights some pitfalls.
-
-use node_runtime::{
-    forum::{
-        Category, CategoryId, Post, PostId, Thread, ThreadId,
-    },
-    AccountId, BlockNumber, ForumConfig, Moment,
-};
-use serde::Deserialize;
-use serde_json::Result;
-use super::new_validation;
-
-use codec::Decode;
-
-#[derive(Deserialize)]
-struct ForumData {
-    /// hex encoded categories
-    categories: Vec<(CategoryId, String)>,
-    /// hex encoded posts
-    posts: Vec<(PostId, String)>,
-    /// hex encoded threads
-    threads: Vec<(ThreadId, String)>,
-}
-
-fn decode_post(encoded: String) -> Post<BlockNumber, Moment, AccountId> {
-    // hex string must not include '0x' prefix!
-    let encoded = hex::decode(encoded.as_bytes()).expect("failed to parse hex string");
-    Decode::decode(&mut encoded.as_slice()).unwrap()
-}
-
-fn decode_category(encoded: String) -> Category<BlockNumber, Moment, AccountId> {
-    // hex string must not include '0x' prefix!
-    let encoded = hex::decode(encoded.as_bytes()).expect("failed to parse hex string");
-    Decode::decode(&mut encoded.as_slice()).unwrap()
-}
-
-fn decode_thread(encoded: String) -> Thread<BlockNumber, Moment, AccountId> {
-    // hex string must not include '0x' prefix!
-    let encoded = hex::decode(encoded.as_bytes()).expect("failed to parse hex string");
-    Decode::decode(&mut encoded.as_slice()).unwrap()
-}
-
-fn parse_forum_json() -> Result<ForumData> {
-    let data = include_str!("../../res/forum_data_acropolis_encoded.json");
-    serde_json::from_str(data)
-}
-
-pub fn create(forum_sudo: AccountId) -> ForumConfig {
-    let forum_data = parse_forum_json().expect("failed loading forum data");
-
-    let next_category_id: CategoryId = forum_data
-        .categories
-        .last()
-        .map_or(1, |category| category.0 + 1);
-    let next_thread_id: ThreadId = forum_data.threads.last().map_or(1, |thread| thread.0 + 1);
-    let next_post_id: PostId = forum_data.posts.last().map_or(1, |post| post.0 + 1);
-
-    ForumConfig {
-        // Decoding will fail because of differnt type used for
-        // BlockNumber between Acropolis (u64) and Rome (u32)
-        // As long as types between chains are identical this approach works nicely
-        // since we don't need to use an intermediate format or do any transformation on source data.
-        category_by_id: forum_data
-            .categories
-            .into_iter()
-            .map(|category| (category.0, decode_category(category.1)))
-            .collect(),
-        thread_by_id: forum_data
-            .threads
-            .into_iter()
-            .map(|thread| (thread.0, decode_thread(thread.1)))
-            .collect(),
-        post_by_id: forum_data
-            .posts
-            .into_iter()
-            .map(|post| (post.0, decode_post(post.1)))
-            .collect(),
-        next_category_id,
-        next_thread_id,
-        next_post_id,
-        forum_sudo,
-        category_title_constraint: new_validation(10, 90),
-        category_description_constraint: new_validation(10, 490),
-        thread_title_constraint: new_validation(10, 90),
-        post_text_constraint: new_validation(10, 990),
-        thread_moderation_rationale_constraint: new_validation(10, 290),
-        post_moderation_rationale_constraint: new_validation(10, 290),
-    }
-}

node/src/forum_config/from_serialized.rs

@@ -1,51 +0,0 @@
-#![allow(clippy::type_complexity)]
-
-use super::new_validation;
-use node_runtime::{
-    forum::{Category, CategoryId, Post, Thread},
-    AccountId, BlockNumber, ForumConfig, Moment, PostId, ThreadId,
-};
-use serde::Deserialize;
-use serde_json::Result;
-
-#[derive(Deserialize)]
-struct ForumData {
-    categories: Vec<(CategoryId, Category<BlockNumber, Moment, AccountId>)>,
-    posts: Vec<(
-        PostId,
-        Post<BlockNumber, Moment, AccountId, ThreadId, PostId>,
-    )>,
-    threads: Vec<(ThreadId, Thread<BlockNumber, Moment, AccountId, ThreadId>)>,
-}
-
-fn parse_forum_json() -> Result<ForumData> {
-    let data = include_str!("../../res/forum_data_empty.json");
-    serde_json::from_str(data)
-}
-
-pub fn create(forum_sudo: AccountId) -> ForumConfig {
-    let forum_data = parse_forum_json().expect("failed loading forum data");
-
-    let next_category_id: CategoryId = forum_data
-        .categories
-        .last()
-        .map_or(1, |category| category.0 + 1);
-    let next_thread_id: ThreadId = forum_data.threads.last().map_or(1, |thread| thread.0 + 1);
-    let next_post_id: PostId = forum_data.posts.last().map_or(1, |post| post.0 + 1);
-
-    ForumConfig {
-        category_by_id: forum_data.categories,
-        thread_by_id: forum_data.threads,
-        post_by_id: forum_data.posts,
-        next_category_id,
-        next_thread_id,
-        next_post_id,
-        category_title_constraint: new_validation(10, 90),
-        category_description_constraint: new_validation(10, 490),
-        thread_title_constraint: new_validation(10, 90),
-        post_text_constraint: new_validation(10, 990),
-        thread_moderation_rationale_constraint: new_validation(10, 290),
-        post_moderation_rationale_constraint: new_validation(10, 290),
-        forum_sudo,
-    }
-}

node/src/forum_config/mod.rs

@@ -1,10 +0,0 @@
-pub mod from_serialized;
-
-// Not exported - only here as sample code
-// mod from_encoded;
-
-use node_runtime::common::constraints::InputValidationLengthConstraint;
-
-pub fn new_validation(min: u16, max_min_diff: u16) -> InputValidationLengthConstraint {
-    InputValidationLengthConstraint { min, max_min_diff }
-}

node/src/lib.rs

@@ -1,6 +1,7 @@
 pub mod chain_spec;
 pub mod cli;
-pub mod forum_config;
-pub mod members_config;
-pub mod proposals_config;
+#[macro_use]
 pub mod service;
+pub mod command;
+pub mod node_executor;
+pub mod node_rpc;

node/src/members_config.rs

@@ -1,50 +0,0 @@
-use serde::Deserialize;
-use serde_json::Result;
-
-use primitives::crypto::{AccountId32, Ss58Codec};
-
-#[derive(Deserialize)]
-struct Member {
-    /// SS58 Encoded public key
-    address: String,
-    handle: String,
-    avatar_uri: String,
-    about: String,
-}
-
-// fn test_load_members() -> Result<Vec<Member>> {
-//     let data = r#"
-//         [{
-//             "address": "5Gn9n7SDJ7VgHqHQWYzkSA4vX6DCmS5TFWdHxikTXp9b4L32",
-//             "handle": "mokhtar",
-//             "avatar_uri": "http://mokhtar.net/avatar.png",
-//             "about": "Mokhtar"
-//         }]"#;
-
-//     serde_json::from_str(data)
-// }
-
-fn parse_members_json() -> Result<Vec<Member>> {
-    let data = include_str!("../res/acropolis_members.json");
-    serde_json::from_str(data)
-}
-
-pub fn decode_address(address: String) -> AccountId32 {
-    AccountId32::from_ss58check(address.as_ref()).expect("failed to decode account id")
-}
-
-pub fn initial_members() -> Vec<(AccountId32, String, String, String)> {
-    let members = parse_members_json().expect("failed parsing members data");
-
-    members
-        .into_iter()
-        .map(|member| {
-            (
-                decode_address(member.address),
-                member.handle,
-                member.avatar_uri,
-                member.about,
-            )
-        })
-        .collect()
-}

node/src/node_executor.rs

@@ -0,0 +1,10 @@
+use sc_executor::native_executor_instance;
+
+// Declare an instance of the native executor named `Executor`. Include the wasm binary as the
+// equivalent wasm code.
+native_executor_instance!(
+    pub Executor,
+    node_runtime::api::dispatch,
+    node_runtime::native_version,
+    frame_benchmarking::benchmarking::HostFunctions,
+);

node/src/node_rpc.rs

@@ -0,0 +1,188 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! A collection of node-specific RPC methods.
+//!
+//! Since `substrate` core functionality makes no assumptions
+//! about the modules used inside the runtime, so do
+//! RPC methods defined in `sc-rpc` crate.
+//! It means that `client/rpc` can't have any methods that
+//! need some strong assumptions about the particular runtime.
+//!
+//! The RPCs available in this crate however can make some assumptions
+//! about how the runtime is constructed and what FRAME pallets
+//! are part of it. Therefore all node-runtime-specific RPCs can
+//! be placed here or imported from corresponding FRAME RPC definitions.
+
+#![warn(missing_docs)]
+
+use std::sync::Arc;
+
+use node_runtime::UncheckedExtrinsic;
+use node_runtime::{opaque::Block, AccountId, Balance, BlockNumber, Hash, Index};
+use sc_consensus_babe::{Config, Epoch};
+use sc_consensus_babe_rpc::BabeRpcHandler;
+use sc_consensus_epochs::SharedEpochChanges;
+use sc_finality_grandpa::{SharedAuthoritySet, SharedVoterState};
+use sc_finality_grandpa_rpc::GrandpaRpcHandler;
+use sc_keystore::KeyStorePtr;
+use sc_rpc_api::DenyUnsafe;
+use sp_api::ProvideRuntimeApi;
+use sp_block_builder::BlockBuilder;
+use sp_blockchain::{Error as BlockChainError, HeaderBackend, HeaderMetadata};
+use sp_consensus::SelectChain;
+use sp_consensus_babe::BabeApi;
+use sp_transaction_pool::TransactionPool;
+
+/// Light client extra dependencies.
+pub struct LightDeps<C, F, P> {
+    /// The client instance to use.
+    pub client: Arc<C>,
+    /// Transaction pool instance.
+    pub pool: Arc<P>,
+    /// Remote access to the blockchain (async).
+    pub remote_blockchain: Arc<dyn sc_client_api::light::RemoteBlockchain<Block>>,
+    /// Fetcher instance.
+    pub fetcher: Arc<F>,
+}
+
+/// Extra dependencies for BABE.
+pub struct BabeDeps {
+    /// BABE protocol config.
+    pub babe_config: Config,
+    /// BABE pending epoch changes.
+    pub shared_epoch_changes: SharedEpochChanges<Block, Epoch>,
+    /// The keystore that manages the keys of the node.
+    pub keystore: KeyStorePtr,
+}
+
+/// Extra dependencies for GRANDPA
+pub struct GrandpaDeps {
+    /// Voting round info.
+    pub shared_voter_state: SharedVoterState,
+    /// Authority set info.
+    pub shared_authority_set: SharedAuthoritySet<Hash, BlockNumber>,
+}
+
+/// Full client dependencies.
+pub struct FullDeps<C, P, SC> {
+    /// The client instance to use.
+    pub client: Arc<C>,
+    /// Transaction pool instance.
+    pub pool: Arc<P>,
+    /// The SelectChain Strategy
+    pub select_chain: SC,
+    /// Whether to deny unsafe calls
+    pub deny_unsafe: DenyUnsafe,
+    /// BABE specific dependencies.
+    pub babe: BabeDeps,
+    /// GRANDPA specific dependencies.
+    pub grandpa: GrandpaDeps,
+}
+
+/// Instantiate all Full RPC extensions.
+pub fn create_full<C, P, M, SC>(deps: FullDeps<C, P, SC>) -> jsonrpc_core::IoHandler<M>
+where
+    C: ProvideRuntimeApi<Block>,
+    C: HeaderBackend<Block> + HeaderMetadata<Block, Error = BlockChainError> + 'static,
+    C: Send + Sync + 'static,
+    C::Api: substrate_frame_rpc_system::AccountNonceApi<Block, AccountId, Index>,
+    C::Api: pallet_transaction_payment_rpc::TransactionPaymentRuntimeApi<
+        Block,
+        Balance,
+        UncheckedExtrinsic,
+    >,
+    C::Api: BabeApi<Block>,
+    C::Api: BlockBuilder<Block>,
+    P: TransactionPool + 'static,
+    M: jsonrpc_core::Metadata + Default,
+    SC: SelectChain<Block> + 'static,
+{
+    use pallet_transaction_payment_rpc::{TransactionPayment, TransactionPaymentApi};
+    use substrate_frame_rpc_system::{FullSystem, SystemApi};
+
+    let mut io = jsonrpc_core::IoHandler::default();
+    let FullDeps {
+        client,
+        pool,
+        select_chain,
+        deny_unsafe,
+        babe,
+        grandpa,
+    } = deps;
+    let BabeDeps {
+        keystore,
+        babe_config,
+        shared_epoch_changes,
+    } = babe;
+    let GrandpaDeps {
+        shared_voter_state,
+        shared_authority_set,
+    } = grandpa;
+
+    io.extend_with(SystemApi::to_delegate(FullSystem::new(
+        client.clone(),
+        pool,
+        deny_unsafe,
+    )));
+    // Making synchronous calls in light client freezes the browser currently,
+    // more context: https://github.com/paritytech/substrate/pull/3480
+    // These RPCs should use an asynchronous caller instead.
+    io.extend_with(TransactionPaymentApi::to_delegate(TransactionPayment::new(
+        client.clone(),
+    )));
+    io.extend_with(sc_consensus_babe_rpc::BabeApi::to_delegate(
+        BabeRpcHandler::new(
+            client,
+            shared_epoch_changes,
+            keystore,
+            babe_config,
+            select_chain,
+            deny_unsafe,
+        ),
+    ));
+    io.extend_with(sc_finality_grandpa_rpc::GrandpaApi::to_delegate(
+        GrandpaRpcHandler::new(shared_authority_set, shared_voter_state),
+    ));
+
+    io
+}
+
+/// Instantiate all Light RPC extensions.
+pub fn create_light<C, P, M, F>(deps: LightDeps<C, F, P>) -> jsonrpc_core::IoHandler<M>
+where
+    C: sp_blockchain::HeaderBackend<Block>,
+    C: Send + Sync + 'static,
+    F: sc_client_api::light::Fetcher<Block> + 'static,
+    P: TransactionPool + 'static,
+    M: jsonrpc_core::Metadata + Default,
+{
+    use substrate_frame_rpc_system::{LightSystem, SystemApi};
+
+    let LightDeps {
+        client,
+        pool,
+        remote_blockchain,
+        fetcher,
+    } = deps;
+    let mut io = jsonrpc_core::IoHandler::default();
+    io.extend_with(SystemApi::<Hash, AccountId, Index>::to_delegate(
+        LightSystem::new(client, remote_blockchain, fetcher, pool),
+    ));
+
+    io
+}

node/src/proposals_config.rs

@@ -1,17 +0,0 @@
-use node_runtime::ProposalsConfigParameters;
-
-/// Development chain config. 0 grace period for all proposals, ie.
-/// proposals executed immediatly. Short voting period.
-pub fn development() -> ProposalsConfigParameters {
-    ProposalsConfigParameters::with_grace_and_voting_periods(0, 200)
-}
-
-/// Staging chain config. Shorter grace periods and voting periods than default.
-pub fn staging() -> ProposalsConfigParameters {
-    ProposalsConfigParameters::with_grace_and_voting_periods(200, 600)
-}
-
-/// The default configuration as defined in the runtime module
-pub fn default() -> ProposalsConfigParameters {
-    ProposalsConfigParameters::default()
-}

node/src/service.rs

@@ -16,107 +16,141 @@
 
 #![warn(unused_extern_crates)]
 
-// Clippy linter warning.
-#![allow(clippy::type_complexity)] // disable it because this is foreign code and can be changed any time
-
-// Clippy linter warning.
-#![allow(clippy::redundant_closure_call)] // disable it because of the substrate lib design
-
-//! Service and ServiceFactory implementation. Specialized wrapper over substrate service.
-
-use client_db::Backend;
-use grandpa::{self, FinalityProofProvider as GrandpaFinalityProofProvider};
-use inherents::InherentDataProviders;
-use network::{construct_simple_protocol, NetworkService};
-use node_runtime::{self, opaque::Block, GenesisConfig, RuntimeApi};
-use offchain::OffchainWorkers;
-use primitives::Blake2Hasher;
-use runtime_primitives::traits::Block as BlockT;
-use std::sync::Arc;
-use substrate_client::{Client, LocalCallExecutor, LongestChain};
-pub use substrate_executor::{native_executor_instance, NativeExecutor};
-use substrate_service::{
-    error::Error as ServiceError, AbstractService, Configuration, NetworkStatus, Service,
-    ServiceBuilder,
-};
-use transaction_pool::{self, txpool::Pool as TransactionPool};
+// Substrate implementation issue.
+#![allow(clippy::redundant_closure_call)]
 
-construct_simple_protocol! {
-    /// Demo protocol attachment for substrate.
-    pub struct NodeProtocol where Block = Block { }
-}
+//! Service implementation. Specialized wrapper over substrate service.
+
+use node_runtime::opaque::Block;
+use node_runtime::RuntimeApi;
+use sc_consensus::LongestChain;
+use sc_finality_grandpa::{
+    self as grandpa, FinalityProofProvider as GrandpaFinalityProofProvider, StorageAndProofProvider,
+};
+use sc_service::{
+    config::Configuration, error::Error as ServiceError, AbstractService, ServiceBuilder,
+};
+use sp_inherents::InherentDataProviders;
+use std::sync::Arc;
 
-// Declare an instance of the native executor named `Executor`. Include the wasm binary as the
-// equivalent wasm code.
-native_executor_instance!(
-    pub Executor,
-    node_runtime::api::dispatch,
-    node_runtime::native_version
-);
+use crate::node_executor;
+use crate::node_rpc;
 
 /// Starts a `ServiceBuilder` for a full service.
 ///
 /// Use this macro if you don't actually need the full service, but just the builder in order to
 /// be able to perform chain operations.
-#[macro_export]
 macro_rules! new_full_start {
     ($config:expr) => {{
-        // type RpcExtension = jsonrpc_core::IoHandler<substrate_rpc::Metadata>;
+        use std::sync::Arc;
+
         let mut import_setup = None;
-        let inherent_data_providers = inherents::InherentDataProviders::new();
+        let mut rpc_setup = None;
+        let inherent_data_providers = sp_inherents::InherentDataProviders::new();
 
-        let builder = substrate_service::ServiceBuilder::new_full::<
-            node_runtime::opaque::Block,
-            node_runtime::RuntimeApi,
-            crate::service::Executor,
+        let builder = sc_service::ServiceBuilder::new_full::<
+            Block,
+            RuntimeApi,
+            node_executor::Executor,
         >($config)?
-        .with_select_chain(|_config, backend| {
-            Ok(substrate_client::LongestChain::new(backend.clone()))
-        })?
-        .with_transaction_pool(|config, client| {
-            Ok(transaction_pool::txpool::Pool::new(
-                config,
-                transaction_pool::FullChainApi::new(client),
+        .with_select_chain(|_config, backend| Ok(sc_consensus::LongestChain::new(backend.clone())))?
+        .with_transaction_pool(|builder| {
+            let pool_api = sc_transaction_pool::FullChainApi::new(builder.client().clone());
+            let config = builder.config();
+
+            Ok(sc_transaction_pool::BasicPool::new(
+                config.transaction_pool.clone(),
+                std::sync::Arc::new(pool_api),
+                builder.prometheus_registry(),
             ))
         })?
-        .with_import_queue(|_config, client, mut select_chain, _transaction_pool| {
-            let select_chain = select_chain
-                .take()
-                .ok_or_else(|| substrate_service::Error::SelectChainRequired)?;
-            let (grandpa_block_import, grandpa_link) =
-                grandpa::block_import::<_, _, _, node_runtime::RuntimeApi, _>(
+        .with_import_queue(
+            |_config,
+             client,
+             mut select_chain,
+             _transaction_pool,
+             spawn_task_handle,
+             prometheus_registry| {
+                let select_chain = select_chain
+                    .take()
+                    .ok_or_else(|| sc_service::Error::SelectChainRequired)?;
+                let (grandpa_block_import, grandpa_link) = grandpa::block_import(
                     client.clone(),
-                    &*client,
+                    &(client.clone() as Arc<_>),
                     select_chain,
                 )?;
-            let justification_import = grandpa_block_import.clone();
-
-            let (block_import, babe_link) = babe::block_import(
-                babe::Config::get_or_compute(&*client)?,
-                grandpa_block_import,
-                client.clone(),
-                client.clone(),
-            )?;
-
-            let import_queue = babe::import_queue(
-                babe_link.clone(),
-                block_import.clone(),
-                Some(Box::new(justification_import)),
-                None,
-                client.clone(),
-                client,
-                inherent_data_providers.clone(),
-            )?;
-
-            import_setup = Some((block_import, grandpa_link, babe_link));
-            Ok(import_queue)
+                let justification_import = grandpa_block_import.clone();
+
+                let (block_import, babe_link) = sc_consensus_babe::block_import(
+                    sc_consensus_babe::Config::get_or_compute(&*client)?,
+                    grandpa_block_import,
+                    client.clone(),
+                )?;
+
+                let import_queue = sc_consensus_babe::import_queue(
+                    babe_link.clone(),
+                    block_import.clone(),
+                    Some(Box::new(justification_import)),
+                    None,
+                    client,
+                    inherent_data_providers.clone(),
+                    spawn_task_handle,
+                    prometheus_registry,
+                )?;
+
+                import_setup = Some((block_import, grandpa_link, babe_link));
+                Ok(import_queue)
+            },
+        )?
+        .with_rpc_extensions_builder(|builder| {
+            let grandpa_link = import_setup
+                .as_ref()
+                .map(|s| &s.1)
+                .expect("GRANDPA LinkHalf is present for full services or set up failed; qed.");
+
+            let shared_authority_set = grandpa_link.shared_authority_set().clone();
+            let shared_voter_state = grandpa::SharedVoterState::empty();
+
+            rpc_setup = Some((shared_voter_state.clone()));
+
+            let babe_link = import_setup
+                .as_ref()
+                .map(|s| &s.2)
+                .expect("BabeLink is present for full services or set up failed; qed.");
+
+            let babe_config = babe_link.config().clone();
+            let shared_epoch_changes = babe_link.epoch_changes().clone();
+
+            let client = builder.client().clone();
+            let pool = builder.pool().clone();
+            let select_chain = builder
+                .select_chain()
+                .cloned()
+                .expect("SelectChain is present for full services or set up failed; qed.");
+            let keystore = builder.keystore().clone();
+
+            Ok(move |deny_unsafe| {
+                let deps = node_rpc::FullDeps {
+                    client: client.clone(),
+                    pool: pool.clone(),
+                    select_chain: select_chain.clone(),
+                    deny_unsafe,
+                    babe: node_rpc::BabeDeps {
+                        babe_config: babe_config.clone(),
+                        shared_epoch_changes: shared_epoch_changes.clone(),
+                        keystore: keystore.clone(),
+                    },
+                    grandpa: node_rpc::GrandpaDeps {
+                        shared_voter_state: shared_voter_state.clone(),
+                        shared_authority_set: shared_authority_set.clone(),
+                    },
+                };
+
+                node_rpc::create_full(deps)
+            })
         })?;
-        // We don't have any custom rpc commands...
-        // .with_rpc_extensions(|client, pool| -> RpcExtension {
-        // 	node_rpc::create(client, pool)
-        // })?;
 
-        (builder, import_setup, inherent_data_providers)
+        (builder, import_setup, inherent_data_providers, rpc_setup)
     }};
 }
 
@@ -126,58 +160,57 @@ macro_rules! new_full_start {
 /// concrete types instead.
 macro_rules! new_full {
 	($config:expr, $with_startup_data: expr) => {{
-		use futures::sync::mpsc;
-		use network::DhtEvent;
+		use futures::prelude::*;
+		use sc_network::Event;
+		use sc_client_api::ExecutorProvider;
+		use sp_core::traits::BareCryptoStorePtr;
 
 		let (
-			is_authority,
+			role,
 			force_authoring,
 			name,
-			disable_grandpa
+			disable_grandpa,
 		) = (
-			$config.roles.is_authority(),
+			$config.role.clone(),
 			$config.force_authoring,
-			$config.name.clone(),
-			$config.disable_grandpa
+			$config.network.node_name.clone(),
+			$config.disable_grandpa,
 		);
 
-        // sentry nodes announce themselves as authorities to the network
-		// and should run the same protocols authorities do, but it should
-		// never actively participate in any consensus process.
-        let participates_in_consensus = is_authority && !$config.sentry_mode;
+		let (builder, mut import_setup, inherent_data_providers, mut rpc_setup) =
+			new_full_start!($config);
 
-		let (builder, mut import_setup, inherent_data_providers) = new_full_start!($config);
-
-		// Dht event channel from the network to the authority discovery module. Use bounded channel to ensure
-		// back-pressure. Authority discovery is triggering one event per authority within the current authority set.
-		// This estimates the authority set size to be somewhere below 10 000 thereby setting the channel buffer size to
-		// 10 000.
-		let (dht_event_tx, _dht_event_rx) =
-			mpsc::channel::<DhtEvent>(10_000);
-
-		let service = builder.with_network_protocol(|_| Ok(crate::service::NodeProtocol::new()))?
-			.with_finality_proof_provider(|client, backend|
-				Ok(Arc::new(grandpa::FinalityProofProvider::new(backend, client)) as _)
-			)?
-			.with_dht_event_tx(dht_event_tx)?
-			.build()?;
+		let service = builder
+			.with_finality_proof_provider(|client, backend| {
+				// GenesisAuthoritySetProvider is implemented for StorageAndProofProvider
+				let provider = client as Arc<dyn grandpa::StorageAndProofProvider<_, _>>;
+				Ok(Arc::new(grandpa::FinalityProofProvider::new(backend, provider)) as _)
+			})?
+			.build_full()?;
 
 		let (block_import, grandpa_link, babe_link) = import_setup.take()
-				.expect("Link Half and Block Import are present for Full Services or setup failed before. qed");
+			.expect("Link Half and Block Import are present for Full Services or setup failed before. qed");
+
+		let shared_voter_state = rpc_setup.take()
+			.expect("The SharedVoterState is present for Full Services or setup failed before. qed");
 
 		($with_startup_data)(&block_import, &babe_link);
 
-		if participates_in_consensus {
-			let proposer = substrate_basic_authorship::ProposerFactory {
-				client: service.client(),
-				transaction_pool: service.transaction_pool(),
-			};
+		if let sc_service::config::Role::Authority { .. } = &role {
+			let proposer = sc_basic_authorship::ProposerFactory::new(
+				service.client(),
+				service.transaction_pool(),
+				service.prometheus_registry().as_ref(),
+			);
 
 			let client = service.client();
 			let select_chain = service.select_chain()
-				.ok_or(substrate_service::Error::SelectChainRequired)?;
+				.ok_or(sc_service::Error::SelectChainRequired)?;
+
+			let can_author_with =
+				sp_consensus::CanAuthorWithNativeVersion::new(client.executor().clone());
 
-			let babe_config = babe::BabeParams {
+			let babe_config = sc_consensus_babe::BabeParams {
 				keystore: service.keystore(),
 				client,
 				select_chain,
@@ -187,62 +220,95 @@ macro_rules! new_full {
 				inherent_data_providers: inherent_data_providers.clone(),
 				force_authoring,
 				babe_link,
+				can_author_with,
 			};
 
-			let babe = babe::start_babe(babe_config)?;
-			service.spawn_essential_task(babe);
-        }
+			let babe = sc_consensus_babe::start_babe(babe_config)?;
+			service.spawn_essential_task_handle().spawn_blocking("babe-proposer", babe);
+		}
+
+		// Spawn authority discovery module.
+		if matches!(role, sc_service::config::Role::Authority{..} | sc_service::config::Role::Sentry {..}) {
+			let (sentries, authority_discovery_role) = match role {
+				sc_service::config::Role::Authority { ref sentry_nodes } => (
+					sentry_nodes.clone(),
+					sc_authority_discovery::Role::Authority (
+						service.keystore(),
+					),
+				),
+				sc_service::config::Role::Sentry {..} => (
+					vec![],
+					sc_authority_discovery::Role::Sentry,
+				),
+				_ => unreachable!("Due to outer matches! constraint; qed.")
+			};
+
+			let network = service.network();
+			let dht_event_stream = network.event_stream("authority-discovery").filter_map(|e| async move { match e {
+				Event::Dht(e) => Some(e),
+				_ => None,
+			}}).boxed();
+			let authority_discovery = sc_authority_discovery::AuthorityDiscovery::new(
+				service.client(),
+				network,
+				sentries,
+				dht_event_stream,
+				authority_discovery_role,
+				service.prometheus_registry(),
+			);
+
+			service.spawn_task_handle().spawn("authority-discovery", authority_discovery);
+		}
 
-        // if the node isn't actively participating in consensus then it doesn't
+		// if the node isn't actively participating in consensus then it doesn't
 		// need a keystore, regardless of which protocol we use below.
-		let keystore = if participates_in_consensus {
-			Some(service.keystore())
+		let keystore = if role.is_authority() {
+			Some(service.keystore() as BareCryptoStorePtr)
 		} else {
 			None
-        };
-
-        let config = grandpa::Config {
-            // FIXME #1578 make this available through chainspec
-            gossip_duration: std::time::Duration::from_millis(333),
-            justification_period: 512,
-            name: Some(name),
-            observer_enabled: true,
-            keystore,
-            is_authority,
-        };
-
-		match (is_authority, disable_grandpa) {
-			(false, false) => {
-				// start the lightweight GRANDPA observer
-				service.spawn_task(Box::new(grandpa::run_grandpa_observer(
-					config,
-					grandpa_link,
-					service.network(),
-					service.on_exit(),
-				)?));
-			},
-			(true, false) => {
-				// start the full GRANDPA voter
-				let grandpa_config = grandpa::GrandpaParams {
-					config,
-					link: grandpa_link,
-					network: service.network(),
-					inherent_data_providers: inherent_data_providers.clone(),
-					on_exit: service.on_exit(),
-					telemetry_on_connect: Some(service.telemetry_on_connect_stream()),
-					voting_rule: grandpa::VotingRulesBuilder::default().build(),
-                };
-                // the GRANDPA voter task is considered infallible, i.e.
-				// if it fails we take down the service with it.
-				service.spawn_essential_task(grandpa::run_grandpa_voter(grandpa_config)?);
-			},
-			(_, true) => {
-				grandpa::setup_disabled_grandpa(
-					service.client(),
-					&inherent_data_providers,
-					service.network(),
-				)?;
-			},
+		};
+
+		let config = grandpa::Config {
+			// FIXME #1578 make this available through chainspec
+			gossip_duration: std::time::Duration::from_millis(333),
+			justification_period: 512,
+			name: Some(name),
+			observer_enabled: false,
+			keystore,
+			is_authority: role.is_network_authority(),
+		};
+
+		let enable_grandpa = !disable_grandpa;
+		if enable_grandpa {
+			// start the full GRANDPA voter
+			// NOTE: non-authorities could run the GRANDPA observer protocol, but at
+			// this point the full voter should provide better guarantees of block
+			// and vote data availability than the observer. The observer has not
+			// been tested extensively yet and having most nodes in a network run it
+			// could lead to finality stalls.
+			let grandpa_config = grandpa::GrandpaParams {
+				config,
+				link: grandpa_link,
+				network: service.network(),
+				inherent_data_providers: inherent_data_providers.clone(),
+				telemetry_on_connect: Some(service.telemetry_on_connect_stream()),
+				voting_rule: grandpa::VotingRulesBuilder::default().build(),
+				prometheus_registry: service.prometheus_registry(),
+				shared_voter_state,
+			};
+
+			// the GRANDPA voter task is considered infallible, i.e.
+			// if it fails we take down the service with it.
+			service.spawn_essential_task_handle().spawn_blocking(
+				"grandpa-voter",
+				grandpa::run_grandpa_voter(grandpa_config)?
+			);
+		} else {
+			grandpa::setup_disabled_grandpa(
+				service.client(),
+				&inherent_data_providers,
+				service.network(),
+			)?;
 		}
 
 		Ok((service, inherent_data_providers))
@@ -252,70 +318,49 @@ macro_rules! new_full {
 	}}
 }
 
-#[allow(dead_code)]
-type ConcreteBlock = node_runtime::opaque::Block;
-#[allow(dead_code)]
-type ConcreteClient = Client<
-    Backend<ConcreteBlock>,
-    LocalCallExecutor<Backend<ConcreteBlock>, NativeExecutor<Executor>>,
-    ConcreteBlock,
-    node_runtime::RuntimeApi,
->;
-#[allow(dead_code)]
-type ConcreteBackend = Backend<ConcreteBlock>;
-
-/// A specialized configuration object for setting up the node..
-pub type NodeConfiguration<C> =
-    Configuration<C, GenesisConfig /*, crate::chain_spec::Extensions*/>;
-
 /// Builds a new service for a full client.
-pub fn new_full<C: Send + Default + 'static>(config: NodeConfiguration<C>)
--> Result<
-	Service<
-		ConcreteBlock,
-		ConcreteClient,
-		LongestChain<ConcreteBackend, ConcreteBlock>,
-		NetworkStatus<ConcreteBlock>,
-		NetworkService<ConcreteBlock, crate::service::NodeProtocol, <ConcreteBlock as BlockT>::Hash>,
-		TransactionPool<transaction_pool::FullChainApi<ConcreteClient, ConcreteBlock>>,
-		OffchainWorkers<
-			ConcreteClient,
-			<ConcreteBackend as substrate_client::backend::Backend<Block, Blake2Hasher>>::OffchainStorage,
-			ConcreteBlock,
-		>
-	>,
-	ServiceError,
->
-{
+pub fn new_full(config: Configuration) -> Result<impl AbstractService, ServiceError> {
     new_full!(config).map(|(service, _)| service)
 }
 
 /// Builds a new service for a light client.
-pub fn new_light<C: Send + Default + 'static>(
-    config: NodeConfiguration<C>,
-) -> Result<impl AbstractService, ServiceError> {
-    // type RpcExtension = jsonrpc_core::IoHandler<substrate_rpc::Metadata>;
+pub fn new_light(config: Configuration) -> Result<impl AbstractService, ServiceError> {
     let inherent_data_providers = InherentDataProviders::new();
 
-    let service = ServiceBuilder::new_light::<Block, RuntimeApi, Executor>(config)?
+    let service = ServiceBuilder::new_light::<Block, RuntimeApi, node_executor::Executor>(config)?
         .with_select_chain(|_config, backend| Ok(LongestChain::new(backend.clone())))?
-        .with_transaction_pool(|config, client| {
-            Ok(TransactionPool::new(
-                config,
-                transaction_pool::FullChainApi::new(client),
-            ))
+        .with_transaction_pool(|builder| {
+            let fetcher = builder
+                .fetcher()
+                .ok_or_else(|| "Trying to start light transaction pool without active fetcher")?;
+            let pool_api =
+                sc_transaction_pool::LightChainApi::new(builder.client().clone(), fetcher);
+            let pool = sc_transaction_pool::BasicPool::with_revalidation_type(
+                builder.config().transaction_pool.clone(),
+                Arc::new(pool_api),
+                builder.prometheus_registry(),
+                sc_transaction_pool::RevalidationType::Light,
+            );
+            Ok(pool)
         })?
         .with_import_queue_and_fprb(
-            |_config, client, backend, fetcher, _select_chain, _tx_pool| {
+            |_config,
+             client,
+             backend,
+             fetcher,
+             _select_chain,
+             _tx_pool,
+             spawn_task_handle,
+             registry| {
                 let fetch_checker = fetcher
                     .map(|fetcher| fetcher.checker().clone())
                     .ok_or_else(|| {
                         "Trying to start light import queue without active fetch checker"
                     })?;
-                let grandpa_block_import = grandpa::light_block_import::<_, _, _, RuntimeApi>(
+                let grandpa_block_import = grandpa::light_block_import(
                     client.clone(),
                     backend,
-                    &*client,
+                    &(client.clone() as Arc<_>),
                     Arc::new(fetch_checker),
                 )?;
 
@@ -323,35 +368,294 @@ pub fn new_light<C: Send + Default + 'static>(
                 let finality_proof_request_builder =
                     finality_proof_import.create_finality_proof_request_builder();
 
-                let (babe_block_import, babe_link) = babe::block_import(
-                    babe::Config::get_or_compute(&*client)?,
+                let (babe_block_import, babe_link) = sc_consensus_babe::block_import(
+                    sc_consensus_babe::Config::get_or_compute(&*client)?,
                     grandpa_block_import,
                     client.clone(),
-                    client.clone(),
                 )?;
 
-                let import_queue = babe::import_queue(
+                let import_queue = sc_consensus_babe::import_queue(
                     babe_link,
                     babe_block_import,
                     None,
                     Some(Box::new(finality_proof_import)),
-                    client.clone(),
                     client,
                     inherent_data_providers.clone(),
+                    spawn_task_handle,
+                    registry,
                 )?;
 
                 Ok((import_queue, finality_proof_request_builder))
             },
         )?
-        .with_network_protocol(|_| Ok(NodeProtocol::new()))?
         .with_finality_proof_provider(|client, backend| {
-            Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, client)) as _)
+            // GenesisAuthoritySetProvider is implemented for StorageAndProofProvider
+            let provider = client as Arc<dyn StorageAndProofProvider<_, _>>;
+            Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, provider)) as _)
+        })?
+        .with_rpc_extensions(|builder| {
+            let fetcher = builder
+                .fetcher()
+                .ok_or_else(|| "Trying to start node RPC without active fetcher")?;
+            let remote_blockchain = builder
+                .remote_backend()
+                .ok_or_else(|| "Trying to start node RPC without active remote blockchain")?;
+
+            let light_deps = node_rpc::LightDeps {
+                remote_blockchain,
+                fetcher,
+                client: builder.client().clone(),
+                pool: builder.pool(),
+            };
+
+            Ok(node_rpc::create_light(light_deps))
         })?
-        // We don't have any custom rpc extensions
-        // .with_rpc_extensions(|client, pool| -> RpcExtension {
-        // 	node_rpc::create(client, pool)
-        // })?
-        .build()?;
+        .build_light()?;
 
     Ok(service)
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::node_executor;
+    use crate::node_rpc;
+    use crate::service::{new_full, new_light};
+    use codec::{Decode, Encode};
+    use node_runtime::RuntimeApi;
+    use node_runtime::{currency::CENTS, SLOT_DURATION};
+    use node_runtime::{opaque::Block, AccountId, DigestItem, Signature};
+    use node_runtime::{BalancesCall, Call, UncheckedExtrinsic};
+    use sc_consensus_babe::{BabeIntermediate, CompatibleDigestItem, INTERMEDIATE_KEY};
+    use sc_consensus_epochs::descendent_query;
+    use sc_finality_grandpa::{self as grandpa};
+    use sc_service::AbstractService;
+    use sp_consensus::{
+        BlockImport, BlockImportParams, BlockOrigin, Environment, ForkChoiceStrategy, Proposer,
+        RecordProof,
+    };
+    use sp_core::{crypto::Pair as CryptoPair, H256};
+    use sp_finality_tracker;
+    use sp_keyring::AccountKeyring;
+    use sp_runtime::traits::IdentifyAccount;
+    use sp_runtime::{
+        generic::{BlockId, Digest, Era, SignedPayload},
+        traits::Verify,
+        traits::{Block as BlockT, Header as HeaderT},
+        OpaqueExtrinsic,
+    };
+    use sp_timestamp;
+    use sp_transaction_pool::{ChainEvent, MaintainedTransactionPool};
+    use std::{any::Any, borrow::Cow, sync::Arc};
+
+    type AccountPublic = <Signature as Verify>::Signer;
+
+    // Long running test. Run it locally only after the node changes.
+    #[test]
+    // It is "ignored", but the node-cli ignored tests are running on the CI.
+    // This can be run locally with `cargo test --release -p node-cli test_sync -- --ignored`.
+    #[ignore]
+    fn test_sync() {
+        let keystore_path = tempfile::tempdir().expect("Creates keystore path");
+        let keystore =
+            sc_keystore::Store::open(keystore_path.path(), None).expect("Creates keystore");
+        let alice = keystore
+            .write()
+            .insert_ephemeral_from_seed::<sc_consensus_babe::AuthorityPair>("//Alice")
+            .expect("Creates authority pair");
+
+        let chain_spec = crate::chain_spec::tests::integration_test_config_with_single_authority();
+
+        // For the block factory
+        let mut slot_num = 1u64;
+
+        // For the extrinsics factory
+        let bob = Arc::new(AccountKeyring::Bob.pair());
+        let charlie = Arc::new(AccountKeyring::Charlie.pair());
+        let mut index = 0;
+
+        sc_service_test::sync(
+            chain_spec,
+            |config| {
+                let mut setup_handles = None;
+                new_full!(
+                    config,
+                    |block_import: &sc_consensus_babe::BabeBlockImport<Block, _, _>,
+                     babe_link: &sc_consensus_babe::BabeLink<Block>| {
+                        setup_handles = Some((block_import.clone(), babe_link.clone()));
+                    }
+                )
+                .map(move |(node, x)| (node, (x, setup_handles.unwrap())))
+            },
+            |config| new_light(config),
+            |service, &mut (ref inherent_data_providers, (ref mut block_import, ref babe_link))| {
+                let mut inherent_data = inherent_data_providers
+                    .create_inherent_data()
+                    .expect("Creates inherent data.");
+                inherent_data.replace_data(sp_finality_tracker::INHERENT_IDENTIFIER, &1u64);
+
+                let parent_id = BlockId::number(service.client().chain_info().best_number);
+                let parent_header = service.client().header(&parent_id).unwrap().unwrap();
+                let parent_hash = parent_header.hash();
+                let parent_number = *parent_header.number();
+
+                futures::executor::block_on(service.transaction_pool().maintain(
+                    ChainEvent::NewBlock {
+                        is_new_best: true,
+                        hash: parent_header.hash(),
+                        tree_route: None,
+                        header: parent_header.clone(),
+                    },
+                ));
+
+                let mut proposer_factory = sc_basic_authorship::ProposerFactory::new(
+                    service.client(),
+                    service.transaction_pool(),
+                    None,
+                );
+
+                let epoch_descriptor = babe_link
+                    .epoch_changes()
+                    .lock()
+                    .epoch_descriptor_for_child_of(
+                        descendent_query(&*service.client()),
+                        &parent_hash,
+                        parent_number,
+                        slot_num,
+                    )
+                    .unwrap()
+                    .unwrap();
+
+                let mut digest = Digest::<H256>::default();
+
+                // even though there's only one authority some slots might be empty,
+                // so we must keep trying the next slots until we can claim one.
+                let babe_pre_digest = loop {
+                    inherent_data.replace_data(
+                        sp_timestamp::INHERENT_IDENTIFIER,
+                        &(slot_num * SLOT_DURATION),
+                    );
+                    if let Some(babe_pre_digest) = sc_consensus_babe::test_helpers::claim_slot(
+                        slot_num,
+                        &parent_header,
+                        &*service.client(),
+                        &keystore,
+                        &babe_link,
+                    ) {
+                        break babe_pre_digest;
+                    }
+
+                    slot_num += 1;
+                };
+
+                digest.push(<DigestItem as CompatibleDigestItem>::babe_pre_digest(
+                    babe_pre_digest,
+                ));
+
+                let new_block = futures::executor::block_on(async move {
+                    let proposer = proposer_factory.init(&parent_header).await;
+                    proposer
+                        .unwrap()
+                        .propose(
+                            inherent_data,
+                            digest,
+                            std::time::Duration::from_secs(1),
+                            RecordProof::Yes,
+                        )
+                        .await
+                })
+                .expect("Error making test block")
+                .block;
+
+                let (new_header, new_body) = new_block.deconstruct();
+                let pre_hash = new_header.hash();
+                // sign the pre-sealed hash of the block and then
+                // add it to a digest item.
+                let to_sign = pre_hash.encode();
+                let signature = alice.sign(&to_sign[..]);
+                let item = <DigestItem as CompatibleDigestItem>::babe_seal(signature.into());
+                slot_num += 1;
+
+                let mut params = BlockImportParams::new(BlockOrigin::File, new_header);
+                params.post_digests.push(item);
+                params.body = Some(new_body);
+                params.intermediates.insert(
+                    Cow::from(INTERMEDIATE_KEY),
+                    Box::new(BabeIntermediate::<Block> { epoch_descriptor }) as Box<dyn Any>,
+                );
+                params.fork_choice = Some(ForkChoiceStrategy::LongestChain);
+
+                block_import
+                    .import_block(params, Default::default())
+                    .expect("error importing test block");
+            },
+            |service, _| {
+                let amount = 5 * CENTS;
+                let to: AccountId = AccountPublic::from(bob.public()).into_account().into();
+                let from: AccountId = AccountPublic::from(charlie.public()).into_account().into();
+                let genesis_hash = service.client().block_hash(0).unwrap().unwrap();
+                let best_block_id = BlockId::number(service.client().chain_info().best_number);
+                let (spec_version, transaction_version) = {
+                    let version = service.client().runtime_version_at(&best_block_id).unwrap();
+                    (version.spec_version, version.transaction_version)
+                };
+                let signer = charlie.clone();
+
+                let function = Call::Balances(BalancesCall::transfer(to.into(), amount));
+
+                let check_spec_version = frame_system::CheckSpecVersion::new();
+                let check_tx_version = frame_system::CheckTxVersion::new();
+                let check_genesis = frame_system::CheckGenesis::new();
+                let check_era = frame_system::CheckEra::from(Era::Immortal);
+                let check_nonce = frame_system::CheckNonce::from(index);
+                let check_weight = frame_system::CheckWeight::new();
+                let payment = pallet_transaction_payment::ChargeTransactionPayment::from(0);
+                let validate_grandpa_equivocation =
+                    pallet_grandpa::ValidateEquivocationReport::new();
+                let extra = (
+                    check_spec_version,
+                    check_tx_version,
+                    check_genesis,
+                    check_era,
+                    check_nonce,
+                    check_weight,
+                    payment,
+                    validate_grandpa_equivocation,
+                );
+                let raw_payload = SignedPayload::from_raw(
+                    function,
+                    extra,
+                    (
+                        spec_version,
+                        transaction_version,
+                        genesis_hash,
+                        genesis_hash,
+                        (),
+                        (),
+                        (),
+                        (),
+                    ),
+                );
+                let signature = raw_payload.using_encoded(|payload| signer.sign(payload));
+                let (function, extra, _) = raw_payload.deconstruct();
+                let xt =
+                    UncheckedExtrinsic::new_signed(function, from.into(), signature.into(), extra)
+                        .encode();
+                let v: Vec<u8> = Decode::decode(&mut xt.as_slice()).unwrap();
+
+                index += 1;
+                OpaqueExtrinsic(v)
+            },
+        );
+    }
+
+    #[test]
+    #[ignore]
+    fn test_consensus() {
+        sc_service_test::consensus(
+            crate::chain_spec::tests::integration_test_config_with_two_authorities(),
+            |config| new_full(config),
+            |config| new_light(config),
+            vec!["//Alice".into(), "//Bob".into()],
+        )
+    }
+}

package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "test": "yarn && yarn workspaces run test",
     "test-migration": "yarn && yarn workspaces run test-migration",
-    "postinstall": "yarn workspace @joystream/types build && yarn workspace storage-node run build",
+    "postinstall": "yarn workspace @joystream/types build",
     "cargo-checks": "devops/git-hooks/pre-commit && devops/git-hooks/pre-push",
     "cargo-build": "scripts/cargo-build.sh",
     "lint": "yarn workspaces run lint"
@@ -15,27 +15,30 @@
     "tests/network-tests",
     "cli",
     "types",
-    "pioneer",
-    "pioneer/packages/*",
     "storage-node",
     "storage-node/packages/*",
     "devops/eslint-config",
-    "devops/prettier-config"
+    "devops/prettier-config",
+    "pioneer",
+    "pioneer/packages/*",
+    "utils/api-examples"
   ],
   "resolutions": {
-    "@polkadot/api": "^0.96.1",
-    "@polkadot/api-contract": "^0.96.1",
-    "@polkadot/keyring": "^1.7.0-beta.5",
-    "@polkadot/types": "^0.96.1",
-    "@polkadot/util": "^1.7.0-beta.5",
-    "@polkadot/util-crypto": "^1.7.0-beta.5",
+    "@polkadot/api": "1.26.1",
+    "@polkadot/api-contract": "1.26.1",
+    "@polkadot/keyring": "^3.0.1",
+    "@polkadot/types": "1.26.1",
+    "@polkadot/util": "^3.0.1",
+    "@polkadot/util-crypto": "^3.0.1",
+    "@polkadot/wasm-crypto": "^1.2.1",
     "babel-core": "^7.0.0-bridge.0",
-    "typescript": "^3.7.2"
+    "typescript": "^3.9.7",
+    "bn.js": "^5.1.2"
   },
   "devDependencies": {
     "husky": "^4.2.5",
     "prettier": "2.0.2",
-    "eslint": "^5.16.0"
+    "eslint": "^7.6.0"
   },
   "husky": {
     "hooks": {

pioneer/.123trigger

@@ -1 +1,2 @@
-5
+11
+0.45.2

pioneer/.dockerignore

@@ -1 +1,3 @@
 node_modules
+build
+.git

pioneer/.env-example

@@ -0,0 +1,4 @@
+# You can define all your ENV in such a file and run docker as:
+# docker run ... --env-file .env ...
+WS_URL=ws://localhost:9944
+POLKADOT_UI_SAMPLE=42

pioneer/.eslintignore

@@ -1,4 +1,5 @@
 **/build/*
 **/coverage/*
 **/node_modules/*
+.eslintrc.js
 i18next-scanner.config.js

pioneer/.eslintrc.js

@@ -1,5 +1,5 @@
 // At some point don't depend on @polkadot rules and use @joystream/eslint-config
-const base = require('@polkadot/dev-react/config/eslint');
+const base = require('@polkadot/dev/config/eslint');
 
 // add override for any (a metric ton of them, initial conversion)
 module.exports = {
@@ -13,14 +13,22 @@ module.exports = {
   rules: {
     ...base.rules,
     '@typescript-eslint/no-explicit-any': 'off',
-    '@typescript-eslint/camelcase': 'off',
-    'react/prop-types': 'off',
     'new-cap': 'off',
     '@typescript-eslint/interface-name-prefix': 'off',
     '@typescript-eslint/ban-ts-comment': 'error',
     // why only required in VSCode!?!? is eslint plugin not working like eslint commandline?
     // Or are we having to add this because of new versions of eslint-config-* ?
     'no-console': 'off',
+    // Override some extended config rules:
+    'camelcase': 'off',
+    'header/header': 'off',
+    'sort-keys': 'off',
+    'react/jsx-sort-props': 'off',
+    'react/jsx-max-props-per-line': 'off',
+    'sort-destructure-keys/sort-destructure-keys': 'off',
+    '@typescript-eslint/unbound-method': 'warn', // Doesn't work well with our version of Formik, see: https://github.com/formium/formik/issues/2589
+    'react-hooks/exhaustive-deps': 'warn', // Causes more issues than it solves currently
+    'no-void': 'off' // Otherwise we cannot mark unhandles promises
   },
   // isolate pioneer from monorepo eslint rules
   root: true

pioneer/.gitignore

@@ -10,13 +10,16 @@ tmp/
 .env.test.local
 .env.production.local
 .npmrc
+.yarn/*
+!.yarn/releases
+!.yarn/plugins
+.pnp.*
 cc-test-reporter
 package-lock.json
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 !patches/**
-.idea/
 
 # Built Joystream types:
 packages/joy-types/lib/

pioneer/.storybook/webpack.config.js

@@ -1,81 +1,47 @@
 const path = require('path')
 const TsconfigPathsPlugin = require('tsconfig-paths-webpack-plugin');
 module.exports = ({ config }) => {
+  // Styles (replace the provided rule):
+  const originalCssRuleIndex = config.module.rules.findIndex(rule => rule.test.source.includes('.css'));
+  config.module.rules[originalCssRuleIndex] = {
+    test: /\.(sa|sc|c)ss$/i,
+    use: [
+      // Creates `style` nodes from JS strings
+      'style-loader',
+      // Translates CSS into CommonJS
+      'css-loader',
+      // Compiles Sass to CSS
+      'sass-loader'
+    ]
+  };
 
-// Post CSS loader for sources:
-config.module.rules.push({
-  test: /\.css$/,
-  include: path.resolve(__dirname, '../packages'),
-  exclude: /(node_modules)/,
-  use: [
-    {
-      loader: require.resolve('postcss-loader'),
-      options: {
-        // Set postcss.config.js config path && ctx
-        config: {
-          path: '../postcss.config.js',
-        },
-        ident: 'postcss',
-        plugins: () => [
-          require('precss'),
-          require('autoprefixer'),
-          require('postcss-simple-vars'),
-          require('postcss-nested'),
-          require('postcss-import'),
-          require('postcss-clean')(),
-          require('postcss-flexbugs-fixes')
-        ]
-      }
-    }
-  ]
-});
+  // TypeScript loader (via Babel to match polkadot/apps)
+  config.module.rules.push({
+    test: /\.(js|ts|tsx)$/,
+    exclude: /(node_modules)/,
+    use: [
+      {
+        loader: require.resolve('babel-loader'),
+        options: require('@polkadot/dev/config/babel')
+      },
+    ],
+  });
+  config.resolve.extensions.push('.js', '.ts', '.tsx');
 
-// TypeScript loader (via Babel to match polkadot/apps)
-config.module.rules.push({
-  test: /\.(js|ts|tsx)$/,
-  exclude: /(node_modules)/,
-  use: [
-    {
-      loader: require.resolve('babel-loader'),
-      options: require('@polkadot/dev-react/config/babel')
-    },
-  ],
-});
-config.resolve.extensions.push('.js', '.ts', '.tsx');
+  // TSConfig, uses the same file as packages
+  config.resolve.plugins = config.resolve.plugins || [];
+  config.resolve.plugins.push(
+    new TsconfigPathsPlugin({
+      configFile: path.resolve(__dirname, '../tsconfig.json'),
+    })
+  );
 
-// TSConfig, uses the same file as packages
-config.resolve.plugins = config.resolve.plugins || [];
-config.resolve.plugins.push(
-  new TsconfigPathsPlugin({
-    configFile: path.resolve(__dirname, '../tsconfig.json'),
-  })
-);
+  // Stories parser
+  config.module.rules.push({
+      test: /\.stories\.tsx?$/,
+      loaders: [require.resolve('@storybook/source-loader')],
+      enforce: 'pre',
+  });
 
-// Stories parser
-config.module.rules.push({
-    test: /\.stories\.tsx?$/,
-    loaders: [require.resolve('@storybook/source-loader')],
-    enforce: 'pre',
-});
-
-// CSS preprocessors
-config.module.rules.push(
-    {
-        test: /\.s[ac]ss$/i,
-        use: [
-            // Creates `style` nodes from JS strings
-            'style-loader',
-            // Translates CSS into CommonJS
-            'css-loader',
-            // Compiles Sass to CSS
-            'sass-loader',
-        ],
-    },
-    {
-        test: /\.less$/,
-        loaders: [ 'style-loader', 'css-loader', 'less-loader' ]
-    }
-);
-
-return config;
+  return config;
 };

pioneer/.stylelintrc

@@ -0,0 +1,7 @@
+{
+  "extends": [
+    "stylelint-config-recommended",
+    "stylelint-config-styled-components"
+  ],
+  "defaultSeverity": "warning"
+}

pioneer/CHANGELOG.md

@@ -1,4 +1,358 @@
-# 0.36.1
+# CHANGELOG
+
+## 0.51.1 Jul 27, 2020
+
+- Support for Acala chain and types (Thanks to https://github.com/aniiantt)
+- First batch of i18n for Arabic (Thanks to https://github.com/nightwolf3)
+- Update for Polkadot council thresholds
+- Council motion adjustment to cater for current and previous generations
+- Adjust imminent proposals to not need own type adjustments
+- Cleanup voting totals to only take free into account
+- Support JSON v3 formats with kdf
+- Always display time left in countdowns, not blocks
+- Adjust progress component to be circular
+- Rename "Deposit" button to "Send" button (less confusion)
+- Ensure new generation tip cancel works for council & users
+- Split council votes in backing & number of votes
+- Adjust council motions to have the correct max display for nay votes
+- Adjust payout toggles with increasing day increments
+- Adjust button formats & layouts
+- Adjust council slashing params to cater for current generation
+- Fixed for keyboard locale detection
+- Don't allow display in an iframe
+
+## 0.50.1 Jul 20, 2020
+
+- Adjust CSPs for Electron (Thanks to https://github.com/EthWorks)
+- Move delegation column to badge (Thanks to https://github.com/Tbaut)
+- Display costs for preimage calls
+- Add buttons for bid/unbid on Society
+- Allow tip endorsements while in countdown
+- Flatten events to group by same-kind in the same block
+- Adjust identity validation to check for whitespacing
+- Allow the use of VecFixed params with type detection/inputs
+- Adjust controller changing to filter when stash === controller (no error, but warn)
+- Allow the poll module via Governance proxy
+- Expand proxy detection to deep-inspect batched calls
+- Ensure various APIs are available (filtering) before attempting to render
+- Ensure tooltips are correctly displayed on account hovers
+- Cleanup council display for candidates
+- Cleanup and simplify QR import logic (& always set genesisHash, even when not supplied)
+- Always set genesisHash when accounts are derived
+- Adjust breakpoints for `AccountName` via `AddressShort`
+- Cleanup SUI dependencies with unused components, Progress, Responsive, Toggle
+- Adjust module bundling splits, remove unused libraries & fonts
+- Remove unmaintained page-123code (& references)
+- Don't display finalized blocks when grandpa is not available
+
+## 0.49.1 Jul 13, 2020
+
+- Update Subscan links with supported chains (https://github.com/carumusan)
+- Enhance Electron desktop with CSP & best practices (Thanks to https://github.com/EthWorks)
+- Activate Electron update feature by default (Thanks to https://github.com/EthWorks)
+- Use external browser for embedded links in Electron (Thanks to https://github.com/EthWorks)
+- Add testing around Electron features (Thanks to https://github.com/EthWorks)
+- Support democracy account delegation (Thanks to https://github.com/Tbaut)
+- Don't filter selected on multi account selector (Thanks to https://github.com/Tbaut)
+- Add support for Polkadot denomination poll
+- Allow input & display of OpaqueCall type from multisig
+- Indicate own nominees on targets page (re-added with badges)
+- Re-add indicator for own nominators in staking targets
+- Add `?filter=<string>` query param support on staking URLs
+- Add generator for Kusama society designs
+- Handle OpaqueCall in inputs and well as displays (multisig)
+- Small layout adjustments for address display components
+- Performance improvements on wrapped styles, component libraries
+- Add support for display detected ASCII bytes as text
+- Adjust type injection to override on-connect API defaults
+- Adjust attestation display with no-balance filters
+
+## 0.48.1 Jul 6, 2020
+
+- Fix for electron package build (Thanks to https://github.com/EthWorks)
+- Allow for setting of sub identities via account action
+- Adjust known account icons (Society & Treasury)
+- Add Westend chain to Subscan link generator
+- Display a warning with extensions and no injected accounts
+- Retrieve all tips at once and sort by closing
+- Fix identity set dialogs to never pass empty fields
+- Optimize favorites retrieval & selection for staking (shared between)
+- Support new registrar ProxyType in the signer
+- Hide funds unbonding when non bonded
+- Add withdraw action to staking menu (as available)
+- Fix InputAddress component with state change warnings
+- Bump to latest API and utilities
+
+## 0.47.1 Jul 1, 2020
+
+- Update zh translation (Thanks to https://github.com/dushaobindoudou)
+- Add DataHighway Harbour testnet endpoint (Thanks to https://github.com/ltfschoen)
+- Small I18N key fix (Thanks to https://github.com/ltfschoen)
+- Allow for Electron auto-update on Mac (Thanks to https://github.com/EthWorks)
+- Swap to default conviction of 1x (Thanks to https://github.com/Tbaut)
+- Make preimage hash selectable on FF (Thanks to https://github.com/Tbaut)
+- show unbonding value in staking actions (Thanks to https://github.com/Tbaut)
+- Default (via toggle) to only last 25% of eras for payouts
+- Allow retracting of tips by proposer
+- Allow tipping with new Substrate types (dual old/new support)
+- Enable the full retrieval of all Map/Doublemap entries
+- Support correct display of vesting with locks (& unlock via account)
+- Adjust on-chain identity inputs with field validation
+- Enable grouping of democracy locks by type
+- Resolve identity links starting with https://twitter
+- Display voted & unvoted council motions, referendums & tips
+- Adjust toggles for file/bytes uploads
+- Correct handling of recursive param structures in extrinsics
+- Swap icons to use font-awesome directly, including official components
+- Additional small UI cleanups and fixes
+
+## 0.46.1 Jun 22, 2020
+
+- I18n for es (Thanks to https://github.com/wimel)
+- Support for importing mini secrets via QR (Thanks to https://github.com/hanwencheng)
+- Update SubstrateTEE types (Thanks to https://github.com/brenzi)
+- Support for multisig calls with new weight parameters
+- Split sign and send updates in the signer modal for better UI tracking
+- Hide zero nonce of accounts/contracts pages
+- Display API extrinsic construction errors in the extrinsics app
+- Do not display signer proxies when there are none matching against accounts
+- Sort recovery addresses to align with the Substrate implementation
+- Check for funded controller on bonding
+- Suggest max values for bonding (& bonding extra), adjusting checks
+- Handle isForceEra to adjust era displays
+- Display candidacy bond on council submission
+- Adjust AddressMni & AddressSmall components to take advantage of bigger screens
+- Display referendum & treasury tips voting status
+- Add tips close buttons & countdown timer
+- Disabled nominations via targets when in election
+- Expand targets page to include waiting validators (full overview of all)
+- Apply shared filters (name, toggles) on all validator lists
+- Display balances in account view sidebar
+- Adjust signer dialog ith split sign/send (better status displays)
+- Adjust proxy checks for sudo calls to closer align with Polkadot
+- Apply i18n caching, with no reload on translation page
+- Add "Apply" i18n button to reflect editing changes in the UI
+- Support Tuple inputs (params/extrinsics) for custom names
+- `@polkadot/api` 1.20.1
+- `@polkadot/util` 2.15.1
+
+## 0.45.2 Jun 16, 2020
+
+- I18n for ja (Thanks to https://github.com/SotaWatanabe)
+- I18n for pt (thanks to https://github.com/laurogripa)
+- I18n for ru (Thanks to https://github.com/illlefr4u)
+- Update Encointer types (thanks to https://github.com/brenzi)
+- Improve Electron app security settings (Thanks to https://github.com/EthWorks)
+- Rework signer dialog to cater for proxies (and multisig/proxy combinations)
+- Construct payouts with oldest eras first (expire first)
+- Show outstanding multisig approvals on accounts page
+- Allow for addition of proxied accounts (access to proxy account only)
+- Change claims to handle no statements required (new module now on Kusama)
+- Publish docker image on release
+- update collective calls to handle weights enhancements for latest Substrate
+- Allow for tip endorsements with 0 value
+- add Centrifuge live as a connection option
+- Adjust Polkascan links with current active chains
+- When collective proposal is in close state, hide vote buttons
+- Cleanup technical committee display (header alignment)
+- Adjust IPFS/IPNS network extraction for local gateways
+- `@polkadot/api` 1.19.1
+- `@polkadot/util` 2.14.1
+
+## 0.44.1 Jun 10, 2020
+
+- Publish electron images on release (Thanks to https://github.com/EthWorks)
+- Adjust with latest Arcardia types (Thanks to https://github.com/ETeissonniere)
+- Extensions and fixes to the russian translations (Thanks to https://github.com/illlefr4u)
+- Rewrite of the contracts app (Thanks to https://github.com/kwingram25)
+- New types for SubstrateTEE (Thanks to https://github.com/brenzi)
+- Adjust for new Polkadot CC1 & Kusama types
+- Cater for new multisig module location
+- Filter multisig signatories based on approvals, set final state based on threshold
+- Adapt QR codes to cater for hashing on large payloads
+- Adjust collective extrinsics to cater for weight & lengths
+- Allow bonding with full free amount (this fixes bonding for vesting)
+- Fixes for Kusama as well as Polkadot claims
+- Allow Polkadot CC1 links to Polkascan & Subscan
+- Update Polkascan links with new formats
+- Don't display era progress when Forcing `isForceNone`
+- Overall styling adjustments
+- Cater for `{kusama,polkadot,westend}.dotapps.io` redirects
+- `@polkadot/api` 1.18.1
+- `@polkadot/util` 2.13.1
+
+## 0.43.1 May 26, 2020
+
+- Support for Polkadot CC1 Claims (Thanks to https://github.com/amaurymartiny & https://github.com/Tbaut)
+- Small typo fixes (Thanks to https://github.com/Swader)
+- updates to russian translation (Thanks to https://github.com/illlefr4u)
+- Adjustments to Electron build support (Thanks to https://github.com/EthWorks)
+- Support for Polkadot CC1 types & RPC endpoints
+- Detect & support new proposal close process in Substrate
+- Adjust checks for on-click validator (immediate isActive)
+- `@polkadot/api` 1.16.1
+- `@polkadot/util` 2.11.1
+
+## 0.42.1 May 22, 2020
+
+- Adjust Subscan proposal links (Thanks to https://github.com/illlefr4u)
+- Add environment suport for docker images (Thanks to https://github.com/chevdor)
+- Adjust overflows on small screens (Thanks to https://github.com/dushaobindoudou)
+- Add links to Polkaassembly (Thanks to https://github.com/Tbaut)
+- Address popup with detailed info (Thanks to https://github.com/kwingram25)
+- Add Russian translation (Thanks to https://github.com/gregzaitsev)
+- Russian translation adjustments (Thanks to https://github.com/illlefr4u)
+- Add Nodle RPC endpoint (Thanks to https://github.com/ETeissonniere)
+- Update Kulupu types (Thanks to https://github.com/sorpaas)
+- Update Edgeware types (Thanks to https://github.com/drewstone)
+- Update Encointer types (Thanks to https://github.com/brenzi)
+- Update node-template types (Thanks to https://github.com/shawntabrizi)
+- Update node-template types (Thanks to https://github.com/JoshOrndorff)
+- Higher default contracts gas limit (Thanks to https://github.com/Stefie)
+- Add block number display to event overview (Thanks to https://github.com/danforbes)
+- Basic Electron support (Thanks to https://github.com/EthWorks)
+- Documentation around IPFS pinning (Thanks to https://github.com/chevdor)
+- Added IPFS/IPNS publishing (ipns via dotapps.io)
+- Support for multisig wallets
+- Ledger address on-wallet display option
+- Add support for new per-staker payouts
+- Allow for "best" selection in staking
+- Simplified nominator & validator creation flows
+- Display >64 nominators on staking pages (clipped payouts)
+- Remove tooltips on staking and elsewhere (large number causes performance issues)
+- Council isMember checks uses council in addition to elections
+- Allow closing of council proposals
+- Expand Treasury proposal inline in council (for approve/reject)
+- Expand external proposals in council (preimage lookups)
+- Allow for sudo with unchecked weight
+- Adjust referendums to display turnout and sentiment
+- Add columar modals with extra info
+- Add table summaries with totals for free, bonded & stash payouts
+- Add images to metadata update dropdowns
+- Ecdsa keypair support
+- Display delegations in voting breakdowns
+- Adjust registrar modal with per-account filters
+- Add i18n linting script
+- Add i18n editor with translation file generator
+- Custom i18n loader with caching
+- Add JS sample for storage key generation
+- Misc UI fixes & adjustments throughout
+- Allow for tabes with aliasses (on renames) & redirects
+- Align types and calls with latest substrate
+
+## 0.41.1 Apr 20, 2020
+
+- Fix for searching child identities on parent name (Thanks to https://github.com/krogla)
+- Support chains with no balances module (Thanks to https://github.com/Voxelot)
+- Add out-of-the-box support for Encointer (Thanks to https://github.com/brenzi)
+- Add ava.do endpoint for Kusama (Thanks to https://github.com/Swader)
+- Show remaining time on staking payout actions, link payouts from actions
+- Display per-validator nominators on waiting list
+- Add support for Treasury tipping (display of available & creation)
+- Adjust display of passing/failing calcs in democracy (incl. no display when other side is 0)
+- Enable use of `system_chainType` to detect development chains
+- Adjust Expander display for balances as used in accounts
+- Adjust formatting outputs (via cleanup) for state queries
+- Cleanup nowrap on Extender as part of tables
+- Optimize retrieval of old-style validator/nominator payouts (not full historyDepth)
+- Optimize AccountName with caching & when used in lists (no lookup information attached, but not shown)
+- Optimize IdentityIcon with removal of extra queries
+- Optimize Transfers, no unneeded useEffect
+- Cleanups, remove unused components with no references (dropped in earlier refactoring)
+- More components to functional, specific focus on TxModal extends
+- Bumps to all @polkadot/* packages for latest support everywhere
+
+## 0.40.1 Apr 9, 2020
+
+- Swap voting to aye/nay toggles (Thanks to https://github.com/Lowhearth)
+- Cater for chains where no tip is present (Thanks to https://github.com/Sushisource)
+- Export chain-specific settings via QR (thanks to https://github.com/hanwencheng)
+- Improve support for WS_URL usage (Thanks to https://github.com/chevdor)
+- Add out-of-the-box support for Centrifuge (Thanks to https://github.com/philipstanislaus)
+- Cleanup docker image construction (Thanks to https://github.com/philipstanislaus)
+- Add out-of-the-box support for node template (Thanks to https://github.com/JoshOrndorff)
+- Text cleanups (Thanks to https://github.com/x5engine)
+- Text cleanups (Thanks to https://github.com/ltfschoen)
+- update Parachains to support latest Polkadot (Thanks to https://github.com/kwingram25)
+- Rework multi address inputs (e.g. nominations & council) (Thanks to https://github.com/kwingram25)
+- Introduce apps-config as a single source of config information
+- Cater for metadata updates to extensions
+- Rework explorer layouts, combining extrinsics & events into a single view
+- Swap all layouts to be explicitly table-based (instead of table-like)
+- Cater for latest Substrate referendum updates
+- Allow for fast-tracking proposals
+- Time countdowns where applicable, e.g. referendums
+- Show referendum pass/fail status as well as change information
+- Combine Accounts & Contacts into a single app
+- Support for display of parent/child relationships in accounts
+- Add ErrorBoundary around components
+- Update Westend after reset
+- Enable Subscan explorer
+- Support for simple payouts on Substrate, with Payouts screen
+- Extensive use of useCallback & React.memo for functional components
+- Add Expander component for consistent UI
+- Loading spinners used consistently
+- Specific names for society & treasury addresses
+- Cleanup all voting lock, consistent display
+- ... loads of other under-the-hood improvements and cleanups
+
+## 0.39.1 Jan 31, 2020
+
+- **Breaking** Drop support for V1 Substrate chains
+- Translation into Chinese (Thanks to https://github.com/dushaobindoudou)
+- Support for sign-only transactions (Thanks to https://github.com/mzolkiewski)
+- Add support for WestEnd testnet
+- Add support for social recovery in accounts
+- Add counters for all proposal-based apps
+- Disable spellcheck on all input fields (privacy)
+- Query the paymentInfo API to get weight fee information
+- Remove FF warning with https:// -> ws://localhost
+- Staking now supports where the controller or stash accounts are not local
+- Social app
+- Add support for identity setting (via identity module)
+- Add support for registrars to hand out identity judgements
+- Use both internal and lib hooks as applicable (refactoring)
+- Support QR codes (accounts) with optional names
+- Cleanup all Modals, simplify
+- Adjust balance display formats
+- Update to latest libraries (incl. util 2.0 & api 1.0)
+
+## 0.38.1
+
+- Fix summarybar in 123-code (Thanks to https://github.com/anakornk)
+- Update Edgeware with correct keys (Thanks to https://github.com/drewstone)
+- Add InputAddressMulti inputs, both to council and staking nominators
+- Rework all layouts, removing cards for table-ike-layouts
+- Technical comittee app
+- Allow for external proposal and queued for dispatch in democracy
+- Add pre-image support to democracy proposals (including imminent)
+- Improved staking page rendering (background)
+- Update to latest libraries
+
+## 0.37.1
+
+- Support for Kusama CC3
+- Support for contracts with new ABI v2 (Thanks to https://github.com/kwingram25)
+- Support for on-chain nicks
+- Speed improvements for the staking pages
+- Add account derivation from existing account
+- Council voting with runner up & phragmen
+- Allow favorites in validators pages
+- Rework nominations to take favorites & current into account
+- Enhance AddressCard with additional info (incl. vested)
+- Move account/address actions to popup menu
+- Convert a large number of components to use hooks
+- Display validator graphs
+- Refactor of backup modal (Thanks to @LukeSugiura)
+- Enable language setting options (Thanks to @LukeSugiura)
+- Allow for signRaw to be used in the signing toolbox (injected accounts)
+- Display account names in status events
+- Nomination targets dashboard
+- Validator preferences are expressed as commission % as supported by chains
+- Account locks are applied on a genesis range (e.g. CC2 & CC3 allow availability)
+- ... lots of smaller enhancements & bug fixes
+
+## 0.36.1
 
 - Api 0.95.1, Util 1.6.1, Extension 0.13.1
 - Support latest contracts ABI (via API), incl. rework of contracts UI
@@ -20,7 +374,7 @@
 - Make the network selection clickable on network name (via bounty)
 - ... and a number of cleanups all around
 
-# 0.35.1
+## 0.35.1
 
 - Api 0.91.1, Util 1.2.1, Extension 0.10.1
 - Support for accounts added via Qr (for instance, the Parity Signer)
@@ -33,7 +387,7 @@
 - Fix account derivation with `///password`
 - Lots of component & maintainability cleanups
 
-# 0.34.1
+## 0.34.1
 
 - Kusama support
 - Full support for Substrate 2.x & Polkadot 0.5.0 networks
@@ -42,7 +396,7 @@
 - Basic Council, Parachains & Treasury apps
 - Moved ui-* packages to react-*
 
-# 0.33.1
+## 0.33.1
 
 - Allow for externally injected accounts (i.e. via extension, polkadot-js & SingleSource)
 - Links to extrnisics & addresses on Polkascan
@@ -60,39 +414,39 @@
 - Latest @polkadot/util & @polkadot/api
 - A large number of optimizations and smaller fixes
 
-# 0.32.1
+## 0.32.1
 
 - Support for Substrate 1.0 release & metadata v4
 - @polkadot/api 0.77.1
 
-# 0.31.1
+## 0.31.1
 
 - Cleanups, fixes and features around the poc-4 staking module
 - Number of UI enhancements
 
-# 0.30.1
+## 0.30.1
 
 - Staking page indicator for offline nodes (count & block)
 - Rework page tabs and content layouts
 - Cleanup of all UI summary headers
 - Emberic Elem support (replaces Dried Danta)
 
-# 0.29.1
+## 0.29.1
 
 - @polkadot/util & @polkadot/api 0.75.1
 
-# 0.28.1
+## 0.28.1
 
 - Support for substrate 1.0-rc
 
-# 0.27.1
+## 0.27.1
 
 - Bring in new staking & nominating functions
 - Swap default keyring accounts (on creation) to sr25519
 - New faster crypto algorithms
 - Misc. bug fixes all around
 
-# 0.26.1
+## 0.26.1
 
 - Swap keyring to HDKD derivation, mnemonic keys are now not backwards compatible with those created earlier. (Defaults are still for ed25519)
 - Swap crypto to new WASM-backed version (and remove libsodium dependency)
@@ -100,23 +454,23 @@
 - New mobile-friendly sidebar
 - Fix issues with nominating (old non-bonds interface)
 
-# 0.25.1
+## 0.25.1
 
 - Swap to publishing -beta.x on merge (non-breaking testing)
 
- # 0.24.1
+ ## 0.24.1
 
  Storage now handles Option type properly
 
- # 0.23.1
+ ## 0.23.1
 
  JavaScript console introduced
 
-# 0.22.1
+## 0.22.1
 
 - Use new Compact<Index> transaction format - this requires the latest binaries from either Polkadot or Substrate
 
-# 0.21.1
+## 0.21.1
 
 - PoC-3 support with latest Substrate master & Polkadot master
 - Add support for Charred Cherry (Substrate) and Alexander (Polkadot) testnets

pioneer/I18N.md

@@ -0,0 +1,22 @@
+# I18N
+
+The apps UI allows all strings to be translated. Additionally it has a basic UI that allows for the creation of the required translation files, which will give an overview on the progress for a specific language.
+
+## Updating translations
+
+To update translations, the following process is required.
+
+- launch the apps UI, either locally or via https://polkadot.js.org/apps
+- explicitly navigate to the i18n page, https://polkadot.js.org/apps/#/settings/i18n
+
+Here you will find a dropdown of all the available languages and all the modules that maps to the UI. On a single screen you will be able to see all the available strings for a specific module.
+
+- adjust any strings as required
+- once completed with the changes, click the `Generate translation.json` button to download the translation file
+- this file can now be added to the repo with a PR to https://github.com/polkadot-js/apps/tree/master/packages/apps/public/locales
+
+## Adding a new language (if not in dropdown above)
+
+The process is similar for the above, but does require a new folder with the language identifier to be added. Create [packages/apps/public/locales/<id>](https://github.com/polkadot-js/apps/tree/master/packages/apps/public/locales) folder with an empty `translation.json` (containing only `{}`). After addition of the folder, run `yarn build:i18n` and then the new language will be available for update as per the process in the previous section.
+
+In addition to the language folder, the language also needs to be added to the dropdown for available languages, this can be found in [packages/apps-config/src/settings.languages.ts](https://github.com/polkadot-js/apps/blob/master/packages/apps-config/src/settings/languages.ts)

pioneer/README.md

@@ -11,16 +11,17 @@ This can be accessed as a hosted application via [https://testnet.joystream.org]
 The repo is split into a number of packages, each representing an application. These are -
 
 - [apps](packages/apps/) This is the main entry point. It handles the selection sidebar and routing to the specific application being displayed.
-- [app-accounts](packages/app-accounts/) A basic account management app.
-- [app-address-book](packages/app-address-book/) A basic address management app.
-- [app-explorer](packages/app-explorer/) A simple block explorer. It only shows the most recent blocks, updating as they become available.
-- [app-extrinsics](packages/app-extrinsics/) Submission of extrinsics to a node.
-- [app-js](packages/app-js/) An online code editor with [@polkadot-js/api](https://github.com/polkadot-js/api/tree/master/packages/api) access to the currently connected node.
-- [app-settings](packages/app-settings/) A basic settings management app, allowing choice of language, node to connect to, and theme
-- [app-staking](packages/app-staking/) A basic staking management app, allowing staking and nominations.
-- [app-storage](packages/app-storage/) A simple node storage query application. Multiple queries can be queued and updates as new values become available.
-- [app-toolbox](packages/app-toolbox/) Submission of raw data to RPC endpoints and utility hashing functions.
-- [app-transfer](packages/app-transfer/) A basic account management app, allowing transfer of Units/DOTs between accounts.
+- [apps-electron](packages/apps-electron/) Desktop app running [apps](packages/apps/).
+- [page-accounts](packages/page-accounts/) A basic account management app.
+- [page-address-book](packages/page-address-book/) A basic address management app.
+- [page-explorer](packages/page-explorer/) A simple block explorer. It only shows the most recent blocks, updating as they become available.
+- [page-extrinsics](packages/page-extrinsics/) Submission of extrinsics to a node.
+- [page-js](packages/page-js/) An online code editor with [@polkadot-js/api](https://github.com/polkadot-js/api/tree/master/packages/api) access to the currently connected node.
+- [page-settings](packages/page-settings/) A basic settings management app, allowing choice of language, node to connect to, and theme
+- [page-staking](packages/page-staking/) A basic staking management app, allowing staking and nominations.
+- [page-storage](packages/page-storage/) A simple node storage query application. Multiple queries can be queued and updates as new values become available.
+- [page-toolbox](packages/page-toolbox/) Submission of raw data to RPC endpoints and utility hashing functions.
+- [page-transfer](packages/page-transfer/) A basic account management app, allowing transfer of Units/DOTs between accounts.
 
 In addition the following libraries are also included in the repo. These are to be moved to the [@polkadot/ui](https://github.com/polkadot-js/ui/) repository once it reaches a base level of stability and usability. (At this point with the framework being tested on the apps above, it makes development easier having it close)
 

pioneer/babel.config.js

@@ -1,4 +1,5 @@
-module.exports = {
-  extends: '@polkadot/dev-react/config/babel',
-  sourceType: 'unambiguous'
-};
+// Copyright 2017-2020 @polkadot/apps authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+module.exports = require('@polkadot/dev/config/babel');

pioneer/docker/nginx.conf

@@ -0,0 +1,27 @@
+user  nginx;
+worker_processes  1;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+    gzip  on;
+    include /etc/nginx/conf.d/*.conf;
+}

pioneer/env.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# This script is used when the docker container starts and does the magic to
+# bring the ENV variables to the generated static UI.
+
+TARGET=./env-config.js
+
+# Recreate config file
+echo -n > $TARGET
+
+declare -a vars=(
+  "WS_URL"
+  "SAMPLE"
+)
+
+echo "window.process_env = {" >> $TARGET
+for VAR in ${vars[@]}; do
+  echo "  $VAR: \"${!VAR}\"," >> $TARGET
+done
+echo "}" >> $TARGET

pioneer/i18next-scanner.config.js

@@ -1,7 +1,44 @@
+// Copyright 2017-2020 @polkadot/apps authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
 const fs = require('fs');
 const path = require('path');
 const typescript = require('typescript');
 
+const findPackages = require('./scripts/findPackages');
+
+function transform (file, enc, done) {
+  const { ext } = path.parse(file.path);
+
+  if (ext === '.tsx') {
+    const content = fs.readFileSync(file.path, enc);
+
+    const { outputText } = typescript.transpileModule(content, {
+      compilerOptions: {
+        target: 'es2018'
+      },
+      fileName: path.basename(file.path)
+    });
+
+    const parserHandler = (key, options) => {
+      options.defaultValue = key;
+
+      if (process.platform !== 'win32') {
+        options.ns = /packages\/(.*?)\/src/g.exec(file.path)[1].replace('page-', 'app-');
+      } else {
+        options.ns = /packages\\(.*?)\\src/g.exec(file.path)[1].replace('page-', 'app-');
+      }
+
+      this.parser.set(key, options);
+    };
+
+    this.parser.parseFuncFromString(outputText, parserHandler);
+  }
+
+  done();
+}
+
 module.exports = {
   input: [
     'packages/*/src/**/*.{ts,tsx}',
@@ -10,80 +47,27 @@ module.exports = {
     '!packages/*/src/i18n/**',
     '!**/node_modules/**'
   ],
-  output: './',
   options: {
     debug: true,
+    defaultLng: 'en',
     func: {
-      list: ['t', 'i18next.t', 'i18n.t'],
-      extensions: ['.tsx']
-    },
-    trans: {
-      component: 'Trans'
+      extensions: ['.tsx', '.ts'],
+      list: ['t', 'i18next.t', 'i18n.t']
     },
+    keySeparator: false, // key separator
     lngs: ['en'],
-    defaultLng: 'en',
-    ns: [
-      'app-123code',
-      'app-accounts',
-      'app-address-book',
-      'app-claims',
-      'app-contracts',
-      'app-council',
-      'app-dashboard',
-      'app-democracy',
-      'app-explorer',
-      'app-extrinsics',
-      'app-generic-asset',
-      'app-js',
-      'app-parachains',
-      'app-settings',
-      'app-staking',
-      'app-storage',
-      'app-sudo',
-      'app-toolbox',
-      'app-transfer',
-      'app-treasury',
-      'apps',
-      'apps-routing',
-      'react-api',
-      'react-components',
-      'react-params',
-      'react-query',
-      'react-signer',
-      'ui'
-    ],
-    defaultNs: 'ui',
+    ns: findPackages().map(({ dir }) => dir.replace('page-', 'app-')),
+    nsSeparator: false, // namespace separator
     resource: {
-      loadPath: 'packages/apps/public/locales/{{lng}}/{{ns}}.json',
-      savePath: 'packages/apps/public/locales/{{lng}}/{{ns}}.json',
       jsonIndent: 2,
-      lineEnding: '\n'
+      lineEnding: '\n',
+      loadPath: 'packages/apps/public/locales/{{lng}}/{{ns}}.json',
+      savePath: 'packages/apps/public/locales/{{lng}}/{{ns}}.json'
     },
-    nsSeparator: false, // namespace separator
-    keySeparator: false // key separator
-  },
-  transform: function transform (file, enc, done) {
-    const { ext } = path.parse(file.path);
-
-    if (ext === '.tsx') {
-      const content = fs.readFileSync(file.path, enc);
-
-      const { outputText } = typescript.transpileModule(content, {
-        compilerOptions: {
-          target: 'es2018'
-        },
-        fileName: path.basename(file.path)
-      });
-
-      const parserHandler = (key, options) => {
-        options.defaultValue = key;
-        options.ns = /packages\/(.*?)\/src/g.exec(file.path)[1];
-        this.parser.set(key, options);
-      };
-
-      this.parser.parseFuncFromString(outputText, parserHandler);
+    trans: {
+      component: 'Trans'
     }
-
-    done();
-  }
+  },
+  output: './',
+  transform
 };

pioneer/jest.config.js

@@ -1,17 +1,26 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
-const config = require('@polkadot/dev-react/config/jest');
+// Copyright 2017-2020 @polkadot/apps authors & contributors
+// This software may be modified and distributed under the terms
+// of the Apache-2.0 license. See the LICENSE file for details.
+
+const config = require('@polkadot/dev/config/jest');
+
 const findPackages = require('./scripts/findPackages');
 
-const internalModules = findPackages().reduce((modules, { dir, name }) => {
-  modules[`${name}(.*)$`] = `<rootDir>/packages/${dir}/src/$1`;
+const internalModules = findPackages()
+  .filter(({ name }) => !['@polkadot/apps'].includes(name))
+  .reduce((modules, { dir, name }) => {
+    modules[`${name}(.*)$`] = `<rootDir>/packages/${dir}/src/$1`;
 
-  return modules;
-}, {});
+    return modules;
+  }, {});
 
 module.exports = Object.assign({}, config, {
   moduleNameMapper: {
     ...internalModules,
-    '\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$': 'empty/object',
-    '\\.(css|less)$': 'empty/object'
-  }
+    '\\.(css|less)$': 'empty/object',
+    '\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$': 'empty/object'
+  },
+  transformIgnorePatterns: [
+    '<rootDir>/node_modules'
+  ]
 });

pioneer/lerna.json

@@ -10,5 +10,5 @@
   "packages": [
     "packages/*"
   ],
-  "version": "0.37.0-beta.63"
+  "version": "0.51.1"
 }

pioneer/package.json

@@ -1,5 +1,6 @@
 {
-  "version": "0.37.0-beta.63",
+  "version": "0.51.1",
+  "license": "Apache-2",
   "private": true,
   "engines": {
     "node": ">=12.18.0",
@@ -8,70 +9,79 @@
   "homepage": ".",
   "name": "pioneer",
   "scripts": {
-    "analyze": "yarn run build && cd packages/apps && yarn run source-map-explorer build/main.*.js",
-    "build": "yarn run build:code && yarn run build:i18n",
-    "build:code": "NODE_ENV=production polkadot-dev-build-ts",
-    "build:i18n": "i18next-scanner --config i18next-scanner.config.js",
+    "analyze": "yarn clean && BUILD_ANALYZE=1 yarn run build:code && yarn source-map-explorer packages/apps/build/main.*.js",
+    "build": "yarn run build:i18n && yarn run build:code",
+    "build:code": "NODE_ENV=production node ./scripts/dev-build-ts.js",
+    "build:i18n": "i18next-scanner --config i18next-scanner.config.js && node ./scripts/i18nSort.js",
+    "build:www": "rm -rf packages/apps/build && mkdir -p packages/apps/build && yarn run build:i18n && cd packages/apps && NODE_ENV=production webpack --config webpack.config.js",
     "docs": "echo \"skipping docs\"",
     "clean": "polkadot-dev-clean-build",
     "clean:i18n": "rm -rf packages/apps/public/locales/en && mkdir -p packages/apps/public/locales/en",
     "lint": "eslint --ext .js,.jsx,.ts,.tsx . && tsc --noEmit --pretty",
+    "lint:css": "stylelint './packages/**/src/**/*.tsx'",
     "lint-only-errors": "eslint --quiet --ext .js,.jsx,.ts,.tsx . && tsc --noEmit --pretty",
     "lint-autofix": "eslint --fix --ext .js,.jsx,.ts,.tsx . && tsc --noEmit --pretty",
     "postinstall": "polkadot-dev-yarn-only",
     "test": "echo \"skipping tests\"",
     "vanitygen": "node packages/app-accounts/scripts/vanitygen.js",
-    "start": "cd packages/apps && webpack --config webpack.config.js",
-    "generate-schemas": "json2ts -i packages/joy-types/src/schemas/role.schema.json -o packages/joy-types/src/schemas/role.schema.ts",
+    "start": "yarn clean && cd packages/apps && webpack --config webpack.config.js",
     "build-storybook": "build-storybook -c .storybook",
     "storybook": "start-storybook -s ./packages/apps/public -p 3001"
   },
   "devDependencies": {
-    "@babel/core": "^7.7.0",
-    "@babel/runtime": "^7.7.1",
-    "@babel/cli": "^7.7.4",
-    "@polkadot/dev-react": "^0.32.0-beta.13",
-    "@polkadot/ts": "^0.1.84",
-    "@polkadot/dev": "^0.32.0-beta.15",
+    "@babel/core": "^7.10.5",
+    "@babel/register": "^7.10.5",
+    "@babel/runtime": "^7.10.5",
+    "@pinata/sdk": "^1.1.10",
+    "@polkadot/dev": "^0.55.28",
+    "@polkadot/ts": "^0.3.29",
+    "@types/bn.js": "^4.11.6",
+    "@types/chart.js": "^2.9.23",
+    "@types/file-saver": "^2.0.1",
+    "@types/i18next": "^13.0.0",
+    "@types/jest": "^26.0.10",
+    "@types/react-beautiful-dnd": "^13.0.0",
+    "@types/react-copy-to-clipboard": "^4.3.0",
+    "@types/react-dom": "^16.9.8",
+    "@types/react-router-dom": "^5.1.5",
+    "@types/react-tooltip": "^4.2.4",
+    "@types/store": "^2.0.2",
+    "@types/styled-components": "^5.1.1",
+    "@types/styled-theming": "^2.2.4",
+    "concurrently": "^5.2.0",
+    "devtron": "^1.4.0",
+    "dnslink-cloudflare": "^2.0.4",
+    "electron": "^9.1.1",
+    "electron-builder": "^22.8.0",
+    "electron-builder-notarize": "^1.2.0",
+    "i18next-scanner": "^2.11.0",
+    "react": "^16.13.1",
+    "react-dom": "^16.13.1",
+    "source-map-explorer": "^2.4.2",
+    "stylelint": "^13.6.1",
+    "stylelint-config-recommended": "^3.0.0",
+    "stylelint-config-styled-components": "^0.1.1",
+    "terser-webpack-plugin": "^3.0.7",
+    "webpack": "^4.44.0",
+    "webpack-cli": "^3.3.12",
+    "webpack-merge": "^4.2.2",
+    "webpack-plugin-serve": "^1.0.1",
+    "//": "Joystream-specific",
+    "react-i18next": "^11.7.0",
     "@storybook/addon-knobs": "^5.2.5",
     "@storybook/addon-storysource": "^5.2.5",
-    "@types/jest": "^24.0.22",
-    "@types/react-router-dom": "^5.1.4",
-    "@types/yup": "^0.26.36",
-    "autoprefixer": "^9.7.1",
-    "empty": "^0.10.1",
-    "html-loader": "^0.5.5",
-    "i18next-scanner": "^2.10.3",
-    "json-schema-to-typescript": "^7.1.0",
-    "markdown-loader": "^5.1.0",
-    "postcss": "^7.0.21",
-    "postcss-clean": "^1.1.0",
-    "postcss-flexbugs-fixes": "^4.1.0",
-    "postcss-import": "^12.0.0",
-    "postcss-loader": "^3.0.0",
-    "postcss-nested": "^4.2.1",
-    "postcss-sass": "^0.4.1",
-    "postcss-simple-vars": "^5.0.0",
-    "precss": "^4.0.0",
-    "source-map-explorer": "^2.0.1",
-    "storybook-react-router": "^1.0.8",
-    "ts-jest": "^24.1.0",
-    "tsconfig-paths-webpack-plugin": "^3.2.0",
-    "webpack": "^4.33.0",
-    "typescript": "3.7.2",
-    "cpx": "^1.5.0",
-    "eslint-config-semistandard": "^15.0.0",
-    "eslint-config-standard": "^14.1.1",
-    "eslint-plugin-import": "^2.20.2",
-    "eslint-plugin-node": "^11.1.0",
-    "eslint-plugin-promise": "^4.2.1",
-    "eslint-plugin-standard": "^4.0.1"
-  },
-  "dependencies": {
-    "@polkadot/ui-settings": "^0.47.0-beta.3",
     "@storybook/addon-actions": "^5.2.5",
     "@storybook/addon-console": "^1.2.1",
     "@storybook/react": "^5.2.5",
+    "storybook-react-router": "^1.0.8",
+    "typescript": "^3.9.7",
+    "eslint-plugin-header": "^3.0.0",
+    "eslint-plugin-sort-destructure-keys": "^1.3.5",
+    "jest": "^26.4.1",
+    "ts-jest": "^26.2.0",
+    "tsconfig-paths-webpack-plugin": "^3.2.0"
+  },
+  "dependencies": {
     "@types/lodash": "^4.14.138",
     "@types/marked": "^0.7.0",
     "ajv": "^6.10.2",

pioneer/packages/app-123code/README.md

@@ -1,22 +0,0 @@
-# @polkadot/app-123code
-
-A simple template to get started with adding an "app" to this UI. It contains the bare minimum for a nicely hackable app (if you just want to code _somewhere_) and the steps needed to create, add and register an new app that appears in the UI.
-
-## adding an app
-
-If you want to add a new app to the UI, this is the place to start.
-
-1. Duplicate this `app-123code` folder and give it an appropriate name, in this case we will select `app-example` to keep things clear.
-2. Edit the `apps-example/package.json` app description, i.e. the name, author and relevant overview.
-
-And we have the basic app source setup, time to get the tooling correct.
-
-3. Add the new app to the TypeScript config in root, `tsconfig.json`, i.e. an entry such as `"@polkadot/app-example/*": [ "packages/app-example/src/*" ],`
-
-At this point the app should be buildable, but not quite reachable. The final step is to add it to the actual sidebar in `apps`.
-
-4. In `apps-routing/src` duplicate the `123code.ts` file to `example.ts` and edit it with the appropriate information, including the hash link, name and icon (any icon name from semantic-ui-react/font-awesome 4 should be appropriate).
-5. In the above description file, the `isHidden` field needs to be toggled to make it appear - the base template is hidden by default.
-6. Finally add the `template` to the `apps-routing/src/index.ts` file at the appropriate place for both full and light mode (either optional)
-
-Yes. After all that we have things hooked up. Run `yarn start` and your new app (non-coded) should show up. Now start having fun and building something great.

pioneer/packages/app-123code/package.json

@@ -1,16 +0,0 @@
-{
-  "name": "@polkadot/app-123code",
-  "version": "0.37.0-beta.63",
-  "description": "A basic app that shows the ropes on customisation",
-  "main": "index.js",
-  "scripts": {},
-  "author": "Jaco Greeff <jacogr@gmail.com>",
-  "maintainers": [
-    "Jaco Greeff <jacogr@gmail.com>"
-  ],
-  "license": "Apache-2.0",
-  "dependencies": {
-    "@babel/runtime": "^7.7.1",
-    "@polkadot/react-components": "^0.37.0-beta.63"
-  }
-}

pioneer/packages/app-123code/src/AccountSelector.tsx

@@ -1,49 +0,0 @@
-// Copyright 2017-2019 @polkadot/app-123code authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import React, { useEffect, useState } from 'react';
-import styled from 'styled-components';
-import { Bubble, InputAddress } from '@polkadot/react-components';
-import { AccountIndex, Balance, Nonce } from '@polkadot/react-query';
-
-interface Props {
-  className?: string;
-  onChange: (accountId: string | null) => void;
-}
-
-function AccountSelector ({ className, onChange }: Props): React.ReactElement<Props> {
-  const [accountId, setAccountId] = useState<string | null>(null);
-
-  useEffect((): void => onChange(accountId), [accountId]);
-
-  return (
-    <section className={`template--AccountSelector ui--row ${className}`}>
-      <InputAddress
-        className='medium'
-        label='my default account'
-        onChange={setAccountId}
-        type='account'
-      />
-      <div className='medium'>
-        <Bubble color='teal' icon='address card' label='index'>
-          <AccountIndex params={accountId} />
-        </Bubble>
-        <Bubble color='yellow' icon='adjust' label='balance'>
-          <Balance params={accountId} />
-        </Bubble>
-        <Bubble color='yellow' icon='target' label='transactions'>
-          <Nonce params={accountId} />
-        </Bubble>
-      </div>
-    </section>
-  );
-}
-
-export default styled(AccountSelector)`
-  align-items: flex-end;
-
-  .summary {
-    text-align: center;
-  }
-`;

pioneer/packages/app-123code/src/Summary.tsx

@@ -1,28 +0,0 @@
-// Copyright 2017-2019 @polkadot/app-123code authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { BareProps } from '@polkadot/react-components/types';
-
-import React from 'react';
-import styled from 'styled-components';
-
-interface Props extends BareProps {
-  children: React.ReactNode;
-}
-
-function Summary ({ children, className, style }: Props): React.ReactElement<Props> {
-  return (
-    <div
-      className={className}
-      style={style}
-    >
-      {children}
-    </div>
-  );
-}
-
-export default styled(Summary)`
-  opacity: 0.5;
-  padding: 1rem 1.5rem;
-`;

pioneer/packages/app-123code/src/SummaryBar.tsx

@@ -1,65 +0,0 @@
-/* eslint-disable @typescript-eslint/camelcase */
-// Copyright 2017-2019 @polkadot/app-123code authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { AccountId } from '@polkadot/types/interfaces';
-import { BareProps, I18nProps } from '@polkadot/react-components/types';
-
-import BN from 'bn.js';
-import React, { useContext } from 'react';
-import { ApiContext, withCalls } from '@polkadot/react-api';
-import { Bubble, IdentityIcon } from '@polkadot/react-components';
-import { formatBalance, formatNumber } from '@polkadot/util';
-
-import translate from './translate';
-
-interface Props extends BareProps, I18nProps {
-  balances_totalIssuance?: BN;
-  chain_bestNumber?: BN;
-  chain_bestNumberLag?: BN;
-  staking_validators?: AccountId[];
-}
-
-function SummaryBar ({ balances_totalIssuance, chain_bestNumber, chain_bestNumberLag, staking_validators }: Props): React.ReactElement<Props> {
-  const { api, systemChain, systemName, systemVersion } = useContext(ApiContext);
-
-  return (
-    <summary>
-      <div>
-        <Bubble icon='tty' label='node'>
-          {systemName} v{systemVersion}
-        </Bubble>
-        <Bubble icon='chain' label='chain'>
-          {systemChain}
-        </Bubble>
-        <Bubble icon='code' label='runtime'>
-          {api.runtimeVersion.implName} v{api.runtimeVersion.implVersion}
-        </Bubble>
-        <Bubble icon='bullseye' label='best #'>
-          {formatNumber(chain_bestNumber)} ({formatNumber(chain_bestNumberLag)} lag)
-        </Bubble>
-        {staking_validators && (
-          <Bubble icon='chess queen' label='validators'>{
-            staking_validators.map((accountId, index): React.ReactNode => (
-              <IdentityIcon key={index} value={accountId} size={20} />
-            ))
-          }</Bubble>
-        )}
-        <Bubble icon='circle' label='total tokens'>
-          {formatBalance(balances_totalIssuance)}
-        </Bubble>
-      </div>
-    </summary>
-  );
-}
-
-// inject the actual API calls automatically into props
-export default translate(
-  withCalls<Props>(
-    'derive.chain.bestNumber',
-    'derive.chain.bestNumberLag',
-    'derive.staking.validators',
-    'query.balances.totalIssuance'
-  )(SummaryBar)
-);

pioneer/packages/app-123code/src/Transfer.tsx

@@ -1,47 +0,0 @@
-// Copyright 2017-2019 @polkadot/app-123code authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import BN from 'bn.js';
-import React, { useState } from 'react';
-import { Button, InputAddress, InputBalance, TxButton } from '@polkadot/react-components';
-
-import Summary from './Summary';
-
-interface Props {
-  accountId?: string | null;
-}
-
-export default function Transfer ({ accountId }: Props): React.ReactElement<Props> {
-  const [amount, setAmount] = useState<BN | undefined | null>(null);
-  const [recipientId, setRecipientId] = useState<string | null>(null);
-
-  return (
-    <section>
-      <h1>transfer</h1>
-      <div className='ui--row'>
-        <div className='large'>
-          <InputAddress
-            label='recipient address for this transfer'
-            onChange={setRecipientId}
-            type='all'
-          />
-          <InputBalance
-            label='amount to transfer'
-            onChange={setAmount}
-          />
-          <Button.Group>
-            <TxButton
-              accountId={accountId}
-              icon='send'
-              label='make transfer'
-              params={[recipientId, amount]}
-              tx='balances.transfer'
-            />
-          </Button.Group>
-        </div>
-        <Summary className='small'>Make a transfer from any account you control to another account. Transfer fees and per-transaction fees apply and will be calculated upon submission.</Summary>
-      </div>
-    </section>
-  );
-}

pioneer/packages/app-123code/src/index.tsx

@@ -1,38 +0,0 @@
-// Copyright 2017-2019 @polkadot/app-123code authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-// some types, AppProps for the app and I18nProps to indicate
-// translatable strings. Generally the latter is quite "light",
-// `t` is inject into props (see the HOC export) and `t('any text')
-// does the translation
-import { AppProps, I18nProps } from '@polkadot/react-components/types';
-
-// external imports (including those found in the packages/*
-// of this repo)
-import React, { useState } from 'react';
-
-// local imports and components
-import AccountSelector from './AccountSelector';
-import SummaryBar from './SummaryBar';
-import Transfer from './Transfer';
-import translate from './translate';
-
-// define our internal types
-interface Props extends AppProps, I18nProps {}
-
-function App ({ className }: Props): React.ReactElement<Props> {
-  const [accountId, setAccountId] = useState<string | null>(null);
-
-  return (
-    // in all apps, the main wrapper is setup to allow the padding
-    // and margins inside the application. (Just from a consistent pov)
-    <main className={className}>
-      <SummaryBar />
-      <AccountSelector onChange={setAccountId} />
-      <Transfer accountId={accountId} />
-    </main>
-  );
-}
-
-export default translate(App);

pioneer/packages/app-123code/src/translate.ts

@@ -1,7 +0,0 @@
-// Copyright 2017-2019 @polkadot/app-123code authors & contributors
-// This software may be modified and distributed under the terms
-// of the Apache-2.0 license. See the LICENSE file for details.
-
-import { withTranslation } from 'react-i18next';
-
-export default withTranslation(['app-123code']);