Merge pull request #92 from mnaamani/upgrade-libp2p

Rome + upgrade libp2p to v0.13.0

.dockerignore

@@ -0,0 +1 @@
+target/

.travis.yml

@@ -0,0 +1,44 @@
+language: rust
+
+rust:
+  - 1.41.1
+
+cache:
+  cargo
+
+os:
+  - linux
+  - osx
+  - windows
+
+before_script:
+  - if [ "$TRAVIS_OS_NAME" = "osx" ]; then export TARGET=x86_64-apple-darwin  ; fi
+  - if [ "$TRAVIS_OS_NAME" = "linux" ]; then export TARGET=x86_64-unknown-linux-gnu  ; fi
+  - if [ "$TRAVIS_OS_NAME" = "windows" ]; then export TARGET=x86_64-pc-windows-msvc  ; fi
+  - rustup component add rustfmt
+  - rustup update nightly
+  - rustup target add ${TARGET}
+  - rustup target add wasm32-unknown-unknown --toolchain nightly
+  - rustup show
+  - cat ~/.cargo/config
+
+script:
+  - cargo fmt --all -- --check
+  - cargo build --release --target=${TARGET}
+  - mv ./target/${TARGET}/release/joystream-node .
+  - export FILENAME=`./joystream-node --version | sed -e "s/ /-/g"`
+  - tar -cf ${FILENAME}.tar ./joystream-node
+  - gzip ${FILENAME}.tar
+
+deploy:
+  provider: releases
+  api_key:
+    secure: icPiO8fnOgkHlo2MpUJvH+sZSaxJqzHif/1Pndlxg2jM/jrvLbUCr9UvtLaRQYPi2BVnR41t2m56uMJD3LktKMzAwUP74JcdX7iw6tq+iyvuU4UBsn5+k8pgue+eE34PVN0vNbDS+RVzzc6wG/vhUyKNKz6M1pOuEBx0s6UX3oED16W1LEN8/CHmFVBxQehlGv40Mje6tSmfZMw++efAmFm9vd0Fx8ub8as5ZqyjGv2IwbQlyIGT3SF9yx2va90F2wOxyZM7wgWzA/XIzlDGy+zvZssEYNGQhepUdGwTwwe+YLvZWuttTC218K4H8aireO2vCMti58veX7hVuDz6j6ETVxhr6Nv47HfxQF4IywDTUH1JGlLVadgcKQXUlVRrfy1tBKDq76ppMul8TdS0LgojeTgcJOblKw0V5TMwx+IYmMabVIZQ7Q7xKqf4SeXxh/YL3EUWBvzu1TUn946CB3jLATUxjGyjAZ00BN01QL7chrkcewInlKxJ/wjUqM34KnHNXXgPNADW+c5IIsl82BwnPsPK37WMfW5FaKtOn0YXQWj9QwtEX7VakNnPZZRvpsl2Ftd1As7hPgufMk8NbXmiQH8i0h1W0xcHQ38hvegIpf1fa8JA7VIFBP6ElymMTIus5xD5rsLGzPqgazm/328p2DthJWebyLAZIbEX+I4=
+  file: ${FILENAME}.tar.gz
+  skip_cleanup: true
+  on:
+    tags: true
+    branch: development
+  draft: true
+  prerelease: true
+  overwrite: true

Cargo.toml

@@ -7,86 +7,151 @@ authors = ['Joystream']
 build = 'build.rs'
 edition = '2018'
 name = 'joystream-node'
-version = '1.0.0'
+version = '2.1.2'
 
 [dependencies]
-error-chain = '0.12'
-exit-future = '0.1'
-futures = '0.1'
-hex-literal = '0.1'
-log = '0.4'
-parity-codec = '3.2'
-parking_lot = '0.7.1'
-tokio = '0.1'
-trie-root = '0.12.0'
-
-[dependencies.basic-authorship]
-git = 'https://github.com/joystream/substrate.git'
+hex-literal = '0.2.1'
+derive_more = '0.14.0'
+exit-future = '0.1.4'
+futures = '0.1.29'
+log = '0.4.8'
+parking_lot = '0.9.0'
+tokio = '0.1.22'
+jsonrpc-core = '13.2.0'
+rand = '0.7.2'
+structopt = '=0.3.5'
+serde_json = '1.0'
+serde = '1.0'
+hex = '0.4'
+# https://users.rust-lang.org/t/failure-derive-compilation-error/39062
+# quote = '<=1.0.2'
+
+[dependencies.node-runtime]
+package = 'joystream-node-runtime'
+git = 'https://github.com/joystream/substrate-runtime-joystream'
+tag = 'v6.8.0'
+# rev = '620094ef5f393180284aab2e5516f854694f009b' # development branch: v6.8.0
+# branch = 'development'
+# local development...
+# path = '/Users/mokhtar/joystream/runtime'
+
+[dependencies.substrate-basic-authorship]
+git = 'https://github.com/paritytech/substrate.git'
 package = 'substrate-basic-authorship'
-rev = '6dfc3e8b057bb00322136251a0f10305fbb1ad8f'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
+
+[dependencies.babe]
+git = 'https://github.com/paritytech/substrate.git'
+package = 'substrate-consensus-babe'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
 
-[dependencies.consensus]
-git = 'https://github.com/joystream/substrate.git'
-package = 'substrate-consensus-aura'
-rev = '6dfc3e8b057bb00322136251a0f10305fbb1ad8f'
+[dependencies.babe-primitives]
+git = 'https://github.com/paritytech/substrate.git'
+package = 'substrate-consensus-babe-primitives'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
+
+[dependencies.codec]
+package = 'parity-scale-codec'
+version = '1.0.0'
 
 [dependencies.ctrlc]
 features = ['termination']
 version = '3.0'
 
 [dependencies.inherents]
-git = 'https://github.com/joystream/substrate.git'
+git = 'https://github.com/paritytech/substrate.git'
 package = 'substrate-inherents'
-rev = '6dfc3e8b057bb00322136251a0f10305fbb1ad8f'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
 
 [dependencies.network]
-git = 'https://github.com/joystream/substrate.git'
+git = 'https://github.com/paritytech/substrate.git'
 package = 'substrate-network'
-rev = '6dfc3e8b057bb00322136251a0f10305fbb1ad8f'
-
-[dependencies.joystream-node-runtime]
-# clone https://github.com/joystream/substrate-runtime-joystream to this path:
-path = 'substrate-runtime-joystream'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
 
 [dependencies.primitives]
-git = 'https://github.com/joystream/substrate.git'
+git = 'https://github.com/paritytech/substrate.git'
 package = 'substrate-primitives'
-rev = '6dfc3e8b057bb00322136251a0f10305fbb1ad8f'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
 
 [dependencies.sr-io]
-git = 'https://github.com/joystream/substrate.git'
-rev = '6dfc3e8b057bb00322136251a0f10305fbb1ad8f'
+git = 'https://github.com/paritytech/substrate.git'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
 
 [dependencies.substrate-cli]
-git = 'https://github.com/joystream/substrate.git'
-rev = '6dfc3e8b057bb00322136251a0f10305fbb1ad8f'
+git = 'https://github.com/paritytech/substrate.git'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
 
 [dependencies.substrate-client]
-git = 'https://github.com/joystream/substrate.git'
-rev = '6dfc3e8b057bb00322136251a0f10305fbb1ad8f'
+git = 'https://github.com/paritytech/substrate.git'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
 
 [dependencies.substrate-executor]
-git = 'https://github.com/joystream/substrate.git'
-rev = '6dfc3e8b057bb00322136251a0f10305fbb1ad8f'
+git = 'https://github.com/paritytech/substrate.git'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
 
 [dependencies.substrate-service]
-git = 'https://github.com/joystream/substrate.git'
-rev = '6dfc3e8b057bb00322136251a0f10305fbb1ad8f'
+git = 'https://github.com/paritytech/substrate.git'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
 
 [dependencies.transaction-pool]
-git = 'https://github.com/joystream/substrate.git'
+git = 'https://github.com/paritytech/substrate.git'
 package = 'substrate-transaction-pool'
-rev = '6dfc3e8b057bb00322136251a0f10305fbb1ad8f'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
 
 [dependencies.substrate-telemetry]
-git = 'https://github.com/joystream/substrate.git'
+git = 'https://github.com/paritytech/substrate.git'
 package = 'substrate-telemetry'
-rev = '6dfc3e8b057bb00322136251a0f10305fbb1ad8f'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
 
 [dependencies.grandpa]
-git = 'https://github.com/joystream/substrate.git'
+git = 'https://github.com/paritytech/substrate.git'
 package = 'substrate-finality-grandpa'
-rev = '6dfc3e8b057bb00322136251a0f10305fbb1ad8f'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
+
+[dependencies.grandpa-primitives]
+git = 'https://github.com/paritytech/substrate.git'
+package = 'substrate-finality-grandpa-primitives'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
+
+[dependencies.im-online]
+default_features = false
+git = 'https://github.com/paritytech/substrate.git'
+package = 'srml-im-online'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
+
+[dependencies.substrate-rpc]
+default_features = false
+git = 'https://github.com/paritytech/substrate.git'
+package = 'substrate-rpc'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
+
+[dependencies.authority-discovery]
+default_features = false
+git = 'https://github.com/paritytech/substrate.git'
+package = 'substrate-authority-discovery'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
+
+[dependencies.client-db]
+default_features = false
+git = 'https://github.com/paritytech/substrate.git'
+package = 'substrate-client-db'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
+
+[dependencies.runtime-primitives]
+default_features = false
+git = 'https://github.com/paritytech/substrate.git'
+package = 'sr-primitives'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
+
+[dependencies.offchain]
+default_features = false
+git = 'https://github.com/paritytech/substrate.git'
+package = 'substrate-offchain'
+rev = 'c37bb08535c49a12320af7facfd555ce05cce2e8'
+
+[dependencies.libp2p]
+version = '0.13.2'
+default-features = false
 
 [profile.release]
 panic = 'unwind'

Dockerfile

@@ -0,0 +1,26 @@
+FROM joystream/rust-builder AS builder
+LABEL description="compiles and caches dependencies, artifacts and node"
+WORKDIR /joystream
+COPY . /joystream
+
+RUN cargo build --release
+
+FROM debian:stretch
+LABEL description="Joystream node"
+WORKDIR /joystream
+COPY --from=builder /joystream/target/release/joystream-node /joystream/node
+
+# confirm it works
+RUN /joystream/node --version
+
+EXPOSE 30333 9933 9944
+
+# Use these volumes to persits chain state and keystore, eg.:
+# --base-path /data
+# optionally separate keystore (otherwise it will be stored in the base path)
+# --keystore-path /keystore
+# if base-path isn't specified, chain state is stored inside container in ~/.local/share/joystream-node/
+# which is not ideal
+VOLUME ["/data", "/keystore"]
+
+ENTRYPOINT ["/joystream/node"]

Dockerfile_experimental

@@ -0,0 +1,30 @@
+# syntax=docker/dockerfile:experimental
+# must enable experimental features in docker daemon and set DOCKER_BUILDKIT=1 env variable
+# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/experimental.md
+FROM joystream/rust-builder AS builder
+LABEL description="compiles and caches dependencies, artifacts and node"
+WORKDIR /joystream
+COPY . /joystream
+RUN mkdir /build-output
+
+RUN --mount=type=cache,target=/joystream/target \
+    --mount=type=cache,target=/root/.cargo/git \
+    --mount=type=cache,target=/root/.cargo/registry \
+    cargo build --release \
+    && cp ./target/release/joystream-node /build-output/joystream-node
+# copy in last part could be done with nightly option --out-dir
+
+FROM debian:stretch
+LABEL description="Joystream node"
+WORKDIR /joystream
+COPY --from=builder /build-output/joystream-node /joystream/node
+
+# Use these volumes to persits chain state and keystore, eg.:
+# --base-path /data
+# optionally separate keystore (otherwise it will be stored in the base path)
+# --keystore-path /keystore
+# if base-path isn't specified, chain state is stored inside container in ~/.local/share/joystream-node/
+# which is not ideal
+VOLUME ["/data", "/keystore"]
+
+ENTRYPOINT ["/joystream/node"]

README.md

@@ -13,37 +13,24 @@ Downloads are available in [releases](https://github.com/Joystream/substrate-nod
 ## Building from source
 
 ### Initial setup
-If you want to build from source you will need the Rust [toolchain](https://rustup.rs/), openssl and llvm/libclang.
+If you want to build from source you will need the Rust [toolchain](https://rustup.rs/), openssl and llvm/libclang. You can install the required dependencies with:
 
 ```bash
 git clone https://github.com/Joystream/substrate-node-joystream.git
-```
-
-Initialise the WASM build environment:
-
-```bash
 cd substrate-node-joystream/
-./init-wasm.sh
+./setup.sh
 ```
 
-### Building
-Clone the joystream runtime into the substrate-node-joystream directory:
-
-```bash
-git clone https://github.com/Joystream/substrate-runtime-joystream.git
-```
+If you prefer to use docker see [building with docker](Docker).
 
-Build the WASM runtime library:
-```bash
-./build-runtime.sh
-```
+### Building
 
-Build the node (native code):
 ```bash
 cargo build --release
 ```
 
 ### Running a public node
+
 Run the node and connect to the public testnet
 ```bash
 cargo run --release
@@ -77,6 +64,36 @@ When making changes to the runtime library remember to purge the chain after reb
 cargo run --release -- purge-chain --dev
 ```
 
-### Substrate node template
+### Docker
+
+#### Building localy
+
+A joystream-node can be compiled with give [Dockerfile](Dockerfile) file:
+
+```bash
+# Build and tag a new image, which will compile joystream-node from source
+docker build . -t joystream-node
+
+# run a development chain with the image just created publishing the websocket port
+docker run -p 9944:9944 joystream-node --dev --ws-external
+```
+
+#### Downloading joystream pre-built images from Docker Hub
+
+```bash
+docker pull joystream/node
+```
+
+#### Running a public node as a service
+
+```bash
+docker run -d -p 30333:30333 --name my-node joystream/node
+
+# check status
+docker ps
+
+# monitor logs
+docker logs --tail 100 -f my-node
+```
 
-The full node is built based on the substrate node [template](https://github.com/shawntabrizi/substrate-package/tree/master/substrate-node-template)
+[More advanced guide]()

build-clean-start.sh

@@ -1,6 +1,5 @@
 #!/usr/bin/env bash
 
-./build-runtime.sh
 cargo build
 cargo run -- purge-chain --dev
 cargo run -- --dev

build-runtime.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-PROJECT_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-
-export CARGO_INCREMENTAL=0
-
-bold=$(tput bold)
-normal=$(tput sgr0)
-
-# Save current directory.
-pushd . >/dev/null
-
-for SRC in substrate-runtime-joystream/
-do
-  echo "${bold}Building webassembly binary in $SRC...${normal}"
-  cd "$PROJECT_ROOT/$SRC"
-
-  ./build.sh
-
-  cd - >> /dev/null
-done
-
-# Restore initial directory.
-popd >/dev/null

build.rs

@@ -1,8 +1,27 @@
+use std::{env, path::PathBuf};
+
 use vergen::{generate_cargo_keys, ConstantsFlags};
 
-const ERROR_MSG: &'static str = "Failed to generate metadata files";
+const ERROR_MSG: &str = "Failed to generate metadata files";
 
 fn main() {
-    generate_cargo_keys(ConstantsFlags::all()).expect(ERROR_MSG);
-    println!("cargo:rerun-if-changed=.git/HEAD");
+    generate_cargo_keys(ConstantsFlags::SHA_SHORT).expect(ERROR_MSG);
+
+    let mut manifest_dir = PathBuf::from(
+        env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is always set by cargo."),
+    );
+
+    while manifest_dir.parent().is_some() {
+        if manifest_dir.join(".git/HEAD").exists() {
+            println!(
+                "cargo:rerun-if-changed={}",
+                manifest_dir.join(".git/HEAD").display()
+            );
+            return;
+        }
+
+        manifest_dir.pop();
+    }
+
+    println!("cargo:warning=Could not find `.git/HEAD` from manifest dir!");
 }

raspberry-cross-build.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+### Cross build for Raspberry Pi - using docker ###
+docker pull joystream/rust-raspberry
+
+docker run \
+    --volume ${PWD}/:/home/cross/project \
+    --volume ${HOME}/.cargo/registry:/home/cross/.cargo/registry \
+    joystream/rust-raspberry \
+    build --release
+
+# output will be in project folder:
+# target/arm-unknown-linux-gnueabihf/joystream-node

res/dummy.json

@@ -0,0 +1 @@
+{}

res/forum_data_empty.json

@@ -0,0 +1 @@
+{ "categories":[], "posts":[], "threads":[] }

init-wasm.sh → setup.sh

@@ -2,6 +2,9 @@
 
 set -e
 
+# Install OS dependencies
+curl https://getsubstrate.io -sSf | bash -s -- --fast
+
 echo "*** Initialising WASM build environment"
 
 if [ -z $CI_PROJECT_NAME ] ; then
@@ -13,4 +16,4 @@ rustup target add wasm32-unknown-unknown --toolchain nightly
 
 # Install wasm-gc. It's useful for stripping slimming down wasm binaries.
 command -v wasm-gc || \
-	cargo +nightly install --git https://github.com/alexcrichton/wasm-gc --force
+	cargo +nightly install --git https://github.com/alexcrichton/wasm-gc --force

src/chain_spec.rs

@@ -14,24 +14,31 @@
 // You should have received a copy of the GNU General Public License
 // along with Joystream node.  If not, see <http://www.gnu.org/licenses/>.
 
-use hex_literal::{hex, hex_impl};
-use joystream_node_runtime::{
-    forum::InputValidationLengthConstraint, AccountId, ActorsConfig, BalancesConfig,
-    ConsensusConfig, CouncilConfig, CouncilElectionConfig, DataObjectStorageRegistryConfig,
-    DataObjectTypeRegistryConfig, DownloadSessionsConfig, ForumConfig, GenesisConfig,
-    GrandpaConfig, IndicesConfig, MembersConfig, Perbill, ProposalsConfig, SessionConfig,
-    StakerStatus, StakingConfig, SudoConfig, TimestampConfig,
+use hex_literal::hex;
+use node_runtime::{
+    versioned_store::InputValidationLengthConstraint as VsInputValidation, AccountId, ActorsConfig,
+    AuthorityDiscoveryConfig, BabeConfig, Balance, BalancesConfig, ContentWorkingGroupConfig,
+    CouncilConfig, CouncilElectionConfig, DataObjectStorageRegistryConfig,
+    DataObjectTypeRegistryConfig, GenesisConfig, GrandpaConfig, ImOnlineConfig, IndicesConfig,
+    MembersConfig, Perbill, ProposalsConfig, SessionConfig, SessionKeys, Signature, StakerStatus,
+    StakingConfig, SudoConfig, SystemConfig, VersionedStoreConfig, DAYS, WASM_BINARY,
 };
-use primitives::{crypto::UncheckedInto, ed25519, sr25519, Pair};
+use primitives::{crypto::UncheckedInto, sr25519, Pair, Public};
+use runtime_primitives::traits::{IdentifyAccount, Verify};
+
+use babe_primitives::AuthorityId as BabeId;
+use grandpa_primitives::AuthorityId as GrandpaId;
+use im_online::sr25519::AuthorityId as ImOnlineId;
+use serde_json as json;
 use substrate_service;
 use substrate_telemetry::TelemetryEndpoints;
 
-use ed25519::Public as AuthorityId;
+type AccountPublic = <Signature as Verify>::Signer;
 
 // Note this is the URL for the telemetry server
 const STAGING_TELEMETRY_URL: &str = "wss://telemetry.polkadot.io/submit/";
 
-/// Specialised `ChainSpec`. This is a specialisation of the general Substrate ChainSpec type.
+/// Specialized `ChainSpec`. This is a specialization of the general Substrate ChainSpec type.
 pub type ChainSpec = substrate_service::ChainSpec<GenesisConfig>;
 
 /// The chain specification option. This is expected to come in from the CLI and
@@ -49,16 +56,40 @@ pub enum Alternative {
     LiveTestnet,
 }
 
-fn authority_key(s: &str) -> AuthorityId {
-    ed25519::Pair::from_string(&format!("//{}", s), None)
+/// Helper function to generate a crypto pair from seed
+pub fn get_from_seed<TPublic: Public>(seed: &str) -> <TPublic::Pair as Pair>::Public {
+    TPublic::Pair::from_string(&format!("//{}", seed), None)
         .expect("static values are valid; qed")
         .public()
 }
 
-fn account_key(s: &str) -> AccountId {
-    sr25519::Pair::from_string(&format!("//{}", s), None)
-        .expect("static values are valid; qed")
-        .public()
+/// Helper function to generate an account ID from seed
+pub fn get_account_id_from_seed<TPublic: Public>(seed: &str) -> AccountId
+where
+    AccountPublic: From<<TPublic::Pair as Pair>::Public>,
+{
+    AccountPublic::from(get_from_seed::<TPublic>(seed)).into_account()
+}
+
+/// Helper function to generate stash, controller and session key from seed
+pub fn get_authority_keys_from_seed(
+    seed: &str,
+) -> (AccountId, AccountId, GrandpaId, BabeId, ImOnlineId) {
+    (
+        get_account_id_from_seed::<sr25519::Public>(&format!("{}//stash", seed)),
+        get_account_id_from_seed::<sr25519::Public>(seed),
+        get_from_seed::<GrandpaId>(seed),
+        get_from_seed::<BabeId>(seed),
+        get_from_seed::<ImOnlineId>(seed),
+    )
+}
+
+fn session_keys(grandpa: GrandpaId, babe: BabeId, im_online: ImOnlineId) -> SessionKeys {
+    SessionKeys {
+        grandpa,
+        babe,
+        im_online,
+    }
 }
 
 impl Alternative {
@@ -70,26 +101,20 @@ impl Alternative {
                 "dev",
                 || {
                     testnet_genesis(
+                        vec![get_authority_keys_from_seed("Alice")],
+                        get_account_id_from_seed::<sr25519::Public>("Alice"),
                         vec![
-                            // stash, controller, authority
-                            (
-                                account_key("Alice//stash"),
-                                account_key("Alice"),
-                                authority_key("Alice"),
-                            ),
-                        ],
-                        vec![
-                            // endowed account
-                            account_key("Alice"),
+                            get_account_id_from_seed::<sr25519::Public>("Alice"),
+                            get_account_id_from_seed::<sr25519::Public>("Bob"),
+                            get_account_id_from_seed::<sr25519::Public>("Alice//stash"),
+                            get_account_id_from_seed::<sr25519::Public>("Bob//stash"),
                         ],
-                        // sudo key
-                        account_key("Alice"),
                     )
                 },
                 vec![],
                 None,
                 None,
-                None,
+                Some(chain_spec_properties()),
                 None,
             ),
             Alternative::LocalTestnet => ChainSpec::from_genesis(
@@ -98,32 +123,30 @@ impl Alternative {
                 || {
                     testnet_genesis(
                         vec![
-                            (
-                                account_key("Alice//stash"),
-                                account_key("Alice"),
-                                authority_key("Alice"),
-                            ),
-                            (
-                                account_key("Bob//stash"),
-                                account_key("Bob"),
-                                authority_key("Bob"),
-                            ),
+                            get_authority_keys_from_seed("Alice"),
+                            get_authority_keys_from_seed("Bob"),
                         ],
+                        get_account_id_from_seed::<sr25519::Public>("Alice"),
                         vec![
-                            account_key("Alice"),
-                            account_key("Bob"),
-                            account_key("Charlie"),
-                            account_key("Dave"),
-                            account_key("Eve"),
-                            account_key("Ferdie"),
+                            get_account_id_from_seed::<sr25519::Public>("Alice"),
+                            get_account_id_from_seed::<sr25519::Public>("Bob"),
+                            get_account_id_from_seed::<sr25519::Public>("Charlie"),
+                            get_account_id_from_seed::<sr25519::Public>("Dave"),
+                            get_account_id_from_seed::<sr25519::Public>("Eve"),
+                            get_account_id_from_seed::<sr25519::Public>("Ferdie"),
+                            get_account_id_from_seed::<sr25519::Public>("Alice//stash"),
+                            get_account_id_from_seed::<sr25519::Public>("Bob//stash"),
+                            get_account_id_from_seed::<sr25519::Public>("Charlie//stash"),
+                            get_account_id_from_seed::<sr25519::Public>("Dave//stash"),
+                            get_account_id_from_seed::<sr25519::Public>("Eve//stash"),
+                            get_account_id_from_seed::<sr25519::Public>("Ferdie//stash"),
                         ],
-                        account_key("Alice"),
                     )
                 },
                 vec![],
                 None,
                 None,
-                None,
+                Some(chain_spec_properties()),
                 None,
             ),
             Alternative::StagingTestnet => staging_testnet_config(),
@@ -136,285 +159,352 @@ impl Alternative {
             "dev" => Some(Alternative::Development),
             "local" => Some(Alternative::LocalTestnet),
             "staging" => Some(Alternative::StagingTestnet),
-            "" | "testnet" => Some(Alternative::LiveTestnet),
+            "rome-experimental" => Some(Alternative::LiveTestnet),
+            // "" | "tesnet" => Some(Alternative::LiveTestnet),
             _ => None,
         }
     }
 }
 
-/// LiveTestnet generator
+fn new_vs_validation(min: u16, max_min_diff: u16) -> VsInputValidation {
+    return VsInputValidation { min, max_min_diff };
+}
+
+/// Joystream LiveTestnet generator
 pub fn live_testnet_config() -> Result<ChainSpec, String> {
-    ChainSpec::from_embedded(include_bytes!("../res/joy_testnet_2.json"))
+    ChainSpec::from_json_bytes(&include_bytes!("../res/rome-experimental.json")[..])
+}
+
+pub fn chain_spec_properties() -> json::map::Map<String, json::Value> {
+    let mut properties: json::map::Map<String, json::Value> = json::map::Map::new();
+    properties.insert(
+        String::from("tokenDecimals"),
+        json::Value::Number(json::Number::from(0)),
+    );
+    properties.insert(
+        String::from("tokenSymbol"),
+        json::Value::String(String::from("JOY")),
+    );
+    properties
 }
 
 /// Staging testnet config
 pub fn staging_testnet_config() -> ChainSpec {
     let boot_nodes = vec![
-		String::from("/dns4/bootnode1.joystream.org/tcp/30333/p2p/QmeDa8jASqMRpTh4YCkeVEuHo6nbMcFDzD9pkUxTr3WxhM"),
-		String::from("/dns4/bootnode2.joystream.org/tcp/30333/p2p/QmbjzmNMjzQUMHpzqcPHW5DnFeUjM3x4hbiDSMkYv1McD3"),
-	];
+        String::from("/dns4/rome-reckless.joystream.org/tcp/30333/p2p/QmaTTdEF6YVCtynSjsXmGPSGcEesAahoZ8pmcCmmBwSE7S")
+    ];
+
     ChainSpec::from_genesis(
-        "Joystream Staging Testnet",
-        "joy_staging_5",
+        "Joystream Rome Reckless Testnet",
+        "joy_rome_reckless_N",
         staging_testnet_config_genesis,
         boot_nodes,
         Some(TelemetryEndpoints::new(vec![(
             STAGING_TELEMETRY_URL.to_string(),
             0,
         )])),
-        None,
-        None,
+        // protocol_id
+        Some(&*"/joy/rome/reckless/N"),
+        // Properties
+        Some(chain_spec_properties()),
+        // Extensions
         None,
     )
 }
 
-fn new_validation(min: u16, max_min_diff: u16) -> InputValidationLengthConstraint {
-    return InputValidationLengthConstraint { min, max_min_diff };
-}
-
 fn staging_testnet_config_genesis() -> GenesisConfig {
-    let initial_authorities: Vec<(AccountId, AccountId, AuthorityId)> = vec![(
-        hex!["0610d1a2b1d704723e588c842a934737491688b18b052baae1286f12e96adb65"].unchecked_into(), // stash
-        hex!["609cee3edd9900e69be44bcbf7a1892cad10408840a2d72d563811d72d9bb339"].unchecked_into(), // controller
-        hex!["65179fd9c39ec301457d1ee47a13f3bb0fef65812a57b6c93212e609b10d35d2"].unchecked_into(), // session key
+    let initial_authorities: Vec<(AccountId, AccountId, GrandpaId, BabeId, ImOnlineId)> = vec![(
+        hex!["4430a31121fc174b1c361b365580c54ef393813171e59542f5d2ce3d8b171a2d"].into(), // stash
+        hex!["58a743f1bab2f472fb99af98b6591e23a56fd84bc9c2a62037ed8867caae7c21"].into(), // controller
+        hex!["af5286fb1e403afd44d92ae3fb0b371a0f4f8faf3e6b2ff50ea91fb426b0015f"].unchecked_into(), // session - grandpa
+        hex!["d69529ed1549644977cec8dc027e71e1e2ae7aab99833a7f7dc08677a8d36307"].unchecked_into(), // session - babe
+        hex!["56bfd27715ce6c76e4d884c31374b9928391e461727ffaf27b94b6ce48570d39"].unchecked_into(), // session - im-online
     )];
-    let endowed_accounts = vec![hex![
-        "0ae55282e669fc55cb9529c0b12b989f2c5bf636d0de7630b5a4850055ed9c30"
-    ]
-    .unchecked_into()];
+    let endowed_accounts =
+        vec![hex!["00680fb81473784017291ef0afd968b986966daa7842d5b5063c8427c2b62577"].into()];
 
-    const CENTS: u128 = 1;
-    const DOLLARS: u128 = 100 * CENTS;
-
-    const SECS_PER_BLOCK: u64 = 6;
-    const MINUTES: u64 = 60 / SECS_PER_BLOCK;
-    const HOURS: u64 = MINUTES * 60;
-    const DAYS: u64 = HOURS * 24;
-    const STASH: u128 = 50 * DOLLARS;
-    const ENDOWMENT: u128 = 100_000_000 * DOLLARS;
+    const CENTS: Balance = 1;
+    const DOLLARS: Balance = 100 * CENTS;
+    const STASH: Balance = 20 * DOLLARS;
+    const ENDOWMENT: Balance = 100_000 * DOLLARS;
 
     GenesisConfig {
-		consensus: Some(ConsensusConfig {
-			code: include_bytes!("../substrate-runtime-joystream/wasm/target/wasm32-unknown-unknown/release/joystream_node_runtime_wasm.compact.wasm").to_vec(),
-			authorities: initial_authorities.iter().map(|x| x.2.clone()).collect(),
-		}),
-		system: None,
-		timestamp: Some(TimestampConfig {
-			minimum_period: SECS_PER_BLOCK / 2, // due to the nature of aura the slots are 2*period
-		}),
-		indices: Some(IndicesConfig {
-			ids: vec![],
-		}),
-		balances: Some(BalancesConfig {
-			balances: endowed_accounts.iter().cloned()
-				.map(|k| (k, ENDOWMENT))
-				.chain(initial_authorities.iter().map(|x| (x.0.clone(), STASH)))
-				.collect(),
-			existential_deposit: 0,
-			transfer_fee: 0,
-			creation_fee: 0,
-			vesting: vec![],
-			transaction_base_fee: 1,
-			transaction_byte_fee: 0,
-		}),
-		sudo: Some(SudoConfig {
-			key: endowed_accounts[0].clone(),
-		}),
-		session: Some(SessionConfig {
-			validators: initial_authorities.iter().map(|x| x.1.clone()).collect(),
-			session_length: 10 * MINUTES,
-			keys: initial_authorities.iter().map(|x| (x.1.clone(), x.2.clone())).collect::<Vec<_>>(),
-		}),
-		staking: Some(StakingConfig {
-			current_era: 0,
-			offline_slash: Perbill::from_millionths(10_000),  // 1/ 100 => 1%
-			session_reward: Perbill::from_millionths(1_000),  // 1/1000 => 0.1% (min stake -> 1000 units for reward to be GT 0)
-			current_session_reward: 0,
-			validator_count: 20,
-			sessions_per_era: 6,
-			bonding_duration: 1, // Number of ERAs
-			offline_slash_grace: 4,
-			minimum_validator_count: 1,
-			stakers: initial_authorities.iter().map(|x| (x.0.clone(), x.1.clone(), STASH, StakerStatus::Validator)).collect(),
-			invulnerables: initial_authorities.iter().map(|x| x.1.clone()).collect(),
-		}),
-		grandpa: Some(GrandpaConfig {
-			authorities: initial_authorities.iter().map(|x| (x.2.clone(), 1)).collect(),
-		}),
-		council: Some(CouncilConfig {
-			active_council: vec![],
-			term_ends_at: 1,
-		}),
-		election: Some(CouncilElectionConfig {
-			auto_start: true,
-			announcing_period: 3 * DAYS,
-			voting_period: 1 * DAYS,
-			revealing_period: 1 * DAYS,
-			council_size: 12,
-			candidacy_limit: 25,
-			min_council_stake: 10 * DOLLARS,
-			new_term_duration: 14 * DAYS,
-			min_voting_stake: 1 * DOLLARS,
-		}),
-		proposals: Some(ProposalsConfig {
-			approval_quorum: 66,
-			min_stake: 2 * DOLLARS,
-			cancellation_fee: 10 * CENTS,
-			rejection_fee: 1 * DOLLARS,
-			voting_period: 2 * DAYS,
-			name_max_len: 512,
-			description_max_len: 10_000,
-			wasm_code_max_len: 2_000_000,
-		}),
-		members: Some(MembersConfig {
-			default_paid_membership_fee: 100u128,
-			first_member_id: 1,
-		}),
-		forum: Some(ForumConfig {
-			category_by_id: vec![],
-			thread_by_id: vec![],
-			post_by_id: vec![],
-			next_category_id: 1,
-			next_thread_id: 1,
-			next_post_id: 1,
-			forum_sudo: endowed_accounts[0].clone(),
-			category_title_constraint: new_validation(10, 90),
-			category_description_constraint: new_validation(10, 490),
-			thread_title_constraint: new_validation(10, 90),
-			post_text_constraint: new_validation(10, 990),
-			thread_moderation_rationale_constraint: new_validation(10, 290),
-			post_moderation_rationale_constraint: new_validation(10, 290)
-		}),
-		data_object_type_registry: Some(DataObjectTypeRegistryConfig {
-			first_data_object_type_id: 1,
-		}),
-		data_object_storage_registry: Some(DataObjectStorageRegistryConfig{
-			first_relationship_id: 1,
-		}),
-		downloads: Some(DownloadSessionsConfig{
-			first_download_session_id: 1,
-		}),
-		actors: Some(ActorsConfig{
-			enable_storage_role: true,
-			request_life_time: 300,
-			_genesis_phantom_data: Default::default(),
-		})
-	}
+        system: Some(SystemConfig {
+            code: WASM_BINARY.to_vec(),
+            changes_trie_config: Default::default(),
+        }),
+        balances: Some(BalancesConfig {
+            balances: endowed_accounts
+                .iter()
+                .cloned()
+                .map(|k| (k, ENDOWMENT))
+                .chain(initial_authorities.iter().map(|x| (x.0.clone(), STASH)))
+                .collect(),
+            vesting: vec![],
+        }),
+        indices: Some(IndicesConfig { ids: vec![] }),
+        session: Some(SessionConfig {
+            keys: initial_authorities
+                .iter()
+                .map(|x| {
+                    (
+                        x.0.clone(),
+                        session_keys(x.2.clone(), x.3.clone(), x.4.clone()),
+                    )
+                })
+                .collect::<Vec<_>>(),
+        }),
+        staking: Some(StakingConfig {
+            current_era: 0,
+            validator_count: 20,
+            minimum_validator_count: 1,
+            stakers: initial_authorities
+                .iter()
+                .map(|x| (x.0.clone(), x.1.clone(), STASH, StakerStatus::Validator))
+                .collect(),
+            invulnerables: initial_authorities.iter().map(|x| x.0.clone()).collect(),
+            slash_reward_fraction: Perbill::from_percent(10),
+            ..Default::default()
+        }),
+        sudo: Some(SudoConfig {
+            key: endowed_accounts[0].clone(),
+        }),
+        babe: Some(BabeConfig {
+            authorities: vec![],
+        }),
+        im_online: Some(ImOnlineConfig { keys: vec![] }),
+        authority_discovery: Some(AuthorityDiscoveryConfig { keys: vec![] }),
+        grandpa: Some(GrandpaConfig {
+            authorities: vec![],
+        }),
+        council: Some(CouncilConfig {
+            active_council: vec![],
+            term_ends_at: 1,
+        }),
+        election: Some(CouncilElectionConfig {
+            auto_start: true,
+            announcing_period: 3 * DAYS,
+            voting_period: 1 * DAYS,
+            revealing_period: 1 * DAYS,
+            council_size: 12,
+            candidacy_limit: 25,
+            min_council_stake: 10 * DOLLARS,
+            new_term_duration: 14 * DAYS,
+            min_voting_stake: 1 * DOLLARS,
+        }),
+        proposals: Some(ProposalsConfig {
+            approval_quorum: 66,
+            min_stake: 2 * DOLLARS,
+            cancellation_fee: 10 * CENTS,
+            rejection_fee: 1 * DOLLARS,
+            voting_period: 2 * DAYS,
+            name_max_len: 512,
+            description_max_len: 10_000,
+            wasm_code_max_len: 2_000_000,
+        }),
+        members: Some(MembersConfig {
+            default_paid_membership_fee: 100u128,
+            members: crate::members_config::initial_members(),
+        }),
+        forum: Some(crate::forum_config::from_serialized::create(
+            endowed_accounts[0].clone(),
+        )),
+        data_object_type_registry: Some(DataObjectTypeRegistryConfig {
+            first_data_object_type_id: 1,
+        }),
+        data_object_storage_registry: Some(DataObjectStorageRegistryConfig {
+            first_relationship_id: 1,
+        }),
+        actors: Some(ActorsConfig {
+            enable_storage_role: true,
+            request_life_time: 300,
+        }),
+        versioned_store: Some(VersionedStoreConfig {
+            class_by_id: vec![],
+            entity_by_id: vec![],
+            next_class_id: 1,
+            next_entity_id: 1,
+            property_name_constraint: new_vs_validation(1, 99),
+            property_description_constraint: new_vs_validation(1, 999),
+            class_name_constraint: new_vs_validation(1, 99),
+            class_description_constraint: new_vs_validation(1, 999),
+        }),
+        content_wg: Some(ContentWorkingGroupConfig {
+            mint_capacity: 100000,
+            curator_opening_by_id: vec![],
+            next_curator_opening_id: 0,
+            curator_application_by_id: vec![],
+            next_curator_application_id: 0,
+            channel_by_id: vec![],
+            next_channel_id: 1,
+            channel_id_by_handle: vec![],
+            curator_by_id: vec![],
+            next_curator_id: 0,
+            principal_by_id: vec![],
+            next_principal_id: 0,
+            channel_creation_enabled: true, // there is no extrinsic to change it so enabling at genesis
+            unstaker_by_stake_id: vec![],
+            channel_handle_constraint: crate::forum_config::new_validation(5, 20),
+            channel_description_constraint: crate::forum_config::new_validation(1, 1024),
+            opening_human_readable_text: crate::forum_config::new_validation(1, 2048),
+            curator_application_human_readable_text: crate::forum_config::new_validation(1, 2048),
+            curator_exit_rationale_text: crate::forum_config::new_validation(1, 2048),
+            channel_avatar_constraint: crate::forum_config::new_validation(5, 1024),
+            channel_banner_constraint: crate::forum_config::new_validation(5, 1024),
+            channel_title_constraint: crate::forum_config::new_validation(5, 1024),
+        }),
+    }
 }
 
-fn testnet_genesis(
-    initial_authorities: Vec<(AccountId, AccountId, AuthorityId)>,
-    endowed_accounts: Vec<AccountId>,
+/// Helper function to create GenesisConfig for testing
+pub fn testnet_genesis(
+    initial_authorities: Vec<(AccountId, AccountId, GrandpaId, BabeId, ImOnlineId)>,
     root_key: AccountId,
+    endowed_accounts: Vec<AccountId>,
 ) -> GenesisConfig {
-    const STASH: u128 = 100;
-    const ENDOWMENT: u128 = 100_000_000;
+    const STASH: Balance = 2000;
+    const ENDOWMENT: Balance = 10_000_000;
+
+    // Static members
+    let initial_members = crate::members_config::initial_members();
+
+    // let mut additional_members = vec![
+    //     // Make Root a member
+    //     (
+    //         root_key.clone(),
+    //         String::from("system"),
+    //         String::from("http://joystream.org/avatar.png"),
+    //         String::from("I am root!"),
+    //     ),
+    // ];
+
+    // // Additional members
+    // initial_members.append(&mut additional_members);
 
     GenesisConfig {
-		consensus: Some(ConsensusConfig {
-			code: include_bytes!("../substrate-runtime-joystream/wasm/target/wasm32-unknown-unknown/release/joystream_node_runtime_wasm.compact.wasm").to_vec(),
-			authorities: initial_authorities.iter().map(|x| x.2.clone()).collect(),
-		}),
-		system: None,
-		timestamp: Some(TimestampConfig {
-			minimum_period: 3,                    // 3*2=6 second block time.
-		}),
-		indices: Some(IndicesConfig {
-			ids: vec![]
-		}),
-		balances: Some(BalancesConfig {
-			existential_deposit: 0,
-			transfer_fee: 0,
-			creation_fee: 0,
-			balances: endowed_accounts.iter().cloned()
-				.map(|k| (k, ENDOWMENT))
-				.chain(initial_authorities.iter().map(|x| (x.0.clone(), STASH)))
-				.collect(),
-			vesting: vec![],
-			transaction_base_fee: 1,
-			transaction_byte_fee: 0,
-		}),
-		sudo: Some(SudoConfig {
-			key: root_key.clone(),
-		}),
-		session: Some(SessionConfig {
-			validators: initial_authorities.iter().map(|x| x.1.clone()).collect(),
-			session_length: 10,
-			keys: initial_authorities.iter().map(|x| (x.1.clone(), x.2.clone())).collect::<Vec<_>>(),
-		}),
-		staking: Some(StakingConfig {
-			current_era: 0,
-			minimum_validator_count: 1,
-			validator_count: 2,
-			sessions_per_era: 5,
-			bonding_duration: 1, // Number of Eras
-			offline_slash: Perbill::zero(),
-			session_reward: Perbill::zero(),
-			current_session_reward: 0,
-			offline_slash_grace: 0,
-			stakers: initial_authorities.iter().map(|x| (x.0.clone(), x.1.clone(), STASH, StakerStatus::Validator)).collect(),
-			invulnerables: initial_authorities.iter().map(|x| x.1.clone()).collect(),
-		}),
-		grandpa: Some(GrandpaConfig {
-			authorities: initial_authorities.iter().map(|x| (x.2.clone(), 1)).collect(),
-		}),
-		council: Some(CouncilConfig {
-			active_council: vec![],
-			term_ends_at: 1,
-		}),
-		election: Some(CouncilElectionConfig {
-			auto_start: true,
-			announcing_period: 50,
-			voting_period: 50,
-			revealing_period: 50,
-			council_size: 2,
-			candidacy_limit: 25,
-			min_council_stake: 100,
-			new_term_duration: 1000,
-			min_voting_stake: 10,
-		}),
-		proposals: Some(ProposalsConfig {
-			approval_quorum: 66,
-			min_stake: 100,
-			cancellation_fee: 5,
-			rejection_fee: 10,
-			voting_period: 100,
-			name_max_len: 100,
-			description_max_len: 10_000,
-			wasm_code_max_len: 2_000_000,
-		}),
-		members: Some(MembersConfig {
-			default_paid_membership_fee: 100u128,
-			first_member_id: 1,
-		}),
-		forum: Some(ForumConfig {
-			category_by_id: vec![],
-			thread_by_id: vec![],
-			post_by_id: vec![],
-			next_category_id: 1,
-			next_thread_id: 1,
-			next_post_id: 1,
-			forum_sudo: root_key,
-			category_title_constraint: new_validation(10, 90),
-			category_description_constraint: new_validation(10, 490),
-			thread_title_constraint: new_validation(10, 90),
-			post_text_constraint: new_validation(10, 990),
-			thread_moderation_rationale_constraint: new_validation(10, 290),
-			post_moderation_rationale_constraint: new_validation(10, 290)
-		}),
-		data_object_type_registry: Some(DataObjectTypeRegistryConfig {
-			first_data_object_type_id: 1,
-		}),
-		data_object_storage_registry: Some(DataObjectStorageRegistryConfig{
-			first_relationship_id: 1,
-		}),
-		downloads: Some(DownloadSessionsConfig{
-			first_download_session_id: 1,
-		}),
-		actors: Some(ActorsConfig{
-			enable_storage_role: true,
-			request_life_time: 300,
-			_genesis_phantom_data: Default::default(),
-		})
-	}
+        //substrate modules
+        system: Some(SystemConfig {
+            code: WASM_BINARY.to_vec(),
+            changes_trie_config: Default::default(),
+        }),
+        balances: Some(BalancesConfig {
+            balances: endowed_accounts
+                .iter()
+                .map(|k| (k.clone(), ENDOWMENT))
+                .collect(),
+            vesting: vec![],
+        }),
+        indices: Some(IndicesConfig { ids: vec![] }),
+        staking: Some(StakingConfig {
+            current_era: 0,
+            minimum_validator_count: 1,
+            validator_count: 2,
+            stakers: initial_authorities
+                .iter()
+                .map(|x| (x.0.clone(), x.1.clone(), STASH, StakerStatus::Validator))
+                .collect(),
+            invulnerables: initial_authorities.iter().map(|x| x.0.clone()).collect(),
+            slash_reward_fraction: Perbill::from_percent(10),
+            ..Default::default()
+        }),
+        session: Some(SessionConfig {
+            keys: initial_authorities
+                .iter()
+                .map(|x| {
+                    (
+                        x.0.clone(),
+                        session_keys(x.2.clone(), x.3.clone(), x.4.clone()),
+                    )
+                })
+                .collect::<Vec<_>>(),
+        }),
+        sudo: Some(SudoConfig {
+            key: root_key.clone(),
+        }),
+        babe: Some(BabeConfig {
+            authorities: vec![],
+        }),
+        im_online: Some(ImOnlineConfig { keys: vec![] }),
+        authority_discovery: Some(AuthorityDiscoveryConfig { keys: vec![] }),
+        grandpa: Some(GrandpaConfig {
+            authorities: vec![],
+        }),
+        // joystream modules
+        council: Some(CouncilConfig {
+            active_council: vec![],
+            term_ends_at: 1,
+        }),
+        election: Some(CouncilElectionConfig {
+            auto_start: true,
+            announcing_period: 50,
+            voting_period: 50,
+            revealing_period: 50,
+            council_size: 2,
+            candidacy_limit: 25,
+            min_council_stake: 100,
+            new_term_duration: 1000,
+            min_voting_stake: 10,
+        }),
+        proposals: Some(ProposalsConfig {
+            approval_quorum: 66,
+            min_stake: 100,
+            cancellation_fee: 5,
+            rejection_fee: 10,
+            voting_period: 100,
+            name_max_len: 100,
+            description_max_len: 10_000,
+            wasm_code_max_len: 2_000_000,
+        }),
+        members: Some(MembersConfig {
+            default_paid_membership_fee: 100u128,
+            members: initial_members,
+        }),
+        forum: Some(crate::forum_config::from_serialized::create(
+            root_key.clone(),
+        )),
+        data_object_type_registry: Some(DataObjectTypeRegistryConfig {
+            first_data_object_type_id: 1,
+        }),
+        data_object_storage_registry: Some(DataObjectStorageRegistryConfig {
+            first_relationship_id: 1,
+        }),
+        actors: Some(ActorsConfig {
+            enable_storage_role: true,
+            request_life_time: 300,
+        }),
+        versioned_store: Some(VersionedStoreConfig {
+            class_by_id: vec![],
+            entity_by_id: vec![],
+            next_class_id: 1,
+            next_entity_id: 1,
+            property_name_constraint: new_vs_validation(1, 99),
+            property_description_constraint: new_vs_validation(1, 999),
+            class_name_constraint: new_vs_validation(1, 99),
+            class_description_constraint: new_vs_validation(1, 999),
+        }),
+        content_wg: Some(ContentWorkingGroupConfig {
+            mint_capacity: 100000,
+            curator_opening_by_id: vec![],
+            next_curator_opening_id: 0,
+            curator_application_by_id: vec![],
+            next_curator_application_id: 0,
+            channel_by_id: vec![],
+            next_channel_id: 1,
+            channel_id_by_handle: vec![],
+            curator_by_id: vec![],
+            next_curator_id: 0,
+            principal_by_id: vec![],
+            next_principal_id: 0,
+            channel_creation_enabled: true, // there is no extrinsic to change it so enabling at genesis
+            unstaker_by_stake_id: vec![],
+            channel_handle_constraint: crate::forum_config::new_validation(5, 20),
+            channel_description_constraint: crate::forum_config::new_validation(1, 1024),
+            opening_human_readable_text: crate::forum_config::new_validation(1, 2048),
+            curator_application_human_readable_text: crate::forum_config::new_validation(1, 2048),
+            curator_exit_rationale_text: crate::forum_config::new_validation(1, 2048),
+            channel_avatar_constraint: crate::forum_config::new_validation(5, 1024),
+            channel_banner_constraint: crate::forum_config::new_validation(5, 1024),
+            channel_title_constraint: crate::forum_config::new_validation(5, 1024),
+        }),
+    }
 }

src/cli.rs

@@ -1,28 +1,12 @@
-// Copyright 2019 Joystream Contributors
-// This file is part of Joystream node.
-
-// Joystream node is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Joystream node is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Joystream node.  If not, see <http://www.gnu.org/licenses/>.
-
 use crate::chain_spec;
+use crate::new_full_start;
 use crate::service;
 use futures::{future, sync::oneshot, Future};
 use log::info;
 use std::cell::RefCell;
-use std::ops::Deref;
 pub use substrate_cli::{error, IntoExit, VersionInfo};
-use substrate_cli::{informant, parse_and_execute, NoCustom};
-use substrate_service::{Roles as ServiceRoles, ServiceFactory};
+use substrate_cli::{informant, parse_and_prepare, NoCustom, ParseAndPrepare};
+use substrate_service::{AbstractService, Configuration, Roles as ServiceRoles};
 use tokio::runtime::Runtime;
 
 /// Parse command line arguments into service configuration.
@@ -32,39 +16,53 @@ where
     T: Into<std::ffi::OsString> + Clone,
     E: IntoExit,
 {
-    parse_and_execute::<service::Factory, NoCustom, NoCustom, _, _, _, _, _>(
-        load_spec,
-        &version,
-        "joystream-node",
-        args,
-        exit,
-        |exit, _custom_args, config| {
-            info!("{}", version.name);
-            info!("  version {}", config.full_version());
-            info!("  by {}, 2019", version.author);
-            info!("Chain specification: {}", config.chain_spec.name());
-            info!("Node name: {}", config.name);
-            info!("Roles: {:?}", config.roles);
-            let runtime = Runtime::new().map_err(|e| format!("{:?}", e))?;
-            let executor = runtime.executor();
-            match config.roles {
-                ServiceRoles::LIGHT => run_until_exit(
-                    runtime,
-                    service::Factory::new_light(config, executor)
-                        .map_err(|e| format!("{:?}", e))?,
-                    exit,
-                ),
-                _ => run_until_exit(
-                    runtime,
-                    service::Factory::new_full(config, executor).map_err(|e| format!("{:?}", e))?,
-                    exit,
-                ),
-            }
-            .map_err(|e| format!("{:?}", e))
-        },
-    )
-    .map_err(Into::into)
-    .map(|_| ())
+    type Config<T> = Configuration<(), T>;
+    match parse_and_prepare::<NoCustom, NoCustom, _>(&version, "joystream-node", args) {
+        ParseAndPrepare::Run(cmd) => cmd.run(
+            load_spec,
+            exit,
+            |exit, _cli_args, _custom_args, config: Config<_>| {
+                info!("{}", version.name);
+                info!("  version {}", config.full_version());
+                info!("  by {}, 2019", version.author);
+                info!("Chain specification: {}", config.chain_spec.name());
+                info!("Node name: {}", config.name);
+                info!("Roles: {:?}", config.roles);
+                let runtime = Runtime::new().map_err(|e| format!("{:?}", e))?;
+                match config.roles {
+                    ServiceRoles::LIGHT => run_until_exit(
+                        runtime,
+                        service::new_light(config).map_err(|e| format!("{:?}", e))?,
+                        exit,
+                    ),
+                    _ => run_until_exit(
+                        runtime,
+                        service::new_full(config).map_err(|e| format!("{:?}", e))?,
+                        exit,
+                    ),
+                }
+                .map_err(|e| format!("{:?}", e))
+            },
+        ),
+        ParseAndPrepare::BuildSpec(cmd) => cmd.run::<NoCustom, _, _, _>(load_spec),
+        ParseAndPrepare::ExportBlocks(cmd) => cmd.run_with_builder(
+            |config: Config<_>| Ok(new_full_start!(config).0),
+            load_spec,
+            exit,
+        ),
+        ParseAndPrepare::ImportBlocks(cmd) => cmd.run_with_builder(
+            |config: Config<_>| Ok(new_full_start!(config).0),
+            load_spec,
+            exit,
+        ),
+        ParseAndPrepare::PurgeChain(cmd) => cmd.run(load_spec),
+        ParseAndPrepare::RevertChain(cmd) => {
+            cmd.run_with_builder(|config: Config<_>| Ok(new_full_start!(config).0), load_spec)
+        }
+        ParseAndPrepare::CustomCommand(_) => Ok(()),
+    }?;
+
+    Ok(())
 }
 
 fn load_spec(id: &str) -> Result<Option<chain_spec::ChainSpec>, String> {
@@ -74,25 +72,35 @@ fn load_spec(id: &str) -> Result<Option<chain_spec::ChainSpec>, String> {
     })
 }
 
-fn run_until_exit<T, C, E>(mut runtime: Runtime, service: T, e: E) -> error::Result<()>
+fn run_until_exit<T, E>(mut runtime: Runtime, service: T, e: E) -> error::Result<()>
 where
-    T: Deref<Target = substrate_service::Service<C>>,
-    C: substrate_service::Components,
+    T: AbstractService,
     E: IntoExit,
 {
     let (exit_send, exit) = exit_future::signal();
 
-    let executor = runtime.executor();
-    informant::start(&service, exit.clone(), executor.clone());
-
-    let _ = runtime.block_on(e.into_exit());
-    exit_send.fire();
+    let informant = informant::build(&service);
+    runtime.executor().spawn(exit.until(informant).map(|_| ()));
 
     // we eagerly drop the service so that the internal exit future is fired,
     // but we need to keep holding a reference to the global telemetry guard
     let _telemetry = service.telemetry();
-    drop(service);
-    Ok(())
+
+    let service_res = {
+        let exit = e
+            .into_exit()
+            .map_err(|_| error::Error::Other("Exit future failed.".into()));
+        let service = service.map_err(|err| error::Error::Service(err));
+        let select = service.select(exit).map(|_| ()).map_err(|(err, _)| err);
+        runtime.block_on(select)
+    };
+
+    exit_send.fire();
+
+    // TODO [andre]: timeout this future #1318
+    let _ = runtime.shutdown_on_idle().wait();
+
+    service_res
 }
 
 // handles ctrl-c
@@ -105,11 +113,11 @@ impl IntoExit for Exit {
 
         let exit_send_cell = RefCell::new(Some(exit_send));
         ctrlc::set_handler(move || {
-            if let Some(exit_send) = exit_send_cell
+            let exit_send = exit_send_cell
                 .try_borrow_mut()
                 .expect("signal handler not reentrant; qed")
-                .take()
-            {
+                .take();
+            if let Some(exit_send) = exit_send {
                 exit_send.send(()).expect("Error sending exit notification");
             }
         })

src/error.rs

@@ -1,29 +0,0 @@
-// Copyright 2019 Joystream Contributors
-// This file is part of Joystream node.
-
-// Joystream node is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Joystream node is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Joystream node.  If not, see <http://www.gnu.org/licenses/>.
-
-//! Initialization errors.
-
-use client;
-
-error_chain! {
-    foreign_links {
-        Io(::std::io::Error) #[doc="IO error"];
-        Cli(::clap::Error) #[doc="CLI error"];
-    }
-    links {
-        Client(client::error::Error, client::error::ErrorKind) #[doc="Client error"];
-    }
-}

src/forum_config/from_encoded.rs

@@ -0,0 +1,90 @@
+// This module is not used but included as sample code
+// and highlights some pitfalls.
+
+use node_runtime::{
+    forum::{
+        Category, CategoryId, Post, PostId, Thread, ThreadId,
+    },
+    AccountId, BlockNumber, ForumConfig, Moment,
+};
+use serde::Deserialize;
+use serde_json::Result;
+use super::new_validation;
+
+use codec::Decode;
+
+#[derive(Deserialize)]
+struct ForumData {
+    /// hex encoded categories
+    categories: Vec<(CategoryId, String)>,
+    /// hex encoded posts
+    posts: Vec<(PostId, String)>,
+    /// hex encoded threads
+    threads: Vec<(ThreadId, String)>,
+}
+
+fn decode_post(encoded: String) -> Post<BlockNumber, Moment, AccountId> {
+    // hex string must not include '0x' prefix!
+    let encoded = hex::decode(encoded.as_bytes()).expect("failed to parse hex string");
+    Decode::decode(&mut encoded.as_slice()).unwrap()
+}
+
+fn decode_category(encoded: String) -> Category<BlockNumber, Moment, AccountId> {
+    // hex string must not include '0x' prefix!
+    let encoded = hex::decode(encoded.as_bytes()).expect("failed to parse hex string");
+    Decode::decode(&mut encoded.as_slice()).unwrap()
+}
+
+fn decode_thread(encoded: String) -> Thread<BlockNumber, Moment, AccountId> {
+    // hex string must not include '0x' prefix!
+    let encoded = hex::decode(encoded.as_bytes()).expect("failed to parse hex string");
+    Decode::decode(&mut encoded.as_slice()).unwrap()
+}
+
+fn parse_forum_json() -> Result<ForumData> {
+    let data = include_str!("../../res/forum_data_acropolis_encoded.json");
+    serde_json::from_str(data)
+}
+
+pub fn create(forum_sudo: AccountId) -> ForumConfig {
+    let forum_data = parse_forum_json().expect("failed loading forum data");
+
+    let next_category_id: CategoryId = forum_data
+        .categories
+        .last()
+        .map_or(1, |category| category.0 + 1);
+    let next_thread_id: ThreadId = forum_data.threads.last().map_or(1, |thread| thread.0 + 1);
+    let next_post_id: PostId = forum_data.posts.last().map_or(1, |post| post.0 + 1);
+
+    ForumConfig {
+        // Decoding will fail because of differnt type used for
+        // BlockNumber between Acropolis (u64) and Rome (u32)
+        // As long as types between chains are identical this approach works nicely
+        // since we don't need to use an intermediate format or do any transformation on source data.
+        category_by_id: forum_data
+            .categories
+            .into_iter()
+            .map(|category| (category.0, decode_category(category.1)))
+            .collect(),
+        thread_by_id: forum_data
+            .threads
+            .into_iter()
+            .map(|thread| (thread.0, decode_thread(thread.1)))
+            .collect(),
+        post_by_id: forum_data
+            .posts
+            .into_iter()
+            .map(|post| (post.0, decode_post(post.1)))
+            .collect(),
+        next_category_id,
+        next_thread_id,
+        next_post_id,
+        forum_sudo,
+        category_title_constraint: new_validation(10, 90),
+        category_description_constraint: new_validation(10, 490),
+        thread_title_constraint: new_validation(10, 90),
+        post_text_constraint: new_validation(10, 990),
+        thread_moderation_rationale_constraint: new_validation(10, 290),
+        post_moderation_rationale_constraint: new_validation(10, 290),
+    }
+}

src/forum_config/from_serialized.rs

@@ -0,0 +1,46 @@
+use super::new_validation;
+use node_runtime::{
+    forum::{Category, CategoryId, Post, PostId, Thread, ThreadId},
+    AccountId, BlockNumber, ForumConfig, Moment,
+};
+use serde::Deserialize;
+use serde_json::Result;
+
+#[derive(Deserialize)]
+struct ForumData {
+    categories: Vec<(CategoryId, Category<BlockNumber, Moment, AccountId>)>,
+    posts: Vec<(PostId, Post<BlockNumber, Moment, AccountId>)>,
+    threads: Vec<(ThreadId, Thread<BlockNumber, Moment, AccountId>)>,
+}
+
+fn parse_forum_json() -> Result<ForumData> {
+    let data = include_str!("../../res/forum_data_acropolis_serialized.json");
+    serde_json::from_str(data)
+}
+
+pub fn create(forum_sudo: AccountId) -> ForumConfig {
+    let forum_data = parse_forum_json().expect("failed loading forum data");
+
+    let next_category_id: CategoryId = forum_data
+        .categories
+        .last()
+        .map_or(1, |category| category.0 + 1);
+    let next_thread_id: ThreadId = forum_data.threads.last().map_or(1, |thread| thread.0 + 1);
+    let next_post_id: PostId = forum_data.posts.last().map_or(1, |post| post.0 + 1);
+
+    ForumConfig {
+        category_by_id: forum_data.categories,
+        thread_by_id: forum_data.threads,
+        post_by_id: forum_data.posts,
+        next_category_id,
+        next_thread_id,
+        next_post_id,
+        forum_sudo,
+        category_title_constraint: new_validation(10, 90),
+        category_description_constraint: new_validation(10, 490),
+        thread_title_constraint: new_validation(10, 90),
+        post_text_constraint: new_validation(10, 990),
+        thread_moderation_rationale_constraint: new_validation(10, 290),
+        post_moderation_rationale_constraint: new_validation(10, 290),
+    }
+}

src/forum_config/mod.rs

@@ -0,0 +1,10 @@
+pub mod from_serialized;
+
+// Not exported - only here as sample code
+// mod from_encoded;
+
+use node_runtime::forum::InputValidationLengthConstraint;
+
+pub fn new_validation(min: u16, max_min_diff: u16) -> InputValidationLengthConstraint {
+    return InputValidationLengthConstraint { min, max_min_diff };
+}

src/main.rs

@@ -21,11 +21,13 @@
 
 mod chain_spec;
 mod cli;
+mod forum_config;
+mod members_config;
 mod service;
 
 pub use substrate_cli::{error, IntoExit, VersionInfo};
 
-fn run() -> cli::error::Result<()> {
+fn main() {
     let version = VersionInfo {
         name: "Joystream Node",
         commit: env!("VERGEN_SHA_SHORT"),
@@ -35,7 +37,9 @@ fn run() -> cli::error::Result<()> {
         description: "Joystream substrate node",
         support_url: "https://www.joystream.org/",
     };
-    cli::run(::std::env::args(), cli::Exit, version)
-}
 
-error_chain::quick_main!(run);
+    if let Err(e) = cli::run(::std::env::args(), cli::Exit, version) {
+        eprintln!("Fatal error: {}\n\n{:?}", e, e);
+        std::process::exit(1)
+    }
+}

src/members_config.rs

@@ -0,0 +1,50 @@
+use serde::Deserialize;
+use serde_json::Result;
+
+use primitives::crypto::{AccountId32, Ss58Codec};
+
+#[derive(Deserialize)]
+struct Member {
+    /// SS58 Encoded public key
+    address: String,
+    handle: String,
+    avatar_uri: String,
+    about: String,
+}
+
+// fn test_load_members() -> Result<Vec<Member>> {
+//     let data = r#"
+//         [{
+//             "address": "5Gn9n7SDJ7VgHqHQWYzkSA4vX6DCmS5TFWdHxikTXp9b4L32",
+//             "handle": "mokhtar",
+//             "avatar_uri": "http://mokhtar.net/avatar.png",
+//             "about": "Mokhtar"
+//         }]"#;
+
+//     serde_json::from_str(data)
+// }
+
+fn parse_members_json() -> Result<Vec<Member>> {
+    let data = include_str!("../res/acropolis_members.json");
+    serde_json::from_str(data)
+}
+
+pub fn decode_address(address: String) -> AccountId32 {
+    AccountId32::from_ss58check(address.as_ref()).expect("failed to decode account id")
+}
+
+pub fn initial_members() -> Vec<(AccountId32, String, String, String)> {
+    let members = parse_members_json().expect("failed parsing members data");
+
+    members
+        .into_iter()
+        .map(|member| {
+            (
+                decode_address(member.address),
+                member.handle,
+                member.avatar_uri,
+                member.about,
+            )
+        })
+        .collect()
+}

src/service.rs

@@ -14,176 +14,338 @@
 // You should have received a copy of the GNU General Public License
 // along with Joystream node.  If not, see <http://www.gnu.org/licenses/>.
 
-//! Service and ServiceFactory implementation. Specialized wrapper over Substrate service.
-
 #![warn(unused_extern_crates)]
 
-use basic_authorship::ProposerFactory;
-use consensus::{import_queue, start_aura, AuraImportQueue, NothingExtra, SlotDuration};
-use grandpa;
+//! Service and ServiceFactory implementation. Specialized wrapper over substrate service.
+
+use client_db::Backend;
+use grandpa::{self, FinalityProofProvider as GrandpaFinalityProofProvider};
 use inherents::InherentDataProviders;
-use joystream_node_runtime::{self, opaque::Block, GenesisConfig, RuntimeApi};
-use log::info;
-use network::construct_simple_protocol;
-use primitives::{crypto::Ss58Codec, ed25519::Pair, sr25519::Public as SrPublic, Pair as PairT};
+use network::{construct_simple_protocol, NetworkService};
+use node_runtime::{self, opaque::Block, GenesisConfig, RuntimeApi};
+use offchain::OffchainWorkers;
+use primitives::Blake2Hasher;
+use runtime_primitives::traits::Block as BlockT;
 use std::sync::Arc;
-use std::time::Duration;
-use substrate_client as client;
-use substrate_executor::native_executor_instance;
-use substrate_service::construct_service_factory;
+use substrate_client::{Client, LocalCallExecutor, LongestChain};
+pub use substrate_executor::{native_executor_instance, NativeExecutor};
 use substrate_service::{
-    FactoryFullConfiguration, FullBackend, FullClient, FullComponents, FullExecutor, LightBackend,
-    LightClient, LightComponents, LightExecutor, TaskExecutor,
+    error::Error as ServiceError, AbstractService, Configuration, NetworkStatus, Service,
+    ServiceBuilder,
 };
 use transaction_pool::{self, txpool::Pool as TransactionPool};
 
-// Get new prefixed ss58 address encoding for ed25519 public keys
-fn ed_ss58check(public_key: &primitives::ed25519::Public) -> String {
-    let raw_bytes: &[u8; 32] = public_key.as_ref();
-    // Interpret bytes as sr25519 public key
-    let v: SrPublic = SrPublic::from_raw(raw_bytes.clone());
-    v.to_ss58check()
+construct_simple_protocol! {
+    /// Demo protocol attachment for substrate.
+    pub struct NodeProtocol where Block = Block { }
 }
 
-pub use substrate_executor::NativeExecutor;
-// Our native executor instance.
+// Declare an instance of the native executor named `Executor`. Include the wasm binary as the
+// equivalent wasm code.
 native_executor_instance!(
 	pub Executor,
-	joystream_node_runtime::api::dispatch,
-	joystream_node_runtime::native_version,
-	include_bytes!("../substrate-runtime-joystream/wasm/target/wasm32-unknown-unknown/release/joystream_node_runtime_wasm.compact.wasm")
+	node_runtime::api::dispatch,
+	node_runtime::native_version
 );
 
-pub struct NodeConfig<F: substrate_service::ServiceFactory> {
-    pub grandpa_import_setup: Option<(
-        Arc<grandpa::BlockImportForService<F>>,
-        grandpa::LinkHalfForService<F>,
-    )>,
-    inherent_data_providers: InherentDataProviders,
+/// Starts a `ServiceBuilder` for a full service.
+///
+/// Use this macro if you don't actually need the full service, but just the builder in order to
+/// be able to perform chain operations.
+#[macro_export]
+macro_rules! new_full_start {
+    ($config:expr) => {{
+        // type RpcExtension = jsonrpc_core::IoHandler<substrate_rpc::Metadata>;
+        let mut import_setup = None;
+        let inherent_data_providers = inherents::InherentDataProviders::new();
+
+        let builder = substrate_service::ServiceBuilder::new_full::<
+            node_runtime::opaque::Block,
+            node_runtime::RuntimeApi,
+            crate::service::Executor,
+        >($config)?
+        .with_select_chain(|_config, backend| {
+            Ok(substrate_client::LongestChain::new(backend.clone()))
+        })?
+        .with_transaction_pool(|config, client| {
+            Ok(transaction_pool::txpool::Pool::new(
+                config,
+                transaction_pool::FullChainApi::new(client),
+            ))
+        })?
+        .with_import_queue(|_config, client, mut select_chain, _transaction_pool| {
+            let select_chain = select_chain
+                .take()
+                .ok_or_else(|| substrate_service::Error::SelectChainRequired)?;
+            let (grandpa_block_import, grandpa_link) =
+                grandpa::block_import::<_, _, _, node_runtime::RuntimeApi, _>(
+                    client.clone(),
+                    &*client,
+                    select_chain,
+                )?;
+            let justification_import = grandpa_block_import.clone();
+
+            let (block_import, babe_link) = babe::block_import(
+                babe::Config::get_or_compute(&*client)?,
+                grandpa_block_import,
+                client.clone(),
+                client.clone(),
+            )?;
+
+            let import_queue = babe::import_queue(
+                babe_link.clone(),
+                block_import.clone(),
+                Some(Box::new(justification_import)),
+                None,
+                client.clone(),
+                client,
+                inherent_data_providers.clone(),
+            )?;
+
+            import_setup = Some((block_import, grandpa_link, babe_link));
+            Ok(import_queue)
+        })?;
+        // We don't have any custom rpc commands...
+        // .with_rpc_extensions(|client, pool| -> RpcExtension {
+        // 	node_rpc::create(client, pool)
+        // })?;
+
+        (builder, import_setup, inherent_data_providers)
+    }};
 }
 
-impl<F> Default for NodeConfig<F>
-where
-    F: substrate_service::ServiceFactory,
-{
-    fn default() -> NodeConfig<F> {
-        NodeConfig {
-            grandpa_import_setup: None,
-            inherent_data_providers: InherentDataProviders::new(),
+/// Creates a full service from the configuration.
+///
+/// We need to use a macro because the test suit doesn't work with an opaque service. It expects
+/// concrete types instead.
+macro_rules! new_full {
+	($config:expr, $with_startup_data: expr) => {{
+		use futures::sync::mpsc;
+		use network::DhtEvent;
+
+		let (
+			is_authority,
+			force_authoring,
+			name,
+			disable_grandpa
+		) = (
+			$config.roles.is_authority(),
+			$config.force_authoring,
+			$config.name.clone(),
+			$config.disable_grandpa
+		);
+
+        // sentry nodes announce themselves as authorities to the network
+		// and should run the same protocols authorities do, but it should
+		// never actively participate in any consensus process.
+        let participates_in_consensus = is_authority && !$config.sentry_mode;
+
+		let (builder, mut import_setup, inherent_data_providers) = new_full_start!($config);
+
+		// Dht event channel from the network to the authority discovery module. Use bounded channel to ensure
+		// back-pressure. Authority discovery is triggering one event per authority within the current authority set.
+		// This estimates the authority set size to be somewhere below 10 000 thereby setting the channel buffer size to
+		// 10 000.
+		let (dht_event_tx, _dht_event_rx) =
+			mpsc::channel::<DhtEvent>(10_000);
+
+		let service = builder.with_network_protocol(|_| Ok(crate::service::NodeProtocol::new()))?
+			.with_finality_proof_provider(|client, backend|
+				Ok(Arc::new(grandpa::FinalityProofProvider::new(backend, client)) as _)
+			)?
+			.with_dht_event_tx(dht_event_tx)?
+			.build()?;
+
+		let (block_import, grandpa_link, babe_link) = import_setup.take()
+				.expect("Link Half and Block Import are present for Full Services or setup failed before. qed");
+
+		($with_startup_data)(&block_import, &babe_link);
+
+		if participates_in_consensus {
+			let proposer = substrate_basic_authorship::ProposerFactory {
+				client: service.client(),
+				transaction_pool: service.transaction_pool(),
+			};
+
+			let client = service.client();
+			let select_chain = service.select_chain()
+				.ok_or(substrate_service::Error::SelectChainRequired)?;
+
+			let babe_config = babe::BabeParams {
+				keystore: service.keystore(),
+				client,
+				select_chain,
+				env: proposer,
+				block_import,
+				sync_oracle: service.network(),
+				inherent_data_providers: inherent_data_providers.clone(),
+				force_authoring,
+				babe_link,
+			};
+
+			let babe = babe::start_babe(babe_config)?;
+			service.spawn_essential_task(babe);
         }
-    }
+
+        // if the node isn't actively participating in consensus then it doesn't
+		// need a keystore, regardless of which protocol we use below.
+		let keystore = if participates_in_consensus {
+			Some(service.keystore())
+		} else {
+			None
+        };
+
+        let config = grandpa::Config {
+            // FIXME #1578 make this available through chainspec
+            gossip_duration: std::time::Duration::from_millis(333),
+            justification_period: 512,
+            name: Some(name),
+            observer_enabled: true,
+            keystore,
+            is_authority,
+        };
+
+		match (is_authority, disable_grandpa) {
+			(false, false) => {
+				// start the lightweight GRANDPA observer
+				service.spawn_task(Box::new(grandpa::run_grandpa_observer(
+					config,
+					grandpa_link,
+					service.network(),
+					service.on_exit(),
+				)?));
+			},
+			(true, false) => {
+				// start the full GRANDPA voter
+				let grandpa_config = grandpa::GrandpaParams {
+					config: config,
+					link: grandpa_link,
+					network: service.network(),
+					inherent_data_providers: inherent_data_providers.clone(),
+					on_exit: service.on_exit(),
+					telemetry_on_connect: Some(service.telemetry_on_connect_stream()),
+					voting_rule: grandpa::VotingRulesBuilder::default().build(),
+                };
+                // the GRANDPA voter task is considered infallible, i.e.
+				// if it fails we take down the service with it.
+				service.spawn_essential_task(grandpa::run_grandpa_voter(grandpa_config)?);
+			},
+			(_, true) => {
+				grandpa::setup_disabled_grandpa(
+					service.client(),
+					&inherent_data_providers,
+					service.network(),
+				)?;
+			},
+		}
+
+		Ok((service, inherent_data_providers))
+	}};
+	($config:expr) => {{
+		new_full!($config, |_, _| {})
+	}}
 }
 
-construct_simple_protocol! {
-    /// Demo protocol attachment for substrate.
-    pub struct NodeProtocol where Block = Block { }
+#[allow(dead_code)]
+type ConcreteBlock = node_runtime::opaque::Block;
+#[allow(dead_code)]
+type ConcreteClient = Client<
+    Backend<ConcreteBlock>,
+    LocalCallExecutor<Backend<ConcreteBlock>, NativeExecutor<Executor>>,
+    ConcreteBlock,
+    node_runtime::RuntimeApi,
+>;
+#[allow(dead_code)]
+type ConcreteBackend = Backend<ConcreteBlock>;
+
+/// A specialized configuration object for setting up the node..
+pub type NodeConfiguration<C> =
+    Configuration<C, GenesisConfig /*, crate::chain_spec::Extensions*/>;
+
+/// Builds a new service for a full client.
+pub fn new_full<C: Send + Default + 'static>(config: NodeConfiguration<C>)
+-> Result<
+	Service<
+		ConcreteBlock,
+		ConcreteClient,
+		LongestChain<ConcreteBackend, ConcreteBlock>,
+		NetworkStatus<ConcreteBlock>,
+		NetworkService<ConcreteBlock, crate::service::NodeProtocol, <ConcreteBlock as BlockT>::Hash>,
+		TransactionPool<transaction_pool::FullChainApi<ConcreteClient, ConcreteBlock>>,
+		OffchainWorkers<
+			ConcreteClient,
+			<ConcreteBackend as substrate_client::backend::Backend<Block, Blake2Hasher>>::OffchainStorage,
+			ConcreteBlock,
+		>
+	>,
+	ServiceError,
+>
+{
+    new_full!(config).map(|(service, _)| service)
 }
 
-construct_service_factory! {
-    struct Factory {
-        Block = Block,
-        RuntimeApi = RuntimeApi,
-        NetworkProtocol = NodeProtocol { |config| Ok(NodeProtocol::new()) },
-        RuntimeDispatch = Executor,
-        FullTransactionPoolApi = transaction_pool::ChainApi<client::Client<FullBackend<Self>, FullExecutor<Self>, Block, RuntimeApi>, Block>
-            { |config, client| Ok(TransactionPool::new(config, transaction_pool::ChainApi::new(client))) },
-        LightTransactionPoolApi = transaction_pool::ChainApi<client::Client<LightBackend<Self>, LightExecutor<Self>, Block, RuntimeApi>, Block>
-            { |config, client| Ok(TransactionPool::new(config, transaction_pool::ChainApi::new(client))) },
-        Genesis = GenesisConfig,
-        Configuration = NodeConfig<Self>,
-        FullService = FullComponents<Self>
-            { |config: FactoryFullConfiguration<Self>, executor: TaskExecutor|
-                FullComponents::<Factory>::new(config, executor)
-            },
-        AuthoritySetup = {
-            |mut service: Self::FullService, executor: TaskExecutor, key: Option<Arc<Pair>>| {
-                let (block_import, link_half) = service.config.custom.grandpa_import_setup.take()
-                    .expect("Link Half and Block Import are present for Full Services or setup failed before. qed");
-
-                if let Some(ref key) = key {
-                    info!("Using authority key {}", ed_ss58check(&key.public()));
-                    let proposer = Arc::new(ProposerFactory {
-                        client: service.client(),
-                        transaction_pool: service.transaction_pool(),
-                        inherents_pool: service.inherents_pool(),
-                    });
-
-                    let client = service.client();
-                    executor.spawn(start_aura(
-                        SlotDuration::get_or_compute(&*client)?,
-                        key.clone(),
-                        client,
-                        block_import.clone(),
-                        proposer,
-                        service.network(),
-                        service.on_exit(),
-                        service.config.custom.inherent_data_providers.clone(),
-                        service.config.force_authoring,
-                    )?);
-
-                    info!("Running Grandpa session as Authority {}", ed_ss58check(&key.public()));
-                }
-
-                let key = if service.config.disable_grandpa {
-                    None
-                } else {
-                    key
-                };
+/// Builds a new service for a light client.
+pub fn new_light<C: Send + Default + 'static>(
+    config: NodeConfiguration<C>,
+) -> Result<impl AbstractService, ServiceError> {
+    // type RpcExtension = jsonrpc_core::IoHandler<substrate_rpc::Metadata>;
+    let inherent_data_providers = InherentDataProviders::new();
 
-                executor.spawn(grandpa::run_grandpa(
-                    grandpa::Config {
-                        local_key: key,
-                        // FIXME #1578 make this available through chainspec
-                        gossip_duration: Duration::from_millis(333),
-                        justification_period: 4096,
-                        name: Some(service.config.name.clone())
-                    },
-                    link_half,
-                    grandpa::NetworkBridge::new(service.network()),
-                    service.config.custom.inherent_data_providers.clone(),
-                    service.on_exit(),
-                )?);
-
-                Ok(service)
-            }
-        },
-        LightService = LightComponents<Self>
-            { |config, executor| <LightComponents<Factory>>::new(config, executor) },
-        FullImportQueue = AuraImportQueue<
-            Self::Block,
-        >
-            { |config: &mut FactoryFullConfiguration<Self> , client: Arc<FullClient<Self>>| {
-                let slot_duration = SlotDuration::get_or_compute(&*client)?;
-                let (block_import, link_half) =
-                    grandpa::block_import::<_, _, _, RuntimeApi, FullClient<Self>>(
-                        client.clone(), client.clone()
-                    )?;
-                let block_import = Arc::new(block_import);
-                let justification_import = block_import.clone();
-
-                config.custom.grandpa_import_setup = Some((block_import.clone(), link_half));
-
-                import_queue::<_, _, _, Pair>(
-                    slot_duration,
-                    block_import,
-                    Some(justification_import),
-                    client,
-                    NothingExtra,
-                    config.custom.inherent_data_providers.clone(),
-                ).map_err(Into::into)
-            }},
-        LightImportQueue = AuraImportQueue<
-            Self::Block,
-        >
-            { |config: &mut FactoryFullConfiguration<Self>, client: Arc<LightClient<Self>>|
-                import_queue::<_, _, _, Pair>(
-                    SlotDuration::get_or_compute(&*client)?,
+    let service = ServiceBuilder::new_light::<Block, RuntimeApi, Executor>(config)?
+        .with_select_chain(|_config, backend| Ok(LongestChain::new(backend.clone())))?
+        .with_transaction_pool(|config, client| {
+            Ok(TransactionPool::new(
+                config,
+                transaction_pool::FullChainApi::new(client),
+            ))
+        })?
+        .with_import_queue_and_fprb(
+            |_config, client, backend, fetcher, _select_chain, _tx_pool| {
+                let fetch_checker = fetcher
+                    .map(|fetcher| fetcher.checker().clone())
+                    .ok_or_else(|| {
+                        "Trying to start light import queue without active fetch checker"
+                    })?;
+                let grandpa_block_import = grandpa::light_block_import::<_, _, _, RuntimeApi>(
+                    client.clone(),
+                    backend,
+                    &*client,
+                    Arc::new(fetch_checker),
+                )?;
+
+                let finality_proof_import = grandpa_block_import.clone();
+                let finality_proof_request_builder =
+                    finality_proof_import.create_finality_proof_request_builder();
+
+                let (babe_block_import, babe_link) = babe::block_import(
+                    babe::Config::get_or_compute(&*client)?,
+                    grandpa_block_import,
                     client.clone(),
+                    client.clone(),
+                )?;
+
+                let import_queue = babe::import_queue(
+                    babe_link,
+                    babe_block_import,
                     None,
+                    Some(Box::new(finality_proof_import)),
+                    client.clone(),
                     client,
-                    NothingExtra,
-                    config.custom.inherent_data_providers.clone(),
-                ).map_err(Into::into)
+                    inherent_data_providers.clone(),
+                )?;
+
+                Ok((import_queue, finality_proof_request_builder))
             },
-    }
+        )?
+        .with_network_protocol(|_| Ok(NodeProtocol::new()))?
+        .with_finality_proof_provider(|client, backend| {
+            Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, client)) as _)
+        })?
+        // We don't have any custom rpc extensions
+        // .with_rpc_extensions(|client, pool| -> RpcExtension {
+        // 	node_rpc::create(client, pool)
+        // })?
+        .build()?;
+
+    Ok(service)
 }