Merge pull request #1717 from mnaamani/runtime_upgrade_ci_and_build_update

Runtime upgrade ci and build update

.dockerignore

@@ -1,4 +1,4 @@
-**target*
+target/
 **node_modules*
 .tmp/
 .vscode/

.env

@@ -0,0 +1,57 @@
+COMPOSE_PROJECT_NAME=joystream
+
+###########################
+#     Common settings     #
+###########################
+
+# The env variables below are by default used by all services and should be
+# overriden in local env files (e.g. ./generated/indexer) if needed
+# DB config
+DB_NAME=query_node
+DB_USER=postgres
+DB_PASS=postgres
+DB_HOST=localhost
+DB_PORT=5432
+DEBUG=index-builder:*
+TYPEORM_LOGGING=error
+
+###########################
+#    Indexer options      #
+###########################
+
+# Substrate endpoint to source events from
+WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944/
+# Block height to start indexing from.
+# Note, that if there are already some indexed events, this setting is ignored
+BLOCK_HEIGHT=0
+
+# Custom types to register for Substrate API
+# TYPE_REGISTER_PACKAGE_NAME=
+# TYPE_REGISTER_PACKAGE_VERSION=
+# TYPE_REGISTER_FUNCTION=
+
+# Redis cache server
+REDIS_URI=redis://localhost:6379/0
+
+###########################
+#    Processor options    #
+###########################
+
+# Where the mapping scripts are located, relative to ./generated/indexer
+TYPES_JSON=../../typedefs.json
+
+# Indexer GraphQL API endpoint to fetch indexed events
+INDEXER_ENDPOINT_URL=http://localhost:4100/graphql
+
+# Block height from which the processor starts. Note that if
+# there are already processed events in the database, this setting is ignored
+BLOCK_HEIGHT=0
+
+###############################
+#    Processor GraphQL API    #
+###############################
+
+GRAPHQL_SERVER_PORT=4002
+GRAPHQL_SERVER_HOST=localhost
+WARTHOG_APP_PORT=4002
+WARTHOG_APP_HOST=localhost

.github/workflows/run-network-tests.yml

@@ -100,7 +100,7 @@ jobs:
       - name: Ensure tests are runnable
         run: yarn workspace network-tests build
       - name: Execute network tests
-        run: RUNTIME=alexandria tests/network-tests/run-tests.sh full
+        run: RUNTIME=babylon tests/network-tests/run-tests.sh full
 
   basic_runtime:
     name: Integration Tests (New Chain)
@@ -148,7 +148,7 @@ jobs:
       - name: Ensure tests are runnable
         run: yarn workspace cd-schemas checks --quiet
       - name: Start chain
-        run: docker-compose up -d
+        run: docker-compose up -d joystream-node
       - name: Initialize the content directory
         run: yarn workspace cd-schemas initialize:dev
 
@@ -202,10 +202,14 @@ jobs:
       - name: Build storage node
         run: yarn workspace storage-node build
       - name: Start Services
-        run: docker-compose --file docker-compose-with-storage.yml up -d
-      - name: Add development storage node and initialize content directory
-        run: DEBUG=* yarn storage-cli dev-init
-      - name: Try uploading
+        run: |
+          docker-compose up -d ipfs
+          docker-compose up -d joystream-node
+      - name: Configure and start development storage node
+        run: |
+          DEBUG=* yarn storage-cli dev-init
+          docker-compose up -d colossus
+      - name: Test uploading
         run: |
           WAIT_TIME=90
           export DEBUG=joystream:*

README.md

@@ -108,9 +108,7 @@ A step by step guide to setup a full node and validator on the Joystream testnet
 ### Integration tests
 
 ```bash
-docker-compose up -d
-DEBUG=* yarn workspace network-tests test-run src/scenarios/full.ts
-docker-compose down
+tests/network-tests/run-tests.sh
 ```
 
 ### Contributing

apps.Dockerfile

@@ -7,9 +7,7 @@ COPY . /joystream
 # to ensure dev dependencies are installed.
 RUN yarn install --frozen-lockfile
 
-# Pioneer is failing to build only on github actions workflow runner
-# Error: packages/page-staking/src/index.tsx(24,21): error TS2307: Cannot find module './Targets' or its corresponding type declarations.
-# RUN yarn workspace pioneer build
+RUN yarn workspace pioneer build
 RUN yarn workspace storage-node build
 RUN yarn workspace query-node-root build
 

build.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+set -e
+
+yarn
+yarn workspace @joystream/types build
+yarn workspace cd-schemas generate:all
+yarn workspace cd-schemas build
+yarn workspace @joystream/cli build
+yarn workspace query-node-root build
+yarn workspace storage-node build
+# Not strictly needed during development, we run "yarn workspace pioneer start" to start
+# a dev instance, but will show highlight build issues
+yarn workspace pioneer build
+
+if ! command -v docker-compose &> /dev/null
+then
+  echo "docker-compose not found, skipping docker build!"
+else
+  # Build joystream/apps docker image
+  docker-compose build pioneer
+
+  # Optionally build joystream/node docker image
+  # TODO: Try to fetch a cached joystream/node image
+  # if one is found matching code shasum instead of building
+  while true
+  do
+    read -p "Rebuild joystream/node docker image? (y/N): " answer2
+
+    case $answer2 in
+    [yY]* ) docker-compose build joystream-node
+            break;;
+
+    [nN]* ) break;;
+
+    * )     break;;
+    esac
+  done
+fi
+
+# Build cargo crates: native binaries joystream/node, wasm runtime, and chainspec builder.
+while true
+do
+  read -p "Compile joystream node native binary? (y/N): " answer1
+
+  case $answer1 in
+   [yY]* ) yarn cargo-checks
+           yarn cargo-build
+           break;;
+
+   [nN]* ) break;;
+
+   * )     break;;
+  esac
+done

cli/package.json

@@ -125,5 +125,9 @@
     "format": "prettier ./ --write",
     "generate:schema-typings": "rm -rf ./src/json-schemas/typings && json2ts -i ./src/json-schemas/ -o ./src/json-schemas/typings/"
   },
-  "types": "lib/index.d.ts"
+  "types": "lib/index.d.ts",
+  "volta": {
+    "node": "12.18.2",
+    "yarn": "1.22.4"
+  }
 }

content-directory-schemas/package.json

@@ -44,5 +44,9 @@
   "bugs": {
     "url": "https://github.com/Joystream/joystream/issues"
   },
-  "homepage": "https://github.com/Joystream/joystream"
+  "homepage": "https://github.com/Joystream/joystream",
+  "volta": {
+    "node": "12.18.2",
+    "yarn": "1.22.4"
+  }
 }

docker-compose-with-storage.yml

@@ -1,39 +0,0 @@
-version: '3'
-services:
-  ipfs:
-    image: ipfs/go-ipfs:latest
-    ports:
-      - '127.0.0.1:5001:5001'
-      - '127.0.0.1:8080:8080'
-    entrypoint: ''
-    command: |
-      /bin/sh -c "
-        set -e
-        /usr/local/bin/start_ipfs config profile apply lowpower
-        /usr/local/bin/start_ipfs config --json Gateway.PublicGateways '{\"localhost\": null }'
-        /sbin/tini -- /usr/local/bin/start_ipfs daemon --migrate=true
-      "
-  chain:
-    image: joystream/node
-    build:
-      context: .
-      dockerfile: joystream-node.Dockerfile
-    ports:
-      - '127.0.0.1:9944:9944'
-    command: --dev --ws-external --base-path /data --log runtime
-
-  colossus:
-    image: joystream/apps
-    restart: on-failure
-    depends_on:
-      - "chain"
-      - "ipfs"
-    build:
-      context: .
-      dockerfile: apps.Dockerfile
-    ports:
-      - '127.0.0.1:3001:3001'
-    command: colossus --dev --ws-provider ws://chain:9944 --ipfs-host ipfs
-    environment:
-      - DEBUG=*
-

docker-compose.yml

@@ -1,17 +1,150 @@
-# Compiles new joystream node image if local image not found,
-# and runs local development chain.
-# To prevent build run docker-compose with "--no-build" arg
-version: "3"
+# Compiles new joystream/node and joystream/apps images if local images not found
+# and runs a complete joystream development network
+# To prevent build of docker images run docker-compose with "--no-build" arg
+version: "3.4"
 services:
   joystream-node:
-    image: joystream/node
+    image: joystream/node:latest
     build:
       # context is relative to the compose file
       context: .
       # dockerfile is relative to the context
       dockerfile: joystream-node.Dockerfile
     container_name: joystream-node
-    command: --dev --alice --validator --unsafe-ws-external --rpc-cors=all --log runtime
+    volumes:
+      - /data
+    command: --dev --alice --validator --unsafe-ws-external --rpc-cors=all --log runtime --base-path /data
     ports:
-      - "9944:9944"
-  
+      - "127.0.0.1:9944:9944"
+
+  ipfs:
+    image: ipfs/go-ipfs:latest
+    ports:
+      - '127.0.0.1:5001:5001'
+      - '127.0.0.1:8080:8080'
+    volumes:
+      - /data/ipfs
+    entrypoint: ''
+    command: |
+      /bin/sh -c "
+        set -e
+        /usr/local/bin/start_ipfs config profile apply lowpower
+        /usr/local/bin/start_ipfs config --json Gateway.PublicGateways '{\"localhost\": null }'
+        /sbin/tini -- /usr/local/bin/start_ipfs daemon --migrate=true
+      "
+
+  colossus:
+    image: joystream/apps
+    restart: on-failure
+    depends_on:
+      - "joystream-node"
+      - "ipfs"
+    build:
+      context: .
+      dockerfile: apps.Dockerfile
+    ports:
+      - '127.0.0.1:3001:3001'
+    command: colossus --dev --ws-provider ${WS_PROVIDER_ENDPOINT_URI} --ipfs-host ipfs
+    environment:
+      - DEBUG=*
+
+  db:
+    image: postgres:12
+    restart: always
+    ports:
+      - "127.0.0.1:${DB_PORT}:5432"
+    volumes:
+      - /var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: ${DB_USER}
+      POSTGRES_PASSWORD: ${DB_PASS}
+      POSTGRES_DB: ${DB_NAME}
+
+  graphql-server:
+    image: joystream/apps
+    restart: unless-stopped
+    build: 
+      context: .
+      dockerfile: apps.Dockerfile
+    env_file:
+      # relative to working directory where docker-compose was run from 
+      - .env
+    environment:
+      - DB_HOST=db
+    ports:
+      - "127.0.0.1:8081:${GRAPHQL_SERVER_PORT}"
+    depends_on: 
+      - db
+    command: ["workspace", "query-node-root", "server:start:prod"]
+
+  processor:
+    image: joystream/apps
+    restart: unless-stopped
+    build: 
+      context: .
+      dockerfile: apps.Dockerfile
+    env_file:
+      # relative to working directory where docker-compose was run from 
+      - .env
+    environment:
+      - INDEXER_ENDPOINT_URL=http://indexer-api-gateway:4000/graphql
+      - DB_HOST=db
+      - TYPEORM_HOST=db
+      - DEBUG=index-builder:*
+      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
+    depends_on:
+      - indexer-api-gateway
+    command: ["workspace", "query-node-root", "processor:start"]
+
+  indexer:
+    image: joystream/apps
+    restart: unless-stopped
+    build: 
+      context: .
+      dockerfile: apps.Dockerfile
+    env_file:
+      # relative to working directory where docker-compose was run from 
+      - .env 
+    environment:
+      - TYPEORM_HOST=db
+      - INDEXER_WORKERS=5
+      - PROCESSOR_POLL_INTERVAL=1000 # refresh every second 
+      - REDIS_URI=redis://redis:6379/0
+      - DEBUG=index-builder:*
+      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
+    depends_on: 
+      - db
+    command: ["workspace", "query-node-root", "indexer:start"] 
+
+  indexer-api-gateway:
+    image: joystream/hydra-indexer-gateway:latest
+    restart: unless-stopped
+    environment:
+      - WARTHOG_STARTER_DB_DATABASE=${DB_NAME}
+      - WARTHOG_STARTER_DB_HOST=db 
+      - WARTHOG_STARTER_DB_PASSWORD=${DB_PASS}
+      - WARTHOG_STARTER_DB_PORT=${DB_PORT}
+      - WARTHOG_STARTER_DB_USERNAME=${DB_USER}
+      - WARTHOG_STARTER_REDIS_URI=redis://redis:6379/0 
+      - PORT=4000
+    ports:
+      - "127.0.0.1:4000:4000"
+    depends_on:
+      - redis
+      - db
+      - indexer
+
+  redis:
+    image: redis:6.0-alpine
+    restart: always
+    ports:
+      - "127.0.0.1:6379:6379"
+
+  pioneer:
+    image: joystream/apps
+    build:
+      context: .
+      dockerfile: apps.Dockerfile
+    ports:
+      - "127.0.0.1:3000:3000"
+    command: workspace pioneer start

joystream-node.Dockerfile

@@ -1,4 +1,11 @@
-FROM joystream/rust-builder AS builder
+FROM liuchong/rustup:1.47.0 AS rustup
+RUN rustup component add rustfmt clippy
+RUN rustup install nightly-2020-10-06 --force
+RUN rustup target add wasm32-unknown-unknown --toolchain nightly-2020-10-06
+RUN apt-get update && \
+  apt-get install -y curl git gcc xz-utils sudo pkg-config unzip clang libc6-dev-i386
+
+FROM rustup AS builder
 LABEL description="Compiles all workspace artifacts"
 WORKDIR /joystream
 COPY . /joystream

package.json

@@ -5,6 +5,8 @@
   "license": "GPL-3.0-only",
   "scripts": {
     "postinstall": "yarn workspace @joystream/types build && yarn workspace cd-schemas generate:all && yarn workspace cd-schemas build && yarn workspace @joystream/cli build",
+    "build": "./build.sh",
+    "start": "./start.sh",
     "cargo-checks": "devops/git-hooks/pre-commit && devops/git-hooks/pre-push",
     "cargo-build": "scripts/cargo-build.sh"
   },
@@ -50,5 +52,9 @@
   "engines": {
     "node": ">=12.18.0",
     "yarn": "^1.22.0"
+  },
+  "volta": {
+    "node": "12.18.2",
+    "yarn": "1.22.4"
   }
 }

pioneer/package.json

@@ -93,5 +93,8 @@
     "sass-loader": "^8.0.0",
     "style-loader": "^1.0.0",
     "@joystream/types": "link:../types"
+  },
+  "volta": {
+    "extends": "../package.json"
   }
 }

query-node/.env

@@ -1,3 +1,5 @@
+COMPOSE_PROJECT_NAME=joystream
+
 # Project name
 PROJECT_NAME=query_node
 

query-node/docker-compose.yml

@@ -1,91 +0,0 @@
-version: "3.4"
-
-services:
-  db:
-    image: postgres:12
-    restart: always
-    ports:
-      - "${DB_PORT}:5432"
-    volumes:
-      - /var/lib/postgresql/data
-    environment:
-      POSTGRES_USER: ${DB_USER}
-      POSTGRES_PASSWORD: ${DB_PASS}
-      POSTGRES_DB: ${DB_NAME}
-
-  graphql-server:
-    image: joystream/apps
-    restart: unless-stopped
-    build: 
-      context: ../
-      dockerfile: apps.Dockerfile
-    env_file:
-      - .env
-    environment:
-      - DB_HOST=db
-    ports:
-      - "8080:${GRAPHQL_SERVER_PORT}"
-    depends_on: 
-      - db
-    command: ["workspace", "query-node-root", "server:start:prod"]
-
-  processor:
-    image: joystream/apps
-    restart: unless-stopped
-    build: 
-      context: ../
-      dockerfile: apps.Dockerfile
-    env_file:
-      - .env
-    environment:
-      - INDEXER_ENDPOINT_URL=http://indexer-api-gateway:4000/graphql
-      - DB_HOST=db
-      - TYPEORM_HOST=db
-      - DEBUG=index-builder:*
-      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
-    depends_on:
-      - indexer-api-gateway
-    command: ["workspace", "query-node-root", "processor:start"]
-  
-  indexer:
-    image: joystream/apps
-    restart: unless-stopped
-    build: 
-      context: ../
-      dockerfile: apps.Dockerfile
-    env_file:
-      - .env 
-    environment:
-      - TYPEORM_HOST=db
-      - INDEXER_WORKERS=5
-      - PROCESSOR_POLL_INTERVAL=1000 # refresh every second 
-      - REDIS_URI=redis://redis:6379/0
-      - DEBUG=index-builder:*
-      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
-    depends_on: 
-      - db
-    command: ["workspace", "query-node-root", "indexer:start"] 
-  
-  indexer-api-gateway:
-    image: joystream/hydra-indexer-gateway:latest
-    restart: unless-stopped
-    environment:
-      - WARTHOG_STARTER_DB_DATABASE=${DB_NAME}
-      - WARTHOG_STARTER_DB_HOST=db 
-      - WARTHOG_STARTER_DB_PASSWORD=${DB_PASS}
-      - WARTHOG_STARTER_DB_PORT=${DB_PORT}
-      - WARTHOG_STARTER_DB_USERNAME=${DB_USER}
-      - WARTHOG_STARTER_REDIS_URI=redis://redis:6379/0 
-      - PORT=4000
-    ports:
-      - "4000:4000"
-    depends_on:
-      - redis
-      - db
-      - indexer
-    
-  redis:
-    image: redis:6.0-alpine
-    restart: always
-    ports:
-      - "6379:6379"

query-node/package.json

@@ -16,23 +16,27 @@
 		"db:schema:migrate": "(cd ./generated/graphql-server && yarn db:create && yarn db:sync && yarn db:migrate)",
 		"db:indexer:migrate": "(cd ./generated/indexer && yarn db:migrate)",
 		"db:migrate": "yarn db:schema:migrate && yarn db:indexer:migrate",
-		"codegen:all": "yarn hydra-cli codegen && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
-		"codegen:indexer": "yarn hydra-cli codegen --no-graphql && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
-		"codegen:server": "yarn hydra-cli codegen --no-indexer",
-		"docker:up": "docker-compose up -d",
-    "cd-classes": "ts-node scripts/get-class-id-and-name.ts"
+		"codegen:all": "yarn hydra-cli codegen --no-install && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
+		"codegen:indexer": "yarn hydra-cli codegen --no-install --no-graphql && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
+		"codegen:server": "yarn hydra-cli codegen --no-install --no-indexer",
+		"cd-classes": "ts-node scripts/get-class-id-and-name.ts"
 	},
 	"author": "",
 	"license": "ISC",
 	"devDependencies": {
-		"@dzlzv/hydra-cli": "^0.0.19"
+		"@dzlzv/hydra-cli": "^0.0.20"
 	},
 	"dependencies": {
+		"@dzlzv/hydra-indexer-lib": "^0.0.19-legacy.1.26.1",
 		"@joystream/types": "^0.14.0",
 		"@types/bn.js": "^4.11.6",
 		"@types/debug": "^4.1.5",
 		"bn.js": "^5.1.2",
 		"debug": "^4.2.0",
+		"dotenvi": "^0.9.1",
 		"tslib": "^2.0.0"
+	},
+	"volta": {
+		"extends": "../package.json"
 	}
 }

query-node/run-tests.sh

@@ -7,8 +7,8 @@ cd $SCRIPT_PATH
 function cleanup() {
     # Show tail end of logs for the processor and indexer containers to
     # see any possible errors
-    (echo "## Processor Logs ##" && docker logs query-node_processor_1 --tail 50) || :
-    (echo "## Indexer Logs ##" && docker logs query-node_indexer_1 --tail 50) || :
+    (echo "## Processor Logs ##" && docker logs joystream_processor_1 --tail 50) || :
+    (echo "## Indexer Logs ##" && docker logs joystream_indexer_1 --tail 50) || :
     docker-compose down -v
 }
 
@@ -23,9 +23,14 @@ export WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944/
 # typeorm commandline is used by db:migrate step below.
 ln -s ../../../../../node_modules/typeorm/cli.js generated/graphql-server/node_modules/.bin/typeorm || :
 
-yarn db:up
+# clean start
+docker-compose down -v
+
+docker-compose up -d db
 yarn db:migrate
-yarn docker:up
+docker-compose up -d graphql-server
+# Starting up processor will bring up all services it depends on
+docker-compose up -d processor
 
 # Run tests
-ATTACH_TO_NETWORK=query-node_default ../tests/network-tests/run-tests.sh content-directory
+ATTACH_TO_NETWORK=joystream_default ../tests/network-tests/run-tests.sh content-directory

rust-builder.Dockerfile

@@ -1,8 +0,0 @@
-FROM liuchong/rustup:1.46.0 AS builder
-LABEL description="Rust and WASM build environment for joystream and substrate"
-
-WORKDIR /setup
-COPY setup.sh /setup
-ENV TERM=xterm
-
-RUN ./setup.sh

scripts/runtime-code-shasum.sh

@@ -7,17 +7,16 @@ export WORKSPACE_ROOT=`cargo metadata --offline --no-deps --format-version 1 | j
 
 cd ${WORKSPACE_ROOT}
 
-# srot/owner/group/mtime arguments only work with gnu version of tar.
-# So if you run this on Mac the default version of tar is `bsdtar`
-# and you will not get an idempotent result.
-# Install gnu-tar with brew
-#   brew install gnu-tar
-#   export PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH"
-tar -c --sort=name --owner=root:0 --group=root:0 --mtime='UTC 2020-01-01' \
+TAR=tar
+if [[ "$OSTYPE" == "darwin"* ]]; then
+	TAR=gtar
+fi
+
+# sort/owner/group/mtime arguments only work with gnu version of tar!
+${TAR} -c --sort=name --owner=root:0 --group=root:0 --mtime='UTC 2020-01-01' \
     Cargo.lock \
     Cargo.toml \
     runtime \
     runtime-modules \
     utils/chain-spec-builder \
     joystream-node.Dockerfile | shasum | cut -d " " -f 1
-

setup.sh

@@ -2,11 +2,8 @@
 
 set -e
 
-# If OS is supported will install:
-#  - build tools and any other dependencies required for rust and substrate
-#  - rustup - rust insaller
-#  - rust compiler and toolchains
-#  - skips installing substrate and subkey
+# If OS is supported will install build tools for rust and substrate.
+# Skips installing substrate itself and subkey
 curl https://getsubstrate.io -sSf | bash -s -- --fast
 
 source ~/.cargo/env
@@ -19,13 +16,21 @@ rustup component add rustfmt clippy
 rustup install nightly-2020-10-06 --force
 rustup target add wasm32-unknown-unknown --toolchain nightly-2020-10-06
 
-# Ensure the stable toolchain is still the default
-rustup default stable
+# Sticking with older version of compiler to ensure working build
+rustup install 1.47.0
+rustup default 1.47.0
 
-# TODO: Install additional tools...
+if [[ "$OSTYPE" == "linux-gnu" ]]; then
+    apt-get install -y coreutils clang jq curl gcc xz-utils sudo pkg-config unzip clang libc6-dev-i386
+    apt-get install -y docker.io docker-compose
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+    brew install b2sum gnu-tar jq curl
+    echo "It is recommended to setup Docker desktop from: https://www.docker.com/products/docker-desktop"
+fi
 
-# - b2sum
-# - nodejs
-# - npm
-# - yarn
-# .... ?
+# Volta nodejs, npm, yarn tools manager
+curl https://get.volta.sh | bash
+
+volta install node@12
+volta install yarn
+volta install npx

start.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -e
+
+# Run a complete joystream development network on your machine using docker.
+# Make sure to run build.sh prior to running this script.
+
+# Clean start!
+docker-compose down -v
+
+function down()
+{
+    # Stop containers and clear volumes
+    docker-compose down -v
+}
+
+trap down EXIT
+
+# Run a local development chain
+docker-compose up -d joystream-node
+
+## Storage Infrastructure
+# Configure a dev storage node and start storage node
+DEBUG=joystream:storage-cli:dev yarn storage-cli dev-init
+docker-compose up -d colossus
+# Initialise the content directory with standard classes, schemas and initial entities
+yarn workspace cd-schemas initialize:dev
+
+## Query Node Infrastructure
+# Initialize a new database for the query node infrastructure
+docker-compose up -d db
+yarn workspace query-node-root db:migrate
+# Startup all query-node infrastructure services
+docker-compose up -d graphql-server
+docker-compose up -d processor
+
+echo "press Ctrl+C to shutdown"
+
+# Start a dev instance of pioneer and wait for exit
+docker-compose up pioneer

storage-node/README.md

@@ -35,6 +35,7 @@ _Building_
 
 ```bash
 $ yarn install
+$ yarn build
 ```
 
 The command will install dependencies, and make a `colossus` executable available:

storage-node/docker-compose.yaml

@@ -1,29 +0,0 @@
-version: '3'
-services:
-  ipfs:
-    image: ipfs/go-ipfs:latest
-    ports:
-      - '127.0.0.1:5001:5001'
-      - '127.0.0.1:8080:8080'
-    volumes:
-      - ipfs-data:/data/ipfs
-    entrypoint: ''
-    command: |
-      /bin/sh -c "
-        set -e
-        /usr/local/bin/start_ipfs config profile apply lowpower
-        /usr/local/bin/start_ipfs config --json Gateway.PublicGateways '{\"localhost\": null }'
-        /sbin/tini -- /usr/local/bin/start_ipfs daemon --migrate=true
-      "
-  chain:
-    image: joystream/node:latest
-    ports:
-      - '127.0.0.1:9944:9944'
-    volumes:
-      - chain-data:/data
-    command: --dev --ws-external --base-path /data
-volumes:
-  ipfs-data:
-    driver: local
-  chain-data:
-    driver: local

storage-node/package.json

@@ -48,5 +48,8 @@
     "prettier": "^2.0.5",
     "typescript": "^3.9.6",
     "wsrun": "^3.6.5"
+  },
+  "volta": {
+    "extends": "../package.json"
   }
 }

storage-node/packages/cli/package.json

@@ -26,6 +26,9 @@
   "engines": {
     "node": ">=12.18.0"
   },
+  "volta": {
+    "extends": "../package.json"
+  },
   "scripts": {
     "test": "mocha 'dist/test/**/*.js'",
     "lint": "eslint --ext .js,.ts . && tsc --noEmit --pretty",

storage-node/packages/colossus/package.json

@@ -31,6 +31,9 @@
   "engines": {
     "node": ">=12.18.0"
   },
+  "volta": {
+    "extends": "../package.json"
+  },
   "scripts": {
     "test": "mocha 'test/**/*.js'",
     "lint": "eslint 'paths/**/*.js' 'lib/**/*.js'",

storage-node/packages/helios/package.json

@@ -14,5 +14,8 @@
     "@types/bn.js": "^4.11.5",
     "axios": "^0.19.0",
     "bn.js": "^4.11.8"
+  },
+  "volta": {
+    "extends": "../package.json"
   }
 }

storage-node/start-dev.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-# Avoid pulling joystream/node from docker hub. It is most likely
-# not the version that we want to work with. Either you should
-# build it locally or pull it down manually.
-if ! docker inspect joystream/node:latest > /dev/null 2>&1;
-then
-  echo "Didn't find local joystream/node:latest docker image."
-  exit 1
-fi
-
-SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
-cd $SCRIPT_PATH
-
-# stop prior run and clear volumes
-# docker-compose down -v
-
-# Run a development joystream-node chain and ipfs daemon in the background
-docker-compose up -d
-
-function down()
-{
-    # Stop containers and clear volumes
-    docker-compose down -v
-}
-
-trap down EXIT
-
-# configure the dev chain
-DEBUG=joystream:storage-cli:dev yarn storage-cli dev-init
-
-# Run the tests
-# Tests sometimes fail, so skip for now
-# yarn workspace storage-node test
-
-# Start Colossus storage-node
-DEBUG=joystream:* yarn colossus --dev

storage-node/stop-dev.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-# stop prior run and clear volumes
-docker-compose down -v

tests/network-tests/.env

@@ -1,7 +1,7 @@
 # Address of the Joystream node.
 NODE_URL = ws://127.0.0.1:9944
 # Address of the Joystream query node.
-QUERY_NODE_URL = http://127.0.0.1:8080/graphql
+QUERY_NODE_URL = http://127.0.0.1:8081/graphql
 # Account which is expected to provide sufficient funds to test accounts.
 TREASURY_ACCOUNT_URI = //Alice
 # Sudo Account

tests/network-tests/package.json

@@ -32,5 +32,8 @@
     "prettier": "2.0.2",
     "ts-node": "^8.8.1",
     "typescript": "^3.8.3"
+  },
+  "volta": {
+    "extends": "../../package.json"
   }
 }

types/package.json

@@ -59,5 +59,9 @@
   "bugs": {
     "url": "https://github.com/Joystream/joystream/issues"
   },
-  "homepage": "https://github.com/Joystream/joystream"
+  "homepage": "https://github.com/Joystream/joystream",
+  "volta": {
+    "node": "12.18.2",
+    "yarn": "1.22.4"
+  }
 }

utils/api-scripts/package.json

@@ -22,5 +22,8 @@
     "@polkadot/ts": "^0.1.56",
     "typescript": "^3.9.7",
     "ts-node": "^8.6.2"
+  },
+  "volta": {
+    "extends": "../../package.json"
   }
 }

utils/api-scripts/src/dev-set-runtime-code.ts

@@ -32,7 +32,7 @@ async function main() {
   const provider = new WsProvider('ws://127.0.0.1:9944')
 
   let api: ApiPromise
-  let retry = 3
+  let retry = 6
   while (true) {
     try {
       api = await ApiPromise.create({ provider, types })

yarn.lock

@@ -1388,10 +1388,10 @@
     ajv "^6.12.0"
     ajv-keywords "^3.4.1"
 
-"@dzlzv/hydra-cli@^0.0.17":
-  version "0.0.17"
-  resolved "https://registry.yarnpkg.com/@dzlzv/hydra-cli/-/hydra-cli-0.0.17.tgz#56ccae132f76e738724cdc5f0abcd47ff25df530"
-  integrity sha512-ixrjGn6a7UG7ecHYKWTHpcxbdi6X32NbtyCuewm4YGFdb+v0/Eg5zWhFbg1PbMUW9GllC4MiIjDF7Bh1fh9t7Q==
+"@dzlzv/hydra-cli@^0.0.20":
+  version "0.0.20"
+  resolved "https://registry.yarnpkg.com/@dzlzv/hydra-cli/-/hydra-cli-0.0.20.tgz#8837ab287966ba30130cecfa77f27a2d6aad7061"
+  integrity sha512-pFDL2ZkbUyvSGgklmFNAxA2Y/afCJSxvKQHspjuxwhscsjTq0ay05mEtARbvDhLI1yN7jWedfyzQNUtQPCYHuA==
   dependencies:
     "@oclif/command" "^1.5.20"
     "@oclif/config" "^1"
@@ -1415,7 +1415,7 @@
     typeorm-model-generator "^0.4.2"
     warthog "https://github.com/metmirr/warthog/releases/download/v2.20.0/warthog-v2.20.0.tgz"
 
-"@dzlzv/hydra-indexer-lib@^0.0.19-legacy.1.26.1":
+"@dzlzv/hydra-indexer-lib@0.0.19-legacy.1.26.1", "@dzlzv/hydra-indexer-lib@^0.0.19-legacy.1.26.1":
   version "0.0.19-legacy.1.26.1"
   resolved "https://registry.yarnpkg.com/@dzlzv/hydra-indexer-lib/-/hydra-indexer-lib-0.0.19-legacy.1.26.1.tgz#346b564845b2014f7a4d9b3976c03e30da8dd309"
   integrity sha512-4pwaSDRIo/1MqxjfSotjv91fkIj/bfZcZx5nqjB9gRT85X28b3WqkqTFrzlGsGGbvUFWAx4WIeQKnY1yrpX89Q==
@@ -3400,7 +3400,7 @@
     is-ipfs "^0.6.0"
     recursive-fs "^1.1.2"
 
-"@polkadot/api-contract@^1.26.1":
+"@polkadot/api-contract@1.26.1", "@polkadot/api-contract@^1.26.1":
   version "1.26.1"
   resolved "https://registry.yarnpkg.com/@polkadot/api-contract/-/api-contract-1.26.1.tgz#a8b52ef469ab8bbddb83191f8d451e31ffd76142"
   integrity sha512-zLGA/MHUJf12vanUEUBBRqpHVAONHWztoHS0JTIWUUS2+3GEXk6hGw+7PPtBDfDsLj0LgU/Qna1bLalC/zyl5w==
@@ -4938,7 +4938,7 @@
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.159.tgz#61089719dc6fdd9c5cb46efc827f2571d1517065"
   integrity sha512-gF7A72f7WQN33DpqOWw9geApQPh4M3PxluMtaHxWHXEGSN12/WbcEk/eNSqWNQcQhF66VSZ06vCF94CrHwXJDg==
 
-"@types/lodash@^4.14.148", "@types/lodash@^4.14.161":
+"@types/lodash@^4.14.148":
   version "4.14.164"
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.164.tgz#52348bcf909ac7b4c1bcbeda5c23135176e5dfa0"
   integrity sha512-fXCEmONnrtbYUc5014avwBeMdhHHO8YJCkOBflUL9EoJBSKZ1dei+VO74fA7JkTHZ1GvZack2TyIw5U+1lT8jg==
@@ -4948,6 +4948,11 @@
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.157.tgz#fdac1c52448861dfde1a2e1515dbc46e54926dc8"
   integrity sha512-Ft5BNFmv2pHDgxV5JDsndOWTRJ+56zte0ZpYLowp03tW+K+t8u8YMOzAnpuqPgzX6WO1XpDIUm7u04M8vdDiVQ==
 
+"@types/lodash@^4.14.161":
+  version "4.14.165"
+  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.165.tgz#74d55d947452e2de0742bad65270433b63a8c30f"
+  integrity sha512-tjSSOTHhI5mCHTy/OOXYIhi2Wt1qcbHmuXD1Ha7q70CgI/I71afO4XtLb/cVexki1oVYchpul/TOuu3Arcdxrg==
+
 "@types/long@^4.0.0":
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9"
@@ -7879,21 +7884,11 @@ bluebird@^3.1.1, bluebird@^3.3.5, bluebird@^3.5.1, bluebird@^3.5.3, bluebird@^3.
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
   integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
 
-bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.1.1, bn.js@^4.11.8, bn.js@^4.4.0:
-  version "4.11.9"
-  resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.11.9.tgz#26d556829458f9d1e81fc48952493d0ba3507828"
-  integrity sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==
-
-bn.js@^5.1.1, bn.js@^5.1.2:
+bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.1.1, bn.js@^4.11.8, bn.js@^4.4.0, bn.js@^5.1.1, bn.js@^5.1.2, bn.js@^5.1.3:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-5.1.2.tgz#c9686902d3c9a27729f43ab10f9d79c2004da7b0"
   integrity sha512-40rZaf3bUNKTVYu9sIeeEGOg7g14Yvnj9kH7b50EiwX0Q7A6umbvfI5tvHaOERH0XigqKkfLkFQxzb4e6CIXnA==
 
-bn.js@^5.1.3:
-  version "5.1.3"
-  resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-5.1.3.tgz#beca005408f642ebebea80b042b4d18d2ac0ee6b"
-  integrity sha512-GkTiFpjFtUzU9CbMeJ5iazkCzGL3jrhzerzZIuqLABjbwRaFt33I9tUdSNryIptM+RxDet6OKm2WnLXzW51KsQ==
-
 body-parser@1.19.0, body-parser@^1.18.3, body-parser@^1.19.0:
   version "1.19.0"
   resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.19.0.tgz#96b2709e57c9c4e09a6fd66a8fd979844f69f08a"
@@ -11341,7 +11336,7 @@ dotenv@^6.2.0:
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-6.2.0.tgz#941c0410535d942c8becf28d3f357dbd9d476064"
   integrity sha512-HygQCKUBSFl8wKQZBSemMywRWcEDNidvNbjGVyZu3nbZ8qq9ubiPoGLMdRDpfSrpkkm9BXYFkpKxxFX38o/76w==
 
-dotenvi@^0.9.0:
+dotenvi@^0.9.0, dotenvi@^0.9.1:
   version "0.9.1"
   resolved "https://registry.yarnpkg.com/dotenvi/-/dotenvi-0.9.1.tgz#e280012ee9d201a0c57cb1f6e43559603b6f0fb4"
   integrity sha512-gM9HKu6P8BS+jBQRcJRdWKkbIA35Ztszr2FEqp1oKYLMfdTWDumLNi9xlIeEAFc2C4DeOwsYcNi+mMl5OWGtcw==
@@ -16146,10 +16141,10 @@ is-color-stop@^1.0.0:
     rgb-regex "^1.0.1"
     rgba-regex "^1.0.0"
 
-is-core-module@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.0.0.tgz#58531b70aed1db7c0e8d4eb1a0a2d1ddd64bd12d"
-  integrity sha512-jq1AH6C8MuteOoBPwkxHafmByhL9j5q4OaPGdbuD+ZtQJVzH+i6E3BJDQcBA09k57i2Hh2yQbEG8yObZ0jdlWw==
+is-core-module@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.1.0.tgz#a4cc031d9b1aca63eecbd18a650e13cb4eeab946"
+  integrity sha512-YcV7BgVMRFRua2FqQzKtTDMz8iCuLEyGKjr70q8Zm1yy2qKcurbFEd79PAdHV77oL3NrAaOVQIbMmiHQCHB7ZA==
   dependencies:
     has "^1.0.3"
 
@@ -22401,6 +22396,11 @@ pg-protocol@^1.3.0:
   resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.3.0.tgz#3c8fb7ca34dbbfcc42776ce34ac5f537d6e34770"
   integrity sha512-64/bYByMrhWULUaCd+6/72c9PMWhiVFs3EVxl9Ct6a3v/U8+rKgqP2w+kKg/BIGgMJyB+Bk/eNivT32Al+Jghw==
 
+pg-protocol@^1.4.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.4.0.tgz#43a71a92f6fe3ac559952555aa3335c8cb4908be"
+  integrity sha512-El+aXWcwG/8wuFICMQjM5ZSAm6OWiJicFdNYo+VY3QP+8vI4SvLIWVe51PppTzMhikUJR+PsyIFKqfdXPz/yxA==
+
 pg-types@1.*:
   version "1.13.0"
   resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.13.0.tgz#75f490b8a8abf75f1386ef5ec4455ecf6b345c63"
@@ -22451,7 +22451,20 @@ pg@^7.12.1:
     pgpass "1.x"
     semver "4.3.2"
 
-pg@^8.0.3, pg@^8.3.3:
+pg@^8.0.3:
+  version "8.5.0"
+  resolved "https://registry.yarnpkg.com/pg/-/pg-8.5.0.tgz#c29332763ffd51ce52b07dc20dc2337f4d213d08"
+  integrity sha512-h+KHEwce67pAQilZhMCpCx1RC7rR1US7mdjwvKzHRaRxKQxbbFtv5UlwjzqILQ1dwhK+RVGqOVcahE/2KOcaeA==
+  dependencies:
+    buffer-writer "2.0.0"
+    packet-reader "1.0.0"
+    pg-connection-string "^2.4.0"
+    pg-pool "^3.2.2"
+    pg-protocol "^1.4.0"
+    pg-types "^2.1.0"
+    pgpass "1.x"
+
+pg@^8.3.3:
   version "8.4.2"
   resolved "https://registry.yarnpkg.com/pg/-/pg-8.4.2.tgz#2aa58166a23391e91d56a7ea57c6d99931c0642a"
   integrity sha512-E9FlUrrc7w3+sbRmL1CSw99vifACzB2TjhMM9J5w9D1LIg+6un0jKkpHS1EQf2CWhKhec2bhrBLVMmUBDbjPRQ==
@@ -25101,11 +25114,11 @@ resolve@1.1.7:
   integrity sha1-IDEU2CrSxe2ejgQRs5ModeiJ6Xs=
 
 resolve@1.x, resolve@^1.0.0:
-  version "1.18.1"
-  resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.18.1.tgz#018fcb2c5b207d2a6424aee361c5a266da8f4130"
-  integrity sha512-lDfCPaMKfOJXjy0dPayzPdF1phampNWr3qFCjAu+rw/qbQmr5jWH5xN2hwh9QKfw9E5v4hwV7A+jrCmL8yjjqA==
+  version "1.19.0"
+  resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.19.0.tgz#1af5bf630409734a067cae29318aac7fa29a267c"
+  integrity sha512-rArEXAgsBG4UgRGcynxWIWKFvh/XZCcS8UJdHhwy91zwAvCZIbcs+vAbflgBnNjYMs/i/i+/Ux6IZhML1yPvxg==
   dependencies:
-    is-core-module "^2.0.0"
+    is-core-module "^2.1.0"
     path-parse "^1.0.6"
 
 resolve@^1.1.6, resolve@^1.1.7, resolve@^1.11.0, resolve@^1.11.1, resolve@^1.17.0, resolve@^1.2.0:
@@ -28163,12 +28176,7 @@ typescript-formatter@^7.2.2:
     commandpost "^1.0.0"
     editorconfig "^0.15.0"
 
-typescript@3.5.2:
-  version "3.5.2"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.5.2.tgz#a09e1dc69bc9551cadf17dba10ee42cf55e5d56c"
-  integrity sha512-7KxJovlYhTX5RaRbUdkAXN1KUZ8PwWlTzQdHV6xNqvuFOs7+WBo10TQUqT19Q/Jz2hk5v9TQDIhyLhhJY4p5AA==
-
-typescript@^3.0.3, typescript@^3.7.2, typescript@^3.7.5, typescript@^3.8.3, typescript@^3.9.5, typescript@^3.9.6, typescript@^3.9.7:
+typescript@3.5.2, typescript@^3.0.3, typescript@^3.7.2, typescript@^3.7.5, typescript@^3.8.3, typescript@^3.9.5, typescript@^3.9.6, typescript@^3.9.7:
   version "3.9.7"
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.7.tgz#98d600a5ebdc38f40cb277522f12dc800e9e25fa"
   integrity sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==