Merge branch 'docker/unified-docker-compose' into hydra-rc-1

.dockerignore

@@ -1,4 +1,7 @@
-**target*
-**node_modules*
-.tmp/
-.vscode/
+target/
+**node_modules*
+.tmp/
+.vscode/
+query-node/generated
+query-node/**/dist
+query-node/lib

.env

@@ -0,0 +1,57 @@
+COMPOSE_PROJECT_NAME=joystream
+
+###########################
+#     Common settings     #
+###########################
+
+# The env variables below are by default used by all services and should be
+# overriden in local env files (e.g. ./generated/indexer) if needed
+# DB config
+DB_NAME=query_node
+DB_USER=postgres
+DB_PASS=postgres
+DB_HOST=localhost
+DB_PORT=5432
+DEBUG=index-builder:*
+TYPEORM_LOGGING=error
+
+###########################
+#    Indexer options      #
+###########################
+
+# Substrate endpoint to source events from
+WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944/
+# Block height to start indexing from.
+# Note, that if there are already some indexed events, this setting is ignored
+BLOCK_HEIGHT=0
+
+# Custom types to register for Substrate API
+# TYPE_REGISTER_PACKAGE_NAME=
+# TYPE_REGISTER_PACKAGE_VERSION=
+# TYPE_REGISTER_FUNCTION=
+
+# Redis cache server
+REDIS_URI=redis://localhost:6379/0
+
+###########################
+#    Processor options    #
+###########################
+
+# Where the mapping scripts are located, relative to ./generated/indexer
+TYPES_JSON=../../typedefs.json
+
+# Indexer GraphQL API endpoint to fetch indexed events
+INDEXER_ENDPOINT_URL=http://localhost:4100/graphql
+
+# Block height from which the processor starts. Note that if
+# there are already processed events in the database, this setting is ignored
+BLOCK_HEIGHT=0
+
+###############################
+#    Processor GraphQL API    #
+###############################
+
+GRAPHQL_SERVER_PORT=4002
+GRAPHQL_SERVER_HOST=localhost
+WARTHOG_APP_PORT=4002
+WARTHOG_APP_HOST=localhost

.github/workflows/run-network-tests.yml

@@ -148,7 +148,7 @@ jobs:
       - name: Ensure tests are runnable
         run: yarn workspace cd-schemas checks --quiet
       - name: Start chain
-        run: docker-compose up -d
+        run: docker-compose up -d joystream-node
       - name: Initialize the content directory
         run: yarn workspace cd-schemas initialize:dev
 
@@ -202,10 +202,14 @@ jobs:
       - name: Build storage node
         run: yarn workspace storage-node build
       - name: Start Services
-        run: docker-compose --file docker-compose-with-storage.yml up -d
-      - name: Add development storage node and initialize content directory
-        run: DEBUG=* yarn storage-cli dev-init
-      - name: Try uploading
+        run: |
+          docker-compose up -d ipfs
+          docker-compose up -d joystream-node
+      - name: Configure and start development storage node
+        run: |
+          DEBUG=* yarn storage-cli dev-init
+          docker-compose up -d colossus
+      - name: Test uploading
         run: |
           WAIT_TIME=90
           export DEBUG=joystream:*

README.md

@@ -108,9 +108,7 @@ A step by step guide to setup a full node and validator on the Joystream testnet
 ### Integration tests
 
 ```bash
-docker-compose up -d
-DEBUG=* yarn workspace network-tests test-run src/scenarios/full.ts
-docker-compose down
+tests/network-tests/run-tests.sh
 ```
 
 ### Contributing

apps.Dockerfile

@@ -7,9 +7,7 @@ COPY . /joystream
 # to ensure dev dependencies are installed.
 RUN yarn install --frozen-lockfile
 
-# Pioneer is failing to build only on github actions workflow runner
-# Error: packages/page-staking/src/index.tsx(24,21): error TS2307: Cannot find module './Targets' or its corresponding type declarations.
-# RUN yarn workspace pioneer build
+RUN yarn workspace pioneer build
 RUN yarn workspace storage-node build
 RUN yarn workspace query-node-root build
 

build.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+set -e
+
+yarn
+yarn workspace @joystream/types build
+yarn workspace cd-schemas generate:all
+yarn workspace cd-schemas build
+yarn workspace @joystream/cli build
+yarn workspace query-node-root build
+yarn workspace storage-node build
+# Not strictly needed during development, we run "yarn workspace pioneer start" to start
+# a dev instance, but will show highlight build issues
+yarn workspace pioneer build 
+
+# Build cargo crates: native binaries joystream/node, wasm runtime, and chainspec builder.
+while true
+do
+  read -p "Compile joystream node native binary? (y/N): " answer1
+
+  case $answer1 in
+   [yY]* ) yarn cargo-checks
+           yarn cargo-build
+           break;;
+
+   [nN]* ) break;;
+
+   * )     break;;
+  esac
+done
+
+if ! command -v docker-compose &> /dev/null
+then
+    echo "docker-compose not found, skipping docker images build"
+    exit
+fi
+
+# Optionally build joystream/node docker image
+# TODO: Try to fetch a cached joystream/node image
+# if one is found matching code shasum instead of building
+while true
+do
+  read -p "Rebuild joystream/node docker image? (y/N): " answer2
+
+  case $answer2 in
+   [yY]* ) docker-compose build joystream-node
+           break;;
+
+   [nN]* ) break;;
+
+   * )     break;;
+  esac
+done
+
+# Optionlly Build joystream/apps docker image
+while true
+do
+  read -p "Rebuild joystream/apps docker image? (y/N): " answer3
+
+  case $answer3 in
+   [yY]* ) docker-compose build pioneer
+           break;;
+
+   [nN]* ) break;;
+
+   * )     break;;
+  esac
+done

docker-compose-with-storage.yml

@@ -1,39 +0,0 @@
-version: '3'
-services:
-  ipfs:
-    image: ipfs/go-ipfs:latest
-    ports:
-      - '127.0.0.1:5001:5001'
-      - '127.0.0.1:8080:8080'
-    entrypoint: ''
-    command: |
-      /bin/sh -c "
-        set -e
-        /usr/local/bin/start_ipfs config profile apply lowpower
-        /usr/local/bin/start_ipfs config --json Gateway.PublicGateways '{\"localhost\": null }'
-        /sbin/tini -- /usr/local/bin/start_ipfs daemon --migrate=true
-      "
-  chain:
-    image: joystream/node
-    build:
-      context: .
-      dockerfile: joystream-node.Dockerfile
-    ports:
-      - '127.0.0.1:9944:9944'
-    command: --dev --ws-external --base-path /data --log runtime
-
-  colossus:
-    image: joystream/apps
-    restart: on-failure
-    depends_on:
-      - "chain"
-      - "ipfs"
-    build:
-      context: .
-      dockerfile: apps.Dockerfile
-    ports:
-      - '127.0.0.1:3001:3001'
-    command: colossus --dev --ws-provider ws://chain:9944 --ipfs-host ipfs
-    environment:
-      - DEBUG=*
-

docker-compose.yml

@@ -1,17 +1,150 @@
-# Compiles new joystream node image if local image not found,
-# and runs local development chain.
-# To prevent build run docker-compose with "--no-build" arg
-version: "3"
+# Compiles new joystream/node and joystream/apps images if local images not found
+# and runs a complete joystream development network
+# To prevent build of docker images run docker-compose with "--no-build" arg
+version: "3.4"
 services:
   joystream-node:
-    image: joystream/node
+    image: joystream/node:latest
     build:
       # context is relative to the compose file
       context: .
       # dockerfile is relative to the context
       dockerfile: joystream-node.Dockerfile
     container_name: joystream-node
-    command: --dev --alice --validator --unsafe-ws-external --rpc-cors=all --log runtime
+    volumes:
+      - /data
+    command: --dev --alice --validator --unsafe-ws-external --rpc-cors=all --log runtime --base-path /data
     ports:
-      - "9944:9944"
-  
+      - "127.0.0.1:9944:9944"
+
+  ipfs:
+    image: ipfs/go-ipfs:latest
+    ports:
+      - '127.0.0.1:5001:5001'
+      - '127.0.0.1:8080:8080'
+    volumes:
+      - /data/ipfs
+    entrypoint: ''
+    command: |
+      /bin/sh -c "
+        set -e
+        /usr/local/bin/start_ipfs config profile apply lowpower
+        /usr/local/bin/start_ipfs config --json Gateway.PublicGateways '{\"localhost\": null }'
+        /sbin/tini -- /usr/local/bin/start_ipfs daemon --migrate=true
+      "
+
+  colossus:
+    image: joystream/apps
+    restart: on-failure
+    depends_on:
+      - "joystream-node"
+      - "ipfs"
+    build:
+      context: .
+      dockerfile: apps.Dockerfile
+    ports:
+      - '127.0.0.1:3001:3001'
+    command: colossus --dev --ws-provider ${WS_PROVIDER_ENDPOINT_URI} --ipfs-host ipfs
+    environment:
+      - DEBUG=*
+
+  db:
+    image: postgres:12
+    restart: always
+    ports:
+      - "127.0.0.1:${DB_PORT}:5432"
+    volumes:
+      - /var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: ${DB_USER}
+      POSTGRES_PASSWORD: ${DB_PASS}
+      POSTGRES_DB: ${DB_NAME}
+
+  graphql-server:
+    image: joystream/apps
+    restart: unless-stopped
+    build: 
+      context: .
+      dockerfile: apps.Dockerfile
+    env_file:
+      # relative to working directory where docker-compose was run from 
+      - .env
+    environment:
+      - DB_HOST=db
+    ports:
+      - "127.0.0.1:8081:${GRAPHQL_SERVER_PORT}"
+    depends_on: 
+      - db
+    command: ["workspace", "query-node-root", "server:start:prod"]
+
+  processor:
+    image: joystream/apps
+    restart: unless-stopped
+    build: 
+      context: .
+      dockerfile: apps.Dockerfile
+    env_file:
+      # relative to working directory where docker-compose was run from 
+      - .env
+    environment:
+      - INDEXER_ENDPOINT_URL=http://indexer-api-gateway:4000/graphql
+      - DB_HOST=db
+      - TYPEORM_HOST=db
+      - DEBUG=index-builder:*
+      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
+    depends_on:
+      - indexer-api-gateway
+    command: ["workspace", "query-node-root", "processor:start"]
+
+  indexer:
+    image: joystream/apps
+    restart: unless-stopped
+    build: 
+      context: .
+      dockerfile: apps.Dockerfile
+    env_file:
+      # relative to working directory where docker-compose was run from 
+      - .env 
+    environment:
+      - TYPEORM_HOST=db
+      - INDEXER_WORKERS=5
+      - PROCESSOR_POLL_INTERVAL=1000 # refresh every second 
+      - REDIS_URI=redis://redis:6379/0
+      - DEBUG=index-builder:*
+      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
+    depends_on: 
+      - db
+    command: ["workspace", "query-node-root", "indexer:start"] 
+
+  indexer-api-gateway:
+    image: joystream/hydra-indexer-gateway:latest
+    restart: unless-stopped
+    environment:
+      - WARTHOG_STARTER_DB_DATABASE=${DB_NAME}
+      - WARTHOG_STARTER_DB_HOST=db 
+      - WARTHOG_STARTER_DB_PASSWORD=${DB_PASS}
+      - WARTHOG_STARTER_DB_PORT=${DB_PORT}
+      - WARTHOG_STARTER_DB_USERNAME=${DB_USER}
+      - WARTHOG_STARTER_REDIS_URI=redis://redis:6379/0 
+      - PORT=4000
+    ports:
+      - "127.0.0.1:4000:4000"
+    depends_on:
+      - redis
+      - db
+      - indexer
+
+  redis:
+    image: redis:6.0-alpine
+    restart: always
+    ports:
+      - "127.0.0.1:6379:6379"
+
+  pioneer:
+    image: joystream/apps
+    build:
+      context: .
+      dockerfile: apps.Dockerfile
+    ports:
+      - "127.0.0.1:3000:3000"
+    command: workspace pioneer start

joystream-node.Dockerfile

@@ -1,4 +1,11 @@
-FROM joystream/rust-builder AS builder
+FROM liuchong/rustup:1.46.0 AS rustup
+RUN rustup component add rustfmt clippy
+RUN rustup install nightly-2020-05-23 --force
+RUN rustup target add wasm32-unknown-unknown --toolchain nightly-2020-05-23
+RUN apt-get update && \
+  apt-get install -y curl git gcc xz-utils sudo pkg-config unzip clang libc6-dev-i386
+
+FROM rustup AS builder
 LABEL description="Compiles all workspace artifacts"
 WORKDIR /joystream
 COPY . /joystream

package.json

@@ -5,6 +5,8 @@
   "license": "GPL-3.0-only",
   "scripts": {
     "postinstall": "yarn workspace @joystream/types build && yarn workspace cd-schemas generate:all && yarn workspace cd-schemas build && yarn workspace @joystream/cli build",
+    "build": "./build.sh",
+    "start": "./start.sh",
     "cargo-checks": "devops/git-hooks/pre-commit && devops/git-hooks/pre-push",
     "cargo-build": "scripts/cargo-build.sh"
   },
@@ -37,9 +39,9 @@
     "@dzlzv/hydra-indexer-lib": "0.0.19-legacy.1.26.1"
   },
   "devDependencies": {
+    "eslint": "^7.6.0",
     "husky": "^4.2.5",
-    "prettier": "2.0.2",
-    "eslint": "^7.6.0"
+    "prettier": "2.0.2"
   },
   "husky": {
     "hooks": {
@@ -50,5 +52,9 @@
   "engines": {
     "node": ">=12.18.0",
     "yarn": "^1.22.0"
+  },
+  "volta": {
+    "node": "12.18.2",
+    "yarn": "1.22.4"
   }
 }

query-node/.dockerignore

@@ -1,4 +0,0 @@
-generated
-**/node_modules
-**/dist
-./lib/

query-node/.env

@@ -1,3 +1,5 @@
+COMPOSE_PROJECT_NAME=joystream
+
 # Project name
 PROJECT_NAME=query_node
 

query-node/build.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -e
+
+SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+cd $SCRIPT_PATH
+
+yarn clean
+yarn codegen:all
+yarn
+ln -s ../../../../../node_modules/typeorm/cli.js generated/graphql-server/node_modules/.bin/typeorm || :
+yarn tsc --build tsconfig.json

query-node/docker-compose.yml

@@ -1,91 +0,0 @@
-version: "3.4"
-
-services:
-  db:
-    image: postgres:12
-    restart: always
-    ports:
-      - "${DB_PORT}:5432"
-    volumes:
-      - /var/lib/postgresql/data
-    environment:
-      POSTGRES_USER: ${DB_USER}
-      POSTGRES_PASSWORD: ${DB_PASS}
-      POSTGRES_DB: ${DB_NAME}
-
-  graphql-server:
-    image: joystream/apps
-    restart: unless-stopped
-    build: 
-      context: ../
-      dockerfile: apps.Dockerfile
-    env_file:
-      - .env
-    environment:
-      - DB_HOST=db
-    ports:
-      - "8080:${GRAPHQL_SERVER_PORT}"
-    depends_on: 
-      - db
-    command: ["workspace", "query-node-root", "server:start:prod"]
-
-  processor:
-    image: joystream/apps
-    restart: unless-stopped
-    build: 
-      context: ../
-      dockerfile: apps.Dockerfile
-    env_file:
-      - .env
-    environment:
-      - INDEXER_ENDPOINT_URL=http://indexer-api-gateway:4000/graphql
-      - DB_HOST=db
-      - TYPEORM_HOST=db
-      - DEBUG=index-builder:*
-      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
-    depends_on:
-      - indexer-api-gateway
-    command: ["workspace", "query-node-root", "processor:start"]
-  
-  indexer:
-    image: joystream/apps
-    restart: unless-stopped
-    build: 
-      context: ../
-      dockerfile: apps.Dockerfile
-    env_file:
-      - .env 
-    environment:
-      - TYPEORM_HOST=db
-      - INDEXER_WORKERS=5
-      - PROCESSOR_POLL_INTERVAL=1000 # refresh every second 
-      - REDIS_URI=redis://redis:6379/0
-      - DEBUG=index-builder:*
-      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
-    depends_on: 
-      - db
-    command: ["workspace", "query-node-root", "indexer:start"] 
-  
-  indexer-api-gateway:
-    image: joystream/hydra-indexer-gateway:latest
-    restart: unless-stopped
-    environment:
-      - WARTHOG_STARTER_DB_DATABASE=${DB_NAME}
-      - WARTHOG_STARTER_DB_HOST=db 
-      - WARTHOG_STARTER_DB_PASSWORD=${DB_PASS}
-      - WARTHOG_STARTER_DB_PORT=${DB_PORT}
-      - WARTHOG_STARTER_DB_USERNAME=${DB_USER}
-      - WARTHOG_STARTER_REDIS_URI=redis://redis:6379/0 
-      - PORT=4000
-    ports:
-      - "4000:4000"
-    depends_on:
-      - redis
-      - db
-      - indexer
-    
-  redis:
-    image: redis:6.0-alpine
-    restart: always
-    ports:
-      - "6379:6379"

query-node/package.json

@@ -3,7 +3,7 @@
 	"version": "1.0.0",
 	"description": "GraphQL server and Substrate indexer. Generated with ♥ by Hydra-CLI",
 	"scripts": {
-		"build": "yarn codegen:all && (ln -s ../../../../../node_modules/typeorm/cli.js generated/graphql-server/node_modules/.bin/typeorm || :) && tsc --build tsconfig.json",
+		"build": "./build.sh",
 		"test": "echo \"Error: no test specified\" && exit 1",
 		"clean": "rm -rf ./generated",
 		"processor:start": "(cd ./generated/indexer && yarn && DEBUG=${DEBUG} yarn start:processor)",
@@ -16,11 +16,10 @@
 		"db:schema:migrate": "(cd ./generated/graphql-server && yarn db:create && yarn db:sync && yarn db:migrate)",
 		"db:indexer:migrate": "(cd ./generated/indexer && yarn db:migrate)",
 		"db:migrate": "yarn db:schema:migrate && yarn db:indexer:migrate",
-		"codegen:all": "yarn hydra-cli codegen && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
-		"codegen:indexer": "yarn hydra-cli codegen --no-graphql && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
-		"codegen:server": "yarn hydra-cli codegen --no-indexer",
-		"docker:up": "docker-compose up -d",
-    "cd-classes": "ts-node scripts/get-class-id-and-name.ts"
+		"codegen:all": "yarn hydra-cli codegen --no-install && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
+		"codegen:indexer": "yarn hydra-cli codegen --no-install --no-graphql && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
+		"codegen:server": "yarn hydra-cli codegen --no-install --no-indexer",
+		"cd-classes": "ts-node scripts/get-class-id-and-name.ts"
 	},
 	"author": "",
 	"license": "ISC",
@@ -28,11 +27,13 @@
 		"@dzlzv/hydra-cli": "^0.0.19"
 	},
 	"dependencies": {
+		"@dzlzv/hydra-indexer-lib": "^0.0.19-legacy.1.26.1",
 		"@joystream/types": "^0.14.0",
 		"@types/bn.js": "^4.11.6",
 		"@types/debug": "^4.1.5",
 		"bn.js": "^5.1.2",
 		"debug": "^4.2.0",
+		"dotenvi": "^0.9.1",
 		"tslib": "^2.0.0"
 	}
 }

query-node/run-tests.sh

@@ -7,8 +7,8 @@ cd $SCRIPT_PATH
 function cleanup() {
     # Show tail end of logs for the processor and indexer containers to
     # see any possible errors
-    (echo "## Processor Logs ##" && docker logs query-node_processor_1 --tail 50) || :
-    (echo "## Indexer Logs ##" && docker logs query-node_indexer_1 --tail 50) || :
+    (echo "## Processor Logs ##" && docker logs joystream_processor_1 --tail 50) || :
+    (echo "## Indexer Logs ##" && docker logs joystream_indexer_1 --tail 50) || :
     docker-compose down -v
 }
 
@@ -23,9 +23,17 @@ export WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944/
 # typeorm commandline is used by db:migrate step below.
 ln -s ../../../../../node_modules/typeorm/cli.js generated/graphql-server/node_modules/.bin/typeorm || :
 
-yarn db:up
+# clean start
+docker-compose down -v
+
+docker-compose up -d db
 yarn db:migrate
-yarn docker:up
+# docker-compose up -d redis
+# docker-compose up -d indexer
+# docker-compose up -d graphql-server
+# docker-compose up -d indexer-api-gateway
+# Starting up processor will bring up all services it depends on
+docker-compose up -d processor
 
 # Run tests
-ATTACH_TO_NETWORK=query-node_default ../tests/network-tests/run-tests.sh content-directory
+ATTACH_TO_NETWORK=joystream_default ../tests/network-tests/run-tests.sh content-directory

rust-builder.Dockerfile

@@ -1,8 +0,0 @@
-FROM liuchong/rustup:1.46.0 AS builder
-LABEL description="Rust and WASM build environment for joystream and substrate"
-
-WORKDIR /setup
-COPY setup.sh /setup
-ENV TERM=xterm
-
-RUN ./setup.sh

scripts/runtime-code-shasum.sh

@@ -7,17 +7,16 @@ export WORKSPACE_ROOT=`cargo metadata --offline --no-deps --format-version 1 | j
 
 cd ${WORKSPACE_ROOT}
 
-# srot/owner/group/mtime arguments only work with gnu version of tar.
-# So if you run this on Mac the default version of tar is `bsdtar`
-# and you will not get an idempotent result.
-# Install gnu-tar with brew
-#   brew install gnu-tar
-#   export PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH"
-tar -c --sort=name --owner=root:0 --group=root:0 --mtime='UTC 2020-01-01' \
+TAR=tar
+if [[ "$OSTYPE" == "darwin"* ]]; then
+	TAR=gtar
+fi
+
+# sort/owner/group/mtime arguments only work with gnu version of tar!
+${TAR} -c --sort=name --owner=root:0 --group=root:0 --mtime='UTC 2020-01-01' \
     Cargo.lock \
     Cargo.toml \
     runtime \
     runtime-modules \
     utils/chain-spec-builder \
     joystream-node.Dockerfile | shasum | cut -d " " -f 1
-

setup.sh

@@ -2,11 +2,8 @@
 
 set -e
 
-# If OS is supported will install:
-#  - build tools and any other dependencies required for rust and substrate
-#  - rustup - rust insaller
-#  - rust compiler and toolchains
-#  - skips installing substrate and subkey
+# If OS is supported will install build tools for rust and substrate.
+# Skips installing substrate itself and subkey
 curl https://getsubstrate.io -sSf | bash -s -- --fast
 
 source ~/.cargo/env
@@ -19,13 +16,16 @@ rustup component add rustfmt clippy
 rustup install nightly-2020-05-23 --force
 rustup target add wasm32-unknown-unknown --toolchain nightly-2020-05-23
 
-# Ensure the stable toolchain is still the default
-rustup default stable
+# Sticking with older version of compiler to ensure working build
+rustup install 1.46.0
+rustup default 1.46.0
 
-# TODO: Install additional tools...
+if [[ "$OSTYPE" == "linux-gnu" ]]; then
+    apt-get install coreutils
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+	brew install b2sum gnu-tar
+fi
+
+# Volta nodejs, npm, yarn tools manager
+curl https://get.volta.sh | bash
 
-# - b2sum
-# - nodejs
-# - npm
-# - yarn
-# .... ?

start.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -e
+
+# Run a complete joystream development network on your machine using docker.
+# Make sure to run build.sh prior to running this script.
+
+# Clean start!
+docker-compose down -v
+
+function down()
+{
+    # Stop containers and clear volumes
+    docker-compose down -v
+}
+
+trap down EXIT
+
+# Run a local development chain
+docker-compose up -d joystream-node
+
+## Storage Infrastructure
+# Configure a dev storage node and start storage node
+DEBUG=joystream:storage-cli:dev yarn storage-cli dev-init
+docker-compose up -d colossus
+# Initialise the content directory with standard classes, schemas and initial entities
+yarn workspace cd-schemas initialize:dev
+
+## Query Node Infrastructure
+# Initialize a new database for the query node infrastructure
+docker-compose up -d db
+yarn workspace query-node-root db:migrate
+# Startup all query-node infrastructure services
+docker-compose up -d processor
+
+echo "press Ctrl+C to shutdown"
+
+# Start a dev instance of pioneer and wait for exit
+docker-compose up pioneer

storage-node/README.md

@@ -35,6 +35,7 @@ _Building_
 
 ```bash
 $ yarn install
+$ yarn build
 ```
 
 The command will install dependencies, and make a `colossus` executable available:

storage-node/docker-compose.yaml

@@ -1,29 +0,0 @@
-version: '3'
-services:
-  ipfs:
-    image: ipfs/go-ipfs:latest
-    ports:
-      - '127.0.0.1:5001:5001'
-      - '127.0.0.1:8080:8080'
-    volumes:
-      - ipfs-data:/data/ipfs
-    entrypoint: ''
-    command: |
-      /bin/sh -c "
-        set -e
-        /usr/local/bin/start_ipfs config profile apply lowpower
-        /usr/local/bin/start_ipfs config --json Gateway.PublicGateways '{\"localhost\": null }'
-        /sbin/tini -- /usr/local/bin/start_ipfs daemon --migrate=true
-      "
-  chain:
-    image: joystream/node:latest
-    ports:
-      - '127.0.0.1:9944:9944'
-    volumes:
-      - chain-data:/data
-    command: --dev --ws-external --base-path /data
-volumes:
-  ipfs-data:
-    driver: local
-  chain-data:
-    driver: local

storage-node/start-dev.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-# Avoid pulling joystream/node from docker hub. It is most likely
-# not the version that we want to work with. Either you should
-# build it locally or pull it down manually.
-if ! docker inspect joystream/node:latest > /dev/null 2>&1;
-then
-  echo "Didn't find local joystream/node:latest docker image."
-  exit 1
-fi
-
-SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
-cd $SCRIPT_PATH
-
-# stop prior run and clear volumes
-# docker-compose down -v
-
-# Run a development joystream-node chain and ipfs daemon in the background
-docker-compose up -d
-
-function down()
-{
-    # Stop containers and clear volumes
-    docker-compose down -v
-}
-
-trap down EXIT
-
-# configure the dev chain
-DEBUG=joystream:storage-cli:dev yarn storage-cli dev-init
-
-# Run the tests
-# Tests sometimes fail, so skip for now
-# yarn workspace storage-node test
-
-# Start Colossus storage-node
-DEBUG=joystream:* yarn colossus --dev

storage-node/stop-dev.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-# stop prior run and clear volumes
-docker-compose down -v

tests/network-tests/.env

@@ -1,5 +1,7 @@
 # Address of the Joystream node.
 NODE_URL = ws://127.0.0.1:9944
+# Address of the Joystream query node.
+QUERY_NODE_URL = http://127.0.0.1:8081/graphql
 # Account which is expected to provide sufficient funds to test accounts.
 TREASURY_ACCOUNT_URI = //Alice
 # Sudo Account

tests/network-tests/package.json

@@ -21,7 +21,8 @@
     "bn.js": "^4.11.8",
     "dotenv": "^8.2.0",
     "fs": "^0.0.1-security",
-    "uuid": "^7.0.3"
+    "uuid": "^7.0.3",
+    "@apollo/client": "^3.2.5"
   },
   "devDependencies": {
     "@polkadot/ts": "^0.3.14",

tests/network-tests/src/Api.ts

@@ -13,7 +13,7 @@ import {
   Opening as WorkingGroupOpening,
 } from '@joystream/types/working-group'
 import { ElectionStake, Seat } from '@joystream/types/council'
-import { AccountInfo, Balance, BalanceOf, BlockNumber, Event, EventRecord } from '@polkadot/types/interfaces'
+import { AccountInfo, Hash, Balance, BalanceOf, BlockNumber, Event, EventRecord } from '@polkadot/types/interfaces'
 import BN from 'bn.js'
 import { SubmittableExtrinsic } from '@polkadot/api/types'
 import { Sender } from './sender'
@@ -30,6 +30,12 @@ import {
 } from '@joystream/types/hiring'
 import { FillOpeningParameters, ProposalId } from '@joystream/types/proposals'
 import { v4 as uuid } from 'uuid'
+import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
+import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
+import { initializeContentDir, InputParser, ExtrinsicsHelper } from 'cd-schemas'
+import { OperationType } from '@joystream/types/content-directory'
+import { gql, ApolloClient, ApolloQueryResult, NormalizedCacheObject } from '@apollo/client'
+
 import Debugger from 'debug'
 const debug = Debugger('api')
 
@@ -39,11 +45,11 @@ export enum WorkingGroups {
 }
 
 export class Api {
-  private readonly api: ApiPromise
-  private readonly sender: Sender
-  private readonly keyring: Keyring
+  protected readonly api: ApiPromise
+  protected readonly sender: Sender
+  protected readonly keyring: Keyring
   // source of funds for all new accounts
-  private readonly treasuryAccount: string
+  protected readonly treasuryAccount: string
 
   public static async create(provider: WsProvider, treasuryAccountUri: string, sudoAccountUri: string): Promise<Api> {
     let connectAttempts = 0
@@ -1707,6 +1713,7 @@ export class Api {
     ).filter((addr) => addr !== '')
   }
   */
+
   public async terminateApplication(
     leader: string,
     applicationId: ApplicationId,
@@ -1918,4 +1925,131 @@ export class Api {
   public getMaxWorkersCount(module: WorkingGroups): BN {
     return this.api.createType('u32', this.api.consts[module].maxWorkerNumberLimit)
   }
+
+  async sendContentDirectoryTransaction(operations: OperationType[]): Promise<void> {
+    const transaction = this.api.tx.contentDirectory.transaction(
+      { Lead: null }, // We use member with id 0 as actor (in this case we assume this is Alice)
+      operations // We provide parsed operations as second argument
+    )
+    const lead = (await this.getGroupLead(WorkingGroups.ContentDirectoryWorkingGroup)) as Worker
+    await this.sender.signAndSend(transaction, lead.role_account_id, false)
+  }
+
+  public async createChannelEntity(channel: ChannelEntity): Promise<void> {
+    // Create the parser with known entity schemas (the ones in content-directory-schemas/inputs)
+    const parser = InputParser.createWithKnownSchemas(
+      this.api,
+      // The second argument is an array of entity batches, following standard entity batch syntax ({ className, entries }):
+      [
+        {
+          className: 'Channel',
+          entries: [channel], // We could specify multiple entries here, but in this case we only need one
+        },
+      ]
+    )
+    // We parse the input into CreateEntity and AddSchemaSupportToEntity operations
+    const operations = await parser.getEntityBatchOperations()
+    return await this.sendContentDirectoryTransaction(operations)
+  }
+
+  public async createVideoEntity(video: VideoEntity): Promise<void> {
+    // Create the parser with known entity schemas (the ones in content-directory-schemas/inputs)
+    const parser = InputParser.createWithKnownSchemas(
+      this.api,
+      // The second argument is an array of entity batches, following standard entity batch syntax ({ className, entries }):
+      [
+        {
+          className: 'Video',
+          entries: [video], // We could specify multiple entries here, but in this case we only need one
+        },
+      ]
+    )
+    // We parse the input into CreateEntity and AddSchemaSupportToEntity operations
+    const operations = await parser.getEntityBatchOperations()
+    return await this.sendContentDirectoryTransaction(operations)
+  }
+
+  public async updateChannelEntity(
+    channelUpdateInput: Record<string, any>,
+    uniquePropValue: Record<string, any>
+  ): Promise<void> {
+    // Create the parser with known entity schemas (the ones in content-directory-schemas/inputs)
+    const parser = InputParser.createWithKnownSchemas(this.api)
+
+    // We can reuse InputParser's `findEntityIdByUniqueQuery` method to find entityId of the channel we
+    // created in ./createChannel.ts example (normally we would probably use some other way to do it, ie.: query node)
+    const CHANNEL_ID = await parser.findEntityIdByUniqueQuery(uniquePropValue, 'Channel') // Use getEntityUpdateOperations to parse the update input
+    const updateOperations = await parser.getEntityUpdateOperations(
+      channelUpdateInput,
+      'Channel', // Class name
+      CHANNEL_ID // Id of the entity we want to update
+    )
+    return await this.sendContentDirectoryTransaction(updateOperations)
+  }
+
+  public async initializeContentDirectory(leadKeyPair: KeyringPair) {
+    await initializeContentDir(this.api, leadKeyPair)
+  }
+}
+
+export class QueryNodeApi extends Api {
+  private readonly queryNodeProvider: ApolloClient<NormalizedCacheObject>
+
+  public static async new(
+    provider: WsProvider,
+    queryNodeProvider: ApolloClient<NormalizedCacheObject>,
+    treasuryAccountUri: string,
+    sudoAccountUri: string
+  ): Promise<QueryNodeApi> {
+    let connectAttempts = 0
+    while (true) {
+      connectAttempts++
+      debug(`Connecting to chain, attempt ${connectAttempts}..`)
+      try {
+        const api = await ApiPromise.create({ provider, types })
+
+        // Wait for api to be connected and ready
+        await api.isReady
+
+        // If a node was just started up it might take a few seconds to start producing blocks
+        // Give it a few seconds to be ready.
+        await Utils.wait(5000)
+
+        return new QueryNodeApi(api, queryNodeProvider, treasuryAccountUri, sudoAccountUri)
+      } catch (err) {
+        if (connectAttempts === 3) {
+          throw new Error('Unable to connect to chain')
+        }
+      }
+      await Utils.wait(5000)
+    }
+  }
+
+  constructor(
+    api: ApiPromise,
+    queryNodeProvider: ApolloClient<NormalizedCacheObject>,
+    treasuryAccountUri: string,
+    sudoAccountUri: string
+  ) {
+    super(api, treasuryAccountUri, sudoAccountUri)
+    this.queryNodeProvider = queryNodeProvider
+  }
+
+  public async getChannelbyTitle(title: string): Promise<ApolloQueryResult<any>> {
+    const GET_CHANNEL_BY_TITLE = gql`
+      query($title: String!) {
+        channels(where: { title_eq: $title }) {
+          title
+          description
+          coverPhotoUrl
+          avatarPhotoUrl
+          isPublic
+          isCurated
+          languageId
+        }
+      }
+    `
+
+    return await this.queryNodeProvider.query({ query: GET_CHANNEL_BY_TITLE, variables: { title } })
+  }
 }

tests/network-tests/src/fixtures/contentDirectoryModule.ts

@@ -0,0 +1,65 @@
+import { QueryNodeApi } from '../Api'
+import BN from 'bn.js'
+import { assert } from 'chai'
+import { Seat } from '@joystream/types/council'
+import { v4 as uuid } from 'uuid'
+import { Utils } from '../utils'
+import { Fixture } from '../Fixture'
+import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
+import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
+
+export class CreateChannelFixture implements Fixture {
+  private api: QueryNodeApi
+  public channelEntity: ChannelEntity
+
+  public constructor(api: QueryNodeApi, channelEntity: ChannelEntity) {
+    this.api = api
+    this.channelEntity = channelEntity
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    await this.api.createChannelEntity(this.channelEntity)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class CreateVideoFixture implements Fixture {
+  private api: QueryNodeApi
+  private videoEntity: VideoEntity
+
+  public constructor(api: QueryNodeApi, videoEntity: VideoEntity) {
+    this.api = api
+    this.videoEntity = videoEntity
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    await this.api.createVideoEntity(this.videoEntity)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class UpdateChannelFixture implements Fixture {
+  private api: QueryNodeApi
+  private channelUpdateInput: Record<string, any>
+  private uniquePropValue: Record<string, any>
+
+  public constructor(api: QueryNodeApi, channelUpdateInput: Record<string, any>, uniquePropValue: Record<string, any>) {
+    this.api = api
+    this.channelUpdateInput = channelUpdateInput
+    this.uniquePropValue = uniquePropValue
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    await this.api.updateChannelEntity(this.channelUpdateInput, this.uniquePropValue)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}

tests/network-tests/src/flows/contentDirectory/contentDirectoryInitialization.ts

@@ -0,0 +1,7 @@
+import { Api, WorkingGroups } from '../../Api'
+import { assert } from 'chai'
+import { KeyringPair } from '@polkadot/keyring/types'
+
+export default async function initializeContentDirectory(api: Api, leadKeyPair: KeyringPair) {
+  await api.initializeContentDirectory(leadKeyPair)
+}

tests/network-tests/src/flows/contentDirectory/creatingChannel.ts

@@ -0,0 +1,43 @@
+import { QueryNodeApi } from '../../Api'
+import { Utils } from '../../utils'
+import { CreateChannelFixture } from '../../fixtures/contentDirectoryModule'
+import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
+import { assert } from 'chai'
+import { KeyringPair } from '@polkadot/keyring/types'
+
+export function createSimpleChannelFixture(api: QueryNodeApi): CreateChannelFixture {
+  const channelEntity: ChannelEntity = {
+    title: 'Example channel',
+    description: 'This is an example channel',
+    // We can use "existing" syntax to reference either an on-chain entity or other entity that's part of the same batch.
+    // Here we reference language that we assume was added by initialization script (initialize:dev), as it is part of
+    // input/entityBatches/LanguageBatch.json
+    language: { existing: { code: 'EN' } },
+    coverPhotoUrl: '',
+    avatarPhotoURL: '',
+    isPublic: true,
+  }
+  return new CreateChannelFixture(api, channelEntity)
+}
+
+export default async function channelCreation(api: QueryNodeApi) {
+  const createChannelHappyCaseFixture = createSimpleChannelFixture(api)
+
+  await createChannelHappyCaseFixture.runner(false)
+
+  // Temporary solution (wait 2 minutes)
+  await Utils.wait(120000)
+
+  // Ensure newly created channel was parsed by query node
+  const result = await api.getChannelbyTitle(createChannelHappyCaseFixture.channelEntity.title)
+  const queriedChannel = result.data.channels[0]
+
+  assert(queriedChannel.title === createChannelHappyCaseFixture.channelEntity.title, 'Should be equal')
+  assert(queriedChannel.description === createChannelHappyCaseFixture.channelEntity.description, 'Should be equal')
+  assert(queriedChannel.coverPhotoUrl === createChannelHappyCaseFixture.channelEntity.coverPhotoUrl, 'Should be equal')
+  assert(
+    queriedChannel.avatarPhotoUrl === createChannelHappyCaseFixture.channelEntity.avatarPhotoURL,
+    'Should be equal'
+  )
+  assert(queriedChannel.isPublic === createChannelHappyCaseFixture.channelEntity.isPublic, 'Should be equal')
+}

tests/network-tests/src/flows/contentDirectory/creatingVideo.ts

@@ -0,0 +1,49 @@
+import { QueryNodeApi, WorkingGroups } from '../../Api'
+import { CreateVideoFixture } from '../../fixtures/contentDirectoryModule'
+import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
+import { assert } from 'chai'
+
+export function createVideoReferencingChannelFixture(api: QueryNodeApi): CreateVideoFixture {
+  const videoEntity: VideoEntity = {
+    title: 'Example video',
+    description: 'This is an example video',
+    // We reference existing language and category by their unique properties with "existing" syntax
+    // (those referenced here are part of inputs/entityBatches)
+    language: { existing: { code: 'EN' } },
+    category: { existing: { name: 'Education' } },
+    // We use the same "existing" syntax to reference a channel by unique property (title)
+    // In this case it's a channel that we created in createChannel example
+    channel: { existing: { title: 'Example channel' } },
+    media: {
+      // We use "new" syntax to sygnalize we want to create a new VideoMedia entity that will be related to this Video entity
+      new: {
+        // We use "exisiting" enconding from inputs/entityBatches/VideoMediaEncodingBatch.json
+        encoding: { existing: { name: 'H.263_MP4' } },
+        pixelHeight: 600,
+        pixelWidth: 800,
+        // We create nested VideoMedia->MediaLocation->HttpMediaLocation relations using the "new" syntax
+        location: { new: { httpMediaLocation: { new: { url: 'https://testnet.joystream.org/' } } } },
+      },
+    },
+    // Here we use combined "new" and "existing" syntaxes to create Video->License->KnownLicense relations
+    license: {
+      new: {
+        knownLicense: {
+          // This license can be found in inputs/entityBatches/KnownLicenseBatch.json
+          existing: { code: 'CC_BY' },
+        },
+      },
+    },
+    duration: 3600,
+    thumbnailURL: '',
+    isExplicit: false,
+    isPublic: true,
+  }
+  return new CreateVideoFixture(api, videoEntity)
+}
+
+export default async function createVideo(api: QueryNodeApi) {
+  const createVideoHappyCaseFixture = createVideoReferencingChannelFixture(api)
+
+  await createVideoHappyCaseFixture.runner(false)
+}

tests/network-tests/src/flows/contentDirectory/updatingChannel.ts

@@ -0,0 +1,21 @@
+import { QueryNodeApi, WorkingGroups } from '../../Api'
+import { UpdateChannelFixture } from '../../fixtures/contentDirectoryModule'
+import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
+import { assert } from 'chai'
+
+export function createUpdateChannelTitleFixture(api: QueryNodeApi): UpdateChannelFixture {
+  // Create partial channel entity, only containing the fields we wish to update
+  const channelUpdateInput: Partial<ChannelEntity> = {
+    title: 'Updated channel title',
+  }
+
+  const uniquePropVal: Record<string, any> = { title: 'Example channel' }
+
+  return new UpdateChannelFixture(api, channelUpdateInput, uniquePropVal)
+}
+
+export default async function updateChannel(api: QueryNodeApi) {
+  const createVideoHappyCaseFixture = createUpdateChannelTitleFixture(api)
+
+  await createVideoHappyCaseFixture.runner(false)
+}

tests/network-tests/src/flows/workingGroup/leaderSetup.ts

@@ -3,13 +3,17 @@ import BN from 'bn.js'
 import { PaidTermId } from '@joystream/types/members'
 import { SudoHireLeadFixture } from '../../fixtures/sudoHireLead'
 import { assert } from 'chai'
+import { KeyringPair } from '@polkadot/keyring/types'
 
 // Worker application happy case scenario
-export default async function leaderSetup(api: Api, env: NodeJS.ProcessEnv, group: WorkingGroups) {
+export default async function leaderSetup(
+  api: Api,
+  env: NodeJS.ProcessEnv,
+  group: WorkingGroups
+): Promise<KeyringPair> {
   const lead = await api.getGroupLead(group)
-  if (lead) {
-    return
-  }
+
+  assert(!lead, `Lead is already set`)
 
   const leadKeyPair = api.createKeyPairs(1)[0]
   const paidTerms: PaidTermId = api.createPaidTermId(new BN(+env.MEMBERSHIP_PAID_TERMS!))
@@ -37,4 +41,6 @@ export default async function leaderSetup(api: Api, env: NodeJS.ProcessEnv, grou
   const hiredLead = await api.getGroupLead(group)
   assert(hiredLead, `${group} group Lead was not hired!`)
   assert(hiredLead!.role_account_id.eq(leadKeyPair.address))
+
+  return leadKeyPair
 }

tests/network-tests/src/scenarios/content-directory.ts

@@ -1,7 +1,12 @@
 import { WsProvider } from '@polkadot/api'
-import { Api, WorkingGroups } from '../Api'
+import { Api, QueryNodeApi, WorkingGroups } from '../Api'
 import { config } from 'dotenv'
 import leaderSetup from '../flows/workingGroup/leaderSetup'
+import initializeContentDirectory from '../flows/contentDirectory/contentDirectoryInitialization'
+import createChannel from '../flows/contentDirectory/creatingChannel'
+import createVideo from '../flows/contentDirectory/creatingVideo'
+import updateChannel from '../flows/contentDirectory/updatingChannel'
+import { ApolloClient, InMemoryCache } from '@apollo/client'
 
 const scenario = async () => {
   // Load env variables
@@ -11,13 +16,34 @@ const scenario = async () => {
   // Connect api to the chain
   const nodeUrl: string = env.NODE_URL || 'ws://127.0.0.1:9944'
   const provider = new WsProvider(nodeUrl)
-  const api: Api = await Api.create(provider, env.TREASURY_ACCOUNT_URI || '//Alice', env.SUDO_ACCOUNT_URI || '//Alice')
 
-  await leaderSetup(api, env, WorkingGroups.ContentDirectoryWorkingGroup)
+  const queryNodeUrl: string = env.QUERY_NODE_URL || 'http://127.0.0.1:8080/graphql'
+
+  const queryNodeProvider = new ApolloClient({
+    uri: queryNodeUrl,
+    cache: new InMemoryCache(),
+  })
+
+  const api: QueryNodeApi = await QueryNodeApi.new(
+    provider,
+    queryNodeProvider,
+    env.TREASURY_ACCOUNT_URI || '//Alice',
+    env.SUDO_ACCOUNT_URI || '//Alice'
+  )
+
+  const leadKeyPair = await leaderSetup(api, env, WorkingGroups.ContentDirectoryWorkingGroup)
 
   // Some flows that use the curator lead to perform some tests...
   //
 
+  await initializeContentDirectory(api, leadKeyPair)
+
+  await createChannel(api)
+
+  await createVideo(api)
+
+  await updateChannel(api)
+
   // Note: disconnecting and then reconnecting to the chain in the same process
   // doesn't seem to work!
   api.close()

utils/api-scripts/src/status.ts

@@ -12,7 +12,7 @@ async function main() {
 
   // Create the API and wait until ready
   let api: ApiPromise
-  let retry = 3
+  let retry = 6
   while (true) {
     try {
       api = await ApiPromise.create({ provider, types })

yarn.lock

@@ -23,6 +23,25 @@
     call-me-maybe "^1.0.1"
     js-yaml "^3.13.1"
 
+"@apollo/client@^3.2.5":
+  version "3.2.5"
+  resolved "https://registry.yarnpkg.com/@apollo/client/-/client-3.2.5.tgz#24e0a6faa1d231ab44807af237c6227410c75c4d"
+  integrity sha512-zpruxnFMz6K94gs2pqc3sidzFDbQpKT5D6P/J/I9s8ekHZ5eczgnRp6pqXC86Bh7+44j/btpmOT0kwiboyqTnA==
+  dependencies:
+    "@graphql-typed-document-node/core" "^3.0.0"
+    "@types/zen-observable" "^0.8.0"
+    "@wry/context" "^0.5.2"
+    "@wry/equality" "^0.2.0"
+    fast-json-stable-stringify "^2.0.0"
+    graphql-tag "^2.11.0"
+    hoist-non-react-statics "^3.3.2"
+    optimism "^0.13.0"
+    prop-types "^15.7.2"
+    symbol-observable "^2.0.0"
+    ts-invariant "^0.4.4"
+    tslib "^1.10.0"
+    zen-observable "^0.8.14"
+
 "@apollo/protobufjs@^1.0.3":
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/@apollo/protobufjs/-/protobufjs-1.0.5.tgz#a78b726147efc0795e74c8cb8a11aafc6e02f773"
@@ -1369,10 +1388,10 @@
     ajv "^6.12.0"
     ajv-keywords "^3.4.1"
 
-"@dzlzv/hydra-cli@^0.0.17":
-  version "0.0.17"
-  resolved "https://registry.yarnpkg.com/@dzlzv/hydra-cli/-/hydra-cli-0.0.17.tgz#56ccae132f76e738724cdc5f0abcd47ff25df530"
-  integrity sha512-ixrjGn6a7UG7ecHYKWTHpcxbdi6X32NbtyCuewm4YGFdb+v0/Eg5zWhFbg1PbMUW9GllC4MiIjDF7Bh1fh9t7Q==
+"@dzlzv/hydra-cli@^0.0.19":
+  version "0.0.19"
+  resolved "https://registry.yarnpkg.com/@dzlzv/hydra-cli/-/hydra-cli-0.0.19.tgz#bd7711cb62a8eb71e447538dcdc2790f8a71001b"
+  integrity sha512-F2Jg9hmLmEOsjrKUp0aVr3VfD5tYTN94RgH0gQnN6I/syq9OtkjZdoV9d3iq6Ujzw7VvAksboytLsa+GViLlZA==
   dependencies:
     "@oclif/command" "^1.5.20"
     "@oclif/config" "^1"
@@ -1683,6 +1702,11 @@
   dependencies:
     prop-types "^15.7.2"
 
+"@graphql-typed-document-node/core@^3.0.0":
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/@graphql-typed-document-node/core/-/core-3.1.0.tgz#0eee6373e11418bfe0b5638f654df7a4ca6a3950"
+  integrity sha512-wYn6r8zVZyQJ6rQaALBEln5B1pzxb9shV5Ef97kTvn6yVGrqyXVnDqnU24MXnFubR+rZjBY9NWuxX3FB2sTsjg==
+
 "@hapi/address@2.x.x":
   version "2.1.4"
   resolved "https://registry.yarnpkg.com/@hapi/address/-/address-2.1.4.tgz#5d67ed43f3fd41a69d4b9ff7b56e7c0d1d0a81e5"
@@ -4914,7 +4938,7 @@
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.159.tgz#61089719dc6fdd9c5cb46efc827f2571d1517065"
   integrity sha512-gF7A72f7WQN33DpqOWw9geApQPh4M3PxluMtaHxWHXEGSN12/WbcEk/eNSqWNQcQhF66VSZ06vCF94CrHwXJDg==
 
-"@types/lodash@^4.14.148", "@types/lodash@^4.14.161":
+"@types/lodash@^4.14.148":
   version "4.14.164"
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.164.tgz#52348bcf909ac7b4c1bcbeda5c23135176e5dfa0"
   integrity sha512-fXCEmONnrtbYUc5014avwBeMdhHHO8YJCkOBflUL9EoJBSKZ1dei+VO74fA7JkTHZ1GvZack2TyIw5U+1lT8jg==
@@ -4924,6 +4948,11 @@
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.157.tgz#fdac1c52448861dfde1a2e1515dbc46e54926dc8"
   integrity sha512-Ft5BNFmv2pHDgxV5JDsndOWTRJ+56zte0ZpYLowp03tW+K+t8u8YMOzAnpuqPgzX6WO1XpDIUm7u04M8vdDiVQ==
 
+"@types/lodash@^4.14.161":
+  version "4.14.165"
+  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.165.tgz#74d55d947452e2de0742bad65270433b63a8c30f"
+  integrity sha512-tjSSOTHhI5mCHTy/OOXYIhi2Wt1qcbHmuXD1Ha7q70CgI/I71afO4XtLb/cVexki1oVYchpul/TOuu3Arcdxrg==
+
 "@types/long@^4.0.0":
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9"
@@ -5458,6 +5487,11 @@
   resolved "https://registry.yarnpkg.com/@types/yup/-/yup-0.26.37.tgz#7a52854ac602ba0dc969bebc960559f7464a1686"
   integrity sha512-cDqR/ez4+iAVQYOwadXjKX4Dq1frtnDGV2GNVKj3aUVKVCKRvsr8esFk66j+LgeeJGmrMcBkkfCf3zk13MjV7A==
 
+"@types/zen-observable@^0.8.0":
+  version "0.8.1"
+  resolved "https://registry.yarnpkg.com/@types/zen-observable/-/zen-observable-0.8.1.tgz#5668c0bce55a91f2b9566b1d8a4c0a8dbbc79764"
+  integrity sha512-wmk0xQI6Yy7Fs/il4EpOcflG4uonUpYGqvZARESLc2oy4u69fkatFLbJOeW4Q6awO15P4rduAe6xkwHevpXcUQ==
+
 "@typescript-eslint/eslint-plugin@3.8.0":
   version "3.8.0"
   resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-3.8.0.tgz#f82947bcdd9a4e42be7ad80dfd61f1dc411dd1df"
@@ -5937,6 +5971,13 @@
     "@webassemblyjs/wast-parser" "1.9.0"
     "@xtuc/long" "4.2.2"
 
+"@wry/context@^0.5.2":
+  version "0.5.2"
+  resolved "https://registry.yarnpkg.com/@wry/context/-/context-0.5.2.tgz#f2a5d5ab9227343aa74c81e06533c1ef84598ec7"
+  integrity sha512-B/JLuRZ/vbEKHRUiGj6xiMojST1kHhu4WcreLfNN7q9DqQFrb97cWgf/kiYsPSUCAMVN0HzfFc8XjJdzgZzfjw==
+  dependencies:
+    tslib "^1.9.3"
+
 "@wry/equality@^0.1.2":
   version "0.1.11"
   resolved "https://registry.yarnpkg.com/@wry/equality/-/equality-0.1.11.tgz#35cb156e4a96695aa81a9ecc4d03787bc17f1790"
@@ -5944,6 +5985,13 @@
   dependencies:
     tslib "^1.9.3"
 
+"@wry/equality@^0.2.0":
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/@wry/equality/-/equality-0.2.0.tgz#a312d1b6a682d0909904c2bcd355b02303104fb7"
+  integrity sha512-Y4d+WH6hs+KZJUC8YKLYGarjGekBrhslDbf/R20oV+AakHPINSitHfDRQz3EGcEWc1luXYNUvMhawWtZVWNGvQ==
+  dependencies:
+    tslib "^1.9.3"
+
 "@xtuc/ieee754@^1.2.0":
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz#eef014a3145ae477a1cbc00cd1e552336dceb790"
@@ -10486,7 +10534,7 @@ date-utils@*:
   resolved "https://registry.yarnpkg.com/date-utils/-/date-utils-1.2.21.tgz#61fb16cdc1274b3c9acaaffe9fc69df8720a2b64"
   integrity sha1-YfsWzcEnSzyayq/+n8ad+HIKK2Q=
 
-dateformat@^3.0.0:
+dateformat@^3.0.0, dateformat@^3.0.2:
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-3.0.3.tgz#a6e37499a4d9a9cf85ef5872044d62901c9889ae"
   integrity sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==
@@ -10499,6 +10547,11 @@ dateformat@~1.0.4-1.2.3:
     get-stdin "^4.0.1"
     meow "^3.3.0"
 
+datejs@^1.0.0-rc3:
+  version "1.0.0-rc3"
+  resolved "https://registry.yarnpkg.com/datejs/-/datejs-1.0.0-rc3.tgz#bffa1efedefeb41fdd8a242af55afa01fb58de57"
+  integrity sha1-v/oe/t7+tB/diiQq9Vr6AftY3lc=
+
 de-indent@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/de-indent/-/de-indent-1.0.2.tgz#b2038e846dc33baa5796128d0804b455b8c1e21d"
@@ -11283,7 +11336,7 @@ dotenv@^6.2.0:
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-6.2.0.tgz#941c0410535d942c8becf28d3f357dbd9d476064"
   integrity sha512-HygQCKUBSFl8wKQZBSemMywRWcEDNidvNbjGVyZu3nbZ8qq9ubiPoGLMdRDpfSrpkkm9BXYFkpKxxFX38o/76w==
 
-dotenvi@^0.9.0:
+dotenvi@^0.9.0, dotenvi@^0.9.1:
   version "0.9.1"
   resolved "https://registry.yarnpkg.com/dotenvi/-/dotenvi-0.9.1.tgz#e280012ee9d201a0c57cb1f6e43559603b6f0fb4"
   integrity sha512-gM9HKu6P8BS+jBQRcJRdWKkbIA35Ztszr2FEqp1oKYLMfdTWDumLNi9xlIeEAFc2C4DeOwsYcNi+mMl5OWGtcw==
@@ -14306,7 +14359,7 @@ graphql-subscriptions@^1.0.0, graphql-subscriptions@^1.1.0:
   dependencies:
     iterall "^1.2.1"
 
-graphql-tag@^2.9.2:
+graphql-tag@^2.11.0, graphql-tag@^2.9.2:
   version "2.11.0"
   resolved "https://registry.yarnpkg.com/graphql-tag/-/graphql-tag-2.11.0.tgz#1deb53a01c46a7eb401d6cb59dec86fa1cccbffd"
   integrity sha512-VmsD5pJqWJnQZMUeRwrDhfgoyqcfwEkvtpANqcoUG8/tOLkwNgU9mzub/Mc78OJMhHjx7gfAMTxzdG43VGg3bA==
@@ -14658,7 +14711,7 @@ hogan.js@^3.0.2:
     mkdirp "0.3.0"
     nopt "1.0.10"
 
-hoist-non-react-statics@^3.0.0:
+hoist-non-react-statics@^3.0.0, hoist-non-react-statics@^3.3.2:
   version "3.3.2"
   resolved "https://registry.yarnpkg.com/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz#ece0acaf71d62c2969c2ec59feff42a4b1a85b45"
   integrity sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==
@@ -19205,7 +19258,7 @@ lodash@^4.17.11, lodash@^4.17.12, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548"
   integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==
 
-lodash@^4.17.20:
+lodash@^4.17.20, lodash@^4.3.0:
   version "4.17.20"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52"
   integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==
@@ -21572,6 +21625,13 @@ opn@^5.5.0:
   dependencies:
     is-wsl "^1.1.0"
 
+optimism@^0.13.0:
+  version "0.13.0"
+  resolved "https://registry.yarnpkg.com/optimism/-/optimism-0.13.0.tgz#c08904e1439a0eb9e7f86183dafa06cc715ff351"
+  integrity sha512-6JAh3dH+YUE4QUdsgUw8nUQyrNeBKfAEKOHMlLkQ168KhIYFIxzPsHakWrRXDnTO+x61RJrS3/2uEt6W0xlocA==
+  dependencies:
+    "@wry/context" "^0.5.2"
+
 optimist@^0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/optimist/-/optimist-0.6.1.tgz#da3ea74686fa21a19a111c326e90eb15a0196686"
@@ -27018,6 +27078,11 @@ symbol-observable@^1.0.4, symbol-observable@^1.1.0, symbol-observable@^1.2.0:
   resolved "https://registry.yarnpkg.com/symbol-observable/-/symbol-observable-1.2.0.tgz#c22688aed4eab3cdc2dfeacbb561660560a00804"
   integrity sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==
 
+symbol-observable@^2.0.0:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/symbol-observable/-/symbol-observable-2.0.3.tgz#5b521d3d07a43c351055fa43b8355b62d33fd16a"
+  integrity sha512-sQV7phh2WCYAn81oAkakC5qjq2Ml0g8ozqz03wOGnx9dDlG1de6yrF+0RAzSJD8fPUow3PTSMf2SAbOGxb93BA==
+
 symbol-tree@^3.2.2, symbol-tree@^3.2.4:
   version "3.2.4"
   resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2"
@@ -27697,7 +27762,7 @@ ts-dedent@^1.1.0:
   resolved "https://registry.yarnpkg.com/ts-dedent/-/ts-dedent-1.1.1.tgz#68fad040d7dbd53a90f545b450702340e17d18f3"
   integrity sha512-UGTRZu1evMw4uTPyYF66/KFd22XiU+jMaIuHrkIHQ2GivAXVlLV0v/vHrpOuTRf9BmpNHi/SO7Vd0rLu0y57jg==
 
-ts-invariant@^0.4.0:
+ts-invariant@^0.4.0, ts-invariant@^0.4.4:
   version "0.4.4"
   resolved "https://registry.yarnpkg.com/ts-invariant/-/ts-invariant-0.4.4.tgz#97a523518688f93aafad01b0e80eb803eb2abd86"
   integrity sha512-uEtWkFM/sdZvRNNDL3Ehu4WVpwaulhwQszV8mrtcdeE8nN00BV9mAmQ88RkrBhFgl9gMgvjJLAQcZbnPXI9mlA==
@@ -30131,7 +30196,7 @@ zen-observable-ts@^0.8.21:
     tslib "^1.9.3"
     zen-observable "^0.8.0"
 
-zen-observable@^0.8.0:
+zen-observable@^0.8.0, zen-observable@^0.8.14:
   version "0.8.15"
   resolved "https://registry.yarnpkg.com/zen-observable/-/zen-observable-0.8.15.tgz#96415c512d8e3ffd920afd3889604e30b9eaac15"
   integrity sha512-PQ2PC7R9rslx84ndNBZB/Dkv8V8fZEpk83RLgXtYd0fwUgEjseMn1Dgajh2x6S8QbZAFa9p2qVCEuYZNgve0dQ==