query-node: configure with two separate databases

.env

@@ -1,17 +1,14 @@
 COMPOSE_PROJECT_NAME=joystream
+PROJECT_NAME=query_node
 
-###########################
-#     Common settings     #
-###########################
-
-# The env variables below are by default used by all services and should be
-# overriden in local env files (e.g. ./generated/indexer) if needed
-# DB config
-DB_NAME=query_node
+# We will use a single postgres service with multiple databases
+INDEXER_DB_NAME=query_node_indexer
+PROCESSOR_DB_NAME=query_node_processor
 DB_USER=postgres
 DB_PASS=postgres
 DB_HOST=localhost
 DB_PORT=5432
+
 DEBUG=index-builder:*
 TYPEORM_LOGGING=error
 
@@ -20,16 +17,12 @@ TYPEORM_LOGGING=error
 ###########################
 
 # Substrate endpoint to source events from
-WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944/
+WS_PROVIDER_ENDPOINT_URI=ws://localhost:9944/
+
 # Block height to start indexing from.
 # Note, that if there are already some indexed events, this setting is ignored
 BLOCK_HEIGHT=0
 
-# Custom types to register for Substrate API
-# TYPE_REGISTER_PACKAGE_NAME=
-# TYPE_REGISTER_PACKAGE_VERSION=
-# TYPE_REGISTER_FUNCTION=
-
 # Redis cache server
 REDIS_URI=redis://localhost:6379/0
 

docker-compose.yml

@@ -44,7 +44,7 @@ services:
       dockerfile: apps.Dockerfile
     ports:
       - '127.0.0.1:3001:3001'
-    command: colossus --dev --ws-provider ${WS_PROVIDER_ENDPOINT_URI} --ipfs-host ipfs
+    command: colossus --dev --ws-provider ws://joystream-node:9944 --ipfs-host ipfs
     environment:
       - DEBUG=*
 
@@ -58,7 +58,7 @@ services:
     environment:
       POSTGRES_USER: ${DB_USER}
       POSTGRES_PASSWORD: ${DB_PASS}
-      POSTGRES_DB: ${DB_NAME}
+      POSTGRES_DB: ${INDEXER_DB_NAME}
 
   graphql-server:
     image: joystream/apps
@@ -91,7 +91,7 @@ services:
       - DB_HOST=db
       - TYPEORM_HOST=db
       - DEBUG=index-builder:*
-      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
+      - WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944
     depends_on:
       - indexer-api-gateway
     command: ["workspace", "query-node-root", "processor:start"]
@@ -104,14 +104,15 @@ services:
       dockerfile: apps.Dockerfile
     env_file:
       # relative to working directory where docker-compose was run from 
-      - .env 
+      - .env
     environment:
       - TYPEORM_HOST=db
+      - TYPEORM_DATABASE=${INDEXER_DB_NAME}
       - INDEXER_WORKERS=5
       - PROCESSOR_POLL_INTERVAL=1000 # refresh every second 
       - REDIS_URI=redis://redis:6379/0
       - DEBUG=index-builder:*
-      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
+      - WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944
     depends_on: 
       - db
     command: ["workspace", "query-node-root", "indexer:start"] 
@@ -120,12 +121,12 @@ services:
     image: joystream/hydra-indexer-gateway:latest
     restart: unless-stopped
     environment:
-      - WARTHOG_STARTER_DB_DATABASE=${DB_NAME}
-      - WARTHOG_STARTER_DB_HOST=db 
+      - WARTHOG_STARTER_DB_DATABASE=${INDEXER_DB_NAME}
+      - WARTHOG_STARTER_DB_HOST=db
       - WARTHOG_STARTER_DB_PASSWORD=${DB_PASS}
       - WARTHOG_STARTER_DB_PORT=${DB_PORT}
       - WARTHOG_STARTER_DB_USERNAME=${DB_USER}
-      - WARTHOG_STARTER_REDIS_URI=redis://redis:6379/0 
+      - WARTHOG_STARTER_REDIS_URI=redis://redis:6379/0
       - PORT=4000
     ports:
       - "127.0.0.1:4000:4000"

query-node/.env

@@ -1,62 +0,0 @@
-COMPOSE_PROJECT_NAME=joystream
-
-# Project name
-PROJECT_NAME=query_node
-
-###########################
-#     Common settings     #
-###########################
-
-# The env variables below are by default used by all services and should be
-# overriden in local env files (e.g. ./generated/indexer) if needed
-# DB config
-DB_NAME=query_node
-DB_USER=postgres
-DB_PASS=postgres
-DB_HOST=localhost
-DB_PORT=5432
-DEBUG=index-builder:*
-TYPEORM_LOGGING=error
-
-###########################
-#    Indexer options      #
-###########################
-
-# Substrate endpoint to source events from
-WS_PROVIDER_ENDPOINT_URI=ws://localhost:9944/
-# Block height to start indexing from.
-# Note, that if there are already some indexed events, this setting is ignored
-BLOCK_HEIGHT=0
-
-# Custom types to register for Substrate API
-# TYPE_REGISTER_PACKAGE_NAME=
-# TYPE_REGISTER_PACKAGE_VERSION=
-# TYPE_REGISTER_FUNCTION=
-
-# Redis cache server
-REDIS_URI=redis://localhost:6379/0
-
-###########################
-#    Processor options    #
-###########################
-
-# Where the mapping scripts are located, relative to ./generated/indexer
-# this env var is deprecated!
-MAPPINGS_LOCATION=../../src
-TYPES_JSON=../../typedefs.json
-
-# Indexer GraphQL API endpoint to fetch indexed events
-INDEXER_ENDPOINT_URL=http://localhost:4100/graphql
-
-# Block height from which the processor starts. Note that if
-# there are already processed events in the database, this setting is ignored
-BLOCK_HEIGHT=0
-
-###############################
-#    Processor GraphQL API    #
-###############################
-
-GRAPHQL_SERVER_PORT=4002
-GRAPHQL_SERVER_HOST=localhost
-WARTHOG_APP_PORT=4002
-WARTHOG_APP_HOST=localhost

query-node/README.md

@@ -11,6 +11,10 @@ $ cd query-node
 $ yarn build
 ```
 
+## Starting services
+
+To start services defined in the project docker-compose.yml, you should run docker-compose from the project root folder to use the correct .env file
+
 ## Run mapping processor
 
 Before running mappings make sure indexer(`yarn indexer:start`) and indexer-api-server (mappings get the chain data from this graphql server) are both running:
@@ -26,7 +30,7 @@ Once processor start to store event data you will be able to query this data fro
 ```graphql
 query {
   channels {
-    title
+    handle
   }
 }
 ```

query-node/build.sh

@@ -4,8 +4,26 @@ set -e
 SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
 cd $SCRIPT_PATH
 
+# Load and export variables from root .env file into shell environment
+set -a
+. ../.env
+set +a
+
 yarn clean
-yarn codegen:all
+
+# We generate the code for each service separately to be able to specify
+# separate database names.
+
+# Build indexer customizing DB name and port
+DB_NAME=${INDEXER_DB_NAME} yarn codegen:indexer
+
+# Build graphql-server customizing DB name and port
+DB_NAME=${PROCESSOR_DB_NAME} yarn codegen:server
+
+# We run yarn again to ensure processor and indexer dependencies are installed
+# and are inline with root workspace resolutions
 yarn
+
 ln -s ../../../../../node_modules/typeorm/cli.js generated/graphql-server/node_modules/.bin/typeorm || :
+
 yarn tsc --build tsconfig.json

query-node/package.json

@@ -11,12 +11,11 @@
 		"server:start:dev": "(cd ./generated/graphql-server && yarn start:dev)",
 		"server:start:prod": "(cd ./generated/graphql-server && yarn start:prod)",
 		"configure": "(cd ./generated/graphql-server && yarn config:dev)",
-		"db:up": "docker-compose up -d db",
+		"db:up": "(cd ../ && docker-compose up -d db)",
 		"db:drop": "(cd ./generated/graphql-server && yarn db:drop)",
+		"db:migrate": "yarn db:schema:migrate && yarn db:indexer:migrate",
 		"db:schema:migrate": "(cd ./generated/graphql-server && yarn db:create && yarn db:sync && yarn db:migrate)",
 		"db:indexer:migrate": "(cd ./generated/indexer && yarn db:migrate)",
-		"db:migrate": "yarn db:schema:migrate && yarn db:indexer:migrate",
-		"codegen:all": "yarn hydra-cli codegen --no-install && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
 		"codegen:indexer": "yarn hydra-cli codegen --no-install --no-graphql && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
 		"codegen:server": "yarn hydra-cli codegen --no-install --no-indexer",
 		"cd-classes": "ts-node scripts/get-class-id-and-name.ts"

query-node/run-tests.sh

@@ -4,6 +4,19 @@ set -e
 SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
 cd $SCRIPT_PATH
 
+# Only run codegen if no generated files found
+[ ! -d "generated/" ] && yarn build
+
+# Make sure typeorm is available.. it get removed when yarn is run again
+# typeorm commandline is used by db:migrate step below.
+ln -s ../../../../../node_modules/typeorm/cli.js generated/graphql-server/node_modules/.bin/typeorm || :
+
+# Move to root directory so docker-compose uses .env file from the project root directory
+cd ../
+
+# Clean start
+docker-compose down -v
+
 function cleanup() {
     # Show tail end of logs for the processor and indexer containers to
     # see any possible errors
@@ -14,23 +27,20 @@ function cleanup() {
 
 trap cleanup EXIT
 
+# We expect docker image to be started by test runner
 export WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944/
 
-# Only run codegen if no generated files found
-[ ! -d "generated/" ] && yarn build
-
-# Make sure typeorm is available.. it get removed again when yarn is run again
-# typeorm commandline is used by db:migrate step below.
-ln -s ../../../../../node_modules/typeorm/cli.js generated/graphql-server/node_modules/.bin/typeorm || :
+# Bring up db
+docker-compose up -d db
 
-# clean start
-docker-compose down -v
+# Migrate the databases
+yarn workspace query-node-root db:indexer:migrate
+yarn workspace query-node-root db:schema:migrate
 
-docker-compose up -d db
-yarn db:migrate
 docker-compose up -d graphql-server
+
 # Starting up processor will bring up all services it depends on
 docker-compose up -d processor
 
 # Run tests
-ATTACH_TO_NETWORK=joystream_default ../tests/network-tests/run-tests.sh content-directory
+ATTACH_TO_NETWORK=joystream_default tests/network-tests/run-tests.sh content-directory

setup.sh

@@ -21,8 +21,8 @@ rustup install 1.46.0
 rustup default 1.46.0
 
 if [[ "$OSTYPE" == "linux-gnu" ]]; then
-    apt-get install -y coreutils clang jq curl gcc xz-utils sudo pkg-config unzip clang libc6-dev-i386
-    apt-get install -y docker.io docker-compose
+    sudo apt-get install -y coreutils clang jq curl gcc xz-utils sudo pkg-config unzip clang libc6-dev-i386
+    sudo apt-get install -y docker.io docker-compose
 elif [[ "$OSTYPE" == "darwin"* ]]; then
     brew install b2sum gnu-tar jq curl
     echo "It is recommended to setup Docker desktop from: https://www.docker.com/products/docker-desktop"

start.sh

@@ -17,6 +17,7 @@ trap down EXIT
 
 # Run a local development chain
 docker-compose up -d joystream-node
+export WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944/
 
 ## Storage Infrastructure
 # Configure a dev storage node and start storage node