Improve query-node docker setup

.dockerignore

@@ -7,3 +7,4 @@ query-node/lib
 cli/
 tests/
 devops/
+metadata-protobuf/lib

.env

@@ -2,14 +2,14 @@ COMPOSE_PROJECT_NAME=joystream
 PROJECT_NAME=query_node
 
 # We will use a single postgres service with multiple databases
-# The env variables below are by default used by all services and should be 
+# The env variables below are by default used by all services and should be
 # overriden in local env files
 # DB config
 INDEXER_DB_NAME=query_node_indexer
 DB_NAME=query_node_processor
 DB_USER=postgres
 DB_PASS=postgres
-DB_HOST=localhost
+DB_HOST=db
 DB_PORT=5432
 DEBUG=index-builder:*
 TYPEORM_LOGGING=error
@@ -30,9 +30,10 @@ BLOCK_HEIGHT=0
 ###############################
 
 GRAPHQL_SERVER_PORT=4002
-GRAPHQL_SERVER_HOST=localhost
+GRAPHQL_SERVER_HOST=graphql-server
+
 WARTHOG_APP_PORT=4002
-WARTHOG_APP_HOST=localhost
+WARTHOG_APP_HOST=hydra-indexer-gateway
 
 # Default configuration is to use the docker container
 WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944/

apps.Dockerfile

@@ -1,17 +1,25 @@
+FROM mikefarah/yq as manifest-maker
+# Change metadata.source in manifest file. It's not possible to override it via flag/env.
+USER root
+ARG WS_PROVIDER_ENDPOINT_URI
+COPY ./query-node/manifest.yml /joystream/qn-manifest.yml
+RUN yq e -i ".typegen.metadata.source = \"$WS_PROVIDER_ENDPOINT_URI\"" /joystream/qn-manifest.yml
+
 FROM --platform=linux/x86-64 node:14 as builder
 
 WORKDIR /joystream
 COPY . /joystream
-RUN  rm -fr /joystream/pioneer
+COPY --from=manifest-maker /joystream/qn-manifest.yml /joystream/query-node/manifest.yml
+
+RUN rm -fr /joystream/pioneer
 
 # Do not set NODE_ENV=production until after running yarn install
 # to ensure dev dependencies are installed.
 RUN yarn --forzen-lockfile
 
 RUN yarn workspace @joystream/types build
-RUN yarn workspace @joystream/content-metadata-protobuf build:ts
+RUN yarn workspace @joystream/metadata-protobuf build
 RUN yarn workspace query-node-root build
-RUN yarn workspace storage-node build
 
 # Second stage to reduce image size, enable it when
 # all packages have correctly identified what is a devDependency and what is not.

docker-compose.yml

@@ -45,8 +45,6 @@ services:
     env_file:
       # relative to working directory where docker-compose was run from
       - .env
-    environment:
-      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
     ports:
       - '127.0.0.1:3001:3001'
     command: colossus --dev --ws-provider ${WS_PROVIDER_ENDPOINT_URI} --ipfs-host ipfs
@@ -74,12 +72,12 @@ services:
     build:
       context: .
       dockerfile: apps.Dockerfile
+      network: joystream_default
+      args:
+        - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
     env_file:
       # relative to working directory where docker-compose was run from
       - .env
-    environment:
-      - DB_HOST=db
-      - DB_NAME=${DB_NAME}
     ports:
       - "127.0.0.1:8081:${GRAPHQL_SERVER_PORT}"
     depends_on:
@@ -92,9 +90,6 @@ services:
     env_file:
       # relative to working directory where docker-compose was run from
       - .env
-    environment:
-      - DB_HOST=db
-      - DB_NAME=${DB_NAME}
     ports:
       - "127.0.0.1:8081:${GRAPHQL_SERVER_PORT}"
     depends_on:
@@ -112,15 +107,17 @@ services:
     build:
       context: .
       dockerfile: apps.Dockerfile
+      network: joystream_default
+      args:
+        - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
     env_file:
       # relative to working directory where docker-compose was run from
       - .env
     environment:
       - INDEXER_ENDPOINT_URL=http://hydra-indexer-gateway:${WARTHOG_APP_PORT}/graphql
-      - TYPEORM_HOST=db
+      - TYPEORM_HOST=${DB_HOST}
       - TYPEORM_DATABASE=${DB_NAME}
-      - DEBUG=index-builder:*
-      - WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944
+      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
     volumes:
       - ./types/augment/all/defs.json:/joystream/query-node/mappings/lib/generated/types/typedefs.json
     depends_on:
@@ -135,10 +132,8 @@ services:
       - .env
     environment:
       - INDEXER_ENDPOINT_URL=http://hydra-indexer-gateway:${WARTHOG_APP_PORT}/graphql
-      - TYPEORM_HOST=db
+      - TYPEORM_HOST=${DB_HOST}
       - TYPEORM_DATABASE=${DB_NAME}
-      - DEBUG=index-builder:*
-      - WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944
     depends_on:
       - hydra-indexer-gateway
     volumes:
@@ -155,12 +150,9 @@ services:
       # relative to working directory where docker-compose was run from
       - .env
     environment:
-      - DB_HOST=db
       - DB_NAME=${INDEXER_DB_NAME}
       - INDEXER_WORKERS=5
       - REDIS_URI=redis://redis:6379/0
-      - DEBUG=index-builder:*
-      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
       - TYPES_JSON=types.json
     depends_on:
       - db
@@ -187,7 +179,7 @@ services:
       - PORT=${WARTHOG_APP_PORT}
       - DEBUG=*
     ports:
-      - "127.0.0.1:4000:4002"
+      - "127.0.0.1:4000:${WARTHOG_APP_PORT}"
     depends_on:
       - redis
       - db

metadata-protobuf/package.json

@@ -20,7 +20,7 @@
   "license": "MIT",
   "private": false,
   "scripts": {
-    "build": "yarn compile && tsc",
+    "build": "yarn compile && rm -rf lib && tsc",
     "compile": "yarn ts-node ./scripts/compile.ts",
     "generate-doc": "./generate-md-doc.sh",
     "test": "env TS_NODE_COMPILER_OPTIONS='{\"module\": \"commonjs\" }' mocha --inline-diffs -r ts-node/register 'test/**/*.ts'",

query-node/build.sh

@@ -9,7 +9,6 @@ set -a
 . ../.env
 set +a
 
-# only use this when new Hydra releases and contents of `generated/` folder needs to be refreshed
 yarn clean
 yarn codegen:noinstall
 yarn typegen # if this fails try to run this command outside of yarn workspaces

query-node/kill-img.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+cd $SCRIPT_PATH
+
+set -a
+. ../.env
+set +a
+
+# Only remove query-node related services
+docker-compose rm -vsf processor
+docker-compose rm -vsf graphql-server
+docker-compose rm -vsf indexer
+docker-compose rm -vsf hydra-indexer-gateway
+docker-compose rm -vsf redis
+docker-compose rm -vsf db

query-node/package.json

@@ -28,10 +28,10 @@
     "typegen:configure": "NODE_URL=${NODE_URL:-ws://localhost:9000} envsub typegen.template.yml typegen.yml",
     "typegen": "rm -rf ./mappings/generated && hydra-typegen typegen manifest.yml --debug",
     "mappings:build": "yarn workspace query-node-mappings build",
-    "docker:build": "docker build . -f docker/Dockerfile.hydra -t hydra-kit:latest",
-    "docker:db:up": "(cd ../ && docker-compose up -d db)",
-    "docker:db:migrate": "docker run --env-file .env --env DB_HOST=db --env TYPEORM_HOST=db --network container:${PWD##*/}_db_1 hydra-kit:latest yarn db:migrate",
-    "docker:up": "docker-compose up -d"
+    "start:dev": "./start.sh",
+    "start": "./start-img.sh",
+    "kill:dev": "./kill.sh",
+    "kill": "./kill-img.sh"
   },
   "author": "",
   "license": "ISC",

query-node/start-img.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -e
+
+SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+cd $SCRIPT_PATH
+
+set -a
+. ../.env
+set +a
+
+# Start the joystream-node first to allow fetching Olympia metadata during build (typegen)
+docker-compose up -d joystream-node
+
+# Bring up db
+docker-compose up -d db
+
+# Setup the db
+docker run --rm --env-file ../.env --network joystream_default joystream/apps workspace query-node-root db:prepare
+docker run --rm --env-file ../.env --network joystream_default joystream/apps workspace query-node-root db:migrate
+
+# Start processor and graphql server
+docker-compose up -d processor
+docker-compose up -d graphql-server

query-node/start.sh

@@ -8,6 +8,9 @@ set -a
 . ../.env
 set +a
 
+# Override DB_HOST
+export DB_HOST=localhost
+
 # Start the joystream-node first to allow fetching Olympia metadata during build (typegen)
 docker-compose up -d joystream-node