docker: consolidate docker-compose files

.env

@@ -0,0 +1,55 @@
+###########################
+#     Common settings     #
+###########################
+
+# The env variables below are by default used by all services and should be
+# overriden in local env files (e.g. ./generated/indexer) if needed
+# DB config
+DB_NAME=query_node
+DB_USER=postgres
+DB_PASS=postgres
+DB_HOST=localhost
+DB_PORT=5432
+DEBUG=index-builder:*
+TYPEORM_LOGGING=error
+
+###########################
+#    Indexer options      #
+###########################
+
+# Substrate endpoint to source events from
+WS_PROVIDER_ENDPOINT_URI=ws://localhost:9944/
+# Block height to start indexing from.
+# Note, that if there are already some indexed events, this setting is ignored
+BLOCK_HEIGHT=0
+
+# Custom types to register for Substrate API
+# TYPE_REGISTER_PACKAGE_NAME=
+# TYPE_REGISTER_PACKAGE_VERSION=
+# TYPE_REGISTER_FUNCTION=
+
+# Redis cache server
+REDIS_URI=redis://localhost:6379/0
+
+###########################
+#    Processor options    #
+###########################
+
+# Where the mapping scripts are located, relative to ./generated/indexer
+TYPES_JSON=../../typedefs.json
+
+# Indexer GraphQL API endpoint to fetch indexed events
+INDEXER_ENDPOINT_URL=http://localhost:4100/graphql
+
+# Block height from which the processor starts. Note that if
+# there are already processed events in the database, this setting is ignored
+BLOCK_HEIGHT=0
+
+###############################
+#    Processor GraphQL API    #
+###############################
+
+GRAPHQL_SERVER_PORT=4002
+GRAPHQL_SERVER_HOST=localhost
+WARTHOG_APP_PORT=4002
+WARTHOG_APP_HOST=localhost

docker-compose-with-storage.yml

@@ -1,39 +0,0 @@
-version: '3'
-services:
-  ipfs:
-    image: ipfs/go-ipfs:latest
-    ports:
-      - '127.0.0.1:5001:5001'
-      - '127.0.0.1:8080:8080'
-    entrypoint: ''
-    command: |
-      /bin/sh -c "
-        set -e
-        /usr/local/bin/start_ipfs config profile apply lowpower
-        /usr/local/bin/start_ipfs config --json Gateway.PublicGateways '{\"localhost\": null }'
-        /sbin/tini -- /usr/local/bin/start_ipfs daemon --migrate=true
-      "
-  chain:
-    image: joystream/node
-    build:
-      context: .
-      dockerfile: joystream-node.Dockerfile
-    ports:
-      - '127.0.0.1:9944:9944'
-    command: --dev --ws-external --base-path /data --log runtime
-
-  colossus:
-    image: joystream/apps
-    restart: on-failure
-    depends_on:
-      - "chain"
-      - "ipfs"
-    build:
-      context: .
-      dockerfile: apps.Dockerfile
-    ports:
-      - '127.0.0.1:3001:3001'
-    command: colossus --dev --ws-provider ws://chain:9944 --ipfs-host ipfs
-    environment:
-      - DEBUG=*
-

docker-compose.yml

@@ -1,17 +1,150 @@
-# Compiles new joystream node image if local image not found,
-# and runs local development chain.
-# To prevent build run docker-compose with "--no-build" arg
-version: "3"
+# Compiles new joystream/node and joystream/apps images if local images not found
+# and runs a complete joystream development network
+# To prevent build of docker images run docker-compose with "--no-build" arg
+version: "3.4"
 services:
   joystream-node:
-    image: joystream/node
+    image: joystream/node:latest
     build:
       # context is relative to the compose file
       context: .
       # dockerfile is relative to the context
       dockerfile: joystream-node.Dockerfile
     container_name: joystream-node
-    command: --dev --alice --validator --unsafe-ws-external --rpc-cors=all --log runtime
+    volumes:
+      - /data
+    command: --dev --alice --validator --unsafe-ws-external --rpc-cors=all --log runtime --base-path /data
     ports:
-      - "9944:9944"
-  
+      - "127.0.0.1:9944:9944"
+
+  ipfs:
+    image: ipfs/go-ipfs:latest
+    ports:
+      - '127.0.0.1:5001:5001'
+      - '127.0.0.1:8080:8080'
+    volumes:
+      - /data/ipfs
+    entrypoint: ''
+    command: |
+      /bin/sh -c "
+        set -e
+        /usr/local/bin/start_ipfs config profile apply lowpower
+        /usr/local/bin/start_ipfs config --json Gateway.PublicGateways '{\"localhost\": null }'
+        /sbin/tini -- /usr/local/bin/start_ipfs daemon --migrate=true
+      "
+
+  colossus:
+    image: joystream/apps
+    restart: on-failure
+    depends_on:
+      - "joystream-node"
+      - "ipfs"
+    build:
+      context: .
+      dockerfile: apps.Dockerfile
+    ports:
+      - '127.0.0.1:3001:3001'
+    command: colossus --dev --ws-provider ws://joystream-node:9944 --ipfs-host ipfs
+    environment:
+      - DEBUG=*
+
+  db:
+    image: postgres:12
+    restart: always
+    ports:
+      - "${DB_PORT}:5432"
+    volumes:
+      - /var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: ${DB_USER}
+      POSTGRES_PASSWORD: ${DB_PASS}
+      POSTGRES_DB: ${DB_NAME}
+
+  graphql-server:
+    image: joystream/apps
+    restart: unless-stopped
+    build: 
+      context: .
+      dockerfile: apps.Dockerfile
+    env_file:
+      # relative to working directory where docker-compose was run from 
+      - .env
+    environment:
+      - DB_HOST=db
+    ports:
+      - "8080:${GRAPHQL_SERVER_PORT}"
+    depends_on: 
+      - db
+    command: ["workspace", "query-node-root", "server:start:prod"]
+
+  processor:
+    image: joystream/apps
+    restart: unless-stopped
+    build: 
+      context: .
+      dockerfile: apps.Dockerfile
+    env_file:
+      # relative to working directory where docker-compose was run from 
+      - .env
+    environment:
+      - INDEXER_ENDPOINT_URL=http://indexer-api-gateway:4000/graphql
+      - DB_HOST=db
+      - TYPEORM_HOST=db
+      - DEBUG=index-builder:*
+      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
+    depends_on:
+      - indexer-api-gateway
+    command: ["workspace", "query-node-root", "processor:start"]
+
+  indexer:
+    image: joystream/apps
+    restart: unless-stopped
+    build: 
+      context: .
+      dockerfile: apps.Dockerfile
+    env_file:
+      # relative to working directory where docker-compose was run from 
+      - .env 
+    environment:
+      - TYPEORM_HOST=db
+      - INDEXER_WORKERS=5
+      - PROCESSOR_POLL_INTERVAL=1000 # refresh every second 
+      - REDIS_URI=redis://redis:6379/0
+      - DEBUG=index-builder:*
+      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
+    depends_on: 
+      - db
+    command: ["workspace", "query-node-root", "indexer:start"] 
+
+  indexer-api-gateway:
+    image: joystream/hydra-indexer-gateway:latest
+    restart: unless-stopped
+    environment:
+      - WARTHOG_STARTER_DB_DATABASE=${DB_NAME}
+      - WARTHOG_STARTER_DB_HOST=db 
+      - WARTHOG_STARTER_DB_PASSWORD=${DB_PASS}
+      - WARTHOG_STARTER_DB_PORT=${DB_PORT}
+      - WARTHOG_STARTER_DB_USERNAME=${DB_USER}
+      - WARTHOG_STARTER_REDIS_URI=redis://redis:6379/0 
+      - PORT=4000
+    ports:
+      - "4000:4000"
+    depends_on:
+      - redis
+      - db
+      - indexer
+
+  redis:
+    image: redis:6.0-alpine
+    restart: always
+    ports:
+      - "6379:6379"
+
+  pioneer:
+    image: joystream/apps
+    build:
+      context: .
+      dockerfile: apps.Dockerfile
+    ports:
+      - "127.0.0.1:3000:3000"
+    command: workspace pioneer start

package.json

@@ -34,7 +34,7 @@
     "babel-core": "^7.0.0-bridge.0",
     "typescript": "^3.9.7",
     "bn.js": "^5.1.2",
-    "@dzlzv/hydra-indexer-lib": "0.0.19-legacy.1.26.1"
+    "@dzlzv/hydra-indexer-lib": "^0.0.19-legacy.1.26.1"
   },
   "devDependencies": {
     "eslint": "^7.6.0",

query-node/docker-compose.yml

@@ -1,91 +0,0 @@
-version: "3.4"
-
-services:
-  db:
-    image: postgres:12
-    restart: always
-    ports:
-      - "${DB_PORT}:5432"
-    volumes:
-      - /var/lib/postgresql/data
-    environment:
-      POSTGRES_USER: ${DB_USER}
-      POSTGRES_PASSWORD: ${DB_PASS}
-      POSTGRES_DB: ${DB_NAME}
-
-  graphql-server:
-    image: joystream/apps
-    restart: unless-stopped
-    build: 
-      context: ../
-      dockerfile: apps.Dockerfile
-    env_file:
-      - .env
-    environment:
-      - DB_HOST=db
-    ports:
-      - "8080:${GRAPHQL_SERVER_PORT}"
-    depends_on: 
-      - db
-    command: ["workspace", "query-node-root", "server:start:prod"]
-
-  processor:
-    image: joystream/apps
-    restart: unless-stopped
-    build: 
-      context: ../
-      dockerfile: apps.Dockerfile
-    env_file:
-      - .env
-    environment:
-      - INDEXER_ENDPOINT_URL=http://indexer-api-gateway:4000/graphql
-      - DB_HOST=db
-      - TYPEORM_HOST=db
-      - DEBUG=index-builder:*
-      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
-    depends_on:
-      - indexer-api-gateway
-    command: ["workspace", "query-node-root", "processor:start"]
-  
-  indexer:
-    image: joystream/apps
-    restart: unless-stopped
-    build: 
-      context: ../
-      dockerfile: apps.Dockerfile
-    env_file:
-      - .env 
-    environment:
-      - TYPEORM_HOST=db
-      - INDEXER_WORKERS=5
-      - PROCESSOR_POLL_INTERVAL=1000 # refresh every second 
-      - REDIS_URI=redis://redis:6379/0
-      - DEBUG=index-builder:*
-      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
-    depends_on: 
-      - db
-    command: ["workspace", "query-node-root", "indexer:start"] 
-  
-  indexer-api-gateway:
-    image: joystream/hydra-indexer-gateway:latest
-    restart: unless-stopped
-    environment:
-      - WARTHOG_STARTER_DB_DATABASE=${DB_NAME}
-      - WARTHOG_STARTER_DB_HOST=db 
-      - WARTHOG_STARTER_DB_PASSWORD=${DB_PASS}
-      - WARTHOG_STARTER_DB_PORT=${DB_PORT}
-      - WARTHOG_STARTER_DB_USERNAME=${DB_USER}
-      - WARTHOG_STARTER_REDIS_URI=redis://redis:6379/0 
-      - PORT=4000
-    ports:
-      - "4000:4000"
-    depends_on:
-      - redis
-      - db
-      - indexer
-    
-  redis:
-    image: redis:6.0-alpine
-    restart: always
-    ports:
-      - "6379:6379"

query-node/package.json

@@ -16,22 +16,23 @@
 		"db:schema:migrate": "(cd ./generated/graphql-server && yarn db:create && yarn db:sync && yarn db:migrate)",
 		"db:indexer:migrate": "(cd ./generated/indexer && yarn db:migrate)",
 		"db:migrate": "yarn db:schema:migrate && yarn db:indexer:migrate",
-		"codegen:all": "yarn hydra-cli codegen && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
-		"codegen:indexer": "yarn hydra-cli codegen --no-graphql && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
-		"codegen:server": "yarn hydra-cli codegen --no-indexer",
-		"docker:up": "docker-compose up -d"
+		"codegen:all": "yarn hydra-cli codegen --no-install && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
+		"codegen:indexer": "yarn hydra-cli codegen --no-install --no-graphql && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
+		"codegen:server": "yarn hydra-cli codegen --no-install --no-indexer"
 	},
 	"author": "",
 	"license": "ISC",
 	"devDependencies": {
-		"@dzlzv/hydra-cli": "^0.0.17"
+		"@dzlzv/hydra-cli": "^0.0.19"
 	},
 	"dependencies": {
+		"@dzlzv/hydra-indexer-lib": "^0.0.19-legacy.1.26.1",
 		"@joystream/types": "^0.14.0",
 		"@types/bn.js": "^4.11.6",
 		"@types/debug": "^4.1.5",
 		"bn.js": "^5.1.2",
 		"debug": "^4.2.0",
+		"dotenvi": "^0.9.1",
 		"tslib": "^2.0.0"
 	}
 }

query-node/run-tests.sh

@@ -23,9 +23,13 @@ export WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944/
 # typeorm commandline is used by db:migrate step below.
 ln -s ../../../../../node_modules/typeorm/cli.js generated/graphql-server/node_modules/.bin/typeorm || :
 
-yarn db:up
+docker-compose up -d db
 yarn db:migrate
-yarn docker:up
+docker-compose up -d redis
+docker-compose up -d indexer
+docker-compose up -d graphql-server
+docker-compose up -d indexer-api-gateway
+docker-compose up -d processor
 
 # Run tests
 ATTACH_TO_NETWORK=query-node_default ../tests/network-tests/run-tests.sh content-directory

storage-node/docker-compose.yaml

@@ -1,29 +0,0 @@
-version: '3'
-services:
-  ipfs:
-    image: ipfs/go-ipfs:latest
-    ports:
-      - '127.0.0.1:5001:5001'
-      - '127.0.0.1:8080:8080'
-    volumes:
-      - ipfs-data:/data/ipfs
-    entrypoint: ''
-    command: |
-      /bin/sh -c "
-        set -e
-        /usr/local/bin/start_ipfs config profile apply lowpower
-        /usr/local/bin/start_ipfs config --json Gateway.PublicGateways '{\"localhost\": null }'
-        /sbin/tini -- /usr/local/bin/start_ipfs daemon --migrate=true
-      "
-  chain:
-    image: joystream/node:latest
-    ports:
-      - '127.0.0.1:9944:9944'
-    volumes:
-      - chain-data:/data
-    command: --dev --ws-external --base-path /data
-volumes:
-  ipfs-data:
-    driver: local
-  chain-data:
-    driver: local

storage-node/start-dev.sh

@@ -16,15 +16,16 @@ cd $SCRIPT_PATH
 # stop prior run and clear volumes
 # docker-compose down -v
 
-# Run a development joystream-node chain and ipfs daemon in the background
-docker-compose up -d
-
 function down()
 {
     # Stop containers and clear volumes
     docker-compose down -v
 }
 
+# Run a development joystream-node chain and ipfs daemon in the background
+docker-compose up -d ipfs
+docker-compose up -d joystream-node
+
 trap down EXIT
 
 # configure the dev chain

yarn.lock

@@ -1388,10 +1388,10 @@
     ajv "^6.12.0"
     ajv-keywords "^3.4.1"
 
-"@dzlzv/hydra-cli@^0.0.17":
-  version "0.0.17"
-  resolved "https://registry.yarnpkg.com/@dzlzv/hydra-cli/-/hydra-cli-0.0.17.tgz#56ccae132f76e738724cdc5f0abcd47ff25df530"
-  integrity sha512-ixrjGn6a7UG7ecHYKWTHpcxbdi6X32NbtyCuewm4YGFdb+v0/Eg5zWhFbg1PbMUW9GllC4MiIjDF7Bh1fh9t7Q==
+"@dzlzv/hydra-cli@^0.0.19":
+  version "0.0.19"
+  resolved "https://registry.yarnpkg.com/@dzlzv/hydra-cli/-/hydra-cli-0.0.19.tgz#bd7711cb62a8eb71e447538dcdc2790f8a71001b"
+  integrity sha512-F2Jg9hmLmEOsjrKUp0aVr3VfD5tYTN94RgH0gQnN6I/syq9OtkjZdoV9d3iq6Ujzw7VvAksboytLsa+GViLlZA==
   dependencies:
     "@oclif/command" "^1.5.20"
     "@oclif/config" "^1"
@@ -4938,7 +4938,7 @@
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.159.tgz#61089719dc6fdd9c5cb46efc827f2571d1517065"
   integrity sha512-gF7A72f7WQN33DpqOWw9geApQPh4M3PxluMtaHxWHXEGSN12/WbcEk/eNSqWNQcQhF66VSZ06vCF94CrHwXJDg==
 
-"@types/lodash@^4.14.148", "@types/lodash@^4.14.161":
+"@types/lodash@^4.14.148":
   version "4.14.164"
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.164.tgz#52348bcf909ac7b4c1bcbeda5c23135176e5dfa0"
   integrity sha512-fXCEmONnrtbYUc5014avwBeMdhHHO8YJCkOBflUL9EoJBSKZ1dei+VO74fA7JkTHZ1GvZack2TyIw5U+1lT8jg==
@@ -4948,6 +4948,11 @@
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.157.tgz#fdac1c52448861dfde1a2e1515dbc46e54926dc8"
   integrity sha512-Ft5BNFmv2pHDgxV5JDsndOWTRJ+56zte0ZpYLowp03tW+K+t8u8YMOzAnpuqPgzX6WO1XpDIUm7u04M8vdDiVQ==
 
+"@types/lodash@^4.14.161":
+  version "4.14.165"
+  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.165.tgz#74d55d947452e2de0742bad65270433b63a8c30f"
+  integrity sha512-tjSSOTHhI5mCHTy/OOXYIhi2Wt1qcbHmuXD1Ha7q70CgI/I71afO4XtLb/cVexki1oVYchpul/TOuu3Arcdxrg==
+
 "@types/long@^4.0.0":
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9"
@@ -11341,7 +11346,7 @@ dotenv@^6.2.0:
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-6.2.0.tgz#941c0410535d942c8becf28d3f357dbd9d476064"
   integrity sha512-HygQCKUBSFl8wKQZBSemMywRWcEDNidvNbjGVyZu3nbZ8qq9ubiPoGLMdRDpfSrpkkm9BXYFkpKxxFX38o/76w==
 
-dotenvi@^0.9.0:
+dotenvi@^0.9.0, dotenvi@^0.9.1:
   version "0.9.1"
   resolved "https://registry.yarnpkg.com/dotenvi/-/dotenvi-0.9.1.tgz#e280012ee9d201a0c57cb1f6e43559603b6f0fb4"
   integrity sha512-gM9HKu6P8BS+jBQRcJRdWKkbIA35Ztszr2FEqp1oKYLMfdTWDumLNi9xlIeEAFc2C4DeOwsYcNi+mMl5OWGtcw==