Merge pull request #3427 from Joystream/olympia-devops-updates

Olympia devops updates

.env

@@ -22,6 +22,7 @@ TYPEORM_LOGGING=error
 BLOCK_HEIGHT=0
 
 # Query node GraphQL server port
+# Remember to change it in COLOSSUS_QUERY_NODE_URL and DISTRIBUTOR_QUERY_NODE_URL as well
 GRAPHQL_SERVER_PORT=8081
 
 # Query node playground endpoint
@@ -36,6 +37,7 @@ GRAPHQL_PLAYGROUND_ENDPOINT=""
 GRAPHQL_PLAYGROUND_CDN_URL=""
 
 # Hydra indexer gateway GraphQL server port
+# Remember to change it in PROCESSOR_INDEXER_GATEWAY as well
 HYDRA_INDEXER_GATEWAY_PORT=4000
 
 # Default GraphQL server host. It is required during "query-node config:dev"
@@ -45,13 +47,13 @@ GRAPHQL_SERVER_HOST=localhost
 JOYSTREAM_NODE_WS=ws://joystream-node:9944/
 
 # Query node which colossus will use
-COLOSSUS_QUERY_NODE_URL=http://graphql-server:${GRAPHQL_SERVER_PORT}/graphql
+COLOSSUS_QUERY_NODE_URL=http://graphql-server:8081/graphql
 
 # Query node which distributor will use
-DISTRIBUTOR_QUERY_NODE_URL=http://graphql-server:${GRAPHQL_SERVER_PORT}/graphql
+DISTRIBUTOR_QUERY_NODE_URL=http://graphql-server:8081/graphql
 
 # Indexer gateway used by processor. If you don't use the local indexer set this to a remote gateway
-PROCESSOR_INDEXER_GATEWAY=http://hydra-indexer-gateway:${HYDRA_INDEXER_GATEWAY_PORT}/graphql
+PROCESSOR_INDEXER_GATEWAY=http://hydra-indexer-gateway:4000/graphql
 
 # Colossus services identities
 COLOSSUS_1_WORKER_ID=0
@@ -64,10 +66,10 @@ COLOSSUS_2_TRANSACTOR_URI=//Colossus2
 
 # Distributor node services identities
 DISTRIBUTOR_1_WORKER_ID=0
-DISTRIBUTOR_1_ACCOUNT_URI=//testing//worker//Distribution//${DISTRIBUTOR_1_WORKER_ID}
+DISTRIBUTOR_1_ACCOUNT_URI=//testing//worker//Distribution//0
 
 DISTRIBUTOR_2_WORKER_ID=1
-DISTRIBUTOR_2_ACCOUNT_URI=//testing//worker//Distribution//${DISTRIBUTOR_2_WORKER_ID}
+DISTRIBUTOR_2_ACCOUNT_URI=//testing//worker//Distribution//1
 
 # Membership Faucet
 SCREENING_AUTHORITY_SEED=//Alice

.github/workflows/deploy-node-network.yml

@@ -126,7 +126,7 @@ jobs:
           echo -e "[rpc]\n${{ steps.deploy_stack.outputs.RPCPublicIp }}" >> inventory
           cat inventory
 
-      - name: Run playbook to compile joystream-node on build server
+      - name: Run playbook to setup all hosts and compile joystream-node
         uses: dawidd6/action-ansible-playbook@v2
         # Build binaries if AMI not specified or a custom proposals parameter is passed
         if: steps.myoutputs.outputs.ec2AMI == '' || steps.myoutputs.outputs.proposalParameters != ''

devops/aws/build-code.yml

@@ -1,7 +1,7 @@
 ---
 
-- name: Get latest Joystream code, build it and copy binary to local
-  hosts: build
+- name: Get latest Joystream code and run setup on all hosts
+  hosts: build,rpc
   gather_facts: no
   tasks:
     - name: Get code from local or git repo
@@ -9,10 +9,19 @@
         name: common
         tasks_from: "{{ 'get-code-local' if build_local_code|bool else 'get-code-git' }}"
 
-    - name: Run setup and build
+    - name: Run setup
+      include_role:
+        name: common
+        tasks_from: run-setup
+
+- name: Build joystream-node on build host and copy binary to local
+  hosts: build
+  gather_facts: no
+  tasks:
+    - name: Run build
       include_role:
         name: common
-        tasks_from: run-setup-build
+        tasks_from: run-build
 
     - name: Copy joystream-node binary from build to local
       fetch:
@@ -20,7 +29,7 @@
         dest: "{{ data_path }}/joystream-node"
         flat: yes
 
-- name: Copy binary to remote servers
+- name: Copy joystream-node binary to all hosts
   hosts: all
   gather_facts: no
   tasks:

devops/aws/create-joystream-node-ami-playbook.yml

@@ -10,10 +10,15 @@
         name: common
         tasks_from: get-code-git
 
-    - name: Run setup and build
+    - name: Run setup
       include_role:
         name: common
-        tasks_from: run-setup-build
+        tasks_from: run-setup
+
+    - name: Run build
+      include_role:
+        name: common
+        tasks_from: run-build
 
     - name: Install subkey
       include_role:

devops/aws/deploy-infra.sample.cfg

@@ -9,6 +9,9 @@ DEFAULT_EC2_INSTANCE_TYPE=t2.medium
 VALIDATOR_EC2_INSTANCE_TYPE=t2.medium
 BUILD_EC2_INSTANCE_TYPE=t2.large
 RPC_EC2_INSTANCE_TYPE=t2.medium
+# Validator volume size
+VOLUME_SIZE=120
+RPC_VOLUME_SIZE=120
 
 # prebuilt AMI with joystream-node, chain-spec and subkey already built
 EC2_AMI_ID="ami-0ce5f13e91397239a"

devops/aws/deploy-infra.sh

@@ -39,7 +39,9 @@ aws cloudformation deploy \
     BuildEC2InstanceType=$BUILD_EC2_INSTANCE_TYPE \
     KeyName=$AWS_KEY_PAIR_NAME \
     EC2AMI=$EC2_AMI_ID \
-    NumberOfValidators=$NUMBER_OF_VALIDATORS
+    NumberOfValidators=$NUMBER_OF_VALIDATORS \
+    VolumeSize=$VOLUME_SIZE \
+    RPCVolumeSize=$RPC_VOLUME_SIZE
 
 # If the deploy succeeded, get the IP, create inventory and configure the created instances
 if [ $? -eq 0 ]; then

devops/aws/roles/build-server/tasks/main.yml

@@ -1,16 +1,6 @@
 ---
 # Configure build server to be able to create chain-spec file and subkey commands
 
-- name: Copy bash_profile content
-  shell: cat ~/.bash_profile
-  register: bash_data
-
-- name: Copy bash_profile content to bashrc for non-interactive sessions
-  blockinfile:
-    block: '{{ bash_data.stdout }}'
-    path: ~/.bashrc
-    insertbefore: BOF
-
 - name: Get dependencies for subkey
   shell: curl https://getsubstrate.io -sSf | bash -s -- --fast
 

devops/aws/roles/common/tasks/chain-spec-node-keys.yml

@@ -51,11 +51,23 @@
 
 - name: Save output of chain spec to local file
   copy:
-    content: '{{ chain_spec_output.stdout | regex_replace("\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[mGK]", "") }}'
+    content: '{{ chain_spec_output.stdout }}'
     dest: '{{ remote_data_path }}/chain_spec_output.txt'
   delegate_to: '{{ build_instance }}'
   run_once: true
 
+- name: Format chain spec output
+  set_fact:
+    chain_spec_output_formatted: '{{ chain_spec_output.stdout | regex_replace("=", ": ") | from_yaml }}'
+  run_once: true
+
+- name: Extract keys from chain spec output
+  set_fact:
+    sudo_key: '{{ chain_spec_output_formatted.sudo }}'
+    endowed_key: '{{ chain_spec_output_formatted.endowed_0 }}'
+  delegate_to: '{{ build_instance }}'
+  run_once: true
+
 - name: Change chain spec name, id, protocolId
   json_modify:
     chain_spec_path: '{{ chain_spec_path }}'

devops/aws/roles/common/tasks/run-setup-build.yml → devops/aws/roles/common/tasks/run-build.yml

@@ -1,13 +1,5 @@
 ---
-# Run setup and build code
-
-- name: Create bash profile file
-  command: 'touch /home/ubuntu/.bash_profile'
-
-- name: Run setup script
-  command: ./setup.sh
-  args:
-    chdir: '{{ remote_code_path }}'
+# Build code
 
 - name: Build joystream node
   shell: . ~/.bash_profile && yarn cargo-build

devops/aws/roles/common/tasks/run-setup.yml

@@ -0,0 +1,20 @@
+---
+# Run setup
+
+- name: Create bash profile file
+  command: 'touch /home/ubuntu/.bash_profile'
+
+- name: Run setup script
+  command: ./setup.sh
+  args:
+    chdir: '{{ remote_code_path }}'
+
+- name: Copy bash_profile content
+  shell: cat ~/.bash_profile
+  register: bash_data
+
+- name: Copy bash_profile content to bashrc for non-interactive sessions
+  blockinfile:
+    block: '{{ bash_data.stdout }}'
+    path: ~/.bashrc
+    insertbefore: BOF

devops/aws/roles/rpc/tasks/main.yml

@@ -41,6 +41,10 @@
   # Check for the status every 100s
   delay: 100
 
+- name: Make sure docker is running
+  command: systemctl start docker
+  become: yes
+
 - name: Run service containers
   command: yarn start
   environment:
@@ -50,9 +54,11 @@
     DISTRIBUTOR_1_URL: 'https://{{ inventory_hostname }}.nip.io/distributor-1/'
     SKIP_NODE: 'true'
     SKIP_CHAIN_SETUP: '{{ skip_chain_setup }}'
+    TREASURY_ACCOUNT_URI: '{{ endowed_key }}'
+    SUDO_ACCOUNT_URI: '{{ sudo_key }}'
   args:
     chdir: '{{ remote_code_path }}'
-  async: 1800
+  async: 3600
   poll: 0
   register: compose_result
 
@@ -62,7 +68,7 @@
   register: job_result
   until: job_result.finished
   # Max number of times to check for status
-  retries: 18
+  retries: 36
   # Check for the status every 100s
   delay: 100
 

start.sh

@@ -36,10 +36,6 @@ docker-compose up -d orion
 
 ## Init the chain with some state
 if [[ $SKIP_CHAIN_SETUP != 'true' ]]; then
-  set -a
-  . ./.env
-  set +a
-
   export SKIP_MOCK_CONTENT=true
   export SKIP_QUERY_NODE_CHECKS=true
   HOST_IP=$(tests/network-tests/get-host-ip.sh)

tests/network-tests/.env

@@ -6,6 +6,14 @@ QUERY_NODE_URL=http://127.0.0.1:8081/graphql
 TREASURY_ACCOUNT_URI=//Alice
 # Sudo Account
 SUDO_ACCOUNT_URI=//Alice
+# to be able to use all accounts generated in a prior scenario run against the same chain
+START_KEY_ID=0
+# Mini-secret or mnemonic used in SURI for deterministic key derivation
+SURI_MINI_SECRET=""
+
+## TODO: Do the below settings really make sense as env variables?
+## Probably a configure them in flows or fixutres or maybe a general test-config.yml file
+
 # Amount of members able to buy membership in membership creation test.
 MEMBERSHIP_CREATION_N=2
 # Amount of members able to invite in members invite test.
@@ -20,7 +28,4 @@ APPLICATION_STATUS_WITHDRAW_N=3
 WORKER_ACTIONS_WORKERS_N=6
 # Amount of workers to terminate in workerActions flow
 WORKER_ACTIONS_TERMINATE_N=3
-# to be able to use all accounts generated in a prior scenario run against the same chain
-START_KEY_ID=0
-# Mini-secret or mnemonic used in SURI for deterministic key derivation
-SURI_MINI_SECRET=""
+

tests/network-tests/run-full-tests.sh

@@ -4,10 +4,6 @@ set -e
 SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
 cd $SCRIPT_PATH
 
-set -a
-. ../../.env
-set +a
-
 # Clean start
 docker-compose -f ../../docker-compose.yml down -v
 

tests/network-tests/run-tests.sh

@@ -29,13 +29,6 @@ fi
 
 # Execute tests
 
-# We can load env config used to start docker services and pass them on to the
-# tests. This could be useful to capture keys used or URLs.
-# We just have to watchout for clashing env var names.
-set -a
-. ../../.env
-set +a
-
 if [ "${NO_STORAGE}" != true ]
 then
   ./start-storage.sh

tests/network-tests/src/Scenario.ts

@@ -11,6 +11,7 @@ import { ResourceManager } from './Resources'
 import fetch from 'cross-fetch'
 import fs, { existsSync, readFileSync } from 'fs'
 import { KeyGenInfo, FaucetInfo } from './types'
+import path from 'path'
 
 export type ScenarioProps = {
   env: NodeJS.ProcessEnv
@@ -48,8 +49,11 @@ function writeOutput(api: Api, miniSecret: string) {
 }
 
 export async function scenario(label: string, scene: (props: ScenarioProps) => Promise<void>): Promise<void> {
-  // Load env variables
-  config()
+  // Load env variables - test framework specific
+  config({ path: path.join(__dirname, '../.env') })
+  // root workspace .env used by docker-compose services
+  config({ path: path.join(__dirname, '../../../.env') })
+
   const env = process.env
 
   // Connect api to the chain

tests/network-tests/start-storage.sh

@@ -1,11 +1,6 @@
 TMP=$0
 THIS_DIR=`dirname $TMP`
 
-# make sure env variables are loaded before calling this script
-#set -a
-#. ../../.env
-#set +a
-
 HOST_IP=`$THIS_DIR/get-host-ip.sh`
 export COLOSSUS_1_URL="http://${HOST_IP}:3333"
 export DISTRIBUTOR_1_URL="http://${HOST_IP}:3334"