Merge branch 'devops/olympia-playground/infra-fixes' into olympia-devops-updates

devops/aws/build-code.yml

@@ -1,7 +1,7 @@
 ---
 
 - name: Get latest Joystream code, build it and copy binary to local
-  hosts: build
+  hosts: build,rpc
   gather_facts: no
   tasks:
     - name: Get code from local or git repo
@@ -9,10 +9,19 @@
         name: common
         tasks_from: "{{ 'get-code-local' if build_local_code|bool else 'get-code-git' }}"
 
-    - name: Run setup and build
+    - name: Run setup
+      include_role:
+        name: common
+        tasks_from: run-setup
+
+- name: Build code and copy binary to local
+  hosts: build
+  gather_facts: no
+  tasks:
+    - name: Run build
       include_role:
         name: common
-        tasks_from: run-setup-build
+        tasks_from: run-build
 
     - name: Copy joystream-node binary from build to local
       fetch:

devops/aws/create-joystream-node-ami-playbook.yml

@@ -10,10 +10,15 @@
         name: common
         tasks_from: get-code-git
 
-    - name: Run setup and build
+    - name: Run setup
       include_role:
         name: common
-        tasks_from: run-setup-build
+        tasks_from: run-setup
+
+    - name: Run build
+      include_role:
+        name: common
+        tasks_from: run-build
 
     - name: Install subkey
       include_role:

devops/aws/deploy-infra.sample.cfg

@@ -9,6 +9,9 @@ DEFAULT_EC2_INSTANCE_TYPE=t2.medium
 VALIDATOR_EC2_INSTANCE_TYPE=t2.medium
 BUILD_EC2_INSTANCE_TYPE=t2.large
 RPC_EC2_INSTANCE_TYPE=t2.medium
+# Validator volume size
+VOLUME_SIZE=120
+RPC_VOLUME_SIZE=120
 
 # prebuilt AMI with joystream-node, chain-spec and subkey already built
 EC2_AMI_ID="ami-0ce5f13e91397239a"

devops/aws/deploy-infra.sh

@@ -39,7 +39,9 @@ aws cloudformation deploy \
     BuildEC2InstanceType=$BUILD_EC2_INSTANCE_TYPE \
     KeyName=$AWS_KEY_PAIR_NAME \
     EC2AMI=$EC2_AMI_ID \
-    NumberOfValidators=$NUMBER_OF_VALIDATORS
+    NumberOfValidators=$NUMBER_OF_VALIDATORS \
+    VolumeSize=$VOLUME_SIZE \
+    RPCVolumeSize=$RPC_VOLUME_SIZE
 
 # If the deploy succeeded, get the IP, create inventory and configure the created instances
 if [ $? -eq 0 ]; then

devops/aws/roles/build-server/tasks/main.yml

@@ -1,16 +1,6 @@
 ---
 # Configure build server to be able to create chain-spec file and subkey commands
 
-- name: Copy bash_profile content
-  shell: cat ~/.bash_profile
-  register: bash_data
-
-- name: Copy bash_profile content to bashrc for non-interactive sessions
-  blockinfile:
-    block: '{{ bash_data.stdout }}'
-    path: ~/.bashrc
-    insertbefore: BOF
-
 - name: Get dependencies for subkey
   shell: curl https://getsubstrate.io -sSf | bash -s -- --fast
 

devops/aws/roles/common/tasks/chain-spec-node-keys.yml

@@ -51,11 +51,23 @@
 
 - name: Save output of chain spec to local file
   copy:
-    content: '{{ chain_spec_output.stdout | regex_replace("\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[mGK]", "") }}'
+    content: '{{ chain_spec_output.stdout }}'
     dest: '{{ remote_data_path }}/chain_spec_output.txt'
   delegate_to: '{{ build_instance }}'
   run_once: true
 
+- name: Format chain spec output
+  set_fact:
+    chain_spec_output_formatted: '{{ chain_spec_output.stdout | regex_replace("=", ": ") | from_yaml }}'
+  run_once: true
+
+- name: Extract keys from chain spec output
+  set_fact:
+    sudo_key: '{{ chain_spec_output_formatted.sudo }}'
+    endowed_key: '{{ chain_spec_output_formatted.endowed_0 }}'
+  delegate_to: '{{ build_instance }}'
+  run_once: true
+
 - name: Change chain spec name, id, protocolId
   json_modify:
     chain_spec_path: '{{ chain_spec_path }}'

devops/aws/roles/common/tasks/run-setup-build.yml → devops/aws/roles/common/tasks/run-build.yml

@@ -1,13 +1,5 @@
 ---
-# Run setup and build code
-
-- name: Create bash profile file
-  command: 'touch /home/ubuntu/.bash_profile'
-
-- name: Run setup script
-  command: ./setup.sh
-  args:
-    chdir: '{{ remote_code_path }}'
+# Build code
 
 - name: Build joystream node
   shell: . ~/.bash_profile && yarn cargo-build

devops/aws/roles/common/tasks/run-setup.yml

@@ -0,0 +1,20 @@
+---
+# Run setup
+
+- name: Create bash profile file
+  command: 'touch /home/ubuntu/.bash_profile'
+
+- name: Run setup script
+  command: ./setup.sh
+  args:
+    chdir: '{{ remote_code_path }}'
+
+- name: Copy bash_profile content
+  shell: cat ~/.bash_profile
+  register: bash_data
+
+- name: Copy bash_profile content to bashrc for non-interactive sessions
+  blockinfile:
+    block: '{{ bash_data.stdout }}'
+    path: ~/.bashrc
+    insertbefore: BOF

devops/aws/roles/rpc/tasks/main.yml

@@ -41,6 +41,10 @@
   # Check for the status every 100s
   delay: 100
 
+- name: Make sure docker is running
+  command: systemctl start docker
+  become: yes
+
 - name: Run service containers
   command: yarn start
   environment:
@@ -48,11 +52,14 @@
     JOYSTREAM_NODE_WS: 'ws://{{ inventory_hostname }}:9944/'
     COLOSSUS_1_URL: 'https://{{ inventory_hostname }}.nip.io/colossus-1/'
     DISTRIBUTOR_1_URL: 'https://{{ inventory_hostname }}.nip.io/distributor-1/'
+    COLOSSUS_1_TRANSACTOR_URI: '//Colossus1'
     SKIP_NODE: 'true'
     SKIP_CHAIN_SETUP: '{{ skip_chain_setup }}'
+    TREASURY_ACCOUNT_URI: '{{ endowed_key }}'
+    SUDO_ACCOUNT_URI: '{{ sudo_key }}'
   args:
     chdir: '{{ remote_code_path }}'
-  async: 1800
+  async: 3600
   poll: 0
   register: compose_result
 
@@ -62,7 +69,7 @@
   register: job_result
   until: job_result.finished
   # Max number of times to check for status
-  retries: 18
+  retries: 36
   # Check for the status every 100s
   delay: 100
 

start.sh

@@ -36,9 +36,12 @@ docker-compose up -d orion
 
 ## Init the chain with some state
 if [[ $SKIP_CHAIN_SETUP != 'true' ]]; then
-  set -a
-  . ./.env
-  set +a
+  if [ "${SKIP_NODE}" != true ]
+  then
+    set -a
+    . ./.env
+    set +a
+  fi
 
   export SKIP_MOCK_CONTENT=true
   export SKIP_QUERY_NODE_CHECKS=true