Remove admin param, add raw chain spec, sample config

devops/infrastructure/bash-config.cfg → devops/infrastructure/bash-config.sample.cfg

@@ -3,13 +3,10 @@
 STACK_NAME=joystream-node
 REGION=us-east-1
 CLI_PROFILE=joystream-user
-KEY_PATH="/Users/anuj/Joystream/anuj-key.pem"
-AWS_KEY_PAIR_NAME="anuj-key"
+KEY_PATH="/Users/joystream/Joystream/joystream-key.pem"
+AWS_KEY_PAIR_NAME="joystream-key"
 EC2_INSTANCE_TYPE=t2.xlarge
 
-# If true will create another instance to generate chain spec file and keys
-CREATE_ADMIN_SERVER=true
-
 # Set a prebuilt AMI if required
 EC2_AMI_ID="ami-064f99551235fb1ac"
 

devops/infrastructure/chain-spec-configuration.yml

@@ -11,7 +11,7 @@
         name: common
         tasks_from: chain-spec-node-keys
       vars:
-        local_or_admin: "{{ groups['admin'][0] if run_on_admin_server|bool else 'localhost' }}"
+        local_or_admin: "{{ groups['build'][0] if run_on_admin_server|bool else 'localhost' }}"
         admin_code_dir: "{{ remote_code_path if run_on_admin_server|bool else local_dir }}"
 
 - name: Copy secret, auth and start joystream-node service for validators

devops/infrastructure/deploy-infra.sh

@@ -2,7 +2,14 @@
 
 set -e
 
-source bash-config.cfg
+if [ -z "$1" ]; then
+  echo "ERROR: Configuration file not passed"
+  echo "Please use ./deploy-infra.sh PATH/TO/CONFIG to run this script"
+  exit 1
+else
+  echo "Using $1 file for config"
+  source $1
+fi
 
 ACCOUNT_ID=$(aws sts get-caller-identity --profile $CLI_PROFILE --query Account --output text)
 
@@ -34,8 +41,7 @@ aws cloudformation deploy \
   --parameter-overrides \
     EC2InstanceType=$EC2_INSTANCE_TYPE \
     KeyName=$AWS_KEY_PAIR_NAME \
-    EC2AMI=$EC2_AMI_ID \
-    CreateAdminServer=$CREATE_ADMIN_SERVER
+    EC2AMI=$EC2_AMI_ID
 
 # If the deploy succeeded, get the IP, create inventory and configure the created instances
 if [ $? -eq 0 ]; then
@@ -49,18 +55,14 @@ if [ $? -eq 0 ]; then
     --query "Exports[?starts_with(Name,'${NEW_STACK_NAME}RPCPublicIp')].Value" \
     --output text | sed 's/\t\t*/\n/g')
 
-  if [ "$CREATE_ADMIN_SERVER" = true ] ; then
-    ADMIN_SERVER=$(aws cloudformation list-exports \
-      --profile $CLI_PROFILE \
-      --query "Exports[?starts_with(Name,'${NEW_STACK_NAME}AdminPublicIp')].Value" \
-      --output text | sed 's/\t\t*/\n/g')
-    ADMIN_INVENTORY="[admin]\n$ADMIN_SERVER\n\n"
-    HOST="admin"
-  fi
+  BUILD_SERVER=$(aws cloudformation list-exports \
+    --profile $CLI_PROFILE \
+    --query "Exports[?starts_with(Name,'${NEW_STACK_NAME}BuildPublicIp')].Value" \
+    --output text | sed 's/\t\t*/\n/g')
 
   mkdir -p $DATA_PATH
 
-  echo -e "$ADMIN_INVENTORY[validators]\n$VALIDATORS\n\n[rpc]\n$RPC_NODES" > $INVENTORY_PATH
+  echo -e "[build]\n$BUILD_SERVER\n\n[validators]\n$VALIDATORS\n\n[rpc]\n$RPC_NODES" > $INVENTORY_PATH
 
   if [ -z "$EC2_AMI_ID" ]
   then
@@ -68,11 +70,11 @@ if [ $? -eq 0 ]; then
     ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH build-code.yml --extra-vars "branch_name=$BRANCH_NAME git_repo=$GIT_REPO build_local_code=$BUILD_LOCAL_CODE"
   fi
 
-  echo -e "\n\n=========== Configuring the Admin server ==========="
+  echo -e "\n\n=========== Configuring the Build server ==========="
   ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH setup-admin.yml \
-    --extra-vars "local_dir=$LOCAL_CODE_PATH run_on_admin_server=$CREATE_ADMIN_SERVER build_local_code=$BUILD_LOCAL_CODE"
+    --extra-vars "local_dir=$LOCAL_CODE_PATH build_local_code=$BUILD_LOCAL_CODE"
 
   echo -e "\n\n=========== Configuring the chain spec file ==========="
   ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH chain-spec-configuration.yml \
-    --extra-vars "local_dir=$LOCAL_CODE_PATH network_suffix=$NETWORK_SUFFIX data_path=data-$NEW_STACK_NAME run_on_admin_server=$CREATE_ADMIN_SERVER"
+    --extra-vars "local_dir=$LOCAL_CODE_PATH network_suffix=$NETWORK_SUFFIX data_path=data-$NEW_STACK_NAME"
 fi

devops/infrastructure/group_vars/all

@@ -11,7 +11,8 @@ network_suffix: "{{ 10000 | random(1000) }}"
 
 data_path: ./data
 chain_spec_path: "{{ data_path }}/chainspec.json"
+raw_chain_spec_path: "{{ data_path }}/chainspec-raw.json"
 remote_code_path: "/home/ubuntu/joystream"
 remote_chain_spec_path: "{{ remote_code_path }}/chainspec.json"
-run_on_admin_server: false
+run_on_admin_server: true
 build_local_code: false

devops/infrastructure/library/json_modify.py

@@ -31,7 +31,12 @@ def main():
             public_key = all_nodes[key]["subkey_output"]["stderr"]
             boot_node_list.append(f"/ip4/{key}/tcp/30333/p2p/{public_key}")
 
+    telemetry_endpoints = data["telemetryEndpoints"]
+    telemetry_endpoints.append([
+        "/dns/telemetry.joystream.org/tcp/443/x-parity-wss/%2Fsubmit%2F", 0])
+
     response["bootNodes"] = boot_node_list
+    response["telemetryEndpoints"] = telemetry_endpoints
 
     data.update(response)
     with open(chain_spec_path, 'w') as outfile:

devops/infrastructure/main.yml

@@ -14,16 +14,8 @@ Parameters:
     Type: 'AWS::EC2::KeyPair::KeyName'
     Default: 'joystream-key'
     ConstraintDescription: must be the name of an existing EC2 KeyPair.
-  CreateAdminServer:
-    Description: Whether or not to create the admin instance for creating keys and chain-spec file
-    Type: String
-    AllowedValues: [true, false]
-    Default: false
 
 Conditions:
-  CreateAdminServerCondition: !Equals
-    - !Ref CreateAdminServer
-    - true
   HasAMIId: !Not [!Equals [!Ref EC2AMI, ""]]
 
 Resources:
@@ -141,16 +133,15 @@ Resources:
         - Key: Name
           Value: !Sub '${AWS::StackName}_rpc'
 
-  AdminInstance:
+  BuildInstance:
     Type: AWS::EC2::Instance
-    Condition: CreateAdminServerCondition
     Properties:
       LaunchTemplate:
         LaunchTemplateId: !Ref InstanceLaunchTemplate
         Version: !GetAtt InstanceLaunchTemplate.LatestVersionNumber
       Tags:
         - Key: Name
-          Value: !Sub '${AWS::StackName}_admin'
+          Value: !Sub '${AWS::StackName}_build'
 
   WaitHandle:
     Type: AWS::CloudFormation::WaitConditionHandle
@@ -160,8 +151,7 @@ Resources:
     Properties:
       Handle: !Ref 'WaitHandle'
       Timeout: '600'
-      Count:
-        !If [CreateAdminServerCondition, 4, 3]
+      Count: 4
 
 Outputs:
   PublicIp:
@@ -182,9 +172,8 @@ Outputs:
     Export:
       Name: !Sub "${AWS::StackName}RPCPublicIp"
 
-  AdminPublicIp:
+  BuildPublicIp:
     Description: The DNS name for the created instance
-    Value:  !Sub "${AdminInstance.PublicIp}"
-    Condition: CreateAdminServerCondition
+    Value:  !Sub "${BuildInstance.PublicIp}"
     Export:
-      Name: !Sub "${AWS::StackName}AdminPublicIp"
+      Name: !Sub "${AWS::StackName}BuildPublicIp"

devops/infrastructure/roles/common/tasks/chain-spec-node-keys.yml

@@ -48,6 +48,11 @@
     var: result.result
   run_once: true
 
+- name: Run build-spec to generate raw chainspec file
+  shell: "{{ admin_code_dir }}/target/release/joystream-node build-spec --chain {{ chain_spec_path }} --raw > {{ raw_chain_spec_path }}"
+  delegate_to: "{{ local_or_admin }}"
+  run_once: true
+
 - name: Copying chain spec files to localhost
   synchronize:
     src: "/home/ubuntu/{{ data_path }}/"
@@ -65,7 +70,7 @@
   run_once: true
   when: run_on_admin_server|bool
 
-- name: Copying chain spec file to all servers
+- name: Copying raw chain spec file to all servers
   copy:
-    src: "{{ chain_spec_path }}"
+    src: "{{ raw_chain_spec_path }}"
     dest: "{{ remote_chain_spec_path }}"

devops/infrastructure/setup-admin.yml

@@ -1,8 +1,8 @@
 ---
-# Setup Admin server, copy code, install subkey
+# Setup Build server install subkey
 
-- name: Setup Admin server, install subkey
-  hosts: admin
+- name: Setup build server, install subkey
+  hosts: build
 
   roles:
     - role: admin