Merge pull request #2459 from ahhda/deploy-improvements

DevOps - Deployment improvements

devops/infrastructure/bash-config.sample.cfg

@@ -5,7 +5,10 @@ REGION=us-east-1
 CLI_PROFILE=joystream-user
 KEY_PATH="/Users/joystream/Joystream/joystream-key.pem"
 AWS_KEY_PAIR_NAME="joystream-key"
-EC2_INSTANCE_TYPE=t2.micro
+DEFAULT_EC2_INSTANCE_TYPE=t2.micro
+VALIDATOR_EC2_INSTANCE_TYPE=t2.micro
+BUILD_EC2_INSTANCE_TYPE=t2.xlarge
+RPC_EC2_INSTANCE_TYPE=t2.micro
 
 # Set a prebuilt AMI if required
 EC2_AMI_ID="ami-064f99551235fb1ac"
@@ -18,6 +21,8 @@ DATA_PATH="data-$NEW_STACK_NAME"
 
 INVENTORY_PATH="$DATA_PATH/inventory"
 
+NUMBER_OF_VALIDATORS=2
+
 #### PARAMETERS USED BY ANSIBLE
 
 LOCAL_CODE_PATH="~/Joystream/joystream"

devops/infrastructure/build-code.yml

@@ -1,7 +1,8 @@
 ---
 
-- name: Get latest Joystream code and build it
-  hosts: all
+- name: Get latest Joystream code, build it and copy binary to local
+  hosts: build
+  gather_facts: no
   tasks:
     - name: Get code from local or git repo
       include_role:
@@ -12,3 +13,24 @@
       include_role:
         name: common
         tasks_from: run-setup-build
+
+    - name: Copy joystream-node binary from build to local
+      fetch:
+        src: "{{ remote_code_path }}/target/release/joystream-node"
+        dest: "{{ data_path }}/joystream-node"
+        flat: yes
+
+- name: Copy binary to remote servers
+  hosts: all
+  gather_facts: no
+  tasks:
+    - name: Create release directory
+      file:
+        path: "{{ remote_code_path }}/target/release"
+        state: directory
+
+    - name: Copying joystream-node binary to all servers
+      copy:
+        src: "{{ data_path }}/joystream-node"
+        dest: "{{ remote_code_path }}/target/release/joystream-node"
+        mode: "0775"

devops/infrastructure/delete-stack.sh

@@ -25,7 +25,7 @@ BUCKET_NAME=$(get_aws_export "S3BucketName")
 
 echo -e "\n\n=========== Emptying bucket $BUCKET_NAME ==========="
 
-aws s3 rm s3://$BUCKET_NAME --recursive --profile $CLI_PROFILE
+aws s3 rm s3://$BUCKET_NAME --recursive --profile $CLI_PROFILE || echo "No bucket"
 
 echo -e "\n\n=========== Deleting stack $NEW_STACK_NAME ==========="
 

devops/infrastructure/deploy-infra.sh

@@ -35,20 +35,35 @@ aws cloudformation deploy \
   --region $REGION \
   --profile $CLI_PROFILE \
   --stack-name $NEW_STACK_NAME \
-  --template-file main.yml \
+  --template-file infrastructure.yml \
   --no-fail-on-empty-changeset \
   --capabilities CAPABILITY_NAMED_IAM \
   --parameter-overrides \
-    EC2InstanceType=$EC2_INSTANCE_TYPE \
+    EC2InstanceType=$DEFAULT_EC2_INSTANCE_TYPE \
+    ValidatorEC2InstanceType=$VALIDATOR_EC2_INSTANCE_TYPE \
+    RPCEC2InstanceType=$RPC_EC2_INSTANCE_TYPE \
+    BuildEC2InstanceType=$BUILD_EC2_INSTANCE_TYPE \
     KeyName=$AWS_KEY_PAIR_NAME \
-    EC2AMI=$EC2_AMI_ID
+    EC2AMI=$EC2_AMI_ID \
+    NumberOfValidators=$NUMBER_OF_VALIDATORS
 
 # If the deploy succeeded, get the IP, create inventory and configure the created instances
 if [ $? -eq 0 ]; then
   # Install additional Ansible roles from requirements
   ansible-galaxy install -r requirements.yml
 
-  VALIDATORS=$(get_aws_export "PublicIp")
+  ASG=$(get_aws_export "AutoScalingGroup")
+
+  VALIDATORS=""
+
+  INSTANCES=$(aws autoscaling describe-auto-scaling-instances --profile $CLI_PROFILE \
+    --query "AutoScalingInstances[?AutoScalingGroupName=='${ASG}'].InstanceId" --output text);
+
+  for ID in $INSTANCES
+  do
+    IP=$(aws ec2 describe-instances --instance-ids $ID --query "Reservations[].Instances[].PublicIpAddress" --profile $CLI_PROFILE --output text)
+    VALIDATORS+="$IP\n"
+  done
 
   RPC_NODES=$(get_aws_export "RPCPublicIp")
 
@@ -60,12 +75,13 @@ if [ $? -eq 0 ]; then
 
   mkdir -p $DATA_PATH
 
-  echo -e "[build]\n$BUILD_SERVER\n\n[validators]\n$VALIDATORS\n\n[rpc]\n$RPC_NODES" > $INVENTORY_PATH
+  echo -e "[build]\n$BUILD_SERVER\n\n[validators]\n$VALIDATORS\n[rpc]\n$RPC_NODES" > $INVENTORY_PATH
 
   if [ -z "$EC2_AMI_ID" ]
   then
     echo -e "\n\n=========== Configuring the node servers ==========="
-    ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH build-code.yml --extra-vars "branch_name=$BRANCH_NAME git_repo=$GIT_REPO build_local_code=$BUILD_LOCAL_CODE"
+    ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH build-code.yml \
+      --extra-vars "branch_name=$BRANCH_NAME git_repo=$GIT_REPO build_local_code=$BUILD_LOCAL_CODE data_path=data-$NEW_STACK_NAME"
   fi
 
   echo -e "\n\n=========== Configuring the Build server ==========="
@@ -74,7 +90,8 @@ if [ $? -eq 0 ]; then
 
   echo -e "\n\n=========== Configuring the chain spec file and Pioneer app ==========="
   ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH chain-spec-pioneer.yml \
-    --extra-vars "local_dir=$LOCAL_CODE_PATH network_suffix=$NETWORK_SUFFIX data_path=data-$NEW_STACK_NAME bucket_name=$BUCKET_NAME"
+    --extra-vars "local_dir=$LOCAL_CODE_PATH network_suffix=$NETWORK_SUFFIX
+                  data_path=data-$NEW_STACK_NAME bucket_name=$BUCKET_NAME number_of_validators=$NUMBER_OF_VALIDATORS"
 
   echo -e "\n\n Pioneer URL: https://$DOMAIN_NAME"
 fi

devops/infrastructure/group_vars/all

@@ -16,5 +16,6 @@ remote_code_path: "/home/ubuntu/joystream"
 remote_chain_spec_path: "{{ remote_code_path }}/chainspec.json"
 run_on_admin_server: true
 build_local_code: false
+number_of_validators: 2
 
 bucket_name: s3-bucket-joystream

devops/infrastructure/main.yml → devops/infrastructure/infrastructure.yml

@@ -3,6 +3,16 @@ AWSTemplateFormatVersion: 2010-09-09
 Parameters:
   EC2InstanceType:
     Type: String
+    Default: t2.micro
+  ValidatorEC2InstanceType:
+    Type: String
+    Default: t2.micro
+  RPCEC2InstanceType:
+    Type: String
+    Default: t2.micro
+  BuildEC2InstanceType:
+    Type: String
+    Default: t2.micro
   EC2AMI:
     Type: String
     Default: 'ami-09e67e426f25ce0d7'
@@ -14,6 +24,10 @@ Parameters:
     Type: 'AWS::EC2::KeyPair::KeyName'
     Default: 'joystream-key'
     ConstraintDescription: must be the name of an existing EC2 KeyPair.
+  NumberOfValidators:
+    Description: Number of validator instances to launch
+    Type: Number
+    Default: 2
 
 Conditions:
   HasAMIId: !Not [!Equals [!Ref EC2AMI, ""]]
@@ -109,31 +123,33 @@ Resources:
 
             /opt/aws/bin/cfn-signal -e $? -r "Instance Created" '${WaitHandle}'
 
-  Instance:
-    Type: AWS::EC2::Instance
+  AutoScalingGroup:
+    Type: AWS::AutoScaling::AutoScalingGroup
     Properties:
-      LaunchTemplate:
-        LaunchTemplateId: !Ref InstanceLaunchTemplate
-        Version: !GetAtt InstanceLaunchTemplate.LatestVersionNumber
+      MinSize: '0'
+      MaxSize: '10'
+      DesiredCapacity: !Ref NumberOfValidators
+      AvailabilityZones:
+        Fn::GetAZs:
+          Ref: "AWS::Region"
+      MixedInstancesPolicy:
+        LaunchTemplate:
+          LaunchTemplateSpecification:
+            LaunchTemplateId: !Ref InstanceLaunchTemplate
+            Version: !GetAtt InstanceLaunchTemplate.LatestVersionNumber
+          Overrides:
+            - InstanceType: !Ref ValidatorEC2InstanceType
       Tags:
         - Key: Name
-          Value: !Sub '${AWS::StackName}_1'
-
-  Instance2:
-    Type: AWS::EC2::Instance
-    Properties:
-      LaunchTemplate:
-        LaunchTemplateId: !Ref InstanceLaunchTemplate
-        Version: !GetAtt InstanceLaunchTemplate.LatestVersionNumber
-      Tags:
-        - Key: Name
-          Value: !Sub '${AWS::StackName}_2'
+          Value: !Sub '${AWS::StackName}'
+          PropagateAtLaunch: "true"
 
   RPCInstance:
     Type: AWS::EC2::Instance
     Properties:
       SecurityGroupIds:
         - !GetAtt RPCSecurityGroup.GroupId
+      InstanceType: !Ref RPCEC2InstanceType
       LaunchTemplate:
         LaunchTemplateId: !Ref InstanceLaunchTemplate
         Version: !GetAtt InstanceLaunchTemplate.LatestVersionNumber
@@ -144,6 +160,7 @@ Resources:
   BuildInstance:
     Type: AWS::EC2::Instance
     Properties:
+      InstanceType: !Ref BuildEC2InstanceType
       LaunchTemplate:
         LaunchTemplateId: !Ref InstanceLaunchTemplate
         Version: !GetAtt InstanceLaunchTemplate.LatestVersionNumber
@@ -159,7 +176,7 @@ Resources:
     Properties:
       Handle: !Ref 'WaitHandle'
       Timeout: '600'
-      Count: 4
+      Count: !Ref NumberOfValidators
 
   S3Bucket:
     Type: AWS::S3::Bucket
@@ -200,17 +217,11 @@ Resources:
         HttpVersion: http2
 
 Outputs:
-  PublicIp:
-    Description: The DNS name for the created instance
-    Value:  !Sub "${Instance.PublicIp}"
-    Export:
-      Name: !Sub "${AWS::StackName}PublicIp"
-
-  PublicIp2:
-    Description: The DNS name for the created instance
-    Value:  !Sub "${Instance2.PublicIp}"
+  AutoScalingId:
+    Description: The Auto Scaling ID
+    Value:  !Ref AutoScalingGroup
     Export:
-      Name: !Sub "${AWS::StackName}PublicIp2"
+      Name: !Sub "${AWS::StackName}AutoScalingGroup"
 
   RPCPublicIp:
     Description: The DNS name for the created instance

devops/infrastructure/roles/admin/tasks/deploy-pioneer.yml

@@ -9,6 +9,10 @@
   shell: "WS_URL=wss://{{ ws_rpc }} yarn && yarn workspace @joystream/types build && yarn workspace pioneer build"
   args:
     chdir: "{{ remote_code_path }}"
+  retries: 3
+  delay: 5
+  register: result
+  until: result is not failed
 
 - name: Copying build files to local
   synchronize:

devops/infrastructure/roles/common/tasks/chain-spec-node-keys.yml

@@ -7,7 +7,7 @@
   run_once: true
 
 - name: Run chain-spec-builder to generate chainspec.json file
-  command: "{{ admin_code_dir }}/target/release/chain-spec-builder generate -a 2 --chain-spec-path {{ chain_spec_path }} --deployment live --endowed 1 --keystore-path {{ data_path }}"
+  command: "{{ admin_code_dir }}/target/release/chain-spec-builder generate -a {{ number_of_validators }} --chain-spec-path {{ chain_spec_path }} --deployment live --endowed 1 --keystore-path {{ data_path }}"
   register: chain_spec_output
   delegate_to: "{{ local_or_admin }}"
   run_once: true