Merge pull request #2473 from ahhda/deploy-independent-node

DevOps - Deploy a new joystream-node to an existing network

devops/infrastructure/bash-config.sample.cfg

@@ -11,7 +11,7 @@ BUILD_EC2_INSTANCE_TYPE=t2.xlarge
 RPC_EC2_INSTANCE_TYPE=t2.micro
 
 # Set a prebuilt AMI if required
-EC2_AMI_ID="ami-064f99551235fb1ac"
+EC2_AMI_ID="ami-08ffec5991ca99db9"
 
 ACCOUNT_ID=$(aws sts get-caller-identity --profile $CLI_PROFILE --query Account --output text)
 
@@ -23,6 +23,14 @@ INVENTORY_PATH="$DATA_PATH/inventory"
 
 NUMBER_OF_VALIDATORS=2
 
+## Used for Deploying a new node
+DATE_TIME=$(date +"%d-%b-%Y-%H-%M-%S")
+
+SINGLE_NODE_STACK_NAME="new-node-$DATE_TIME"
+
+BINARY_FILE="https://github.com/Joystream/joystream/releases/download/v9.3.0/joystream-node-5.1.0-9d9e77751-x86_64-linux-gnu.tar.gz"
+CHAIN_SPEC_FILE="https://github.com/Joystream/joystream/releases/download/v9.3.0/joy-testnet-5.json"
+
 #### PARAMETERS USED BY ANSIBLE
 
 LOCAL_CODE_PATH="~/Joystream/joystream"

devops/infrastructure/common.sh

@@ -0,0 +1,9 @@
+get_aws_export () {
+  STACK=$1
+  PROPERTY=$2
+  RESULT=$(aws cloudformation list-exports \
+    --profile $CLI_PROFILE \
+    --query "Exports[?starts_with(Name,'$STACK$PROPERTY')].Value" \
+    --output text | sed 's/\t\t*/\n/g')
+  echo -e $RESULT | tr " " "\n"
+}

devops/infrastructure/delete-stack.sh

@@ -2,6 +2,8 @@
 
 set -e
 
+source common.sh
+
 if [ -z "$1" ]; then
   echo "ERROR: Configuration file not passed"
   echo "Please use ./delete-stack.sh PATH/TO/CONFIG to run this script"
@@ -11,15 +13,7 @@ else
   source $1
 fi
 
-get_aws_export () {
-  RESULT=$(aws cloudformation list-exports \
-    --profile $CLI_PROFILE \
-    --query "Exports[?starts_with(Name,'${NEW_STACK_NAME}$1')].Value" \
-    --output text | sed 's/\t\t*/\n/g')
-  echo -e $RESULT | tr " " "\n"
-}
-
-BUCKET_NAME=$(get_aws_export "S3BucketName")
+BUCKET_NAME=$(get_aws_export $NEW_STACK_NAME "S3BucketName")
 
 # Delete the CloudFormation stack
 

devops/infrastructure/deploy-infra.sh

@@ -2,6 +2,8 @@
 
 set -e
 
+source common.sh
+
 if [ -z "$1" ]; then
   echo "ERROR: Configuration file not passed"
   echo "Please use ./deploy-infra.sh PATH/TO/CONFIG to run this script"
@@ -21,14 +23,6 @@ if [ ! -f "$KEY_PATH" ]; then
     exit 1
 fi
 
-get_aws_export () {
-  RESULT=$(aws cloudformation list-exports \
-    --profile $CLI_PROFILE \
-    --query "Exports[?starts_with(Name,'${NEW_STACK_NAME}$1')].Value" \
-    --output text | sed 's/\t\t*/\n/g')
-  echo -e $RESULT | tr " " "\n"
-}
-
 # Deploy the CloudFormation template
 echo -e "\n\n=========== Deploying main.yml ==========="
 aws cloudformation deploy \
@@ -52,7 +46,7 @@ if [ $? -eq 0 ]; then
   # Install additional Ansible roles from requirements
   ansible-galaxy install -r requirements.yml
 
-  ASG=$(get_aws_export "AutoScalingGroup")
+  ASG=$(get_aws_export $NEW_STACK_NAME "AutoScalingGroup")
 
   VALIDATORS=""
 
@@ -65,13 +59,13 @@ if [ $? -eq 0 ]; then
     VALIDATORS+="$IP\n"
   done
 
-  RPC_NODES=$(get_aws_export "RPCPublicIp")
+  RPC_NODES=$(get_aws_export $NEW_STACK_NAME "RPCPublicIp")
 
-  BUILD_SERVER=$(get_aws_export "BuildPublicIp")
+  BUILD_SERVER=$(get_aws_export $NEW_STACK_NAME "BuildPublicIp")
 
-  BUCKET_NAME=$(get_aws_export "S3BucketName")
+  BUCKET_NAME=$(get_aws_export $NEW_STACK_NAME "S3BucketName")
 
-  DOMAIN_NAME=$(get_aws_export "DomainName")
+  DOMAIN_NAME=$(get_aws_export $NEW_STACK_NAME "DomainName")
 
   mkdir -p $DATA_PATH
 

devops/infrastructure/deploy-single-node.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -e
+
+source common.sh
+
+if [ -z "$1" ]; then
+  echo "ERROR: Configuration file not passed"
+  echo "Please use ./deploy-infra.sh PATH/TO/CONFIG to run this script"
+  exit 1
+else
+  echo "Using $1 file for config"
+  source $1
+fi
+
+if [ $ACCOUNT_ID == None ]; then
+    echo "Couldn't find Account ID, please check if AWS Profile $CLI_PROFILE is set"
+    exit 1
+fi
+
+if [ ! -f "$KEY_PATH" ]; then
+    echo "Key file not found at $KEY_PATH"
+    exit 1
+fi
+
+# # Deploy the CloudFormation template
+echo -e "\n\n=========== Deploying single instance ==========="
+aws cloudformation deploy \
+  --region $REGION \
+  --profile $CLI_PROFILE \
+  --stack-name $SINGLE_NODE_STACK_NAME \
+  --template-file single-instance.yml \
+  --no-fail-on-empty-changeset \
+  --capabilities CAPABILITY_NAMED_IAM \
+  --parameter-overrides \
+    EC2InstanceType=$DEFAULT_EC2_INSTANCE_TYPE \
+    KeyName=$AWS_KEY_PAIR_NAME
+
+# If the deploy succeeded, get the IP and configure the created instance
+if [ $? -eq 0 ]; then
+  # Install additional Ansible roles from requirements
+  ansible-galaxy install -r requirements.yml
+
+  SERVER_IP=$(get_aws_export $SINGLE_NODE_STACK_NAME "PublicIp")
+
+  echo -e "New Node Public IP: $SERVER_IP"
+
+  echo -e "\n\n=========== Configuring the chain spec file and Pioneer app ==========="
+  ansible-playbook -i $SERVER_IP, --private-key $KEY_PATH new-node-playbook.yml \
+    --extra-vars "binary_file=$BINARY_FILE chain_spec_file=$CHAIN_SPEC_FILE"
+fi

devops/infrastructure/new-node-playbook.yml

@@ -0,0 +1,48 @@
+---
+# Configure chain spec file, copy joystream-node binary and run the service
+
+- name: Create and copy the chain-spec file
+  hosts: all
+  gather_facts: no
+
+  tasks:
+    - name: Download chain spec file using link
+      get_url:
+        url: "{{ chain_spec_file }}"
+        dest: ~/chain-spec.json
+      when: chain_spec_file is search("http")
+
+    - name: Copy chain spec file from local
+      copy:
+        src: "{{ chain_spec_file }}"
+        dest: ~/chain-spec.json
+      when: chain_spec_file is not search("http")
+
+    - name: Download and unarchive binary using link
+      unarchive:
+        src: "{{ binary_file }}"
+        dest: ~/
+        remote_src: yes
+      when: binary_file is search("http")
+
+    - name: Copy binary from local
+      copy:
+        src: "{{ binary_file }}"
+        dest: ~/joystream-node
+        mode: "0775"
+      when: binary_file is not search("http")
+
+    - name: Create a service file
+      template:
+        src: roles/node/templates/joystream-node.service.j2
+        dest: /etc/systemd/system/joystream-node.service
+      vars:
+        template_remote_chain_spec_path: "/home/ubuntu/chain-spec.json"
+        template_binary_path: "/home/ubuntu/joystream-node"
+      become: yes
+
+    - name: Start service joystream-node, if not started
+      service:
+        name: joystream-node
+        state: started
+      become: yes

devops/infrastructure/roles/node/templates/joystream-node.service.j2

@@ -0,0 +1,18 @@
+[Unit]
+Description=Joystream Node
+After=network.target
+
+[Service]
+Type=simple
+User=ubuntu
+WorkingDirectory=/home/ubuntu/
+ExecStart={{ template_binary_path }} \
+        --chain {{ template_remote_chain_spec_path }} \
+        --pruning archive \
+        --log runtime,txpool,transaction-pool,trace=sync
+Restart=on-failure
+RestartSec=3
+LimitNOFILE=10000
+
+[Install]
+WantedBy=multi-user.target

devops/infrastructure/single-instance.yml

@@ -7,18 +7,12 @@ Parameters:
   EC2AMI:
     Type: String
     Default: 'ami-09e67e426f25ce0d7'
-  DefaultAMI:
-    Type: String
-    Default: 'ami-09e67e426f25ce0d7'
   KeyName:
     Description: Name of an existing EC2 KeyPair to enable SSH access to the instance
     Type: 'AWS::EC2::KeyPair::KeyName'
     Default: 'joystream-key'
     ConstraintDescription: must be the name of an existing EC2 KeyPair.
 
-Conditions:
-  HasAMIId: !Not [!Equals [!Ref EC2AMI, ""]]
-
 Resources:
   SecurityGroup:
     Type: AWS::EC2::SecurityGroup
@@ -46,7 +40,7 @@ Resources:
     Properties:
       LaunchTemplateName: !Sub 'LaunchTemplate_${AWS::StackName}'
       LaunchTemplateData:
-        ImageId: !If [HasAMIId, !Ref EC2AMI, !Ref DefaultAMI]
+        ImageId: !Ref EC2AMI
         InstanceType: !Ref EC2InstanceType
         KeyName: !Ref KeyName
         SecurityGroupIds:
@@ -100,6 +94,8 @@ Outputs:
   PublicIp:
     Description: The DNS name for the created instance
     Value:  !Sub "${Instance.PublicIp}"
+    Export:
+      Name: !Sub "${AWS::StackName}PublicIp"
 
   InstanceId:
     Description: The Instance ID