Create bash config, add gitignore, add ansible params

.gitignore

@@ -39,3 +39,8 @@ yarn*
 # test data for local node tests
 test-data/
 tmp.*
+
+# Ignore files created by deployment scripts
+inventory
+devops/infrastructure/data-*
+chain_spec_output.txt

devops/infrastructure/README.md

@@ -26,7 +26,7 @@ On Mac run the command:
 Follow [the official installation guide](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) for your system.
 
 # How to run
-Edit the file `deploy-infra.sh` and update parameters like AWS_KEY_PAIR_NAME, KEY_PATH
+Edit the file `bash-config.cfg` and update parameters like AWS_KEY_PAIR_NAME, KEY_PATH
 Run the `deploy-infra.sh` script to deploy the infrastructure
 
 ```

devops/infrastructure/bash-config.cfg

@@ -0,0 +1,11 @@
+STACK_NAME=joystream-node
+REGION=us-east-1
+CLI_PROFILE=joystream-user
+KEY_PATH="~/Joystream/joystream-key.pem"
+AWS_KEY_PAIR_NAME="joystream-key"
+BRANCH_NAME=sumer
+LOCAL_CODE_PATH="~/Joystream/joystream"
+EC2_INSTANCE_TYPE=t2.xlarge
+NETWORK_SUFFIX=7891
+# Set a prebuilt AMI if required
+EC2_AMI_ID=""

devops/infrastructure/chain-spec-configuration.yml

@@ -3,7 +3,7 @@
   vars:
     local_dir: ~/Joystream/joystream
     # Generates random number between 1000..9999
-    random_suffix: "{{ 10000 | random(1000) }}"
+    network_suffix: "{{ 10000 | random(1000) }}"
     data_path: ./data
     change_spec_path: "{{ data_path }}/chainspec.json"
     remote_code_path: "/home/ubuntu/joystream"
@@ -35,7 +35,7 @@
     delegate_to: localhost
     json_modify:
       change_spec_path: "{{ change_spec_path }}"
-      prefix: "{{ random_suffix }}"
+      prefix: "{{ network_suffix }}"
       all_nodes: "{{ hostvars }}"
     register: result
     run_once: true
@@ -81,9 +81,9 @@
       src: joystream-node.service.j2
       dest: /etc/systemd/system/joystream-node.service
     vars:
-      keystore_path: "{{ keystore_path }}"
-      secret_path: "{{ secret_path }}"
-      keystore_path: "{{ remote_chain_spec_path }}"
+      template_keystore_path: "{{ keystore_path }}"
+      template_secret_path: "{{ secret_path }}"
+      template_remote_chain_spec_path: "{{ remote_chain_spec_path }}"
     become: yes
 
   - name: Start service joystream-node, if not started

devops/infrastructure/deploy-infra.sh

@@ -1,17 +1,8 @@
 #!/bin/bash
 
-STACK_NAME=joystream-node
-REGION=us-east-1
-CLI_PROFILE=joystream-user
-KEY_PATH=""
-AWS_KEY_PAIR_NAME=""
-BRANCH_NAME=sumer
-LOCAL_CODE_PATH="~/Joystream/joystream"
-EC2_INSTANCE_TYPE=t2.xlarge
-# Set a prebuilt AMI if required
-EC2_AMI_ID=""
+source bash-config.cfg
 
-ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+ACCOUNT_ID=$(aws sts get-caller-identity --profile $CLI_PROFILE --query Account --output text)
 
 NEW_STACK_NAME="${STACK_NAME}-${ACCOUNT_ID}"
 
@@ -36,8 +27,6 @@ if [ $? -eq 0 ]; then
     --query "Exports[?starts_with(Name,'${NEW_STACK_NAME}PublicIp')].Value" \
     --output text | sed 's/\t\t*/\n/g' > inventory
 
-  sleep 15s
-
   if [ -z "$EC2_AMI_ID" ]
   then
     echo -e "\n\n=========== Configuring the servers ==========="
@@ -45,5 +34,6 @@ if [ $? -eq 0 ]; then
   fi
 
   echo -e "\n\n=========== Configuring the chain spec file ==========="
-  ansible-playbook -i inventory --private-key $KEY_PATH chain-spec-configuration.yml --extra-vars "local_dir=$LOCAL_CODE_PATH"
+  ansible-playbook -i inventory --private-key $KEY_PATH chain-spec-configuration.yml \
+    --extra-vars "local_dir=$LOCAL_CODE_PATH network_suffix=$NETWORK_SUFFIX data_path=data-$NEW_STACK_NAME"
 fi

devops/infrastructure/joystream-node.service.j2

@@ -7,10 +7,10 @@ Type=simple
 User=ubuntu
 WorkingDirectory=/home/ubuntu/joystream/
 ExecStart=/home/ubuntu/joystream/target/release/joystream-node \
-        --chain {{ remote_chain_spec_path }} \
+        --chain {{ template_remote_chain_spec_path }} \
         --pruning archive \
-        --node-key-file {{ secret_path }} \
-        --keystore-path {{ keystore_path }}\
+        --node-key-file {{ template_secret_path }} \
+        --keystore-path {{ template_keystore_path }} \
         --validator \
         --log runtime,txpool,transaction-pool,trace=sync
 Restart=on-failure

devops/infrastructure/library/json_modify.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 
-from ansible.module_utils.basic import *
 import json
+from ansible.module_utils.basic import AnsibleModule
 
 
 def main():
@@ -9,14 +9,7 @@ def main():
         "change_spec_path": {"required": True, "type": "str"},
         "file_content": {"required": False, "type": "str" },
         "prefix": {"required": False, "type": "str" },
-        "all_nodes": {"required": False, "type": "dict" },
-        # "description": {"required": False, "type": "str"},
-        # "private": {"default": False, "type": "bool" },
-        # "state": {
-        # 	"default": "present", 
-        # 	"choices": ['present', 'absent'],  
-        # 	"type": 'str' 
-        # },
+        "all_nodes": {"required": False, "type": "dict" }
     }
     module = AnsibleModule(argument_spec=fields)
     prefix = module.params["prefix"]
@@ -25,9 +18,7 @@ def main():
 
     with open(change_spec_path) as f:
         data = json.load(f)
-    # data = json.loads(module.params["file_content"])
-    # print(data)
-    # response = {"hello": "world"}
+
     response = {
         "name": f'{data["name"]} {prefix}',
         "id": f'{data["id"]}_{prefix}',

devops/infrastructure/main.yml

@@ -62,21 +62,13 @@ Resources:
             # Update all packages
             apt-get update -y
 
-            # Get latest cfn scripts;
-            # apt-get install -y python3-pip
+            # Get latest cfn scripts and install them;
             apt-get install -y python3-setuptools
             mkdir -p /opt/aws/bin
-
-            # pip3 install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz
             wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz
             python3 -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-py3-latest.tar.gz
 
-            # curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
-            # echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
-            # apt-get update -y
-            # apt-get install -y yarn
-
-            ln -s /root/aws-cfn-bootstrap-latest/init/ubuntu/cfn-hup /etc/init.d/cfn-hup
+            /opt/aws/bin/cfn-signal -e $? -r "Instance Created" '${WaitHandle}'
 
   Instance:
     Type: AWS::EC2::Instance
@@ -98,6 +90,16 @@ Resources:
         - Key: Name
           Value: !Sub '${AWS::StackName}_2'
 
+  WaitHandle:
+    Type: AWS::CloudFormation::WaitConditionHandle
+
+  WaitCondition:
+    Type: AWS::CloudFormation::WaitCondition
+    Properties:
+      Handle: !Ref 'WaitHandle'
+      Timeout: '300'
+      Count: 2
+
 Outputs:
   PublicIp:
     Description: The DNS name for the created instance