Add RPC instance, add Ansible roles for validator and rpc

devops/infrastructure/bash-config.cfg

@@ -1,11 +1,11 @@
 STACK_NAME=joystream-node
 REGION=us-east-1
 CLI_PROFILE=joystream-user
-KEY_PATH="/Users/myuser/Joystream/joystream-key.pem"
-AWS_KEY_PAIR_NAME="joystream-key"
+KEY_PATH="/Users/anuj/Joystream/anuj-key.pem"
+AWS_KEY_PAIR_NAME="anuj-key"
 BRANCH_NAME=sumer
 LOCAL_CODE_PATH="~/Joystream/joystream"
 EC2_INSTANCE_TYPE=t2.xlarge
 NETWORK_SUFFIX=7891
 # Set a prebuilt AMI if required
-EC2_AMI_ID=""
+EC2_AMI_ID="ami-064f99551235fb1ac"

devops/infrastructure/chain-spec-configuration.yml

@@ -1,93 +1,20 @@
-- name: Configure chain spec and start joystream-node service on the servers
-  hosts: all
-  vars:
-    local_dir: ~/Joystream/joystream
-    # Generates random number between 1000..9999
-    network_suffix: "{{ 10000 | random(1000) }}"
-    data_path: ./data
-    change_spec_path: "{{ data_path }}/chainspec.json"
-    remote_code_path: "/home/ubuntu/joystream"
-    remote_chain_spec_path: "{{ remote_code_path }}/chainspec.json"
-  tasks:
-  - name: Run subkey to generate node keys
-    local_action: ansible.builtin.command {{ local_dir }}/target/release/chain-spec-builder generate -a 2 --chain-spec-path {{ change_spec_path }} --deployment live --endowed 1 --keystore-path {{ data_path }}
-    register: chain_spec_output
-    run_once: true
-
-  - name: Run subkey to generate node keys
-    local_action: ansible.builtin.command subkey generate-node-key
-    register: subkey_output
-
-  - name: Print to stdout
-    debug:
-      msg:
-      - "Public Key: {{ subkey_output.stderr }}"
-      - "Private Key: {{ subkey_output.stdout }}"
-
-  - name: Print to stdout chain spec
-    debug: var=chain_spec_output.stdout
-    run_once: true
-
-  - name: Save output of chain spec to local file
-    local_action: copy content={{ chain_spec_output.stdout }} dest="{{ data_path }}/chain_spec_output.txt"
+---
+# Configure chain spec and start joystream-node service on the servers
 
-  - name: Change chain spec name, id, protocolId
-    delegate_to: localhost
-    json_modify:
-      change_spec_path: "{{ change_spec_path }}"
-      prefix: "{{ network_suffix }}"
-      all_nodes: "{{ hostvars }}"
-    register: result
-    run_once: true
-
-  - name: Print output of modified chainspec
-    debug:
-      var: result.result
-
-  - name: Copying chain spec file to server
-    copy:
-      src: "{{ change_spec_path }}"
-      dest: "{{ remote_chain_spec_path }}"
-
-  - set_fact:
-      chain_path: "{{ remote_code_path }}/chains/{{ result.result.id }}"
-
-  - set_fact:
-      network_path: "{{ chain_path }}/network"
-      keystore_path: "{{ chain_path }}/keystore/"
-  
-  - set_fact:
-      secret_path: "{{ network_path }}/secret"
+- name: Create and copy the chain-spec file
+  hosts: all
 
-  - name: Creating chains directory
-    file:
-      path: "{{ item }}"
-      state: directory
-    loop:
-      - "{{ network_path }}"
+  roles:
+    - common
 
-  - name: Copy secret to remote host
-    copy:
-      dest: "{{ secret_path }}"
-      content: "{{ subkey_output.stdout }}"
+- name: Copy secret, auth and start joystream-node service for validators
+  hosts: validators
 
-  - name: Copy auth directory to remote host
-    copy:
-      src: "{{ data_path }}/auth-{{ ansible_play_batch.index(inventory_hostname) }}/"
-      dest: "{{ keystore_path }}"
+  roles:
+    - validators
 
-  - name: Create a service file
-    template:
-      src: joystream-node.service.j2
-      dest: /etc/systemd/system/joystream-node.service
-    vars:
-      template_keystore_path: "{{ keystore_path }}"
-      template_secret_path: "{{ secret_path }}"
-      template_remote_chain_spec_path: "{{ remote_chain_spec_path }}"
-    become: yes
+- name: Configure RPC service and start it
+  hosts: rpc
 
-  - name: Start service joystream-node, if not started
-    service:
-      name: joystream-node
-      state: started
-    become: yes
+  roles:
+    - rpc

devops/infrastructure/deploy-infra.sh

@@ -34,10 +34,17 @@ aws cloudformation deploy \
 
 # If the deploy succeeded, get the IP, create inventory and configure the created instances
 if [ $? -eq 0 ]; then
-  aws cloudformation list-exports \
+  VALIDATORS=$(aws cloudformation list-exports \
     --profile $CLI_PROFILE \
     --query "Exports[?starts_with(Name,'${NEW_STACK_NAME}PublicIp')].Value" \
-    --output text | sed 's/\t\t*/\n/g' > inventory
+    --output text | sed 's/\t\t*/\n/g')
+
+  RPC_NODES=$(aws cloudformation list-exports \
+    --profile $CLI_PROFILE \
+    --query "Exports[?starts_with(Name,'${NEW_STACK_NAME}RPCPublicIp')].Value" \
+    --output text | sed 's/\t\t*/\n/g')
+
+  echo -e "[validators]\n$VALIDATORS\n\n[rpc]\n$RPC_NODES" > inventory
 
   if [ -z "$EC2_AMI_ID" ]
   then

devops/infrastructure/group_vars/all

@@ -0,0 +1,10 @@
+---
+# Variables applicable to all hosts
+
+local_dir: ~/Joystream/joystream
+# Generates random number between 1000..9999
+network_suffix: "{{ 10000 | random(1000) }}"
+data_path: ./data
+change_spec_path: "{{ data_path }}/chainspec.json"
+remote_code_path: "/home/ubuntu/joystream"
+remote_chain_spec_path: "{{ remote_code_path }}/chainspec.json"

devops/infrastructure/library/json_modify.py

@@ -27,8 +27,9 @@ def main():
 
     boot_node_list = data["bootNodes"]
     for key in all_nodes:
-        public_key = all_nodes[key]["subkey_output"]["stderr"]
-        boot_node_list.append(f"/ip4/{key}/tcp/30333/p2p/{public_key}")
+        if "validators" in all_nodes[key]["group_names"]:
+            public_key = all_nodes[key]["subkey_output"]["stderr"]
+            boot_node_list.append(f"/ip4/{key}/tcp/30333/p2p/{public_key}")
 
     response["bootNodes"] = boot_node_list
 

devops/infrastructure/main.yml

@@ -17,7 +17,7 @@ Resources:
     Type: AWS::EC2::SecurityGroup
     Properties:
       GroupDescription:
-        !Sub 'Internal Security group for ${AWS::StackName}'
+        !Sub 'Internal Security group for validator nodes ${AWS::StackName}'
       SecurityGroupIngress:
         - IpProtocol: tcp
           FromPort: 30333
@@ -29,7 +29,29 @@ Resources:
           CidrIp: 0.0.0.0/0
       Tags:
         - Key: Name
-          Value: !Ref AWS::StackName
+          Value: !Sub '${AWS::StackName}_validator'
+
+  RPCSecurityGroup:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription:
+        !Sub 'Internal Security group for RPC nodes ${AWS::StackName}'
+      SecurityGroupIngress:
+        - IpProtocol: tcp
+          FromPort: 9933
+          ToPort: 9933
+          CidrIp: 0.0.0.0/0
+        - IpProtocol: tcp
+          FromPort: 9944
+          ToPort: 9944
+          CidrIp: 0.0.0.0/0
+        - IpProtocol: tcp
+          FromPort: 22
+          ToPort: 22
+          CidrIp: 0.0.0.0/0
+      Tags:
+        - Key: Name
+          Value: !Sub '${AWS::StackName}_rpc'
 
   InstanceLaunchTemplate:
     Type: AWS::EC2::LaunchTemplate
@@ -90,6 +112,18 @@ Resources:
         - Key: Name
           Value: !Sub '${AWS::StackName}_2'
 
+  RPCInstance:
+    Type: AWS::EC2::Instance
+    Properties:
+      SecurityGroupIds:
+        - !GetAtt RPCSecurityGroup.GroupId
+      LaunchTemplate:
+        LaunchTemplateId: !Ref InstanceLaunchTemplate
+        Version: !GetAtt InstanceLaunchTemplate.LatestVersionNumber
+      Tags:
+        - Key: Name
+          Value: !Sub '${AWS::StackName}_rpc'
+
   WaitHandle:
     Type: AWS::CloudFormation::WaitConditionHandle
 
@@ -98,7 +132,7 @@ Resources:
     Properties:
       Handle: !Ref 'WaitHandle'
       Timeout: '300'
-      Count: 2
+      Count: 3
 
 Outputs:
   PublicIp:
@@ -112,3 +146,9 @@ Outputs:
     Value:  !Sub "${Instance2.PublicIp}"
     Export:
       Name: !Sub "${AWS::StackName}PublicIp2"
+
+  RPCPublicIp:
+    Description: The DNS name for the created instance
+    Value:  !Sub "${RPCInstance.PublicIp}"
+    Export:
+      Name: !Sub "${AWS::StackName}RPCPublicIp"

devops/infrastructure/roles/common/tasks/main.yml

@@ -0,0 +1,43 @@
+---
+# Configure chain spec and start joystream-node service on the servers
+
+- name: Run subkey to generate node keys
+  local_action: command {{ local_dir }}/target/release/chain-spec-builder generate -a 2 --chain-spec-path {{ change_spec_path }} --deployment live --endowed 1 --keystore-path {{ data_path }}
+  register: chain_spec_output
+  run_once: true
+
+- name: Run subkey to generate node keys
+  local_action: command subkey generate-node-key
+  register: subkey_output
+
+- name: Print to stdout
+  debug:
+    msg:
+    - "Public Key: {{ subkey_output.stderr }}"
+    - "Private Key: {{ subkey_output.stdout }}"
+
+- name: Print to stdout chain spec
+  debug: var=chain_spec_output.stdout
+  run_once: true
+
+- name: Save output of chain spec to local file
+  local_action: copy content={{ chain_spec_output.stdout }} dest="{{ data_path }}/chain_spec_output.txt"
+
+- name: Change chain spec name, id, protocolId
+  delegate_to: localhost
+  json_modify:
+    change_spec_path: "{{ change_spec_path }}"
+    prefix: "{{ network_suffix }}"
+    all_nodes: "{{ hostvars }}"
+  register: result
+  run_once: true
+
+- name: Print output of modified chainspec
+  debug:
+    var: result.result
+  run_once: true
+
+- name: Copying chain spec file to server
+  copy:
+    src: "{{ change_spec_path }}"
+    dest: "{{ remote_chain_spec_path }}"

devops/infrastructure/roles/rpc/tasks/main.yml

@@ -0,0 +1,22 @@
+---
+# Configure and start joystream-node RPC service on the servers
+
+- name: Print bootNodes
+  debug:
+    var: result.result.bootNodes
+  run_once: true
+
+- name: Create a service file
+  template:
+    src: joystream-node.service.j2
+    dest: /etc/systemd/system/joystream-node.service
+  vars:
+    template_remote_chain_spec_path: "{{ remote_chain_spec_path }}"
+    boot_nodes: "{{ result.result.bootNodes }}"
+  become: yes
+
+- name: Start service joystream-node, if not started
+  service:
+    name: joystream-node
+    state: started
+  become: yes

devops/infrastructure/roles/rpc/templates/joystream-node.service.j2

@@ -0,0 +1,25 @@
+[Unit]
+Description=Joystream Node
+After=network.target
+
+[Service]
+Type=simple
+User=ubuntu
+WorkingDirectory=/home/ubuntu/joystream/
+ExecStart=/home/ubuntu/joystream/target/release/joystream-node \
+        --chain {{ template_remote_chain_spec_path }} \
+        --ws-external \
+        --rpc-cors all \
+        --pruning archive \
+        --ws-max-connections 512 \
+        --telemetry-url "wss://telemetry.joystream.org/submit/ 0" \
+        --telemetry-url "wss://telemetry.polkadot.io/submit/ 0"
+        --reserved-nodes \
+                {{ boot_nodes|join(" ") }}
+
+Restart=on-failure
+RestartSec=3
+LimitNOFILE=16384
+
+[Install]
+WantedBy=multi-user.target

devops/infrastructure/roles/validators/tasks/main.yml

@@ -0,0 +1,45 @@
+---
+# Configure chain spec and start joystream-node service on the servers
+
+- set_fact:
+    chain_path: "{{ remote_code_path }}/chains/{{ result.result.id }}"
+
+- set_fact:
+    network_path: "{{ chain_path }}/network"
+    keystore_path: "{{ chain_path }}/keystore/"
+
+- set_fact:
+    secret_path: "{{ network_path }}/secret"
+
+- name: Creating chains directory
+  file:
+    path: "{{ item }}"
+    state: directory
+  loop:
+    - "{{ network_path }}"
+
+- name: Copy secret to remote host
+  copy:
+    dest: "{{ secret_path }}"
+    content: "{{ subkey_output.stdout }}"
+
+- name: Copy auth directory to remote host
+  copy:
+    src: "{{ data_path }}/auth-{{ ansible_play_batch.index(inventory_hostname) }}/"
+    dest: "{{ keystore_path }}"
+
+- name: Create a service file
+  template:
+    src: joystream-node.service.j2
+    dest: /etc/systemd/system/joystream-node.service
+  vars:
+    template_keystore_path: "{{ keystore_path }}"
+    template_secret_path: "{{ secret_path }}"
+    template_remote_chain_spec_path: "{{ remote_chain_spec_path }}"
+  become: yes
+
+- name: Start service joystream-node, if not started
+  service:
+    name: joystream-node
+    state: started
+  become: yes

devops/infrastructure/roles/validators/templates/joystream-node.service.j2

@@ -0,0 +1,21 @@
+[Unit]
+Description=Joystream Node
+After=network.target
+
+[Service]
+Type=simple
+User=ubuntu
+WorkingDirectory=/home/ubuntu/joystream/
+ExecStart=/home/ubuntu/joystream/target/release/joystream-node \
+        --chain {{ template_remote_chain_spec_path }} \
+        --pruning archive \
+        --node-key-file {{ template_secret_path }} \
+        --keystore-path {{ template_keystore_path }} \
+        --validator \
+        --log runtime,txpool,transaction-pool,trace=sync
+Restart=on-failure
+RestartSec=3
+LimitNOFILE=10000
+
+[Install]
+WantedBy=multi-user.target