Merge pull request #3184 from ahhda/infra-olympia-fixes

DevOps - Deploy Network improvements

.github/workflows/deploy-node-network.yml

@@ -39,8 +39,19 @@ jobs:
           echo ::set-output name=ec2AMI::$(echo $jsonInput | jq -r '.ec2AMI.value')
           echo ::set-output name=networkSuffix::$(echo $jsonInput | jq -r '.networkSuffix.value')
           echo ::set-output name=deploymentType::$(echo $jsonInput | jq -r '.deploymentType.value')
+          echo ::set-output name=volumeSize::$(echo $jsonInput | jq -r '.volumeSize.value')
+          echo ::set-output name=rpcVolumeSize::$(echo $jsonInput | jq -r '.rpcVolumeSize.value')
           initialBalancesFile=$(echo $jsonInput | jq -r '.initialBalancesFile.value')
           initialMembersFile=$(echo $jsonInput | jq -r '.initialMembersFile.value')
+          proposalParametersInput=$(echo $jsonInput | jq -r '.proposalParameters.value')
+
+          if [ $proposalParametersInput = "{}" ]
+          then
+            echo ::set-output name=proposalParameters::''
+          else
+            echo ::set-output name=proposalParameters::$proposalParametersInput
+          fi
+
           if [ -z "$initialBalancesFile" ]
           then
             echo ::set-output name=initialBalancesFilePath::''
@@ -48,6 +59,7 @@ jobs:
             wget $initialBalancesFile -O initial-balances.json
             echo ::set-output name=initialBalancesFilePath::'initial-balances.json'
           fi
+
           if [ -z "$initialMembersFile" ]
           then
             echo ::set-output name=initialMembersFilePath::''
@@ -66,6 +78,16 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
 
+      - name: Check if CloudFormation stack exists
+        id: stack_exists
+        run: |
+          if aws cloudformation describe-stacks --stack-name ${{ env.STACK_NAME }} >/dev/null 2>/dev/null; then
+            echo "Stack already exists"
+            exit 1
+          else
+            echo "Stack does not exist"
+          fi
+
       - name: Deploy to AWS CloudFormation
         uses: aws-actions/aws-cloudformation-github-deploy@v1
         id: deploy_stack
@@ -79,7 +101,9 @@ jobs:
             RPCEC2InstanceType=${{ steps.myoutputs.outputs.rpcInstanceType }},
             BuildEC2InstanceType=${{ steps.myoutputs.outputs.buildInstanceType }},
             EC2AMI=${{ steps.myoutputs.outputs.ec2AMI }},
-            NumberOfValidators=${{ steps.myoutputs.outputs.numberOfValidators }}
+            NumberOfValidators=${{ steps.myoutputs.outputs.numberOfValidators }},
+            VolumeSize=${{ steps.myoutputs.outputs.volumeSize }},
+            RPCVolumeSize=${{ steps.myoutputs.outputs.rpcVolumeSize }}
 
       - name: Prepare inventory for Ansible
         run: |
@@ -103,7 +127,8 @@ jobs:
 
       - name: Run playbook to compile joystream-node on build server
         uses: dawidd6/action-ansible-playbook@v2
-        if: steps.myoutputs.outputs.ec2AMI == ''
+        # Build binaries if AMI not specified or a custom proposals parameter is passed
+        if: steps.myoutputs.outputs.ec2AMI == '' || steps.myoutputs.outputs.proposalParameters != ''
         with:
           playbook: build-code.yml
           directory: devops/aws
@@ -112,23 +137,24 @@ jobs:
           options: |
             --inventory inventory
             --extra-vars "branch_name=${{ steps.myoutputs.outputs.branchName }} \
-                          git_repo=${{ steps.myoutputs.outputs.gitRepo }} data_path=mydata"
+                          git_repo=${{ steps.myoutputs.outputs.gitRepo }} data_path=mydata \
+                          proposal_parameters=${{ steps.myoutputs.outputs.proposalParameters }}"
 
       - name: Run playbook to install additional utils on build server
         uses: dawidd6/action-ansible-playbook@v2
         if: steps.myoutputs.outputs.ec2AMI == ''
         with:
-          playbook: setup-admin.yml
+          playbook: setup-build-server.yml
           directory: devops/aws
           requirements: requirements.yml
           key: ${{ secrets.SSH_PRIVATE_KEY }}
           options: |
             --inventory inventory
 
-      - name: Run playbook to configure chain-spec and pioneer
+      - name: Run playbook to configure chain-spec
         uses: dawidd6/action-ansible-playbook@v2
         with:
-          playbook: chain-spec-pioneer.yml
+          playbook: configure-network.yml
           directory: devops/aws
           requirements: requirements.yml
           key: ${{ secrets.SSH_PRIVATE_KEY }}
@@ -138,11 +164,16 @@ jobs:
                           number_of_validators=${{ steps.myoutputs.outputs.numberOfValidators }} \
                           git_repo=${{ steps.myoutputs.outputs.gitRepo }} \
                           deployment_type=${{ steps.myoutputs.outputs.deploymentType }} \
-                          bucket_name=${{ steps.deploy_stack.outputs.S3BucketName }} \
                           branch_name=${{ steps.myoutputs.outputs.branchName }} \
                           initial_members_file=${{ steps.myoutputs.outputs.initialMembersFilePath }} \
                           initial_balances_file=${{ steps.myoutputs.outputs.initialBalancesFilePath }}"
 
+      - name: Terminate Build instance
+        continue-on-error: true
+        run: |
+          echo "Deleting build instance with id ${{ steps.deploy_stack.outputs.BuildInstanceId }}"
+          aws ec2 terminate-instances --instance-ids ${{ steps.deploy_stack.outputs.BuildInstanceId }}
+
       - name: Encrpyt the artifacts
         run: |
           7z a -p${{ steps.myoutputs.outputs.encryptionKey }} chain-data.7z mydata/*
@@ -153,16 +184,16 @@ jobs:
           name: data-chainspec-auth
           path: devops/aws/chain-data.7z
 
-      - name: Print output URL's
-        run: |
-          echo -e "Pioneer URL: https://${{ steps.deploy_stack.outputs.DomainName }}"
-          echo -e "RPC: wss://${{ steps.deploy_stack.outputs.RPCPublicIp }}.nip.io/ws-rpc"
+      - name: Save the endpoints file as an artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: endpoints
+          path: devops/aws/endpoints.json
 
       - name: Delete CloudFormation Stack if any step failed
-        if: failure()
+        # Skip only if stack already existed or all steps passed succesfully
+        if: ( failure() || cancelled() ) && steps.stack_exists.outcome != 'failure'
         run: |
-          echo "Empty the bucket"
-          aws s3 rm s3://${{ steps.deploy_stack.outputs.S3BucketName }} --recursive || echo "No bucket"
           echo "Deleting ${{ env.STACK_NAME }} stack"
           aws cloudformation delete-stack --stack-name ${{ env.STACK_NAME }}
           echo "Waiting for ${{ env.STACK_NAME }} to be deleted..."

.pipelines/deploy-node-network-inputs.json

@@ -31,6 +31,14 @@
     "description": "Pre-built AMI ID (ami-095792100b6e43a67)",
     "value": "ami-095792100b6e43a67"
   },
+  "volumeSize": {
+    "description": "Validator and Build instance volume size in GB",
+    "value": "120"
+  },
+  "rpcVolumeSize": {
+    "description": "RPC Instance volume size in GB",
+    "value": "120"
+  },
   "networkSuffix": {
     "description": "Network suffix that will be added to the network name",
     "value": "8129"
@@ -50,5 +58,9 @@
   "encryptionKey": {
     "description": "Password to encrypt the artifacts",
     "value": "password"
+  },
+  "proposalParameters": {
+    "description": "Proposal Parameters",
+    "value": {}
   }
 }

devops/aws/chain-spec-pioneer.yml

@@ -1,37 +0,0 @@
----
-# Configure chain spec, start joystream-node service on the servers and build Pioneer
-
-- name: Create and copy the chain-spec file
-  hosts: all
-
-  tasks:
-    - name: Generate chain-spec file and data keys either on localhost or admin server
-      include_role:
-        name: common
-        tasks_from: chain-spec-node-keys
-      vars:
-        local_or_admin: "{{ groups['build'][0] if run_on_admin_server|bool else 'localhost' }}"
-        admin_code_dir: "{{ remote_code_path if run_on_admin_server|bool else local_dir }}"
-
-- name: Copy secret, auth and start joystream-node service for validators
-  hosts: validators
-  gather_facts: no
-
-  roles:
-    - validators
-
-- name: Configure RPC service and start it
-  hosts: rpc
-  gather_facts: no
-
-  roles:
-    - rpc
-
-- name: Build Pioneer and copy artifacts to S3
-  hosts: build
-  gather_facts: no
-
-  tasks:
-    - include_role:
-        name: admin
-        tasks_from: deploy-pioneer

devops/aws/cloudformation/infrastructure.yml

@@ -2,7 +2,7 @@
 # This is comprised of:
 #   - N validators
 #   - One RPC node
-#   - s3 bucket with a build of Pionner
+#   - One Build instance
 
 AWSTemplateFormatVersion: 2010-09-09
 
@@ -34,16 +34,23 @@ Parameters:
     Description: Number of validator instances to launch
     Type: Number
     Default: 2
+  VolumeSize:
+    Description: Validator and Build instance volume size in GB
+    Type: Number
+    Default: 120
+  RPCVolumeSize:
+    Description: RPC Instance volume size in GB
+    Type: Number
+    Default: 120
 
 Conditions:
-  HasAMIId: !Not [!Equals [!Ref EC2AMI, ""]]
+  HasAMIId: !Not [!Equals [!Ref EC2AMI, '']]
 
 Resources:
   SecurityGroup:
     Type: AWS::EC2::SecurityGroup
     Properties:
-      GroupDescription:
-        !Sub 'Internal Security group for validator nodes ${AWS::StackName}'
+      GroupDescription: !Sub 'Internal Security group for validator nodes ${AWS::StackName}'
       SecurityGroupIngress:
         - IpProtocol: tcp
           FromPort: 30333
@@ -60,8 +67,7 @@ Resources:
   RPCSecurityGroup:
     Type: AWS::EC2::SecurityGroup
     Properties:
-      GroupDescription:
-        !Sub 'Internal Security group for RPC nodes ${AWS::StackName}'
+      GroupDescription: !Sub 'Internal Security group for RPC nodes ${AWS::StackName}'
       SecurityGroupIngress:
         - IpProtocol: tcp
           FromPort: 9933
@@ -111,7 +117,7 @@ Resources:
         BlockDeviceMappings:
           - DeviceName: /dev/sda1
             Ebs:
-              VolumeSize: '120'
+              VolumeSize: !Ref VolumeSize
         UserData:
           Fn::Base64: !Sub |
             #!/bin/bash -xe
@@ -161,7 +167,7 @@ Resources:
       DesiredCapacity: !Ref NumberOfValidators
       AvailabilityZones:
         Fn::GetAZs:
-          Ref: "AWS::Region"
+          Ref: 'AWS::Region'
       MixedInstancesPolicy:
         LaunchTemplate:
           LaunchTemplateSpecification:
@@ -172,7 +178,7 @@ Resources:
       Tags:
         - Key: Name
           Value: !Sub '${AWS::StackName}'
-          PropagateAtLaunch: "true"
+          PropagateAtLaunch: 'true'
 
   RPCInstance:
     Type: AWS::EC2::Instance
@@ -183,6 +189,10 @@ Resources:
       LaunchTemplate:
         LaunchTemplateId: !Ref InstanceLaunchTemplate
         Version: !GetAtt InstanceLaunchTemplate.LatestVersionNumber
+      BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+            VolumeSize: !Ref RPCVolumeSize
       Tags:
         - Key: Name
           Value: !Sub '${AWS::StackName}_rpc'
@@ -208,71 +218,27 @@ Resources:
       Timeout: '600'
       Count: !Ref NumberOfValidators
 
-  S3Bucket:
-    Type: AWS::S3::Bucket
-    Properties:
-      AccessControl: PublicRead
-      WebsiteConfiguration:
-        IndexDocument: index.html
-
-  BucketPolicy:
-    Type: AWS::S3::BucketPolicy
-    Properties:
-      PolicyDocument:
-        Id: PublicPolicy
-        Version: 2012-10-17
-        Statement:
-          - Sid: PublicReadForGetBucketObjects
-            Effect: Allow
-            Principal: '*'
-            Action: 's3:GetObject'
-            Resource: !Sub "arn:aws:s3:::${S3Bucket}/*"
-      Bucket: !Ref S3Bucket
-
-  CloudFrontDistribution:
-    Type: AWS::CloudFront::Distribution
-    Properties:
-      DistributionConfig:
-        Origins:
-        - DomainName: !Select [1, !Split ["//", !GetAtt S3Bucket.WebsiteURL]]
-          Id: pioneer-origin-s3
-          CustomOriginConfig:
-            OriginProtocolPolicy: http-only
-        DefaultCacheBehavior:
-          TargetOriginId: pioneer-origin-s3
-          ViewerProtocolPolicy: redirect-to-https
-          ForwardedValues:
-            QueryString: true
-        Enabled: true
-        HttpVersion: http2
-
 Outputs:
   AutoScalingId:
     Description: The Auto Scaling ID
-    Value:  !Ref AutoScalingGroup
+    Value: !Ref AutoScalingGroup
     Export:
-      Name: !Sub "${AWS::StackName}AutoScalingGroup"
+      Name: !Sub '${AWS::StackName}AutoScalingGroup'
 
   RPCPublicIp:
     Description: The DNS name for the created instance
-    Value:  !Sub "${RPCInstance.PublicIp}"
+    Value: !Sub '${RPCInstance.PublicIp}'
     Export:
-      Name: !Sub "${AWS::StackName}RPCPublicIp"
+      Name: !Sub '${AWS::StackName}RPCPublicIp'
 
   BuildPublicIp:
     Description: The DNS name for the created instance
-    Value:  !Sub "${BuildInstance.PublicIp}"
-    Export:
-      Name: !Sub "${AWS::StackName}BuildPublicIp"
-
-  S3BucketName:
-    Value: !Ref S3Bucket
-    Description: Name of S3 bucket to hold website content
+    Value: !Sub '${BuildInstance.PublicIp}'
     Export:
-      Name: !Sub "${AWS::StackName}S3BucketName"
+      Name: !Sub '${AWS::StackName}BuildPublicIp'
 
-  DomainName:
-    Description: CloudFront Domain Name
-    Value:  !Sub "${CloudFrontDistribution.DomainName}"
+  BuildInstanceId:
+    Description: Build instance ID
+    Value: !Ref BuildInstance
     Export:
-      Name: !Sub "${AWS::StackName}DomainName"
+      Name: !Sub '${AWS::StackName}BuildInstanceId'

devops/aws/configure-network.yml

@@ -0,0 +1,27 @@
+---
+# Configure chain spec, start joystream-node and other services on the servers
+
+- name: Create and copy the chain-spec file
+  hosts: all
+
+  tasks:
+    - name: Generate chain-spec file and data keys on build server
+      include_role:
+        name: common
+        tasks_from: chain-spec-node-keys
+      vars:
+        build_instance: "{{ groups['build'][0] }}"
+
+- name: Copy secret, auth and start joystream-node service for validators
+  hosts: validators
+  gather_facts: no
+
+  roles:
+    - validators
+
+- name: Configure RPC service and start it
+  hosts: rpc
+  gather_facts: no
+
+  roles:
+    - rpc

devops/aws/create-joystream-node-ami-playbook.yml

@@ -17,7 +17,7 @@
 
     - name: Install subkey
       include_role:
-        name: admin
+        name: build-server
         tasks_from: main
 
     - name: Basic AMI Creation

devops/aws/deploy-infra.sample.cfg

@@ -35,5 +35,7 @@ INITIAL_BALANCES_PATH=""
 GIT_REPO="https://github.com/Joystream/joystream.git"
 BRANCH_NAME=master
 
+ALL_PROPOSALS_PARAMETERS_JSON=""
+
 # If true will build LOCAL_CODE_PATH otherwise will pull from GIT_REPO:BRANCH_NAME
 BUILD_LOCAL_CODE=false

devops/aws/deploy-infra.sh

@@ -63,29 +63,35 @@ if [ $? -eq 0 ]; then
 
   BUILD_SERVER=$(get_aws_export $NEW_STACK_NAME "BuildPublicIp")
 
-  BUCKET_NAME=$(get_aws_export $NEW_STACK_NAME "S3BucketName")
-
-  DOMAIN_NAME=$(get_aws_export $NEW_STACK_NAME "DomainName")
+  BUILD_INSTANCE_ID=$(get_aws_export $NEW_STACK_NAME "BuildInstanceId")
 
   mkdir -p $DATA_PATH
 
   echo -e "[build]\n$BUILD_SERVER\n\n[validators]\n$VALIDATORS\n[rpc]\n$RPC_NODES" > $INVENTORY_PATH
 
-  if [ -z "$EC2_AMI_ID" ]
+  # Build binaries if AMI not specified or a custom proposals parameter is passed
+  if [ -z "$EC2_AMI_ID" ] || [ -n "$ALL_PROPOSALS_PARAMETERS_JSON" ]
   then
     echo -e "\n\n=========== Compile joystream-node on build server ==========="
     ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH build-code.yml \
-      --extra-vars "branch_name=$BRANCH_NAME git_repo=$GIT_REPO build_local_code=$BUILD_LOCAL_CODE data_path=$DATA_PATH"
+      --extra-vars "branch_name=$BRANCH_NAME git_repo=$GIT_REPO build_local_code=$BUILD_LOCAL_CODE
+                    data_path=$DATA_PATH proposal_parameters=$ALL_PROPOSALS_PARAMETERS_JSON"
+  fi
 
+  if [ -z "$EC2_AMI_ID" ]
+  then
     echo -e "\n\n=========== Install additional utils on build server ==========="
-    ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH setup-admin.yml
+    ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH setup-build-server.yml
   fi
 
-  echo -e "\n\n=========== Configure and start new validators, rpc node and pioneer ==========="
-  ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH chain-spec-pioneer.yml \
+  echo -e "\n\n=========== Configure and start new validators and rpc node ==========="
+  ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH configure-network.yml \
     --extra-vars "local_dir=$LOCAL_CODE_PATH network_suffix=$NETWORK_SUFFIX
-                  data_path=$DATA_PATH bucket_name=$BUCKET_NAME number_of_validators=$NUMBER_OF_VALIDATORS
+                  data_path=$DATA_PATH number_of_validators=$NUMBER_OF_VALIDATORS
                   deployment_type=$DEPLOYMENT_TYPE initial_balances_file=$INITIAL_BALANCES_PATH initial_members_file=$INITIAL_MEMBERS_PATH"
 
-  echo -e "\n\n Pioneer URL: https://$DOMAIN_NAME"
+  echo -e "\n\n=========== Delete Build instance ==========="
+  DELETE_RESULT=$(aws ec2 terminate-instances --instance-ids $BUILD_INSTANCE_ID --profile $CLI_PROFILE)
+  echo $DELETE_RESULT
+
 fi

devops/aws/deploy-playground-playbook.yml

@@ -11,7 +11,7 @@
         name: common
         tasks_from: get-code-git
 
-    - name: Creat bash profile file
+    - name: Create bash profile file
       command: 'touch /home/ubuntu/.bash_profile'
 
     - name: Run setup script

devops/aws/destroy-infra.sh

@@ -13,14 +13,8 @@ else
   source $1
 fi
 
-BUCKET_NAME=$(get_aws_export $NEW_STACK_NAME "S3BucketName")
-
 # Delete the CloudFormation stack
 
-echo -e "\n\n=========== Emptying bucket $BUCKET_NAME ==========="
-
-aws s3 rm s3://$BUCKET_NAME --recursive --profile $CLI_PROFILE || echo "No bucket"
-
 echo -e "\n\n=========== Deleting stack $NEW_STACK_NAME ==========="
 
 aws cloudformation delete-stack --stack-name $NEW_STACK_NAME --profile $CLI_PROFILE

devops/aws/group_vars/all

@@ -15,8 +15,5 @@ raw_chain_spec_path: "{{ remote_data_path }}/chainspec-raw.json"
 local_raw_chain_spec_path: "{{ data_path }}/chainspec-raw.json"
 remote_code_path: "/home/ubuntu/joystream"
 remote_chain_spec_path: "{{ remote_code_path }}/chainspec.json"
-run_on_admin_server: true
 build_local_code: false
 number_of_validators: 2
-
-bucket_name: s3-bucket-joystream

devops/aws/roles/admin/tasks/deploy-pioneer.yml

@@ -1,38 +0,0 @@
----
-# Build Pioneer, copy build artifacts and sync to S3
-
-- name: Set ws_rpc for build node
-  set_fact:
-    ws_rpc: "{{ hostvars[groups['rpc'][0]].ws_rpc }}"
-
-- name: Build Pioneer code
-  shell: "WS_URL=wss://{{ ws_rpc }} yarn && yarn workspace @joystream/types build && yarn workspace pioneer build"
-  args:
-    chdir: "{{ remote_code_path }}"
-  async: 3600
-  poll: 0
-  register: build_result
-
-- name: Check on build async task
-  async_status:
-    jid: '{{ build_result.ansible_job_id }}'
-  register: job_result
-  until: job_result.finished
-  # Max number of times to check for status
-  retries: 36
-  # Check for the status every 100s
-  delay: 100
-
-- name: Copying build files to local
-  synchronize:
-    src: "{{ remote_code_path }}/pioneer/packages/apps/build"
-    dest: "{{ data_path }}"
-    mode: pull
-  run_once: true
-
-- name: Run S3 Sync to upload build files to bucket
-  community.aws.s3_sync:
-    bucket: "{{ bucket_name }}"
-    file_root: "{{ data_path }}/build"
-    region: us-east-1
-  delegate_to: localhost

devops/aws/roles/admin/tasks/main.yml → devops/aws/roles/build-server/tasks/main.yml

@@ -1,5 +1,5 @@
 ---
-# Configure admin server to be able to create chain-spec file and subkey commands
+# Configure build server to be able to create chain-spec file and subkey commands
 
 - name: Copy bash_profile content
   shell: cat ~/.bash_profile

devops/aws/roles/common/tasks/chain-spec-node-keys.yml

@@ -9,32 +9,34 @@
 - name: Copying initial members file to the server
   copy:
     src: '{{ initial_members_file }}'
-    dest: '{{ admin_code_dir }}/initial-members.json'
+    dest: '{{ remote_code_path }}/query-node/mappings/src/bootstrap-data/data/members.json'
   when: initial_members_file is defined and initial_members_file|length > 0
   run_once: true
 
 - name: Copying initial balances file to the server
   copy:
     src: '{{ initial_balances_file }}'
-    dest: '{{ admin_code_dir }}/initial-balances.json'
+    dest: '{{ remote_code_path }}/initial-balances.json'
   when: initial_balances_file is defined and initial_balances_file|length > 0
   run_once: true
 
 - name: Run chain-spec-builder to generate chainspec.json file (with initial data)
   shell: >
-    {{ admin_code_dir }}/target/release/chain-spec-builder generate -a {{ number_of_validators }}
+    {{ remote_code_path }}/target/release/chain-spec-builder generate -a {{ number_of_validators }}
     --chain-spec-path {{ chain_spec_path }}
     --endowed 1 --keystore-path {{ remote_data_path }}
     {% if deployment_type is defined and deployment_type|length > 0 %}--deployment {{ deployment_type }}{% endif %}
-    {% if initial_members_file is defined and initial_members_file|length > 0 %}--initial-balances-path {{ admin_code_dir }}/initial-balances.json{% endif %}
-    {% if initial_balances_file is defined and initial_balances_file|length > 0 %}--initial-members-path {{ admin_code_dir }}/initial-members.json{% endif %}
+    {% if initial_members_file is defined and initial_members_file|length > 0 %}--initial-balances-path {{ remote_code_path }}/initial-balances.json{% endif %}
+    {% if initial_balances_file is defined and initial_balances_file|length > 0 %}
+    --initial-members-path {{ remote_code_path }}/query-node/mappings/src/bootstrap-data/data/members.json
+    {% endif %}
   register: chain_spec_output
-  delegate_to: '{{ local_or_admin }}'
+  delegate_to: '{{ build_instance }}'
   run_once: true
 
 - name: Run subkey to generate node keys
   shell: subkey generate-node-key
-  delegate_to: '{{ local_or_admin }}'
+  delegate_to: '{{ build_instance }}'
   register: subkey_output
 
 - name: Print to stdout
@@ -51,7 +53,7 @@
   copy:
     content: '{{ chain_spec_output.stdout | regex_replace("\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[mGK]", "") }}'
     dest: '{{ remote_data_path }}/chain_spec_output.txt'
-  delegate_to: '{{ local_or_admin }}'
+  delegate_to: '{{ build_instance }}'
   run_once: true
 
 - name: Change chain spec name, id, protocolId
@@ -59,7 +61,7 @@
     chain_spec_path: '{{ chain_spec_path }}'
     prefix: '{{ network_suffix }}'
     all_nodes: '{{ hostvars }}'
-  delegate_to: '{{ local_or_admin }}'
+  delegate_to: '{{ build_instance }}'
   register: result
   run_once: true
 
@@ -69,8 +71,8 @@
   run_once: true
 
 - name: Run build-spec to generate raw chainspec file
-  shell: '{{ admin_code_dir }}/target/release/joystream-node build-spec --chain {{ chain_spec_path }} --raw > {{ raw_chain_spec_path }}'
-  delegate_to: '{{ local_or_admin }}'
+  shell: '{{ remote_code_path }}/target/release/joystream-node build-spec --chain {{ chain_spec_path }} --raw > {{ raw_chain_spec_path }}'
+  delegate_to: '{{ build_instance }}'
   run_once: true
 
 - name: Copying chain spec files to localhost
@@ -79,16 +81,14 @@
     dest: '{{ data_path }}'
     mode: pull
   run_once: true
-  when: run_on_admin_server|bool
 
 - name: Copy joystream-node binary to localhost
   fetch:
-    src: '{{ admin_code_dir }}/target/release/joystream-node'
+    src: '{{ remote_code_path }}/target/release/joystream-node'
     dest: '{{ data_path }}/joystream-node'
     flat: yes
-  delegate_to: '{{ local_or_admin }}'
+  delegate_to: '{{ build_instance }}'
   run_once: true
-  when: run_on_admin_server|bool
 
 - name: Copying raw chain spec file to all servers
   copy:

devops/aws/roles/common/tasks/get-code-git.yml

@@ -1,14 +1,9 @@
 ---
 # Get the latest code
 
-- name: Delete remote code directory if exists
-  file:
-    state: absent
-    path: "{{ remote_code_path }}"
-  become: yes
-
 - name: Git checkout
   git:
-    repo: "{{ git_repo }}"
-    dest: "{{ remote_code_path }}"
-    version: "{{ branch_name }}"
+    repo: '{{ git_repo }}'
+    dest: '{{ remote_code_path }}'
+    version: '{{ branch_name }}'
+    force: yes

devops/aws/roles/common/tasks/run-setup-build.yml

@@ -1,7 +1,7 @@
 ---
 # Run setup and build code
 
-- name: Creat bash profile file
+- name: Create bash profile file
   command: 'touch /home/ubuntu/.bash_profile'
 
 - name: Run setup script
@@ -13,6 +13,8 @@
   shell: . ~/.bash_profile && yarn cargo-build
   args:
     chdir: '{{ remote_code_path }}'
+  environment:
+    ALL_PROPOSALS_PARAMETERS_JSON: '{{ proposal_parameters }}'
   # Run in async fashion for max duration of 1 hr
   async: 3600
   poll: 0

devops/aws/roles/rpc/tasks/main.yml

@@ -11,8 +11,8 @@
     src: joystream-node.service.j2
     dest: /etc/systemd/system/joystream-node.service
   vars:
-    template_remote_chain_spec_path: "{{ remote_chain_spec_path }}"
-    boot_nodes: "{{ result.result.bootNodes }}"
+    template_remote_chain_spec_path: '{{ remote_chain_spec_path }}'
+    boot_nodes: '{{ result.result.bootNodes }}'
   become: yes
 
 - name: Start service joystream-node, if not started
@@ -41,10 +41,14 @@
   # Check for the status every 100s
   delay: 100
 
-- name: Run query node containers
-  command: yarn workspace query-node-root start:dev
+- name: Run service containers
+  command: yarn start
   environment:
+    PERSIST: 'true'
     JOYSTREAM_NODE_WS: 'ws://{{ inventory_hostname }}:9944/'
+    COLOSSUS_1_URL: 'https://{{ inventory_hostname }}.nip.io/colossus-1/'
+    DISTRIBUTOR_1_URL: 'https://{{ inventory_hostname }}.nip.io/distributor-1/'
+    SKIP_NODE: 'true'
   args:
     chdir: '{{ remote_code_path }}'
   async: 1800
@@ -63,9 +67,7 @@
 
 - name: Set websocket and http endpoint variables
   set_fact:
-    ws_rpc: "{{ inventory_hostname }}.nip.io/ws-rpc"
-    http_rpc: "{{ inventory_hostname }}.nip.io/http-rpc"
-    nip_domain: "{{ inventory_hostname }}.nip.io"
+    nip_domain: '{{ inventory_hostname }}.nip.io'
   run_once: yes
 
 - name: Install and configure Caddy
@@ -78,6 +80,32 @@
     caddy_systemd_capabilities_enabled: true
     caddy_update: false
 
-- name: Print RPC node DNS
+- name: Set endpoints
+  set_fact:
+    all_services: |
+      websocket_rpc: wss://{{ nip_domain }}/ws-rpc
+      http_rpc: https://{{ nip_domain }}/http-rpc
+      colossus: https://{{ nip_domain }}/colossus-1
+      distributor: https://{{ nip_domain }}/distributor-1
+      graphql_server: https://{{ nip_domain }}/query-node/server/graphql
+      graphql_server_websocket: wss://{{ nip_domain }}/query-node/server/graphql
+      indexer: https://{{ nip_domain }}/query-node/indexer/graphql
+      member_faucet: https://{{ nip_domain }}/member-faucet/register
+      config: https://{{ nip_domain }}/network/config.json
+  run_once: yes
+
+- name: Print endpoints
   debug:
-    msg: "RPC Endpoint: wss://{{ ws_rpc }}"
+    msg: '{{ all_services | from_yaml }}'
+  run_once: yes
+
+- name: Create config.json to serve as Caddy endpoint
+  copy:
+    content: '{{ all_services | from_yaml | to_json }}'
+    dest: '/home/ubuntu/config.json'
+
+- name: Save output as file on local
+  copy:
+    content: '{{ all_services | from_yaml | to_json }}'
+    dest: 'endpoints.json'
+  delegate_to: localhost

devops/aws/roles/rpc/templates/Caddyfile.j2

@@ -1,11 +1,24 @@
-{{ ws_rpc }} {
+{{ nip_domain }}/ws-rpc* {
+    uri strip_prefix /ws-rpc
     reverse_proxy localhost:9944
 }
 
-{{ http_rpc }} {
+{{ nip_domain }}/http-rpc* {
+    uri strip_prefix /http-rpc
     reverse_proxy localhost:9933
 }
 
+{{ nip_domain }}/colossus-1* {
+    uri strip_prefix /colossus-1
+    reverse_proxy localhost:3333
+}
+
+{{ nip_domain }}/distributor-1* {
+    uri strip_prefix /distributor-1
+    reverse_proxy localhost:3334
+}
+
+# newer versions of graphql-server seems to expect this url also
 {{ nip_domain }}/@apollographql/* {
     reverse_proxy localhost:8081
 }
@@ -15,7 +28,32 @@
     reverse_proxy localhost:8081
 }
 
+wss://{{ nip_domain }}/query-node/server* {
+    uri strip_prefix /query-node/server
+    reverse_proxy localhost:8081
+}
+
 {{ nip_domain }}/query-node/indexer* {
     uri strip_prefix /query-node/indexer
     reverse_proxy localhost:4000
 }
+
+{{ nip_domain }}/orion* {
+    uri strip_prefix /orion
+    reverse_proxy localhost:6116
+}
+
+{{ nip_domain }}/member-faucet* {
+    uri strip_prefix /member-faucet
+    reverse_proxy localhost:3002
+}
+
+{{ nip_domain }}/network/config.json {
+    header /* {
+      Access-Control-Allow-Origin *
+      Cache-Control: no-cache
+    }
+    root * /home/ubuntu
+    rewrite * /config.json
+    file_server
+}

devops/aws/setup-admin.yml → devops/aws/setup-build-server.yml

@@ -5,5 +5,4 @@
   hosts: build
 
   roles:
-    - role: admin
-      when: run_on_admin_server|bool
+    - role: build-server

start.sh

@@ -22,8 +22,11 @@ else
   trap down EXIT
 fi
 
-## Run a local development chain
-docker-compose up -d joystream-node
+if [ "${SKIP_NODE}" != true ]
+then
+  ## Run a local development chain
+  docker-compose up -d joystream-node
+fi
 
 ## Init the chain with some state
 if [[ $SKIP_CHAIN_SETUP != 'true' ]]; then