Prevent deletion of stack, remove pioneer, update build naming

.github/workflows/deploy-node-network.yml

@@ -66,6 +66,16 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
 
+      - name: Check if CloudFormation stack exists
+        id: stack_exists
+        run: |
+          if aws cloudformation describe-stacks --stack-name ${{ env.STACK_NAME }} >/dev/null 2>/dev/null; then
+            echo "Stack already exists"
+            exit 1
+          else
+            echo "Stack does not exist"
+          fi
+
       - name: Deploy to AWS CloudFormation
         uses: aws-actions/aws-cloudformation-github-deploy@v1
         id: deploy_stack
@@ -125,10 +135,10 @@ jobs:
           options: |
             --inventory inventory
 
-      - name: Run playbook to configure chain-spec and pioneer
+      - name: Run playbook to configure chain-spec
         uses: dawidd6/action-ansible-playbook@v2
         with:
-          playbook: chain-spec-pioneer.yml
+          playbook: configure-network.yml
           directory: devops/aws
           requirements: requirements.yml
           key: ${{ secrets.SSH_PRIVATE_KEY }}
@@ -138,7 +148,6 @@ jobs:
                           number_of_validators=${{ steps.myoutputs.outputs.numberOfValidators }} \
                           git_repo=${{ steps.myoutputs.outputs.gitRepo }} \
                           deployment_type=${{ steps.myoutputs.outputs.deploymentType }} \
-                          bucket_name=${{ steps.deploy_stack.outputs.S3BucketName }} \
                           branch_name=${{ steps.myoutputs.outputs.branchName }} \
                           initial_members_file=${{ steps.myoutputs.outputs.initialMembersFilePath }} \
                           initial_balances_file=${{ steps.myoutputs.outputs.initialBalancesFilePath }}"
@@ -155,14 +164,12 @@ jobs:
 
       - name: Print output URL's
         run: |
-          echo -e "Pioneer URL: https://${{ steps.deploy_stack.outputs.DomainName }}"
           echo -e "RPC: wss://${{ steps.deploy_stack.outputs.RPCPublicIp }}.nip.io/ws-rpc"
 
       - name: Delete CloudFormation Stack if any step failed
-        if: failure()
+        # Skip only if stack already existed or all steps passed succesfully
+        if: ( failure() || cancelled() ) && steps.stack_exists.outcome != 'failure'
         run: |
-          echo "Empty the bucket"
-          aws s3 rm s3://${{ steps.deploy_stack.outputs.S3BucketName }} --recursive || echo "No bucket"
           echo "Deleting ${{ env.STACK_NAME }} stack"
           aws cloudformation delete-stack --stack-name ${{ env.STACK_NAME }}
           echo "Waiting for ${{ env.STACK_NAME }} to be deleted..."

devops/aws/chain-spec-pioneer.yml

@@ -1,37 +0,0 @@
----
-# Configure chain spec, start joystream-node service on the servers and build Pioneer
-
-- name: Create and copy the chain-spec file
-  hosts: all
-
-  tasks:
-    - name: Generate chain-spec file and data keys either on localhost or admin server
-      include_role:
-        name: common
-        tasks_from: chain-spec-node-keys
-      vars:
-        local_or_admin: "{{ groups['build'][0] if run_on_admin_server|bool else 'localhost' }}"
-        admin_code_dir: "{{ remote_code_path if run_on_admin_server|bool else local_dir }}"
-
-- name: Copy secret, auth and start joystream-node service for validators
-  hosts: validators
-  gather_facts: no
-
-  roles:
-    - validators
-
-- name: Configure RPC service and start it
-  hosts: rpc
-  gather_facts: no
-
-  roles:
-    - rpc
-
-- name: Build Pioneer and copy artifacts to S3
-  hosts: build
-  gather_facts: no
-
-  tasks:
-    - include_role:
-        name: admin
-        tasks_from: deploy-pioneer

devops/aws/cloudformation/infrastructure.yml

@@ -2,7 +2,7 @@
 # This is comprised of:
 #   - N validators
 #   - One RPC node
-#   - s3 bucket with a build of Pionner
+#   - One Build instance
 
 AWSTemplateFormatVersion: 2010-09-09
 
@@ -208,44 +208,6 @@ Resources:
       Timeout: '600'
       Count: !Ref NumberOfValidators
 
-  S3Bucket:
-    Type: AWS::S3::Bucket
-    Properties:
-      AccessControl: PublicRead
-      WebsiteConfiguration:
-        IndexDocument: index.html
-
-  BucketPolicy:
-    Type: AWS::S3::BucketPolicy
-    Properties:
-      PolicyDocument:
-        Id: PublicPolicy
-        Version: 2012-10-17
-        Statement:
-          - Sid: PublicReadForGetBucketObjects
-            Effect: Allow
-            Principal: '*'
-            Action: 's3:GetObject'
-            Resource: !Sub "arn:aws:s3:::${S3Bucket}/*"
-      Bucket: !Ref S3Bucket
-
-  CloudFrontDistribution:
-    Type: AWS::CloudFront::Distribution
-    Properties:
-      DistributionConfig:
-        Origins:
-        - DomainName: !Select [1, !Split ["//", !GetAtt S3Bucket.WebsiteURL]]
-          Id: pioneer-origin-s3
-          CustomOriginConfig:
-            OriginProtocolPolicy: http-only
-        DefaultCacheBehavior:
-          TargetOriginId: pioneer-origin-s3
-          ViewerProtocolPolicy: redirect-to-https
-          ForwardedValues:
-            QueryString: true
-        Enabled: true
-        HttpVersion: http2
-
 Outputs:
   AutoScalingId:
     Description: The Auto Scaling ID
@@ -264,15 +226,3 @@ Outputs:
     Value:  !Sub "${BuildInstance.PublicIp}"
     Export:
       Name: !Sub "${AWS::StackName}BuildPublicIp"
-
-  S3BucketName:
-    Value: !Ref S3Bucket
-    Description: Name of S3 bucket to hold website content
-    Export:
-      Name: !Sub "${AWS::StackName}S3BucketName"
-
-  DomainName:
-    Description: CloudFront Domain Name
-    Value:  !Sub "${CloudFrontDistribution.DomainName}"
-    Export:
-      Name: !Sub "${AWS::StackName}DomainName"

devops/aws/configure-network.yml

@@ -0,0 +1,27 @@
+---
+# Configure chain spec, start joystream-node and other services on the servers
+
+- name: Create and copy the chain-spec file
+  hosts: all
+
+  tasks:
+    - name: Generate chain-spec file and data keys on build server
+      include_role:
+        name: common
+        tasks_from: chain-spec-node-keys
+      vars:
+        build_instance: "{{ groups['build'][0] }}"
+
+- name: Copy secret, auth and start joystream-node service for validators
+  hosts: validators
+  gather_facts: no
+
+  roles:
+    - validators
+
+- name: Configure RPC service and start it
+  hosts: rpc
+  gather_facts: no
+
+  roles:
+    - rpc

devops/aws/deploy-infra.sh

@@ -63,10 +63,6 @@ if [ $? -eq 0 ]; then
 
   BUILD_SERVER=$(get_aws_export $NEW_STACK_NAME "BuildPublicIp")
 
-  BUCKET_NAME=$(get_aws_export $NEW_STACK_NAME "S3BucketName")
-
-  DOMAIN_NAME=$(get_aws_export $NEW_STACK_NAME "DomainName")
-
   mkdir -p $DATA_PATH
 
   echo -e "[build]\n$BUILD_SERVER\n\n[validators]\n$VALIDATORS\n[rpc]\n$RPC_NODES" > $INVENTORY_PATH
@@ -81,11 +77,10 @@ if [ $? -eq 0 ]; then
     ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH setup-admin.yml
   fi
 
-  echo -e "\n\n=========== Configure and start new validators, rpc node and pioneer ==========="
-  ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH chain-spec-pioneer.yml \
+  echo -e "\n\n=========== Configure and start new validators and rpc node ==========="
+  ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH configure-network.yml \
     --extra-vars "local_dir=$LOCAL_CODE_PATH network_suffix=$NETWORK_SUFFIX
-                  data_path=$DATA_PATH bucket_name=$BUCKET_NAME number_of_validators=$NUMBER_OF_VALIDATORS
+                  data_path=$DATA_PATH number_of_validators=$NUMBER_OF_VALIDATORS
                   deployment_type=$DEPLOYMENT_TYPE initial_balances_file=$INITIAL_BALANCES_PATH initial_members_file=$INITIAL_MEMBERS_PATH"
 
-  echo -e "\n\n Pioneer URL: https://$DOMAIN_NAME"
 fi

devops/aws/destroy-infra.sh

@@ -13,14 +13,8 @@ else
   source $1
 fi
 
-BUCKET_NAME=$(get_aws_export $NEW_STACK_NAME "S3BucketName")
-
 # Delete the CloudFormation stack
 
-echo -e "\n\n=========== Emptying bucket $BUCKET_NAME ==========="
-
-aws s3 rm s3://$BUCKET_NAME --recursive --profile $CLI_PROFILE || echo "No bucket"
-
 echo -e "\n\n=========== Deleting stack $NEW_STACK_NAME ==========="
 
 aws cloudformation delete-stack --stack-name $NEW_STACK_NAME --profile $CLI_PROFILE

devops/aws/group_vars/all

@@ -15,8 +15,5 @@ raw_chain_spec_path: "{{ remote_data_path }}/chainspec-raw.json"
 local_raw_chain_spec_path: "{{ data_path }}/chainspec-raw.json"
 remote_code_path: "/home/ubuntu/joystream"
 remote_chain_spec_path: "{{ remote_code_path }}/chainspec.json"
-run_on_admin_server: true
 build_local_code: false
 number_of_validators: 2
-
-bucket_name: s3-bucket-joystream

devops/aws/roles/admin/tasks/deploy-pioneer.yml

@@ -1,38 +0,0 @@
----
-# Build Pioneer, copy build artifacts and sync to S3
-
-- name: Set ws_rpc for build node
-  set_fact:
-    ws_rpc: "{{ hostvars[groups['rpc'][0]].ws_rpc }}"
-
-- name: Build Pioneer code
-  shell: "WS_URL=wss://{{ ws_rpc }} yarn && yarn workspace @joystream/types build && yarn workspace pioneer build"
-  args:
-    chdir: "{{ remote_code_path }}"
-  async: 3600
-  poll: 0
-  register: build_result
-
-- name: Check on build async task
-  async_status:
-    jid: '{{ build_result.ansible_job_id }}'
-  register: job_result
-  until: job_result.finished
-  # Max number of times to check for status
-  retries: 36
-  # Check for the status every 100s
-  delay: 100
-
-- name: Copying build files to local
-  synchronize:
-    src: "{{ remote_code_path }}/pioneer/packages/apps/build"
-    dest: "{{ data_path }}"
-    mode: pull
-  run_once: true
-
-- name: Run S3 Sync to upload build files to bucket
-  community.aws.s3_sync:
-    bucket: "{{ bucket_name }}"
-    file_root: "{{ data_path }}/build"
-    region: us-east-1
-  delegate_to: localhost

devops/aws/roles/admin/tasks/main.yml

@@ -1,5 +1,5 @@
 ---
-# Configure admin server to be able to create chain-spec file and subkey commands
+# Configure build server to be able to create chain-spec file and subkey commands
 
 - name: Copy bash_profile content
   shell: cat ~/.bash_profile

devops/aws/roles/common/tasks/chain-spec-node-keys.yml

@@ -9,32 +9,32 @@
 - name: Copying initial members file to the server
   copy:
     src: '{{ initial_members_file }}'
-    dest: '{{ admin_code_dir }}/initial-members.json'
+    dest: '{{ remote_code_path }}/initial-members.json'
   when: initial_members_file is defined and initial_members_file|length > 0
   run_once: true
 
 - name: Copying initial balances file to the server
   copy:
     src: '{{ initial_balances_file }}'
-    dest: '{{ admin_code_dir }}/initial-balances.json'
+    dest: '{{ remote_code_path }}/initial-balances.json'
   when: initial_balances_file is defined and initial_balances_file|length > 0
   run_once: true
 
 - name: Run chain-spec-builder to generate chainspec.json file (with initial data)
   shell: >
-    {{ admin_code_dir }}/target/release/chain-spec-builder generate -a {{ number_of_validators }}
+    {{ remote_code_path }}/target/release/chain-spec-builder generate -a {{ number_of_validators }}
     --chain-spec-path {{ chain_spec_path }}
     --endowed 1 --keystore-path {{ remote_data_path }}
     {% if deployment_type is defined and deployment_type|length > 0 %}--deployment {{ deployment_type }}{% endif %}
-    {% if initial_members_file is defined and initial_members_file|length > 0 %}--initial-balances-path {{ admin_code_dir }}/initial-balances.json{% endif %}
-    {% if initial_balances_file is defined and initial_balances_file|length > 0 %}--initial-members-path {{ admin_code_dir }}/initial-members.json{% endif %}
+    {% if initial_members_file is defined and initial_members_file|length > 0 %}--initial-balances-path {{ remote_code_path }}/initial-balances.json{% endif %}
+    {% if initial_balances_file is defined and initial_balances_file|length > 0 %}--initial-members-path {{ remote_code_path }}/initial-members.json{% endif %}
   register: chain_spec_output
-  delegate_to: '{{ local_or_admin }}'
+  delegate_to: '{{ build_instance }}'
   run_once: true
 
 - name: Run subkey to generate node keys
   shell: subkey generate-node-key
-  delegate_to: '{{ local_or_admin }}'
+  delegate_to: '{{ build_instance }}'
   register: subkey_output
 
 - name: Print to stdout
@@ -51,7 +51,7 @@
   copy:
     content: '{{ chain_spec_output.stdout | regex_replace("\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[mGK]", "") }}'
     dest: '{{ remote_data_path }}/chain_spec_output.txt'
-  delegate_to: '{{ local_or_admin }}'
+  delegate_to: '{{ build_instance }}'
   run_once: true
 
 - name: Change chain spec name, id, protocolId
@@ -59,7 +59,7 @@
     chain_spec_path: '{{ chain_spec_path }}'
     prefix: '{{ network_suffix }}'
     all_nodes: '{{ hostvars }}'
-  delegate_to: '{{ local_or_admin }}'
+  delegate_to: '{{ build_instance }}'
   register: result
   run_once: true
 
@@ -69,8 +69,8 @@
   run_once: true
 
 - name: Run build-spec to generate raw chainspec file
-  shell: '{{ admin_code_dir }}/target/release/joystream-node build-spec --chain {{ chain_spec_path }} --raw > {{ raw_chain_spec_path }}'
-  delegate_to: '{{ local_or_admin }}'
+  shell: '{{ remote_code_path }}/target/release/joystream-node build-spec --chain {{ chain_spec_path }} --raw > {{ raw_chain_spec_path }}'
+  delegate_to: '{{ build_instance }}'
   run_once: true
 
 - name: Copying chain spec files to localhost
@@ -79,16 +79,14 @@
     dest: '{{ data_path }}'
     mode: pull
   run_once: true
-  when: run_on_admin_server|bool
 
 - name: Copy joystream-node binary to localhost
   fetch:
-    src: '{{ admin_code_dir }}/target/release/joystream-node'
+    src: '{{ remote_code_path }}/target/release/joystream-node'
     dest: '{{ data_path }}/joystream-node'
     flat: yes
-  delegate_to: '{{ local_or_admin }}'
+  delegate_to: '{{ build_instance }}'
   run_once: true
-  when: run_on_admin_server|bool
 
 - name: Copying raw chain spec file to all servers
   copy:

devops/aws/setup-admin.yml

@@ -6,4 +6,3 @@
 
   roles:
     - role: admin
-      when: run_on_admin_server|bool