Add CloudFront, S3 bucket, Pioneer build automation

devops/infrastructure/chain-spec-configuration.yml

@@ -3,7 +3,6 @@
 
 - name: Create and copy the chain-spec file
   hosts: all
-  gather_facts: no
 
   tasks:
     - name: Generate chain-spec file and data keys either on localhost or admin server
@@ -16,12 +15,23 @@
 
 - name: Copy secret, auth and start joystream-node service for validators
   hosts: validators
+  gather_facts: no
 
   roles:
     - validators
 
 - name: Configure RPC service and start it
   hosts: rpc
+  gather_facts: no
 
   roles:
     - rpc
+
+- name: Build Pioneer and copy artifacts to S3
+  hosts: build
+  gather_facts: no
+
+  tasks:
+    - include_role:
+        name: admin
+        tasks_from: deploy-pioneer

devops/infrastructure/deploy-infra.sh

@@ -29,6 +29,14 @@ if [ ! -f "$KEY_PATH" ]; then
     exit 1
 fi
 
+get_aws_export () {
+  RESULT=$(aws cloudformation list-exports \
+    --profile $CLI_PROFILE \
+    --query "Exports[?starts_with(Name,'${NEW_STACK_NAME}$1')].Value" \
+    --output text | sed 's/\t\t*/\n/g')
+  echo -e $RESULT | tr " " "\n"
+}
+
 # Deploy the CloudFormation template
 echo -e "\n\n=========== Deploying main.yml ==========="
 aws cloudformation deploy \
@@ -48,20 +56,15 @@ if [ $? -eq 0 ]; then
   # Install additional Ansible roles from requirements
   ansible-galaxy install -r requirements.yml
 
-  VALIDATORS=$(aws cloudformation list-exports \
-    --profile $CLI_PROFILE \
-    --query "Exports[?starts_with(Name,'${NEW_STACK_NAME}PublicIp')].Value" \
-    --output text | sed 's/\t\t*/\n/g')
+  VALIDATORS=$(get_aws_export "PublicIp")
 
-  RPC_NODES=$(aws cloudformation list-exports \
-    --profile $CLI_PROFILE \
-    --query "Exports[?starts_with(Name,'${NEW_STACK_NAME}RPCPublicIp')].Value" \
-    --output text | sed 's/\t\t*/\n/g')
+  RPC_NODES=$(get_aws_export "RPCPublicIp")
 
-  BUILD_SERVER=$(aws cloudformation list-exports \
-    --profile $CLI_PROFILE \
-    --query "Exports[?starts_with(Name,'${NEW_STACK_NAME}BuildPublicIp')].Value" \
-    --output text | sed 's/\t\t*/\n/g')
+  BUILD_SERVER=$(get_aws_export "BuildPublicIp")
+
+  BUCKET_NAME=$(get_aws_export "S3BucketName")
+
+  DOMAIN_NAME=$(get_aws_export "DomainName")
 
   mkdir -p $DATA_PATH
 
@@ -77,7 +80,9 @@ if [ $? -eq 0 ]; then
   ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH setup-admin.yml \
     --extra-vars "local_dir=$LOCAL_CODE_PATH build_local_code=$BUILD_LOCAL_CODE"
 
-  echo -e "\n\n=========== Configuring the chain spec file ==========="
+  echo -e "\n\n=========== Configuring the chain spec file and Pioneer app ==========="
   ansible-playbook -i $INVENTORY_PATH --private-key $KEY_PATH chain-spec-configuration.yml \
-    --extra-vars "local_dir=$LOCAL_CODE_PATH network_suffix=$NETWORK_SUFFIX data_path=data-$NEW_STACK_NAME"
+    --extra-vars "local_dir=$LOCAL_CODE_PATH network_suffix=$NETWORK_SUFFIX data_path=data-$NEW_STACK_NAME bucket_name=$BUCKET_NAME"
+
+  echo -e "\n\n Pioneer URL: https://$DOMAIN_NAME"
 fi

devops/infrastructure/group_vars/all

@@ -16,3 +16,5 @@ remote_code_path: "/home/ubuntu/joystream"
 remote_chain_spec_path: "{{ remote_code_path }}/chainspec.json"
 run_on_admin_server: true
 build_local_code: false
+
+bucket_name: s3-bucket-joystream

devops/infrastructure/main.yml

@@ -161,6 +161,44 @@ Resources:
       Timeout: '600'
       Count: 4
 
+  S3Bucket:
+    Type: AWS::S3::Bucket
+    Properties:
+      AccessControl: PublicRead
+      WebsiteConfiguration:
+        IndexDocument: index.html
+
+  BucketPolicy:
+    Type: AWS::S3::BucketPolicy
+    Properties:
+      PolicyDocument:
+        Id: PublicPolicy
+        Version: 2012-10-17
+        Statement:
+          - Sid: PublicReadForGetBucketObjects
+            Effect: Allow
+            Principal: '*'
+            Action: 's3:GetObject'
+            Resource: !Sub "arn:aws:s3:::${S3Bucket}/*"
+      Bucket: !Ref S3Bucket
+
+  CloudFrontDistribution:
+    Type: AWS::CloudFront::Distribution
+    Properties:
+      DistributionConfig:
+        Origins:
+        - DomainName: !Select [1, !Split ["//", !GetAtt S3Bucket.WebsiteURL]]
+          Id: pioneer-origin-s3
+          CustomOriginConfig:
+            OriginProtocolPolicy: http-only
+        DefaultCacheBehavior:
+          TargetOriginId: pioneer-origin-s3
+          ViewerProtocolPolicy: redirect-to-https
+          ForwardedValues:
+            QueryString: true
+        Enabled: true
+        HttpVersion: http2
+
 Outputs:
   PublicIp:
     Description: The DNS name for the created instance
@@ -185,3 +223,15 @@ Outputs:
     Value:  !Sub "${BuildInstance.PublicIp}"
     Export:
       Name: !Sub "${AWS::StackName}BuildPublicIp"
+
+  S3BucketName:
+    Value: !Ref S3Bucket
+    Description: Name of S3 bucket to hold website content
+    Export:
+      Name: !Sub "${AWS::StackName}S3BucketName"
+
+  DomainName:
+    Description: CloudFront Domain Name
+    Value:  !Sub "${CloudFrontDistribution.DomainName}"
+    Export:
+      Name: !Sub "${AWS::StackName}DomainName"

devops/infrastructure/requirements.yml

@@ -1,3 +1,5 @@
 ---
 roles:
 - caddy_ansible.caddy_ansible
+collections:
+- community.aws

devops/infrastructure/roles/admin/tasks/deploy-pioneer.yml

@@ -0,0 +1,26 @@
+---
+# Build Pioneer, copy build artifacts and sync to S3
+
+- name: Set ws_rpc for build node
+  set_fact:
+    ws_rpc: "{{ hostvars[groups['rpc'][0]].ws_rpc }}"
+
+- name: Build Pioneer code
+  shell: "WS_URL=wss://{{ ws_rpc }} yarn && yarn workspace @joystream/types build && yarn workspace pioneer build"
+  args:
+    chdir: "{{ remote_code_path }}"
+
+- name: Copying build files to local
+  synchronize:
+    src: "{{ remote_code_path }}/pioneer/packages/apps/build"
+    dest: "{{ data_path }}"
+    mode: pull
+  run_once: true
+
+- name: Run S3 Sync to upload build files to bucket
+  community.aws.s3_sync:
+    bucket: "{{ bucket_name }}"
+    file_root: "{{ data_path }}/build"
+    profile: joystream-user
+    region: us-east-1
+  delegate_to: localhost

devops/infrastructure/roles/rpc/tasks/main.yml

@@ -21,6 +21,12 @@
     state: started
   become: yes
 
+- name: Set websocket and http endpoint variables
+  set_fact:
+    ws_rpc: "{{ inventory_hostname }}.nip.io/ws-rpc"
+    http_rpc: "{{ inventory_hostname }}.nip.io/http-rpc"
+  run_once: yes
+
 - name: Install and configure Caddy
   include_role:
     name: caddy_ansible.caddy_ansible
@@ -29,9 +35,8 @@
   vars:
     caddy_config: "{{ lookup('template', 'templates/Caddyfile.j2') }}"
     caddy_systemd_capabilities_enabled: true
-    ws_rpc: "{{ inventory_hostname }}.nip.io/ws-rpc"
-    http_rpc: "{{ inventory_hostname }}.nip.io/http-rpc"
+    caddy_update: false
 
 - name: Print RPC node DNS
   debug:
-    msg: "RPC Endpoint: wss://{{ inventory_hostname }}.nip.io/ws-rpc"
+    msg: "RPC Endpoint: wss://{{ ws_rpc }}"