Merge branch 'master' into update-giza-aws-config

.github/workflows/deploy-node-network.yml

@@ -0,0 +1,169 @@
+name: Deploy Network
+
+on:
+  workflow_dispatch:
+    inputs:
+      jsonInput:
+        description: 'Json input (refer to .pipelines/deploy-node-network-inputs.json)'
+        required: true
+        default: ''
+
+defaults:
+  run:
+    working-directory: devops/aws
+
+jobs:
+  deploy-node-network:
+    name: Create CloudFormation stack and run Ansible playbook
+    runs-on: ubuntu-latest
+    env:
+      STACK_NAME: ga-deploy-node-network-${{ github.run_number }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set env variables
+        id: myoutputs # set the outputs
+        run: |
+          jsonInput=$(jq -r '.inputs.jsonInput' $GITHUB_EVENT_PATH)
+          export ENCRYPTION_KEY=$(echo $jsonInput | jq -r '.encryptionKey.value')
+          echo ::set-output name=encryptionKey::$ENCRYPTION_KEY
+          echo "::add-mask::$ENCRYPTION_KEY"
+          echo ::set-output name=gitRepo::$(echo $jsonInput | jq -r '.gitRepo.value')
+          echo ::set-output name=branchName::$(echo $jsonInput | jq -r '.branchName.value')
+          echo ::set-output name=keyName::$(echo $jsonInput | jq -r '.keyName.value')
+          echo ::set-output name=numberOfValidators::$(echo $jsonInput | jq -r '.numberOfValidators.value')
+          echo ::set-output name=validatorInstanceType::$(echo $jsonInput | jq -r '.validatorInstanceType.value')
+          echo ::set-output name=buildInstanceType::$(echo $jsonInput | jq -r '.buildInstanceType.value')
+          echo ::set-output name=rpcInstanceType::$(echo $jsonInput | jq -r '.rpcInstanceType.value')
+          echo ::set-output name=ec2AMI::$(echo $jsonInput | jq -r '.ec2AMI.value')
+          echo ::set-output name=networkSuffix::$(echo $jsonInput | jq -r '.networkSuffix.value')
+          echo ::set-output name=deploymentType::$(echo $jsonInput | jq -r '.deploymentType.value')
+          initialBalancesFile=$(echo $jsonInput | jq -r '.initialBalancesFile.value')
+          initialMembersFile=$(echo $jsonInput | jq -r '.initialMembersFile.value')
+          if [ -z "$initialBalancesFile" ]
+          then
+            echo ::set-output name=initialBalancesFilePath::''
+          else
+            wget $initialBalancesFile -O initial-balances.json
+            echo ::set-output name=initialBalancesFilePath::'initial-balances.json'
+          fi
+          if [ -z "$initialMembersFile" ]
+          then
+            echo ::set-output name=initialMembersFilePath::''
+          else
+            wget $initialMembersFile -O initial-members.json
+            echo ::set-output name=initialMembersFilePath::'initial-members.json'
+          fi
+
+      - name: Install Ansible dependencies
+        run: pipx inject ansible-core boto3 botocore
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - name: Deploy to AWS CloudFormation
+        uses: aws-actions/aws-cloudformation-github-deploy@v1
+        id: deploy_stack
+        with:
+          name: ${{ env.STACK_NAME }}
+          template: devops/aws/cloudformation/infrastructure.yml
+          no-fail-on-empty-changeset: '1'
+          parameter-overrides: >-
+            KeyName=${{ steps.myoutputs.outputs.keyName }},
+            ValidatorEC2InstanceType=${{ steps.myoutputs.outputs.validatorInstanceType }},
+            RPCEC2InstanceType=${{ steps.myoutputs.outputs.rpcInstanceType }},
+            BuildEC2InstanceType=${{ steps.myoutputs.outputs.buildInstanceType }},
+            EC2AMI=${{ steps.myoutputs.outputs.ec2AMI }},
+            NumberOfValidators=${{ steps.myoutputs.outputs.numberOfValidators }}
+
+      - name: Prepare inventory for Ansible
+        run: |
+          ASG=${{ steps.deploy_stack.outputs.AutoScalingId }}
+
+          VALIDATORS=""
+
+          INSTANCES=$(aws autoscaling describe-auto-scaling-instances \
+            --query "AutoScalingInstances[?AutoScalingGroupName=='${ASG}'].InstanceId" --output text);
+
+          for ID in $INSTANCES
+          do
+            IP=$(aws ec2 describe-instances --instance-ids $ID --query "Reservations[].Instances[].PublicIpAddress" --output text)
+            VALIDATORS+="$IP\n"
+          done
+
+          echo -e "[build]\n${{ steps.deploy_stack.outputs.BuildPublicIp }}\n" >> inventory
+          echo -e "[validators]\n$VALIDATORS" >> inventory
+          echo -e "[rpc]\n${{ steps.deploy_stack.outputs.RPCPublicIp }}" >> inventory
+          cat inventory
+
+      - name: Run playbook to compile joystream-node on build server
+        uses: dawidd6/action-ansible-playbook@v2
+        if: steps.myoutputs.outputs.ec2AMI == ''
+        with:
+          playbook: build-code.yml
+          directory: devops/aws
+          requirements: requirements.yml
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          options: |
+            --inventory inventory
+            --extra-vars "branch_name=${{ steps.myoutputs.outputs.branchName }} \
+                          git_repo=${{ steps.myoutputs.outputs.gitRepo }} data_path=mydata"
+
+      - name: Run playbook to install additional utils on build server
+        uses: dawidd6/action-ansible-playbook@v2
+        if: steps.myoutputs.outputs.ec2AMI == ''
+        with:
+          playbook: setup-admin.yml
+          directory: devops/aws
+          requirements: requirements.yml
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          options: |
+            --inventory inventory
+
+      - name: Run playbook to configure chain-spec and pioneer
+        uses: dawidd6/action-ansible-playbook@v2
+        with:
+          playbook: chain-spec-pioneer.yml
+          directory: devops/aws
+          requirements: requirements.yml
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          options: |
+            --inventory inventory
+            --extra-vars "network_suffix=${{ steps.myoutputs.outputs.networkSuffix }} data_path=mydata \
+                          number_of_validators=${{ steps.myoutputs.outputs.numberOfValidators }} \
+                          git_repo=${{ steps.myoutputs.outputs.gitRepo }} \
+                          deployment_type=${{ steps.myoutputs.outputs.deploymentType }} \
+                          bucket_name=${{ steps.deploy_stack.outputs.S3BucketName }} \
+                          branch_name=${{ steps.myoutputs.outputs.branchName }} \
+                          initial_members_file=${{ steps.myoutputs.outputs.initialMembersFilePath }} \
+                          initial_balances_file=${{ steps.myoutputs.outputs.initialBalancesFilePath }}"
+
+      - name: Encrpyt the artifacts
+        run: |
+          7z a -p${{ steps.myoutputs.outputs.encryptionKey }} chain-data.7z mydata/*
+
+      - name: Save the output as an artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: data-chainspec-auth
+          path: devops/aws/chain-data.7z
+
+      - name: Print output URL's
+        run: |
+          echo -e "Pioneer URL: https://${{ steps.deploy_stack.outputs.DomainName }}"
+          echo -e "RPC: wss://${{ steps.deploy_stack.outputs.RPCPublicIp }}.nip.io/ws-rpc"
+
+      - name: Delete CloudFormation Stack if any step failed
+        if: failure()
+        run: |
+          echo "Empty the bucket"
+          aws s3 rm s3://${{ steps.deploy_stack.outputs.S3BucketName }} --recursive || echo "No bucket"
+          echo "Deleting ${{ env.STACK_NAME }} stack"
+          aws cloudformation delete-stack --stack-name ${{ env.STACK_NAME }}
+          echo "Waiting for ${{ env.STACK_NAME }} to be deleted..."
+          aws cloudformation wait stack-delete-complete --stack-name ${{ env.STACK_NAME }}

.pipelines/deploy-node-network-inputs.json

@@ -0,0 +1,54 @@
+{
+  "gitRepo": {
+    "description": "Code repository",
+    "value": "https://github.com/Joystream/joystream.git"
+  },
+  "branchName": {
+    "description": "Branch to deploy",
+    "value": "master"
+  },
+  "keyName": {
+    "description": "SSH key pair on AWS",
+    "value": "joystream-github-action-key"
+  },
+  "numberOfValidators": {
+    "description": "Number of validators to deploy",
+    "value": "2"
+  },
+  "validatorInstanceType": {
+    "description": "AWS EC2 instance type for Validators (t2.micro, t2.large)",
+    "value": "t2.micro"
+  },
+  "buildInstanceType": {
+    "description": "AWS EC2 instance type for Build Instance (t2.micro, t2.large)",
+    "value": "t2.large"
+  },
+  "rpcInstanceType": {
+    "description": "AWS EC2 instance type for RPC (t2.micro, t2.large)",
+    "value": "t2.micro"
+  },
+  "ec2AMI": {
+    "description": "Pre-built AMI ID (ami-0090c9c49dc765fbb)",
+    "value": "ami-0090c9c49dc765fbb"
+  },
+  "networkSuffix": {
+    "description": "Network suffix that will be added to the network name",
+    "value": "8129"
+  },
+  "initialBalancesFile": {
+    "description": "HTTP Link to the Initial Balances file",
+    "value": ""
+  },
+  "initialMembersFile": {
+    "description": "HTTP Link to the Initial Members file",
+    "value": ""
+  },
+  "deploymentType": {
+    "description": "Chain deployment type (live, dev etc.)",
+    "value": "live"
+  },
+  "encryptionKey": {
+    "description": "Password to encrypt the artifacts",
+    "value": "password"
+  }
+}

README.md

@@ -46,13 +46,13 @@ yarn start
 **Server Applications - infrastructure**
 
 - [Storage Node](./storage-node) - Media Storage Infrastructure
-- [Query Node](https://github.com/Joystream/joystream/tree/query_node/query-node) - _under development_
-- [Discovery Node](https://github.com/Joystream/joystream/tree/init_discovery_node/discovery_node) - _under development_
+- [Distributor Node](./distributor-node) - Bandwith Infrastructure
+- [Query Node](./query-node) - GraphQL query infrastructure powering front end applications
 
 **Front-end Applications**
 
 - [Pioneer](./pioneer) - Main UI for accessing all Joystream features
-- [Atlas](https://github.com/Joystream/joystream/tree/init_atlas/atlas) - Media Player- _under development_
+- [Atlas](https://github.com/Joystream/atlas) - Media Player
 
 **Tools and CLI**
 

devops/aws/cloudformation/infrastructure.yml

@@ -128,6 +128,23 @@ Resources:
             # Install the updates
             apt-get upgrade -y
 
+            apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release
+
+            curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+
+            echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+            apt-get update -y
+
+            apt-get install -y docker-ce docker-ce-cli containerd.io
+
+            usermod -aG docker ubuntu
+
+            # Update docker-compose to 1.28+
+            curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+            chmod +x /usr/local/bin/docker-compose
+            ln -sf /usr/local/bin/docker-compose /usr/bin/docker-compose
+
             # Get latest cfn scripts and install them;
             apt-get install -y python3-setuptools
             mkdir -p /opt/aws/bin

devops/aws/roles/admin/tasks/deploy-pioneer.yml

@@ -9,10 +9,19 @@
   shell: "WS_URL=wss://{{ ws_rpc }} yarn && yarn workspace @joystream/types build && yarn workspace pioneer build"
   args:
     chdir: "{{ remote_code_path }}"
-  retries: 3
-  delay: 5
-  register: result
-  until: result is not failed
+  async: 3600
+  poll: 0
+  register: build_result
+
+- name: Check on build async task
+  async_status:
+    jid: '{{ build_result.ansible_job_id }}'
+  register: job_result
+  until: job_result.finished
+  # Max number of times to check for status
+  retries: 36
+  # Check for the status every 100s
+  delay: 100
 
 - name: Copying build files to local
   synchronize:
@@ -25,6 +34,5 @@
   community.aws.s3_sync:
     bucket: "{{ bucket_name }}"
     file_root: "{{ data_path }}/build"
-    profile: joystream-user
     region: us-east-1
   delegate_to: localhost

devops/aws/roles/rpc/tasks/main.yml

@@ -22,10 +22,48 @@
     enabled: yes
   become: yes
 
+- name: Build required packages
+  shell: yarn build:packages
+  args:
+    chdir: '{{ remote_code_path }}'
+  # Run in async fashion for max duration of 1 hr
+  async: 3600
+  poll: 0
+  register: build_result
+
+- name: Check on build async task
+  async_status:
+    jid: '{{ build_result.ansible_job_id }}'
+  register: job_result
+  until: job_result.finished
+  # Max number of times to check for status
+  retries: 36
+  # Check for the status every 100s
+  delay: 100
+
+- name: Run query node containers
+  command: yarn workspace query-node-root start:dev
+  args:
+    chdir: '{{ remote_code_path }}'
+  async: 1800
+  poll: 0
+  register: compose_result
+
+- name: Check on yarn start task
+  async_status:
+    jid: '{{ compose_result.ansible_job_id }}'
+  register: job_result
+  until: job_result.finished
+  # Max number of times to check for status
+  retries: 18
+  # Check for the status every 100s
+  delay: 100
+
 - name: Set websocket and http endpoint variables
   set_fact:
     ws_rpc: "{{ inventory_hostname }}.nip.io/ws-rpc"
     http_rpc: "{{ inventory_hostname }}.nip.io/http-rpc"
+    nip_domain: "{{ inventory_hostname }}.nip.io"
   run_once: yes
 
 - name: Install and configure Caddy

devops/aws/roles/rpc/templates/Caddyfile.j2

@@ -5,3 +5,17 @@
 {{ http_rpc }} {
     reverse_proxy localhost:9933
 }
+
+{{ nip_domain }}/@apollographql/* {
+    reverse_proxy localhost:8081
+}
+
+{{ nip_domain }}/query-node/server* {
+    uri strip_prefix /query-node/server
+    reverse_proxy localhost:8081
+}
+
+{{ nip_domain }}/query-node/indexer* {
+    uri strip_prefix /query-node/indexer
+    reverse_proxy localhost:4000
+}