Update caddy, delete build instance, add parameters support

.github/workflows/deploy-node-network.yml

@@ -42,6 +42,15 @@ jobs:
           echo ::set-output name=volumeSize::$(echo $jsonInput | jq -r '.volumeSize.value')
           initialBalancesFile=$(echo $jsonInput | jq -r '.initialBalancesFile.value')
           initialMembersFile=$(echo $jsonInput | jq -r '.initialMembersFile.value')
+          proposalParametersInput=$(echo $jsonInput | jq -r '.proposalParameters.value')
+
+          if [ $proposalParametersInput = "{}" ]
+          then
+            echo ::set-output name=proposalParameters::''
+          else
+            echo ::set-output name=proposalParameters::'$proposalParametersInput'
+          fi
+
           if [ -z "$initialBalancesFile" ]
           then
             echo ::set-output name=initialBalancesFilePath::''
@@ -49,6 +58,7 @@ jobs:
             wget $initialBalancesFile -O initial-balances.json
             echo ::set-output name=initialBalancesFilePath::'initial-balances.json'
           fi
+
           if [ -z "$initialMembersFile" ]
           then
             echo ::set-output name=initialMembersFilePath::''
@@ -76,6 +86,7 @@ jobs:
           else
             echo "Stack does not exist"
           fi
+          echo ${{ steps.myoutputs.outputs.proposalParameters }}
 
       - name: Deploy to AWS CloudFormation
         uses: aws-actions/aws-cloudformation-github-deploy@v1
@@ -115,7 +126,7 @@ jobs:
 
       - name: Run playbook to compile joystream-node on build server
         uses: dawidd6/action-ansible-playbook@v2
-        if: steps.myoutputs.outputs.ec2AMI == ''
+        if: steps.myoutputs.outputs.ec2AMI == '' || steps.myoutputs.outputs.proposalParameters != ''
         with:
           playbook: build-code.yml
           directory: devops/aws
@@ -124,7 +135,8 @@ jobs:
           options: |
             --inventory inventory
             --extra-vars "branch_name=${{ steps.myoutputs.outputs.branchName }} \
-                          git_repo=${{ steps.myoutputs.outputs.gitRepo }} data_path=mydata"
+                          git_repo=${{ steps.myoutputs.outputs.gitRepo }} data_path=mydata \
+                          proposal_parameters=${{ steps.myoutputs.outputs.proposalParameters }}"
 
       - name: Run playbook to install additional utils on build server
         uses: dawidd6/action-ansible-playbook@v2
@@ -154,6 +166,12 @@ jobs:
                           initial_members_file=${{ steps.myoutputs.outputs.initialMembersFilePath }} \
                           initial_balances_file=${{ steps.myoutputs.outputs.initialBalancesFilePath }}"
 
+      - name: Terminate Build instance
+        continue-on-error: true
+        run: |
+          echo "Deleting build instance with id ${{ steps.deploy_stack.outputs.BuildInstanceId }}"
+          aws ec2 terminate-instances --instance-ids ${{ steps.deploy_stack.outputs.BuildInstanceId }}
+
       - name: Encrpyt the artifacts
         run: |
           7z a -p${{ steps.myoutputs.outputs.encryptionKey }} chain-data.7z mydata/*

.pipelines/deploy-node-network-inputs.json

@@ -54,5 +54,9 @@
   "encryptionKey": {
     "description": "Password to encrypt the artifacts",
     "value": "password"
+  },
+  "proposalParameters": {
+    "description": "Proposal Parameters",
+    "value": {}
   }
 }

devops/aws/cloudformation/infrastructure.yml

@@ -40,14 +40,13 @@ Parameters:
     Default: 120
 
 Conditions:
-  HasAMIId: !Not [!Equals [!Ref EC2AMI, ""]]
+  HasAMIId: !Not [!Equals [!Ref EC2AMI, '']]
 
 Resources:
   SecurityGroup:
     Type: AWS::EC2::SecurityGroup
     Properties:
-      GroupDescription:
-        !Sub 'Internal Security group for validator nodes ${AWS::StackName}'
+      GroupDescription: !Sub 'Internal Security group for validator nodes ${AWS::StackName}'
       SecurityGroupIngress:
         - IpProtocol: tcp
           FromPort: 30333
@@ -64,8 +63,7 @@ Resources:
   RPCSecurityGroup:
     Type: AWS::EC2::SecurityGroup
     Properties:
-      GroupDescription:
-        !Sub 'Internal Security group for RPC nodes ${AWS::StackName}'
+      GroupDescription: !Sub 'Internal Security group for RPC nodes ${AWS::StackName}'
       SecurityGroupIngress:
         - IpProtocol: tcp
           FromPort: 9933
@@ -165,7 +163,7 @@ Resources:
       DesiredCapacity: !Ref NumberOfValidators
       AvailabilityZones:
         Fn::GetAZs:
-          Ref: "AWS::Region"
+          Ref: 'AWS::Region'
       MixedInstancesPolicy:
         LaunchTemplate:
           LaunchTemplateSpecification:
@@ -176,7 +174,7 @@ Resources:
       Tags:
         - Key: Name
           Value: !Sub '${AWS::StackName}'
-          PropagateAtLaunch: "true"
+          PropagateAtLaunch: 'true'
 
   RPCInstance:
     Type: AWS::EC2::Instance
@@ -215,18 +213,24 @@ Resources:
 Outputs:
   AutoScalingId:
     Description: The Auto Scaling ID
-    Value:  !Ref AutoScalingGroup
+    Value: !Ref AutoScalingGroup
     Export:
-      Name: !Sub "${AWS::StackName}AutoScalingGroup"
+      Name: !Sub '${AWS::StackName}AutoScalingGroup'
 
   RPCPublicIp:
     Description: The DNS name for the created instance
-    Value:  !Sub "${RPCInstance.PublicIp}"
+    Value: !Sub '${RPCInstance.PublicIp}'
     Export:
-      Name: !Sub "${AWS::StackName}RPCPublicIp"
+      Name: !Sub '${AWS::StackName}RPCPublicIp'
 
   BuildPublicIp:
     Description: The DNS name for the created instance
-    Value:  !Sub "${BuildInstance.PublicIp}"
+    Value: !Sub '${BuildInstance.PublicIp}'
     Export:
-      Name: !Sub "${AWS::StackName}BuildPublicIp"
+      Name: !Sub '${AWS::StackName}BuildPublicIp'
+
+  BuildInstanceId:
+    Description: Build instance ID
+    Value: !Ref BuildInstance
+    Export:
+      Name: !Sub '${AWS::StackName}BuildInstanceId'

devops/aws/deploy-infra.sh

@@ -63,6 +63,8 @@ if [ $? -eq 0 ]; then
 
   BUILD_SERVER=$(get_aws_export $NEW_STACK_NAME "BuildPublicIp")
 
+  BUILD_INSTANCE_ID=$(get_aws_export $NEW_STACK_NAME "BuildInstanceId")
+
   mkdir -p $DATA_PATH
 
   echo -e "[build]\n$BUILD_SERVER\n\n[validators]\n$VALIDATORS\n[rpc]\n$RPC_NODES" > $INVENTORY_PATH
@@ -83,4 +85,8 @@ if [ $? -eq 0 ]; then
                   data_path=$DATA_PATH number_of_validators=$NUMBER_OF_VALIDATORS
                   deployment_type=$DEPLOYMENT_TYPE initial_balances_file=$INITIAL_BALANCES_PATH initial_members_file=$INITIAL_MEMBERS_PATH"
 
+  echo -e "\n\n=========== Delete Build instance ==========="
+  DELETE_RESULT=$(aws ec2 terminate-instances --instance-ids $BUILD_INSTANCE_ID --profile $CLI_PROFILE)
+  echo $DELETE_RESULT
+
 fi

devops/aws/roles/common/tasks/run-setup-build.yml

@@ -13,6 +13,8 @@
   shell: . ~/.bash_profile && yarn cargo-build
   args:
     chdir: '{{ remote_code_path }}'
+  environment:
+    ALL_PROPOSALS_PARAMETERS_JSON: '{{ proposal_parameters }}'
   # Run in async fashion for max duration of 1 hr
   async: 3600
   poll: 0

devops/aws/roles/rpc/templates/Caddyfile.j2

@@ -8,6 +8,17 @@
     reverse_proxy localhost:9933
 }
 
+{{ nip_domain }}/colossus-1* {
+    uri strip_prefix /colossus-1
+    reverse_proxy localhost:3333
+}
+
+{{ nip_domain }}/distributor-1* {
+    uri strip_prefix /distributor-1
+    reverse_proxy localhost:3334
+}
+
+# newer versions of graphql-server seems to expect this url also
 {{ nip_domain }}/@apollographql/* {
     reverse_proxy localhost:8081
 }
@@ -17,17 +28,30 @@
     reverse_proxy localhost:8081
 }
 
+wss://{{ nip_domain }}/query-node/server* {
+    uri strip_prefix /query-node/server
+    reverse_proxy localhost:8081
+}
+
 {{ nip_domain }}/query-node/indexer* {
     uri strip_prefix /query-node/indexer
     reverse_proxy localhost:4000
 }
 
+{{ nip_domain }}/orion* {
+    uri strip_prefix /orion
+    reverse_proxy localhost:6116
+}
+
 {{ nip_domain }}/member-faucet* {
     uri strip_prefix /member-faucet
     reverse_proxy localhost:3002
 }
 
 {{ nip_domain }}/network/config.json {
+    header /* {
+      Access-Control-Allow-Origin *
+    }
     root * /home/ubuntu
     rewrite * /config.json
     file_server