Merge branch 'giza-storage-playground-config' into deploy-giza-playground

docker-compose.yml

@@ -14,10 +14,10 @@ services:
     container_name: joystream-node
     volumes:
       - chain-data:/data
-    command: --dev --alice --validator --unsafe-ws-external --unsafe-rpc-external --rpc-methods Unsafe --rpc-cors=all --log runtime --base-path /data
+    command: --dev --alice --validator --unsafe-ws-external --unsafe-rpc-external --rpc-methods Safe --rpc-cors=all --log runtime --base-path /data
     ports:
-      - '127.0.0.1:9944:9944'
+      - 9944:9944
-      - '127.0.0.1:9933:9933'
+      - 9933:9933
 
   colossus-1:
     image: node:14
@@ -31,7 +31,7 @@ services:
         target: /joystream
     working_dir: /joystream/storage-node-v2
     ports:
-      - '127.0.0.1:3333:3333'
+      - 3333:3333
     env_file:
       # relative to working directory where docker-compose was run from
       - .env
@@ -59,7 +59,7 @@ services:
     # let the working_dir be the distributor node to pickup the config.yml file
     working_dir: /joystream/distributor-node
     ports:
-      - 127.0.0.1:3334:3334
+      - 3334:3334
     env_file:
       # relative to working directory where docker-compose was run from
       - .env
@@ -94,7 +94,7 @@ services:
         target: /joystream
     working_dir: /joystream/storage-node-v2
     ports:
-      - '127.0.0.1:3335:3333'
+      - 3335:3333
     env_file:
       # relative to working directory where docker-compose was run from
       - .env
@@ -122,7 +122,7 @@ services:
     # let the working_dir be the distributor node to pickup the config.yml file
     working_dir: /joystream/distributor-node
     ports:
-      - 127.0.0.1:3336:3334
+      - 3336:3334
     env_file:
       # relative to working directory where docker-compose was run from
       - .env

start.sh

@@ -31,18 +31,23 @@ export SKIP_MOCK_CONTENT=true
 ## Set sudo as the membership screening authority
 yarn workspace api-scripts set-sudo-as-screening-auth
 
+## Member faucet
 docker-compose up -d faucet
 
+## Storage Infrastructure Configuration
+./storage-playground-config.sh
+
 ## Query Node Infrastructure
 ./query-node/start.sh
 
-## Storage Infrastructure
+## Storage Infrastructure Nodes
 docker-compose up -d colossus-1
 docker-compose up -d distributor-1
 
 ## Pioneer UI
 docker-compose up -d pioneer
 
+## Orion
 docker-compose up -d orion
 
 if [ "${PERSIST}" == true ]

storage-playground-config.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -e
+
+SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+cd $SCRIPT_PATH
+
+# Basic script to be run to configure dev chain storage infrastructure.
+# Assumes one storage lead and distributor lead are already hired
+# and their identity information are stored in .env
+
+# Load the worker ids and SURIs as local variables only
+. .env
+
+HOST_IP=$(tests/network-tests/get-host-ip.sh)
+
+## Colossus 1
+CLI=storage-node-v2/bin/run
+
+${CLI} leader:update-bag-limit -l 10 --accountUri ${COLOSSUS_1_ACCOUNT_URI}
+${CLI} leader:update-voucher-limits -o 10000 -s 1000000000000 --accountUri ${COLOSSUS_1_ACCOUNT_URI}
+BUCKET_ID=`${CLI} leader:create-bucket -i=${COLOSSUS_1_WORKER_ID} -a -n=10000 -s=1000000000000  --accountUri ${COLOSSUS_1_ACCOUNT_URI}`
+${CLI} operator:accept-invitation -w=${COLOSSUS_1_WORKER_ID} -i=${BUCKET_ID} --accountUri ${COLOSSUS_1_ACCOUNT_URI}
+${CLI} leader:update-dynamic-bag-policy -n 1 -t Channel --accountUri ${COLOSSUS_1_ACCOUNT_URI}
+${CLI} leader:update-data-fee -f 10 --accountUri ${COLOSSUS_1_ACCOUNT_URI} # Optionally - set some data fee per megabyte
+
+# The node uri should be an accessible endpoint from within a container as well as the host machine.
+# In production it would most likely be the reverse proxy endpoint. If not specified we
+# set it to the host machine address.
+COLOSSUS_1_NODE_URI=${COLOSSUS_1_NODE_URI:="http://${HOST_IP}:3333"}
+${CLI} operator:set-metadata -w=${COLOSSUS_1_WORKER_ID} -i=${BUCKET_ID} -e="${COLOSSUS_1_NODE_URI}" --accountUri ${COLOSSUS_1_ACCOUNT_URI}
+
+echo "Colossus 1 BUCKET_ID=${BUCKET_ID}"
+
+## Distributor 1
+export AUTO_CONFIRM=true
+export CONFIG_PATH=$(pwd)/distributor-node/config.yml
+export JOYSTREAM_DISTRIBUTOR__KEYS="[{\"suri\":\"${DISTRIBUTOR_1_ACCOUNT_URI}\"}]"
+CLI=distributor-node/bin/run
+
+${CLI} leader:set-buckets-per-bag-limit -l 10
+FAMILY_ID=`${CLI} leader:create-bucket-family`
+BUCKET_ID=`${CLI} leader:create-bucket -f ${FAMILY_ID} -a yes`
+${CLI} leader:update-bucket-mode -f ${FAMILY_ID} -B ${BUCKET_ID} --mode on
+${CLI} leader:update-dynamic-bag-policy -t Channel -p ${FAMILY_ID}:5
+${CLI} leader:invite-bucket-operator -f ${FAMILY_ID} -B ${BUCKET_ID} -w ${DISTRIBUTOR_1_WORKER_ID}
+${CLI} operator:accept-invitation -f ${FAMILY_ID} -B ${BUCKET_ID} -w ${DISTRIBUTOR_1_WORKER_ID}
+
+# The node uri should be an accessible endpoint from within a container as well as the host machine.
+# In production it would most likely be the reverse proxy endpoint. If not specified we
+# set it to the host machine address.
+DISTRIBUTOR_1_NODE_URI=${DISTRIBUTOR_1_NODE_URI:="http://${HOST_IP}:3334"}
+${CLI} operator:set-metadata -f ${FAMILY_ID} -B ${BUCKET_ID} -w ${DISTRIBUTOR_1_WORKER_ID} -e="${DISTRIBUTOR_1_NODE_URI}"
+
+echo "Distributor 1 FAMILY_ID=${FAMILY_ID}"
+echo "Distributor 1 BUCKET_ID=${BUCKET_ID}"

tests/network-tests/get-host-ip.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -e
+
+if [[ "$OSTYPE" == "linux-gnu" ]]; then
+    # We could get the specific network address for the docker network with
+    #   docker network inspect --format='{{range .IPAM.Config}}{{.Gateway}}{{end}}' joystream_default
+    ip route | awk '/default/ {print $3}'
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+    # try en0 if not set then en1 (Wired or Wireless)
+    ipconfig getifaddr en0 || ipconfig getifaddr en1
+fi
+
+# Some alternative approaches (cross platform)
+# ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1'