Merge pull request #558 from traumschule/giza-testing

KPI 37.OP-1: Giza Testing

working-groups/operations-group/Giza-testing/.env

@@ -0,0 +1,62 @@
+COMPOSE_PROJECT_NAME=joystream
+PROJECT_NAME=query_node
+
+# We will use a single postgres service with multiple databases
+# The env variables below are by default used by all services and should be
+# overriden in local env files
+# DB config
+INDEXER_DB_NAME=query_node_indexer
+DB_NAME=query_node_processor
+DB_USER=postgres
+DB_PASS=postgres
+# This value will not be used by query-node docker containers.
+# When running query-node with docker these services will always use the db service
+DB_HOST=localhost
+DB_PORT=5432
+DEBUG=index-builder:*
+TYPEORM_LOGGING=error
+
+## Indexer options
+# Block height to start indexing from.
+# Note, that if there are already some indexed events, this setting is ignored
+BLOCK_HEIGHT=13876
+
+# Query node GraphQL server port
+GRAPHQL_SERVER_PORT=8081
+
+# Hydra indexer gateway GraphQL server port
+HYDRA_INDEXER_GATEWAY_PORT=4000
+
+# Default GraphQL server host. It is required during "query-node config:dev"
+GRAPHQL_SERVER_HOST=localhost
+
+# Websocket RPC endpoint containers will use.
+JOYSTREAM_NODE_WS=ws://localhost:9944
+
+# Query node which colossus will use
+COLOSSUS_QUERY_NODE_URL=http://graphql-server:4000/graphql
+
+# Query node which distributor will use
+DISTRIBUTOR_QUERY_NODE_URL=http://graphql-server:4000/graphql
+
+# Indexer gateway used by processor. If you don't use the local indexer set this to a remote gateway
+PROCESSOR_INDEXER_GATEWAY=http://hydra-indexer-gateway:4000/graphql
+
+# Colossus services identities
+COLOSSUS_1_WORKER_ID=0
+COLOSSUS_1_WORKER_URI=//testing//worker//Storage//${COLOSSUS_1_WORKER_ID}
+COLOSSUS_1_TRANSACTOR_URI=//Colossus1
+
+COLOSSUS_2_WORKER_ID=1
+COLOSSUS_2_WORKER_URI=//testing//worker//Storage//${COLOSSUS_2_WORKER_ID}
+COLOSSUS_2_TRANSACTOR_URI=//Colossus2
+
+# Distributor node services identities
+DISTRIBUTOR_1_WORKER_ID=0
+DISTRIBUTOR_1_ACCOUNT_URI=//testing//worker//Distribution//${DISTRIBUTOR_1_WORKER_ID}
+
+DISTRIBUTOR_2_WORKER_ID=1
+DISTRIBUTOR_2_ACCOUNT_URI=//testing//worker//Distribution//${DISTRIBUTOR_2_WORKER_ID}
+
+# joystream/node docker image tag
+JOYSTREAM_NODE_TAG=latest

working-groups/operations-group/Giza-testing/Commands.md

@@ -0,0 +1,956 @@
+- [Storage node](#storage)
+- [Distributor node](#distributor)
+
+# [Storage node](https://github.com/Joystream/joystream/tree/giza_staging/storage-node-v2#cli-commands)
+
+commands-storage.txt
+> dev:init dev:multihash dev:sync dev:upload dev:verify-bag-id help leader:cancel-invite leader:create-bucket leader:delete-bucket leader:invite-operator leader:remove-operator leader:set-bucket-limits leader:set-global-uploading-status leader:update-bag leader:update-bag-limit leader:update-blacklist leader:update-bucket-status leader:update-data-fee leader:update-dynamic-bag-policy leader:update-voucher-limits operator:accept-invitation operator:set-metadata server
+
+`for cmd in $(cat commands-storage.txt);do echo $cmd ; ~/joystream/storage-node-v2/bin/run $cmd --help ; done`
+
+## dev:init
+
+Initialize development environment. Sets Alice as storage working group leader.
+
+```
+USAGE
+  $ storage-node dev:init
+
+OPTIONS
+  -h, --help                   show CLI help
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## dev:multihash
+
+Creates a multihash (blake3) for a file.
+
+```
+USAGE
+  $ storage-node dev:multihash
+
+OPTIONS
+  -f, --file=file  (required) Path for a hashing file.
+  -h, --help       show CLI help
+```
+
+## dev:sync
+
+Synchronizes the data - it fixes the differences between local data folder and worker ID obligations from the runtime.
+
+```
+USAGE
+  $ storage-node dev:sync
+
+OPTIONS
+  -d, --uploads=uploads                              (required) Data uploading directory (absolute path).
+  -h, --help                                         show CLI help
+
+  -o, --dataSourceOperatorUrl=dataSourceOperatorUrl  [default: http://localhost:3333] Storage node url
+                                                     base (e.g.: http://some.com:3333) to get data from.
+
+  -p, --syncWorkersNumber=syncWorkersNumber          [default: 20] Sync workers number (max async
+                                                     operations in progress).
+
+  -q, --queryNodeEndpoint=queryNodeEndpoint          [default: http://localhost:8081/graphql] Query node
+                                                     endpoint (e.g.: http://some.com:8081/graphql)
+
+  -t, --syncWorkersTimeout=syncWorkersTimeout        [default: 30] Asset downloading timeout for the
+                                                     syncronization (in minutes).
+
+  -w, --workerId=workerId                            (required) Storage node operator worker ID.
+```
+
+## dev:upload
+
+Upload data object (development mode only).
+
+```
+USAGE
+  $ storage-node dev:upload
+
+OPTIONS
+  -c, --cid=cid                (required) Data object IPFS content ID.
+  -h, --help                   show CLI help
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -s, --size=size              (required) Data object size.
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## dev:verify-bag-id
+The command verifies bag id supported by the storage node. Requires chain connection.
+
+```
+USAGE
+  $ storage-node dev:verify-bag-id
+
+OPTIONS
+  -h, --help
+      show CLI help
+
+  -i, --bagId=bagId
+      (required) Bag ID. Format: {bag_type}:{sub_type}:{id}.
+           - Bag types: 'static', 'dynamic'
+           - Sub types: 'static:council', 'static:wg', 'dynamic:member', 'dynamic:channel'
+           - Id:
+             - absent for 'static:council'
+             - working group name for 'static:wg'
+             - integer for 'dynamic:member' and 'dynamic:channel'
+           Examples:
+           - static:council
+           - static:wg:storage
+           - dynamic:member:4
+```
+
+## help
+
+Joystream storage subsystem.
+
+```
+VERSION
+  storage-node-v2/2.0.0 linux-x64 node-v14.18.2
+
+USAGE
+  $ storage-node [COMMAND]
+
+TOPICS
+  dev       Development mode commands.
+  leader    Storage working group leader commands.
+  operator  Storage provider(operator) commands.
+
+COMMANDS
+  help    display help for storage-node
+  server  Starts the storage node server.
+```
+
+## leader:cancel-invite
+
+Cancel a storage bucket operator invite. Requires storage working group leader permissions.
+
+```
+USAGE
+  $ storage-node leader:cancel-invite
+
+OPTIONS
+  -h, --help                   show CLI help
+  -i, --bucketId=bucketId      (required) Storage bucket ID
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## leader:create-bucket
+
+Create new storage bucket. Requires storage working group leader permissions.
+
+```
+USAGE
+  $ storage-node leader:create-bucket
+
+OPTIONS
+  -a, --allow                  Accepts new bags
+  -h, --help                   show CLI help
+  -i, --invited=invited        Invited storage operator ID (storage WG worker ID)
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+  -n, --number=number          Storage bucket max total objects number
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -s, --size=size              Storage bucket max total objects size
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## leader:delete-bucket
+
+Delete a storage bucket. Requires storage working group leader permissions.
+
+```
+USAGE
+  $ storage-node leader:delete-bucket
+
+OPTIONS
+  -h, --help                   show CLI help
+  -i, --bucketId=bucketId      (required) Storage bucket ID
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## leader:invite-operator
+
+Invite a storage bucket operator. Requires storage working group leader permissions.
+
+```
+USAGE
+  $ storage-node leader:invite-operator
+
+OPTIONS
+  -h, --help                   show CLI help
+  -i, --bucketId=bucketId      (required) Storage bucket ID
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -w, --operatorId=operatorId  (required) Storage bucket operator ID (storage group worker ID)
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## leader:remove-operator
+
+Remove a storage bucket operator. Requires storage working group leader permissions.
+
+```
+USAGE
+  $ storage-node leader:remove-operator
+
+OPTIONS
+  -h, --help                   show CLI help
+  -i, --bucketId=bucketId      (required) Storage bucket ID
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## leader:set-bucket-limits
+
+Set VoucherObjectsSizeLimit and VoucherObjectsNumberLimit for the storage bucket.
+
+```
+USAGE
+  $ storage-node leader:set-bucket-limits
+
+OPTIONS
+  -h, --help                   show CLI help
+  -i, --bucketId=bucketId      (required) Storage bucket ID
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+  -o, --objects=objects        (required) New 'voucher object number limit' value
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -s, --size=size              (required) New 'voucher object size limit' value
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## leader:set-global-uploading-status
+
+Set global uploading block. Requires storage working group leader permissions.
+
+```
+USAGE
+  $ storage-node leader:set-global-uploading-status
+
+OPTIONS
+  -h, --help                   show CLI help
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -s, --set=(on|off)           (required) Sets global uploading block (on/off).
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## leader:update-bag
+
+Add/remove a storage bucket from a bag (adds by default).
+
+```
+USAGE
+  $ storage-node leader:update-bag
+
+OPTIONS
+  -a, --add=add
+      [default: ] ID of a bucket to add to bag
+
+  -h, --help
+      show CLI help
+
+  -i, --bagId=bagId
+      (required) Bag ID. Format: {bag_type}:{sub_type}:{id}.
+           - Bag types: 'static', 'dynamic'
+           - Sub types: 'static:council', 'static:wg', 'dynamic:member', 'dynamic:channel'
+           - Id:
+             - absent for 'static:council'
+             - working group name for 'static:wg'
+             - integer for 'dynamic:member' and 'dynamic:channel'
+           Examples:
+           - static:council
+           - static:wg:storage
+           - dynamic:member:4
+
+  -k, --keyFile=keyFile
+      Key file for the account. Mandatory in non-dev environment.
+
+  -m, --dev
+      Use development mode
+
+  -p, --password=password
+      Key file password (optional). Could be overriden by ACCOUNT_PWD environment variable.
+
+  -r, --remove=remove
+      [default: ] ID of a bucket to remove from bag
+
+  -u, --apiUrl=apiUrl
+      [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev environment.
+
+  -y, --accountUri=accountUri
+      Account URI (optional). Has a priority over the keyFile and password flags. Could be overriden by 
+      ACCOUNT_URI environment variable.
+```
+
+## leader:update-bag-limit
+
+Update StorageBucketsPerBagLimit variable in the Joystream node storage.
+
+```
+USAGE
+  $ storage-node leader:update-bag-limit
+
+OPTIONS
+  -h, --help                   show CLI help
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -l, --limit=limit            (required) New StorageBucketsPerBagLimit value
+  -m, --dev                    Use development mode
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## leader:update-blacklist
+
+Add/remove a content ID from the blacklist (adds by default).
+
+```
+USAGE
+  $ storage-node leader:update-blacklist
+
+OPTIONS
+  -a, --add=add                [default: ] Content ID to add
+  -h, --help                   show CLI help
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -r, --remove=remove          [default: ] Content ID to remove
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## leader:update-bucket-status
+
+Update storage bucket status (accepting new bags).
+
+```
+USAGE
+  $ storage-node leader:update-bucket-status
+
+OPTIONS
+  -h, --help                   show CLI help
+  -i, --bucketId=bucketId      (required) Storage bucket ID
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -s, --set=(on|off)           (required) Sets 'accepting new bags' parameter for the bucket (on/off).
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## leader:update-data-fee
+
+Update data size fee. Requires storage working group leader permissions.
+
+```
+USAGE
+  $ storage-node leader:update-data-fee
+
+OPTIONS
+  -f, --fee=fee                (required) New data size fee
+  -h, --help                   show CLI help
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## leader:update-dynamic-bag-policy
+
+Update number of storage buckets used in the dynamic bag creation policy.
+
+```
+USAGE
+  $ storage-node leader:update-dynamic-bag-policy
+
+OPTIONS
+  -h, --help                      show CLI help
+  -k, --keyFile=keyFile           Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                       Use development mode
+  -n, --number=number             (required) New storage buckets number
+
+  -p, --password=password         Key file password (optional). Could be overriden by ACCOUNT_PWD
+                                  environment variable.
+
+  -t, --bagType=(Channel|Member)  (required) Dynamic bag type (Channel, Member).
+
+  -u, --apiUrl=apiUrl             [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                                  environment.
+
+  -y, --accountUri=accountUri     Account URI (optional). Has a priority over the keyFile and password
+                                  flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## leader:update-voucher-limits
+
+Update VoucherMaxObjectsSizeLimit and VoucherMaxObjectsNumberLimit for the Joystream node storage.
+
+```
+USAGE
+  $ storage-node leader:update-voucher-limits
+
+OPTIONS
+  -h, --help                   show CLI help
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+  -o, --objects=objects        (required) New 'max voucher object number limit' value
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -s, --size=size              (required) New 'max voucher object size limit' value
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## operator:accept-invitation
+
+Accept pending storage bucket invitation.
+
+```
+USAGE
+  $ storage-node operator:accept-invitation
+
+OPTIONS
+  -h, --help                                     show CLI help
+  -i, --bucketId=bucketId                        (required) Storage bucket ID
+
+  -k, --keyFile=keyFile                          Key file for the account. Mandatory in non-dev
+                                                 environment.
+
+  -m, --dev                                      Use development mode
+
+  -p, --password=password                        Key file password (optional). Could be overriden by
+                                                 ACCOUNT_PWD environment variable.
+
+  -t, --transactorAccountId=transactorAccountId  (required) Transactor account ID (public key)
+
+  -u, --apiUrl=apiUrl                            [default: ws://localhost:9944] Runtime API URL.
+                                                 Mandatory in non-dev environment.
+
+  -w, --workerId=workerId                        (required) Storage operator worker ID
+
+  -y, --accountUri=accountUri                    Account URI (optional). Has a priority over the keyFile
+                                                 and password flags. Could be overriden by ACCOUNT_URI
+                                                 environment variable.
+```
+
+## operator:set-metadata
+
+Set metadata for the storage bucket.
+
+```
+USAGE
+  $ storage-node operator:set-metadata
+
+OPTIONS
+  -e, --endpoint=endpoint      Root distribution node endpoint
+  -h, --help                   show CLI help
+  -i, --bucketId=bucketId      (required) Storage bucket ID
+  -j, --jsonFile=jsonFile      Path to JSON metadata file
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -w, --operatorId=operatorId  (required) Storage bucket operator ID (storage group worker ID)
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## server
+
+Starts the storage node server.
+
+```
+USAGE
+  $ storage-node server
+
+OPTIONS
+  -d, --uploads=uploads                              (required) Data uploading directory (absolute path).
+
+  -e, --elasticSearchEndpoint=elasticSearchEndpoint  Elasticsearch endpoint (e.g.: http://some.com:8081).
+                                                     Log level could be set using the ELASTIC_LOG_LEVEL
+                                                     enviroment variable.
+                                                     Supported values: warn, error, debug, info.
+                                                     Default:debug
+
+  -h, --help                                         show CLI help
+
+  -i, --syncInterval=syncInterval                    [default: 1] Interval between synchronizations (in
+                                                     minutes)
+
+  -k, --keyFile=keyFile                              Key file for the account. Mandatory in non-dev
+                                                     environment.
+
+  -m, --dev                                          Use development mode
+
+  -o, --port=port                                    (required) Server port.
+
+  -p, --password=password                            Key file password (optional). Could be overriden by
+                                                     ACCOUNT_PWD environment variable.
+
+  -q, --queryNodeEndpoint=queryNodeEndpoint          (required) [default: http://localhost:8081/graphql]
+                                                     Query node endpoint (e.g.:
+                                                     http://some.com:8081/graphql)
+
+  -r, --syncWorkersNumber=syncWorkersNumber          [default: 20] Sync workers number (max async
+                                                     operations in progress).
+
+  -s, --sync                                         Enable data synchronization.
+
+  -t, --syncWorkersTimeout=syncWorkersTimeout        [default: 30] Asset downloading timeout for the
+                                                     syncronization (in minutes).
+
+  -u, --apiUrl=apiUrl                                [default: ws://localhost:9944] Runtime API URL.
+                                                     Mandatory in non-dev environment.
+
+  -w, --worker=worker                                (required) Storage provider worker ID
+
+  -y, --accountUri=accountUri                        Account URI (optional). Has a priority over the
+                                                     keyFile and password flags. Could be overriden by
+                                                     ACCOUNT_URI environment variable.
+```
+
+
+# [Distributor node](https://github.com/Joystream/joystream/tree/giza_staging/distributor-node/docs/commands)
+
+commands-distributor.txt
+> start help dev:init dev:batchUpload operator:accept-invitation operator:set-metadata node:set-buckets node:set-worker node:shutdown node:start-public-api node:stop-public-api leader:cancel-invitation leader:create-bucket leader:create-bucket-family leader:delete-bucket leader:delete-bucket-family leader:invite-bucket-operator leader:remove-bucket-operator leader:set-bucket-family-metadata leader:set-buckets-per-bag-limit leader:update-bag leader:update-bucket-mode leader:update-bucket-status leader:update-dynamic-bag-policy
+
+`for cmd in $(cat commands-distributor.txt);do echo $cmd ; ~/joystream/storage-node-v2/bin/run $cmd --help ; done`
+
+## start
+
+>  ›   Error: command start not found
+
+## help
+
+Joystream storage subsystem.
+
+```
+VERSION
+  storage-node-v2/2.0.0 linux-x64 node-v14.18.2
+
+USAGE
+  $ storage-node [COMMAND]
+
+TOPICS
+  dev       Development mode commands.
+  leader    Storage working group leader commands.
+  operator  Storage provider(operator) commands.
+
+COMMANDS
+  help    display help for storage-node
+  server  Starts the storage node server.
+```
+
+## dev:init
+
+Initialize development environment. Sets Alice as storage working group leader.
+
+```
+USAGE
+  $ storage-node dev:init
+
+OPTIONS
+  -h, --help                   show CLI help
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## dev:batchUpload
+
+>  ›   Error: command dev:batchUpload not found
+
+## operator:accept-invitation
+
+Accept pending storage bucket invitation.
+
+```
+USAGE
+  $ storage-node operator:accept-invitation
+
+OPTIONS
+  -h, --help                                     show CLI help
+  -i, --bucketId=bucketId                        (required) Storage bucket ID
+
+  -k, --keyFile=keyFile                          Key file for the account. Mandatory in non-dev
+                                                 environment.
+
+  -m, --dev                                      Use development mode
+
+  -p, --password=password                        Key file password (optional). Could be overriden by
+                                                 ACCOUNT_PWD environment variable.
+
+  -t, --transactorAccountId=transactorAccountId  (required) Transactor account ID (public key)
+
+  -u, --apiUrl=apiUrl                            [default: ws://localhost:9944] Runtime API URL.
+                                                 Mandatory in non-dev environment.
+
+  -w, --workerId=workerId                        (required) Storage operator worker ID
+
+  -y, --accountUri=accountUri                    Account URI (optional). Has a priority over the keyFile
+                                                 and password flags. Could be overriden by ACCOUNT_URI
+                                                 environment variable.
+```
+
+## operator:set-metadata
+
+Set metadata for the storage bucket.
+
+```
+USAGE
+  $ storage-node operator:set-metadata
+
+OPTIONS
+  -e, --endpoint=endpoint      Root distribution node endpoint
+  -h, --help                   show CLI help
+  -i, --bucketId=bucketId      (required) Storage bucket ID
+  -j, --jsonFile=jsonFile      Path to JSON metadata file
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -w, --operatorId=operatorId  (required) Storage bucket operator ID (storage group worker ID)
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## node:set-buckets
+
+>  ›   Error: command node:set-buckets not found
+
+## node:set-worker
+
+> ›   Error: command node:set-worker not found
+
+## node:shutdown
+
+> ›   Error: command node:shutdown not found
+
+## node:start-public-api
+
+> ›   Error: command node:start-public-api not found
+
+## node:stop-public-api
+
+> ›   Error: command node:stop-public-api not found
+
+## leader:cancel-invitation
+
+> ›   Error: command leader:cancel-invitation not found
+
+## leader:create-bucket
+
+Create new storage bucket. Requires storage working group leader permissions.
+
+```
+USAGE
+  $ storage-node leader:create-bucket
+
+OPTIONS
+  -a, --allow                  Accepts new bags
+  -h, --help                   show CLI help
+  -i, --invited=invited        Invited storage operator ID (storage WG worker ID)
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+  -n, --number=number          Storage bucket max total objects number
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -s, --size=size              Storage bucket max total objects size
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## leader:create-bucket-family
+
+>  ›   Error: command leader:create-bucket-family not found
+
+## leader:delete-bucket
+
+Delete a storage bucket. Requires storage working group leader permissions.
+
+```
+USAGE
+  $ storage-node leader:delete-bucket
+
+OPTIONS
+  -h, --help                   show CLI help
+  -i, --bucketId=bucketId      (required) Storage bucket ID
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## leader:delete-bucket-family
+
+> ›   Error: command leader:delete-bucket-family not found
+
+## leader:invite-bucket-operator
+> ›   Error: command leader:invite-bucket-operator not found
+
+## leader:remove-bucket-operator
+> ›   Error: command leader:remove-bucket-operator not found
+
+## leader:set-bucket-family-metadata
+> ›   Error: command leader:set-bucket-family-metadata not found
+
+## leader:set-buckets-per-bag-limit
+> ›   Error: command leader:set-buckets-per-bag-limit not found
+ 
+## leader:update-bag
+
+Add/remove a storage bucket from a bag (adds by default).
+
+```
+USAGE
+  $ storage-node leader:update-bag
+
+OPTIONS
+  -a, --add=add
+      [default: ] ID of a bucket to add to bag
+
+  -h, --help
+      show CLI help
+
+  -i, --bagId=bagId
+      (required) Bag ID. Format: {bag_type}:{sub_type}:{id}.
+           - Bag types: 'static', 'dynamic'
+           - Sub types: 'static:council', 'static:wg', 'dynamic:member', 'dynamic:channel'
+           - Id:
+             - absent for 'static:council'
+             - working group name for 'static:wg'
+             - integer for 'dynamic:member' and 'dynamic:channel'
+           Examples:
+           - static:council
+           - static:wg:storage
+           - dynamic:member:4
+
+  -k, --keyFile=keyFile
+      Key file for the account. Mandatory in non-dev environment.
+
+  -m, --dev
+      Use development mode
+
+  -p, --password=password
+      Key file password (optional). Could be overriden by ACCOUNT_PWD environment variable.
+
+  -r, --remove=remove
+      [default: ] ID of a bucket to remove from bag
+
+  -u, --apiUrl=apiUrl
+      [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev environment.
+
+  -y, --accountUri=accountUri
+      Account URI (optional). Has a priority over the keyFile and password flags. Could be overriden by 
+      ACCOUNT_URI environment variable.
+```
+
+## leader:update-bucket-mode
+
+> ›   Error: command leader:update-bucket-mode not found
+
+## leader:update-bucket-status
+
+Update storage bucket status (accepting new bags).
+
+```
+USAGE
+  $ storage-node leader:update-bucket-status
+
+OPTIONS
+  -h, --help                   show CLI help
+  -i, --bucketId=bucketId      (required) Storage bucket ID
+  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                    Use development mode
+
+  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD
+                               environment variable.
+
+  -s, --set=(on|off)           (required) Sets 'accepting new bags' parameter for the bucket (on/off).
+
+  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                               environment.
+
+  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password
+                               flags. Could be overriden by ACCOUNT_URI environment variable.
+```
+
+## leader:update-dynamic-bag-policy
+
+Update number of storage buckets used in the dynamic bag creation policy.
+
+```
+USAGE
+  $ storage-node leader:update-dynamic-bag-policy
+
+OPTIONS
+  -h, --help                      show CLI help
+  -k, --keyFile=keyFile           Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                       Use development mode
+  -n, --number=number             (required) New storage buckets number
+
+  -p, --password=password         Key file password (optional). Could be overriden by ACCOUNT_PWD
+                                  environment variable.
+
+  -t, --bagType=(Channel|Member)  (required) Dynamic bag type (Channel, Member).
+
+  -u, --apiUrl=apiUrl             [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                                  environment.
+
+  -y, --accountUri=accountUri     Account URI (optional). Has a priority over the keyFile and password
+                                  flags. Could be overriden by ACCOUNT_URI environment variable.
+```

working-groups/operations-group/Giza-testing/README.md

@@ -0,0 +1,401 @@
+# [KPI 36.OP-1](https://blog.joystream.org/sumer-kpis/#36.OP-1)
+
+The goal is to install a [GIZA](https://github.com/Joystream/joystream/tree/giza_staging) validator, query node, storage provider and distributor on a fresh VPS.
+
+## Setup
+
+### Node
+
+See [Network deployment](../Network-deployment)
+
+### [Query-node](https://github.com/Joystream/joystream/tree/giza_staging/query-node)
+
+Edit: Some instructions have been simplified since the test ([README olympia QN](https://github.com/Lezek123/substrate-runtime-joystream/tree/olympia-readme/query-node)).
+
+Download these to `~/joystream` (or your repository root):
+- [.env](.env)
+- [import-qn-data.sh](import-qn-data.sh)
+- [deploy-qn.sh](deploy-qn.sh)
+- [reset-qn.sh](reset-qn.sh)
+
+Adapt `JOYSTREAM_NODE_WS` in `.env` and `import-qn-data.sh` if necessary.
+
+```
+cd ~/joystream/query-node
+yarn build
+nano manifest.yml
+# change second to last line to:
+---
+           height: "[13876,13876]"
+---
+cd ..
+sudo docker-compose up
+chmod +x import-qn-data.sh deploy-qn.sh reset-qn.sh
+./reset-qn.sh
+```
+
+### Storage
+
+See [storage-node](https://github.com/Joystream/joystream/tree/giza_staging/storage-node).
+
+### Distributor
+
+See [distributor-node](https://github.com/Joystream/joystream/tree/giza_staging/distributor-node).
+
+### Pioneer
+
+```
+cd ~/joystream/pioneer
+yarn
+yarn build:www
+rsync -aP packages/apps/build/ /var/www/pioneer
+```
+
+To pre-select your endpoint, edit `packages/apps-config/src/settings/endpoints.js` and add
+```
+  {
+    info: 'Joystream Giza',
+    text: t('giza-l1dev', 'Joystream Giza (l1dev)', { ns: 'apps-config' }),
+    value: 'wss://giza-l1dev.joystream.app/rpc'
+  },
+```
+as first array entry for `createLive` (line 23). Run `yarn build:www` again and you should see:
+> i18next-scanner: Added a new translation key { "giza-l1dev": "Joystream Giza (l1dev)" } to "packages/apps/public/locales/en/apps-config.json"
+
+Alternatively it is possible to set `WS_URL` during build time: `WS_URL=wss://myendpoint yarn build:www`
+
+### Atlas
+
+```
+git clone htps://github.com/joystream/atlas
+cd atlas
+yarn
+yarn build
+rsync -aP dist/ /var/www/atlas
+```
+
+- Set DNS record for `atlas` subdomain (depends on your registrar)
+
+- Create `/etc/nginx/sites-available/atlas` to configure nginx site
+```
+server {
+    listen 443 ssl;
+    server_name atlas.DOMAIN;
+
+    root /var/www/atlas;
+    index index.html;
+
+    include /etc/letsencrypt/options-ssl-nginx.conf;
+    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
+}
+```
+
+- Activate site
+```
+cd /etc/nginx/sites-enabled
+ln -s ../sites-available/atlas
+```
+
+- Request TLS certificate: `certbot run`, select the subdomain and hit `enter`.
+
+### Nginx
+
+- Install dependencies:
+`apt install nginx certbot python3-certbot-nginx`
+
+- Create `/etc/nginx/sites-available/joysream` to configure nginx site
+```
+server {
+    listen 80;
+    return 301 https://$host$request_uri;
+}
+
+server {
+    listen 443 ssl;
+    server_name giza-l1dev.joystream.app;
+
+    root /var/www/joystream; # build atlas or pioneer and move it there
+    index index.html;
+
+    location /rpc {
+      proxy_pass http://localhost:9944;
+      proxy_redirect off;
+      proxy_http_version 1.1;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "Upgrade";
+      proxy_set_header Host $host;
+      proxy_set_header    X-Real-IP $remote_addr;
+    }
+
+    # query node
+    location /graphql {
+      rewrite /graphql(/.+) $1 break;
+      proxy_pass http://localhost:8081;
+      proxy_set_header Access-Control-Allow-Methods "GET, PUT, HEAD, OPTIONS";
+    }
+    location /@apollographql { rewrite (.*) /graphql$1 last; }
+
+    # storage provider
+    location /storage {
+      rewrite /storage/?(.*) /$1 break;
+      proxy_pass http://localhost:3333;
+      proxy_set_header Access-Control-Allow-Methods "GET, PUT, HEAD, OPTIONS";
+      proxy_set_header Access-Control-Allow-Headers "GET, PUT, HEAD, OPTIONS";
+      client_max_body_size 50G;
+    }
+
+    # distributor
+    location /distributor {
+      rewrite /distributor/?(.*) /$1 break;
+      proxy_pass http://localhost:3334;
+      proxy_set_header Access-Control-Allow-Methods "*";
+      proxy_set_header Access-Control-Allow-Headers "*";
+    }
+
+    include /etc/letsencrypt/options-ssl-nginx.conf;
+    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
+}
+```
+
+- Activate site
+```
+cd /etc/nginx/sites-enabled
+ln -s ../sites-available/joystream
+rm default
+```
+
+- Request TLS certificate: `certbot run`.
+Enter email address, accept the TOS to create a certbot account. The rest happens automatically.
+No need to add a redirect because it is already configured above.
+
+Test:
+- open Pioneer and choose as custom endpoint: `wss://giza-l1dev.joystream.app/rpc`
+- https://giza-l1dev.joystream.app (if pioneer or atlas are set up)
+- https://giza-l1dev.joystream.app/graphql
+- https://giza-l1dev.joystream.app/storage/api/v1/version
+- https://giza-l1dev.joystream.app/storage/api/v1/state/data-objects
+- https://giza-l1dev.joystream.app/distributor/
+
+
+## Configuration
+
+1. Apply as storage provider and distributor, save files to ~/keys/
+2. Have the leads fill the openings
+3. Get invited to a bucket
+4. Verify group memberships
+```
+$ ~/joystream/cli/bin/run working-groups:overview
+
+Current Group: storageProviders
+
+___________________ Group lead ___________________
+
+Member id:                    2010                                             
+Member handle:                storage_lead_giza                                
+Role account:                 5GHPwGqcVjt3KKrEUhkgLJaizRymegx5astt4fekGVmL7F4q 
+
+____________________ Members _____________________
+
+Worker id     Member id     Member handle         Stake            Earned     Role account                  
+3             515           l1dev                 10.0000 JOY      0          5DNe6u...ATpk38               
+2             2012          storage_2             100.0000 JOY     0          5DGQLX...Fai2MW               
+1             2012          storage_2             100.0000 JOY     0          5H8CPz...hoWPtP               
+0             2010          storage_lead_giza     100.0000 JOY     0          5GHPwG...mL7F4q
+
+$ ~/joystream/cli/bin/run working-groups:overview -g distributors
+Current Group: distributors
+
+___________________ Group lead ___________________
+
+Member id:                    2015                                             
+Member handle:                dist_lead                                        
+Role account:                 5GJWFApQiMqFjhiLWhxtVAjva7fVy7kYAf8mBgME4rGXX4Vy 
+
+____________________ Members _____________________
+
+Worker id     Member id     Member handle     Stake            Earned     Role account                  
+2             515           l1dev             100.0000 JOY     0          5DCCUr...qjEvK9               
+1             2016          dist_1            100.0000 JOY     0          5GQZGW...m9z7VA               
+0             2015          dist_lead         100.0000 JOY     0          5GJWFA...GXX4Vy
+```
+
+### Storage Provider
+
+- [Accept invitation](https://github.com/Joystream/joystream/tree/giza_staging/storage-node-v2#storage-node-operatoraccept-invitation)
+```
+$ ~/joystream/storage-node-v2/bin/run operator:accept-invitation -i 2 -w 3 -k ~/keys/giza-storage-l1dev.json -t 5DNe6ubsQhmcSZRZdBLJroe9qKThW9CsSGRKHiZx8pATpk38
+2022-01-04 15:43:26:4326 info: Initialized runtime connection: ws://localhost:9944
+2022-01-04 15:43:30:4330 info: Waiting for chain to be synced before proceeding.
+2022-01-04 15:43:30:4330 info: Accepting pending storage bucket invitation...
+2022-01-04 15:43:30:4330 debug: Sending storage.acceptStorageBucketInvitation extrinsic...
+2022-01-04 15:43:36:4336 debug: Extrinsic successful!
+```
+
+- set [metadata](https://github.com/Joystream/joystream/tree/giza_staging/storage-node-v2#metadata): [`storage-node operator:set-metadata`](https://github.com/Joystream/joystream/tree/giza_staging/storage-node-v2#storage-node-operatorset-metadata) ([example json](https://github.com/Joystream/joystream/blob/giza_staging/storage-node-v2/scripts/operatorMetadata.json) - [fields](https://github.com/Joystream/joystream/blob/giza_staging/metadata-protobuf/proto/Storage.proto))
+```
+$ ~/joystream/storage-node-v2/bin/run operator:set-metadata -i 2 -w 3 -j ~/bucket-2-metadata.json -k ~/keys/giza-storage-l1dev.json
+2022-01-04 15:46:21:4621 info: Initialized runtime connection: ws://localhost:9944
+2022-01-04 15:46:24:4624 info: Waiting for chain to be synced before proceeding.
+2022-01-04 15:46:24:4624 info: Setting the storage operator metadata...
+2022-01-04 15:46:24:4624 debug: Sending storage.setStorageOperatorMetadata extrinsic...
+2022-01-04 15:46:30:4630 debug: Extrinsic successful!
+```
+
+### Distributor
+
+See [docs/node](https://github.com/Joystream/joystream/blob/giza_staging/distributor-node/docs/node/index.md)
+
+- Accept invitaiton
+```
+~/joystream/distributor-node/bin/run operator:accept-invitation -B 2:0 -w 2 -c ~/joystream/distributor-node/config.yml
+2022-01-04 16:08:16:816 CLI info: Accepting distribution bucket operator invitation...
+{
+    "bucketId": {
+        "distribution_bucket_family_id": "2",
+        "distribution_bucket_index": "0"
+    },
+    "workerId": 2
+}
+? Tx fee of 0 will be deduced from you account, do you confirm the transfer? (y/N) y
+? Tx fee of 0 will be deduced from you account, do you confirm the transfer? (y/N) y? Tx fee of 0 will be deduced from you account, do you confirm the transfer? Yes
+2022-01-04 16:08:22:822 SubstrateApi info: Sending storage.acceptDistributionBucketInvitation extrinsic from 5DCCUrwUZuqBHMMcaswK5rk1DpFNGvuZvoZHWiHYNyqjEvK9
+2022-01-04 16:08:24:824 CLI info: Invitation succesfully accepted!
+```
+
+- set [metadata](https://github.com/Joystream/joystream/blob/giza_staging/distributor-node/docs/node/index.md#metadata) (or use the [API](https://github.com/Joystream/joystream/blob/giza_staging/distributor-node/docs/api/operator/index.md))
+```
+~/joystream/distributor-node/bin/run operator:set-metadata -B 2:0 -w 2 -c ~/joystream/distributor-node/config.yml -i ~/distributor-metadata.json
+2022-01-04 16:25:29:2529 CLI info: Setting bucket operator metadata...
+{
+    "bucketId": {
+        "distribution_bucket_family_id": "2",
+        "distribution_bucket_index": "0"
+    },
+    "workerId": 2,
+    "metadata": {
+        "endpoint": "https://giza-l1dev.joystream.app/distributor",
+        "location": {
+            "countryCode": "US-NJ",
+            "city": "Newark",
+            "coordinates": {
+                "latitude": 40,
+                "longitude": 74
+            }
+        },
+        "extra": "Welcome to Joystream GIZA - America branch! -- operated by l1dev (dev@joystreamstats.live)"
+    }
+}
+? Tx fee of 0 will be deduced from you account, do you confirm the transfer? (y/N) y
+? Tx fee of 0 will be deduced from you account, do you confirm the transfer? (y/N) y? Tx fee of 0 will be deduced from you account, do you confirm the transfer? Yes
+2022-01-04 16:25:39:2539 SubstrateApi info: Sending storage.setDistributionOperatorMetadata extrinsic from 5DCCUrwUZuqBHMMcaswK5rk1DpFNGvuZvoZHWiHYNyqjEvK9
+2022-01-04 16:25:42:2542 CLI info: Bucket operator metadata succesfully set/updated!
+```
+
+## Runtime
+### Staking and Rewards (WGs)
+
+> For each group (incl. Leads), check that:
+> - staking and unstaking works as expected
+> - rewards are paid out as they should
+
+### Validator Rewards
+
+> The runtime upgrade included a change in these params. For each `era`, get:
+>   - total issuance
+>   - total stake
+>   - total reward
+>   - ~blockheight
+
+| Block | Era | Issuance | Stake | % Staked | Era Reward | Action |
+|---|---|---|---|---|---|---|
+| 0 | 0 | 574460220 | 15000 | 0.0 | 3275 | |
+| 6000 | 10 | 574460120 | 15000 | 0.0 | 3275 | |
+| 6600 | 11 | 574455020 | 15000 | 0.0 | 5698 | |
+| 7200 | 12 | 1000000000 | 200000000 | 20.0 | 58841 | |
+| 13200 | 22 | 1000000000 | 200000000 | 20.0 | 58841 | |
+| 13800 | 23 | 999999800 | 200000000 | 20.0 | 14805 | |
+| 14400 | 24 | 999999700 | 200000000 | 20.0 | 14780 | |
+| 15000 | 25 | 999999700 | 100007500 | 10.0 | 10250 | |
+| 15600 | 26 | 999999600 | 100007500 | 10.0 | 10250 | |
+| 16200 | 27 | 999999600 | 100007500 | 10.0 | 10250 | |
+| 16800 | 28 | 999999600 | 200000000 | 20.0 | 14805 | |
+| 151800 | 253 | 999999500 | 200000000 | 20.0 | 14805 | |
+| 152400 | 254 | 999999500 | 200000000 | 20.0 | 14780 | |
+| 153000 | 255 | 999999500 | 200000000 | 20.0 | 14805 | |
+| 161400 | 269 | 999999400 | 200000000 | 20.0 | 14805 | |
+| 162000 | 270 | 999999400 | 200500000 | 20.1 | 14828 | |
+| 187800 | 313 | 999999400 | 200500000 | 20.1 | 14828 | |
+| 188400 | 314 | 999999400 | 200500000 | 20.1 | 15397 | |
+| 192000 | 320 | 1099991546 | 200500000 | 18.2 | 15397 | |
+
+## Storage
+
+The storage tests are delayed to improve the content migration script (for non-sudo uploads and to support dynamic channel-bucket-provider assignment - see ([PR #3034](https://github.com/Joystream/joystream/pull/3034)).
+
+1. What happens when a storage node runs out of (local) storage?
+  - eg. the bucket is configured to accept 100GB, whereas the node runs out at 80GB
+2. What happens when a bucket is full (ref. bucket config on-chain) and you upload more from a channel where it's assigned to accept from?
+3. What happens when a bucket is full (ref. bucket config on-chain) and you assign a new bag to it?
+4. What happens when a SP node, that is the only source of some data goes down?
+  - Cases:
+    1. No distributor has it (or is assigned): Try to assign the channel bag to another SP, and use the re-upload command (cli and/or atlas)
+    2. One or more distributors has it (and is assigned): Try to assign the channel bag to another SP, and use the re-upload command (cli and/or atlas)
+    3. One or more distributors has it (and is assigned): Try to assign the channel bag to another SP, and try to fetch 5it from the distributor
+5. How does the blacklist work?
+  - Cases:
+    1. One distributor has (and is assigned) the file. asset it get censored in atlas?
+      - What happens if you upload the asset again?
+    2. No distributor has the file (or is assigned). When one gets assigned it, does it get it?
+      - What happens if you upload the asset again?
+
+### CLI
+
+> - Check that doing `--help` for all commands
+>  - provides a useful description
+>  - everything is correct
+>  - syntax is consistent
+
+See [Commands](Commands.md#storage)
+
+## Distributor
+
+1. What happens when a distributor node runs out of storage?
+  - eg. the bucket is configured to accept 100GB, whereas the node runs out at 80GB
+
+### CLI
+
+> - Check that doing `--help` for all commands
+>  - provides a useful description
+>  - everything is correct
+>  - syntax is consistent
+>  - all the examples are correct
+
+See [Commands](Commands.md#distributor)
+
+No help:
+- start
+- dev:batchUpload
+- node:*
+- leader:cancel-invitation
+- leader:create-bucket-family
+- leader:delete-bucket-family
+- leader:invite-bucket-operator
+- leader:remove-bucket-operator
+- leader:set-bucket-family-metadata
+- leader:set-buckets-per-bag-limit
+- leader:update-bucket-mode
+
+# Hours
+
+| Date        | Hours | Task                                           |
+|-------------|-------|------------------------------------------------|
+| Jan 2 13-17 |     4 | start node, storage provider, distributor      |
+| Jan 4 15-18 |     3 | storage provider and distributor configuration |
+| Jan 4 19-22 |     3 | check commands, validator eras                 |
+| Jan 5 20-23 |     3 | nginx config                                   |
+| Jan 6 18-23 |     5 | content creation                               |
+| Jan 7 12-15 |     3 | query-node deployment                          |
+| Jan 10 6-11 |     5 | research joystream issue updates               |
+| Jan 13 6-12 |     6 | content-migration script, PR #3034             |
+| Jan 14 2-7  |     5 | content-migration script, PR #3034             |
+|             |    37 |                                                |

working-groups/operations-group/Giza-testing/deploy-qn.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -e
+
+SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+cd $SCRIPT_PATH
+
+# Bring up db
+docker-compose up -d db
+
+# Make sure we use dev config for db migrations (prevents "Cannot create database..." and some other errors)
+docker-compose run --rm --entrypoint sh graphql-server -c "yarn workspace query-node config:dev"
+# Migrate the databases
+docker-compose run --rm --entrypoint sh graphql-server -c "yarn workspace query-node-root db:prepare"
+docker-compose run --rm --entrypoint sh graphql-server -c "yarn workspace query-node-root db:migrate"
+
+# Start indexer and gateway
+docker-compose up -d indexer
+docker-compose up -d hydra-indexer-gateway
+
+# Start processor and graphql server
+docker-compose up -d processor
+docker-compose up -d graphql-server

working-groups/operations-group/Giza-testing/import-qn-data.sh

@@ -0,0 +1,68 @@
+COMPOSE_PROJECT_NAME=joystream
+PROJECT_NAME=query_node
+
+# We will use a single postgres service with multiple databases
+# The env variables below are by default used by all services and should be
+# overriden in local env files
+# DB config
+INDEXER_DB_NAME=query_node_indexer
+DB_NAME=query_node_processor
+DB_USER=postgres
+DB_PASS=postgres
+# This value will not be used by query-node docker containers.
+# When running query-node with docker these services will always use the db service
+DB_HOST=localhost
+DB_PORT=5432
+DEBUG=index-builder:*
+TYPEORM_LOGGING=error
+
+## Indexer options
+# Block height to start indexing from.
+# Note, that if there are already some indexed events, this setting is ignored
+BLOCK_HEIGHT=13876
+#BLOCK_HEIGHT=200000
+
+# Query node GraphQL server port
+GRAPHQL_SERVER_PORT=8081
+
+# Hydra indexer gateway GraphQL server port
+HYDRA_INDEXER_GATEWAY_PORT=4000
+
+# Default GraphQL server host. It is required during "query-node config:dev"
+GRAPHQL_SERVER_HOST=localhost
+
+# Websocket RPC endpoint containers will use.
+#JOYSTREAM_NODE_WS=ws://joystream-node:9944/
+JOYSTREAM_NODE_WS=ws://localhost:9944
+
+# Query node which colossus will use
+#COLOSSUS_QUERY_NODE_URL=http://graphql-server:${GRAPHQL_SERVER_PORT}/graphql
+COLOSSUS_QUERY_NODE_URL=http://graphql-server:4000/graphql
+
+
+# Query node which distributor will use
+#DISTRIBUTOR_QUERY_NODE_URL=http://graphql-server:${GRAPHQL_SERVER_PORT}/graphql
+DISTRIBUTOR_QUERY_NODE_URL=http://graphql-server:4000/graphql
+
+# Indexer gateway used by processor. If you don't use the local indexer set this to a remote gateway
+#PROCESSOR_INDEXER_GATEWAY=http://hydra-indexer-gateway:${HYDRA_INDEXER_GATEWAY_PORT}/graphql
+PROCESSOR_INDEXER_GATEWAY=http://hydra-indexer-gateway:4000/graphql
+
+# Colossus services identities
+COLOSSUS_1_WORKER_ID=0
+COLOSSUS_1_WORKER_URI=//testing//worker//Storage//${COLOSSUS_1_WORKER_ID}
+COLOSSUS_1_TRANSACTOR_URI=//Colossus1
+
+COLOSSUS_2_WORKER_ID=1
+COLOSSUS_2_WORKER_URI=//testing//worker//Storage//${COLOSSUS_2_WORKER_ID}
+COLOSSUS_2_TRANSACTOR_URI=//Colossus2
+
+# Distributor node services identities
+DISTRIBUTOR_1_WORKER_ID=0
+DISTRIBUTOR_1_ACCOUNT_URI=//testing//worker//Distribution//${DISTRIBUTOR_1_WORKER_ID}
+
+DISTRIBUTOR_2_WORKER_ID=1
+DISTRIBUTOR_2_ACCOUNT_URI=//testing//worker//Distribution//${DISTRIBUTOR_2_WORKER_ID}
+
+# joystream/node docker image tag
+JOYSTREAM_NODE_TAG=latest

working-groups/operations-group/Giza-testing/reset-qn.sh

@@ -0,0 +1,8 @@
+docker-compose down -v
+service docker stop
+rm -rf /var/lib/docker/volumes/joystream_query-node-data
+service docker start
+./import-qn-data.sh
+for file in $(rgrep -l mokhtar query-node/); do sed -i s/"\\\u0000"/""/g $file ; done
+./deploy-qn.sh
+docker logs -f -n 100 processor