Merge branch 'giza_staging' into giza-integration-tests

.env

@@ -34,8 +34,7 @@ GRAPHQL_SERVER_HOST=localhost
 JOYSTREAM_NODE_WS=ws://joystream-node:9944/
 
 # Query node which colossus will use
-# TODO: Colossus should take a full Url instead
-COLOSSUS_QUERY_NODE_HOST=graphql-server:${GRAPHQL_SERVER_PORT}
+COLOSSUS_QUERY_NODE_URL=http://graphql-server:${GRAPHQL_SERVER_PORT}/graphql
 
 # Query node which distributor will use
 DISTRIBUTOR_QUERY_NODE_URL=http://graphql-server:${GRAPHQL_SERVER_PORT}/graphql

.github/workflows/deploy-playground.yml

@@ -0,0 +1,68 @@
+name: Deploy Playground
+
+on:
+  workflow_dispatch:
+    inputs:
+      gitRepo:
+        description: 'Code repository'
+        required: false
+        default: 'https://github.com/Joystream/joystream.git'
+      branchName:
+        description: 'Branch to deploy'
+        required: false
+        default: 'master'
+      keyName:
+        description: 'SSH key pair on AWS'
+        required: false
+        default: 'joystream-github-action-key'
+      instanceType:
+        description: 'AWS EC2 instance type (t2.micro, t2.large)'
+        required: false
+        default: 't2.micro'
+
+defaults:
+  run:
+    working-directory: devops/aws
+
+jobs:
+  deploy-playground:
+    name: Create an EC2 instance and configure docker-compose stack
+    runs-on: ubuntu-latest
+    env:
+      STACK_NAME: joystream-playground-${{ github.event.inputs.branchName }}-${{ github.run_number }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install Ansible dependencies
+        run: pipx inject ansible-core boto3 botocore
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - name: Deploy to AWS CloudFormation
+        uses: aws-actions/aws-cloudformation-github-deploy@v1
+        id: deploy_stack
+        with:
+          name: ${{ env.STACK_NAME }}
+          template: devops/aws/cloudformation/single-instance-docker.yml
+          no-fail-on-empty-changeset: '1'
+          parameter-overrides: 'KeyName=${{ github.event.inputs.keyName }},EC2InstanceType=${{ github.event.inputs.instanceType }}'
+
+      - name: Run playbook
+        uses: dawidd6/action-ansible-playbook@v2
+        with:
+          playbook: deploy-playground-playbook.yml
+          directory: devops/aws
+          requirements: requirements.yml
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          inventory: |
+            [all]
+            ${{ steps.deploy_stack.outputs.PublicIp }}
+          options: |
+            --extra-vars "git_repo=${{ github.event.inputs.gitRepo }} \
+                          branch_name=${{ github.event.inputs.branchName }}"

.github/workflows/run-network-tests.yml

@@ -108,7 +108,7 @@ jobs:
         run: npm -g install @joystream/cli
       - name: Execute network tests
         run: |
-          export HOME=$(pwd)
+          export HOME=${PWD}
           mkdir -p ${HOME}/.local/share/joystream-cli
           joystream-cli api:setUri ws://localhost:9944
           export RUNTIME=sumer

README.md

@@ -92,7 +92,7 @@ You can also run your our own joystream-node:
 
 ```sh
 git checkout master
-WASM_BUILD_TOOLCHAIN=nightly-2021-03-24 cargo build --release
+WASM_BUILD_TOOLCHAIN=nightly-2021-02-20 cargo +nightly-2021-02-20 build --release
 ./target/release/joystream-node -- --pruning archive --chain testnets/joy-testnet-5.json
 ```
 

cli/src/base/UploadCommandBase.ts

@@ -343,16 +343,22 @@ export default abstract class UploadCommandBase extends ContentDirectoryCommandB
 
   async prepareAssetsForExtrinsic(resolvedAssets: ResolvedAsset[]): Promise<StorageAssets | undefined> {
     const feePerMB = await this.getOriginalApi().query.storage.dataObjectPerMegabyteFee()
+    const { dataObjectDeletionPrize } = this.getOriginalApi().consts.storage
     if (resolvedAssets.length) {
       const totalBytes = resolvedAssets
         .reduce((a, b) => {
           return a.add(b.parameters.getField('size'))
         }, new BN(0))
         .toNumber()
-      const totalFee = feePerMB.muln(Math.ceil(totalBytes / 1024 / 1024))
+      const totalStorageFee = feePerMB.muln(Math.ceil(totalBytes / 1024 / 1024))
+      const totalDeletionPrize = dataObjectDeletionPrize.muln(resolvedAssets.length)
       await this.requireConfirmation(
-        `Total fee of ${chalk.cyan(formatBalance(totalFee))} ` +
-          `will have to be paid in order to store the provided assets. Are you sure you want to continue?`
+        `Some additional costs will be associated with this operation:\n` +
+          `Total data storage fee: ${chalk.cyan(formatBalance(totalStorageFee))}\n` +
+          `Total deletion prize: ${chalk.cyan(
+            formatBalance(totalDeletionPrize)
+          )} (recoverable on data object(s) removal)\n` +
+          `Are you sure you want to continue?`
       )
       return createTypeFromConstructor(StorageAssets, {
         expected_data_size_fee: feePerMB,

cli/src/commands/api/setQueryNodeEndpoint.ts

@@ -28,7 +28,7 @@ export default class ApiSetQueryNodeEndpoint extends ApiCommandBase {
     } else {
       newEndpoint = await this.promptForQueryNodeUri()
     }
-    await this.setPreservedState({ queryNodeUri: endpoint })
+    await this.setPreservedState({ queryNodeUri: newEndpoint })
     this.log(
       chalk.greenBright('Query node endpoint successfuly changed! New endpoint: ') + chalk.magentaBright(newEndpoint)
     )

cli/src/graphql/generated/queries.ts

@@ -7,7 +7,7 @@ export type StorageNodeInfoFragment = {
 }
 
 export type GetStorageNodesInfoByBagIdQueryVariables = Types.Exact<{
-  bagId?: Types.Maybe<Types.Scalars['String']>
+  bagId?: Types.Maybe<Types.Scalars['ID']>
 }>
 
 export type GetStorageNodesInfoByBagIdQuery = { storageBuckets: Array<StorageNodeInfoFragment> }
@@ -81,11 +81,11 @@ export const DataObjectInfo = gql`
   }
 `
 export const GetStorageNodesInfoByBagId = gql`
-  query getStorageNodesInfoByBagId($bagId: String) {
+  query getStorageNodesInfoByBagId($bagId: ID) {
     storageBuckets(
       where: {
         operatorStatus_json: { isTypeOf_eq: "StorageBucketOperatorStatusActive" }
-        bagAssignments_some: { storageBagId_eq: $bagId }
+        bags_some: { id_eq: $bagId }
         operatorMetadata: { nodeEndpoint_contains: "http" }
       }
     ) {

cli/src/graphql/queries/storage.graphql

@@ -5,11 +5,11 @@ fragment StorageNodeInfo on StorageBucket {
   }
 }
 
-query getStorageNodesInfoByBagId($bagId: String) {
+query getStorageNodesInfoByBagId($bagId: ID) {
   storageBuckets(
     where: {
       operatorStatus_json: { isTypeOf_eq: "StorageBucketOperatorStatusActive" }
-      bagAssignments_some: { storageBagId_eq: $bagId }
+      bags_some: { id_eq: $bagId }
       operatorMetadata: { nodeEndpoint_contains: "http" }
     }
   ) {

colossus.Dockerfile

@@ -3,7 +3,7 @@ FROM --platform=linux/x86-64 node:14 as builder
 WORKDIR /joystream
 COPY . /joystream
 
-RUN yarn
+RUN yarn --frozen-lockfile
 
 RUN yarn workspace @joystream/types build
 RUN yarn workspace @joystream/metadata-protobuf build
@@ -15,14 +15,14 @@ VOLUME ["/data", "/keystore"]
 # Required variables
 ENV WS_PROVIDER_ENDPOINT_URI=ws://not-set
 ENV COLOSSUS_PORT=3333
-ENV QUERY_NODE_HOST=not-set
-ENV WORKER_ID=
+ENV QUERY_NODE_ENDPOINT=http://not-set/graphql
+ENV WORKER_ID=not-set
 # - set external key file using the `/keystore` volume
 ENV ACCOUNT_KEYFILE=
 ENV ACCOUNT_PWD=
 # Optional variables
 ENV SYNC_INTERVAL=1
-ENV ELASTIC_SEARCH_HOST=
+ENV ELASTIC_SEARCH_ENDPOINT=
 # warn, error, debug, info
 ENV ELASTIC_LOG_LEVEL=debug
 # - overrides account key file
@@ -32,7 +32,7 @@ ENV ACCOUNT_URI=
 EXPOSE ${COLOSSUS_PORT}
 
 WORKDIR /joystream/storage-node-v2
-ENTRYPOINT yarn storage-node server --queryNodeHost ${QUERY_NODE_HOST} \
+ENTRYPOINT yarn storage-node server --queryNodeEndpoint ${QUERY_NODE_ENDPOINT} \
     --port ${COLOSSUS_PORT} --uploads /data  \
     --apiUrl ${WS_PROVIDER_ENDPOINT_URI} --sync --syncInterval=${SYNC_INTERVAL} \
     --elasticSearchHost=${ELASTIC_SEARCH_HOST} \

devops/aws/cloudformation/single-instance-docker.yml

@@ -26,6 +26,14 @@ Resources:
           FromPort: 22
           ToPort: 22
           CidrIp: 0.0.0.0/0
+        - IpProtocol: tcp
+          FromPort: 443
+          ToPort: 443
+          CidrIp: 0.0.0.0/0
+        - IpProtocol: tcp
+          FromPort: 80
+          ToPort: 80
+          CidrIp: 0.0.0.0/0
       Tags:
         - Key: Name
           Value: !Sub '${AWS::StackName}_validator'
@@ -71,7 +79,7 @@ Resources:
 
             curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
 
-            echo "deb [arch=arm64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+            echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
 
             apt-get update -y
 
@@ -79,6 +87,11 @@ Resources:
 
             usermod -aG docker ubuntu
 
+            # Update docker-compose to 1.28+
+            curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+            chmod +x /usr/local/bin/docker-compose
+            ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
+
             # Get latest cfn scripts and install them;
             apt-get install -y python3-setuptools
             mkdir -p /opt/aws/bin

devops/aws/deploy-playground-playbook.yml

@@ -0,0 +1,108 @@
+---
+# Run the docker-compose setup on a new EC2 instance
+
+- name: Setup EC2 instance and start docker-compose services
+  hosts: all
+  gather_facts: yes
+
+  tasks:
+    - name: Get code from git repo
+      include_role:
+        name: common
+        tasks_from: get-code-git
+
+    - name: Creat bash profile file
+      command: 'touch /home/ubuntu/.bash_profile'
+
+    - name: Run setup script
+      command: ./setup.sh
+      args:
+        chdir: '{{ remote_code_path }}'
+
+    - name: Copy bash_profile content
+      shell: cat ~/.bash_profile
+      register: bash_data
+
+    - name: Copy bash_profile content to bashrc for non-interactive sessions
+      blockinfile:
+        block: '{{ bash_data.stdout }}'
+        path: ~/.bashrc
+        insertbefore: BOF
+
+    - name: Make sure docker is running
+      command: systemctl start docker
+      become: yes
+
+    - name: Build packages
+      command: yarn build:packages
+      args:
+        chdir: '{{ remote_code_path }}'
+      async: 3600
+      poll: 0
+      register: build_result
+
+    - name: Check on build async task
+      async_status:
+        jid: '{{ build_result.ansible_job_id }}'
+      register: job_result
+      until: job_result.finished
+      # Max number of times to check for status
+      retries: 36
+      # Check for the status every 100s
+      delay: 100
+
+    - name: Build Node image
+      command: yarn build:node:docker
+      args:
+        chdir: '{{ remote_code_path }}'
+
+    - name: Run docker-compose
+      command: yarn start
+      args:
+        chdir: '{{ remote_code_path }}'
+      environment:
+        PERSIST: 'true'
+        COLOSSUS_1_NODE_URI: 'https://{{ inventory_hostname }}.nip.io/colossus-1/'
+        DISTRIBUTOR_1_NODE_URI: 'https://{{ inventory_hostname }}.nip.io/distributor-1/'
+      async: 1800
+      poll: 0
+      register: compose_result
+
+    - name: Check on yarn start task
+      async_status:
+        jid: '{{ compose_result.ansible_job_id }}'
+      register: job_result
+      until: job_result.finished
+      # Max number of times to check for status
+      retries: 18
+      # Check for the status every 100s
+      delay: 100
+
+    - name: Set nip.io domain with IP
+      set_fact:
+        nip_domain: '{{ inventory_hostname }}.nip.io'
+      run_once: yes
+
+    - name: Install and configure Caddy
+      include_role:
+        name: caddy_ansible.caddy_ansible
+        apply:
+          become: yes
+      vars:
+        caddy_config: "{{ lookup('template', 'templates/Playground-Caddyfile.j2') }}"
+        caddy_systemd_capabilities_enabled: true
+        caddy_update: false
+
+    - name: Print endpoints
+      debug:
+        msg:
+          - 'The services should now be accesible at:'
+          - 'Pioneer: https://{{ nip_domain }}/pioneer/'
+          - 'WebSocket RPC: wss://{{ nip_domain }}/ws-rpc'
+          - 'HTTP RPC: https://{{ nip_domain }}/http-rpc'
+          - 'Colossus: https://{{ nip_domain }}/colossus-1'
+          - 'Distributor: https://{{ nip_domain }}/distributor-1'
+          - 'GraphQL server: https://{{ nip_domain }}/query-node/server/graphql'
+          - 'Indexer: https://{{ nip_domain }}/query-node/indexer/graphql'
+          - 'Member Faucet: https://{{ nip_domain }}/member-faucet/register'
+          - 'Orion: https://{{ nip_domain }}/orion/graphql'

devops/aws/deploy-playground.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -e
+
+source common.sh
+
+if [ -z "$1" ]; then
+  echo "ERROR: Configuration file not passed"
+  echo "Please use ./deploy-playground.sh PATH/TO/CONFIG to run this script"
+  exit 1
+else
+  echo "Using $1 file for config"
+  source $1
+fi
+
+if [ ! -f "$KEY_PATH" ]; then
+    echo "Key file not found at $KEY_PATH"
+    exit 1
+fi
+
+# Deploy the CloudFormation template
+echo -e "\n\n=========== Deploying single node ==========="
+aws cloudformation deploy \
+  --region $REGION \
+  --profile $CLI_PROFILE \
+  --stack-name $SINGLE_NODE_STACK_NAME \
+  --template-file cloudformation/single-instance-docker.yml \
+  --no-fail-on-empty-changeset \
+  --capabilities CAPABILITY_NAMED_IAM \
+  --parameter-overrides \
+    EC2InstanceType=$DEFAULT_EC2_INSTANCE_TYPE \
+    KeyName=$AWS_KEY_PAIR_NAME
+
+# If the deploy succeeded, get the IP and configure the created instance
+if [ $? -eq 0 ]; then
+  # Install additional Ansible roles from requirements
+  ansible-galaxy install -r requirements.yml
+
+  SERVER_IP=$(get_aws_export $SINGLE_NODE_STACK_NAME "PublicIp")
+
+  echo -e "New Node Public IP: $SERVER_IP"
+
+  echo -e "\n\n=========== Configuring node ==========="
+  ansible-playbook -i $SERVER_IP, --private-key $KEY_PATH deploy-playground-playbook.yml \
+    --extra-vars "branch_name=$BRANCH_NAME git_repo=$GIT_REPO"
+fi

devops/aws/deploy-single-node.sample.cfg

@@ -7,8 +7,6 @@ AWS_KEY_PAIR_NAME="joystream-key"
 
 DEFAULT_EC2_INSTANCE_TYPE=t2.micro
 
-ACCOUNT_ID=$(aws sts get-caller-identity --profile $CLI_PROFILE --query Account --output text)
-
 ## Used for Deploying a new node
 DATE_TIME=$(date +"%d-%b-%Y-%H-%M-%S")
 
@@ -16,3 +14,6 @@ SINGLE_NODE_STACK_NAME="joystream-node-$DATE_TIME"
 
 BINARY_FILE="https://github.com/Joystream/joystream/releases/download/v9.3.0/joystream-node-5.1.0-9d9e77751-x86_64-linux-gnu.tar.gz"
 CHAIN_SPEC_FILE="https://github.com/Joystream/joystream/releases/download/v9.3.0/joy-testnet-5.json"
+
+GIT_REPO="https://github.com/Joystream/joystream.git"
+BRANCH_NAME="master"

devops/aws/deploy-single-node.sh

@@ -13,11 +13,6 @@ else
   source $1
 fi
 
-if [ $ACCOUNT_ID == None ]; then
-    echo "Couldn't find Account ID, please check if AWS Profile $CLI_PROFILE is set"
-    exit 1
-fi
-
 if [ ! -f "$KEY_PATH" ]; then
     echo "Key file not found at $KEY_PATH"
     exit 1

devops/aws/roles/common/tasks/get-code-git.yml

@@ -5,6 +5,7 @@
   file:
     state: absent
     path: "{{ remote_code_path }}"
+  become: yes
 
 - name: Git checkout
   git:

devops/aws/templates/Playground-Caddyfile.j2

@@ -0,0 +1,49 @@
+{{ nip_domain }}/ws-rpc* {
+    uri strip_prefix /ws-rpc
+    reverse_proxy localhost:9944
+}
+
+{{ nip_domain }}/http-rpc* {
+    uri strip_prefix /http-rpc
+    reverse_proxy localhost:9933
+}
+
+{{ nip_domain }}/pioneer* {
+    uri strip_prefix /pioneer
+    reverse_proxy localhost:3000
+}
+
+{{ nip_domain }}/colossus-1* {
+    uri strip_prefix /colossus-1
+    reverse_proxy localhost:3333
+}
+
+{{ nip_domain }}/distributor-1* {
+    uri strip_prefix /distributor-1
+    reverse_proxy localhost:3334
+}
+
+# newer versions of graphql-server seems to expect this url also
+{{ nip_domain }}/@apollographql/* {
+    reverse_proxy localhost:8081
+}
+
+{{ nip_domain }}/query-node/server* {
+    uri strip_prefix /query-node/server
+    reverse_proxy localhost:8081
+}
+
+{{ nip_domain }}/query-node/indexer* {
+    uri strip_prefix /query-node/indexer
+    reverse_proxy localhost:4000
+}
+
+{{ nip_domain }}/orion* {
+    uri strip_prefix /orion
+    reverse_proxy localhost:6116
+}
+
+{{ nip_domain }}/member-faucet* {
+    uri strip_prefix /member-faucet
+    reverse_proxy localhost:3002
+}

devops/git-hooks/pre-push

@@ -1,13 +1,13 @@
 #!/bin/sh
 set -e
 
-export WASM_BUILD_TOOLCHAIN=nightly-2021-03-24
+export WASM_BUILD_TOOLCHAIN=nightly-2021-02-20
 
 echo 'running clippy (rust linter)'
 # When custom build.rs triggers wasm-build-runner-impl to build we get error:
 # "Rust WASM toolchain not installed, please install it!"
 # So we skip building the WASM binary by setting BUILD_DUMMY_WASM_BINARY=1
-BUILD_DUMMY_WASM_BINARY=1 cargo +nightly-2021-03-24 clippy --release --all -- -D warnings
+BUILD_DUMMY_WASM_BINARY=1 cargo +nightly-2021-02-20 clippy --release --all -- -D warnings
 
 echo 'running cargo unit tests'
-cargo +nightly-2021-03-24 test --release --all
+cargo +nightly-2021-02-20 test --release --all

distributor-node/.eslintignore

@@ -1 +1,2 @@
 src/types/generated
+src/services/networking/query-node/generated

distributor-node/.prettierignore

@@ -1,4 +1,3 @@
-/**/generated
 /**/mock.graphql
 lib
 local

distributor-node/README.md

@@ -27,7 +27,9 @@ To determine environment variable name based on a config key, for example `inter
 - replace all dots with `__`: `INTERVALS.CACHE_CLEANUP` => `INTERVALS__CACHE_CLEANUP`
 - add `JOYSTREAM_DISTRIBUTOR__` prefix: `INTERVALS__CACHE_CLEANUP` => `JOYSTREAM_DISTRIBUTOR__INTERVALS__CACHE_CLEANUP`
 
-In case of arrays, the values must be provided as json string, for example `JOYSTREAM_DISTRIBUTOR__KEYS="[{\"suri\":\"//Bob\"}]"`.
+In case of arrays or `oneOf` objects (ie. `keys`), the values must be provided as json string, for example `JOYSTREAM_DISTRIBUTOR__KEYS="[{\"suri\":\"//Bob\"}]"`.
+
+In order to unset a value you can use one of the following strings as env variable value: `"off"` `"null"`, `"undefined"`, for example: `JOYSTREAM_DISTRIBUTOR__LOGS__FILE="off"`.
 
 For more envirnoment variable examples see the `distributor-node` service configuration in [docker-compose.yml](../docker-compose.yml).
 

distributor-node/config.yml

@@ -2,29 +2,39 @@ id: test-node
 endpoints:
   queryNode: http://localhost:8081/graphql
   joystreamNodeWs: ws://localhost:9944
-  # elasticSearch: http://localhost:9200
 directories:
   assets: ./local/data
   cacheState: ./local/cache
-  logs: ./local/logs
-log:
-  file: debug
-  console: verbose
-  # elastic: info
+logs:
+  file:
+    level: debug
+    path: ./local/logs
+    maxFiles: 5
+    maxSize: 1000000
+  console:
+    level: verbose
+  # elastic:
+  #   level: info
+  #   endpoint: http://localhost:9200
 limits:
   storage: 100G
   maxConcurrentStorageNodeDownloads: 100
   maxConcurrentOutboundConnections: 300
-  outboundRequestsTimeout: 5000
+  outboundRequestsTimeoutMs: 5000
+  pendingDownloadTimeoutSec: 3600
+  maxCachedItemSize: 1G
 intervals:
   saveCacheState: 60
   checkStorageNodeResponseTimes: 60
   cacheCleanup: 60
-port: 3334
+publicApi:
+  port: 3334
+operatorApi:
+  port: 3335
+  hmacSecret: this-is-not-so-secret
 keys:
   - suri: //Alice
   # - mnemonic: "escape naive annual throw tragic achieve grunt verify cram note harvest problem"
   #   type: ed25519
   # - keyfile: "/path/to/keyfile.json"
-buckets: 'all'
 workerId: 0

distributor-node/docs/api/operator/index.md

@@ -0,0 +1,375 @@
+---
+title: Distributor node operator API v0.1.0
+language_tabs:
+  - javascript: JavaScript
+  - shell: Shell
+language_clients:
+  - javascript: ""
+  - shell: ""
+toc_footers: []
+includes: []
+search: true
+highlight_theme: darkula
+headingLevel: 2
+
+---
+
+<!-- AUTO-GENERATED-CONTENT:START (TOC) -->
+<!-- AUTO-GENERATED-CONTENT:END -->
+
+<h1 id="distributor-node-operator-api">Distributor node operator API v0.1.0</h1>
+
+> Scroll down for code samples, example requests and responses.
+
+Distributor node operator API
+
+Base URLs:
+
+* <a href="http://localhost:3335/api/v1/">http://localhost:3335/api/v1/</a>
+
+Email: <a href="mailto:info@joystream.org">Support</a> 
+License: <a href="https://spdx.org/licenses/GPL-3.0-only.html">GPL-3.0-only</a>
+
+undefined
+
+<h1 id="distributor-node-operator-api-default">Default</h1>
+
+## operator.stopApi
+
+<a id="opIdoperator.stopApi"></a>
+
+> Code samples
+
+```javascript
+
+const headers = {
+  'Authorization':'Bearer {access-token}'
+};
+
+fetch('http://localhost:3335/api/v1/stop-api',
+{
+  method: 'POST',
+
+  headers: headers
+})
+.then(function(res) {
+    return res.json();
+}).then(function(body) {
+    console.log(body);
+});
+
+```
+
+```shell
+# You can also use wget
+curl -X POST http://localhost:3335/api/v1/stop-api \
+  -H 'Authorization: Bearer {access-token}'
+
+```
+
+`POST /stop-api`
+
+Turns off the public api.
+
+<h3 id="operator.stopapi-responses">Responses</h3>
+
+|Status|Meaning|Description|Schema|
+|---|---|---|---|
+|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|OK|None|
+|401|[Unauthorized](https://tools.ietf.org/html/rfc7235#section-3.1)|Not authorized|None|
+|409|[Conflict](https://tools.ietf.org/html/rfc7231#section-6.5.8)|Already stopped|None|
+|500|[Internal Server Error](https://tools.ietf.org/html/rfc7231#section-6.6.1)|Unexpected server error|None|
+
+<aside class="warning">
+To perform this operation, you must be authenticated by means of one of the following methods:
+OperatorAuth
+</aside>
+
+## operator.startApi
+
+<a id="opIdoperator.startApi"></a>
+
+> Code samples
+
+```javascript
+
+const headers = {
+  'Authorization':'Bearer {access-token}'
+};
+
+fetch('http://localhost:3335/api/v1/start-api',
+{
+  method: 'POST',
+
+  headers: headers
+})
+.then(function(res) {
+    return res.json();
+}).then(function(body) {
+    console.log(body);
+});
+
+```
+
+```shell
+# You can also use wget
+curl -X POST http://localhost:3335/api/v1/start-api \
+  -H 'Authorization: Bearer {access-token}'
+
+```
+
+`POST /start-api`
+
+Turns on the public api.
+
+<h3 id="operator.startapi-responses">Responses</h3>
+
+|Status|Meaning|Description|Schema|
+|---|---|---|---|
+|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|OK|None|
+|401|[Unauthorized](https://tools.ietf.org/html/rfc7235#section-3.1)|Not authorized|None|
+|409|[Conflict](https://tools.ietf.org/html/rfc7231#section-6.5.8)|Already started|None|
+|500|[Internal Server Error](https://tools.ietf.org/html/rfc7231#section-6.6.1)|Unexpected server error|None|
+
+<aside class="warning">
+To perform this operation, you must be authenticated by means of one of the following methods:
+OperatorAuth
+</aside>
+
+## operator.shutdown
+
+<a id="opIdoperator.shutdown"></a>
+
+> Code samples
+
+```javascript
+
+const headers = {
+  'Authorization':'Bearer {access-token}'
+};
+
+fetch('http://localhost:3335/api/v1/shutdown',
+{
+  method: 'POST',
+
+  headers: headers
+})
+.then(function(res) {
+    return res.json();
+}).then(function(body) {
+    console.log(body);
+});
+
+```
+
+```shell
+# You can also use wget
+curl -X POST http://localhost:3335/api/v1/shutdown \
+  -H 'Authorization: Bearer {access-token}'
+
+```
+
+`POST /shutdown`
+
+Shuts down the node.
+
+<h3 id="operator.shutdown-responses">Responses</h3>
+
+|Status|Meaning|Description|Schema|
+|---|---|---|---|
+|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|OK|None|
+|401|[Unauthorized](https://tools.ietf.org/html/rfc7235#section-3.1)|Not authorized|None|
+|409|[Conflict](https://tools.ietf.org/html/rfc7231#section-6.5.8)|Already shutting down|None|
+|500|[Internal Server Error](https://tools.ietf.org/html/rfc7231#section-6.6.1)|Unexpected server error|None|
+
+<aside class="warning">
+To perform this operation, you must be authenticated by means of one of the following methods:
+OperatorAuth
+</aside>
+
+## operator.setWorker
+
+<a id="opIdoperator.setWorker"></a>
+
+> Code samples
+
+```javascript
+const inputBody = '{
+  "workerId": 0
+}';
+const headers = {
+  'Content-Type':'application/json',
+  'Authorization':'Bearer {access-token}'
+};
+
+fetch('http://localhost:3335/api/v1/set-worker',
+{
+  method: 'POST',
+  body: inputBody,
+  headers: headers
+})
+.then(function(res) {
+    return res.json();
+}).then(function(body) {
+    console.log(body);
+});
+
+```
+
+```shell
+# You can also use wget
+curl -X POST http://localhost:3335/api/v1/set-worker \
+  -H 'Content-Type: application/json' \
+  -H 'Authorization: Bearer {access-token}'
+
+```
+
+`POST /set-worker`
+
+Updates the operator worker id.
+
+> Body parameter
+
+```json
+{
+  "workerId": 0
+}
+```
+
+<h3 id="operator.setworker-parameters">Parameters</h3>
+
+|Name|In|Type|Required|Description|
+|---|---|---|---|---|
+|body|body|[SetWorkerOperation](#schemasetworkeroperation)|false|none|
+
+<h3 id="operator.setworker-responses">Responses</h3>
+
+|Status|Meaning|Description|Schema|
+|---|---|---|---|
+|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|OK|None|
+|401|[Unauthorized](https://tools.ietf.org/html/rfc7235#section-3.1)|Not authorized|None|
+|500|[Internal Server Error](https://tools.ietf.org/html/rfc7231#section-6.6.1)|Unexpected server error|None|
+
+<aside class="warning">
+To perform this operation, you must be authenticated by means of one of the following methods:
+OperatorAuth
+</aside>
+
+## operator.setBuckets
+
+<a id="opIdoperator.setBuckets"></a>
+
+> Code samples
+
+```javascript
+const inputBody = '{
+  "buckets": [
+    0
+  ]
+}';
+const headers = {
+  'Content-Type':'application/json',
+  'Authorization':'Bearer {access-token}'
+};
+
+fetch('http://localhost:3335/api/v1/set-buckets',
+{
+  method: 'POST',
+  body: inputBody,
+  headers: headers
+})
+.then(function(res) {
+    return res.json();
+}).then(function(body) {
+    console.log(body);
+});
+
+```
+
+```shell
+# You can also use wget
+curl -X POST http://localhost:3335/api/v1/set-buckets \
+  -H 'Content-Type: application/json' \
+  -H 'Authorization: Bearer {access-token}'
+
+```
+
+`POST /set-buckets`
+
+Updates buckets supported by the node.
+
+> Body parameter
+
+```json
+{
+  "buckets": [
+    0
+  ]
+}
+```
+
+<h3 id="operator.setbuckets-parameters">Parameters</h3>
+
+|Name|In|Type|Required|Description|
+|---|---|---|---|---|
+|body|body|[SetBucketsOperation](#schemasetbucketsoperation)|false|none|
+
+<h3 id="operator.setbuckets-responses">Responses</h3>
+
+|Status|Meaning|Description|Schema|
+|---|---|---|---|
+|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|OK|None|
+|401|[Unauthorized](https://tools.ietf.org/html/rfc7235#section-3.1)|Not authorized|None|
+|500|[Internal Server Error](https://tools.ietf.org/html/rfc7231#section-6.6.1)|Unexpected server error|None|
+
+<aside class="warning">
+To perform this operation, you must be authenticated by means of one of the following methods:
+OperatorAuth
+</aside>
+
+# Schemas
+
+<h2 id="tocS_SetWorkerOperation">SetWorkerOperation</h2>
+
+<a id="schemasetworkeroperation"></a>
+<a id="schema_SetWorkerOperation"></a>
+<a id="tocSsetworkeroperation"></a>
+<a id="tocssetworkeroperation"></a>
+
+```json
+{
+  "workerId": 0
+}
+
+```
+
+### Properties
+
+|Name|Type|Required|Restrictions|Description|
+|---|---|---|---|---|
+|workerId|integer|true|none|none|
+
+<h2 id="tocS_SetBucketsOperation">SetBucketsOperation</h2>
+
+<a id="schemasetbucketsoperation"></a>
+<a id="schema_SetBucketsOperation"></a>
+<a id="tocSsetbucketsoperation"></a>
+<a id="tocssetbucketsoperation"></a>
+
+```json
+{
+  "buckets": [
+    0
+  ]
+}
+
+```
+
+### Properties
+
+|Name|Type|Required|Restrictions|Description|
+|---|---|---|---|---|
+|buckets|[integer]|false|none|Set of bucket ids to be distributed by the node. If not provided - all buckets assigned to currently configured worker will be distributed.|
+
+undefined
+

distributor-node/docs/api/index.md → distributor-node/docs/api/public/index.md

@@ -1,5 +1,5 @@
 ---
-title: Distributor node API v0.1.0
+title: Distributor node public API v0.1.0
 language_tabs:
   - javascript: JavaScript
   - shell: Shell
@@ -8,7 +8,7 @@ language_clients:
   - shell: ""
 toc_footers:
   - <a href="https://github.com/Joystream/joystream/issues/2224">Distributor
-    node API</a>
+    node public API</a>
 includes: []
 search: true
 highlight_theme: darkula
@@ -17,33 +17,13 @@ headingLevel: 2
 ---
 
 <!-- AUTO-GENERATED-CONTENT:START (TOC) -->
-- [public](#public)
-- [public.status](#publicstatus)
-  - [Responses](#responses)
-  - [Responses](#responses-1)
-- [public.buckets](#publicbuckets)
-- [public.assetHead](#publicassethead)
-  - [Parameters](#parameters)
-  - [Responses](#responses-2)
-  - [Response Headers](#response-headers)
-- [public.asset](#publicasset)
-  - [Parameters](#parameters-1)
-  - [Responses](#responses-3)
-- [ErrorResponse](#errorresponse)
-  - [Response Headers](#response-headers-1)
-- [Schemas](#schemas)
-  - [Properties](#properties)
-- [StatusResponse](#statusresponse)
-  - [Properties](#properties-1)
-- [BucketsResponse](#bucketsresponse)
-  - [Properties](#properties-2)
 <!-- AUTO-GENERATED-CONTENT:END -->
 
-<h1 id="distributor-node-api">Distributor node API v0.1.0</h1>
+<h1 id="distributor-node-public-api">Distributor node public API v0.1.0</h1>
 
 > Scroll down for code samples, example requests and responses.
 
-Distributor node API
+Distributor node public API
 
 Base URLs:
 
@@ -52,9 +32,7 @@ Base URLs:
 Email: <a href="mailto:info@joystream.org">Support</a> 
 License: <a href="https://spdx.org/licenses/GPL-3.0-only.html">GPL-3.0-only</a>
 
-<h1 id="distributor-node-api-public">public</h1>
-
-Public distributor node API
+<h1 id="distributor-node-public-api-default">Default</h1>
 
 ## public.status
 
@@ -187,7 +165,7 @@ This operation does not require authentication
 
 ```javascript
 
-fetch('http://localhost:3334/api/v1/asset/{objectId}',
+fetch('http://localhost:3334/api/v1/assets/{objectId}',
 {
   method: 'HEAD'
 
@@ -202,11 +180,11 @@ fetch('http://localhost:3334/api/v1/asset/{objectId}',
 
 ```shell
 # You can also use wget
-curl -X HEAD http://localhost:3334/api/v1/asset/{objectId}
+curl -X HEAD http://localhost:3334/api/v1/assets/{objectId}
 
 ```
 
-`HEAD /asset/{objectId}`
+`HEAD /assets/{objectId}`
 
 Returns asset response headers (cache status, content type and/or length, accepted ranges etc.)
 
@@ -247,7 +225,7 @@ const headers = {
   'Accept':'image/*'
 };
 
-fetch('http://localhost:3334/api/v1/asset/{objectId}',
+fetch('http://localhost:3334/api/v1/assets/{objectId}',
 {
   method: 'GET',
 
@@ -263,12 +241,12 @@ fetch('http://localhost:3334/api/v1/asset/{objectId}',
 
 ```shell
 # You can also use wget
-curl -X GET http://localhost:3334/api/v1/asset/{objectId} \
+curl -X GET http://localhost:3334/api/v1/assets/{objectId} \
   -H 'Accept: image/*'
 
 ```
 
-`GET /asset/{objectId}`
+`GET /assets/{objectId}`
 
 Returns a media file.
 

distributor-node/docs/commands/dev.md

@@ -9,8 +9,6 @@ Developer utility commands
 ## `joystream-distributor dev:batchUpload`
 
 ```
-undefined
-
 USAGE
   $ joystream-distributor dev:batchUpload
 
@@ -33,8 +31,6 @@ _See code: [src/commands/dev/batchUpload.ts](https://github.com/Joystream/joystr
 Initialize development environment. Sets Alice as distributor working group leader.
 
 ```
-Initialize development environment. Sets Alice as distributor working group leader.
-
 USAGE
   $ joystream-distributor dev:init
 

distributor-node/docs/commands/help.md

@@ -10,8 +10,6 @@ display help for joystream-distributor
 display help for joystream-distributor
 
 ```
-display help for <%= config.bin %>
-
 USAGE
   $ joystream-distributor help [COMMAND]
 
@@ -22,4 +20,4 @@ OPTIONS
   --all  see all commands in CLI
 ```
 
-_See code: [@oclif/plugin-help](https://github.com/oclif/plugin-help/blob/v2.2.3/src/commands/help.ts)_
+_See code: [@oclif/plugin-help](https://github.com/oclif/plugin-help/blob/v3.2.2/src/commands/help.ts)_

distributor-node/docs/commands/leader.md

@@ -22,9 +22,6 @@ Commands for performing Distribution Working Group leader on-chain duties (like
 Cancel pending distribution bucket operator invitation.
 
 ```
-Cancel pending distribution bucket operator invitation.
-  Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:cancel-invitation
 
@@ -51,8 +48,6 @@ _See code: [src/commands/leader/cancel-invitation.ts](https://github.com/Joystre
 Create new distribution bucket. Requires distribution working group leader permissions.
 
 ```
-Create new distribution bucket. Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:create-bucket
 
@@ -74,8 +69,6 @@ _See code: [src/commands/leader/create-bucket.ts](https://github.com/Joystream/j
 Create new distribution bucket family. Requires distribution working group leader permissions.
 
 ```
-Create new distribution bucket family. Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:create-bucket-family
 
@@ -93,8 +86,6 @@ _See code: [src/commands/leader/create-bucket-family.ts](https://github.com/Joys
 Delete distribution bucket. The bucket must have no operators. Requires distribution working group leader permissions.
 
 ```
-Delete distribution bucket. The bucket must have no operators. Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:delete-bucket
 
@@ -116,8 +107,6 @@ _See code: [src/commands/leader/delete-bucket.ts](https://github.com/Joystream/j
 Delete distribution bucket family. Requires distribution working group leader permissions.
 
 ```
-Delete distribution bucket family. Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:delete-bucket-family
 
@@ -137,10 +126,6 @@ _See code: [src/commands/leader/delete-bucket-family.ts](https://github.com/Joys
 Invite distribution bucket operator (distribution group worker).
 
 ```
-Invite distribution bucket operator (distribution group worker).
-  The specified bucket must not have any operator currently.
-  Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:invite-bucket-operator
 
@@ -168,9 +153,6 @@ _See code: [src/commands/leader/invite-bucket-operator.ts](https://github.com/Jo
 Remove distribution bucket operator (distribution group worker).
 
 ```
-Remove distribution bucket operator (distribution group worker).
-  Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:remove-bucket-operator
 
@@ -198,9 +180,6 @@ _See code: [src/commands/leader/remove-bucket-operator.ts](https://github.com/Jo
 Set/update distribution bucket family metadata.
 
 ```
-Set/update distribution bucket family metadata.
-  Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:set-bucket-family-metadata
 
@@ -225,8 +204,6 @@ _See code: [src/commands/leader/set-bucket-family-metadata.ts](https://github.co
 Set max. distribution buckets per bag limit. Requires distribution working group leader permissions.
 
 ```
-Set max. distribution buckets per bag limit. Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:set-buckets-per-bag-limit
 
@@ -246,8 +223,6 @@ _See code: [src/commands/leader/set-buckets-per-bag-limit.ts](https://github.com
 Add/remove distribution buckets from a bag.
 
 ```
-Add/remove distribution buckets from a bag.
-
 USAGE
   $ joystream-distributor leader:update-bag
 
@@ -291,8 +266,6 @@ _See code: [src/commands/leader/update-bag.ts](https://github.com/Joystream/joys
 Update distribution bucket mode ("distributing" flag). Requires distribution working group leader permissions.
 
 ```
-Update distribution bucket mode ("distributing" flag). Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:update-bucket-mode
 
@@ -316,8 +289,6 @@ _See code: [src/commands/leader/update-bucket-mode.ts](https://github.com/Joystr
 Update distribution bucket status ("acceptingNewBags" flag). Requires distribution working group leader permissions.
 
 ```
-Update distribution bucket status ("acceptingNewBags" flag). Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:update-bucket-status
 
@@ -340,9 +311,6 @@ _See code: [src/commands/leader/update-bucket-status.ts](https://github.com/Joys
 Update dynamic bag creation policy (number of buckets by family that should store given dynamic bag type).
 
 ```
-Update dynamic bag creation policy (number of buckets by family that should store given dynamic bag type).
-    Requires distribution working group leader permissions.
-
 USAGE
   $ joystream-distributor leader:update-dynamic-bag-policy
 

distributor-node/docs/commands/node.md

@@ -0,0 +1,120 @@
+`joystream-distributor node`
+============================
+
+Commands for interacting with a running distributor node through OperatorApi
+
+* [`joystream-distributor node:set-buckets`](#joystream-distributor-nodeset-buckets)
+* [`joystream-distributor node:set-worker`](#joystream-distributor-nodeset-worker)
+* [`joystream-distributor node:shutdown`](#joystream-distributor-nodeshutdown)
+* [`joystream-distributor node:start-public-api`](#joystream-distributor-nodestart-public-api)
+* [`joystream-distributor node:stop-public-api`](#joystream-distributor-nodestop-public-api)
+
+## `joystream-distributor node:set-buckets`
+
+Send an api request to change the set of buckets distributed by given distributor node.
+
+```
+USAGE
+  $ joystream-distributor node:set-buckets
+
+OPTIONS
+  -B, --bucketIds=bucketIds    Set of bucket ids to distribute
+  -a, --all                    Distribute all buckets belonging to configured worker
+
+  -c, --configPath=configPath  [default: ./config.yml] Path to config JSON/YAML file (relative to current working
+                               directory)
+
+  -s, --secret=secret          HMAC secret key to use (will default to config.operatorApi.hmacSecret if present)
+
+  -u, --url=url                (required) Distributor node operator api base url (ie. http://localhost:3335)
+
+  -y, --yes                    Answer "yes" to any prompt, skipping any manual confirmations
+```
+
+_See code: [src/commands/node/set-buckets.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/node/set-buckets.ts)_
+
+## `joystream-distributor node:set-worker`
+
+Send an api request to change workerId assigned to given distributor node instance.
+
+```
+USAGE
+  $ joystream-distributor node:set-worker
+
+OPTIONS
+  -c, --configPath=configPath  [default: ./config.yml] Path to config JSON/YAML file (relative to current working
+                               directory)
+
+  -s, --secret=secret          HMAC secret key to use (will default to config.operatorApi.hmacSecret if present)
+
+  -u, --url=url                (required) Distributor node operator api base url (ie. http://localhost:3335)
+
+  -w, --workerId=workerId      (required) New workerId to set
+
+  -y, --yes                    Answer "yes" to any prompt, skipping any manual confirmations
+```
+
+_See code: [src/commands/node/set-worker.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/node/set-worker.ts)_
+
+## `joystream-distributor node:shutdown`
+
+Send an api request to shutdown given distributor node.
+
+```
+USAGE
+  $ joystream-distributor node:shutdown
+
+OPTIONS
+  -c, --configPath=configPath  [default: ./config.yml] Path to config JSON/YAML file (relative to current working
+                               directory)
+
+  -s, --secret=secret          HMAC secret key to use (will default to config.operatorApi.hmacSecret if present)
+
+  -u, --url=url                (required) Distributor node operator api base url (ie. http://localhost:3335)
+
+  -y, --yes                    Answer "yes" to any prompt, skipping any manual confirmations
+```
+
+_See code: [src/commands/node/shutdown.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/node/shutdown.ts)_
+
+## `joystream-distributor node:start-public-api`
+
+Send an api request to start public api of given distributor node.
+
+```
+USAGE
+  $ joystream-distributor node:start-public-api
+
+OPTIONS
+  -c, --configPath=configPath  [default: ./config.yml] Path to config JSON/YAML file (relative to current working
+                               directory)
+
+  -s, --secret=secret          HMAC secret key to use (will default to config.operatorApi.hmacSecret if present)
+
+  -u, --url=url                (required) Distributor node operator api base url (ie. http://localhost:3335)
+
+  -y, --yes                    Answer "yes" to any prompt, skipping any manual confirmations
+```
+
+_See code: [src/commands/node/start-public-api.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/node/start-public-api.ts)_
+
+## `joystream-distributor node:stop-public-api`
+
+Send an api request to stop public api of given distributor node.
+
+```
+USAGE
+  $ joystream-distributor node:stop-public-api
+
+OPTIONS
+  -c, --configPath=configPath  [default: ./config.yml] Path to config JSON/YAML file (relative to current working
+                               directory)
+
+  -s, --secret=secret          HMAC secret key to use (will default to config.operatorApi.hmacSecret if present)
+
+  -u, --url=url                (required) Distributor node operator api base url (ie. http://localhost:3335)
+
+  -y, --yes                    Answer "yes" to any prompt, skipping any manual confirmations
+```
+
+_See code: [src/commands/node/stop-public-api.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/node/stop-public-api.ts)_

distributor-node/docs/commands/operator.md

@@ -11,9 +11,6 @@ Commands for performing node operator (Distribution Working Group worker) on-cha
 Accept pending distribution bucket operator invitation.
 
 ```
-Accept pending distribution bucket operator invitation.
-  Requires the invited distribution group worker role key.
-
 USAGE
   $ joystream-distributor operator:accept-invitation
 
@@ -40,9 +37,6 @@ _See code: [src/commands/operator/accept-invitation.ts](https://github.com/Joyst
 Set/update distribution bucket operator metadata.
 
 ```
-Set/update distribution bucket operator metadata.
-  Requires active distribution bucket operator worker role key.
-
 USAGE
   $ joystream-distributor operator:set-metadata
 

distributor-node/docs/commands/start.md

@@ -10,8 +10,6 @@ Start the node
 Start the node
 
 ```
-Start the node
-
 USAGE
   $ joystream-distributor start
 

distributor-node/docs/node/index.md

@@ -1,5 +1,7 @@
 <!-- AUTO-GENERATED-CONTENT:START (TOC:firsth1=true) -->
-- [The API](#the-api)
+- [API](#api)
+  - [Public API](#public-api)
+  - [Operator API](#operator-api)
   - [Requesting assets](#requesting-assets)
     - [Scenario 1 (cache hit)](#scenario-1-cache-hit)
     - [Scenario 2 (pending)](#scenario-2-pending)
@@ -32,19 +34,32 @@
 
 <a name="the-api"></a>
 
-# The API
+# API
 
-The Distributor Node exposes an HTTP api implemented with [ExpressJS](https://expressjs.com/).
+The Distributor Node, depending on the configuration, can expose either one or two HTTP APIs, both implemented with [ExpressJS](https://expressjs.com/).
 
-The api is described by an [OpenAPI](https://swagger.io/specification/) schema located at _[src/api-spec/openapi.yml](../../src/api-spec/openapi.yml)_
+## Public API
 
-**Current, detailed api documentation can be found [here](../api/index.md)**
+Public API is enabled by default and can be used to retrieve assets from the node as well as some basic information about its current status.
+
+Public API is described by an [OpenAPI](https://swagger.io/specification/) schema located at _[src/api-spec/public.yml](../../src/api-spec/public.yml)_
+
+**Full public API documentation can be found [here](../api/public/index.md)**
+
+## Operator API
+
+Secured operator API can be enabled with [`config.operatorApi`](../schema/definition-properties-operatorapi.md).
+Operator API makes it possible to remotely execute some operations on a running node (ie. changing supported buckets).
+
+Operator API is described by an [OpenAPI](https://swagger.io/specification/) schema located at _[src/api-spec/operator.yml](../../src/api-spec/operator.yml)_
+
+**Full operator API documentation can be found [here](../api/operator/index.md)**
 
 <a name="requesting-assets"></a>
 
 ## Requesting assets
 
-The assets are requested from the distributor node by using a `GET` request to [`/asset/{objectId}`](../api/index.md#opIdpublic.asset) endpoint.
+The assets are requested from the distributor node by using a `GET` request to [`/assets/{objectId}`](../api/index.md#opIdpublic.asset) endpoint.
 
 There are multiple scenarios of how a distributor will act upon that request, depending on its current state:
 
@@ -120,7 +135,7 @@ In this case
 
 ## Checking asset status
 
-It is possible to check an asset status without affecting the distributor node state in any way (for example - by triggering the process of [fetching the missing data object](#data-fetching)), by sending a [`HEAD` request to `/asset/{objectId}`](../api/index.md#opIdpublic.assetHead) endpoint.
+It is possible to check an asset status without affecting the distributor node state in any way (for example - by triggering the process of [fetching the missing data object](#data-fetching)), by sending a [`HEAD` request to `/assets/{objectId}`](../api/index.md#opIdpublic.assetHead) endpoint.
 
 If the request is valid, the node will respond with, among others, the `x-cache`, `content-length`, `cache-control` headers.
 
@@ -378,10 +393,10 @@ No-longer-distributed data objects are dropped from the cache periodically every
 
 The distributor node supports detailed logging with [winston](https://www.npmjs.com/package/winston) library. [NPM log levels](https://www.npmjs.com/package/winston#logging-levels) are used to specify the log priority.
 
-The logs can be directed to some of the 3 available outputs, depending on the [`log`](../schema/definition-properties-log.md) configuration settings:
-- console
-- a log file inside [`directories.logs`](../schema/definition-properties-directories.md#logs)
-- an elasticsearch endpoint specified via [`endpoints.elasticsearch`](../schema/definition-properties-endpoints.md#elasticsearch)
+The logs can be directed to some of the 3 available outputs, depending on the [`logs`](../schema/definition-properties-logs.md) configuration settings:
+- console ([`logs.console`](../schema/definition-properties-logs-properties-console.md))
+- log file(s) ([`logs.file`](../schema/definition-properties-logs-properties-file.md))
+- an elasticsearch endpoint ([`logs.elastic`](../schema/definition-properties-logs-properties-elastic.md))
 
 # Query node integration
 

distributor-node/docs/schema/definition-properties-bucket-ids.md

@@ -0,0 +1,9 @@
+## buckets Type
+
+`integer[]`
+
+## buckets Constraints
+
+**minimum number of items**: the minimum number of items for this array is: `1`
+
+**unique items**: all items in this array must be unique. Duplicates are not allowed.

distributor-node/docs/schema/definition-properties-buckets-oneof-all-buckets.md

@@ -1,11 +0,0 @@
-## 1 Type
-
-`string` ([All buckets](definition-properties-buckets-oneof-all-buckets.md))
-
-## 1 Constraints
-
-**enum**: the value of this property must be equal to one of the following values:
-
-| Value   | Explanation |
-| :------ | :---------- |
-| `"all"` |             |

distributor-node/docs/schema/definition-properties-buckets-oneof-bucket-ids.md

@@ -1,7 +0,0 @@
-## 0 Type
-
-`integer[]`
-
-## 0 Constraints
-
-**minimum number of items**: the minimum number of items for this array is: `1`

distributor-node/docs/schema/definition-properties-buckets.md

@@ -1,9 +0,0 @@
-## buckets Type
-
-merged type ([Details](definition-properties-buckets.md))
-
-one (and only one) of
-
-*   [Bucket ids](definition-properties-buckets-oneof-bucket-ids.md "check type definition")
-
-*   [All buckets](definition-properties-buckets-oneof-all-buckets.md "check type definition")

distributor-node/docs/schema/definition-properties-directories-properties-logs.md

@@ -1,3 +0,0 @@
-## logs Type
-
-`string`

distributor-node/docs/schema/definition-properties-directories.md

@@ -4,11 +4,10 @@
 
 # directories Properties
 
-| Property                  | Type     | Required | Nullable       | Defined by                                                                                                                                             |
-| :------------------------ | :------- | :------- | :------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [assets](#assets)         | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-directories-properties-assets.md "undefined#/properties/directories/properties/assets")         |
-| [cacheState](#cachestate) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-directories-properties-cachestate.md "undefined#/properties/directories/properties/cacheState") |
-| [logs](#logs)             | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-directories-properties-logs.md "undefined#/properties/directories/properties/logs")             |
+| Property                  | Type     | Required | Nullable       | Defined by                                                                                                                                                                              |
+| :------------------------ | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [assets](#assets)         | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-directories-properties-assets.md "https://joystream.org/schemas/argus/config#/properties/directories/properties/assets")         |
+| [cacheState](#cachestate) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-directories-properties-cachestate.md "https://joystream.org/schemas/argus/config#/properties/directories/properties/cacheState") |
 
 ## assets
 
@@ -22,7 +21,7 @@ Path to a directory where all the cached assets will be stored
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-directories-properties-assets.md "undefined#/properties/directories/properties/assets")
+*   defined in: [Distributor node configuration](definition-properties-directories-properties-assets.md "https://joystream.org/schemas/argus/config#/properties/directories/properties/assets")
 
 ### assets Type
 
@@ -40,26 +39,8 @@ Path to a directory where information about the current cache state will be stor
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-directories-properties-cachestate.md "undefined#/properties/directories/properties/cacheState")
+*   defined in: [Distributor node configuration](definition-properties-directories-properties-cachestate.md "https://joystream.org/schemas/argus/config#/properties/directories/properties/cacheState")
 
 ### cacheState Type
 
 `string`
-
-## logs
-
-Path to a directory where logs will be stored if logging to a file was enabled (via `log.file`).
-
-`logs`
-
-*   is optional
-
-*   Type: `string`
-
-*   cannot be null
-
-*   defined in: [Distributor node configuration](definition-properties-directories-properties-logs.md "undefined#/properties/directories/properties/logs")
-
-### logs Type
-
-`string`

distributor-node/docs/schema/definition-properties-endpoints-properties-elasticsearch.md

@@ -1,3 +0,0 @@
-## elasticSearch Type
-
-`string`

distributor-node/docs/schema/definition-properties-endpoints.md

@@ -4,11 +4,10 @@
 
 # endpoints Properties
 
-| Property                            | Type     | Required | Nullable       | Defined by                                                                                                                                                   |
-| :---------------------------------- | :------- | :------- | :------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [queryNode](#querynode)             | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-endpoints-properties-querynode.md "undefined#/properties/endpoints/properties/queryNode")             |
-| [joystreamNodeWs](#joystreamnodews) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-endpoints-properties-joystreamnodews.md "undefined#/properties/endpoints/properties/joystreamNodeWs") |
-| [elasticSearch](#elasticsearch)     | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-endpoints-properties-elasticsearch.md "undefined#/properties/endpoints/properties/elasticSearch")     |
+| Property                            | Type     | Required | Nullable       | Defined by                                                                                                                                                                                    |
+| :---------------------------------- | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [queryNode](#querynode)             | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-endpoints-properties-querynode.md "https://joystream.org/schemas/argus/config#/properties/endpoints/properties/queryNode")             |
+| [joystreamNodeWs](#joystreamnodews) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-endpoints-properties-joystreamnodews.md "https://joystream.org/schemas/argus/config#/properties/endpoints/properties/joystreamNodeWs") |
 
 ## queryNode
 
@@ -22,7 +21,7 @@ Query node graphql server uri (for example: <http://localhost:8081/graphql>)
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-endpoints-properties-querynode.md "undefined#/properties/endpoints/properties/queryNode")
+*   defined in: [Distributor node configuration](definition-properties-endpoints-properties-querynode.md "https://joystream.org/schemas/argus/config#/properties/endpoints/properties/queryNode")
 
 ### queryNode Type
 
@@ -40,26 +39,8 @@ Joystream node websocket api uri (for example: ws\://localhost:9944)
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-endpoints-properties-joystreamnodews.md "undefined#/properties/endpoints/properties/joystreamNodeWs")
+*   defined in: [Distributor node configuration](definition-properties-endpoints-properties-joystreamnodews.md "https://joystream.org/schemas/argus/config#/properties/endpoints/properties/joystreamNodeWs")
 
 ### joystreamNodeWs Type
 
 `string`
-
-## elasticSearch
-
-Elasticsearch uri used for submitting the distributor node logs (if enabled via `log.elastic`)
-
-`elasticSearch`
-
-*   is optional
-
-*   Type: `string`
-
-*   cannot be null
-
-*   defined in: [Distributor node configuration](definition-properties-endpoints-properties-elasticsearch.md "undefined#/properties/endpoints/properties/elasticSearch")
-
-### elasticSearch Type
-
-`string`

distributor-node/docs/schema/definition-properties-intervals.md

@@ -4,11 +4,11 @@
 
 # intervals Properties
 
-| Property                                                        | Type      | Required | Nullable       | Defined by                                                                                                                                                                               |
-| :-------------------------------------------------------------- | :-------- | :------- | :------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [saveCacheState](#savecachestate)                               | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-intervals-properties-savecachestate.md "undefined#/properties/intervals/properties/saveCacheState")                               |
-| [checkStorageNodeResponseTimes](#checkstoragenoderesponsetimes) | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-intervals-properties-checkstoragenoderesponsetimes.md "undefined#/properties/intervals/properties/checkStorageNodeResponseTimes") |
-| [cacheCleanup](#cachecleanup)                                   | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-intervals-properties-cachecleanup.md "undefined#/properties/intervals/properties/cacheCleanup")                                   |
+| Property                                                        | Type      | Required | Nullable       | Defined by                                                                                                                                                                                                                |
+| :-------------------------------------------------------------- | :-------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| [saveCacheState](#savecachestate)                               | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-intervals-properties-savecachestate.md "https://joystream.org/schemas/argus/config#/properties/intervals/properties/saveCacheState")                               |
+| [checkStorageNodeResponseTimes](#checkstoragenoderesponsetimes) | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-intervals-properties-checkstoragenoderesponsetimes.md "https://joystream.org/schemas/argus/config#/properties/intervals/properties/checkStorageNodeResponseTimes") |
+| [cacheCleanup](#cachecleanup)                                   | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-intervals-properties-cachecleanup.md "https://joystream.org/schemas/argus/config#/properties/intervals/properties/cacheCleanup")                                   |
 
 ## saveCacheState
 
@@ -22,7 +22,7 @@ How often, in seconds, will the cache state be saved in `directories.state`. Ind
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-intervals-properties-savecachestate.md "undefined#/properties/intervals/properties/saveCacheState")
+*   defined in: [Distributor node configuration](definition-properties-intervals-properties-savecachestate.md "https://joystream.org/schemas/argus/config#/properties/intervals/properties/saveCacheState")
 
 ### saveCacheState Type
 
@@ -44,7 +44,7 @@ How often, in seconds, will the distributor node attempt to send requests to all
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-intervals-properties-checkstoragenoderesponsetimes.md "undefined#/properties/intervals/properties/checkStorageNodeResponseTimes")
+*   defined in: [Distributor node configuration](definition-properties-intervals-properties-checkstoragenoderesponsetimes.md "https://joystream.org/schemas/argus/config#/properties/intervals/properties/checkStorageNodeResponseTimes")
 
 ### checkStorageNodeResponseTimes Type
 
@@ -66,7 +66,7 @@ How often, in seconds, will the distributor node fetch data about all its distri
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-intervals-properties-cachecleanup.md "undefined#/properties/intervals/properties/cacheCleanup")
+*   defined in: [Distributor node configuration](definition-properties-intervals-properties-cachecleanup.md "https://joystream.org/schemas/argus/config#/properties/intervals/properties/cacheCleanup")
 
 ### cacheCleanup Type
 

distributor-node/docs/schema/definition-properties-keys-items-oneof-json-backup-file.md

@@ -4,9 +4,9 @@
 
 # 2 Properties
 
-| Property            | Type     | Required | Nullable       | Defined by                                                                                                                                                                    |
-| :------------------ | :------- | :------- | :------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [keyfile](#keyfile) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-json-backup-file-properties-keyfile.md "undefined#/properties/keys/items/oneOf/2/properties/keyfile") |
+| Property            | Type     | Required | Nullable       | Defined by                                                                                                                                                                                                     |
+| :------------------ | :------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [keyfile](#keyfile) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-json-backup-file-properties-keyfile.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/2/properties/keyfile") |
 
 ## keyfile
 
@@ -20,7 +20,7 @@
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-json-backup-file-properties-keyfile.md "undefined#/properties/keys/items/oneOf/2/properties/keyfile")
+*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-json-backup-file-properties-keyfile.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/2/properties/keyfile")
 
 ### keyfile Type
 

distributor-node/docs/schema/definition-properties-keys-items-oneof-mnemonic-phrase.md

@@ -4,10 +4,10 @@
 
 # 1 Properties
 
-| Property              | Type     | Required | Nullable       | Defined by                                                                                                                                                                     |
-| :-------------------- | :------- | :------- | :------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [type](#type)         | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-mnemonic-phrase-properties-type.md "undefined#/properties/keys/items/oneOf/1/properties/type")         |
-| [mnemonic](#mnemonic) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-mnemonic-phrase-properties-mnemonic.md "undefined#/properties/keys/items/oneOf/1/properties/mnemonic") |
+| Property              | Type     | Required | Nullable       | Defined by                                                                                                                                                                                                      |
+| :-------------------- | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [type](#type)         | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-mnemonic-phrase-properties-type.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/1/properties/type")         |
+| [mnemonic](#mnemonic) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-mnemonic-phrase-properties-mnemonic.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/1/properties/mnemonic") |
 
 ## type
 
@@ -21,7 +21,7 @@
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-mnemonic-phrase-properties-type.md "undefined#/properties/keys/items/oneOf/1/properties/type")
+*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-mnemonic-phrase-properties-type.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/1/properties/type")
 
 ### type Type
 
@@ -57,7 +57,7 @@ The default value is:
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-mnemonic-phrase-properties-mnemonic.md "undefined#/properties/keys/items/oneOf/1/properties/mnemonic")
+*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-mnemonic-phrase-properties-mnemonic.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/1/properties/mnemonic")
 
 ### mnemonic Type
 

distributor-node/docs/schema/definition-properties-keys-items-oneof-substrate-uri.md

@@ -4,10 +4,10 @@
 
 # 0 Properties
 
-| Property      | Type     | Required | Nullable       | Defined by                                                                                                                                                           |
-| :------------ | :------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [type](#type) | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-substrate-uri-properties-type.md "undefined#/properties/keys/items/oneOf/0/properties/type") |
-| [suri](#suri) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-substrate-uri-properties-suri.md "undefined#/properties/keys/items/oneOf/0/properties/suri") |
+| Property      | Type     | Required | Nullable       | Defined by                                                                                                                                                                                            |
+| :------------ | :------- | :------- | :------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [type](#type) | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-substrate-uri-properties-type.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/0/properties/type") |
+| [suri](#suri) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-keys-items-oneof-substrate-uri-properties-suri.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/0/properties/suri") |
 
 ## type
 
@@ -21,7 +21,7 @@
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-substrate-uri-properties-type.md "undefined#/properties/keys/items/oneOf/0/properties/type")
+*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-substrate-uri-properties-type.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/0/properties/type")
 
 ### type Type
 
@@ -57,7 +57,7 @@ The default value is:
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-substrate-uri-properties-suri.md "undefined#/properties/keys/items/oneOf/0/properties/suri")
+*   defined in: [Distributor node configuration](definition-properties-keys-items-oneof-substrate-uri-properties-suri.md "https://joystream.org/schemas/argus/config#/properties/keys/items/oneOf/0/properties/suri")
 
 ### suri Type
 

distributor-node/docs/schema/definition-properties-limits-properties-dataobjectsourcebyobjectidttl.md

@@ -0,0 +1,15 @@
+## dataObjectSourceByObjectIdTTL Type
+
+`integer`
+
+## dataObjectSourceByObjectIdTTL Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1`
+
+## dataObjectSourceByObjectIdTTL Default Value
+
+The default value is:
+
+```json
+60
+```

distributor-node/docs/schema/definition-properties-limits-properties-maxcacheditemsize.md

@@ -0,0 +1,13 @@
+## maxCachedItemSize Type
+
+`string`
+
+## maxCachedItemSize Constraints
+
+**pattern**: the string must match the following regular expression: 
+
+```regexp
+^[0-9]+(B|K|M|G|T)$
+```
+
+[try pattern](https://regexr.com/?expression=%5E%5B0-9%5D%2B\(B%7CK%7CM%7CG%7CT\)%24 "try regular expression with regexr.com")

distributor-node/docs/schema/definition-properties-limits-properties-outboundrequeststimeoutms.md

@@ -0,0 +1,7 @@
+## outboundRequestsTimeoutMs Type
+
+`integer`
+
+## outboundRequestsTimeoutMs Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1000`

distributor-node/docs/schema/definition-properties-limits-properties-pendingdownloadtimeoutsec.md

@@ -0,0 +1,7 @@
+## pendingDownloadTimeoutSec Type
+
+`integer`
+
+## pendingDownloadTimeoutSec Constraints
+
+**minimum**: the value of this number must greater than or equal to: `60`

distributor-node/docs/schema/definition-properties-limits.md

@@ -4,12 +4,15 @@
 
 # limits Properties
 
-| Property                                                                | Type      | Required | Nullable       | Defined by                                                                                                                                                                                 |
-| :---------------------------------------------------------------------- | :-------- | :------- | :------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [storage](#storage)                                                     | `string`  | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-storage.md "undefined#/properties/limits/properties/storage")                                                     |
-| [maxConcurrentStorageNodeDownloads](#maxconcurrentstoragenodedownloads) | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-maxconcurrentstoragenodedownloads.md "undefined#/properties/limits/properties/maxConcurrentStorageNodeDownloads") |
-| [maxConcurrentOutboundConnections](#maxconcurrentoutboundconnections)   | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-maxconcurrentoutboundconnections.md "undefined#/properties/limits/properties/maxConcurrentOutboundConnections")   |
-| [outboundRequestsTimeout](#outboundrequeststimeout)                     | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-outboundrequeststimeout.md "undefined#/properties/limits/properties/outboundRequestsTimeout")                     |
+| Property                                                                | Type      | Required | Nullable       | Defined by                                                                                                                                                                                                                  |
+| :---------------------------------------------------------------------- | :-------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [storage](#storage)                                                     | `string`  | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-storage.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/storage")                                                     |
+| [maxConcurrentStorageNodeDownloads](#maxconcurrentstoragenodedownloads) | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-maxconcurrentstoragenodedownloads.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/maxConcurrentStorageNodeDownloads") |
+| [maxConcurrentOutboundConnections](#maxconcurrentoutboundconnections)   | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-maxconcurrentoutboundconnections.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/maxConcurrentOutboundConnections")   |
+| [outboundRequestsTimeoutMs](#outboundrequeststimeoutms)                 | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-outboundrequeststimeoutms.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/outboundRequestsTimeoutMs")                 |
+| [pendingDownloadTimeoutSec](#pendingdownloadtimeoutsec)                 | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-limits-properties-pendingdownloadtimeoutsec.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/pendingDownloadTimeoutSec")                 |
+| [maxCachedItemSize](#maxcacheditemsize)                                 | `string`  | Optional | cannot be null | [Distributor node configuration](definition-properties-limits-properties-maxcacheditemsize.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/maxCachedItemSize")                                 |
+| [dataObjectSourceByObjectIdTTL](#dataobjectsourcebyobjectidttl)         | `integer` | Optional | cannot be null | [Distributor node configuration](definition-properties-limits-properties-dataobjectsourcebyobjectidttl.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/dataObjectSourceByObjectIdTTL")         |
 
 ## storage
 
@@ -23,7 +26,7 @@ Maximum total size of all (cached) assets stored in `directories.assets`
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-limits-properties-storage.md "undefined#/properties/limits/properties/storage")
+*   defined in: [Distributor node configuration](definition-properties-limits-properties-storage.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/storage")
 
 ### storage Type
 
@@ -51,7 +54,7 @@ Maximum number of concurrent downloads from the storage node(s)
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-limits-properties-maxconcurrentstoragenodedownloads.md "undefined#/properties/limits/properties/maxConcurrentStorageNodeDownloads")
+*   defined in: [Distributor node configuration](definition-properties-limits-properties-maxconcurrentstoragenodedownloads.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/maxConcurrentStorageNodeDownloads")
 
 ### maxConcurrentStorageNodeDownloads Type
 
@@ -63,7 +66,7 @@ Maximum number of concurrent downloads from the storage node(s)
 
 ## maxConcurrentOutboundConnections
 
-Maximum number of total simultaneous outbound connections to storage node(s)
+Maximum number of total simultaneous outbound connections to storage node(s) (excluding proxy connections)
 
 `maxConcurrentOutboundConnections`
 
@@ -73,7 +76,7 @@ Maximum number of total simultaneous outbound connections to storage node(s)
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-limits-properties-maxconcurrentoutboundconnections.md "undefined#/properties/limits/properties/maxConcurrentOutboundConnections")
+*   defined in: [Distributor node configuration](definition-properties-limits-properties-maxconcurrentoutboundconnections.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/maxConcurrentOutboundConnections")
 
 ### maxConcurrentOutboundConnections Type
 
@@ -83,11 +86,11 @@ Maximum number of total simultaneous outbound connections to storage node(s)
 
 **minimum**: the value of this number must greater than or equal to: `1`
 
-## outboundRequestsTimeout
+## outboundRequestsTimeoutMs
 
 Timeout for all outbound storage node http requests in miliseconds
 
-`outboundRequestsTimeout`
+`outboundRequestsTimeoutMs`
 
 *   is required
 
@@ -95,12 +98,92 @@ Timeout for all outbound storage node http requests in miliseconds
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-limits-properties-outboundrequeststimeout.md "undefined#/properties/limits/properties/outboundRequestsTimeout")
+*   defined in: [Distributor node configuration](definition-properties-limits-properties-outboundrequeststimeoutms.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/outboundRequestsTimeoutMs")
 
-### outboundRequestsTimeout Type
+### outboundRequestsTimeoutMs Type
 
 `integer`
 
-### outboundRequestsTimeout Constraints
+### outboundRequestsTimeoutMs Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1000`
+
+## pendingDownloadTimeoutSec
+
+Timeout for pending storage node downloads in seconds
+
+`pendingDownloadTimeoutSec`
+
+*   is required
+
+*   Type: `integer`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-limits-properties-pendingdownloadtimeoutsec.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/pendingDownloadTimeoutSec")
+
+### pendingDownloadTimeoutSec Type
+
+`integer`
+
+### pendingDownloadTimeoutSec Constraints
+
+**minimum**: the value of this number must greater than or equal to: `60`
+
+## maxCachedItemSize
+
+Maximum size of a data object allowed to be cached by the node
+
+`maxCachedItemSize`
+
+*   is optional
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-limits-properties-maxcacheditemsize.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/maxCachedItemSize")
+
+### maxCachedItemSize Type
+
+`string`
+
+### maxCachedItemSize Constraints
+
+**pattern**: the string must match the following regular expression: 
+
+```regexp
+^[0-9]+(B|K|M|G|T)$
+```
+
+[try pattern](https://regexr.com/?expression=%5E%5B0-9%5D%2B\(B%7CK%7CM%7CG%7CT\)%24 "try regular expression with regexr.com")
+
+## dataObjectSourceByObjectIdTTL
+
+TTL (in seconds) for dataObjectSourceByObjectId cache used when proxying objects of size greater than maxCachedItemSize to the right storage node.
+
+`dataObjectSourceByObjectIdTTL`
+
+*   is optional
+
+*   Type: `integer`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-limits-properties-dataobjectsourcebyobjectidttl.md "https://joystream.org/schemas/argus/config#/properties/limits/properties/dataObjectSourceByObjectIdTTL")
+
+### dataObjectSourceByObjectIdTTL Type
+
+`integer`
+
+### dataObjectSourceByObjectIdTTL Constraints
 
 **minimum**: the value of this number must greater than or equal to: `1`
+
+### dataObjectSourceByObjectIdTTL Default Value
+
+The default value is:
+
+```json
+60
+```

distributor-node/docs/schema/definition-properties-log-properties-console.md

@@ -1,18 +0,0 @@
-## console Type
-
-`string`
-
-## console Constraints
-
-**enum**: the value of this property must be equal to one of the following values:
-
-| Value       | Explanation |
-| :---------- | :---------- |
-| `"error"`   |             |
-| `"warn"`    |             |
-| `"info"`    |             |
-| `"http"`    |             |
-| `"verbose"` |             |
-| `"debug"`   |             |
-| `"silly"`   |             |
-| `"off"`     |             |

distributor-node/docs/schema/definition-properties-log-properties-elastic.md

@@ -1,18 +0,0 @@
-## elastic Type
-
-`string`
-
-## elastic Constraints
-
-**enum**: the value of this property must be equal to one of the following values:
-
-| Value       | Explanation |
-| :---------- | :---------- |
-| `"error"`   |             |
-| `"warn"`    |             |
-| `"info"`    |             |
-| `"http"`    |             |
-| `"verbose"` |             |
-| `"debug"`   |             |
-| `"silly"`   |             |
-| `"off"`     |             |

distributor-node/docs/schema/definition-properties-log.md

@@ -1,110 +0,0 @@
-## log Type
-
-`object` ([Details](definition-properties-log.md))
-
-# log Properties
-
-| Property            | Type     | Required | Nullable       | Defined by                                                                                                                       |
-| :------------------ | :------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------- |
-| [file](#file)       | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-log-properties-file.md "undefined#/properties/log/properties/file")       |
-| [console](#console) | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-log-properties-console.md "undefined#/properties/log/properties/console") |
-| [elastic](#elastic) | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-log-properties-elastic.md "undefined#/properties/log/properties/elastic") |
-
-## file
-
-Minimum level of logs written to a file specified in `directories.logs`
-
-`file`
-
-*   is optional
-
-*   Type: `string`
-
-*   cannot be null
-
-*   defined in: [Distributor node configuration](definition-properties-log-properties-file.md "undefined#/properties/log/properties/file")
-
-### file Type
-
-`string`
-
-### file Constraints
-
-**enum**: the value of this property must be equal to one of the following values:
-
-| Value       | Explanation |
-| :---------- | :---------- |
-| `"error"`   |             |
-| `"warn"`    |             |
-| `"info"`    |             |
-| `"http"`    |             |
-| `"verbose"` |             |
-| `"debug"`   |             |
-| `"silly"`   |             |
-| `"off"`     |             |
-
-## console
-
-Minimum level of logs outputted to a console
-
-`console`
-
-*   is optional
-
-*   Type: `string`
-
-*   cannot be null
-
-*   defined in: [Distributor node configuration](definition-properties-log-properties-console.md "undefined#/properties/log/properties/console")
-
-### console Type
-
-`string`
-
-### console Constraints
-
-**enum**: the value of this property must be equal to one of the following values:
-
-| Value       | Explanation |
-| :---------- | :---------- |
-| `"error"`   |             |
-| `"warn"`    |             |
-| `"info"`    |             |
-| `"http"`    |             |
-| `"verbose"` |             |
-| `"debug"`   |             |
-| `"silly"`   |             |
-| `"off"`     |             |
-
-## elastic
-
-Minimum level of logs sent to elasticsearch endpoint specified in `endpoints.elasticSearch`
-
-`elastic`
-
-*   is optional
-
-*   Type: `string`
-
-*   cannot be null
-
-*   defined in: [Distributor node configuration](definition-properties-log-properties-elastic.md "undefined#/properties/log/properties/elastic")
-
-### elastic Type
-
-`string`
-
-### elastic Constraints
-
-**enum**: the value of this property must be equal to one of the following values:
-
-| Value       | Explanation |
-| :---------- | :---------- |
-| `"error"`   |             |
-| `"warn"`    |             |
-| `"info"`    |             |
-| `"http"`    |             |
-| `"verbose"` |             |
-| `"debug"`   |             |
-| `"silly"`   |             |
-| `"off"`     |             |

distributor-node/docs/schema/definition-properties-logs-properties-console-logging-options.md

@@ -0,0 +1,41 @@
+## console Type
+
+`object` ([Console logging options](definition-properties-logs-properties-console-logging-options.md))
+
+# console Properties
+
+| Property        | Type     | Required | Nullable       | Defined by                                                                                                                                                                                                         |
+| :-------------- | :------- | :------- | :------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [level](#level) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/console/properties/level") |
+
+## level
+
+Minimum level of logs sent to this output
+
+`level`
+
+*   is required
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/console/properties/level")
+
+### level Type
+
+`string`
+
+### level Constraints
+
+**enum**: the value of this property must be equal to one of the following values:
+
+| Value       | Explanation |
+| :---------- | :---------- |
+| `"error"`   |             |
+| `"warn"`    |             |
+| `"info"`    |             |
+| `"http"`    |             |
+| `"verbose"` |             |
+| `"debug"`   |             |
+| `"silly"`   |             |

distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options-properties-endpoint.md

@@ -0,0 +1,3 @@
+## endpoint Type
+
+`string`

distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options.md

@@ -0,0 +1,60 @@
+## elastic Type
+
+`object` ([Elasticsearch logging options](definition-properties-logs-properties-elasticsearch-logging-options.md))
+
+# elastic Properties
+
+| Property              | Type     | Required | Nullable       | Defined by                                                                                                                                                                                                                        |
+| :-------------------- | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [level](#level)       | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/level")                |
+| [endpoint](#endpoint) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-endpoint.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/endpoint") |
+
+## level
+
+Minimum level of logs sent to this output
+
+`level`
+
+*   is required
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/level")
+
+### level Type
+
+`string`
+
+### level Constraints
+
+**enum**: the value of this property must be equal to one of the following values:
+
+| Value       | Explanation |
+| :---------- | :---------- |
+| `"error"`   |             |
+| `"warn"`    |             |
+| `"info"`    |             |
+| `"http"`    |             |
+| `"verbose"` |             |
+| `"debug"`   |             |
+| `"silly"`   |             |
+
+## endpoint
+
+Elastichsearch endpoint to push the logs to (for example: <http://localhost:9200>)
+
+`endpoint`
+
+*   is required
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-endpoint.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/endpoint")
+
+### endpoint Type
+
+`string`

distributor-node/docs/schema/definition-properties-logs-properties-file-logging-options-properties-archive.md

@@ -0,0 +1,3 @@
+## archive Type
+
+`boolean`

distributor-node/docs/schema/definition-properties-logs-properties-file-logging-options-properties-frequency.md

@@ -0,0 +1,22 @@
+## frequency Type
+
+`string`
+
+## frequency Constraints
+
+**enum**: the value of this property must be equal to one of the following values:
+
+| Value       | Explanation |
+| :---------- | :---------- |
+| `"yearly"`  |             |
+| `"monthly"` |             |
+| `"daily"`   |             |
+| `"hourly"`  |             |
+
+## frequency Default Value
+
+The default value is:
+
+```json
+"daily"
+```

distributor-node/docs/schema/definition-properties-log-properties-file.md → distributor-node/docs/schema/definition-properties-logs-properties-file-logging-options-properties-level.md

@@ -1,8 +1,8 @@
-## file Type
+## level Type
 
 `string`
 
-## file Constraints
+## level Constraints
 
 **enum**: the value of this property must be equal to one of the following values:
 
@@ -15,4 +15,3 @@
 | `"verbose"` |             |
 | `"debug"`   |             |
 | `"silly"`   |             |
-| `"off"`     |             |

distributor-node/docs/schema/definition-properties-limits-properties-outboundrequeststimeout.md → distributor-node/docs/schema/definition-properties-logs-properties-file-logging-options-properties-maxfiles.md

@@ -1,7 +1,7 @@
-## outboundRequestsTimeout Type
+## maxFiles Type
 
 `integer`
 
-## outboundRequestsTimeout Constraints
+## maxFiles Constraints
 
 **minimum**: the value of this number must greater than or equal to: `1`

distributor-node/docs/schema/definition-properties-logs-properties-file-logging-options-properties-maxsize.md

@@ -0,0 +1,7 @@
+## maxSize Type
+
+`integer`
+
+## maxSize Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1024`

distributor-node/docs/schema/definition-properties-logs-properties-file-logging-options-properties-path.md

@@ -0,0 +1,3 @@
+## path Type
+
+`string`

distributor-node/docs/schema/definition-properties-logs-properties-file-logging-options.md

@@ -0,0 +1,163 @@
+## file Type
+
+`object` ([File logging options](definition-properties-logs-properties-file-logging-options.md))
+
+# file Properties
+
+| Property                | Type      | Required | Nullable       | Defined by                                                                                                                                                                                                              |
+| :---------------------- | :-------- | :------- | :------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [level](#level)         | `string`  | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/level")         |
+| [path](#path)           | `string`  | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-path.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/path")           |
+| [maxFiles](#maxfiles)   | `integer` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-maxfiles.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/maxFiles")   |
+| [maxSize](#maxsize)     | `integer` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-maxsize.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/maxSize")     |
+| [frequency](#frequency) | `string`  | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-frequency.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/frequency") |
+| [archive](#archive)     | `boolean` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-archive.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/archive")     |
+
+## level
+
+Minimum level of logs sent to this output
+
+`level`
+
+*   is required
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/level")
+
+### level Type
+
+`string`
+
+### level Constraints
+
+**enum**: the value of this property must be equal to one of the following values:
+
+| Value       | Explanation |
+| :---------- | :---------- |
+| `"error"`   |             |
+| `"warn"`    |             |
+| `"info"`    |             |
+| `"http"`    |             |
+| `"verbose"` |             |
+| `"debug"`   |             |
+| `"silly"`   |             |
+
+## path
+
+Path where the logs will be stored (absolute or relative to config file)
+
+`path`
+
+*   is required
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-path.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/path")
+
+### path Type
+
+`string`
+
+## maxFiles
+
+Maximum number of log files to store
+
+`maxFiles`
+
+*   is optional
+
+*   Type: `integer`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-maxfiles.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/maxFiles")
+
+### maxFiles Type
+
+`integer`
+
+### maxFiles Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1`
+
+## maxSize
+
+Maximum size of a single log file in bytes
+
+`maxSize`
+
+*   is optional
+
+*   Type: `integer`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-maxsize.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/maxSize")
+
+### maxSize Type
+
+`integer`
+
+### maxSize Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1024`
+
+## frequency
+
+The frequency of creating new log files (regardless of maxSize)
+
+`frequency`
+
+*   is optional
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-frequency.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/frequency")
+
+### frequency Type
+
+`string`
+
+### frequency Constraints
+
+**enum**: the value of this property must be equal to one of the following values:
+
+| Value       | Explanation |
+| :---------- | :---------- |
+| `"yearly"`  |             |
+| `"monthly"` |             |
+| `"daily"`   |             |
+| `"hourly"`  |             |
+
+### frequency Default Value
+
+The default value is:
+
+```json
+"daily"
+```
+
+## archive
+
+Whether to archive old logs
+
+`archive`
+
+*   is optional
+
+*   Type: `boolean`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-archive.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file/properties/archive")
+
+### archive Type
+
+`boolean`

distributor-node/docs/schema/definition-properties-logs.md

@@ -0,0 +1,65 @@
+## logs Type
+
+`object` ([Details](definition-properties-logs.md))
+
+# logs Properties
+
+| Property            | Type     | Required | Nullable       | Defined by                                                                                                                                                                                |
+| :------------------ | :------- | :------- | :------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [file](#file)       | `object` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file")             |
+| [console](#console) | `object` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-console-logging-options.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/console")       |
+| [elastic](#elastic) | `object` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic") |
+
+## file
+
+
+
+`file`
+
+*   is optional
+
+*   Type: `object` ([File logging options](definition-properties-logs-properties-file-logging-options.md))
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-file-logging-options.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/file")
+
+### file Type
+
+`object` ([File logging options](definition-properties-logs-properties-file-logging-options.md))
+
+## console
+
+
+
+`console`
+
+*   is optional
+
+*   Type: `object` ([Console logging options](definition-properties-logs-properties-console-logging-options.md))
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-console-logging-options.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/console")
+
+### console Type
+
+`object` ([Console logging options](definition-properties-logs-properties-console-logging-options.md))
+
+## elastic
+
+
+
+`elastic`
+
+*   is optional
+
+*   Type: `object` ([Elasticsearch logging options](definition-properties-logs-properties-elasticsearch-logging-options.md))
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic")
+
+### elastic Type
+
+`object` ([Elasticsearch logging options](definition-properties-logs-properties-elasticsearch-logging-options.md))

distributor-node/docs/schema/definition-properties-operatorapi-properties-hmacsecret.md

@@ -0,0 +1,3 @@
+## hmacSecret Type
+
+`string`

distributor-node/docs/schema/definition-properties-operatorapi.md

@@ -0,0 +1,50 @@
+## operatorApi Type
+
+`object` ([Details](definition-properties-operatorapi.md))
+
+# operatorApi Properties
+
+| Property                  | Type      | Required | Nullable       | Defined by                                                                                                                                                                              |
+| :------------------------ | :-------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [port](#port)             | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-operatorapi-properties-port.md "https://joystream.org/schemas/argus/config#/properties/operatorApi/properties/port")             |
+| [hmacSecret](#hmacsecret) | `string`  | Required | cannot be null | [Distributor node configuration](definition-properties-operatorapi-properties-hmacsecret.md "https://joystream.org/schemas/argus/config#/properties/operatorApi/properties/hmacSecret") |
+
+## port
+
+Distributor node operator api port
+
+`port`
+
+*   is required
+
+*   Type: `integer`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-operatorapi-properties-port.md "https://joystream.org/schemas/argus/config#/properties/operatorApi/properties/port")
+
+### port Type
+
+`integer`
+
+### port Constraints
+
+**minimum**: the value of this number must greater than or equal to: `0`
+
+## hmacSecret
+
+HMAC (HS256) secret key used for JWT authorization
+
+`hmacSecret`
+
+*   is required
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-operatorapi-properties-hmacsecret.md "https://joystream.org/schemas/argus/config#/properties/operatorApi/properties/hmacSecret")
+
+### hmacSecret Type
+
+`string`

distributor-node/docs/schema/definition-properties-publicapi-properties-port.md

@@ -0,0 +1,7 @@
+## port Type
+
+`integer`
+
+## port Constraints
+
+**minimum**: the value of this number must greater than or equal to: `0`

distributor-node/docs/schema/definition-properties-publicapi.md

@@ -0,0 +1,31 @@
+## publicApi Type
+
+`object` ([Details](definition-properties-publicapi.md))
+
+# publicApi Properties
+
+| Property      | Type      | Required | Nullable       | Defined by                                                                                                                                                              |
+| :------------ | :-------- | :------- | :------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [port](#port) | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-publicapi-properties-port.md "https://joystream.org/schemas/argus/config#/properties/publicApi/properties/port") |
+
+## port
+
+Distributor node public api port
+
+`port`
+
+*   is required
+
+*   Type: `integer`
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-publicapi-properties-port.md "https://joystream.org/schemas/argus/config#/properties/publicApi/properties/port")
+
+### port Type
+
+`integer`
+
+### port Constraints
+
+**minimum**: the value of this number must greater than or equal to: `0`

distributor-node/docs/schema/definition.md

@@ -4,18 +4,19 @@
 
 # Distributor node configuration Properties
 
-| Property                    | Type      | Required | Nullable       | Defined by                                                                                                 |
-| :-------------------------- | :-------- | :------- | :------------- | :--------------------------------------------------------------------------------------------------------- |
-| [id](#id)                   | `string`  | Required | cannot be null | [Distributor node configuration](definition-properties-id.md "undefined#/properties/id")                   |
-| [endpoints](#endpoints)     | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-endpoints.md "undefined#/properties/endpoints")     |
-| [directories](#directories) | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-directories.md "undefined#/properties/directories") |
-| [log](#log)                 | `object`  | Optional | cannot be null | [Distributor node configuration](definition-properties-log.md "undefined#/properties/log")                 |
-| [limits](#limits)           | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-limits.md "undefined#/properties/limits")           |
-| [intervals](#intervals)     | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-intervals.md "undefined#/properties/intervals")     |
-| [port](#port)               | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-port.md "undefined#/properties/port")               |
-| [keys](#keys)               | `array`   | Required | cannot be null | [Distributor node configuration](definition-properties-keys.md "undefined#/properties/keys")               |
-| [buckets](#buckets)         | Merged    | Required | cannot be null | [Distributor node configuration](definition-properties-buckets.md "undefined#/properties/buckets")         |
-| [workerId](#workerid)       | `integer` | Required | cannot be null | [Distributor node configuration](definition-properties-workerid.md "undefined#/properties/workerId")       |
+| Property                    | Type      | Required | Nullable       | Defined by                                                                                                                                  |
+| :-------------------------- | :-------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------ |
+| [id](#id)                   | `string`  | Required | cannot be null | [Distributor node configuration](definition-properties-id.md "https://joystream.org/schemas/argus/config#/properties/id")                   |
+| [endpoints](#endpoints)     | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-endpoints.md "https://joystream.org/schemas/argus/config#/properties/endpoints")     |
+| [directories](#directories) | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-directories.md "https://joystream.org/schemas/argus/config#/properties/directories") |
+| [logs](#logs)               | `object`  | Optional | cannot be null | [Distributor node configuration](definition-properties-logs.md "https://joystream.org/schemas/argus/config#/properties/logs")               |
+| [limits](#limits)           | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-limits.md "https://joystream.org/schemas/argus/config#/properties/limits")           |
+| [intervals](#intervals)     | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-intervals.md "https://joystream.org/schemas/argus/config#/properties/intervals")     |
+| [publicApi](#publicapi)     | `object`  | Required | cannot be null | [Distributor node configuration](definition-properties-publicapi.md "https://joystream.org/schemas/argus/config#/properties/publicApi")     |
+| [operatorApi](#operatorapi) | `object`  | Optional | cannot be null | [Distributor node configuration](definition-properties-operatorapi.md "https://joystream.org/schemas/argus/config#/properties/operatorApi") |
+| [keys](#keys)               | `array`   | Optional | cannot be null | [Distributor node configuration](definition-properties-keys.md "https://joystream.org/schemas/argus/config#/properties/keys")               |
+| [buckets](#buckets)         | `array`   | Optional | cannot be null | [Distributor node configuration](definition-properties-bucket-ids.md "https://joystream.org/schemas/argus/config#/properties/buckets")      |
+| [workerId](#workerid)       | `integer` | Optional | cannot be null | [Distributor node configuration](definition-properties-workerid.md "https://joystream.org/schemas/argus/config#/properties/workerId")       |
 
 ## id
 
@@ -29,7 +30,7 @@ Node identifier used when sending elasticsearch logs and exposed on /status endp
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-id.md "undefined#/properties/id")
+*   defined in: [Distributor node configuration](definition-properties-id.md "https://joystream.org/schemas/argus/config#/properties/id")
 
 ### id Type
 
@@ -51,7 +52,7 @@ Specifies external endpoints that the distributor node will connect to
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-endpoints.md "undefined#/properties/endpoints")
+*   defined in: [Distributor node configuration](definition-properties-endpoints.md "https://joystream.org/schemas/argus/config#/properties/endpoints")
 
 ### endpoints Type
 
@@ -69,29 +70,29 @@ Specifies paths where node's data will be stored
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-directories.md "undefined#/properties/directories")
+*   defined in: [Distributor node configuration](definition-properties-directories.md "https://joystream.org/schemas/argus/config#/properties/directories")
 
 ### directories Type
 
 `object` ([Details](definition-properties-directories.md))
 
-## log
+## logs
 
-Specifies minimum log levels by supported log outputs
+Specifies the logging configuration
 
-`log`
+`logs`
 
 *   is optional
 
-*   Type: `object` ([Details](definition-properties-log.md))
+*   Type: `object` ([Details](definition-properties-logs.md))
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-log.md "undefined#/properties/log")
+*   defined in: [Distributor node configuration](definition-properties-logs.md "https://joystream.org/schemas/argus/config#/properties/logs")
 
-### log Type
+### logs Type
 
-`object` ([Details](definition-properties-log.md))
+`object` ([Details](definition-properties-logs.md))
 
 ## limits
 
@@ -105,7 +106,7 @@ Specifies node limits w\.r.t. storage, outbound connections etc.
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-limits.md "undefined#/properties/limits")
+*   defined in: [Distributor node configuration](definition-properties-limits.md "https://joystream.org/schemas/argus/config#/properties/limits")
 
 ### limits Type
 
@@ -123,33 +124,47 @@ Specifies how often periodic tasks (for example cache cleanup) are executed by t
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-intervals.md "undefined#/properties/intervals")
+*   defined in: [Distributor node configuration](definition-properties-intervals.md "https://joystream.org/schemas/argus/config#/properties/intervals")
 
 ### intervals Type
 
 `object` ([Details](definition-properties-intervals.md))
 
-## port
+## publicApi
 
-Distributor node http server port
+Public api configuration
 
-`port`
+`publicApi`
 
 *   is required
 
-*   Type: `integer`
+*   Type: `object` ([Details](definition-properties-publicapi.md))
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-port.md "undefined#/properties/port")
+*   defined in: [Distributor node configuration](definition-properties-publicapi.md "https://joystream.org/schemas/argus/config#/properties/publicApi")
 
-### port Type
+### publicApi Type
 
-`integer`
+`object` ([Details](definition-properties-publicapi.md))
 
-### port Constraints
+## operatorApi
 
-**minimum**: the value of this number must greater than or equal to: `0`
+Operator api configuration
+
+`operatorApi`
+
+*   is optional
+
+*   Type: `object` ([Details](definition-properties-operatorapi.md))
+
+*   cannot be null
+
+*   defined in: [Distributor node configuration](definition-properties-operatorapi.md "https://joystream.org/schemas/argus/config#/properties/operatorApi")
+
+### operatorApi Type
+
+`object` ([Details](definition-properties-operatorapi.md))
 
 ## keys
 
@@ -157,13 +172,13 @@ Specifies the keys available within distributor node CLI.
 
 `keys`
 
-*   is required
+*   is optional
 
 *   Type: an array of merged types ([Details](definition-properties-keys-items.md))
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-keys.md "undefined#/properties/keys")
+*   defined in: [Distributor node configuration](definition-properties-keys.md "https://joystream.org/schemas/argus/config#/properties/keys")
 
 ### keys Type
 
@@ -175,27 +190,27 @@ an array of merged types ([Details](definition-properties-keys-items.md))
 
 ## buckets
 
-Specifies the buckets distributed by the node
+Set of bucket ids distributed by the node. If not specified, all buckets currently assigned to worker specified in `config.workerId` will be distributed.
 
 `buckets`
 
-*   is required
+*   is optional
 
-*   Type: merged type ([Details](definition-properties-buckets.md))
+*   Type: `integer[]`
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-buckets.md "undefined#/properties/buckets")
+*   defined in: [Distributor node configuration](definition-properties-bucket-ids.md "https://joystream.org/schemas/argus/config#/properties/buckets")
 
 ### buckets Type
 
-merged type ([Details](definition-properties-buckets.md))
+`integer[]`
 
-one (and only one) of
+### buckets Constraints
 
-*   [Bucket ids](definition-properties-buckets-oneof-bucket-ids.md "check type definition")
+**minimum number of items**: the minimum number of items for this array is: `1`
 
-*   [All buckets](definition-properties-buckets-oneof-all-buckets.md "check type definition")
+**unique items**: all items in this array must be unique. Duplicates are not allowed.
 
 ## workerId
 
@@ -203,13 +218,13 @@ ID of the node operator (distribution working group worker)
 
 `workerId`
 
-*   is required
+*   is optional
 
 *   Type: `integer`
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-workerid.md "undefined#/properties/workerId")
+*   defined in: [Distributor node configuration](definition-properties-workerid.md "https://joystream.org/schemas/argus/config#/properties/workerId")
 
 ### workerId Type
 

distributor-node/package.json

@@ -14,7 +14,7 @@
     "@joystream/types": "^0.17.0",
     "@oclif/command": "^1",
     "@oclif/config": "^1",
-    "@oclif/plugin-help": "^3.2.4",
+    "@oclif/plugin-help": "^3",
     "ajv": "^7",
     "axios": "^0.21.1",
     "blake3-wasm": "^2.1.5",
@@ -41,6 +41,10 @@
     "tslib": "^1",
     "winston": "^3.3.3",
     "winston-elasticsearch": "^0.15.8",
+    "url-join": "^4.0.1",
+    "@types/url-join": "^4.0.1",
+    "winston-daily-rotate-file": "^4.5.5",
+    "jsonwebtoken": "^8.5.1",
     "yaml": "^1.10.2"
   },
   "devDependencies": {
@@ -76,6 +80,10 @@
   "engines": {
     "node": ">=14.16.1"
   },
+  "volta": {
+    "node": "14.16.1",
+    "yarn": "1.22.4"
+  },
   "files": [
     "/bin",
     "/lib",
@@ -101,6 +109,9 @@
       "operator": {
         "description": "Commands for performing node operator (Distribution Working Group worker) on-chain duties (like accepting bucket invitations, setting node metadata)"
       },
+      "node": {
+        "description": "Commands for interacting with a running distributor node through OperatorApi"
+      },
       "dev": {
         "description": "Developer utility commands"
       }
@@ -118,14 +129,17 @@
     "version": "generate:docs:cli && git add docs/cli/*",
     "generate:types:json-schema": "yarn ts-node ./src/schemas/scripts/generateTypes.ts",
     "generate:types:graphql": "yarn graphql-codegen -c ./src/services/networking/query-node/codegen.yml",
-    "generate:types:openapi": "yarn openapi-typescript ./src/api-spec/openapi.yml -o ./src/types/generated/OpenApi.ts -c ../prettierrc.js",
-    "generate:types:all": "yarn generate:types:json-schema && yarn generate:types:graphql && yarn generate:types:openapi",
+    "generate:types:public-api": "yarn openapi-typescript ./src/api-spec/public.yml -o ./src/types/generated/PublicApi.ts -c ../prettierrc.js",
+    "generate:types:operator-api": "yarn openapi-typescript ./src/api-spec/operator.yml -o ./src/types/generated/OperatorApi.ts -c ../prettierrc.js",
+    "generate:types:api": "yarn generate:types:public-api && yarn generate:types:operator-api",
+    "generate:types:all": "yarn generate:types:json-schema && yarn generate:types:graphql && yarn generate:types:api",
     "generate:api:storage-node": "yarn openapi-generator-cli generate -i ../storage-node-v2/src/api-spec/openapi.yaml -g typescript-axios -o ./src/services/networking/storage-node/generated",
-    "generate:api:distributor-node": "yarn openapi-generator-cli generate -i ./src/api-spec/openapi.yml -g typescript-axios -o ./src/services/networking/distributor-node/generated",
-    "generate:api:all": "yarn generate:api:storage-node && yarn generate:api:distributor-node",
+    "generate:api:all": "yarn generate:api:storage-node",
     "generate:docs:cli": "yarn oclif-dev readme --multi --dir ./docs/commands",
     "generate:docs:config": "yarn ts-node --transpile-only ./src/schemas/scripts/generateConfigDoc.ts",
-    "generate:docs:api": "yarn widdershins ./src/api-spec/openapi.yml --language_tabs javascript:JavaScript shell:Shell -o ./docs/api/index.md -u ./docs/api/templates",
+    "generate:docs:public-api": "yarn widdershins ./src/api-spec/public.yml --language_tabs javascript:JavaScript shell:Shell -o ./docs/api/public/index.md -u ./docs/api/templates",
+    "generate:docs:operator-api": "yarn widdershins ./src/api-spec/operator.yml --language_tabs javascript:JavaScript shell:Shell -o ./docs/api/operator/index.md -u ./docs/api/templates",
+    "generate:docs:api": "yarn generate:docs:public-api && yarn generate:docs:operator-api",
     "generate:docs:toc": "yarn md-magic --path ./docs/**/*.md",
     "generate:docs:all": "yarn generate:docs:cli && yarn generate:docs:config && yarn generate:docs:api && yarn generate:docs:toc",
     "generate:all": "yarn generate:types:all && yarn generate:api:all && yarn generate:docs:all",

distributor-node/scripts/init-bucket.sh

@@ -15,3 +15,6 @@ ${CLI} leader:update-bag -b static:council -f ${FAMILY_ID} -a ${BUCKET_ID}
 ${CLI} leader:update-bucket-mode -f ${FAMILY_ID} -B ${BUCKET_ID} --mode on
 ${CLI} leader:invite-bucket-operator -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0
 ${CLI} operator:accept-invitation -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0
+${CLI} operator:set-metadata -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0 -e http://localhost:3334
+${CLI} leader:update-dynamic-bag-policy -t Channel -p ${FAMILY_ID}:1
+${CLI} leader:update-dynamic-bag-policy -t Member -p ${FAMILY_ID}:1

distributor-node/scripts/test-commands.sh

@@ -21,6 +21,7 @@ ${CLI} leader:update-bag -b static:wg:gateway -f ${FAMILY_ID} -a ${BUCKET_ID}
 ${CLI} leader:update-bag -b static:wg:distribution -f ${FAMILY_ID} -a ${BUCKET_ID}
 ${CLI} leader:update-bucket-status -f ${FAMILY_ID} -B ${BUCKET_ID}  --acceptingBags yes
 ${CLI} leader:update-bucket-mode -f ${FAMILY_ID} -B ${BUCKET_ID} --mode on
+${CLI} leader:update-dynamic-bag-policy -t Channel -p ${FAMILY_ID}:5
 ${CLI} leader:update-dynamic-bag-policy -t Member -p ${FAMILY_ID}:5
 ${CLI} leader:update-dynamic-bag-policy -t Member
 ${CLI} leader:invite-bucket-operator -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0
@@ -28,11 +29,13 @@ ${CLI} leader:cancel-invitation -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0
 ${CLI} leader:invite-bucket-operator -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0
 ${CLI} operator:accept-invitation -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0
 ${CLI} operator:set-metadata -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0 -i ./data/operator-metadata.json
-${CLI} leader:remove-bucket-operator -f ${FAMILY_ID} -B ${BUCKET_ID} -w 0
 ${CLI} leader:set-bucket-family-metadata -f ${FAMILY_ID} -i ./data/family-metadata.json
 
-# Deletion commands tested separately, since bucket operator removal is not yet supported
+# Deletion commands tested separately
 FAMILY_TO_DELETE_ID=`${CLI} leader:create-bucket-family`
 BUCKET_TO_DELETE_ID=`${CLI} leader:create-bucket -f ${FAMILY_TO_DELETE_ID} -a yes`
+${CLI} leader:invite-bucket-operator -f ${FAMILY_TO_DELETE_ID} -B ${BUCKET_TO_DELETE_ID} -w 0
+${CLI} operator:accept-invitation -f ${FAMILY_TO_DELETE_ID} -B ${BUCKET_TO_DELETE_ID} -w 0
+${CLI} leader:remove-bucket-operator -f ${FAMILY_TO_DELETE_ID} -B ${BUCKET_TO_DELETE_ID} -w 0
 ${CLI} leader:delete-bucket -f ${FAMILY_TO_DELETE_ID} -B ${BUCKET_TO_DELETE_ID}
 ${CLI} leader:delete-bucket-family -f ${FAMILY_TO_DELETE_ID}

distributor-node/src/api-spec/operator.yml

@@ -0,0 +1,119 @@
+openapi: 3.0.3
+info:
+  title: Distributor node operator API
+  description: Distributor node operator API
+  contact:
+    email: info@joystream.org
+  license:
+    name: GPL-3.0-only
+    url: https://spdx.org/licenses/GPL-3.0-only.html
+  version: 0.1.0
+servers:
+  - url: http://localhost:3335/api/v1/
+
+paths:
+  /stop-api:
+    post:
+      operationId: operator.stopApi
+      description: Turns off the public api.
+      responses:
+        200:
+          description: OK
+        401:
+          description: Not authorized
+        409:
+          description: Already stopped
+        500:
+          description: Unexpected server error
+  /start-api:
+    post:
+      operationId: operator.startApi
+      description: Turns on the public api.
+      responses:
+        200:
+          description: OK
+        401:
+          description: Not authorized
+        409:
+          description: Already started
+        500:
+          description: Unexpected server error
+  /shutdown:
+    post:
+      operationId: operator.shutdown
+      description: Shuts down the node.
+      responses:
+        200:
+          description: OK
+        401:
+          description: Not authorized
+        409:
+          description: Already shutting down
+        500:
+          description: Unexpected server error
+  /set-worker:
+    post:
+      operationId: operator.setWorker
+      description: Updates the operator worker id.
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/SetWorkerOperation'
+      responses:
+        200:
+          description: OK
+        401:
+          description: Not authorized
+        500:
+          description: Unexpected server error
+  /set-buckets:
+    post:
+      operationId: operator.setBuckets
+      description: Updates buckets supported by the node.
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/SetBucketsOperation'
+      responses:
+        200:
+          description: OK
+        401:
+          description: Not authorized
+        500:
+          description: Unexpected server error
+
+components:
+  securitySchemes:
+    OperatorAuth:
+      type: http
+      scheme: bearer
+      bearerFormat:
+        "JWT signed with HMAC (HS256) secret key specified in distributor node's `config.operator.hmacSecret`.
+        The payload should include:
+        - `reqBody` - content of the request body
+        - `reqUrl` - request url (only pathname + query string, without origin. For example: `/api/v1/set-buckets`)"
+  schemas:
+    SetWorkerOperation:
+      type: object
+      required:
+        - workerId
+      properties:
+        workerId:
+          type: integer
+          minimum: 0
+    SetBucketsOperation:
+      type: object
+      properties:
+        buckets:
+          description: 'Set of bucket ids to be distributed by the node.
+            If not provided - all buckets assigned to currently configured worker will be distributed.'
+          type: array
+          minItems: 1
+          items:
+            type: integer
+            minimum: 0
+
+security:
+  - OperatorAuth: []

distributor-node/src/api-spec/openapi.yml → distributor-node/src/api-spec/public.yml

@@ -1,7 +1,7 @@
 openapi: 3.0.3
 info:
-  title: Distributor node API
-  description: Distributor node API
+  title: Distributor node public API
+  description: Distributor node public API
   contact:
     email: info@joystream.org
   license:
@@ -9,22 +9,16 @@ info:
     url: https://spdx.org/licenses/GPL-3.0-only.html
   version: 0.1.0
 externalDocs:
-  description: Distributor node API
+  description: Distributor node public API
   url: https://github.com/Joystream/joystream/issues/2224
 servers:
   - url: http://localhost:3334/api/v1/
 
-tags:
-  - name: public
-    description: Public distributor node API
-
 paths:
   /status:
     get:
       operationId: public.status
       description: Returns json object describing current node status.
-      tags:
-        - public
       responses:
         200:
           description: OK
@@ -38,8 +32,6 @@ paths:
     get:
       operationId: public.buckets
       description: Returns list of distributed buckets
-      tags:
-        - public
       responses:
         200:
           description: OK
@@ -49,12 +41,10 @@ paths:
                 $ref: '#/components/schemas/BucketsResponse'
         500:
           description: Unexpected server error
-  /asset/{objectId}:
+  /assets/{objectId}:
     head:
       operationId: public.assetHead
       description: Returns asset response headers (cache status, content type and/or length, accepted ranges etc.)
-      tags:
-        - public
       parameters:
         - $ref: '#/components/parameters/ObjectId'
       responses:
@@ -72,8 +62,6 @@ paths:
     get:
       operationId: public.asset
       description: Returns a media file.
-      tags:
-        - public
       parameters:
         - $ref: '#/components/parameters/ObjectId'
       responses:
@@ -203,7 +191,6 @@ components:
           properties:
             bucketIds:
               type: array
-              minItems: 1
               items:
                 type: integer
                 minimum: 0

distributor-node/src/app/index.ts

@@ -1,31 +1,37 @@
-import { ReadonlyConfig } from '../types'
+import { Config } from '../types'
 import { NetworkingService } from '../services/networking'
 import { LoggingService } from '../services/logging'
 import { StateCacheService } from '../services/cache/StateCacheService'
 import { ContentService } from '../services/content/ContentService'
-import { ServerService } from '../services/server/ServerService'
 import { Logger } from 'winston'
 import fs from 'fs'
 import nodeCleanup from 'node-cleanup'
 import { AppIntervals } from '../types/app'
+import { PublicApiService } from '../services/httpApi/PublicApiService'
+import { OperatorApiService } from '../services/httpApi/OperatorApiService'
 
 export class App {
-  private config: ReadonlyConfig
+  private config: Config
   private content: ContentService
   private stateCache: StateCacheService
   private networking: NetworkingService
-  private server: ServerService
+  private publicApi: PublicApiService
+  private operatorApi: OperatorApiService | undefined
   private logging: LoggingService
   private logger: Logger
   private intervals: AppIntervals | undefined
+  private isStopping = false
 
-  constructor(config: ReadonlyConfig) {
+  constructor(config: Config) {
     this.config = config
     this.logging = LoggingService.withAppConfig(config)
     this.stateCache = new StateCacheService(config, this.logging)
     this.networking = new NetworkingService(config, this.stateCache, this.logging)
     this.content = new ContentService(config, this.logging, this.networking, this.stateCache)
-    this.server = new ServerService(config, this.stateCache, this.content, this.logging, this.networking)
+    this.publicApi = new PublicApiService(config, this.stateCache, this.content, this.logging, this.networking)
+    if (this.config.operatorApi) {
+      this.operatorApi = new OperatorApiService(config, this, this.logging, this.publicApi)
+    }
     this.logger = this.logging.createLogger('App')
   }
 
@@ -46,30 +52,32 @@ export class App {
     }
   }
 
-  private checkConfigDirectories(): void {
-    Object.entries(this.config.directories).forEach(([name, path]) => {
-      if (path === undefined) {
-        return
-      }
-      const dirInfo = `${name} directory (${path})`
-      if (!fs.existsSync(path)) {
-        try {
-          fs.mkdirSync(path, { recursive: true })
-        } catch (e) {
-          throw new Error(`${dirInfo} doesn't exist and cannot be created!`)
-        }
-      }
-      try {
-        fs.accessSync(path, fs.constants.R_OK)
-      } catch (e) {
-        throw new Error(`${dirInfo} is not readable`)
-      }
+  private checkConfigDir(name: string, path: string): void {
+    const dirInfo = `${name} directory (${path})`
+    if (!fs.existsSync(path)) {
       try {
-        fs.accessSync(path, fs.constants.W_OK)
+        fs.mkdirSync(path, { recursive: true })
       } catch (e) {
-        throw new Error(`${dirInfo} is not writable`)
+        throw new Error(`${dirInfo} doesn't exist and cannot be created!`)
       }
-    })
+    }
+    try {
+      fs.accessSync(path, fs.constants.R_OK)
+    } catch (e) {
+      throw new Error(`${dirInfo} is not readable`)
+    }
+    try {
+      fs.accessSync(path, fs.constants.W_OK)
+    } catch (e) {
+      throw new Error(`${dirInfo} is not writable`)
+    }
+  }
+
+  private checkConfigDirectories(): void {
+    Object.entries(this.config.directories).forEach(([name, path]) => this.checkConfigDir(name, path))
+    if (this.config.logs?.file) {
+      this.checkConfigDir('logs.file.path', this.config.logs.file.path)
+    }
   }
 
   public async start(): Promise<void> {
@@ -79,7 +87,8 @@ export class App {
       this.stateCache.load()
       await this.content.startupInit()
       this.setIntervals()
-      this.server.start()
+      this.publicApi.start()
+      this.operatorApi?.start()
     } catch (err) {
       this.logger.error('Node initialization failed!', { err })
       process.exit(-1)
@@ -87,6 +96,21 @@ export class App {
     nodeCleanup(this.exitHandler.bind(this))
   }
 
+  public stop(timeoutSec?: number): boolean {
+    if (this.isStopping) {
+      return false
+    }
+    this.logger.info(`Stopping the app${timeoutSec ? ` in ${timeoutSec} sec...` : ''}`)
+    this.isStopping = true
+    if (timeoutSec) {
+      setTimeout(() => process.kill(process.pid, 'SIGINT'), timeoutSec * 1000)
+    } else {
+      process.kill(process.pid, 'SIGINT')
+    }
+
+    return true
+  }
+
   private async exitGracefully(): Promise<void> {
     // Async exit handler - ideally should not take more than 10 sec
     // We can try to wait until some pending downloads are finished here etc.
@@ -125,10 +149,15 @@ export class App {
     this.logger.info('Exiting...')
     // Clear intervals
     this.clearIntervals()
-    // Stop the server
-    this.server.stop()
+    // Stop the http apis
+    this.publicApi.stop()
+    this.operatorApi?.stop()
     // Save cache
-    this.stateCache.saveSync()
+    try {
+      this.stateCache.saveSync()
+    } catch (err) {
+      this.logger.error('Failed to save the cache state on exit!', { err })
+    }
     if (signal) {
       // Async exit can be executed
       this.exitGracefully()

distributor-node/src/command-base/accounts.ts

@@ -1,7 +1,7 @@
 import ApiCommandBase from './api'
 import { AccountId } from '@polkadot/types/interfaces'
 import { Keyring } from '@polkadot/api'
-import { KeyringInstance, KeyringOptions, KeyringPair } from '@polkadot/keyring/types'
+import { KeyringInstance, KeyringOptions, KeyringPair, KeyringPair$Json } from '@polkadot/keyring/types'
 import { CLIError } from '@oclif/errors'
 import ExitCodes from './ExitCodes'
 import fs from 'fs'
@@ -30,7 +30,7 @@ export default abstract class AccountsCommandBase extends ApiCommandBase {
         exit: ExitCodes.InvalidFile,
       })
     }
-    let accountJsonObj: any
+    let accountJsonObj: unknown
     try {
       accountJsonObj = require(jsonBackupFilePath)
     } catch (e) {
@@ -48,8 +48,8 @@ export default abstract class AccountsCommandBase extends ApiCommandBase {
     let account: KeyringPair
     try {
       // Try adding and retrieving the keys in order to validate that the backup file is correct
-      keyring.addFromJson(accountJsonObj)
-      account = keyring.getPair(accountJsonObj.address)
+      keyring.addFromJson(accountJsonObj as KeyringPair$Json)
+      account = keyring.getPair((accountJsonObj as KeyringPair$Json).address)
     } catch (e) {
       throw new CLIError(`Keypair backup json file is is not valid: ${jsonBackupFilePath}`, {
         exit: ExitCodes.InvalidFile,
@@ -124,7 +124,7 @@ export default abstract class AccountsCommandBase extends ApiCommandBase {
 
   initKeyring(): void {
     this.keyring = new Keyring(KEYRING_OPTIONS)
-    this.appConfig.keys.forEach((keyData) => {
+    this.appConfig.keys?.forEach((keyData) => {
       if ('suri' in keyData) {
         this.keyring.addFromUri(keyData.suri, undefined, keyData.type)
       }

distributor-node/src/command-base/default.ts

@@ -22,8 +22,8 @@ export const flags = {
   }),
   bagId: oclifFlags.build({
     parse: (value: string) => {
-      const parser = new BagIdParserService()
-      return parser.parseBagId(value)
+      const parser = new BagIdParserService(value)
+      return parser.parse()
     },
     description: `Bag ID. Format: {bag_type}:{sub_type}:{id}.
     - Bag types: 'static', 'dynamic'
@@ -61,8 +61,8 @@ export default abstract class DefaultCommandBase extends Command {
 
   async init(): Promise<void> {
     const { configPath, yes } = this.parse(this.constructor as typeof DefaultCommandBase).flags
-    const configParser = new ConfigParserService()
-    this.appConfig = configParser.loadConfing(configPath) as ReadonlyConfig
+    const configParser = new ConfigParserService(configPath)
+    this.appConfig = configParser.parse() as ReadonlyConfig
     this.logging = LoggingService.withCLIConfig()
     this.logger = this.logging.createLogger('CLI')
     this.autoConfirm = !!(process.env.AUTO_CONFIRM === 'true' || parseInt(process.env.AUTO_CONFIRM || '') || yes)
@@ -89,11 +89,11 @@ export default abstract class DefaultCommandBase extends Command {
     }
   }
 
-  async finally(err: any): Promise<void> {
+  async finally(err: unknown): Promise<void> {
     if (!err) this.exit(ExitCodes.OK)
     if (process.env.DEBUG === 'true') {
       console.error(err)
     }
-    super.finally(err)
+    super.finally(err as Error)
   }
 }

distributor-node/src/command-base/node.ts

@@ -0,0 +1,58 @@
+import axios from 'axios'
+import urljoin from 'url-join'
+import DefaultCommandBase, { flags } from './default'
+import jwt from 'jsonwebtoken'
+import ExitCodes from './ExitCodes'
+
+export default abstract class NodeCommandBase extends DefaultCommandBase {
+  static flags = {
+    url: flags.string({
+      char: 'u',
+      description: 'Distributor node operator api base url (ie. http://localhost:3335)',
+      required: true,
+    }),
+    secret: flags.string({
+      char: 's',
+      description: 'HMAC secret key to use (will default to config.operatorApi.hmacSecret if present)',
+      required: false,
+    }),
+    ...DefaultCommandBase.flags,
+  }
+
+  protected abstract reqUrl(): string
+
+  protected reqBody(): Record<string, unknown> {
+    return {}
+  }
+
+  async run(): Promise<void> {
+    const { url, secret } = this.parse(this.constructor as typeof NodeCommandBase).flags
+
+    const hmacSecret = secret || this.appConfig.operatorApi?.hmacSecret
+
+    if (!hmacSecret) {
+      this.error('No --secret was provided and no config.operatorApi.hmacSecret is set!', {
+        exit: ExitCodes.InvalidInput,
+      })
+    }
+
+    const reqUrl = this.reqUrl()
+    const reqBody = this.reqBody()
+    const payload = { reqUrl, reqBody }
+    try {
+      await axios.post(urljoin(url, reqUrl), reqBody, {
+        headers: {
+          authorization: `bearer ${jwt.sign(payload, hmacSecret, { expiresIn: 60 })}`,
+        },
+      })
+      this.log('Request successful')
+    } catch (e) {
+      if (axios.isAxiosError(e)) {
+        this.error(`Request failed: ${e.response ? JSON.stringify(e.response.data) : e.message}`, {
+          exit: ExitCodes.ApiError,
+        })
+      }
+      this.error(e instanceof Error ? e.message : JSON.stringify(e), { exit: ExitCodes.ApiError })
+    }
+  }
+}

distributor-node/src/commands/dev/batchUpload.ts

@@ -1,18 +1,17 @@
 import AccountsCommandBase from '../../command-base/accounts'
 import DefaultCommandBase, { flags } from '../../command-base/default'
-import { hash } from 'blake3-wasm'
 import { FilesApi, Configuration, TokenRequest } from '../../services/networking/storage-node/generated'
 import { u8aToHex } from '@polkadot/util'
-import * as multihash from 'multihashes'
 import FormData from 'form-data'
 import imgGen from 'js-image-generator'
 import { SubmittableExtrinsic } from '@polkadot/api/types'
 import { BagIdParserService } from '../../services/parsers/BagIdParserService'
 import axios from 'axios'
+import { ContentHash } from '../../services/crypto/ContentHash'
 
 async function generateRandomImage(): Promise<Buffer> {
   return new Promise((resolve, reject) => {
-    imgGen.generateImage(10, 10, 80, function (err: any, image: any) {
+    imgGen.generateImage(10, 10, 80, function (err: unknown, image: { data: Buffer }) {
       if (err) {
         reject(err)
       } else {
@@ -61,7 +60,7 @@ export default class DevBatchUpload extends AccountsCommandBase {
       const batch: [SubmittableExtrinsic<'promise'>, Buffer][] = []
       for (let j = 0; j < batchSize; ++j) {
         const dataObject = await generateRandomImage()
-        const dataHash = multihash.toB58String(multihash.encode(hash(dataObject) as Buffer, 'blake3'))
+        const dataHash = new ContentHash().update(dataObject).digest()
         batch.push([
           api.tx.sudo.sudo(
             api.tx.storage.sudoUploadDataObjects({
@@ -73,7 +72,7 @@ export default class DevBatchUpload extends AccountsCommandBase {
                 },
               ],
               expectedDataSizeFee: dataFee,
-              bagId: new BagIdParserService().parseBagId(bagId),
+              bagId: new BagIdParserService(bagId).parse(),
             })
           ),
           dataObject,
@@ -102,7 +101,7 @@ export default class DevBatchUpload extends AccountsCommandBase {
             signature,
           })
           if (!token) {
-            throw new Error('Recieved empty token!')
+            throw new Error('Received empty token!')
           }
 
           const formData = new FormData()

distributor-node/src/commands/leader/update-dynamic-bag-policy.ts

@@ -2,6 +2,7 @@ import { flags } from '@oclif/command'
 import { DynamicBagTypeKey } from '@joystream/types/storage'
 import AccountsCommandBase from '../../command-base/accounts'
 import DefaultCommandBase from '../../command-base/default'
+import { createType } from '@joystream/types'
 
 export default class LeaderUpdateDynamicBagPolicy extends AccountsCommandBase {
   static description = `Update dynamic bag creation policy (number of buckets by family that should store given dynamic bag type).
@@ -41,11 +42,7 @@ export default class LeaderUpdateDynamicBagPolicy extends AccountsCommandBase {
       await this.getDecodedPair(leadKey),
       this.api.tx.storage.updateFamiliesInDynamicBagCreationPolicy(
         type,
-        // FIXME: https://github.com/polkadot-js/api/pull/3789
-        this.api.createType(
-          'DynamicBagCreationPolicyDistributorFamiliesMap',
-          new Map((policy || []).sort(([keyA], [keyB]) => keyA - keyB))
-        )
+        createType('DynamicBagCreationPolicyDistributorFamiliesMap', new Map(policy))
       )
     )
     this.log('Dynamic bag creation policy succesfully updated!')

distributor-node/src/commands/node/set-buckets.ts

@@ -0,0 +1,41 @@
+import { flags } from '@oclif/command'
+import ExitCodes from '../../command-base/ExitCodes'
+import NodeCommandBase from '../../command-base/node'
+import { SetBucketsOperation } from '../../types'
+
+export default class NodeSetBucketsCommand extends NodeCommandBase {
+  static description = `Send an api request to change the set of buckets distributed by given distributor node.`
+
+  static flags = {
+    all: flags.boolean({
+      char: 'a',
+      description: 'Distribute all buckets belonging to configured worker',
+      exclusive: ['bucketIds'],
+    }),
+    bucketIds: flags.integer({
+      char: 'B',
+      description: 'Set of bucket ids to distribute',
+      exclusive: ['all'],
+      multiple: true,
+    }),
+    ...NodeCommandBase.flags,
+  }
+
+  protected reqUrl(): string {
+    return '/api/v1/set-buckets'
+  }
+
+  protected reqBody(): SetBucketsOperation {
+    const {
+      flags: { all, bucketIds },
+    } = this.parse(NodeSetBucketsCommand)
+    if (!all && !bucketIds) {
+      this.error('You must provide either --bucketIds or --all flag!', { exit: ExitCodes.InvalidInput })
+    }
+    return all
+      ? {}
+      : {
+          buckets: bucketIds,
+        }
+  }
+}

distributor-node/src/commands/node/set-worker.ts

@@ -0,0 +1,29 @@
+import { flags } from '@oclif/command'
+import NodeCommandBase from '../../command-base/node'
+import { SetWorkerOperation } from '../../types'
+
+export default class NodeSetWorkerCommand extends NodeCommandBase {
+  static description = `Send an api request to change workerId assigned to given distributor node instance.`
+
+  static flags = {
+    workerId: flags.integer({
+      char: 'w',
+      description: 'New workerId to set',
+      required: true,
+    }),
+    ...NodeCommandBase.flags,
+  }
+
+  protected reqUrl(): string {
+    return '/api/v1/set-worker'
+  }
+
+  protected reqBody(): SetWorkerOperation {
+    const {
+      flags: { workerId },
+    } = this.parse(NodeSetWorkerCommand)
+    return {
+      workerId,
+    }
+  }
+}

distributor-node/src/commands/node/shutdown.ts

@@ -0,0 +1,17 @@
+import NodeCommandBase from '../../command-base/node'
+
+export default class NodeShutdownCommand extends NodeCommandBase {
+  static description = `Send an api request to shutdown given distributor node.`
+
+  static flags = {
+    ...NodeCommandBase.flags,
+  }
+
+  protected reqUrl(): string {
+    return '/api/v1/shutdown'
+  }
+
+  protected reqBody(): Record<string, unknown> {
+    return {}
+  }
+}

distributor-node/src/commands/node/start-public-api.ts

@@ -0,0 +1,17 @@
+import NodeCommandBase from '../../command-base/node'
+
+export default class NodeStartPublicApiCommand extends NodeCommandBase {
+  static description = `Send an api request to start public api of given distributor node.`
+
+  static flags = {
+    ...NodeCommandBase.flags,
+  }
+
+  protected reqUrl(): string {
+    return '/api/v1/start-api'
+  }
+
+  protected reqBody(): Record<string, unknown> {
+    return {}
+  }
+}

distributor-node/src/commands/node/stop-public-api.ts

@@ -0,0 +1,17 @@
+import NodeCommandBase from '../../command-base/node'
+
+export default class NodeStopPublicApiCommand extends NodeCommandBase {
+  static description = `Send an api request to stop public api of given distributor node.`
+
+  static flags = {
+    ...NodeCommandBase.flags,
+  }
+
+  protected reqUrl(): string {
+    return '/api/v1/stop-api'
+  }
+
+  protected reqBody(): Record<string, unknown> {
+    return {}
+  }
+}

distributor-node/src/commands/start.ts

@@ -1,5 +1,6 @@
 import DefaultCommandBase from '../command-base/default'
 import { App } from '../app'
+import { Config } from '../types'
 
 export default class StartNode extends DefaultCommandBase {
   static description = 'Start the node'
@@ -9,7 +10,7 @@ export default class StartNode extends DefaultCommandBase {
   }
 
   async run(): Promise<void> {
-    const app = new App(this.appConfig)
+    const app = new App(this.appConfig as Config)
     await app.start()
   }
 

distributor-node/src/schemas/configSchema.ts

@@ -1,27 +1,30 @@
 import { JSONSchema4 } from 'json-schema'
 import winston from 'winston'
 import { MAX_CONCURRENT_RESPONSE_TIME_CHECKS } from '../services/networking/NetworkingService'
+import { objectSchema } from './utils'
 
 export const bytesizeUnits = ['B', 'K', 'M', 'G', 'T']
 export const bytesizeRegex = new RegExp(`^[0-9]+(${bytesizeUnits.join('|')})$`)
 
-export const configSchema: JSONSchema4 = {
+const logLevelSchema: JSONSchema4 = {
+  description: 'Minimum level of logs sent to this output',
+  type: 'string',
+  enum: [...Object.keys(winston.config.npm.levels)],
+}
+
+export const configSchema: JSONSchema4 = objectSchema({
+  '$id': 'https://joystream.org/schemas/argus/config',
   title: 'Distributor node configuration',
   description: 'Configuration schema for distirubtor CLI and node',
-  type: 'object',
-  required: ['id', 'endpoints', 'directories', 'buckets', 'keys', 'port', 'workerId', 'limits', 'intervals'],
-  additionalProperties: false,
+  required: ['id', 'endpoints', 'directories', 'limits', 'intervals', 'publicApi'],
   properties: {
     id: {
       type: 'string',
       description: 'Node identifier used when sending elasticsearch logs and exposed on /status endpoint',
       minLength: 1,
     },
-    endpoints: {
-      type: 'object',
+    endpoints: objectSchema({
       description: 'Specifies external endpoints that the distributor node will connect to',
-      additionalProperties: false,
-      required: ['queryNode', 'joystreamNodeWs'],
       properties: {
         queryNode: {
           description: 'Query node graphql server uri (for example: http://localhost:8081/graphql)',
@@ -31,16 +34,10 @@ export const configSchema: JSONSchema4 = {
           description: 'Joystream node websocket api uri (for example: ws://localhost:9944)',
           type: 'string',
         },
-        elasticSearch: {
-          description: 'Elasticsearch uri used for submitting the distributor node logs (if enabled via `log.elastic`)',
-          type: 'string',
-        },
       },
-    },
-    directories: {
-      type: 'object',
-      required: ['assets', 'cacheState'],
-      additionalProperties: false,
+      required: ['queryNode', 'joystreamNodeWs'],
+    }),
+    directories: objectSchema({
       description: "Specifies paths where node's data will be stored",
       properties: {
         assets: {
@@ -52,45 +49,65 @@ export const configSchema: JSONSchema4 = {
             'Path to a directory where information about the current cache state will be stored (LRU-SP cache data, stored assets mime types etc.)',
           type: 'string',
         },
-        logs: {
-          description:
-            'Path to a directory where logs will be stored if logging to a file was enabled (via `log.file`).',
-          type: 'string',
-        },
       },
-    },
-    log: {
-      type: 'object',
-      additionalProperties: false,
-      description: 'Specifies minimum log levels by supported log outputs',
+      required: ['assets', 'cacheState'],
+    }),
+    logs: objectSchema({
+      description: 'Specifies the logging configuration',
       properties: {
-        file: {
-          description: 'Minimum level of logs written to a file specified in `directories.logs`',
-          type: 'string',
-          enum: [...Object.keys(winston.config.npm.levels), 'off'],
-        },
-        console: {
-          description: 'Minimum level of logs outputted to a console',
-          type: 'string',
-          enum: [...Object.keys(winston.config.npm.levels), 'off'],
-        },
-        elastic: {
-          description: 'Minimum level of logs sent to elasticsearch endpoint specified in `endpoints.elasticSearch`',
-          type: 'string',
-          enum: [...Object.keys(winston.config.npm.levels), 'off'],
-        },
+        file: objectSchema({
+          title: 'File logging options',
+          properties: {
+            level: logLevelSchema,
+            path: {
+              description: 'Path where the logs will be stored (absolute or relative to config file)',
+              type: 'string',
+            },
+            maxFiles: {
+              description: 'Maximum number of log files to store',
+              type: 'integer',
+              minimum: 1,
+            },
+            maxSize: {
+              description: 'Maximum size of a single log file in bytes',
+              type: 'integer',
+              minimum: 1024,
+            },
+            frequency: {
+              description: 'The frequency of creating new log files (regardless of maxSize)',
+              default: 'daily',
+              type: 'string',
+              enum: ['yearly', 'monthly', 'daily', 'hourly'],
+            },
+            archive: {
+              description: 'Whether to archive old logs',
+              default: false,
+              type: 'boolean',
+            },
+          },
+          required: ['level', 'path'],
+        }),
+        console: objectSchema({
+          title: 'Console logging options',
+          properties: { level: logLevelSchema },
+          required: ['level'],
+        }),
+        elastic: objectSchema({
+          title: 'Elasticsearch logging options',
+          properties: {
+            level: logLevelSchema,
+            endpoint: {
+              description: 'Elastichsearch endpoint to push the logs to (for example: http://localhost:9200)',
+              type: 'string',
+            },
+          },
+          required: ['level', 'endpoint'],
+        }),
       },
-    },
-    limits: {
-      type: 'object',
-      required: [
-        'storage',
-        'maxConcurrentStorageNodeDownloads',
-        'maxConcurrentOutboundConnections',
-        'outboundRequestsTimeout',
-      ],
+      required: [],
+    }),
+    limits: objectSchema({
       description: 'Specifies node limits w.r.t. storage, outbound connections etc.',
-      additionalProperties: false,
       properties: {
         storage: {
           description: 'Maximum total size of all (cached) assets stored in `directories.assets`',
@@ -103,21 +120,43 @@ export const configSchema: JSONSchema4 = {
           minimum: 1,
         },
         maxConcurrentOutboundConnections: {
-          description: 'Maximum number of total simultaneous outbound connections to storage node(s)',
+          description:
+            'Maximum number of total simultaneous outbound connections to storage node(s) (excluding proxy connections)',
           type: 'integer',
           minimum: 1,
         },
-        outboundRequestsTimeout: {
+        outboundRequestsTimeoutMs: {
           description: 'Timeout for all outbound storage node http requests in miliseconds',
           type: 'integer',
+          minimum: 1000,
+        },
+        pendingDownloadTimeoutSec: {
+          description: 'Timeout for pending storage node downloads in seconds',
+          type: 'integer',
+          minimum: 60,
+        },
+        maxCachedItemSize: {
+          description: 'Maximum size of a data object allowed to be cached by the node',
+          type: 'string',
+          pattern: bytesizeRegex.source,
+        },
+        dataObjectSourceByObjectIdTTL: {
+          description:
+            'TTL (in seconds) for dataObjectSourceByObjectId cache used when proxying objects of size greater than maxCachedItemSize to the right storage node.',
+          default: 60,
+          type: 'integer',
           minimum: 1,
         },
       },
-    },
-    intervals: {
-      type: 'object',
-      required: ['saveCacheState', 'checkStorageNodeResponseTimes', 'cacheCleanup'],
-      additionalProperties: false,
+      required: [
+        'storage',
+        'maxConcurrentStorageNodeDownloads',
+        'maxConcurrentOutboundConnections',
+        'outboundRequestsTimeoutMs',
+        'pendingDownloadTimeoutSec',
+      ],
+    }),
+    intervals: objectSchema({
       description: 'Specifies how often periodic tasks (for example cache cleanup) are executed by the node.',
       properties: {
         saveCacheState: {
@@ -143,66 +182,66 @@ export const configSchema: JSONSchema4 = {
           minimum: 1,
         },
       },
-    },
-    port: { description: 'Distributor node http server port', type: 'integer', minimum: 0 },
+      required: ['saveCacheState', 'checkStorageNodeResponseTimes', 'cacheCleanup'],
+    }),
+    publicApi: objectSchema({
+      description: 'Public api configuration',
+      properties: {
+        port: { description: 'Distributor node public api port', type: 'integer', minimum: 0 },
+      },
+      required: ['port'],
+    }),
+    operatorApi: objectSchema({
+      description: 'Operator api configuration',
+      properties: {
+        port: { description: 'Distributor node operator api port', type: 'integer', minimum: 0 },
+        hmacSecret: { description: 'HMAC (HS256) secret key used for JWT authorization', type: 'string' },
+      },
+      required: ['port', 'hmacSecret'],
+    }),
     keys: {
       description: 'Specifies the keys available within distributor node CLI.',
       type: 'array',
       items: {
         oneOf: [
-          {
-            type: 'object',
+          objectSchema({
             title: 'Substrate uri',
             description: "Keypair's substrate uri (for example: //Alice)",
-            required: ['suri'],
-            additionalProperties: false,
             properties: {
               type: { type: 'string', enum: ['ed25519', 'sr25519', 'ecdsa'], default: 'sr25519' },
               suri: { type: 'string' },
             },
-          },
-          {
-            type: 'object',
+            required: ['suri'],
+          }),
+          objectSchema({
             title: 'Mnemonic phrase',
             description: 'Menomonic phrase',
-            required: ['mnemonic'],
-            additionalProperties: false,
             properties: {
               type: { type: 'string', enum: ['ed25519', 'sr25519', 'ecdsa'], default: 'sr25519' },
               mnemonic: { type: 'string' },
             },
-          },
-          {
-            type: 'object',
+            required: ['mnemonic'],
+          }),
+          objectSchema({
             title: 'JSON backup file',
             description: 'Path to JSON backup file from polkadot signer / polakdot/apps (relative to config file path)',
-            required: ['keyfile'],
-            additionalProperties: false,
             properties: {
               keyfile: { type: 'string' },
             },
-          },
+            required: ['keyfile'],
+          }),
         ],
       },
       minItems: 1,
     },
     buckets: {
-      description: 'Specifies the buckets distributed by the node',
-      oneOf: [
-        {
-          title: 'Bucket ids',
-          description: 'List of distribution bucket ids',
-          type: 'array',
-          items: { type: 'integer', minimum: 0 },
-          minItems: 1,
-        },
-        {
-          title: 'All buckets',
-          description: 'Distribute all buckets assigned to worker specified in `workerId`',
-          type: 'string',
-          enum: ['all'],
-        },
-      ],
+      description:
+        'Set of bucket ids distributed by the node. If not specified, all buckets currently assigned to worker specified in `config.workerId` will be distributed.',
+      title: 'Bucket ids',
+      type: 'array',
+      uniqueItems: true,
+      items: { type: 'integer', minimum: 0 },
+      minItems: 1,
     },
     workerId: {
       description: 'ID of the node operator (distribution working group worker)',
@@ -210,6 +249,6 @@ export const configSchema: JSONSchema4 = {
       minimum: 0,
     },
   },
-}
+})
 
 export default configSchema

distributor-node/src/schemas/scripts/generateTypes.ts

@@ -7,7 +7,10 @@ import { schemas } from '..'
 const prettierConfig = require('@joystream/prettier-config')
 
 Object.entries(schemas).forEach(([schemaKey, schema]) => {
-  compile(schema, `${schemaKey}Json`, { style: prettierConfig })
+  compile(schema, `${schemaKey}Json`, {
+    style: prettierConfig,
+    ignoreMinAndMaxItems: true,
+  })
     .then((output) => fs.writeFileSync(path.resolve(__dirname, `../../types/generated/${schemaKey}Json.d.ts`), output))
     .catch(console.error)
 })

distributor-node/src/schemas/utils.ts

@@ -0,0 +1,15 @@
+import { JSONSchema4 } from 'json-schema'
+
+export function objectSchema<P extends NonNullable<JSONSchema4['properties']>>(props: {
+  $id?: string
+  title?: string
+  description?: string
+  properties: P
+  required: Array<keyof P & string>
+}): JSONSchema4 {
+  return {
+    type: 'object',
+    additionalProperties: false,
+    ...props,
+  }
+}

distributor-node/src/services/cache/StateCacheService.ts

@@ -1,22 +1,17 @@
 import { Logger } from 'winston'
-import { ReadonlyConfig, StorageNodeDownloadResponse } from '../../types'
+import { ReadonlyConfig } from '../../types'
 import { LoggingService } from '../logging'
 import _ from 'lodash'
 import fs from 'fs'
+import NodeCache from 'node-cache'
+import { PendingDownload } from '../networking/PendingDownload'
 
 // LRU-SP cache parameters
 // Since size is in KB, these parameters should be enough for grouping objects of size up to 2^24 KB = 16 GB
-// TODO: Intoduce MAX_CACHED_ITEM_SIZE and skip caching for large objects entirely? (ie. 10 GB objects)
 export const CACHE_GROUP_LOG_BASE = 2
 export const CACHE_GROUPS_COUNT = 24
 
-type PendingDownloadStatus = 'Waiting' | 'LookingForSource' | 'Downloading'
-
-export interface PendingDownloadData {
-  objectSize: number
-  status: PendingDownloadStatus
-  promise: Promise<StorageNodeDownloadResponse>
-}
+export const DEFAULT_DATA_OBJECT_SOURCE_CACHE_TTL = 60
 
 export interface StorageNodeEndpointData {
   last10ResponseTimes: number[]
@@ -34,9 +29,12 @@ export class StateCacheService {
   private cacheFilePath: string
 
   private memoryState = {
-    pendingDownloadsByObjectId: new Map<string, PendingDownloadData>(),
+    pendingDownloadsByObjectId: new Map<string, PendingDownload>(),
     storageNodeEndpointDataByEndpoint: new Map<string, StorageNodeEndpointData>(),
     groupNumberByObjectId: new Map<string, number>(),
+    dataObjectSourceByObjectId: new NodeCache({
+      deleteOnExpire: true,
+    }),
   }
 
   private storedState = {
@@ -149,17 +147,8 @@ export class StateCacheService {
     return bestCandidate
   }
 
-  public newPendingDownload(
-    objectId: string,
-    objectSize: number,
-    promise: Promise<StorageNodeDownloadResponse>
-  ): PendingDownloadData {
-    const pendingDownload: PendingDownloadData = {
-      status: 'Waiting',
-      objectSize,
-      promise,
-    }
-    this.memoryState.pendingDownloadsByObjectId.set(objectId, pendingDownload)
+  public addPendingDownload(pendingDownload: PendingDownload): PendingDownload {
+    this.memoryState.pendingDownloadsByObjectId.set(pendingDownload.getObjectId(), pendingDownload)
     return pendingDownload
   }
 
@@ -167,18 +156,22 @@ export class StateCacheService {
     return this.memoryState.pendingDownloadsByObjectId.size
   }
 
-  public getPendingDownload(objectId: string): PendingDownloadData | undefined {
+  public getPendingDownload(objectId: string): PendingDownload | undefined {
     return this.memoryState.pendingDownloadsByObjectId.get(objectId)
   }
 
   public dropPendingDownload(objectId: string): void {
-    this.memoryState.pendingDownloadsByObjectId.delete(objectId)
+    const pendingDownload = this.memoryState.pendingDownloadsByObjectId.get(objectId)
+    if (pendingDownload) {
+      pendingDownload.cleanup()
+      this.memoryState.pendingDownloadsByObjectId.delete(objectId)
+    }
   }
 
   public dropById(objectId: string): void {
     this.logger.debug('Dropping all state by object id', { objectId })
     this.storedState.mimeTypeByObjectId.delete(objectId)
-    this.memoryState.pendingDownloadsByObjectId.delete(objectId)
+    this.dropPendingDownload(objectId)
     const cacheGroupNumber = this.memoryState.groupNumberByObjectId.get(objectId)
     this.logger.debug('Cache group by object id established', { objectId, cacheGroupNumber })
     if (cacheGroupNumber) {
@@ -210,6 +203,26 @@ export class StateCacheService {
     ])
   }
 
+  public cacheDataObjectSource(objectId: string, source: string): void {
+    this.memoryState.dataObjectSourceByObjectId.set<string>(
+      objectId,
+      source,
+      this.config.limits.dataObjectSourceByObjectIdTTL || DEFAULT_DATA_OBJECT_SOURCE_CACHE_TTL
+    )
+  }
+
+  public getCachedDataObjectSource(objectId: string): string | undefined {
+    return this.memoryState.dataObjectSourceByObjectId.get<string | undefined>(objectId)
+  }
+
+  public dropCachedDataObjectSource(objectId: string, expectedSource?: string): void {
+    const cachedSource = this.memoryState.dataObjectSourceByObjectId.get<string | undefined>(objectId)
+    if (!expectedSource || cachedSource === expectedSource) {
+      this.logger.info('Force-dropping cached dataObjectSource', { objectId, cachedSource, expectedSource })
+      this.memoryState.dataObjectSourceByObjectId.del(objectId)
+    }
+  }
+
   private serializeData() {
     const { lruCacheGroups, mimeTypeByObjectId } = this.storedState
     return JSON.stringify(
@@ -218,7 +231,7 @@ export class StateCacheService {
         mimeTypeByObjectId: Array.from(mimeTypeByObjectId.entries()),
       },
       null,
-      2 // TODO: Only for debugging
+      2
     )
   }
 

distributor-node/src/services/content/ContentService.ts

@@ -1,5 +1,5 @@
 import fs from 'fs'
-import { ReadonlyConfig } from '../../types'
+import { ObjectStatus, ObjectStatusType, ReadonlyConfig } from '../../types'
 import { StateCacheService } from '../cache/StateCacheService'
 import { LoggingService } from '../logging'
 import { Logger } from 'winston'
@@ -7,10 +7,11 @@ import { FileContinousReadStream, FileContinousReadStreamOptions } from './FileC
 import FileType from 'file-type'
 import { Readable, pipeline } from 'stream'
 import { NetworkingService } from '../networking'
-import { createHash } from 'blake3-wasm'
-import * as multihash from 'multihashes'
+import { ContentHash } from '../crypto/ContentHash'
+import readChunk from 'read-chunk'
 
 export const DEFAULT_CONTENT_TYPE = 'application/octet-stream'
+export const MIME_TYPE_DETECTION_CHUNK_SIZE = 4100
 
 export class ContentService {
   private config: ReadonlyConfig
@@ -90,6 +91,12 @@ export class ContentService {
         continue
       }
 
+      // Drop files that are missing in the cache
+      if (!this.stateCache.peekContent(objectId)) {
+        this.drop(objectId, 'Missing cache data')
+        continue
+      }
+
       // Compare file size to expected one
       const { size: dataObjectSize } = dataObject
       if (fileSize !== dataObjectSize) {
@@ -122,11 +129,13 @@ export class ContentService {
     })
   }
 
-  public drop(objectId: string, reason?: string): void {
+  public drop(objectId: string, reason?: string, unreserveSpace = true): void {
     if (this.exists(objectId)) {
       const size = this.fileSize(objectId)
       fs.unlinkSync(this.path(objectId))
-      this.contentSizeSum -= size
+      if (unreserveSpace) {
+        this.contentSizeSum -= size
+      }
       this.logger.debug('Dropping content', { objectId, reason, size, contentSizeSum: this.contentSizeSum })
     } else {
       this.logger.warn('Trying to drop content that no loger exists', { objectId, reason })
@@ -159,8 +168,19 @@ export class ContentService {
   }
 
   public async detectMimeType(objectId: string): Promise<string> {
-    const result = await FileType.fromFile(this.path(objectId))
-    return result?.mime || DEFAULT_CONTENT_TYPE
+    const objectPath = this.path(objectId)
+    try {
+      const buffer = await readChunk(objectPath, 0, MIME_TYPE_DETECTION_CHUNK_SIZE)
+      const result = await FileType.fromBuffer(buffer)
+      return result?.mime || DEFAULT_CONTENT_TYPE
+    } catch (err) {
+      this.logger.error(`Error while trying to detect object mimeType: ${err instanceof Error ? err.message : err}`, {
+        err,
+        objectId,
+        objectPath,
+      })
+      return DEFAULT_CONTENT_TYPE
+    }
   }
 
   private async evictCacheUntilFreeSpaceReached(targetFreeSpace: number): Promise<void> {
@@ -203,44 +223,60 @@ export class ContentService {
       newContentSizeSum: this.contentSizeSum,
     })
 
+    const rejectContent = (reason: string, metadata: Record<string, unknown>) => {
+      const msg = `Content rejected: ${reason}`
+      // Drop (without unreserving space, will do that manually)
+      this.drop(objectId, msg, false)
+      // Unreserve reserved space
+      this.contentSizeSum -= expectedSize
+      // Log the error
+      this.logger.error(msg, { ...metadata })
+    }
+
     // Return a promise that resolves when the new file is created
     return new Promise<void>((resolve, reject) => {
       const fileStream = this.createWriteStream(objectId)
 
-      let bytesRecieved = 0
-      const hash = createHash()
+      let bytesReceived = 0
+      const hash = new ContentHash()
+
+      const onData = (chunk: Buffer) => {
+        bytesReceived += chunk.length
+        hash.update(chunk)
+
+        if (bytesReceived > expectedSize) {
+          dataStream.destroy(new Error('Unexpected content size: Too much data received from source!'))
+        }
+      }
 
       pipeline(dataStream, fileStream, async (err) => {
+        dataStream.off('data', onData)
         const { bytesWritten } = fileStream
-        const finalHash = multihash.toB58String(multihash.encode(hash.digest(), 'blake3'))
+        const finalHash = hash.digest()
         const logMetadata = {
           objectId,
           expectedSize,
-          expectedHash,
-          bytesRecieved,
+          bytesReceived,
           bytesWritten,
+          expectedHash,
+          finalHash,
         }
         if (err) {
-          this.logger.error(`Error while processing content data stream`, {
+          rejectContent(`Error while processing content data stream`, {
             err,
             ...logMetadata,
           })
-          this.drop(objectId)
           reject(err)
           return
         }
 
-        if (bytesWritten !== bytesRecieved || bytesWritten !== expectedSize) {
-          this.logger.error('Content rejected: Bytes written/recieved/expected mismatch!', {
-            ...logMetadata,
-          })
-          this.drop(objectId)
+        if (bytesWritten !== bytesReceived || bytesWritten !== expectedSize) {
+          rejectContent('Bytes written/received/expected mismatch!', { ...logMetadata })
           return
         }
 
         if (finalHash !== expectedHash) {
-          this.logger.error('Content rejected: Hash mismatch!', { ...logMetadata })
-          this.drop(objectId)
+          rejectContent('Hash mismatch!', { ...logMetadata })
           return
         }
 
@@ -255,15 +291,35 @@ export class ContentService {
         // Note: The promise is resolved on "ready" event, since that's what's awaited in the current flow
         resolve()
       })
+      dataStream.on('data', onData)
+    })
+  }
 
-      dataStream.on('data', (chunk) => {
-        bytesRecieved += chunk.length
-        hash.update(chunk)
+  public async objectStatus(objectId: string): Promise<ObjectStatus> {
+    const pendingDownload = this.stateCache.getPendingDownload(objectId)
 
-        if (bytesRecieved > expectedSize) {
-          dataStream.destroy(new Error('Unexpected content size: Too much data recieved from source!'))
-        }
-      })
-    })
+    if (!pendingDownload && this.exists(objectId)) {
+      return { type: ObjectStatusType.Available, path: this.path(objectId) }
+    }
+
+    if (pendingDownload) {
+      return { type: ObjectStatusType.PendingDownload, pendingDownload }
+    }
+
+    const objectInfo = await this.networking.dataObjectInfo(objectId)
+    if (!objectInfo.exists) {
+      return { type: ObjectStatusType.NotFound }
+    }
+
+    if (!objectInfo.isSupported) {
+      return { type: ObjectStatusType.NotSupported }
+    }
+
+    const { data: objectData } = objectInfo
+    if (!objectData) {
+      throw new Error('Missing data object data')
+    }
+
+    return { type: ObjectStatusType.Missing, objectData }
   }
 }

distributor-node/src/services/crypto/ContentHash.ts

@@ -0,0 +1,21 @@
+import { createHash, HashInput, NodeHash } from 'blake3-wasm'
+import { HashReader } from 'blake3-wasm/dist/wasm/nodejs/blake3_js'
+import { toB58String, encode } from 'multihashes'
+
+export class ContentHash {
+  private hash: NodeHash<HashReader>
+  public static readonly algorithm = 'blake3'
+
+  constructor() {
+    this.hash = createHash()
+  }
+
+  update(data: HashInput): this {
+    this.hash.update(data)
+    return this
+  }
+
+  digest(): string {
+    return toB58String(encode(this.hash.digest(), ContentHash.algorithm))
+  }
+}

distributor-node/src/services/httpApi/HttpApiBase.ts

@@ -0,0 +1,145 @@
+import express from 'express'
+import * as OpenApiValidator from 'express-openapi-validator'
+import { HttpError, OpenApiValidatorOpts } from 'express-openapi-validator/dist/framework/types'
+import { ReadonlyConfig } from '../../types/config'
+import expressWinston from 'express-winston'
+import { Logger } from 'winston'
+import { Server } from 'http'
+import cors from 'cors'
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type HttpApiRoute = ['get' | 'head' | 'post', string, express.RequestHandler<any>]
+
+export abstract class HttpApiBase {
+  protected abstract port: number
+  protected expressApp: express.Application
+  protected config: ReadonlyConfig
+  protected logger: Logger
+  private httpServer: Server | undefined
+  private isInitialized = false
+  private isOn = false
+
+  protected routeWrapper(handler: express.RequestHandler) {
+    return async (req: express.Request, res: express.Response, next: express.NextFunction): Promise<void> => {
+      // Fix for express-winston in order to also log prematurely closed requests
+      res.on('close', () => {
+        res.locals.prematurelyClosed = !res.writableFinished
+        res.end()
+      })
+      try {
+        await handler(req, res, next)
+      } catch (err) {
+        next(err)
+      }
+    }
+  }
+
+  public constructor(config: ReadonlyConfig, logger: Logger) {
+    this.expressApp = express()
+    this.logger = logger
+    this.config = config
+  }
+
+  protected createRoutes(routes: HttpApiRoute[]): void {
+    routes.forEach(([type, path, handler]) => {
+      this.expressApp[type](path, this.routeWrapper(handler))
+    })
+  }
+
+  protected abstract routes(): HttpApiRoute[]
+
+  protected defaultOpenApiValidatorConfig(): Partial<OpenApiValidatorOpts> {
+    const isProd = process.env.NODE_ENV === 'prod'
+    return {
+      validateResponses: !isProd,
+    }
+  }
+
+  protected abstract openApiValidatorConfig(): OpenApiValidatorOpts
+
+  protected defaultRequestLoggerConfig(): expressWinston.LoggerOptions {
+    return {
+      winstonInstance: this.logger,
+      level: 'http',
+      dynamicMeta: (req, res) => {
+        return { prematurelyClosed: res.locals.prematurelyClosed ?? false }
+      },
+    }
+  }
+
+  protected requestLoggerConfig(): expressWinston.LoggerOptions {
+    return this.defaultRequestLoggerConfig()
+  }
+
+  protected defaultErrorLoggerConfig(): expressWinston.ErrorLoggerOptions {
+    return {
+      winstonInstance: this.logger,
+      level: 'error',
+      metaField: null,
+      exceptionToMeta: (err) => ({ err }),
+    }
+  }
+
+  protected errorLoggerConfig(): expressWinston.ErrorLoggerOptions {
+    return this.defaultErrorLoggerConfig()
+  }
+
+  protected errorHandler() {
+    return (err: HttpError, req: express.Request, res: express.Response, next: express.NextFunction): void => {
+      if (res.headersSent) {
+        return next(err)
+      }
+      if (err.status && err.status >= 400 && err.status < 500) {
+        res
+          .status(err.status)
+          .json({
+            type: 'request_validation',
+            message: err.message,
+            errors: err.errors,
+          })
+          .end()
+      } else {
+        res.status(err.status || 500).json({ type: 'exception', message: err.message })
+      }
+    }
+  }
+
+  protected initApp(): void {
+    if (this.isInitialized) {
+      return
+    }
+    const { expressApp: app } = this
+    app.use(express.json())
+    app.use(cors())
+    app.use(expressWinston.logger(this.requestLoggerConfig()))
+    app.use(OpenApiValidator.middleware(this.openApiValidatorConfig()))
+    this.createRoutes(this.routes())
+    app.use(expressWinston.errorLogger(this.errorLoggerConfig()))
+    app.use(this.errorHandler())
+    this.isInitialized = true
+  }
+
+  public start(): boolean {
+    if (this.isOn) {
+      return false
+    }
+    if (!this.isInitialized) {
+      this.initApp()
+    }
+    this.httpServer = this.expressApp.listen(this.port, () => {
+      this.logger.info(`Express server started listening on port ${this.port}`)
+    })
+    this.isOn = true
+    return true
+  }
+
+  public stop(): boolean {
+    if (!this.isOn) {
+      return false
+    }
+    this.httpServer?.close()
+    this.logger.info(`Express server stopped`)
+    this.isOn = false
+    return true
+  }
+}

distributor-node/src/services/httpApi/OperatorApiService.ts

@@ -0,0 +1,90 @@
+import express from 'express'
+import path from 'path'
+import { OpenApiValidatorOpts } from 'express-openapi-validator/dist/framework/types'
+import { Config } from '../../types/config'
+import { LoggingService } from '../logging'
+import jwt from 'jsonwebtoken'
+import { OperatorApiController } from './controllers/operator'
+import { HttpApiBase, HttpApiRoute } from './HttpApiBase'
+import { PublicApiService } from './PublicApiService'
+import _ from 'lodash'
+import { App } from '../../app'
+
+const OPENAPI_SPEC_PATH = path.join(__dirname, '../../api-spec/operator.yml')
+const JWT_TOKEN_MAX_AGE = '5m'
+
+export class OperatorApiService extends HttpApiBase {
+  protected port: number
+  protected operatorSecretKey: string
+  protected config: Config
+  protected app: App
+  protected publicApi: PublicApiService
+  protected logging: LoggingService
+
+  public constructor(config: Config, app: App, logging: LoggingService, publicApi: PublicApiService) {
+    super(config, logging.createLogger('OperatorApi'))
+    if (!config.operatorApi) {
+      throw new Error('Cannot construct OperatorApiService - missing operatorApi config!')
+    }
+    this.port = config.operatorApi.port
+    this.operatorSecretKey = config.operatorApi.hmacSecret
+    this.config = config
+    this.app = app
+    this.logging = logging
+    this.publicApi = publicApi
+    this.initApp()
+  }
+
+  protected openApiValidatorConfig(): OpenApiValidatorOpts {
+    return {
+      apiSpec: OPENAPI_SPEC_PATH,
+      validateSecurity: {
+        handlers: {
+          OperatorAuth: this.operatorRequestValidator(),
+        },
+      },
+      ...this.defaultOpenApiValidatorConfig(),
+    }
+  }
+
+  protected routes(): HttpApiRoute[] {
+    const controller = new OperatorApiController(this.config, this.app, this.publicApi, this.logging)
+    return [
+      ['post', '/api/v1/stop-api', controller.stopApi.bind(controller)],
+      ['post', '/api/v1/start-api', controller.startApi.bind(controller)],
+      ['post', '/api/v1/shutdown', controller.shutdown.bind(controller)],
+      ['post', '/api/v1/set-worker', controller.setWorker.bind(controller)],
+      ['post', '/api/v1/set-buckets', controller.setBuckets.bind(controller)],
+    ]
+  }
+
+  private operatorRequestValidator() {
+    return (req: express.Request): boolean => {
+      const authHeader = req.headers.authorization
+      if (!authHeader) {
+        throw new Error('Authrorization header missing')
+      }
+
+      const [authType, token] = authHeader.split(' ')
+      if (authType.toLowerCase() !== 'bearer') {
+        throw new Error(`Unexpected authorization type: ${authType}`)
+      }
+
+      if (!token) {
+        throw new Error(`Bearer token missing`)
+      }
+
+      const decoded = jwt.verify(token, this.operatorSecretKey, { maxAge: JWT_TOKEN_MAX_AGE }) as jwt.JwtPayload
+
+      if (!_.isEqual(req.body, decoded.reqBody)) {
+        throw new Error('Invalid token: Request body does not match')
+      }
+
+      if (req.originalUrl !== decoded.reqUrl) {
+        throw new Error('Invalid token: Request url does not match')
+      }
+
+      return true
+    }
+  }
+}

distributor-node/src/services/httpApi/PublicApiService.ts

@@ -0,0 +1,60 @@
+import path from 'path'
+import { ReadonlyConfig } from '../../types/config'
+import { LoggingService } from '../logging'
+import { PublicApiController } from './controllers/public'
+import { StateCacheService } from '../cache/StateCacheService'
+import { NetworkingService } from '../networking'
+import { ContentService } from '../content/ContentService'
+import { HttpApiBase, HttpApiRoute } from './HttpApiBase'
+import { OpenApiValidatorOpts } from 'express-openapi-validator/dist/openapi.validator'
+
+const OPENAPI_SPEC_PATH = path.join(__dirname, '../../api-spec/public.yml')
+
+export class PublicApiService extends HttpApiBase {
+  protected port: number
+
+  private loggingService: LoggingService
+  private networkingService: NetworkingService
+  private stateCache: StateCacheService
+  private contentService: ContentService
+
+  public constructor(
+    config: ReadonlyConfig,
+    stateCache: StateCacheService,
+    content: ContentService,
+    logging: LoggingService,
+    networking: NetworkingService
+  ) {
+    super(config, logging.createLogger('PublicApi'))
+    this.stateCache = stateCache
+    this.loggingService = logging
+    this.networkingService = networking
+    this.contentService = content
+    this.port = config.publicApi.port
+    this.initApp()
+  }
+
+  protected openApiValidatorConfig(): OpenApiValidatorOpts {
+    return {
+      apiSpec: OPENAPI_SPEC_PATH,
+      ...this.defaultOpenApiValidatorConfig(),
+    }
+  }
+
+  protected routes(): HttpApiRoute[] {
+    const publicController = new PublicApiController(
+      this.config,
+      this.loggingService,
+      this.networkingService,
+      this.stateCache,
+      this.contentService
+    )
+
+    return [
+      ['head', '/api/v1/assets/:objectId', publicController.assetHead.bind(publicController)],
+      ['get', '/api/v1/assets/:objectId', publicController.asset.bind(publicController)],
+      ['get', '/api/v1/status', publicController.status.bind(publicController)],
+      ['get', '/api/v1/buckets', publicController.buckets.bind(publicController)],
+    ]
+  }
+}

distributor-node/src/services/httpApi/controllers/operator.ts

@@ -0,0 +1,79 @@
+import { Logger } from 'winston'
+import * as express from 'express'
+import { PublicApiService } from '../PublicApiService'
+import { LoggingService } from '../../logging'
+import { App } from '../../../app'
+import { Config, SetBucketsOperation, SetWorkerOperation } from '../../../types'
+import { ParamsDictionary } from 'express-serve-static-core'
+
+export class OperatorApiController {
+  private config: Config
+  private app: App
+  private publicApi: PublicApiService
+  private logger: Logger
+
+  public constructor(config: Config, app: App, publicApi: PublicApiService, logging: LoggingService) {
+    this.config = config
+    this.app = app
+    this.publicApi = publicApi
+    this.logger = logging.createLogger('OperatorApiController')
+  }
+
+  public async stopApi(req: express.Request, res: express.Response): Promise<void> {
+    this.logger.info(`Stopping public api on operator request from ${req.ip}`, { ip: req.ip })
+    const stopped = this.publicApi.stop()
+    if (!stopped) {
+      res.status(409).json({ message: 'Already stopped' })
+    }
+    res.status(200).send()
+  }
+
+  public async startApi(req: express.Request, res: express.Response): Promise<void> {
+    this.logger.info(`Starting public api on operator request from ${req.ip}`, { ip: req.ip })
+    const started = this.publicApi.start()
+    if (!started) {
+      res.status(409).json({ message: 'Already started' })
+    }
+    res.status(200).send()
+  }
+
+  public async shutdown(req: express.Request, res: express.Response): Promise<void> {
+    this.logger.info(`Shutting down the app on operator request from ${req.ip}`, { ip: req.ip })
+    const shutdown = this.app.stop(5)
+    if (!shutdown) {
+      res.status(409).json({ message: 'Already shutting down' })
+    }
+    res.status(200).send()
+  }
+
+  public async setWorker(
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    req: express.Request<ParamsDictionary, any, SetWorkerOperation>,
+    res: express.Response
+  ): Promise<void> {
+    const { workerId } = req.body
+    this.logger.info(`Updating workerId to ${workerId} on operator request from ${req.ip}`, {
+      workerId,
+      ip: req.ip,
+    })
+    this.config.workerId = workerId
+    res.status(200).send()
+  }
+
+  public async setBuckets(
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    req: express.Request<ParamsDictionary, any, SetBucketsOperation>,
+    res: express.Response
+  ): Promise<void> {
+    const { buckets } = req.body
+    this.logger.info(
+      `Updating buckets to ${buckets ? JSON.stringify(buckets) : '"all"'} on operator request from ${req.ip}`,
+      {
+        buckets,
+        ip: req.ip,
+      }
+    )
+    this.config.buckets = buckets
+    res.status(200).send()
+  }
+}

distributor-node/src/services/server/controllers/public.ts → distributor-node/src/services/httpApi/controllers/public.ts

@@ -1,13 +1,22 @@
 import * as express from 'express'
 import { Logger } from 'winston'
 import send from 'send'
-import { StateCacheService } from '../../../services/cache/StateCacheService'
-import { NetworkingService } from '../../../services/networking'
-import { AssetRouteParams, BucketsResponse, ErrorResponse, StatusResponse } from '../../../types/api'
+import { StateCacheService } from '../../cache/StateCacheService'
+import { NetworkingService } from '../../networking'
+import {
+  AssetRouteParams,
+  BucketsResponse,
+  ErrorResponse,
+  StatusResponse,
+  DataObjectData,
+  ObjectStatusType,
+  ReadonlyConfig,
+} from '../../../types'
 import { LoggingService } from '../../logging'
 import { ContentService, DEFAULT_CONTENT_TYPE } from '../../content/ContentService'
 import proxy from 'express-http-proxy'
-import { ReadonlyConfig } from '../../../types'
+import { PendingDownloadStatusDownloading, PendingDownloadStatusType } from '../../networking/PendingDownload'
+import urljoin from 'url-join'
 
 const CACHED_MAX_AGE = 31536000
 const PENDING_MAX_AGE = 180
@@ -33,14 +42,75 @@ export class PublicApiController {
     this.content = content
   }
 
+  private createErrorResponse(message: string, type?: string): ErrorResponse {
+    return { type, message }
+  }
+
+  private async proxyAssetRequest(
+    req: express.Request<AssetRouteParams>,
+    res: express.Response,
+    next: express.NextFunction,
+    objectId: string,
+    sourceApiEndpoint: string
+  ) {
+    const sourceObjectUrl = new URL(urljoin(sourceApiEndpoint, `files/${objectId}`))
+    res.setHeader('x-data-source', 'external')
+    this.logger.verbose(`Forwarding request to ${sourceObjectUrl.toString()}`, {
+      objectId,
+      sourceUrl: sourceObjectUrl.href,
+    })
+    return proxy(sourceObjectUrl.origin, {
+      proxyReqPathResolver: () => sourceObjectUrl.pathname,
+      proxyErrorHandler: (err, res, next) => {
+        this.logger.error(`Proxy request to ${sourceObjectUrl} failed!`, {
+          objectId,
+          sourceObjectUrl: sourceObjectUrl.href,
+        })
+        this.stateCache.dropCachedDataObjectSource(objectId, sourceApiEndpoint)
+        next(err)
+      },
+    })(req, res, next)
+  }
+
+  private async serveMissingAsset(
+    req: express.Request<AssetRouteParams>,
+    res: express.Response,
+    next: express.NextFunction,
+    objectData: DataObjectData
+  ): Promise<void> {
+    const { objectId, size, contentHash } = objectData
+
+    const { maxCachedItemSize } = this.config.limits
+    if (maxCachedItemSize && size > maxCachedItemSize) {
+      this.logger.info(`Requested object is above maxCachedItemSize: ${size}/${maxCachedItemSize}`, {
+        objectId,
+        size,
+        maxCachedItemSize,
+      })
+      const source = await this.networking.getDataObjectDownloadSource(objectData)
+      res.setHeader('x-cache', 'miss')
+      return this.proxyAssetRequest(req, res, next, objectId, source)
+    }
+
+    const downloadResponse = await this.networking.downloadDataObject({ objectData })
+
+    if (downloadResponse) {
+      // Note: Await will only wait unil the file is created, so we may serve the response from it
+      await this.content.handleNewContent(objectId, size, contentHash, downloadResponse.data)
+      res.setHeader('x-cache', 'miss')
+    } else {
+      res.setHeader('x-cache', 'pending')
+    }
+    return this.servePendingDownloadAsset(req, res, next, objectId)
+  }
+
   private serveAssetFromFilesystem(
     req: express.Request<AssetRouteParams>,
     res: express.Response,
     next: express.NextFunction,
     objectId: string
   ): void {
-    // TODO: Limit the number of times useContent is trigerred for similar requests?
-    // (for example: same ip, 3 different request within a minute = 1 request)
+    this.logger.verbose('Serving object from filesystem', { objectId })
     this.stateCache.useContent(objectId)
 
     const path = this.content.path(objectId)
@@ -82,12 +152,13 @@ export class PublicApiController {
     if (!pendingDownload) {
       throw new Error('Trying to serve pending download asset that is not pending download!')
     }
+    const status = pendingDownload.getStatus().type
+    this.logger.verbose('Serving object in pending download state', { objectId, status })
 
-    const { promise, objectSize } = pendingDownload
-    const response = await promise
-    const source = new URL(response.config.url!)
-    const contentType = response.headers['content-type'] || DEFAULT_CONTENT_TYPE
-    res.setHeader('content-type', contentType)
+    await pendingDownload.untilStatus(PendingDownloadStatusType.Downloading)
+    const objectSize = pendingDownload.getObjectSize()
+    const { source, contentType } = pendingDownload.getStatus() as PendingDownloadStatusDownloading
+    res.setHeader('content-type', contentType || DEFAULT_CONTENT_TYPE)
     // Allow caching pendingDownload reponse only for very short period of time and requite revalidation,
     // since the data coming from the source may not be valid
     res.setHeader('cache-control', `max-age=${PENDING_MAX_AGE}, must-revalidate`)
@@ -106,9 +177,7 @@ export class PublicApiController {
     }
 
     // Range doesn't start from the beginning of the content or the file was not found - froward request to source storage node
-    this.logger.verbose(`Forwarding request to ${source.href}`, { source: source.href })
-    res.setHeader('x-data-source', 'external')
-    return proxy(source.origin, { proxyReqPathResolver: () => source.pathname })(req, res, next)
+    return this.proxyAssetRequest(req, res, next, objectId, source)
   }
 
   private async servePendingDownloadAssetFromFile(
@@ -134,41 +203,44 @@ export class PublicApiController {
     stream.pipe(res)
     req.on('close', () => {
       stream.destroy()
-      res.destroy()
+      res.end()
     })
   }
 
   public async assetHead(req: express.Request<AssetRouteParams>, res: express.Response): Promise<void> {
-    const objectId = req.params.objectId
-    const pendingDownload = this.stateCache.getPendingDownload(objectId)
+    const { objectId } = req.params
+    const objectStatus = await this.content.objectStatus(objectId)
 
     res.setHeader('timing-allow-origin', '*')
     res.setHeader('accept-ranges', 'bytes')
     res.setHeader('content-disposition', 'inline')
 
-    if (!pendingDownload && this.content.exists(objectId)) {
-      res.status(200)
-      res.setHeader('x-cache', 'hit')
-      res.setHeader('cache-control', `max-age=${CACHED_MAX_AGE}`)
-      res.setHeader('content-type', this.stateCache.getContentMimeType(objectId) || DEFAULT_CONTENT_TYPE)
-      res.setHeader('content-length', this.content.fileSize(objectId))
-    } else if (pendingDownload) {
-      res.status(200)
-      res.setHeader('x-cache', 'pending')
-      res.setHeader('cache-control', `max-age=${PENDING_MAX_AGE}, must-revalidate`)
-      res.setHeader('content-length', pendingDownload.objectSize)
-    } else {
-      const objectInfo = await this.networking.dataObjectInfo(objectId)
-      if (!objectInfo.exists) {
+    switch (objectStatus.type) {
+      case ObjectStatusType.Available:
+        res.status(200)
+        res.setHeader('x-cache', 'hit')
+        res.setHeader('cache-control', `max-age=${CACHED_MAX_AGE}`)
+        res.setHeader('content-type', this.stateCache.getContentMimeType(objectId) || DEFAULT_CONTENT_TYPE)
+        res.setHeader('content-length', this.content.fileSize(objectId))
+        break
+      case ObjectStatusType.PendingDownload:
+        res.status(200)
+        res.setHeader('x-cache', 'pending')
+        res.setHeader('cache-control', `max-age=${PENDING_MAX_AGE}, must-revalidate`)
+        res.setHeader('content-length', objectStatus.pendingDownload.getObjectSize())
+        break
+      case ObjectStatusType.NotFound:
         res.status(404)
-      } else if (!objectInfo.isSupported) {
+        break
+      case ObjectStatusType.NotSupported:
         res.status(421)
-      } else {
+        break
+      case ObjectStatusType.Missing:
         res.status(200)
         res.setHeader('x-cache', 'miss')
         res.setHeader('cache-control', `max-age=${PENDING_MAX_AGE}, must-revalidate`)
-        res.setHeader('content-length', objectInfo.data?.size || 0)
-      }
+        res.setHeader('content-length', objectStatus.objectData.size)
+        break
     }
 
     res.send()
@@ -179,55 +251,30 @@ export class PublicApiController {
     res: express.Response,
     next: express.NextFunction
   ): Promise<void> {
-    const objectId = req.params.objectId
-    const pendingDownload = this.stateCache.getPendingDownload(objectId)
+    const { objectId } = req.params
+    const objectStatus = await this.content.objectStatus(objectId)
 
     this.logger.verbose('Data object requested', {
       objectId,
-      status: pendingDownload && pendingDownload.status,
+      objectStatus,
     })
 
     res.setHeader('timing-allow-origin', '*')
 
-    if (!pendingDownload && this.content.exists(objectId)) {
-      this.logger.verbose('Requested file found in filesystem', { path: this.content.path(objectId) })
-      return this.serveAssetFromFilesystem(req, res, next, objectId)
-    } else if (pendingDownload) {
-      this.logger.verbose('Requested file is in pending download state', { path: this.content.path(objectId) })
-      res.setHeader('x-cache', 'pending')
-      return this.servePendingDownloadAsset(req, res, next, objectId)
-    } else {
-      this.logger.verbose('Requested file not found in filesystem')
-      const objectInfo = await this.networking.dataObjectInfo(objectId)
-      if (!objectInfo.exists) {
-        const errorRes: ErrorResponse = {
-          message: 'Data object does not exist',
-        }
-        res.status(404).json(errorRes)
-      } else if (!objectInfo.isSupported) {
-        const errorRes: ErrorResponse = {
-          message: 'Data object not served by this node',
-        }
-        res.status(421).json(errorRes)
-        // TODO: Try to direct to a node that supports it?
-      } else {
-        const { data: objectData } = objectInfo
-        if (!objectData) {
-          throw new Error('Missing data object data')
-        }
-        const { size, contentHash } = objectData
-
-        const downloadResponse = await this.networking.downloadDataObject({ objectData })
-
-        if (downloadResponse) {
-          // Note: Await will only wait unil the file is created, so we may serve the response from it
-          await this.content.handleNewContent(objectId, size, contentHash, downloadResponse.data)
-          res.setHeader('x-cache', 'miss')
-        } else {
-          res.setHeader('x-cache', 'pending')
-        }
+    switch (objectStatus.type) {
+      case ObjectStatusType.Available:
+        return this.serveAssetFromFilesystem(req, res, next, objectId)
+      case ObjectStatusType.PendingDownload:
+        res.setHeader('x-cache', 'pending')
         return this.servePendingDownloadAsset(req, res, next, objectId)
-      }
+      case ObjectStatusType.NotFound:
+        res.status(404).json(this.createErrorResponse('Data object does not exist'))
+        return
+      case ObjectStatusType.NotSupported:
+        res.status(421).json(this.createErrorResponse('Data object not served by this node'))
+        return
+      case ObjectStatusType.Missing:
+        return this.serveMissingAsset(req, res, next, objectStatus.objectData)
     }
   }
 
@@ -247,9 +294,11 @@ export class PublicApiController {
     res
       .status(200)
       .json(
-        this.config.buckets === 'all'
+        this.config.buckets
+          ? { bucketIds: [...this.config.buckets] }
+          : typeof this.config.workerId === 'number'
           ? { allByWorkerId: this.config.workerId }
-          : { bucketIds: [...this.config.buckets] }
+          : { bucketIds: [] }
       )
   }
 }

distributor-node/src/services/logging/LoggingService.ts

@@ -6,6 +6,8 @@ import { blake2AsHex } from '@polkadot/util-crypto'
 import { Format } from 'logform'
 import stringify from 'fast-safe-stringify'
 import NodeCache from 'node-cache'
+import path from 'path'
+import 'winston-daily-rotate-file'
 
 const cliColors = {
   error: 'red',
@@ -22,7 +24,8 @@ const pausedLogs = new NodeCache({
 })
 
 // Pause log for a specified time period
-const pauseFormat: (opts: { id: string }) => Format = winston.format((info, opts: { id: string }) => {
+type PauseFormatOpts = { id: string }
+const pauseFormat: (opts: PauseFormatOpts) => Format = winston.format((info, opts: PauseFormatOpts) => {
   if (info['@pauseFor']) {
     const messageHash = blake2AsHex(`${opts.id}:${info.level}:${info.message}`)
     if (!pausedLogs.has(messageHash)) {
@@ -37,8 +40,20 @@ const pauseFormat: (opts: { id: string }) => Format = winston.format((info, opts
   return info
 })
 
+// Error format applied to specific log meta field
+type ErrorFormatOpts = { filedName: string }
+const errorFormat: (opts: ErrorFormatOpts) => Format = winston.format((info, opts: ErrorFormatOpts) => {
+  if (!info[opts.filedName]) {
+    return info
+  }
+  const formatter = winston.format.errors({ stack: true })
+  info[opts.filedName] = formatter.transform(info[opts.filedName], formatter.options)
+  return info
+})
+
 const cliFormat = winston.format.combine(
   winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss:ms' }),
+  errorFormat({ filedName: 'err' }),
   winston.format.metadata({ fillExcept: ['label', 'level', 'timestamp', 'message'] }),
   winston.format.colorize({ all: true }),
   winston.format.printf(
@@ -61,39 +76,44 @@ export class LoggingService {
     const transports: winston.LoggerOptions['transports'] = []
 
     let esTransport: ElasticsearchTransport | undefined
-    if (config.log?.elastic && config.log.elastic !== 'off') {
-      if (!config.endpoints.elasticSearch) {
-        throw new Error('config.endpoints.elasticSearch must be provided when elasticSeach logging is enabled!')
-      }
+    if (config.logs?.elastic) {
       esTransport = new ElasticsearchTransport({
-        level: config.log.elastic,
+        index: 'distributor-node',
+        level: config.logs.elastic.level,
         format: winston.format.combine(pauseFormat({ id: 'es' }), escFormat()),
         flushInterval: 5000,
         source: config.id,
         clientOpts: {
           node: {
-            url: new URL(config.endpoints.elasticSearch),
+            url: new URL(config.logs.elastic.endpoint),
           },
         },
       })
       transports.push(esTransport)
     }
 
-    if (config.log?.file && config.log.file !== 'off') {
-      if (!config.directories.logs) {
-        throw new Error('config.directories.logs must be provided when file logging is enabled!')
+    if (config.logs?.file) {
+      const datePatternByFrequency = {
+        yearly: 'YYYY',
+        monthly: 'YYYY-MM',
+        daily: 'YYYY-MM-DD',
+        hourly: 'YYYY-MM-DD-HH',
       }
-      const fileTransport = new winston.transports.File({
-        filename: `${config.directories.logs}/logs.json`,
-        level: config.log.file,
+      const fileTransport = new winston.transports.DailyRotateFile({
+        filename: path.join(config.logs.file.path, 'argus-%DATE%.log'),
+        datePattern: datePatternByFrequency[config.logs.file.frequency || 'daily'],
+        zippedArchive: config.logs.file.archive,
+        maxSize: config.logs.file.maxSize,
+        maxFiles: config.logs.file.maxFiles,
+        level: config.logs.file.level,
         format: winston.format.combine(pauseFormat({ id: 'file' }), escFormat()),
       })
       transports.push(fileTransport)
     }
 
-    if (config.log?.console && config.log.console !== 'off') {
+    if (config.logs?.console) {
       const consoleTransport = new winston.transports.Console({
-        level: config.log.console,
+        level: config.logs.console.level,
         format: winston.format.combine(pauseFormat({ id: 'cli' }), cliFormat),
       })
       transports.push(consoleTransport)

distributor-node/src/services/networking/NetworkingService.ts

@@ -3,9 +3,9 @@ import { QueryNodeApi } from './query-node/api'
 import { Logger } from 'winston'
 import { LoggingService } from '../logging'
 import { StorageNodeApi } from './storage-node/api'
-import { PendingDownloadData, StateCacheService } from '../cache/StateCacheService'
+import { StateCacheService } from '../cache/StateCacheService'
 import { DataObjectDetailsFragment } from './query-node/generated/queries'
-import axios, { AxiosRequestConfig } from 'axios'
+import axios from 'axios'
 import {
   StorageNodeEndpointData,
   DataObjectAccessPoints,
@@ -19,15 +19,15 @@ import { DistributionBucketOperatorStatus } from './query-node/generated/schema'
 import http from 'http'
 import https from 'https'
 import { parseAxiosError } from '../parsers/errors'
+import { PendingDownload, PendingDownloadStatusType } from './PendingDownload'
 
 // Concurrency limits
-export const MAX_CONCURRENT_AVAILABILITY_CHECKS_PER_DOWNLOAD = 10
+export const MAX_CONCURRENT_AVAILABILITY_CHECKS_PER_OBJECT = 10
 export const MAX_CONCURRENT_RESPONSE_TIME_CHECKS = 10
 
 export class NetworkingService {
   private config: ReadonlyConfig
   private queryNodeApi: QueryNodeApi
-  // private runtimeApi: RuntimeApi
   private logging: LoggingService
   private stateCache: StateCacheService
   private logger: Logger
@@ -36,10 +36,10 @@ export class NetworkingService {
   private downloadQueue: queue
 
   constructor(config: ReadonlyConfig, stateCache: StateCacheService, logging: LoggingService) {
-    axios.defaults.timeout = config.limits.outboundRequestsTimeout
+    axios.defaults.timeout = config.limits.outboundRequestsTimeoutMs
     const httpConfig: http.AgentOptions | https.AgentOptions = {
       keepAlive: true,
-      timeout: config.limits.outboundRequestsTimeout,
+      timeout: config.limits.outboundRequestsTimeoutMs,
       maxSockets: config.limits.maxConcurrentOutboundConnections,
     }
     axios.defaults.httpAgent = new http.Agent(httpConfig)
@@ -49,7 +49,6 @@ export class NetworkingService {
     this.stateCache = stateCache
     this.logger = logging.createLogger('NetworkingManager')
     this.queryNodeApi = new QueryNodeApi(config.endpoints.queryNode, this.logging)
-    // this.runtimeApi = new RuntimeApi(config.endpoints.substrateNode)
     void this.checkActiveStorageNodeEndpoints()
     // Queues
     this.testLatencyQueue = queue({ concurrency: MAX_CONCURRENT_RESPONSE_TIME_CHECKS, autostart: true }).on(
@@ -61,6 +60,9 @@ export class NetworkingService {
       }
     )
     this.downloadQueue = queue({ concurrency: config.limits.maxConcurrentStorageNodeDownloads, autostart: true })
+    this.downloadQueue.on('error', (err) => {
+      this.logger.error('Data object download failed', { err })
+    })
   }
 
   private validateNodeEndpoint(endpoint: string): void {
@@ -92,17 +94,13 @@ export class NetworkingService {
   }
 
   private prepareStorageNodeEndpoints(details: DataObjectDetailsFragment) {
-    const endpointsData = details.storageBag.storageAssignments
-      .filter(
-        (a) =>
-          a.storageBucket.operatorStatus.__typename === 'StorageBucketOperatorStatusActive' &&
-          a.storageBucket.operatorMetadata?.nodeEndpoint
-      )
-      .map((a) => {
-        const rootEndpoint = a.storageBucket.operatorMetadata!.nodeEndpoint!
-        const apiEndpoint = this.getApiEndpoint(rootEndpoint)
+    const endpointsData = details.storageBag.storageBuckets
+      .filter((bucket) => bucket.operatorStatus.__typename === 'StorageBucketOperatorStatusActive')
+      .map((bucket) => {
+        const rootEndpoint = bucket.operatorMetadata?.nodeEndpoint
+        const apiEndpoint = rootEndpoint ? this.getApiEndpoint(rootEndpoint) : ''
         return {
-          bucketId: a.storageBucket.id,
+          bucketId: bucket.id,
           endpoint: apiEndpoint,
         }
       })
@@ -116,32 +114,42 @@ export class NetworkingService {
     }
   }
 
+  private getDataObjectActiveDistributorsSet(objectDetails: DataObjectDetailsFragment): Set<number> {
+    const activeDistributorsSet = new Set<number>()
+    const { distributionBuckets } = objectDetails.storageBag
+    for (const bucket of distributionBuckets) {
+      for (const operator of bucket.operators) {
+        if (operator.status === DistributionBucketOperatorStatus.Active) {
+          activeDistributorsSet.add(operator.workerId)
+        }
+      }
+    }
+    return activeDistributorsSet
+  }
+
   public async dataObjectInfo(objectId: string): Promise<DataObjectInfo> {
     const details = await this.queryNodeApi.getDataObjectDetails(objectId)
-    return {
-      exists: !!details,
-      isSupported:
-        (this.config.buckets === 'all' &&
-          details?.storageBag.distirbutionAssignments.some((d) =>
-            d.distributionBucket.operators.some(
-              (o) => o.workerId === this.config.workerId && o.status === DistributionBucketOperatorStatus.Active
-            )
-          )) ||
-        (Array.isArray(this.config.buckets) &&
-          this.config.buckets.some((bucketId) =>
-            details?.storageBag.distirbutionAssignments
-              .map((a) => a.distributionBucket.id)
-              .includes(bucketId.toString())
-          )),
-      data: details
-        ? {
-            objectId,
-            accessPoints: this.parseDataObjectAccessPoints(details),
-            contentHash: details.ipfsHash,
-            size: parseInt(details.size),
-          }
-        : undefined,
+    let exists = false
+    let isSupported = false
+    let data: DataObjectData | undefined
+    if (details) {
+      exists = true
+      if (!this.config.buckets) {
+        const distributors = this.getDataObjectActiveDistributorsSet(details)
+        isSupported = typeof this.config.workerId === 'number' ? distributors.has(this.config.workerId) : false
+      } else {
+        const supportedBucketIds = this.config.buckets.map((id) => id.toString())
+        isSupported = details.storageBag.distributionBuckets.some((b) => supportedBucketIds.includes(b.id))
+      }
+      data = {
+        objectId,
+        accessPoints: this.parseDataObjectAccessPoints(details),
+        contentHash: details.ipfsHash,
+        size: parseInt(details.size),
+      }
     }
+
+    return { exists, isSupported, data }
   }
 
   private sortEndpointsByMeanResponseTime(endpoints: string[]) {
@@ -152,8 +160,93 @@ export class NetworkingService {
     )
   }
 
+  private async checkObjectAvailability(objectId: string, endpoint: string): Promise<void> {
+    const api = new StorageNodeApi(endpoint, this.logging, this.config)
+    const available = await api.isObjectAvailable(objectId)
+    if (!available) {
+      throw new Error('Not available')
+    }
+  }
+
+  private createDataObjectAvailabilityCheckQueue(objectId: string, storageEndpoints: string[]) {
+    const availabilityQueue = queue({
+      concurrency: MAX_CONCURRENT_AVAILABILITY_CHECKS_PER_OBJECT,
+      autostart: true,
+    })
+
+    storageEndpoints.forEach(async (endpoint) => {
+      availabilityQueue.push(async () => {
+        await this.checkObjectAvailability(objectId, endpoint)
+        return endpoint
+      })
+    })
+
+    availabilityQueue.on('error', () => {
+      /*
+      Do nothing.
+      The handler is needed to avoid unhandled promise rejection
+      */
+    })
+
+    return availabilityQueue
+  }
+
+  public async getDataObjectDownloadSource(objectData: DataObjectData): Promise<string> {
+    const { objectId } = objectData
+    const cachedSource = await this.checkCachedDataObjectSource(objectId)
+    if (cachedSource) {
+      this.logger.info(`Found active download source for object ${objectId} in cache`, { objectId, cachedSource })
+      return cachedSource
+    }
+    return this.findDataObjectDownloadSource(objectData)
+  }
+
+  private async checkCachedDataObjectSource(objectId: string): Promise<string | undefined> {
+    const cachedSource = this.stateCache.getCachedDataObjectSource(objectId)
+    if (cachedSource) {
+      try {
+        await this.checkObjectAvailability(objectId, cachedSource)
+      } catch (err) {
+        this.stateCache.dropCachedDataObjectSource(objectId, cachedSource)
+        return undefined
+      }
+      return cachedSource
+    }
+  }
+
+  private findDataObjectDownloadSource({ objectId, accessPoints }: DataObjectData): Promise<string> {
+    return new Promise((resolve, reject) => {
+      const storageEndpoints = this.sortEndpointsByMeanResponseTime(
+        accessPoints?.storageNodes.map((n) => n.endpoint) || []
+      )
+
+      this.logger.info('Looking for data object source', {
+        objectId,
+        possibleSources: storageEndpoints.map((e) => ({
+          endpoint: e,
+          meanResponseTime: this.stateCache.getStorageNodeEndpointMeanResponseTime(e),
+        })),
+      })
+      if (!storageEndpoints.length) {
+        return reject(new Error('No storage endpoints available to download the data object from'))
+      }
+
+      const availabilityQueue = this.createDataObjectAvailabilityCheckQueue(objectId, storageEndpoints)
+
+      availabilityQueue.on('success', (endpoint) => {
+        availabilityQueue.stop()
+        this.stateCache.cacheDataObjectSource(objectId, endpoint)
+        return resolve(endpoint)
+      })
+
+      availabilityQueue.on('end', () => {
+        return reject(new Error('Failed to find data object download source'))
+      })
+    })
+  }
+
   private downloadJob(
-    pendingDownload: PendingDownloadData,
+    pendingDownload: PendingDownload,
     downloadData: DownloadData,
     onSourceFound: (response: StorageNodeDownloadResponse) => void,
     onError: (error: Error) => void,
@@ -164,7 +257,7 @@ export class NetworkingService {
       startAt,
     } = downloadData
 
-    pendingDownload.status = 'LookingForSource'
+    pendingDownload.setStatus({ type: PendingDownloadStatusType.LookingForSource })
 
     return new Promise<void>((resolve, reject) => {
       // Handlers:
@@ -174,9 +267,13 @@ export class NetworkingService {
         reject(new Error(message))
       }
 
-      const sourceFound = (response: StorageNodeDownloadResponse) => {
-        this.logger.info('Download source chosen', { objectId, source: response.config.url })
-        pendingDownload.status = 'Downloading'
+      const sourceFound = (endpoint: string, response: StorageNodeDownloadResponse) => {
+        this.logger.info('Download source chosen', { objectId, source: endpoint })
+        pendingDownload.setStatus({
+          type: PendingDownloadStatusType.Downloading,
+          source: endpoint,
+          contentType: response.headers['content-type'],
+        })
         onSourceFound(response)
       }
 
@@ -197,46 +294,25 @@ export class NetworkingService {
         })),
       })
       if (!storageEndpoints.length) {
-        return fail('No storage endpoints available to download the data object from')
+        return fail(`No storage endpoints available to download the data object: ${objectId}`)
       }
 
-      const availabilityQueue = queue({
-        concurrency: MAX_CONCURRENT_AVAILABILITY_CHECKS_PER_DOWNLOAD,
-        autostart: true,
-      })
+      const availabilityQueue = this.createDataObjectAvailabilityCheckQueue(objectId, storageEndpoints)
       const objectDownloadQueue = queue({ concurrency: 1, autostart: true })
 
-      storageEndpoints.forEach(async (endpoint) => {
-        availabilityQueue.push(async () => {
-          const api = new StorageNodeApi(endpoint, this.logging)
-          const available = await api.isObjectAvailable(objectId)
-          if (!available) {
-            throw new Error('Not avilable')
-          }
-          return endpoint
-        })
-      })
-
       availabilityQueue.on('success', (endpoint) => {
         availabilityQueue.stop()
         const job = async () => {
-          const api = new StorageNodeApi(endpoint, this.logging)
+          const api = new StorageNodeApi(endpoint, this.logging, this.config)
           const response = await api.downloadObject(objectId, startAt)
-          return response
+          return [endpoint, response]
         }
         objectDownloadQueue.push(job)
       })
 
-      availabilityQueue.on('error', () => {
-        /*
-        Do nothing.
-        The handler is needed to avoid unhandled promise rejection
-        */
-      })
-
       availabilityQueue.on('end', () => {
         if (!objectDownloadQueue.length) {
-          fail('Failed to download the object from any availablable storage provider')
+          fail(`Failed to download object ${objectId} from any availablable storage provider`)
         }
       })
 
@@ -248,15 +324,15 @@ export class NetworkingService {
         if (availabilityQueue.length) {
           availabilityQueue.start()
         } else {
-          fail('Failed to download the object from any availablable storage provider')
+          fail(`Failed to download object ${objectId} from any availablable storage provider`)
         }
       })
 
-      objectDownloadQueue.on('success', (response: StorageNodeDownloadResponse) => {
+      objectDownloadQueue.on('success', ([endpoint, response]: [string, StorageNodeDownloadResponse]) => {
         availabilityQueue.removeAllListeners().end()
         objectDownloadQueue.removeAllListeners().end()
         response.data.on('close', finish).on('error', finish).on('end', finish)
-        sourceFound(response)
+        sourceFound(endpoint, response)
       })
     })
   }
@@ -265,34 +341,29 @@ export class NetworkingService {
     const {
       objectData: { objectId, size },
     } = downloadData
-
     if (this.stateCache.getPendingDownload(objectId)) {
       // Already downloading
       return null
     }
-
-    let resolveDownload: (response: StorageNodeDownloadResponse) => void, rejectDownload: (err: Error) => void
-    const downloadPromise = new Promise<StorageNodeDownloadResponse>((resolve, reject) => {
-      resolveDownload = resolve
-      rejectDownload = reject
+    const pendingDownload = this.stateCache.addPendingDownload(new PendingDownload(objectId, size))
+    return new Promise<StorageNodeDownloadResponse>((resolve, reject) => {
+      const onSourceFound = resolve
+      const onError = reject
+      // Queue the download
+      this.downloadQueue.push(() => this.downloadJob(pendingDownload, downloadData, onSourceFound, onError))
     })
-
-    // Queue the download
-    const pendingDownload = this.stateCache.newPendingDownload(objectId, size, downloadPromise)
-    this.downloadQueue.push(() => this.downloadJob(pendingDownload, downloadData, resolveDownload, rejectDownload))
-
-    return downloadPromise
   }
 
   async fetchSupportedDataObjects(): Promise<Map<string, DataObjectData>> {
-    const data =
-      this.config.buckets === 'all'
-        ? await this.queryNodeApi.getDistributionBucketsWithObjectsByWorkerId(this.config.workerId)
-        : await this.queryNodeApi.getDistributionBucketsWithObjectsByIds(this.config.buckets.map((id) => id.toString()))
+    const data = this.config.buckets
+      ? await this.queryNodeApi.getDistributionBucketsWithObjectsByIds(this.config.buckets.map((id) => id.toString()))
+      : typeof this.config.workerId === 'number'
+      ? await this.queryNodeApi.getDistributionBucketsWithObjectsByWorkerId(this.config.workerId)
+      : []
     const objectsData = new Map<string, DataObjectData>()
     data.forEach((bucket) => {
-      bucket.bagAssignments.forEach((a) => {
-        a.storageBag.objects.forEach((object) => {
+      bucket.bags.forEach((bag) => {
+        bag.objects.forEach((object) => {
           const { ipfsHash, id, size } = object
           objectsData.set(id, { contentHash: ipfsHash, objectId: id, size: parseInt(size) })
         })
@@ -308,7 +379,7 @@ export class NetworkingService {
       const endpoints = this.filterStorageNodeEndpoints(
         activeStorageOperators.map(({ id, operatorMetadata }) => ({
           bucketId: id,
-          endpoint: this.getApiEndpoint(operatorMetadata!.nodeEndpoint!),
+          endpoint: operatorMetadata?.nodeEndpoint ? this.getApiEndpoint(operatorMetadata.nodeEndpoint) : '',
         }))
       )
       this.logger.verbose('Checking nearby storage nodes...', { validEndpointsCount: endpoints.length })
@@ -327,9 +398,8 @@ export class NetworkingService {
     const start = Date.now()
     this.logger.debug(`Sending storage node response-time check request to: ${endpoint}`, { endpoint })
     try {
-      const api = new StorageNodeApi(endpoint, this.logging)
-      const reqConfig: AxiosRequestConfig = { headers: { connection: 'close' } }
-      await api.stateApi.stateApiGetVersion(reqConfig)
+      const api = new StorageNodeApi(endpoint, this.logging, this.config)
+      await api.getVersion()
       const responseTime = Date.now() - start
       this.logger.debug(`${endpoint} check request response time: ${responseTime}`, { endpoint, responseTime })
       this.stateCache.setStorageNodeEndpointResponseTime(endpoint, responseTime)