storage-node-v2: Start adding bagId parser.

storage-node-v2/README.md

@@ -30,11 +30,12 @@ USAGE
 <!-- commands -->
 * [`storage-node dev:init`](#storage-node-devinit)
 * [`storage-node dev:upload`](#storage-node-devupload)
+* [`storage-node dev:verify-bag-id [FILE]`](#storage-node-devverify-bag-id-file)
 * [`storage-node help [COMMAND]`](#storage-node-help-command)
 * [`storage-node leader:create-bucket`](#storage-node-leadercreate-bucket)
 * [`storage-node leader:update-bag`](#storage-node-leaderupdate-bag)
 * [`storage-node leader:update-bag-limit`](#storage-node-leaderupdate-bag-limit)
-* [`storage-node leader:update-voucher-limits [FILE]`](#storage-node-leaderupdate-voucher-limits-file)
+* [`storage-node leader:update-voucher-limits`](#storage-node-leaderupdate-voucher-limits)
 * [`storage-node multihash`](#storage-node-multihash)
 * [`storage-node operator:accept-invitation`](#storage-node-operatoraccept-invitation)
 * [`storage-node server [FILE]`](#storage-node-server-file)
@@ -51,7 +52,7 @@ OPTIONS
   -h, --help  show CLI help
 ```
 
-_See code: [src/commands/dev/init.ts](https://github.com/shamil-gadelshin/storage-node-v2/blob/v0.1.0/src/commands/dev/init.ts)_
+_See code: [src/commands/dev/init.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/dev/init.ts)_
 
 ## `storage-node dev:upload`
 
@@ -67,7 +68,23 @@ OPTIONS
   -s, --size=size  (required) Data object size.
 ```
 
-_See code: [src/commands/dev/upload.ts](https://github.com/shamil-gadelshin/storage-node-v2/blob/v0.1.0/src/commands/dev/upload.ts)_
+_See code: [src/commands/dev/upload.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/dev/upload.ts)_
+
+## `storage-node dev:verify-bag-id [FILE]`
+
+describe the command here
+
+```
+USAGE
+  $ storage-node dev:verify-bag-id [FILE]
+
+OPTIONS
+  -f, --force
+  -h, --help       show CLI help
+  -n, --name=name  name to print
+```
+
+_See code: [src/commands/dev/verify-bag-id.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/dev/verify-bag-id.ts)_
 
 ## `storage-node help [COMMAND]`
 
@@ -105,7 +122,7 @@ OPTIONS
   -s, --size=size          Storage bucket max total objects size
 ```
 
-_See code: [src/commands/leader/create-bucket.ts](https://github.com/shamil-gadelshin/storage-node-v2/blob/v0.1.0/src/commands/leader/create-bucket.ts)_
+_See code: [src/commands/leader/create-bucket.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/leader/create-bucket.ts)_
 
 ## `storage-node leader:update-bag`
 
@@ -149,11 +166,11 @@ OPTIONS
       Remove a bucket from the bag
 ```
 
-_See code: [src/commands/leader/update-bag.ts](https://github.com/shamil-gadelshin/storage-node-v2/blob/v0.1.0/src/commands/leader/update-bag.ts)_
+_See code: [src/commands/leader/update-bag.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/leader/update-bag.ts)_
 
 ## `storage-node leader:update-bag-limit`
 
-Updates StorageBucketsPerBagLimit variable in Joystream node storage.
+Updates StorageBucketsPerBagLimit variable in the Joystream node storage.
 
 ```
 USAGE
@@ -167,23 +184,26 @@ OPTIONS
   -p, --password=password  Key file password (optional).
 ```
 
-_See code: [src/commands/leader/update-bag-limit.ts](https://github.com/shamil-gadelshin/storage-node-v2/blob/v0.1.0/src/commands/leader/update-bag-limit.ts)_
+_See code: [src/commands/leader/update-bag-limit.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/leader/update-bag-limit.ts)_
 
-## `storage-node leader:update-voucher-limits [FILE]`
+## `storage-node leader:update-voucher-limits`
 
-describe the command here
+Updates VoucherMaxObjectsSizeLimit and VoucherMaxObjectsNumberLimit the Joystream node storage.
 
 ```
 USAGE
-  $ storage-node leader:update-voucher-limits [FILE]
+  $ storage-node leader:update-voucher-limits
 
 OPTIONS
-  -f, --force
-  -h, --help       show CLI help
-  -n, --name=name  name to print
+  -d, --dev                Use development mode
+  -h, --help               show CLI help
+  -k, --keyfile=keyfile    Key file for the account. Mandatory in non-dev environment.
+  -o, --objects=objects    (required) New 'max voucher object number limit' value
+  -p, --password=password  Key file password (optional).
+  -s, --size=size          (required) New 'max voucher object size limit' value
 ```
 
-_See code: [src/commands/leader/update-voucher-limits.ts](https://github.com/shamil-gadelshin/storage-node-v2/blob/v0.1.0/src/commands/leader/update-voucher-limits.ts)_
+_See code: [src/commands/leader/update-voucher-limits.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/leader/update-voucher-limits.ts)_
 
 ## `storage-node multihash`
 
@@ -198,7 +218,7 @@ OPTIONS
   -h, --help       show CLI help
 ```
 
-_See code: [src/commands/multihash.ts](https://github.com/shamil-gadelshin/storage-node-v2/blob/v0.1.0/src/commands/multihash.ts)_
+_See code: [src/commands/multihash.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/multihash.ts)_
 
 ## `storage-node operator:accept-invitation`
 
@@ -217,7 +237,7 @@ OPTIONS
   -w, --worker=worker      (required) Storage operator worker ID
 ```
 
-_See code: [src/commands/operator/accept-invitation.ts](https://github.com/shamil-gadelshin/storage-node-v2/blob/v0.1.0/src/commands/operator/accept-invitation.ts)_
+_See code: [src/commands/operator/accept-invitation.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/operator/accept-invitation.ts)_
 
 ## `storage-node server [FILE]`
 
@@ -236,5 +256,5 @@ OPTIONS
   -u, --uploads=uploads    (required) Data uploading directory.
 ```
 
-_See code: [src/commands/server.ts](https://github.com/shamil-gadelshin/storage-node-v2/blob/v0.1.0/src/commands/server.ts)_
+_See code: [src/commands/server.ts](https://github.com/Joystream/joystream/blob/v0.1.0/src/commands/server.ts)_
 <!-- commandsstop -->

storage-node-v2/src/commands/multihash.ts → storage-node-v2/src/commands/dev/multihash.ts

@@ -1,7 +1,7 @@
 import { Command, flags } from '@oclif/command'
-import { hashFile } from '../services/hashing'
+import { hashFile } from '../../services/helpers/hashing'
 
-export default class Multihash extends Command {
+export default class DevMultihash extends Command {
   static description = 'Creates a multihash (blake3) for a file.'
 
   static flags = {
@@ -14,7 +14,7 @@ export default class Multihash extends Command {
   }
 
   async run(): Promise<void> {
-    const { flags } = this.parse(Multihash)
+    const { flags } = this.parse(DevMultihash)
 
     console.log(`Hashing ${flags.file}`)
 

storage-node-v2/src/commands/dev/verify-bag-id.ts

@@ -0,0 +1,38 @@
+import { flags } from '@oclif/command'
+import ApiCommandBase from '../../command-base/ApiCommandBase'
+import { parseBagId } from '../../services/helpers/bagIdParser'
+
+export default class DevVerifyBagId extends ApiCommandBase {
+  static description =
+    'The command verifies bag id supported by the storage node. Requires chain connection.'
+
+  static flags = {
+    bagId: flags.string({
+      char: 'i',
+      required: true,
+      description: `
+      Bag ID. Format: {bag_type}:{sub_type}:{id}.
+      - Bag types: 'static', 'dynamic'
+      - Sub types: 'static:council', 'static:wg', 'dynamic:member', 'dynamic:channel'
+      - Id: 
+        - absent for 'static:council'
+        - working group name for 'static:wg'
+        - integer for 'dynamic:member' and 'dynamic:channel'
+      Examples:
+      - static:council
+      - static:wg:storage
+      - dynamic:member:4
+      `,
+    }),
+    ...ApiCommandBase.keyflags,
+  }
+
+  async run(): Promise<void> {
+    const { flags } = this.parse(DevVerifyBagId)
+
+    const api = await this.getApi()
+    parseBagId(api, flags.bagId)
+
+    console.log(`Correct bag id: ${flags.bagId}`)
+  }
+}

storage-node-v2/src/commands/leader/update-bag.ts

@@ -1,6 +1,7 @@
 import { flags } from '@oclif/command'
 import { updateStorageBucketsForBag } from '../../services/runtime/extrinsics'
 import ApiCommandBase from '../../command-base/ApiCommandBase'
+import { parseBagId } from '../../services/helpers/bagIdParser'
 
 export default class LeaderUpdateBag extends ApiCommandBase {
   static description =
@@ -47,9 +48,9 @@ export default class LeaderUpdateBag extends ApiCommandBase {
     }
 
     const account = this.getAccount(flags)
-
-    // TODO: add bag parameter
     const api = await this.getApi()
-    await updateStorageBucketsForBag(api, account, bucket, flags.remove)
+    const bagId = parseBagId(api, flags.bagId)
+
+    await updateStorageBucketsForBag(api, bagId, account, bucket, flags.remove)
   }
 }

storage-node-v2/src/services/helpers/bagIdParser.ts

@@ -0,0 +1,43 @@
+import { BagId, Static } from '@joystream/types/storage'
+import { ApiPromise } from '@polkadot/api'
+
+export function parseBagId(api: ApiPromise, bagId: string): BagId {
+  const bagIdParts = bagId.toLowerCase().split(':')
+
+  if (bagIdParts.length > 3 || bagIdParts.length < 2) {
+    throw new Error(`Invalid bagId: ${bagId}`)
+  }
+
+  if (bagIdParts[0] === 'static') {
+    return parseStaticBagId(api, bagId, bagIdParts)
+  }
+
+  if (bagIdParts[0] === 'dynamic') {
+    return parseDynamicBagId()
+  }
+
+  throw new Error(`Invalid bagId: ${bagId}`)
+}
+
+function parseStaticBagId(
+  api: ApiPromise,
+  bagId: string,
+  bagIdParts: string[]
+): BagId {
+  if (bagIdParts[1] === 'council') {
+    if (bagIdParts.length === 2) {
+      const staticBagId: Static = api.createType('Static', 'Council')
+      const constructedBagId: BagId = api.createType('BagId', {
+        'Static': staticBagId,
+      })
+
+      return constructedBagId
+    }
+  }
+
+  throw new Error(`Invalid bagId: ${bagId}`)
+}
+
+function parseDynamicBagId(): BagId {
+  throw new Error('Function not implemented.')
+}

storage-node-v2/src/services/runtime/extrinsics.ts

@@ -6,6 +6,7 @@ import {
 import { KeyringPair } from '@polkadot/keyring/types'
 import { CodecArg } from '@polkadot/types/types'
 import { ApiPromise } from '@polkadot/api'
+import { BagId } from '@joystream/types/storage'
 
 export async function createStorageBucket(
   api: ApiPromise,
@@ -52,6 +53,7 @@ export async function acceptStorageBucketInvitation(
 
 export async function updateStorageBucketsForBag(
   api: ApiPromise,
+  bagId: BagId,
   account: KeyringPair,
   bucketId: number,
   removeBucket: boolean
@@ -71,7 +73,7 @@ export async function updateStorageBucketsForBag(
       account,
       'storage',
       'updateStorageBucketsForBag',
-      [{ 'Static': 'Council' }, addBuckets, removeBuckets]
+      [bagId, addBuckets, removeBuckets]
     )
   } catch (err) {
     console.error(`Api Error: ${err}`)
@@ -110,14 +112,13 @@ export async function uploadDataObjects(
 
 export async function acceptPendingDataObjects(
   api: ApiPromise,
+  bagId: BagId,
   account: KeyringPair,
   workerId: number,
   storageBucketId: number,
   dataObjects: number[]
 ): Promise<void> {
   try {
-    const bagId = { 'Static': 'Council' }
-
     const dataObjectSet: CodecArg = api.createType(
       'DataObjectIdSet',
       dataObjects

storage-node-v2/src/services/webApi/app.ts

@@ -6,7 +6,7 @@ import * as OpenApiValidator from 'express-openapi-validator'
 import { OpenAPIV3 } from 'express-openapi-validator/dist/framework/types'
 import { KeyringPair } from '@polkadot/keyring/types'
 import { ApiPromise } from '@polkadot/api'
-import { TokenRequest, verifyTokenSignature } from '../auth'
+import { TokenRequest, verifyTokenSignature } from '../helpers/auth'
 import { createStorageBucket } from '../runtime/extrinsics'
 
 // TODO: custom errors (including validation errors)

storage-node-v2/src/services/webApi/controllers/publicApi.ts

@@ -1,20 +1,22 @@
 import * as express from 'express'
 import { acceptPendingDataObjects } from '../../runtime/extrinsics'
-import { TokenRequest, signToken } from '../../auth'
-import { hashFile } from '../../../services/hashing'
+import { TokenRequest, signToken } from '../../helpers/auth'
+import { hashFile } from '../../../services/helpers/hashing'
 import { KeyringPair } from '@polkadot/keyring/types'
 import { ApiPromise } from '@polkadot/api'
+import { parseBagId } from '../../../services/helpers/bagIdParser'
 import fs from 'fs'
 const fsPromises = fs.promises
 
 // TODO: test api connection?
 // TODO: error handling
 // TODO: convert to JSON
-// TODO: bagId
+// TODO: validate bagId
 interface UploadRequest {
   dataObjectId: number
   storageBucketId: number
   workerId: number
+  bagId: string
 }
 
 export async function upload(
@@ -34,8 +36,11 @@ export async function upload(
     // Overwrites existing file.
     await fsPromises.rename(fileObj.path, newPath)
 
+    const api = getApi(res)
+    const bagId = parseBagId(api, uploadRequest.bagId)
     await acceptPendingDataObjects(
-      getApi(res),
+      api,
+      bagId,
       getAccount(res),
       uploadRequest.workerId,
       uploadRequest.storageBucketId,