storage-node-v2: Modify bag parser.

Add all bag types.

runtime-modules/storage/src/lib.rs

@@ -129,6 +129,7 @@ use codec::{Codec, Decode, Encode};
 use frame_support::dispatch::{DispatchError, DispatchResult};
 use frame_support::traits::{Currency, ExistenceRequirement, Get, Randomness};
 use frame_support::{decl_error, decl_event, decl_module, decl_storage, ensure, Parameter};
+use frame_system::ensure_root;
 #[cfg(feature = "std")]
 use serde::{Deserialize, Serialize};
 use sp_arithmetic::traits::{BaseArithmetic, One, Zero};
@@ -139,7 +140,6 @@ use sp_std::collections::btree_set::BTreeSet;
 use sp_std::iter;
 use sp_std::marker::PhantomData;
 use sp_std::vec::Vec;
-use frame_system::ensure_root;
 
 use common::constraints::BoundedValueConstraint;
 use common::origin::ActorOriginValidator;

storage-node-v2/scripts/create-auth-request-signature.ts

@@ -13,7 +13,7 @@ createApi('ws://localhost:9944').then(() => {
   const tokenRequestBody: UploadTokenRequestBody = {
     memberId: 0,
     accountId: alice.address,
-    dataObjectId: 2,
+    dataObjectId: 1,
     storageBucketId: 0,
     bagId: 'static:council'
   }

storage-node-v2/scripts/run-all-commands.sh

@@ -11,6 +11,7 @@ yarn storage-node dev:init # leader workerId = 0
 yarn storage-node leader:update-bag-limit -l 7 --dev
 yarn storage-node leader:update-voucher-limits -o 100 -s 10000000 --dev
 yarn storage-node leader:update-data-fee -f 10000 --dev
+yarn storage-node leader:update-dynamic-bag-policy -n 10 -m
 
 # Create and configure a bucket.
 yarn storage-node leader:create-bucket -i=0 --dev # bucketId = 0

storage-node-v2/src/api-spec/openapi.yaml

@@ -36,15 +36,7 @@ paths:
         200:
           description: Ok
           content:
-            image/*:
-              schema:
-                type: string
-                format: binary
-            audio/*:
-              schema:
-                type: string
-                format: binary
-            video/*:
+            application/octet-stream:
               schema:
                 type: string
                 format: binary

storage-node-v2/src/commands/dev/verify-bag-id.ts

@@ -32,8 +32,9 @@ export default class DevVerifyBagId extends ApiCommandBase {
     const { flags } = this.parse(DevVerifyBagId)
 
     const api = await this.getApi()
-    parseBagId(api, flags.bagId)
+    const parsedBagId = parseBagId(api, flags.bagId)
 
     logger.info(`Correct bag id: ${flags.bagId}`)
+    logger.info(`Parsed: ${parsedBagId}`)
   }
 }

storage-node-v2/src/commands/leader/update-blacklist.ts

@@ -31,14 +31,8 @@ export default class LeaderUpdateBlacklist extends ApiCommandBase {
     const account = this.getAccount(flags)
     const api = await this.getApi()
 
-    const success = await updateBlacklist(
-      api,
-      account,
-      flags.cid,
-      flags.remove
-    )
+    const success = await updateBlacklist(api, account, flags.cid, flags.remove)
 
     this.exitAfterRuntimeCall(success)
   }
 }
-

storage-node-v2/src/services/helpers/bagTypes.ts

@@ -1,60 +1,135 @@
-import { BagId, DynamicBagType, Static } from '@joystream/types/storage'
+import {
+  BagId,
+  DynamicBagType,
+  DynamicBagTypeKey,
+  Static,
+  Dynamic,
+} from '@joystream/types/storage'
+import { WorkingGroup } from '@joystream/types/common'
 import { ApiPromise } from '@polkadot/api'
 import ExitCodes from '../../command-base/ExitCodes'
 import { CLIError } from '@oclif/errors'
 
+export function parseDynamicBagType(
+  api: ApiPromise,
+  bagType: DynamicBagTypeKey
+): DynamicBagType {
+  return api.createType('DynamicBagType', bagType)
+}
+
 export function parseBagId(api: ApiPromise, bagId: string): BagId {
-  const bagIdParts = bagId.toLowerCase().split(':')
+  const parser = new BagIdParser(api, bagId)
+
+  return parser.parse()
+}
+
+class BagIdParser {
+  bagId: string
+  api: ApiPromise
+  bagIdParts: string[]
+
+  constructor(api: ApiPromise, bagId: string) {
+    this.bagId = bagId
+    this.api = api
 
-  if (bagIdParts.length > 3 || bagIdParts.length < 2) {
-    throw new CLIError(`Invalid bagId: ${bagId}`, {
+    this.bagIdParts = bagId.trim().toLowerCase().split(':')
+
+    if (this.bagIdParts.length > 3 || this.bagIdParts.length < 2) {
+      throw new CLIError(`Invalid bagId: ${bagId}`, {
+        exit: ExitCodes.InvalidParameters,
+      })
+    }
+  }
+
+  parse(): BagId {
+    if (this.bagIdParts[0] === 'static') {
+      return this.parseStaticBagId()
+    }
+
+    if (this.bagIdParts[0] === 'dynamic') {
+      return this.parseDynamicBagId()
+    }
+
+    throw new CLIError(`Invalid bagId: ${this.bagId}`, {
       exit: ExitCodes.InvalidParameters,
     })
   }
 
-  if (bagIdParts[0] === 'static') {
-    return parseStaticBagId(api, bagId, bagIdParts)
-  }
+  parseStaticBagId(): BagId {
+    // Try to construct static council bag ID.
+    if (this.bagIdParts[1] === 'council') {
+      if (this.bagIdParts.length === 2) {
+        const staticBagId: Static = this.api.createType('Static', 'Council')
+        const constructedBagId: BagId = this.api.createType('BagId', {
+          'Static': staticBagId,
+        })
 
-  if (bagIdParts[0] === 'dynamic') {
-    return parseDynamicBagId()
-  }
+        return constructedBagId
+      }
+    }
 
-  throw new CLIError(`Invalid bagId: ${bagId}`, {
-    exit: ExitCodes.InvalidParameters,
-  })
-}
+    // Try to construct static working group bag ID.
+    if (this.bagIdParts[1] === 'wg') {
+      if (this.bagIdParts.length === 3) {
+        const groups = Object.keys(WorkingGroup.typeDefinitions)
+        const actualGroup = this.bagIdParts[2]
 
-function parseStaticBagId(
-  api: ApiPromise,
-  bagId: string,
-  bagIdParts: string[]
-): BagId {
-  if (bagIdParts[1] === 'council') {
-    if (bagIdParts.length === 2) {
-      const staticBagId: Static = api.createType('Static', 'Council')
-      const constructedBagId: BagId = api.createType('BagId', {
-        'Static': staticBagId,
-      })
+        for (const group of groups) {
+          if (group.toLowerCase() === actualGroup) {
+            const workingGroup: WorkingGroup = this.api.createType(
+              'WorkingGroup',
+              group
+            )
+            const staticBagId: Static = this.api.createType('Static', {
+              'WorkingGroup': workingGroup,
+            })
+            const constructedBagId: BagId = this.api.createType('BagId', {
+              'Static': staticBagId,
+            })
 
-      return constructedBagId
+            return constructedBagId
+          }
+        }
+      }
     }
+
+    throw new CLIError(`Invalid static bagId: ${this.bagId}`, {
+      exit: ExitCodes.InvalidParameters,
+    })
   }
 
-  throw new CLIError(`Invalid bagId: ${bagId}`, {
-    exit: ExitCodes.InvalidParameters,
-  })
-}
+  parseDynamicBagId(): BagId {
+    if (this.bagIdParts.length === 3) {
+      const idString = this.bagIdParts[2]
+      const parsedId = parseInt(idString)
 
-function parseDynamicBagId(): BagId {
-  throw new CLIError('Function not implemented.', {
-    exit: ExitCodes.InvalidParameters,
-  })
-}
+      // Verify successful entity ID parsing
+      if (!isNaN(parsedId)) {
+        const dynamicBagTypes = Object.keys(DynamicBagType.typeDefinitions)
+        const actualType = this.bagIdParts[1]
 
-export function parseDynamicBagType(
-  api: ApiPromise,
-  bagType: 'Member' | 'Channel'
-): DynamicBagType {
-  return api.createType('DynamicBagType', bagType)
+        // Try to construct dynamic bag ID.
+        for (const dynamicBagType of dynamicBagTypes) {
+          if (dynamicBagType.toLowerCase() === actualType) {
+            const dynamic = {}
+            dynamic[dynamicBagType] = parsedId
+
+            const dynamicBagId: Dynamic = this.api.createType(
+              'Dynamic',
+              dynamic
+            )
+            const constructedBagId: BagId = this.api.createType('BagId', {
+              'Dynamic': dynamicBagId,
+            })
+
+            return constructedBagId
+          }
+        }
+      }
+    }
+
+    throw new CLIError(`Invalid dynamic bagId: ${this.bagId}`, {
+      exit: ExitCodes.InvalidParameters,
+    })
+  }
 }

storage-node-v2/src/services/helpers/tokenNonceKeeper.ts

@@ -1,7 +1,6 @@
 import NodeCache from 'node-cache'
 
-// TODO: move to config or set to 10 seconds
-export const TokenExpirationPeriod: number = 100 * 1000 // seconds
+export const TokenExpirationPeriod: number = 30 * 1000 // seconds
 const MaxNonces = 100000
 
 const nonceCache = new NodeCache({

storage-node-v2/src/services/runtime/extrinsics.ts

@@ -11,8 +11,8 @@ export async function createStorageBucket(
   account: KeyringPair,
   invitedWorker: number | null = null,
   allowedNewBags = true,
-  sizeLimit: number = 0,
-  objectsLimit: number = 0
+  sizeLimit = 0,
+  objectsLimit = 0
 ): Promise<boolean> {
   return await extrinsicWrapper(() => {
     const invitedWorkerValue = api.createType('Option<WorkerId>', invitedWorker)
@@ -347,12 +347,9 @@ export async function updateBlacklist(
       addHashes = api.createType('ContentIdSet', [cid])
     }
 
-    return sendAndFollowNamedTx(
-      api,
-      account,
-      'storage',
-      'updateBlacklist',
-      [removeHashes, addHashes]
-    )
+    return sendAndFollowNamedTx(api, account, 'storage', 'updateBlacklist', [
+      removeHashes,
+      addHashes,
+    ])
   })
-}
+}

types/augment-codec/augment-api-query.ts

@@ -960,7 +960,7 @@ declare module '@polkadot/api/types/storage' {
       /**
        * Blacklisted data object hashes.
        **/
-      blacklist: AugmentedQuery<ApiType, (arg: ContentId | string) => Observable<ITuple<[]>>, [ContentId]>;
+      blacklist: AugmentedQuery<ApiType, (arg: ContentId | string | Uint8Array) => Observable<ITuple<[]>>, [ContentId]>;
       /**
        * Blacklist collection counter.
        **/

types/augment/all/defs.json

@@ -552,7 +552,7 @@
         "identity": "IPNSIdentity",
         "expires_at": "u32"
     },
-    "ContentId": "Text",
+    "ContentId": "[u8;32]",
     "LiaisonJudgement": {
         "_enum": [
             "Pending",
@@ -571,7 +571,7 @@
     },
     "DataObjectStorageRelationshipId": "u64",
     "DataObjectStorageRelationship": {
-        "content_id": "Hash",
+        "content_id": "ContentId",
         "storage_provider": "StorageProviderId",
         "ready": "bool"
     },
@@ -580,7 +580,7 @@
         "description": "Text",
         "active": "bool"
     },
-    "DataObjectsMap": "BTreeMap<[u8;32],DataObject>",
+    "DataObjectsMap": "BTreeMap<ContentId,DataObject>",
     "ProposalId": "u32",
     "ProposalStatus": {
         "_enum": {
@@ -1058,7 +1058,8 @@
     },
     "StorageBucketIdSet": "BTreeSet<StorageBucketId>",
     "DataObjectIdSet": "BTreeSet<DataObjectId>",
-    "ContentIdSet": "BTreeSet<ContentId>",
+    "ContentIdSet": "BTreeSet<Cid>",
+    "Cid": "Text",
     "StorageBucketOperatorStatus": {
         "_enum": {
             "Missing": "Null",

types/augment/all/types.ts

@@ -246,6 +246,9 @@ export interface ChildPositionInParentCategory extends Struct {
   readonly child_nr_in_parent_category: u32;
 }
 
+/** @name Cid */
+export interface Cid extends Text {}
+
 /** @name Class */
 export interface Class extends Struct {
   readonly class_permissions: ClassPermissions;
@@ -288,10 +291,10 @@ export interface ClassPermissionsType extends Null {}
 export interface ClassPropertyValue extends Null {}
 
 /** @name ContentId */
-export interface ContentId extends Text {}
+export interface ContentId extends U8aFixed {}
 
 /** @name ContentIdSet */
-export interface ContentIdSet extends BTreeSet<ContentId> {}
+export interface ContentIdSet extends BTreeSet<Cid> {}
 
 /** @name CreateEntityOperation */
 export interface CreateEntityOperation extends Struct {
@@ -419,11 +422,11 @@ export interface DataObjectId extends u64 {}
 export interface DataObjectIdSet extends BTreeSet<DataObjectId> {}
 
 /** @name DataObjectsMap */
-export interface DataObjectsMap extends BTreeMap<U8aFixed, DataObject> {}
+export interface DataObjectsMap extends BTreeMap<ContentId, DataObject> {}
 
 /** @name DataObjectStorageRelationship */
 export interface DataObjectStorageRelationship extends Struct {
-  readonly content_id: Hash;
+  readonly content_id: ContentId;
   readonly storage_provider: StorageProviderId;
   readonly ready: bool;
 }

types/augment/augment-api-query.ts

@@ -960,7 +960,7 @@ declare module '@polkadot/api/types/storage' {
       /**
        * Blacklisted data object hashes.
        **/
-      blacklist: AugmentedQuery<ApiType, (arg: ContentId | string) => Observable<ITuple<[]>>, [ContentId]>;
+      blacklist: AugmentedQuery<ApiType, (arg: ContentId | string | Uint8Array) => Observable<ITuple<[]>>, [ContentId]>;
       /**
        * Blacklist collection counter.
        **/

types/src/storage.ts

@@ -183,8 +183,8 @@ export class UploadParameters
   })
   implements UploadParametersType {}
 
-export class ContentId extends Text {}
-export class ContentIdSet extends BTreeSet.with(ContentId) {}
+export class Cid extends Text {}
+export class ContentIdSet extends BTreeSet.with(Cid) {}
 
 export const storageTypes: RegistryTypes = {
   StorageBucketId,
@@ -207,7 +207,7 @@ export const storageTypes: RegistryTypes = {
   StorageBucketIdSet,
   DataObjectIdSet,
   ContentIdSet,
-  ContentId,
+  Cid,
   StorageBucketOperatorStatus,
 }
 export default storageTypes