Merge pull request #2173 from iorveth/storage_data_migration

Init data_directory on runtime upgrade

cli/src/Api.ts

@@ -527,8 +527,8 @@ export default class Api {
     return exists ? await this._api.query.contentDirectory.entityById<Entity>(id) : null
   }
 
-  async dataObjectByContentId(contentId: ContentId): Promise<DataObject | null> {
-    const dataObject = await this._api.query.dataDirectory.dataObjectByContentId<Option<DataObject>>(contentId)
+  async dataByContentId(contentId: ContentId): Promise<DataObject | null> {
+    const dataObject = await this._api.query.dataDirectory.dataByContentId<Option<DataObject>>(contentId)
     return dataObject.unwrapOr(null)
   }
 

cli/src/commands/media/uploadVideo.ts

@@ -395,7 +395,7 @@ export default class UploadVideoCommand extends MediaCommandBase {
       ipfsCid,
     ])
 
-    const dataObject = await this.getApi().dataObjectByContentId(contentId)
+    const dataObject = await this.getApi().dataByContentId(contentId)
     if (!dataObject) {
       this.error('Data object could not be retrieved from chain', { exit: ExitCodes.ApiError })
     }

node/src/chain_spec/content_config.rs

@@ -1,7 +1,7 @@
 use codec::Decode;
 use node_runtime::{
     common::storage::StorageObjectOwner,
-    data_directory::{DataObject, Quota},
+    data_directory::*,
     ChannelId, ContentId, DAOId, DataDirectoryConfig, MemberId, Runtime,
 };
 use serde::Deserialize;
@@ -125,10 +125,10 @@ pub fn empty_data_directory_config() -> DataDirectoryConfig {
     DataDirectoryConfig {
         data_object_by_content_id: vec![],
         quotas: vec![],
-        quota_size_limit_upper_bound: 20000,
-        quota_objects_limit_upper_bound: 200,
-        global_quota: Quota::new(2000000, 2000),
-        uploading_blocked: false,
+        quota_size_limit_upper_bound: DEFAULT_QUOTA_SIZE_LIMIT_UPPER_BOUND,
+        quota_objects_limit_upper_bound: DEFAULT_QUOTA_OBJECTS_LIMIT_UPPER_BOUND,
+        global_quota: DEFAULT_GLOBAL_QUOTA,
+        uploading_blocked: DEFAULT_UPLOADING_BLOCKED_STATUS,
     }
 }
 

runtime-modules/storage/src/data_directory.rs

@@ -40,6 +40,11 @@ use crate::data_object_type_registry;
 use crate::data_object_type_registry::IsActiveDataObjectType;
 use crate::*;
 
+pub const DEFAULT_QUOTA_SIZE_LIMIT_UPPER_BOUND: u64 = 20000;
+pub const DEFAULT_QUOTA_OBJECTS_LIMIT_UPPER_BOUND: u64 = 200;
+pub const DEFAULT_GLOBAL_QUOTA: Quota = Quota::new(2000000, 2000);
+pub const DEFAULT_UPLOADING_BLOCKED_STATUS: bool = false;
+
 /// The _Data directory_ main _Trait_.
 pub trait Trait:
     pallet_timestamp::Trait
@@ -247,7 +252,7 @@ decl_storage! {
     trait Store for Module<T: Trait> as DataDirectory {
 
         /// Maps data objects by their content id.
-        pub DataObjectByContentId get(fn data_object_by_content_id) config():
+        pub DataByContentId get(fn data_object_by_content_id) config():
             map hasher(blake2_128_concat) T::ContentId => Option<DataObject<T>>;
 
         /// Maps storage owner to it`s quota. Created when the first upload by the new actor occured.
@@ -255,16 +260,16 @@ decl_storage! {
             map hasher(blake2_128_concat) StorageObjectOwner<MemberId<T>, ChannelId<T>, DAOId<T>> => Quota;
 
         /// Upper bound for the Quota size limit.
-        pub QuotaSizeLimitUpperBound get(fn quota_size_limit_upper_bound) config(): u64;
+        pub QuotaSizeLimitUpperBound get(fn quota_size_limit_upper_bound) config(): u64 = DEFAULT_QUOTA_SIZE_LIMIT_UPPER_BOUND;
 
         /// Upper bound for the Quota objects number limit.
-        pub QuotaObjectsLimitUpperBound get(fn quota_objects_limit_upper_bound) config(): u64;
+        pub QuotaObjectsLimitUpperBound get(fn quota_objects_limit_upper_bound) config(): u64 = DEFAULT_QUOTA_OBJECTS_LIMIT_UPPER_BOUND;
 
         /// Global quota.
-        pub GlobalQuota get(fn global_quota) config(): Quota;
+        pub GlobalQuota get(fn global_quota) config(): Quota = DEFAULT_GLOBAL_QUOTA;
 
         /// If all new uploads blocked
-        pub UploadingBlocked get(fn uploading_blocked) config(): bool;
+        pub UploadingBlocked get(fn uploading_blocked) config(): bool = DEFAULT_UPLOADING_BLOCKED_STATUS;
     }
 }
 
@@ -493,6 +498,26 @@ decl_module! {
 }
 
 impl<T: Trait> Module<T> {
+    pub fn initialize_data_directory(
+        quotas: Vec<(
+            StorageObjectOwner<MemberId<T>, ChannelId<T>, DAOId<T>>,
+            Quota,
+        )>,
+        quota_size_limit_upper_bound: u64,
+        quota_objects_limit_upper_bound: u64,
+        global_quota: Quota,
+        uploading_blocked: bool,
+    ) {
+        for (storage_object_owner, quota) in quotas {
+            <Quotas<T>>::insert(storage_object_owner, quota);
+        }
+
+        <QuotaSizeLimitUpperBound>::put(quota_size_limit_upper_bound);
+        <QuotaObjectsLimitUpperBound>::put(quota_objects_limit_upper_bound);
+        <GlobalQuota>::put(global_quota);
+        <UploadingBlocked>::put(uploading_blocked);
+    }
+
     // Ensure given origin can perform operation under specific storage object owner
     fn ensure_storage_object_owner_origin(
         origin: T::Origin,
@@ -620,7 +645,7 @@ impl<T: Trait> Module<T> {
                 ipfs_content_id: content.ipfs_content_id,
             };
 
-            <DataObjectByContentId<T>>::insert(content.content_id, data);
+            <DataByContentId<T>>::insert(content.content_id, data);
         }
 
         // Updade or create owner quota.
@@ -639,7 +664,7 @@ impl<T: Trait> Module<T> {
         let removal_voucher = Self::calculate_content_voucher(content);
 
         for content_id in content_ids {
-            <DataObjectByContentId<T>>::remove(content_id);
+            <DataByContentId<T>>::remove(content_id);
         }
 
         // Updade owner quota.
@@ -661,7 +686,7 @@ impl<T: Trait> Module<T> {
             );
 
             ensure!(
-                !<DataObjectByContentId<T>>::contains_key(&content.content_id),
+                !<DataByContentId<T>>::contains_key(&content.content_id),
                 Error::<T>::DataObjectAlreadyAdded
             );
         }
@@ -683,7 +708,7 @@ impl<T: Trait> Module<T> {
         );
 
         data.liaison_judgement = judgement;
-        <DataObjectByContentId<T>>::insert(content_id, data);
+        <DataByContentId<T>>::insert(content_id, data);
 
         Ok(())
     }

runtime-modules/storage/src/tests/mock.rs

@@ -278,14 +278,14 @@ pub struct ExtBuilder {
 impl Default for ExtBuilder {
     fn default() -> Self {
         Self {
-            quota_objects_limit_upper_bound: 200,
-            quota_size_limit_upper_bound: 20000,
-            global_quota: Quota::new(2000000, 2000),
+            quota_objects_limit_upper_bound: DEFAULT_QUOTA_SIZE_LIMIT_UPPER_BOUND,
+            quota_size_limit_upper_bound: DEFAULT_QUOTA_OBJECTS_LIMIT_UPPER_BOUND,
+            global_quota: DEFAULT_GLOBAL_QUOTA,
             first_data_object_type_id: 1,
             first_content_id: 2,
             first_relationship_id: 3,
             first_metadata_id: 4,
-            uploading_blocked: false,
+            uploading_blocked: DEFAULT_UPLOADING_BLOCKED_STATUS,
         }
     }
 }

runtime/src/runtime_api.rs

@@ -10,10 +10,12 @@ use sp_runtime::traits::{BlakeTwo256, Block as BlockT, NumberFor};
 use sp_runtime::{generic, ApplyExtrinsicResult};
 use sp_std::vec::Vec;
 
-use crate::{BuilderWorkingGroupInstance, GatewayWorkingGroupInstance};
+use crate::{BuilderWorkingGroupInstance, DataDirectory, GatewayWorkingGroupInstance};
+
 use crate::constants::PRIMARY_PROBABILITY;
+
 use crate::{
-    content, AccountId, AuthorityDiscoveryId, Balance, BlockNumber, EpochDuration,
+    content, data_directory, AccountId, AuthorityDiscoveryId, Balance, BlockNumber, EpochDuration,
     GrandpaAuthorityList, GrandpaId, Hash, Index, RuntimeVersion, Signature, VERSION,
 };
 use crate::{
@@ -87,6 +89,14 @@ impl OnRuntimeUpgrade for CustomOnRuntimeUpgrade {
             default_content_working_group_mint_capacity,
         );
 
+        DataDirectory::initialize_data_directory(
+            Vec::new(),
+            data_directory::DEFAULT_QUOTA_SIZE_LIMIT_UPPER_BOUND,
+            data_directory::DEFAULT_QUOTA_OBJECTS_LIMIT_UPPER_BOUND,
+            data_directory::DEFAULT_GLOBAL_QUOTA,
+            data_directory::DEFAULT_UPLOADING_BLOCKED_STATUS
+        );
+
         // TODO: storage / data_directory migration or clear all data objects
 
         10_000_000 // TODO: adjust weight

storage-node/packages/helios/bin/cli.js

@@ -30,7 +30,7 @@ function makeAssetUrl(contentId, source) {
 }
 
 async function assetRelationshipState(api, contentId, providers) {
-  const dataObject = await api.query.dataDirectory.dataObjectByContentId(contentId)
+  const dataObject = await api.query.dataDirectory.dataByContentId(contentId)
 
   const relationshipIds = await api.query.dataObjectStorageRegistry.relationshipsByContentId(contentId)
 

storage-node/packages/runtime-api/assets.js

@@ -44,7 +44,7 @@ class AssetsApi {
    */
   async getDataObject(contentId) {
     contentId = parseContentId(contentId)
-    return this.base.api.query.dataDirectory.dataObjectByContentId(contentId)
+    return this.base.api.query.dataDirectory.dataByContentId(contentId)
   }
 
   /*

tests/network-tests/src/Api.ts

@@ -1790,8 +1790,8 @@ export class Api {
     return this.sendContentDirectoryTransaction(updateOperations)
   }
 
-  async getDataObjectByContentId(contentId: ContentId): Promise<DataObject | null> {
-    const dataObject = await this.api.query.dataDirectory.dataObjectByContentId<Option<DataObject>>(contentId)
+  async getDataByContentId(contentId: ContentId): Promise<DataObject | null> {
+    const dataObject = await this.api.query.dataDirectory.dataByContentId<Option<DataObject>>(contentId)
     return dataObject.unwrapOr(null)
   }
 

tests/network-tests/src/flows/storageNode/getContentFromStorageNode.ts

@@ -29,7 +29,7 @@ export default async function getContentFromStorageNode({ api, query }: FlowProp
   const contentId = ContentId.decode(registry, dataObjectId)
 
   // Decode data object
-  const dataObject = await api.getDataObjectByContentId(contentId)
+  const dataObject = await api.getDataByContentId(contentId)
 
   assert(dataObject, 'dataObject should not be null')
 

types/augment-codec/augment-api-query.ts

@@ -254,7 +254,7 @@ declare module '@polkadot/api/types/storage' {
       /**
        * Maps data objects by their content id.
        **/
-      dataObjectByContentId: AugmentedQuery<ApiType, (arg: ContentId | string | Uint8Array) => Observable<Option<DataObject>>>;
+      dataByContentId: AugmentedQuery<ApiType, (arg: ContentId | string | Uint8Array) => Observable<Option<DataObject>>>;
       /**
        * Global quota.
        **/

types/augment/augment-api-query.ts

@@ -254,7 +254,7 @@ declare module '@polkadot/api/types/storage' {
       /**
        * Maps data objects by their content id.
        **/
-      dataObjectByContentId: AugmentedQuery<ApiType, (arg: ContentId | string | Uint8Array) => Observable<Option<DataObject>>>;
+      dataByContentId: AugmentedQuery<ApiType, (arg: ContentId | string | Uint8Array) => Observable<Option<DataObject>>>;
       /**
        * Global quota.
        **/

utils/api-scripts/scripts/export-data-directory.js

@@ -15,7 +15,7 @@ const script = async ({ api }) => {
 
   const transformed = await Promise.all(
     ids.map(async (id) => {
-      let obj = await api.query.dataDirectory.dataObjectByContentId(id)
+      let obj = await api.query.dataDirectory.dataByContentId(id)
       if (obj.isNone) {
         return null
       }

utils/api-scripts/scripts/list-data-directory.js

@@ -12,7 +12,7 @@ const script = async ({ api }) => {
 
   await Promise.all(
     ids.map(async (id) => {
-      let obj = await api.query.dataDirectory.dataObjectByContentId(id)
+      let obj = await api.query.dataDirectory.dataByContentId(id)
       if (obj.isNone) {
         return
       }