Merge pull request #2798 from ignazio-bovo/map_clearing_migration_tests

Video / Channel map clearing unrolling implementation

node/src/chain_spec/mod.rs

@@ -366,6 +366,14 @@ pub fn testnet_genesis(
                 next_series_id: 1,
                 next_person_id: 1,
                 next_channel_transfer_request_id: 1,
+                video_migration: node_runtime::content::MigrationConfigRecord {
+                    current_id: 1,
+                    final_id: 1,
+                },
+                channel_migration: node_runtime::content::MigrationConfigRecord {
+                    current_id: 1,
+                    final_id: 1,
+                },
             }
         }),
         proposals_codex: Some(ProposalsCodexConfig {

runtime-modules/content/src/errors.rs

@@ -79,6 +79,10 @@ decl_error! {
         /// Bag Size specified is not valid
         InvalidBagSizeSpecified,
 
+        /// Migration not done yet
+        MigrationNotFinished,
+
+
 
     }
 }

runtime-modules/content/src/lib.rs

@@ -60,6 +60,8 @@ pub trait NumericIdentifier:
     + PartialEq
     + Ord
     + Zero
+    + From<u64>
+    + Into<u64>
 {
 }
 
@@ -101,8 +103,29 @@ pub trait Trait:
 
     /// The storage type used
     type DataObjectStorage: storage::DataObjectStorage<Self>;
+
+    /// Video migrated in each block during migration
+    type VideosMigrationsEachBlock: Get<u64>;
+
+    /// Channel migrated in each block during migration
+    type ChannelsMigrationsEachBlock: Get<u64>;
 }
 
+/// Data structure in order to keep track of the migration
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
+#[derive(Encode, Decode, Default, Clone, PartialEq, Eq, Debug)]
+pub struct MigrationConfigRecord<NumericId> {
+    // at each block the videos/channels removed will be those with id in the
+    // half open range [current_id, final_id).
+    // when migration is triggered final_id will be updated
+    // when migration is performed current_id will be updated
+    pub current_id: NumericId,
+    pub final_id: NumericId,
+}
+
+type VideoMigrationConfig<T> = MigrationConfigRecord<<T as Trait>::VideoId>;
+type ChannelMigrationConfig<T> = MigrationConfigRecord<<T as storage::Trait>::ChannelId>;
+
 /// The owner of a channel, is the authorized "actor" that can update
 /// or delete or transfer a channel and its contents.
 #[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
@@ -490,6 +513,13 @@ decl_storage! {
 
         /// Map, representing  CuratorGroupId -> CuratorGroup relation
         pub CuratorGroupById get(fn curator_group_by_id): map hasher(blake2_128_concat) T::CuratorGroupId => CuratorGroup<T>;
+
+        /// Migration config for channels
+        pub ChannelMigration get(fn channel_migration) config(): ChannelMigrationConfig<T>;
+
+        /// Migration config for videos:
+        pub VideoMigration get(fn video_migration) config(): VideoMigrationConfig<T>;
+
     }
 }
 
@@ -633,6 +663,9 @@ decl_module! {
             actor: ContentActor<T::CuratorGroupId, T::CuratorId, T::MemberId>,
             params: ChannelCreationParameters<T>,
         ) {
+            // ensure migration is done
+             ensure!(Self::is_migration_done(), Error::<T>::MigrationNotFinished);
+
             ensure_actor_authorized_to_create_channel::<T>(
                 origin.clone(),
                 &actor,
@@ -689,7 +722,7 @@ decl_module! {
             params: ChannelUpdateParameters<T>,
         ) {
             // check that channel exists
-            let channel = Self::ensure_channel_exists(&channel_id)?;
+            let channel = Self::ensure_channel_validity(&channel_id)?;
 
             ensure_actor_authorized_to_update_channel::<T>(
                 origin,
@@ -733,8 +766,9 @@ decl_module! {
             channel_id: T::ChannelId,
             num_objects_to_delete: u64,
         ) -> DispatchResult {
+
             // check that channel exists
-            let channel = Self::ensure_channel_exists(&channel_id)?;
+            let channel = Self::ensure_channel_validity(&channel_id)?;
 
             // ensure permissions
             ensure_actor_authorized_to_update_channel::<T>(
@@ -797,7 +831,7 @@ decl_module! {
             rationale: Vec<u8>,
         ) {
             // check that channel exists
-            let channel = Self::ensure_channel_exists(&channel_id)?;
+            let channel = Self::ensure_channel_validity(&channel_id)?;
 
             if channel.is_censored == is_censored {
                 return Ok(())
@@ -919,7 +953,7 @@ decl_module! {
         ) {
 
             // check that channel exists
-            let channel = Self::ensure_channel_exists(&channel_id)?;
+            let channel = Self::ensure_channel_validity(&channel_id)?;
 
             ensure_actor_authorized_to_update_channel::<T>(
                 origin,
@@ -956,10 +990,11 @@ decl_module! {
             // add it to the onchain state
             VideoById::<T>::insert(video_id, video);
 
-            // Only increment next video id if adding content was successful
+            // Only increment next video id
             NextVideoId::<T>::mutate(|id| *id += T::VideoId::one());
 
             // Add recently added video id to the channel
+
             ChannelById::<T>::mutate(channel_id, |channel| {
                 channel.num_videos = channel.num_videos.saturating_add(1);
             });
@@ -975,8 +1010,9 @@ decl_module! {
             video_id: T::VideoId,
             params: VideoUpdateParameters<T>,
         ) {
+
             // check that video exists, retrieve corresponding channel id.
-            let video = Self::ensure_video_exists(&video_id)?;
+            let video = Self::ensure_video_validity(&video_id)?;
 
             let channel_id = video.in_channel;
             let channel = ChannelById::<T>::get(&channel_id);
@@ -1013,15 +1049,13 @@ decl_module! {
             video_id: T::VideoId,
             assets_to_remove: BTreeSet<DataObjectId<T>>,
         ) {
-
             // check that video exists
-            let video = Self::ensure_video_exists(&video_id)?;
+            let video = Self::ensure_video_validity(&video_id)?;
 
             // get information regarding channel
             let channel_id = video.in_channel;
             let channel = ChannelById::<T>::get(channel_id);
 
-
             ensure_actor_authorized_to_update_channel::<T>(
                 origin,
                 &actor,
@@ -1214,7 +1248,7 @@ decl_module! {
             rationale: Vec<u8>,
         ) {
             // check that video exists
-            let video = Self::ensure_video_exists(&video_id)?;
+            let video = Self::ensure_video_validity(&video_id)?;
 
             if video.is_censored == is_censored {
                 return Ok(())
@@ -1267,10 +1301,94 @@ decl_module! {
         ) {
             Self::not_implemented()?;
         }
+
+        fn on_initialize(_n: T::BlockNumber) -> frame_support::weights::Weight {
+            Self::perform_video_migration();
+            Self::perform_channel_migration();
+
+            10_000_000 // TODO: adjust Weight
+        }
     }
 }
 
 impl<T: Trait> Module<T> {
+    /// Migrate Videos
+    fn perform_video_migration() {
+        let MigrationConfigRecord {
+            current_id,
+            final_id,
+        } = <VideoMigration<T>>::get();
+
+        if current_id < final_id {
+            // perform migration procedure
+            let next_id = sp_std::cmp::min(
+                current_id + T::VideosMigrationsEachBlock::get().into(),
+                final_id,
+            );
+
+            //
+            // == MUTATION SAFE ==
+            //
+
+            // clear maps: (iterator are lazy and do nothing unless consumed)
+            for id in current_id.into()..next_id.into() {
+                <VideoById<T>>::remove(T::VideoId::from(id));
+            }
+
+            // edit the current id
+            <VideoMigration<T>>::mutate(|value| value.current_id = next_id);
+        }
+    }
+
+    /// Migrate Channels
+    fn perform_channel_migration() {
+        let MigrationConfigRecord {
+            current_id,
+            final_id,
+        } = <ChannelMigration<T>>::get();
+
+        if current_id < final_id {
+            // perform migration procedure
+            let next_id = sp_std::cmp::min(
+                current_id + T::ChannelsMigrationsEachBlock::get().into(),
+                final_id,
+            );
+
+            //
+            // == MUTATION SAFE ==
+            //
+
+            // clear maps: (iterator are lazy and do nothing unless consumed)
+            for id in current_id.into()..next_id.into() {
+                <ChannelById<T>>::remove(T::ChannelId::from(id));
+            }
+
+            // edit the current id
+            <ChannelMigration<T>>::mutate(|value| value.current_id = next_id);
+        }
+    }
+
+    /// Ensure Channel Migration Finished
+
+    /// Ensure Video Migration Finished
+    fn is_migration_done() -> bool {
+        let MigrationConfigRecord {
+            current_id,
+            final_id,
+        } = <VideoMigration<T>>::get();
+
+        let video_migration_done = current_id == final_id;
+
+        let MigrationConfigRecord {
+            current_id,
+            final_id,
+        } = <ChannelMigration<T>>::get();
+
+        let channel_migration_done = current_id == final_id;
+
+        return video_migration_done && channel_migration_done;
+    }
+
     /// Ensure `CuratorGroup` under given id exists
     fn ensure_curator_group_under_given_id_exists(
         curator_group_id: &T::CuratorGroupId,
@@ -1290,7 +1408,11 @@ impl<T: Trait> Module<T> {
         Ok(Self::curator_group_by_id(curator_group_id))
     }
 
-    fn ensure_channel_exists(channel_id: &T::ChannelId) -> Result<Channel<T>, Error<T>> {
+    fn ensure_channel_validity(channel_id: &T::ChannelId) -> Result<Channel<T>, Error<T>> {
+        // ensure migration is done
+        ensure!(Self::is_migration_done(), Error::<T>::MigrationNotFinished,);
+
+        // ensure channel exists
         ensure!(
             ChannelById::<T>::contains_key(channel_id),
             Error::<T>::ChannelDoesNotExist
@@ -1298,7 +1420,11 @@ impl<T: Trait> Module<T> {
         Ok(ChannelById::<T>::get(channel_id))
     }
 
-    fn ensure_video_exists(video_id: &T::VideoId) -> Result<Video<T>, Error<T>> {
+    fn ensure_video_validity(video_id: &T::VideoId) -> Result<Video<T>, Error<T>> {
+        // ensure migration is done
+        ensure!(Self::is_migration_done(), Error::<T>::MigrationNotFinished,);
+
+        // ensure video exists
         ensure!(
             VideoById::<T>::contains_key(video_id),
             Error::<T>::VideoDoesNotExist
@@ -1413,10 +1539,9 @@ impl<T: Trait> Module<T> {
 // Reset Videos and Channels on runtime upgrade but preserving next ids and categories.
 impl<T: Trait> Module<T> {
     pub fn on_runtime_upgrade() {
-        // Clear VideoById map
-        <VideoById<T>>::remove_all();
-        // Clear ChannelById map
-        <ChannelById<T>>::remove_all();
+        // setting final index triggers migration
+        <VideoMigration<T>>::mutate(|config| config.final_id = <NextVideoId<T>>::get());
+        <ChannelMigration<T>>::mutate(|config| config.final_id = <NextChannelId<T>>::get());
     }
 }
 

runtime-modules/content/src/tests/migration.rs

@@ -0,0 +1,212 @@
+#![cfg(test)]
+
+use super::mock::*;
+use crate::sp_api_hidden_includes_decl_storage::hidden_include::traits::Currency;
+use crate::*;
+use std::ops::Rem;
+
+fn assert_video_and_channel_existrinsics_with(result: DispatchResult) {
+    let params = VideoCreationParametersRecord {
+        assets: None,
+        meta: None,
+    };
+
+    // attempt to create valid channel if result is ok, otherwise id does not matter
+    let channel_id = if result.is_ok() {
+        Content::next_channel_id()
+    } else {
+        <Test as storage::Trait>::ChannelId::one()
+    };
+
+    // attempt to create valid video if result is ok, otherwise id does not matter
+    let video_id = if result.is_ok() {
+        Content::next_video_id()
+    } else {
+        <Test as Trait>::VideoId::one()
+    };
+
+    assert_eq!(
+        Content::create_channel(
+            Origin::signed(FIRST_MEMBER_ORIGIN),
+            ContentActor::Member(FIRST_MEMBER_ID),
+            ChannelCreationParametersRecord {
+                assets: None,
+                meta: Some(vec![]),
+                reward_account: None,
+            },
+        ),
+        result
+    );
+
+    assert_eq!(
+        Content::create_video(
+            Origin::signed(FIRST_MEMBER_ORIGIN),
+            ContentActor::Member(FIRST_MEMBER_ID),
+            channel_id.clone(),
+            params.clone()
+        ),
+        result
+    );
+    assert_eq!(
+        Content::update_channel(
+            Origin::signed(FIRST_MEMBER_ORIGIN),
+            ContentActor::Member(FIRST_MEMBER_ID),
+            channel_id.clone(),
+            ChannelUpdateParametersRecord {
+                assets_to_upload: None,
+                new_meta: Some(vec![]),
+                reward_account: None,
+                assets_to_remove: BTreeSet::new(),
+            },
+        ),
+        result
+    );
+    assert_eq!(
+        Content::update_video(
+            Origin::signed(FIRST_MEMBER_ORIGIN),
+            ContentActor::Member(FIRST_MEMBER_ID),
+            video_id.clone(),
+            VideoUpdateParametersRecord {
+                assets_to_upload: None,
+                new_meta: Some(vec![]),
+                assets_to_remove: BTreeSet::new(),
+            },
+        ),
+        result
+    );
+
+    assert_eq!(
+        Content::update_channel_censorship_status(
+            Origin::signed(FIRST_MEMBER_ORIGIN),
+            ContentActor::Member(FIRST_MEMBER_ID),
+            channel_id.clone(),
+            false,
+            b"test".to_vec()
+        ),
+        result
+    );
+
+    assert_eq!(
+        Content::update_video_censorship_status(
+            Origin::signed(FIRST_MEMBER_ORIGIN),
+            ContentActor::Member(FIRST_MEMBER_ID),
+            video_id.clone(),
+            false,
+            b"test".to_vec()
+        ),
+        result
+    );
+
+    assert_eq!(
+        Content::delete_video(
+            Origin::signed(FIRST_MEMBER_ORIGIN),
+            ContentActor::Member(FIRST_MEMBER_ID),
+            video_id.clone(),
+            BTreeSet::new(),
+        ),
+        result
+    );
+    assert_eq!(
+        Content::delete_channel(
+            Origin::signed(FIRST_MEMBER_ORIGIN),
+            ContentActor::Member(FIRST_MEMBER_ID),
+            channel_id.clone(),
+            0u64,
+        ),
+        result
+    );
+}
+
+fn setup_scenario_with(n_videos: u64, n_channels: u64) -> (u64, u64) {
+    let _ = balances::Module::<Test>::deposit_creating(
+        &FIRST_MEMBER_ORIGIN,
+        <Test as balances::Trait>::Balance::from(10_000u32),
+    );
+
+    // create n_channels channels
+    for _ in 0..n_channels {
+        create_channel_mock(
+            FIRST_MEMBER_ORIGIN,
+            ContentActor::Member(FIRST_MEMBER_ID),
+            ChannelCreationParametersRecord {
+                assets: None,
+                meta: Some(vec![]),
+                reward_account: None,
+            },
+            Ok(()),
+        );
+    }
+
+    let params = VideoCreationParametersRecord {
+        assets: None,
+        meta: None,
+    };
+
+    // create n_videos videos
+    for i in 0..n_videos {
+        create_video_mock(
+            FIRST_MEMBER_ORIGIN,
+            ContentActor::Member(FIRST_MEMBER_ID),
+            i.rem(n_channels) + 1,
+            params.clone(),
+            Ok(()),
+        );
+    }
+
+    // assert that the specified channels have been created
+    assert_eq!(VideoById::<Test>::iter().count() as u64, n_videos);
+    assert_eq!(ChannelById::<Test>::iter().count() as u64, n_channels);
+
+    let channels_migrations_per_block = <Test as Trait>::ChannelsMigrationsEachBlock::get();
+    let videos_migrations_per_block = <Test as Trait>::VideosMigrationsEachBlock::get();
+
+    // return the number of blocks required for migration
+    let divide_with_ceiling =
+        |x: u64, y: u64| (x / y) + ((x.checked_rem(y).unwrap_or_default() > 0u64) as u64);
+    (
+        divide_with_ceiling(n_channels, channels_migrations_per_block),
+        divide_with_ceiling(n_videos, videos_migrations_per_block),
+    )
+}
+
+#[test]
+fn migration_test() {
+    with_default_mock_builder(|| {
+        const START_MIGRATION_AT_BLOCK: u64 = 1;
+        run_to_block(START_MIGRATION_AT_BLOCK);
+
+        // setup scenario
+        let (blocks_channels, blocks_videos) = setup_scenario_with(100u64, 100u64);
+
+        // block at which all migrations should be completed
+        let last_migration_block = std::cmp::max(blocks_channels, blocks_videos);
+
+        // ensure we have setup scenario to properly test migration over multiple blocks
+        assert!(last_migration_block > START_MIGRATION_AT_BLOCK);
+
+        // triggering migration
+        Content::on_runtime_upgrade();
+
+        // migration should have started
+        assert!(!Content::is_migration_done());
+
+        // migration is not complete all extrinsics should fail
+        assert_video_and_channel_existrinsics_with(Err(Error::<Test>::MigrationNotFinished.into()));
+
+        // make progress with migration but should not be complete yet
+        run_to_block(last_migration_block);
+        assert!(!Content::is_migration_done());
+        assert_video_and_channel_existrinsics_with(Err(Error::<Test>::MigrationNotFinished.into()));
+
+        // run migration to expected completion block
+        run_to_block(last_migration_block + 1);
+
+        // assert that maps are cleared & migration is done
+        assert!(Content::is_migration_done());
+        assert_eq!(VideoById::<Test>::iter().count(), 0);
+        assert_eq!(ChannelById::<Test>::iter().count(), 0);
+
+        // video and channel extr. now succeed
+        assert_video_and_channel_existrinsics_with(Ok(()));
+    })
+}

runtime-modules/content/src/tests/mock.rs

@@ -325,6 +325,8 @@ impl common::origin::ActorOriginValidator<Origin, u64, u64> for () {
 parameter_types! {
     pub const MaxNumberOfCuratorsPerGroup: u32 = 10;
     pub const ChannelOwnershipPaymentEscrowId: [u8; 8] = *b"12345678";
+    pub const VideosMigrationsEachBlock: u64 = 20;
+    pub const ChannelsMigrationsEachBlock: u64 = 10;
 }
 
 impl Trait for Test {
@@ -360,6 +362,9 @@ impl Trait for Test {
 
     /// The data object used in storage
     type DataObjectStorage = storage::Module<Self>;
+
+    type VideosMigrationsEachBlock = VideosMigrationsEachBlock;
+    type ChannelsMigrationsEachBlock = ChannelsMigrationsEachBlock;
 }
 
 pub type System = frame_system::Module<Test>;
@@ -375,6 +380,8 @@ pub struct ExtBuilder {
     next_series_id: u64,
     next_channel_transfer_request_id: u64,
     next_curator_group_id: u64,
+    video_migration: VideoMigrationConfig<Test>,
+    channel_migration: ChannelMigrationConfig<Test>,
 }
 
 impl Default for ExtBuilder {
@@ -389,6 +396,14 @@ impl Default for ExtBuilder {
             next_series_id: 1,
             next_channel_transfer_request_id: 1,
             next_curator_group_id: 1,
+            video_migration: MigrationConfigRecord {
+                current_id: 1,
+                final_id: 1,
+            },
+            channel_migration: MigrationConfigRecord {
+                current_id: 1,
+                final_id: 1,
+            },
         }
     }
 }
@@ -409,6 +424,8 @@ impl ExtBuilder {
             next_series_id: self.next_series_id,
             next_channel_transfer_request_id: self.next_channel_transfer_request_id,
             next_curator_group_id: self.next_curator_group_id,
+            video_migration: self.video_migration,
+            channel_migration: self.channel_migration,
         }
         .assimilate_storage(&mut t)
         .unwrap();
@@ -425,9 +442,9 @@ pub fn with_default_mock_builder<R, F: FnOnce() -> R>(f: F) -> R {
 // https://substrate.dev/docs/en/next/development/module/tests
 pub fn run_to_block(n: u64) {
     while System::block_number() < n {
-        <System as OnFinalize<u64>>::on_finalize(System::block_number());
+        <Content as OnFinalize<u64>>::on_finalize(System::block_number());
         System::set_block_number(System::block_number() + 1);
-        <System as OnInitialize<u64>>::on_initialize(System::block_number());
+        <Content as OnInitialize<u64>>::on_initialize(System::block_number());
     }
 }
 

runtime-modules/content/src/tests/mod.rs

@@ -2,5 +2,6 @@
 
 mod channels;
 mod curators;
+mod migration;
 mod mock;
 mod videos;

runtime-modules/storage/src/lib.rs

@@ -275,7 +275,9 @@ pub trait Trait: frame_system::Trait + balances::Trait + membership::Trait {
         + Default
         + Copy
         + MaybeSerialize
-        + PartialEq;
+        + PartialEq
+        + From<u64>
+        + Into<u64>;
 
     /// Distribution bucket operator ID type (relationship between distribution bucket and
     /// distribution operator).

runtime/src/lib.rs

@@ -429,6 +429,8 @@ impl pallet_finality_tracker::Trait for Runtime {
 parameter_types! {
     pub const MaxNumberOfCuratorsPerGroup: MaxNumber = 50;
     pub const ChannelOwnershipPaymentEscrowId: [u8; 8] = *b"chescrow";
+    pub const VideosMigrationsEachBlock: u64 = 20;
+    pub const ChannelsMigrationsEachBlock: u64 = 10;
 }
 
 impl content::Trait for Runtime {
@@ -443,6 +445,8 @@ impl content::Trait for Runtime {
     type ChannelOwnershipTransferRequestId = ChannelOwnershipTransferRequestId;
     type MaxNumberOfCuratorsPerGroup = MaxNumberOfCuratorsPerGroup;
     type DataObjectStorage = Storage;
+    type VideosMigrationsEachBlock = VideosMigrationsEachBlock;
+    type ChannelsMigrationsEachBlock = ChannelsMigrationsEachBlock;
 }
 
 impl hiring::Trait for Runtime {

tests/network-tests/assets/TestChannel.json

@@ -1,9 +1,10 @@
 {
-  "handle": "Storage node channel",
-  "description": "Storage node channel",
-  "language": { "existing": { "code": "EN" } },
-  "coverPhotoUrl": "",
-  "avatarPhotoUrl": "",
+  "title": "Example Joystream Channel",
+  "description": "This is an awesome example channel!",
   "isPublic": true,
-  "isCensored": false
+  "language": "en",
+  "category": 1,
+  "avatarPhotoPath": "./joystream.png",
+  "coverPhotoPath": "./joystream.png",
+  "rewardAccount": "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY"
 }

tests/network-tests/assets/TestVideo.json

@@ -1,22 +1,20 @@
 {
-  "title": "Storage node test",
-  "description": "Storage node test",
-  "thumbnailUrl": "",
+  "title": "Example Joystream Video",
+  "description": "This is an awesome example video!",
+  "videoPath": "./joystream.MOV",
+  "thumbnailPhotoPath": "./joystream.png",
+  "language": "en",
+  "hasMarketing": false,
   "isPublic": true,
-  "isExplicit": true,
-  "hasMarketing": true,
-  "language": { "existing": { "code": "EN" } },
-  "category": { "existing": { "name": "Entertainment" } },
+  "isExplicit": false,
+  "personsList": [],
+  "category": 1,
   "license": {
-    "new": {
-      "knownLicense": {
-        "existing": { "code": "CC_BY" }
-      }
-    }
+    "code": 1001,
+    "attribution": "by Joystream Contributors"
   },
-  "media": {
-    "new": {
-      "encoding": { "existing": { "name": "H.264_MP4" } }
-    }
+  "publishedBeforeJoystream": {
+    "isPublished": true,
+    "date": "2020-01-01"
   }
 }

tests/network-tests/run-migration-tests.sh

@@ -4,10 +4,129 @@ set -e
 SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
 cd $SCRIPT_PATH
 
-# verify existence of the three new groups
+# Location that will be mounted as the /data volume in containers
+# This is how we access the initial members and balances files from
+# the containers and where generated chainspec files will be located.
+DATA_PATH=${DATA_PATH:=~/tmp}
+
+# Initial account balance for Alice
+# Alice is the source of funds for all new accounts that are created in the tests.
+ALICE_INITIAL_BALANCE=${ALICE_INITIAL_BALANCE:=100000000}
+
+# The docker image tag to use for joystream/node as the starting chain
+# that will be upgraded to the latest runtime.
+RUNTIME=${RUNTIME:=latest}
+TARGET_RUNTIME=${TARGET_RUNTIME:=latest}
+
+AUTO_CONFIRM=true
+
+mkdir -p ${DATA_PATH}
+
+echo "{
+  \"balances\":[
+    [\"5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY\", ${ALICE_INITIAL_BALANCE}]
+  ]
+}" > ${DATA_PATH}/initial-balances.json
+
+# Make Alice a member
+echo '
+  [{
+    "member_id":0,
+    "root_account":"5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY",
+    "controller_account":"5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY",
+    "handle":"alice",
+    "avatar_uri":"https://alice.com/avatar.png",
+    "about":"Alice",
+    "registered_at_time":0
+  }]
+' > ${DATA_PATH}/initial-members.json
+
+# Create a chain spec file
+docker run --rm -v ${DATA_PATH}:/data --entrypoint ./chain-spec-builder joystream/node:${RUNTIME} \
+  new \
+  --authority-seeds Alice \
+  --sudo-account  5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY \
+  --deployment dev \
+  --chain-spec-path /data/chain-spec.json \
+  --initial-balances-path /data/initial-balances.json \
+  --initial-members-path /data/initial-members.json
+
+# Convert the chain spec file to a raw chainspec file
+docker run --rm -v ${DATA_PATH}:/data joystream/node:${RUNTIME} build-spec \
+  --raw --disable-default-bootnode \
+  --chain /data/chain-spec.json > ~/tmp/chain-spec-raw.json
+
+NETWORK_ARG=
+if [ "$ATTACH_TO_NETWORK" != "" ]; then
+  NETWORK_ARG="--network ${ATTACH_TO_NETWORK}"
+fi
+
+# Start a chain with generated chain spec
+# Add "-l ws=trace,ws::handler=info" to get websocket trace logs
+CONTAINER_ID=`docker run -d -v ${DATA_PATH}:/data -p 9944:9944 ${NETWORK_ARG} --name joystream-node joystream/node:${RUNTIME} \
+  --validator --alice --unsafe-ws-external --rpc-cors=all -l runtime \
+  --chain /data/chain-spec-raw.json`
+
+function cleanup() {
+    docker logs ${CONTAINER_ID} --tail 15
+    docker stop ${CONTAINER_ID}
+    docker rm ${CONTAINER_ID}
+    rm tests/network-tests/assets/TestChannel__rejectedContent.json
+    rm tests/network-tests/assets/TestVideo__rejectedContent.json
+    
+}
+
+function pre_migration_hook() {
+sleep 5 # needed otherwise docker image won't be ready yet
+joystream-cli account:choose --address 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY # Alice
+echo "creating 1 channel"
+joystream-cli content:createChannel --input=./assets/TestChannel.json --context=Member || true
+echo "adding 1 video to the above channel"
+joystream-cli content:createVideo -c 1 --input=./assets/TestVideo.json || true
+}
+
+function post_migration_hook() {
+echo "*** verify existence of the 5 new groups ***"
 yarn joystream-cli working-groups:overview --group=operationsAlpha
 yarn joystream-cli working-groups:overview --group=operationsBeta
 yarn joystream-cli working-groups:overview --group=operationsGamma
 yarn joystream-cli working-groups:overview --group=curators
 yarn joystream-cli working-groups:overview --group=distributors
 
+echo "*** verify previously created channel and video are cleared ***"
+yarn joystream-cli content:videos 1
+yarn joystream-cli content:channel 1
+}    
+
+trap cleanup EXIT
+
+if [ "$TARGET_RUNTIME" == "$RUNTIME" ]; then
+  echo "Not Performing a runtime upgrade."
+else
+    # pre migration hook
+    pre_migration_hook
+    
+  # Copy new runtime wasm file from target joystream/node image
+  echo "Extracting wasm blob from target joystream/node image."
+  id=`docker create joystream/node:${TARGET_RUNTIME}`
+  docker cp $id:/joystream/runtime.compact.wasm ${DATA_PATH}
+  docker rm $id
+
+  # Display runtime version before runtime upgrade
+  yarn workspace api-scripts tsnode-strict src/status.ts | grep Runtime
+
+  echo "Performing runtime upgrade."
+  yarn workspace api-scripts tsnode-strict \
+    src/dev-set-runtime-code.ts -- ${DATA_PATH}/runtime.compact.wasm
+
+  echo "Runtime upgraded."
+
+  echo "Performing migration tests"
+  # post migration hook
+  post_migration_hook
+  echo "Done with migrations tests"
+fi
+
+# Display runtime version
+yarn workspace api-scripts tsnode-strict src/status.ts | grep Runtime
+

tests/network-tests/run-tests.sh

@@ -16,7 +16,6 @@ ALICE_INITIAL_BALANCE=${ALICE_INITIAL_BALANCE:=100000000}
 # The docker image tag to use for joystream/node as the starting chain
 # that will be upgraded to the latest runtime.
 RUNTIME=${RUNTIME:=latest}
-TARGET_RUNTIME=${TARGET_RUNTIME:=latest}
 
 mkdir -p ${DATA_PATH}
 
@@ -73,30 +72,7 @@ function cleanup() {
 
 trap cleanup EXIT
 
-if [ "$TARGET_RUNTIME" == "$RUNTIME" ]; then
-  echo "Not Performing a runtime upgrade."
-else
-  # Copy new runtime wasm file from target joystream/node image
-  echo "Extracting wasm blob from target joystream/node image."
-  id=`docker create joystream/node:${TARGET_RUNTIME}`
-  docker cp $id:/joystream/runtime.compact.wasm ${DATA_PATH}
-  docker rm $id
-
-  # Display runtime version before runtime upgrade
-  yarn workspace api-scripts tsnode-strict src/status.ts | grep Runtime
-
-  echo "Performing runtime upgrade."
-  yarn workspace api-scripts tsnode-strict \
-    src/dev-set-runtime-code.ts -- ${DATA_PATH}/runtime.compact.wasm
-
-  echo "Runtime upgraded."
-
-  echo "Performing migration tests"
-  ./run-migration-tests.sh $1
-  echo "Done with migrations tests"
-fi
-
 # Display runtime version
 yarn workspace api-scripts tsnode-strict src/status.ts | grep Runtime
 
-# ./run-test-scenario.sh $1
+./run-test-scenario.sh $1

types/augment/all/defs.json

@@ -903,5 +903,13 @@
     },
     "MaxNumber": "u32",
     "IsCensored": "bool",
+    "VideoMigrationConfig": {
+        "current_id": "VideoId",
+        "final_id": "VideoId"
+    },
+    "ChannelMigrationConfig": {
+        "current_id": "ChannelId",
+        "final_id": "ChannelId"
+    },
     "AccountInfo": "AccountInfoWithRefCount"
 }

types/augment/all/types.ts

@@ -245,6 +245,12 @@ export interface ChannelCurationStatus extends Null {}
 /** @name ChannelId */
 export interface ChannelId extends u64 {}
 
+/** @name ChannelMigrationConfig */
+export interface ChannelMigrationConfig extends Struct {
+  readonly current_id: ChannelId;
+  readonly final_id: ChannelId;
+}
+
 /** @name ChannelOwner */
 export interface ChannelOwner extends Enum {
   readonly isMember: boolean;
@@ -1381,6 +1387,12 @@ export interface VideoCreationParameters extends Struct {
 /** @name VideoId */
 export interface VideoId extends u64 {}
 
+/** @name VideoMigrationConfig */
+export interface VideoMigrationConfig extends Struct {
+  readonly current_id: VideoId;
+  readonly final_id: VideoId;
+}
+
 /** @name VideoUpdateParameters */
 export interface VideoUpdateParameters extends Struct {
   readonly assets_to_upload: Option<StorageAssets>;

types/src/content/index.ts

@@ -169,6 +169,15 @@ export class PersonActor extends JoyEnum({
   Curator: CuratorId,
 }) {}
 
+export class VideoMigrationConfig extends JoyStructDecorated({
+  current_id: VideoId,
+  final_id: VideoId,
+}) {}
+export class ChannelMigrationConfig extends JoyStructDecorated({
+  current_id: ChannelId,
+  final_id: ChannelId,
+}) {}
+
 export const contentTypes = {
   CuratorId,
   CuratorGroupId,
@@ -211,6 +220,8 @@ export const contentTypes = {
   EpisodeParemters,
   MaxNumber,
   IsCensored,
+  VideoMigrationConfig,
+  ChannelMigrationConfig,
 }
 
 export default contentTypes