Merge pull request #2266 from mnaamani/drop-ipns-discovery

Move away from IPNS name resolution

.github/workflows/run-network-tests.yml

@@ -219,18 +219,9 @@ jobs:
           docker-compose up -d colossus
       - name: Test uploading
         run: |
-          WAIT_TIME=90
+          sleep 6
           export DEBUG=joystream:*
-          for i in {1..4}; do
-            [ "$i" == "4" ] && exit -1
-            echo "Waiting for ipfs name registration"
-            sleep ${WAIT_TIME}
-            if yarn storage-cli upload ./pioneer/packages/apps/public/images/default-thumbnail.png 1 0; then
-              break
-            else
-              echo "Upload test failed, will retry"
-            fi
-          done
+          yarn storage-cli upload ./pioneer/packages/apps/public/images/default-thumbnail.png 1 0
           # Wait for storage-node to set status Accepted on uploaded content
           sleep 6
           cd utils/api-scripts/

Cargo.lock

@@ -2083,7 +2083,6 @@ dependencies = [
  "pallet-proposals-engine",
  "pallet-randomness-collective-flip",
  "pallet-recurring-reward",
- "pallet-service-discovery",
  "pallet-session",
  "pallet-session-benchmarking",
  "pallet-stake",
@@ -3599,29 +3598,6 @@ dependencies = [
  "sp-runtime",
 ]
 
-[[package]]
-name = "pallet-service-discovery"
-version = "3.0.0"
-dependencies = [
- "frame-support",
- "frame-system",
- "pallet-balances",
- "pallet-common",
- "pallet-hiring",
- "pallet-membership",
- "pallet-recurring-reward",
- "pallet-stake",
- "pallet-timestamp",
- "pallet-token-mint",
- "pallet-working-group",
- "parity-scale-codec",
- "serde",
- "sp-core",
- "sp-io",
- "sp-runtime",
- "sp-std 2.0.0-rc4",
-]
-
 [[package]]
 name = "pallet-session"
 version = "2.0.0-rc4"

Cargo.toml

@@ -11,7 +11,6 @@ members = [
 	"runtime-modules/membership",
 	"runtime-modules/memo",
 	"runtime-modules/recurring-reward",
-	"runtime-modules/service-discovery",
 	"runtime-modules/stake",
 	"runtime-modules/storage",
 	"runtime-modules/token-minting",

cli/src/Api.ts

@@ -557,4 +557,9 @@ export default class Api {
     const bestNumber = await this.bestNumber()
     return !!accounInfoEntries.filter(([, info]) => info.expires_at.toNumber() > bestNumber).length
   }
+
+  async storageProviderEndpoint(storageProviderId: number): Promise<string> {
+    const value = await this._api.query.storageWorkingGroup.workerStorage(storageProviderId)
+    return this._api.createType('Text', value).toString()
+  }
 }

cli/src/commands/media/uploadVideo.ts

@@ -96,66 +96,9 @@ export default class UploadVideoCommand extends MediaCommandBase {
     return hash
   }
 
-  private async getDiscoveryDataViaLocalIpfsNode(ipnsIdentity: string): Promise<any> {
-    const ipfs = ipfsHttpClient({
-      // TODO: Allow customizing node url:
-      // host: 'localhost', port: '5001', protocol: 'http',
-      timeout: 10000,
-    })
-
-    const ipnsAddress = `/ipns/${ipnsIdentity}/`
-    const ipfsName = await last(
-      ipfs.name.resolve(ipnsAddress, {
-        recursive: false,
-        nocache: false,
-      })
-    )
-    const data: any = await first(ipfs.get(ipfsName))
-    const buffer = await toBuffer(data.content)
-
-    return JSON.parse(buffer.toString())
-  }
-
-  private async getDiscoveryDataViaBootstrapEndpoint(storageProviderId: number): Promise<any> {
-    const bootstrapEndpoint = await this.getApi().getRandomBootstrapEndpoint()
-    if (!bootstrapEndpoint) {
-      this.error('No bootstrap endpoints available', { exit: ExitCodes.ApiError })
-    }
-    this.log('Bootstrap endpoint:', bootstrapEndpoint)
-    const discoveryEndpoint = new URL(`discover/v0/${storageProviderId}`, bootstrapEndpoint).toString()
-    try {
-      const data = (await axios.get(discoveryEndpoint)).data
-      return data
-    } catch (e) {
-      this.error(`Cannot retrieve data from bootstrap enpoint (${discoveryEndpoint})`, {
-        exit: ExitCodes.ExternalInfrastructureError,
-      })
-    }
-  }
-
-  private async getUploadUrlFromDiscoveryData(data: any, contentId: ContentId): Promise<string> {
-    if (typeof data === 'object' && data !== null && data.serialized) {
-      const unserialized = JSON.parse(data.serialized)
-      if (unserialized.asset && unserialized.asset.endpoint && typeof unserialized.asset.endpoint === 'string') {
-        return new URL(`asset/v0/${contentId.encode()}`, unserialized.asset.endpoint).toString()
-      }
-    }
-    this.error(`Unexpected discovery data: ${JSON.stringify(data)}`)
-  }
-
-  private async getUploadUrl(ipnsIdentity: string, storageProviderId: number, contentId: ContentId): Promise<string> {
-    let data: any
-    try {
-      this.log('Trying to connect to local ipfs node...')
-      data = await this.getDiscoveryDataViaLocalIpfsNode(ipnsIdentity)
-    } catch (e) {
-      this.warn("Couldn't get data from local ipfs node, resolving to bootstrap endpoint...")
-      data = await this.getDiscoveryDataViaBootstrapEndpoint(storageProviderId)
-    }
-
-    const uploadUrl = await this.getUploadUrlFromDiscoveryData(data, contentId)
-
-    return uploadUrl
+  private async getUploadUrl(storageProviderId: number, contentId: ContentId): Promise<string> {
+    const endpoint = await this.getApi().storageProviderEndpoint(storageProviderId)
+    return new URL(`asset/v0/${contentId.encode()}`, endpoint).toString()
   }
 
   private async getVideoMetadata(filePath: string): Promise<VideoMetadata | null> {
@@ -412,7 +355,7 @@ export default class UploadVideoCommand extends MediaCommandBase {
     }
 
     // Resolve upload url and upload the video
-    const uploadUrl = await this.getUploadUrl(ipnsIdentity, storageProviderId, contentId)
+    const uploadUrl = await this.getUploadUrl(storageProviderId, contentId)
     this.log('Resolved upload url:', uploadUrl)
 
     await this.uploadVideo(filePath, fileSize, uploadUrl)

package.json

@@ -7,6 +7,7 @@
     "build": "./build-npm-packages.sh && ./build-docker-images.sh",
     "build:packages": "./build-npm-packages.sh",
     "build:docker": "./build-docker-images.sh",
+    "setup": "./setup.sh",
     "start": "./start.sh",
     "cargo-checks": "devops/git-hooks/pre-commit && devops/git-hooks/pre-push",
     "cargo-build": "scripts/cargo-build.sh"

runtime-modules/service-discovery/Cargo.toml

@@ -1,39 +0,0 @@
-[package]
-name = 'pallet-service-discovery'
-version = '3.0.0'
-authors = ['Joystream contributors']
-edition = '2018'
-
-[dependencies]
-serde = { version = "1.0.101", optional = true, features = ["derive"] }
-codec = { package = 'parity-scale-codec', version = '1.3.1', default-features = false, features = ['derive'] }
-sp-std = { package = 'sp-std', default-features = false, git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4'}
-frame-support = { package = 'frame-support', default-features = false, git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4'}
-system = { package = 'frame-system', default-features = false, git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4'}
-sp-runtime = { package = 'sp-runtime', default-features = false, git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4'}
-working-group = { package = 'pallet-working-group', default-features = false, path = '../working-group'}
-common = { package = 'pallet-common', default-features = false, path = '../common'}
-
-[dev-dependencies]
-sp-io = { package = 'sp-io', default-features = false, git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4'}
-sp-core = { package = 'sp-core', default-features = false, git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4'}
-pallet-timestamp = { package = 'pallet-timestamp', default-features = false, git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4'}
-balances = { package = 'pallet-balances', default-features = false, git = 'https://github.com/paritytech/substrate.git', rev = '00768a1f21a579c478fe5d4f51e1fa71f7db9fd4'}
-membership = { package = 'pallet-membership', default-features = false, path = '../membership'}
-stake = { package = 'pallet-stake', default-features = false, path = '../stake'}
-hiring = { package = 'pallet-hiring', default-features = false, path = '../hiring'}
-minting = { package = 'pallet-token-mint', default-features = false, path = '../token-minting'}
-recurringrewards = { package = 'pallet-recurring-reward', default-features = false, path = '../recurring-reward'}
-
-[features]
-default = ['std']
-std = [
-	'serde',
-	'codec/std',
-	'sp-std/std',
-	'frame-support/std',
-	'system/std',
-	'sp-runtime/std',
-	'working-group/std',
-	'common/std'
-]

runtime-modules/service-discovery/src/lib.rs

@@ -1,188 +0,0 @@
-//! # Service discovery module
-//! Service discovery module for the Joystream platform supports the storage providers.
-//! It registers their 'pings' in the system with the expiration time, and stores the bootstrap
-//! nodes for the Colossus.
-//!
-//! ## Comments
-//!
-//! Service discovery module uses working group module to authorize actions. It is generally used by
-//! the Colossus service.
-//!
-//! ## Supported extrinsics
-//!
-//! - [set_ipns_id](./struct.Module.html#method.set_ipns_id) - Creates the ServiceProviderRecord to save an IPNS identity for the storage provider.
-//! - [unset_ipns_id](./struct.Module.html#method.unset_ipns_id) - Deletes the ServiceProviderRecord with the IPNS identity for the storage provider.
-//! - [set_default_lifetime](./struct.Module.html#method.set_default_lifetime) - Sets default lifetime for storage providers accounts info.
-//! - [set_bootstrap_endpoints](./struct.Module.html#method.set_bootstrap_endpoints) - Sets bootstrap endpoints for the Colossus.
-//!
-
-// Ensure we're `no_std` when compiling for Wasm.
-#![cfg_attr(not(feature = "std"), no_std)]
-
-mod mock;
-mod tests;
-
-use codec::{Decode, Encode};
-use common::Url;
-
-#[cfg(feature = "std")]
-use serde::{Deserialize, Serialize};
-
-use frame_support::{decl_event, decl_module, decl_storage, ensure};
-use sp_std::vec::Vec;
-use system::ensure_root;
-/*
-  Although there is support for ed25519 keys as the IPNS identity key and we could potentially
-  reuse the same key for the role account and ipns (and make this discovery module obselete)
-  it is probably better to separate concerns.
-  Why not to use a fixed size 32byte -> SHA256 hash of public key: because we would have to force
-  specific key type on ipfs side.
-  pub struct IPNSIdentity(pub [u8; 32]); // we loose the key type!
-  pub type IPNSIdentity(pub u8, pub [u8; 32]); // we could add the keytype?
-  can we use rust library in wasm runtime?
-  https://github.com/multiformats/rust-multihash
-  https://github.com/multiformats/multicodec/
-  https://github.com/multiformats/multihash/
-*/
-/// base58 encoded IPNS identity multihash codec
-pub type IPNSIdentity = Vec<u8>;
-
-// The storage working group instance alias.
-pub(crate) type StorageWorkingGroupInstance = working_group::Instance2;
-
-// Alias for storage working group.
-pub(crate) type StorageWorkingGroup<T> = working_group::Module<T, StorageWorkingGroupInstance>;
-
-/// Storage provider is a worker from the  working_group module.
-pub type StorageProviderId<T> = working_group::WorkerId<T>;
-
-pub(crate) const MINIMUM_LIFETIME: u32 = 600; // 1hr assuming 6s block times
-pub(crate) const DEFAULT_LIFETIME: u32 = MINIMUM_LIFETIME * 24; // 24hr
-
-/// Defines the expiration date for the storage provider.
-#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))]
-#[derive(Encode, Decode, Default, Clone, PartialEq, Eq)]
-pub struct ServiceProviderRecord<BlockNumber> {
-    /// IPNS Identity.
-    pub identity: IPNSIdentity,
-    /// Block at which information expires.
-    pub expires_at: BlockNumber,
-}
-
-/// The _Service discovery_ main _Trait_.
-pub trait Trait: system::Trait + working_group::Trait<StorageWorkingGroupInstance> {
-    /// _Service discovery_ event type.
-    type Event: From<Event<Self>> + Into<<Self as system::Trait>::Event>;
-}
-
-decl_storage! {
-    trait Store for Module<T: Trait> as Discovery {
-        /// Bootstrap endpoints maintained by root
-        pub BootstrapEndpoints get(fn bootstrap_endpoints): Vec<Url>;
-
-        /// Mapping of service providers' storage provider id to their ServiceProviderRecord
-        pub AccountInfoByStorageProviderId get(fn account_info_by_storage_provider_id):
-            map hasher(blake2_128_concat) StorageProviderId<T> => ServiceProviderRecord<T::BlockNumber>;
-
-        /// Lifetime of an ServiceProviderRecord record in AccountInfoByAccountId map
-        pub DefaultLifetime get(fn default_lifetime) config():
-            T::BlockNumber = T::BlockNumber::from(DEFAULT_LIFETIME);
-    }
-}
-
-decl_event! {
-    /// _Service discovery_ events
-    pub enum Event<T> where
-        StorageProviderId = StorageProviderId<T>
-       {
-        /// Emits on updating of the account info.
-        /// Params:
-        /// - Id of the storage provider.
-        /// - Id of the IPNS.
-        AccountInfoUpdated(StorageProviderId, IPNSIdentity),
-
-        /// Emits on removing of the account info.
-        /// Params:
-        /// - Id of the storage provider.
-        AccountInfoRemoved(StorageProviderId),
-    }
-}
-
-decl_module! {
-    /// _Service discovery_ substrate module.
-    pub struct Module<T: Trait> for enum Call where origin: T::Origin {
-        /// Default deposit_event() handler
-        fn deposit_event() = default;
-
-        /// Creates the ServiceProviderRecord to save an IPNS identity for the storage provider.
-        /// Requires signed storage provider credentials.
-        #[weight = 10_000_000] // TODO: adjust weight
-        pub fn set_ipns_id(
-            origin,
-            storage_provider_id: StorageProviderId<T>,
-            id: Vec<u8>,
-        ) {
-            <StorageWorkingGroup<T>>::ensure_worker_signed(origin, &storage_provider_id)?;
-
-            // TODO: ensure id is a valid base58 encoded IPNS identity
-
-            //
-            // == MUTATION SAFE ==
-            //
-
-            <AccountInfoByStorageProviderId<T>>::insert(storage_provider_id, ServiceProviderRecord {
-                identity: id.clone(),
-                expires_at: <system::Module<T>>::block_number() + Self::default_lifetime(),
-            });
-
-            Self::deposit_event(RawEvent::AccountInfoUpdated(storage_provider_id, id));
-        }
-
-        /// Deletes the ServiceProviderRecord with the IPNS identity for the storage provider.
-        /// Requires signed storage provider credentials.
-        #[weight = 10_000_000] // TODO: adjust weight
-        pub fn unset_ipns_id(origin, storage_provider_id: StorageProviderId<T>) {
-            <StorageWorkingGroup<T>>::ensure_worker_signed(origin, &storage_provider_id)?;
-
-            // == MUTATION SAFE ==
-
-            if <AccountInfoByStorageProviderId<T>>::contains_key(storage_provider_id) {
-                <AccountInfoByStorageProviderId<T>>::remove(storage_provider_id);
-                Self::deposit_event(RawEvent::AccountInfoRemoved(storage_provider_id));
-            }
-        }
-
-        // Privileged methods
-
-        /// Sets default lifetime for storage providers accounts info. Requires root privileges.
-        #[weight = 10_000_000] // TODO: adjust weight
-        pub fn set_default_lifetime(origin, lifetime: T::BlockNumber) {
-            ensure_root(origin)?;
-            ensure!(lifetime >= T::BlockNumber::from(MINIMUM_LIFETIME),
-                "discovery: default lifetime must be gte minimum lifetime");
-
-            // == MUTATION SAFE ==
-
-            <DefaultLifetime<T>>::put(lifetime);
-        }
-
-        /// Sets bootstrap endpoints for the Colossus. Requires root privileges.
-        #[weight = 10_000_000] // TODO: adjust weight
-        pub fn set_bootstrap_endpoints(origin, endpoints: Vec<Url>) {
-            ensure_root(origin)?;
-
-            // == MUTATION SAFE ==
-
-            BootstrapEndpoints::put(endpoints);
-        }
-    }
-}
-
-impl<T: Trait> Module<T> {
-    /// Verifies that account info for the storage provider is still valid.
-    pub fn is_account_info_expired(storage_provider_id: &StorageProviderId<T>) -> bool {
-        !<AccountInfoByStorageProviderId<T>>::contains_key(storage_provider_id)
-            || <system::Module<T>>::block_number()
-                > <AccountInfoByStorageProviderId<T>>::get(storage_provider_id).expires_at
-    }
-}

runtime-modules/service-discovery/src/mock.rs

@@ -1,183 +0,0 @@
-#![cfg(test)]
-
-pub use crate::*;
-
-use frame_support::{impl_outer_event, impl_outer_origin, parameter_types};
-use sp_core::H256;
-use sp_runtime::{
-    testing::Header,
-    traits::{BlakeTwo256, IdentityLookup},
-    Perbill,
-};
-
-// The storage working group instance alias.
-pub type StorageWorkingGroupInstance = working_group::Instance2;
-
-mod working_group_mod {
-    pub use super::StorageWorkingGroupInstance;
-    pub use working_group::Event;
-    pub use working_group::Trait;
-}
-
-mod membership_mod {
-    pub use membership::Event;
-}
-
-mod discovery {
-    pub use crate::Event;
-}
-
-impl_outer_origin! {
-    pub enum Origin for Test {}
-}
-
-impl_outer_event! {
-    pub enum MetaEvent for Test {
-        discovery<T>,
-        balances<T>,
-        membership_mod<T>,
-        working_group_mod StorageWorkingGroupInstance <T>,
-        system<T>,
-    }
-}
-
-// Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted.
-#[derive(Clone, PartialEq, Eq, Debug)]
-pub struct Test;
-parameter_types! {
-    pub const BlockHashCount: u64 = 250;
-    pub const MaximumBlockWeight: u32 = 1024;
-    pub const MaximumBlockLength: u32 = 2 * 1024;
-    pub const AvailableBlockRatio: Perbill = Perbill::one();
-    pub const MinimumPeriod: u64 = 5;
-    pub const StakePoolId: [u8; 8] = *b"joystake";
-    pub const ExistentialDeposit: u32 = 0;
-}
-
-impl system::Trait for Test {
-    type BaseCallFilter = ();
-    type Origin = Origin;
-    type Call = ();
-    type Index = u64;
-    type BlockNumber = u64;
-    type Hash = H256;
-    type Hashing = BlakeTwo256;
-    type AccountId = u64;
-    type Lookup = IdentityLookup<Self::AccountId>;
-    type Header = Header;
-    type Event = MetaEvent;
-    type BlockHashCount = BlockHashCount;
-    type MaximumBlockWeight = MaximumBlockWeight;
-    type DbWeight = ();
-    type BlockExecutionWeight = ();
-    type ExtrinsicBaseWeight = ();
-    type MaximumExtrinsicWeight = ();
-    type MaximumBlockLength = MaximumBlockLength;
-    type AvailableBlockRatio = AvailableBlockRatio;
-    type Version = ();
-    type ModuleToIndex = ();
-    type AccountData = balances::AccountData<u64>;
-    type OnNewAccount = ();
-    type OnKilledAccount = ();
-}
-
-impl Trait for Test {
-    type Event = MetaEvent;
-}
-
-impl hiring::Trait for Test {
-    type OpeningId = u64;
-    type ApplicationId = u64;
-    type ApplicationDeactivatedHandler = ();
-    type StakeHandlerProvider = hiring::Module<Self>;
-}
-
-impl minting::Trait for Test {
-    type Currency = Balances;
-    type MintId = u64;
-}
-
-impl stake::Trait for Test {
-    type Currency = Balances;
-    type StakePoolId = StakePoolId;
-    type StakingEventsHandler = ();
-    type StakeId = u64;
-    type SlashId = u64;
-}
-
-parameter_types! {
-    pub const ScreenedMemberMaxInitialBalance: u64 = 500;
-}
-
-impl membership::Trait for Test {
-    type Event = MetaEvent;
-    type MemberId = u64;
-    type PaidTermId = u64;
-    type SubscriptionId = u64;
-    type ActorId = u64;
-    type ScreenedMemberMaxInitialBalance = ScreenedMemberMaxInitialBalance;
-}
-
-impl common::currency::GovernanceCurrency for Test {
-    type Currency = Balances;
-}
-
-impl balances::Trait for Test {
-    type Balance = u64;
-    type DustRemoval = ();
-    type Event = MetaEvent;
-    type ExistentialDeposit = ExistentialDeposit;
-    type AccountStore = System;
-}
-
-impl recurringrewards::Trait for Test {
-    type PayoutStatusHandler = ();
-    type RecipientId = u64;
-    type RewardRelationshipId = u64;
-}
-
-parameter_types! {
-    pub const MaxWorkerNumberLimit: u32 = 3;
-}
-
-impl working_group::Trait<StorageWorkingGroupInstance> for Test {
-    type Event = MetaEvent;
-    type MaxWorkerNumberLimit = MaxWorkerNumberLimit;
-}
-
-impl pallet_timestamp::Trait for Test {
-    type Moment = u64;
-    type OnTimestampSet = ();
-    type MinimumPeriod = MinimumPeriod;
-}
-
-pub fn initial_test_ext() -> sp_io::TestExternalities {
-    let t = system::GenesisConfig::default()
-        .build_storage::<Test>()
-        .unwrap();
-
-    t.into()
-}
-
-pub type Balances = balances::Module<Test>;
-pub type System = system::Module<Test>;
-pub type Discovery = Module<Test>;
-
-pub(crate) fn hire_storage_provider() -> (u64, u64) {
-    let storage_provider_id = 1;
-    let role_account_id = 1;
-
-    let storage_provider = working_group::Worker {
-        member_id: 1,
-        role_account_id,
-        reward_relationship: None,
-        role_stake_profile: None,
-    };
-
-    <working_group::WorkerById<Test, StorageWorkingGroupInstance>>::insert(
-        storage_provider_id,
-        storage_provider,
-    );
-
-    (role_account_id, storage_provider_id)
-}

runtime-modules/service-discovery/src/tests.rs

@@ -1,178 +0,0 @@
-#![cfg(test)]
-
-use super::mock::*;
-
-use system::{EventRecord, Phase, RawOrigin};
-
-#[test]
-fn set_ipns_id() {
-    initial_test_ext().execute_with(|| {
-        let current_block_number = 1000;
-        System::set_block_number(current_block_number);
-
-        let (storage_provider_account_id, storage_provider_id) = hire_storage_provider();
-
-        let identity = "alice".as_bytes().to_vec();
-        let ttl = <Test as system::Trait>::BlockNumber::from(DEFAULT_LIFETIME);
-        assert!(Discovery::set_ipns_id(
-            Origin::signed(storage_provider_account_id),
-            storage_provider_id,
-            identity.clone(),
-        )
-        .is_ok());
-
-        assert!(<AccountInfoByStorageProviderId<Test>>::contains_key(
-            &storage_provider_id
-        ));
-        let account_info = Discovery::account_info_by_storage_provider_id(&storage_provider_id);
-        assert_eq!(
-            account_info,
-            ServiceProviderRecord {
-                identity: identity.clone(),
-                expires_at: current_block_number + ttl
-            }
-        );
-
-        assert_eq!(
-            *System::events().last().unwrap(),
-            EventRecord {
-                phase: Phase::Initialization,
-                event: MetaEvent::discovery(RawEvent::AccountInfoUpdated(
-                    storage_provider_id,
-                    identity.clone()
-                )),
-                topics: vec![]
-            }
-        );
-
-        // Invalid storage provider data
-        let invalid_storage_provider_id = 2;
-        let invalid_storage_provider_account_id = 2;
-        assert!(Discovery::set_ipns_id(
-            Origin::signed(invalid_storage_provider_id),
-            invalid_storage_provider_account_id,
-            identity.clone(),
-        )
-        .is_err());
-        assert!(!<AccountInfoByStorageProviderId<Test>>::contains_key(
-            &invalid_storage_provider_id
-        ));
-    });
-}
-
-#[test]
-fn unset_ipns_id() {
-    initial_test_ext().execute_with(|| {
-        let current_block_number = 1000;
-        System::set_block_number(current_block_number);
-
-        let (storage_provider_account_id, storage_provider_id) = hire_storage_provider();
-
-        <AccountInfoByStorageProviderId<Test>>::insert(
-            &storage_provider_id,
-            ServiceProviderRecord {
-                expires_at: 1000,
-                identity: "alice".as_bytes().to_vec(),
-            },
-        );
-
-        assert!(<AccountInfoByStorageProviderId<Test>>::contains_key(
-            &storage_provider_account_id
-        ));
-
-        assert!(Discovery::unset_ipns_id(
-            Origin::signed(storage_provider_account_id),
-            storage_provider_id
-        )
-        .is_ok());
-        assert!(!<AccountInfoByStorageProviderId<Test>>::contains_key(
-            &storage_provider_account_id
-        ));
-
-        assert_eq!(
-            *System::events().last().unwrap(),
-            EventRecord {
-                phase: Phase::Initialization,
-                event: MetaEvent::discovery(RawEvent::AccountInfoRemoved(storage_provider_id)),
-                topics: vec![]
-            }
-        );
-
-        // Invalid storage provider data
-        let invalid_storage_provider_id = 2;
-        let invalid_storage_provider_account_id = 2;
-        assert!(Discovery::unset_ipns_id(
-            Origin::signed(invalid_storage_provider_id),
-            invalid_storage_provider_account_id,
-        )
-        .is_err());
-        assert!(!<AccountInfoByStorageProviderId<Test>>::contains_key(
-            &invalid_storage_provider_id
-        ));
-    });
-}
-
-#[test]
-fn is_account_info_expired() {
-    initial_test_ext().execute_with(|| {
-        let storage_provider_id = 1;
-        let expires_at = 1000;
-        let id = "alice".as_bytes().to_vec();
-        <AccountInfoByStorageProviderId<Test>>::insert(
-            &storage_provider_id,
-            ServiceProviderRecord {
-                expires_at,
-                identity: id.clone(),
-            },
-        );
-
-        System::set_block_number(expires_at - 10);
-        assert!(!Discovery::is_account_info_expired(&storage_provider_id));
-
-        System::set_block_number(expires_at + 10);
-        assert!(Discovery::is_account_info_expired(&storage_provider_id));
-    });
-}
-
-#[test]
-fn set_default_lifetime() {
-    initial_test_ext().execute_with(|| {
-        let lifetime = <Test as system::Trait>::BlockNumber::from(MINIMUM_LIFETIME + 2000);
-        // privileged method should fail if not from root origin
-        assert!(
-            Discovery::set_default_lifetime(Origin::signed(1), lifetime).is_err(),
-            ""
-        );
-        assert!(
-            Discovery::set_default_lifetime(RawOrigin::Root.into(), lifetime).is_ok(),
-            ""
-        );
-        assert_eq!(Discovery::default_lifetime(), lifetime, "");
-
-        // cannot set default lifetime to less than minimum
-        let less_than_min_lifetime =
-            <Test as system::Trait>::BlockNumber::from(MINIMUM_LIFETIME - 1);
-        assert!(
-            Discovery::set_default_lifetime(RawOrigin::Root.into(), less_than_min_lifetime)
-                .is_err(),
-            ""
-        );
-    });
-}
-
-#[test]
-fn set_bootstrap_endpoints() {
-    initial_test_ext().execute_with(|| {
-        let endpoints = vec!["endpoint1".as_bytes().to_vec()];
-        // privileged method should fail if not from root origin
-        assert!(
-            Discovery::set_bootstrap_endpoints(Origin::signed(1), endpoints.clone()).is_err(),
-            ""
-        );
-        assert!(
-            Discovery::set_bootstrap_endpoints(RawOrigin::Root.into(), endpoints.clone()).is_ok(),
-            ""
-        );
-        assert_eq!(Discovery::bootstrap_endpoints(), endpoints, "");
-    });
-}

runtime/Cargo.toml

@@ -71,7 +71,6 @@ minting = { package = 'pallet-token-mint', default-features = false, path = '../
 recurring-rewards = { package = 'pallet-recurring-reward', default-features = false, path = '../runtime-modules/recurring-reward'}
 working-group = { package = 'pallet-working-group', default-features = false, path = '../runtime-modules/working-group'}
 storage = { package = 'pallet-storage', default-features = false, path = '../runtime-modules/storage'}
-service-discovery = { package = 'pallet-service-discovery', default-features = false, path = '../runtime-modules/service-discovery'}
 proposals-engine = { package = 'pallet-proposals-engine', default-features = false, path = '../runtime-modules/proposals/engine'}
 proposals-discussion = { package = 'pallet-proposals-discussion', default-features = false, path = '../runtime-modules/proposals/discussion'}
 proposals-codex = { package = 'pallet-proposals-codex', default-features = false, path = '../runtime-modules/proposals/codex'}
@@ -145,7 +144,6 @@ std = [
     'recurring-rewards/std',
     'working-group/std',
     'storage/std',
-    'service-discovery/std',
     'proposals-engine/std',
     'proposals-discussion/std',
     'proposals-codex/std',

runtime/src/integration/storage.rs

@@ -10,16 +10,17 @@ impl storage::data_directory::StorageProviderHelper<Runtime> for StorageProvider
     fn get_random_storage_provider() -> Result<ActorId, storage::data_directory::Error<Runtime>> {
         let ids = crate::StorageWorkingGroup::get_all_worker_ids();
 
-        let live_ids: Vec<ActorId> = ids
+        // Filter workers that have set value for their storage value
+        let ids: Vec<ActorId> = ids
             .into_iter()
-            .filter(|id| !<service_discovery::Module<Runtime>>::is_account_info_expired(id))
+            .filter(|id| !crate::StorageWorkingGroup::worker_storage(id).is_empty())
             .collect();
 
-        if live_ids.is_empty() {
+        if ids.is_empty() {
             Err(storage::data_directory::Error::<Runtime>::NoProviderAvailable)
         } else {
-            let index = Self::random_index(live_ids.len());
-            Ok(live_ids[index])
+            let index = Self::random_index(ids.len());
+            Ok(ids[index])
         }
     }
 }

runtime/src/lib.rs

@@ -529,10 +529,6 @@ impl working_group::Trait<GatewayWorkingGroupInstance> for Runtime {
     type MaxWorkerNumberLimit = MaxWorkerNumberLimit;
 }
 
-impl service_discovery::Trait for Runtime {
-    type Event = Event;
-}
-
 parameter_types! {
     pub const ProposalCancellationFee: u64 = 10000;
     pub const ProposalRejectionFee: u64 = 5000;
@@ -654,7 +650,6 @@ construct_runtime!(
         DataObjectTypeRegistry: data_object_type_registry::{Module, Call, Storage, Event<T>, Config<T>},
         DataDirectory: data_directory::{Module, Call, Storage, Event<T>, Config<T>},
         DataObjectStorageRegistry: data_object_storage_registry::{Module, Call, Storage, Event<T>, Config<T>},
-        Discovery: service_discovery::{Module, Call, Storage, Event<T>},
         // --- Proposals
         ProposalsEngine: proposals_engine::{Module, Call, Storage, Event<T>},
         ProposalsDiscussion: proposals_discussion::{Module, Call, Storage, Event<T>},

runtime/src/tests/storage_integration.rs

@@ -24,18 +24,13 @@ fn storage_provider_helper_succeeds() {
 		<working_group::WorkerById<Runtime, Instance2>>::insert(worker_id2, Worker::default());
 		<working_group::WorkerById<Runtime, Instance2>>::insert(worker_id3, Worker::default());
 
-		// Still error - not registered in the service discovery.
+		// Still error - endpoints not set in worker storage value.
 		let random_provider_result = <StorageProviderHelper as storage::data_directory::StorageProviderHelper<Runtime>>::get_random_storage_provider();
 		assert!(random_provider_result.is_err());
 
-		let account_info = service_discovery::ServiceProviderRecord{
-			identity: Vec::new(),
-			expires_at: 1000
-		};
-
-		<service_discovery::AccountInfoByStorageProviderId<Runtime>>::insert(worker_id1,account_info.clone());
-		<service_discovery::AccountInfoByStorageProviderId<Runtime>>::insert(worker_id2,account_info.clone());
-		<service_discovery::AccountInfoByStorageProviderId<Runtime>>::insert(worker_id3,account_info);
+		<working_group::WorkerStorage<Runtime, Instance2>>::insert(worker_id1, b"http://storage1.net/".to_vec());
+		<working_group::WorkerStorage<Runtime, Instance2>>::insert(worker_id2, b"http://storage2.net/".to_vec());
+		<working_group::WorkerStorage<Runtime, Instance2>>::insert(worker_id3, b"http://storage3.net/".to_vec());
 
 		// Should work now.
 		let worker_ids = vec![worker_id1, worker_id2, worker_id3];

storage-node/packages/cli/package.json

@@ -45,7 +45,6 @@
   },
   "dependencies": {
     "@joystream/storage-runtime-api": "^0.1.0",
-    "@joystream/service-discovery": "^0.1.0",
     "@joystream/storage-utils": "^0.1.0",
     "@joystream/types": "^0.16.0",
     "axios": "^0.21.1",

storage-node/packages/cli/src/commands/dev.ts

@@ -102,13 +102,6 @@ const init = async (api: RuntimeApi): Promise<any> => {
     debug('Alice is already a member.')
   }
 
-  // set localhost colossus as discovery provider
-  // assuming pioneer dev server is running on port 3000 we should run
-  // the storage dev server on a different port than the default for colossus which is also
-  // 3000
-  debug('Setting Local development node as bootstrap endpoint.')
-  await api.discovery.setBootstrapEndpoints(alice, [`http://localhost:${developmentPort()}/`])
-
   debug('Transferring tokens to storage role account.')
   // Give role account some tokens to work with
   api.balances.transfer(alice, roleAccount, 100000)

storage-node/packages/cli/src/commands/upload.ts

@@ -4,7 +4,6 @@ import ipfsHash from 'ipfs-only-hash'
 import { ContentId, DataObject } from '@joystream/types/storage'
 import BN from 'bn.js'
 import { BaseCommand } from './base'
-import { DiscoveryClient } from '@joystream/service-discovery'
 import Debug from 'debug'
 import chalk from 'chalk'
 import { aliceKeyPair } from './dev'
@@ -28,7 +27,6 @@ export class UploadCommand extends BaseCommand {
   private readonly keyFile: string
   private readonly passPhrase: string
   private readonly memberId: string
-  private readonly discoveryClient: DiscoveryClient
 
   constructor(
     api: any,
@@ -41,7 +39,6 @@ export class UploadCommand extends BaseCommand {
     super()
 
     this.api = api
-    this.discoveryClient = new DiscoveryClient({ api })
     this.mediaSourceFilePath = mediaSourceFilePath
     this.dataObjectTypeId = dataObjectTypeId
     this.memberId = memberId
@@ -159,19 +156,13 @@ export class UploadCommand extends BaseCommand {
   // Requests the runtime and obtains the storage node endpoint URL.
   private async discoverStorageProviderEndpoint(storageProviderId: string): Promise<string> {
     try {
-      const serviceInfo = await this.discoveryClient.discover(storageProviderId)
+      const endpoint = await this.api.workers.getWorkerStorageValue(storageProviderId)
 
-      if (serviceInfo === null) {
-        this.fail('Storage node discovery failed.')
-      }
-      debug(`Discovered service info object: ${serviceInfo}`)
-
-      const dataWrapper = JSON.parse(serviceInfo)
-      const assetWrapper = JSON.parse(dataWrapper.serialized)
+      debug(`Resolved endpoint: ${endpoint}`)
 
-      return assetWrapper.asset.endpoint
+      return endpoint
     } catch (err) {
-      this.fail(`Could not get asset endpoint: ${err}`)
+      this.fail(`Could not get provider endpoint: ${err}`)
     }
   }
 

storage-node/packages/colossus/bin/cli.js

@@ -83,9 +83,7 @@ const cli = meow(
     $ colossus [command] [arguments]
 
   Commands:
-    server        Runs a production server instance. (discovery and storage services)
-                  This is the default command if not specified.
-    discovery     Run the discovery service only.
+    server        Runs a production server instance
 
   Arguments (required for with server command, unless --dev or --anonymous args are used):
     --provider-id ID, -i ID     StorageProviderId assigned to you in working group.
@@ -128,14 +126,8 @@ function startExpressApp(app, port) {
 }
 
 // Start app
-function startAllServices({ store, api, port, discoveryClient, ipfsHttpGatewayUrl, anonymous }) {
-  const app = require('../lib/app')(PROJECT_ROOT, store, api, discoveryClient, ipfsHttpGatewayUrl, anonymous)
-  return startExpressApp(app, port)
-}
-
-// Start discovery service app only
-function startDiscoveryService({ port, discoveryClient }) {
-  const app = require('../lib/discovery')(PROJECT_ROOT, discoveryClient)
+function startAllServices({ store, api, port, ipfsHttpGatewayUrl, anonymous }) {
+  const app = require('../lib/app')(PROJECT_ROOT, store, api, ipfsHttpGatewayUrl, anonymous)
   return startExpressApp(app, port)
 }
 
@@ -214,26 +206,12 @@ async function initApiDevelopment({ wsProvider }) {
   return api
 }
 
-function getServiceInformation(publicUrl) {
-  // For now assume we run all services on the same endpoint
-  return {
-    asset: {
-      version: 1, // spec version
-      endpoint: publicUrl,
-    },
-    discover: {
-      version: 1, // spec version
-      endpoint: publicUrl,
-    },
-  }
-}
-
 // TODO: instead of recursion use while/async-await and use promise/setTimout based sleep
 // or cleaner code with generators?
-async function announcePublicUrl(api, publicUrl, publisherClient) {
+async function announcePublicUrl(api, publicUrl) {
   // re-announce in future
   const reannounce = function (timeoutMs) {
-    setTimeout(announcePublicUrl, timeoutMs, api, publicUrl, publisherClient)
+    setTimeout(announcePublicUrl, timeoutMs, api, publicUrl)
   }
 
   const chainIsSyncing = await api.chainIsSyncing()
@@ -257,20 +235,9 @@ async function announcePublicUrl(api, publicUrl, publisherClient) {
   debug('announcing public url')
 
   try {
-    const serviceInformation = getServiceInformation(publicUrl)
+    await api.workers.setWorkerStorageValue(publicUrl)
 
-    const keyId = await publisherClient.publish(serviceInformation)
-
-    await api.discovery.setAccountInfo(keyId)
-
-    debug('publishing complete, scheduling next update')
-
-    // >> sometimes after tx is finalized.. we are not reaching here!
-
-    // Reannounce before expiery. Here we are concerned primarily
-    // with keeping the account information refreshed and 'available' in
-    // the ipfs network. our record on chain is valid for 24hr
-    reannounce(50 * 60 * 1000) // in 50 minutes
+    debug('announcing complete.')
   } catch (err) {
     debug(`announcing public url failed: ${err.stack}`)
 
@@ -306,34 +273,16 @@ const commands = {
     const store = getStorage(api, cli.flags)
 
     const ipfsHost = cli.flags.ipfsHost
-    const ipfs = require('ipfs-http-client')(ipfsHost, '5001', { protocol: 'http' })
     const ipfsHttpGatewayUrl = `http://${ipfsHost}:8080/`
 
     const { startSyncing } = require('../lib/sync')
     startSyncing(api, { syncPeriod: SYNC_PERIOD_MS, anonymous: cli.flags.anonymous }, store)
 
     if (!cli.flags.anonymous) {
-      const { PublisherClient } = require('@joystream/service-discovery')
-      announcePublicUrl(api, publicUrl, new PublisherClient(ipfs))
+      announcePublicUrl(api, publicUrl)
     }
 
-    const { DiscoveryClient } = require('@joystream/service-discovery')
-    const discoveryClient = new DiscoveryClient({ ipfs, api })
-    return startAllServices({ store, api, port, discoveryClient, ipfsHttpGatewayUrl, anonymous: cli.flags.anonymous })
-  },
-  discovery: async () => {
-    banner()
-    debug('Starting Joystream Discovery Service')
-    const { RuntimeApi } = require('@joystream/storage-runtime-api')
-    const wsProvider = cli.flags.wsProvider
-    const api = await RuntimeApi.create({ provider_url: wsProvider })
-    const port = cli.flags.port
-    const ipfsHost = cli.flags.ipfsHost
-    const ipfs = require('ipfs-http-client')(ipfsHost, '5001', { protocol: 'http' })
-    const { DiscoveryClient } = require('@joystream/service-discovery')
-    const discoveryClient = new DiscoveryClient({ ipfs, api })
-    await api.untilChainIsSynced()
-    await startDiscoveryService({ api, port, discoveryClient })
+    return startAllServices({ store, api, port, ipfsHttpGatewayUrl, anonymous: cli.flags.anonymous })
   },
 }
 

storage-node/packages/colossus/lib/app.js

@@ -35,7 +35,7 @@ const fileUploads = require('./middleware/file_uploads')
 const pagination = require('@joystream/storage-utils/pagination')
 
 // Configure app
-function createApp(projectRoot, storage, runtime, discoveryClient, ipfsHttpGatewayUrl, anonymous) {
+function createApp(projectRoot, storage, runtime, ipfsHttpGatewayUrl, anonymous) {
   const app = express()
   app.use(cors())
   app.use(bodyParser.json())
@@ -59,7 +59,6 @@ function createApp(projectRoot, storage, runtime, discoveryClient, ipfsHttpGatew
     dependencies: {
       storage,
       runtime,
-      discoveryClient,
       ipfsHttpGatewayUrl,
       anonymous,
     },

storage-node/packages/colossus/lib/discovery.js

@@ -1,70 +0,0 @@
-/*
- * This file is part of the storage node for the Joystream project.
- * Copyright (C) 2019 Joystream Contributors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-'use strict'
-
-// npm requires
-const express = require('express')
-const openapi = require('express-openapi')
-const bodyParser = require('body-parser')
-const cors = require('cors')
-const yaml = require('js-yaml')
-
-// Node requires
-const fs = require('fs')
-const path = require('path')
-
-// Project requires
-const validateResponses = require('./middleware/validate_responses')
-
-// Configure app
-function createApp(projectRoot, discoveryClient) {
-  const app = express()
-  app.use(cors())
-  app.use(bodyParser.json())
-  // FIXME app.use(bodyParser.urlencoded({ extended: true }));
-
-  // Load & extend/configure API docs
-  const api = yaml.safeLoad(fs.readFileSync(path.resolve(projectRoot, 'api-base.yml')))
-  api['x-express-openapi-additional-middleware'] = [validateResponses]
-  api['x-express-openapi-validation-strict'] = true
-
-  openapi.initialize({
-    apiDoc: api,
-    app,
-    // paths: path.resolve(projectRoot, 'discovery_app_paths'),
-    paths: {
-      path: '/discover/v0/{id}',
-      module: require('../paths/discover/v0/{id}'),
-    },
-    docsPath: '/swagger.json',
-    dependencies: {
-      discoveryClient,
-    },
-  })
-
-  // If no other handler gets triggered (errors), respond with the
-  // error serialized to JSON.
-  app.use(function (err, req, res) {
-    res.status(err.status).json(err)
-  })
-
-  return app
-}
-
-module.exports = createApp

storage-node/packages/colossus/paths/discover/v0/{id}.js

@@ -1,95 +0,0 @@
-const debug = require('debug')('joystream:colossus:api:discovery')
-
-const MAX_CACHE_AGE = 30 * 60 * 1000
-const USE_CACHE = true
-
-module.exports = function (discoveryClient) {
-  const doc = {
-    // parameters for all operations in this path
-    parameters: [
-      {
-        name: 'id',
-        in: 'path',
-        required: true,
-        description: 'Storage Provider Id',
-        schema: {
-          type: 'string', // integer ?
-        },
-      },
-    ],
-
-    // Resolve Service Information
-    async get(req, res) {
-      let parsedId
-      try {
-        parsedId = parseInt(req.params.id)
-      } catch (err) {
-        return res.status(400).end()
-      }
-
-      const id = parsedId
-      let cacheMaxAge = req.query.max_age
-
-      if (cacheMaxAge) {
-        try {
-          cacheMaxAge = parseInt(cacheMaxAge)
-        } catch (err) {
-          cacheMaxAge = MAX_CACHE_AGE
-        }
-      } else {
-        cacheMaxAge = 0
-      }
-
-      // todo - validate id before querying
-
-      try {
-        debug(`resolving ${id}`)
-        // Storage providers discoveryClient must use ipfs client and not rely
-        // on joystream http discovery to avoid potentially an infinite request loop
-        // back to our own api endpoint.
-        if (!discoveryClient.ipfs) {
-          return res.status(500)
-        }
-        const info = await discoveryClient.discover(id, USE_CACHE, cacheMaxAge)
-        if (info === null) {
-          debug('info not found')
-          res.status(404).end()
-        } else {
-          res.status(200).send(info)
-        }
-      } catch (err) {
-        debug(`${err}`)
-        res.status(404).end()
-      }
-    },
-  }
-
-  // OpenAPI specs
-  doc.get.apiDoc = {
-    description: 'Resolve Service Information',
-    operationId: 'discover',
-    // tags: ['asset', 'data'],
-    responses: {
-      200: {
-        description: 'Wrapped JSON Service Information',
-        content: {
-          'application/json': {
-            schema: {
-              required: ['serialized'],
-              properties: {
-                serialized: {
-                  type: 'string',
-                },
-                signature: {
-                  type: 'string',
-                },
-              },
-            },
-          },
-        },
-      },
-    },
-  }
-
-  return doc
-}

storage-node/packages/discovery/README.md

@@ -1,120 +0,0 @@
-# Discovery
-
-The `@joystream/service-discovery` package provides an API for role services to publish
-discovery information about themselves, and for consumers to resolve this
-information.
-
-In the Joystream network, services are provided by having members stake for a
-role. The role is identified by a worker id. Resolving service information
-associated with the worker id is the main purpose of this module.
-
-This implementation is based on [IPNS](https://docs.ipfs.io/guides/concepts/ipns/)
-as well as runtime information.
-
-## Discovery Workflow
-
-The discovery workflow provides worker id to the `discover()` function, which
-will eventually return structured data.
-
-Under the hood, `discover()` the bootstrap nodes from the runtime are
-used in a browser environment, or the local ipfs node otherwise.
-
-There is a distinction in the discovery workflow:
-
-1. If run in the browser environment, a HTTP request to a participating node
-   is performed to discover nodes.
-2. If run in a node.js process, instead:
-
-- A trusted (local) IPFS node must be configured.
-- The chain is queried to resolve a worker id to an IPNS id.
-- The trusted IPFS node is used to resolve the IPNS id to an IPFS
-  file.
-- The IPFS file is fetched; this contains the structured data.
-
-Web services providing the HTTP endpoint used in the first approach will
-themselves use the second approach for fulfilling queries.
-
-## Publishing Workflow
-
-The publishing workflow is a little more involved, and requires more interaction
-with the runtime and the trusted IPFS node.
-
-1. A service information file is created.
-1. The file is published on IPFS, using the IPNS self key of the local node.
-1. The IPNS name of the trusted IPFS node is updated to refer to the published
-   file.
-1. The runtime mapping from the worker ID to the IPNS name is updated.
-
-## Published Information
-
-Any JSON data can theoretically be published with this system; however, the
-following structure is currently imposed:
-
-- The JSON must be an Object at the top-level, not an Array.
-- Each key must correspond to a [service spec](../../docs/json-signing/README.md).
-
-## Service Info Specifications
-
-Service specifications are JSON Objects, not Arrays. All service specifications
-come with their own `version` field which should be intepreted by clients making
-use of the information.
-
-Additionally, some services may only provide an `endpoint` value, as defined
-here:
-
-- `version`: A numeric version identifier for the service info field.
-- `endpoint`: A publicly accessible base URL for a service API.
-
-The `endpoint` should include a scheme and full authority, such that appending
-`swagger.json` to the path resolves the OpenAPI definition of the API served
-at this endpoint.
-
-The OpenAPI definition must include a top level path component corresponding
-to the service name, followed by an API version component. The remainder of the
-provided paths are dependent on the specific version of the API provided.
-
-For example, for an endpoint value of `https://user:password@host:port/` the
-following must hold:
-
-- `https://user:password@host:port/swagger.json` resolves to the OpenAPI
-  definition of the API(s) provided by this endpoint.
-- The OpenAPI definitions include paths prefixed by
-  `https://user:password@host:port/XXX/vYYY` where
-  - `XXX` is the service name, identical to the field name of the service spec
-    in the published service information.
-  - `YYY` the version identifier for the published service API.
-
-**Note:** The `version` field in the spec indicates the version of the spec.
-The `YYY` path component above indicates the version of the published OpenAPI.
-
-### Discovery Service
-
-Publishes `version` and `endpoint` as above; the `version` field is currently
-always `1`.
-
-### Asset Service
-
-Publishes `version` and `endpoint` as above; the `version` field is currently
-always `1`.
-
-### Example
-
-```json
-{
-  "asset": {
-    "version": 1,
-    "endpoint": "https://foo.bar/"
-  },
-  "discovery": {
-    "version": 1,
-    "endpoint": "http://quux.io/"
-  }
-}
-```
-
-Here, the following must be true:
-
-- `https://foo.bar/swagger.json` must include paths beginning with `https://foo.bar/asset/vXXX`
-  where `XXX` is the API version of the asset API.
-- `https://quux.io/swagger.json` must include paths beginning with `https://foo.bar/discovery/vYYY`
-  where `XXX` is the API version of the asset API.

storage-node/packages/discovery/discover.js

@@ -1,276 +0,0 @@
-const axios = require('axios')
-const debug = require('debug')('joystream:discovery:discover')
-const stripEndingSlash = require('@joystream/storage-utils/stripEndingSlash')
-
-const BN = require('bn.js')
-const { newExternallyControlledPromise } = require('@joystream/storage-utils/externalPromise')
-
-/**
- * Determines if code is running in a browser by testing for the global window object.
- * @return {boolean} returns result check.
- */
-function inBrowser() {
-  return typeof window !== 'undefined'
-}
-
-/**
- * After what period of time a cached record is considered stale, and would
- * trigger a re-discovery, but only if a query is made for the same provider.
- */
-const CACHE_TTL = 60 * 60 * 1000
-
-class DiscoveryClient {
-  /**
-   * Map storage-provider id to a Promise of a discovery result. The purpose
-   * is to avoid concurrent active discoveries for the same provider.
-   */
-  activeDiscoveries = {}
-
-  /**
-   * Map of storage provider id to string
-   * Cache of past discovery lookup results
-   */
-  accountInfoCache = {}
-
-  /*
-   * @param {RuntimeApi} api - api instance to query the chain
-   * @param {string} ipfsHttpGatewayUrl - optional ipfs http gateway
-   * @param {IpfsClient} ipfs - optinoal instance of an ipfs-http-client
-   */
-  constructor({ api, ipfs, ipfsHttpGatewayUrl }) {
-    this.runtimeApi = api
-    this.ipfs = ipfs
-    this.ipfsHttpGatewayUrl = ipfsHttpGatewayUrl
-  }
-
-  /**
-   * Queries the ipns id (service key) of the storage provider from the blockchain.
-   * If the storage provider is not registered it will return null.
-   * @param {number | BN | u64} storageProviderId - the provider id to lookup
-   * @returns { Promise<string | null> } - ipns multiformat address
-   */
-  async getIpnsIdentity(storageProviderId) {
-    storageProviderId = new BN(storageProviderId)
-    // lookup ipns identity from chain corresponding to storageProviderId
-    const info = await this.runtimeApi.discovery.getAccountInfo(storageProviderId)
-
-    if (info === null) {
-      // no identity found on chain for account
-      return null
-    }
-    return info.identity.toString()
-  }
-
-  /**
-   * Resolves provider id to its service information.
-   * Will use an IPFS HTTP gateway. If caller doesn't provide a url the default gateway on
-   * the local ipfs node will be used.
-   * If the storage provider is not registered it will throw an error
-   * @param {number | BN | u64} storageProviderId - the provider id to lookup
-   * @param {string} ipfsHttpGatewayUrl - optional ipfs http gateway url to perform ipfs queries
-   * @returns { Promise<object> } - the published service information
-   */
-  async discoverOverIpfsHttpGateway(storageProviderId, ipfsHttpGatewayUrl) {
-    let gateway = ipfsHttpGatewayUrl || this.ipfsHttpGatewayUrl || 'http://localhost:8080'
-    storageProviderId = new BN(storageProviderId)
-    const isProvider = await this.runtimeApi.workers.isStorageProvider(storageProviderId)
-
-    if (!isProvider) {
-      throw new Error('Cannot discover non storage providers')
-    }
-
-    const identity = await this.getIpnsIdentity(storageProviderId)
-
-    if (identity === null) {
-      // dont waste time trying to resolve if no identity was found
-      throw new Error('no identity to resolve')
-    }
-
-    gateway = stripEndingSlash(gateway)
-
-    const url = `${gateway}/ipns/${identity}`
-
-    const response = await axios.get(url)
-
-    return response.data
-  }
-
-  /**
-   * Resolves id of provider to its service information.
-   * Will use the provided colossus discovery api endpoint. If no api endpoint
-   * is provided it attempts to use the configured endpoints from the chain.
-   * If the storage provider is not registered it will throw an error
-   * @param {number | BN | u64 } storageProviderId - provider id to lookup
-   * @param {string} discoverApiEndpoint - url for a colossus discovery api endpoint
-   * @returns { Promise<object> } - the published service information
-   */
-  async discoverOverJoystreamDiscoveryService(storageProviderId, discoverApiEndpoint) {
-    storageProviderId = new BN(storageProviderId)
-    const isProvider = await this.runtimeApi.workers.isStorageProvider(storageProviderId)
-
-    if (!isProvider) {
-      throw new Error('Cannot discover non storage providers')
-    }
-
-    const identity = await this.getIpnsIdentity(storageProviderId)
-
-    // dont waste time trying to resolve if no identity was found
-    if (identity === null) {
-      throw new Error('no identity to resolve')
-    }
-
-    if (!discoverApiEndpoint) {
-      // Use bootstrap nodes
-      const discoveryBootstrapNodes = await this.runtimeApi.discovery.getBootstrapEndpoints()
-
-      if (discoveryBootstrapNodes.length) {
-        discoverApiEndpoint = stripEndingSlash(discoveryBootstrapNodes[0].toString())
-      } else {
-        throw new Error('No known discovery bootstrap nodes found on network')
-      }
-    }
-
-    const url = `${discoverApiEndpoint}/discover/v0/${storageProviderId.toNumber()}`
-
-    // should have parsed if data was json?
-    const response = await axios.get(url)
-
-    return response.data
-  }
-
-  /**
-   * Resolves id of provider to its service information.
-   * Will use the local IPFS node over RPC interface.
-   * If the storage provider is not registered it will throw an error.
-   * @param {number | BN | u64 } storageProviderId - provider id to lookup
-   * @returns { Promise<object> } - the published service information
-   */
-  async discoverOverLocalIpfsNode(storageProviderId) {
-    storageProviderId = new BN(storageProviderId)
-    const isProvider = await this.runtimeApi.workers.isStorageProvider(storageProviderId)
-
-    if (!isProvider) {
-      throw new Error('Cannot discover non storage providers')
-    }
-
-    const identity = await this.getIpnsIdentity(storageProviderId)
-
-    if (identity === null) {
-      // dont waste time trying to resolve if no identity was found
-      throw new Error('no identity to resolve')
-    }
-
-    const ipnsAddress = `/ipns/${identity}/`
-
-    debug('resolved ipns to ipfs object')
-    // Can this call hang forever!? can/should we set a timeout?
-    const ipfsName = await this.ipfs.name.resolve(ipnsAddress, {
-      // don't recurse, there should only be one indirection to the service info file
-      recursive: false,
-      nocache: false,
-    })
-
-    debug('getting ipfs object', ipfsName)
-    const data = await this.ipfs.get(ipfsName) // this can sometimes hang forever!?! can we set a timeout?
-
-    // there should only be one file published under the resolved path
-    const content = data[0].content
-
-    return JSON.parse(content)
-  }
-
-  /**
-   * Internal method that handles concurrent discoveries and caching of results. Will
-   * select the appropriate discovery protocol based on browser environment or not,
-   * and if an ipfs client was passed in the constructor.
-   * @param {number | BN | u64} storageProviderId - ID of the storage provider
-   * @returns { Promise<object | null> } - the published service information
-   */
-  async _discover(storageProviderId) {
-    storageProviderId = new BN(storageProviderId)
-    const id = storageProviderId.toNumber()
-
-    const discoveryResult = this.activeDiscoveries[id]
-    if (discoveryResult) {
-      debug('discovery in progress waiting for result for', id)
-      return discoveryResult
-    }
-
-    debug('starting new discovery for', id)
-    const deferredDiscovery = newExternallyControlledPromise()
-    this.activeDiscoveries[id] = deferredDiscovery.promise
-
-    let result
-    try {
-      if (inBrowser() || !this.ipfs) {
-        result = await this.discoverOverJoystreamDiscoveryService(storageProviderId)
-      } else {
-        result = await this.discoverOverLocalIpfsNode(storageProviderId)
-      }
-
-      debug(result)
-      result = JSON.stringify(result)
-      this.accountInfoCache[id] = {
-        value: result,
-        updated: Date.now(),
-      }
-
-      deferredDiscovery.resolve(result)
-      delete this.activeDiscoveries[id]
-      return result
-    } catch (err) {
-      // we catch the error so we can update all callers
-      // and throw again to inform the first caller.
-      debug(err.message)
-      delete this.activeDiscoveries[id]
-      // deferredDiscovery.reject(err)
-      deferredDiscovery.resolve(null) // resolve to null until we figure out the issue below
-      // throw err // <-- throwing but this isn't being
-      // caught correctly in express server! Is it because there is an uncaught promise somewhere
-      // in the prior .reject() call ?
-      // I've only seen this behaviour when error is from ipfs-client
-      // ... is this unique to errors thrown from ipfs-client?
-      // Problem is its crashing the node so just return null for now
-      return null
-    }
-  }
-
-  /**
-   * Cached discovery of storage provider service information. If useCachedValue is
-   * set to true, will always return the cached result if found. New discovery will be triggered
-   * if record is found to be stale. If a stale record is not desired (CACHE_TTL old) pass a non zero
-   * value for maxCacheAge, which will force a new discovery and return the new resolved value.
-   * This method in turn calls _discovery which handles concurrent discoveries and selects the appropriate
-   * protocol to perform the query.
-   * If the storage provider is not registered it will resolve to null
-   * @param {number | BN | u64} storageProviderId - provider to discover
-   * @param {bool} useCachedValue - optionaly use chached queries
-   * @param {number} maxCacheAge - maximum age of a cached query that triggers automatic re-discovery
-   * @returns { Promise<object | null> } - the published service information
-   */
-  async discover(storageProviderId, useCachedValue = false, maxCacheAge = 0) {
-    storageProviderId = new BN(storageProviderId)
-    const id = storageProviderId.toNumber()
-    const cached = this.accountInfoCache[id]
-
-    if (cached && useCachedValue) {
-      if (maxCacheAge > 0) {
-        // get latest value
-        if (Date.now() > cached.updated + maxCacheAge) {
-          return this._discover(storageProviderId)
-        }
-      }
-      // refresh if cache if stale, new value returned on next cached query
-      if (Date.now() > cached.updated + CACHE_TTL) {
-        this._discover(storageProviderId)
-      }
-      // return best known value
-      return cached.value
-    }
-    return this._discover(storageProviderId)
-  }
-}
-
-module.exports = {
-  DiscoveryClient,
-}

storage-node/packages/discovery/index.js

@@ -1,7 +0,0 @@
-const { PublisherClient } = require('./publish')
-const { DiscoveryClient } = require('./discover')
-
-module.exports = {
-  PublisherClient,
-  DiscoveryClient,
-}

storage-node/packages/discovery/package.json

@@ -1,63 +0,0 @@
-{
-  "name": "@joystream/service-discovery",
-  "private": true,
-  "version": "0.1.0",
-  "description": "Service Discovery - Joystream Storage Node",
-  "author": "Joystream",
-  "homepage": "https://github.com/Joystream/joystream",
-  "bugs": {
-    "url": "https://github.com/Joystream/joystream/issues"
-  },
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/Joystream/joystream.git"
-  },
-  "license": "GPL-3.0-only",
-  "contributors": [
-    {
-      "name": "Joystream",
-      "url": "https://joystream.org"
-    }
-  ],
-  "keywords": [
-    "joystream",
-    "storage",
-    "node"
-  ],
-  "os": [
-    "darwin",
-    "linux"
-  ],
-  "engines": {
-    "node": ">=12.18.0"
-  },
-  "volta": {
-    "extends": "../../package.json"
-  },
-  "main": "./index.js",
-  "scripts": {
-    "test": "mocha 'test/**/*.js'",
-    "lint": "eslint 'paths/**/*.js' 'lib/**/*.js'"
-  },
-  "devDependencies": {
-    "chai": "^4.2.0",
-    "eslint": "^7.6.0",
-    "mocha": "^5.2.0",
-    "supertest": "^3.4.2",
-    "temp": "^0.9.0"
-  },
-  "dependencies": {
-    "@joystream/storage-runtime-api": "^0.1.0",
-    "@joystream/storage-utils": "^0.1.0",
-    "async-lock": "^1.2.0",
-    "axios": "^0.18.0",
-    "chalk": "^2.4.2",
-    "configstore": "^4.0.0",
-    "figlet": "^1.2.1",
-    "ipfs-http-client": "^32.0.1",
-    "js-yaml": "^3.13.1",
-    "meow": "^5.0.0",
-    "multer": "^1.4.1",
-    "si-prefix": "^0.2.0"
-  }
-}

storage-node/packages/discovery/publish.js

@@ -1,93 +0,0 @@
-const debug = require('debug')('joystream:discovery:publish')
-
-/**
- * The name of the key used for publishing. We use same key used by the ipfs node
- * for the network identitiy, to make it possible to identify the ipfs node of the storage
- * provider and use `ipfs ping` to check on the uptime of a particular node.
- */
-const PUBLISH_KEY = 'self'
-
-/**
- * Applies JSON serialization on the data object and converts the utf-8
- * string to a Buffer.
- * @param {object} data - json object
- * @returns {Buffer} returns buffer from UTF-8 json
- */
-function bufferFrom(data) {
-  return Buffer.from(JSON.stringify(data), 'utf-8')
-}
-
-/**
- * Encodes the service info into a standard format see. /storage-node/docs/json-signing.md
- * To be able to add a signature over the json data. Signing is not currently implemented.
- * @param {object} info - json object
- * @returns {Buffer} return buffer.
- */
-function encodeServiceInfo(info) {
-  return bufferFrom({
-    serialized: JSON.stringify(info),
-  })
-}
-/**
- * A PublisherClient is used to store a JSON serializable piece of "service information" in the ipfs network
- * using the `self` key of the ipfs node. This makes looking up that information available through IPNS.
- */
-class PublisherClient {
-  /**
-   * Create an instance of a PublisherClient, taking an optional ipfs client instance. If not provided
-   * a default client using default localhost node will be used.
-   * @param {IpfsClient} ipfs - optional instance of an ipfs-http-client.
-   */
-  constructor(ipfs) {
-    this.ipfs = ipfs || require('ipfs-http-client')('localhost', '5001', { protocol: 'http' })
-  }
-
-  /**
-   * Publishes the service information, encoded using the standard defined in encodeServiceInfo()
-   * to ipfs, using the local ipfs node's PUBLISH_KEY, and returns the key id used to publish.
-   * What we refer to as the ipns id.
-   * @param {object} serviceInfo - the service information to publish
-   * @return {string} - the ipns id
-   */
-  async publish(serviceInfo) {
-    const keys = await this.ipfs.key.list()
-    let servicesKey = keys.find((key) => key.name === PUBLISH_KEY)
-
-    // An ipfs node will always have the self key.
-    // If the publish key is specified as anything else and it doesn't exist
-    // we create it.
-    if (PUBLISH_KEY !== 'self' && !servicesKey) {
-      debug('generating ipns services key')
-      servicesKey = await this.ipfs.key.gen(PUBLISH_KEY, {
-        type: 'rsa',
-        size: 2048,
-      })
-    }
-
-    if (!servicesKey) {
-      throw new Error('No IPFS publishing key available!')
-    }
-
-    debug('adding service info file to node')
-    const files = await this.ipfs.add(encodeServiceInfo(serviceInfo))
-
-    debug('publishing...')
-    const { name, value } = await this.ipfs.name.publish(files[0].hash, {
-      key: PUBLISH_KEY,
-      resolve: false,
-      // lifetime: // string - Time duration of the record. Default: 24h
-      // ttl:      // string - Time duration this record should be cached
-    })
-
-    debug(`published ipns name: ${name} -> ${value}`)
-
-    // Return the key id under which the content was published. Which is used
-    // to lookup the actual ipfs content id of the published service information
-    // Note: name === servicesKey.id
-    return servicesKey.id
-  }
-}
-
-module.exports = {
-  PublisherClient,
-}

storage-node/packages/discovery/test/index.js

@@ -1 +0,0 @@
-// Add Tests!

storage-node/packages/helios/bin/cli.js

@@ -2,28 +2,9 @@
 
 const { RuntimeApi } = require('@joystream/storage-runtime-api')
 const { encodeAddress } = require('@polkadot/keyring')
-const { DiscoveryClient } = require('@joystream/service-discovery')
 const axios = require('axios')
 const stripEndingSlash = require('@joystream/storage-utils/stripEndingSlash')
 
-function mapInfoToStatus(providers, currentHeight) {
-  return providers.map(({ providerId, info }) => {
-    if (info) {
-      return {
-        providerId,
-        identity: info.identity.toString(),
-        expiresIn: info.expires_at.sub(currentHeight).toNumber(),
-        expired: currentHeight.gte(info.expires_at),
-      }
-    }
-    return {
-      providerId,
-      identity: null,
-      status: 'down',
-    }
-  })
-}
-
 function makeAssetUrl(contentId, source) {
   source = stripEndingSlash(source)
   return `${source}/asset/v0/${encodeAddress(contentId)}`
@@ -78,69 +59,17 @@ async function main() {
   const runtime = await RuntimeApi.create()
   const { api } = runtime
 
-  // get current blockheight
-  const currentHeader = await api.rpc.chain.getHeader()
-  const currentHeight = currentHeader.number.toBn()
-
   // get all providers
   const { ids: storageProviders } = await runtime.workers.getAllProviders()
   console.log(`Found ${storageProviders.length} staked providers`)
 
-  const storageProviderAccountInfos = await Promise.all(
-    storageProviders.map(async (providerId) => {
-      return {
-        providerId,
-        info: await runtime.discovery.getAccountInfo(providerId),
-      }
-    })
-  )
-
-  // providers that have updated their account info and published ipfs id
-  // considered live if the record hasn't expired yet
-  const liveProviders = storageProviderAccountInfos.filter(({ info }) => {
-    return info && info.expires_at.gte(currentHeight)
-  })
-
-  const downProviders = storageProviderAccountInfos.filter(({ info }) => {
-    return info === null
-  })
-
-  const expiredTtlProviders = storageProviderAccountInfos.filter(({ info }) => {
-    return info && currentHeight.gte(info.expires_at)
-  })
-
-  const providersStatuses = mapInfoToStatus(liveProviders, currentHeight)
-  console.log('\n== Live Providers\n', providersStatuses)
-
-  const expiredProviderStatuses = mapInfoToStatus(expiredTtlProviders, currentHeight)
-  console.log('\n== Expired Providers\n', expiredProviderStatuses)
-
-  console.log(
-    '\n== Down Providers!\n',
-    downProviders.map((provider) => {
-      return {
-        providerId: provider.providerId,
-      }
-    })
-  )
-
-  const discoveryClient = new DiscoveryClient({ api: runtime })
-
-  // Resolve IPNS identities of providers
+  // Resolve Endpoints of providers
   console.log('\nResolving live provider API Endpoints...')
   const endpoints = await Promise.all(
-    providersStatuses.map(async ({ providerId }) => {
+    storageProviders.map(async (providerId) => {
       try {
-        const serviceInfo = await discoveryClient.discoverOverJoystreamDiscoveryService(providerId)
-
-        if (serviceInfo === null) {
-          console.log(`provider ${providerId} has not published service information`)
-          return { providerId, endpoint: null }
-        }
-
-        const info = JSON.parse(serviceInfo.serialized)
-        console.log(`${providerId} -> ${info.asset.endpoint}`)
-        return { providerId, endpoint: info.asset.endpoint }
+        const endpoint = (await runtime.workers.getWorkerStorageValue(providerId)).toString()
+        return { providerId, endpoint }
       } catch (err) {
         console.log('resolve failed for id', providerId, err.message)
         return { providerId, endpoint: null }
@@ -152,7 +81,7 @@ async function main() {
   await Promise.all(
     endpoints.map(async (provider) => {
       if (!provider.endpoint) {
-        console.log('skipping', provider.address)
+        console.log(provider.providerId, 'No url set, skipping')
         return
       }
       const swaggerUrl = `${stripEndingSlash(provider.endpoint)}/swagger.json`
@@ -164,22 +93,20 @@ async function main() {
       } catch (err) {
         error = err
       }
-      console.log(`${provider.endpoint} - ${error ? error.message : 'OK'}`)
+      console.log(`${provider.providerId}`, `${provider.endpoint} - ${error ? error.message : 'OK'}`)
     })
   )
 
   const knownContentIds = await runtime.assets.getKnownContentIds()
-  console.log(`\nData Directory has ${knownContentIds.length} assets`)
-
-  // Check which providers are reporting a ready relationship for each asset
+  const assetStatuses = {}
   await Promise.all(
     knownContentIds.map(async (contentId) => {
-      const [relationshipsCount, judgement] = await assetRelationshipState(api, contentId, storageProviders)
-      console.log(
-        `${encodeAddress(contentId)} replication ${relationshipsCount}/${storageProviders.length} - ${judgement}`
-      )
+      const [, judgement] = await assetRelationshipState(api, contentId, storageProviders)
+      const j = judgement.toString()
+      assetStatuses[j] = assetStatuses[j] ? assetStatuses[j] + 1 : 1
     })
   )
+  console.log(`\nData Directory has ${knownContentIds.length} assets:`, assetStatuses)
 
   // interesting disconnect doesn't work unless an explicit provider was created
   // for underlying api instance

storage-node/packages/runtime-api/discovery.js

@@ -1,72 +0,0 @@
-'use strict'
-
-const debug = require('debug')('joystream:runtime:discovery')
-
-/*
- * Add discovery related functionality to the substrate API.
- */
-class DiscoveryApi {
-  static async create(base) {
-    const ret = new DiscoveryApi()
-    ret.base = base
-    await DiscoveryApi.init()
-    return ret
-  }
-
-  static async init() {
-    debug('Init')
-  }
-
-  /*
-   * Get Bootstrap endpoints
-   */
-  async getBootstrapEndpoints() {
-    return this.base.api.query.discovery.bootstrapEndpoints()
-  }
-
-  /*
-   * Set Bootstrap endpoints, requires the sudo account to be provided and unlocked
-   */
-  async setBootstrapEndpoints(sudoAccount, endpoints) {
-    const tx = this.base.api.tx.discovery.setBootstrapEndpoints(endpoints)
-    // make sudo call
-    return this.base.signAndSend(sudoAccount, this.base.api.tx.sudo.sudo(tx))
-  }
-
-  /*
-   * Get AccountInfo of a storage provider
-   */
-  async getAccountInfo(storageProviderId) {
-    const info = await this.base.api.query.discovery.accountInfoByStorageProviderId(storageProviderId)
-    // Not an Option so we use default value check to know if info was found
-    return info.expires_at.eq(0) ? null : info
-  }
-
-  /*
-   * Set AccountInfo of our storage provider
-   */
-  async setAccountInfo(ipnsId) {
-    const roleAccountId = this.base.identities.key.address
-    const storageProviderId = this.base.storageProviderId
-    const isProvider = await this.base.workers.isStorageProvider(storageProviderId)
-    if (isProvider) {
-      const tx = this.base.api.tx.discovery.setIpnsId(storageProviderId, ipnsId)
-      return this.base.signAndSend(roleAccountId, tx)
-    }
-    throw new Error('Cannot set AccountInfo, id is not a storage provider')
-  }
-
-  /*
-   * Clear AccountInfo of our storage provider
-   */
-  async unsetAccountInfo() {
-    const roleAccountId = this.base.identities.key.address
-    const storageProviderId = this.base.storageProviderId
-    const tx = this.base.api.tx.discovery.unsetIpnsId(storageProviderId)
-    return this.base.signAndSend(roleAccountId, tx)
-  }
-}
-
-module.exports = {
-  DiscoveryApi,
-}

storage-node/packages/runtime-api/index.js

@@ -27,7 +27,6 @@ const { IdentitiesApi } = require('@joystream/storage-runtime-api/identities')
 const { BalancesApi } = require('@joystream/storage-runtime-api/balances')
 const { WorkersApi } = require('@joystream/storage-runtime-api/workers')
 const { AssetsApi } = require('@joystream/storage-runtime-api/assets')
-const { DiscoveryApi } = require('@joystream/storage-runtime-api/discovery')
 const { SystemApi } = require('@joystream/storage-runtime-api/system')
 const AsyncLock = require('async-lock')
 const Promise = require('bluebird')
@@ -89,7 +88,6 @@ class RuntimeApi {
     this.balances = await BalancesApi.create(this)
     this.workers = await WorkersApi.create(this)
     this.assets = await AssetsApi.create(this)
-    this.discovery = await DiscoveryApi.create(this)
     this.system = await SystemApi.create(this)
   }
 

storage-node/packages/runtime-api/workers.js

@@ -20,7 +20,7 @@
 
 const debug = require('debug')('joystream:runtime:roles')
 const BN = require('bn.js')
-const { Worker } = require('@joystream/types/working-group')
+const { Text } = require('@polkadot/types')
 
 /*
  * Finds assigned worker id corresponding to the application id from the resulting
@@ -101,6 +101,24 @@ class WorkersApi {
     return providers[id.toNumber()] || null
   }
 
+  /*
+   * Returns storage provider's general purpose storage value from chain
+   */
+  async getWorkerStorageValue(id) {
+    const value = await this.base.api.query.storageWorkingGroup.workerStorage(id)
+    return new Text(this.base.api.registry, value).toString()
+  }
+
+  /*
+   * Set storage provider's general purpose storage value on chain
+   */
+  async setWorkerStorageValue(value) {
+    const id = this.base.storageProviderId
+    const tx = this.base.api.tx.storageWorkingGroup.updateRoleStorage(id, value)
+    const senderAccount = await this.storageProviderRoleAccount(id)
+    return this.base.signAndSend(senderAccount, tx)
+  }
+
   /*
    * Returns the the first found provider id with a role account or null if not found
    */

tests/network-tests/run-storage-node-tests.sh

@@ -35,8 +35,8 @@ docker-compose up -d graphql-server
 docker-compose up -d processor
 
 # Fixes Error: No active storage providers available
-echo "Waiting for ipfs name registration"
-sleep 120
+echo "Wait for colossus to announce public url"
+sleep 6
 
 echo "Creating channel..."
 yarn joystream-cli media:createChannel \

types/augment-codec/augment-api-query.ts

@@ -4,7 +4,7 @@
 import { AnyNumber, ITuple, Observable } from '@polkadot/types/types';
 import { Option, Vec } from '@polkadot/types/codec';
 import { Bytes, bool, u16, u32, u64 } from '@polkadot/types/primitive';
-import { Application, ApplicationId, ApplicationOf, Category, CategoryId, Channel, ChannelCategory, ChannelCategoryId, ChannelId, ChannelOwnershipTransferRequest, ChannelOwnershipTransferRequestId, ContentId, CuratorGroup, CuratorGroupId, DataObject, DataObjectStorageRelationship, DataObjectStorageRelationshipId, DataObjectType, DataObjectTypeId, DiscussionPost, DiscussionThread, ElectionStage, ElectionStake, HiringApplicationId, InputValidationLengthConstraint, MemberId, Membership, MemoText, Mint, MintId, ObjectOwner, Opening, OpeningId, OpeningOf, PaidMembershipTerms, PaidTermId, Person, PersonId, Playlist, PlaylistId, Post, PostId, ProposalDetailsOf, ProposalId, ProposalOf, Recipient, RecipientId, RewardRelationship, RewardRelationshipId, SealedVote, Seats, Series, SeriesId, ServiceProviderRecord, Stake, StakeId, StorageProviderId, Thread, ThreadCounter, ThreadId, TransferableStake, Url, Video, VideoCategory, VideoCategoryId, VideoId, VoteKind, Voucher, WorkerId, WorkerOf } from './all';
+import { Application, ApplicationId, ApplicationOf, Category, CategoryId, Channel, ChannelCategory, ChannelCategoryId, ChannelId, ChannelOwnershipTransferRequest, ChannelOwnershipTransferRequestId, ContentId, CuratorGroup, CuratorGroupId, DataObject, DataObjectStorageRelationship, DataObjectStorageRelationshipId, DataObjectType, DataObjectTypeId, DiscussionPost, DiscussionThread, ElectionStage, ElectionStake, HiringApplicationId, InputValidationLengthConstraint, MemberId, Membership, MemoText, Mint, MintId, ObjectOwner, Opening, OpeningId, OpeningOf, PaidMembershipTerms, PaidTermId, Person, PersonId, Playlist, PlaylistId, Post, PostId, ProposalDetailsOf, ProposalId, ProposalOf, Recipient, RecipientId, RewardRelationship, RewardRelationshipId, SealedVote, Seats, Series, SeriesId, Stake, StakeId, Thread, ThreadCounter, ThreadId, TransferableStake, Video, VideoCategory, VideoCategoryId, VideoId, VoteKind, Voucher, WorkerId, WorkerOf } from './all';
 import { UncleEntryItem } from '@polkadot/types/interfaces/authorship';
 import { BabeAuthorityWeight, MaybeRandomness, NextConfigDescriptor, Randomness } from '@polkadot/types/interfaces/babe';
 import { AccountData, BalanceLock } from '@polkadot/types/interfaces/balances';
@@ -383,20 +383,6 @@ declare module '@polkadot/api/types/storage' {
        **/
       nextDataObjectTypeId: AugmentedQuery<ApiType, () => Observable<DataObjectTypeId>>;
     };
-    discovery: {
-      /**
-       * Mapping of service providers' storage provider id to their ServiceProviderRecord
-       **/
-      accountInfoByStorageProviderId: AugmentedQuery<ApiType, (arg: StorageProviderId | AnyNumber | Uint8Array) => Observable<ServiceProviderRecord>>;
-      /**
-       * Bootstrap endpoints maintained by root
-       **/
-      bootstrapEndpoints: AugmentedQuery<ApiType, () => Observable<Vec<Url>>>;
-      /**
-       * Lifetime of an ServiceProviderRecord record in AccountInfoByAccountId map
-       **/
-      defaultLifetime: AugmentedQuery<ApiType, () => Observable<BlockNumber>>;
-    };
     forum: {
       /**
        * Map category identifier to corresponding category.

types/augment-codec/augment-api-tx.ts

@@ -4,7 +4,7 @@
 import { AnyNumber } from '@polkadot/types/types';
 import { Compact, Option, Vec } from '@polkadot/types/codec';
 import { Bytes, bool, u16, u32, u64 } from '@polkadot/types/primitive';
-import { ActivateOpeningAt, AddOpeningParameters, ApplicationId, ApplicationIdSet, BalanceOfMint, CategoryId, ChannelCategoryCreationParameters, ChannelCategoryId, ChannelCategoryUpdateParameters, ChannelCreationParameters, ChannelId, ChannelOwnershipTransferRequest, ChannelOwnershipTransferRequestId, ChannelUpdateParameters, ContentActor, ContentId, ContentParameters, CuratorGroupId, CuratorId, DataObjectStorageRelationshipId, DataObjectType, DataObjectTypeId, ElectionParameters, FillOpeningParameters, MemberId, MemoText, ObjectOwner, OpeningId, OpeningPolicyCommitment, OpeningType, PaidTermId, PersonActor, PersonCreationParameters, PersonId, PersonUpdateParameters, PlaylistCreationParameters, PlaylistId, PlaylistUpdateParameters, PostId, ProposalId, RewardPolicy, SeriesId, SeriesParameters, StorageProviderId, TerminateRoleParameters, ThreadId, Url, VideoCategoryCreationParameters, VideoCategoryId, VideoCategoryUpdateParameters, VideoCreationParameters, VideoId, VideoUpdateParameters, VoteKind, WorkerId, WorkingGroup } from './all';
+import { ActivateOpeningAt, AddOpeningParameters, ApplicationId, ApplicationIdSet, BalanceOfMint, CategoryId, ChannelCategoryCreationParameters, ChannelCategoryId, ChannelCategoryUpdateParameters, ChannelCreationParameters, ChannelId, ChannelOwnershipTransferRequest, ChannelOwnershipTransferRequestId, ChannelUpdateParameters, ContentActor, ContentId, ContentParameters, CuratorGroupId, CuratorId, DataObjectStorageRelationshipId, DataObjectType, DataObjectTypeId, ElectionParameters, FillOpeningParameters, MemberId, MemoText, ObjectOwner, OpeningId, OpeningPolicyCommitment, OpeningType, PaidTermId, PersonActor, PersonCreationParameters, PersonId, PersonUpdateParameters, PlaylistCreationParameters, PlaylistId, PlaylistUpdateParameters, PostId, ProposalId, RewardPolicy, SeriesId, SeriesParameters, StorageProviderId, TerminateRoleParameters, ThreadId, VideoCategoryCreationParameters, VideoCategoryId, VideoCategoryUpdateParameters, VideoCreationParameters, VideoId, VideoUpdateParameters, VoteKind, WorkerId, WorkingGroup } from './all';
 import { Extrinsic, Signature } from '@polkadot/types/interfaces/extrinsics';
 import { GrandpaEquivocationProof, KeyOwnerProof } from '@polkadot/types/interfaces/grandpa';
 import { Heartbeat } from '@polkadot/types/interfaces/imOnline';
@@ -433,26 +433,6 @@ declare module '@polkadot/api/types/submittable' {
        **/
       updateDataObjectType: AugmentedSubmittable<(id: DataObjectTypeId | AnyNumber | Uint8Array, dataObjectType: DataObjectType | { description?: any; active?: any } | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
     };
-    discovery: {
-      /**
-       * Sets bootstrap endpoints for the Colossus. Requires root privileges.
-       **/
-      setBootstrapEndpoints: AugmentedSubmittable<(endpoints: Vec<Url> | (Url | string)[]) => SubmittableExtrinsic<ApiType>>;
-      /**
-       * Sets default lifetime for storage providers accounts info. Requires root privileges.
-       **/
-      setDefaultLifetime: AugmentedSubmittable<(lifetime: BlockNumber | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
-      /**
-       * Creates the ServiceProviderRecord to save an IPNS identity for the storage provider.
-       * Requires signed storage provider credentials.
-       **/
-      setIpnsId: AugmentedSubmittable<(storageProviderId: StorageProviderId | AnyNumber | Uint8Array, id: Bytes | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
-      /**
-       * Deletes the ServiceProviderRecord with the IPNS identity for the storage provider.
-       * Requires signed storage provider credentials.
-       **/
-      unsetIpnsId: AugmentedSubmittable<(storageProviderId: StorageProviderId | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
-    };
     finalityTracker: {
       /**
        * Hint that the author of this block thinks the best finalized

types/augment-codec/augment-types.ts

@@ -2060,6 +2060,12 @@ declare module '@polkadot/types/types/registry' {
     FailedAt: FailedAt;
     'Option<FailedAt>': Option<FailedAt>;
     'Vec<FailedAt>': Vec<FailedAt>;
+    IPNSIdentity: IPNSIdentity;
+    'Option<IPNSIdentity>': Option<IPNSIdentity>;
+    'Vec<IPNSIdentity>': Vec<IPNSIdentity>;
+    ServiceProviderRecord: ServiceProviderRecord;
+    'Option<ServiceProviderRecord>': Option<ServiceProviderRecord>;
+    'Vec<ServiceProviderRecord>': Vec<ServiceProviderRecord>;
     BlockAndTime: BlockAndTime;
     'Option<BlockAndTime>': Option<BlockAndTime>;
     'Vec<BlockAndTime>': Vec<BlockAndTime>;
@@ -2345,12 +2351,6 @@ declare module '@polkadot/types/types/registry' {
     RoleStakeProfile: RoleStakeProfile;
     'Option<RoleStakeProfile>': Option<RoleStakeProfile>;
     'Vec<RoleStakeProfile>': Vec<RoleStakeProfile>;
-    IPNSIdentity: IPNSIdentity;
-    'Option<IPNSIdentity>': Option<IPNSIdentity>;
-    'Vec<IPNSIdentity>': Vec<IPNSIdentity>;
-    ServiceProviderRecord: ServiceProviderRecord;
-    'Option<ServiceProviderRecord>': Option<ServiceProviderRecord>;
-    'Vec<ServiceProviderRecord>': Vec<ServiceProviderRecord>;
     ContentId: ContentId;
     'Option<ContentId>': Option<ContentId>;
     'Vec<ContentId>': Vec<ContentId>;

types/augment/all/defs.json

@@ -68,6 +68,8 @@
     "Operation": "Null",
     "ReferenceConstraint": "Null",
     "FailedAt": "Null",
+    "IPNSIdentity": "Null",
+    "ServiceProviderRecord": "Null",
     "BlockAndTime": {
         "block": "u32",
         "time": "u64"
@@ -479,11 +481,6 @@
         "termination_unstaking_period": "Option<u32>",
         "exit_unstaking_period": "Option<u32>"
     },
-    "IPNSIdentity": "Text",
-    "ServiceProviderRecord": {
-        "identity": "IPNSIdentity",
-        "expires_at": "u32"
-    },
     "ContentId": "[u8;32]",
     "LiaisonJudgement": {
         "_enum": [

types/augment/all/types.ts

@@ -520,7 +520,7 @@ export interface InputValidationLengthConstraint extends Struct {
 export interface InputValue extends Null {}
 
 /** @name IPNSIdentity */
-export interface IPNSIdentity extends Text {}
+export interface IPNSIdentity extends Null {}
 
 /** @name Lead */
 export interface Lead extends Null {}
@@ -1052,10 +1052,7 @@ export interface SeriesParameters extends Struct {
 }
 
 /** @name ServiceProviderRecord */
-export interface ServiceProviderRecord extends Struct {
-  readonly identity: IPNSIdentity;
-  readonly expires_at: u32;
-}
+export interface ServiceProviderRecord extends Null {}
 
 /** @name SetLeadParams */
 export interface SetLeadParams extends ITuple<[MemberId, GenericAccountId]> {}

types/augment/augment-api-query.ts

@@ -4,7 +4,7 @@
 import { AnyNumber, ITuple, Observable } from '@polkadot/types/types';
 import { Option, Vec } from '@polkadot/types/codec';
 import { Bytes, bool, u16, u32, u64 } from '@polkadot/types/primitive';
-import { Application, ApplicationId, ApplicationOf, Category, CategoryId, Channel, ChannelCategory, ChannelCategoryId, ChannelId, ChannelOwnershipTransferRequest, ChannelOwnershipTransferRequestId, ContentId, CuratorGroup, CuratorGroupId, DataObject, DataObjectStorageRelationship, DataObjectStorageRelationshipId, DataObjectType, DataObjectTypeId, DiscussionPost, DiscussionThread, ElectionStage, ElectionStake, HiringApplicationId, InputValidationLengthConstraint, MemberId, Membership, MemoText, Mint, MintId, ObjectOwner, Opening, OpeningId, OpeningOf, PaidMembershipTerms, PaidTermId, Person, PersonId, Playlist, PlaylistId, Post, PostId, ProposalDetailsOf, ProposalId, ProposalOf, Recipient, RecipientId, RewardRelationship, RewardRelationshipId, SealedVote, Seats, Series, SeriesId, ServiceProviderRecord, Stake, StakeId, StorageProviderId, Thread, ThreadCounter, ThreadId, TransferableStake, Url, Video, VideoCategory, VideoCategoryId, VideoId, VoteKind, Voucher, WorkerId, WorkerOf } from './all';
+import { Application, ApplicationId, ApplicationOf, Category, CategoryId, Channel, ChannelCategory, ChannelCategoryId, ChannelId, ChannelOwnershipTransferRequest, ChannelOwnershipTransferRequestId, ContentId, CuratorGroup, CuratorGroupId, DataObject, DataObjectStorageRelationship, DataObjectStorageRelationshipId, DataObjectType, DataObjectTypeId, DiscussionPost, DiscussionThread, ElectionStage, ElectionStake, HiringApplicationId, InputValidationLengthConstraint, MemberId, Membership, MemoText, Mint, MintId, ObjectOwner, Opening, OpeningId, OpeningOf, PaidMembershipTerms, PaidTermId, Person, PersonId, Playlist, PlaylistId, Post, PostId, ProposalDetailsOf, ProposalId, ProposalOf, Recipient, RecipientId, RewardRelationship, RewardRelationshipId, SealedVote, Seats, Series, SeriesId, Stake, StakeId, Thread, ThreadCounter, ThreadId, TransferableStake, Video, VideoCategory, VideoCategoryId, VideoId, VoteKind, Voucher, WorkerId, WorkerOf } from './all';
 import { UncleEntryItem } from '@polkadot/types/interfaces/authorship';
 import { BabeAuthorityWeight, MaybeRandomness, NextConfigDescriptor, Randomness } from '@polkadot/types/interfaces/babe';
 import { AccountData, BalanceLock } from '@polkadot/types/interfaces/balances';
@@ -383,20 +383,6 @@ declare module '@polkadot/api/types/storage' {
        **/
       nextDataObjectTypeId: AugmentedQuery<ApiType, () => Observable<DataObjectTypeId>>;
     };
-    discovery: {
-      /**
-       * Mapping of service providers' storage provider id to their ServiceProviderRecord
-       **/
-      accountInfoByStorageProviderId: AugmentedQuery<ApiType, (arg: StorageProviderId | AnyNumber | Uint8Array) => Observable<ServiceProviderRecord>>;
-      /**
-       * Bootstrap endpoints maintained by root
-       **/
-      bootstrapEndpoints: AugmentedQuery<ApiType, () => Observable<Vec<Url>>>;
-      /**
-       * Lifetime of an ServiceProviderRecord record in AccountInfoByAccountId map
-       **/
-      defaultLifetime: AugmentedQuery<ApiType, () => Observable<BlockNumber>>;
-    };
     forum: {
       /**
        * Map category identifier to corresponding category.

types/augment/augment-api-tx.ts

@@ -4,7 +4,7 @@
 import { AnyNumber } from '@polkadot/types/types';
 import { Compact, Option, Vec } from '@polkadot/types/codec';
 import { Bytes, bool, u16, u32, u64 } from '@polkadot/types/primitive';
-import { ActivateOpeningAt, AddOpeningParameters, ApplicationId, ApplicationIdSet, BalanceOfMint, CategoryId, ChannelCategoryCreationParameters, ChannelCategoryId, ChannelCategoryUpdateParameters, ChannelCreationParameters, ChannelId, ChannelOwnershipTransferRequest, ChannelOwnershipTransferRequestId, ChannelUpdateParameters, ContentActor, ContentId, ContentParameters, CuratorGroupId, CuratorId, DataObjectStorageRelationshipId, DataObjectType, DataObjectTypeId, ElectionParameters, FillOpeningParameters, MemberId, MemoText, ObjectOwner, OpeningId, OpeningPolicyCommitment, OpeningType, PaidTermId, PersonActor, PersonCreationParameters, PersonId, PersonUpdateParameters, PlaylistCreationParameters, PlaylistId, PlaylistUpdateParameters, PostId, ProposalId, RewardPolicy, SeriesId, SeriesParameters, StorageProviderId, TerminateRoleParameters, ThreadId, Url, VideoCategoryCreationParameters, VideoCategoryId, VideoCategoryUpdateParameters, VideoCreationParameters, VideoId, VideoUpdateParameters, VoteKind, WorkerId, WorkingGroup } from './all';
+import { ActivateOpeningAt, AddOpeningParameters, ApplicationId, ApplicationIdSet, BalanceOfMint, CategoryId, ChannelCategoryCreationParameters, ChannelCategoryId, ChannelCategoryUpdateParameters, ChannelCreationParameters, ChannelId, ChannelOwnershipTransferRequest, ChannelOwnershipTransferRequestId, ChannelUpdateParameters, ContentActor, ContentId, ContentParameters, CuratorGroupId, CuratorId, DataObjectStorageRelationshipId, DataObjectType, DataObjectTypeId, ElectionParameters, FillOpeningParameters, MemberId, MemoText, ObjectOwner, OpeningId, OpeningPolicyCommitment, OpeningType, PaidTermId, PersonActor, PersonCreationParameters, PersonId, PersonUpdateParameters, PlaylistCreationParameters, PlaylistId, PlaylistUpdateParameters, PostId, ProposalId, RewardPolicy, SeriesId, SeriesParameters, StorageProviderId, TerminateRoleParameters, ThreadId, VideoCategoryCreationParameters, VideoCategoryId, VideoCategoryUpdateParameters, VideoCreationParameters, VideoId, VideoUpdateParameters, VoteKind, WorkerId, WorkingGroup } from './all';
 import { Extrinsic, Signature } from '@polkadot/types/interfaces/extrinsics';
 import { GrandpaEquivocationProof, KeyOwnerProof } from '@polkadot/types/interfaces/grandpa';
 import { Heartbeat } from '@polkadot/types/interfaces/imOnline';
@@ -433,26 +433,6 @@ declare module '@polkadot/api/types/submittable' {
        **/
       updateDataObjectType: AugmentedSubmittable<(id: DataObjectTypeId | AnyNumber | Uint8Array, dataObjectType: DataObjectType | { description?: any; active?: any } | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
     };
-    discovery: {
-      /**
-       * Sets bootstrap endpoints for the Colossus. Requires root privileges.
-       **/
-      setBootstrapEndpoints: AugmentedSubmittable<(endpoints: Vec<Url> | (Url | string)[]) => SubmittableExtrinsic<ApiType>>;
-      /**
-       * Sets default lifetime for storage providers accounts info. Requires root privileges.
-       **/
-      setDefaultLifetime: AugmentedSubmittable<(lifetime: BlockNumber | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
-      /**
-       * Creates the ServiceProviderRecord to save an IPNS identity for the storage provider.
-       * Requires signed storage provider credentials.
-       **/
-      setIpnsId: AugmentedSubmittable<(storageProviderId: StorageProviderId | AnyNumber | Uint8Array, id: Bytes | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
-      /**
-       * Deletes the ServiceProviderRecord with the IPNS identity for the storage provider.
-       * Requires signed storage provider credentials.
-       **/
-      unsetIpnsId: AugmentedSubmittable<(storageProviderId: StorageProviderId | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
-    };
     finalityTracker: {
       /**
        * Hint that the author of this block thinks the best finalized

types/augment/augment-types.ts

@@ -2060,6 +2060,12 @@ declare module '@polkadot/types/types/registry' {
     FailedAt: FailedAt;
     'Option<FailedAt>': Option<FailedAt>;
     'Vec<FailedAt>': Vec<FailedAt>;
+    IPNSIdentity: IPNSIdentity;
+    'Option<IPNSIdentity>': Option<IPNSIdentity>;
+    'Vec<IPNSIdentity>': Vec<IPNSIdentity>;
+    ServiceProviderRecord: ServiceProviderRecord;
+    'Option<ServiceProviderRecord>': Option<ServiceProviderRecord>;
+    'Vec<ServiceProviderRecord>': Vec<ServiceProviderRecord>;
     BlockAndTime: BlockAndTime;
     'Option<BlockAndTime>': Option<BlockAndTime>;
     'Vec<BlockAndTime>': Vec<BlockAndTime>;
@@ -2345,12 +2351,6 @@ declare module '@polkadot/types/types/registry' {
     RoleStakeProfile: RoleStakeProfile;
     'Option<RoleStakeProfile>': Option<RoleStakeProfile>;
     'Vec<RoleStakeProfile>': Vec<RoleStakeProfile>;
-    IPNSIdentity: IPNSIdentity;
-    'Option<IPNSIdentity>': Option<IPNSIdentity>;
-    'Vec<IPNSIdentity>': Vec<IPNSIdentity>;
-    ServiceProviderRecord: ServiceProviderRecord;
-    'Option<ServiceProviderRecord>': Option<ServiceProviderRecord>;
-    'Vec<ServiceProviderRecord>': Vec<ServiceProviderRecord>;
     ContentId: ContentId;
     'Option<ContentId>': Option<ContentId>;
     'Vec<ContentId>': Vec<ContentId>;

types/src/discovery.ts

@@ -1,17 +0,0 @@
-import { u32, Text } from '@polkadot/types'
-import { RegistryTypes } from '@polkadot/types/types'
-import { JoyStructDecorated } from './common'
-
-export class IPNSIdentity extends Text {}
-
-export class ServiceProviderRecord extends JoyStructDecorated({
-  identity: IPNSIdentity,
-  expires_at: u32, // BlockNumber
-}) {}
-
-export const discoveryTypes: RegistryTypes = {
-  IPNSIdentity,
-  ServiceProviderRecord,
-}
-
-export default discoveryTypes

types/src/index.ts

@@ -9,7 +9,6 @@ import mint from './mint'
 import recurringRewards from './recurring-rewards'
 import hiring from './hiring'
 import workingGroup from './working-group'
-import discovery from './discovery'
 import storage from './storage'
 import proposals from './proposals'
 import contentDirectory from './content'
@@ -31,7 +30,6 @@ export {
   recurringRewards,
   hiring,
   workingGroup,
-  discovery,
   storage,
   proposals,
   contentDirectory,
@@ -50,7 +48,6 @@ export const types: RegistryTypes = {
   ...recurringRewards,
   ...hiring,
   ...workingGroup,
-  ...discovery,
   ...storage,
   ...proposals,
   ...contentDirectory,

types/src/legacy.ts

@@ -89,6 +89,10 @@ export class ReferenceConstraint extends Null {}
 export class InputEntityValuesMap extends Null {}
 export class FailedAt extends Null {}
 
+// From discovery_service
+export class IPNSIdentity extends Null {}
+export class ServiceProviderRecord extends Null {}
+
 export const legacyTypes: RegistryTypes = {
   ChannelContentType,
   ChannelCurationStatus,
@@ -159,6 +163,8 @@ export const legacyTypes: RegistryTypes = {
   Operation,
   ReferenceConstraint,
   FailedAt,
+  IPNSIdentity,
+  ServiceProviderRecord,
 }
 
 export default legacyTypes

types/src/scripts/generateAugmentCodec.ts

@@ -18,7 +18,6 @@ import mint from '../mint'
 import recurringRewards from '../recurring-rewards'
 import hiring from '../hiring'
 import workingGroup from '../working-group'
-import discovery from '../discovery'
 import storage from '../storage'
 import proposals from '../proposals'
 import contentDirectory from '../content'
@@ -41,7 +40,6 @@ const typesByModule = {
   'recurring-rewards': recurringRewards,
   'hiring': hiring,
   'working-group': workingGroup,
-  'discovery': discovery,
   'storage': storage,
   'proposals': proposals,
   'content': contentDirectory,

yarn.lock

@@ -9672,18 +9672,6 @@ configstore@^3.0.0:
     write-file-atomic "^2.0.0"
     xdg-basedir "^3.0.0"
 
-configstore@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/configstore/-/configstore-4.0.0.tgz#5933311e95d3687efb592c528b922d9262d227e7"
-  integrity sha512-CmquAXFBocrzaSM8mtGPMM/HiWmyIpr4CcJl/rgY2uCObZ/S7cKU0silxslqJejl+t/T9HS8E0PUNQD81JGUEQ==
-  dependencies:
-    dot-prop "^4.1.0"
-    graceful-fs "^4.1.2"
-    make-dir "^1.0.0"
-    unique-string "^1.0.0"
-    write-file-atomic "^2.0.0"
-    xdg-basedir "^3.0.0"
-
 configstore@^5.0.1:
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/configstore/-/configstore-5.0.1.tgz#d365021b5df4b98cdd187d6a3b0e3f6a7cc5ed96"