Merge branch 'nicaea' into working_group_proposals

.github/workflows/joystream-cli.yml

@@ -0,0 +1,39 @@
+name: joystream-cli
+on: [pull_request, push]
+
+jobs:
+  cli_build_ubuntu:
+    name: Ubuntu Build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [12.x]
+    steps:
+    - uses: actions/checkout@v1
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: build
+      run: |
+        yarn install --frozen-lockfile
+        yarn madge --circular types/
+        yarn workspace joystream-cli build
+
+  cli_build_osx:
+    name: MacOS Build
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        node-version: [12.x]
+    steps:
+    - uses: actions/checkout@v1
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: build
+      run: |
+        yarn install --frozen-lockfile --network-timeout 120000
+        yarn madge --circular types/
+        yarn workspace joystream-cli build

.github/workflows/pioneer-pr.yml → .github/workflows/pioneer.yml

@@ -17,6 +17,7 @@ jobs:
     - name: build
       run: |
         yarn install --frozen-lockfile
+        yarn madge --circular types/
         yarn workspace pioneer build
 
   pioneer_build_osx:
@@ -33,7 +34,8 @@ jobs:
         node-version: ${{ matrix.node-version }}
     - name: build
       run: |
-        yarn install --frozen-lockfile
+        yarn install --frozen-lockfile --network-timeout 120000
+        yarn madge --circular types/
         yarn workspace pioneer build
 
   pioneer_lint_ubuntu:
@@ -51,6 +53,7 @@ jobs:
     - name: lint
       run: |
         yarn install --frozen-lockfile
+        yarn madge --circular types/
         yarn workspace pioneer lint
 
   pioneer_lint_osx:
@@ -67,5 +70,6 @@ jobs:
         node-version: ${{ matrix.node-version }}
     - name: lint
       run: |
-        yarn install --frozen-lockfile
+        yarn install --frozen-lockfile --network-timeout 120000
+        yarn madge --circular types/
         yarn workspace pioneer lint

Cargo.lock

@@ -1569,7 +1569,7 @@ dependencies = [
 
 [[package]]
 name = "joystream-node"
-version = "2.4.0"
+version = "2.4.1"
 dependencies = [
  "ctrlc",
  "derive_more 0.14.1",

README.md

@@ -52,7 +52,7 @@ cargo build --release
 Run the node and connect to the public testnet.
 
 ```bash
-cargo run --release -- --chain ./rome-tesnet.json
+cargo run --release -- --chain ./rome-testnet.json
 ```
 
 The `rome-testnet.json` chain file can be obtained from the [releases page](https://github.com/Joystream/joystream/releases/tag/v6.8.0)
@@ -68,7 +68,7 @@ cargo install joystream-node --path node/
 Now you can run
 
 ```bash
-joystream-node --chain rome-testnet.json
+joystream-node --chain ./rome-testnet.json
 ```
 
 ### Local development
@@ -149,15 +149,15 @@ cargo-fmt
 
 ## Contributing
 
-Please see our [contributing guidlines](https://github.com/Joystream/joystream#contribute) for details on our code of conduct, and the process for submitting pull requests to us.
+Please see our [contributing guidlines](./CONTRIBUTING.md) for details on our code of conduct, and the process for submitting pull requests to us.
 
 ## Authors
 
-See also the list of [CONTRIBUTORS](./CONTRIBUTORS) who participated in this project.
+See also the list of [CONTRIBUTORS](https://github.com/Joystream/joystream/graphs/contributors) who participated in this project.
 
 ## License
 
-This project is licensed under the GPLv3 License - see the [LICENSE](LICENSE) file for details
+This project is licensed under the GPLv3 License - see the [LICENSE](./LICENSE) file for details
 
 ## Acknowledgments
 

cli/src/Api.ts

@@ -18,10 +18,10 @@ import {
 import { DerivedFees, DerivedBalances } from '@polkadot/api-derive/types';
 import { CLIError } from '@oclif/errors';
 import ExitCodes from './ExitCodes';
-import { Worker, Lead as WorkerLead, WorkerId, WorkerRoleStakeProfile } from '@joystream/types/lib/bureaucracy';
-import { MemberId, Profile } from '@joystream/types/lib/members';
-import { RewardRelationship, RewardRelationshipId } from '@joystream/types/lib/recurring-rewards';
-import { Stake, StakeId } from '@joystream/types/lib/stake';
+import { Worker, Lead as WorkerLead, WorkerId, WorkerRoleStakeProfile } from '@joystream/types/working-group';
+import { MemberId, Profile } from '@joystream/types/members';
+import { RewardRelationship, RewardRelationshipId } from '@joystream/types/recurring-rewards';
+import { Stake, StakeId } from '@joystream/types/stake';
 import { LinkageResult } from '@polkadot/types/codec/Linkage';
 
 export const DEFAULT_API_URI = 'wss://rome-rpc-endpoint.joystream.org:9944/';
@@ -29,7 +29,7 @@ const DEFAULT_DECIMALS = new u32(12);
 
 // Mapping of working group to api module
 const apiModuleByGroup: { [key in WorkingGroups]: string } = {
-    [WorkingGroups.StorageProviders]: 'storageBureaucracy'
+    [WorkingGroups.StorageProviders]: 'storageWorkingGroup'
 };
 
 // Api wrapper for handling most common api calls and allowing easy API implementation switch in the future

cli/src/Types.ts

@@ -1,11 +1,11 @@
 import BN from 'bn.js';
-import { ElectionStage, Seat } from '@joystream/types/lib/council';
+import { ElectionStage, Seat } from '@joystream/types/council';
 import { Option } from '@polkadot/types';
 import { BlockNumber, Balance, AccountId } from '@polkadot/types/interfaces';
 import { DerivedBalances } from '@polkadot/api-derive/types';
 import { KeyringPair } from '@polkadot/keyring/types';
-import { WorkerId, Lead } from '@joystream/types/lib/bureaucracy';
-import { Profile, MemberId } from '@joystream/types/lib/members';
+import { WorkerId, Lead } from '@joystream/types/working-group';
+import { Profile, MemberId } from '@joystream/types/members';
 
 // KeyringPair type extended with mandatory "meta.name"
 // It's used for accounts/keys management within CLI.

cli/src/commands/council/info.ts

@@ -1,4 +1,4 @@
-import { ElectionStage } from '@joystream/types/lib/council';
+import { ElectionStage } from '@joystream/types/council';
 import { formatNumber, formatBalance } from '@polkadot/util';
 import { BlockNumber } from '@polkadot/types/interfaces';
 import { CouncilInfoObj, NameValueObj } from '../../Types';

node/Cargo.toml

@@ -3,7 +3,7 @@ authors = ['Joystream']
 build = 'build.rs'
 edition = '2018'
 name = 'joystream-node'
-version = '2.4.0'
+version = '2.4.1'
 default-run = "joystream-node"
 
 [[bin]]

node/src/chain_spec.rs

@@ -257,7 +257,7 @@ pub fn testnet_genesis(
         }),
         members: Some(MembersConfig {
             default_paid_membership_fee: 100u128,
-            members: crate::members_config::initial_members(),
+            members: vec![],
         }),
         forum: Some(crate::forum_config::from_serialized::create(
             endowed_accounts[0].clone(),

node/src/forum_config/from_serialized.rs

@@ -19,7 +19,7 @@ struct ForumData {
 }
 
 fn parse_forum_json() -> Result<ForumData> {
-    let data = include_str!("../../res/forum_data_acropolis_serialized.json");
+    let data = include_str!("../../res/forum_data_empty.json");
     serde_json::from_str(data)
 }
 

package.json

@@ -14,7 +14,9 @@
 		"cli",
 		"types",
 		"pioneer",
-		"pioneer/packages/*"
+		"pioneer/packages/*",
+		"storage-node/",
+		"storage-node/packages/*"
 	],
 	"resolutions": {
 		"@polkadot/api": "^0.96.1",

pioneer/packages/app-accounts/src/Overview.tsx

@@ -7,6 +7,8 @@ import { SubjectInfo } from '@polkadot/ui-keyring/observable/types';
 import { ComponentProps } from './types';
 
 import React, { useState } from 'react';
+import { Link, useLocation } from 'react-router-dom';
+import { Button as SUIButton } from 'semantic-ui-react';
 import keyring from '@polkadot/ui-keyring';
 import accountObservable from '@polkadot/ui-keyring/observable/accounts';
 import { getLedger, isLedger, withMulti, withObservable } from '@polkadot/react-api';
@@ -35,6 +37,7 @@ async function queryLedger (): Promise<void> {
 }
 
 function Overview ({ accounts, onStatusChange, t }: Props): React.ReactElement<Props> {
+  const { pathname } = useLocation();
   const [isCreateOpen, setIsCreateOpen] = useState(false);
   const [isImportOpen, setIsImportOpen] = useState(false);
   const emptyScreen = !(isCreateOpen || isImportOpen) && accounts && (Object.keys(accounts).length === 0);
@@ -44,6 +47,9 @@ function Overview ({ accounts, onStatusChange, t }: Props): React.ReactElement<P
 
   return (
     <CardGrid
+      topButtons={
+        !emptyScreen && <SUIButton as={Link} to={`${pathname}/vanity`}>Generate a vanity address</SUIButton>
+      }
       buttons={
         <Button.Group>
           <Button

pioneer/packages/app-accounts/src/Vanity/index.tsx

@@ -18,6 +18,7 @@ import matchRegex from '../vanitygen/regex';
 import generatorSort from '../vanitygen/sort';
 import Match from './Match';
 import translate from './translate';
+import Section from '@polkadot/joy-utils/Section';
 
 interface Props extends ComponentProps, I18nProps {}
 
@@ -42,6 +43,10 @@ const BOOL_OPTIONS = [
   { text: 'Yes', value: true }
 ];
 
+const SectionContentWrapper = styled.div`
+  margin-left: -2rem;
+`;
+
 class VanityApp extends TxComponent<Props, State> {
   private results: GeneratorResult[] = [];
 
@@ -72,18 +77,22 @@ class VanityApp extends TxComponent<Props, State> {
 
     return (
       <div className={className}>
-        {this.renderOptions()}
-        {this.renderButtons()}
-        {this.renderStats()}
-        {this.renderMatches()}
-        {createSeed && (
-          <CreateModal
-            onClose={this.closeCreate}
-            onStatusChange={onStatusChange}
-            seed={createSeed}
-            type={type}
-          />
-        )}
+        <Section title="Generate a vanity address">
+          <SectionContentWrapper>
+            {this.renderOptions()}
+            {this.renderButtons()}
+            {this.renderStats()}
+            {this.renderMatches()}
+            {createSeed && (
+              <CreateModal
+                onClose={this.closeCreate}
+                onStatusChange={onStatusChange}
+                seed={createSeed}
+                type={type}
+              />
+            )}
+          </SectionContentWrapper>
+        </Section>
       </div>
     );
   }

pioneer/packages/app-accounts/src/index.tsx

@@ -62,10 +62,6 @@ function AccountsApp ({ allAccounts = {}, basePath, location, onStatusChange, t
               name: 'overview',
               text: t('My accounts')
             },
-            {
-              name: 'vanity',
-              text: t('Vanity address')
-            },
             {
               name: 'memo',
               text: t('My memo')

pioneer/packages/app-address-book/src/Overview.tsx

@@ -7,7 +7,9 @@ import { SubjectInfo } from '@polkadot/ui-keyring/observable/types';
 import { ComponentProps } from './types';
 
 import React, { useState } from 'react';
+import { Link, useLocation } from 'react-router-dom';
 import { Button, CardGrid } from '@polkadot/react-components';
+import { Button as SUIButton, Icon } from 'semantic-ui-react';
 import addressObservable from '@polkadot/ui-keyring/observable/addresses';
 import { withMulti, withObservable } from '@polkadot/react-api';
 
@@ -20,6 +22,7 @@ interface Props extends ComponentProps, I18nProps {
 }
 
 function Overview ({ addresses, onStatusChange, t }: Props): React.ReactElement<Props> {
+  const { pathname } = useLocation();
   const [isCreateOpen, setIsCreateOpen] = useState(false);
   const emptyScreen = !isCreateOpen && (!addresses || Object.keys(addresses).length === 0);
 
@@ -27,15 +30,19 @@ function Overview ({ addresses, onStatusChange, t }: Props): React.ReactElement<
 
   return (
     <CardGrid
+      topButtons={
+        <SUIButton as={Link} to={`${pathname}/memo`}>
+          <Icon name="search" />
+          View memo
+        </SUIButton>
+      }
       buttons={
-        <Button.Group>
-          <Button
-            icon='add'
-            isPrimary
-            label={t('Add contact')}
-            onClick={_toggleCreate}
-          />
-        </Button.Group>
+        <Button
+          icon='add'
+          isPrimary
+          label={t('Add contact')}
+          onClick={_toggleCreate}
+        />
       }
       isEmpty={emptyScreen}
       emptyText={t('No contacts found.')}

pioneer/packages/app-address-book/src/index.tsx

@@ -8,8 +8,10 @@ import { ComponentProps } from './types';
 
 import React from 'react';
 import { Route, Switch } from 'react-router';
+import { Link } from 'react-router-dom';
+import styled from 'styled-components';
+import { Breadcrumb } from 'semantic-ui-react';
 import { HelpOverlay } from '@polkadot/react-components';
-import Tabs from '@polkadot/react-components/Tabs';
 
 import basicMd from './md/basic.md';
 import Overview from './Overview';
@@ -21,7 +23,16 @@ interface Props extends AppProps, I18nProps {
   location: any;
 }
 
-function AddressBookApp ({ basePath, onStatusChange, t }: Props): React.ReactElement<Props> {
+const StyledHeader = styled.header`
+  text-align: left;
+
+  .ui.breadcrumb {
+    padding: 1.4rem 0 0 .4rem;
+    font-size: 1.4rem;
+  }
+`;
+
+function AddressBookApp ({ basePath, onStatusChange }: Props): React.ReactElement<Props> {
   const _renderComponent = (Component: React.ComponentType<ComponentProps>): () => React.ReactNode => {
     // eslint-disable-next-line react/display-name
     return (): React.ReactNode =>
@@ -32,27 +43,27 @@ function AddressBookApp ({ basePath, onStatusChange, t }: Props): React.ReactEle
       />;
   };
 
+  const viewMemoPath = `${basePath}/memo/:accountId?`;
+
   return (
     <main className='address-book--App'>
       <HelpOverlay md={basicMd} />
-      <header>
-        <Tabs
-          basePath={basePath}
-          items={[
-            {
-              isRoot: true,
-              name: 'overview',
-              text: t('My contacts')
-            },
-            {
-              name: 'memo',
-              text: t('View memo')
-            }
-          ]}
-        />
-      </header>
+      <StyledHeader>
+        <Breadcrumb>
+          <Switch>
+            <Route path={viewMemoPath}>
+              <Breadcrumb.Section link as={Link} to={basePath}>Contacts</Breadcrumb.Section>
+              <Breadcrumb.Divider icon="right angle" />
+              <Breadcrumb.Section active>View memo</Breadcrumb.Section>
+            </Route>
+            <Route>
+              <Breadcrumb.Section active>Contacts</Breadcrumb.Section>
+            </Route>
+          </Switch>
+        </Breadcrumb>
+      </StyledHeader>
       <Switch>
-        <Route path={`${basePath}/memo/:accountId?`} component={MemoByAccount} />
+        <Route path={viewMemoPath} component={MemoByAccount} />
         <Route render={_renderComponent(Overview)} />
       </Switch>
     </main>

pioneer/packages/apps-routing/src/joy-roles.ts

@@ -7,7 +7,8 @@ export default ([
     Component: Roles,
     display: {
       needsApi: [
-        'query.actors.actorAccountIds'
+        'query.contentWorkingGroup.mint',
+        'query.storageWorkingGroup.mint'
       ]
     },
     i18n: {

pioneer/packages/joy-election/src/SealedVotes.tsx

@@ -1,4 +1,6 @@
 import React from 'react';
+import { Link } from 'react-router-dom';
+import { Button } from 'semantic-ui-react';
 
 import { I18nProps } from '@polkadot/react-components/types';
 import { ApiProps } from '@polkadot/react-api/types';
@@ -42,11 +44,14 @@ class Comp extends React.PureComponent<Props> {
           ? <em>No votes by the current account found on the current browser.</em>
           : this.renderVotes(myVotes)
       }</Section>
-      <Section title={`Other votes (${otherVotes.length})`}>{
-        !otherVotes.length
-          ? <em>No votes submitted by other accounts yet.</em>
-          : this.renderVotes(otherVotes)
-      }</Section>
+      <Section title={`Other votes (${otherVotes.length})`}>
+        <Button primary as={Link} to="reveals">Reveal a vote</Button>
+        {
+          !otherVotes.length
+            ? <em>No votes submitted by other accounts yet.</em>
+            : this.renderVotes(otherVotes)
+        }
+      </Section>
     </>;
   }
 }

pioneer/packages/joy-election/src/index.tsx

@@ -51,10 +51,6 @@ class App extends React.PureComponent<Props, State> {
       {
         name: 'votes',
         text: t('Votes') + ` (${commitments.length})`
-      },
-      {
-        name: 'reveals',
-        text: t('Reveal a vote')
       }
     ];
   }

pioneer/packages/joy-media/src/DiscoveryProvider.tsx

@@ -2,7 +2,7 @@ import React, { useState, useEffect, useContext, createContext } from 'react';
 import { Message } from 'semantic-ui-react';
 import axios, { CancelToken } from 'axios';
 
-import { StorageProviderId } from '@joystream/types/bureaucracy';
+import { StorageProviderId } from '@joystream/types/working-group';
 import { Vec } from '@polkadot/types';
 import { Url } from '@joystream/types/discovery';
 import ApiContext from '@polkadot/react-api/ApiContext';

pioneer/packages/joy-media/src/Upload.tsx

@@ -15,7 +15,7 @@ import { formatNumber } from '@polkadot/util';
 import translate from './translate';
 import { fileNameWoExt } from './utils';
 import { ContentId, DataObject } from '@joystream/types/media';
-import { withMembershipRequired } from '@polkadot/joy-utils/MyAccount';
+import { withOnlyMembers, MyAccountProps } from '@polkadot/joy-utils/MyAccount';
 import { DiscoveryProviderProps, withDiscoveryProvider } from './DiscoveryProvider';
 import TxButton from '@polkadot/joy-utils/TxButton';
 import IpfsHash from 'ipfs-only-hash';
@@ -23,12 +23,12 @@ import { ChannelId } from '@joystream/types/content-working-group';
 import { EditVideoView } from './upload/EditVideo.view';
 import { JoyInfo } from '@polkadot/joy-utils/JoyStatus';
 import { IterableFile } from './IterableFile';
-import { StorageProviderId } from '@joystream/types/bureaucracy';
+import { StorageProviderId } from '@joystream/types/working-group';
 
 const MAX_FILE_SIZE_MB = 500;
 const MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024;
 
-type Props = ApiProps & I18nProps & DiscoveryProviderProps & {
+type Props = ApiProps & I18nProps & DiscoveryProviderProps & MyAccountProps & {
   channelId: ChannelId;
   history?: History;
   match: {
@@ -62,7 +62,7 @@ const defaultState = (): State => ({
   cancelSource: axios.CancelToken.source()
 });
 
-class Component extends React.PureComponent<Props, State> {
+class Upload extends React.PureComponent<Props, State> {
   state = defaultState();
 
   componentWillUnmount () {
@@ -248,8 +248,8 @@ class Component extends React.PureComponent<Props, State> {
 
     // TODO get corresponding data type id based on file content
     const dataObjectTypeId = new BN(1);
-
-    return [newContentId, dataObjectTypeId, new BN(file.size), ipfs_cid];
+    const { myMemberId } = this.props;
+    return [myMemberId, newContentId, dataObjectTypeId, new BN(file.size), ipfs_cid];
   }
 
   private onDataObjectCreated = async (_txResult: SubmittableResult) => {
@@ -349,9 +349,9 @@ class Component extends React.PureComponent<Props, State> {
 }
 
 export const UploadWithRouter = withMulti(
-  Component,
+  Upload,
   translate,
   withApi,
-  withMembershipRequired,
+  withOnlyMembers,
   withDiscoveryProvider
 );

pioneer/packages/joy-media/src/channels/EditChannel.tsx

@@ -17,6 +17,7 @@ import { TxCallback } from '@polkadot/react-components/Status/types';
 import { SubmittableResult } from '@polkadot/api';
 import { ChannelValidationConstraints } from '../transport';
 import { JoyError } from '@polkadot/joy-utils/JoyStatus';
+import Section from '@polkadot/joy-utils/Section';
 
 export type OuterProps = {
   history?: History;
@@ -176,21 +177,23 @@ const InnerForm = (props: MediaFormProps<OuterProps, FormValues>) => {
       {avatar && <img src={avatar} onError={onImageError} />}
     </div>
 
-    <Form className='ui form JoyForm EditMetaForm'>
-
-      {formFields()}
-
-      <LabelledField style={{ marginTop: '1rem' }} {...props}>
-        {renderMainButton()}
-        <Button
-          type='button'
-          size='large'
-          disabled={!dirty || isSubmitting}
-          onClick={() => resetForm()}
-          content='Reset form'
-        />
-      </LabelledField>
-    </Form>
+    <Section title={isNew ? 'Create a channel' : 'Edit a channel'}>
+      <Form className='ui form JoyForm EditMetaForm'>
+
+        {formFields()}
+
+        <LabelledField style={{ marginTop: '1rem' }} {...props}>
+          {renderMainButton()}
+          <Button
+            type='button'
+            size='large'
+            disabled={!dirty || isSubmitting}
+            onClick={() => resetForm()}
+            content='Reset form'
+          />
+        </LabelledField>
+      </Form>
+    </Section>
   </div>;
 };
 

pioneer/packages/joy-media/src/common/MediaPlayerWithResolver.tsx

@@ -6,7 +6,7 @@ import { ApiProps } from '@polkadot/react-api/types';
 import { I18nProps } from '@polkadot/react-components/types';
 import { withMulti } from '@polkadot/react-api/with';
 import { Option } from '@polkadot/types/codec';
-import { StorageProviderId, Worker } from '@joystream/types/bureaucracy';
+import { StorageProviderId, Worker } from '@joystream/types/working-group';
 
 import translate from '../translate';
 import { DiscoveryProviderProps, withDiscoveryProvider } from '../DiscoveryProvider';
@@ -31,7 +31,7 @@ function InnerComponent (props: Props) {
   const [cancelSource, setCancelSource] = useState<CancelTokenSource>(newCancelSource());
 
   const getActiveStorageProviderIds = async (): Promise<StorageProviderId[]> => {
-    const nextId = await api.query.storageBureaucracy.nextWorkerId() as StorageProviderId;
+    const nextId = await api.query.storageWorkingGroup.nextWorkerId() as StorageProviderId;
     // This is chain specfic, but if next id is still 0, it means no workers have been added,
     // so the workerById is empty
     if (nextId.eq(0)) {
@@ -41,7 +41,7 @@ function InnerComponent (props: Props) {
     const workers = new MultipleLinkedMapEntry<StorageProviderId, Worker>(
       StorageProviderId,
       Worker,
-      await api.query.storageBureaucracy.workerById()
+      await api.query.storageWorkingGroup.workerById()
     );
 
     return workers.linked_keys;

pioneer/packages/joy-media/src/index.css

@@ -124,7 +124,7 @@
     }
   }
 
-  .EditMetaForm {
+  .JoySection {
     width: 100%;
     max-width: 600px;
   }

pioneer/packages/joy-media/src/index.tsx

@@ -40,10 +40,6 @@ function App (props: Props) {
     !myAddress ? undefined : {
       name: `account/${myAddress}/channels`,
       text: t('My channels')
-    },
-    {
-      name: 'channels/new',
-      text: t('New channel')
     }
     // !myAddress ? undefined : {
     //   name: `account/${myAddress}/videos`,

pioneer/packages/joy-members/src/index.tsx

@@ -37,7 +37,7 @@ class App extends React.PureComponent<Props> {
       },
       {
         name: 'edit',
-        text: iAmMember ? t('Edit my profile') : t('Register')
+        text: iAmMember ? 'My profile' : t('Register')
       },
       {
         name: 'dashboard',

pioneer/packages/joy-proposals/src/Proposal/ProposalPreviewList.tsx

@@ -1,5 +1,7 @@
 import React, { useState } from 'react';
-import { Card, Container, Menu } from 'semantic-ui-react';
+import { Button, Card, Container, Icon } from 'semantic-ui-react';
+import styled from 'styled-components';
+import { Link, useLocation } from 'react-router-dom';
 
 import ProposalPreview from './ProposalPreview';
 import { ParsedProposal } from '@polkadot/joy-utils/types/proposals';
@@ -7,6 +9,7 @@ import { useTransport, usePromise } from '@polkadot/joy-utils/react/hooks';
 import { PromiseComponent } from '@polkadot/joy-utils/react/components';
 import { withCalls } from '@polkadot/react-api';
 import { BlockNumber } from '@polkadot/types/interfaces';
+import { Dropdown } from '@polkadot/react-components';
 
 const filters = ['All', 'Active', 'Canceled', 'Approved', 'Rejected', 'Slashed', 'Expired'] as const;
 
@@ -50,31 +53,84 @@ type ProposalPreviewListProps = {
   bestNumber?: BlockNumber;
 };
 
+const FilterContainer = styled.div`
+  display: flex;
+  align-items: flex-start;
+  justify-content: space-between;
+  margin-bottom: 1.75rem;
+`;
+const FilterOption = styled.span`
+  display: inline-flex;
+  align-items: center;
+`;
+const ProposalFilterCountBadge = styled.span`
+  background-color: rgba(0, 0, 0, .3);
+  color: #fff;
+
+  border-radius: 10px;
+  height: 19px;
+  min-width: 19px;
+  padding: 0 4px;
+
+  font-size: .8rem;
+  font-weight: 500;
+  line-height: 1;
+
+  display: flex;
+  align-items: center;
+  justify-content: center;
+
+  margin-left: 6px;
+`;
+const StyledDropdown = styled(Dropdown)`
+  .dropdown {
+    width: 200px;
+  }
+`;
+
 function ProposalPreviewList ({ bestNumber }: ProposalPreviewListProps) {
+  const { pathname } = useLocation();
   const transport = useTransport();
   const [proposals, error, loading] = usePromise<ParsedProposal[]>(() => transport.proposals.proposals(), []);
   const [activeFilter, setActiveFilter] = useState<ProposalFilter>('All');
 
   const proposalsMap = mapFromProposals(proposals);
   const filteredProposals = proposalsMap.get(activeFilter) as ParsedProposal[];
+  const sortedProposals = filteredProposals.sort((p1, p2) => p2.id.cmp(p1.id));
+
+  const filterOptions = filters.map(filter => ({
+    text: (
+      <FilterOption>
+        {filter}
+        <ProposalFilterCountBadge>{(proposalsMap.get(filter) as ParsedProposal[]).length}</ProposalFilterCountBadge>
+      </FilterOption>
+    ),
+    value: filter
+  }));
+
+  const _onChangePrefix = (f: ProposalFilter) => setActiveFilter(f);
 
   return (
     <Container className="Proposal" fluid>
+      <FilterContainer>
+        <Button primary as={Link} to={`${pathname}/new`}>
+          <Icon name="add" />
+          New proposal
+        </Button>
+        {!loading && (
+          <StyledDropdown
+            label="Proposal state"
+            options={filterOptions}
+            value={activeFilter}
+            onChange={_onChangePrefix}
+          />
+        )}
+      </FilterContainer>
       <PromiseComponent error={ error } loading={ loading } message="Fetching proposals...">
-        <Menu tabular className="list-menu">
-          {filters.map((filter, idx) => (
-            <Menu.Item
-              key={`${filter} - ${idx}`}
-              name={`${filter.toLowerCase()} - ${(proposalsMap.get(filter) as ParsedProposal[]).length}`}
-              active={activeFilter === filter}
-              onClick={() => setActiveFilter(filter)}
-            />
-          ))}
-        </Menu>
         {
-          filteredProposals.length ? (
+          sortedProposals.length ? (
             <Card.Group>
-              {filteredProposals.map((prop: ParsedProposal, idx: number) => (
+              {sortedProposals.map((prop: ParsedProposal, idx: number) => (
                 <ProposalPreview key={`${prop.title}-${idx}`} proposal={prop} bestNumber={bestNumber} />
               ))}
             </Card.Group>

pioneer/packages/joy-proposals/src/index.tsx

@@ -1,8 +1,10 @@
 import React from 'react';
 import { Route, Switch } from 'react-router';
+import { Link } from 'react-router-dom';
+import styled from 'styled-components';
+import { Breadcrumb } from 'semantic-ui-react';
 
 import { AppProps, I18nProps } from '@polkadot/react-components/types';
-import Tabs, { TabItem } from '@polkadot/react-components/Tabs';
 import { TransportProvider } from '@polkadot/joy-utils/react/context';
 import { ProposalPreviewList, ProposalFromId, ChooseProposalType } from './Proposal';
 
@@ -24,27 +26,35 @@ import {
 
 interface Props extends AppProps, I18nProps {}
 
-function App (props: Props): React.ReactElement<Props> {
-  const { t, basePath } = props;
+const StyledHeader = styled.header`
+  text-align: left;
+
+  .ui.breadcrumb {
+    padding: 1.4rem 0 0 .4rem;
+    font-size: 1.4rem;
+  }
+`;
 
-  const tabs: TabItem[] = [
-    {
-      isRoot: true,
-      name: 'proposals',
-      text: t('Proposals')
-    },
-    {
-      name: 'new',
-      text: t('New Proposal')
-    }
-  ];
+function App (props: Props): React.ReactElement<Props> {
+  const { basePath } = props;
 
   return (
     <TransportProvider>
       <main className="proposal--App">
-        <header>
-          <Tabs basePath={basePath} items={tabs} />
-        </header>
+        <StyledHeader>
+          <Breadcrumb>
+            <Switch>
+              <Route path={`${basePath}/new`}>
+                <Breadcrumb.Section link as={Link} to={basePath}>Proposals</Breadcrumb.Section>
+                <Breadcrumb.Divider icon="right angle" />
+                <Breadcrumb.Section active>New proposal</Breadcrumb.Section>
+              </Route>
+              <Route>
+                <Breadcrumb.Section active>Proposals</Breadcrumb.Section>
+              </Route>
+            </Switch>
+          </Breadcrumb>
+        </StyledHeader>
         <Switch>
           <Route exact path={`${basePath}/new`} component={ChooseProposalType} />
           <Route exact path={`${basePath}/new/text`} component={SignalForm} />

pioneer/packages/joy-roles/src/elements.stories.tsx

@@ -1,84 +0,0 @@
-// @ts-nocheck
-import React from 'react';
-import { boolean, number, text, withKnobs } from '@storybook/addon-knobs';
-import { Table } from 'semantic-ui-react';
-
-import { u128, Text } from '@polkadot/types';
-
-import { Actor } from '@joystream/types/roles';
-
-import { BalanceView, GroupMemberView, HandleView, MemberView, MemoView } from './elements';
-
-import 'semantic-ui-css/semantic.min.css';
-import '@polkadot/joy-roles/index.sass';
-
-export default {
-  title: 'Roles / Elements',
-  decorators: [withKnobs]
-};
-
-export const Balance = () => {
-  return (
-    <BalanceView balance={new u128(number('Balance', 10))} />
-  );
-};
-
-export const Memo = () => {
-  const actor = new Actor({ member_id: 1, account: '5HZ6GtaeyxagLynPryM7ZnmLzoWFePKuDrkb4AT8rT4pU1fp' });
-  const memo = new Text(text('Memo text', 'This is a memo'));
-
-  return (
-    <MemoView actor={actor} memo={memo} />
-  );
-};
-
-export const Handle = () => {
-  const profile = {
-    handle: new Text(text('Handle', 'benholdencrowther'))
-  };
-
-  return (
-    <HandleView profile={profile} />
-  );
-};
-
-export const Member = () => {
-  const actor = new Actor({ member_id: 1, account: '5HZ6GtaeyxagLynPryM7ZnmLzoWFePKuDrkb4AT8rT4pU1fp' });
-  const profile = {
-    handle: new Text(text('Handle', 'benholdencrowther'))
-  };
-
-  return (
-    <Table basic='very'>
-      <Table.Body>
-        <Table.Row>
-          <Table.Cell>
-            <MemberView
-              actor={actor}
-              balance={new u128(number('Balance', 10))}
-              profile={profile}
-            />
-          </Table.Cell>
-        </Table.Row>
-      </Table.Body>
-    </Table>
-  );
-};
-
-export const GroupMember = () => {
-  const actor = new Actor({ member_id: 1, account: '5HZ6GtaeyxagLynPryM7ZnmLzoWFePKuDrkb4AT8rT4pU1fp' });
-  const profile = {
-    handle: new Text(text('Handle', 'benholdencrowther'))
-  };
-
-  return (
-    <GroupMemberView
-      actor={actor}
-      profile={profile}
-      title={text('Title', 'Group lead')}
-      lead={boolean('Lead member', true)}
-      stake={new u128(number('Stake', 10))}
-      earned={new u128(number('Earned', 10))}
-    />
-  );
-};

pioneer/packages/joy-roles/src/elements.tsx

@@ -1,20 +1,15 @@
 import React, { useEffect, useState } from 'react';
 import moment from 'moment';
-import { Header, Card, Icon, Image, Label, Statistic } from 'semantic-ui-react';
+import { Card, Icon, Image, Label, Statistic } from 'semantic-ui-react';
 import { Link } from 'react-router-dom';
 
 import { Balance } from '@polkadot/types/interfaces';
 import { formatBalance } from '@polkadot/util';
 import Identicon from '@polkadot/react-identicon';
-import { Actor } from '@joystream/types/roles';
 import { IProfile, MemberId } from '@joystream/types/members';
-import { Text, GenericAccountId } from '@polkadot/types';
+import { GenericAccountId } from '@polkadot/types';
 import { LeadRoleState } from '@joystream/types/content-working-group';
 
-type ActorProps = {
-  actor: Actor;
-}
-
 type BalanceProps = {
   balance?: Balance;
 }
@@ -27,23 +22,6 @@ export function BalanceView (props: BalanceProps) {
   );
 }
 
-type MemoProps = ActorProps & {
-  memo?: Text;
-}
-
-export function MemoView (props: MemoProps) {
-  if (typeof props.memo === 'undefined') {
-    return null;
-  }
-
-  return (
-    <div className="memo">
-      <span>Memo:</span> {props.memo.toString()}
-      <Link to={`/addressbook/memo/${props.actor.account.toString()}`}>{' view full memo'}</Link>
-    </div>
-  );
-}
-
 type ProfileProps = {
   profile: IProfile;
 }
@@ -58,36 +36,6 @@ export function HandleView (props: ProfileProps) {
   );
 }
 
-type MemberProps = ActorProps & BalanceProps & ProfileProps
-
-export function MemberView (props: MemberProps) {
-  let avatar = <Identicon value={props.actor.account.toString()} size={50} />;
-  if (typeof props.profile.avatar_uri !== 'undefined' && props.profile.avatar_uri.toString() !== '') {
-    avatar = <Image src={props.profile.avatar_uri.toString()} circular className='avatar' />;
-  }
-
-  return (
-    <Header as='h4' image>
-      {avatar}
-      <Header.Content>
-        <HandleView profile={props.profile} />
-        <BalanceView balance={props.balance} />
-      </Header.Content>
-    </Header>
-  );
-}
-
-type ActorDetailsProps = MemoProps & BalanceProps
-
-export function ActorDetailsView (props: ActorDetailsProps) {
-  return (
-    <div className="actor-summary" id={props.actor.account.toString()}>
-      {props.actor.account.toString()}
-      <MemoView actor={props.actor} memo={props.memo} />
-    </div>
-  );
-}
-
 export type GroupMember = {
   memberId: MemberId;
   roleAccount: GenericAccountId;

pioneer/packages/joy-roles/src/transport.substrate.ts

@@ -29,7 +29,7 @@ import {
   Worker, WorkerId,
   WorkerRoleStakeProfile,
   Lead as LeadOf
-} from '@joystream/types/bureaucracy';
+} from '@joystream/types/working-group';
 
 import { Application, Opening, OpeningId } from '@joystream/types/hiring';
 import { Stake, StakeId } from '@joystream/types/stake';
@@ -97,7 +97,7 @@ type WGApiMapping = {
 
 const workingGroupsApiMapping: WGApiMapping = {
   [WorkingGroups.StorageProviders]: {
-    module: 'storageBureaucracy',
+    module: 'storageWorkingGroup',
     methods: {
       nextOpeningId: 'nextWorkerOpeningId',
       openingById: 'workerOpeningById',
@@ -291,7 +291,7 @@ export class Transport extends TransportBase implements ITransport {
   }
 
   protected async currentStorageLead (): Promise <GroupLeadWithMemberId | null> {
-    const optLead = (await this.cachedApi.query.storageBureaucracy.currentLead()) as Option<LeadOf>;
+    const optLead = (await this.cachedApi.query.storageWorkingGroup.currentLead()) as Option<LeadOf>;
 
     if (!optLead.isSome) {
       return null;

pioneer/packages/react-components/src/CardGrid.tsx

@@ -15,7 +15,7 @@ class CardGrid extends Collection<Props, State> {
 
     return {
       ...state,
-      showHeader: !state.isEmpty || !!props.headerText
+      showFullHeader: !state.isEmpty
     };
   }
 

pioneer/packages/react-components/src/Collection.tsx

@@ -9,6 +9,7 @@ import React from 'react';
 export interface CollectionProps extends I18nProps {
   banner?: React.ReactNode;
   buttons?: React.ReactNode;
+  topButtons?: React.ReactNode;
   children: React.ReactNode;
   className?: string;
   headerText?: React.ReactNode;
@@ -19,7 +20,7 @@ export interface CollectionProps extends I18nProps {
 
 export interface CollectionState {
   isEmpty: boolean;
-  showHeader?: boolean;
+  showFullHeader?: boolean;
 }
 
 export const collectionStyles = `
@@ -33,6 +34,12 @@ export const collectionStyles = `
       margin: 0;
       text-transform: lowercase;
     }
+
+    .ui--Collection-buttons {
+      flex: 1;
+      display: flex;
+      justify-content: space-between;
+    }
   }
 
   .ui--Collection-lowercase {
@@ -61,11 +68,11 @@ export default class Collection<P extends CollectionProps, S extends CollectionS
 
   public render (): React.ReactNode {
     const { banner, className } = this.props;
-    const { isEmpty, showHeader } = this.state;
+    const { isEmpty } = this.state;
 
     return (
       <div className={className}>
-        {showHeader && this.renderHeader()}
+        {this.renderHeader()}
         {banner}
         {isEmpty
           ? this.renderEmpty()
@@ -76,18 +83,24 @@ export default class Collection<P extends CollectionProps, S extends CollectionS
   }
 
   protected renderHeader (): React.ReactNode {
-    const { buttons, headerText } = this.props;
+    const { buttons, topButtons, headerText } = this.props;
+    const { showFullHeader } = this.state;
 
-    if (!headerText && !buttons) {
+    if (!headerText && !buttons && !topButtons) {
       return null;
     }
 
     return (
       <div className='ui--Collection-header'>
-        <h1>{headerText}</h1>
-        {buttons && (
+        {headerText && <h1>{headerText}</h1>}
+        {(buttons || topButtons) && (
           <div className='ui--Collection-buttons'>
-            {buttons}
+            <div>
+              {topButtons}
+            </div>
+            <div>
+              {showFullHeader && buttons}
+            </div>
           </div>
         )}
       </div>

runtime-modules/hiring/src/hiring/staking_policy.rs

@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};
 
 /// Policy for staking
 #[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
-#[derive(Encode, Decode, Debug, Eq, PartialEq, Clone)]
+#[derive(Encode, Decode, Debug, Eq, PartialEq, Clone, Default)]
 pub struct StakingPolicy<Balance, BlockNumber> {
     /// Staking amount
     pub amount: Balance,
@@ -77,3 +77,9 @@ pub enum StakingAmountLimitMode {
     /// Stake should be equal to provided value
     Exact,
 }
+
+impl Default for StakingAmountLimitMode {
+    fn default() -> Self {
+        StakingAmountLimitMode::Exact
+    }
+}

runtime-modules/service-discovery/src/mock.rs

@@ -130,8 +130,13 @@ impl recurringrewards::Trait for Test {
     type RewardRelationshipId = u64;
 }
 
+parameter_types! {
+    pub const MaxWorkerNumberLimit: u32 = 3;
+}
+
 impl working_group::Trait<StorageWorkingGroupInstance> for Test {
     type Event = MetaEvent;
+    type MaxWorkerNumberLimit = MaxWorkerNumberLimit;
 }
 
 impl timestamp::Trait for Test {
@@ -158,7 +163,7 @@ pub(crate) fn hire_storage_provider() -> (u64, u64) {
 
     let storage_provider = working_group::Worker {
         member_id: 1,
-        role_account: role_account_id,
+        role_account_id,
         reward_relationship: None,
         role_stake_profile: None,
     };

runtime-modules/storage/src/tests/data_object_type_registry.rs

@@ -1,7 +1,7 @@
 #![cfg(test)]
 
 use super::mock::*;
-use srml_support::StorageValue;
+use srml_support::{StorageLinkedMap, StorageValue};
 use system::{self, EventRecord, Phase, RawOrigin};
 
 const DEFAULT_LEADER_ACCOUNT_ID: u64 = 1;
@@ -11,15 +11,23 @@ const DEFAULT_LEADER_WORKER_ID: u32 = 1;
 struct SetLeadFixture;
 impl SetLeadFixture {
     fn set_default_lead() {
-        // Construct lead
-        let new_lead = working_group::Lead {
+        let worker = working_group::Worker {
             member_id: DEFAULT_LEADER_MEMBER_ID,
             role_account_id: DEFAULT_LEADER_ACCOUNT_ID,
-            worker_id: DEFAULT_LEADER_WORKER_ID,
+            reward_relationship: None,
+            role_stake_profile: None,
         };
 
-        // Update current lead
-        <working_group::CurrentLead<Test, StorageWorkingGroupInstance>>::put(new_lead);
+        // Create the worker.
+        <working_group::WorkerById<Test, StorageWorkingGroupInstance>>::insert(
+            DEFAULT_LEADER_WORKER_ID,
+            worker,
+        );
+
+        // Update current lead.
+        <working_group::CurrentLead<Test, StorageWorkingGroupInstance>>::put(
+            DEFAULT_LEADER_WORKER_ID,
+        );
     }
 }
 

runtime-modules/storage/src/tests/mock.rs

@@ -146,8 +146,13 @@ impl GovernanceCurrency for Test {
     type Currency = balances::Module<Self>;
 }
 
+parameter_types! {
+    pub const MaxWorkerNumberLimit: u32 = 3;
+}
+
 impl working_group::Trait<StorageWorkingGroupInstance> for Test {
     type Event = MetaEvent;
+    type MaxWorkerNumberLimit = MaxWorkerNumberLimit;
 }
 
 impl data_object_type_registry::Trait for Test {
@@ -304,7 +309,7 @@ pub(crate) fn hire_storage_provider() -> (u64, u32) {
 
     let storage_provider = working_group::Worker {
         member_id: 1,
-        role_account: role_account_id,
+        role_account_id,
         reward_relationship: None,
         role_stake_profile: None,
     };

runtime-modules/working-group/src/errors.rs

@@ -17,8 +17,8 @@ decl_error! {
         /// There is leader already, cannot hire another one.
         CannotHireLeaderWhenLeaderExists,
 
-        /// Cannot fill opening with several applications.
-        CannotHireSeveralLeader,
+        /// Cannot fill opening with multiple applications.
+        CannotHireMultipleLeaders,
 
         /// Not a lead account.
         IsNotLeadAccount,
@@ -244,6 +244,9 @@ decl_error! {
 
         /// Require signed origin in extrinsics.
         RequireSignedOrigin,
+
+        /// Working group size limit exceeded.
+        MaxActiveWorkerNumberExceeded,
     }
 }
 

runtime-modules/working-group/src/lib.rs

@@ -57,8 +57,8 @@ use rstd::collections::btree_set::BTreeSet;
 use rstd::prelude::*;
 use rstd::vec::Vec;
 use sr_primitives::traits::{Bounded, One, Zero};
-use srml_support::traits::{Currency, ExistenceRequirement, Imbalance, WithdrawReasons};
-use srml_support::{decl_event, decl_module, decl_storage, ensure, print};
+use srml_support::traits::{Currency, ExistenceRequirement, Get, Imbalance, WithdrawReasons};
+use srml_support::{decl_event, decl_module, decl_storage, ensure, print, StorageValue};
 use system::{ensure_root, ensure_signed};
 
 use crate::types::ExitInitiationOrigin;
@@ -67,13 +67,10 @@ use errors::WrappedError;
 
 pub use errors::Error;
 pub use types::{
-    Application, Lead, Opening, OpeningPolicyCommitment, OpeningType, RewardPolicy,
-    RoleStakeProfile, Worker,
+    Application, Opening, OpeningPolicyCommitment, OpeningType, RewardPolicy, RoleStakeProfile,
+    Worker,
 };
 
-/// Alias for the _Lead_ type
-pub type LeadOf<T> = Lead<MemberId<T>, <T as system::Trait>::AccountId, WorkerId<T>>;
-
 /// Stake identifier in staking module
 pub type StakeId<T> = <T as stake::Trait>::StakeId;
 
@@ -115,25 +112,17 @@ pub type HiringApplicationId<T> = <T as hiring::Trait>::ApplicationId;
 
 // Type simplification
 type OpeningInfo<T> = (
-    Opening<
-        <T as hiring::Trait>::OpeningId,
-        <T as system::Trait>::BlockNumber,
-        BalanceOf<T>,
-        ApplicationId<T>,
-    >,
+    OpeningOf<T>,
     hiring::Opening<BalanceOf<T>, <T as system::Trait>::BlockNumber, HiringApplicationId<T>>,
 );
 
 // Type simplification
-type ApplicationInfo<T> = (
-    Application<<T as system::Trait>::AccountId, OpeningId<T>, MemberId<T>, HiringApplicationId<T>>,
-    ApplicationId<T>,
-    Opening<
-        <T as hiring::Trait>::OpeningId,
-        <T as system::Trait>::BlockNumber,
-        BalanceOf<T>,
-        ApplicationId<T>,
-    >,
+type ApplicationInfo<T> = (ApplicationOf<T>, ApplicationId<T>, OpeningOf<T>);
+
+// Type simplification
+type RewardSettings<T> = (
+    <T as minting::Trait>::MintId,
+    RewardPolicy<BalanceOfMint<T>, <T as system::Trait>::BlockNumber>,
 );
 
 // Type simplification
@@ -145,6 +134,18 @@ type WorkerOf<T> = Worker<
     MemberId<T>,
 >;
 
+// Type simplification
+type OpeningOf<T> = Opening<
+    <T as hiring::Trait>::OpeningId,
+    <T as system::Trait>::BlockNumber,
+    BalanceOf<T>,
+    ApplicationId<T>,
+>;
+
+// Type simplification
+type ApplicationOf<T> =
+    Application<<T as system::Trait>::AccountId, OpeningId<T>, MemberId<T>, HiringApplicationId<T>>;
+
 /// The _Working group_ main _Trait_
 pub trait Trait<I: Instance>:
     system::Trait
@@ -156,13 +157,15 @@ pub trait Trait<I: Instance>:
 {
     /// _Working group_ event type.
     type Event: From<Event<Self, I>> + Into<<Self as system::Trait>::Event>;
+
+    /// Defines max workers number in the working group.
+    type MaxWorkerNumberLimit: Get<u32>;
 }
 
 decl_event!(
     /// _Working group_ events
     pub enum Event<T, I>
     where
-        MemberId = MemberId<T>,
         WorkerId = WorkerId<T>,
         <T as membership::members::Trait>::ActorId,
         <T as system::Trait>::AccountId,
@@ -175,16 +178,12 @@ decl_event!(
     {
         /// Emits on setting the leader.
         /// Params:
-        /// - Member id of the leader.
-        /// - Role account id of the leader.
         /// - Worker id.
-        LeaderSet(MemberId, AccountId, WorkerId),
+        LeaderSet(WorkerId),
 
         /// Emits on un-setting the leader.
         /// Params:
-        /// - Member id of the leader.
-        /// - Role account id of the leader.
-        LeaderUnset(MemberId, AccountId),
+        LeaderUnset(),
 
         /// Emits on terminating the worker.
         /// Params:
@@ -287,19 +286,19 @@ decl_storage! {
         pub Mint get(mint) : <T as minting::Trait>::MintId;
 
         /// The current lead.
-        pub CurrentLead get(current_lead) : Option<LeadOf<T>>;
+        pub CurrentLead get(current_lead) : Option<WorkerId<T>>;
 
         /// Next identifier value for new worker opening.
         pub NextOpeningId get(next_opening_id): OpeningId<T>;
 
         /// Maps identifier to worker opening.
-        pub OpeningById get(opening_by_id): linked_map OpeningId<T> => Opening<T::OpeningId, T::BlockNumber, BalanceOf<T>, ApplicationId<T>>;
+        pub OpeningById get(opening_by_id): linked_map OpeningId<T> => OpeningOf<T>;
 
         /// Opening human readable text length limits
         pub OpeningHumanReadableText get(opening_human_readable_text): InputValidationLengthConstraint;
 
         /// Maps identifier to worker application on opening.
-        pub ApplicationById get(application_by_id) : linked_map ApplicationId<T> => Application<T::AccountId, OpeningId<T>, T::MemberId, T::ApplicationId>;
+        pub ApplicationById get(application_by_id) : linked_map ApplicationId<T> => ApplicationOf<T>;
 
         /// Next identifier value for new worker application.
         pub NextApplicationId get(next_application_id) : ApplicationId<T>;
@@ -310,6 +309,9 @@ decl_storage! {
         /// Maps identifier to corresponding worker.
         pub WorkerById get(worker_by_id) : linked_map WorkerId<T> => WorkerOf<T>;
 
+        /// Count of active workers.
+        pub ActiveWorkerCount get(fn active_worker_count): u32;
+
         /// Next identifier for new worker.
         pub NextWorkerId get(next_worker_id) : WorkerId<T>;
 
@@ -346,6 +348,9 @@ decl_module! {
         /// Predefined errors
         type Error = Error;
 
+        /// Exports const -  max simultaneous active worker number.
+        const MaxWorkerNumberLimit: u32 = T::MaxWorkerNumberLimit::get();
+
         // ****************** Roles lifecycle **********************
 
         /// Update the associated role account of the active worker/lead.
@@ -368,21 +373,9 @@ decl_module! {
 
             // Update role account
             WorkerById::<T, I>::mutate(worker_id, |worker| {
-                worker.role_account = new_role_account_id.clone()
+                worker.role_account_id = new_role_account_id.clone()
             });
 
-            // Update lead data if it is necessary.
-            let lead = <CurrentLead::<T, I>>::get();
-            if let Some(lead) = lead {
-                if lead.worker_id == worker_id {
-                    let new_lead = Lead{
-                        role_account_id: new_role_account_id.clone(),
-                        ..lead
-                    };
-                    <CurrentLead::<T, I>>::put(new_lead);
-                }
-            }
-
             // Trigger event
             Self::deposit_event(RawEvent::WorkerRoleAccountUpdated(worker_id, new_role_account_id));
         }
@@ -523,6 +516,7 @@ decl_module! {
 
             Self::ensure_opening_human_readable_text_is_valid(&human_readable_text)?;
 
+
             // Add opening
             // NB: This call can in principle fail, because the staking policies
             // may not respect the minimum currency requirement.
@@ -594,7 +588,7 @@ decl_module! {
             origin,
             member_id: T::MemberId,
             opening_id: OpeningId<T>,
-            role_account: T::AccountId,
+            role_account_id: T::AccountId,
             opt_role_stake_balance: Option<BalanceOf<T>>,
             opt_application_stake_balance: Option<BalanceOf<T>>,
             human_readable_text: Vec<u8>
@@ -666,7 +660,7 @@ decl_module! {
             let new_application_id = NextApplicationId::<T, I>::get();
 
             // Make worker/lead application
-            let application = Application::new(&role_account, &opening_id, &member_id, &hiring_application_id);
+            let application = Application::new(&role_account_id, &opening_id, &member_id, &hiring_application_id);
 
             // Store application
             ApplicationById::<T, I>::insert(new_application_id, application);
@@ -696,7 +690,7 @@ decl_module! {
 
             // Ensure that signer is applicant role account
             ensure!(
-                signer_account == application.role_account,
+                signer_account == application.role_account_id,
                 Error::OriginIsNotApplicant
             );
 
@@ -786,6 +780,14 @@ decl_module! {
 
             Self::ensure_origin_for_opening_type(origin, opening.opening_type)?;
 
+            let potential_worker_number =
+                Self::active_worker_count() + (successful_application_ids.len() as u32);
+
+            ensure!(
+                potential_worker_number <= T::MaxWorkerNumberLimit::get(),
+                Error::MaxActiveWorkerNumberExceeded
+            );
+
             // Cannot hire a lead when another leader exists.
             if matches!(opening.opening_type, OpeningType::Leader) {
                 ensure!(!<CurrentLead<T,I>>::exists(), Error::CannotHireLeaderWhenLeaderExists);
@@ -836,7 +838,7 @@ decl_module! {
 
             // Check for a single application for a leader.
             if matches!(opening.opening_type, OpeningType::Leader) {
-                ensure!(successful_application_ids.len() == 1, Error::CannotHireSeveralLeader);
+                ensure!(successful_application_ids.len() == 1, Error::CannotHireMultipleLeaders);
             }
 
             // NB: Combined ensure check and mutation in hiring module
@@ -854,79 +856,12 @@ decl_module! {
             // == MUTATION SAFE ==
             //
 
-            let mut application_id_to_worker_id = BTreeMap::new();
-
-            successful_iter
-            .clone()
-            .for_each(|(successful_application, id, _)| {
-                // Create a reward relationship.
-                let reward_relationship = if let Some((mint_id, checked_policy)) = create_reward_settings.clone() {
-
-                    // Create a new recipient for the new relationship.
-                    let recipient = <recurringrewards::Module<T>>::add_recipient();
-
-                    // Member must exist, since it was checked that it can enter the role.
-                    let member_profile = <membership::members::Module<T>>::member_profile(successful_application.member_id).unwrap();
-
-                    // Rewards are deposited in the member's root account.
-                    let reward_destination_account = member_profile.root_account;
-
-                    // Values have been checked so this should not fail!
-                    let relationship_id = <recurringrewards::Module<T>>::add_reward_relationship(
-                        mint_id,
-                        recipient,
-                        reward_destination_account,
-                        checked_policy.amount_per_payout,
-                        checked_policy.next_payment_at_block,
-                        checked_policy.payout_interval,
-                    ).expect("Failed to create reward relationship!");
-
-                    Some(relationship_id)
-                } else {
-                    None
-                };
-
-                // Get possible stake for role
-                let application = hiring::ApplicationById::<T>::get(successful_application.hiring_application_id);
-
-                // Staking profile for worker
-                let stake_profile =
-                    if let Some(ref stake_id) = application.active_role_staking_id {
-                        Some(
-                            RoleStakeProfile::new(
-                                stake_id,
-                                &opening.policy_commitment.terminate_role_stake_unstaking_period,
-                                &opening.policy_commitment.exit_role_stake_unstaking_period
-                            )
-                        )
-                    } else {
-                        None
-                    };
-
-                // Get worker id
-                let new_worker_id = <NextWorkerId<T, I>>::get();
-
-                // Construct worker
-                let worker = Worker::new(
-                    &successful_application.member_id,
-                    &successful_application.role_account,
-                    &reward_relationship,
-                    &stake_profile,
-                );
-
-                // Store a worker
-                <WorkerById<T, I>>::insert(new_worker_id, worker.clone());
-
-                // Update next worker id
-                <NextWorkerId<T, I>>::mutate(|id| *id += <WorkerId<T> as One>::one());
-
-                application_id_to_worker_id.insert(id, new_worker_id);
-
-                // Sets a leader on successful opening when opening is for leader.
-                if matches!(opening.opening_type, OpeningType::Leader) {
-                    Self::set_lead(worker.member_id, worker.role_account, new_worker_id);
-                }
-            });
+            // Process successful applications
+            let application_id_to_worker_id = Self::fulfill_successful_applications(
+                &opening,
+                create_reward_settings,
+                successful_iter.collect()
+            );
 
             // Trigger event
             Self::deposit_event(RawEvent::OpeningFilled(opening_id, application_id_to_worker_id));
@@ -964,7 +899,7 @@ decl_module! {
             Self::deposit_event(RawEvent::StakeSlashed(worker_id));
         }
 
-        /// Decreases the worker/lead stake and returns the remainder to the worker role_account.
+        /// Decreases the worker/lead stake and returns the remainder to the worker role_account_id.
         /// Can be decreased to zero, no actions on zero stake.
         /// Require signed leader origin or the root (to decrease the leader stake).
         pub fn decrease_stake(origin, worker_id: WorkerId<T>, balance: BalanceOf<T>) {
@@ -985,7 +920,7 @@ decl_module! {
             ensure_on_wrapped_error!(
                 <stake::Module<T>>::decrease_stake_to_account(
                     &stake_profile.stake_id,
-                    &worker.role_account,
+                    &worker.role_account_id,
                     balance
                 )
             )?;
@@ -994,7 +929,7 @@ decl_module! {
         }
 
         /// Increases the worker/lead stake, demands a worker origin. Transfers tokens from the worker
-        /// role_account to the stake. No limits on the stake.
+        /// role_account_id to the stake. No limits on the stake.
         pub fn increase_stake(origin, worker_id: WorkerId<T>, balance: BalanceOf<T>) {
             // Checks worker origin, worker existence
             let worker = Self::ensure_worker_signed(origin, &worker_id)?;
@@ -1011,7 +946,7 @@ decl_module! {
             ensure_on_wrapped_error!(
                 <stake::Module<T>>::increase_stake_from_account(
                     &stake_profile.stake_id,
-                    &worker.role_account,
+                    &worker.role_account_id,
                     balance
                 )
             )?;
@@ -1074,9 +1009,9 @@ impl<T: Trait<I>, I: Instance> Module<T, I> {
         origin: T::Origin,
         worker_id: WorkerId<T>,
     ) -> Result<ExitInitiationOrigin, Error> {
-        let lead = Self::ensure_lead_is_set()?;
+        let leader_worker_id = Self::ensure_lead_is_set()?;
 
-        let (worker_opening_type, exit_origin) = if lead.worker_id == worker_id {
+        let (worker_opening_type, exit_origin) = if leader_worker_id == worker_id {
             (OpeningType::Leader, ExitInitiationOrigin::Sudo)
         } else {
             (OpeningType::Worker, ExitInitiationOrigin::Lead)
@@ -1087,16 +1022,29 @@ impl<T: Trait<I>, I: Instance> Module<T, I> {
         Ok(exit_origin)
     }
 
-    fn ensure_lead_is_set() -> Result<LeadOf<T>, Error> {
-        let lead = <CurrentLead<T, I>>::get();
+    fn ensure_lead_is_set() -> Result<WorkerId<T>, Error> {
+        let leader_worker_id = Self::current_lead();
 
-        if let Some(lead) = lead {
-            Ok(lead)
+        if let Some(leader_worker_id) = leader_worker_id {
+            Ok(leader_worker_id)
         } else {
             Err(Error::CurrentLeadNotSet)
         }
     }
 
+    // Checks that provided lead account id belongs to the current working group leader
+    fn ensure_is_lead_account(lead_account_id: T::AccountId) -> Result<(), Error> {
+        let leader_worker_id = Self::ensure_lead_is_set()?;
+
+        let leader = Self::worker_by_id(leader_worker_id);
+
+        if leader.role_account_id != lead_account_id {
+            return Err(Error::IsNotLeadAccount);
+        }
+
+        Ok(())
+    }
+
     fn ensure_opening_human_readable_text_is_valid(text: &[u8]) -> Result<(), Error> {
         <OpeningHumanReadableText<I>>::get()
             .ensure_valid(
@@ -1229,7 +1177,7 @@ impl<T: Trait<I>, I: Instance> Module<T, I> {
 
         // Ensure that signer is actually role account of worker
         ensure!(
-            signer_account == worker.role_account,
+            signer_account == worker.role_account_id,
             Error::SignerIsNotWorkerRoleAccount
         );
 
@@ -1312,35 +1260,22 @@ impl<T: Trait<I>, I: Instance> Module<T, I> {
         <NegativeImbalance<T>>::zero()
     }
 
-    /// Checks that provided lead account id belongs to the current working group leader
-    pub fn ensure_is_lead_account(lead_account_id: T::AccountId) -> Result<(), Error> {
-        let lead = <CurrentLead<T, I>>::get();
-
-        if let Some(lead) = lead {
-            if lead.role_account_id != lead_account_id {
-                return Err(Error::IsNotLeadAccount);
-            }
-        } else {
-            return Err(Error::CurrentLeadNotSet);
-        }
-
-        Ok(())
-    }
-
     /// Returns all existing worker id list excluding the current leader worker id.
     pub fn get_regular_worker_ids() -> Vec<WorkerId<T>> {
-        let lead = Self::current_lead();
+        let lead_worker_id = Self::current_lead();
 
         <WorkerById<T, I>>::enumerate()
             .filter_map(|(worker_id, _)| {
                 // Filter the leader worker id if the leader is set.
-                lead.clone().map_or(Some(worker_id), |lead| {
-                    if worker_id == lead.worker_id {
-                        None
-                    } else {
-                        Some(worker_id)
-                    }
-                })
+                lead_worker_id
+                    .clone()
+                    .map_or(Some(worker_id), |lead_worker_id| {
+                        if worker_id == lead_worker_id {
+                            None
+                        } else {
+                            Some(worker_id)
+                        }
+                    })
             })
             .collect()
     }
@@ -1396,15 +1331,16 @@ impl<T: Trait<I>, I: Instance> Module<T, I> {
         }
 
         // Unset lead if the leader is leaving.
-        let lead = <CurrentLead<T, I>>::get();
-        if let Some(lead) = lead {
-            if lead.worker_id == *worker_id {
+        let leader_worker_id = <CurrentLead<T, I>>::get();
+        if let Some(leader_worker_id) = leader_worker_id {
+            if leader_worker_id == *worker_id {
                 Self::unset_lead();
             }
         }
 
         // Remove the worker from the storage.
         WorkerById::<T, I>::remove(worker_id);
+        Self::decrease_active_worker_counter();
 
         // Trigger the event
         let event = match exit_initiation_origin {
@@ -1447,33 +1383,123 @@ impl<T: Trait<I>, I: Instance> Module<T, I> {
         <WorkerExitRationaleText<I>>::put(worker_exit_rationale_text_constraint);
     }
 
-    // Introduce a lead.
-    pub(crate) fn set_lead(
-        member_id: T::MemberId,
-        role_account_id: T::AccountId,
-        worker_id: WorkerId<T>,
-    ) {
-        // Construct lead
-        let new_lead = Lead {
-            member_id,
-            role_account_id: role_account_id.clone(),
-            worker_id,
-        };
-
+    // Set worker id as a leader id.
+    pub(crate) fn set_lead(worker_id: WorkerId<T>) {
         // Update current lead
-        <CurrentLead<T, I>>::put(new_lead);
+        <CurrentLead<T, I>>::put(worker_id);
 
         // Trigger an event
-        Self::deposit_event(RawEvent::LeaderSet(member_id, role_account_id, worker_id));
+        Self::deposit_event(RawEvent::LeaderSet(worker_id));
     }
 
     // Evict the currently set lead.
     pub(crate) fn unset_lead() {
-        if let Ok(lead) = Self::ensure_lead_is_set() {
+        if Self::ensure_lead_is_set().is_ok() {
             // Update current lead
             <CurrentLead<T, I>>::kill();
 
-            Self::deposit_event(RawEvent::LeaderUnset(lead.member_id, lead.role_account_id));
+            Self::deposit_event(RawEvent::LeaderUnset());
         }
     }
+
+    // Processes successful application during the fill_opening().
+    fn fulfill_successful_applications(
+        opening: &OpeningOf<T>,
+        reward_settings: Option<RewardSettings<T>>,
+        successful_applications_info: Vec<ApplicationInfo<T>>,
+    ) -> BTreeMap<ApplicationId<T>, WorkerId<T>> {
+        let mut application_id_to_worker_id = BTreeMap::new();
+
+        successful_applications_info
+            .iter()
+            .for_each(|(successful_application, id, _)| {
+                // Create a reward relationship.
+                let reward_relationship = if let Some((mint_id, checked_policy)) =
+                    reward_settings.clone()
+                {
+                    // Create a new recipient for the new relationship.
+                    let recipient = <recurringrewards::Module<T>>::add_recipient();
+
+                    // Member must exist, since it was checked that it can enter the role.
+                    let member_profile = <membership::members::Module<T>>::member_profile(
+                        successful_application.member_id,
+                    )
+                    .unwrap();
+
+                    // Rewards are deposited in the member's root account.
+                    let reward_destination_account = member_profile.root_account;
+
+                    // Values have been checked so this should not fail!
+                    let relationship_id = <recurringrewards::Module<T>>::add_reward_relationship(
+                        mint_id,
+                        recipient,
+                        reward_destination_account,
+                        checked_policy.amount_per_payout,
+                        checked_policy.next_payment_at_block,
+                        checked_policy.payout_interval,
+                    )
+                    .expect("Failed to create reward relationship!");
+
+                    Some(relationship_id)
+                } else {
+                    None
+                };
+
+                // Get possible stake for role
+                let application =
+                    hiring::ApplicationById::<T>::get(successful_application.hiring_application_id);
+
+                // Staking profile for worker
+                let stake_profile = if let Some(ref stake_id) = application.active_role_staking_id {
+                    Some(RoleStakeProfile::new(
+                        stake_id,
+                        &opening
+                            .policy_commitment
+                            .terminate_role_stake_unstaking_period,
+                        &opening.policy_commitment.exit_role_stake_unstaking_period,
+                    ))
+                } else {
+                    None
+                };
+
+                // Get worker id
+                let new_worker_id = <NextWorkerId<T, I>>::get();
+
+                // Construct worker
+                let worker = Worker::new(
+                    &successful_application.member_id,
+                    &successful_application.role_account_id,
+                    &reward_relationship,
+                    &stake_profile,
+                );
+
+                // Store a worker
+                <WorkerById<T, I>>::insert(new_worker_id, worker);
+                Self::increase_active_worker_counter();
+
+                // Update next worker id
+                <NextWorkerId<T, I>>::mutate(|id| *id += <WorkerId<T> as One>::one());
+
+                application_id_to_worker_id.insert(*id, new_worker_id);
+
+                // Sets a leader on successful opening when opening is for leader.
+                if matches!(opening.opening_type, OpeningType::Leader) {
+                    Self::set_lead(new_worker_id);
+                }
+            });
+
+        application_id_to_worker_id
+    }
+
+    // Increases active worker counter (saturating).
+    fn increase_active_worker_counter() {
+        let next_active_worker_count_value = Self::active_worker_count().saturating_add(1);
+        <ActiveWorkerCount<I>>::put(next_active_worker_count_value);
+    }
+
+    // Decreases active worker counter (saturating).
+    fn decrease_active_worker_counter() {
+        let next_active_worker_count_value = Self::active_worker_count().saturating_sub(1);
+        <ActiveWorkerCount<I>>::put(next_active_worker_count_value);
+    }
 }

runtime-modules/working-group/src/tests/fixtures.rs

@@ -163,10 +163,12 @@ pub struct UpdateWorkerRewardAmountFixture {
 
 impl UpdateWorkerRewardAmountFixture {
     pub fn default_for_worker_id(worker_id: u64) -> Self {
+        let lead_account_id = get_current_lead_account_id();
+
         Self {
             worker_id,
             amount: 120,
-            origin: RawOrigin::Signed(1),
+            origin: RawOrigin::Signed(lead_account_id),
         }
     }
     pub fn with_origin(self, origin: RawOrigin<u64>) -> Self {
@@ -259,7 +261,7 @@ impl UpdateWorkerRoleAccountFixture {
         if actual_result.is_ok() {
             let worker = TestWorkingGroup::worker_by_id(self.worker_id);
 
-            assert_eq!(worker.role_account, self.new_role_account_id);
+            assert_eq!(worker.role_account_id, self.new_role_account_id);
         }
     }
 }
@@ -276,7 +278,7 @@ pub struct FillWorkerOpeningFixture {
     origin: RawOrigin<u64>,
     opening_id: u64,
     successful_application_ids: BTreeSet<u64>,
-    role_account: u64,
+    role_account_id: u64,
     reward_policy: Option<RewardPolicy<u64, u64>>,
 }
 
@@ -288,7 +290,7 @@ impl FillWorkerOpeningFixture {
             origin: RawOrigin::Signed(1),
             opening_id,
             successful_application_ids: application_ids,
-            role_account: 1,
+            role_account_id: 1,
             reward_policy: None,
         }
     }
@@ -348,7 +350,7 @@ impl FillWorkerOpeningFixture {
 
             let expected_worker = Worker {
                 member_id: 1,
-                role_account: self.role_account,
+                role_account_id: self.role_account_id,
                 reward_relationship,
                 role_stake_profile,
             };
@@ -476,7 +478,7 @@ pub struct ApplyOnWorkerOpeningFixture {
     origin: RawOrigin<u64>,
     member_id: u64,
     worker_opening_id: u64,
-    role_account: u64,
+    role_account_id: u64,
     opt_role_stake_balance: Option<u64>,
     opt_application_stake_balance: Option<u64>,
     human_readable_text: Vec<u8>,
@@ -517,7 +519,7 @@ impl ApplyOnWorkerOpeningFixture {
             origin: RawOrigin::Signed(1),
             member_id: 1,
             worker_opening_id: opening_id,
-            role_account: 1,
+            role_account_id: 1,
             opt_role_stake_balance: None,
             opt_application_stake_balance: None,
             human_readable_text: b"human_text".to_vec(),
@@ -530,7 +532,7 @@ impl ApplyOnWorkerOpeningFixture {
             self.origin.clone().into(),
             self.member_id,
             self.worker_opening_id,
-            self.role_account,
+            self.role_account_id,
             self.opt_role_stake_balance,
             self.opt_application_stake_balance,
             self.human_readable_text.clone(),
@@ -554,7 +556,7 @@ impl ApplyOnWorkerOpeningFixture {
             let actual_application = TestWorkingGroup::application_by_id(application_id);
 
             let expected_application = Application {
-                role_account: self.role_account,
+                role_account_id: self.role_account_id,
                 opening_id: self.worker_opening_id,
                 member_id: self.member_id,
                 hiring_application_id: application_id,
@@ -592,14 +594,14 @@ impl AcceptWorkerApplicationsFixture {
 
 pub struct SetLeadFixture {
     pub member_id: u64,
-    pub role_account: u64,
+    pub role_account_id: u64,
     pub worker_id: u64,
 }
 impl Default for SetLeadFixture {
     fn default() -> Self {
         SetLeadFixture {
             member_id: 1,
-            role_account: 1,
+            role_account_id: 1,
             worker_id: 1,
         }
     }
@@ -611,12 +613,12 @@ impl SetLeadFixture {
     }
 
     pub fn set_lead(self) {
-        TestWorkingGroup::set_lead(self.member_id, self.role_account, self.worker_id);
+        TestWorkingGroup::set_lead(self.worker_id);
     }
-    pub fn set_lead_with_ids(member_id: u64, role_account: u64, worker_id: u64) {
+    pub fn set_lead_with_ids(member_id: u64, role_account_id: u64, worker_id: u64) {
         Self {
             member_id,
-            role_account,
+            role_account_id,
             worker_id,
         }
         .set_lead();
@@ -774,7 +776,6 @@ impl EventFixture {
             u64,
             u64,
             u64,
-            u64,
             std::collections::BTreeMap<u64, u64>,
             Vec<u8>,
             u64,
@@ -808,8 +809,11 @@ pub struct DecreaseWorkerStakeFixture {
 impl DecreaseWorkerStakeFixture {
     pub fn default_for_worker_id(worker_id: u64) -> Self {
         let account_id = 1;
+
+        let lead_account_id = get_current_lead_account_id();
+
         Self {
-            origin: RawOrigin::Signed(account_id),
+            origin: RawOrigin::Signed(lead_account_id),
             worker_id,
             balance: 10,
             account_id,
@@ -860,6 +864,17 @@ pub(crate) fn get_stake_balance(stake: stake::Stake<u64, u64, u64>) -> u64 {
     panic!("Not staked.");
 }
 
+fn get_current_lead_account_id() -> u64 {
+    let leader_worker_id = TestWorkingGroup::current_lead();
+
+    if let Some(leader_worker_id) = leader_worker_id {
+        let leader = TestWorkingGroup::worker_by_id(leader_worker_id);
+        leader.role_account_id
+    } else {
+        0 // return invalid lead_account_id for testing
+    }
+}
+
 pub struct SlashWorkerStakeFixture {
     origin: RawOrigin<u64>,
     worker_id: u64,
@@ -870,8 +885,11 @@ pub struct SlashWorkerStakeFixture {
 impl SlashWorkerStakeFixture {
     pub fn default_for_worker_id(worker_id: u64) -> Self {
         let account_id = 1;
+
+        let lead_account_id = get_current_lead_account_id();
+
         Self {
-            origin: RawOrigin::Signed(account_id),
+            origin: RawOrigin::Signed(lead_account_id),
             worker_id,
             balance: 10,
             account_id,

runtime-modules/working-group/src/tests/hiring_workflow.rs

@@ -3,6 +3,7 @@ use crate::tests::fixtures::{
     AddWorkerOpeningFixture, ApplyOnWorkerOpeningFixture, BeginReviewWorkerApplicationsFixture,
     FillWorkerOpeningFixture, SetLeadFixture,
 };
+use crate::tests::mock::TestWorkingGroup;
 use crate::Error;
 use crate::{OpeningPolicyCommitment, OpeningType, RewardPolicy};
 use system::RawOrigin;
@@ -112,7 +113,7 @@ impl HiringWorkflow {
             SetLeadFixture::default().set_lead();
         }
         increase_total_balance_issuance_using_account_id(1, 10000);
-        setup_members(3);
+        setup_members(4);
         set_mint_id(create_mint());
     }
 
@@ -131,11 +132,15 @@ impl HiringWorkflow {
     }
 
     fn fill_worker_position(&self) -> Result<u64, Error> {
-        let lead_account_id = SetLeadFixture::default().role_account;
-
         let origin = match self.opening_type {
             OpeningType::Leader => RawOrigin::Root,
-            OpeningType::Worker => RawOrigin::Signed(lead_account_id),
+            OpeningType::Worker => {
+                let leader_worker_id = TestWorkingGroup::current_lead().unwrap();
+                let leader = TestWorkingGroup::worker_by_id(leader_worker_id);
+                let lead_account_id = leader.role_account_id;
+
+                RawOrigin::Signed(lead_account_id)
+            }
         };
 
         // create the opening

runtime-modules/working-group/src/tests/mock.rs

@@ -127,8 +127,13 @@ impl recurringrewards::Trait for Test {
 pub type Balances = balances::Module<Test>;
 pub type System = system::Module<Test>;
 
+parameter_types! {
+    pub const MaxWorkerNumberLimit: u32 = 3;
+}
+
 impl Trait<TestWorkingGroupInstance> for Test {
     type Event = TestEvent;
+    type MaxWorkerNumberLimit = MaxWorkerNumberLimit;
 }
 
 pub type Membership = membership::members::Module<Test>;

runtime-modules/working-group/src/tests/mod.rs

@@ -3,7 +3,7 @@ mod hiring_workflow;
 mod mock;
 
 use crate::types::{OpeningPolicyCommitment, OpeningType, RewardPolicy};
-use crate::{Error, Lead, RawEvent};
+use crate::{Error, RawEvent, Worker};
 use common::constraints::InputValidationLengthConstraint;
 use mock::{
     build_test_externalities, Test, TestWorkingGroup, TestWorkingGroupInstance,
@@ -50,7 +50,7 @@ fn hire_lead_fails_multiple_applications() {
             .with_opening_type(OpeningType::Leader)
             .add_application_with_origin(b"leader_handle".to_vec(), RawOrigin::Signed(1), 1)
             .add_application_with_origin(b"leader_handle2".to_vec(), RawOrigin::Signed(2), 2)
-            .expect(Err(Error::CannotHireSeveralLeader));
+            .expect(Err(Error::CannotHireMultipleLeaders));
 
         hiring_workflow.execute();
     });
@@ -59,7 +59,7 @@ fn hire_lead_fails_multiple_applications() {
 #[test]
 fn add_worker_opening_succeeds() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
 
@@ -72,7 +72,7 @@ fn add_worker_opening_succeeds() {
 #[test]
 fn add_leader_opening_succeeds_fails_with_incorrect_origin_for_opening_type() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture =
             AddWorkerOpeningFixture::default().with_opening_type(OpeningType::Leader);
@@ -84,7 +84,7 @@ fn add_leader_opening_succeeds_fails_with_incorrect_origin_for_opening_type() {
 #[test]
 fn add_leader_opening_succeeds() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default()
             .with_opening_type(OpeningType::Leader)
@@ -106,7 +106,7 @@ fn add_worker_opening_fails_with_lead_is_not_set() {
 #[test]
 fn add_worker_opening_fails_with_invalid_human_readable_text() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         <crate::OpeningHumanReadableText<TestWorkingGroupInstance>>::put(
             InputValidationLengthConstraint {
@@ -129,7 +129,7 @@ fn add_worker_opening_fails_with_invalid_human_readable_text() {
 #[test]
 fn add_worker_opening_fails_with_hiring_error() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default()
             .with_activate_at(hiring::ActivateOpeningAt::ExactBlock(0));
@@ -141,7 +141,7 @@ fn add_worker_opening_fails_with_hiring_error() {
 #[test]
 fn accept_worker_applications_succeeds() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default()
             .with_activate_at(hiring::ActivateOpeningAt::ExactBlock(5));
@@ -158,7 +158,7 @@ fn accept_worker_applications_succeeds() {
 #[test]
 fn accept_worker_applications_fails_for_invalid_opening_type() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default()
             .with_origin(RawOrigin::Root)
@@ -175,7 +175,7 @@ fn accept_worker_applications_fails_for_invalid_opening_type() {
 #[test]
 fn accept_worker_applications_fails_with_hiring_error() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -191,7 +191,7 @@ fn accept_worker_applications_fails_with_hiring_error() {
 #[test]
 fn accept_worker_applications_fails_with_not_lead() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -207,7 +207,7 @@ fn accept_worker_applications_fails_with_not_lead() {
 #[test]
 fn accept_worker_applications_fails_with_no_opening() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let opening_id = 55; // random opening id
 
@@ -220,8 +220,7 @@ fn accept_worker_applications_fails_with_no_opening() {
 #[test]
 fn apply_on_worker_opening_succeeds() {
     build_test_externalities().execute_with(|| {
-        setup_members(2);
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -240,10 +239,9 @@ fn apply_on_worker_opening_succeeds() {
 #[test]
 fn apply_on_worker_opening_fails_with_no_opening() {
     build_test_externalities().execute_with(|| {
-        setup_members(2);
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
-        let opening_id = 0; // random opening id
+        let opening_id = 123; // random opening id
 
         let appy_on_worker_opening_fixture =
             ApplyOnWorkerOpeningFixture::default_for_opening_id(opening_id);
@@ -254,13 +252,14 @@ fn apply_on_worker_opening_fails_with_no_opening() {
 #[test]
 fn apply_on_worker_opening_fails_with_not_set_members() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
 
         let appy_on_worker_opening_fixture =
-            ApplyOnWorkerOpeningFixture::default_for_opening_id(opening_id);
+            ApplyOnWorkerOpeningFixture::default_for_opening_id(opening_id)
+                .with_origin(RawOrigin::Signed(55), 55);
         appy_on_worker_opening_fixture
             .call_and_assert(Err(Error::OriginIsNeitherMemberControllerOrRoot));
     });
@@ -270,8 +269,7 @@ fn apply_on_worker_opening_fails_with_not_set_members() {
 fn apply_on_worker_opening_fails_with_hiring_error() {
     build_test_externalities().execute_with(|| {
         increase_total_balance_issuance_using_account_id(1, 500000);
-        setup_members(2);
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -287,15 +285,24 @@ fn apply_on_worker_opening_fails_with_hiring_error() {
 #[test]
 fn apply_on_worker_opening_fails_with_invalid_application_stake() {
     build_test_externalities().execute_with(|| {
-        setup_members(2);
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
-        let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
+        let stake = 100;
+
+        let add_worker_opening_fixture =
+            AddWorkerOpeningFixture::default().with_policy_commitment(OpeningPolicyCommitment {
+                application_staking_policy: Some(hiring::StakingPolicy {
+                    amount: stake,
+                    ..hiring::StakingPolicy::default()
+                }),
+                ..OpeningPolicyCommitment::default()
+            });
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
 
         let appy_on_worker_opening_fixture =
             ApplyOnWorkerOpeningFixture::default_for_opening_id(opening_id)
-                .with_application_stake(100);
+                .with_origin(RawOrigin::Signed(2), 2)
+                .with_application_stake(stake);
         appy_on_worker_opening_fixture.call_and_assert(Err(Error::InsufficientBalanceToApply));
     });
 }
@@ -303,15 +310,24 @@ fn apply_on_worker_opening_fails_with_invalid_application_stake() {
 #[test]
 fn apply_on_worker_opening_fails_with_invalid_role_stake() {
     build_test_externalities().execute_with(|| {
-        setup_members(2);
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
-        let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
+        let stake = 100;
+
+        let add_worker_opening_fixture =
+            AddWorkerOpeningFixture::default().with_policy_commitment(OpeningPolicyCommitment {
+                role_staking_policy: Some(hiring::StakingPolicy {
+                    amount: stake,
+                    ..hiring::StakingPolicy::default()
+                }),
+                ..OpeningPolicyCommitment::default()
+            });
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
 
         let appy_on_worker_opening_fixture =
             ApplyOnWorkerOpeningFixture::default_for_opening_id(opening_id)
-                .with_role_stake(Some(100));
+                .with_role_stake(Some(stake))
+                .with_origin(RawOrigin::Signed(2), 2);
         appy_on_worker_opening_fixture.call_and_assert(Err(Error::InsufficientBalanceToApply));
     });
 }
@@ -319,8 +335,7 @@ fn apply_on_worker_opening_fails_with_invalid_role_stake() {
 #[test]
 fn apply_on_worker_opening_fails_with_invalid_text() {
     build_test_externalities().execute_with(|| {
-        setup_members(2);
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -348,8 +363,7 @@ fn apply_on_worker_opening_fails_with_invalid_text() {
 #[test]
 fn apply_on_worker_opening_fails_with_already_active_application() {
     build_test_externalities().execute_with(|| {
-        setup_members(2);
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -366,8 +380,7 @@ fn apply_on_worker_opening_fails_with_already_active_application() {
 #[test]
 fn withdraw_worker_application_succeeds() {
     build_test_externalities().execute_with(|| {
-        setup_members(2);
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -398,8 +411,7 @@ fn withdraw_worker_application_fails_invalid_application_id() {
 #[test]
 fn withdraw_worker_application_fails_invalid_origin() {
     build_test_externalities().execute_with(|| {
-        setup_members(2);
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -418,8 +430,7 @@ fn withdraw_worker_application_fails_invalid_origin() {
 #[test]
 fn withdraw_worker_application_fails_with_invalid_application_author() {
     build_test_externalities().execute_with(|| {
-        setup_members(2);
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -439,8 +450,7 @@ fn withdraw_worker_application_fails_with_invalid_application_author() {
 #[test]
 fn withdraw_worker_application_fails_with_hiring_error() {
     build_test_externalities().execute_with(|| {
-        setup_members(2);
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -460,8 +470,7 @@ fn withdraw_worker_application_fails_with_hiring_error() {
 #[test]
 fn terminate_worker_application_succeeds() {
     build_test_externalities().execute_with(|| {
-        setup_members(2);
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -481,8 +490,7 @@ fn terminate_worker_application_succeeds() {
 #[test]
 fn terminate_worker_application_fails_with_invalid_application_author() {
     build_test_externalities().execute_with(|| {
-        setup_members(2);
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -502,8 +510,7 @@ fn terminate_worker_application_fails_with_invalid_application_author() {
 #[test]
 fn terminate_worker_application_fails_invalid_origin() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
-        setup_members(2);
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -522,7 +529,7 @@ fn terminate_worker_application_fails_invalid_origin() {
 #[test]
 fn terminate_worker_application_fails_invalid_application_id() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let invalid_application_id = 6;
 
@@ -535,8 +542,7 @@ fn terminate_worker_application_fails_invalid_application_id() {
 #[test]
 fn terminate_worker_application_fails_with_hiring_error() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
-        setup_members(2);
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -556,7 +562,7 @@ fn terminate_worker_application_fails_with_hiring_error() {
 #[test]
 fn begin_review_worker_applications_succeeds() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -572,7 +578,7 @@ fn begin_review_worker_applications_succeeds() {
 #[test]
 fn begin_review_worker_applications_fails_with_invalid_origin_for_opening_type() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default()
             .with_origin(RawOrigin::Root)
@@ -588,7 +594,7 @@ fn begin_review_worker_applications_fails_with_invalid_origin_for_opening_type()
 #[test]
 fn begin_review_worker_applications_fails_with_not_a_lead() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -604,7 +610,7 @@ fn begin_review_worker_applications_fails_with_not_a_lead() {
 #[test]
 fn begin_review_worker_applications_fails_with_invalid_opening() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let invalid_opening_id = 6;
 
@@ -617,7 +623,7 @@ fn begin_review_worker_applications_fails_with_invalid_opening() {
 #[test]
 fn begin_review_worker_applications_with_hiring_error() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -634,7 +640,7 @@ fn begin_review_worker_applications_with_hiring_error() {
 #[test]
 fn begin_review_worker_applications_fails_with_invalid_origin() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -649,8 +655,7 @@ fn begin_review_worker_applications_fails_with_invalid_origin() {
 #[test]
 fn fill_worker_opening_succeeds() {
     build_test_externalities().execute_with(|| {
-        setup_members(2);
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
         increase_total_balance_issuance_using_account_id(1, 10000);
 
         let add_worker_opening_fixture =
@@ -699,8 +704,7 @@ fn fill_worker_opening_succeeds() {
 #[test]
 fn fill_worker_opening_fails_with_invalid_origin_for_opening_type() {
     build_test_externalities().execute_with(|| {
-        setup_members(2);
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
         increase_total_balance_issuance_using_account_id(1, 10000);
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default()
@@ -743,7 +747,7 @@ fn fill_worker_opening_fails_with_invalid_origin_for_opening_type() {
 #[test]
 fn fill_worker_opening_fails_with_invalid_origin() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -758,7 +762,7 @@ fn fill_worker_opening_fails_with_invalid_origin() {
 #[test]
 fn fill_worker_opening_fails_with_not_a_lead() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -774,7 +778,7 @@ fn fill_worker_opening_fails_with_not_a_lead() {
 #[test]
 fn fill_worker_opening_fails_with_invalid_opening() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let invalid_opening_id = 6;
 
@@ -787,8 +791,7 @@ fn fill_worker_opening_fails_with_invalid_opening() {
 #[test]
 fn fill_worker_opening_fails_with_invalid_application_list() {
     build_test_externalities().execute_with(|| {
-        setup_members(2);
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -814,7 +817,7 @@ fn fill_worker_opening_fails_with_invalid_application_list() {
 #[test]
 fn fill_worker_opening_fails_with_invalid_application_with_hiring_error() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -829,8 +832,7 @@ fn fill_worker_opening_fails_with_invalid_application_with_hiring_error() {
 #[test]
 fn fill_worker_opening_fails_with_invalid_reward_policy() {
     build_test_externalities().execute_with(|| {
-        setup_members(2);
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
 
         let add_worker_opening_fixture = AddWorkerOpeningFixture::default();
         let opening_id = add_worker_opening_fixture.call_and_assert(Ok(()));
@@ -879,26 +881,22 @@ fn update_worker_role_account_by_leader_succeeds() {
         let new_account_id = 10;
         let worker_id = HireLeadFixture::default().hire_lead();
 
-        // Default ids.
-        let mut lead = Lead {
-            member_id: 1,
-            role_account_id: 1,
-            worker_id: 0,
-        };
-
-        assert_eq!(TestWorkingGroup::current_lead(), Some(lead));
+        let old_lead = TestWorkingGroup::worker_by_id(worker_id);
 
         let update_worker_account_fixture =
             UpdateWorkerRoleAccountFixture::default_with_ids(worker_id, new_account_id);
 
         update_worker_account_fixture.call_and_assert(Ok(()));
 
-        lead = Lead {
-            role_account_id: new_account_id,
-            ..lead
-        };
+        let new_lead = TestWorkingGroup::worker_by_id(worker_id);
 
-        assert_eq!(TestWorkingGroup::current_lead(), Some(lead));
+        assert_eq!(
+            new_lead,
+            Worker {
+                role_account_id: new_account_id,
+                ..old_lead
+            }
+        );
     });
 }
 
@@ -969,7 +967,7 @@ fn update_worker_reward_account_fails_with_invalid_origin_signed_account() {
 
         let invalid_role_account = 23333;
         let update_worker_account_fixture =
-            UpdateWorkerRewardAccountFixture::default_with_ids(worker_id, worker.role_account)
+            UpdateWorkerRewardAccountFixture::default_with_ids(worker_id, worker.role_account_id)
                 .with_origin(RawOrigin::Signed(invalid_role_account));
 
         update_worker_account_fixture.call_and_assert(Err(Error::SignerIsNotWorkerRoleAccount));
@@ -1634,7 +1632,7 @@ fn decrease_worker_stake_fails_with_zero_balance() {
 #[test]
 fn decrease_worker_stake_fails_with_invalid_worker_id() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
         let invalid_worker_id = 11;
 
         let decrease_stake_fixture =
@@ -1751,7 +1749,7 @@ fn slash_worker_stake_fails_with_zero_balance() {
 #[test]
 fn slash_worker_stake_fails_with_invalid_worker_id() {
     build_test_externalities().execute_with(|| {
-        SetLeadFixture::default().set_lead();
+        HireLeadFixture::default().hire_lead();
         let invalid_worker_id = 11;
 
         let slash_stake_fixture = SlashWorkerStakeFixture::default_for_worker_id(invalid_worker_id);
@@ -1874,3 +1872,76 @@ fn ensure_setting_genesis_constraints_succeeds() {
         assert_eq!(worker_exit_text_constraint, default_constraint);
     });
 }
+
+#[test]
+fn active_worker_counter_works_successfully() {
+    build_test_externalities().execute_with(|| {
+        assert_eq!(TestWorkingGroup::active_worker_count(), 0);
+
+        let leader_id = HireLeadFixture::default().hire_lead();
+        assert_eq!(TestWorkingGroup::active_worker_count(), 1);
+
+        let worker_id1 = fill_worker_position(
+            None,
+            None,
+            false,
+            OpeningType::Worker,
+            Some(b"worker1".to_vec()),
+        );
+        assert_eq!(TestWorkingGroup::active_worker_count(), 2);
+
+        let worker_id2 = fill_worker_position(
+            None,
+            None,
+            false,
+            OpeningType::Worker,
+            Some(b"worker1".to_vec()),
+        );
+        assert_eq!(TestWorkingGroup::active_worker_count(), 3);
+
+        TerminateWorkerRoleFixture::default_for_worker_id(worker_id1).call_and_assert(Ok(()));
+        assert_eq!(TestWorkingGroup::active_worker_count(), 2);
+
+        TerminateWorkerRoleFixture::default_for_worker_id(worker_id2).call_and_assert(Ok(()));
+        assert_eq!(TestWorkingGroup::active_worker_count(), 1);
+
+        TerminateWorkerRoleFixture::default_for_worker_id(leader_id)
+            .with_origin(RawOrigin::Root)
+            .call_and_assert(Ok(()));
+        assert_eq!(TestWorkingGroup::active_worker_count(), 0);
+    });
+}
+
+#[test]
+fn adding_too_much_workers_fails_with_single_application_out_of_limit() {
+    build_test_externalities().execute_with(|| {
+        HireLeadFixture::default().hire_lead();
+
+        fill_worker_position(None, None, false, OpeningType::Worker, None);
+        fill_worker_position(None, None, false, OpeningType::Worker, None);
+
+        let hiring_workflow = HiringWorkflow::default()
+            .disable_setup_environment()
+            .add_default_application()
+            .expect(Err(Error::MaxActiveWorkerNumberExceeded));
+
+        hiring_workflow.execute()
+    });
+}
+
+#[test]
+fn fill_opening_cannot_hire_more_workers_using_several_applicationst_han_allows_worker_limit() {
+    build_test_externalities().execute_with(|| {
+        HireLeadFixture::default().hire_lead();
+
+        fill_worker_position(None, None, false, OpeningType::Worker, None);
+
+        let hiring_workflow = HiringWorkflow::default()
+            .disable_setup_environment()
+            .add_application_with_origin(b"Some1".to_vec(), RawOrigin::Signed(2), 2)
+            .add_application_with_origin(b"Some2".to_vec(), RawOrigin::Signed(3), 3)
+            .expect(Err(Error::MaxActiveWorkerNumberExceeded));
+
+        hiring_workflow.execute()
+    });
+}

runtime-modules/working-group/src/types.rs

@@ -115,26 +115,12 @@ impl Default for OpeningType {
     }
 }
 
-/// Working group lead.
-#[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
-#[derive(Encode, Decode, Default, Debug, Clone, PartialEq, Copy)]
-pub struct Lead<MemberId, AccountId, WorkerId> {
-    /// Member id of the leader.
-    pub member_id: MemberId,
-
-    /// Account used to authenticate in this role.
-    pub role_account_id: AccountId,
-
-    /// Leader worker id.
-    pub worker_id: WorkerId,
-}
-
 /// An application for the worker/lead role.
 #[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
 #[derive(Encode, Decode, Default, Debug, Clone, PartialEq)]
 pub struct Application<AccountId, OpeningId, MemberId, ApplicationId> {
     /// Account used to authenticate in this role.
-    pub role_account: AccountId,
+    pub role_account_id: AccountId,
 
     /// Opening on which this application applies.
     pub opening_id: OpeningId,
@@ -151,13 +137,13 @@ impl<AccountId: Clone, OpeningId: Clone, MemberId: Clone, ApplicationId: Clone>
 {
     /// Creates a new worker application using parameters.
     pub fn new(
-        role_account: &AccountId,
+        role_account_id: &AccountId,
         opening_id: &OpeningId,
         member_id: &MemberId,
         application_id: &ApplicationId,
     ) -> Self {
         Application {
-            role_account: role_account.clone(),
+            role_account_id: role_account_id.clone(),
             opening_id: opening_id.clone(),
             member_id: member_id.clone(),
             hiring_application_id: application_id.clone(),
@@ -203,7 +189,7 @@ pub struct Worker<AccountId, RewardRelationshipId, StakeId, BlockNumber, MemberI
     pub member_id: MemberId,
 
     /// Account used to authenticate in this role.
-    pub role_account: AccountId,
+    pub role_account_id: AccountId,
 
     /// Whether the role has recurring reward, and if so an identifier for this.
     pub reward_relationship: Option<RewardRelationshipId>,
@@ -223,13 +209,13 @@ impl<
     /// Creates a new _Worker_ using parameters.
     pub fn new(
         member_id: &MemberId,
-        role_account: &AccountId,
+        role_account_id: &AccountId,
         reward_relationship: &Option<RewardRelationshipId>,
         role_stake_profile: &Option<RoleStakeProfile<StakeId, BlockNumber>>,
     ) -> Self {
         Worker {
             member_id: member_id.clone(),
-            role_account: role_account.clone(),
+            role_account_id: role_account_id.clone(),
             reward_relationship: reward_relationship.clone(),
             role_stake_profile: role_stake_profile.clone(),
         }

runtime/src/lib.rs

@@ -631,8 +631,13 @@ impl migration::Trait for Runtime {
 // The storage working group instance alias.
 pub type StorageWorkingGroupInstance = working_group::Instance2;
 
+parameter_types! {
+    pub const MaxWorkerNumberLimit: u32 = 100;
+}
+
 impl working_group::Trait<StorageWorkingGroupInstance> for Runtime {
     type Event = Event;
+    type MaxWorkerNumberLimit = MaxWorkerNumberLimit;
 }
 
 impl service_discovery::Trait for Runtime {

scripts/run-test-chain.sh

@@ -7,8 +7,6 @@ perl -i -pe's/"setElectionParametersProposalGracePeriod":.*/"setElectionParamete
 perl -i -pe's/"textProposalGracePeriod":.*/"textProposalGracePeriod": 0,/' .tmp/chainspec.json
 perl -i -pe's/"setContentWorkingGroupMintCapacityProposalGracePeriod":.*/"setContentWorkingGroupMintCapacityProposalGracePeriod": 0,/' .tmp/chainspec.json
 perl -i -pe's/"setLeadProposalGracePeriod":.*/"setLeadProposalGracePeriod": 0,/' .tmp/chainspec.json
-perl -i -pe's/"spendingProposalGracePeriod":.*/"spendingProposalGracePeriod": 0,/' .tmp/chainspec.json
-perl -i -pe's/"evictStorageProviderProposalGracePeriod":.*/"evictStorageProviderProposalGracePeriod": 0,/' .tmp/chainspec.json
-perl -i -pe's/"setStorageRoleParametersProposalGracePeriod":.*/"setStorageRoleParametersProposalGracePeriod": 0/' .tmp/chainspec.json
+perl -i -pe's/"spendingProposalGracePeriod":.*/"spendingProposalGracePeriod": 0/' .tmp/chainspec.json
 yes | cargo run --release -p joystream-node -- purge-chain --dev
 cargo run --release -p joystream-node -- --chain=.tmp/chainspec.json --alice --validator

storage-node/.eslintrc.js

@@ -0,0 +1,290 @@
+module.exports = {
+    "env": {
+        "es6": true,
+        "node": true
+    },
+    "extends": "eslint:recommended",
+    "parserOptions": {
+        "ecmaVersion": 2018
+    },
+    "rules": {
+        "accessor-pairs": "error",
+        "array-bracket-newline": "off",
+        "array-bracket-spacing": [
+            "error",
+            "never",
+        ],
+        "array-callback-return": "error",
+        "array-element-newline": [
+          "error",
+          "consistent",
+        ],
+        "arrow-body-style": [
+          "warn",
+          "as-needed"
+        ],
+        "arrow-parens": [
+            "error",
+            "always"
+        ],
+        "arrow-spacing": [
+            "error",
+            {
+                "after": true,
+                "before": true
+            }
+        ],
+        "block-scoped-var": "error",
+        "block-spacing": "error",
+        "brace-style": "off",
+        "callback-return": "error",
+        "camelcase": "off",
+        "capitalized-comments": "off",
+        "class-methods-use-this": "error",
+        "comma-dangle": "off",
+        "comma-spacing": "off",
+        "comma-style": [
+            "error",
+            "last"
+        ],
+        "complexity": "error",
+        "computed-property-spacing": [
+            "error",
+            "never"
+        ],
+        "consistent-return": "error",
+        "consistent-this": "error",
+        "curly": "error",
+        "default-case": "error",
+        "dot-location": "error",
+        "dot-notation": "off",
+        "eol-last": "error",
+        "eqeqeq": "off",
+        "func-call-spacing": "error",
+        "func-name-matching": "off",
+        "func-names": "off",
+        "func-style": "off",
+        "function-paren-newline": "off",
+        "generator-star-spacing": "error",
+        "global-require": "off",
+        "guard-for-in": "warn",
+        "handle-callback-err": "error",
+        "id-blacklist": "error",
+        "id-length": "off",
+        "id-match": "error",
+        "implicit-arrow-linebreak": "off",
+        "indent": "off",
+        "indent-legacy": "off",
+        "init-declarations": "off",
+        "jsx-quotes": "error",
+        "key-spacing": "error",
+        "keyword-spacing": [
+            "error",
+            {
+                "after": true,
+                "before": true
+            }
+        ],
+        "line-comment-position": "off",
+        "linebreak-style": [
+            "error",
+            "unix"
+        ],
+        "lines-around-comment": "error",
+        "lines-around-directive": "error",
+        "lines-between-class-members": "error",
+        "max-classes-per-file": "error",
+        "max-depth": "error",
+        "max-len": "off",
+        "max-lines": "off",
+        "max-lines-per-function": "off",
+        "max-nested-callbacks": "error",
+        "max-params": "off",
+        "max-statements": "off",
+        "max-statements-per-line": "error",
+        "multiline-comment-style": "off",
+        "new-cap": "error",
+        "new-parens": "error",
+        "newline-after-var": "off",
+        "newline-before-return": "off",
+        "newline-per-chained-call": "off",
+        "no-alert": "error",
+        "no-array-constructor": "error",
+        "no-async-promise-executor": "error",
+        "no-await-in-loop": "error",
+        "no-bitwise": "error",
+        "no-buffer-constructor": "error",
+        "no-caller": "error",
+        "no-catch-shadow": "error",
+        "no-confusing-arrow": "error",
+        "no-continue": "off",
+        "no-constant-condition": "off",
+        "no-div-regex": "error",
+        "no-duplicate-imports": "error",
+        "no-else-return": "off",
+        "no-empty-function": "error",
+        "no-eq-null": "error",
+        "no-eval": "error",
+        "no-extend-native": "error",
+        "no-extra-bind": "error",
+        "no-extra-label": "error",
+        "no-extra-parens": "off",
+        "no-floating-decimal": "error",
+        "no-implicit-globals": "error",
+        "no-implied-eval": "error",
+        "no-inline-comments": "off",
+        "no-invalid-this": "error",
+        "no-iterator": "error",
+        "no-label-var": "error",
+        "no-labels": "error",
+        "no-lone-blocks": "error",
+        "no-lonely-if": "error",
+        "no-loop-func": "error",
+        "no-magic-numbers": "off",
+        "no-misleading-character-class": "error",
+        "no-mixed-operators": "error",
+        "no-mixed-requires": "error",
+        "no-multi-assign": "error",
+        "no-multi-spaces": "off",
+        "no-multi-str": "error",
+        "no-multiple-empty-lines": "error",
+        "no-native-reassign": "error",
+        "no-negated-condition": "error",
+        "no-negated-in-lhs": "error",
+        "no-nested-ternary": "error",
+        "no-new": "error",
+        "no-new-func": "error",
+        "no-new-object": "error",
+        "no-new-require": "error",
+        "no-new-wrappers": "error",
+        "no-octal-escape": "error",
+        "no-param-reassign": "error",
+        "no-path-concat": "error",
+        "no-plusplus": "off",
+        "no-process-env": "error",
+        "no-process-exit": "error",
+        "no-proto": "error",
+        "no-prototype-builtins": "error",
+        "no-restricted-globals": "error",
+        "no-restricted-imports": "error",
+        "no-restricted-modules": "error",
+        "no-restricted-properties": "error",
+        "no-restricted-syntax": "error",
+        "no-return-assign": "error",
+        "no-return-await": "error",
+        "no-script-url": "error",
+        "no-self-compare": "error",
+        "no-sequences": "error",
+        "no-shadow": "error",
+        "no-shadow-restricted-names": "error",
+        "no-spaced-func": "error",
+        "no-sync": "warn",
+        "no-tabs": "error",
+        "no-template-curly-in-string": "error",
+        "no-ternary": "off",
+        "no-throw-literal": "error",
+        "no-trailing-spaces": "error",
+        "no-undef-init": "error",
+        "no-undefined": "off",
+        "no-underscore-dangle": "off",
+        "no-unmodified-loop-condition": "error",
+        "no-unneeded-ternary": "off",
+        "no-unused-expressions": "error",
+        "no-unused-vars": [
+          "error",
+          {
+            "argsIgnorePattern": "^_",
+          },
+        ],
+        "no-use-before-define": "error",
+        "no-useless-call": "error",
+        "no-useless-catch": "error",
+        "no-useless-computed-key": "error",
+        "no-useless-concat": "error",
+        "no-useless-constructor": "error",
+        "no-useless-rename": "error",
+        "no-useless-return": "error",
+        "no-useless-escape": "off",
+        "no-var": "off",
+        "no-void": "error",
+        "no-warning-comments": "warn",
+        "no-whitespace-before-property": "error",
+        "no-with": "error",
+        "nonblock-statement-body-position": "error",
+        "object-curly-newline": "error",
+        "object-curly-spacing": [
+            "error",
+            "always"
+        ],
+        "object-shorthand": "off",
+        "one-var": "off",
+        "one-var-declaration-per-line": "error",
+        "operator-assignment": "error",
+        "operator-linebreak": "error",
+        "padded-blocks": "off",
+        "padding-line-between-statements": "error",
+        "prefer-arrow-callback": "off",
+        "prefer-const": "error",
+        "prefer-destructuring": "off",
+        "prefer-numeric-literals": "error",
+        "prefer-object-spread": "error",
+        "prefer-promise-reject-errors": "error",
+        "prefer-reflect": "off",
+        "prefer-rest-params": "error",
+        "prefer-spread": "error",
+        "prefer-template": "off",
+        "quote-props": "off",
+        "quotes": "off",
+        "radix": "error",
+        "require-atomic-updates": "error",
+        "require-await": "error",
+        "require-jsdoc": "warn",
+        "require-unicode-regexp": "error",
+        "rest-spread-spacing": [
+            "error",
+            "never"
+        ],
+        "semi": "off",
+        "semi-spacing": "error",
+        "semi-style": [
+            "error",
+            "last"
+        ],
+        "sort-imports": "error",
+        "sort-keys": "off",
+        "sort-vars": "error",
+        "space-before-blocks": "error",
+        "space-before-function-paren": "off",
+        "space-in-parens": [
+            "error",
+            "never"
+        ],
+        "space-infix-ops": "error",
+        "space-unary-ops": "error",
+        "spaced-comment": [
+            "error",
+            "always"
+        ],
+        "strict": "error",
+        "switch-colon-spacing": "error",
+        "symbol-description": "error",
+        "template-curly-spacing": [
+            "error",
+            "never"
+        ],
+        "template-tag-spacing": "error",
+        "unicode-bom": [
+            "error",
+            "never"
+        ],
+        "valid-jsdoc": "error",
+        "vars-on-top": "off",
+        "wrap-iife": "error",
+        "wrap-regex": "error",
+        "yield-star-spacing": "error",
+        "yoda": [
+            "error",
+            "never"
+        ]
+    }
+};

storage-node/.gitignore

@@ -0,0 +1,27 @@
+build/
+coverage/
+dist
+tmp/
+.DS_Store
+
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+.npmrc
+package-lock.json
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# IDEs
+.idea
+.vscode
+.*.sw*
+
+# Node modules
+node_modules/
+
+# Ignore nvm config file
+.nvmrc

storage-node/.travis.yml

@@ -0,0 +1,15 @@
+language: node_js
+
+node_js:
+    - 10
+    - 12
+    - 13
+
+services:
+  - docker
+
+script:
+  - docker-compose -f ./scripts/compose/devchain-and-ipfs-node/docker-compose.yaml up -d
+  - yarn test
+  - docker-compose -f ./scripts/compose/devchain-and-ipfs-node/docker-compose.yaml stop
+

storage-node/LICENSE.md

@@ -0,0 +1,675 @@
+### GNU GENERAL PUBLIC LICENSE
+
+Version 3, 29 June 2007
+
+Copyright (C) 2007 Free Software Foundation, Inc.
+<https://fsf.org/>
+
+Everyone is permitted to copy and distribute verbatim copies of this
+license document, but changing it is not allowed.
+
+### Preamble
+
+The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+the GNU General Public License is intended to guarantee your freedom
+to share and change all versions of a program--to make sure it remains
+free software for all its users. We, the Free Software Foundation, use
+the GNU General Public License for most of our software; it applies
+also to any other work released this way by its authors. You can apply
+it to your programs, too.
+
+When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights. Therefore, you
+have certain responsibilities if you distribute copies of the
+software, or if you modify it: responsibilities to respect the freedom
+of others.
+
+For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received. You must make sure that they, too, receive
+or can get the source code. And you must show them these terms so they
+know their rights.
+
+Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software. For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the
+manufacturer can do so. This is fundamentally incompatible with the
+aim of protecting users' freedom to change the software. The
+systematic pattern of such abuse occurs in the area of products for
+individuals to use, which is precisely where it is most unacceptable.
+Therefore, we have designed this version of the GPL to prohibit the
+practice for those products. If such problems arise substantially in
+other domains, we stand ready to extend this provision to those
+domains in future versions of the GPL, as needed to protect the
+freedom of users.
+
+Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish
+to avoid the special danger that patents applied to a free program
+could make it effectively proprietary. To prevent this, the GPL
+assures that patents cannot be used to render the program non-free.
+
+The precise terms and conditions for copying, distribution and
+modification follow.
+
+### TERMS AND CONDITIONS
+
+#### 0. Definitions.
+
+"This License" refers to version 3 of the GNU General Public License.
+
+"Copyright" also means copyright-like laws that apply to other kinds
+of works, such as semiconductor masks.
+
+"The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of
+an exact copy. The resulting work is called a "modified version" of
+the earlier work or a work "based on" the earlier work.
+
+A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user
+through a computer network, with no transfer of a copy, is not
+conveying.
+
+An interactive user interface displays "Appropriate Legal Notices" to
+the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+#### 1. Source Code.
+
+The "source code" for a work means the preferred form of the work for
+making modifications to it. "Object code" means any non-source form of
+a work.
+
+A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+The Corresponding Source need not include anything that users can
+regenerate automatically from other parts of the Corresponding Source.
+
+The Corresponding Source for a work in source code form is that same
+work.
+
+#### 2. Basic Permissions.
+
+All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+You may make, run and propagate covered works that you do not convey,
+without conditions so long as your license otherwise remains in force.
+You may convey covered works to others for the sole purpose of having
+them make modifications exclusively for you, or provide you with
+facilities for running those works, provided that you comply with the
+terms of this License in conveying all material for which you do not
+control copyright. Those thus making or running the covered works for
+you must do so exclusively on your behalf, under your direction and
+control, on terms that prohibit them from making any copies of your
+copyrighted material outside their relationship with you.
+
+Conveying under any other circumstances is permitted solely under the
+conditions stated below. Sublicensing is not allowed; section 10 makes
+it unnecessary.
+
+#### 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such
+circumvention is effected by exercising rights under this License with
+respect to the covered work, and you disclaim any intention to limit
+operation or modification of the work as a means of enforcing, against
+the work's users, your or third parties' legal rights to forbid
+circumvention of technological measures.
+
+#### 4. Conveying Verbatim Copies.
+
+You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+#### 5. Conveying Modified Source Versions.
+
+You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these
+conditions:
+
+-   a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+-   b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under
+    section 7. This requirement modifies the requirement in section 4
+    to "keep intact all notices".
+-   c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy. This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged. This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+-   d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+#### 6. Conveying Non-Source Forms.
+
+You may convey a covered work in object code form under the terms of
+sections 4 and 5, provided that you also convey the machine-readable
+Corresponding Source under the terms of this License, in one of these
+ways:
+
+-   a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+-   b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the Corresponding
+    Source from a network server at no charge.
+-   c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source. This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+-   d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge. You need not require recipients to copy the
+    Corresponding Source along with the object code. If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source. Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+-   e) Convey the object code using peer-to-peer transmission,
+    provided you inform other peers where the object code and
+    Corresponding Source of the work are being offered to the general
+    public at no charge under subsection 6d.
+
+A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal,
+family, or household purposes, or (2) anything designed or sold for
+incorporation into a dwelling. In determining whether a product is a
+consumer product, doubtful cases shall be resolved in favor of
+coverage. For a particular product received by a particular user,
+"normally used" refers to a typical or common use of that class of
+product, regardless of the status of the particular user or of the way
+in which the particular user actually uses, or expects or is expected
+to use, the product. A product is a consumer product regardless of
+whether the product has substantial commercial, industrial or
+non-consumer uses, unless such uses represent the only significant
+mode of use of the product.
+
+"Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to
+install and execute modified versions of a covered work in that User
+Product from a modified version of its Corresponding Source. The
+information must suffice to ensure that the continued functioning of
+the modified object code is in no case prevented or interfered with
+solely because modification has been made.
+
+If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or
+updates for a work that has been modified or installed by the
+recipient, or for the User Product in which it has been modified or
+installed. Access to a network may be denied when the modification
+itself materially and adversely affects the operation of the network
+or violates the rules and protocols for communication across the
+network.
+
+Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+#### 7. Additional Terms.
+
+"Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders
+of that material) supplement the terms of this License with terms:
+
+-   a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+-   b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+-   c) Prohibiting misrepresentation of the origin of that material,
+    or requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+-   d) Limiting the use for publicity purposes of names of licensors
+    or authors of the material; or
+-   e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+-   f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions
+    of it) with contractual assumptions of liability to the recipient,
+    for any liability that these contractual assumptions directly
+    impose on those licensors and authors.
+
+All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions; the
+above requirements apply either way.
+
+#### 8. Termination.
+
+You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+However, if you cease all violation of this License, then your license
+from a particular copyright holder is reinstated (a) provisionally,
+unless and until the copyright holder explicitly and finally
+terminates your license, and (b) permanently, if the copyright holder
+fails to notify you of the violation by some reasonable means prior to
+60 days after the cessation.
+
+Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+#### 9. Acceptance Not Required for Having Copies.
+
+You are not required to accept this License in order to receive or run
+a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+#### 10. Automatic Licensing of Downstream Recipients.
+
+Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+#### 11. Patents.
+
+A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+A contributor's "essential patent claims" are all patent claims owned
+or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+A patent license is "discriminatory" if it does not include within the
+scope of its coverage, prohibits the exercise of, or is conditioned on
+the non-exercise of one or more of the rights that are specifically
+granted under this License. You may not convey a covered work if you
+are a party to an arrangement with a third party that is in the
+business of distributing software, under which you make payment to the
+third party based on the extent of your activity of conveying the
+work, and under which the third party grants, to any of the parties
+who would receive the covered work from you, a discriminatory patent
+license (a) in connection with copies of the covered work conveyed by
+you (or copies made from those copies), or (b) primarily for and in
+connection with specific products or compilations that contain the
+covered work, unless you entered into that arrangement, or that patent
+license was granted, prior to 28 March 2007.
+
+Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+#### 12. No Surrender of Others' Freedom.
+
+If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under
+this License and any other pertinent obligations, then as a
+consequence you may not convey it at all. For example, if you agree to
+terms that obligate you to collect a royalty for further conveying
+from those to whom you convey the Program, the only way you could
+satisfy both those terms and this License would be to refrain entirely
+from conveying the Program.
+
+#### 13. Use with the GNU Affero General Public License.
+
+Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+#### 14. Revised Versions of this License.
+
+The Free Software Foundation may publish revised and/or new versions
+of the GNU General Public License from time to time. Such new versions
+will be similar in spirit to the present version, but may differ in
+detail to address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program
+specifies that a certain numbered version of the GNU General Public
+License "or any later version" applies to it, you have the option of
+following the terms and conditions either of that numbered version or
+of any later version published by the Free Software Foundation. If the
+Program does not specify a version number of the GNU General Public
+License, you may choose any version ever published by the Free
+Software Foundation.
+
+If the Program specifies that a proxy can decide which future versions
+of the GNU General Public License can be used, that proxy's public
+statement of acceptance of a version permanently authorizes you to
+choose that version for the Program.
+
+Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+#### 15. Disclaimer of Warranty.
+
+THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT
+WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND
+PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE
+DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR
+CORRECTION.
+
+#### 16. Limitation of Liability.
+
+IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR
+CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES
+ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT
+NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR
+LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM
+TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER
+PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+#### 17. Interpretation of Sections 15 and 16.
+
+If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+END OF TERMS AND CONDITIONS
+
+### How to Apply These Terms to Your New Programs
+
+If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these
+terms.
+
+To do so, attach the following notices to the program. It is safest to
+attach them to the start of each source file to most effectively state
+the exclusion of warranty; and each file should have at least the
+"copyright" line and a pointer to where the full notice is found.
+
+        <one line to give the program's name and a brief idea of what it does.>
+        Copyright (C) <year>  <name of author>
+
+        This program is free software: you can redistribute it and/or modify
+        it under the terms of the GNU General Public License as published by
+        the Free Software Foundation, either version 3 of the License, or
+        (at your option) any later version.
+
+        This program is distributed in the hope that it will be useful,
+        but WITHOUT ANY WARRANTY; without even the implied warranty of
+        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+        GNU General Public License for more details.
+
+        You should have received a copy of the GNU General Public License
+        along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper
+mail.
+
+If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+        <program>  Copyright (C) <year>  <name of author>
+        This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+        This is free software, and you are welcome to redistribute it
+        under certain conditions; type `show c' for details.
+
+The hypothetical commands \`show w' and \`show c' should show the
+appropriate parts of the General Public License. Of course, your
+program's commands might be different; for a GUI interface, you would
+use an "about box".
+
+You should also get your employer (if you work as a programmer) or
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary. For more information on this, and how to apply and follow
+the GNU GPL, see <https://www.gnu.org/licenses/>.
+
+The GNU General Public License does not permit incorporating your
+program into proprietary programs. If your program is a subroutine
+library, you may consider it more useful to permit linking proprietary
+applications with the library. If this is what you want to do, use the
+GNU Lesser General Public License instead of this License. But first,
+please read <https://www.gnu.org/licenses/why-not-lgpl.html>.

storage-node/README.md

@@ -0,0 +1,56 @@
+![Storage Nodes for Joystream](./storage-node_new.svg)
+
+This repository contains several Node packages, located under the `packages/`
+subdirectory. See each individual package for details:
+
+* [colossus](./packages/colossus/README.md) - the main colossus app.
+* [storage](./packages/storage/README.md) - abstraction over the storage backend.
+* [runtime-api](./packages/runtime-api/README.md) - convenience wrappers for the runtime API.
+* [crypto](./packages/crypto/README.md) - cryptographic utility functions.
+* [util](./packages/util/README.md) - general utility functions.
+* [discovery](./packages/discovery/README.md) - service discovery using IPNS.
+
+Installation
+------------
+
+*Requirements*
+
+This project uses [yarn](https://yarnpkg.com/) as Node package manager. It also
+uses some node packages with native components, so make sure to install your
+system's basic build tools.
+
+On Debian-based systems:
+
+```bash
+$ apt install build-essential
+```
+
+On Mac OS (using [homebrew](https://brew.sh/)):
+
+```bash
+$ brew install libtool automake autoconf
+```
+
+*Building*
+
+```bash
+$ yarn install
+```
+
+The command will install dependencies, and make a `colossus` executable available:
+
+```bash
+$ yarn run colossus --help
+```
+
+*Testing*
+
+Running tests from the repository root will run tests from all packages:
+
+```
+$ yarn run test
+```
+
+
+## Detailed Setup and Configuration Guide
+For details on how to setup a storage node on the Joystream network, follow this [step by step guide](https://github.com/Joystream/helpdesk/tree/master/roles/storage-providers).

storage-node/docs/json-signing.md

@@ -0,0 +1,54 @@
+# JSON Data Signing
+
+As serializing and deserializing JSON is not deterministic, but may depend
+on the order in which keys are added or even the system's collation method,
+signing JSON cryptographically is fraught with issues. We circumvent them
+by wrapping any JSON to be signed in another JSON object:
+
+* `version` contains the version of the wrapper JSON, currently always `1`.
+* `serialized` contains the serialized version of the data, currently this
+  will be the base64 encoded, serialized JSON payload.
+* `signature` contains the base64 encoded signature of the `serialized` field
+  value prior to its base64 encoding.
+* `payload` [optional] contains the deserialized JSON object corresponding
+  to the `serialized` payload.
+
+For signing and verification, we'll use polkadot's *ed25519* or *sr25519* keys
+directly.
+
+## Signing Process
+
+Given some structured data:
+
+1. Serialize the structured data into a JSON string.
+1. Create a signature over the serialized JSON string.
+1. Create a new structured data with the appropriate `version` field.
+1. Add a base64 encoded version of the serialized JSON string as the `serialized` field.
+1. Add a base64 encoded version of the signature as the `signature` field.
+1. Optionally add the original structured data as the `payload` field.
+
+## Verification Process
+
+1. Verify data contains a `version`, `serialized` and `signature` field.
+1. Currently, verify that the `version` field's value is `1`.
+1. Try to base64 decode the `serialized` and `signature` fields.
+1. Verify that the decoded `signature` is valid for the decoded `serialized`
+  field.
+1. JSON deserialize the decoded `serialized` field.
+1. Add the resulting structured data as the `payload` field, and return the
+  modified object.
+
+# Alternatives
+
+There are alternative schemes available for signing JSON objects, but they
+have specific issues we'd like to avoid.
+
+* [JOSE](https://jose.readthedocs.io/en/latest/) has no support for the *ed25519*
+  or *sr25519* keys used in polkadot apps, and
+  [appears to be fraught with security issues](https://paragonie.com/blog/2017/03/jwt-json-web-tokens-is-bad-standard-that-everyone-should-avoid).
+  Either makes its use hard to justify.
+* While [PASETO](https://paseto.io/) does use *ed25519* keys and seems to have
+  a reasonably robuts JavaScript implementation, it requires its secret keys to
+  be 512 bits long, while polkadot provides 256 bit secret keys. The implication
+  is that we would have to manage 512 bit keys and their corresponding public
+  keys as linked to polkadot's keys, which is cumbersome at the very least.

storage-node/license_header.txt

@@ -0,0 +1,18 @@
+/*
+ * This file is part of the storage node for the Joystream project.
+ * Copyright (C) 2019 Joystream Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+

storage-node/package.json

@@ -0,0 +1,43 @@
+{
+  "private": true,
+  "name": "@joystream/storage-node",
+  "version": "1.0.0",
+  "engines": {
+    "node": ">=10.15.3",
+    "yarn": "^1.15.2"
+  },
+  "homepage": "https://github.com/Joystream/joystream/",
+  "bugs": {
+    "url": "https://github.com/Joystream/joystream/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Joystream/joystream.git"
+  },
+  "license": "GPL-3.0",
+  "contributors": [
+    {
+      "name": "Joystream",
+      "url": "https://joystream.org"
+    }
+  ],
+  "keywords": [
+    "joystream",
+    "storage",
+    "node"
+  ],
+  "os": [
+    "darwin",
+    "linux"
+  ],
+  "workspaces": [
+    "packages/*"
+  ],
+  "scripts": {
+    "test": "wsrun --serial test",
+    "lint": "wsrun --serial lint"
+  },
+  "devDependencies": {
+    "wsrun": "^3.6.5"
+  }
+}

storage-node/packages/cli/README.md

@@ -0,0 +1,5 @@
+# A CLI for the Joystream Runtime & Colossus
+
+- CLI access for some functionality from `@joystream/runtime-api`
+- Colossus/storage node functionality:
+  - File uploads

storage-node/packages/cli/bin/cli.js

@@ -0,0 +1,230 @@
+#!/usr/bin/env node
+/*
+ * This file is part of the storage node for the Joystream project.
+ * Copyright (C) 2019 Joystream Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+const assert = require('assert');
+
+const { RuntimeApi } = require('@joystream/runtime-api');
+
+const meow = require('meow');
+const chalk = require('chalk');
+const _ = require('lodash');
+
+const debug = require('debug')('joystream:cli');
+
+// Project root
+const project_root = path.resolve(__dirname, '..');
+
+// Configuration (default)
+const pkg = require(path.resolve(project_root, 'package.json'));
+
+// Parse CLI
+const FLAG_DEFINITIONS = {
+  // TODO
+};
+
+const cli = meow(`
+  Usage:
+    $ joystream key_file command [options]
+
+  All commands require a key file holding the identity for interacting with the
+  runtime API.
+
+  Commands:
+    upload            Upload a file to a Colossus storage node. Requires a
+                      storage node URL, and a local file name to upload. As
+                      an optional third parameter, you can provide a Data
+                      Object Type ID - this defaults to "1" if not provided.
+    download          Retrieve a file. Requires a storage node URL and a content
+                      ID, as well as an output filename.
+    head              Send a HEAD request for a file, and print headers.
+                      Requires a storage node URL and a content ID.
+  `,
+  { flags: FLAG_DEFINITIONS });
+
+function assert_file(name, filename)
+{
+  assert(filename, `Need a ${name} parameter to proceed!`);
+  assert(fs.statSync(filename).isFile(), `Path "${filename}" is not a file, aborting!`);
+}
+
+const commands = {
+  'upload': async (runtime_api, url, filename, do_type_id) => {
+    // Check parameters
+    assert_file('file', filename);
+
+    const size = fs.statSync(filename).size;
+    console.log(`File "${filename}" is ` + chalk.green(size) + ' Bytes.');
+
+    if (!do_type_id) {
+      do_type_id = 1;
+    }
+    console.log('Data Object Type ID is: ' + chalk.green(do_type_id));
+
+    // Generate content ID
+    // FIXME this require path is like this because of
+    // https://github.com/Joystream/apps/issues/207
+    const { ContentId } = require('@joystream/types/lib/media');
+    var cid = ContentId.generate();
+    cid = cid.encode().toString();
+    console.log('Generated content ID: ' + chalk.green(cid));
+
+    // Create Data Object
+    const data_object = await runtime_api.assets.createDataObject(
+      runtime_api.identities.key.address, cid, do_type_id, size);
+    console.log('Data object created.');
+
+    // TODO in future, optionally contact liaison here?
+    const request = require('request');
+    url = `${url}asset/v0/${cid}`;
+    console.log('Uploading to URL', chalk.green(url));
+
+    const f = fs.createReadStream(filename);
+    const opts = {
+      url: url,
+      headers: {
+        'content-type': '',
+        'content-length': `${size}`,
+      },
+      json: true,
+    };
+    return new Promise((resolve, reject) => {
+      const r = request.put(opts, (error, response, body) => {
+        if (error) {
+          reject(error);
+          return;
+        }
+
+        if (response.statusCode / 100 != 2) {
+          reject(new Error(`${response.statusCode}: ${body.message || 'unknown reason'}`));
+          return;
+        }
+        console.log('Upload successful:', body.message);
+        resolve();
+      });
+      f.pipe(r);
+    });
+  },
+
+  'download': async (runtime_api, url, content_id, filename) => {
+    const request = require('request');
+    url = `${url}asset/v0/${content_id}`;
+    console.log('Downloading URL', chalk.green(url), 'to', chalk.green(filename));
+
+    const f = fs.createWriteStream(filename);
+    const opts = {
+      url: url,
+      json: true,
+    };
+    return new Promise((resolve, reject) => {
+      const r = request.get(opts, (error, response, body) => {
+        if (error) {
+          reject(error);
+          return;
+        }
+
+        console.log('Downloading', chalk.green(response.headers['content-type']), 'of size', chalk.green(response.headers['content-length']), '...');
+
+        f.on('error', (err) => {
+          reject(err);
+        });
+
+        f.on('finish', () => {
+          if (response.statusCode / 100 != 2) {
+            reject(new Error(`${response.statusCode}: ${body.message || 'unknown reason'}`));
+            return;
+          }
+          console.log('Download completed.');
+          resolve();
+        });
+      });
+      r.pipe(f);
+    });
+  },
+
+  'head': async (runtime_api, url, content_id) => {
+    const request = require('request');
+    url = `${url}asset/v0/${content_id}`;
+    console.log('Checking URL', chalk.green(url), '...');
+
+    const opts = {
+      url: url,
+      json: true,
+    };
+    return new Promise((resolve, reject) => {
+      const r = request.head(opts, (error, response, body) => {
+        if (error) {
+          reject(error);
+          return;
+        }
+
+        if (response.statusCode / 100 != 2) {
+          reject(new Error(`${response.statusCode}: ${body.message || 'unknown reason'}`));
+          return;
+        }
+
+        for (var propname in response.headers) {
+          console.log(`  ${chalk.yellow(propname)}: ${response.headers[propname]}`);
+        }
+
+        resolve();
+      });
+    });
+  },
+
+};
+
+
+async function main()
+{
+  // Key file is at the first instance.
+  const key_file = cli.input[0];
+  assert_file('key file', key_file);
+
+  // Create runtime API.
+  const runtime_api = await RuntimeApi.create({ account_file: key_file });
+
+  // Simple CLI commands
+  const command = cli.input[1];
+  if (!command) {
+    throw new Error('Need a command to run!');
+  }
+
+  if (commands.hasOwnProperty(command)) {
+    // Command recognized
+    const args = _.clone(cli.input).slice(2);
+    await commands[command](runtime_api, ...args);
+  }
+  else {
+    throw new Error(`Command "${command}" not recognized, aborting!`);
+  }
+}
+
+main()
+  .then(() => {
+    console.log('Process exiting gracefully.');
+    process.exit(0);
+  })
+  .catch((err) => {
+    console.error(chalk.red(err.stack));
+    process.exit(-1);
+  });

storage-node/packages/cli/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@joystream/storage-cli",
+  "version": "0.1.0",
+  "description": "Joystream tool for uploading and downloading files to the network",
+  "author": "Joystream",
+  "homepage": "https://github.com/Joystream/joystream",
+  "bugs": {
+    "url": "https://github.com/Joystream/joystream/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Joystream/joystream.git"
+  },
+  "license": "GPL-3.0",
+  "contributors": [
+    {
+      "name": "Joystream",
+      "url": "https://joystream.org"
+    }
+  ],
+  "os": [
+    "darwin",
+    "linux"
+  ],
+  "engines": {
+    "node": ">=10.15.3"
+  },
+  "scripts": {
+    "test": "mocha 'test/**/*.js'",
+    "lint": "eslint 'paths/**/*.js' 'lib/**/*.js'"
+  },
+  "bin": {
+    "joystream": "bin/cli.js"
+  },
+  "devDependencies": {
+    "chai": "^4.2.0",
+    "eslint": "^5.13.0",
+    "mocha": "^5.2.0",
+    "temp": "^0.9.0"
+  },
+  "dependencies": {
+    "@joystream/runtime-api": "^0.1.0",
+    "chalk": "^2.4.2",
+    "lodash": "^4.17.11",
+    "meow": "^5.0.0",
+    "request": "^2.88.0"
+  }
+}

storage-node/packages/cli/test/index.js

@@ -0,0 +1 @@
+// Add Tests!

storage-node/packages/colossus/.eslintrc.js

@@ -0,0 +1 @@
+../../.eslintrc.js

storage-node/packages/colossus/README.md

@@ -0,0 +1,94 @@
+![Storage Nodes for Joystream](../../banner.svg)
+
+Development
+-----------
+
+Run a development server:
+
+```bash
+$ yarn run dev --config myconfig.json
+```
+
+Command-Line
+------------
+
+Running a storage server is (almost) as easy as running the bundled `colossus`
+executable:
+
+```bash
+$ colossus --storage=/path/to/storage/directory
+```
+
+Run with `--help` to see a list of available CLI options.
+
+You need to stake as a storage provider to run a storage node.
+
+Configuration
+-------------
+
+Most common configuration options are available as command-line options
+for the CLI.
+
+However, some advanced configuration options are only possible to set
+via the configuration file.
+
+* `filter` is a hash of upload filtering options.
+  * `max_size` sets the maximum permissible file upload size. If unset,
+    this defaults to 100 MiB.
+  * `mime` is a hash of...
+    * `accept` is an Array of mime types that are acceptable for uploads,
+      such as `text/plain`, etc. Mime types can also be specified for
+      wildcard matching, such as `video/*`.
+    * `reject` is an Array of mime types that are unacceptable for uploads.
+
+Upload Filtering
+----------------
+
+The upload filtering logic first tests whether any of the `accept` mime types
+are matched. If none are matched, the upload is rejected. If any is matched,
+then the upload is still rejected if any of the `reject` mime types are
+matched.
+
+This allows inclusive and exclusive filtering.
+
+* `{ accept: ['text/plain', 'text/html'] }` accepts *only* the two given mime types.
+* `{ accept: ['text/*'], reject: ['text/plain'] }` accepts any `text/*` that is not
+  `text/plain`.
+
+More advanced filtering is currently not available.
+
+API Packages
+------------
+
+Since it's not entirely clear yet how APIs will develop in future, the approach
+taken here is to package individual APIs up individually. That is, instead of
+providing an overall API version in `api-base.yml`, it should be part of each
+API package's path.
+
+For example, for a `foo` API in its version `v1`, its definitions should live
+in `./paths/foo/v1.js` and `./paths/foo/v1/*.js` respectively.
+
+*Note:* until a reasonably stable API is reached, this project uses a `v0`
+version prefix.
+
+Interface/implementation
+------------------------
+
+For reusability across API versions, it's best to keep files in the `paths`
+subfolder very thin, and instead inject implementations via the `dependencies`
+configuration value of `express-openapi`.
+
+These implementations line to the `./lib` subfolder. Adjust `server.js` as
+needed to make them available to API packages.
+
+Streaming Notes
+---------------
+
+For streaming content, it is required that stream metadata is located at the
+start of the stream. Most software writes metadata at the end of the stream,
+because it is when the stream is committed to disk that the entirety of the
+metadata is known.
+
+To move metadata to the start of the stream, a CLI tool such as
+[qtfaststart](https://github.com/danielgtaylor/qtfaststart) for MP4 files might
+be used.

storage-node/packages/colossus/api-base.yml

@@ -0,0 +1,33 @@
+openapi: '3.0.0'
+info:
+  title: 'Joystream Storage Node API.'
+  version: '1.0.0'
+paths: {}  # Will be populated by express-openapi
+
+components:
+  # Re-usable parameter definitions
+  parameters: {}
+
+  # Re-usable (response) object definitions
+  schemas:
+    Error:
+      required:
+        - message
+      properties:
+        code:
+          type: integer
+          format: int32
+        message:
+          type: string
+
+    ContentDirectoryEntry: # TODO implement
+      required:
+        - name
+      properties:
+        name:
+          type: string
+
+    ContentDirectoryEntries:
+      type: array
+      items:
+        $ref: '#/components/schemas/ContentDirectoryEntry'

storage-node/packages/colossus/bin/cli.js

@@ -0,0 +1,397 @@
+#!/usr/bin/env node
+'use strict';
+
+// Node requires
+const path = require('path');
+
+// npm requires
+const meow = require('meow');
+const configstore = require('configstore');
+const chalk = require('chalk');
+const figlet = require('figlet');
+const _ = require('lodash');
+
+const debug = require('debug')('joystream:cli');
+
+// Project root
+const PROJECT_ROOT = path.resolve(__dirname, '..');
+
+// Configuration (default)
+const pkg = require(path.resolve(PROJECT_ROOT, 'package.json'));
+const default_config = new configstore(pkg.name);
+
+// Parse CLI
+const FLAG_DEFINITIONS = {
+  port: {
+    type: 'integer',
+    alias: 'p',
+    _default: 3000,
+  },
+  'syncPeriod': {
+    type: 'integer',
+    _default: 120000,
+  },
+  keyFile: {
+    type: 'string',
+  },
+  config: {
+    type: 'string',
+    alias: 'c',
+  },
+  'publicUrl': {
+    type: 'string',
+    alias: 'u'
+  },
+  'passphrase': {
+    type: 'string'
+  },
+  'wsProvider': {
+    type: 'string',
+    _default: 'ws://localhost:9944'
+  }
+};
+
+const cli = meow(`
+  Usage:
+    $ colossus [command] [options]
+
+  Commands:
+    server [default]  Run a server instance with the given configuration.
+    signup            Sign up as a storage provider. Requires that you provide
+                      a JSON account file of an account that is a member, and has
+                      sufficient balance for staking as a storage provider.
+                      Writes a new account file that should be used to run the
+                      storage node.
+    down              Signal to network that all services are down. Running
+                      the server will signal that services as online again.
+    discovery         Run the discovery service only.
+
+  Options:
+    --config=PATH, -c PATH  Configuration file path. Defaults to
+                            "${default_config.path}".
+    --port=PORT, -p PORT    Port number to listen on, defaults to 3000.
+    --sync-period           Number of milliseconds to wait between synchronization
+                            runs. Defaults to 30,000 (30s).
+    --key-file              JSON key export file to use as the storage provider.
+    --passphrase            Optional passphrase to use to decrypt the key-file (if its encrypted).
+    --public-url            API Public URL to announce. No URL will be announced if not specified.
+    --ws-provider           Joystream Node websocket provider url, eg: "ws://127.0.0.1:9944"
+  `,
+  { flags: FLAG_DEFINITIONS });
+
+// Create configuration
+function create_config(pkgname, flags)
+{
+  // Create defaults from flag definitions
+  const defaults = {};
+  for (var key in FLAG_DEFINITIONS) {
+    const defs = FLAG_DEFINITIONS[key];
+    if (defs._default) {
+      defaults[key] = defs._default;
+    }
+  }
+
+  // Provide flags as defaults. Anything stored in the config overrides.
+  var config = new configstore(pkgname, defaults, { configPath: flags.config });
+
+  // But we want the flags to also override what's stored in the config, so
+  // set them all.
+  for (var key in flags) {
+    // Skip aliases and self-referential config flag
+    if (key.length == 1 || key === 'config') continue;
+    // Skip sensitive flags
+    if (key == 'passphrase') continue;
+    // Skip unset flags
+    if (!flags[key]) continue;
+    // Otherwise set.
+    config.set(key, flags[key]);
+  }
+
+  debug('Configuration at', config.path, config.all);
+  return config;
+}
+
+// All-important banner!
+function banner()
+{
+  console.log(chalk.blue(figlet.textSync('joystream', 'Speed')));
+}
+
+function start_express_app(app, port) {
+  const http = require('http');
+  const server = http.createServer(app);
+
+  return new Promise((resolve, reject) => {
+    server.on('error', reject);
+    server.on('close', (...args) => {
+      console.log('Server closed, shutting down...');
+      resolve(...args);
+    });
+    server.on('listening', () => {
+      console.log('API server started.', server.address());
+    });
+    server.listen(port, '::');
+    console.log('Starting API server...');
+  });
+}
+// Start app
+function start_all_services(store, api, config)
+{
+  const app = require('../lib/app')(PROJECT_ROOT, store, api, config);
+  const port = config.get('port');
+  return start_express_app(app, port);
+}
+
+// Start discovery service app
+function start_discovery_service(api, config)
+{
+  const app = require('../lib/discovery')(PROJECT_ROOT, api, config);
+  const port = config.get('port');
+  return start_express_app(app, port);
+}
+
+// Get an initialized storage instance
+function get_storage(runtime_api, config)
+{
+  // TODO at some point, we can figure out what backend-specific connection
+  // options make sense. For now, just don't use any configuration.
+  const { Storage } = require('@joystream/storage');
+
+  const options = {
+    resolve_content_id: async (content_id) => {
+      // Resolve via API
+      const obj = await runtime_api.assets.getDataObject(content_id);
+      if (!obj || obj.isNone) {
+        return;
+      }
+
+      return obj.unwrap().ipfs_content_id.toString();
+    },
+  };
+
+  return Storage.create(options);
+}
+
+async function run_signup(account_file, provider_url)
+{
+  if (!account_file) {
+    console.log('Cannot proceed without keyfile');
+    return
+  }
+
+  const { RuntimeApi } = require('@joystream/runtime-api');
+  const api = await RuntimeApi.create({account_file, canPromptForPassphrase: true, provider_url});
+
+  if (!api.identities.key) {
+    console.log('Cannot proceed without a member account');
+    return
+  }
+
+  // Check there is an opening
+  let availableSlots = await api.roles.availableSlotsForRole(api.roles.ROLE_STORAGE);
+
+  if (availableSlots == 0) {
+    console.log(`
+      There are no open storage provider slots available at this time.
+      Please try again later.
+    `);
+    return;
+  } else {
+    console.log(`There are still ${availableSlots} slots available, proceeding`);
+  }
+
+  const member_address = api.identities.key.address;
+
+  // Check if account works
+  const min = await api.roles.requiredBalanceForRoleStaking(api.roles.ROLE_STORAGE);
+  console.log(`Account needs to be a member and have a minimum balance of ${min.toString()}`);
+  const check = await api.roles.checkAccountForStaking(member_address);
+  if (check) {
+    console.log('Account is working for staking, proceeding.');
+  }
+
+  // Create a role key
+  const role_key = await api.identities.createRoleKey(member_address);
+  const role_address = role_key.address;
+  console.log('Generated', role_address, '- this is going to be exported to a JSON file.\n',
+    ' You can provide an empty passphrase to make starting the server easier,\n',
+    ' but you must keep the file very safe, then.');
+  const filename = await api.identities.writeKeyPairExport(role_address);
+  console.log('Identity stored in', filename);
+
+  // Ok, transfer for staking.
+  await api.roles.transferForStaking(member_address, role_address, api.roles.ROLE_STORAGE);
+  console.log('Funds transferred.');
+
+  // Now apply for the role
+  await api.roles.applyForRole(role_address, api.roles.ROLE_STORAGE, member_address);
+  console.log('Role application sent.\nNow visit Roles > My Requests in the app.');
+}
+
+async function wait_for_role(config)
+{
+  // Load key information
+  const { RuntimeApi } = require('@joystream/runtime-api');
+  const keyFile = config.get('keyFile');
+  if (!keyFile) {
+    throw new Error("Must specify a key file for running a storage node! Sign up for the role; see `colussus --help' for details.");
+  }
+  const wsProvider = config.get('wsProvider');
+
+  const api = await RuntimeApi.create({
+    account_file: keyFile,
+    passphrase: cli.flags.passphrase,
+    provider_url: wsProvider,
+  });
+
+  if (!api.identities.key) {
+    throw new Error('Failed to unlock storage provider account');
+  }
+
+  // Wait for the account role to be finalized
+  console.log('Waiting for the account to be staked as a storage provider role...');
+  const result = await api.roles.waitForRole(api.identities.key.address, api.roles.ROLE_STORAGE);
+  return [result, api];
+}
+
+function get_service_information(config) {
+  // For now assume we run all services on the same endpoint
+  return({
+    asset: {
+      version: 1, // spec version
+      endpoint: config.get('publicUrl')
+    },
+    discover: {
+      version: 1, // spec version
+      endpoint: config.get('publicUrl')
+    }
+  })
+}
+
+async function announce_public_url(api, config) {
+  // re-announce in future
+  const reannounce = function (timeoutMs) {
+    setTimeout(announce_public_url, timeoutMs, api, config);
+  }
+
+  debug('announcing public url')
+  const { publish } = require('@joystream/discovery')
+
+  const accountId = api.identities.key.address
+
+  try {
+    const serviceInformation = get_service_information(config)
+
+    let keyId = await publish.publish(serviceInformation);
+
+    const expiresInBlocks = 600; // ~ 1 hour (6s block interval)
+    await api.discovery.setAccountInfo(accountId, keyId, expiresInBlocks);
+
+    debug('publishing complete, scheduling next update')
+
+// >> sometimes after tx is finalized.. we are not reaching here!
+
+    // Reannounce before expiery
+    reannounce(50 * 60 * 1000); // in 50 minutes
+
+  } catch (err) {
+    debug(`announcing public url failed: ${err.stack}`)
+
+    // On failure retry sooner
+    debug(`announcing failed, retrying in: 2 minutes`)
+    reannounce(120 * 1000)
+  }
+}
+
+function go_offline(api) {
+  return api.discovery.unsetAccountInfo(api.identities.key.address)
+}
+
+// Simple CLI commands
+var command = cli.input[0];
+if (!command) {
+  command = 'server';
+}
+
+const commands = {
+  'server': async () => {
+    const cfg = create_config(pkg.name, cli.flags);
+
+    // Load key information
+    const values = await wait_for_role(cfg);
+    const result = values[0]
+    const api = values[1];
+    if (!result) {
+      throw new Error(`Not staked as storage role.`);
+    }
+    console.log('Staked, proceeding.');
+
+    // Make sure a public URL is configured
+    if (!cfg.get('publicUrl')) {
+      throw new Error('publicUrl not configured')
+    }
+
+    // Continue with server setup
+    const store = get_storage(api, cfg);
+    banner();
+
+    const { start_syncing } = require('../lib/sync');
+    start_syncing(api, cfg, store);
+
+    announce_public_url(api, cfg);
+    await start_all_services(store, api, cfg);
+  },
+  'signup': async (account_file) => {
+    const cfg = create_config(pkg.name, cli.flags);
+    await run_signup(account_file, cfg.get('wsProvider'));
+  },
+  'down': async () => {
+    const cfg = create_config(pkg.name, cli.flags);
+
+    const values = await wait_for_role(cfg);
+    const result = values[0]
+    const api = values[1];
+    if (!result) {
+      throw new Error(`Not staked as storage role.`);
+    }
+
+    await go_offline(api)
+  },
+  'discovery': async () => {
+    debug("Starting Joystream Discovery Service")
+    const { RuntimeApi } = require('@joystream/runtime-api')
+    const cfg = create_config(pkg.name, cli.flags)
+    const wsProvider = cfg.get('wsProvider');
+    const api = await RuntimeApi.create({ provider_url: wsProvider });
+    await start_discovery_service(api, cfg)
+  }
+};
+
+
+async function main()
+{
+  // Simple CLI commands
+  var command = cli.input[0];
+  if (!command) {
+    command = 'server';
+  }
+
+  if (commands.hasOwnProperty(command)) {
+    // Command recognized
+    const args = _.clone(cli.input).slice(1);
+    await commands[command](...args);
+  }
+  else {
+    throw new Error(`Command "${command}" not recognized, aborting!`);
+  }
+}
+
+main()
+  .then(() => {
+    console.log('Process exiting gracefully.');
+    process.exit(0);
+  })
+  .catch((err) => {
+    console.error(chalk.red(err.stack));
+    process.exit(-1);
+  });

storage-node/packages/colossus/lib/app.js

@@ -0,0 +1,78 @@
+/*
+ * This file is part of the storage node for the Joystream project.
+ * Copyright (C) 2019 Joystream Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+'use strict';
+
+// Node requires
+const fs = require('fs');
+const path = require('path');
+
+// npm requires
+const express = require('express');
+const openapi = require('express-openapi');
+const bodyParser = require('body-parser');
+const cors = require('cors');
+const yaml = require('js-yaml');
+
+// Project requires
+const validateResponses = require('./middleware/validate_responses');
+const fileUploads = require('./middleware/file_uploads');
+const pagination = require('@joystream/util/pagination');
+const storage = require('@joystream/storage');
+
+// Configure app
+function create_app(project_root, storage, runtime, config)
+{
+  const app = express();
+  app.use(cors());
+  app.use(bodyParser.json());
+  // FIXME app.use(bodyParser.urlencoded({ extended: true }));
+
+  // Load & extend/configure API docs
+  var api = yaml.safeLoad(fs.readFileSync(
+    path.resolve(project_root, 'api-base.yml')));
+  api['x-express-openapi-additional-middleware'] = [validateResponses];
+  api['x-express-openapi-validation-strict'] = true;
+
+  api = pagination.openapi(api);
+
+  openapi.initialize({
+    apiDoc: api,
+    app: app,
+    paths: path.resolve(project_root, 'paths'),
+    docsPath: '/swagger.json',
+    consumesMiddleware: {
+      'multipart/form-data': fileUploads
+    },
+    dependencies: {
+      config: config,
+      storage: storage,
+      runtime: runtime,
+    },
+  });
+
+  // If no other handler gets triggered (errors), respond with the
+  // error serialized to JSON.
+  app.use(function(err, req, res, next) {
+    res.status(err.status).json(err);
+  });
+
+  return app;
+}
+
+module.exports = create_app;

storage-node/packages/colossus/lib/discovery.js

@@ -0,0 +1,73 @@
+/*
+ * This file is part of the storage node for the Joystream project.
+ * Copyright (C) 2019 Joystream Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+'use strict';
+
+// npm requires
+const express = require('express');
+const openapi = require('express-openapi');
+const bodyParser = require('body-parser');
+const cors = require('cors');
+const yaml = require('js-yaml');
+
+// Node requires
+const fs = require('fs');
+const path = require('path');
+
+// Project requires
+const validateResponses = require('./middleware/validate_responses');
+
+// Configure app
+function create_app(project_root, runtime, config)
+{
+  const app = express();
+  app.use(cors());
+  app.use(bodyParser.json());
+  // FIXME app.use(bodyParser.urlencoded({ extended: true }));
+
+  // Load & extend/configure API docs
+  var api = yaml.safeLoad(fs.readFileSync(
+    path.resolve(project_root, 'api-base.yml')));
+  api['x-express-openapi-additional-middleware'] = [validateResponses];
+  api['x-express-openapi-validation-strict'] = true;
+
+  openapi.initialize({
+    apiDoc: api,
+    app: app,
+    //paths: path.resolve(project_root, 'discovery_app_paths'),
+    paths: {
+      path: '/discover/v0/{id}',
+      module: require('../paths/discover/v0/{id}')
+    },
+    docsPath: '/swagger.json',
+    dependencies: {
+      config: config,
+      runtime: runtime,
+    },
+  });
+
+  // If no other handler gets triggered (errors), respond with the
+  // error serialized to JSON.
+  app.use(function(err, req, res, next) {
+    res.status(err.status).json(err);
+  });
+
+  return app;
+}
+
+module.exports = create_app;

storage-node/packages/colossus/lib/middleware/file_uploads.js

@@ -0,0 +1,44 @@
+/*
+ * This file is part of the storage node for the Joystream project.
+ * Copyright (C) 2019 Joystream Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+'use strict';
+
+const multer = require('multer');
+
+// Taken from express-openapi examples
+module.exports = function(req, res, next)
+{
+  multer().any()(req, res, function(err) {
+    if (err) {
+      return next(err);
+    }
+    // Handle both single and multiple files
+    const filesMap = req.files.reduce(
+      (acc, f) =>
+        Object.assign(acc, {
+          [f.fieldname]: (acc[f.fieldname] || []).concat(f)
+        }),
+      {}
+    );
+    Object.keys(filesMap).forEach((fieldname) => {
+      const files = filesMap[fieldname];
+      req.body[fieldname] = files.length > 1 ? files.map(() => '') : '';
+    });
+    return next();
+  });
+}

storage-node/packages/colossus/lib/middleware/validate_responses.js

@@ -0,0 +1,61 @@
+/*
+ * This file is part of the storage node for the Joystream project.
+ * Copyright (C) 2019 Joystream Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+'use strict';
+
+const debug = require('debug')('joystream:middleware:validate');
+
+// Function taken directly from https://github.com/kogosoftwarellc/open-api/tree/master/packages/express-openapi
+module.exports = function(req, res, next)
+{
+  const strictValidation = req.apiDoc['x-express-openapi-validation-strict'] ? true : false;
+  if (typeof res.validateResponse === 'function') {
+    const send = res.send;
+    res.send = function expressOpenAPISend(...args) {
+      const onlyWarn = !strictValidation;
+      if (res.get('x-express-openapi-validation-error-for') !== undefined) {
+        return send.apply(res, args);
+      }
+      if (res.get('x-express-openapi-validation-for') !== undefined) {
+        return send.apply(res, args);
+      }
+
+      const body = args[0];
+      let validation = res.validateResponse(res.statusCode, body);
+      let validationMessage;
+      if (validation === undefined) {
+        validation = { message: undefined, errors: undefined };
+      }
+      if (validation.errors) {
+        const errorList = Array.from(validation.errors).map((_) => _.message).join(',');
+        validationMessage = `Invalid response for status code ${res.statusCode}: ${errorList}`;
+        debug(validationMessage);
+        // Set to avoid a loop, and to provide the original status code
+        res.set('x-express-openapi-validation-error-for', res.statusCode.toString());
+      }
+      if ((onlyWarn || !validation.errors) && res.statusCode) {
+        res.set('x-express-openapi-validation-for', res.statusCode.toString());
+        return send.apply(res, args);
+      } else {
+        res.status(500);
+        return res.json({ error: validationMessage });
+      }
+    }
+  }
+  next();
+}

storage-node/packages/colossus/lib/sync.js

@@ -0,0 +1,108 @@
+/*
+ * This file is part of the storage node for the Joystream project.
+ * Copyright (C) 2019 Joystream Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+'use strict';
+
+const debug = require('debug')('joystream:sync');
+
+async function sync_callback(api, config, storage)
+{
+  debug('Starting sync run...');
+
+  // The first step is to gather all data objects from chain.
+  // TODO: in future, limit to a configured tranche
+  // FIXME this isn't actually on chain yet, so we'll fake it.
+  const knownContentIds = await api.assets.getKnownContentIds() || [];
+
+  const role_addr = api.identities.key.address;
+
+  // Iterate over all sync objects, and ensure they're synced.
+  const allChecks = knownContentIds.map(async (content_id) => {
+    let { relationship, relationshipId } = await api.assets.getStorageRelationshipAndId(role_addr, content_id);
+
+    let fileLocal;
+    try {
+      // check if we have content or not
+      let stats = await storage.stat(content_id);
+      fileLocal = stats.local;
+    } catch (err) {
+      // on error stating or timeout
+      debug(err.message);
+      // we don't have content if we can't stat it
+      fileLocal = false;
+    }
+
+    if (!fileLocal) {
+      try {
+        await storage.synchronize(content_id);
+      } catch (err) {
+        debug(err.message)
+      }
+      return;
+    }
+
+    if (!relationship) {
+      // create relationship
+      debug(`Creating new storage relationship for ${content_id.encode()}`);
+      try {
+        relationshipId = await api.assets.createAndReturnStorageRelationship(role_addr, content_id);
+        await api.assets.toggleStorageRelationshipReady(role_addr, relationshipId, true);
+      } catch (err) {
+        debug(`Error creating new storage relationship ${content_id.encode()}: ${err.stack}`);
+        return;
+      }
+    } else if (!relationship.ready) {
+      debug(`Updating storage relationship to ready for ${content_id.encode()}`);
+      // update to ready. (Why would there be a relationship set to ready: false?)
+      try {
+        await api.assets.toggleStorageRelationshipReady(role_addr, relationshipId, true);
+      } catch(err) {
+        debug(`Error setting relationship ready ${content_id.encode()}: ${err.stack}`);
+      }
+    } else {
+      // we already have content and a ready relationship set. No need to do anything
+      // debug(`content already stored locally ${content_id.encode()}`);
+    }
+  });
+
+
+  await Promise.all(allChecks);
+  debug('sync run complete');
+}
+
+
+async function sync_periodic(api, config, storage)
+{
+  try {
+    await sync_callback(api, config, storage);
+  } catch (err) {
+    debug(`Error in sync_periodic ${err.stack}`);
+  }
+  // always try again
+  setTimeout(sync_periodic, config.get('syncPeriod'), api, config, storage);
+}
+
+
+function start_syncing(api, config, storage)
+{
+  sync_periodic(api, config, storage);
+}
+
+module.exports = {
+  start_syncing: start_syncing,
+}

storage-node/packages/colossus/package.json

@@ -0,0 +1,67 @@
+{
+  "name": "@joystream/colossus",
+  "version": "0.1.0",
+  "description": "Colossus - Joystream Storage Node",
+  "author": "Joystream",
+  "homepage": "https://github.com/Joystream/joystream",
+  "bugs": {
+    "url": "https://github.com/Joystream/joystream/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Joystream/joystream.git"
+  },
+  "license": "GPL-3.0",
+  "contributors": [
+    {
+      "name": "Joystream",
+      "url": "https://joystream.org"
+    }
+  ],
+  "keywords": [
+    "joystream",
+    "storage",
+    "node"
+  ],
+  "os": [
+    "darwin",
+    "linux"
+  ],
+  "engines": {
+    "node": ">=10.15.3"
+  },
+  "scripts": {
+    "test": "mocha 'test/**/*.js'",
+    "lint": "eslint 'paths/**/*.js' 'lib/**/*.js'",
+    "dev": "nodemon --watch api-base.yml --watch bin/ --watch paths/ --watch lib/ --verbose --ext js --exec node bin/cli.js --"
+  },
+  "bin": {
+    "colossus": "bin/cli.js"
+  },
+  "devDependencies": {
+    "chai": "^4.2.0",
+    "eslint": "^5.13.0",
+    "express": "^4.16.4",
+    "mocha": "^5.2.0",
+    "node-mocks-http": "^1.7.3",
+    "nodemon": "^1.18.10",
+    "supertest": "^3.4.2",
+    "temp": "^0.9.0"
+  },
+  "dependencies": {
+    "@joystream/runtime-api": "^0.1.0",
+    "@joystream/storage": "^0.1.0",
+    "@joystream/util": "^0.1.0",
+    "body-parser": "^1.19.0",
+    "chalk": "^2.4.2",
+    "configstore": "^4.0.0",
+    "cors": "^2.8.5",
+    "express-openapi": "^4.6.1",
+    "figlet": "^1.2.1",
+    "js-yaml": "^3.13.1",
+    "lodash": "^4.17.11",
+    "meow": "^5.0.0",
+    "multer": "^1.4.1",
+    "si-prefix": "^0.2.0"
+  }
+}

storage-node/packages/colossus/paths/asset/v0/{id}.js

@@ -0,0 +1,361 @@
+/*
+ * This file is part of the storage node for the Joystream project.
+ * Copyright (C) 2019 Joystream Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+'use strict';
+
+const path = require('path');
+
+const file_type = require('file-type');
+const mime_types = require('mime-types');
+
+const debug = require('debug')('joystream:api:asset');
+
+const util_ranges = require('@joystream/util/ranges');
+const filter = require('@joystream/storage/filter');
+
+function error_handler(response, err, code)
+{
+  debug(err);
+  response.status((err.code || code) || 500).send({ message: err.toString() });
+}
+
+
+module.exports = function(config, storage, runtime)
+{
+  var doc = {
+    // parameters for all operations in this path
+    parameters: [
+      {
+        name: 'id',
+        in: 'path',
+        required: true,
+        description: 'Joystream Content ID',
+        schema: {
+          type: 'string',
+        },
+      },
+    ],
+
+    // Head: report that ranges are OK
+    head: async function(req, res, _next)
+    {
+      const id = req.params.id;
+
+      // Open file
+      try {
+        const size = await storage.size(id);
+        const stream = await storage.open(id, 'r');
+        const type = stream.file_info.mime_type;
+
+        // Close the stream; we don't need to fetch the file (if we haven't
+        // already). Then return result.
+        stream.destroy();
+
+        res.status(200);
+        res.contentType(type);
+        res.header('Content-Disposition', 'inline');
+        res.header('Content-Transfer-Encoding', 'binary');
+        res.header('Accept-Ranges', 'bytes');
+        if (size > 0) {
+          res.header('Content-Length', size);
+        }
+        res.send();
+      } catch (err) {
+        error_handler(res, err, err.code);
+      }
+    },
+
+    // Put for uploads
+    put: async function(req, res, _next)
+    {
+      const id = req.params.id;
+
+      // First check if we're the liaison for the name, otherwise we can bail
+      // out already.
+      const role_addr = runtime.identities.key.address;
+      let dataObject;
+      try {
+        debug('calling checkLiaisonForDataObject')
+        dataObject = await runtime.assets.checkLiaisonForDataObject(role_addr, id);
+        debug('called checkLiaisonForDataObject')
+      } catch (err) {
+        error_handler(res, err, 403);
+        return;
+      }
+
+      // We'll open a write stream to the backend, but reserve the right to
+      // abort upload if the filters don't smell right.
+      var stream;
+      try {
+        stream = await storage.open(id, 'w');
+
+        // We don't know whether the filtering occurs before or after the
+        // stream was finished, and can only commit if both passed.
+        var finished = false;
+        var accepted = false;
+        const possibly_commit = () => {
+          if (finished && accepted) {
+            debug('Stream is finished and passed filters; committing.');
+            stream.commit();
+          }
+        };
+
+
+        stream.on('file_info', async (info) => {
+          try {
+            debug('Detected file info:', info);
+
+            // Filter
+            const filter_result = filter(config, req.headers, info.mime_type);
+            if (200 != filter_result.code) {
+              debug('Rejecting content', filter_result.message);
+              stream.end();
+              res.status(filter_result.code).send({ message: filter_result.message });
+
+              // Reject the content
+              await runtime.assets.rejectContent(role_addr, id);
+              return;
+            }
+            debug('Content accepted.');
+            accepted = true;
+
+            // We may have to commit the stream.
+            possibly_commit();
+          } catch (err) {
+            error_handler(res, err);
+          }
+        });
+
+        stream.on('finish', () => {
+          try {
+            finished = true;
+            possibly_commit();
+          } catch (err) {
+            error_handler(res, err);
+          }
+        });
+
+        stream.on('committed', async (hash) => {
+          console.log('commited', dataObject)
+          try {
+            if (hash !== dataObject.ipfs_content_id.toString()) {
+              debug('Rejecting content. IPFS hash does not match value in objectId');
+              await runtime.assets.rejectContent(role_addr, id);
+              res.status(400).send({ message: "Uploaded content doesn't match IPFS hash" });
+              return;
+            }
+
+            debug('accepting Content')
+            await runtime.assets.acceptContent(role_addr, id);
+
+            debug('creating storage relationship for newly uploaded content')
+            // Create storage relationship and flip it to ready.
+            const dosr_id = await runtime.assets.createAndReturnStorageRelationship(role_addr, id);
+
+            debug('toggling storage relationship for newly uploaded content')
+            await runtime.assets.toggleStorageRelationshipReady(role_addr, dosr_id, true);
+
+            debug('Sending OK response.');
+            res.status(200).send({ message: 'Asset uploaded.' });
+          } catch (err) {
+            debug(`${err.message}`);
+            error_handler(res, err);
+          }
+        });
+
+        stream.on('error', (err) => error_handler(res, err));
+        req.pipe(stream);
+
+      } catch (err) {
+        error_handler(res, err);
+        return;
+      }
+    },
+
+    // Get content
+    get: async function(req, res, _next)
+    {
+      const id = req.params.id;
+      const download = req.query.download;
+
+      // Parse range header
+      var ranges;
+      if (!download) {
+        try {
+          var range_header = req.headers['range'];
+          ranges = util_ranges.parse(range_header);
+        } catch (err) {
+          // Do nothing; it's ok to ignore malformed ranges and respond with the
+          // full content according to https://www.rfc-editor.org/rfc/rfc7233.txt
+        }
+        if (ranges && ranges.unit != 'bytes') {
+          // Ignore ranges that are not byte units.
+          ranges = undefined;
+        }
+      }
+      debug('Requested range(s) is/are', ranges);
+
+      // Open file
+      try {
+        const size = await storage.size(id);
+        const stream = await storage.open(id, 'r');
+
+        // Add a file extension to download requests if necessary. If the file
+        // already contains an extension, don't add one.
+        var send_name = id;
+        const type = stream.file_info.mime_type;
+        if (download) {
+          var ext = path.extname(send_name);
+          if (!ext) {
+            ext = stream.file_info.ext;
+            if (ext) {
+              send_name = `${send_name}.${ext}`;
+            }
+          }
+        }
+
+        var opts = {
+          name: send_name,
+          type: type,
+          size: size,
+          ranges: ranges,
+          download: download,
+        };
+        util_ranges.send(res, stream, opts);
+
+
+      } catch (err) {
+        error_handler(res, err, err.code);
+      }
+    }
+  };
+
+  // OpenAPI specs
+  doc.get.apiDoc =
+  {
+    description: 'Download an asset.',
+    operationId: 'assetData',
+    tags: ['asset', 'data'],
+    parameters: [
+      {
+        name: 'download',
+        in: 'query',
+        description: 'Download instead of streaming inline.',
+        required: false,
+        allowEmptyValue: true,
+        schema: {
+          type: 'boolean',
+          default: false,
+        },
+      },
+    ],
+    responses: {
+      200: {
+        description: 'Asset download.',
+        content: {
+          default: {
+            schema: {
+              type: 'string',
+              format: 'binary',
+            },
+          },
+        },
+      },
+      default: {
+        description: 'Unexpected error',
+        content: {
+          'application/json': {
+            schema: {
+              '$ref': '#/components/schemas/Error'
+            },
+          },
+        },
+      },
+    },
+  };
+
+  doc.put.apiDoc =
+  {
+    description: 'Asset upload.',
+    operationId: 'assetUpload',
+    tags: ['asset', 'data'],
+    requestBody: {
+      content: {
+        '*/*': {
+          schema: {
+            type: 'string',
+            format: 'binary',
+          },
+        },
+      },
+    },
+    responses: {
+      200: {
+        description: 'Asset upload.',
+        content: {
+          'application/json': {
+            schema: {
+              type: 'object',
+              required: ['message'],
+              properties: {
+                message: {
+                  type: 'string',
+                }
+              },
+            },
+          },
+        },
+      },
+      default: {
+        description: 'Unexpected error',
+        content: {
+          'application/json': {
+            schema: {
+              '$ref': '#/components/schemas/Error'
+            },
+          },
+        },
+      },
+    },
+  };
+
+
+  doc.head.apiDoc =
+  {
+    description: 'Asset download information.',
+    operationId: 'assetInfo',
+    tags: ['asset', 'metadata'],
+    responses: {
+      200: {
+        description: 'Asset info.',
+      },
+      default: {
+        description: 'Unexpected error',
+        content: {
+          'application/json': {
+            schema: {
+              '$ref': '#/components/schemas/Error'
+            },
+          },
+        },
+      },
+    },
+  };
+
+  return doc;
+};

storage-node/packages/colossus/paths/discover/v0/{id}.js

@@ -0,0 +1,86 @@
+const { discover } = require('@joystream/discovery')
+const debug = require('debug')('joystream:api:discovery');
+
+const MAX_CACHE_AGE = 30 * 60 * 1000;
+const USE_CACHE = true;
+
+module.exports = function(config, runtime)
+{
+  var doc = {
+    // parameters for all operations in this path
+    parameters: [
+      {
+        name: 'id',
+        in: 'path',
+        required: true,
+        description: 'Actor accouuntId',
+        schema: {
+          type: 'string',
+        },
+      },
+    ],
+
+    // Resolve Service Information
+    get: async function(req, res)
+    {
+        const id = req.params.id;
+        let cacheMaxAge = req.query.max_age;
+
+        if (cacheMaxAge) {
+          try {
+            cacheMaxAge = parseInt(cacheMaxAge);
+          } catch(err) {
+            cacheMaxAge = MAX_CACHE_AGE
+          }
+        } else {
+          cacheMaxAge = 0
+        }
+
+        // todo - validate id before querying
+
+        try {
+          debug(`resolving ${id}`);
+          const info = await discover.discover(id, runtime, USE_CACHE, cacheMaxAge);
+          if (info == null) {
+            debug('info not found');
+            res.status(404).end();
+          } else {
+            res.status(200).send(info);
+          }
+
+        } catch (err) {
+          debug(`${err}`);
+          res.status(400).end()
+        }
+    }
+  };
+
+    // OpenAPI specs
+    doc.get.apiDoc = {
+        description: 'Resolve Service Information',
+        operationId: 'discover',
+        //tags: ['asset', 'data'],
+        responses: {
+            200: {
+                description: 'Wrapped JSON Service Information',
+                content: {
+                  'application/json': {
+                    schema: {
+                      required: ['serialized'],
+                      properties: {
+                        'serialized': {
+                          type: 'string'
+                        },
+                        'signature': {
+                          type: 'string'
+                        }
+                      },
+                    },
+                  }
+                }
+            }
+        }
+    }
+
+    return doc;
+};

storage-node/packages/colossus/test/index.js

@@ -0,0 +1 @@
+// Add Tests!

storage-node/packages/discovery/IpfsResolver.js

@@ -0,0 +1,68 @@
+const IpfsClient = require('ipfs-http-client')
+const axios = require('axios')
+const { Resolver } = require('./Resolver')
+
+class IpfsResolver extends Resolver {
+    constructor({
+        host = 'localhost',
+        port,
+        mode = 'rpc', // rpc or gateway
+        protocol = 'http', // http or https
+        ipfs,
+        runtime
+    }) {
+        super({runtime})
+
+        if (ipfs) {
+            // use an existing ipfs client instance
+            this.ipfs = ipfs
+        } else if (mode == 'rpc') {
+            port = port || '5001'
+            this.ipfs = IpfsClient(host, port, { protocol })
+        } else if (mode === 'gateway') {
+            port = port || '8080'
+            this.gateway = this.constructUrl(protocol, host, port)
+        } else {
+            throw new Error('Invalid IPFS Resolver options')
+        }
+    }
+
+    async _resolveOverRpc(identity) {
+        const ipnsPath = `/ipns/${identity}/`
+
+        const ipfsName = await this.ipfs.name.resolve(ipnsPath, {
+            recursive: false, // there should only be one indirection to service info file
+            nocache: false,
+        })
+
+        const data = await this.ipfs.get(ipfsName)
+
+        // there should only be one file published under the resolved path
+        const content = data[0].content
+
+        return JSON.parse(content)
+    }
+
+    async _resolveOverGateway(identity) {
+        const url = `${this.gateway}/ipns/${identity}`
+
+        // expected JSON object response
+        const response = await axios.get(url)
+
+        return response.data
+    }
+
+    resolve(accountId) {
+        const identity = this.resolveIdentity(accountId)
+
+        if (this.ipfs) {
+            return this._resolveOverRpc(identity)
+        } else {
+            return this._resolveOverGateway(identity)
+        }
+    }
+}
+
+module.exports = {
+    IpfsResolver
+}

storage-node/packages/discovery/JdsResolver.js

@@ -0,0 +1,28 @@
+const axios = require('axios')
+const { Resolver } = require('./Resolver')
+
+class JdsResolver extends Resolver {
+    constructor({
+        protocol = 'http', // http or https
+        host = 'localhost',
+        port,
+        runtime
+    }) {
+        super({runtime})
+
+        this.baseUrl = this.constructUrl(protocol, host, port)
+    }
+
+    async resolve(accountId) {
+        const url = `${this.baseUrl}/discover/v0/${accountId}`
+
+        // expected JSON object response
+        const response = await axios.get(url)
+
+        return response.data
+    }
+}
+
+module.exports = {
+    JdsResolver
+}

storage-node/packages/discovery/README.md

@@ -0,0 +1,129 @@
+# Discovery
+
+The `@joystream/discovery` package provides an API for role services to publish
+discovery information about themselves, and for consumers to resolve this
+information.
+
+In the Joystream network, services are provided by having members stake for a
+role. The role is identified by a unique actor key. Resolving service information
+associated with the actor key is the main purpose of this module.
+
+This implementation is based on [IPNS](https://docs.ipfs.io/guides/concepts/ipns/)
+as well as runtime information.
+
+## Discovery Workflow
+
+The discovery workflow provides an actor public key to the `discover()` function, which
+will eventually return structured data.
+
+Clients can verify that the structured data has been signed by the identifying
+actor. This is normally done automatically, unless a `verify: false` option is
+passed to `discover()`. Then, a separate `verify()` call can be used for
+verification.
+
+Under the hood, `discover()` uses any known participating node in the discovery
+network. If no other nodes are known, the bootstrap nodes from the runtime are
+used.
+
+There is a distinction in the discovery workflow:
+
+1. If run in the browser environment, a HTTP request to a participating node
+  is performed to discover nodes.
+2. If run in a node.js process, instead:
+  - A trusted (local) IPFS node must be configured.
+  - The chain is queried to resolve an actor key to an IPNS peer ID.
+  - The trusted IPFS node is used to resolve the IPNS peer ID to an IPFS
+    file.
+  - The IPFS file is fetched; this contains the structured data.
+
+Web services providing the HTTP endpoint used in the first approach will
+themselves use the second approach for fulfilling queries.
+
+## Publishing Workflow
+
+The publishing workflow is a little more involved, and requires more interaction
+with the runtime and the trusted IPFS node.
+
+1. A service information file is created.
+1. The file is signed with the actor key (see below).
+1. The file is published on IPFS.
+1. The IPNS name of the trusted IPFS node is updated to refer to the published
+   file.
+1. The runtime mapping from the actor ID to the IPNS name is updated.
+
+## Published Information
+
+Any JSON data can theoretically be published with this system; however, the
+following structure is currently imposed:
+
+- The JSON must be an Object at the top-level, not an Array.
+- Each key must correspond to a service spec (below).
+
+The data is signed using the [@joystream/json-signing](../json-signing/README.md)
+package.
+
+## Service Info Specifications
+
+Service specifications are JSON Objects, not Arrays. All service specifications
+come with their own `version` field which should be intepreted by clients making
+use of the information.
+
+Additionally, some services may only provide an `endpoint` value, as defined
+here:
+
+* `version`: A numeric version identifier for the service info field.
+* `endpoint`: A publicly accessible base URL for a service API.
+
+The `endpoint` should include a scheme and full authority, such that appending
+`swagger.json` to the path resolves the OpenAPI definition of the API served
+at this endpoint.
+
+The OpenAPI definition must include a top level path component corresponding
+to the service name, followed by an API version component. The remainder of the
+provided paths are dependent on the specific version of the API provided.
+
+For example, for an endpoint value of `https://user:password@host:port/` the
+following must hold:
+
+- `https://user:password@host:port/swagger.json` resolves to the OpenAPI
+  definition of the API(s) provided by this endpoint.
+- The OpenAPI definitions include paths prefixed by
+  `https://user:password@host:port/XXX/vYYY` where
+  - `XXX` is the service name, identical to the field name of the service spec
+    in the published service information.
+  - `YYY` the version identifier for the published service API.
+
+**Note:** The `version` field in the spec indicates the version of the spec.
+The `YYY` path component above indicates the version of the published OpenAPI.
+
+### Discovery Service
+
+Publishes `version` and `endpoint` as above; the `version` field is currently
+always `1`.
+
+### Asset Service
+
+Publishes `version` and `endpoint` as above; the `version` field is currently
+always `1`.
+
+### Example
+
+```json
+{
+  "asset": {
+    "version": 1,
+    "endpoint": "https://foo.bar/"
+  },
+  "discovery": {
+    "version": 1,
+    "endpoint": "http://quux.io/"
+  },
+}
+```
+
+Here, the following must be true:
+
+- `https://foo.bar/swagger.json` must include paths beginning with `https://foo.bar/asset/vXXX`
+  where `XXX` is the API version of the asset API.
+- `https://quux.io/swagger.json` must include paths beginning with `https://foo.bar/discovery/vYYY`
+  where `XXX` is the API version of the asset API.

storage-node/packages/discovery/Resolver.js

@@ -0,0 +1,48 @@
+class Resolver {
+    constructor ({
+        runtime
+    }) {
+        this.runtime = runtime
+    }
+
+    constructUrl (protocol, host, port) {
+        port = port ? `:${port}` : ''
+        return `${protocol}:://${host}${port}`
+    }
+
+    async resolveServiceInformation(accountId) {
+        let isActor = await this.runtime.identities.isActor(accountId)
+
+        if (!isActor) {
+            throw new Error('Cannot discover non actor account service info')
+        }
+
+        const identity = await this.resolveIdentity(accountId)
+
+        if (identity == null) {
+            // dont waste time trying to resolve if no identity was found
+            throw new Error('no identity to resolve');
+        }
+
+        return this.resolve(accountId)
+    }
+
+    // lookup ipns identity from chain corresponding to accountId
+    // return null if no identity found or record is expired
+    async resolveIdentity(accountId) {
+        const info = await this.runtime.discovery.getAccountInfo(accountId)
+        return info ? info.identity.toString() : null
+    }
+}
+
+Resolver.Error = {};
+Resolver.Error.UnrecognizedProtocol = class UnrecognizedProtocol extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'UnrecognizedProtocol';
+    }
+}
+
+module.exports = {
+    Resolver
+}

storage-node/packages/discovery/discover.js

@@ -0,0 +1,182 @@
+const axios = require('axios')
+const debug = require('debug')('discovery::discover')
+const stripEndingSlash = require('@joystream/util/stripEndingSlash')
+
+const ipfs = require('ipfs-http-client')('localhost', '5001', { protocol: 'http' })
+
+function inBrowser() {
+    return typeof window !== 'undefined'
+}
+
+var activeDiscoveries = {};
+var accountInfoCache = {};
+const CACHE_TTL = 60 * 60 * 1000;
+
+async function getIpnsIdentity (actorAccountId, runtimeApi) {
+    // lookup ipns identity from chain corresponding to actorAccountId
+    const info = await runtimeApi.discovery.getAccountInfo(actorAccountId)
+
+    if (info == null) {
+        // no identity found on chain for account
+        return null
+    } else {
+        return info.identity.toString()
+    }
+}
+
+async function discover_over_ipfs_http_gateway(actorAccountId, runtimeApi, gateway) {
+    let isActor = await runtimeApi.identities.isActor(actorAccountId)
+
+    if (!isActor) {
+        throw new Error('Cannot discover non actor account service info')
+    }
+
+    const identity = await getIpnsIdentity(actorAccountId, runtimeApi)
+
+    gateway = gateway || 'http://localhost:8080'
+
+    const url = `${gateway}/ipns/${identity}`
+
+    const response = await axios.get(url)
+
+    return response.data
+}
+
+async function discover_over_joystream_discovery_service(actorAccountId, runtimeApi, discoverApiEndpoint) {
+    let isActor = await runtimeApi.identities.isActor(actorAccountId)
+
+    if (!isActor) {
+        throw new Error('Cannot discover non actor account service info')
+    }
+
+    const identity = await getIpnsIdentity(actorAccountId, runtimeApi)
+
+    if (identity == null) {
+        // dont waste time trying to resolve if no identity was found
+        throw new Error('no identity to resolve');
+    }
+
+    if (!discoverApiEndpoint) {
+        // Use bootstrap nodes
+        let discoveryBootstrapNodes = await runtimeApi.discovery.getBootstrapEndpoints()
+
+        if (discoveryBootstrapNodes.length) {
+            discoverApiEndpoint = stripEndingSlash(discoveryBootstrapNodes[0].toString())
+        } else {
+            throw new Error('No known discovery bootstrap nodes found on network');
+        }
+    }
+
+    const url = `${discoverApiEndpoint}/discover/v0/${actorAccountId}`
+
+    // should have parsed if data was json?
+    const response = await axios.get(url)
+
+    return response.data
+}
+
+async function discover_over_local_ipfs_node(actorAccountId, runtimeApi) {
+    let isActor = await runtimeApi.identities.isActor(actorAccountId)
+
+    if (!isActor) {
+        throw new Error('Cannot discover non actor account service info')
+    }
+
+    const identity = await getIpnsIdentity(actorAccountId, runtimeApi)
+
+    const ipns_address = `/ipns/${identity}/`
+
+    debug('resolved ipns to ipfs object')
+    let ipfs_name = await ipfs.name.resolve(ipns_address, {
+        recursive: false, // there should only be one indirection to service info file
+        nocache: false,
+    }) // this can hang forever!? can we set a timeout?
+
+    debug('getting ipfs object', ipfs_name)
+    let data = await ipfs.get(ipfs_name) // this can sometimes hang forever!?! can we set a timeout?
+
+    // there should only be one file published under the resolved path
+    let content = data[0].content
+
+    // verify information and if 'discovery' service found
+    // add it to our list of bootstrap nodes
+
+    // TODO cache result or flag
+    return JSON.parse(content)
+}
+
+async function discover (actorAccountId, runtimeApi, useCachedValue = false, maxCacheAge = 0) {
+    const id = actorAccountId.toString();
+    const cached = accountInfoCache[id];
+
+    if (cached && useCachedValue) {
+        if (maxCacheAge > 0) {
+            // get latest value
+            if (Date.now() > (cached.updated + maxCacheAge)) {
+                return _discover(actorAccountId, runtimeApi);
+            }
+        }
+        // refresh if cache is stale, new value returned on next cached query
+        if (Date.now() > (cached.updated + CACHE_TTL)) {
+            _discover(actorAccountId, runtimeApi);
+        }
+        // return best known value
+        return cached.value;
+    } else {
+        return _discover(actorAccountId, runtimeApi);
+    }
+}
+
+function createExternallyControlledPromise() {
+    let resolve, reject;
+    const promise = new Promise((_resolve, _reject) => {
+        resolve = _resolve;
+        reject = _reject;
+    });
+    return ({ resolve, reject, promise });
+}
+
+async function _discover(actorAccountId, runtimeApi) {
+    const id = actorAccountId.toString();
+
+    const discoveryResult = activeDiscoveries[id];
+    if (discoveryResult) {
+        debug('discovery in progress waiting for result for',id);
+        return discoveryResult
+    }
+
+    debug('starting new discovery for', id);
+    const deferredDiscovery = createExternallyControlledPromise();
+    activeDiscoveries[id] = deferredDiscovery.promise;
+
+    let result;
+    try {
+        if (inBrowser()) {
+            result = await discover_over_joystream_discovery_service(actorAccountId, runtimeApi)
+        } else {
+            result = await discover_over_local_ipfs_node(actorAccountId, runtimeApi)
+        }
+        debug(result)
+        result = JSON.stringify(result)
+        accountInfoCache[id] = {
+            value: result,
+            updated: Date.now()
+        };
+
+        deferredDiscovery.resolve(result);
+        delete activeDiscoveries[id];
+        return result;
+    } catch (err) {
+        debug(err.message);
+        deferredDiscovery.reject(err);
+        delete activeDiscoveries[id];
+        throw err;
+    }
+}
+
+module.exports = {
+    discover,
+    discover_over_joystream_discovery_service,
+    discover_over_ipfs_http_gateway,
+    discover_over_local_ipfs_node,
+}

storage-node/packages/discovery/example.js

@@ -0,0 +1,34 @@
+const { RuntimeApi } = require('@joystream/runtime-api')
+
+const { discover, publish } = require('./')
+
+async function main() {
+    const runtimeApi = await RuntimeApi.create({
+        account_file: "/Users/mokhtar/Downloads/5Gn9n7SDJ7VgHqHQWYzkSA4vX6DCmS5TFWdHxikTXp9b4L32.json"
+    })
+
+    let published = await publish.publish(
+        "5Gn9n7SDJ7VgHqHQWYzkSA4vX6DCmS5TFWdHxikTXp9b4L32",
+        {
+            asset: {
+                version: 1,
+                endpoint: 'http://endpoint.com'
+            }
+        },
+        runtimeApi
+    )
+
+    console.log(published)
+
+    // let serviceInfo = await discover('5Gn9n7SDJ7VgHqHQWYzkSA4vX6DCmS5TFWdHxikTXp9b4L32', { runtimeApi })
+    let serviceInfo = await discover.discover(
+        '5Gn9n7SDJ7VgHqHQWYzkSA4vX6DCmS5TFWdHxikTXp9b4L32',
+        runtimeApi
+    )
+
+    console.log(serviceInfo)
+
+    runtimeApi.api.disconnect()
+}
+
+main()

storage-node/packages/discovery/index.js

@@ -0,0 +1,5 @@
+
+module.exports = {
+    discover : require('./discover'),
+    publish : require('./publish'),
+}

storage-node/packages/discovery/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@joystream/discovery",
+  "version": "0.1.0",
+  "description": "Service Discovery - Joystream Storage Node",
+  "author": "Joystream",
+  "homepage": "https://github.com/Joystream/joystream",
+  "bugs": {
+    "url": "https://github.com/Joystream/joystream/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Joystream/joystream.git"
+  },
+  "license": "GPL-3.0",
+  "contributors": [
+    {
+      "name": "Joystream",
+      "url": "https://joystream.org"
+    }
+  ],
+  "keywords": [
+    "joystream",
+    "storage",
+    "node"
+  ],
+  "os": [
+    "darwin",
+    "linux"
+  ],
+  "engines": {
+    "node": ">=10.15.3"
+  },
+  "main": "./index.js",
+  "scripts": {
+    "test": "mocha 'test/**/*.js'",
+    "lint": "eslint 'paths/**/*.js' 'lib/**/*.js'"
+  },
+  "devDependencies": {
+    "chai": "^4.2.0",
+    "eslint": "^5.13.0",
+    "mocha": "^5.2.0",
+    "supertest": "^3.4.2",
+    "temp": "^0.9.0"
+  },
+  "dependencies": {
+    "@joystream/runtime-api": "^0.1.0",
+    "@joystream/util": "^0.1.0",
+    "async-lock": "^1.2.0",
+    "axios": "^0.18.0",
+    "chalk": "^2.4.2",
+    "configstore": "^4.0.0",
+    "figlet": "^1.2.1",
+    "ipfs-http-client": "^32.0.1",
+    "js-yaml": "^3.13.1",
+    "meow": "^5.0.0",
+    "multer": "^1.4.1",
+    "si-prefix": "^0.2.0"
+  }
+}

storage-node/packages/discovery/publish.js

@@ -0,0 +1,53 @@
+const ipfsClient = require('ipfs-http-client')
+const ipfs = ipfsClient('localhost', '5001', { protocol: 'http' })
+
+const debug = require('debug')('discovery::publish')
+
+const PUBLISH_KEY = 'self'; // 'services';
+
+function bufferFrom(data) {
+    return Buffer.from(JSON.stringify(data), 'utf-8')
+}
+
+function encodeServiceInfo(info) {
+    return bufferFrom({
+        serialized: JSON.stringify(info),
+        // signature: ''
+    })
+}
+
+async function publish (service_info) {
+    const keys = await ipfs.key.list()
+    let services_key = keys.find((key) => key.name === PUBLISH_KEY)
+
+    // generate a new services key if not found
+    if (PUBLISH_KEY !== 'self' && !services_key) {
+        debug('generating ipns services key')
+        services_key = await ipfs.key.gen(PUBLISH_KEY, {
+          type: 'rsa',
+          size: 2048
+        });
+    }
+
+    if (!services_key) {
+        throw new Error('No IPFS publishing key available!')
+    }
+
+    debug('adding service info file to node')
+    const files = await ipfs.add(encodeServiceInfo(service_info))
+
+    debug('publishing...')
+    const published = await ipfs.name.publish(files[0].hash, {
+        key: PUBLISH_KEY,
+        resolve: false,
+        // lifetime: // string - Time duration of the record. Default: 24h
+        // ttl:      // string - Time duration this record should be cached
+    })
+
+    debug(published)
+    return services_key.id;
+}
+
+module.exports = {
+    publish
+}

storage-node/packages/discovery/test/index.js

@@ -0,0 +1 @@
+// Add Tests!

storage-node/packages/helios/.gitignore

@@ -0,0 +1,3 @@
+node_modules/
+lib/
+

storage-node/packages/helios/README.md

@@ -0,0 +1,12 @@
+# Joystream Helios
+
+A basic tool to scan the joystream storage network to get a birds eye view of the health of the storage providers and content replication status.
+
+
+## Scanning
+
+```
+yarn
+yarn run helios
+```
+

storage-node/packages/helios/bin/cli.js

@@ -0,0 +1,166 @@
+#!/usr/bin/env node
+
+const { RuntimeApi } = require('@joystream/runtime-api');
+const { encodeAddress } = require('@polkadot/keyring')
+const { discover } = require('@joystream/discovery');
+const axios = require('axios');
+const stripEndingSlash = require('@joystream/util/stripEndingSlash');
+
+(async function main () {
+
+  const runtime = await RuntimeApi.create();
+  const api  = runtime.api;
+
+  // get current blockheight
+  const currentHeader = await api.rpc.chain.getHeader();
+  const currentHeight = currentHeader.number.toBn();
+
+  // get all providers
+  const storageProviders = await api.query.actors.accountIdsByRole(0);
+
+  const storageProviderAccountInfos = await Promise.all(storageProviders.map(async (account) => {
+    return ({
+      account,
+      info: await runtime.discovery.getAccountInfo(account),
+      joined: (await api.query.actors.actorByAccountId(account)).unwrap().joined_at
+    });
+  }));
+
+  const liveProviders = storageProviderAccountInfos.filter(({account, info}) => {
+    return info && info.expires_at.gte(currentHeight)
+  });
+
+  const downProviders = storageProviderAccountInfos.filter(({account, info}) => {
+    return info == null
+  });
+
+  const expiredTtlProviders = storageProviderAccountInfos.filter(({account, info}) => {
+    return info && currentHeight.gte(info.expires_at)
+  });
+
+  let providersStatuses = mapInfoToStatus(liveProviders, currentHeight);
+  console.log('\n== Live Providers\n', providersStatuses);
+
+  let expiredProviderStatuses = mapInfoToStatus(expiredTtlProviders, currentHeight)
+  console.log('\n== Expired Providers\n', expiredProviderStatuses);
+
+  // check when actor account was created consider grace period before removing
+  console.log('\n== Down Providers!\n', downProviders.map(provider => {
+    return ({
+      account: provider.account.toString(),
+      age: currentHeight.sub(provider.joined).toNumber()
+    })
+  }));
+
+  // Resolve IPNS identities of providers
+  console.log('\nResolving live provider API Endpoints...')
+  //providersStatuses = providersStatuses.concat(expiredProviderStatuses);
+  let endpoints = await Promise.all(providersStatuses.map(async (status) => {
+    try {
+      let serviceInfo = await discover.discover_over_joystream_discovery_service(status.address, runtime);
+      let info = JSON.parse(serviceInfo.serialized);
+      console.log(`${status.address} -> ${info.asset.endpoint}`);
+      return { address: status.address, endpoint: info.asset.endpoint};
+    } catch (err) {
+      console.log('resolve failed', status.address, err.message);
+      return { address: status.address, endpoint: null};
+    }
+  }));
+
+  console.log('\nChecking API Endpoint is online')
+  await Promise.all(endpoints.map(async (provider) => {
+    if (!provider.endpoint) {
+      console.log('skipping', provider.address);
+      return
+    }
+    const swaggerUrl = `${stripEndingSlash(provider.endpoint)}/swagger.json`;
+    let error;
+    try {
+      await axios.get(swaggerUrl)
+    } catch (err) {error = err}
+    console.log(`${provider.endpoint} - ${error ? error.message : 'OK'}`);
+  }));
+
+  // after resolving for each resolved provider, HTTP HEAD with axios all known content ids
+  // report available/known
+  let knownContentIds = await runtime.assets.getKnownContentIds()
+
+  console.log(`\nContent Directory has ${knownContentIds.length} assets`);
+
+  await Promise.all(knownContentIds.map(async (contentId) => {
+    let [relationships, judgement] = await assetRelationshipState(api, contentId, storageProviders);
+    console.log(`${encodeAddress(contentId)} replication ${relationships}/${storageProviders.length} - ${judgement}`);
+  }));
+
+  console.log('\nChecking available assets on providers...');
+
+  endpoints.map(async ({address, endpoint}) => {
+    if (!endpoint) { return }
+    let { found, content } = await countContentAvailability(knownContentIds, endpoint);
+    console.log(`${address}: has ${found} assets`);
+    return content
+  });
+
+
+  // interesting disconnect doesn't work unless an explicit provider was created
+  // for underlying api instance
+  runtime.api.disconnect();
+})();
+
+function mapInfoToStatus(providers, currentHeight) {
+  return providers.map(({account, info, joined}) => {
+    if (info) {
+      return {
+        address: account.toString(),
+        age: currentHeight.sub(joined).toNumber(),
+        identity: info.identity.toString(),
+        expiresIn: info.expires_at.sub(currentHeight).toNumber(),
+        expired: currentHeight.gte(info.expires_at),
+      }
+    } else {
+      return {
+        address: account.toString(),
+        identity: null,
+        status: 'down'
+      }
+    }
+  })
+}
+
+async function countContentAvailability(contentIds, source) {
+  let content = {}
+  let found = 0;
+  for(let i = 0; i < contentIds.length; i++) {
+    const assetUrl = makeAssetUrl(contentIds[i], source);
+    try {
+      let info = await axios.head(assetUrl)
+      content[encodeAddress(contentIds[i])] = {
+        type: info.headers['content-type'],
+        bytes: info.headers['content-length']
+      }
+      found++
+    } catch(err) { console.log(`${assetUrl} ${err.message}`); continue; }
+  }
+  console.log(content);
+  return { found, content };
+}
+
+function makeAssetUrl(contentId, source) {
+  source = stripEndingSlash(source);
+  return `${source}/asset/v0/${encodeAddress(contentId)}`
+}
+
+async function assetRelationshipState(api, contentId, providers) {
+  let dataObject = await api.query.dataDirectory.dataObjectByContentId(contentId);
+
+  // how many relationships out of active providers?
+  let relationshipIds = await api.query.dataObjectStorageRegistry.relationshipsByContentId(contentId);
+
+  let activeRelationships = await Promise.all(relationshipIds.map(async (id) => {
+    let relationship = await api.query.dataObjectStorageRegistry.relationships(id);
+    relationship = relationship.unwrap()
+    return providers.find((provider) => relationship.storage_provider.eq(provider))
+  }));
+
+  return [activeRelationships.filter(active => active).length, dataObject.unwrap().liaison_judgement]
+}

storage-node/packages/helios/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@joystream/helios",
+  "version": "0.1.0",
+  "bin": {
+    "helios": "bin/cli.js"
+  },
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 0"
+  },
+  "license": "MIT",
+  "dependencies": {
+    "@joystream/runtime-api": "^0.1.0",
+    "@types/bn.js": "^4.11.5",
+    "axios": "^0.19.0",
+    "bn.js": "^4.11.8"
+  }
+}

storage-node/packages/helios/test/index.js

@@ -0,0 +1 @@
+// Add Tests!

storage-node/packages/runtime-api/.eslintrc.js

@@ -0,0 +1 @@
+../../.eslintrc.js

storage-node/packages/runtime-api/.gitignore

@@ -0,0 +1,3 @@
+# Generated JS files
+types/*.js
+!types/index.js

storage-node/packages/runtime-api/README.md

@@ -0,0 +1,7 @@
+Summary
+=======
+
+This package contains convenience functions for the runtime API.
+
+The main entry point creates and initializes a `@polkadot/api` instance, and
+provides more workflow oriented functions than the underlying API exposes.

storage-node/packages/runtime-api/assets.js

@@ -0,0 +1,176 @@
+'use strict';
+
+const debug = require('debug')('joystream:runtime:assets');
+
+const { Null } = require('@polkadot/types/primitive');
+
+const { _ } = require('lodash');
+
+const { decodeAddress, encodeAddress } = require('@polkadot/keyring');
+
+function parseContentId(contentId) {
+  try {
+    return decodeAddress(contentId)
+  } catch (err) {
+    return contentId
+  }
+}
+
+/*
+ * Add asset related functionality to the substrate API.
+ */
+class AssetsApi
+{
+  static async create(base)
+  {
+    const ret = new AssetsApi();
+    ret.base = base;
+    await ret.init();
+    return ret;
+  }
+
+  async init(account_file)
+  {
+    debug('Init');
+  }
+
+  /*
+   * Create a data object.
+   */
+  async createDataObject(accountId, contentId, doTypeId, size)
+  {
+    contentId = parseContentId(contentId)
+    const tx = this.base.api.tx.dataDirectory.addContent(contentId, doTypeId, size);
+    await this.base.signAndSend(accountId, tx);
+
+    // If the data object constructed properly, we should now be able to return
+    // the data object from the state.
+    return await this.getDataObject(contentId);
+  }
+
+  /*
+   * Return the Data Object for a CID
+   */
+  async getDataObject(contentId)
+  {
+    contentId = parseContentId(contentId)
+    const obj = await this.base.api.query.dataDirectory.dataObjectByContentId(contentId);
+    return obj;
+  }
+
+  /*
+   * Verify the liaison state for a DO:
+   * - Check the content ID has a DO
+   * - Check the account is the liaison
+   * - Check the liaison state is pending
+   *
+   * Each failure errors out, success returns the data object.
+   */
+  async checkLiaisonForDataObject(accountId, contentId)
+  {
+    contentId = parseContentId(contentId)
+
+    let obj = await this.getDataObject(contentId);
+
+    if (obj.isNone) {
+      throw new Error(`No DataObject created for content ID: ${contentId}`);
+    }
+
+    const encoded = encodeAddress(obj.raw.liaison);
+    if (encoded != accountId) {
+      throw new Error(`This storage node is not liaison for the content ID: ${contentId}`);
+    }
+
+    if (obj.raw.liaison_judgement.type != 'Pending') {
+      throw new Error(`Expected Pending judgement, but found: ${obj.raw.liaison_judgement.type}`);
+    }
+
+    return obj.unwrap();
+  }
+
+  /*
+   * Changes a data object liaison judgement.
+   */
+  async acceptContent(accountId, contentId)
+  {
+    contentId = parseContentId(contentId)
+    const tx = this.base.api.tx.dataDirectory.acceptContent(contentId);
+    return await this.base.signAndSend(accountId, tx);
+  }
+
+  /*
+   * Changes a data object liaison judgement.
+   */
+  async rejectContent(accountId, contentId)
+  {
+    contentId = parseContentId(contentId)
+    const tx = this.base.api.tx.dataDirectory.rejectContent(contentId);
+    return await this.base.signAndSend(accountId, tx);
+  }
+
+  /*
+   * Create storage relationship
+   */
+  async createStorageRelationship(accountId, contentId, callback)
+  {
+    contentId = parseContentId(contentId)
+    const tx = this.base.api.tx.dataObjectStorageRegistry.addRelationship(contentId);
+
+    const subscribed = [['dataObjectStorageRegistry', 'DataObjectStorageRelationshipAdded']];
+    return await this.base.signAndSend(accountId, tx, 3, subscribed, callback);
+  }
+
+  /*
+   * Get storage relationship for contentId
+   */
+  async getStorageRelationshipAndId(accountId, contentId) {
+    contentId = parseContentId(contentId)
+    let rids = await this.base.api.query.dataObjectStorageRegistry.relationshipsByContentId(contentId);
+
+    while(rids.length) {
+      const relationshipId = rids.shift();
+      let relationship = await this.base.api.query.dataObjectStorageRegistry.relationships(relationshipId);
+      relationship = relationship.unwrap();
+      if (relationship.storage_provider.eq(decodeAddress(accountId))) {
+        return ({ relationship, relationshipId });
+      }
+    }
+
+    return {};
+  }
+
+  async createAndReturnStorageRelationship(accountId, contentId)
+  {
+    contentId = parseContentId(contentId)
+    return new Promise(async (resolve, reject) => {
+      try {
+        await this.createStorageRelationship(accountId, contentId, (events) => {
+          events.forEach((event) => {
+            resolve(event[1].DataObjectStorageRelationshipId);
+          });
+        });
+      } catch (err) {
+        reject(err);
+      }
+    });
+  }
+
+  /*
+   * Toggle ready state for DOSR.
+   */
+  async toggleStorageRelationshipReady(accountId, dosrId, ready)
+  {
+    var tx = ready
+      ? this.base.api.tx.dataObjectStorageRegistry.setRelationshipReady(dosrId)
+      : this.base.api.tx.dataObjectStorageRegistry.unsetRelationshipReady(dosrId);
+    return await this.base.signAndSend(accountId, tx);
+  }
+
+  async getKnownContentIds() {
+    return this.base.api.query.dataDirectory.knownContentIds();
+  }
+}
+
+module.exports = {
+  AssetsApi: AssetsApi,
+}

storage-node/packages/runtime-api/balances.js

@@ -0,0 +1,90 @@
+/*
+ * This file is part of the storage node for the Joystream project.
+ * Copyright (C) 2019 Joystream Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+'use strict';
+
+const debug = require('debug')('joystream:runtime:balances');
+
+const { IdentitiesApi } = require('@joystream/runtime-api/identities');
+
+/*
+ * Bundle API calls related to account balances.
+ */
+class BalancesApi
+{
+  static async create(base)
+  {
+    const ret = new BalancesApi();
+    ret.base = base;
+    await ret.init();
+    return ret;
+  }
+
+  async init(account_file)
+  {
+    debug('Init');
+  }
+
+  /*
+   * Return true/false if the account has the minimum balance given.
+   */
+  async hasMinimumBalanceOf(accountId, min)
+  {
+    const balance = await this.freeBalance(accountId);
+    if (typeof min === 'number') {
+      return balance.cmpn(min) >= 0;
+    }
+    else {
+      return balance.cmp(min) >= 0;
+    }
+  }
+
+  /*
+   * Return the account's current free balance.
+   */
+  async freeBalance(accountId)
+  {
+    const decoded = this.base.identities.keyring.decodeAddress(accountId, true);
+    return this.base.api.query.balances.freeBalance(decoded);
+  }
+
+  /*
+   * Return the base transaction fee.
+   */
+  baseTransactionFee()
+  {
+    return this.base.api.consts.transactionPayment.transactionBaseFee;
+  }
+
+  /*
+   * Transfer amount currency from one address to another. The sending
+   * address must be an unlocked key pair!
+   */
+  async transfer(from, to, amount)
+  {
+    const decode = require('@polkadot/keyring').decodeAddress;
+    const to_decoded = decode(to, true);
+
+    const tx = this.base.api.tx.balances.transfer(to_decoded, amount);
+    return this.base.signAndSend(from, tx);
+  }
+}
+
+module.exports = {
+  BalancesApi: BalancesApi,
+}

storage-node/packages/runtime-api/discovery.js

@@ -0,0 +1,64 @@
+'use strict';
+
+const debug = require('debug')('joystream:runtime:discovery');
+
+/*
+ * Add discovery related functionality to the substrate API.
+ */
+class DiscoveryApi
+{
+  static async create(base)
+  {
+    const ret = new DiscoveryApi();
+    ret.base = base;
+    await ret.init();
+    return ret;
+  }
+
+  async init(account_file)
+  {
+    debug('Init');
+  }
+
+  /*
+   * Get Bootstrap endpoints
+   */
+  async getBootstrapEndpoints() {
+    return this.base.api.query.discovery.bootstrapEndpoints()
+  }
+
+  /*
+   * Get AccountInfo of an accountId
+   */
+  async getAccountInfo(accountId) {
+    const decoded = this.base.identities.keyring.decodeAddress(accountId, true)
+    const info = await this.base.api.query.discovery.accountInfoByAccountId(decoded)
+    // Not an Option so we use default value check to know if info was found
+    return info.expires_at.eq(0) ? null : info
+  }
+
+  /*
+   * Set AccountInfo of an accountId
+   */
+  async setAccountInfo(accountId, ipnsId, ttl) {
+    const isActor = await this.base.identities.isActor(accountId)
+    if (isActor) {
+      const tx = this.base.api.tx.discovery.setIpnsId(ipnsId, ttl)
+      return this.base.signAndSend(accountId, tx)
+    } else {
+      throw new Error('Cannot set AccountInfo for non actor account')
+    }
+  }
+
+  /*
+   * Clear AccountInfo of an accountId
+   */
+  async unsetAccountInfo(accountId) {
+    var tx = this.base.api.tx.discovery.unsetIpnsId()
+    return this.base.signAndSend(accountId, tx)
+  }
+}
+
+module.exports = {
+  DiscoveryApi: DiscoveryApi,
+}

storage-node/packages/runtime-api/identities.js

@@ -0,0 +1,235 @@
+/*
+ * This file is part of the storage node for the Joystream project.
+ * Copyright (C) 2019 Joystream Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+// const readline = require('readline');
+
+const debug = require('debug')('joystream:runtime:identities');
+
+const { Keyring } = require('@polkadot/keyring');
+// const { Null } = require('@polkadot/types/primitive');
+const util_crypto = require('@polkadot/util-crypto');
+
+// const { _ } = require('lodash');
+
+/*
+ * Add identity management to the substrate API.
+ *
+ * This loosely groups: accounts, key management, and membership.
+ */
+class IdentitiesApi
+{
+  static async create(base, {account_file, passphrase, canPromptForPassphrase})
+  {
+    const ret = new IdentitiesApi();
+    ret.base = base;
+    await ret.init(account_file, passphrase, canPromptForPassphrase);
+    return ret;
+  }
+
+  async init(account_file, passphrase, canPromptForPassphrase)
+  {
+    debug('Init');
+
+    // Creatre keyring
+    this.keyring = new Keyring();
+
+    this.canPromptForPassphrase = canPromptForPassphrase || false;
+
+    // Load account file, if possible.
+    try {
+      this.key = await this.loadUnlock(account_file, passphrase);
+    } catch (err) {
+      debug('Error loading account file:', err.message);
+    }
+  }
+
+  /*
+   * Load a key file and unlock it if necessary.
+   */
+  async loadUnlock(account_file, passphrase)
+  {
+    const fullname = path.resolve(account_file);
+    debug('Initializing key from', fullname);
+    const key = this.keyring.addFromJson(require(fullname));
+    await this.tryUnlock(key, passphrase);
+    debug('Successfully initialized with address', key.address);
+    return key;
+  }
+
+  /*
+   * Try to unlock a key if it isn't already unlocked.
+   * passphrase should be supplied as argument.
+   */
+  async tryUnlock(key, passphrase)
+  {
+    if (!key.isLocked) {
+      debug('Key is not locked, not attempting to unlock')
+      return;
+    }
+
+    // First try with an empty passphrase - for convenience
+    try {
+      key.decodePkcs8('');
+
+      if (passphrase) {
+        debug('Key was not encrypted, supplied passphrase was ignored');
+      }
+
+      return;
+    } catch (err) {
+      // pass
+    }
+
+    // Then with supplied passphrase
+    try {
+      debug('Decrypting with supplied passphrase');
+      key.decodePkcs8(passphrase);
+      return;
+    } catch (err) {
+      // pass
+    }
+
+    // If that didn't work, ask for a passphrase if appropriate
+    if (this.canPromptForPassphrase) {
+      passphrase = await this.askForPassphrase(key.address);
+      key.decodePkcs8(passphrase);
+      return
+    }
+
+    throw new Error('invalid passphrase supplied');
+  }
+
+  /*
+   * Ask for a passphrase
+   */
+  askForPassphrase(address)
+  {
+    // Query for passphrase
+    const prompt = require('password-prompt');
+    return prompt(`Enter passphrase for ${address}: `, { required: false });
+  }
+
+  /*
+   * Return true if the account is a member
+   */
+  async isMember(accountId)
+  {
+    const memberIds = await this.memberIdsOf(accountId); // return array of member ids
+    return memberIds.length > 0 // true if at least one member id exists for the acccount
+  }
+
+  /*
+   * Return true if the account is an actor/role account
+   */
+  async isActor(accountId)
+  {
+    const decoded = this.keyring.decodeAddress(accountId);
+    const actor = await this.base.api.query.actors.actorByAccountId(decoded)
+    return actor.isSome
+  }
+
+  /*
+   * Return the member IDs of an account
+   */
+  async memberIdsOf(accountId)
+  {
+    const decoded = this.keyring.decodeAddress(accountId);
+    return await this.base.api.query.members.memberIdsByRootAccountId(decoded);
+  }
+
+  /*
+   * Return the first member ID of an account, or undefined if not a member.
+   */
+  async firstMemberIdOf(accountId)
+  {
+    const decoded = this.keyring.decodeAddress(accountId);
+    let ids = await this.base.api.query.members.memberIdsByRootAccountId(decoded);
+    return ids[0]
+  }
+
+  /*
+   * Create a new key for the given role *name*. If no name is given,
+   * default to 'storage'.
+   */
+  async createRoleKey(accountId, role)
+  {
+    role = role || 'storage';
+
+    // Generate new key pair
+    const keyPair = util_crypto.naclKeypairFromRandom();
+
+    // Encode to an address.
+    const addr = this.keyring.encodeAddress(keyPair.publicKey);
+    debug('Generated new key pair with address', addr);
+
+    // Add to key wring. We set the meta to identify the account as
+    // a role key.
+    const meta = {
+      name: `${role} role account for ${accountId}`,
+    };
+
+    const createPair = require('@polkadot/keyring/pair').default;
+    const pair = createPair('ed25519', keyPair, meta);
+
+    this.keyring.addPair(pair);
+
+    return pair;
+  }
+
+  /*
+   * Export a key pair to JSON. Will ask for a passphrase.
+   */
+  async exportKeyPair(accountId)
+  {
+    const passphrase = await this.askForPassphrase(accountId);
+
+    // Produce JSON output
+    return this.keyring.toJson(accountId, passphrase);
+  }
+
+  /*
+   * Export a key pair and write it to a JSON file with the account ID as the
+   * name.
+   */
+  async writeKeyPairExport(accountId, prefix)
+  {
+    // Generate JSON
+    const data = await this.exportKeyPair(accountId);
+
+    // Write JSON
+    var filename = `${data.address}.json`;
+    if (prefix) {
+      const path = require('path');
+      filename = path.resolve(prefix, filename);
+    }
+    fs.writeFileSync(filename, JSON.stringify(data), {
+      encoding: 'utf8',
+      mode: 0o600,
+    });
+
+    return filename;
+  }
+}
+
+module.exports = {
+  IdentitiesApi: IdentitiesApi,
+}

storage-node/packages/runtime-api/index.js

@@ -0,0 +1,291 @@
+/*
+ * This file is part of the storage node for the Joystream project.
+ * Copyright (C) 2019 Joystream Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+'use strict';
+
+const debug = require('debug')('joystream:runtime:base');
+
+const { registerJoystreamTypes } = require('@joystream/types');
+const { ApiPromise, WsProvider } = require('@polkadot/api');
+
+const { IdentitiesApi } = require('@joystream/runtime-api/identities');
+const { BalancesApi } = require('@joystream/runtime-api/balances');
+const { RolesApi } = require('@joystream/runtime-api/roles');
+const { AssetsApi } = require('@joystream/runtime-api/assets');
+const { DiscoveryApi } = require('@joystream/runtime-api/discovery');
+const AsyncLock = require('async-lock');
+
+/*
+ * Initialize runtime (substrate) API and keyring.
+ */
+class RuntimeApi
+{
+  static async create(options)
+  {
+    const runtime_api = new RuntimeApi();
+    await runtime_api.init(options || {});
+    return runtime_api;
+  }
+
+  async init(options)
+  {
+    debug('Init');
+
+    options = options || {};
+
+    // Register joystream types
+    registerJoystreamTypes();
+
+    const provider = new WsProvider(options.provider_url || 'ws://localhost:9944');
+
+    // Create the API instrance
+    this.api = await ApiPromise.create({ provider });
+
+    this.asyncLock = new AsyncLock();
+
+    // Keep track locally of account nonces.
+    this.nonces = {};
+
+    // Ok, create individual APIs
+    this.identities = await IdentitiesApi.create(this, {
+      account_file: options.account_file,
+      passphrase: options.passphrase,
+      canPromptForPassphrase: options.canPromptForPassphrase
+    });
+    this.balances = await BalancesApi.create(this);
+    this.roles = await RolesApi.create(this);
+    this.assets = await AssetsApi.create(this);
+    this.discovery = await DiscoveryApi.create(this);
+  }
+
+  disconnect()
+  {
+    this.api.disconnect();
+  }
+
+  executeWithAccountLock(account_id, func) {
+    return this.asyncLock.acquire(`${account_id}`, func);
+  }
+
+  /*
+   * Wait for an event. Filters out any events that don't match the module and
+   * event name.
+   *
+   * The result of the Promise is an array containing first the full event
+   * name, and then the event fields as an object.
+   */
+  async waitForEvent(module, name)
+  {
+    return this.waitForEvents([[module, name]]);
+  }
+
+  _matchingEvents(subscribed, events)
+  {
+    debug(`Number of events: ${events.length}; subscribed to ${subscribed}`);
+
+    const filtered = events.filter((record) => {
+      const { event, phase } = record;
+
+      // Show what we are busy with
+      debug(`\t${event.section}:${event.method}:: (phase=${phase.toString()})`);
+      debug(`\t\t${event.meta.documentation.toString()}`);
+
+      // Skip events we're not interested in.
+      const matching = subscribed.filter((value) => {
+        return event.section == value[0] && event.method == value[1];
+      });
+      return matching.length > 0;
+    });
+    debug(`Filtered: ${filtered.length}`);
+
+    const mapped = filtered.map((record) => {
+      const { event } = record;
+      const types = event.typeDef;
+
+      // Loop through each of the parameters, displaying the type and data
+      const payload = {};
+      event.data.forEach((data, index) => {
+        debug(`\t\t\t${types[index].type}: ${data.toString()}`);
+        payload[types[index].type] = data;
+      });
+
+      const full_name = `${event.section}.${event.method}`;
+      return [full_name, payload];
+    });
+    debug('Mapped', mapped);
+
+    return mapped;
+  }
+
+  /*
+   * Same as waitForEvent, but filter on multiple events. The parameter is an
+   * array of arrays containing module and name. Calling waitForEvent is
+   * identical to calling this with [[module, name]].
+   *
+   * Returns the first matched event *only*.
+   */
+  async waitForEvents(subscribed)
+  {
+    return new Promise((resolve, reject) => {
+      this.api.query.system.events((events) => {
+        const matches = this._matchingEvents(subscribed, events);
+        if (matches && matches.length) {
+          resolve(matches);
+        }
+      });
+    });
+  }
+
+  /*
+   * Nonce-aware signAndSend(). Also allows you to use the accountId instead
+   * of the key, making calls a little simpler. Will lock to prevent concurrent
+   * calls so correct nonce is used.
+   *
+   * If the subscribed events are given, and a callback as well, then the
+   * callback is invoked with matching events.
+   */
+  async signAndSend(accountId, tx, attempts, subscribed, callback)
+  {
+    // Prepare key
+    const from_key = this.identities.keyring.getPair(accountId);
+
+    if (from_key.isLocked) {
+      throw new Error('Must unlock key before using it to sign!');
+    }
+
+    const finalizedPromise = newExternallyControlledPromise();
+
+    let unsubscribe = await this.executeWithAccountLock(accountId,  async () => {
+      // Try to get the next nonce to use
+      let nonce = this.nonces[accountId];
+
+      let incrementNonce = () => {
+        // only increment once
+        incrementNonce = () => {}; // turn it into a no-op
+        nonce = nonce.addn(1);
+        this.nonces[accountId] = nonce;
+      }
+
+      // If the nonce isn't available, get it from chain.
+      if (!nonce) {
+        // current nonce
+        nonce = await this.api.query.system.accountNonce(accountId);
+        debug(`Got nonce for ${accountId} from chain: ${nonce}`);
+      }
+
+      return new Promise((resolve, reject) => {
+        debug('Signing and sending tx');
+        // send(statusUpdates) returns a function for unsubscribing from status updates
+        let unsubscribe = tx.sign(from_key, { nonce })
+          .send(({events = [], status}) => {
+            debug(`TX status: ${status.type}`);
+
+            // Whatever events we get, process them if there's someone interested.
+            // It is critical that this event handling doesn't prevent
+            try {
+              if (subscribed && callback) {
+                const matched = this._matchingEvents(subscribed, events);
+                debug('Matching events:', matched);
+                if (matched.length) {
+                  callback(matched);
+                }
+              }
+            } catch(err) {
+              debug(`Error handling events ${err.stack}`)
+            }
+
+            // We want to release lock as early as possible, sometimes Ready status
+            // doesn't occur, so we do it on Broadcast instead
+            if (status.isReady) {
+              debug('TX Ready.');
+              incrementNonce();
+              resolve(unsubscribe); //releases lock
+            } else if (status.isBroadcast) {
+              debug('TX Broadcast.');
+              incrementNonce();
+              resolve(unsubscribe); //releases lock
+            } else if (status.isFinalized) {
+              debug('TX Finalized.');
+              finalizedPromise.resolve(status)
+            } else if (status.isFuture) {
+              // comes before ready.
+              // does that mean it will remain in mempool or in api internal queue?
+              // nonce was set in the future. Treating it as an error for now.
+              debug('TX Future!')
+              // nonce is likely out of sync, delete it so we reload it from chain on next attempt
+              delete this.nonces[accountId];
+              const err = new Error('transaction nonce set in future');
+              finalizedPromise.reject(err);
+              reject(err);
+            }
+
+            /* why don't we see these status updates on local devchain (single node)
+            isUsurped
+            isBroadcast
+            isDropped
+            isInvalid
+            */
+          })
+          .catch((err) => {
+            // 1014 error: Most likely you are sending transaction with the same nonce,
+            // so it assumes you want to replace existing one, but the priority is too low to replace it (priority = fee = len(encoded_transaction) currently)
+            // Remember this can also happen if in the past we sent a tx with a future nonce, and the current nonce
+            // now matches it.
+            if (err) {
+              const errstr = err.toString();
+              // not the best way to check error code.
+              // https://github.com/polkadot-js/api/blob/master/packages/rpc-provider/src/coder/index.ts#L52
+              if (errstr.indexOf('Error: 1014:') < 0 && // low priority
+                  errstr.indexOf('Error: 1010:') < 0) // bad transaction
+              {
+                // Error but not nonce related. (bad arguments maybe)
+                debug('TX error', err);
+              } else {
+                // nonce is likely out of sync, delete it so we reload it from chain on next attempt
+                delete this.nonces[accountId];
+              }
+            }
+
+            finalizedPromise.reject(err);
+            // releases lock
+            reject(err);
+          });
+      });
+    })
+
+    // when does it make sense to manyally unsubscribe?
+    // at this point unsubscribe.then and unsubscribe.catch have been deleted
+    // unsubscribe(); // don't unsubscribe if we want to wait for additional status
+    // updates to know when the tx has been finalized
+    return finalizedPromise.promise;
+  }
+}
+
+module.exports = {
+  RuntimeApi: RuntimeApi,
+}
+
+function newExternallyControlledPromise () {
+  // externally controller promise
+  let resolve, reject;
+  const promise = new Promise((res, rej) => {
+    resolve = res;
+    reject = rej;
+  });
+  return ({resolve, reject, promise});
+}

storage-node/packages/runtime-api/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@joystream/runtime-api",
+  "version": "0.1.0",
+  "description": "Runtime API abstraction for Joystream Storage Node",
+  "author": "Joystream",
+  "homepage": "https://github.com/Joystream/joystream",
+  "bugs": {
+    "url": "https://github.com/Joystream/joystream/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Joystream/joystream.git"
+  },
+  "license": "GPL-3.0",
+  "contributors": [
+    {
+      "name": "Joystream",
+      "url": "https://joystream.org/"
+    }
+  ],
+  "keywords": [
+    "joystream",
+    "storage",
+    "node",
+    "runtime"
+  ],
+  "os": [
+    "darwin",
+    "linux"
+  ],
+  "engines": {
+    "node": ">=10.15.3"
+  },
+  "scripts": {
+    "test": "mocha 'test/**/*.js' --exit",
+    "lint": "eslint '**/*.js' --ignore-pattern 'test/**/*.js'"
+  },
+  "devDependencies": {
+    "chai": "^4.2.0",
+    "eslint": "^5.13.0",
+    "mocha": "^5.2.0",
+    "sinon": "^7.3.2",
+    "sinon-chai": "^3.3.0",
+    "temp": "^0.9.0"
+  },
+  "dependencies": {
+    "@joystream/types": "^0.10.0",
+    "@polkadot/api": "^0.96.1",
+    "async-lock": "^1.2.0",
+    "lodash": "^4.17.11",
+    "password-prompt": "^1.1.2"
+  }
+}

storage-node/packages/runtime-api/roles.js

@@ -0,0 +1,186 @@
+/*
+ * This file is part of the storage node for the Joystream project.
+ * Copyright (C) 2019 Joystream Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+'use strict';
+
+const debug = require('debug')('joystream:runtime:roles');
+
+const { Null, u64 } = require('@polkadot/types');
+
+const { _ } = require('lodash');
+
+/*
+ * Add role related functionality to the substrate API.
+ */
+class RolesApi
+{
+  static async create(base)
+  {
+    const ret = new RolesApi();
+    ret.base = base;
+    await ret.init();
+    return ret;
+  }
+
+  async init()
+  {
+    debug('Init');
+
+    // Constants
+    this.ROLE_STORAGE = 'StorageProvider'; // new u64(0x00);
+  }
+
+  /*
+   * Raises errors if the given account ID is not valid for staking as the given
+   * role. The role should be one of the ROLE_* constants above.
+   */
+  async checkAccountForStaking(accountId, role)
+  {
+    role = role || this.ROLE_STORAGE;
+
+    if (!await this.base.identities.isMember(accountId)) {
+      const msg = `Account with id "${accountId}" is not a member!`;
+      debug(msg);
+      throw new Error(msg);
+    }
+
+    if (!await this.hasBalanceForRoleStaking(accountId, role)) {
+      const msg = `Account with id "${accountId}" does not have sufficient free balance for role staking!`;
+      debug(msg);
+      throw new Error(msg);
+    }
+
+    debug(`Account with id "${accountId}" is a member with sufficient free balance, able to proceed.`);
+    return true;
+  }
+
+  /*
+   * Returns the required balance for staking for a role.
+   */
+  async requiredBalanceForRoleStaking(role)
+  {
+    const params = await this.base.api.query.actors.parameters(role);
+    if (params.isNone) {
+      throw new Error(`Role ${role} is not defined!`);
+    }
+    const result = params.raw.min_stake
+      .add(params.raw.entry_request_fee)
+      .add(await this.base.balances.baseTransactionFee());
+    return result;
+  }
+
+  /*
+   * Returns true/false if the given account has the balance required for
+   * staking for the given role.
+   */
+  async hasBalanceForRoleStaking(accountId, role)
+  {
+    const required = await this.requiredBalanceForRoleStaking(role);
+    return await this.base.balances.hasMinimumBalanceOf(accountId, required);
+  }
+
+  /*
+   * Transfer enough funds to allow the recipient to stake for the given role.
+   */
+  async transferForStaking(from, to, role)
+  {
+    const required = await this.requiredBalanceForRoleStaking(role);
+    return await this.base.balances.transfer(from, to, required);
+  }
+
+  /*
+   * Return current accounts holding a role.
+   */
+  async accountIdsByRole(role)
+  {
+    const ids = await this.base.api.query.actors.accountIdsByRole(role);
+    return ids.map(id => id.toString());
+  }
+
+  /*
+   * Returns the number of slots available for a role
+   */
+  async availableSlotsForRole(role)
+  {
+    let params = await this.base.api.query.actors.parameters(role);
+    if (params.isNone) {
+      throw new Error(`Role ${role} is not defined!`);
+    }
+    params = params.unwrap();
+    const slots = params.max_actors;
+    const active = await this.accountIdsByRole(role);
+    return (slots.subn(active.length)).toNumber();
+  }
+
+  /*
+   * Send a role application.
+   * - The role account must not be a member, but have sufficient funds for
+   *   staking.
+   * - The member account must be a member.
+   *
+   * After sending this application, the member account will have role request
+   * in the 'My Requests' tab of the app.
+   */
+  async applyForRole(roleAccountId, role, memberAccountId)
+  {
+    const memberId = await this.base.identities.firstMemberIdOf(memberAccountId);
+    if (memberId == undefined) {
+      throw new Error('Account is not a member!');
+    }
+
+    const tx = this.base.api.tx.actors.roleEntryRequest(role, memberId);
+    return await this.base.signAndSend(roleAccountId, tx);
+  }
+
+  /*
+   * Check whether the given role is occupying the given role.
+   */
+  async checkForRole(roleAccountId, role)
+  {
+    const actor = await this.base.api.query.actors.actorByAccountId(roleAccountId);
+    return !_.isEqual(actor.raw, new Null());
+  }
+
+  /*
+   * Same as checkForRole(), but if the account is not currently occupying the
+   * role, wait for the appropriate `actors.Staked` event to be emitted.
+   */
+  async waitForRole(roleAccountId, role)
+  {
+    if (await this.checkForRole(roleAccountId, role)) {
+      return true;
+    }
+
+    return new Promise((resolve, reject) => {
+      this.base.waitForEvent('actors', 'Staked').then((values) => {
+        const name = values[0][0];
+        const payload = values[0][1];
+
+        if (payload.AccountId == roleAccountId) {
+          resolve(true);
+        } else {
+          // reject() ?
+        }
+      });
+    });
+  }
+}
+
+module.exports = {
+  RolesApi: RolesApi,
+}

storage-node/packages/runtime-api/test/assets.js

@@ -0,0 +1,52 @@
+/*
+ * This file is part of the storage node for the Joystream project.
+ * Copyright (C) 2019 Joystream Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+'use strict';
+
+const mocha = require('mocha');
+const expect = require('chai').expect;
+const sinon = require('sinon');
+
+const { RuntimeApi } = require('@joystream/runtime-api');
+
+describe('Assets', () => {
+  var api;
+  var key;
+  before(async () => {
+    api = await RuntimeApi.create();
+    key = await api.identities.loadUnlock('test/data/edwards_unlocked.json');
+  });
+
+  it('returns DataObjects for a content ID', async () => {
+    const obj = await api.assets.getDataObject('foo');
+    expect(obj.isNone).to.be.true;
+  });
+
+  it('can check the liaison for a DataObject', async () => {
+    expect(async _ => {
+      await api.assets.checkLiaisonForDataObject('foo', 'bar');
+    }).to.throw;
+  });
+
+  // Needs properly staked accounts
+  it('can accept content');
+  it('can reject content');
+  it('can create a storage relationship for content');
+  it('can create a storage relationship for content and return it');
+  it('can toggle a storage relatsionship to ready state');
+});

storage-node/packages/runtime-api/test/balances.js

@@ -0,0 +1,55 @@
+/*
+ * This file is part of the storage node for the Joystream project.
+ * Copyright (C) 2019 Joystream Contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+'use strict';
+
+const mocha = require('mocha');
+const expect = require('chai').expect;
+const sinon = require('sinon');
+
+const { RuntimeApi } = require('@joystream/runtime-api');
+
+describe('Balances', () => {
+  var api;
+  var key;
+  before(async () => {
+    api = await RuntimeApi.create();
+    key = await api.identities.loadUnlock('test/data/edwards_unlocked.json');
+  });
+
+  it('returns free balance for an account', async () => {
+    const balance = await api.balances.freeBalance(key.address);
+    // Should be exactly zero
+    expect(balance.cmpn(0)).to.equal(0);
+  });
+
+  it('checks whether a minimum balance exists', async () => {
+    // A minimum of 0 should exist, but no more.
+    expect(await api.balances.hasMinimumBalanceOf(key.address, 0)).to.be.true;
+    expect(await api.balances.hasMinimumBalanceOf(key.address, 1)).to.be.false;
+  });
+
+  it('returns the base transaction fee of the chain', async () => {
+    const fee = await api.balances.baseTransactionFee();
+    // >= 0 comparison works
+    expect(fee.cmpn(0)).to.be.at.least(0);
+  });
+
+  // TODO implemtable only with accounts with balance
+  it('can transfer funds');
+});

storage-node/packages/runtime-api/test/data/edwards.json

@@ -0,0 +1 @@
+{"address":"5HDnLpCjdbUBR6eyuz5geBJWzoZdXmWFXahEYrLg44rvToCK","encoded":"0x475f0c37c7893517f5a93c88b81208346211dfa9b0fd09e08bfd34f6e14da5468f48c6d9b0b4cbfbd7dd03a6f0730f5ee9a01b0cd30265e6b1b9fb652958889d5b174624568f49f3a671b8c330c3920814e938383749aa9046366ae6881281e0d053a9aa913a54ad53bd2f1dcf6c26e6b476495ea058832a36f122d09c18154577f951298ac72e6f471a6dca41e4d5741ed5db966001ae5ffd2b99d4c7","encoding":{"content":["pkcs8","ed25519"],"type":"xsalsa20-poly1305","version":"2"},"meta":{"name":"Edwards keypair for testing","whenCreated":1558974074691}}

storage-node/packages/runtime-api/test/data/edwards_unlocked.json

@@ -0,0 +1 @@
+{"address":"5EZxbX2arChvhYL7cEgSybJL3kzEeuPqqNYyLqRBJxZx7Mao","encoded":"0x3053020101300506032b65700422042071f2096e5857177f03768478d0c006f60d1ee684f14feaede0f9c17e139e65586ec832e5db75112b0a4585b6a9ffe58fa056e5b1228f02663e9e64743e65c9a5a1230321006ec832e5db75112b0a4585b6a9ffe58fa056e5b1228f02663e9e64743e65c9a5","encoding":{"content":["pkcs8","ed25519"],"type":"none","version":"2"},"meta":{"name":"Unlocked keypair for testing","whenCreated":1558975434890}}