Merge branch 'master' into integration-tests-refactoring

README.md

@@ -1,4 +1,4 @@
-# Joystream [![Build Status](https://travis-ci.org/Joystream/joystream.svg?branch=master)](https://travis-ci.org/Joystream/joystream)
+# Joystream
 
 This is the main code repository for all Joystream software. In this mono-repo you will find all the software required to run a Joystream network: The Joystream full node, runtime and all reusable substrate runtime modules that make up the Joystream runtime. In addition to all front-end apps and infrastructure servers necessary for operating the network.
 
@@ -7,10 +7,6 @@ This is the main code repository for all Joystream software. In this mono-repo y
 The Joystream network builds on a pre-release version of [substrate v2.0](https://substrate.dev/) and adds additional
 functionality to support the [various roles](https://www.joystream.org/roles) that can be entered into on the platform.
 
-## Build Status
-
-Development [![Development Branch Build Status](https://travis-ci.org/Joystream/joystream.svg?branch=development)](https://travis-ci.org/Joystream/joystream) - build history on [Travis](https://travis-ci.org/github/Joystream/joystream/builds)
-
 ## Development Tools
 
 The following tools are required for building, testing and contributing to this repo:

cli/README.md

@@ -44,7 +44,7 @@ $ npm install -g @joystream/cli
 $ joystream-cli COMMAND
 running command...
 $ joystream-cli (-v|--version|version)
-@joystream/cli/0.2.0 linux-x64 node-v13.12.0
+@joystream/cli/0.3.0 linux-x64 node-v12.18.2
 $ joystream-cli --help [COMMAND]
 USAGE
   $ joystream-cli COMMAND
@@ -83,6 +83,7 @@ When using the CLI for the first time there are a few common steps you might wan
 * [`joystream-cli content-directory:classes`](#joystream-cli-content-directoryclasses)
 * [`joystream-cli content-directory:createClass`](#joystream-cli-content-directorycreateclass)
 * [`joystream-cli content-directory:createCuratorGroup`](#joystream-cli-content-directorycreatecuratorgroup)
+* [`joystream-cli content-directory:createEntity CLASSNAME`](#joystream-cli-content-directorycreateentity-classname)
 * [`joystream-cli content-directory:curatorGroup ID`](#joystream-cli-content-directorycuratorgroup-id)
 * [`joystream-cli content-directory:curatorGroups`](#joystream-cli-content-directorycuratorgroups)
 * [`joystream-cli content-directory:entities CLASSNAME [PROPERTIES]`](#joystream-cli-content-directoryentities-classname-properties)
@@ -94,6 +95,7 @@ When using the CLI for the first time there are a few common steps you might wan
 * [`joystream-cli content-directory:removeMaintainerFromClass [CLASSNAME] [GROUPID]`](#joystream-cli-content-directoryremovemaintainerfromclass-classname-groupid)
 * [`joystream-cli content-directory:setCuratorGroupStatus [ID] [STATUS]`](#joystream-cli-content-directorysetcuratorgroupstatus-id-status)
 * [`joystream-cli content-directory:updateClassPermissions [CLASSNAME]`](#joystream-cli-content-directoryupdateclasspermissions-classname)
+* [`joystream-cli content-directory:updateEntityPropertyValues ID`](#joystream-cli-content-directoryupdateentitypropertyvalues-id)
 * [`joystream-cli council:info`](#joystream-cli-councilinfo)
 * [`joystream-cli help [COMMAND]`](#joystream-cli-help-command)
 * [`joystream-cli media:createChannel`](#joystream-cli-mediacreatechannel)
@@ -423,6 +425,23 @@ ALIASES
 
 _See code: [src/commands/content-directory/createCuratorGroup.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/createCuratorGroup.ts)_
 
+## `joystream-cli content-directory:createEntity CLASSNAME`
+
+Creates a new entity in the specified class (can be executed in Member, Curator or Lead context)
+
+```
+USAGE
+  $ joystream-cli content-directory:createEntity CLASSNAME
+
+ARGUMENTS
+  CLASSNAME  Name or ID of the Class
+
+OPTIONS
+  --context=(Member|Curator|Lead)  Actor context to execute the command in (Member/Curator/Lead)
+```
+
+_See code: [src/commands/content-directory/createEntity.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/createEntity.ts)_
+
 ## `joystream-cli content-directory:curatorGroup ID`
 
 Show Curator Group details by ID.
@@ -588,6 +607,23 @@ ARGUMENTS
 
 _See code: [src/commands/content-directory/updateClassPermissions.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/updateClassPermissions.ts)_
 
+## `joystream-cli content-directory:updateEntityPropertyValues ID`
+
+Updates the property values of the specified entity (can be executed in Member, Curator or Lead context)
+
+```
+USAGE
+  $ joystream-cli content-directory:updateEntityPropertyValues ID
+
+ARGUMENTS
+  ID  ID of the Entity
+
+OPTIONS
+  --context=(Member|Curator|Lead)  Actor context to execute the command in (Member/Curator/Lead)
+```
+
+_See code: [src/commands/content-directory/updateEntityPropertyValues.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/updateEntityPropertyValues.ts)_
+
 ## `joystream-cli council:info`
 
 Get current council and council elections information

cli/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@joystream/cli",
   "description": "Command Line Interface for Joystream community and governance activities",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "author": "Leszek Wiesner",
   "bin": {
     "joystream-cli": "./bin/run"

cli/src/base/ContentDirectoryCommandBase.ts

@@ -12,6 +12,7 @@ import {
   EntityId,
   Actor,
   PropertyType,
+  Property,
 } from '@joystream/types/content-directory'
 import { Worker } from '@joystream/types/working-group'
 import { CLIError } from '@oclif/errors'
@@ -23,6 +24,7 @@ import { RolesCommandBase } from './WorkingGroupsCommandBase'
 import { createType } from '@joystream/types'
 import chalk from 'chalk'
 import { flags } from '@oclif/command'
+import { DistinctQuestion } from 'inquirer'
 
 const CONTEXTS = ['Member', 'Curator', 'Lead'] as const
 type Context = typeof CONTEXTS[number]
@@ -247,7 +249,7 @@ export default abstract class ContentDirectoryCommandBase extends RolesCommandBa
 
   async getAndParseKnownEntity<T>(id: string | number, className?: string): Promise<FlattenRelations<T>> {
     const entity = await this.getEntity(id, className)
-    return this.parseToKnownEntityJson<T>(entity)
+    return this.parseToEntityJson<T>(entity)
   }
 
   async entitiesByClassAndOwner(classNameOrId: number | string, ownerMemberId?: number): Promise<[EntityId, Entity][]> {
@@ -339,7 +341,7 @@ export default abstract class ContentDirectoryCommandBase extends RolesCommandBa
     }, {} as Record<string, ParsedPropertyValue>)
   }
 
-  async parseToKnownEntityJson<T>(entity: Entity): Promise<FlattenRelations<T>> {
+  async parseToEntityJson<T = unknown>(entity: Entity): Promise<FlattenRelations<T>> {
     const entityClass = (await this.classEntryByNameOrId(entity.class_id.toString()))[1]
     return (_.mapValues(this.parseEntityPropertyValues(entity, entityClass), (v) =>
       this.parseStoredPropertyInnerValue(v.value)
@@ -376,4 +378,124 @@ export default abstract class ContentDirectoryCommandBase extends RolesCommandBa
 
     return parsedEntities.filter((entity) => filters.every(([pName, pValue]) => entity[pName] === pValue))
   }
+
+  async getActor(context: typeof CONTEXTS[number], pickedClass: Class) {
+    let actor: Actor
+    if (context === 'Member') {
+      const memberId = await this.getRequiredMemberId()
+      actor = this.createType('Actor', { Member: memberId })
+    } else if (context === 'Curator') {
+      actor = await this.getCuratorContext([pickedClass.name.toString()])
+    } else {
+      await this.getRequiredLead()
+
+      actor = this.createType('Actor', { Lead: null })
+    }
+
+    return actor
+  }
+
+  isActorEntityController(actor: Actor, entity: Entity, isMaintainer: boolean): boolean {
+    const entityController = entity.entity_permissions.controller
+    return (
+      (isMaintainer && entityController.isOfType('Maintainers')) ||
+      (entityController.isOfType('Member') &&
+        actor.isOfType('Member') &&
+        entityController.asType('Member').eq(actor.asType('Member'))) ||
+      (entityController.isOfType('Lead') && actor.isOfType('Lead'))
+    )
+  }
+
+  async isEntityPropertyEditableByActor(entity: Entity, classPropertyId: number, actor: Actor): Promise<boolean> {
+    const [, entityClass] = await this.classEntryByNameOrId(entity.class_id.toString())
+
+    const isActorMaintainer =
+      actor.isOfType('Curator') &&
+      entityClass.class_permissions.maintainers.toArray().some((groupId) => groupId.eq(actor.asType('Curator')[0]))
+
+    const isActorController = this.isActorEntityController(actor, entity, isActorMaintainer)
+
+    const {
+      is_locked_from_controller: isLockedFromController,
+      is_locked_from_maintainer: isLockedFromMaintainer,
+    } = entityClass.properties[classPropertyId].locking_policy
+
+    return (
+      (isActorController && !isLockedFromController.valueOf()) ||
+      (isActorMaintainer && !isLockedFromMaintainer.valueOf())
+    )
+  }
+
+  getQuestionsFromProperties(properties: Property[], defaults?: { [key: string]: unknown }): DistinctQuestion[] {
+    return properties.reduce((previousValue, { name, property_type: propertyType, required }) => {
+      const propertySubtype = propertyType.subtype
+      const questionType = propertySubtype === 'Bool' ? 'list' : 'input'
+      const isSubtypeNumber = propertySubtype.toLowerCase().includes('int')
+      const isSubtypeReference = propertyType.isOfType('Single') && propertyType.asType('Single').isOfType('Reference')
+
+      const validate = async (answer: string | number | null) => {
+        if (answer === null) {
+          return true // Can only happen through "filter" if property is not required
+        }
+
+        if ((isSubtypeNumber || isSubtypeReference) && parseInt(answer.toString()).toString() !== answer.toString()) {
+          return `Expected integer value!`
+        }
+
+        if (isSubtypeReference) {
+          try {
+            await this.getEntity(+answer, propertyType.asType('Single').asType('Reference')[0].toString())
+          } catch (e) {
+            return e.message || JSON.stringify(e)
+          }
+        }
+
+        return true
+      }
+
+      const optionalQuestionProperties = {
+        ...{
+          filter: async (answer: string) => {
+            if (required.isFalse && !answer) {
+              return null
+            }
+
+            // Only cast to number if valid
+            // Prevents inquirer bug not allowing to edit invalid values when casted to number
+            // See: https://github.com/SBoudrias/Inquirer.js/issues/866
+            if ((isSubtypeNumber || isSubtypeReference) && (await validate(answer)) === true) {
+              return parseInt(answer)
+            }
+
+            return answer
+          },
+          validate,
+        },
+        ...(propertySubtype === 'Bool' && {
+          choices: ['true', 'false'],
+          filter: (answer: string) => {
+            return answer === 'true' || false
+          },
+        }),
+      }
+
+      const isQuestionOptional = propertySubtype === 'Bool' ? '' : required.isTrue ? '(required)' : '(optional)'
+      const classId = isSubtypeReference
+        ? ` [Class Id: ${propertyType.asType('Single').asType('Reference')[0].toString()}]`
+        : ''
+
+      return [
+        ...previousValue,
+        {
+          name: name.toString(),
+          message: `${name} - ${propertySubtype}${classId} ${isQuestionOptional}`,
+          type: questionType,
+          ...optionalQuestionProperties,
+          ...(defaults && {
+            default: propertySubtype === 'Bool' ? JSON.stringify(defaults[name.toString()]) : defaults[name.toString()],
+          }),
+        },
+      ]
+    }, [] as DistinctQuestion[])
+  }
 }

cli/src/base/MediaCommandBase.ts

@@ -23,7 +23,7 @@ export default abstract class MediaCommandBase extends ContentDirectoryCommandBa
     })
     if (licenseType === 'known') {
       const [id, knownLicenseEntity] = await this.promptForEntityEntry('Choose License', 'KnownLicense', 'code')
-      const knownLicense = await this.parseToKnownEntityJson<KnownLicenseEntity>(knownLicenseEntity)
+      const knownLicense = await this.parseToEntityJson<KnownLicenseEntity>(knownLicenseEntity)
       licenseInput = { knownLicense: id.toNumber() }
       if (knownLicense.attributionRequired) {
         licenseInput.attribution = await this.simplePrompt({ message: 'Attribution' })

cli/src/commands/content-directory/createEntity.ts

@@ -0,0 +1,58 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import inquirer from 'inquirer'
+import { InputParser } from '@joystream/cd-schemas'
+import ExitCodes from '../../ExitCodes'
+
+export default class CreateEntityCommand extends ContentDirectoryCommandBase {
+  static description =
+    'Creates a new entity in the specified class (can be executed in Member, Curator or Lead context)'
+
+  static args = [
+    {
+      name: 'className',
+      required: true,
+      description: 'Name or ID of the Class',
+    },
+  ]
+
+  static flags = {
+    context: ContentDirectoryCommandBase.contextFlag,
+  }
+
+  async run() {
+    const { className } = this.parse(CreateEntityCommand).args
+    let { context } = this.parse(CreateEntityCommand).flags
+
+    if (!context) {
+      context = await this.promptForContext()
+    }
+
+    const currentAccount = await this.getRequiredSelectedAccount()
+    await this.requestAccountDecoding(currentAccount)
+    const [, entityClass] = await this.classEntryByNameOrId(className)
+
+    const actor = await this.getActor(context, entityClass)
+
+    if (actor.isOfType('Member') && entityClass.class_permissions.any_member.isFalse) {
+      this.error('Choosen actor has no access to create an entity of this type', { exit: ExitCodes.AccessDenied })
+    }
+
+    const answers: {
+      [key: string]: string | number | null
+    } = await inquirer.prompt(this.getQuestionsFromProperties(entityClass.properties.toArray()))
+
+    this.jsonPrettyPrint(JSON.stringify(answers))
+    await this.requireConfirmation('Do you confirm the provided input?')
+
+    const inputParser = InputParser.createWithKnownSchemas(this.getOriginalApi(), [
+      {
+        className: entityClass.name.toString(),
+        entries: [answers],
+      },
+    ])
+
+    const operations = await inputParser.getEntityBatchOperations()
+
+    await this.sendAndFollowNamedTx(currentAccount, 'contentDirectory', 'transaction', [actor, operations])
+  }
+}

cli/src/commands/content-directory/removeEntity.ts

@@ -1,6 +1,5 @@
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
 import { Actor } from '@joystream/types/content-directory'
-import { createType } from '@joystream/types'
 import ExitCodes from '../../ExitCodes'
 
 export default class RemoveEntityCommand extends ContentDirectoryCommandBase {
@@ -31,20 +30,9 @@ export default class RemoveEntityCommand extends ContentDirectoryCommandBase {
     }
 
     const account = await this.getRequiredSelectedAccount()
-    let actor: Actor
-    if (context === 'Curator') {
-      actor = await this.getCuratorContext([entityClass.name.toString()])
-    } else if (context === 'Member') {
-      const memberId = await this.getRequiredMemberId()
-      if (
-        !entity.entity_permissions.controller.isOfType('Member') ||
-        entity.entity_permissions.controller.asType('Member').toNumber() !== memberId
-      ) {
-        this.error('You are not the entity controller!', { exit: ExitCodes.AccessDenied })
-      }
-      actor = createType('Actor', { Member: memberId })
-    } else {
-      actor = createType('Actor', { Lead: null })
+    const actor: Actor = await this.getActor(context, entityClass)
+    if (!actor.isOfType('Curator') && !this.isActorEntityController(actor, entity, false)) {
+      this.error('You are not the entity controller!', { exit: ExitCodes.AccessDenied })
     }
 
     await this.requireConfirmation(

cli/src/commands/content-directory/updateEntityPropertyValues.ts

@@ -0,0 +1,61 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import inquirer from 'inquirer'
+import { InputParser } from '@joystream/cd-schemas'
+import ExitCodes from '../../ExitCodes'
+
+export default class UpdateEntityPropertyValues extends ContentDirectoryCommandBase {
+  static description =
+    'Updates the property values of the specified entity (can be executed in Member, Curator or Lead context)'
+
+  static args = [
+    {
+      name: 'id',
+      required: true,
+      description: 'ID of the Entity',
+    },
+  ]
+
+  static flags = {
+    context: ContentDirectoryCommandBase.contextFlag,
+  }
+
+  async run() {
+    const { id } = this.parse(UpdateEntityPropertyValues).args
+    let { context } = this.parse(UpdateEntityPropertyValues).flags
+
+    if (!context) {
+      context = await this.promptForContext()
+    }
+
+    const currentAccount = await this.getRequiredSelectedAccount()
+    await this.requestAccountDecoding(currentAccount)
+
+    const entity = await this.getEntity(id)
+    const [, entityClass] = await this.classEntryByNameOrId(entity.class_id.toString())
+    const defaults = await this.parseToEntityJson(entity)
+
+    const actor = await this.getActor(context, entityClass)
+
+    const isPropertEditableByIndex = await Promise.all(
+      entityClass.properties.map((p, i) => this.isEntityPropertyEditableByActor(entity, i, actor))
+    )
+    const filteredProperties = entityClass.properties.filter((p, i) => isPropertEditableByIndex[i])
+
+    if (!filteredProperties.length) {
+      this.error('No entity properties are editable by choosen actor', { exit: ExitCodes.AccessDenied })
+    }
+
+    const answers: {
+      [key: string]: string | number | null
+    } = await inquirer.prompt(this.getQuestionsFromProperties(filteredProperties, defaults))
+
+    this.jsonPrettyPrint(JSON.stringify(answers))
+    await this.requireConfirmation('Do you confirm the provided input?')
+
+    const inputParser = InputParser.createWithKnownSchemas(this.getOriginalApi())
+
+    const operations = await inputParser.getEntityUpdateOperations(answers, entityClass.name.toString(), +id)
+
+    await this.sendAndFollowNamedTx(currentAccount, 'contentDirectory', 'transaction', [actor, operations])
+  }
+}

cli/src/commands/media/featuredVideos.ts

@@ -11,7 +11,7 @@ export default class FeaturedVideosCommand extends ContentDirectoryCommandBase {
     const featured = await Promise.all(
       featuredEntries
         .filter(([, entity]) => entity.supported_schemas.toArray().length) // Ignore FeaturedVideo entities without schema
-        .map(([, entity]) => this.parseToKnownEntityJson<FeaturedVideoEntity>(entity))
+        .map(([, entity]) => this.parseToEntityJson<FeaturedVideoEntity>(entity))
     )
 
     const videoIds: number[] = featured.map(({ video: videoId }) => videoId)

cli/src/commands/media/removeChannel.ts

@@ -33,7 +33,7 @@ export default class RemoveChannelCommand extends ContentDirectoryCommandBase {
       channelId = id.toNumber()
       channelEntity = channel
     }
-    const channel = await this.parseToKnownEntityJson<ChannelEntity>(channelEntity)
+    const channel = await this.parseToEntityJson<ChannelEntity>(channelEntity)
 
     await this.requireConfirmation(`Are you sure you want to remove "${channel.handle}" channel?`)
 

cli/src/commands/media/removeVideo.ts

@@ -34,7 +34,7 @@ export default class RemoveVideoCommand extends ContentDirectoryCommandBase {
       videoEntity = video
     }
 
-    const video = await this.parseToKnownEntityJson<VideoEntity>(videoEntity)
+    const video = await this.parseToEntityJson<VideoEntity>(videoEntity)
 
     await this.requireConfirmation(`Are you sure you want to remove the "${video.title}" video?`)
 

cli/src/commands/media/updateChannel.ts

@@ -57,7 +57,7 @@ export default class UpdateChannelCommand extends ContentDirectoryCommandBase {
       channelEntity = channel
     }
 
-    const currentValues = await this.parseToKnownEntityJson<ChannelEntity>(channelEntity)
+    const currentValues = await this.parseToEntityJson<ChannelEntity>(channelEntity)
     this.jsonPrettyPrint(JSON.stringify(currentValues))
 
     const channelJsonSchema = (ChannelEntitySchema as unknown) as JSONSchema

cli/src/commands/media/updateVideo.ts

@@ -55,7 +55,7 @@ export default class UpdateVideoCommand extends MediaCommandBase {
       videoEntity = video
     }
 
-    const currentValues = await this.parseToKnownEntityJson<VideoEntity>(videoEntity)
+    const currentValues = await this.parseToEntityJson<VideoEntity>(videoEntity)
     const videoJsonSchema = (VideoEntitySchema as unknown) as JSONSchema
 
     const {

cli/src/commands/media/updateVideoLicense.ts

@@ -37,7 +37,7 @@ export default class UpdateVideoLicenseCommand extends MediaCommandBase {
       videoEntity = video
     }
 
-    const video = await this.parseToKnownEntityJson<VideoEntity>(videoEntity)
+    const video = await this.parseToEntityJson<VideoEntity>(videoEntity)
     const currentLicense = await this.getAndParseKnownEntity<LicenseEntity>(video.license)
 
     this.log('Current license:', currentLicense)

content-directory-schemas/package.json

@@ -14,9 +14,9 @@
     "generate:types": "ts-node --files ./scripts/schemasToTS.ts",
     "generate:entity-schemas": "ts-node ./scripts/inputSchemasToEntitySchemas.ts",
     "generate:all": "yarn generate:entity-schemas && yarn generate:types",
-    "initialize:alice-as-lead": "ts-node ./scripts/devInitAliceLead.ts",
+    "initialize:lead": "ts-node ./scripts/devInitContentLead.ts",
     "initialize:content-dir": "ts-node ./scripts/initializeContentDir.ts",
-    "initialize:dev": "yarn initialize:alice-as-lead && yarn initialize:content-dir",
+    "initialize:dev": "yarn initialize:lead && yarn initialize:content-dir",
     "example:createChannel": "ts-node ./examples/createChannel.ts",
     "example:createVideo": "ts-node ./examples/createVideo.ts",
     "example:updateChannelHandle": "ts-node ./examples/updateChannelHandle.ts",

content-directory-schemas/scripts/devInitAliceLead.ts

@@ -1,91 +0,0 @@
-import { types } from '@joystream/types'
-import { ApiPromise, WsProvider } from '@polkadot/api'
-import { SubmittableExtrinsic } from '@polkadot/api/types'
-import { ExtrinsicsHelper, getAlicePair } from '../src/helpers/extrinsics'
-
-async function main() {
-  // Init api
-  const WS_URI = process.env.WS_URI || 'ws://127.0.0.1:9944'
-  console.log(`Initializing the api (${WS_URI})...`)
-  const provider = new WsProvider(WS_URI)
-  const api = await ApiPromise.create({ provider, types })
-
-  const ALICE = getAlicePair()
-
-  const txHelper = new ExtrinsicsHelper(api)
-
-  const sudo = (tx: SubmittableExtrinsic<'promise'>) => api.tx.sudo.sudo(tx)
-  const extrinsics: SubmittableExtrinsic<'promise'>[] = []
-
-  // Create membership if not already created
-  let aliceMemberId: number | undefined = (await api.query.members.memberIdsByControllerAccountId(ALICE.address))
-    .toArray()[0]
-    ?.toNumber()
-
-  if (aliceMemberId === undefined) {
-    console.log('Perparing Alice member account creation extrinsic...')
-    aliceMemberId = (await api.query.members.nextMemberId()).toNumber()
-    extrinsics.push(api.tx.members.buyMembership(0, 'alice', null, null))
-  } else {
-    console.log(`Alice member id found: ${aliceMemberId}...`)
-  }
-
-  // Set Alice as lead if lead not already set
-  if ((await api.query.contentDirectoryWorkingGroup.currentLead()).isNone) {
-    const newOpeningId = (await api.query.contentDirectoryWorkingGroup.nextOpeningId()).toNumber()
-    const newApplicationId = (await api.query.contentDirectoryWorkingGroup.nextApplicationId()).toNumber()
-    // Create curator lead opening
-    console.log('Perparing Create Curator Lead Opening extrinsic...')
-    extrinsics.push(
-      sudo(
-        api.tx.contentDirectoryWorkingGroup.addOpening(
-          { CurrentBlock: null }, // activate_at
-          { max_review_period_length: 9999 }, // OpeningPolicyCommitment
-          'api-examples curator opening', // human_readable_text
-          'Leader' // opening_type
-        )
-      )
-    )
-
-    // Apply to lead opening
-    console.log('Perparing Apply to Curator Lead Opening as Alice extrinsic...')
-    extrinsics.push(
-      api.tx.contentDirectoryWorkingGroup.applyOnOpening(
-        aliceMemberId, // member id
-        newOpeningId, // opening id
-        ALICE.address, // address
-        null, // opt role stake
-        null, // opt appl. stake
-        'api-examples curator opening appl.' // human_readable_text
-      )
-    )
-
-    // Begin review period
-    console.log('Perparing Begin Applicant Review extrinsic...')
-    extrinsics.push(sudo(api.tx.contentDirectoryWorkingGroup.beginApplicantReview(newOpeningId)))
-
-    // Fill opening
-    console.log('Perparing Fill Opening extrinsic...')
-    extrinsics.push(
-      sudo(
-        api.tx.contentDirectoryWorkingGroup.fillOpening(
-          newOpeningId, // opening id
-          api.createType('ApplicationIdSet', [newApplicationId]), // succesful applicants
-          null // reward policy
-        )
-      )
-    )
-
-    console.log('Sending extrinsics...')
-    await txHelper.sendAndCheck(ALICE, extrinsics, 'Failed to initialize Alice as Content Curators Lead!')
-  } else {
-    console.log('Curators lead already exists, skipping...')
-  }
-}
-
-main()
-  .then(() => process.exit())
-  .catch((e) => {
-    console.error(e)
-    process.exit(-1)
-  })

content-directory-schemas/scripts/devInitContentLead.ts

@@ -0,0 +1,105 @@
+import { types } from '@joystream/types'
+import { ApiPromise, WsProvider } from '@polkadot/api'
+import { SubmittableExtrinsic } from '@polkadot/api/types'
+import { ExtrinsicsHelper, getAlicePair, getKeyFromSuri } from '../src/helpers/extrinsics'
+
+async function main() {
+  // Init api
+  const WS_URI = process.env.WS_URI || 'ws://127.0.0.1:9944'
+  console.log(`Initializing the api (${WS_URI})...`)
+  const provider = new WsProvider(WS_URI)
+  const api = await ApiPromise.create({ provider, types })
+
+  const LeadKeyPair = process.env.LEAD_URI ? getKeyFromSuri(process.env.LEAD_URI) : getAlicePair()
+  const SudoKeyPair = process.env.SUDO_URI ? getKeyFromSuri(process.env.SUDO_URI) : getAlicePair()
+
+  const txHelper = new ExtrinsicsHelper(api)
+
+  const sudo = (tx: SubmittableExtrinsic<'promise'>) => api.tx.sudo.sudo(tx)
+
+  // Create membership if not already created
+  let memberId: number | undefined = (await api.query.members.memberIdsByControllerAccountId(LeadKeyPair.address))
+    .toArray()[0]
+    ?.toNumber()
+
+  // Only buy membership if LEAD_URI is not provided
+  if (memberId === undefined && process.env.LEAD_URI) {
+    throw new Error('Make sure Controller key LEAD_URI is for a member')
+  }
+
+  if (memberId === undefined) {
+    console.log('Perparing member account creation extrinsic...')
+    memberId = (await api.query.members.nextMemberId()).toNumber()
+    await txHelper.sendAndCheck(
+      LeadKeyPair,
+      [api.tx.members.buyMembership(0, 'alice', null, null)],
+      'Failed to setup member account'
+    )
+  }
+
+  console.log(`Making member id: ${memberId} the content lead.`)
+
+  // Create a new lead opening
+  if ((await api.query.contentDirectoryWorkingGroup.currentLead()).isNone) {
+    const newOpeningId = (await api.query.contentDirectoryWorkingGroup.nextOpeningId()).toNumber()
+    const newApplicationId = (await api.query.contentDirectoryWorkingGroup.nextApplicationId()).toNumber()
+    // Create curator lead opening
+    console.log('Perparing Create Curator Lead Opening extrinsic...')
+    await txHelper.sendAndCheck(
+      SudoKeyPair,
+      [
+        sudo(
+          api.tx.contentDirectoryWorkingGroup.addOpening(
+            { CurrentBlock: null }, // activate_at
+            { max_review_period_length: 9999 }, // OpeningPolicyCommitment
+            'bootstrap curator opening', // human_readable_text
+            'Leader' // opening_type
+          )
+        ),
+      ],
+      'Failed to create Content Curators Lead opening!'
+    )
+
+    // Apply to lead opening
+    console.log('Perparing Apply to Curator Lead Opening as extrinsic...')
+    await txHelper.sendAndCheck(
+      LeadKeyPair,
+      [
+        api.tx.contentDirectoryWorkingGroup.applyOnOpening(
+          memberId, // member id
+          newOpeningId, // opening id
+          LeadKeyPair.address, // address
+          null, // opt role stake
+          null, // opt appl. stake
+          'bootstrap curator opening' // human_readable_text
+        ),
+      ],
+      'Failed to apply on lead opening!'
+    )
+
+    const extrinsics: SubmittableExtrinsic<'promise'>[] = []
+    // Begin review period
+    console.log('Perparing Begin Applicant Review extrinsic...')
+    extrinsics.push(sudo(api.tx.contentDirectoryWorkingGroup.beginApplicantReview(newOpeningId)))
+
+    // Fill opening
+    console.log('Perparing Fill Opening extrinsic...')
+    extrinsics.push(
+      sudo(
+        api.tx.contentDirectoryWorkingGroup.fillOpening(
+          newOpeningId, // opening id
+          api.createType('ApplicationIdSet', [newApplicationId]), // succesful applicants
+          null // reward policy
+        )
+      )
+    )
+
+    await txHelper.sendAndCheck(SudoKeyPair, extrinsics, 'Failed to initialize Content Curators Lead!')
+  } else {
+    console.log('Curators lead already exists, skipping...')
+  }
+}
+
+main()
+  .then(() => process.exit())
+  .catch((e) => console.error(e))

content-directory-schemas/scripts/initializeContentDir.ts

@@ -4,7 +4,7 @@ import { getInitializationInputs } from '../src/helpers/inputs'
 import fs from 'fs'
 import path from 'path'
 import { InputParser } from '../src/helpers/InputParser'
-import { ExtrinsicsHelper, getAlicePair } from '../src/helpers/extrinsics'
+import { ExtrinsicsHelper, getAlicePair, getKeyFromSuri } from '../src/helpers/extrinsics'
 
 // Save entity operations output here for easier debugging
 const ENTITY_OPERATIONS_OUTPUT_PATH = path.join(__dirname, '../operations.json')
@@ -18,7 +18,7 @@ async function main() {
   const provider = new WsProvider(WS_URI)
   const api = await ApiPromise.create({ provider, types })
 
-  const ALICE = getAlicePair()
+  const LeadKeyPair = process.env.LEAD_URI ? getKeyFromSuri(process.env.LEAD_URI) : getAlicePair()
 
   // Emptiness check
   if ((await api.query.contentDirectory.classById.keys()).length > 0) {
@@ -31,11 +31,11 @@ async function main() {
 
   console.log(`Initializing classes (${classInputs.length} input files found)...\n`)
   const classExtrinsics = parser.getCreateClassExntrinsics()
-  await txHelper.sendAndCheck(ALICE, classExtrinsics, 'Class initialization failed!')
+  await txHelper.sendAndCheck(LeadKeyPair, classExtrinsics, 'Class initialization failed!')
 
   console.log(`Initializing schemas (${schemaInputs.length} input files found)...\n`)
   const schemaExtrinsics = await parser.getAddSchemaExtrinsics()
-  await txHelper.sendAndCheck(ALICE, schemaExtrinsics, 'Schemas initialization failed!')
+  await txHelper.sendAndCheck(LeadKeyPair, schemaExtrinsics, 'Schemas initialization failed!')
 
   console.log(`Initializing entities (${entityBatchInputs.length} input files found)`)
   const entityOperations = await parser.getEntityBatchOperations()
@@ -51,7 +51,7 @@ async function main() {
   )
   console.log(`Sending Transaction extrinsic (${entityOperations.length} operations)...`)
   await txHelper.sendAndCheck(
-    ALICE,
+    LeadKeyPair,
     [api.tx.contentDirectory.transaction({ Lead: null }, entityOperations)],
     'Entity initialization failed!'
   )

content-directory-schemas/src/helpers/extrinsics.ts

@@ -7,7 +7,7 @@ import { TypeRegistry } from '@polkadot/types'
 
 // TODO: Move to @joystream/js soon
 
-export function getAlicePair() {
+export function getAlicePair(): KeyringPair {
   const keyring = new Keyring({ type: 'sr25519' })
   keyring.addFromUri('//Alice', { name: 'Alice' })
   const ALICE = keyring.getPairs()[0]
@@ -15,6 +15,13 @@ export function getAlicePair() {
   return ALICE
 }
 
+export function getKeyFromSuri(suri: string): KeyringPair {
+  const keyring = new Keyring({ type: 'sr25519' })
+
+  // Assume a SURI, add to keyring and return keypair
+  return keyring.addFromUri(suri)
+}
+
 export class ExtrinsicsHelper {
   api: ApiPromise
   noncesByAddress: Map<string, number>

pioneer/packages/apps-routing/src/index.ts

@@ -27,10 +27,12 @@ import proposals from './joy-proposals';
 import roles from './joy-roles';
 import forum from './joy-forum';
 import tokenomics from './joy-tokenomics';
+import media from './joy-media';
 
 export default function create (t: <T = string> (key: string, text: string, options: { ns: string }) => T): Routes {
   return appSettings.uiMode === 'light'
     ? [
+      media(t),
       tokenomics(t),
       members(t),
       roles(t),
@@ -47,6 +49,7 @@ export default function create (t: <T = string> (key: string, text: string, opti
       privacyPolicy(t)
     ]
     : [
+      media(t),
       tokenomics(t),
       members(t),
       roles(t),

pioneer/packages/apps-routing/src/joy-media.ts

@@ -0,0 +1,13 @@
+import { Route } from './types';
+
+import Media from '@polkadot/joy-media';
+
+export default function create (t: <T = string> (key: string, text: string, options: { ns: string }) => T): Route {
+  return {
+    Component: Media,
+    display: {},
+    text: t<string>('nav.media', 'Media', { ns: 'apps-routing' }),
+    icon: 'play-circle',
+    name: 'media'
+  };
+}

pioneer/packages/apps/src/wp-jpg.d.ts

@@ -0,0 +1,5 @@
+declare module '*.jpg' {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const content: any;
+  export default content;
+}

pioneer/packages/joy-media/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "@polkadot/joy-media",
+  "version": "0.1.0",
+  "description": "Pioneer's media page",
+  "main": "index.js",
+  "scripts": {},
+  "author": "Joystream contributors",
+  "maintainers": [],
+  "dependencies": {
+    "@babel/runtime": "^7.10.5",
+    "@polkadot/react-components": "0.51.1"
+  }
+}

pioneer/packages/joy-media/src/index.tsx

@@ -0,0 +1,118 @@
+import React from 'react';
+import styled from 'styled-components';
+
+import { I18nProps } from '@polkadot/react-components/types';
+import _ from 'lodash';
+
+import { RouteProps as AppMainRouteProps } from '@polkadot/apps-routing/types';
+import translate from './translate';
+import { Button, Grid, Message, Icon, Image } from 'semantic-ui-react';
+
+import AtlasScreenShot from './assets/atlas-screenshot.jpg';
+
+const MediaMain = styled.main`
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  padding-top: 2em;
+  font-size: 1.2em;
+  p {
+    margin: 0.25em;
+    padding: 0;
+  }
+`;
+
+const Header = styled.header`
+  margin-bottom: 1em;
+  h1 {
+    color: #222 !important;
+  }
+`;
+
+const StyledMessage = styled(Message)`
+  font-size: 1.2em;
+  display: flex;
+  flex-direction: column;
+  background: #fff !important;
+  .header, .content {
+    margin-bottom: 0.5em !important;
+  }
+  .button {
+    margin-top: auto;
+    margin-right: auto !important;
+  }
+`;
+
+const Screenshot = styled(Image)`
+  margin: 0.5em 0;
+  transition: opacity 0.5s;
+  :hover { opacity: 0.7; }
+`;
+
+interface Props extends AppMainRouteProps, I18nProps {}
+
+const App: React.FC<Props> = () => {
+  return (
+    <MediaMain>
+      <Header>
+        <h1>Hello there!</h1>
+        <p>
+          We have now upgraded to the Babylon chain.
+        </p>
+        <p>
+          Pioneer consequently <b>no longer supports</b> media uploads and consumption.
+        </p>
+      </Header>
+      <Grid stackable>
+        <Grid.Row columns={2}>
+          <Grid.Column>
+            <StyledMessage>
+              <Message.Header>Media consumption</Message.Header>
+              <Message.Content>
+                Media consumption has been migrated over to our new consumer interface.
+                <Screenshot
+                  src={AtlasScreenShot as string}
+                  href='https://play.joystream.org'
+                  target='_blank'
+                  rel='noopener noreferrer'/>
+              </Message.Content>
+              <Button
+                size='big'
+                primary
+                icon
+                labelPosition='right'
+                href='https://play.joystream.org'
+                target='_blank'
+                rel='noopener noreferrer'>
+                Launch Atlas
+                <Icon name='arrow right' />
+              </Button>
+            </StyledMessage>
+          </Grid.Column>
+          <Grid.Column>
+            <StyledMessage>
+              <Message.Header>Uploading content</Message.Header>
+              <Message.Content>
+                Uploading has been migrated over to the Joystream CLI.
+                Instructions on how to use the CLI can be found in our helpdesk.
+              </Message.Content>
+              <Button
+                size='big'
+                primary
+                href='https://github.com/Joystream/helpdesk/tree/master/roles/content-creators'
+                icon
+                labelPosition='right'
+                target='_blank'
+                rel='noopener noreferrer'>
+                Explore Joystream CLI
+                <Icon name='arrow right' />
+              </Button>
+            </StyledMessage>
+          </Grid.Column>
+        </Grid.Row>
+      </Grid>
+    </MediaMain>
+  );
+};
+
+export default translate(App);

pioneer/packages/joy-media/src/translate.ts

@@ -0,0 +1,3 @@
+import { withTranslation } from 'react-i18next';
+
+export default withTranslation(['media', 'ui']);

pioneer/tsconfig.json

@@ -28,6 +28,10 @@
       "@polkadot/joy-utils/*": [ "packages/joy-utils/src/*" ],
       "@polkadot/joy-forum": [ "packages/joy-forum/src/" ],
       "@polkadot/joy-forum/*": [ "packages/joy-forum/src/*" ],
+      "@polkadot/joy-media": [ "packages/joy-media/src" ],
+      "@polkadot/joy-media/*": [ "packages/joy-media/src/*" ],
+      "@polkadot/joy-tokenomics": [ "packages/joy-tokenomics/src" ],
+      "@polkadot/joy-tokenomics/*": [ "packages/joy-tokenomics/src/*" ],
       "@polkadot/apps/*": ["packages/apps/src/*"],
       "@polkadot/apps": ["packages/apps/src"],
       "@polkadot/apps-config/*": [ "packages/apps-config/src/*" ],
@@ -86,9 +90,7 @@
       "@polkadot/react-query": [ "packages/react-query/src" ],
       "@polkadot/react-query/*": [ "packages/react-query/src/*" ],
       "@polkadot/react-signer": [ "packages/react-signer/src" ],
-      "@polkadot/react-signer/*": [ "packages/react-signer/src/*" ],
-      "@polkadot/joy-tokenomics": [ "packages/joy-tokenomics/src" ],
-      "@polkadot/joy-tokenomics/*": [ "packages/joy-tokenomics/src/*" ]
+      "@polkadot/react-signer/*": [ "packages/react-signer/src/*" ]
     },
     "skipLibCheck": true,
     "typeRoots": [

runtime/CHANGELOG.md

@@ -1,3 +1,6 @@
+### Version 7.9.0 - Babylon - runtime upgrade - December 21 2020
+- Introduction of new and improved content directory
+
 ### Version 7.4.0 - Alexandria - new chain - September 21 2020
 - Update to substrate v2.0.0-rc4
 

setup.sh

@@ -16,7 +16,10 @@ rustup component add rustfmt clippy
 rustup install nightly-2020-05-23 --force
 rustup target add wasm32-unknown-unknown --toolchain nightly-2020-05-23
 
-# Sticking with older version of compiler to ensure working build
+# Latest clippy linter which comes with 1.47.0 fails on some subtrate modules
+# Also note combination of newer versions of toolchain with the above nightly
+# toolchain to build wasm seems to fail.
+# So we need to stick with an older version until we update substrate
 rustup install 1.46.0
 rustup default 1.46.0
 

storage-node/packages/cli/src/cli.ts

@@ -49,7 +49,7 @@ const usage = `
   Dev Commands:       Commands to run on a development chain.
     dev-init          Setup chain with Alice as lead and storage provider.
     dev-check         Check the chain is setup with Alice as lead and storage provider.
-    vstore-init      Initialize versioned store, Requires SURI of ContentWorking Lead.
+    sudo-create-sp    Initialize the chain with a lead storage provider.
     
   Type 'storage-cli command' for the exact command usage examples.
   `
@@ -72,6 +72,9 @@ const commands = {
   'dev-check': async (api) => {
     return dev.check(api)
   },
+  'sudo-create-sp': async (api) => {
+    return dev.makeMemberInitialLeadAndStorageProvider(api)
+  },
   // Uploads the file to the system. Registers new data object in the runtime, obtains proper colossus instance URL.
   upload: async (
     api: any,

storage-node/packages/cli/src/commands/dev.ts

@@ -3,6 +3,8 @@
 import dbug from 'debug'
 import { KeyringPair } from '@polkadot/keyring/types'
 import { RuntimeApi } from '@joystream/storage-runtime-api'
+import { GenericJoyStreamRoleSchema as HRTJson } from '@joystream/types/hiring/schemas/role.schema.typings'
+
 const debug = dbug('joystream:storage-cli:dev')
 
 // Derivation path appended to well known development seed used on
@@ -142,4 +144,129 @@ const init = async (api: RuntimeApi): Promise<any> => {
   return check(api)
 }
 
-export { init, check, aliceKeyPair, roleKeyPair, developmentPort }
+// Using sudo to create initial storage lead and worker with given keys taken from env variables.
+// Used to quickly setup a storage provider on a new chain before a council is ready.
+const makeMemberInitialLeadAndStorageProvider = async (api: RuntimeApi): Promise<any> => {
+  if (api.workers.getLeadRoleAccount()) {
+    throw new Error('The Storage Lead is already set!')
+  }
+
+  if (!process.env.SUDO_URI) {
+    throw new Error('required SUDO_URI env variable was not set')
+  }
+
+  if (!process.env.MEMBER_ID) {
+    throw new Error('required MEMBER_ID env variable was not set')
+  }
+
+  if (!process.env.MEMBER_CONTROLLER_URI) {
+    throw new Error('required MEMBER_CONTROLLER_URI env variable was not set')
+  }
+
+  if (!process.env.STORAGE_WORKER_ADDRESS) {
+    throw new Error('required STORAGE_WORKER_ADDRESS env variable was not set')
+  }
+
+  const sudoKey = getKeyFromAddressOrSuri(api, process.env.SUDO_URI)
+  const memberId = parseInt(process.env.MEMBER_ID)
+  const memberController = getKeyFromAddressOrSuri(api, process.env.MEMBER_CONTROLLER_URI).address
+  const leadAccount = memberController
+  const workerAccount = process.env.STORAGE_WORKER_ADDRESS
+
+  const sudo = await api.identities.getSudoAccount()
+
+  // Ensure correct sudo key was provided
+  if (!sudo.eq(sudoKey.address)) {
+    throw new Error('Provided SUDO_URI is not the chain sudo')
+  }
+
+  // Ensure MEMBER_ID and MEMBER_CONTROLLER_URI are valid
+  const memberIds = await api.identities.memberIdsOfController(memberController)
+  if (memberIds.find((id) => id.eq(memberId)) === undefined) {
+    throw new Error(
+      'MEMBER_ID and MEMBER_CONTROLLER_URI do not correspond to a registered member and their controller account'
+    )
+  }
+
+  // Ensure STORAGE_WORKER_ADDRESS is a valid Address
+  api.identities.keyring.decodeAddress(workerAccount)
+
+  debug(`Creating Leader with role key: ${leadAccount}`)
+  debug('Creating Lead Opening')
+  const leadOpeningId = await api.workers.devAddStorageLeadOpening(JSON.stringify(getLeadOpeningInfo()))
+  debug('Applying')
+  const leadApplicationId = await api.workers.devApplyOnOpening(leadOpeningId, memberId, memberController, leadAccount)
+  debug('Starting Review')
+  api.workers.devBeginLeadOpeningReview(leadOpeningId)
+  debug('Filling Opening')
+  await api.workers.devFillLeadOpening(leadOpeningId, leadApplicationId)
+
+  const setLeadAccount = await api.workers.getLeadRoleAccount()
+  if (!setLeadAccount.eq(leadAccount)) {
+    throw new Error('Setting Lead failed!')
+  }
+
+  // Create a storage openinging, apply, start review, and fill opening
+  debug(`Making ${workerAccount} account a storage provider.`)
+
+  const openingId = await api.workers.devAddStorageOpening(JSON.stringify(getWorkerOpeningInfo()))
+  debug(`Created new storage opening: ${openingId}`)
+
+  const applicationId = await api.workers.devApplyOnOpening(openingId, memberId, memberController, workerAccount)
+  debug(`Applied with application id: ${applicationId}`)
+
+  api.workers.devBeginStorageOpeningReview(openingId)
+
+  debug(`Filling storage opening.`)
+  const providerId = await api.workers.devFillStorageOpening(openingId, applicationId)
+
+  debug(`Assigned storage provider id: ${providerId}`)
+}
+
+function getLeadOpeningInfo(): HRTJson {
+  return {
+    'version': 1,
+    'headline': 'Initial Storage Lead',
+    'job': {
+      'title': 'Bootstrap Lead',
+      'description': 'Starting opportunity to bootstrap the network',
+    },
+    'application': {
+      'sections': [],
+    },
+    'reward': 'None',
+    'creator': {
+      'membership': {
+        'handle': 'mokhtar',
+      },
+    },
+    'process': {
+      'details': ['automated'],
+    },
+  }
+}
+
+function getWorkerOpeningInfo(): HRTJson {
+  return {
+    'version': 1,
+    'headline': 'Initial Storage Worker',
+    'job': {
+      'title': 'Bootstrap Worker',
+      'description': 'Starting opportunity to bootstrap the network',
+    },
+    'application': {
+      'sections': [],
+    },
+    'reward': 'None',
+    'creator': {
+      'membership': {
+        'handle': 'mokhtar',
+      },
+    },
+    'process': {
+      'details': ['automated'],
+    },
+  }
+}
+
+export { init, check, aliceKeyPair, roleKeyPair, developmentPort, makeMemberInitialLeadAndStorageProvider }

storage-node/packages/colossus/bin/cli.js

@@ -31,18 +31,22 @@ const FLAG_DEFINITIONS = {
   keyFile: {
     type: 'string',
     isRequired: (flags, input) => {
-      // Only required if running server command and not in dev mode
-      const serverCmd = input[0] === 'server'
-      return !flags.dev && serverCmd
+      // Only required if running server command and not in dev or anonymous mode
+      if (flags.anonymous || flags.dev) {
+        return false
+      }
+      return input[0] === 'server'
     },
   },
   publicUrl: {
     type: 'string',
     alias: 'u',
     isRequired: (flags, input) => {
-      // Only required if running server command and not in dev mode
-      const serverCmd = input[0] === 'server'
-      return !flags.dev && serverCmd
+      // Only required if running server command and not in dev or anonymous mode
+      if (flags.anonymous || flags.dev) {
+        return false
+      }
+      return input[0] === 'server'
     },
   },
   passphrase: {
@@ -56,15 +60,21 @@ const FLAG_DEFINITIONS = {
     type: 'number',
     alias: 'i',
     isRequired: (flags, input) => {
-      // Only required if running server command and not in dev mode
-      const serverCmd = input[0] === 'server'
-      return !flags.dev && serverCmd
+      // Only required if running server command and not in dev or anonymous mode
+      if (flags.anonymous || flags.dev) {
+        return false
+      }
+      return input[0] === 'server'
     },
   },
   ipfsHost: {
     type: 'string',
     default: 'localhost',
   },
+  anonymous: {
+    type: 'boolean',
+    default: false,
+  },
 }
 
 const cli = meow(
@@ -77,7 +87,7 @@ const cli = meow(
                   This is the default command if not specified.
     discovery     Run the discovery service only.
 
-  Arguments (required for server. Ignored if running server with --dev option):
+  Arguments (required for with server command, unless --dev or --anonymous args are used):
     --provider-id ID, -i ID     StorageProviderId assigned to you in working group.
     --key-file FILE             JSON key export file to use as the storage provider (role account).
     --public-url=URL, -u URL    API Public URL to announce.
@@ -88,6 +98,8 @@ const cli = meow(
     --port=PORT, -p PORT    Port number to listen on, defaults to 3000.
     --ws-provider WS_URL    Joystream-node websocket provider, defaults to ws://localhost:9944
     --ipfs-host   hostname  ipfs host to use, default to 'localhost'. Default port 5001 is always used
+    --anonymous             Runs server in anonymous mode. Replicates content without need to register
+                            on-chain, and can serve content. Cannot be used to upload content.
   `,
   { flags: FLAG_DEFINITIONS }
 )
@@ -116,8 +128,8 @@ function startExpressApp(app, port) {
 }
 
 // Start app
-function startAllServices({ store, api, port, discoveryClient, ipfsHttpGatewayUrl }) {
-  const app = require('../lib/app')(PROJECT_ROOT, store, api, discoveryClient, ipfsHttpGatewayUrl)
+function startAllServices({ store, api, port, discoveryClient, ipfsHttpGatewayUrl, anonymous }) {
+  const app = require('../lib/app')(PROJECT_ROOT, store, api, discoveryClient, ipfsHttpGatewayUrl, anonymous)
   return startExpressApp(app, port)
 }
 
@@ -149,7 +161,7 @@ function getStorage(runtimeApi, { ipfsHost }) {
   return Storage.create(options)
 }
 
-async function initApiProduction({ wsProvider, providerId, keyFile, passphrase }) {
+async function initApiProduction({ wsProvider, providerId, keyFile, passphrase, anonymous }) {
   // Load key information
   const { RuntimeApi } = require('@joystream/storage-runtime-api')
 
@@ -160,7 +172,7 @@ async function initApiProduction({ wsProvider, providerId, keyFile, passphrase }
     storageProviderId: providerId,
   })
 
-  if (!api.identities.key) {
+  if (!anonymous && !api.identities.key) {
     throw new Error('Failed to unlock storage provider account')
   }
 
@@ -168,7 +180,7 @@ async function initApiProduction({ wsProvider, providerId, keyFile, passphrase }
 
   // We allow the node to startup without correct provider id and account, but syncing and
   // publishing of identity will be skipped.
-  if (!(await api.providerIsActiveWorker())) {
+  if (!anonymous && !(await api.providerIsActiveWorker())) {
     debug('storage provider role account and storageProviderId are not associated with a worker')
   }
 
@@ -295,17 +307,19 @@ const commands = {
 
     const ipfsHost = cli.flags.ipfsHost
     const ipfs = require('ipfs-http-client')(ipfsHost, '5001', { protocol: 'http' })
-    const { PublisherClient, DiscoveryClient } = require('@joystream/service-discovery')
-    const publisherClient = new PublisherClient(ipfs)
-    const discoveryClient = new DiscoveryClient({ ipfs, api })
     const ipfsHttpGatewayUrl = `http://${ipfsHost}:8080/`
 
     const { startSyncing } = require('../lib/sync')
-    startSyncing(api, { syncPeriod: SYNC_PERIOD_MS }, store)
+    startSyncing(api, { syncPeriod: SYNC_PERIOD_MS, anonymous: cli.flags.anonymous }, store)
 
-    announcePublicUrl(api, publicUrl, publisherClient)
+    if (!cli.flags.anonymous) {
+      const { PublisherClient } = require('@joystream/service-discovery')
+      announcePublicUrl(api, publicUrl, new PublisherClient(ipfs))
+    }
 
-    return startAllServices({ store, api, port, discoveryClient, ipfsHttpGatewayUrl })
+    const { DiscoveryClient } = require('@joystream/service-discovery')
+    const discoveryClient = new DiscoveryClient({ ipfs, api })
+    return startAllServices({ store, api, port, discoveryClient, ipfsHttpGatewayUrl, anonymous: cli.flags.anonymous })
   },
   discovery: async () => {
     banner()

storage-node/packages/colossus/lib/app.js

@@ -35,7 +35,7 @@ const fileUploads = require('./middleware/file_uploads')
 const pagination = require('@joystream/storage-utils/pagination')
 
 // Configure app
-function createApp(projectRoot, storage, runtime, discoveryClient, ipfsHttpGatewayUrl) {
+function createApp(projectRoot, storage, runtime, discoveryClient, ipfsHttpGatewayUrl, anonymous) {
   const app = express()
   app.use(cors())
   app.use(bodyParser.json())
@@ -61,6 +61,7 @@ function createApp(projectRoot, storage, runtime, discoveryClient, ipfsHttpGatew
       runtime,
       discoveryClient,
       ipfsHttpGatewayUrl,
+      anonymous,
     },
   })
 

storage-node/packages/colossus/lib/sync.js

@@ -132,30 +132,35 @@ async function syncPeriodic({ api, flags, storage, contentBeingSynced, contentCo
       return retry()
     }
 
-    // Retry later if provider is not active
-    if (!(await api.providerIsActiveWorker())) {
-      debug(
-        'storage provider role account and storageProviderId are not associated with a worker. Postponing sync run.'
-      )
-      return retry()
-    }
+    if (!flags.anonymous) {
+      // Retry later if provider is not active
+      if (!(await api.providerIsActiveWorker())) {
+        debug(
+          'storage provider role account and storageProviderId are not associated with a worker. Postponing sync run.'
+        )
+        return retry()
+      }
 
-    const recommendedBalance = await api.providerHasMinimumBalance(300)
-    if (!recommendedBalance) {
-      debug('Warning: Provider role account is running low on balance.')
-    }
+      const recommendedBalance = await api.providerHasMinimumBalance(300)
+      if (!recommendedBalance) {
+        debug('Warning: Provider role account is running low on balance.')
+      }
 
-    const sufficientBalance = await api.providerHasMinimumBalance(100)
-    if (!sufficientBalance) {
-      debug('Provider role account does not have sufficient balance. Postponing sync run!')
-      return retry()
+      const sufficientBalance = await api.providerHasMinimumBalance(100)
+      if (!sufficientBalance) {
+        debug('Provider role account does not have sufficient balance. Postponing sync run!')
+        return retry()
+      }
     }
 
     await syncContent({ api, storage, contentBeingSynced, contentCompleteSynced })
-    const relationshipIds = await createNewRelationships({ api, contentCompleteSynced })
-    await setRelationshipsReady({ api, relationshipIds })
 
-    debug(`Sync run completed, set ${relationshipIds.length} new relationships to ready`)
+    // Only update on chain state if not in anonymous mode
+    if (!flags.anonymous) {
+      const relationshipIds = await createNewRelationships({ api, contentCompleteSynced })
+      await setRelationshipsReady({ api, relationshipIds })
+      debug(`Sync run completed, set ${relationshipIds.length} new relationships to ready`)
+    }
   } catch (err) {
     debug(`Error in sync run ${err.stack}`)
   }

storage-node/packages/colossus/paths/asset/v0/{id}.js

@@ -27,7 +27,7 @@ function errorHandler(response, err, code) {
   response.status(err.code || code || 500).send({ message: err.toString() })
 }
 
-module.exports = function (storage, runtime, ipfsHttpGatewayUrl) {
+module.exports = function (storage, runtime, ipfsHttpGatewayUrl, anonymous) {
   // Creat the IPFS HTTP Gateway proxy middleware
   const proxy = ipfsProxy.createProxy(storage, ipfsHttpGatewayUrl)
 
@@ -47,6 +47,11 @@ module.exports = function (storage, runtime, ipfsHttpGatewayUrl) {
 
     // Put for uploads
     async put(req, res) {
+      if (anonymous) {
+        errorHandler(res, 'Uploads Not Permitted in Anonymous Mode', 400)
+        return
+      }
+
       const id = req.params.id // content id
 
       // First check if we're the liaison for the name, otherwise we can bail

storage-node/packages/runtime-api/identities.js

@@ -135,6 +135,14 @@ class IdentitiesApi {
     return this.base.api.query.members.memberIdsByRootAccountId(decoded)
   }
 
+  /*
+   * Return all the member IDs of an account by the controller account id
+   */
+  async memberIdsOfController(accountId) {
+    const decoded = this.keyring.decodeAddress(accountId)
+    return this.base.api.query.members.memberIdsByControllerAccountId(decoded)
+  }
+
   /*
    * Return the first member ID of an account, or undefined if not a member root account.
    */

storage-node/packages/runtime-api/workers.js

@@ -150,8 +150,8 @@ class WorkersApi {
    * Add a new storage group opening using the lead account. Returns the
    * new opening id.
    */
-  async devAddStorageOpening() {
-    const openTx = this.devMakeAddOpeningTx('Worker')
+  async devAddStorageOpening(info) {
+    const openTx = this.devMakeAddOpeningTx('Worker', info)
     return this.devSubmitAddOpeningTx(openTx, await this.getLeadRoleAccount())
   }
 
@@ -159,8 +159,8 @@ class WorkersApi {
    * Add a new storage working group lead opening using sudo account. Returns the
    * new opening id.
    */
-  async devAddStorageLeadOpening() {
-    const openTx = this.devMakeAddOpeningTx('Leader')
+  async devAddStorageLeadOpening(info) {
+    const openTx = this.devMakeAddOpeningTx('Leader', info)
     const sudoTx = this.base.api.tx.sudo.sudo(openTx)
     return this.devSubmitAddOpeningTx(sudoTx, await this.base.identities.getSudoAccount())
   }
@@ -168,17 +168,17 @@ class WorkersApi {
   /*
    * Constructs an addOpening tx of openingType
    */
-  devMakeAddOpeningTx(openingType) {
+  devMakeAddOpeningTx(openingType, info) {
     return this.base.api.tx.storageWorkingGroup.addOpening(
       'CurrentBlock',
       {
         application_rationing_policy: {
           max_active_applicants: 1,
         },
-        max_review_period_length: 1000,
+        max_review_period_length: 10,
         // default values for everything else..
       },
-      'dev-opening',
+      info || 'dev-opening',
       openingType
     )
   }