storage-node: anonymous (replication only) mode

storage-node/packages/colossus/bin/cli.js

@@ -31,18 +31,22 @@ const FLAG_DEFINITIONS = {
   keyFile: {
     type: 'string',
     isRequired: (flags, input) => {
-      // Only required if running server command and not in dev mode
-      const serverCmd = input[0] === 'server'
-      return !flags.dev && serverCmd
+      // Only required if running server command and not in dev or anonymous mode
+      if (flags.anonymous || flags.dev) {
+        return false
+      }
+      return input[0] === 'server'
     },
   },
   publicUrl: {
     type: 'string',
     alias: 'u',
     isRequired: (flags, input) => {
-      // Only required if running server command and not in dev mode
-      const serverCmd = input[0] === 'server'
-      return !flags.dev && serverCmd
+      // Only required if running server command and not in dev or anonymous mode
+      if (flags.anonymous || flags.dev) {
+        return false
+      }
+      return input[0] === 'server'
     },
   },
   passphrase: {
@@ -56,15 +60,21 @@ const FLAG_DEFINITIONS = {
     type: 'number',
     alias: 'i',
     isRequired: (flags, input) => {
-      // Only required if running server command and not in dev mode
-      const serverCmd = input[0] === 'server'
-      return !flags.dev && serverCmd
+      // Only required if running server command and not in dev or anonymous mode
+      if (flags.anonymous || flags.dev) {
+        return false
+      }
+      return input[0] === 'server'
     },
   },
   ipfsHost: {
     type: 'string',
     default: 'localhost',
   },
+  anonymous: {
+    type: 'boolean',
+    default: false,
+  },
 }
 
 const cli = meow(
@@ -77,7 +87,7 @@ const cli = meow(
                   This is the default command if not specified.
     discovery     Run the discovery service only.
 
-  Arguments (required for server. Ignored if running server with --dev option):
+  Arguments (required for with server command, unless --dev or --anonymous args are used):
     --provider-id ID, -i ID     StorageProviderId assigned to you in working group.
     --key-file FILE             JSON key export file to use as the storage provider (role account).
     --public-url=URL, -u URL    API Public URL to announce.
@@ -88,6 +98,8 @@ const cli = meow(
     --port=PORT, -p PORT    Port number to listen on, defaults to 3000.
     --ws-provider WS_URL    Joystream-node websocket provider, defaults to ws://localhost:9944
     --ipfs-host   hostname  ipfs host to use, default to 'localhost'. Default port 5001 is always used
+    --anonymous             Runs server in anonymous mode. Replicates content without need to register
+                            on-chain, and can serve content. Cannot be used to upload content.
   `,
   { flags: FLAG_DEFINITIONS }
 )
@@ -116,8 +128,8 @@ function startExpressApp(app, port) {
 }
 
 // Start app
-function startAllServices({ store, api, port, discoveryClient, ipfsHttpGatewayUrl }) {
-  const app = require('../lib/app')(PROJECT_ROOT, store, api, discoveryClient, ipfsHttpGatewayUrl)
+function startAllServices({ store, api, port, discoveryClient, ipfsHttpGatewayUrl, anonymous }) {
+  const app = require('../lib/app')(PROJECT_ROOT, store, api, discoveryClient, ipfsHttpGatewayUrl, anonymous)
   return startExpressApp(app, port)
 }
 
@@ -149,7 +161,7 @@ function getStorage(runtimeApi, { ipfsHost }) {
   return Storage.create(options)
 }
 
-async function initApiProduction({ wsProvider, providerId, keyFile, passphrase }) {
+async function initApiProduction({ wsProvider, providerId, keyFile, passphrase, anonymous }) {
   // Load key information
   const { RuntimeApi } = require('@joystream/storage-runtime-api')
 
@@ -160,7 +172,7 @@ async function initApiProduction({ wsProvider, providerId, keyFile, passphrase }
     storageProviderId: providerId,
   })
 
-  if (!api.identities.key) {
+  if (!anonymous && !api.identities.key) {
     throw new Error('Failed to unlock storage provider account')
   }
 
@@ -168,7 +180,7 @@ async function initApiProduction({ wsProvider, providerId, keyFile, passphrase }
 
   // We allow the node to startup without correct provider id and account, but syncing and
   // publishing of identity will be skipped.
-  if (!(await api.providerIsActiveWorker())) {
+  if (!anonymous && !(await api.providerIsActiveWorker())) {
     debug('storage provider role account and storageProviderId are not associated with a worker')
   }
 
@@ -295,17 +307,19 @@ const commands = {
 
     const ipfsHost = cli.flags.ipfsHost
     const ipfs = require('ipfs-http-client')(ipfsHost, '5001', { protocol: 'http' })
-    const { PublisherClient, DiscoveryClient } = require('@joystream/service-discovery')
-    const publisherClient = new PublisherClient(ipfs)
-    const discoveryClient = new DiscoveryClient({ ipfs, api })
     const ipfsHttpGatewayUrl = `http://${ipfsHost}:8080/`
 
     const { startSyncing } = require('../lib/sync')
-    startSyncing(api, { syncPeriod: SYNC_PERIOD_MS }, store)
+    startSyncing(api, { syncPeriod: SYNC_PERIOD_MS }, store, cli.flags.anonymous)
 
-    announcePublicUrl(api, publicUrl, publisherClient)
+    if (!cli.flags.anonymous) {
+      const { PublisherClient } = require('@joystream/service-discovery')
+      announcePublicUrl(api, publicUrl, new PublisherClient(ipfs))
+    }
 
-    return startAllServices({ store, api, port, discoveryClient, ipfsHttpGatewayUrl })
+    const { DiscoveryClient } = require('@joystream/service-discovery')
+    const discoveryClient = new DiscoveryClient({ ipfs, api })
+    return startAllServices({ store, api, port, discoveryClient, ipfsHttpGatewayUrl, anonymous: cli.flags.anonymous })
   },
   discovery: async () => {
     banner()

storage-node/packages/colossus/lib/app.js

@@ -35,7 +35,7 @@ const fileUploads = require('./middleware/file_uploads')
 const pagination = require('@joystream/storage-utils/pagination')
 
 // Configure app
-function createApp(projectRoot, storage, runtime, discoveryClient, ipfsHttpGatewayUrl) {
+function createApp(projectRoot, storage, runtime, discoveryClient, ipfsHttpGatewayUrl, anonymous) {
   const app = express()
   app.use(cors())
   app.use(bodyParser.json())
@@ -61,6 +61,7 @@ function createApp(projectRoot, storage, runtime, discoveryClient, ipfsHttpGatew
       runtime,
       discoveryClient,
       ipfsHttpGatewayUrl,
+      anonymous,
     },
   })
 

storage-node/packages/colossus/lib/sync.js

@@ -112,7 +112,7 @@ async function setRelationshipsReady({ api, relationshipIds }) {
   )
 }
 
-async function syncPeriodic({ api, flags, storage, contentBeingSynced, contentCompleteSynced }) {
+async function syncPeriodic({ api, flags, storage, contentBeingSynced, contentCompleteSynced, anonymous }) {
   const retry = () => {
     setTimeout(syncPeriodic, flags.syncPeriod, {
       api,
@@ -120,6 +120,7 @@ async function syncPeriodic({ api, flags, storage, contentBeingSynced, contentCo
       storage,
       contentBeingSynced,
       contentCompleteSynced,
+      anonymous,
     })
   }
 
@@ -132,30 +133,35 @@ async function syncPeriodic({ api, flags, storage, contentBeingSynced, contentCo
       return retry()
     }
 
-    // Retry later if provider is not active
-    if (!(await api.providerIsActiveWorker())) {
-      debug(
-        'storage provider role account and storageProviderId are not associated with a worker. Postponing sync run.'
-      )
-      return retry()
-    }
+    if (!anonymous) {
+      // Retry later if provider is not active
+      if (!(await api.providerIsActiveWorker())) {
+        debug(
+          'storage provider role account and storageProviderId are not associated with a worker. Postponing sync run.'
+        )
+        return retry()
+      }
 
-    const recommendedBalance = await api.providerHasMinimumBalance(300)
-    if (!recommendedBalance) {
-      debug('Warning: Provider role account is running low on balance.')
-    }
+      const recommendedBalance = await api.providerHasMinimumBalance(300)
+      if (!recommendedBalance) {
+        debug('Warning: Provider role account is running low on balance.')
+      }
 
-    const sufficientBalance = await api.providerHasMinimumBalance(100)
-    if (!sufficientBalance) {
-      debug('Provider role account does not have sufficient balance. Postponing sync run!')
-      return retry()
+      const sufficientBalance = await api.providerHasMinimumBalance(100)
+      if (!sufficientBalance) {
+        debug('Provider role account does not have sufficient balance. Postponing sync run!')
+        return retry()
+      }
     }
 
     await syncContent({ api, storage, contentBeingSynced, contentCompleteSynced })
-    const relationshipIds = await createNewRelationships({ api, contentCompleteSynced })
-    await setRelationshipsReady({ api, relationshipIds })
 
-    debug(`Sync run completed, set ${relationshipIds.length} new relationships to ready`)
+    // Only update on chain state if not in anonymous mode
+    if (!anonymous) {
+      const relationshipIds = await createNewRelationships({ api, contentCompleteSynced })
+      await setRelationshipsReady({ api, relationshipIds })
+      debug(`Sync run completed, set ${relationshipIds.length} new relationships to ready`)
+    }
   } catch (err) {
     debug(`Error in sync run ${err.stack}`)
   }
@@ -164,13 +170,13 @@ async function syncPeriodic({ api, flags, storage, contentBeingSynced, contentCo
   retry()
 }
 
-function startSyncing(api, flags, storage) {
+function startSyncing(api, flags, storage, anonymous) {
   // ids of content currently being synced
   const contentBeingSynced = new Map()
   // ids of content that completed sync and may require creating a new relationship
   const contentCompleteSynced = new Map()
 
-  syncPeriodic({ api, flags, storage, contentBeingSynced, contentCompleteSynced })
+  syncPeriodic({ api, flags, storage, contentBeingSynced, contentCompleteSynced, anonymous })
 }
 
 module.exports = {

storage-node/packages/colossus/paths/asset/v0/{id}.js

@@ -27,7 +27,7 @@ function errorHandler(response, err, code) {
   response.status(err.code || code || 500).send({ message: err.toString() })
 }
 
-module.exports = function (storage, runtime, ipfsHttpGatewayUrl) {
+module.exports = function (storage, runtime, ipfsHttpGatewayUrl, anonymous) {
   // Creat the IPFS HTTP Gateway proxy middleware
   const proxy = ipfsProxy.createProxy(storage, ipfsHttpGatewayUrl)
 
@@ -47,6 +47,11 @@ module.exports = function (storage, runtime, ipfsHttpGatewayUrl) {
 
     // Put for uploads
     async put(req, res) {
+      if (anonymous) {
+        errorHandler(res, 'Uploads Not Permitted in Anonymous Mode', 400)
+        return
+      }
+
       const id = req.params.id // content id
 
       // First check if we're the liaison for the name, otherwise we can bail