Merge branch 'iznik' into colossus/minor-performance-enhancements

pioneer/packages/joy-media/src/DiscoveryProvider.tsx

@@ -44,7 +44,11 @@ type ProviderStats = {
 function newDiscoveryProvider ({ bootstrapNodes }: BootstrapNodes): DiscoveryProvider {
   const stats = new Map<string, ProviderStats>();
 
-  const resolveAssetEndpoint = async (storageProvider: StorageProviderId, contentId?: string, cancelToken?: CancelToken) => {
+  const resolveAssetEndpoint = async (
+    storageProvider: StorageProviderId,
+    contentId?: string,
+    cancelToken?: CancelToken
+  ) => {
     const providerKey = storageProvider.toString();
 
     let stat = stats.get(providerKey);

storage-node/packages/colossus/lib/middleware/ipfs_proxy.js

@@ -0,0 +1,85 @@
+const { createProxyMiddleware } = require('http-proxy-middleware')
+const debug = require('debug')('joystream:ipfs-proxy')
+const mime = require('mime-types')
+
+/* 
+For this proxying to work correctly, ensure IPFS HTTP Gateway is configured as a path gateway:
+This can be done manually with the following command:
+
+  $ ipfs config --json Gateway.PublicGateways '{"localhost": null }' 
+  
+The implicit default config is below which is not what we want!
+
+  $ ipfs config --json Gateway.PublicGateways '{
+    "localhost": {
+        "Paths": ["/ipfs", "/ipns"],
+        "UseSubdomains": true
+      }
+    }'
+
+https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#gateway
+*/
+
+const pathFilter = function (path, req) {
+  // we get the full path here so it needs to match the path where
+  // it is used by the openapi initializer
+  return path.match('^/asset/v0') && (req.method === 'GET' || req.method === 'HEAD')
+}
+
+const createPathRewriter = (resolve) => {
+  return async (_path, req) => {
+    // we expect the handler was used in openapi/express path with an id in the path:
+    // "/asset/v0/:id"
+    // TODO: catch and deal with hash == undefined if content id not found
+    const contentId = req.params.id
+    const hash = await resolve(contentId)
+    return `/ipfs/${hash}`
+  }
+}
+
+const createResolver = (storage) => {
+  return async (id) => await storage.resolveContentIdWithTimeout(5000, id)
+}
+
+const createProxy = (storage) => {
+  const pathRewrite = createPathRewriter(createResolver(storage))
+
+  return createProxyMiddleware(pathFilter, {
+    // Default path to local IPFS HTTP GATEWAY
+    target: 'http://localhost:8080/',
+    pathRewrite,
+    onProxyRes: function (proxRes, req, res) {
+      /*
+        Make sure the reverse proxy used infront of colosss (nginx/caddy) Does not duplicate
+        these headers to prevent some browsers getting confused especially
+        with duplicate access-control-allow-origin headers!
+        'accept-ranges': 'bytes',
+        'access-control-allow-headers': 'Content-Type, Range, User-Agent, X-Requested-With',
+        'access-control-allow-methods': 'GET',
+        'access-control-allow-origin': '*',
+        'access-control-expose-headers': 'Content-Range, X-Chunked-Output, X-Stream-Output',
+      */
+
+      if (proxRes.statusCode === 301) {
+        // capture redirect when IPFS HTTP Gateway is configured with 'UseDomains':true
+        // and treat it as an error.
+        console.error('IPFS HTTP Gateway is configured for "UseSubdomains". Killing stream')
+        res.status(500).end()
+        proxRes.destroy()
+      } else {
+        // Handle downloading as attachment /asset/v0/:id?download
+        if (req.query.download) {
+          const contentId = req.params.id
+          const contentType = proxRes.headers['content-type']
+          const ext = mime.extension(contentType) || 'bin'
+          const fileName = `${contentId}.${ext}`
+          proxRes.headers['Content-Disposition'] = `attachment; filename=${fileName}`
+        }
+      }
+    },
+  })
+}
+
+module.exports = {
+  createProxy,
+}

storage-node/packages/colossus/package.json

@@ -50,17 +50,19 @@
     "temp": "^0.9.0"
   },
   "dependencies": {
-    "@joystream/storage-runtime-api": "^0.1.0",
     "@joystream/storage-node-backend": "^0.1.0",
+    "@joystream/storage-runtime-api": "^0.1.0",
     "@joystream/storage-utils": "^0.1.0",
     "body-parser": "^1.19.0",
     "chalk": "^2.4.2",
     "cors": "^2.8.5",
     "express-openapi": "^4.6.1",
     "figlet": "^1.2.1",
+    "http-proxy-middleware": "^1.0.5",
     "js-yaml": "^3.13.1",
     "lodash": "^4.17.11",
     "meow": "^7.0.1",
+    "mime-types": "^2.1.27",
     "multer": "^1.4.1",
     "si-prefix": "^0.2.0"
   }

storage-node/packages/colossus/paths/asset/v0/{id}.js

@@ -18,12 +18,9 @@
 
 'use strict'
 
-const path = require('path')
-
 const debug = require('debug')('joystream:colossus:api:asset')
-
-const utilRanges = require('@joystream/storage-utils/ranges')
 const filter = require('@joystream/storage-node-backend/filter')
+const ipfsProxy = require('../../../lib/middleware/ipfs_proxy')
 
 function errorHandler(response, err, code) {
   debug(err)
@@ -31,6 +28,9 @@ function errorHandler(response, err, code) {
 }
 
 module.exports = function (storage, runtime) {
+  // Creat the IPFS HTTP Gateway proxy middleware
+  const proxy = ipfsProxy.createProxy(storage)
+
   const doc = {
     // parameters for all operations in this path
     parameters: [
@@ -45,34 +45,6 @@ module.exports = function (storage, runtime) {
       },
     ],
 
-    // Head: report that ranges are OK
-    async head(req, res) {
-      const id = req.params.id
-
-      // Open file
-      try {
-        const size = await storage.size(id)
-        const stream = await storage.open(id, 'r')
-        const type = stream.fileInfo.mimeType
-
-        // Close the stream; we don't need to fetch the file (if we haven't
-        // already). Then return result.
-        stream.destroy()
-
-        res.status(200)
-        res.contentType(type)
-        res.header('Content-Disposition', 'inline')
-        res.header('Content-Transfer-Encoding', 'binary')
-        res.header('Accept-Ranges', 'bytes')
-        if (size > 0) {
-          res.header('Content-Length', size)
-        }
-        res.send()
-      } catch (err) {
-        errorHandler(res, err, err.code)
-      }
-    },
-
     // Put for uploads
     async put(req, res) {
       const id = req.params.id // content id
@@ -184,61 +156,21 @@ module.exports = function (storage, runtime) {
       }
     },
 
-    // Get content
     async get(req, res) {
-      const id = req.params.id
-      const download = req.query.download
-
-      // Parse range header
-      let ranges
-      if (!download) {
-        try {
-          const rangeHeader = req.headers.range
-          ranges = utilRanges.parse(rangeHeader)
-        } catch (err) {
-          // Do nothing; it's ok to ignore malformed ranges and respond with the
-          // full content according to https://www.rfc-editor.org/rfc/rfc7233.txt
-        }
-        if (ranges && ranges.unit !== 'bytes') {
-          // Ignore ranges that are not byte units.
-          ranges = undefined
-        }
-      }
-      debug('Requested range(s) is/are', ranges)
-
-      // Open file
-      try {
-        const size = await storage.size(id)
-        const stream = await storage.open(id, 'r')
-
-        // Add a file extension to download requests if necessary. If the file
-        // already contains an extension, don't add one.
-        let sendName = id
-        const type = stream.fileInfo.mimeType
-        if (download) {
-          let ext = path.extname(sendName)
-          if (!ext) {
-            ext = stream.fileInfo.ext
-            if (ext) {
-              sendName = `${sendName}.${ext}`
-            }
-          }
-        }
+      proxy(req, res)
+    },
 
-        const opts = {
-          name: sendName,
-          type,
-          size,
-          ranges,
-          download,
-        }
-        utilRanges.send(res, stream, opts)
-      } catch (err) {
-        errorHandler(res, err, err.code)
-      }
+    async head(req, res) {
+      proxy(req, res)
     },
   }
 
+  // doc.get = proxy
+  // doc.head = proxy
+  // Note: Adding the middleware this way is causing problems!
+  // We are loosing some information from the request, specifically req.query.download parameters for some reason.
+  // Does it have to do with how/when the apiDoc is being processed? binding issue?
+
   // OpenAPI specs
   doc.get.apiDoc = {
     description: 'Download an asset.',

storage-node/packages/colossus/paths/discover/v0/{id}.js

@@ -12,7 +12,7 @@ module.exports = function (runtime) {
         name: 'id',
         in: 'path',
         required: true,
-        description: 'Actor accouuntId',
+        description: 'Storage Provider Id',
         schema: {
           type: 'string', // integer ?
         },

storage-node/packages/helios/bin/cli.js

@@ -26,7 +26,7 @@ function mapInfoToStatus(providers, currentHeight) {
 
 function makeAssetUrl(contentId, source) {
   source = stripEndingSlash(source)
-  return `${source}/asset/v0/${encodeAddress(contentId)}`
+  return `${source}/asset/v1/${encodeAddress(contentId)}`
 }
 
 async function assetRelationshipState(api, contentId, providers) {

storage-node/packages/runtime-api/index.js

@@ -61,9 +61,6 @@ class RuntimeApi {
 
     this.asyncLock = new AsyncLock()
 
-    // Keep track locally of account nonces.
-    this.nonces = {}
-
     // The storage provider id to use
     this.storageProviderId = parseInt(options.storageProviderId) // u64 instead ?
 
@@ -148,28 +145,6 @@ class RuntimeApi {
     })
   }
 
-  // Get cached nonce and use unless system nonce is greater, to avoid stale nonce if
-  // there was a long gap in time between calls to signAndSend during which an external app
-  // submitted a transaction.
-  async selectBestNonce(accountId) {
-    const cachedNonce = this.nonces[accountId]
-    // In future use this rpc method to take the pending tx pool into account when fetching the nonce
-    // const nonce = await this.api.rpc.system.accountNextIndex(accountId)
-    const { nonce } = await this.api.query.system.account(accountId)
-
-    const systemNonce = nonce
-
-    const bestNonce = cachedNonce && cachedNonce.gte(systemNonce) ? cachedNonce : systemNonce
-
-    this.nonces[accountId] = bestNonce
-
-    return bestNonce.toNumber()
-  }
-
-  incrementAndSaveNonce(accountId) {
-    this.nonces[accountId] = this.nonces[accountId].addn(1)
-  }
-
   /*
    * signAndSend() with nonce tracking, to enable concurrent sending of transacctions
    * so that they can be included in the same block. Allows you to use the accountId instead
@@ -213,7 +188,7 @@ class RuntimeApi {
 
     // synchronize access to nonce
     await this.executeWithAccountLock(accountId, async () => {
-      const nonce = await this.selectBestNonce(accountId)
+      const nonce = await this.api.rpc.system.accountNextIndex(accountId)
       const signed = tx.sign(fromKey, { nonce })
       const txhash = signed.hash
 
@@ -237,9 +212,6 @@ class RuntimeApi {
         } else {
           debugTx(`Submitted: ${serialized}`)
         }
-
-        // transaction submitted successfully, increment and save nonce.
-        this.incrementAndSaveNonce(accountId)
       } catch (err) {
         const errstr = err.toString()
         debugTx(`Rejected: ${errstr} txhash: ${txhash} nonce: ${nonce}`)

storage-node/packages/storage/storage.js

@@ -225,6 +225,10 @@ class Storage {
         debug(`Warning IPFS daemon not running: ${err.message}`)
       } else {
         debug(`IPFS node is up with identity: ${identity.id}`)
+        // TODO: wait for IPFS daemon to be online for this to be effective..?
+        // set the IPFS HTTP Gateway config we desire.. operator might need
+        // to restart their daemon if the config was changed.
+        this.ipfs.config.set('Gateway.PublicGateways', { 'localhost': null })
       }
     })
   }

storage-node/scripts/compose/devchain-and-ipfs-node/docker-compose.yaml

@@ -4,8 +4,16 @@ services:
     image: ipfs/go-ipfs:latest
     ports:
       - '127.0.0.1:5001:5001'
+      - '127.0.0.1:8080:8080'
     volumes:
       - ipfs-data:/data/ipfs
+    entrypoint: ''
+    command: |
+      /bin/sh -c "
+        set -e
+        /usr/local/bin/start_ipfs config --json Gateway.PublicGateways '{\"localhost\": null }'
+        /sbin/tini -- /usr/local/bin/start_ipfs daemon --migrate=true
+      "
   chain:
     image: joystream/node:latest
     ports:

yarn.lock

@@ -13595,7 +13595,7 @@ http-proxy-middleware@0.19.1:
     lodash "^4.17.11"
     micromatch "^3.1.10"
 
-http-proxy-middleware@^1.0.3:
+http-proxy-middleware@^1.0.3, http-proxy-middleware@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-1.0.5.tgz#4c6e25d95a411e3d750bc79ccf66290675176dc2"
   integrity sha512-CKzML7u4RdGob8wuKI//H8Ein6wNTEQR7yjVEzPbhBLGdOfkfvgTnp2HLnniKBDP9QW4eG10/724iTWLBeER3g==
@@ -17753,7 +17753,7 @@ mime-types@^2.1.12, mime-types@~2.1.19, mime-types@~2.1.24:
   dependencies:
     mime-db "1.42.0"
 
-mime-types@^2.1.18, mime-types@^2.1.22, mime-types@^2.1.26, mime-types@~2.1.17:
+mime-types@^2.1.18, mime-types@^2.1.22, mime-types@^2.1.26, mime-types@^2.1.27, mime-types@~2.1.17:
   version "2.1.27"
   resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f"
   integrity sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==