storage-node: refactor ipfs gateway proxy to be under the openapi paths

pioneer/packages/joy-media/src/DiscoveryProvider.tsx

@@ -16,7 +16,7 @@ export type BootstrapNodes = {
 };
 
 export type DiscoveryProvider = {
-  resolveAssetEndpoint: (provider: StorageProviderId, operation: 'download'| 'upload', contentId?: string, cancelToken?: CancelToken) => Promise<string>;
+  resolveAssetEndpoint: (provider: StorageProviderId, contentId?: string, cancelToken?: CancelToken) => Promise<string>;
   reportUnreachable: (provider: StorageProviderId) => void;
 };
 
@@ -46,7 +46,6 @@ function newDiscoveryProvider ({ bootstrapNodes }: BootstrapNodes): DiscoveryPro
 
   const resolveAssetEndpoint = async (
     storageProvider: StorageProviderId,
-    operation: 'download' | 'upload',
     contentId?: string,
     cancelToken?: CancelToken
   ) => {
@@ -118,9 +117,7 @@ function newDiscoveryProvider ({ bootstrapNodes }: BootstrapNodes): DiscoveryPro
     console.log(stat);
 
     if (stat) {
-      const ver = operation === 'download' ? 'v1' : 'v0';
-
-      return `${stat.assetApiEndpoint}/asset/${ver}/${contentId || ''}`;
+      return `${stat.assetApiEndpoint}/asset/v0/${contentId || ''}`;
     }
 
     throw new Error('Resolving failed.');

pioneer/packages/joy-media/src/Upload.tsx

@@ -350,7 +350,7 @@ class Upload extends React.PureComponent<Props, State> {
     let url: string;
 
     try {
-      url = await discoveryProvider.resolveAssetEndpoint(storageProvider, 'upload', contentId, cancelSource.token);
+      url = await discoveryProvider.resolveAssetEndpoint(storageProvider, contentId, cancelSource.token);
     } catch (err) {
       return this.setState({
         error: `Failed to contact storage provider: ${normalizeError(err)}`,

pioneer/packages/joy-media/src/common/MediaPlayerWithResolver.tsx

@@ -80,7 +80,7 @@ function InnerComponent (props: Props) {
       let assetUrl: string | undefined;
 
       try {
-        assetUrl = await discoveryProvider.resolveAssetEndpoint(provider, 'download', contentId.encode(), cancelSource.token);
+        assetUrl = await discoveryProvider.resolveAssetEndpoint(provider, contentId.encode(), cancelSource.token);
       } catch (err) {
         if (axios.isCancel(err)) {
           return;

storage-node/packages/cli/src/commands/base.ts

@@ -5,7 +5,7 @@ import { ContentId } from '@joystream/types/media'
 // Commands base abstract class. Contains reusable methods.
 export abstract class BaseCommand {
   // Creates the Colossus asset URL and logs it.
-  protected createAndLogAssetUrl(url: string, operation: 'download' | 'upload', contentId: string | ContentId): string {
+  protected createAndLogAssetUrl(url: string, contentId: string | ContentId): string {
     let normalizedContentId: string
 
     if (typeof contentId === 'string') {
@@ -13,9 +13,9 @@ export abstract class BaseCommand {
     } else {
       normalizedContentId = contentId.encode()
     }
-    const ver = operation === 'download' ? 'v1' : 'v0'
+
     const normalizedUrl = removeEndingForwardSlash(url)
-    const assetUrl = `${normalizedUrl}/asset/${ver}/${normalizedContentId}`
+    const assetUrl = `${normalizedUrl}/asset/v0/${normalizedContentId}`
     console.log(chalk.yellow('Generated asset URL:', assetUrl))
 
     return assetUrl

storage-node/packages/cli/src/commands/download.ts

@@ -46,7 +46,7 @@ export class DownloadCommand extends BaseCommand {
     // Checks for input parameters, shows usage if they are invalid.
     if (!this.assertParameters()) return
 
-    const assetUrl = this.createAndLogAssetUrl(this.storageNodeUrl, 'download', this.contentId)
+    const assetUrl = this.createAndLogAssetUrl(this.storageNodeUrl, this.contentId)
     console.log(chalk.yellow('File path:', this.outputFilePath))
 
     // Create file write stream and set error handler.

storage-node/packages/cli/src/commands/head.ts

@@ -36,7 +36,7 @@ export class HeadCommand extends BaseCommand {
     // Checks for input parameters, shows usage if they are invalid.
     if (!this.assertParameters()) return
 
-    const assetUrl = this.createAndLogAssetUrl(this.storageNodeUrl, 'download', this.contentId)
+    const assetUrl = this.createAndLogAssetUrl(this.storageNodeUrl, this.contentId)
 
     try {
       const response = await axios.head(assetUrl)

storage-node/packages/cli/src/commands/upload.ts

@@ -214,7 +214,7 @@ export class UploadCommand extends BaseCommand {
     const colossusEndpoint = await this.discoverStorageProviderEndpoint(dataObject.liaison.toString())
     debug(`Discovered storage node endpoint: ${colossusEndpoint}`)
 
-    const assetUrl = this.createAndLogAssetUrl(colossusEndpoint, 'upload', addContentParams.contentId)
+    const assetUrl = this.createAndLogAssetUrl(colossusEndpoint, addContentParams.contentId)
     await this.uploadFile(assetUrl)
   }
 }

storage-node/packages/colossus/lib/app.js

@@ -32,7 +32,6 @@ const yaml = require('js-yaml')
 // Project requires
 const validateResponses = require('./middleware/validate_responses')
 const fileUploads = require('./middleware/file_uploads')
-const ipfsGateway = require('./middleware/ipfs_proxy')
 const pagination = require('@joystream/storage-utils/pagination')
 
 // Configure app
@@ -63,9 +62,6 @@ function createApp(projectRoot, storage, runtime) {
     },
   })
 
-  // Proxy asset GET and HEAD directly to the IPFS gateway
-  app.use('/asset/v1/:id', ipfsGateway.createProxy(storage))
-
   // If no other handler gets triggered (errors), respond with the
   // error serialized to JSON.
   // Disable lint because we need such function signature.

storage-node/packages/colossus/lib/middleware/ipfs_proxy.js

@@ -1,5 +1,6 @@
 const { createProxyMiddleware } = require('http-proxy-middleware')
 const debug = require('debug')('joystream:ipfs-proxy')
+const mime = require('mime-types')
 
 /* 
 For this proxying to work correctly, ensure IPFS HTTP Gateway is configured as a path gateway:
@@ -20,12 +21,18 @@ https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#gateway
 */
 
 const pathFilter = function (path, req) {
-  return path.match('^/asset/v1/') && (req.method === 'GET' || req.method === 'HEAD')
+  // we get the full path here so it needs to match the path where
+  // it is used by the openapi initializer
+  return path.match('^/asset/v0') && (req.method === 'GET' || req.method === 'HEAD')
 }
 
 const createPathRewriter = (resolve) => {
   return async (_path, req) => {
-    const hash = await resolve(req.params.id)
+    // we expect the handler was used in openapi/express path with an id in the path:
+    // "/asset/v0/:id"
+    // TODO: catch and deal with hash == undefined if content id not found
+    const contentId = req.params.id
+    const hash = await resolve(contentId)
     return `/ipfs/${hash}`
   }
 }
@@ -41,14 +48,33 @@ const createProxy = (storage) => {
     // Default path to local IPFS HTTP GATEWAY
     target: 'http://localhost:8080/',
     pathRewrite,
-    // capture redirect when IPFS HTTP Gateway is configured with 'UseDomains':true
-    // and treat it as an error.
-    onProxyRes: function (proxRes, _req, res) {
+    onProxyRes: function (proxRes, req, res) {
+      /*
+        Make sure the reverse proxy used infront of colosss (nginx/caddy) Does not duplicate
+        these headers to prevent some browsers getting confused especially
+        with duplicate access-control-allow-origin headers!
+        'accept-ranges': 'bytes',
+        'access-control-allow-headers': 'Content-Type, Range, User-Agent, X-Requested-With',
+        'access-control-allow-methods': 'GET',
+        'access-control-allow-origin': '*',
+        'access-control-expose-headers': 'Content-Range, X-Chunked-Output, X-Stream-Output',
+      */
+
       if (proxRes.statusCode === 301) {
-        debug('IPFS HTTP Gateway is allowing UseSubdomains. Killing stream')
-        proxRes.destroy()
-        // TODO: Maybe redirect - temporary to /v0/asset/contentId ?
+        // capture redirect when IPFS HTTP Gateway is configured with 'UseDomains':true
+        // and treat it as an error.
+        console.error('IPFS HTTP Gateway is configured for "UseSubdomains". Killing stream')
         res.status(500).end()
+        proxRes.destroy()
+      } else {
+        // Handle downloading as attachment /asset/v0/:id?download
+        if (req.query.download) {
+          const contentId = req.params.id
+          const contentType = proxRes.headers['content-type']
+          const ext = mime.extension(contentType) || 'bin'
+          const fileName = `${contentId}.${ext}`
+          proxRes.headers['Content-Disposition'] = `attachment; filename=${fileName}`
+        }
       }
     },
   })

storage-node/packages/colossus/package.json

@@ -62,6 +62,7 @@
     "js-yaml": "^3.13.1",
     "lodash": "^4.17.11",
     "meow": "^7.0.1",
+    "mime-types": "^2.1.27",
     "multer": "^1.4.1",
     "si-prefix": "^0.2.0"
   }

storage-node/packages/colossus/paths/asset/v0/{id}.js

@@ -18,12 +18,9 @@
 
 'use strict'
 
-const path = require('path')
-
 const debug = require('debug')('joystream:colossus:api:asset')
-
-const utilRanges = require('@joystream/storage-utils/ranges')
 const filter = require('@joystream/storage-node-backend/filter')
+const ipfsProxy = require('../../../lib/middleware/ipfs_proxy')
 
 function errorHandler(response, err, code) {
   debug(err)
@@ -31,6 +28,9 @@ function errorHandler(response, err, code) {
 }
 
 module.exports = function (storage, runtime) {
+  // Creat the IPFS HTTP Gateway proxy middleware
+  const proxy = ipfsProxy.createProxy(storage)
+
   const doc = {
     // parameters for all operations in this path
     parameters: [
@@ -45,34 +45,6 @@ module.exports = function (storage, runtime) {
       },
     ],
 
-    // Head: report that ranges are OK
-    async head(req, res) {
-      const id = req.params.id
-
-      // Open file
-      try {
-        const size = await storage.size(id)
-        const stream = await storage.open(id, 'r')
-        const type = stream.fileInfo.mimeType
-
-        // Close the stream; we don't need to fetch the file (if we haven't
-        // already). Then return result.
-        stream.destroy()
-
-        res.status(200)
-        res.contentType(type)
-        res.header('Content-Disposition', 'inline')
-        res.header('Content-Transfer-Encoding', 'binary')
-        res.header('Accept-Ranges', 'bytes')
-        if (size > 0) {
-          res.header('Content-Length', size)
-        }
-        res.send()
-      } catch (err) {
-        errorHandler(res, err, err.code)
-      }
-    },
-
     // Put for uploads
     async put(req, res) {
       const id = req.params.id // content id
@@ -184,61 +156,21 @@ module.exports = function (storage, runtime) {
       }
     },
 
-    // Get content
     async get(req, res) {
-      const id = req.params.id
-      const download = req.query.download
-
-      // Parse range header
-      let ranges
-      if (!download) {
-        try {
-          const rangeHeader = req.headers.range
-          ranges = utilRanges.parse(rangeHeader)
-        } catch (err) {
-          // Do nothing; it's ok to ignore malformed ranges and respond with the
-          // full content according to https://www.rfc-editor.org/rfc/rfc7233.txt
-        }
-        if (ranges && ranges.unit !== 'bytes') {
-          // Ignore ranges that are not byte units.
-          ranges = undefined
-        }
-      }
-      debug('Requested range(s) is/are', ranges)
-
-      // Open file
-      try {
-        const size = await storage.size(id)
-        const stream = await storage.open(id, 'r')
-
-        // Add a file extension to download requests if necessary. If the file
-        // already contains an extension, don't add one.
-        let sendName = id
-        const type = stream.fileInfo.mimeType
-        if (download) {
-          let ext = path.extname(sendName)
-          if (!ext) {
-            ext = stream.fileInfo.ext
-            if (ext) {
-              sendName = `${sendName}.${ext}`
-            }
-          }
-        }
+      proxy(req, res)
+    },
 
-        const opts = {
-          name: sendName,
-          type,
-          size,
-          ranges,
-          download,
-        }
-        utilRanges.send(res, stream, opts)
-      } catch (err) {
-        errorHandler(res, err, err.code)
-      }
+    async head(req, res) {
+      proxy(req, res)
     },
   }
 
+  // doc.get = proxy
+  // doc.head = proxy
+  // Note: Adding the middleware this way is causing problems!
+  // We are loosing some information from the request, specifically req.query.download parameters for some reason.
+  // Does it have to do with how/when the apiDoc is being processed? binding issue?
+
   // OpenAPI specs
   doc.get.apiDoc = {
     description: 'Download an asset.',

yarn.lock

@@ -17753,7 +17753,7 @@ mime-types@^2.1.12, mime-types@~2.1.19, mime-types@~2.1.24:
   dependencies:
     mime-db "1.42.0"
 
-mime-types@^2.1.18, mime-types@^2.1.22, mime-types@^2.1.26, mime-types@~2.1.17:
+mime-types@^2.1.18, mime-types@^2.1.22, mime-types@^2.1.26, mime-types@^2.1.27, mime-types@~2.1.17:
   version "2.1.27"
   resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f"
   integrity sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==