upload.ts 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220
  1. import axios, { AxiosRequestConfig } from 'axios'
  2. import fs from 'fs'
  3. import ipfsHash from 'ipfs-only-hash'
  4. import { ContentId, DataObject } from '@joystream/types/media'
  5. import BN from 'bn.js'
  6. import { Option } from '@polkadot/types/codec'
  7. import { BaseCommand } from './base'
  8. import { discover } from '@joystream/service-discovery/discover'
  9. import Debug from 'debug'
  10. import chalk from 'chalk'
  11. import {aliceKeyPair} from './dev'
  12. const debug = Debug('joystream:storage-cli:upload')
  13. // Defines maximum content length for the assets (files). Limits the upload.
  14. const MAX_CONTENT_LENGTH = 500 * 1024 * 1024 // 500Mb
  15. // Defines the necessary parameters for the AddContent runtime tx.
  16. interface AddContentParams {
  17. accountId: string
  18. ipfsCid: string
  19. contentId: ContentId
  20. fileSize: BN
  21. dataObjectTypeId: number
  22. memberId: number
  23. }
  24. // Upload command class. Validates input parameters and uploads the asset to the storage node and runtime.
  25. export class UploadCommand extends BaseCommand {
  26. private readonly api: any
  27. private readonly mediaSourceFilePath: string
  28. private readonly dataObjectTypeId: string
  29. private readonly keyFile: string
  30. private readonly passPhrase: string
  31. private readonly memberId: string
  32. constructor(
  33. api: any,
  34. mediaSourceFilePath: string,
  35. dataObjectTypeId: string,
  36. memberId: string,
  37. keyFile: string,
  38. passPhrase: string
  39. ) {
  40. super()
  41. this.api = api
  42. this.mediaSourceFilePath = mediaSourceFilePath
  43. this.dataObjectTypeId = dataObjectTypeId
  44. this.memberId = memberId
  45. this.keyFile = keyFile
  46. this.passPhrase = passPhrase
  47. }
  48. // Provides parameter validation. Overrides the abstract method from the base class.
  49. protected validateParameters(): boolean {
  50. return (
  51. this.mediaSourceFilePath &&
  52. this.mediaSourceFilePath !== '' &&
  53. this.dataObjectTypeId &&
  54. this.dataObjectTypeId !== '' &&
  55. this.memberId &&
  56. this.memberId !== ''
  57. )
  58. }
  59. // Reads the file from the filesystem and computes IPFS hash.
  60. private async computeIpfsHash(): Promise<string> {
  61. const file = fs.createReadStream(this.mediaSourceFilePath).on('error', (err) => {
  62. this.fail(`File read failed: ${err}`)
  63. })
  64. return await ipfsHash.of(file)
  65. }
  66. // Read the file size from the file system.
  67. private getFileSize(): number {
  68. const stats = fs.statSync(this.mediaSourceFilePath)
  69. return stats.size
  70. }
  71. // Creates parameters for the AddContent runtime tx.
  72. private async getAddContentParams(): Promise<AddContentParams> {
  73. const identity = await this.loadIdentity()
  74. const accountId = identity.address
  75. const dataObjectTypeId: number = parseInt(this.dataObjectTypeId)
  76. if (isNaN(dataObjectTypeId)) {
  77. this.fail(`Cannot parse dataObjectTypeId: ${this.dataObjectTypeId}`)
  78. }
  79. const memberId: number = parseInt(this.memberId)
  80. if (isNaN(dataObjectTypeId)) {
  81. this.fail(`Cannot parse memberIdString: ${this.memberId}`)
  82. }
  83. return {
  84. accountId,
  85. ipfsCid: await this.computeIpfsHash(),
  86. contentId: ContentId.generate(),
  87. fileSize: new BN(this.getFileSize()),
  88. dataObjectTypeId,
  89. memberId,
  90. }
  91. }
  92. // Creates the DataObject in the runtime.
  93. private async createContent(p: AddContentParams): Promise<DataObject> {
  94. try {
  95. const dataObject: Option<DataObject> = await this.api.assets.createDataObject(
  96. p.accountId,
  97. p.memberId,
  98. p.contentId,
  99. p.dataObjectTypeId,
  100. p.fileSize,
  101. p.ipfsCid
  102. )
  103. if (dataObject.isNone) {
  104. this.fail('Cannot create data object: got None object')
  105. }
  106. return dataObject.unwrap()
  107. } catch (err) {
  108. this.fail(`Cannot create data object: ${err}`)
  109. }
  110. }
  111. // Uploads file to given asset URL.
  112. private async uploadFile(assetUrl: string) {
  113. // Create file read stream and set error handler.
  114. const file = fs.createReadStream(this.mediaSourceFilePath).on('error', (err) => {
  115. this.fail(`File read failed: ${err}`)
  116. })
  117. // Upload file from the stream.
  118. try {
  119. const fileSize = this.getFileSize()
  120. const config: AxiosRequestConfig = {
  121. headers: {
  122. 'Content-Type': '', // https://github.com/Joystream/storage-node-joystream/issues/16
  123. 'Content-Length': fileSize.toString(),
  124. },
  125. maxContentLength: MAX_CONTENT_LENGTH,
  126. }
  127. await axios.put(assetUrl, file, config)
  128. console.log('File uploaded.')
  129. } catch (err) {
  130. this.fail(err.toString())
  131. }
  132. }
  133. // Requests the runtime and obtains the storage node endpoint URL.
  134. private async discoverStorageProviderEndpoint(storageProviderId: string): Promise<string> {
  135. try {
  136. const serviceInfo = await discover(storageProviderId, this.api)
  137. if (serviceInfo === null) {
  138. this.fail('Storage node discovery failed.')
  139. }
  140. debug(`Discovered service info object: ${serviceInfo}`)
  141. const dataWrapper = JSON.parse(serviceInfo)
  142. const assetWrapper = JSON.parse(dataWrapper.serialized)
  143. return assetWrapper.asset.endpoint
  144. } catch (err) {
  145. this.fail(`Could not get asset endpoint: ${err}`)
  146. }
  147. }
  148. // Loads and unlocks the runtime identity using the key file and pass phrase.
  149. private async loadIdentity(): Promise<any> {
  150. const noKeyFileProvided = !this.keyFile || this.keyFile === ''
  151. const useAlice = noKeyFileProvided && await this.api.system.isDevelopmentChain()
  152. if (useAlice) {
  153. debug('Discovered \'development\' chain.')
  154. return aliceKeyPair(this.api)
  155. }
  156. try {
  157. await fs.promises.access(this.keyFile)
  158. } catch (error) {
  159. this.fail(`Cannot read file "${this.keyFile}".`)
  160. }
  161. return this.api.identities.loadUnlock(this.keyFile, this.passPhrase)
  162. }
  163. // Shows command usage. Overrides the abstract method from the base class.
  164. protected showUsage() {
  165. console.log(
  166. chalk.yellow(`
  167. Usage: storage-cli upload mediaSourceFilePath dataObjectTypeId memberId [keyFilePath] [passPhrase]
  168. Example: storage-cli upload ./movie.mp4 1 1 ./keyFile.json secretPhrase
  169. Development: storage-cli upload ./movie.mp4 1 0
  170. `)
  171. )
  172. }
  173. // Command executor.
  174. async run() {
  175. // Checks for input parameters, shows usage if they are invalid.
  176. if (!this.assertParameters()) return
  177. const addContentParams = await this.getAddContentParams()
  178. debug(`AddContent Tx params: ${JSON.stringify(addContentParams)}`)
  179. debug(`Decoded CID: ${addContentParams.contentId.toString()}`)
  180. const dataObject = await this.createContent(addContentParams)
  181. debug(`Received data object: ${dataObject.toString()}`)
  182. const colossusEndpoint = await this.discoverStorageProviderEndpoint(dataObject.liaison.toString())
  183. debug(`Discovered storage node endpoint: ${colossusEndpoint}`)
  184. const assetUrl = this.createAndLogAssetUrl(colossusEndpoint, addContentParams.contentId)
  185. await this.uploadFile(assetUrl)
  186. }
  187. }