Merge pull request #383 from freakstatic/sp-downloader-tester

Storage_Provider_Download_Tester

contributions/community-contributions/tech/sp-downloader-tester/.gitignore

@@ -0,0 +1,122 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+.env.production
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+.idea/*
+
+assets.txt

contributions/community-contributions/tech/sp-downloader-tester/README.md

@@ -0,0 +1,28 @@
+# Storage Provider Download Tester
+This tool helps to check the response time of the Storage Providers Workers.  
+It selects a list of files and downloads all of them at the same time and outputs some stats about the downloads. 
+
+## Install dependencies
+ ```
+ yarn 
+ ```
+
+## Usage
+ ```
+  -w, --workers <workerId>        the Worker id's to perform the tests, separated by comma. Ex: 4,5,6
+  -f, --asset-file [path]         a list of assets ids to download the same files for different workers
+  -s, --nr-small-assets [number]  the number of small files to download (default: 9)
+  -b, --nr-big-assets [number]    the number of big files to download (default: 1)
+  -h, --help                      display help for command
+ ```
+
+## Examples
+Run script without the list of files to download (it will generate an assets.txt file for future use)
+ ```
+yarn run sp-downloader-tester -w 3
+```
+Run script with the list of files to download
+```
+yarn run sp-downloader-tester -w 3 -f assets.txt
+```
+

contributions/community-contributions/tech/sp-downloader-tester/bin/cli.js

@@ -0,0 +1,137 @@
+#!/usr/bin/env node
+
+const { ContentId } = require('@joystream/types/storage')
+const { ApiPromise, WsProvider } = require('@polkadot/api')
+const { types } = require('@joystream/types')
+
+const fsPromises = require('fs/promises')
+const fs = require('fs')
+const Path = require('path')
+
+const {
+  getActiveWorkersIds,
+  getWorkerEndpoint,
+  generateListOfDataObjectsToDownload,
+  makeAssetUrl,
+  downloadFile,
+  clearFolder,
+  computeMedian
+} = require('./utils')
+
+const { program } = require('commander')
+
+const TEMP_FOLDER = Path.resolve(__dirname, '../', 'temp')
+const PROVIDER_URL = 'wss://rome-rpc-endpoint.joystream.org:9944'
+const NR_DEFAULT_SMALL_ASSETS = 9
+const NR_BIG_ASSETS = 1
+
+async function main() {
+  program
+    .option('-w, --workers <workerId>', `the Worker id's to perform the tests, separated by comma. Ex: 4,5,6`)
+    .option('-f, --asset-file [path]', 'a list of assets ids to download the same files for different providers')
+    .option('-s, --nr-small-assets [number]', 'the number of small files to download', NR_DEFAULT_SMALL_ASSETS)
+    .option('-b, --nr-big-assets [number]', 'the number of big files to download', NR_BIG_ASSETS)
+    .parse()
+
+  const provider = new WsProvider(process.env.PROVIDER_URL || PROVIDER_URL)
+  const api = await ApiPromise.create({ provider, types })
+  await api.isReady
+
+  const args = program.opts()
+
+  let dataObjectsIds = []
+  const assetsFilePath = args.assetFile
+  if (assetsFilePath) {
+    try {
+      await fsPromises.access(assetsFilePath, fs.constants.R_OK)
+    } catch {
+      console.error('Unable to read ' + assetsFilePath)
+      process.exit(1)
+    }
+
+    const data = (await fsPromises.readFile(assetsFilePath, 'utf8')).toString()
+    dataObjectsIds = data
+      .split('\n')
+      .filter((line) => line)
+      .map((line) => ContentId.decode(api.registry, line))
+  } else {
+    dataObjectsIds = await generateListOfDataObjectsToDownload(api, args.nrSmallAssets, args.nrBigAssets)
+    const writeStream = fs.createWriteStream(Path.join(__dirname, '..', 'assets.txt'))
+    for (const id of dataObjectsIds) {
+      writeStream.write(id.encode() + '\n')
+    }
+    writeStream.close()
+  }
+
+  const dataObjects = await Promise.all(
+    dataObjectsIds.map(async (id) => {
+      const dataObject = await api.query.dataDirectory.dataByContentId(id)
+      dataObject.contentId = id
+      return dataObject
+    })
+  )
+
+  let workerIds = args.workers?.split(',')
+  if (!workerIds) {
+    workerIds = await getActiveWorkersIds(api)
+  }
+
+  let success = true
+  try {
+    for (const workerId of workerIds) {
+      try {
+        await testWorker(api, workerId, dataObjects)
+      } catch (e) {
+        console.error(e)
+      }
+      await clearFolder(TEMP_FOLDER)
+    }
+  } catch {
+    success = false
+  } finally {
+    await api.disconnect()
+  }
+
+  if (!success) {
+    process.exit(2)
+  }
+}
+
+async function testWorker(api, workerId, dataObjects) {
+  const endpoint = await getWorkerEndpoint(api, workerId)
+  if (!endpoint) {
+    throw new Error(`Worker ${workerId} doesn't have an endpoint defined`)
+  }
+
+  const promises = []
+  const startRequests = process.hrtime.bigint()
+  for (const dataObject of dataObjects) {
+    const url = makeAssetUrl(dataObject.contentId, endpoint)
+    promises.push(downloadFile(url, TEMP_FOLDER, dataObject.contentId.toString()))
+  }
+  try {
+    const times = await Promise.all(promises)
+    const endRequests = process.hrtime.bigint()
+    const totalTime = Number((endRequests - startRequests) / BigInt(1000000))
+    const totalSizeInMegas =
+      dataObjects.reduce((accumulator, dataObject) => accumulator + Number(dataObject.size_in_bytes), 0) / 1024 / 1024
+
+    const average = times.reduce((accumulator, time) => accumulator + time, 0) / times.length
+    console.log(
+      JSON.stringify({
+        averageMs: Number(average.toFixed(2)),
+        medianMs: Number(computeMedian(times).toFixed(2)),
+        maxMs: Number(Math.max.apply(Math, times).toFixed(2)),
+        minMs: Number(Math.min.apply(Math, times).toFixed(2)),
+        totalTimeMs: totalTime,
+        averageSpeedMBpS: Number((totalSizeInMegas / (totalTime / 100)).toFixed(3)),
+        nrFilesDownloaded: times.length,
+        workerId: workerId,
+      })
+    )
+  } catch (e) {
+    throw new Error('Fail to download files from worker')
+  }
+}
+
+main()

contributions/community-contributions/tech/sp-downloader-tester/bin/utils.js

@@ -0,0 +1,121 @@
+const { Text } = require('@polkadot/types')
+const fsPromises = require('fs/promises')
+const Path = require('path')
+const { encodeAddress } = require('@polkadot/keyring')
+const fs = require('fs')
+const axios = require('axios')
+
+function computeMedian(numbers) {
+  const sorted = numbers.slice().sort((a, b) => a - b)
+  const middle = Math.floor(sorted.length / 2)
+
+  if (sorted.length % 2 === 0) {
+    return (sorted[middle - 1] + sorted[middle]) / 2
+  }
+
+  return sorted[middle]
+}
+
+async function generateListOfDataObjectsToDownload(api, nrSmall, nrBig) {
+  const dataObjects = await getAcceptedDataObjects(api)
+
+  const dataObjectsSortedBySize = dataObjects.sort((dataObject) => dataObject.size_in_bytes)
+
+  const smallFiles = dataObjectsSortedBySize.slice(0, nrSmall)
+  const bigFiles = dataObjectsSortedBySize.slice(-nrBig)
+  return smallFiles.concat(bigFiles).map((dataObject) => dataObject.contentId)
+}
+
+async function getWorkerEndpoint(api, workerId) {
+  const value = await api.query.storageWorkingGroup.workerStorage(workerId)
+  return new Text(api.registry, value).toString()
+}
+
+async function getAcceptedDataObjects(api) {
+  let mapObjects = await api.query.dataDirectory.dataByContentId.entries()
+
+  mapObjects = mapObjects.filter(([, dataObject]) => dataObject.liaison_judgement.type === 'Accepted')
+  const ids = mapToContentId(mapObjects)
+  const dataObjects = mapObjects.map(([, dataObject]) => dataObject)
+  const dataObjectsWithIds = []
+  for (let i = 0; i < dataObjects.length; i++) {
+    dataObjects[i].contentId = ids[i]
+    dataObjectsWithIds.push(dataObjects[i])
+  }
+  return dataObjectsWithIds
+}
+
+async function clearFolder(folder) {
+  const files = await fsPromises.readdir(folder)
+
+  for (const file of files) {
+    await fsPromises.unlink(Path.join(folder, file))
+  }
+}
+
+function makeAssetUrl(contentId, source) {
+  source = removeEndingForwardSlash(source)
+  return `${source}/asset/v0/${encodeAddress(contentId)}`
+}
+
+function mapToContentId(dataObjects) {
+  return dataObjects.map(
+    ([
+      {
+        args: [contentId],
+      },
+    ]) => contentId
+  )
+}
+
+async function downloadFile(url, outputFolder, outputFilename) {
+  const path = Path.join(outputFolder, outputFilename)
+  const writer = fs.createWriteStream(path)
+
+  const start = process.hrtime.bigint()
+  const response = await axios({
+    url,
+    method: 'GET',
+    responseType: 'stream',
+  })
+
+  response.data.pipe(writer)
+
+  return new Promise((resolve, reject) => {
+    writer.on('finish', () => {
+      const end = process.hrtime.bigint()
+      resolve(Number(end - start) / 1000000)
+    })
+    writer.on('error', reject)
+  })
+}
+
+async function getActiveWorkersIds(api) {
+  const ids = []
+  const entries = await api.query.storageWorkingGroup.workerById.entries()
+  entries.forEach(([storageKey, worker]) => {
+    if (worker.is_active) {
+      const id = storageKey.args[0].toNumber()
+      ids.push(id)
+    }
+  })
+
+  return ids
+}
+
+function removeEndingForwardSlash(url) {
+  if (url.endsWith('/')) {
+    return url.substring(0, url.length - 1)
+  }
+  return url.toString()
+}
+
+module.exports = {
+  getWorkerEndpoint,
+  generateListOfDataObjectsToDownload,
+  makeAssetUrl,
+  downloadFile,
+  computeMedian,
+  clearFolder,
+  getActiveWorkersIds
+}

contributions/community-contributions/tech/sp-downloader-tester/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "sp-downloader-tester",
+  "version": "1.0.0",
+  "description": "",
+  "bin": {
+    "sp-downloader-tester": "bin/cli.js"
+  },
+  "scripts": {
+    "sp-downloader-tester": "bin/cli.js"
+  },
+  "author": "freakstatic",
+  "license": "GPL-3.0-only",
+  "dependencies": {
+    "@joystream/types": "^0.16.1",
+    "@polkadot/api": "4.2.1",
+    "@polkadot/api-contract": "4.2.1",
+    "@polkadot/keyring": "^6.0.5",
+    "@polkadot/metadata": "4.2.1",
+    "@polkadot/types": "4.2.1",
+    "@polkadot/util": "^6.0.5",
+    "@polkadot/util-crypto": "^6.0.5",
+    "@polkadot/wasm-crypto": "^4.0.2",
+    "axios": "^0.21.4",
+    "commander": "^8.2.0"
+  }
+}