joystream/storage-node/packages/helios/bin/cli.js

#!/usr/bin/env node

const yargs = require('yargs')
const argv = yargs.scriptName('helios.js').option('testAssets', {
  alias: 'a',
  description: 'test for all assets on all providers',
  type: 'boolean',
}).argv

const { RuntimeApi } = require('@joystream/storage-runtime-api')
const { encodeAddress } = require('@polkadot/keyring')
const axios = require('axios')
const stripEndingSlash = require('@joystream/storage-utils/stripEndingSlash')

function makeAssetUrl(contentId, source) {
  source = stripEndingSlash(source)
  return `${source}/asset/v0/${encodeAddress(contentId)}`
}

async function assetRelationshipState(api, contentId, providers) {
  const dataObject = await api.query.dataDirectory.dataByContentId(contentId)

  const relationshipIds = await api.query.dataObjectStorageRegistry.relationshipsByContentId(contentId)

  // how many relationships associated with active providers and in ready state
  const activeRelationships = await Promise.all(
    relationshipIds.map(async (id) => {
      let relationship = await api.query.dataObjectStorageRegistry.relationships(id)
      relationship = relationship.unwrap()
      // only interested in ready relationships
      if (!relationship.ready) {
        return undefined
      }
      // Does the relationship belong to an active provider ?
      return providers.find((provider) => relationship.storage_provider.eq(provider))
    })
  )

  return [activeRelationships.filter((active) => active).length, dataObject.liaison_judgement]
}

// HTTP HEAD with axios all known content ids on each provider
async function countContentAvailability(contentIds, source) {
  const content = {}
  let found = 0
  let missing = 0
  for (let i = 0; i < contentIds.length; i++) {
    const assetUrl = makeAssetUrl(contentIds[i], source)
    try {
      const info = await axios.head(assetUrl)
      content[encodeAddress(contentIds[i])] = {
        type: info.headers['content-type'],
        bytes: info.headers['content-length'],
      }
      // TODO: cross check against dataobject size
      found++
    } catch (err) {
      missing++
    }
  }

  return { found, missing, content }
}

async function testAssets(runtime, api, endpoints, storageProviders) {
  console.log(`\nIndexing Data Directory ..,`)
  const knownContentIds = await runtime.assets.getKnownContentIds()
  const assetStatuses = {}
  await Promise.all(
    knownContentIds.map(async (contentId) => {
      const [, judgement] = await assetRelationshipState(api, contentId, storageProviders)
      const j = judgement.toString()
      assetStatuses[j] = assetStatuses[j] ? assetStatuses[j] + 1 : 1
    })
  )
  console.log(`\nData Directory has ${knownContentIds.length} assets:`, assetStatuses)

  // interesting disconnect doesn't work unless an explicit provider was created
  // for underlying api instance
  // We no longer need a connection to the chain
  api.disconnect()

  console.log(`\nChecking available assets on providers (this can take some time)...`)
  endpoints.forEach(async ({ providerId, endpoint }) => {
    if (!endpoint) {
      return
    }
    const total = knownContentIds.length
    const { found } = await countContentAvailability(knownContentIds, endpoint)
    console.log(`provider ${providerId}: has ${found} out of ${total}`)
  })
}

async function main() {
  const runtime = await RuntimeApi.create()
  const { api } = runtime

  // get all providers
  const { ids: storageProviders } = await runtime.workers.getAllProviders()

  // Resolve Endpoints of providers
  console.log(`Testing ${storageProviders.length} staked providers:`)
  const endpoints = await Promise.all(
    storageProviders.map(async (providerId) => {
      try {
        const endpoint = (await runtime.workers.getWorkerStorageValue(providerId)).toString()
        return { providerId, endpoint }
      } catch (err) {
        console.log('resolve failed for id', providerId, err.message)
        return { providerId, endpoint: null }
      }
    })
  )

  //console.log('\nChecking API Endpoints are online')
  await Promise.all(
    endpoints.map(async (provider) => {
      if (!provider.endpoint) {
        console.log(` - ${provider.providerId} No public url set.`)
        return
      }
      const swaggerUrl = `${stripEndingSlash(provider.endpoint)}/swagger.json`
      let error
      try {
        await axios.get(swaggerUrl)
        // maybe print out api version information to detect which version of colossus is running?
        // or add anothe api endpoint for diagnostics information
      } catch (err) {
        error = err
      }
      console.log(` - ${provider.providerId}`, `${provider.endpoint} - ${error ? error.message : 'OK'}`)
    })
  )

  if (argv.testAssets) testAssets(runtime, api, endpoints, storageProviders)
  else api.disconnect()
}

main()