storage-node: bump version and improve logging in helios

storage-node/packages/colossus/api-base.yml

@@ -1,7 +1,7 @@
 openapi: '3.0.0'
 info:
-  title: 'Joystream Storage Node API.'
-  version: '1.0.0'
+  title: 'Colossus - Joystream Storage Node'
+  version: '1.1.0'
 paths: {} # Will be populated by express-openapi
 
 components:

storage-node/packages/colossus/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@joystream/colossus",
   "private": true,
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Colossus - Joystream Storage Node",
   "author": "Joystream",
   "homepage": "https://github.com/Joystream/joystream",

storage-node/packages/helios/bin/cli.js

@@ -10,34 +10,44 @@ function makeAssetUrl(contentId, source) {
   return `${source}/asset/v0/${encodeAddress(contentId)}`
 }
 
-// HTTP HEAD with axios all known content ids on each provider
-async function countContentAvailability(contentIds, source) {
+// HTTP HEAD with axios all known content ids on given endpoint
+async function countContentAvailability(contentIds, endpoint) {
   let found = 0
   let errored = 0
+  let requestsSent = 0
+  // Avoid opening too many connections, do it in chunks.. otherwise we get
+  // "Client network socket disconnected before secure TLS connection was established" errors
+  while (contentIds.length) {
+    const chunk = contentIds.splice(0, 500)
+    requestsSent += chunk.length
+    const results = await Promise.allSettled(chunk.map((id) => axios.head(makeAssetUrl(id, endpoint))))
+
+    results.forEach((result, _ix) => {
+      if (result.status === 'rejected') {
+        errored++
+      } else {
+        found++
+      }
+    })
 
-  // TODO: To avoid opening too many connections do it in chunks.. otherwise were are getting
-  // Error: Client network socket disconnected before secure TLS connection was established
-  contentIds = contentIds.slice(0, 200)
-
-  // use axios.all() instead ?
-  const results = await Promise.allSettled(contentIds.map((id) => axios.head(makeAssetUrl(id, source))))
-
-  results.forEach((result, _ix) => {
-    if (result.status === 'rejected') {
-      errored++
-    } else {
-      found++
-    }
-  })
+    // show some progress
+    console.error(`${endpoint}:`, `total requests sent: ${requestsSent}`, `found: ${found}`, `failed: ${errored}`)
+  }
 
   return { found, errored }
 }
 
 async function testProviderHasAssets(providerId, endpoint, contentIds) {
   const total = contentIds.length
+  const startedAt = Date.now()
   const { found, errored } = await countContentAvailability(contentIds, endpoint)
-  console.log(`provider ${providerId}: has ${errored} errored assets`)
-  console.log(`provider ${providerId}: has ${found} out of ${total}`)
+  const completedAt = Date.now()
+  console.log(`
+    Final Result for provider ${providerId}
+    fetched: ${found}/${total}
+    failed: ${errored}
+    took: ${(completedAt - startedAt) / 1000}s
+  `)
 }
 
 async function main() {
@@ -70,15 +80,17 @@ async function main() {
         return
       }
       const swaggerUrl = `${stripEndingSlash(provider.endpoint)}/swagger.json`
-      let error
       try {
-        await axios.get(swaggerUrl)
-        // maybe print out api version information to detect which version of colossus is running?
-        // or add anothe api endpoint for diagnostics information
+        const { data } = await axios.get(swaggerUrl)
+        console.log(
+          `${provider.providerId}:`,
+          `${provider.endpoint}`,
+          '- OK -',
+          `storage node version ${data.info.version}`
+        )
       } catch (err) {
-        error = err
+        console.log(`${provider.providerId}`, `${provider.endpoint} - ${err.message}`)
       }
-      console.log(`${provider.providerId}`, `${provider.endpoint} - ${error ? error.message : 'OK'}`)
     })
   )
 
@@ -87,20 +99,19 @@ async function main() {
 
   console.log(`\nData Directory has ${acceptedContentIds.length} 'Accepted' objects out of ${allContentIds.length}`)
 
-  // interesting disconnect doesn't work unless an explicit provider was created
-  // for underlying api instance
   // We no longer need a connection to the chain
   api.disconnect()
 
-  console.log(`\nChecking available assets on providers (this can take some time)...`)
+  console.log(`
+    Checking available assets on providers (this can take some time)
+    This is done by sending HEAD requests for all 'Accepted' assets.
+  `)
 
-  // TODO: Do it sequentially one provider at a time.. to control connections/s to avoid
-  // connection resets?
   endpoints.forEach(async ({ providerId, endpoint }) => {
     if (!endpoint) {
       return
     }
-    return testProviderHasAssets(providerId, endpoint, acceptedContentIds)
+    return testProviderHasAssets(providerId, endpoint, acceptedContentIds.slice())
   })
 }