|
@@ -10,34 +10,44 @@ function makeAssetUrl(contentId, source) {
|
|
|
return `${source}/asset/v0/${encodeAddress(contentId)}`
|
|
|
}
|
|
|
|
|
|
-// HTTP HEAD with axios all known content ids on each provider
|
|
|
-async function countContentAvailability(contentIds, source) {
|
|
|
+// HTTP HEAD with axios all known content ids on given endpoint
|
|
|
+async function countContentAvailability(contentIds, endpoint) {
|
|
|
let found = 0
|
|
|
let errored = 0
|
|
|
+ let requestsSent = 0
|
|
|
+ // Avoid opening too many connections, do it in chunks.. otherwise we get
|
|
|
+ // "Client network socket disconnected before secure TLS connection was established" errors
|
|
|
+ while (contentIds.length) {
|
|
|
+ const chunk = contentIds.splice(0, 500)
|
|
|
+ requestsSent += chunk.length
|
|
|
+ const results = await Promise.allSettled(chunk.map((id) => axios.head(makeAssetUrl(id, endpoint))))
|
|
|
+
|
|
|
+ results.forEach((result, _ix) => {
|
|
|
+ if (result.status === 'rejected') {
|
|
|
+ errored++
|
|
|
+ } else {
|
|
|
+ found++
|
|
|
+ }
|
|
|
+ })
|
|
|
|
|
|
- // TODO: To avoid opening too many connections do it in chunks.. otherwise were are getting
|
|
|
- // Error: Client network socket disconnected before secure TLS connection was established
|
|
|
- contentIds = contentIds.slice(0, 200)
|
|
|
-
|
|
|
- // use axios.all() instead ?
|
|
|
- const results = await Promise.allSettled(contentIds.map((id) => axios.head(makeAssetUrl(id, source))))
|
|
|
-
|
|
|
- results.forEach((result, _ix) => {
|
|
|
- if (result.status === 'rejected') {
|
|
|
- errored++
|
|
|
- } else {
|
|
|
- found++
|
|
|
- }
|
|
|
- })
|
|
|
+ // show some progress
|
|
|
+ console.error(`${endpoint}:`, `total requests sent: ${requestsSent}`, `found: ${found}`, `failed: ${errored}`)
|
|
|
+ }
|
|
|
|
|
|
return { found, errored }
|
|
|
}
|
|
|
|
|
|
async function testProviderHasAssets(providerId, endpoint, contentIds) {
|
|
|
const total = contentIds.length
|
|
|
+ const startedAt = Date.now()
|
|
|
const { found, errored } = await countContentAvailability(contentIds, endpoint)
|
|
|
- console.log(`provider ${providerId}: has ${errored} errored assets`)
|
|
|
- console.log(`provider ${providerId}: has ${found} out of ${total}`)
|
|
|
+ const completedAt = Date.now()
|
|
|
+ console.log(`
|
|
|
+ Final Result for provider ${providerId}
|
|
|
+ fetched: ${found}/${total}
|
|
|
+ failed: ${errored}
|
|
|
+ took: ${(completedAt - startedAt) / 1000}s
|
|
|
+ `)
|
|
|
}
|
|
|
|
|
|
async function main() {
|
|
@@ -70,15 +80,17 @@ async function main() {
|
|
|
return
|
|
|
}
|
|
|
const swaggerUrl = `${stripEndingSlash(provider.endpoint)}/swagger.json`
|
|
|
- let error
|
|
|
try {
|
|
|
- await axios.get(swaggerUrl)
|
|
|
- // maybe print out api version information to detect which version of colossus is running?
|
|
|
- // or add anothe api endpoint for diagnostics information
|
|
|
+ const { data } = await axios.get(swaggerUrl)
|
|
|
+ console.log(
|
|
|
+ `${provider.providerId}:`,
|
|
|
+ `${provider.endpoint}`,
|
|
|
+ '- OK -',
|
|
|
+ `storage node version ${data.info.version}`
|
|
|
+ )
|
|
|
} catch (err) {
|
|
|
- error = err
|
|
|
+ console.log(`${provider.providerId}`, `${provider.endpoint} - ${err.message}`)
|
|
|
}
|
|
|
- console.log(`${provider.providerId}`, `${provider.endpoint} - ${error ? error.message : 'OK'}`)
|
|
|
})
|
|
|
)
|
|
|
|
|
@@ -87,20 +99,19 @@ async function main() {
|
|
|
|
|
|
console.log(`\nData Directory has ${acceptedContentIds.length} 'Accepted' objects out of ${allContentIds.length}`)
|
|
|
|
|
|
- // interesting disconnect doesn't work unless an explicit provider was created
|
|
|
- // for underlying api instance
|
|
|
// We no longer need a connection to the chain
|
|
|
api.disconnect()
|
|
|
|
|
|
- console.log(`\nChecking available assets on providers (this can take some time)...`)
|
|
|
+ console.log(`
|
|
|
+ Checking available assets on providers (this can take some time)
|
|
|
+ This is done by sending HEAD requests for all 'Accepted' assets.
|
|
|
+ `)
|
|
|
|
|
|
- // TODO: Do it sequentially one provider at a time.. to control connections/s to avoid
|
|
|
- // connection resets?
|
|
|
endpoints.forEach(async ({ providerId, endpoint }) => {
|
|
|
if (!endpoint) {
|
|
|
return
|
|
|
}
|
|
|
- return testProviderHasAssets(providerId, endpoint, acceptedContentIds)
|
|
|
+ return testProviderHasAssets(providerId, endpoint, acceptedContentIds.slice())
|
|
|
})
|
|
|
}
|
|
|
|