storage-node-v2: Add custom logger.

- Add winston logger
- Add winston express integration

storage-node-v2/.eslintrc.js

@@ -7,6 +7,7 @@ module.exports = {
   },
   extends: ['@joystream/eslint-config'],
   rules: {
+    'no-console': 'warn', // use dedicated logger
     'no-unused-vars': 'off', // Required by the typescript rule below
     '@typescript-eslint/no-unused-vars': ['error'],
     '@typescript-eslint/no-floating-promises': 'error',

storage-node-v2/package.json

@@ -15,13 +15,16 @@
     "@polkadot/api": "4.2.1",
     "@types/express": "4.17.1",
     "@types/multer": "^1.4.5",
+    "@types/winston": "^2.4.4",
     "await-lock": "^2.1.0",
     "blake3": "^2.1.4",
     "express": "4.17.1",
     "express-openapi-validator": "^4.12.4",
+    "express-winston": "^4.1.0",
     "multihashes": "^4.0.2",
     "openapi-editor": "^0.3.0",
-    "tslib": "^1"
+    "tslib": "^1",
+    "winston": "^3.3.3"
   },
   "devDependencies": {
     "@joystream/eslint-config": "^1.0.0",

storage-node-v2/src/command-base/ApiCommandBase.ts

@@ -3,6 +3,7 @@ import { createApi, getAlicePair } from '../services/runtime/api'
 import { getAccountFromJsonFile } from '../services/runtime/accounts'
 import { KeyringPair } from '@polkadot/keyring/types'
 import { ApiPromise } from '@polkadot/api'
+import logger from '../services/logger'
 
 export default abstract class ApiCommandBase extends Command {
   private api: ApiPromise | null = null
@@ -52,7 +53,7 @@ export default abstract class ApiCommandBase extends Command {
       throw new Error('This command should only be run on a Development chain.')
     }
 
-    this.log('Development mode is ON.')
+    logger.info('Development mode is ON.')
   }
 
   getAccount(flags: {

storage-node-v2/src/commands/dev/multihash.ts

@@ -1,5 +1,6 @@
 import { Command, flags } from '@oclif/command'
 import { hashFile } from '../../services/helpers/hashing'
+import logger from '../../services/logger'
 
 export default class DevMultihash extends Command {
   static description = 'Creates a multihash (blake3) for a file.'
@@ -16,10 +17,10 @@ export default class DevMultihash extends Command {
   async run(): Promise<void> {
     const { flags } = this.parse(DevMultihash)
 
-    console.log(`Hashing ${flags.file}`)
+    logger.info(`Hashing ${flags.file} ....`)
 
     const multi = await hashFile(flags.file)
 
-    console.log(multi)
+    logger.info(`Hash: ${multi}`)
   }
 }

storage-node-v2/src/commands/dev/upload.ts

@@ -1,6 +1,7 @@
 import { flags } from '@oclif/command'
 import { uploadDataObjects } from '../../services/runtime/extrinsics'
 import ApiCommandBase from '../../command-base/ApiCommandBase'
+import logger from '../../services/logger'
 
 export default class DevUpload extends ApiCommandBase {
   static description = 'Upload data object (development mode only).'
@@ -27,7 +28,7 @@ export default class DevUpload extends ApiCommandBase {
     const objectSize = flags.size ?? 0
     const objectCid = flags.cid
 
-    this.log('Uploading data objects...')
+    logger.info('Uploading data objects...')
 
     const api = await this.getApi()
     await uploadDataObjects(api, objectSize, objectCid)

storage-node-v2/src/commands/dev/verify-bag-id.ts

@@ -1,6 +1,7 @@
 import { flags } from '@oclif/command'
 import ApiCommandBase from '../../command-base/ApiCommandBase'
 import { parseBagId } from '../../services/helpers/bagIdParser'
+import logger from '../../services/logger'
 
 export default class DevVerifyBagId extends ApiCommandBase {
   static description =
@@ -33,6 +34,6 @@ export default class DevVerifyBagId extends ApiCommandBase {
     const api = await this.getApi()
     parseBagId(api, flags.bagId)
 
-    console.log(`Correct bag id: ${flags.bagId}`)
+    logger.info(`Correct bag id: ${flags.bagId}`)
   }
 }

storage-node-v2/src/commands/leader/create-bucket.ts

@@ -1,6 +1,7 @@
 import { createStorageBucket } from '../../services/runtime/extrinsics'
 import { flags } from '@oclif/command'
 import ApiCommandBase from '../../command-base/ApiCommandBase'
+import logger from '../../services/logger'
 
 export default class LeaderCreateBucket extends ApiCommandBase {
   static description = `Create new storage bucket. Requires storage working group leader permissions.`
@@ -30,7 +31,7 @@ export default class LeaderCreateBucket extends ApiCommandBase {
     const allowNewBags = flags.allow ?? false
     const invitedWorker = flags.invited
 
-    this.log('Creating storage bucket...')
+    logger.info('Creating storage bucket...')
     if (flags.dev) {
       await this.ensureDevelopmentChain()
     }

storage-node-v2/src/commands/leader/update-bag-limit.ts

@@ -1,6 +1,7 @@
 import ApiCommandBase from '../../command-base/ApiCommandBase'
 import { updateStorageBucketsPerBagLimit } from '../../services/runtime/extrinsics'
 import { flags } from '@oclif/command'
+import logger from '../../services/logger'
 
 export default class LeaderUpdateBagLimit extends ApiCommandBase {
   static description =
@@ -18,7 +19,7 @@ export default class LeaderUpdateBagLimit extends ApiCommandBase {
   async run(): Promise<void> {
     const { flags } = this.parse(LeaderUpdateBagLimit)
 
-    this.log('Update "Storage buckets per bag" number limit....')
+    logger.info('Update "Storage buckets per bag" number limit....')
     if (flags.dev) {
       await this.ensureDevelopmentChain()
     }

storage-node-v2/src/commands/leader/update-bag.ts

@@ -2,6 +2,7 @@ import { flags } from '@oclif/command'
 import { updateStorageBucketsForBag } from '../../services/runtime/extrinsics'
 import ApiCommandBase from '../../command-base/ApiCommandBase'
 import { parseBagId } from '../../services/helpers/bagIdParser'
+import logger from '../../services/logger'
 
 export default class LeaderUpdateBag extends ApiCommandBase {
   static description =
@@ -42,7 +43,7 @@ export default class LeaderUpdateBag extends ApiCommandBase {
 
     const bucket = flags.bucket ?? 0
 
-    this.log('Update bag - add/remove storage buckets...')
+    logger.info('Update bag - add/remove storage buckets...')
     if (flags.dev) {
       await this.ensureDevelopmentChain()
     }

storage-node-v2/src/commands/leader/update-voucher-limits.ts

@@ -1,6 +1,7 @@
 import ApiCommandBase from '../../command-base/ApiCommandBase'
 import { updateStorageBucketsVoucherMaxLimits } from '../../services/runtime/extrinsics'
 import { flags } from '@oclif/command'
+import logger from '../../services/logger'
 
 export default class LeaderUpdateVoucherLimits extends ApiCommandBase {
   static description =
@@ -23,7 +24,7 @@ export default class LeaderUpdateVoucherLimits extends ApiCommandBase {
   async run(): Promise<void> {
     const { flags } = this.parse(LeaderUpdateVoucherLimits)
 
-    this.log('Update "Storage buckets per bag" number limit....')
+    logger.info('Update "Storage buckets per bag" number limit....')
     if (flags.dev) {
       await this.ensureDevelopmentChain()
     }

storage-node-v2/src/commands/operator/accept-invitation.ts

@@ -1,6 +1,7 @@
 import { flags } from '@oclif/command'
 import { acceptStorageBucketInvitation } from '../../services/runtime/extrinsics'
 import ApiCommandBase from '../../command-base/ApiCommandBase'
+import logger from '../../services/logger'
 
 export default class OperatorAcceptInvitation extends ApiCommandBase {
   static description = 'Accept pending storage bucket invitation.'
@@ -25,7 +26,7 @@ export default class OperatorAcceptInvitation extends ApiCommandBase {
     const worker = flags.worker ?? 0 // TODO: don't require on dev???
     const bucket = flags.bucket ?? 0
 
-    this.log('Accepting pending storage bucket invitation...')
+    logger.info('Accepting pending storage bucket invitation...')
     if (flags.dev) {
       await this.ensureDevelopmentChain()
     }

storage-node-v2/src/commands/server.ts

@@ -1,6 +1,7 @@
 import { flags } from '@oclif/command'
 import { createApp } from '../services/webApi/app'
 import ApiCommandBase from '../command-base/ApiCommandBase'
+import logger from '../services/logger'
 
 // TODO: fix command not found error (error handling)
 // TODO: custom IP address?
@@ -43,10 +44,10 @@ export default class Server extends ApiCommandBase {
       const port = flags.port
       const workerId = flags.worker ?? 0 // TODO: don't require on dev???
       const app = await createApp(api, account, workerId, flags.uploads)
-      console.info(`Listening on http://localhost:${port}`)
+      logger.info(`Listening on http://localhost:${port}`)
       app.listen(port)
     } catch (err) {
-      console.error(`Error: ${err}`)
+      logger.error(`Error: ${err}`)
     }
   }
 

storage-node-v2/src/services/logger.ts

@@ -0,0 +1,68 @@
+import winston from 'winston'
+import expressWinston from 'express-winston'
+import { Handler } from 'express'
+
+// Creates basic winston logger.
+function createDefaultLogger(): winston.Logger {
+  const levels = {
+    error: 0,
+    warn: 1,
+    info: 2,
+    http: 3,
+    debug: 4,
+  }
+
+  const level = () => {
+    const env = process.env.NODE_ENV || 'development'
+    const isDevelopment = env === 'development'
+    return isDevelopment ? 'debug' : 'warn'
+  }
+
+  const colors = {
+    error: 'red',
+    warn: 'yellow',
+    info: 'green',
+    http: 'magenta',
+    debug: 'white',
+  }
+
+  winston.addColors(colors)
+
+  const format = winston.format.combine(
+    winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss:ms' }),
+    winston.format.colorize({ all: true }),
+    winston.format.printf(
+      (info) => `${info.timestamp} ${info.level}: ${info.message}`
+    )
+  )
+
+  const transports = [new winston.transports.Console()]
+
+  return winston.createLogger({
+    level: level(),
+    levels,
+    format,
+    transports,
+  })
+}
+
+const Logger = createDefaultLogger()
+
+export default Logger
+
+// Creates Express-Winston logger handler.
+export function getHttpLogger(): Handler {
+  const opts: expressWinston.LoggerOptions = {
+    transports: [new winston.transports.Console()],
+    format: winston.format.combine(
+      winston.format.colorize(),
+      winston.format.json()
+    ),
+    meta: true,
+    msg: 'HTTP {{req.method}} {{req.url}}',
+    expressFormat: true,
+    colorize: false,
+  }
+
+  return expressWinston.logger(opts)
+}

storage-node-v2/src/services/runtime/api.ts

@@ -4,7 +4,6 @@ import { CodecArg, ISubmittableResult } from '@polkadot/types/types'
 import { types } from '@joystream/types/'
 import { TypeRegistry } from '@polkadot/types'
 import { KeyringPair } from '@polkadot/keyring/types'
-import chalk from 'chalk'
 import { SubmittableExtrinsic } from '@polkadot/api/types'
 import {
   DispatchError,
@@ -12,6 +11,8 @@ import {
 } from '@polkadot/types/interfaces/system'
 import { Keyring } from '@polkadot/keyring'
 import { getNonce } from './nonceKeeper'
+import logger from '../../services/logger'
+
 // TODO: ApiHelper class container for functions ???
 
 export class ExtrinsicFailedError extends Error {}
@@ -102,21 +103,16 @@ function sendExtrinsic(
 export async function sendAndFollowTx(
   api: ApiPromise,
   account: KeyringPair,
-  tx: SubmittableExtrinsic<'promise'>,
-  warnOnly = false // If specified - only warning will be displayed in case of failure (instead of error beeing thrown)
+  tx: SubmittableExtrinsic<'promise'>
 ): Promise<boolean> {
   try {
-    // TODO: use async-lock package
     const nonce = await getNonce(api, account)
 
     await sendExtrinsic(api, account, tx, nonce)
-    console.log(chalk.green(`Extrinsic successful!`))
+    logger.debug(`Extrinsic successful!`)
     return true
   } catch (e) {
-    if (e instanceof ExtrinsicFailedError && warnOnly) {
-      console.warn(`Extrinsic failed! ${e.message}`)
-      return false
-    } else if (e instanceof ExtrinsicFailedError) {
+    if (e instanceof ExtrinsicFailedError) {
       throw new ExtrinsicFailedError(`Extrinsic failed! ${e.message}`)
     } else {
       throw e
@@ -129,12 +125,11 @@ export async function sendAndFollowNamedTx(
   account: KeyringPair,
   module: string,
   method: string,
-  params: CodecArg[],
-  warnOnly = false
+  params: CodecArg[]
 ): Promise<boolean> {
-  console.log(chalk.white(`\nSending ${module}.${method} extrinsic...`))
+  logger.debug(`Sending ${module}.${method} extrinsic...`)
   const tx = api.tx[module][method](...params)
-  return await sendAndFollowTx(api, account, tx, warnOnly)
+  return await sendAndFollowTx(api, account, tx)
 }
 
 export async function sendAndFollowSudoNamedTx(
@@ -142,12 +137,11 @@ export async function sendAndFollowSudoNamedTx(
   account: KeyringPair,
   module: string,
   method: string,
-  params: CodecArg[],
-  warnOnly = false
+  params: CodecArg[]
 ): Promise<boolean> {
-  console.log(chalk.white(`\nSending ${module}.${method} extrinsic...`))
+  logger.debug(`Sending ${module}.${method} extrinsic...`)
   const tx = api.tx.sudo.sudo(api.tx[module][method](...params))
-  return await sendAndFollowTx(api, account, tx, warnOnly)
+  return await sendAndFollowTx(api, account, tx)
 }
 
 export function getAlicePair(): KeyringPair {

storage-node-v2/src/services/runtime/extrinsics.ts

@@ -7,6 +7,7 @@ import { KeyringPair } from '@polkadot/keyring/types'
 import { CodecArg } from '@polkadot/types/types'
 import { ApiPromise } from '@polkadot/api'
 import { BagId } from '@joystream/types/storage'
+import logger from '../../services/logger'
 
 export async function createStorageBucket(
   api: ApiPromise,
@@ -26,7 +27,7 @@ export async function createStorageBucket(
       objectsLimit,
     ])
   } catch (err) {
-    console.error(`Api Error: ${err}`)
+    logger.error(`Api Error: ${err}`)
   }
 }
 
@@ -45,7 +46,7 @@ export async function acceptStorageBucketInvitation(
       [workerId, storageBucketId]
     )
   } catch (err) {
-    console.error(`Api Error: ${err}`)
+    logger.error(`Api Error: ${err}`)
   }
 }
 
@@ -76,7 +77,7 @@ export async function updateStorageBucketsForBag(
       [bagId, addBuckets, removeBuckets]
     )
   } catch (err) {
-    console.error(`Api Error: ${err}`)
+    logger.error(`Api Error: ${err}`)
   }
 }
 
@@ -106,7 +107,7 @@ export async function uploadDataObjects(
       [data]
     )
   } catch (err) {
-    console.error(`Api Error: ${err}`)
+    logger.error(`Api Error: ${err}`)
   }
 }
 
@@ -132,7 +133,7 @@ export async function acceptPendingDataObjects(
       [workerId, storageBucketId, bagId, dataObjectSet]
     )
   } catch (err) {
-    console.error(`Api Error: ${err}`)
+    logger.error(`Api Error: ${err}`)
     throw err
   }
 }
@@ -151,7 +152,7 @@ export async function updateStorageBucketsPerBagLimit(
       [newLimit]
     )
   } catch (err) {
-    console.error(`Api Error: ${err}`)
+    logger.error(`Api Error: ${err}`)
   }
 }
 
@@ -170,6 +171,6 @@ export async function updateStorageBucketsVoucherMaxLimits(
       [newSizeLimit, newObjectLimit]
     )
   } catch (err) {
-    console.error(`Api Error: ${err}`)
+    logger.error(`Api Error: ${err}`)
   }
 }

storage-node-v2/src/services/runtime/hireLead.ts

@@ -12,6 +12,7 @@ import {
 } from '@joystream/types/working-group'
 import { MemberId } from '@joystream/types/members'
 import { ApiPromise } from '@polkadot/api'
+import logger from '../../services/logger'
 
 export async function hireStorageWorkingGroupLead(
   api: ApiPromise
@@ -29,7 +30,7 @@ export async function hireStorageWorkingGroupLead(
   let memberId: MemberId | undefined = members.toArray()[0] as MemberId
 
   if (memberId === undefined) {
-    console.log('Preparing member account creation extrinsic...')
+    logger.info('Preparing member account creation extrinsic...')
     memberId = (await api.query.members.nextMemberId()) as MemberId
     await sendAndFollowNamedTx(api, LeadKeyPair, 'members', 'buyMembership', [
       0,
@@ -43,11 +44,11 @@ export async function hireStorageWorkingGroupLead(
   const currentLead =
     (await api.query.storageWorkingGroup.currentLead()) as Option<WorkerId>
   if (currentLead.isSome) {
-    console.log('Storage lead already exists, skipping...')
+    logger.info('Storage lead already exists, skipping...')
     return
   }
 
-  console.log(`Making member id: ${memberId} the content lead.`)
+  logger.info(`Making member id: ${memberId} the content lead.`)
 
   const newOpeningId =
     (await api.query.storageWorkingGroup.nextOpeningId()) as OpeningId
@@ -55,7 +56,7 @@ export async function hireStorageWorkingGroupLead(
     (await api.query.storageWorkingGroup.nextApplicationId()) as ApplicationId
 
   // Create curator lead opening
-  console.log('Preparing Create Storage Lead Opening extrinsic...')
+  logger.info('Preparing Create Storage Lead Opening extrinsic...')
   await sendAndFollowSudoNamedTx(
     api,
     SudoKeyPair,
@@ -70,7 +71,7 @@ export async function hireStorageWorkingGroupLead(
   )
 
   // Apply to lead opening
-  console.log('Preparing Apply to Storage Lead Opening extrinsic...')
+  logger.info('Preparing Apply to Storage Lead Opening extrinsic...')
   await sendAndFollowNamedTx(
     api,
     LeadKeyPair,
@@ -87,7 +88,7 @@ export async function hireStorageWorkingGroupLead(
   )
 
   // Begin review period
-  console.log('Preparing Begin Applicant Review extrinsic...')
+  logger.info('Preparing Begin Applicant Review extrinsic...')
   await sendAndFollowSudoNamedTx(
     api,
     SudoKeyPair,
@@ -97,7 +98,7 @@ export async function hireStorageWorkingGroupLead(
   )
 
   // Fill opening
-  console.log('Preparing Fill Opening extrinsic...')
+  logger.info('Preparing Fill Opening extrinsic...')
   await sendAndFollowSudoNamedTx(
     api,
     SudoKeyPair,

storage-node-v2/src/services/runtime/nonceKeeper.ts

@@ -3,6 +3,7 @@ import type { Index } from '@polkadot/types/interfaces/runtime'
 import BN from 'bn.js'
 import AwaitLock from 'await-lock'
 import { ApiPromise } from '@polkadot/api'
+import logger from '../../services/logger'
 
 let nonce: Index | null = null
 const lock = new AwaitLock()
@@ -22,7 +23,7 @@ export async function getNonce(
     lock.release()
   }
 
-  console.debug(`Last nonce:${nonce}`)
+  logger.debug(`Last transaction nonce:${nonce}`)
 
   return nonce as Index
 }

storage-node-v2/src/services/webApi/app.ts

@@ -9,6 +9,7 @@ import { ApiPromise } from '@polkadot/api'
 import { TokenRequest, verifyTokenSignature } from '../helpers/auth'
 import { createStorageBucket } from '../runtime/extrinsics'
 import { parseBagId } from '../../services/helpers/bagIdParser'
+import { getHttpLogger } from '../../services/logger'
 
 // TODO: custom errors (including validation errors)
 // TODO: custom authorization errors
@@ -25,6 +26,7 @@ export async function createApp(
 
   app.use(cors())
   app.use(express.json())
+  app.use(getHttpLogger())
 
   // TODO: check path
   app.use('/files', express.static(uploadsDir))