storage-node-v2: Add elasticSearch log-formatter.

storage-node-v2/package.json

@@ -9,6 +9,7 @@
   "bugs": "https://github.com/Joystream/joystream/issues",
   "dependencies": {
     "@apollo/client": "^3.3.21",
+    "@elastic/ecs-winston-format": "^1.3.1",
     "@joystream/metadata-protobuf": "^1.0.0",
     "@joystream/types": "^0.17.0",
     "@oclif/command": "^1",

storage-node-v2/scripts/init-dev-bucket.sh

@@ -11,8 +11,8 @@ CLI=../bin/run
 
 ${CLI} dev:init
 ${CLI} leader:update-bag-limit -l 7 --dev
-${CLI} leader:update-voucher-limits -o 100 -s 10000000 --dev
-BUCKET_ID=`${CLI} leader:create-bucket -i=0 -a -n=100 -s=10000000  --dev` 
+${CLI} leader:update-voucher-limits -o 10000 -s 1000000000 --dev
+BUCKET_ID=`${CLI} leader:create-bucket -i=0 -a -n=10000 -s=1000000000  --dev` 
 ${CLI} operator:accept-invitation -w=0 -i=${BUCKET_ID} --dev
 ${CLI} leader:update-bag -a=${BUCKET_ID} -i static:council --dev 
 ${CLI} operator:set-metadata -w 0 -i=${BUCKET_ID} -e http://localhost:3333 --dev

storage-node-v2/src/services/logger.ts

@@ -1,13 +1,9 @@
 import winston, { transport } from 'winston'
+import ecsformat from '@elastic/ecs-winston-format'
 import expressWinston from 'express-winston'
 import { Handler, ErrorRequestHandler } from 'express'
 import { ElasticsearchTransport } from 'winston-elasticsearch'
 
-/**
- * ElasticSearch server date format.
- */
-const elasticDateFormat = 'YYYY-MM-DDTHH:mm:ss'
-
 /**
  * Possible log levels.
  */
@@ -50,12 +46,11 @@ function createDefaultLoggerOptions(): winston.LoggerOptions {
   )
 
   // Redirect all logs to the stderr
-  const transports = [new winston.transports.Console({ stderrLevels: Object.keys(levels) })]
+  const transports = [new winston.transports.Console({ stderrLevels: Object.keys(levels), format })]
 
   return {
     level: level(),
     levels,
-    format,
     transports,
   }
 }
@@ -95,7 +90,14 @@ export default proxy
  *
  */
 export function httpLogger(elasticSearchEndpoint?: string): Handler {
-  const transports: winston.transport[] = [new winston.transports.Console()]
+  // ElasticSearch server date format.
+  const elasticDateFormat = 'YYYY-MM-DDTHH:mm:ss'
+
+  const transports: winston.transport[] = [
+    new winston.transports.Console({
+      format: winston.format.combine(winston.format.timestamp({ format: elasticDateFormat }), winston.format.json()),
+    }),
+  ]
 
   if (elasticSearchEndpoint) {
     const esTransport = createElasticTransport(elasticSearchEndpoint)
@@ -104,7 +106,6 @@ export function httpLogger(elasticSearchEndpoint?: string): Handler {
 
   const opts: expressWinston.LoggerOptions = {
     transports,
-    format: winston.format.combine(winston.format.timestamp({ format: elasticDateFormat }), winston.format.json()),
     meta: true,
     msg: 'HTTP {{req.method}} {{req.url}}',
     expressFormat: true,
@@ -153,12 +154,6 @@ export function createStdConsoleLogger(): winston.Logger {
 function createElasticLogger(elasticSearchEndpoint: string): winston.Logger {
   const loggerOptions = createDefaultLoggerOptions()
 
-  // Formats
-  loggerOptions.format = winston.format.combine(
-    winston.format.timestamp({ format: elasticDateFormat }),
-    winston.format.printf((info) => `${info.timestamp} ${info.level}: ${info.message}`)
-  )
-
   // Transports
   let transports: transport[] = []
   if (loggerOptions.transports !== undefined) {
@@ -210,6 +205,7 @@ function createElasticTransport(elasticSearchEndpoint: string): winston.transpor
     level: elasticLogLevel,
     clientOpts: { node: elasticSearchEndpoint, maxRetries: 5 },
     index: 'storage-node',
+    format: ecsformat(),
   }
   return new ElasticsearchTransport(esTransportOpts)
 }

yarn.lock

@@ -1720,6 +1720,13 @@
   dependencies:
     "@elastic/ecs-helpers" "^1.1.0"
 
+"@elastic/ecs-winston-format@^1.3.1":
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/@elastic/ecs-winston-format/-/ecs-winston-format-1.3.1.tgz#da37d3f6e19ee7bdc076d9e3f2110c96602238a5"
+  integrity sha512-cbDaTU6zUXNpAZSJoLUgNqB0kq2YZ1hmDePVdKrJmw7OBDbzUUMfaK+7S21yFpln/tUF4KRSSPbSwEBgtTLp7Q==
+  dependencies:
+    "@elastic/ecs-helpers" "^1.1.0"
+
 "@elastic/elasticsearch@^7.14.1":
   version "7.14.1"
   resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-7.14.1.tgz#da2f86a550e75c093cc1114115e8719cc7edbfc7"