More extensive logging configuration

distributor-node/config.yml

@@ -2,15 +2,20 @@ id: test-node
 endpoints:
   queryNode: http://localhost:8081/graphql
   joystreamNodeWs: ws://localhost:9944
-  # elasticSearch: http://localhost:9200
 directories:
   assets: ./local/data
   cacheState: ./local/cache
-  logs: ./local/logs
-log:
-  file: debug
-  console: verbose
-  # elastic: info
+logs:
+  file:
+    level: debug
+    path: ./local/logs
+    maxFiles: 5
+    maxSize: 1000000
+  console:
+    level: verbose
+  # elastic:
+  #   level: info
+  #   endpoint: http://localhost:9200
 limits:
   storage: 100G
   maxConcurrentStorageNodeDownloads: 100

distributor-node/config/docker/config.docker.yml

@@ -2,14 +2,12 @@ id: distributor-node-docker
 endpoints:
   queryNode: http://graphql-server-mnt:4002/graphql
   joystreamNodeWs: ws://joystream-node:9944
-  # elasticSearch: http://elasticsearch:9200
 directories:
   assets: /data
   cacheState: /cache
-  logs: /logs
 log:
-  console: info
-  # elastic: info
+  console:
+    level: info
 limits:
   storage: 100G
   maxConcurrentStorageNodeDownloads: 100

distributor-node/package.json

@@ -43,7 +43,8 @@
     "blake3": "^2.1.4",
     "js-image-generator": "^1.0.3",
     "url-join": "^4.0.1",
-    "@types/url-join": "^4.0.1"
+    "@types/url-join": "^4.0.1",
+    "winston-daily-rotate-file": "^4.5.5"
   },
   "devDependencies": {
     "@graphql-codegen/cli": "^1.21.4",

distributor-node/src/command-base/default.ts

@@ -61,8 +61,8 @@ export default abstract class DefaultCommandBase extends Command {
 
   async init(): Promise<void> {
     const { configPath, yes } = this.parse(this.constructor as typeof DefaultCommandBase).flags
-    const configParser = new ConfigParserService()
-    this.appConfig = configParser.loadConfig(configPath) as ReadonlyConfig
+    const configParser = new ConfigParserService(configPath)
+    this.appConfig = configParser.parse() as ReadonlyConfig
     this.logging = LoggingService.withCLIConfig()
     this.logger = this.logging.createLogger('CLI')
     this.autoConfirm = !!(process.env.AUTO_CONFIRM === 'true' || parseInt(process.env.AUTO_CONFIRM || '') || yes)

distributor-node/src/schemas/configSchema.ts

@@ -31,10 +31,6 @@ export const configSchema: JSONSchema4 = {
           description: 'Joystream node websocket api uri (for example: ws://localhost:9944)',
           type: 'string',
         },
-        elasticSearch: {
-          description: 'Elasticsearch uri used for submitting the distributor node logs (if enabled via `log.elastic`)',
-          type: 'string',
-        },
       },
     },
     directories: {
@@ -52,32 +48,92 @@ export const configSchema: JSONSchema4 = {
             'Path to a directory where information about the current cache state will be stored (LRU-SP cache data, stored assets mime types etc.)',
           type: 'string',
         },
-        logs: {
-          description:
-            'Path to a directory where logs will be stored if logging to a file was enabled (via `log.file`).',
-          type: 'string',
-        },
       },
     },
-    log: {
+    logs: {
       type: 'object',
       additionalProperties: false,
-      description: 'Specifies minimum log levels by supported log outputs',
+      description: 'Specifies the logging configuration',
       properties: {
         file: {
-          description: 'Minimum level of logs written to a file specified in `directories.logs`',
-          type: 'string',
-          enum: [...Object.keys(winston.config.npm.levels), 'off'],
+          oneOf: [
+            {
+              type: 'object',
+              additionalProperties: false,
+              required: ['level', 'path'],
+              description: 'File logging options',
+              properties: {
+                level: { $ref: '#/definitions/logLevel' },
+                path: {
+                  description: 'Path where the logs will be stored (absolute or relative to config file)',
+                  type: 'string',
+                },
+                maxFiles: {
+                  description: 'Maximum number of log files to store',
+                  type: 'integer',
+                  minimum: 1,
+                },
+                maxSize: {
+                  description: 'Maximum size of a single log file in bytes',
+                  type: 'integer',
+                  minimum: 1024,
+                },
+                frequency: {
+                  description: 'The frequency of creating new log files (regardless of maxSize)',
+                  default: 'daily',
+                  type: 'string',
+                  enum: ['yearly', 'monthly', 'daily', 'hourly'],
+                },
+                archive: {
+                  description: 'Whether to archive old logs',
+                  default: false,
+                  type: 'boolean',
+                },
+              },
+            },
+            {
+              type: 'string',
+              enum: ['off'],
+            },
+          ],
         },
         console: {
-          description: 'Minimum level of logs outputted to a console',
-          type: 'string',
-          enum: [...Object.keys(winston.config.npm.levels), 'off'],
+          oneOf: [
+            {
+              type: 'object',
+              additionalProperties: false,
+              required: ['level'],
+              description: 'Console logging options',
+              properties: {
+                level: { $ref: '#/definitions/logLevel' },
+              },
+            },
+            {
+              type: 'string',
+              enum: ['off'],
+            },
+          ],
         },
         elastic: {
-          description: 'Minimum level of logs sent to elasticsearch endpoint specified in `endpoints.elasticSearch`',
-          type: 'string',
-          enum: [...Object.keys(winston.config.npm.levels), 'off'],
+          oneOf: [
+            {
+              type: 'object',
+              additionalProperties: false,
+              required: ['level', 'endpoint'],
+              description: 'Elasticsearch logging options',
+              properties: {
+                level: { $ref: '#/definitions/logLevel' },
+                endpoint: {
+                  description: 'Elastichsearch endpoint to push the logs to (for example: http://localhost:9200)',
+                  type: 'string',
+                },
+              },
+            },
+            {
+              type: 'string',
+              enum: ['off'],
+            },
+          ],
         },
       },
     },
@@ -228,6 +284,13 @@ export const configSchema: JSONSchema4 = {
       minimum: 0,
     },
   },
+  definitions: {
+    logLevel: {
+      description: 'Minimum level of logs sent to this output',
+      type: 'string',
+      enum: [...Object.keys(winston.config.npm.levels)],
+    },
+  },
 }
 
 export default configSchema

distributor-node/src/schemas/scripts/generateTypes.ts

@@ -7,7 +7,7 @@ import { schemas } from '..'
 const prettierConfig = require('@joystream/prettier-config')
 
 Object.entries(schemas).forEach(([schemaKey, schema]) => {
-  compile(schema, `${schemaKey}Json`, { style: prettierConfig })
+  compile(schema, `${schemaKey}Json`, { style: prettierConfig, ignoreMinAndMaxItems: true })
     .then((output) => fs.writeFileSync(path.resolve(__dirname, `../../types/generated/${schemaKey}Json.d.ts`), output))
     .catch(console.error)
 })

distributor-node/src/services/logging/LoggingService.ts

@@ -6,6 +6,8 @@ import { blake2AsHex } from '@polkadot/util-crypto'
 import { Format } from 'logform'
 import stringify from 'fast-safe-stringify'
 import NodeCache from 'node-cache'
+import path from 'path'
+import 'winston-daily-rotate-file'
 
 const cliColors = {
   error: 'red',
@@ -74,40 +76,44 @@ export class LoggingService {
     const transports: winston.LoggerOptions['transports'] = []
 
     let esTransport: ElasticsearchTransport | undefined
-    if (config.log?.elastic && config.log.elastic !== 'off') {
-      if (!config.endpoints.elasticSearch) {
-        throw new Error('config.endpoints.elasticSearch must be provided when elasticSeach logging is enabled!')
-      }
+    if (config.logs?.elastic && config.logs.elastic !== 'off') {
       esTransport = new ElasticsearchTransport({
         index: 'distributor-node',
-        level: config.log.elastic,
+        level: config.logs.elastic.level,
         format: winston.format.combine(pauseFormat({ id: 'es' }), escFormat()),
         flushInterval: 5000,
         source: config.id,
         clientOpts: {
           node: {
-            url: new URL(config.endpoints.elasticSearch),
+            url: new URL(config.logs.elastic.endpoint),
           },
         },
       })
       transports.push(esTransport)
     }
 
-    if (config.log?.file && config.log.file !== 'off') {
-      if (!config.directories.logs) {
-        throw new Error('config.directories.logs must be provided when file logging is enabled!')
+    if (config.logs?.file && config.logs.file !== 'off') {
+      const datePatternByFrequency = {
+        yearly: 'YYYY',
+        monthly: 'YYYY-MM',
+        daily: 'YYYY-MM-DD',
+        hourly: 'YYYY-MM-DD-HH',
       }
-      const fileTransport = new winston.transports.File({
-        filename: `${config.directories.logs}/logs.json`,
-        level: config.log.file,
+      const fileTransport = new winston.transports.DailyRotateFile({
+        filename: path.join(config.logs.file.path, 'argus-%DATE%.log'),
+        datePattern: datePatternByFrequency[config.logs.file.frequency || 'daily'],
+        zippedArchive: config.logs.file.archive,
+        maxSize: config.logs.file.maxSize,
+        maxFiles: config.logs.file.maxFiles,
+        level: config.logs.file.level,
         format: winston.format.combine(pauseFormat({ id: 'file' }), escFormat()),
       })
       transports.push(fileTransport)
     }
 
-    if (config.log?.console && config.log.console !== 'off') {
+    if (config.logs?.console && config.logs.console !== 'off') {
       const consoleTransport = new winston.transports.Console({
-        level: config.log.console,
+        level: config.logs.console.level,
         format: winston.format.combine(pauseFormat({ id: 'cli' }), cliFormat),
       })
       transports.push(consoleTransport)

distributor-node/src/services/parsers/ConfigParserService.ts

@@ -11,22 +11,24 @@ const MIN_CACHE_SIZE = '20G'
 const MIN_MAX_CACHED_ITEM_SIZE = '1M'
 
 export class ConfigParserService {
-  validator: ValidationService
+  private configPath: string
+  private validator: ValidationService
 
-  constructor() {
+  constructor(configPath: string) {
     this.validator = new ValidationService()
+    this.configPath = configPath
   }
 
-  public resolveConfigDirectoryPaths(paths: Config['directories'], configFilePath: string): Config['directories'] {
-    return _.mapValues(paths, (v) =>
-      typeof v === 'string' ? path.resolve(path.dirname(configFilePath), v) : v
-    ) as Config['directories']
+  public resolvePath(p: string): string {
+    return path.resolve(path.dirname(this.configPath), p)
   }
 
-  public resolveConfigKeysPaths(keys: Config['keys'], configFilePath: string): Config['keys'] {
-    return keys.map((k) =>
-      'keyfile' in k ? { keyfile: path.resolve(path.dirname(configFilePath), k.keyfile) } : k
-    ) as Config['keys']
+  public resolveConfigDirectoryPaths(paths: Config['directories']): Config['directories'] {
+    return _.mapValues(paths, (p) => this.resolvePath(p))
+  }
+
+  public resolveConfigKeysPaths(keys: Config['keys']): Config['keys'] {
+    return keys.map((k) => ('keyfile' in k ? { keyfile: this.resolvePath(k.keyfile) } : k))
   }
 
   private parseBytesize(bytesize: string) {
@@ -120,7 +122,8 @@ export class ConfigParserService {
       })
   }
 
-  public loadConfig(configPath: string): Config {
+  public parse(): Config {
+    const { configPath } = this
     let inputConfig: Record<string, unknown> = {}
     // Try to load config from file if exists
     if (fs.existsSync(configPath)) {
@@ -141,13 +144,14 @@ export class ConfigParserService {
     const configJson = this.validator.validate('Config', inputConfig)
 
     // Normalize values
-    const directories = this.resolveConfigDirectoryPaths(configJson.directories, configPath)
-    const keys = this.resolveConfigKeysPaths(configJson.keys, configPath)
+    const directories = this.resolveConfigDirectoryPaths(configJson.directories)
+    const keys = this.resolveConfigKeysPaths(configJson.keys)
     const storageLimit = this.parseBytesize(configJson.limits.storage)
     const maxCachedItemSize = configJson.limits.maxCachedItemSize
       ? this.parseBytesize(configJson.limits.maxCachedItemSize)
       : undefined
 
+    // Additional validation:
     if (storageLimit < this.parseBytesize(MIN_CACHE_SIZE)) {
       throw new Error(`Config.limits.storage should be at least ${MIN_CACHE_SIZE}!`)
     }

distributor-node/src/types/generated/ConfigJson.d.ts

@@ -5,10 +5,14 @@
  * and run json-schema-to-typescript to regenerate this file.
  */
 
+/**
+ * Minimum level of logs sent to this output
+ */
+export type LogLevel = 'error' | 'warn' | 'info' | 'http' | 'verbose' | 'debug' | 'silly'
 /**
  * List of distribution bucket ids
  */
-export type BucketIds = [number, ...number[]]
+export type BucketIds = number[]
 /**
  * Distribute all buckets assigned to worker specified in `workerId`
  */
@@ -34,10 +38,6 @@ export interface DistributorNodeConfiguration {
      * Joystream node websocket api uri (for example: ws://localhost:9944)
      */
     joystreamNodeWs: string
-    /**
-     * Elasticsearch uri used for submitting the distributor node logs (if enabled via `log.elastic`)
-     */
-    elasticSearch?: string
   }
   /**
    * Specifies paths where node's data will be stored
@@ -51,27 +51,50 @@ export interface DistributorNodeConfiguration {
      * Path to a directory where information about the current cache state will be stored (LRU-SP cache data, stored assets mime types etc.)
      */
     cacheState: string
-    /**
-     * Path to a directory where logs will be stored if logging to a file was enabled (via `log.file`).
-     */
-    logs?: string
   }
   /**
-   * Specifies minimum log levels by supported log outputs
+   * Specifies the logging configuration
    */
-  log?: {
-    /**
-     * Minimum level of logs written to a file specified in `directories.logs`
-     */
-    file?: 'error' | 'warn' | 'info' | 'http' | 'verbose' | 'debug' | 'silly' | 'off'
-    /**
-     * Minimum level of logs outputted to a console
-     */
-    console?: 'error' | 'warn' | 'info' | 'http' | 'verbose' | 'debug' | 'silly' | 'off'
-    /**
-     * Minimum level of logs sent to elasticsearch endpoint specified in `endpoints.elasticSearch`
-     */
-    elastic?: 'error' | 'warn' | 'info' | 'http' | 'verbose' | 'debug' | 'silly' | 'off'
+  logs?: {
+    file?:
+      | {
+          level: LogLevel
+          /**
+           * Path where the logs will be stored (absolute or relative to config file)
+           */
+          path: string
+          /**
+           * Maximum number of log files to create
+           */
+          maxFiles?: number
+          /**
+           * Maximum size of a single log file in bytes
+           */
+          maxSize?: number
+          /**
+           * The frequency of creating new log files (regardless of maxSize)
+           */
+          frequency?: 'yearly' | 'monthly' | 'daily' | 'hourly'
+          /**
+           * Whether to archive old logs
+           */
+          archive?: boolean
+        }
+      | 'off'
+    console?:
+      | {
+          level: LogLevel
+        }
+      | 'off'
+    elastic?:
+      | {
+          level: LogLevel
+          /**
+           * Elastichsearch endpoint to push the logs to (for example: http://localhost:9200)
+           */
+          endpoint: string
+        }
+      | 'off'
   }
   /**
    * Specifies node limits w.r.t. storage, outbound connections etc.
@@ -130,7 +153,7 @@ export interface DistributorNodeConfiguration {
   /**
    * Specifies the keys available within distributor node CLI.
    */
-  keys: [SubstrateUri | MnemonicPhrase | JSONBackupFile, ...(SubstrateUri | MnemonicPhrase | JSONBackupFile)[]]
+  keys: (SubstrateUri | MnemonicPhrase | JSONBackupFile)[]
   /**
    * Specifies the buckets distributed by the node
    */

docker-compose.yml

@@ -67,13 +67,11 @@ services:
     #   JOYSTREAM_DISTRIBUTOR__ID: node-id
     #   JOYSTREAM_DISTRIBUTOR__ENDPOINTS__QUERY_NODE: qn-endpoint
     #   JOYSTREAM_DISTRIBUTOR__ENDPOINTS__JOYSTREAM_NODE_WS: sn-endpoint
-    #   JOYSTREAM_DISTRIBUTOR__ENDPOINTS__ELASTIC_SEARCH: es-endpoint
     #   JOYSTREAM_DISTRIBUTOR__DIRECTORIES__ASSETS: assets-dir
     #   JOYSTREAM_DISTRIBUTOR__DIRECTORIES__CACHE_STATE: cache-state-dir
-    #   JOYSTREAM_DISTRIBUTOR__DIRECTORIES__LOGS: logs-dir
-    #   JOYSTREAM_DISTRIBUTOR__LOG__CONSOLE: "off"
-    #   JOYSTREAM_DISTRIBUTOR__LOG__FILE: "off"
-    #   JOYSTREAM_DISTRIBUTOR__LOG__ELASTIC: "off"
+    #   JOYSTREAM_DISTRIBUTOR__LOGS__CONSOLE: "off"
+    #   JOYSTREAM_DISTRIBUTOR__LOGS__FILE: "{\"level\":\"debug\",\"path\":\"/tmp\"}"
+    #   JOYSTREAM_DISTRIBUTOR__LOGS__ELASTIC: "off"
     #   JOYSTREAM_DISTRIBUTOR__LIMITS__STORAGE: 50G
     #   JOYSTREAM_DISTRIBUTOR__PORT: 1234
     #   JOYSTREAM_DISTRIBUTOR__KEYS: "[{\"suri\":\"//Bob\"}]"

yarn.lock

@@ -14481,6 +14481,13 @@ file-selector@^0.2.2:
   dependencies:
     tslib "^2.0.3"
 
+file-stream-rotator@^0.5.7:
+  version "0.5.7"
+  resolved "https://registry.yarnpkg.com/file-stream-rotator/-/file-stream-rotator-0.5.7.tgz#868a2e5966f7640a17dd86eda0e4467c089f6286"
+  integrity sha512-VYb3HZ/GiAGUCrfeakO8Mp54YGswNUHvL7P09WQcXAJNSj3iQ5QraYSp3cIn1MUyw6uzfgN/EFOarCNa4JvUHQ==
+  dependencies:
+    moment "^2.11.2"
+
 file-system-cache@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/file-system-cache/-/file-system-cache-1.0.5.tgz#84259b36a2bbb8d3d6eb1021d3132ffe64cfff4f"
@@ -22113,7 +22120,7 @@ module-lookup-amd@^6.1.0:
     requirejs "^2.3.5"
     requirejs-config-file "^3.1.1"
 
-moment@^2.10.2, moment@^2.22.1, moment@^2.24.0:
+moment@^2.10.2, moment@^2.11.2, moment@^2.22.1, moment@^2.24.0:
   version "2.29.1"
   resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3"
   integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==
@@ -23258,6 +23265,11 @@ object-hash@2.1.1:
   resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.1.1.tgz#9447d0279b4fcf80cff3259bf66a1dc73afabe09"
   integrity sha512-VOJmgmS+7wvXf8CjbQmimtCnEx3IAoLxI3fp2fbWehxrWBcAQFbk+vcwb6vzR0VZv/eNCJ/27j151ZTwqW/JeQ==
 
+object-hash@^2.0.1:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.2.0.tgz#5ad518581eefc443bd763472b8ff2e9c2c0d54a5"
+  integrity sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==
+
 object-identity-map@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/object-identity-map/-/object-identity-map-1.0.2.tgz#2b4213a4285ca3a8cd2e696782c9964f887524e7"
@@ -32089,6 +32101,16 @@ windows-release@^3.1.0:
   dependencies:
     execa "^1.0.0"
 
+winston-daily-rotate-file@^4.5.5:
+  version "4.5.5"
+  resolved "https://registry.yarnpkg.com/winston-daily-rotate-file/-/winston-daily-rotate-file-4.5.5.tgz#cfa3a89f4eb0e4126917592b375759b772bcd972"
+  integrity sha512-ds0WahIjiDhKCiMXmY799pDBW+58ByqIBtUcsqr4oDoXrAI3Zn+hbgFdUxzMfqA93OG0mPLYVMiotqTgE/WeWQ==
+  dependencies:
+    file-stream-rotator "^0.5.7"
+    object-hash "^2.0.1"
+    triple-beam "^1.3.0"
+    winston-transport "^4.4.0"
+
 winston-elasticsearch@^0.15.8:
   version "0.15.8"
   resolved "https://registry.yarnpkg.com/winston-elasticsearch/-/winston-elasticsearch-0.15.8.tgz#4ec0fa295c75187d5992a074e221c793c76f7b02"