Browse Source

Merge pull request #2990 from Lezek123/storage_node_v2_file_logging

Giza: Storage node - folder structure, readme, docker, file logging.
Mokhtar Naamani 3 years ago
parent
commit
1b8fe2dcd8
76 changed files with 255 additions and 125 deletions
  1. 1 1
      build-npm-packages.sh
  2. 4 4
      colossus.Dockerfile
  3. 1 0
      docker-compose.yml
  4. 0 2
      package.json
  5. 0 0
      storage-node/.eslintignore
  6. 0 0
      storage-node/.eslintrc.js
  7. 0 0
      storage-node/.gitignore
  8. 0 0
      storage-node/.prettierignore
  9. 55 36
      storage-node/README.md
  10. 0 0
      storage-node/bin/run
  11. 0 0
      storage-node/bin/run.cmd
  12. 4 3
      storage-node/package.json
  13. 0 0
      storage-node/scripts/create-auth-request-signature.ts
  14. 0 0
      storage-node/scripts/init-dev-bucket.sh
  15. 0 0
      storage-node/scripts/operatorMetadata.json
  16. 0 0
      storage-node/scripts/run-all-commands.sh
  17. 0 0
      storage-node/src/api-spec/openapi.yaml
  18. 0 0
      storage-node/src/command-base/ApiCommandBase.ts
  19. 0 0
      storage-node/src/command-base/ExitCodes.ts
  20. 0 0
      storage-node/src/commands/dev/init.ts
  21. 3 4
      storage-node/src/commands/dev/multihash.ts
  22. 0 0
      storage-node/src/commands/dev/sync.ts
  23. 0 0
      storage-node/src/commands/dev/upload.ts
  24. 0 0
      storage-node/src/commands/dev/verify-bag-id.ts
  25. 0 0
      storage-node/src/commands/leader/cancel-invite.ts
  26. 4 4
      storage-node/src/commands/leader/create-bucket.ts
  27. 0 0
      storage-node/src/commands/leader/delete-bucket.ts
  28. 0 0
      storage-node/src/commands/leader/invite-operator.ts
  29. 0 0
      storage-node/src/commands/leader/remove-operator.ts
  30. 0 0
      storage-node/src/commands/leader/set-bucket-limits.ts
  31. 0 0
      storage-node/src/commands/leader/set-global-uploading-status.ts
  32. 0 0
      storage-node/src/commands/leader/update-bag-limit.ts
  33. 0 0
      storage-node/src/commands/leader/update-bag.ts
  34. 0 0
      storage-node/src/commands/leader/update-blacklist.ts
  35. 0 0
      storage-node/src/commands/leader/update-bucket-status.ts
  36. 0 0
      storage-node/src/commands/leader/update-data-fee.ts
  37. 0 0
      storage-node/src/commands/leader/update-dynamic-bag-policy.ts
  38. 0 0
      storage-node/src/commands/leader/update-voucher-limits.ts
  39. 0 0
      storage-node/src/commands/operator/accept-invitation.ts
  40. 0 0
      storage-node/src/commands/operator/set-metadata.ts
  41. 34 5
      storage-node/src/commands/server.ts
  42. 0 0
      storage-node/src/index.ts
  43. 0 0
      storage-node/src/services/caching/localDataObjects.ts
  44. 0 0
      storage-node/src/services/caching/newUploads.ts
  45. 0 0
      storage-node/src/services/caching/tokenNonceKeeper.ts
  46. 0 0
      storage-node/src/services/helpers/auth.ts
  47. 0 0
      storage-node/src/services/helpers/bagTypes.ts
  48. 0 0
      storage-node/src/services/helpers/fileInfo.ts
  49. 0 0
      storage-node/src/services/helpers/hashing.ts
  50. 9 0
      storage-node/src/services/helpers/stdout.ts
  51. 106 52
      storage-node/src/services/logger.ts
  52. 0 0
      storage-node/src/services/metadata/generateTypes.ts
  53. 0 0
      storage-node/src/services/metadata/generated/OperatorMetadataJson.d.ts
  54. 0 0
      storage-node/src/services/metadata/schemas/index.ts
  55. 0 0
      storage-node/src/services/metadata/schemas/operatorMetadataSchema.ts
  56. 0 0
      storage-node/src/services/metadata/validationService.ts
  57. 0 0
      storage-node/src/services/queryNode/api.ts
  58. 0 0
      storage-node/src/services/queryNode/codegen.yml
  59. 0 0
      storage-node/src/services/queryNode/generated/queries.ts
  60. 0 0
      storage-node/src/services/queryNode/generated/schema.ts
  61. 0 0
      storage-node/src/services/queryNode/queries/queries.graphql
  62. 0 0
      storage-node/src/services/runtime/accounts.ts
  63. 0 0
      storage-node/src/services/runtime/api.ts
  64. 0 0
      storage-node/src/services/runtime/extrinsics.ts
  65. 0 0
      storage-node/src/services/runtime/hireLead.ts
  66. 0 0
      storage-node/src/services/sync/remoteStorageData.ts
  67. 0 0
      storage-node/src/services/sync/storageObligations.ts
  68. 0 0
      storage-node/src/services/sync/synchronizer.ts
  69. 0 0
      storage-node/src/services/sync/tasks.ts
  70. 0 0
      storage-node/src/services/sync/workingProcess.ts
  71. 11 3
      storage-node/src/services/webApi/app.ts
  72. 0 10
      storage-node/src/services/webApi/controllers/common.ts
  73. 0 0
      storage-node/src/services/webApi/controllers/filesApi.ts
  74. 0 0
      storage-node/src/services/webApi/controllers/stateApi.ts
  75. 0 0
      storage-node/tsconfig.json
  76. 23 1
      yarn.lock

+ 1 - 1
build-npm-packages.sh

@@ -8,5 +8,5 @@ yarn workspace @joystream/metadata-protobuf build
 yarn workspace query-node-root build
 yarn workspace @joystream/cli build
 yarn workspace storage-node build
-yarn workspace storage-node-v2 build
+yarn workspace storage-node build
 yarn workspace pioneer build

+ 4 - 4
colossus.Dockerfile

@@ -7,10 +7,10 @@ RUN yarn --frozen-lockfile
 
 RUN yarn workspace @joystream/types build
 RUN yarn workspace @joystream/metadata-protobuf build
-RUN yarn workspace storage-node-v2 build
+RUN yarn workspace storage-node build
 
 # Use these volumes to persist uploading data and to pass the keyfile.
-VOLUME ["/data", "/keystore"]
+VOLUME ["/data", "/keystore", "/logs"]
 
 # Required variables
 ENV WS_PROVIDER_ENDPOINT_URI=ws://not-set
@@ -31,5 +31,5 @@ ENV ACCOUNT_URI=
 # Colossus node port
 EXPOSE ${COLOSSUS_PORT}
 
-WORKDIR /joystream/storage-node-v2
-ENTRYPOINT yarn storage-node server --queryNodeEndpoint ${QUERY_NODE_ENDPOINT} --port ${COLOSSUS_PORT} --uploads /data --worker ${WORKER_ID} --apiUrl ${WS_PROVIDER_ENDPOINT_URI} --sync --syncInterval=${SYNC_INTERVAL} --keyFile=${ACCOUNT_KEYFILE} --elasticSearchEndpoint=${ELASTIC_SEARCH_ENDPOINT}
+WORKDIR /joystream/storage-node
+ENTRYPOINT yarn storage-node server --queryNodeEndpoint ${QUERY_NODE_ENDPOINT} --port ${COLOSSUS_PORT} --uploads /data --worker ${WORKER_ID} --apiUrl ${WS_PROVIDER_ENDPOINT_URI} --sync --syncInterval=${SYNC_INTERVAL} --keyFile=${ACCOUNT_KEYFILE} --elasticSearchEndpoint=${ELASTIC_SEARCH_ENDPOINT} --logFileName=/logs/log.txt

+ 1 - 0
docker-compose.yml

@@ -29,6 +29,7 @@ services:
     volumes:
       - /data
       - /keystore
+      - /logs
     ports:
       - '127.0.0.1:3333:3333'
     env_file:

+ 0 - 2
package.json

@@ -18,9 +18,7 @@
     "tests/network-tests",
     "cli",
     "types",
-    "storage-node-v2",
     "storage-node",
-    "storage-node/packages/*",
     "devops/eslint-config",
     "devops/prettier-config",
     "pioneer",

+ 0 - 0
storage-node-v2/.eslintignore → storage-node/.eslintignore


+ 0 - 0
storage-node-v2/.eslintrc.js → storage-node/.eslintrc.js


+ 0 - 0
storage-node-v2/.gitignore → storage-node/.gitignore


+ 0 - 0
storage-node-v2/.prettierignore → storage-node/.prettierignore


+ 55 - 36
storage-node-v2/README.md → storage-node/README.md

@@ -111,10 +111,10 @@ bash
 yarn
 yarn workspace @joystream/types build
 yarn workspace @joystream/metadata-protobuf build
-yarn workspace storage-node-v2 build
+yarn workspace storage-node build
 
 # Verify installation
-cd storage-node-v2
+cd storage-node
 yarn storage-node version
 ```
 # Usage
@@ -650,16 +650,23 @@ USAGE
   $ storage-node operator:accept-invitation
 
 OPTIONS
-  -h, --help                   show CLI help
-  -i, --bucketId=bucketId      (required) Storage bucket ID
-  -k, --keyFile=keyFile        Key file for the account. Mandatory in non-dev environment.
-  -m, --dev                    Use development mode
-  -p, --password=password      Key file password (optional). Could be overriden by ACCOUNT_PWD environment variable.
-  -u, --apiUrl=apiUrl          [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev environment.
-  -w, --workerId=workerId      (required) Storage operator worker ID
+  -h, --help                                     show CLI help
+  -i, --bucketId=bucketId                        (required) Storage bucket ID
+  -k, --keyFile=keyFile                          Key file for the account. Mandatory in non-dev environment.
+  -m, --dev                                      Use development mode
 
-  -y, --accountUri=accountUri  Account URI (optional). Has a priority over the keyFile and password flags. Could be
-                               overriden by ACCOUNT_URI environment variable.
+  -p, --password=password                        Key file password (optional). Could be overriden by ACCOUNT_PWD
+                                                 environment variable.
+
+  -t, --transactorAccountId=transactorAccountId  (required) Transactor account ID (public key)
+
+  -u, --apiUrl=apiUrl                            [default: ws://localhost:9944] Runtime API URL. Mandatory in non-dev
+                                                 environment.
+
+  -w, --workerId=workerId                        (required) Storage operator worker ID
+
+  -y, --accountUri=accountUri                    Account URI (optional). Has a priority over the keyFile and password
+                                                 flags. Could be overriden by ACCOUNT_URI environment variable.
 ```
 
 _See code: [src/commands/operator/accept-invitation.ts](https://github.com/Joystream/joystream/blob/v2.0.0/src/commands/operator/accept-invitation.ts)_
@@ -698,46 +705,58 @@ USAGE
   $ storage-node server
 
 OPTIONS
-  -d, --uploads=uploads                              (required) Data uploading directory (absolute path).
+  -d, --uploads=uploads                                       (required) Data uploading directory (absolute path).
 
-  -e, --elasticSearchEndpoint=elasticSearchEndpoint  Elasticsearch endpoint (e.g.: http://some.com:8081).
-                                                     Log level could be set using the ELASTIC_LOG_LEVEL enviroment
-                                                     variable.
-                                                     Supported values: warn, error, debug, info. Default:debug
+  -e, --elasticSearchEndpoint=elasticSearchEndpoint           Elasticsearch endpoint (e.g.: http://some.com:8081).
+                                                              Log level could be set using the ELASTIC_LOG_LEVEL
+                                                              enviroment variable.
+                                                              Supported values: warn, error, debug, info. Default:debug
 
-  -h, --help                                         show CLI help
+  -h, --help                                                  show CLI help
 
-  -i, --syncInterval=syncInterval                    [default: 1] Interval between synchronizations (in minutes)
+  -i, --syncInterval=syncInterval                             [default: 1] Interval between synchronizations (in
+                                                              minutes)
 
-  -k, --keyFile=keyFile                              Key file for the account. Mandatory in non-dev environment.
+  -k, --keyFile=keyFile                                       Key file for the account. Mandatory in non-dev
+                                                              environment.
 
-  -m, --dev                                          Use development mode
+  -l, --logFilePath=logFilePath                               Absolute path to the rolling log files.
 
-  -o, --port=port                                    (required) Server port.
+  -m, --dev                                                   Use development mode
 
-  -p, --password=password                            Key file password (optional). Could be overriden by ACCOUNT_PWD
-                                                     environment variable.
+  -n, --logMaxFileNumber=logMaxFileNumber                     [default: 7] Maximum rolling log files number.
 
-  -q, --queryNodeEndpoint=queryNodeEndpoint          (required) [default: http://localhost:8081/graphql] Query node
-                                                     endpoint (e.g.: http://some.com:8081/graphql)
+  -o, --port=port                                             (required) Server port.
 
-  -r, --syncWorkersNumber=syncWorkersNumber          [default: 20] Sync workers number (max async operations in
-                                                     progress).
+  -p, --password=password                                     Key file password (optional). Could be overriden by
+                                                              ACCOUNT_PWD environment variable.
 
-  -s, --sync                                         Enable data synchronization.
+  -q, --queryNodeEndpoint=queryNodeEndpoint                   (required) [default: http://localhost:8081/graphql] Query
+                                                              node endpoint (e.g.: http://some.com:8081/graphql)
 
-  -t, --syncWorkersTimeout=syncWorkersTimeout        [default: 30] Asset downloading timeout for the syncronization (in
-                                                     minutes).
+  -r, --syncWorkersNumber=syncWorkersNumber                   [default: 20] Sync workers number (max async operations in
+                                                              progress).
 
-  -u, --apiUrl=apiUrl                                [default: ws://localhost:9944] Runtime API URL. Mandatory in
-                                                     non-dev environment.
+  -s, --sync                                                  Enable data synchronization.
 
-  -w, --worker=worker                                (required) Storage provider worker ID
+  -t, --syncWorkersTimeout=syncWorkersTimeout                 [default: 30] Asset downloading timeout for the
+                                                              syncronization (in minutes).
 
-  -y, --accountUri=accountUri                        Account URI (optional). Has a priority over the keyFile and
-                                                     password flags. Could be overriden by ACCOUNT_URI environment
-                                                     variable.
+  -u, --apiUrl=apiUrl                                         [default: ws://localhost:9944] Runtime API URL. Mandatory
+                                                              in non-dev environment.
+
+  -w, --worker=worker                                         (required) Storage provider worker ID
+
+  -x, --logMaxFileSize=logMaxFileSize                         [default: 50000000] Maximum rolling log files size in
+                                                              bytes.
+
+  -y, --accountUri=accountUri                                 Account URI (optional). Has a priority over the keyFile
+                                                              and password flags. Could be overriden by ACCOUNT_URI
+                                                              environment variable.
+
+  -z, --logFileChangeFrequency=(yearly|monthly|daily|hourly)  [default: daily] Log files update frequency.
 ```
 
 _See code: [src/commands/server.ts](https://github.com/Joystream/joystream/blob/v2.0.0/src/commands/server.ts)_
+
 <!-- commandsstop -->

+ 0 - 0
storage-node-v2/bin/run → storage-node/bin/run


+ 0 - 0
storage-node-v2/bin/run.cmd → storage-node/bin/run.cmd


+ 4 - 3
storage-node-v2/package.json → storage-node/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "storage-node-v2",
+  "name": "storage-node",
   "description": "Joystream storage subsystem.",
   "version": "2.0.0",
   "author": "Joystream contributors",
@@ -55,7 +55,8 @@
     "url-join": "^4.0.1",
     "uuid": "^8.3.2",
     "winston": "^3.3.3",
-    "winston-elasticsearch": "^0.15.8"
+    "winston-elasticsearch": "^0.15.8",
+    "winston-daily-rotate-file": "^4.5.5"
   },
   "devDependencies": {
     "@graphql-codegen/cli": "^1.21.4",
@@ -129,7 +130,7 @@
   "repository": {
     "type": "git",
     "url": "https://github.com/Joystream/joystream",
-    "directory": "storage-node-v2"
+    "directory": "storage-node"
   },
   "scripts": {
     "postpack": "rm -f oclif.manifest.json",

+ 0 - 0
storage-node-v2/scripts/create-auth-request-signature.ts → storage-node/scripts/create-auth-request-signature.ts


+ 0 - 0
storage-node-v2/scripts/init-dev-bucket.sh → storage-node/scripts/init-dev-bucket.sh


+ 0 - 0
storage-node-v2/scripts/operatorMetadata.json → storage-node/scripts/operatorMetadata.json


+ 0 - 0
storage-node-v2/scripts/run-all-commands.sh → storage-node/scripts/run-all-commands.sh


+ 0 - 0
storage-node-v2/src/api-spec/openapi.yaml → storage-node/src/api-spec/openapi.yaml


+ 0 - 0
storage-node-v2/src/command-base/ApiCommandBase.ts → storage-node/src/command-base/ApiCommandBase.ts


+ 0 - 0
storage-node-v2/src/command-base/ExitCodes.ts → storage-node/src/command-base/ExitCodes.ts


+ 0 - 0
storage-node-v2/src/commands/dev/init.ts → storage-node/src/commands/dev/init.ts


+ 3 - 4
storage-node-v2/src/commands/dev/multihash.ts → storage-node/src/commands/dev/multihash.ts

@@ -1,6 +1,7 @@
 import { Command, flags } from '@oclif/command'
 import { hashFile } from '../../services/helpers/hashing'
-import logger, { createStdConsoleLogger } from '../../services/logger'
+import logger from '../../services/logger'
+import { print } from '../../services/helpers/stdout'
 
 /**
  * CLI command:
@@ -32,8 +33,6 @@ export default class DevMultihash extends Command {
 
     logger.info(`Hash: ${multi}`)
 
-    const stdConsoleLogger = createStdConsoleLogger()
-
-    stdConsoleLogger.info(multi)
+    print(multi)
   }
 }

+ 0 - 0
storage-node-v2/src/commands/dev/sync.ts → storage-node/src/commands/dev/sync.ts


+ 0 - 0
storage-node-v2/src/commands/dev/upload.ts → storage-node/src/commands/dev/upload.ts


+ 0 - 0
storage-node-v2/src/commands/dev/verify-bag-id.ts → storage-node/src/commands/dev/verify-bag-id.ts


+ 0 - 0
storage-node-v2/src/commands/leader/cancel-invite.ts → storage-node/src/commands/leader/cancel-invite.ts


+ 4 - 4
storage-node-v2/src/commands/leader/create-bucket.ts → storage-node/src/commands/leader/create-bucket.ts

@@ -1,7 +1,8 @@
 import { createStorageBucket } from '../../services/runtime/extrinsics'
 import { flags } from '@oclif/command'
 import ApiCommandBase from '../../command-base/ApiCommandBase'
-import logger, { createStdConsoleLogger } from '../../services/logger'
+import logger from '../../services/logger'
+import { print } from '../../services/helpers/stdout'
 
 /**
  * CLI command:
@@ -56,9 +57,8 @@ export default class LeaderCreateBucket extends ApiCommandBase {
       objectNumber
     )
     if (success) {
-      const stdConsoleLogger = createStdConsoleLogger()
-
-      stdConsoleLogger.info(bucketId)
+      const castedBucketId = bucketId as number
+      print(castedBucketId.toString())
     }
     this.exitAfterRuntimeCall(success)
   }

+ 0 - 0
storage-node-v2/src/commands/leader/delete-bucket.ts → storage-node/src/commands/leader/delete-bucket.ts


+ 0 - 0
storage-node-v2/src/commands/leader/invite-operator.ts → storage-node/src/commands/leader/invite-operator.ts


+ 0 - 0
storage-node-v2/src/commands/leader/remove-operator.ts → storage-node/src/commands/leader/remove-operator.ts


+ 0 - 0
storage-node-v2/src/commands/leader/set-bucket-limits.ts → storage-node/src/commands/leader/set-bucket-limits.ts


+ 0 - 0
storage-node-v2/src/commands/leader/set-global-uploading-status.ts → storage-node/src/commands/leader/set-global-uploading-status.ts


+ 0 - 0
storage-node-v2/src/commands/leader/update-bag-limit.ts → storage-node/src/commands/leader/update-bag-limit.ts


+ 0 - 0
storage-node-v2/src/commands/leader/update-bag.ts → storage-node/src/commands/leader/update-bag.ts


+ 0 - 0
storage-node-v2/src/commands/leader/update-blacklist.ts → storage-node/src/commands/leader/update-blacklist.ts


+ 0 - 0
storage-node-v2/src/commands/leader/update-bucket-status.ts → storage-node/src/commands/leader/update-bucket-status.ts


+ 0 - 0
storage-node-v2/src/commands/leader/update-data-fee.ts → storage-node/src/commands/leader/update-data-fee.ts


+ 0 - 0
storage-node-v2/src/commands/leader/update-dynamic-bag-policy.ts → storage-node/src/commands/leader/update-dynamic-bag-policy.ts


+ 0 - 0
storage-node-v2/src/commands/leader/update-voucher-limits.ts → storage-node/src/commands/leader/update-voucher-limits.ts


+ 0 - 0
storage-node-v2/src/commands/operator/accept-invitation.ts → storage-node/src/commands/operator/accept-invitation.ts


+ 0 - 0
storage-node-v2/src/commands/operator/set-metadata.ts → storage-node/src/commands/operator/set-metadata.ts


+ 34 - 5
storage-node-v2/src/commands/server.ts → storage-node/src/commands/server.ts

@@ -1,7 +1,7 @@
 import { flags } from '@oclif/command'
 import { createApp } from '../services/webApi/app'
 import ApiCommandBase from '../command-base/ApiCommandBase'
-import logger, { initElasticLogger } from '../services/logger'
+import logger, { initNewLogger, DatePatternByFrequency, Frequency } from '../services/logger'
 import { loadDataObjectIdCache } from '../services/caching/localDataObjects'
 import { ApiPromise } from '@polkadot/api'
 import { performSync, TempDirName } from '../services/sync/synchronizer'
@@ -75,6 +75,30 @@ export default class Server extends ApiCommandBase {
 Log level could be set using the ELASTIC_LOG_LEVEL enviroment variable.
 Supported values: warn, error, debug, info. Default:debug`,
     }),
+    logFilePath: flags.string({
+      char: 'l',
+      required: false,
+      description: `Absolute path to the rolling log files.`,
+    }),
+    logMaxFileNumber: flags.integer({
+      char: 'n',
+      required: false,
+      default: 7,
+      description: `Maximum rolling log files number.`,
+    }),
+    logMaxFileSize: flags.integer({
+      char: 'x',
+      required: false,
+      default: 50000000,
+      description: `Maximum rolling log files size in bytes.`,
+    }),
+    logFileChangeFrequency: flags.enum({
+      char: 'z',
+      description: `Log files update frequency.`,
+      options: Object.keys(DatePatternByFrequency),
+      default: 'daily',
+      required: false,
+    }),
     ...ApiCommandBase.flags,
   }
 
@@ -89,8 +113,15 @@ Supported values: warn, error, debug, info. Default:debug`,
       await loadDataObjectIdCache(flags.uploads, TempDirName)
     }
 
-    if (!_.isEmpty(flags.elasticSearchEndpoint)) {
-      initElasticLogger(logSource, flags.elasticSearchEndpoint ?? '')
+    if (!_.isEmpty(flags.elasticSearchEndpoint) || !_.isEmpty(flags.logFilePath)) {
+      initNewLogger({
+        elasticSearchlogSource: logSource,
+        elasticSearchEndpoint: flags.elasticSearchEndpoint,
+        filePath: flags.logFilePath,
+        maxFileNumber: flags.logMaxFileNumber,
+        maxFileSize: flags.logMaxFileSize,
+        fileFrequency: flags.logFileChangeFrequency as Frequency, // type checked in the flags.enum
+      })
     }
 
     logger.info(`Query node endpoint set: ${flags.queryNodeEndpoint}`)
@@ -138,8 +169,6 @@ Supported values: warn, error, debug, info. Default:debug`,
         process: this.config,
         queryNodeEndpoint: flags.queryNodeEndpoint,
         enableUploadingAuth: false,
-        elasticSearchEndpoint: flags.elasticSearchEndpoint,
-        logSource,
       })
       logger.info(`Listening on http://localhost:${port}`)
       app.listen(port)

+ 0 - 0
storage-node-v2/src/index.ts → storage-node/src/index.ts


+ 0 - 0
storage-node-v2/src/services/caching/localDataObjects.ts → storage-node/src/services/caching/localDataObjects.ts


+ 0 - 0
storage-node-v2/src/services/caching/newUploads.ts → storage-node/src/services/caching/newUploads.ts


+ 0 - 0
storage-node-v2/src/services/caching/tokenNonceKeeper.ts → storage-node/src/services/caching/tokenNonceKeeper.ts


+ 0 - 0
storage-node-v2/src/services/helpers/auth.ts → storage-node/src/services/helpers/auth.ts


+ 0 - 0
storage-node-v2/src/services/helpers/bagTypes.ts → storage-node/src/services/helpers/bagTypes.ts


+ 0 - 0
storage-node-v2/src/services/helpers/fileInfo.ts → storage-node/src/services/helpers/fileInfo.ts


+ 0 - 0
storage-node-v2/src/services/helpers/hashing.ts → storage-node/src/services/helpers/hashing.ts


+ 9 - 0
storage-node/src/services/helpers/stdout.ts

@@ -0,0 +1,9 @@
+/**
+ * Prints message to console. We don't use logger in some cases to avoid metadata printing.
+ *
+ * @param msg message to output
+ */
+export function print(msg: string): void {
+  /* eslint-disable no-console */
+  console.log(msg)
+}

+ 106 - 52
storage-node-v2/src/services/logger.ts → storage-node/src/services/logger.ts

@@ -3,6 +3,8 @@ import ecsformat from '@elastic/ecs-winston-format'
 import expressWinston from 'express-winston'
 import { Handler, ErrorRequestHandler } from 'express'
 import { ElasticsearchTransport } from 'winston-elasticsearch'
+import 'winston-daily-rotate-file'
+import path from 'path'
 
 /**
  * Possible log levels.
@@ -83,76 +85,57 @@ const proxy = new Proxy(InnerLogger, {
 export default proxy
 
 /**
- * Creates Express-Winston logger handler.
- * @param logSource - source tag for log entries.
- * @param elasticSearchEndpoint - elastic search engine endpoint (optional).
- * @returns  Express-Winston logger handler
+ * Creates Express-Winston default logger options.
  *
  */
-export function httpLogger(logSource: string, elasticSearchEndpoint?: string): Handler {
-  // ElasticSearch server date format.
-  const elasticDateFormat = 'YYYY-MM-DDTHH:mm:ss'
-
-  const transports: winston.transport[] = [
-    new winston.transports.Console({
-      format: winston.format.combine(winston.format.timestamp({ format: elasticDateFormat }), winston.format.json()),
-    }),
-  ]
-
-  if (elasticSearchEndpoint) {
-    const esTransport = createElasticTransport(logSource, elasticSearchEndpoint)
-    transports.push(esTransport)
+export function createExpressDefaultLoggerOptions(): expressWinston.LoggerOptions {
+  return {
+    winstonInstance: proxy,
+    level: 'http',
   }
+}
 
-  const opts: expressWinston.LoggerOptions = {
-    transports,
-    meta: true,
-    msg: 'HTTP {{req.method}} {{req.url}}',
-    expressFormat: true,
-    colorize: false,
+/**
+ * Creates Express-Winston error logger options.
+ *
+ */
+export function createExpressErrorLoggerOptions(): expressWinston.LoggerOptions {
+  return {
+    winstonInstance: proxy,
+    level: 'error',
+    msg: '{{req.method}} {{req.path}}: Error {{res.statusCode}}: {{err.message}}',
   }
-
-  return expressWinston.logger(opts)
 }
 
 /**
  * Creates Express-Winston error logger.
  *
+ * @param options - express winston logger options.
  * @returns  Express-Winston error logger
  *
  */
-export function errorLogger(): ErrorRequestHandler {
-  return expressWinston.errorLogger({
-    transports: [new winston.transports.Console()],
-    format: winston.format.combine(winston.format.json()),
-  })
+export function errorLogger(options: expressWinston.LoggerOptions): ErrorRequestHandler {
+  return expressWinston.errorLogger(options)
 }
 
 /**
- * Creates clean Console Winston logger for standard output.
+ * Creates Express-Winston logger handler.
  *
- * @returns Winston logger
+ * @param options - express winston logger options.
+ * @returns  Express-Winston logger handler
  *
  */
-export function createStdConsoleLogger(): winston.Logger {
-  const format = winston.format.printf((info) => `${info.message}`)
-
-  const transports = [new winston.transports.Console()]
-
-  return winston.createLogger({
-    levels,
-    format,
-    transports,
-  })
+export function httpLogger(options: expressWinston.LoggerOptions): Handler {
+  return expressWinston.logger(options)
 }
+
 /**
- * Creates Winston logger with Elastic search.
- * @param logSource - source tag for log entries.
- * @param elasticSearchEndpoint - elastic search engine endpoint.
+ * Creates Winston logger with ElasticSearch and File transports.
+ * @param customOptions - logger options
  * @returns Winston logger
  *
  */
-function createElasticLogger(logSource: string, elasticSearchEndpoint: string): winston.Logger {
+function createCustomLogger(customOptions: LogConfig): winston.Logger {
   const loggerOptions = createDefaultLoggerOptions()
 
   // Transports
@@ -161,8 +144,19 @@ function createElasticLogger(logSource: string, elasticSearchEndpoint: string):
     transports = Array.isArray(loggerOptions.transports) ? loggerOptions.transports : [loggerOptions.transports]
   }
 
-  const esTransport = createElasticTransport(logSource, elasticSearchEndpoint)
-  transports.push(esTransport)
+  if (customOptions.elasticSearchEndpoint) {
+    transports.push(createElasticTransport(customOptions.elasticSearchlogSource, customOptions.elasticSearchEndpoint))
+  }
+  if (customOptions.filePath) {
+    transports.push(
+      createFileTransport(
+        customOptions.filePath,
+        customOptions.fileFrequency,
+        customOptions.maxFileNumber,
+        customOptions.maxFileSize
+      )
+    )
+  }
 
   // Logger
   const logger = winston.createLogger(loggerOptions)
@@ -180,11 +174,10 @@ function createElasticLogger(logSource: string, elasticSearchEndpoint: string):
 /**
  * Updates the default system logger with elastic search capabilities.
  *
- * @param logSource - source tag for log entries.
- * @param elasticSearchEndpoint - elastic search engine endpoint.
+ * @param customOptions - logger options
  */
-export function initElasticLogger(logSource: string, elasticSearchEndpoint: string): void {
-  InnerLogger = createElasticLogger(logSource, elasticSearchEndpoint)
+export function initNewLogger(options: LogConfig): void {
+  InnerLogger = createCustomLogger(options)
 }
 
 /**
@@ -210,6 +203,67 @@ function createElasticTransport(logSource: string, elasticSearchEndpoint: string
     index: 'storage-node',
     format: ecsformat(),
     source: logSource,
+    retryLimit: 10,
   }
   return new ElasticsearchTransport(esTransportOpts)
 }
+
+/**
+ * Creates winston logger file transport.
+ *
+ * @param fileName - log file path
+ * @param fileFrequency - file frequence (daily,montly, etc.)
+ * @param maxFiles - maximum number of the log files
+ * @param maxSize - maximum log file size
+ * @returns winston file transport
+ */
+function createFileTransport(
+  filepath: string,
+  fileFrequency: Frequency,
+  maxFiles: number,
+  maxSize: number
+): winston.transport {
+  const options = {
+    filename: path.join(filepath, 'colossus-%DATE%.log'),
+    datePattern: DatePatternByFrequency[fileFrequency || 'daily'],
+    maxSize,
+    maxFiles,
+    level: 'debug',
+    format: ecsformat(),
+  }
+
+  return new winston.transports.DailyRotateFile(options)
+}
+
+export const DatePatternByFrequency = {
+  yearly: 'YYYY',
+  monthly: 'YYYY-MM',
+  daily: 'YYYY-MM-DD',
+  hourly: 'YYYY-MM-DD-HH',
+}
+
+/** File frequency for  */
+export type Frequency = keyof typeof DatePatternByFrequency
+
+/**
+ * Configuration for the ElasticSearch and File loggers
+ */
+export type LogConfig = {
+  /** Path to log files */
+  filePath?: string
+
+  /** Maximum log file size */
+  maxFileSize: number
+
+  /** Maximum number of the log files */
+  maxFileNumber: number
+
+  /** Log files update frequency (yearly, monthly, daily, hourly) */
+  fileFrequency: Frequency
+
+  /** Source tag for log entries. */
+  elasticSearchlogSource: string
+
+  /** Elastic search engine endpoint */
+  elasticSearchEndpoint?: string
+}

+ 0 - 0
storage-node-v2/src/services/metadata/generateTypes.ts → storage-node/src/services/metadata/generateTypes.ts


+ 0 - 0
storage-node-v2/src/services/metadata/generated/OperatorMetadataJson.d.ts → storage-node/src/services/metadata/generated/OperatorMetadataJson.d.ts


+ 0 - 0
storage-node-v2/src/services/metadata/schemas/index.ts → storage-node/src/services/metadata/schemas/index.ts


+ 0 - 0
storage-node-v2/src/services/metadata/schemas/operatorMetadataSchema.ts → storage-node/src/services/metadata/schemas/operatorMetadataSchema.ts


+ 0 - 0
storage-node-v2/src/services/metadata/validationService.ts → storage-node/src/services/metadata/validationService.ts


+ 0 - 0
storage-node-v2/src/services/queryNode/api.ts → storage-node/src/services/queryNode/api.ts


+ 0 - 0
storage-node-v2/src/services/queryNode/codegen.yml → storage-node/src/services/queryNode/codegen.yml


+ 0 - 0
storage-node-v2/src/services/queryNode/generated/queries.ts → storage-node/src/services/queryNode/generated/queries.ts


+ 0 - 0
storage-node-v2/src/services/queryNode/generated/schema.ts → storage-node/src/services/queryNode/generated/schema.ts


+ 0 - 0
storage-node-v2/src/services/queryNode/queries/queries.graphql → storage-node/src/services/queryNode/queries/queries.graphql


+ 0 - 0
storage-node-v2/src/services/runtime/accounts.ts → storage-node/src/services/runtime/accounts.ts


+ 0 - 0
storage-node-v2/src/services/runtime/api.ts → storage-node/src/services/runtime/api.ts


+ 0 - 0
storage-node-v2/src/services/runtime/extrinsics.ts → storage-node/src/services/runtime/extrinsics.ts


+ 0 - 0
storage-node-v2/src/services/runtime/hireLead.ts → storage-node/src/services/runtime/hireLead.ts


+ 0 - 0
storage-node-v2/src/services/sync/remoteStorageData.ts → storage-node/src/services/sync/remoteStorageData.ts


+ 0 - 0
storage-node-v2/src/services/sync/storageObligations.ts → storage-node/src/services/sync/storageObligations.ts


+ 0 - 0
storage-node-v2/src/services/sync/synchronizer.ts → storage-node/src/services/sync/synchronizer.ts


+ 0 - 0
storage-node-v2/src/services/sync/tasks.ts → storage-node/src/services/sync/tasks.ts


+ 0 - 0
storage-node-v2/src/services/sync/workingProcess.ts → storage-node/src/services/sync/workingProcess.ts


+ 11 - 3
storage-node-v2/src/services/webApi/app.ts → storage-node/src/services/webApi/app.ts

@@ -8,8 +8,13 @@ import { KeyringPair } from '@polkadot/keyring/types'
 import { ApiPromise } from '@polkadot/api'
 import { RequestData, verifyTokenSignature, parseUploadToken, UploadToken } from '../helpers/auth'
 import { checkRemoveNonce } from '../caching/tokenNonceKeeper'
-import { httpLogger, errorLogger } from '../../services/logger'
 import { AppConfig } from './controllers/common'
+import {
+  createExpressErrorLoggerOptions,
+  createExpressDefaultLoggerOptions,
+  httpLogger,
+  errorLogger,
+} from '../../services/logger'
 
 /**
  * Creates Express web application. Uses the OAS spec file for the API.
@@ -20,10 +25,11 @@ import { AppConfig } from './controllers/common'
 export async function createApp(config: AppConfig): Promise<Express> {
   const spec = path.join(__dirname, './../../api-spec/openapi.yaml')
   const app = express()
+  const expressLoggerOptions = createExpressDefaultLoggerOptions()
 
   app.use(cors())
   app.use(express.json())
-  app.use(httpLogger(config.logSource, config.elasticSearchEndpoint))
+  app.use(httpLogger(expressLoggerOptions))
 
   app.use(
     // Set parameters for each request.
@@ -56,7 +62,9 @@ export async function createApp(config: AppConfig): Promise<Express> {
     })
   ) // Required signature.
 
-  app.use(errorLogger())
+  // Error logger
+  const errorLoggerOptions = createExpressErrorLoggerOptions()
+  app.use(errorLogger(errorLoggerOptions))
 
   /* eslint-disable @typescript-eslint/no-unused-vars */
   app.use((err: Error, req: express.Request, res: express.Response, next: express.NextFunction) => {

+ 0 - 10
storage-node-v2/src/services/webApi/controllers/common.ts → storage-node/src/services/webApi/controllers/common.ts

@@ -132,16 +132,6 @@ export type AppConfig = {
    */
   enableUploadingAuth: boolean
 
-  /**
-   * Source tag for log entries for ElasticSearch.
-   */
-  logSource: string
-
-  /**
-   * ElasticSearch logging endpoint URL
-   */
-  elasticSearchEndpoint?: string
-
   /**
    * Max file size for uploading limit.
    */

+ 0 - 0
storage-node-v2/src/services/webApi/controllers/filesApi.ts → storage-node/src/services/webApi/controllers/filesApi.ts


+ 0 - 0
storage-node-v2/src/services/webApi/controllers/stateApi.ts → storage-node/src/services/webApi/controllers/stateApi.ts


+ 0 - 0
storage-node-v2/tsconfig.json → storage-node/tsconfig.json


+ 23 - 1
yarn.lock

@@ -14459,6 +14459,13 @@ file-selector@^0.2.2:
   dependencies:
     tslib "^2.0.3"
 
+file-stream-rotator@^0.5.7:
+  version "0.5.7"
+  resolved "https://registry.yarnpkg.com/file-stream-rotator/-/file-stream-rotator-0.5.7.tgz#868a2e5966f7640a17dd86eda0e4467c089f6286"
+  integrity sha512-VYb3HZ/GiAGUCrfeakO8Mp54YGswNUHvL7P09WQcXAJNSj3iQ5QraYSp3cIn1MUyw6uzfgN/EFOarCNa4JvUHQ==
+  dependencies:
+    moment "^2.11.2"
+
 file-system-cache@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/file-system-cache/-/file-system-cache-1.0.5.tgz#84259b36a2bbb8d3d6eb1021d3132ffe64cfff4f"
@@ -21508,7 +21515,7 @@ module-lookup-amd@^6.1.0:
     requirejs "^2.3.5"
     requirejs-config-file "^3.1.1"
 
-moment@^2.10.2, moment@^2.22.1, moment@^2.24.0, moment@^2.29.1:
+moment@^2.10.2, moment@^2.11.2, moment@^2.22.1, moment@^2.24.0, moment@^2.29.1:
   version "2.29.1"
   resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3"
   integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==
@@ -22496,6 +22503,11 @@ object-filter-sequence@^1.0.0:
   resolved "https://registry.yarnpkg.com/object-filter-sequence/-/object-filter-sequence-1.0.0.tgz#10bb05402fff100082b80d7e83991b10db411692"
   integrity sha512-CsubGNxhIEChNY4cXYuA6KXafztzHqzLLZ/y3Kasf3A+sa3lL9thq3z+7o0pZqzEinjXT6lXDPAfVWI59dUyzQ==
 
+object-hash@^2.0.1:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.2.0.tgz#5ad518581eefc443bd763472b8ff2e9c2c0d54a5"
+  integrity sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==
+
 object-identity-map@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/object-identity-map/-/object-identity-map-1.0.2.tgz#2b4213a4285ca3a8cd2e696782c9964f887524e7"
@@ -31031,6 +31043,16 @@ windows-release@^3.1.0:
   dependencies:
     execa "^1.0.0"
 
+winston-daily-rotate-file@^4.5.5:
+  version "4.5.5"
+  resolved "https://registry.yarnpkg.com/winston-daily-rotate-file/-/winston-daily-rotate-file-4.5.5.tgz#cfa3a89f4eb0e4126917592b375759b772bcd972"
+  integrity sha512-ds0WahIjiDhKCiMXmY799pDBW+58ByqIBtUcsqr4oDoXrAI3Zn+hbgFdUxzMfqA93OG0mPLYVMiotqTgE/WeWQ==
+  dependencies:
+    file-stream-rotator "^0.5.7"
+    object-hash "^2.0.1"
+    triple-beam "^1.3.0"
+    winston-transport "^4.4.0"
+
 winston-elasticsearch@^0.15.8:
   version "0.15.9"
   resolved "https://registry.yarnpkg.com/winston-elasticsearch/-/winston-elasticsearch-0.15.9.tgz#a9490614f6b92d5e1977df927c77c29b3a66f247"