storage-node-v2: Add data stats endpoint.

storage-node-v2/package.json

@@ -34,6 +34,7 @@
     "express": "4.17.1",
     "express-openapi-validator": "^4.12.4",
     "express-winston": "^4.1.0",
+    "fast-folder-size": "^1.4.0",
     "file-type": "^16.5.0",
     "lodash": "^4.17.21",
     "multihashes": "^4.0.2",
@@ -114,7 +115,7 @@
       },
       "leader": {
         "description": "Storage working group leader commands."
-      },      
+      },
       "operator": {
         "description": "Storage provider(operator) commands."
       }

storage-node-v2/scripts/generate-test-data.ts

@@ -7,10 +7,10 @@ import { Client, ClientConfig,QueryResult } from 'pg'
 import { exit } from 'process'
 
 async function doJob(): Promise<void> {
-  const uploadDirectory = '/Users/shamix/uploads2'
-  const fileSize = 100000000
+  const uploadDirectory = '/Users/shamix/uploads5'
+  const fileSize = 1000
 
-  const objectNumber = 100
+  const objectNumber = 10000
   const bagNumber = 10
   const bucketNumber = 10
 
@@ -20,8 +20,8 @@ async function doJob(): Promise<void> {
     `http://localhost:3335/`,
   ]
 
-  const updateDb = true
-  const generateFiles = false
+  const updateDb = false
+  const generateFiles = true
 
   if (updateDb) {
     const config : ClientConfig = {

storage-node-v2/src/api-spec/openapi.yaml

@@ -224,6 +224,19 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/VersionResponse'
+  /state/data:
+    get:
+      operationId: stateApi.getLocalDataStats
+      description: Returns local uploading directory stats.
+      tags:
+        - state
+      responses:
+        200:
+          description: Ok
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/DataStatsResponse'
 
 components:
   securitySchemes:
@@ -271,6 +284,24 @@ components:
           type: string
         message:
           type: string
+    DataStatsResponse:
+      type: object
+      required:
+        - totalSize
+        - objectNumber
+      properties:
+        totalSize:
+          type: integer
+          format: int64
+        objectNumber:
+          type: integer
+          format: int64
+        tempDirSize:
+          type: integer
+          format: int64
+        tempDownloads:
+          type: integer
+          format: int64
     VersionResponse:
       type: object
       required:

storage-node-v2/src/services/webApi/app.ts

@@ -87,6 +87,7 @@ export async function createApp(config: AppConfig): Promise<Express> {
     // Set parameters for each request.
     (req: express.Request, res: express.Response, next: NextFunction) => {
       res.locals.uploadsDir = config.uploadsDir
+      res.locals.tempFileUploadingDir = tempFileUploadingDir
       res.locals.storageProviderAccount = config.account
       res.locals.workerId = config.workerId
       res.locals.api = config.api

storage-node-v2/src/services/webApi/controllers/common.ts

@@ -15,6 +15,21 @@ export function getUploadsDir(res: express.Response): string {
   throw new Error('No upload directory path loaded.')
 }
 
+/**
+ * Returns a directory for temporary file uploading from the response.
+ *
+ * @remarks
+ * This is a helper function. It parses the response object for a variable and
+ * throws an error on failure.
+ */
+export function getTempFileUploadingDir(res: express.Response): string {
+  if (res.locals.tempFileUploadingDir) {
+    return res.locals.tempFileUploadingDir
+  }
+
+  throw new Error('No temporary uploading directory path loaded.')
+}
+
 /**
  * Returns worker ID from the response.
  *

storage-node-v2/src/services/webApi/controllers/stateApi.ts

@@ -2,9 +2,13 @@ import { getLocalDataObjects } from '../../../services/sync/synchronizer'
 import * as express from 'express'
 import _ from 'lodash'
 import { getDataObjectIDsByBagId } from '../../sync/storageObligations'
-import { getUploadsDir } from './common'
+import { getUploadsDir, getTempFileUploadingDir } from './common'
+import fastFolderSize from 'fast-folder-size'
+import { promisify } from 'util'
+import fs from 'fs'
 
 import NodeCache from 'node-cache'
+const fsPromises = fs.promises
 
 // Expiration period in seconds for the local cache.
 const ExpirationPeriod = 30 // minutes
@@ -15,9 +19,6 @@ const dataCache = new NodeCache({
   deleteOnExpire: true,
 })
 
-/**
- * A public endpoint: serves files by CID.
-
 /**
  * A public endpoint: return all local data objects.
  */
@@ -39,6 +40,60 @@ export async function getAllLocalDataObjects(
   }
 }
 
+/**
+ * A public endpoint: serves local data uploading directory stats.
+ *
+ *  @return total size and count of the data objects.
+ */
+export async function getLocalDataStats(
+  req: express.Request,
+  res: express.Response
+): Promise<void> {
+  try {
+    const uploadsDir = getUploadsDir(res)
+    const tempFileDir = getTempFileUploadingDir(res)
+    const fastFolderSizeAsync = promisify(fastFolderSize)
+
+    const tempFolderExists = fs.existsSync(tempFileDir)
+    const statsPromise = fsPromises.readdir(uploadsDir)
+    const sizePromise = fastFolderSizeAsync(uploadsDir)
+
+    const [stats, totalSize] = await Promise.all([statsPromise, sizePromise])
+
+    let objectNumber = stats.length
+    let tempDownloads = 0
+    let tempDirSize = 0
+    if (tempFolderExists) {
+      if (objectNumber > 0) {
+        objectNumber--
+      }
+
+      const tempDirStatsPromise = fsPromises.readdir(tempFileDir)
+      const tempDirSizePromise = fastFolderSizeAsync(tempFileDir)
+
+      const [tempDirStats, tempSize] = await Promise.all([
+        tempDirStatsPromise,
+        tempDirSizePromise,
+      ])
+
+      tempDirSize = tempSize ?? 0
+      tempDownloads = tempDirStats.length
+    }
+
+    res.status(200).json({
+      objectNumber,
+      totalSize,
+      tempDownloads,
+      tempDirSize,
+    })
+  } catch (err) {
+    res.status(500).json({
+      type: 'local_data_stats',
+      message: err.toString(),
+    })
+  }
+}
+
 /**
  * A public endpoint: return local data objects for the bag.
  */

yarn.lock

@@ -9188,6 +9188,11 @@ before-after-hook@^2.0.0:
   resolved "https://registry.yarnpkg.com/before-after-hook/-/before-after-hook-2.1.0.tgz#b6c03487f44e24200dd30ca5e6a1979c5d2fb635"
   integrity sha512-IWIbu7pMqyw3EAJHzzHbWa85b6oud/yfKYg5rqB5hNE8CeMi3nX+2C2sj0HswfblST86hpVEOAb9x34NZd6P7A==
 
+big-integer@^1.6.17:
+  version "1.6.48"
+  resolved "https://registry.yarnpkg.com/big-integer/-/big-integer-1.6.48.tgz#8fd88bd1632cba4a1c8c3e3d7159f08bb95b4b9e"
+  integrity sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w==
+
 big.js@^3.1.3:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/big.js/-/big.js-3.2.0.tgz#a5fc298b81b9e0dca2e458824784b65c52ba588e"
@@ -9223,6 +9228,14 @@ binary-search@^1.3.3:
   resolved "https://registry.yarnpkg.com/binary-search/-/binary-search-1.3.6.tgz#e32426016a0c5092f0f3598836a1c7da3560565c"
   integrity sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==
 
+binary@~0.3.0:
+  version "0.3.0"
+  resolved "https://registry.yarnpkg.com/binary/-/binary-0.3.0.tgz#9f60553bc5ce8c3386f3b553cff47462adecaa79"
+  integrity sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=
+  dependencies:
+    buffers "~0.1.1"
+    chainsaw "~0.1.0"
+
 bindings@^1.2.1, bindings@^1.3.0, bindings@^1.5.0:
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df"
@@ -9318,6 +9331,11 @@ bluebird@^3.1.1, bluebird@^3.3.5, bluebird@^3.5.1, bluebird@^3.5.3, bluebird@^3.
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
   integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
 
+bluebird@~3.4.1:
+  version "3.4.7"
+  resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.4.7.tgz#f72d760be09b7f76d08ed8fae98b289a8d05fab3"
+  integrity sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM=
+
 bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.1.1, bn.js@^4.11.8, bn.js@^4.11.9, bn.js@^4.4.0, bn.js@^5.1.2, bn.js@^5.1.3:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-5.1.2.tgz#c9686902d3c9a27729f43ab10f9d79c2004da7b0"
@@ -9673,6 +9691,11 @@ buffer-from@1.x, buffer-from@^1.0.0, buffer-from@^1.1.0:
   resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"
   integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==
 
+buffer-indexof-polyfill@~1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz#d2732135c5999c64b277fcf9b1abe3498254729c"
+  integrity sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==
+
 buffer-indexof@^1.0.0:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/buffer-indexof/-/buffer-indexof-1.1.1.tgz#52fabcc6a606d1a00302802648ef68f639da268c"
@@ -9739,6 +9762,11 @@ buffer@^6.0.3:
     base64-js "^1.3.1"
     ieee754 "^1.2.1"
 
+buffers@~0.1.1:
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/buffers/-/buffers-0.1.1.tgz#b24579c3bed4d6d396aeee6d9a8ae7f5482ab7bb"
+  integrity sha1-skV5w77U1tOWru5tmorn9Ugqt7s=
+
 bufferutil@^4.0.1:
   version "4.0.3"
   resolved "https://registry.yarnpkg.com/bufferutil/-/bufferutil-4.0.3.tgz#66724b756bed23cd7c28c4d306d7994f9943cc6b"
@@ -10192,6 +10220,13 @@ chai@^4.2.0:
     pathval "^1.1.0"
     type-detect "^4.0.5"
 
+chainsaw@~0.1.0:
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/chainsaw/-/chainsaw-0.1.0.tgz#5eab50b28afe58074d0d58291388828b5e5fbc98"
+  integrity sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=
+  dependencies:
+    traverse ">=0.3.0 <0.4"
+
 chalk@*, chalk@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.0.tgz#4e14870a618d9e2edd97dd8345fd9d9dc315646a"
@@ -13013,6 +13048,13 @@ duplexer2@~0.0.2:
   dependencies:
     readable-stream "~1.1.9"
 
+duplexer2@~0.1.4:
+  version "0.1.4"
+  resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1"
+  integrity sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=
+  dependencies:
+    readable-stream "^2.0.2"
+
 duplexer3@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2"
@@ -14646,6 +14688,13 @@ fast-fifo@^1.0.0:
   resolved "https://registry.yarnpkg.com/fast-fifo/-/fast-fifo-1.0.0.tgz#9bc72e6860347bb045a876d1c5c0af11e9b984e7"
   integrity sha512-4VEXmjxLj7sbs8J//cn2qhRap50dGzF5n8fjay8mau+Jn4hxSeR3xPFwxMaQq/pDaq7+KQk0PAbC2+nWDkJrmQ==
 
+fast-folder-size@^1.4.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/fast-folder-size/-/fast-folder-size-1.4.0.tgz#566934c2071ac7571c54b99d4fc136556ca0c266"
+  integrity sha512-1U38nr6DgoP0YOD1KjAG4wsgZZUpICCqlUFiiTdY4Zc2nM3ioNmgsBToy5yf3+/AT3Do6fb0v/b2ozVXuZX2Ng==
+  dependencies:
+    unzipper "^0.10.11"
+
 fast-glob@^2.0.2, fast-glob@^2.2.6:
   version "2.2.7"
   resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-2.2.7.tgz#6953857c3afa475fff92ee6015d52da70a4cd39d"
@@ -20783,6 +20832,11 @@ linkify-it@^2.0.0:
   dependencies:
     uc.micro "^1.0.1"
 
+listenercount@~1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/listenercount/-/listenercount-1.0.1.tgz#84c8a72ab59c4725321480c975e6508342e70937"
+  integrity sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc=
+
 listr-silent-renderer@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/listr-silent-renderer/-/listr-silent-renderer-1.1.1.tgz#924b5a3757153770bf1a8e3fbf74b8bbf3f9242e"
@@ -28011,7 +28065,7 @@ set-value@^2.0.0, set-value@^2.0.1:
     is-plain-object "^2.0.3"
     split-string "^3.0.1"
 
-setimmediate@^1.0.4, setimmediate@^1.0.5:
+setimmediate@^1.0.4, setimmediate@^1.0.5, setimmediate@~1.0.4:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285"
   integrity sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=
@@ -30174,6 +30228,11 @@ traverse-chain@~0.1.0:
   resolved "https://registry.yarnpkg.com/traverse-chain/-/traverse-chain-0.1.0.tgz#61dbc2d53b69ff6091a12a168fd7d433107e40f1"
   integrity sha1-YdvC1Ttp/2CRoSoWj9fUMxB+QPE=
 
+"traverse@>=0.3.0 <0.4":
+  version "0.3.9"
+  resolved "https://registry.yarnpkg.com/traverse/-/traverse-0.3.9.tgz#717b8f220cc0bb7b44e40514c22b2e8bbc70d8b9"
+  integrity sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=
+
 traverse@^0.6.6, traverse@~0.6.6:
   version "0.6.6"
   resolved "https://registry.yarnpkg.com/traverse/-/traverse-0.6.6.tgz#cbdf560fd7b9af632502fed40f918c157ea97137"
@@ -31009,6 +31068,22 @@ unzip-response@^2.0.1:
   resolved "https://registry.yarnpkg.com/unzip-response/-/unzip-response-2.0.1.tgz#d2f0f737d16b0615e72a6935ed04214572d56f97"
   integrity sha1-0vD3N9FrBhXnKmk17QQhRXLVb5c=
 
+unzipper@^0.10.11:
+  version "0.10.11"
+  resolved "https://registry.yarnpkg.com/unzipper/-/unzipper-0.10.11.tgz#0b4991446472cbdb92ee7403909f26c2419c782e"
+  integrity sha512-+BrAq2oFqWod5IESRjL3S8baohbevGcVA+teAIOYWM3pDVdseogqbzhhvvmiyQrUNKFUnDMtELW3X8ykbyDCJw==
+  dependencies:
+    big-integer "^1.6.17"
+    binary "~0.3.0"
+    bluebird "~3.4.1"
+    buffer-indexof-polyfill "~1.0.0"
+    duplexer2 "~0.1.4"
+    fstream "^1.0.12"
+    graceful-fs "^4.2.2"
+    listenercount "~1.0.1"
+    readable-stream "~2.3.6"
+    setimmediate "~1.0.4"
+
 upath@^1.1.0, upath@^1.1.1, upath@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/upath/-/upath-1.2.0.tgz#8f66dbcd55a883acdae4408af8b035a5044c1894"