storage-node-v2: Change data ID handling.

- migrate data quering from the CID to the data object IDs

storage-node-v2/src/api-spec/openapi.yaml

@@ -21,14 +21,14 @@ tags:
     description: Storage node State API
 
 paths:
-  /files/{cid}:
+  /files/{id}:
     get:
       operationId: publicApi.getFile
       description: Returns a media file.
       tags:
         - files
       parameters:
-        - name: cid
+        - name: id
           required: true
           in: path
           description: Content ID
@@ -80,7 +80,7 @@ paths:
       tags:
         - files
       parameters:
-        - name: cid
+        - name: id
           required: true
           in: path
           description: Content ID

storage-node-v2/src/services/metadata/generateTypes.ts

@@ -1,5 +1,5 @@
 /**
- * OperatorMetadataJson file generating script. 
+ * OperatorMetadataJson file generating script.
  */
 
 import fs from 'fs'

storage-node-v2/src/services/queryNode/generated/queries.ts

@@ -24,7 +24,7 @@ export type GetStorageBagDetailsQueryVariables = Types.Exact<{
 
 export type GetStorageBagDetailsQuery = { storageBags: Array<StorageBagDetailsFragment> }
 
-export type DataObjectDetailsFragment = { ipfsHash: string; storageBag: { id: string } }
+export type DataObjectDetailsFragment = { id: string; storageBagId: string }
 
 export type GetDataObjectDetailsQueryVariables = Types.Exact<{
   bagIds?: Types.Maybe<Types.StorageBagWhereInput>
@@ -63,10 +63,8 @@ export const StorageBagDetails = gql`
 `
 export const DataObjectDetails = gql`
   fragment DataObjectDetails on StorageDataObject {
-    ipfsHash
-    storageBag {
-      id
-    }
+    id
+    storageBagId
   }
 `
 export const GetStorageBucketDetails = gql`

storage-node-v2/src/services/queryNode/queries/queries.graphql

@@ -36,10 +36,8 @@ query getStorageBagDetails($bucketIds: [String!], $offset: Int, $limit: Int) {
 }
 
 fragment DataObjectDetails on StorageDataObject {
-  ipfsHash
-  storageBag {
-    id
-  }
+  id
+  storageBagId
 }
 
 query getDataObjectDetails($bagIds: StorageBagWhereInput, $offset: Int, $limit: Int) {

storage-node-v2/src/services/sync/remoteStorageData.ts

@@ -9,8 +9,8 @@ const ExpirationPeriod: number = 5 * 60 // minutes
 // Max data entries in local cache
 const MaxEntries = 10000
 
-// Local in-memory cache for CIDs by operator URL.
-const availableCidsCache = new NodeCache({
+// Local in-memory cache for data object IDs by operator URL.
+const availableIDsCache = new NodeCache({
   stdTTL: ExpirationPeriod,
   deleteOnExpire: true,
   maxKeys: MaxEntries,
@@ -39,7 +39,7 @@ export async function getRemoteDataObjects(operatorUrl: string): Promise<string[
     return []
   }
 
-  const cachedData = availableCidsCache.get<string[]>(url)
+  const cachedData = availableIDsCache.get<string[]>(url)
   if (cachedData) {
     logger.debug(`Sync - getting from cache available data for ${url}`)
     return cachedData
@@ -51,7 +51,7 @@ export async function getRemoteDataObjects(operatorUrl: string): Promise<string[
     const response = await superagent.get(url).timeout(timeoutMs)
 
     const data = response.body
-    availableCidsCache.set(url, data, ExpirationPeriod)
+    availableIDsCache.set(url, data, ExpirationPeriod)
 
     return data
   } catch (err) {

storage-node-v2/src/services/sync/storageObligations.ts

@@ -67,9 +67,9 @@ type Bag = {
  */
 type DataObject = {
   /**
-   * Content ID (IPFS hash)
+   * Data object ID
    */
-  cid: string
+  id: string
 
   /**
    * Assigned bag ID
@@ -112,8 +112,8 @@ export async function getStorageObligationsFromRuntime(
       buckets: bag.storageAssignments.map((bucketInBag) => bucketInBag.storageBucket.id),
     })),
     dataObjects: assignedDataObjects.map((dataObject) => ({
-      cid: dataObject.ipfsHash,
-      bagId: dataObject.storageBag.id,
+      id: dataObject.id,
+      bagId: dataObject.storageBagId,
     })),
   }
 
@@ -149,7 +149,7 @@ export async function getDataObjectIDsByBagId(queryNodeUrl: string, bagId: strin
   const api = new QueryNodeApi(queryNodeUrl)
   const dataObjects = await getAllAssignedDataObjects(api, [bagId])
 
-  return dataObjects.map((obj) => obj.ipfsHash)
+  return dataObjects.map((obj) => obj.id)
 }
 
 /**

storage-node-v2/src/services/sync/synchronizer.ts

@@ -33,10 +33,10 @@ export async function performSync(
     getLocalFileNames(uploadDirectory),
   ])
 
-  const requiredCids = model.dataObjects.map((obj) => obj.cid)
+  const requiredIds = model.dataObjects.map((obj) => obj.id)
 
-  const added = _.difference(requiredCids, files)
-  const deleted = _.difference(files, requiredCids)
+  const added = _.difference(requiredIds, files)
+  const deleted = _.difference(files, requiredIds)
 
   logger.debug(`Sync - added objects: ${added.length}`)
   logger.debug(`Sync - deleted objects: ${deleted.length}`)
@@ -75,19 +75,19 @@ async function getLocalFileNames(directory: string): Promise<string[]> {
  * Creates the download preparation tasks.
  *
  * @param dataObligations - defines the current data obligations for the node
- * @param addedCids - data object IDs to download
+ * @param addedIds - data object IDs to download
  * @param uploadDirectory - local directory for data uploading
  * @param taskSink - a destination for the newly created tasks
  */
 async function getPrepareDownloadTasks(
   dataObligations: DataObligations,
-  addedCids: string[],
+  addedIds: string[],
   uploadDirectory: string,
   taskSink: TaskSink
 ): Promise<PrepareDownloadFileTask[]> {
-  const cidMap = new Map()
+  const idMap = new Map()
   for (const entry of dataObligations.dataObjects) {
-    cidMap.set(entry.cid, entry.bagId)
+    idMap.set(entry.id, entry.bagId)
   }
 
   const bucketMap = new Map()
@@ -111,16 +111,16 @@ async function getPrepareDownloadTasks(
     bagMap.set(entry.id, operatorUrls)
   }
 
-  const tasks = addedCids.map((cid) => {
+  const tasks = addedIds.map((id) => {
     let operatorUrls: string[] = [] // can be empty after look up
-    if (cidMap.has(cid)) {
-      const bagid = cidMap.get(cid)
+    if (idMap.has(id)) {
+      const bagid = idMap.get(id)
       if (bagMap.has(bagid)) {
         operatorUrls = bagMap.get(bagid)
       }
     }
 
-    return new PrepareDownloadFileTask(operatorUrls, cid, uploadDirectory, taskSink)
+    return new PrepareDownloadFileTask(operatorUrls, id, uploadDirectory, taskSink)
   })
 
   return tasks
@@ -130,15 +130,15 @@ async function getPrepareDownloadTasks(
  * Creates the download file tasks.
  *
  * @param operatorUrl - defines the data source URL.
- * @param addedCids - data object IDs to download
+ * @param addedIds - data object IDs to download
  * @param uploadDirectory - local directory for data uploading
  */
 async function getDownloadTasks(
   operatorUrl: string,
-  addedCids: string[],
+  addedIds: string[],
   uploadDirectory: string
 ): Promise<DownloadFileTask[]> {
-  const addedTasks = addedCids.map((fileName) => new DownloadFileTask(operatorUrl, fileName, uploadDirectory))
+  const addedTasks = addedIds.map((fileName) => new DownloadFileTask(operatorUrl, fileName, uploadDirectory))
 
   return addedTasks
 }
@@ -149,7 +149,7 @@ async function getDownloadTasks(
  * @param uploadDirectory - local directory to get file names from
  */
 export async function getLocalDataObjects(uploadDirectory: string): Promise<string[]> {
-  const localCids = await getLocalFileNames(uploadDirectory)
+  const localIds = await getLocalFileNames(uploadDirectory)
 
-  return localCids
+  return localIds
 }

storage-node-v2/src/services/sync/tasks.ts

@@ -99,20 +99,20 @@ export class DownloadFileTask implements SyncTask {
  * Resolve remote storage node URLs and creates file downloading tasks (DownloadFileTask).
  */
 export class PrepareDownloadFileTask implements SyncTask {
-  cid: string
+  dataObjectId: string
   operatorUrlCandidates: string[]
   taskSink: TaskSink
   uploadsDirectory: string
 
-  constructor(operatorUrlCandidates: string[], cid: string, uploadsDirectory: string, taskSink: TaskSink) {
-    this.cid = cid
+  constructor(operatorUrlCandidates: string[], dataObjectId: string, uploadsDirectory: string, taskSink: TaskSink) {
+    this.dataObjectId = dataObjectId
     this.taskSink = taskSink
     this.operatorUrlCandidates = operatorUrlCandidates
     this.uploadsDirectory = uploadsDirectory
   }
 
   description(): string {
-    return `Sync - preparing for download of: ${this.cid} ....`
+    return `Sync - preparing for download of: ${this.dataObjectId} ....`
   }
 
   async execute(): Promise<void> {
@@ -136,18 +136,18 @@ export class PrepareDownloadFileTask implements SyncTask {
 
       try {
         const chosenBaseUrl = randomUrl
-        const remoteOperatorCids: string[] = await getRemoteDataObjects(chosenBaseUrl)
+        const remoteOperatorIds: string[] = await getRemoteDataObjects(chosenBaseUrl)
 
-        if (remoteOperatorCids.includes(this.cid)) {
-          const newTask = new DownloadFileTask(chosenBaseUrl, this.cid, this.uploadsDirectory)
+        if (remoteOperatorIds.includes(this.dataObjectId)) {
+          const newTask = new DownloadFileTask(chosenBaseUrl, this.dataObjectId, this.uploadsDirectory)
 
           return this.taskSink.add([newTask])
         }
       } catch (err) {
-        logger.error(`Sync - fetching data error for ${this.cid}: ${err}`)
+        logger.error(`Sync - fetching data error for ${this.dataObjectId}: ${err}`)
       }
     }
 
-    logger.warn(`Sync - cannot get operator URLs for ${this.cid}`)
+    logger.warn(`Sync - cannot get operator URLs for ${this.dataObjectId}`)
   }
 }

storage-node-v2/src/services/webApi/controllers/publicApi.ts

@@ -35,13 +35,13 @@ import { Membership } from '@joystream/types/members'
 const fsPromises = fs.promises
 
 /**
- * A public endpoint: serves files by CID.
+ * A public endpoint: serves files by data object ID.
  */
 export async function getFile(req: express.Request, res: express.Response): Promise<void> {
   try {
-    const cid = getCid(req)
+    const dataObjectId = getDataObjectId(req)
     const uploadsDir = getUploadsDir(res)
-    const fullPath = path.resolve(uploadsDir, cid)
+    const fullPath = path.resolve(uploadsDir, dataObjectId)
 
     const fileInfo = await getFileInfo(fullPath)
     const fileStats = await fsPromises.stat(fullPath)
@@ -66,13 +66,13 @@ export async function getFile(req: express.Request, res: express.Response): Prom
 }
 
 /**
- * A public endpoint: sends file headers by CID.
+ * A public endpoint: sends file headers by data object ID.
  */
 export async function getFileHeaders(req: express.Request, res: express.Response): Promise<void> {
   try {
-    const cid = getCid(req)
+    const dataObjectId = getDataObjectId(req)
     const uploadsDir = getUploadsDir(res)
-    const fullPath = path.resolve(uploadsDir, cid)
+    const fullPath = path.resolve(uploadsDir, dataObjectId)
     const fileInfo = await getFileInfo(fullPath)
     const fileStats = await fsPromises.stat(fullPath)
 
@@ -112,7 +112,7 @@ export async function uploadFile(req: express.Request, res: express.Response): P
 
     // Prepare new file name
     const uploadsDir = getUploadsDir(res)
-    const newPath = path.join(uploadsDir, hash)
+    const newPath = path.join(uploadsDir, uploadRequest.dataObjectId.toString())
 
     // Overwrites existing file.
     await fsPromises.rename(fileObj.path, newPath)
@@ -214,19 +214,19 @@ function getApi(res: express.Response): ApiPromise {
 }
 
 /**
- * Returns Content ID from the request.
+ * Returns data object ID from the request.
  *
  * @remarks
  * This is a helper function. It parses the request object for a variable and
  * throws an error on failure.
  */
-function getCid(req: express.Request): string {
-  const cid = req.params.cid || ''
-  if (cid.length > 0) {
-    return cid
+function getDataObjectId(req: express.Request): string {
+  const id = req.params.id || ''
+  if (id.length > 0) {
+    return id
   }
 
-  throw new WebApiError('No CID provided.', 400)
+  throw new WebApiError('No data object ID provided.', 400)
 }
 
 /**

storage-node-v2/src/services/webApi/controllers/stateApi.ts

@@ -34,9 +34,9 @@ export async function getAllLocalDataObjects(req: express.Request, res: express.
     const uploadsDir = getUploadsDir(res)
     const tempFileDir = getTempFileUploadingDir(res)
 
-    const cids = await getCachedLocalDataObjects(uploadsDir, tempFileDir)
+    const ids = await getCachedLocalDataObjects(uploadsDir, tempFileDir)
 
-    res.status(200).json(cids)
+    res.status(200).json(ids)
   } catch (err) {
     sendResponseWithError(res, err, 'all_data_objects')
   }
@@ -98,12 +98,12 @@ export async function getLocalDataObjectsByBagId(req: express.Request, res: expr
     const queryNodeUrl = getQueryNodeUrl(res)
     const bagId = getBagId(req)
 
-    const [cids, requiredCids] = await Promise.all([
+    const [ids, requiredIds] = await Promise.all([
       getCachedLocalDataObjects(uploadsDir, tempFileDir),
       getCachedDataObjectsObligations(queryNodeUrl, bagId),
     ])
 
-    const localDataForBag = _.intersection(cids, requiredCids)
+    const localDataForBag = _.intersection(ids, requiredIds)
 
     res.status(200).json(localDataForBag)
   } catch (err) {
@@ -157,7 +157,7 @@ async function getCachedLocalDataObjects(uploadsDir: string, tempDirName: string
 
     // Filter temporary directory name.
     const tempDirectoryName = path.parse(tempDirName).name
-    data = data.filter((cid) => cid !== tempDirectoryName)
+    data = data.filter((dataObjectId) => dataObjectId !== tempDirectoryName)
 
     dataCache.set(entryName, data)
   }