storage-node: some notes on improving sync

storage-node/packages/colossus/bin/cli.js

@@ -126,7 +126,7 @@ function get_storage (runtime_api) {
       if (!obj || obj.isNone) {
         return
       }
-
+      // if obj.liaison_judgement !== Accepted .. throw ?
       return obj.unwrap().ipfs_content_id.toString()
     }
   }

storage-node/packages/colossus/lib/sync.js

@@ -20,10 +20,7 @@
 
 const debug = require('debug')('joystream:sync');
 
-async function sync_callback(api, storage)
-{
-  debug('Starting sync run...');
-
+async function sync_callback(api, storage) {
   // The first step is to gather all data objects from chain.
   // TODO: in future, limit to a configured tranche
   // FIXME this isn't actually on chain yet, so we'll fake it.
@@ -36,6 +33,10 @@ async function sync_callback(api, storage)
   const allChecks = knownContentIds.map(async (content_id) => {
     let { relationship, relationshipId } = await api.assets.getStorageRelationshipAndId(providerId, content_id);
 
+    // get the data object
+    // make sure the data object was Accepted by the liaison,
+    // don't just blindly attempt to fetch them
+
     let fileLocal;
     try {
       // check if we have content or not
@@ -52,8 +53,11 @@ async function sync_callback(api, storage)
       try {
         await storage.synchronize(content_id);
       } catch (err) {
-        debug(err.message)
+        // duplicate logging
+        // debug(err.message)
+        return
       }
+      // why are we returning, if we synced the file
       return;
     }
 
@@ -82,15 +86,16 @@ async function sync_callback(api, storage)
   });
 
 
-  await Promise.all(allChecks);
-  debug('sync run complete');
+  return Promise.all(allChecks);
 }
 
 
 async function sync_periodic(api, flags, storage)
 {
   try {
-    await sync_callback(api, storage);
+    debug('Starting sync run...')
+    await sync_callback(api, storage)
+    debug('sync run complete')
   } catch (err) {
     debug(`Error in sync_periodic ${err.stack}`);
   }

storage-node/packages/storage/storage.js

@@ -383,6 +383,8 @@ class Storage
   {
     const resolved = await this._resolve_content_id_with_timeout(this._timeout, content_id);
 
+    // validate resolved id is proper ipfs_cid, not null or empty string
+
     if (this.pins[resolved]) {
       return;
     }
@@ -396,6 +398,7 @@ class Storage
         delete this.pins[resolved];
       } else {
         debug(`Pinned ${resolved}`);
+        // why aren't we doing this.pins[resolved] = true
       }
     });
   }