A better way to handle "doubleMapEntries"

pioneer/packages/joy-utils/src/transport/base.ts

@@ -1,9 +1,5 @@
 import { ApiPromise } from '@polkadot/api';
-import { Observable } from 'rxjs';
-import { StorageEntryBase } from '@polkadot/api/types';
-import { CodecArg, Codec } from '@polkadot/types/types';
-
-type ApiMethod = StorageEntryBase<'promise', (arg1?: CodecArg, arg2?: CodecArg) => Observable<Codec>>;
+import { Codec } from '@polkadot/types/types';
 
 export default abstract class BaseTransport {
   protected api: ApiPromise;
@@ -48,6 +44,11 @@ export default abstract class BaseTransport {
     return this.api.query.minting;
   }
 
+  protected queryMethodByName (name: string) {
+    const [module, method] = name.split('.');
+    return this.api.query[module][method];
+  }
+
   // Fetch all double map entries using only the first key
   //
   // TODO: FIXME: This may be a risky implementation, because it relies on a few assumptions about how the data is stored etc.
@@ -55,19 +56,32 @@ export default abstract class BaseTransport {
   // 32-bytes prefix assuming a given (fixed) value of the first key (ie. for all values like map[x][y], the storage key starts
   // with the same prefix as long as x remains the same. Changing y will not affect this prefix)
   protected async doubleMapEntries<T extends Codec> (
-    method: ApiMethod,
+    methodName: string,
     firstKey: Codec,
-    valueConverter: (hex: string) => T
-  ): Promise<{ storageKey: string; value: T}[]> {
-    const entryKey = method.key(firstKey, 0);
-    const entryKeyPrefix = entryKey.toString().substr(0, 66); // "0x" + 64 hex characters (32 bytes)
-    const allEntryKeys = await this.api.rpc.state.getKeys(entryKeyPrefix);
-    const entries: { storageKey: string; value: T }[] = [];
-    for (const key of allEntryKeys) {
+    valueConverter: (hex: string) => T,
+    getEntriesCount: () => Promise<number>,
+    secondKeyStart = 1
+  ): Promise<{ secondKey: number; value: T}[]> {
+    // Get prefix and storage keys of all entries
+    const firstEntryStorageKey = this.queryMethodByName(methodName).key(firstKey, secondKeyStart);
+    const entryStorageKeyPrefix = firstEntryStorageKey.substr(0, 66); // "0x" + 64 hex characters (32 bytes)
+    const allEntriesStorageKeys = await this.api.rpc.state.getKeys(entryStorageKeyPrefix);
+
+    // Create storageKey-to-secondKey map
+    const maxSecondKey = (await getEntriesCount()) - 1 + secondKeyStart;
+    const storageKeyToSecondKey: { [key: string]: number } = {};
+    for (let secondKey = secondKeyStart; secondKey <= maxSecondKey; ++secondKey) {
+      const storageKey = this.queryMethodByName(methodName).key(firstKey, secondKey);
+      storageKeyToSecondKey[storageKey] = secondKey;
+    }
+
+    // Create the resulting entries array
+    const entries: { secondKey: number; value: T }[] = [];
+    for (const key of allEntriesStorageKeys) {
       const value: any = await this.api.rpc.state.getStorage(key);
       if (typeof value === 'object' && value !== null && value.raw) {
         entries.push({
-          storageKey: key.toString(),
+          secondKey: storageKeyToSecondKey[key.toString()],
           value: valueConverter(value.raw.toString())
         });
       }

pioneer/packages/joy-utils/src/transport/proposals.ts

@@ -14,8 +14,8 @@ import BaseTransport from './base';
 import { ThreadId, PostId } from '@joystream/types/forum';
 import { Proposal, ProposalId, VoteKind, DiscussionThread, DiscussionPost } from '@joystream/types/proposals';
 import { MemberId } from '@joystream/types/members';
-import { u32 } from '@polkadot/types/';
-import { BalanceOf, EventRecord } from '@polkadot/types/interfaces';
+import { u32, u64 } from '@polkadot/types/';
+import { BalanceOf } from '@polkadot/types/interfaces';
 
 import { includeKeys, bytesToString } from '../functions/misc';
 import _ from 'lodash';
@@ -26,8 +26,6 @@ import MembersTransport from './members';
 import ChainTransport from './chain';
 import CouncilTransport from './council';
 
-import { Vec } from '@polkadot/types/codec';
-
 export default class ProposalsTransport extends BaseTransport {
   private membersT: MembersTransport;
   private chainT: ChainTransport;
@@ -181,25 +179,6 @@ export default class ProposalsTransport extends BaseTransport {
     return this.proposalsEngine.proposals(id, callback);
   }
 
-  // Find postId having only the object and storage key
-  // FIXME: TODO: This is necessary because of the "hacky" workaround described in ./base.ts
-  // (in order to avoid fetching all posts ever created)
-  async findPostId (post: DiscussionPost, storageKey: string): Promise<PostId | null> {
-    const blockHash = await this.api.rpc.chain.getBlockHash(post.created_at);
-    const events = await this.api.query.system.events.at(blockHash) as Vec<EventRecord>;
-    const postIds: PostId[] = events
-      .filter(({ event }) => event.section === 'proposalsDiscussion' && event.method === 'PostCreated')
-      .map(({ event }) => event.data[0] as PostId);
-
-    // Just in case there were multiple posts created in this block...
-    for (const postId of postIds) {
-      const foundPostKey = this.proposalsDiscussion.postThreadIdByPostId.key(post.thread_id, postId);
-      if (foundPostKey === storageKey) return postId;
-    }
-
-    return null;
-  }
-
   async discussion (id: number|ProposalId): Promise<ParsedDiscussion | null> {
     const threadId = (await this.proposalsCodex.threadIdByProposalId(id)) as ThreadId;
     if (!threadId.toNumber()) {
@@ -207,15 +186,16 @@ export default class ProposalsTransport extends BaseTransport {
     }
     const thread = (await this.proposalsDiscussion.threadById(threadId)) as DiscussionThread;
     const postEntries = await this.doubleMapEntries(
-      this.proposalsDiscussion.postThreadIdByPostId,
+      'proposalsDiscussion.postThreadIdByPostId',
       threadId,
-      (v) => new DiscussionPost(v)
+      (v) => new DiscussionPost(v),
+      async () => ((await this.proposalsDiscussion.postCount()) as u64).toNumber()
     );
 
     const parsedPosts: ParsedPost[] = [];
-    for (const { storageKey, value: post } of postEntries) {
+    for (const { secondKey: postId, value: post } of postEntries) {
       parsedPosts.push({
-        postId: await this.findPostId(post, storageKey),
+        postId: new PostId(postId),
         threadId: post.thread_id,
         text: bytesToString(post.text),
         createdAt: await this.chainT.blockTimestamp(post.created_at.toNumber()),