query node - accommodating membership mappings

query-node/mappings/bootstrap/members.ts

@@ -1,33 +1,27 @@
+// TODO: add logger
+
 import BN from 'bn.js'
 import { ApiPromise } from '@polkadot/api'
 import { Option } from '@polkadot/types/codec'
 import { Codec } from '@polkadot/types/types'
 import { Membership as Profile } from '@joystream/types/members'
 
-import { DB, getLogger } from '../../generated/hydra-processor'
+import { DatabaseManager } from '@dzlzv/hydra-db-utils'
+import { prepareBlock } from '../common'
+//import { DB, getLogger } from '../../generated/hydra-processor'
 import { Block, Network } from '../../generated/graphql-server/src/modules/block/block.model'
 import { Membership } from '../../generated/graphql-server/src/modules/membership/membership.model'
 
-const logger = getLogger()
+//const logger = getLogger()
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
-export async function members_bootMembers(api: ApiPromise, db: DB): Promise<void> {
+export async function members_bootMembers(api: ApiPromise, db: DatabaseManager): Promise<void> {
   const blkHeight = process.env.BLOCK_HEIGHT ? parseInt(process.env.BLOCK_HEIGHT) : 0
   const blkHash = await api.rpc.chain.getBlockHash(blkHeight)
   const ids = await api.query.members.membersCreated.at(blkHash)
   const num: number = parseInt(ids.toString())
 
-  let block = await db.get(Block, { where: { block: blkHeight } })
-
-  if (!block) {
-    block = new Block({
-      network: Network.BABYLON,
-      block: blkHeight,
-      // TODO: upgrade indexer-lib which support block timestamp: substrateEvent.timestamp
-      timestamp: new BN(Date.now()),
-    })
-    await db.save<Block>(block)
-  }
+  let block = await prepareBlock(db, {blockNumber: blkHeight, blockTimestamp: new BN(Date.now())} as any)
 
   for (let i = 0; i < num; i++) {
     const profileOpt = (await api.query.members.memberProfile.at(blkHash, i)) as Option<Profile & Codec>
@@ -47,9 +41,9 @@ export async function members_bootMembers(api: ApiPromise, db: DB): Promise<void
     member.controllerAccount = profile.controller_account.toString()
     member.registeredAtBlock = block
 
-    logger.trace(`Saving member: ${JSON.stringify(member, null, 2)}`)
+    //logger.trace(`Saving member: ${JSON.stringify(member, null, 2)}`)
     await db.save<Membership>(member)
-    logger.info(`Saved members: ${i}/${num}`)
+    //logger.info(`Saved members: ${i}/${num}`)
   }
-  logger.info(`Done bootstrapping members!`)
+  //logger.info(`Done bootstrapping members!`)
 }

query-node/mappings/common.ts

@@ -0,0 +1,22 @@
+import { SubstrateEvent } from '@dzlzv/hydra-common'
+import { DatabaseManager } from '@dzlzv/hydra-db-utils'
+
+import { Block } from 'query-node/dist/src/modules/block/block.model'
+import { Network } from 'query-node/src/modules/enums/enums'
+
+const currentNetwork = Network.BABYLON
+
+// prepare block record
+export async function prepareBlock(db: DatabaseManager, event: SubstrateEvent): Promise<Block> {
+  let block = await db.get(Block, { where: { block: event.blockNumber } })
+
+  if (block) {
+      return block
+  }
+
+  return new Block({
+    block: event.blockNumber,
+    executedAt: new Date(event.blockTimestamp.toNumber()),
+    network: currentNetwork,
+  })
+}

query-node/mappings/index.ts

@@ -1,3 +1,2 @@
-export * from './mappings'
 export * from './mappingsContent'
 export * from './membership'

query-node/mappings/mappings.ts

@@ -1,26 +0,0 @@
-import { SubstrateEvent } from '@dzlzv/hydra-common'
-import { DatabaseManager } from '@dzlzv/hydra-db-utils'
-
-// eslint-disable-next-line @typescript-eslint/naming-convention
-export async function members_MemberRegistered(db: DatabaseManager, event_: SubstrateEvent): Promise<void> {
-}
-
-// eslint-disable-next-line @typescript-eslint/naming-convention
-export async function members_MemberUpdatedAboutText(db: DatabaseManager, event_: SubstrateEvent): Promise<void> {
-}
-
-// eslint-disable-next-line @typescript-eslint/naming-convention
-export async function members_MemberUpdatedAvatar(db: DatabaseManager, event_: SubstrateEvent): Promise<void> {
-}
-
-// eslint-disable-next-line @typescript-eslint/naming-convention
-export async function members_MemberUpdatedHandle(db: DatabaseManager, event_: SubstrateEvent): Promise<void> {
-}
-
-// eslint-disable-next-line @typescript-eslint/naming-convention
-export async function members_MemberSetRootAccount(db: DatabaseManager, event_: SubstrateEvent): Promise<void> {
-}
-
-// eslint-disable-next-line @typescript-eslint/naming-convention
-export async function members_MemberSetControllerAccount(db: DatabaseManager, event_: SubstrateEvent): Promise<void> {
-}

query-node/mappings/mappingsContent.ts

@@ -44,8 +44,9 @@ import {
 } from 'query-node'
 */
 
+import { prepareBlock } from './common'
+
 // primary entities
-import { Network } from 'query-node/src/modules/enums/enums'
 import { Block } from 'query-node/dist/src/modules/block/block.model'
 import { Channel } from 'query-node/dist/src/modules/channel/channel.model'
 import { ChannelCategory } from 'query-node/dist/src/modules/channel-category/channel-category.model'
@@ -80,8 +81,6 @@ import {
 type RawAsset = ReturnType<typeof contentDirectory.NewAsset.create>
 type RawAssetTypes = typeof contentDirectory.NewAsset.typeDefinitions
 
-const currentNetwork = Network.BABYLON
-
 /////////////////// Utils //////////////////////////////////////////////////////
 
 async function readProtobuf(
@@ -89,6 +88,7 @@ async function readProtobuf(
   metadata: Uint8Array,
   assets: RawAsset[],
   db: DatabaseManager,
+  event: SubstrateEvent,
 ): Promise<Partial<typeof type>> {
   // process channel
   if (type instanceof Channel) {
@@ -98,12 +98,12 @@ async function readProtobuf(
 
     // prepare cover photo asset if needed
     if (metaAsObject.coverPhoto !== undefined) {
-      result.coverPhoto = extractAsset(metaAsObject.coverPhoto, assets)
+      result.coverPhoto = await extractAsset(metaAsObject.coverPhoto, assets, db, event)
     }
 
     // prepare avatar photo asset if needed
     if (metaAsObject.avatarPhoto !== undefined) {
-      result.avatarPhoto = extractAsset(metaAsObject.avatarPhoto, assets)
+      result.avatarPhoto = await extractAsset(metaAsObject.avatarPhoto, assets, db, event)
     }
 
     // prepare language if needed
@@ -144,12 +144,12 @@ async function readProtobuf(
 
     // prepare thumbnail photo asset if needed
     if (metaAsObject.thumbnailPhoto !== undefined) {
-      result.thumbnail = extractAsset(metaAsObject.thumbnailPhoto, assets)
+      result.thumbnail = await extractAsset(metaAsObject.thumbnailPhoto, assets, db, event)
     }
 
     // prepare video asset if needed
     if (metaAsObject.video !== undefined) {
-      result.media = extractAsset(metaAsObject.video, assets)
+      result.media = await extractAsset(metaAsObject.video, assets, db, event)
     }
 
     // prepare language if needed
@@ -179,16 +179,7 @@ async function readProtobuf(
   throw `Not implemented type: ${type}`
 }
 
-// temporary function used before proper block is retrieved
-function convertBlockNumberToBlock(block: number): Block {
-  return new Block({
-    block: block,
-    executedAt: new Date(), // TODO get real block execution time
-    network: currentNetwork,
-  })
-}
-
-function convertAsset(rawAsset: RawAsset): typeof Asset {
+async function convertAsset(rawAsset: RawAsset, db: DatabaseManager, event: SubstrateEvent): Promise<typeof Asset> {
   if (rawAsset.type == 'Urls') {
     const assetUrl = new AssetUrl()
     assetUrl.url = rawAsset.asType('Urls').toArray()[0].toString() // TODO: find out why asUrl() returns array
@@ -206,7 +197,7 @@ function convertAsset(rawAsset: RawAsset): typeof Asset {
 
   const assetDataObject = new AssetDataObject({
     owner: assetOwner,
-    addedAt: convertBlockNumberToBlock(0), // TODO: proper addedAt
+    addedAt: await prepareBlock(db, event),
     typeId: contentParameters.type_id.toNumber(),
     size: 0, // TODO: retrieve proper file size
     liaisonId: 0, // TODO: proper id
@@ -227,7 +218,12 @@ function convertAsset(rawAsset: RawAsset): typeof Asset {
   return assetStorage
 }
 
-function extractAsset(assetIndex: number | undefined, assets: RawAsset[]): typeof Asset | undefined {
+async function extractAsset(
+  assetIndex: number | undefined,
+  assets: RawAsset[],
+  db: DatabaseManager,
+  event: SubstrateEvent,
+): Promise<typeof Asset | undefined> {
   if (assetIndex === undefined) {
     return undefined
   }
@@ -236,7 +232,7 @@ function extractAsset(assetIndex: number | undefined, assets: RawAsset[]): typeo
     throw 'Inconsistent state' // TODO: more sophisticated inconsistency handling; unify handling with other critical errors
   }
 
-  return convertAsset(assets[assetIndex])
+  return convertAsset(assets[assetIndex], db, event)
 }
 
 async function prepareLanguage(languageIso: string, db: DatabaseManager): Promise<Language> {
@@ -307,13 +303,19 @@ export async function content_ChannelCreated(db: DatabaseManager, event: Substra
   */
 
   //const protobufContent = await readProtobuf(ProtobufEntity.Channel, (event.params[3].value as any).meta, event.params[2].value as any[], db) // TODO: get rid of `any` typecast
-  const protobufContent = await readProtobuf(new Channel(), (event.params[3].value as any).meta, event.params[2].value as any[], db) // TODO: get rid of `any` typecast
+  const protobufContent = await readProtobuf(
+    new Channel(),
+    (event.params[3].value as any).meta,
+    event.params[2].value as any[],
+    db,
+    event,
+  ) // TODO: get rid of `any` typecast
 
   const channel = new Channel({
     id: event.params[0].value.toString(), // ChannelId
     isCensored: false,
     videos: [],
-    happenedIn: convertBlockNumberToBlock(event.blockNumber),
+    happenedIn: await prepareBlock(db, event),
     ...Object(protobufContent)
   })
 
@@ -343,7 +345,8 @@ export async function content_ChannelUpdated(
     new Channel(),
     (event.params[3].value as any).new_meta,
     (event.params[3].value as any).assets,
-    db
+    db,
+    event,
   ) // TODO: get rid of `any` typecast
 
   // update all fields read from protobuf
@@ -448,13 +451,14 @@ export async function content_ChannelCategoryCreated(
     new ChannelCategory(),
     (event.params[2].value as any).meta,
     [],
-    db
+    db,
+    event,
   ) // TODO: get rid of `any` typecast
 
   const channelCategory = new ChannelCategory({
     id: event.params[0].value.toString(), // ChannelCategoryId
     channels: [],
-    happenedIn: convertBlockNumberToBlock(event.blockNumber),
+    happenedIn: await prepareBlock(db, event),
     ...Object(protobufContent)
   })
 
@@ -483,7 +487,8 @@ export async function content_ChannelCategoryUpdated(
     new ChannelCategory(),
     (event.params[2].value as any).meta,
     [],
-    db
+    db,
+    event,
   ) // TODO: get rid of `any` typecast
 
   // update all fields read from protobuf
@@ -526,13 +531,19 @@ export async function content_VideoCategoryCreated(
   VideoCategoryCreationParameters,
   */
 
-  const protobufContent = readProtobuf(new VideoCategory(), (event.params[2].value as any).meta, [], db) // TODO: get rid of `any` typecast
+  const protobufContent = readProtobuf(
+    new VideoCategory(),
+    (event.params[2].value as any).meta,
+    [],
+    db,
+    event
+  ) // TODO: get rid of `any` typecast
 
   const videoCategory = new VideoCategory({
     id: event.params[0].value.toString(), // ChannelId
     isCensored: false,
     videos: [],
-    happenedIn: convertBlockNumberToBlock(event.blockNumber),
+    happenedIn: await prepareBlock(db, event),
     ...Object(protobufContent)
   })
 
@@ -561,7 +572,8 @@ export async function content_VideoCategoryUpdated(
     new VideoCategory(),
     (event.params[2].value as any).meta,
     [],
-    db
+    db,
+    event,
   ) // TODO: get rid of `any` typecast
 
   // update all fields read from protobuf
@@ -610,14 +622,15 @@ export async function content_VideoCreated(
     new Video(),
     (event.params[3].value as any).meta,
     (event.params[3].value as any).assets,
-    db
+    db,
+    event,
   ) // TODO: get rid of `any` typecast
 
   const channel = new Video({
     id: event.params[2].toString(), // ChannelId
     isCensored: false,
     channel: event.params[1],
-    happenedIn: convertBlockNumberToBlock(event.blockNumber),
+    happenedIn: await prepareBlock(db, event),
     ...Object(protobufContent)
   })
 
@@ -645,7 +658,8 @@ export async function content_VideoUpdated(
     new Video(),
     (event.params[2].value as any).meta,
     (event.params[2].value as any).assets,
-    db
+    db,
+    event,
   ) // TODO: get rid of `any` typecast
 
   // update all fields read from protobuf

query-node/mappings/membership.ts

@@ -1,11 +1,13 @@
 import BN from 'bn.js'
 import { Bytes } from '@polkadot/types'
 import { MemberId } from '@joystream/types/members'
-import { DatabaseManager, SubstrateEvent } from '@dzlzv/hydra-indexer-lib/lib'
+import { SubstrateEvent } from '@dzlzv/hydra-common'
+import { DatabaseManager } from '@dzlzv/hydra-db-utils'
 
+import { prepareBlock } from './common'
 import { Members } from '../generated/types'
-import { EntryMethod, Membership } from '../generated/graphql-server/src/modules/membership/membership.model'
-import { Block, Network } from '../generated/graphql-server/src/modules/block/block.model'
+import { MembershipEntryMethod, Membership } from 'query-node/dist/src/modules/membership/membership.model'
+import { Block } from 'query-node/dist/src/modules/block/block.model'
 
 async function getMemberById(db: DatabaseManager, id: MemberId): Promise<Membership> {
   const member = await db.get(Membership, { where: { id: id.toString() } })
@@ -22,18 +24,6 @@ export async function members_MemberRegistered(db: DatabaseManager, event_: Subs
   const { accountId, memberId } = new Members.MemberRegisteredEvent(event_).data
   const { avatarUri, about, handle } = new Members.BuyMembershipCall(event_).args
 
-  let block = await db.get(Block, { where: { block: event_.blockNumber } })
-
-  if (!block) {
-    block = new Block({
-      network: Network.BABYLON,
-      block: event_.blockNumber,
-      // TODO: upgrade indexer-lib which support block timestamp: substrateEvent.timestamp
-      timestamp: new BN(Date.now()),
-    })
-    await db.save<Block>(block)
-  }
-
   const member = new Membership({
     id: memberId.toString(),
     rootAccount: accountId.toString(),
@@ -41,11 +31,10 @@ export async function members_MemberRegistered(db: DatabaseManager, event_: Subs
     handle: convertBytesToString(handle.unwrap()),
     about: convertBytesToString(about.unwrap()),
     avatarUri: convertBytesToString(avatarUri.unwrap()),
-    registeredAtBlock: block,
+    registeredAtBlock: await prepareBlock(db, event_),
     // TODO: upgrade indexer-lib which support block timestamp: substrateEvent.timestamp
-    registeredAtTime: new Date(),
-    entry: EntryMethod.PAID, // TODO?: callArgs.paidTermsId
-    suspended: false,
+    registeredAtTime: new Date(event_.blockTimestamp.toNumber()),
+    entry: MembershipEntryMethod.PAID, // TODO?: callArgs.paidTermsId
   })
 
   await db.save<Membership>(member)