query node mappings - rough implementation II

query-node/mappings/mappingsContent.ts

@@ -1,11 +1,23 @@
 // TODO: add logging of mapping events (entity found/not found, entity updated/deleted, etc.)
 // TODO: update event list - some events were added/removed recently and are missing in this file
 // TODO: handling of Language, MediaType, etc.
+// TODO: fix TS imports from joystream packages
+// TODO: split file into multiple files
 
 import { SubstrateEvent } from '@dzlzv/hydra-common'
 import { DatabaseManager } from '@dzlzv/hydra-db-utils'
 
 // protobuf definitions
+import {
+  ChannelMetadata,
+  ChannelCategoryMetadata,
+  PublishedBeforeJoystream as PublishedBeforeJoystreamMetadata,
+  License as LicenseMetadata,
+  MediaType as MediaTypeMetadata,
+  VideoMetadata,
+  VideoCategoryMetadata,
+} from '@joystream/content-metadata-protobuf'
+/*
 import {
   ChannelMetadata,
   ChannelCategoryMetadata
@@ -17,7 +29,39 @@ import {
   VideoMetadata,
   VideoCategoryMetadata,
 } from '../../content-metadata-protobuf/compiled/proto/Video_pb'
+*/
 
+import {
+  // primary entites
+  Network,
+  Block,
+  Channel,
+  ChannelCategory,
+  Video,
+  VideoCategory,
+
+  // secondary entities
+  Language,
+  License,
+  MediaType,
+  VideoMediaEncoding,
+  VideoMediaMetadata,
+
+  // Asset
+  Asset,
+  AssetUrl,
+  AssetUploadStatus,
+  AssetDataObject,
+  LiaisonJudgement,
+  AssetStorage,
+  AssetOwner,
+  AssetOwnerMember,
+} from 'query-node'
+
+import {
+  contentDirectory
+} from '@joystream/types'
+/*
 // enums
 import { Network } from '../generated/graphql-server/src/modules/enums/enums'
 
@@ -27,10 +71,12 @@ import { Channel } from '../generated/graphql-server/src/modules/channel/channel
 import { ChannelCategory } from '../generated/graphql-server/src/modules/channelCategory/channelCategory.model'
 import { Video } from '../generated/graphql-server/src/modules/video/video.model'
 import { VideoCategory } from '../generated/graphql-server/src/modules/videoCategory/videoCategory.model'
-
+*/
 
 const currentNetwork = Network.BABYLON
 
+/////////////////// Utils //////////////////////////////////////////////////////
+
 enum ProtobufEntity {
   Channel,
   ChannelCategory,
@@ -38,41 +84,104 @@ enum ProtobufEntity {
   VideoCategory,
 }
 
-function readProtobuf(type: ProtobufEntity, metadata: Uint8Array) {
+// TODO: tweak generic types to make them actually work
+//function readProtobuf(type: ProtobufEntity, metadata: Uint8Array) {
+async function readProtobuf<T extends ProtobufEntity>(
+  type: ProtobufEntity,
+  metadata: Uint8Array,
+  assets: contentDirectory.RawAsset[],
+  db: DatabaseManager,
+): Promise<Partial<T>> {
   // TODO: consider getting rid of this function - it makes sense to keep it only complex logic will be executed here
   //       for example retriving language for channel, retrieving new assets (channel photo), etc.
 
+  // process channel
   if (type == ProtobufEntity.Channel) {
-    return ChannelMetadata.deserializeBinary(metadata)
-    /*
-    return {
-      title: 'TODO handle', // TODO: read from protobuf
-      description: 'TODO description', // TODO: read from protobuf
-      coverPhoto: undefined, // TODO: read from protobuf
-      avatarPhoto: undefined, // TODO: read from protobuf
-      isPublic: true, // TODO: read from protobuf
-      language: undefined, // TODO: read language from protobuf and connect it with existing Language (if any)
+    const meta = ChannelMetadata.deserializeBinary(metadata)
+    const result = meta.toObject()
+
+    // prepare cover photo asset if needed
+    if (result.coverPhoto !== undefined) {
+      result.coverPhoto = extractAsset(result.coverPhoto, assets)
+    }
+
+    // prepare avatar photo asset if needed
+    if (result.avatarPhoto !== undefined) {
+      result.avatarPhoto = extractAsset(result.avatarPhoto, assets)
+    }
+
+    // prepare language if needed
+    if (result.language) {
+      result.language = await prepareLanguage(result.language, db)
     }
-    */
+
+    return result
   }
 
+  // process channel category
   if (type == ProtobufEntity.ChannelCategory) {
-    return ChannelCategoryMetadata.deserializeBinary(metadata)
+    return ChannelCategoryMetadata.deserializeBinary(metadata).toObject()
   }
 
+  // process video
   if (type == ProtobufEntity.Video) {
-    return VideoMetadata.deserializeBinary(metadata)
+    const meta = VideoMetadata.deserializeBinary(metadata)
+    const result = meta.toObject()
+
+    // prepare video category if needed
+    if (result.category !== undefined) {
+      result.category = prepareVideoCategory(result.category, db)
+    }
+
+    // prepare media meta information if needed
+    if (result.mediaType) {
+      result.mediaType = prepareVideoMetadata(result)
+    }
+
+    // prepare license if needed
+    if (result.license) {
+      result.license = prepareLicense(result.license)
+    }
+
+    // prepare thumbnail photo asset if needed
+    if (result.thumbnail !== undefined) {
+      result.thumbnail = extractAsset(result.thumbnail, assets)
+    }
+
+    // prepare video asset if needed
+    if (result.media !== undefined) {
+      result.media = extractAsset(result.media, assets)
+    }
+
+    // prepare language if needed
+    if (result.language) {
+      result.language = await prepareLanguage(result.language, db)
+    }
+
+    // prepare information about media published somewhere else before Joystream if needed.
+    if (result.publishedBeforeJoystream) {
+      // TODO: is ok to just ignore `isPublished?: boolean` here?
+      if (result.publishedBeforeJoystream.hasDate()) {
+        result.publishedBeforeJoystream = new Date(result.publishedBeforeJoystream.getDate())
+      } else {
+        delete result.publishedBeforeJoystream
+      }
+    }
+
+    return result
   }
 
+  // process video category
   if (type == ProtobufEntity.VideoCategory) {
-    return VideoCategoryMetadata.deserializeBinary(metadata)
+    return VideoCategoryMetadata.deserializeBinary(metadata).toObject()
   }
 
+  // this should never happen
   throw `Not implemented type: ${type}`
 }
 
 // temporary function used before proper block is retrieved
-function convertblockNumberToBlock(block: number): Block {
+function convertBlockNumberToBlock(block: number): Block {
   return new Block({
     block: block,
     executedAt: new Date(), // TODO get real block execution time
@@ -80,6 +189,110 @@ function convertblockNumberToBlock(block: number): Block {
   })
 }
 
+function convertAsset(rawAsset: contentDirectory.RawAsset): Asset {
+  if (rawAsset.isUrl) {
+    const assetUrl = new AssetUrl({
+      url: rawAsset.asUrl()[0] // TODO: find out why asUrl() returns array
+    })
+
+    const asset = new Asset(assetUrl) // TODO: make sure this is a proper way to initialize Asset (on all places)
+
+    return asset
+  }
+
+  // !rawAsset.isUrl && rawAsset.isUpload
+
+  const contentParameters: contentDirectory.ContentParameters = rawAsset.asStorage()
+
+  const assetOwner = new AssetOwner(new AssetOwnerMember(0)) // TODO: proper owner
+  const assetDataObject = new AssetDataObject({
+    owner: new AssetOwner(),
+    addedAt: convertBlockNumberToBlock(0), // TODO: proper addedAt
+    typeId: contentParameters.type_id,
+    size: 0, // TODO: retrieve proper file size
+    liaisonId: 0, // TODO: proper id
+    liaisonJudgement: LiaisonJudgement.PENDING, // TODO: proper judgement
+    ipfsContentId: contentParameters.ipfs_content_id,
+    joystreamContentId: contentParameters.content_id,
+  })
+  // TODO: handle `AssetNeverProvided` and `AssetDeleted` states
+  const uploadingStatus = new AssetUploadStatus({
+    dataObject: new AssetDataObject,
+    oldDataObject: undefined // TODO: handle oldDataObject
+  })
+
+  const assetStorage = new AssetStorage({
+    uploadStatus: uploadingStatus
+  })
+  const asset = new Asset(assetStorage)
+
+  return asset
+}
+
+function extractAsset(assetIndex: number | undefined, assets: contentDirectory.RawAsset[]): Asset | undefined {
+  if (assetIndex === undefined) {
+    return undefined
+  }
+
+  if (assetIndex > assets.length) {
+    throw 'Inconsistent state' // TODO: more sophisticated inconsistency handling; unify handling with other critical errors
+  }
+
+  return convertAsset(assets[assetIndex])
+}
+
+async function prepareLanguage(languageIso: string, db: DatabaseManager): Promise<Language> {
+  // TODO: ensure language is ISO name
+  const isValidIso = true;
+
+  if (!isValidIso) {
+    throw // TODO: create a proper way of handling inconsistent state
+  }
+
+  const language = await db.get(Language, { where: { iso: languageIso }})
+
+  if (language) {
+    return language;
+  }
+
+  const newLanguage = new Language({
+    iso: languageIso
+  })
+
+  return newLanguage
+}
+
+async function prepareLicense(licenseProtobuf: LicenseMetadata.AsObject): Promise<License> {
+  // TODO: add old license removal (when existing) or rework the whole function
+
+  const license = new License(licenseProtobuf.toObject())
+
+  return license
+}
+
+async function prepareVideoMetadata(videoProtobuf: VideoMetadata.AsObject): Promise<MediaType> {
+  const encoding = new VideoMediaEncoding(videoProtobuf.mediaType)
+
+  const videoMeta = new VideoMediaMetadata({
+    encoding,
+    pixelWidth: videoProtobuf.mediaPixelWidth,
+    pixelHeight: videoProtobuf.mediaPixelHeight,
+    size: 0, // TODO: retrieve proper file size
+  })
+
+  return videoMeta
+}
+
+async function prepareVideoCategory(categoryId: number, db: DatabaseManager): Promise<VideoCategory> {
+  const category = await db.get(VideoCategory, { where: { id: categoryId }})
+
+  if (!category) {
+    throw // TODO: create a proper way of handling inconsistent state
+  }
+
+  return category
+}
+
 /////////////////// Channel ////////////////////////////////////////////////////
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
@@ -91,14 +304,14 @@ export async function content_ChannelCreated(db: DatabaseManager, event: Substra
   ChannelCreationParameters<ContentParameters>,
   */
 
-  const protobufContent = readProtobuf(ProtobufEntity.Channel, (event.params[3] as any).meta) // TODO: get rid of `any` typecast
+  const protobufContent = await readProtobuf(ProtobufEntity.Channel, (event.params[3].value as any).meta, event.params[2].value as any[], db) // TODO: get rid of `any` typecast
 
   const channel = new Channel({
-    id: event.params[0].toString(), // ChannelId
+    id: event.params[0].value.toString(), // ChannelId
     isCensored: false,
     videos: [],
-    happenedIn: convertblockNumberToBlock(event.blockNumber),
-    ...protobufContent.toObject()
+    happenedIn: convertBlockNumberToBlock(event.blockNumber),
+    ...Object(protobufContent)
   })
 
   await db.save<Channel>(channel)
@@ -116,18 +329,20 @@ export async function content_ChannelUpdated(
   ChannelUpdateParameters<ContentParameters, AccountId>,
   */
 
-  const channelId = event.params[1].toString()
+  const channelId = event.params[1].value.toString()
   const channel = await db.get(Channel, { where: { id: channelId } })
 
-  const protobufContent = readProtobuf(ProtobufEntity.Channel, (event.params[3] as any).meta) // TODO: get rid of `any` typecast
+  if (!channel) {
+    throw // TODO: create a proper way of handling inconsistent state
+  }
 
-  const updatedChannel = new Channel({
-    // TODO: check that this kind of new updated channel construction is valid or it should rather be edited as `channel.myProperty = something`
-    ...channel,
-    ...protobufContent.toObject()
-  })
+  const protobufContent = await readProtobuf(ProtobufEntity.Channel, (event.params[3].value as any).new_meta, (event.params[3].value as any).assets, db) // TODO: get rid of `any` typecast
 
-  await db.save<Channel>(updatedChannel)
+  for (let [key, value] of Object(protobufContent).entries()) {
+    channel[key] = value
+  }
+
+  await db.save<Channel>(channel)
 }
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
@@ -135,7 +350,7 @@ export async function content_ChannelDeleted(
   db: DatabaseManager,
   event: SubstrateEvent
 ) {
-  const channelId = event.params[1].toString()
+  const channelId = event.params[1].value.toString()
   const channel = await db.get(Channel, { where: { id: channelId } })
 
   await db.remove<Channel>(channel)
@@ -152,9 +367,13 @@ export async function content_ChannelCensored(
   Vec<u8>
   */
 
-  const channelId = event.params[1].toString()
+  const channelId = event.params[1].value.toString()
   const channel = await db.get(Channel, { where: { id: channelId } })
 
+  if (!channel) {
+    throw // TODO: create a proper way of handling inconsistent state
+  }
+
   channel.isCensored = true;
 
   await db.save<Channel>(channel)
@@ -171,9 +390,13 @@ export async function content_ChannelUncensored(
   Vec<u8>
   */
 
-  const channelId = event.params[1].toString()
+  const channelId = event.params[1].value.toString()
   const channel = await db.get(Channel, { where: { id: channelId } })
 
+  if (!channel) {
+    throw // TODO: create a proper way of handling inconsistent state
+  }
+
   channel.isCensored = false;
 
   await db.save<Channel>(channel)
@@ -216,13 +439,13 @@ export async function content_ChannelCategoryCreated(
   ChannelCategoryCreationParameters,
   */
 
-  const protobufContent = readProtobuf(ProtobufEntity.ChannelCategory, (event.params[2] as any).meta) // TODO: get rid of `any` typecast
+  const protobufContent = await readProtobuf(ProtobufEntity.ChannelCategory, (event.params[2].value as any).meta, [], db) // TODO: get rid of `any` typecast
 
   const channelCategory = new ChannelCategory({
-    id: event.params[0].toString(), // ChannelCategoryId
+    id: event.params[0].value.toString(), // ChannelCategoryId
     channels: [],
-    happenedIn: convertblockNumberToBlock(event.blockNumber),
-    ...protobufContent.toObject()
+    happenedIn: convertBlockNumberToBlock(event.blockNumber),
+    ...Object(protobufContent)
   })
 
   await db.save<ChannelCategory>(channelCategory)
@@ -239,16 +462,18 @@ export async function content_ChannelCategoryUpdated(
   ChannelCategoryUpdateParameters,
   */
 
-  const channelCategoryId = event.params[1].toString()
+  const channelCategoryId = event.params[1].value.toString()
   const channelCategory = await db.get(ChannelCategory, { where: { id: channelCategoryId } })
 
-  const protobufContent = readProtobuf(ProtobufEntity.ChannelCategory, (event.params[2] as any).meta) // TODO: get rid of `any` typecast
+  if (!channelCategory) {
+    throw // TODO: create a proper way of handling inconsistent state
+  }
 
-  const updatedChannelCategory = new ChannelCategory({
-    // TODO: check that this kind of new updated channel construction is valid or it should rather be edited as `channel.myProperty = something`
-    ...channelCategory,
-    ...protobufContent.toObject()
-  })
+  const protobufContent = await readProtobuf(ProtobufEntity.ChannelCategory, (event.params[2].value as any).meta, [], db) // TODO: get rid of `any` typecast
+
+  for (let [key, value] of Object(protobufContent).entries()) {
+    channelCategory[key] = value
+  }
 
   await db.save<ChannelCategory>(channelCategory)
 }
@@ -262,7 +487,7 @@ export async function content_ChannelCategoryDeleted(
   ContentActor,
   ChannelCategoryId
   */
-  const channelCategoryId = event.params[1].toString()
+  const channelCategoryId = event.params[1].value.toString()
   const channelCategory = await db.get(ChannelCategory, { where: { id: channelCategoryId } })
 
   await db.remove<ChannelCategory>(channelCategory)
@@ -281,14 +506,14 @@ export async function content_VideoCategoryCreated(
   VideoCategoryCreationParameters,
   */
 
-  const protobufContent = readProtobuf(ProtobufEntity.VideoCategory, (event.params[2] as any).meta) // TODO: get rid of `any` typecast
+  const protobufContent = readProtobuf(ProtobufEntity.VideoCategory, (event.params[2].value as any).meta, [], db) // TODO: get rid of `any` typecast
 
   const videoCategory = new VideoCategory({
-    id: event.params[0].toString(), // ChannelId
-    isCensored: false, // TODO: where this value comes from?
+    id: event.params[0].value.toString(), // ChannelId
+    isCensored: false,
     videos: [],
-    happenedIn: convertblockNumberToBlock(event.blockNumber),
-    ...protobufContent.toObject()
+    happenedIn: convertBlockNumberToBlock(event.blockNumber),
+    ...Object(protobufContent)
   })
 
   await db.save<VideoCategory>(videoCategory)
@@ -308,13 +533,15 @@ export async function content_VideoCategoryUpdated(
   const videoCategoryId = event.params[1].toString()
   const videoCategory = await db.get(VideoCategory, { where: { id: videoCategoryId } })
 
-  const protobufContent = readProtobuf(ProtobufEntity.VideoCategory, (event.params[2] as any).meta) // TODO: get rid of `any` typecast
+  if (!videoCategory) {
+    throw // TODO: create a proper way of handling inconsistent state
+  }
 
-  const updatedVideoCategory = new VideoCategory({
-    // TODO: check that this kind of new updated channel construction is valid or it should rather be edited as `channel.myProperty = something`
-    ...videoCategory,
-    ...protobufContent.toObject()
-  })
+  const protobufContent = await readProtobuf(ProtobufEntity.VideoCategory, (event.params[2].value as any).meta, [], db) // TODO: get rid of `any` typecast
+
+  for (let [key, value] of Object(protobufContent).entries()) {
+    videoCategory[key] = value
+  }
 
   await db.save<VideoCategory>(videoCategory)
 }
@@ -349,14 +576,14 @@ export async function content_VideoCreated(
   VideoCreationParameters<ContentParameters>,
   */
 
-  const protobufContent = readProtobuf(ProtobufEntity.Video, (event.params[3] as any).meta) // TODO: get rid of `any` typecast
+  const protobufContent = await readProtobuf(ProtobufEntity.Video, (event.params[3].value as any).meta, (event.params[3].value as any).assets, db) // TODO: get rid of `any` typecast
 
   const channel = new Video({
-    id: event.params[1].toString(), // ChannelId
+    id: event.params[2].toString(), // ChannelId
     isCensored: false,
     channel: event.params[1],
-    happenedIn: convertblockNumberToBlock(event.blockNumber),
-    ...protobufContent.toObject()
+    happenedIn: convertBlockNumberToBlock(event.blockNumber),
+    ...Object(protobufContent)
   })
 
   await db.save<Video>(channel)
@@ -375,13 +602,15 @@ export async function content_VideoUpdated(
   const videoId = event.params[1].toString()
   const video = await db.get(Video, { where: { id: videoId } })
 
-  const protobufContent = readProtobuf(ProtobufEntity.Video, (event.params[2] as any).meta) // TODO: get rid of `any` typecast
+  if (!video) {
+    throw // TODO: create a proper way of handling inconsistent state
+  }
 
-  const updatedVideo = new Video({
-    // TODO: check that this kind of new updated channel construction is valid or it should rather be edited as `channel.myProperty = something`
-    ...video,
-    ...protobufContent.toObject()
-  })
+  const protobufContent = await readProtobuf(ProtobufEntity.Video, (event.params[2].value as any).meta, (event.params[2].value as any).assets, db) // TODO: get rid of `any` typecast
+
+  for (let [key, value] of Object(protobufContent).entries()) {
+    video[key] = value
+  }
 
   await db.save<Video>(video)
 }
@@ -450,15 +679,23 @@ export async function content_FeaturedVideosSet(
   Vec<VideoId>,
   */
 
-  const videoIds = event.params[1]
+  const videoIds = event.params[1].value as string[]
+  const existingFeaturedVideos = await db.getMany(Video, { where: { isFeatured: true } })
 
-  for (let videoId in videoIds) {
-    const video = await db.get(Video, { where: { id: videoId } })
+  const isSame = (videoA: Video) => (videoB: Video) => videoA.id == videoB
 
-    video.isFeatured = true;
+  const toRemove = existingFeaturedVideos.filter(existingFV => !videoIds.some(isSame(existingFV)))
+  const toAdd = videoIds.filter(video => !existingFeaturedVideos.some(isSame(video)))
+
+  for (let video in toRemove) {
+    video.isFeatured = false;
 
     await db.save<Video>(video)
   }
 
-  // TODO: remove featured flag from previously featured videos
+  for (let video in toAdd) {
+    video.isFeatured = true;
+
+    await db.save<Video>(video)
+  }
 }

query-node/mappings/package.json

@@ -12,6 +12,9 @@
   "dependencies": {
     "@dzlzv/hydra-common": "0.0.3",
     "@polkadot/types": "~2.10.2-7",
+    "@joystream/content-metadata-protobuf": "^1.0.0",
+    "@joystream/types": "^0.16.0",
+    "query-node": "^0.0.0",
     "warthog": "https://github.com/metmirr/warthog/releases/download/v2.23.0/warthog-v2.23.0.tgz"
   },
   "devDependencies": {

query-node/package.json

@@ -39,9 +39,9 @@
     "bn.js": "^5.1.2"
   },
   "devDependencies": {
-    "@dzlzv/hydra-cli": "2.1.0-beta.2",
-    "@dzlzv/hydra-processor": "0.1.1",
-    "@dzlzv/hydra-typegen": "0.0.3-1"
+    "@dzlzv/hydra-cli": "2.0.1-beta.3",
+    "@dzlzv/hydra-processor": "2.0.1-beta.3",
+    "@dzlzv/hydra-typegen": "2.0.1-beta.3"
   },
   "volta": {
 		"extends": "../package.json"