Merge branch 'babylon' into babylon-schema-adjustments

.dockerignore

@@ -1,4 +1,7 @@
-**target*
-**node_modules*
-.tmp/
-.vscode/
+**target*
+**node_modules*
+.tmp/
+.vscode/
+query-node/generated
+query-node/**/dist
+query-node/lib

package.json

@@ -37,9 +37,9 @@
     "@dzlzv/hydra-indexer-lib": "0.0.19-legacy.1.26.1"
   },
   "devDependencies": {
+    "eslint": "^7.6.0",
     "husky": "^4.2.5",
-    "prettier": "2.0.2",
-    "eslint": "^7.6.0"
+    "prettier": "2.0.2"
   },
   "husky": {
     "hooks": {

query-node/.dockerignore

@@ -1,4 +0,0 @@
-generated
-**/node_modules
-**/dist
-./lib/

query-node/build.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -e
+
+SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+cd $SCRIPT_PATH
+
+yarn clean
+yarn codegen:all
+yarn
+ln -s ../../../../../node_modules/typeorm/cli.js generated/graphql-server/node_modules/.bin/typeorm || :
+yarn tsc --build tsconfig.json

query-node/indexer-tsconfig.json

@@ -14,7 +14,8 @@
     "baseUrl": ".",
     "paths": {
       "@polkadot/types/augment": ["../../../types/augment-codec/augment-types.ts"]
-    }
+    },
+    "esModuleInterop": true
   },
   "exclude": ["node_modules"]
 }

query-node/mappings/content-directory/content-dir-consts.ts

@@ -4,13 +4,15 @@ import { IPropertyIdWithName } from '../types'
 export enum ContentDirectoryKnownClasses {
   CHANNEL = 'Channel',
   CATEGORY = 'Category',
+  HTTPMEDIALOCATION = 'HttpMediaLocation',
+  JOYSTREAMMEDIALOCATION = 'JoystreamMediaLocation',
   KNOWNLICENSE = 'KnownLicense',
+  LANGUAGE = 'Language',
+  LICENSE = 'License',
+  MEDIALOCATION = 'MediaLocation',
   USERDEFINEDLICENSE = 'UserDefinedLicense',
-  JOYSTREAMMEDIALOCATION = 'JoystreamMediaLocation',
-  HTTPMEDIALOCATION = 'HttpMediaLocation',
-  VIDEOMEDIA = 'VideoMedia',
   VIDEO = 'Video',
-  LANGUAGE = 'Language',
+  VIDEOMEDIA = 'VideoMedia',
   VIDEOMEDIAENCODING = 'VideoMediaEncoding',
 }
 
@@ -18,13 +20,15 @@ export enum ContentDirectoryKnownClasses {
 export const contentDirectoryClassNamesWithId: { classId: number; name: string }[] = [
   { name: ContentDirectoryKnownClasses.CHANNEL, classId: 1 },
   { name: ContentDirectoryKnownClasses.CATEGORY, classId: 2 },
+  { name: ContentDirectoryKnownClasses.HTTPMEDIALOCATION, classId: 3 },
+  { name: ContentDirectoryKnownClasses.JOYSTREAMMEDIALOCATION, classId: 4 },
   { name: ContentDirectoryKnownClasses.KNOWNLICENSE, classId: 5 },
-  { name: ContentDirectoryKnownClasses.USERDEFINEDLICENSE, classId: 9 },
   { name: ContentDirectoryKnownClasses.LANGUAGE, classId: 6 },
-  { name: ContentDirectoryKnownClasses.JOYSTREAMMEDIALOCATION, classId: 4 },
-  { name: ContentDirectoryKnownClasses.HTTPMEDIALOCATION, classId: 3 },
-  { name: ContentDirectoryKnownClasses.VIDEOMEDIA, classId: 11 },
+  { name: ContentDirectoryKnownClasses.LICENSE, classId: 7 },
+  { name: ContentDirectoryKnownClasses.MEDIALOCATION, classId: 8 },
+  { name: ContentDirectoryKnownClasses.USERDEFINEDLICENSE, classId: 9 },
   { name: ContentDirectoryKnownClasses.VIDEO, classId: 10 },
+  { name: ContentDirectoryKnownClasses.VIDEOMEDIA, classId: 11 },
   { name: ContentDirectoryKnownClasses.VIDEOMEDIAENCODING, classId: 12 },
 ]
 
@@ -43,6 +47,11 @@ export const channelPropertyNamesWithId: IPropertyIdWithName = {
   6: 'language',
 }
 
+export const licensePropertyNamesWithId: IPropertyIdWithName = {
+  0: 'knownLicense',
+  1: 'userDefinedLicense',
+}
+
 export const knownLicensePropertyNamesWIthId: IPropertyIdWithName = {
   0: 'code',
   1: 'name',
@@ -59,6 +68,11 @@ export const userDefinedLicensePropertyNamesWithId: IPropertyIdWithName = {
   0: 'content',
 }
 
+export const mediaLocationPropertyNamesWithId: IPropertyIdWithName = {
+  0: 'httpMediaLocation',
+  1: 'joystreamMediaLocation',
+}
+
 export const joystreamMediaLocationPropertyNamesWithId: IPropertyIdWithName = {
   0: 'dataObjectId',
 }

query-node/mappings/content-directory/decode.ts

@@ -6,11 +6,15 @@ import {
   IBatchOperation,
   ICreateEntityOperation,
   IEntity,
+  IReference,
 } from '../types'
+import Debug from 'debug'
 
 import { ParametrizedClassPropertyValue, UpdatePropertyValuesOperation } from '@joystream/types/content-directory'
 import { createType } from '@joystream/types'
 
+const debug = Debug('mappings:cd:decode')
+
 function stringIfyEntityId(event: SubstrateEvent): string {
   const { 1: entityId } = event.params
   return entityId.value as string
@@ -20,16 +24,18 @@ function setProperties<T>({ extrinsic, blockNumber }: SubstrateEvent, propNamesW
   if (extrinsic === undefined) throw Error('Undefined extrinsic')
 
   const { 3: newPropertyValues } = extrinsic!.args
-  const properties: { [key: string]: any } = {}
+  const properties: { [key: string]: any; reference?: IReference } = {}
 
   for (const [k, v] of Object.entries(newPropertyValues.value)) {
     const propertyName = propNamesWithId[k]
-    const propertyValue = createType('InputPropertyValue', v as any)
-      .asType('Single')
-      .value.toJSON()
-    properties[propertyName] = propertyValue
+    const singlePropVal = createType('InputPropertyValue', v as any).asType('Single')
+    properties[propertyName] = singlePropVal.isOfType('Reference')
+      ? { entityId: singlePropVal.asType('Reference').toJSON(), existing: true }
+      : singlePropVal.value.toJSON()
   }
   properties.version = blockNumber
+
+  debug(`Entity properties: ${JSON.stringify(properties)}`)
   return properties as T
 }
 
@@ -49,15 +55,15 @@ function getClassEntity(event: SubstrateEvent): IClassEntity {
  * @param propertyNamesWithId
  */
 function setEntityPropertyValues<T>(properties: IProperty[], propertyNamesWithId: IPropertyIdWithName): T {
-  const entityProperties: { [key: string]: any } = {}
+  const entityProperties: { [key: string]: any; reference?: IReference } = {}
 
   for (const [propId, propName] of Object.entries(propertyNamesWithId)) {
     // get the property value by id
-    const p = properties.find((p) => p.propertyId === propId)
-    const propertyValue = p ? p.value : undefined
-    entityProperties[propName] = propertyValue
+    const p = properties.find((p) => p.id === propId)
+    if (!p) continue
+    entityProperties[propName] = p.reference ? p.reference : p.value
   }
-  // console.log(entityProperties);
+  // debug(`Entity properties ${JSON.stringify(entityProperties)}`)
   return entityProperties as T
 }
 
@@ -70,20 +76,27 @@ function getEntityProperties(propertyValues: ParametrizedClassPropertyValue[]):
     const v = createType('ParametrizedPropertyValue', pv.value)
     const propertyId = pv.in_class_index.toJSON()
 
+    let reference
     let value
     if (v.isOfType('InputPropertyValue')) {
       const inputPropVal = v.asType('InputPropertyValue')
       value = inputPropVal.isOfType('Single')
         ? inputPropVal.asType('Single').value.toJSON()
         : inputPropVal.asType('Vector').value.toJSON()
+
+      if (inputPropVal.isOfType('Single')) {
+        if (inputPropVal.asType('Single').isOfType('Reference')) {
+          reference = { entityId: value as number, existing: true }
+        }
+      }
     } else if (v.isOfType('InternalEntityJustAdded')) {
-      // const inputPropVal = v.asType('InternalEntityJustAdded');
       value = v.asType('InternalEntityJustAdded').toJSON()
+      reference = { entityId: value as number, existing: false }
     } else {
       // TODO: Add support for v.asType('InternalEntityVec')
       throw Error('InternalEntityVec property type is not supported yet!')
     }
-    properties.push({ propertyId: `${propertyId}`, value })
+    properties.push({ id: `${propertyId}`, value, reference })
   })
   return properties
 }

query-node/mappings/content-directory/entity-helper.ts

@@ -1,476 +0,0 @@
-import { DB, SubstrateEvent } from '../../generated/indexer'
-import { Channel } from '../../generated/graphql-server/src/modules/channel/channel.model'
-import { Category } from '../../generated/graphql-server/src/modules/category/category.model'
-import { KnownLicense } from '../../generated/graphql-server/src/modules/known-license/known-license.model'
-import { UserDefinedLicense } from '../../generated/graphql-server/src/modules/user-defined-license/user-defined-license.model'
-import { JoystreamMediaLocation } from '../../generated/graphql-server/src/modules/joystream-media-location/joystream-media-location.model'
-import { HttpMediaLocation } from '../../generated/graphql-server/src/modules/http-media-location/http-media-location.model'
-import { VideoMedia } from '../../generated/graphql-server/src/modules/video-media/video-media.model'
-import { Video } from '../../generated/graphql-server/src/modules/video/video.model'
-import { Block, Network } from '../../generated/graphql-server/src/modules/block/block.model'
-import { Language } from '../../generated/graphql-server/src/modules/language/language.model'
-import { VideoMediaEncoding } from '../../generated/graphql-server/src/modules/video-media-encoding/video-media-encoding.model'
-import { ClassEntity } from '../../generated/graphql-server/src/modules/class-entity/class-entity.model'
-import { decode } from './decode'
-import {
-  CategoryPropertyNamesWithId,
-  channelPropertyNamesWithId,
-  httpMediaLocationPropertyNamesWithId,
-  joystreamMediaLocationPropertyNamesWithId,
-  knownLicensePropertyNamesWIthId,
-  languagePropertyNamesWIthId,
-  userDefinedLicensePropertyNamesWithId,
-  videoMediaEncodingPropertyNamesWithId,
-  videoPropertyNamesWithId,
-  contentDirectoryClassNamesWithId,
-  ContentDirectoryKnownClasses,
-} from './content-dir-consts'
-import {
-  ICategory,
-  IChannel,
-  ICreateEntityOperation,
-  IDBBlockId,
-  IEntity,
-  IHttpMediaLocation,
-  IJoystreamMediaLocation,
-  IKnownLicense,
-  ILanguage,
-  IUserDefinedLicense,
-  IVideo,
-  IVideoMedia,
-  IVideoMediaEncoding,
-  IWhereCond,
-} from '../types'
-
-async function createBlockOrGetFromDatabase(db: DB, blockNumber: number): Promise<Block> {
-  let b = await db.get(Block, { where: { block: blockNumber } })
-  if (b === undefined) {
-    // TODO: get timestamp from the event or extrinsic
-    b = new Block({ block: blockNumber, nework: Network.BABYLON, timestamp: 123 })
-    await db.save<Block>(b)
-  }
-  return b
-}
-
-async function createChannel({ db, block, id }: IDBBlockId, p: IChannel): Promise<void> {
-  // const { properties: p } = decode.channelEntity(event);
-  const channel = new Channel()
-
-  channel.version = block
-  channel.id = id
-  channel.title = p.title
-  channel.description = p.description
-  channel.isCurated = p.isCurated || false
-  channel.isPublic = p.isPublic
-  channel.coverPhotoUrl = p.coverPhotoURL
-  channel.avatarPhotoUrl = p.avatarPhotoURL
-  channel.languageId = p.language
-  channel.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save(channel)
-}
-
-async function createCategory({ db, block, id }: IDBBlockId, p: ICategory): Promise<void> {
-  // const p = decode.categoryEntity(event);
-  const category = new Category()
-
-  category.id = id
-  category.name = p.name
-  category.description = p.description
-  category.version = block
-  category.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save(category)
-}
-
-async function createKnownLicense({ db, block, id }: IDBBlockId, p: IKnownLicense): Promise<void> {
-  const knownLicence = new KnownLicense()
-
-  knownLicence.id = id
-  knownLicence.code = p.code
-  knownLicence.name = p.name
-  knownLicence.description = p.description
-  knownLicence.url = p.url
-  knownLicence.version = block
-  knownLicence.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save(knownLicence)
-}
-
-async function createUserDefinedLicense({ db, block, id }: IDBBlockId, p: IUserDefinedLicense): Promise<void> {
-  const userDefinedLicense = new UserDefinedLicense()
-
-  userDefinedLicense.id = id
-  userDefinedLicense.content = p.content
-  userDefinedLicense.version = block
-  userDefinedLicense.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save(userDefinedLicense)
-}
-
-async function createJoystreamMediaLocation({ db, block, id }: IDBBlockId, p: IJoystreamMediaLocation): Promise<void> {
-  const joyMediaLoc = new JoystreamMediaLocation()
-
-  joyMediaLoc.id = id
-  joyMediaLoc.dataObjectId = p.dataObjectId
-  joyMediaLoc.version = block
-  joyMediaLoc.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save(joyMediaLoc)
-}
-
-async function createHttpMediaLocation({ db, block, id }: IDBBlockId, p: IHttpMediaLocation): Promise<void> {
-  const httpMediaLoc = new HttpMediaLocation()
-
-  httpMediaLoc.id = id
-  httpMediaLoc.url = p.url
-  httpMediaLoc.port = p.port
-  httpMediaLoc.version = block
-  httpMediaLoc.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save(httpMediaLoc)
-}
-
-async function createVideoMedia({ db, block, id }: IDBBlockId, p: IVideoMedia): Promise<void> {
-  const videoMedia = new VideoMedia()
-
-  videoMedia.id = id
-  videoMedia.encodingId = p.encoding
-  videoMedia.locationId = p.location
-  videoMedia.pixelHeight = p.pixelHeight
-  videoMedia.pixelWidth = p.pixelWidth
-  videoMedia.size = p.size
-  videoMedia.version = block
-  videoMedia.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save(videoMedia)
-}
-
-async function createVideo({ db, block, id }: IDBBlockId, p: IVideo): Promise<void> {
-  const video = new Video()
-
-  video.id = id
-  video.title = p.title
-  video.description = p.description
-  video.categoryId = p.category
-  video.channelId = p.channel
-  video.duration = p.duration
-  video.hasMarketing = p.hasMarketing
-  // TODO: needs to be handled correctly, from runtime CurationStatus is coming
-  video.isCurated = p.isCurated || true
-  video.isExplicit = p.isExplicit
-  video.isPublic = p.isPublic
-  video.languageId = p.language
-  video.licenseId = p.license
-  video.videoMediaId = p.media
-  video.publishedBeforeJoystream = p.publishedBeforeJoystream
-  video.skippableIntroDuration = p.skippableIntroDuration
-  video.thumbnailUrl = p.thumbnailURL
-  video.version = block
-  video.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save<Video>(video)
-}
-
-async function createLanguage({ db, block, id }: IDBBlockId, p: ILanguage): Promise<void> {
-  const language = new Language()
-  language.id = id
-  language.name = p.name
-  language.code = p.code
-  language.version = block
-  language.happenedIn = await createBlockOrGetFromDatabase(db, block)
-
-  await db.save<Language>(language)
-}
-
-async function createVideoMediaEncoding({ db, block, id }: IDBBlockId, p: IVideoMediaEncoding): Promise<void> {
-  const encoding = new VideoMediaEncoding()
-
-  encoding.id = id
-  encoding.name = p.name
-  encoding.version = block
-  // happenedIn is not defined in the graphql schema!
-  // encoding.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save<VideoMediaEncoding>(encoding)
-}
-
-async function batchCreateClassEntities(db: DB, block: number, operations: ICreateEntityOperation[]): Promise<void> {
-  // Create entities before adding schema support
-  operations.map(async ({ classId }, index) => {
-    const c = new ClassEntity()
-    c.id = index.toString()
-    c.classId = classId
-    c.version = block
-    c.happenedIn = await createBlockOrGetFromDatabase(db, block)
-    await db.save<ClassEntity>(c)
-  })
-}
-
-async function getClassName(
-  db: DB,
-  entity: IEntity,
-  createEntityOperations: ICreateEntityOperation[]
-): Promise<string | undefined> {
-  const { entityId, indexOf } = entity
-  if (entityId === undefined && indexOf === undefined) {
-    throw Error(`Can not determine class of the entity`)
-  }
-
-  let classId: number | undefined
-  // Is newly created entity in the same transaction
-  if (indexOf !== undefined) {
-    classId = createEntityOperations[indexOf].classId
-  } else {
-    const ce = await db.get(ClassEntity, { where: { id: entityId } })
-    if (ce === undefined) console.log(`Class not found for the entity: ${entityId}`)
-    classId = ce ? ce.classId : undefined
-  }
-
-  const c = contentDirectoryClassNamesWithId.find((c) => c.classId === classId)
-  // TODO: stop execution, class should be created before entity creation
-  if (c === undefined) console.log(`Not recognized class id: ${classId}`)
-  return c ? c.name : undefined
-}
-
-async function removeChannel(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(Channel, where)
-  if (record === undefined) throw Error(`Channel not found`)
-  await db.remove<Channel>(record)
-}
-async function removeCategory(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(Category, where)
-  if (record === undefined) throw Error(`Category not found`)
-  await db.remove<Category>(record)
-}
-async function removeVideoMedia(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(VideoMedia, where)
-  if (record === undefined) throw Error(`VideoMedia not found`)
-  await db.remove<VideoMedia>(record)
-}
-async function removeVideo(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(Video, where)
-  if (record === undefined) throw Error(`Video not found`)
-  await db.remove<Video>(record)
-}
-async function removeUserDefinedLicense(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(UserDefinedLicense, where)
-  if (record === undefined) throw Error(`UserDefinedLicense not found`)
-  await db.remove<UserDefinedLicense>(record)
-}
-async function removeKnownLicense(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(KnownLicense, where)
-  if (record === undefined) throw Error(`KnownLicense not found`)
-  await db.remove<KnownLicense>(record)
-}
-async function removeHttpMediaLocation(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(HttpMediaLocation, where)
-  if (record === undefined) throw Error(`HttpMediaLocation not found`)
-  await db.remove<HttpMediaLocation>(record)
-}
-async function removeJoystreamMediaLocation(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(JoystreamMediaLocation, where)
-  if (record === undefined) throw Error(`JoystreamMediaLocation not found`)
-  await db.remove<JoystreamMediaLocation>(record)
-}
-async function removeLanguage(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(Language, where)
-  if (record === undefined) throw Error(`Language not found`)
-  await db.remove<Language>(record)
-}
-async function removeVideoMediaEncoding(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(VideoMediaEncoding, where)
-  if (record === undefined) throw Error(`Language not found`)
-  await db.remove<VideoMediaEncoding>(record)
-}
-
-// ========Entity property value updates========
-
-async function updateCategoryEntityPropertyValues(db: DB, where: IWhereCond, props: ICategory): Promise<void> {
-  const record = await db.get(Category, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<Category>(record)
-}
-async function updateChannelEntityPropertyValues(db: DB, where: IWhereCond, props: IChannel): Promise<void> {
-  const record = await db.get(Channel, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<Channel>(record)
-}
-async function updateVideoMediaEntityPropertyValues(db: DB, where: IWhereCond, props: IVideoMedia): Promise<void> {
-  const record = await db.get(VideoMedia, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<VideoMedia>(record)
-}
-async function updateVideoEntityPropertyValues(db: DB, where: IWhereCond, props: IVideo): Promise<void> {
-  const record = await db.get(Video, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<Video>(record)
-}
-async function updateUserDefinedLicenseEntityPropertyValues(
-  db: DB,
-  where: IWhereCond,
-  props: IUserDefinedLicense
-): Promise<void> {
-  const record = await db.get(UserDefinedLicense, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<UserDefinedLicense>(record)
-}
-async function updateKnownLicenseEntityPropertyValues(db: DB, where: IWhereCond, props: IKnownLicense): Promise<void> {
-  const record = await db.get(KnownLicense, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<KnownLicense>(record)
-}
-async function updateHttpMediaLocationEntityPropertyValues(
-  db: DB,
-  where: IWhereCond,
-  props: IHttpMediaLocation
-): Promise<void> {
-  const record = await db.get(HttpMediaLocation, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<HttpMediaLocation>(record)
-}
-async function updateJoystreamMediaLocationEntityPropertyValues(
-  db: DB,
-  where: IWhereCond,
-  props: IJoystreamMediaLocation
-): Promise<void> {
-  const record = await db.get(JoystreamMediaLocation, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<JoystreamMediaLocation>(record)
-}
-async function updateLanguageEntityPropertyValues(db: DB, where: IWhereCond, props: ILanguage): Promise<void> {
-  const record = await db.get(Language, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<Language>(record)
-}
-async function updateVideoMediaEncodingEntityPropertyValues(
-  db: DB,
-  where: IWhereCond,
-  props: IVideoMediaEncoding
-): Promise<void> {
-  const record = await db.get(VideoMediaEncoding, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<VideoMediaEncoding>(record)
-}
-
-async function updateEntityPropertyValues(
-  db: DB,
-  event: SubstrateEvent,
-  where: IWhereCond,
-  className: string
-): Promise<void> {
-  switch (className) {
-    case ContentDirectoryKnownClasses.CHANNEL:
-      updateChannelEntityPropertyValues(db, where, decode.setProperties<IChannel>(event, channelPropertyNamesWithId))
-      break
-
-    case ContentDirectoryKnownClasses.CATEGORY:
-      await updateCategoryEntityPropertyValues(
-        db,
-        where,
-        decode.setProperties<ICategory>(event, CategoryPropertyNamesWithId)
-      )
-      break
-
-    case ContentDirectoryKnownClasses.KNOWNLICENSE:
-      await updateKnownLicenseEntityPropertyValues(
-        db,
-        where,
-        decode.setProperties<IKnownLicense>(event, knownLicensePropertyNamesWIthId)
-      )
-      break
-
-    case ContentDirectoryKnownClasses.USERDEFINEDLICENSE:
-      await updateUserDefinedLicenseEntityPropertyValues(
-        db,
-        where,
-        decode.setProperties<IUserDefinedLicense>(event, userDefinedLicensePropertyNamesWithId)
-      )
-      break
-
-    case ContentDirectoryKnownClasses.JOYSTREAMMEDIALOCATION:
-      await updateJoystreamMediaLocationEntityPropertyValues(
-        db,
-        where,
-        decode.setProperties<IJoystreamMediaLocation>(event, joystreamMediaLocationPropertyNamesWithId)
-      )
-      break
-
-    case ContentDirectoryKnownClasses.HTTPMEDIALOCATION:
-      await updateHttpMediaLocationEntityPropertyValues(
-        db,
-        where,
-        decode.setProperties<IHttpMediaLocation>(event, httpMediaLocationPropertyNamesWithId)
-      )
-      break
-
-    case ContentDirectoryKnownClasses.VIDEOMEDIA:
-      await updateVideoMediaEntityPropertyValues(
-        db,
-        where,
-        decode.setProperties<IVideoMedia>(event, videoPropertyNamesWithId)
-      )
-      break
-
-    case ContentDirectoryKnownClasses.VIDEO:
-      await updateVideoEntityPropertyValues(db, where, decode.setProperties<IVideo>(event, videoPropertyNamesWithId))
-      break
-
-    case ContentDirectoryKnownClasses.LANGUAGE:
-      await updateLanguageEntityPropertyValues(
-        db,
-        where,
-        decode.setProperties<ILanguage>(event, languagePropertyNamesWIthId)
-      )
-      break
-
-    case ContentDirectoryKnownClasses.VIDEOMEDIAENCODING:
-      await updateVideoMediaEncodingEntityPropertyValues(
-        db,
-        where,
-        decode.setProperties<IVideoMediaEncoding>(event, videoMediaEncodingPropertyNamesWithId)
-      )
-      break
-
-    default:
-      throw new Error(`Unknown class name: ${className}`)
-  }
-}
-
-export {
-  createCategory,
-  createChannel,
-  createVideoMedia,
-  createVideo,
-  createUserDefinedLicense,
-  createKnownLicense,
-  createHttpMediaLocation,
-  createJoystreamMediaLocation,
-  createLanguage,
-  createVideoMediaEncoding,
-  removeCategory,
-  removeChannel,
-  removeVideoMedia,
-  removeVideo,
-  removeUserDefinedLicense,
-  removeKnownLicense,
-  removeHttpMediaLocation,
-  removeJoystreamMediaLocation,
-  removeLanguage,
-  removeVideoMediaEncoding,
-  createBlockOrGetFromDatabase,
-  batchCreateClassEntities,
-  getClassName,
-  updateCategoryEntityPropertyValues,
-  updateChannelEntityPropertyValues,
-  updateVideoMediaEntityPropertyValues,
-  updateVideoEntityPropertyValues,
-  updateUserDefinedLicenseEntityPropertyValues,
-  updateHttpMediaLocationEntityPropertyValues,
-  updateJoystreamMediaLocationEntityPropertyValues,
-  updateKnownLicenseEntityPropertyValues,
-  updateLanguageEntityPropertyValues,
-  updateVideoMediaEncodingEntityPropertyValues,
-  updateEntityPropertyValues,
-}

query-node/mappings/content-directory/entity/create.ts

@@ -0,0 +1,397 @@
+import { DB } from '../../../generated/indexer'
+import { Channel } from '../../../generated/graphql-server/src/modules/channel/channel.model'
+import { Category } from '../../../generated/graphql-server/src/modules/category/category.model'
+import { KnownLicense } from '../../../generated/graphql-server/src/modules/known-license/known-license.model'
+import { UserDefinedLicense } from '../../../generated/graphql-server/src/modules/user-defined-license/user-defined-license.model'
+import { JoystreamMediaLocation } from '../../../generated/graphql-server/src/modules/joystream-media-location/joystream-media-location.model'
+import { HttpMediaLocation } from '../../../generated/graphql-server/src/modules/http-media-location/http-media-location.model'
+import { VideoMedia } from '../../../generated/graphql-server/src/modules/video-media/video-media.model'
+import { Video } from '../../../generated/graphql-server/src/modules/video/video.model'
+import { Block, Network } from '../../../generated/graphql-server/src/modules/block/block.model'
+import { Language } from '../../../generated/graphql-server/src/modules/language/language.model'
+import { VideoMediaEncoding } from '../../../generated/graphql-server/src/modules/video-media-encoding/video-media-encoding.model'
+import { ClassEntity } from '../../../generated/graphql-server/src/modules/class-entity/class-entity.model'
+import { License } from '../../../generated/graphql-server/src/modules/license/license.model'
+import { MediaLocation } from '../../../generated/graphql-server/src/modules/media-location/media-location.model'
+
+import { contentDirectoryClassNamesWithId } from '../content-dir-consts'
+import {
+  ClassEntityMap,
+  ICategory,
+  IChannel,
+  ICreateEntityOperation,
+  IDBBlockId,
+  IEntity,
+  IHttpMediaLocation,
+  IJoystreamMediaLocation,
+  IKnownLicense,
+  ILanguage,
+  ILicense,
+  IMediaLocation,
+  IUserDefinedLicense,
+  IVideo,
+  IVideoMedia,
+  IVideoMediaEncoding,
+} from '../../types'
+import { getOrCreate } from '../get-or-create'
+import BN from 'bn.js'
+
+async function createBlockOrGetFromDatabase(db: DB, blockNumber: number): Promise<Block> {
+  let b = await db.get(Block, { where: { block: blockNumber } })
+  if (b === undefined) {
+    // TODO: get timestamp from the event or extrinsic
+    b = new Block({ block: blockNumber, network: Network.BABYLON, timestamp: new BN(Date.now()) })
+    await db.save<Block>(b)
+  }
+  return b
+}
+
+async function createChannel(
+  { db, block, id }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  p: IChannel,
+  nextEntityIdBeforeTransaction: number
+): Promise<Channel> {
+  const record = await db.get(Channel, { where: { id } })
+  if (record) return record
+
+  const channel = new Channel()
+
+  channel.version = block
+  channel.id = id
+  channel.title = p.title
+  channel.description = p.description
+  channel.isCurated = p.isCurated || false
+  channel.isPublic = p.isPublic
+  channel.coverPhotoUrl = p.coverPhotoURL
+  channel.avatarPhotoUrl = p.avatarPhotoURL
+
+  channel.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  const { language } = p
+  if (language !== undefined) {
+    channel.language = await getOrCreate.language(
+      { db, block, id },
+      classEntityMap,
+      language,
+      nextEntityIdBeforeTransaction
+    )
+  }
+  await db.save(channel)
+  return channel
+}
+
+async function createCategory({ db, block, id }: IDBBlockId, p: ICategory): Promise<Category> {
+  const record = await db.get(Category, { where: { id } })
+  if (record) return record
+
+  const category = new Category()
+
+  category.id = id
+  category.name = p.name
+  category.description = p.description
+  category.version = block
+  category.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save(category)
+  return category
+}
+
+async function createKnownLicense({ db, block, id }: IDBBlockId, p: IKnownLicense): Promise<KnownLicense> {
+  const record = await db.get(KnownLicense, { where: { id } })
+  if (record) return record
+
+  const knownLicence = new KnownLicense()
+
+  knownLicence.id = id
+  knownLicence.code = p.code
+  knownLicence.name = p.name
+  knownLicence.description = p.description
+  knownLicence.url = p.url
+  knownLicence.version = block
+  knownLicence.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save(knownLicence)
+  return knownLicence
+}
+
+async function createUserDefinedLicense(
+  { db, block, id }: IDBBlockId,
+  p: IUserDefinedLicense
+): Promise<UserDefinedLicense> {
+  const record = await db.get(UserDefinedLicense, { where: { id } })
+  if (record) return record
+
+  const userDefinedLicense = new UserDefinedLicense()
+
+  userDefinedLicense.id = id
+  userDefinedLicense.content = p.content
+  userDefinedLicense.version = block
+  userDefinedLicense.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save<UserDefinedLicense>(userDefinedLicense)
+  return userDefinedLicense
+}
+
+async function createJoystreamMediaLocation(
+  { db, block, id }: IDBBlockId,
+  p: IJoystreamMediaLocation
+): Promise<JoystreamMediaLocation> {
+  const record = await db.get(JoystreamMediaLocation, { where: { id } })
+  if (record) return record
+
+  const joyMediaLoc = new JoystreamMediaLocation()
+
+  joyMediaLoc.id = id
+  joyMediaLoc.dataObjectId = p.dataObjectId
+  joyMediaLoc.version = block
+  joyMediaLoc.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save(joyMediaLoc)
+  return joyMediaLoc
+}
+
+async function createHttpMediaLocation(
+  { db, block, id }: IDBBlockId,
+  p: IHttpMediaLocation
+): Promise<HttpMediaLocation> {
+  const record = await db.get(HttpMediaLocation, { where: { id } })
+  if (record) return record
+
+  const httpMediaLoc = new HttpMediaLocation()
+
+  httpMediaLoc.id = id
+  httpMediaLoc.url = p.url
+  httpMediaLoc.port = p.port
+  httpMediaLoc.version = block
+  httpMediaLoc.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save(httpMediaLoc)
+  return httpMediaLoc
+}
+
+async function createVideoMedia(
+  { db, block, id }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  p: IVideoMedia,
+  nextEntityIdBeforeTransaction: number
+): Promise<VideoMedia> {
+  const videoMedia = new VideoMedia()
+
+  videoMedia.id = id
+  videoMedia.pixelHeight = p.pixelHeight
+  videoMedia.pixelWidth = p.pixelWidth
+  videoMedia.size = p.size
+  videoMedia.version = block
+  const { encoding, location } = p
+  if (encoding !== undefined) {
+    videoMedia.encoding = await getOrCreate.videoMediaEncoding(
+      { db, block, id },
+      classEntityMap,
+      encoding,
+      nextEntityIdBeforeTransaction
+    )
+  }
+  if (location !== undefined) {
+    videoMedia.location = await getOrCreate.mediaLocation(
+      { db, block, id },
+      classEntityMap,
+      location,
+      nextEntityIdBeforeTransaction
+    )
+  }
+
+  videoMedia.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save(videoMedia)
+  return videoMedia
+}
+
+async function createVideo(
+  { db, block, id }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  p: IVideo,
+  nextEntityIdBeforeTransaction: number
+): Promise<Video> {
+  const record = await db.get(Video, { where: { id } })
+  if (record) return record
+
+  const video = new Video()
+
+  video.id = id
+  video.title = p.title
+  video.description = p.description
+  video.duration = p.duration
+  video.hasMarketing = p.hasMarketing
+  // TODO: needs to be handled correctly, from runtime CurationStatus is coming
+  video.isCurated = p.isCurated || true
+  video.isExplicit = p.isExplicit
+  video.isPublic = p.isPublic
+  video.publishedBeforeJoystream = p.publishedBeforeJoystream
+  video.skippableIntroDuration = p.skippableIntroDuration
+  video.thumbnailUrl = p.thumbnailURL
+  video.version = block
+
+  const { language, license, category, channel, media } = p
+  if (language !== undefined) {
+    video.language = await getOrCreate.language(
+      { db, block, id },
+      classEntityMap,
+      language,
+      nextEntityIdBeforeTransaction
+    )
+  }
+  if (license !== undefined) {
+    video.license = await getOrCreate.license({ db, block, id }, classEntityMap, license, nextEntityIdBeforeTransaction)
+  }
+  if (category !== undefined) {
+    video.category = await getOrCreate.category(
+      { db, block, id },
+      classEntityMap,
+      category,
+      nextEntityIdBeforeTransaction
+    )
+  }
+  if (channel !== undefined) {
+    video.channel = await getOrCreate.channel({ db, block, id }, classEntityMap, channel, nextEntityIdBeforeTransaction)
+  }
+  if (media !== undefined) {
+    video.media = await getOrCreate.videoMedia({ db, block, id }, classEntityMap, media, nextEntityIdBeforeTransaction)
+  }
+
+  video.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save<Video>(video)
+  return video
+}
+
+async function createLanguage({ db, block, id }: IDBBlockId, p: ILanguage): Promise<Language> {
+  const record = await db.get(Language, { where: { id } })
+  if (record) return record
+
+  const language = new Language()
+  language.id = id
+  language.name = p.name
+  language.code = p.code
+  language.version = block
+  language.happenedIn = await createBlockOrGetFromDatabase(db, block)
+
+  await db.save<Language>(language)
+  return language
+}
+
+async function createVideoMediaEncoding(
+  { db, block, id }: IDBBlockId,
+  p: IVideoMediaEncoding
+): Promise<VideoMediaEncoding> {
+  const record = await db.get(VideoMediaEncoding, { where: { id } })
+  if (record) return record
+
+  const encoding = new VideoMediaEncoding()
+  encoding.id = id
+  encoding.name = p.name
+  encoding.version = block
+  encoding.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save<VideoMediaEncoding>(encoding)
+  return encoding
+}
+
+async function createLicense(
+  { db, block, id }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  p: ILicense,
+  nextEntityIdBeforeTransaction: number
+): Promise<License> {
+  const record = await db.get(License, { where: { id } })
+  if (record) return record
+
+  const { knownLicense, userDefinedLicense } = p
+
+  const license = new License()
+  license.id = id
+  if (knownLicense !== undefined) {
+    license.knownLicense = await getOrCreate.knownLicense(
+      { db, block, id },
+      classEntityMap,
+      knownLicense,
+      nextEntityIdBeforeTransaction
+    )
+  }
+  if (userDefinedLicense !== undefined) {
+    license.userdefinedLicense = await getOrCreate.userDefinedLicense(
+      { db, block, id },
+      classEntityMap,
+      userDefinedLicense,
+      nextEntityIdBeforeTransaction
+    )
+  }
+  license.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save<License>(license)
+  return license
+}
+
+async function createMediaLocation(
+  { db, block, id }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  p: IMediaLocation,
+  nextEntityIdBeforeTransaction: number
+): Promise<MediaLocation> {
+  const { httpMediaLocation, joystreamMediaLocation } = p
+
+  const location = new MediaLocation()
+  location.id = id
+  if (httpMediaLocation !== undefined) {
+    location.httpMediaLocation = await getOrCreate.httpMediaLocation(
+      { db, block, id },
+      classEntityMap,
+      httpMediaLocation,
+      nextEntityIdBeforeTransaction
+    )
+  }
+  if (joystreamMediaLocation !== undefined) {
+    location.joystreamMediaLocation = await getOrCreate.joystreamMediaLocation(
+      { db, block, id },
+      classEntityMap,
+      joystreamMediaLocation,
+      nextEntityIdBeforeTransaction
+    )
+  }
+  location.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save<License>(location)
+  return location
+}
+
+async function getClassName(
+  db: DB,
+  entity: IEntity,
+  createEntityOperations: ICreateEntityOperation[]
+): Promise<string | undefined> {
+  const { entityId, indexOf } = entity
+  if (entityId === undefined && indexOf === undefined) {
+    throw Error(`Can not determine class of the entity`)
+  }
+
+  let classId: number | undefined
+  // Is newly created entity in the same transaction
+  if (indexOf !== undefined) {
+    classId = createEntityOperations[indexOf].classId
+  } else {
+    const ce = await db.get(ClassEntity, { where: { id: entityId } })
+    if (ce === undefined) console.log(`Class not found for the entity: ${entityId}`)
+    classId = ce ? ce.classId : undefined
+  }
+
+  const c = contentDirectoryClassNamesWithId.find((c) => c.classId === classId)
+  // TODO: stop execution, class should be created before entity creation
+  if (c === undefined) console.log(`Not recognized class id: ${classId}`)
+  return c ? c.name : undefined
+}
+
+export {
+  createCategory,
+  createChannel,
+  createVideoMedia,
+  createVideo,
+  createUserDefinedLicense,
+  createKnownLicense,
+  createHttpMediaLocation,
+  createJoystreamMediaLocation,
+  createLanguage,
+  createVideoMediaEncoding,
+  createLicense,
+  createMediaLocation,
+  createBlockOrGetFromDatabase,
+  getClassName,
+}

query-node/mappings/content-directory/entity.ts → query-node/mappings/content-directory/entity/index.ts

@@ -1,17 +1,23 @@
 import Debug from 'debug'
-import { DB, SubstrateEvent } from '../../generated/indexer'
-import { ClassEntity } from '../../generated/graphql-server/src/modules/class-entity/class-entity.model'
+import { DB, SubstrateEvent } from '../../../generated/indexer'
+import { ClassEntity } from '../../../generated/graphql-server/src/modules/class-entity/class-entity.model'
 
-import { decode } from './decode'
+import { decode } from '../decode'
+import {
+  updateCategoryEntityPropertyValues,
+  updateChannelEntityPropertyValues,
+  updateVideoMediaEntityPropertyValues,
+  updateVideoEntityPropertyValues,
+  updateUserDefinedLicenseEntityPropertyValues,
+  updateHttpMediaLocationEntityPropertyValues,
+  updateJoystreamMediaLocationEntityPropertyValues,
+  updateKnownLicenseEntityPropertyValues,
+  updateLanguageEntityPropertyValues,
+  updateVideoMediaEncodingEntityPropertyValues,
+  updateLicenseEntityPropertyValues,
+  updateMediaLocationEntityPropertyValues,
+} from './update'
 import {
-  createCategory,
-  createChannel,
-  createVideoMedia,
-  createVideo,
-  createUserDefinedLicense,
-  createKnownLicense,
-  createHttpMediaLocation,
-  createJoystreamMediaLocation,
   removeCategory,
   removeChannel,
   removeVideoMedia,
@@ -22,20 +28,22 @@ import {
   removeJoystreamMediaLocation,
   removeLanguage,
   removeVideoMediaEncoding,
+  removeLicense,
+  removeMediaLocation,
+} from './remove'
+import {
+  createCategory,
+  createChannel,
+  createVideoMedia,
+  createVideo,
+  createUserDefinedLicense,
+  createKnownLicense,
+  createHttpMediaLocation,
+  createJoystreamMediaLocation,
   createLanguage,
   createVideoMediaEncoding,
-  updateCategoryEntityPropertyValues,
-  updateChannelEntityPropertyValues,
-  updateVideoMediaEntityPropertyValues,
-  updateVideoEntityPropertyValues,
-  updateUserDefinedLicenseEntityPropertyValues,
-  updateHttpMediaLocationEntityPropertyValues,
-  updateJoystreamMediaLocationEntityPropertyValues,
-  updateKnownLicenseEntityPropertyValues,
-  updateLanguageEntityPropertyValues,
-  updateVideoMediaEncodingEntityPropertyValues,
   createBlockOrGetFromDatabase,
-} from './entity-helper'
+} from './create'
 import {
   CategoryPropertyNamesWithId,
   channelPropertyNamesWithId,
@@ -48,7 +56,7 @@ import {
   videoPropertyNamesWithId,
   contentDirectoryClassNamesWithId,
   ContentDirectoryKnownClasses,
-} from './content-dir-consts'
+} from '../content-dir-consts'
 
 import {
   IChannel,
@@ -63,7 +71,11 @@ import {
   IVideoMediaEncoding,
   IDBBlockId,
   IWhereCond,
-} from '../types'
+  IEntity,
+  ILicense,
+  IMediaLocation,
+} from '../../types'
+import { getOrCreate } from '../get-or-create'
 
 const debug = Debug('mappings:content-directory')
 
@@ -91,7 +103,12 @@ async function contentDirectory_EntitySchemaSupportAdded(db: DB, event: Substrat
 
   switch (cls.name) {
     case ContentDirectoryKnownClasses.CHANNEL:
-      await createChannel(arg, decode.setProperties<IChannel>(event, channelPropertyNamesWithId))
+      await createChannel(
+        arg,
+        new Map<string, IEntity[]>(),
+        decode.setProperties<IChannel>(event, channelPropertyNamesWithId),
+        0 // ignored
+      )
       break
 
     case ContentDirectoryKnownClasses.CATEGORY:
@@ -124,11 +141,21 @@ async function contentDirectory_EntitySchemaSupportAdded(db: DB, event: Substrat
       break
 
     case ContentDirectoryKnownClasses.VIDEOMEDIA:
-      await createVideoMedia(arg, decode.setProperties<IVideoMedia>(event, videoPropertyNamesWithId))
+      await createVideoMedia(
+        arg,
+        new Map<string, IEntity[]>(),
+        decode.setProperties<IVideoMedia>(event, videoPropertyNamesWithId),
+        0 // ignored
+      )
       break
 
     case ContentDirectoryKnownClasses.VIDEO:
-      await createVideo(arg, decode.setProperties<IVideo>(event, videoPropertyNamesWithId))
+      await createVideo(
+        arg,
+        new Map<string, IEntity[]>(),
+        decode.setProperties<IVideo>(event, videoPropertyNamesWithId),
+        0 // ignored
+      )
       break
 
     case ContentDirectoryKnownClasses.LANGUAGE:
@@ -162,7 +189,7 @@ async function contentDirectory_EntityRemoved(db: DB, event: SubstrateEvent): Pr
 
   const cls = contentDirectoryClassNamesWithId.find((c) => c.classId === classEntity.classId)
   if (cls === undefined) {
-    console.log('Undefined class')
+    console.log('Unknown class')
     return
   }
 
@@ -206,6 +233,14 @@ async function contentDirectory_EntityRemoved(db: DB, event: SubstrateEvent): Pr
       await removeVideoMediaEncoding(db, where)
       break
 
+    case ContentDirectoryKnownClasses.LICENSE:
+      await removeLicense(db, where)
+      break
+
+    case ContentDirectoryKnownClasses.MEDIALOCATION:
+      await removeMediaLocation(db, where)
+      break
+
     default:
       throw new Error(`Unknown class name: ${cls.name}`)
   }
@@ -224,17 +259,18 @@ async function contentDirectory_EntityCreated(db: DB, event: SubstrateEvent): Pr
   classEntity.version = event.blockNumber
   classEntity.happenedIn = await createBlockOrGetFromDatabase(db, event.blockNumber)
   await db.save<ClassEntity>(classEntity)
+
+  await getOrCreate.nextEntityId(db, c.entityId + 1)
 }
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
 async function contentDirectory_EntityPropertyValuesUpdated(db: DB, event: SubstrateEvent): Promise<void> {
-  debug(`EntityPropertyValuesUpdated event: ${JSON.stringify(event)}`)
-
   const { extrinsic } = event
-
   if (extrinsic && extrinsic.method === 'transaction') return
   if (extrinsic === undefined) throw Error(`Extrinsic data not found for event: ${event.id}`)
 
+  debug(`EntityPropertyValuesUpdated event: ${JSON.stringify(event)}`)
+
   const { 2: newPropertyValues } = extrinsic.args
   const entityId = decode.stringIfyEntityId(event)
 
@@ -253,7 +289,7 @@ async function contentDirectory_EntityPropertyValuesUpdated(db: DB, event: Subst
 
   switch (cls.name) {
     case ContentDirectoryKnownClasses.CHANNEL:
-      updateChannelEntityPropertyValues(db, where, decode.setProperties<IChannel>(event, channelPropertyNamesWithId))
+      updateChannelEntityPropertyValues(db, where, decode.setProperties<IChannel>(event, channelPropertyNamesWithId), 0)
       break
 
     case ContentDirectoryKnownClasses.CATEGORY:
@@ -300,12 +336,13 @@ async function contentDirectory_EntityPropertyValuesUpdated(db: DB, event: Subst
       await updateVideoMediaEntityPropertyValues(
         db,
         where,
-        decode.setProperties<IVideoMedia>(event, videoPropertyNamesWithId)
+        decode.setProperties<IVideoMedia>(event, videoPropertyNamesWithId),
+        0
       )
       break
 
     case ContentDirectoryKnownClasses.VIDEO:
-      await updateVideoEntityPropertyValues(db, where, decode.setProperties<IVideo>(event, videoPropertyNamesWithId))
+      await updateVideoEntityPropertyValues(db, where, decode.setProperties<IVideo>(event, videoPropertyNamesWithId), 0)
       break
 
     case ContentDirectoryKnownClasses.LANGUAGE:
@@ -324,6 +361,24 @@ async function contentDirectory_EntityPropertyValuesUpdated(db: DB, event: Subst
       )
       break
 
+    case ContentDirectoryKnownClasses.LICENSE:
+      await updateLicenseEntityPropertyValues(
+        db,
+        where,
+        decode.setProperties<ILicense>(event, videoMediaEncodingPropertyNamesWithId),
+        0
+      )
+      break
+
+    case ContentDirectoryKnownClasses.MEDIALOCATION:
+      await updateMediaLocationEntityPropertyValues(
+        db,
+        where,
+        decode.setProperties<IMediaLocation>(event, videoMediaEncodingPropertyNamesWithId),
+        0
+      )
+      break
+
     default:
       throw new Error(`Unknown class name: ${cls.name}`)
   }

query-node/mappings/content-directory/entity/remove.ts

@@ -0,0 +1,108 @@
+import { DB } from '../../../generated/indexer'
+import { Channel } from '../../../generated/graphql-server/src/modules/channel/channel.model'
+import { Category } from '../../../generated/graphql-server/src/modules/category/category.model'
+import { KnownLicense } from '../../../generated/graphql-server/src/modules/known-license/known-license.model'
+import { UserDefinedLicense } from '../../../generated/graphql-server/src/modules/user-defined-license/user-defined-license.model'
+import { JoystreamMediaLocation } from '../../../generated/graphql-server/src/modules/joystream-media-location/joystream-media-location.model'
+import { HttpMediaLocation } from '../../../generated/graphql-server/src/modules/http-media-location/http-media-location.model'
+import { VideoMedia } from '../../../generated/graphql-server/src/modules/video-media/video-media.model'
+import { Video } from '../../../generated/graphql-server/src/modules/video/video.model'
+import { Language } from '../../../generated/graphql-server/src/modules/language/language.model'
+import { VideoMediaEncoding } from '../../../generated/graphql-server/src/modules/video-media-encoding/video-media-encoding.model'
+import { License } from '../../../generated/graphql-server/src/modules/license/license.model'
+import { MediaLocation } from '../../../generated/graphql-server/src/modules/media-location/media-location.model'
+
+import { IWhereCond } from '../../types'
+
+async function removeChannel(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(Channel, where)
+  if (record === undefined) throw Error(`Channel not found`)
+  if (record.videos) record.videos.map(async (v) => await removeVideo(db, { where: { id: v.id } }))
+  await db.remove<Channel>(record)
+}
+async function removeCategory(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(Category, where)
+  if (record === undefined) throw Error(`Category not found`)
+  if (record.videos) record.videos.map(async (v) => await removeVideo(db, { where: { id: v.id } }))
+  await db.remove<Category>(record)
+}
+async function removeVideoMedia(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(VideoMedia, where)
+  if (record === undefined) throw Error(`VideoMedia not found`)
+  if (record.video) await db.remove<Video>(record.video)
+  await db.remove<VideoMedia>(record)
+}
+async function removeVideo(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(Video, where)
+  if (record === undefined) throw Error(`Video not found`)
+  await db.remove<Video>(record)
+}
+
+async function removeLicense(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(License, where)
+  if (record === undefined) throw Error(`License not found`)
+  // Remove all the videos under this license
+  if (record.videolicense) record.videolicense.map(async (v) => await removeVideo(db, { where: { id: v.id } }))
+  await db.remove<License>(record)
+}
+async function removeUserDefinedLicense(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(UserDefinedLicense, where)
+  if (record === undefined) throw Error(`UserDefinedLicense not found`)
+  if (record.licenseuserdefinedLicense)
+    record.licenseuserdefinedLicense.map(async (l) => await removeLicense(db, { where: { id: l.id } }))
+  await db.remove<UserDefinedLicense>(record)
+}
+async function removeKnownLicense(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(KnownLicense, where)
+  if (record === undefined) throw Error(`KnownLicense not found`)
+  if (record.licenseknownLicense)
+    record.licenseknownLicense.map(async (k) => await removeLicense(db, { where: { id: k.id } }))
+  await db.remove<KnownLicense>(record)
+}
+async function removeMediaLocation(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(MediaLocation, where)
+  if (record === undefined) throw Error(`MediaLocation not found`)
+  if (record.videoMedia) await removeVideo(db, { where: { id: record.videoMedia.id } })
+  await db.remove<MediaLocation>(record)
+}
+async function removeHttpMediaLocation(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(HttpMediaLocation, where)
+  if (record === undefined) throw Error(`HttpMediaLocation not found`)
+  if (record.medialocationhttpMediaLocation)
+    record.medialocationhttpMediaLocation.map(async (v) => await removeMediaLocation(db, { where: { id: v.id } }))
+  await db.remove<HttpMediaLocation>(record)
+}
+async function removeJoystreamMediaLocation(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(JoystreamMediaLocation, where)
+  if (record === undefined) throw Error(`JoystreamMediaLocation not found`)
+  if (record.medialocationjoystreamMediaLocation)
+    record.medialocationjoystreamMediaLocation.map(async (v) => await removeVideo(db, { where: { id: v.id } }))
+  await db.remove<JoystreamMediaLocation>(record)
+}
+async function removeLanguage(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(Language, where)
+  if (record === undefined) throw Error(`Language not found`)
+  if (record.channellanguage) record.channellanguage.map(async (c) => await removeChannel(db, { where: { id: c.id } }))
+  if (record.videolanguage) record.videolanguage.map(async (v) => await removeVideo(db, { where: { id: v.id } }))
+  await db.remove<Language>(record)
+}
+async function removeVideoMediaEncoding(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(VideoMediaEncoding, where)
+  if (record === undefined) throw Error(`Language not found`)
+  await db.remove<VideoMediaEncoding>(record)
+}
+
+export {
+  removeCategory,
+  removeChannel,
+  removeVideoMedia,
+  removeVideo,
+  removeUserDefinedLicense,
+  removeKnownLicense,
+  removeHttpMediaLocation,
+  removeJoystreamMediaLocation,
+  removeLanguage,
+  removeVideoMediaEncoding,
+  removeMediaLocation,
+  removeLicense,
+}

query-node/mappings/content-directory/entity/update.ts

@@ -0,0 +1,260 @@
+import { DB } from '../../../generated/indexer'
+import { Channel } from '../../../generated/graphql-server/src/modules/channel/channel.model'
+import { Category } from '../../../generated/graphql-server/src/modules/category/category.model'
+import { KnownLicense } from '../../../generated/graphql-server/src/modules/known-license/known-license.model'
+import { UserDefinedLicense } from '../../../generated/graphql-server/src/modules/user-defined-license/user-defined-license.model'
+import { JoystreamMediaLocation } from '../../../generated/graphql-server/src/modules/joystream-media-location/joystream-media-location.model'
+import { HttpMediaLocation } from '../../../generated/graphql-server/src/modules/http-media-location/http-media-location.model'
+import { VideoMedia } from '../../../generated/graphql-server/src/modules/video-media/video-media.model'
+import { Video } from '../../../generated/graphql-server/src/modules/video/video.model'
+import { Language } from '../../../generated/graphql-server/src/modules/language/language.model'
+import { VideoMediaEncoding } from '../../../generated/graphql-server/src/modules/video-media-encoding/video-media-encoding.model'
+import { License } from '../../../generated/graphql-server/src/modules/license/license.model'
+import { MediaLocation } from '../../../generated/graphql-server/src/modules/media-location/media-location.model'
+
+import {
+  ICategory,
+  IChannel,
+  IHttpMediaLocation,
+  IJoystreamMediaLocation,
+  IKnownLicense,
+  ILanguage,
+  ILicense,
+  IMediaLocation,
+  IReference,
+  IUserDefinedLicense,
+  IVideo,
+  IVideoMedia,
+  IVideoMediaEncoding,
+  IWhereCond,
+} from '../../types'
+
+function getEntityIdFromReferencedField(ref: IReference, entityIdBeforeTransaction: number): string {
+  const { entityId, existing } = ref
+  const id = existing ? entityId : entityIdBeforeTransaction + entityId
+  return id.toString()
+}
+
+async function updateMediaLocationEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: IMediaLocation,
+  entityIdBeforeTransaction: number
+): Promise<void> {
+  const { httpMediaLocation, joystreamMediaLocation } = props
+  const record = await db.get(MediaLocation, where)
+  if (record === undefined) throw Error(`MediaLocation entity not found: ${where.where.id}`)
+
+  if (httpMediaLocation) {
+    const id = getEntityIdFromReferencedField(httpMediaLocation, entityIdBeforeTransaction)
+    record.httpMediaLocation = await db.get(HttpMediaLocation, { where: { id } })
+  }
+  if (joystreamMediaLocation) {
+    const id = getEntityIdFromReferencedField(joystreamMediaLocation, entityIdBeforeTransaction)
+    record.joystreamMediaLocation = await db.get(JoystreamMediaLocation, { where: { id } })
+  }
+  await db.save<MediaLocation>(record)
+}
+
+async function updateLicenseEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: ILicense,
+  entityIdBeforeTransaction: number
+): Promise<void> {
+  const record = await db.get(License, where)
+  if (record === undefined) throw Error(`License entity not found: ${where.where.id}`)
+
+  const { knownLicense, userDefinedLicense } = props
+  if (knownLicense) {
+    const id = getEntityIdFromReferencedField(knownLicense, entityIdBeforeTransaction)
+    record.knownLicense = await db.get(KnownLicense, { where: { id } })
+  }
+  if (userDefinedLicense) {
+    const id = getEntityIdFromReferencedField(userDefinedLicense, entityIdBeforeTransaction)
+    record.userdefinedLicense = await db.get(UserDefinedLicense, { where: { id } })
+  }
+  await db.save<License>(record)
+}
+
+async function updateCategoryEntityPropertyValues(db: DB, where: IWhereCond, props: ICategory): Promise<void> {
+  const record = await db.get(Category, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+  Object.assign(record, props)
+  await db.save<Category>(record)
+}
+async function updateChannelEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: IChannel,
+  entityIdBeforeTransaction: number
+): Promise<void> {
+  const record = await db.get(Channel, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+
+  let lang: Language | undefined
+  if (props.language !== undefined) {
+    const id = getEntityIdFromReferencedField(props.language, entityIdBeforeTransaction)
+    lang = await db.get(Language, { where: { id } })
+    if (lang === undefined) throw Error(`Language entity not found: ${id}`)
+    props.language = undefined
+  }
+  Object.assign(record, props)
+
+  record.language = lang || record.language
+  await db.save<Channel>(record)
+}
+async function updateVideoMediaEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: IVideoMedia,
+  entityIdBeforeTransaction: number
+): Promise<void> {
+  const record = await db.get(VideoMedia, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+
+  let enco: VideoMediaEncoding | undefined
+  let mediaLoc: MediaLocation | undefined
+  const { encoding, location } = props
+  if (encoding) {
+    const id = getEntityIdFromReferencedField(encoding, entityIdBeforeTransaction)
+    enco = await db.get(VideoMediaEncoding, { where: { id } })
+    if (enco === undefined) throw Error(`VideoMediaEncoding entity not found: ${id}`)
+    props.encoding = undefined
+  }
+  if (location) {
+    const id = getEntityIdFromReferencedField(location, entityIdBeforeTransaction)
+    mediaLoc = await db.get(MediaLocation, { where: { id } })
+    if (!mediaLoc) throw Error(`MediaLocation entity not found: ${id}`)
+    props.location = undefined
+  }
+  Object.assign(record, props)
+
+  record.encoding = enco || record.encoding
+  record.location = mediaLoc || record.location
+  await db.save<VideoMedia>(record)
+}
+async function updateVideoEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: IVideo,
+  entityIdBeforeTransaction: number
+): Promise<void> {
+  const record = await db.get<Video>(Video, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+
+  let chann: Channel | undefined
+  let cat: Category | undefined
+  let lang: Language | undefined
+  let vMedia: VideoMedia | undefined
+  let lic: License | undefined
+  const { channel, category, language, media, license } = props
+  if (channel) {
+    const id = getEntityIdFromReferencedField(channel, entityIdBeforeTransaction)
+    chann = await db.get(Channel, { where: { id } })
+    if (!chann) throw Error(`Channel entity not found: ${id}`)
+    props.channel = undefined
+  }
+  if (category) {
+    const id = getEntityIdFromReferencedField(category, entityIdBeforeTransaction)
+    cat = await db.get(Category, { where: { id } })
+    if (!cat) throw Error(`Category entity not found: ${id}`)
+    props.category = undefined
+  }
+  if (media) {
+    const id = getEntityIdFromReferencedField(media, entityIdBeforeTransaction)
+    vMedia = await db.get(VideoMedia, { where: { id } })
+    if (!vMedia) throw Error(`VideoMedia entity not found: ${id}`)
+    props.media = undefined
+  }
+  if (license) {
+    const id = getEntityIdFromReferencedField(license, entityIdBeforeTransaction)
+    lic = await db.get(License, { where: { id } })
+    if (!lic) throw Error(`License entity not found: ${id}`)
+    props.license = undefined
+  }
+  if (language) {
+    const id = getEntityIdFromReferencedField(language, entityIdBeforeTransaction)
+    lang = await db.get(Language, { where: { id } })
+    if (!lang) throw Error(`Language entity not found: ${id}`)
+    props.language = undefined
+  }
+
+  Object.assign(record, props)
+
+  record.channel = chann || record.channel
+  record.category = cat || record.category
+  record.media = vMedia || record.media
+  record.license = lic || record.license
+  record.language = lang
+
+  await db.save<Video>(record)
+}
+async function updateUserDefinedLicenseEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: IUserDefinedLicense
+): Promise<void> {
+  const record = await db.get(UserDefinedLicense, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+  Object.assign(record, props)
+  await db.save<UserDefinedLicense>(record)
+}
+async function updateKnownLicenseEntityPropertyValues(db: DB, where: IWhereCond, props: IKnownLicense): Promise<void> {
+  const record = await db.get(KnownLicense, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+  Object.assign(record, props)
+  await db.save<KnownLicense>(record)
+}
+async function updateHttpMediaLocationEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: IHttpMediaLocation
+): Promise<void> {
+  const record = await db.get(HttpMediaLocation, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+  Object.assign(record, props)
+  await db.save<HttpMediaLocation>(record)
+}
+
+async function updateJoystreamMediaLocationEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: IJoystreamMediaLocation
+): Promise<void> {
+  const record = await db.get(JoystreamMediaLocation, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+  Object.assign(record, props)
+  await db.save<JoystreamMediaLocation>(record)
+}
+async function updateLanguageEntityPropertyValues(db: DB, where: IWhereCond, props: ILanguage): Promise<void> {
+  const record = await db.get(Language, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+  Object.assign(record, props)
+  await db.save<Language>(record)
+}
+async function updateVideoMediaEncodingEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: IVideoMediaEncoding
+): Promise<void> {
+  const record = await db.get(VideoMediaEncoding, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+  Object.assign(record, props)
+  await db.save<VideoMediaEncoding>(record)
+}
+
+export {
+  updateCategoryEntityPropertyValues,
+  updateChannelEntityPropertyValues,
+  updateVideoMediaEntityPropertyValues,
+  updateVideoEntityPropertyValues,
+  updateUserDefinedLicenseEntityPropertyValues,
+  updateHttpMediaLocationEntityPropertyValues,
+  updateJoystreamMediaLocationEntityPropertyValues,
+  updateKnownLicenseEntityPropertyValues,
+  updateLanguageEntityPropertyValues,
+  updateVideoMediaEncodingEntityPropertyValues,
+  updateLicenseEntityPropertyValues,
+  updateMediaLocationEntityPropertyValues,
+}

query-node/mappings/content-directory/get-or-create.ts

@@ -0,0 +1,411 @@
+import { Channel } from '../../generated/graphql-server/src/modules/channel/channel.model'
+import { Category } from '../../generated/graphql-server/src/modules/category/category.model'
+import { KnownLicense } from '../../generated/graphql-server/src/modules/known-license/known-license.model'
+import { UserDefinedLicense } from '../../generated/graphql-server/src/modules/user-defined-license/user-defined-license.model'
+import { JoystreamMediaLocation } from '../../generated/graphql-server/src/modules/joystream-media-location/joystream-media-location.model'
+import { HttpMediaLocation } from '../../generated/graphql-server/src/modules/http-media-location/http-media-location.model'
+import { VideoMedia } from '../../generated/graphql-server/src/modules/video-media/video-media.model'
+import { Language } from '../../generated/graphql-server/src/modules/language/language.model'
+import { VideoMediaEncoding } from '../../generated/graphql-server/src/modules/video-media-encoding/video-media-encoding.model'
+import { License } from '../../generated/graphql-server/src/modules/license/license.model'
+import { MediaLocation } from '../../generated/graphql-server/src/modules/media-location/media-location.model'
+import { NextEntityId } from '../../generated/graphql-server/src/modules/next-entity-id/next-entity-id.model'
+
+import { decode } from './decode'
+import {
+  CategoryPropertyNamesWithId,
+  channelPropertyNamesWithId,
+  httpMediaLocationPropertyNamesWithId,
+  joystreamMediaLocationPropertyNamesWithId,
+  knownLicensePropertyNamesWIthId,
+  languagePropertyNamesWIthId,
+  licensePropertyNamesWithId,
+  mediaLocationPropertyNamesWithId,
+  userDefinedLicensePropertyNamesWithId,
+  videoMediaEncodingPropertyNamesWithId,
+  videoPropertyNamesWithId,
+} from './content-dir-consts'
+import {
+  ClassEntityMap,
+  ICategory,
+  IChannel,
+  IDBBlockId,
+  IEntity,
+  IHttpMediaLocation,
+  IJoystreamMediaLocation,
+  IKnownLicense,
+  ILanguage,
+  ILicense,
+  IMediaLocation,
+  IReference,
+  IUserDefinedLicense,
+  IVideoMedia,
+  IVideoMediaEncoding,
+} from '../types'
+
+import {
+  createCategory,
+  createChannel,
+  createVideoMedia,
+  createUserDefinedLicense,
+  createKnownLicense,
+  createHttpMediaLocation,
+  createJoystreamMediaLocation,
+  createLanguage,
+  createVideoMediaEncoding,
+  createLicense,
+  createMediaLocation,
+} from './entity/create'
+
+import { DB } from '../../generated/indexer'
+
+// Keep track of the next entity id
+async function nextEntityId(db: DB, nextEntityId: number): Promise<void> {
+  let e = await db.get(NextEntityId, { where: { id: '1' } })
+  if (!e) e = new NextEntityId({ id: '1' })
+  e.nextId = nextEntityId
+  await db.save<NextEntityId>(e)
+}
+
+function generateEntityIdFromIndex(index: number): string {
+  return `${index}`
+}
+
+function findEntity(entityId: number, className: string, classEntityMap: ClassEntityMap): IEntity {
+  const newlyCreatedEntities = classEntityMap.get(className)
+  if (newlyCreatedEntities === undefined) throw Error(`Couldn't find '${className}' entities in the classEntityMap`)
+  const entity = newlyCreatedEntities.find((e) => e.indexOf === entityId)
+  if (!entity) throw Error(`Unknown ${className} entity id: ${entityId}`)
+  removeInsertedEntity(className, entityId, classEntityMap)
+  return entity
+}
+
+async function language(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  language: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<Language> {
+  let lang
+  const { entityId, existing } = language
+  if (existing) {
+    lang = await db.get(Language, { where: { id: entityId.toString() } })
+    if (!lang) throw Error(`Language entity not found`)
+    return lang
+  }
+
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+  // could be created in the transaction
+  lang = await db.get(Language, { where: { id } })
+  if (lang) return lang
+
+  // get the entity from list of newly created entities and insert into db
+  const { properties } = findEntity(entityId, 'Language', classEntityMap)
+  return await createLanguage(
+    { db, block, id },
+    decode.setEntityPropertyValues<ILanguage>(properties, languagePropertyNamesWIthId)
+  )
+}
+
+async function videoMediaEncoding(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  encoding: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<VideoMediaEncoding> {
+  let vmEncoding
+  const { entityId, existing } = encoding
+  if (existing) {
+    vmEncoding = await db.get(VideoMediaEncoding, { where: { id: entityId.toString() } })
+    if (!vmEncoding) throw Error(`VideoMediaEncoding entity not found`)
+    return vmEncoding
+  }
+
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+
+  // could be created in the transaction
+  vmEncoding = await db.get(VideoMediaEncoding, { where: { id } })
+  if (vmEncoding) return vmEncoding
+
+  const { properties } = findEntity(entityId, 'VideoMediaEncoding', classEntityMap)
+  return await createVideoMediaEncoding(
+    { db, block, id },
+    decode.setEntityPropertyValues<IVideoMediaEncoding>(properties, videoMediaEncodingPropertyNamesWithId)
+  )
+}
+
+async function videoMedia(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  media: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<VideoMedia> {
+  let videoM: VideoMedia | undefined
+  const { entityId, existing } = media
+  if (existing) {
+    videoM = await db.get(VideoMedia, { where: { id: entityId.toString() } })
+    if (!videoM) throw Error(`VideoMedia entity not found`)
+    return videoM
+  }
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+
+  // could be created in the transaction
+  videoM = await db.get(VideoMedia, { where: { id } })
+  if (videoM) return videoM
+
+  const { properties } = findEntity(entityId, 'VideoMedia', classEntityMap)
+  return await createVideoMedia(
+    { db, block, id },
+    classEntityMap,
+    decode.setEntityPropertyValues<IVideoMedia>(properties, videoPropertyNamesWithId),
+    nextEntityIdBeforeTransaction
+  )
+}
+
+async function knownLicense(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  knownLicense: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<KnownLicense> {
+  let kLicense: KnownLicense | undefined
+  const { entityId, existing } = knownLicense
+  if (existing) {
+    kLicense = await db.get(KnownLicense, { where: { id: entityId.toString() } })
+    if (!kLicense) throw Error(`KnownLicense entity not found`)
+    return kLicense
+  }
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+  // could be created in the transaction
+  kLicense = await db.get(KnownLicense, { where: { id } })
+  if (kLicense) return kLicense
+
+  const { properties } = findEntity(entityId, 'KnownLicense', classEntityMap)
+  return await createKnownLicense(
+    { db, block, id },
+    decode.setEntityPropertyValues<IKnownLicense>(properties, knownLicensePropertyNamesWIthId)
+  )
+}
+async function userDefinedLicense(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  userDefinedLicense: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<UserDefinedLicense> {
+  let udLicense: UserDefinedLicense | undefined
+  const { entityId, existing } = userDefinedLicense
+  if (existing) {
+    udLicense = await db.get(UserDefinedLicense, { where: { id: entityId.toString() } })
+    if (!udLicense) throw Error(`UserDefinedLicense entity not found`)
+    return udLicense
+  }
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+  // could be created in the transaction
+  udLicense = await db.get(UserDefinedLicense, {
+    where: { id },
+  })
+  if (udLicense) return udLicense
+
+  const { properties } = findEntity(entityId, 'UserDefinedLicense', classEntityMap)
+  return await createUserDefinedLicense(
+    { db, block, id },
+    decode.setEntityPropertyValues<IUserDefinedLicense>(properties, userDefinedLicensePropertyNamesWithId)
+  )
+}
+
+async function channel(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  channel: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<Channel> {
+  let chann: Channel | undefined
+  const { entityId, existing } = channel
+
+  if (existing) {
+    chann = await db.get(Channel, { where: { id: entityId.toString() } })
+    if (!chann) throw Error(`Channel entity not found`)
+    return chann
+  }
+
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+  // could be created in the transaction
+  chann = await db.get(Channel, { where: { id } })
+  if (chann) return chann
+
+  const { properties } = findEntity(entityId, 'Channel', classEntityMap)
+  return await createChannel(
+    { db, block, id },
+    classEntityMap,
+    decode.setEntityPropertyValues<IChannel>(properties, channelPropertyNamesWithId),
+    nextEntityIdBeforeTransaction
+  )
+}
+
+async function category(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  category: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<Category> {
+  let cat: Category | undefined
+  const { entityId, existing } = category
+
+  if (existing) {
+    cat = await db.get(Category, { where: { id: entityId.toString() } })
+    if (!cat) throw Error(`Category entity not found`)
+    return cat
+  }
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+  // could be created in the transaction
+  cat = await db.get(Category, { where: { id } })
+  if (cat) return cat
+
+  const { properties } = findEntity(entityId, 'Category', classEntityMap)
+  return await createCategory(
+    { db, block, id },
+    decode.setEntityPropertyValues<ICategory>(properties, CategoryPropertyNamesWithId)
+  )
+}
+
+async function httpMediaLocation(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  httpMediaLoc: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<HttpMediaLocation | undefined> {
+  let loc: HttpMediaLocation | undefined
+  const { entityId, existing } = httpMediaLoc
+
+  if (existing) {
+    loc = await db.get(HttpMediaLocation, { where: { id: entityId.toString() } })
+    if (!loc) throw Error(`HttpMediaLocation entity not found`)
+    return loc
+  }
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+
+  // could be created in the transaction
+  loc = await db.get(HttpMediaLocation, {
+    where: { id },
+  })
+  if (loc) return loc
+
+  const { properties } = findEntity(entityId, 'HttpMediaLocation', classEntityMap)
+  return await createHttpMediaLocation(
+    { db, block, id },
+    decode.setEntityPropertyValues<IHttpMediaLocation>(properties, httpMediaLocationPropertyNamesWithId)
+  )
+}
+
+async function joystreamMediaLocation(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  joyMediaLoc: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<JoystreamMediaLocation | undefined> {
+  let loc: JoystreamMediaLocation | undefined
+  const { entityId, existing } = joyMediaLoc
+
+  if (existing) {
+    loc = await db.get(JoystreamMediaLocation, { where: { id: entityId.toString() } })
+    if (!loc) throw Error(`JoystreamMediaLocation entity not found`)
+    return loc
+  }
+
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+
+  // could be created in the transaction
+  loc = await db.get(JoystreamMediaLocation, {
+    where: { id },
+  })
+  if (loc) return loc
+
+  const { properties } = findEntity(entityId, 'JoystreamMediaLocation', classEntityMap)
+  return await createJoystreamMediaLocation(
+    { db, block, id },
+    decode.setEntityPropertyValues<IJoystreamMediaLocation>(properties, joystreamMediaLocationPropertyNamesWithId)
+  )
+}
+
+async function license(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  license: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<License> {
+  let lic: License | undefined
+  const { entityId, existing } = license
+
+  if (existing) {
+    lic = await db.get(License, { where: { id: entityId.toString() } })
+    if (!lic) throw Error(`License entity not found`)
+    return lic
+  }
+
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+  // could be created in the transaction
+  lic = await db.get(License, { where: { id } })
+  if (lic) return lic
+
+  const { properties } = findEntity(entityId, 'License', classEntityMap)
+  return await createLicense(
+    { db, block, id },
+    classEntityMap,
+    decode.setEntityPropertyValues<ILicense>(properties, licensePropertyNamesWithId),
+    nextEntityIdBeforeTransaction
+  )
+}
+
+async function mediaLocation(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  location: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<MediaLocation> {
+  let loc: MediaLocation | undefined
+  const { entityId, existing } = location
+  if (existing) {
+    loc = await db.get(MediaLocation, { where: { id: entityId.toString() } })
+    if (!loc) throw Error(`MediaLocation entity not found`)
+    return loc
+  }
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+
+  // could be created in the transaction
+  loc = await db.get(MediaLocation, {
+    where: { id },
+  })
+  if (loc) return loc
+
+  const { properties } = findEntity(entityId, 'MediaLocation', classEntityMap)
+  return await createMediaLocation(
+    { db, block, id },
+    classEntityMap,
+    decode.setEntityPropertyValues<IMediaLocation>(properties, mediaLocationPropertyNamesWithId),
+    nextEntityIdBeforeTransaction
+  )
+}
+
+function removeInsertedEntity(key: string, insertedEntityId: number, classEntityMap: ClassEntityMap) {
+  const newlyCreatedEntities = classEntityMap.get(key)
+  // Remove the inserted entity from the list
+  classEntityMap.set(
+    key,
+    newlyCreatedEntities!.filter((e) => e.entityId !== insertedEntityId)
+  )
+}
+
+export const getOrCreate = {
+  language,
+  videoMediaEncoding,
+  videoMedia,
+  knownLicense,
+  userDefinedLicense,
+  channel,
+  category,
+  joystreamMediaLocation,
+  httpMediaLocation,
+  license,
+  mediaLocation,
+  nextEntityId,
+}

query-node/mappings/content-directory/transaction.ts

@@ -1,8 +1,12 @@
 import Debug from 'debug'
 
 import { DB, SubstrateEvent } from '../../generated/indexer'
+import { NextEntityId } from '../../generated/graphql-server/src/modules/next-entity-id/next-entity-id.model'
+import { ClassEntity } from '../../generated/graphql-server/src/modules/class-entity/class-entity.model'
+
 import { decode } from './decode'
 import {
+  ClassEntityMap,
   ICategory,
   IChannel,
   ICreateEntityOperation,
@@ -12,6 +16,8 @@ import {
   IJoystreamMediaLocation,
   IKnownLicense,
   ILanguage,
+  ILicense,
+  IMediaLocation,
   IUserDefinedLicense,
   IVideo,
   IVideoMedia,
@@ -30,19 +36,10 @@ import {
   videoPropertyNamesWithId,
   languagePropertyNamesWIthId,
   ContentDirectoryKnownClasses,
+  licensePropertyNamesWithId,
+  mediaLocationPropertyNamesWithId,
 } from './content-dir-consts'
 import {
-  createCategory,
-  createChannel,
-  createVideoMedia,
-  createVideo,
-  createUserDefinedLicense,
-  createKnownLicense,
-  createHttpMediaLocation,
-  createJoystreamMediaLocation,
-  createLanguage,
-  createVideoMediaEncoding,
-  getClassName,
   updateCategoryEntityPropertyValues,
   updateChannelEntityPropertyValues,
   updateVideoMediaEntityPropertyValues,
@@ -53,10 +50,36 @@ import {
   updateKnownLicenseEntityPropertyValues,
   updateLanguageEntityPropertyValues,
   updateVideoMediaEncodingEntityPropertyValues,
-  batchCreateClassEntities,
-} from './entity-helper'
+  updateLicenseEntityPropertyValues,
+  updateMediaLocationEntityPropertyValues,
+} from './entity/update'
 
-const debug = Debug('mappings:content-directory')
+import {
+  createCategory,
+  createChannel,
+  createVideoMedia,
+  createVideo,
+  createUserDefinedLicense,
+  createKnownLicense,
+  createHttpMediaLocation,
+  createJoystreamMediaLocation,
+  createLanguage,
+  createVideoMediaEncoding,
+  getClassName,
+  createLicense,
+  createMediaLocation,
+  createBlockOrGetFromDatabase,
+} from './entity/create'
+import { getOrCreate } from './get-or-create'
+
+const debug = Debug('mappings:cd:transaction')
+
+async function getNextEntityId(db: DB): Promise<number> {
+  const e = await db.get(NextEntityId, { where: { id: '1' } })
+  // Entity creation happens before addSchemaSupport so this should never happen
+  if (!e) throw Error(`NextEntityId table doesn't have any record`)
+  return e.nextId
+}
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
 export async function contentDirectory_TransactionCompleted(db: DB, event: SubstrateEvent): Promise<void> {
@@ -80,11 +103,30 @@ export async function contentDirectory_TransactionCompleted(db: DB, event: Subst
 
   // Create entities before adding schema support
   // We need this to know which entity belongs to which class(we will need to know to update/create
-  // Channel, Video etc.). For example if there is
-  // a property update operation there is no class id
+  // Channel, Video etc.). For example if there is a property update operation there is no class id
   await batchCreateClassEntities(db, block, createEntityOperations)
-  await batchUpdatePropertyValue(db, createEntityOperations, updatePropertyValuesOperations)
+
   await batchAddSchemaSupportToEntity(db, createEntityOperations, addSchemaSupportToEntityOperations, block)
+
+  await batchUpdatePropertyValue(db, createEntityOperations, updatePropertyValuesOperations)
+}
+
+async function batchCreateClassEntities(db: DB, block: number, operations: ICreateEntityOperation[]): Promise<void> {
+  const nId = await db.get(NextEntityId, { where: { id: '1' } })
+  let nextId = nId ? nId.nextId : 1 // start entity id from 1
+
+  for (const { classId } of operations) {
+    const c = new ClassEntity({
+      id: nextId.toString(), // entity id
+      classId: classId,
+      version: block,
+      happenedIn: await createBlockOrGetFromDatabase(db, block),
+    })
+    await db.save<ClassEntity>(c)
+    nextId++
+  }
+
+  await getOrCreate.nextEntityId(db, nextId)
 }
 
 /**
@@ -100,80 +142,124 @@ async function batchAddSchemaSupportToEntity(
   entities: IEntity[],
   block: number
 ) {
-  // find the related entity ie. Channel, Video etc
-  for (const entity of entities) {
-    const { entityId, indexOf, properties } = entity
-
-    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-    const id = entityId ? entityId.toString() : indexOf!.toString()
+  const classEntityMap: ClassEntityMap = new Map<string, IEntity[]>()
 
+  for (const entity of entities) {
     const className = await getClassName(db, entity, createEntityOperations)
-    if (className === undefined) continue
-
-    const arg: IDBBlockId = { db, block, id }
-
-    switch (className) {
-      case ContentDirectoryKnownClasses.CATEGORY:
-        await createCategory(arg, decode.setEntityPropertyValues<ICategory>(properties, CategoryPropertyNamesWithId))
-        break
-
-      case ContentDirectoryKnownClasses.CHANNEL:
-        await createChannel(arg, decode.setEntityPropertyValues<IChannel>(properties, channelPropertyNamesWithId))
-        break
-
-      case ContentDirectoryKnownClasses.KNOWNLICENSE:
-        await createKnownLicense(
-          arg,
-          decode.setEntityPropertyValues<IKnownLicense>(properties, knownLicensePropertyNamesWIthId)
-        )
-        break
-
-      case ContentDirectoryKnownClasses.USERDEFINEDLICENSE:
-        await createUserDefinedLicense(
-          arg,
-          decode.setEntityPropertyValues<IUserDefinedLicense>(properties, userDefinedLicensePropertyNamesWithId)
-        )
-        break
-
-      case ContentDirectoryKnownClasses.JOYSTREAMMEDIALOCATION:
-        await createJoystreamMediaLocation(
-          arg,
-          decode.setEntityPropertyValues<IJoystreamMediaLocation>(properties, joystreamMediaLocationPropertyNamesWithId)
-        )
-        break
-
-      case ContentDirectoryKnownClasses.HTTPMEDIALOCATION:
-        await createHttpMediaLocation(
-          arg,
-          decode.setEntityPropertyValues<IHttpMediaLocation>(properties, httpMediaLocationPropertyNamesWithId)
-        )
-        break
-
-      case ContentDirectoryKnownClasses.VIDEOMEDIA:
-        await createVideoMedia(
-          arg,
-          decode.setEntityPropertyValues<IVideoMedia>(properties, videoMediaPropertyNamesWithId)
-        )
-        break
-
-      case ContentDirectoryKnownClasses.VIDEO:
-        await createVideo(arg, decode.setEntityPropertyValues<IVideo>(properties, videoPropertyNamesWithId))
-        break
-
-      case ContentDirectoryKnownClasses.LANGUAGE:
-        await createLanguage(arg, decode.setEntityPropertyValues<ILanguage>(properties, languagePropertyNamesWIthId))
-        break
-
-      case ContentDirectoryKnownClasses.VIDEOMEDIAENCODING:
-        await createVideoMediaEncoding(
-          arg,
-          decode.setEntityPropertyValues<IVideoMediaEncoding>(properties, videoMediaEncodingPropertyNamesWithId)
-        )
-        break
+    if (className !== undefined) {
+      const es = classEntityMap.get(className)
+      classEntityMap.set(className, es ? [...es, entity] : [entity])
+    }
+  }
 
-      default:
-        console.log(`Unknown class name: ${className}`)
-        break
+  // This is a copy of classEntityMap, we will use it to keep track of items.
+  // We will remove items from this list whenever we insert them into db
+  const doneList: ClassEntityMap = new Map(classEntityMap.entries())
+
+  const nextEntityIdBeforeTransaction = (await getNextEntityId(db)) - createEntityOperations.length
+
+  for (const [className, entities] of classEntityMap) {
+    for (const entity of entities) {
+      const { entityId, indexOf, properties } = entity
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      const id = entityId !== undefined ? entityId : indexOf! + nextEntityIdBeforeTransaction
+      const arg: IDBBlockId = { db, block, id: id.toString() }
+
+      switch (className) {
+        case ContentDirectoryKnownClasses.CATEGORY:
+          await createCategory(arg, decode.setEntityPropertyValues<ICategory>(properties, CategoryPropertyNamesWithId))
+          break
+
+        case ContentDirectoryKnownClasses.CHANNEL:
+          await createChannel(
+            arg,
+            doneList,
+            decode.setEntityPropertyValues<IChannel>(properties, channelPropertyNamesWithId),
+            nextEntityIdBeforeTransaction
+          )
+          break
+
+        case ContentDirectoryKnownClasses.KNOWNLICENSE:
+          await createKnownLicense(
+            arg,
+            decode.setEntityPropertyValues<IKnownLicense>(properties, knownLicensePropertyNamesWIthId)
+          )
+          break
+
+        case ContentDirectoryKnownClasses.USERDEFINEDLICENSE:
+          await createUserDefinedLicense(
+            arg,
+            decode.setEntityPropertyValues<IUserDefinedLicense>(properties, userDefinedLicensePropertyNamesWithId)
+          )
+          break
+
+        case ContentDirectoryKnownClasses.JOYSTREAMMEDIALOCATION:
+          await createJoystreamMediaLocation(
+            arg,
+            decode.setEntityPropertyValues<IJoystreamMediaLocation>(
+              properties,
+              joystreamMediaLocationPropertyNamesWithId
+            )
+          )
+          break
+
+        case ContentDirectoryKnownClasses.HTTPMEDIALOCATION:
+          await createHttpMediaLocation(
+            arg,
+            decode.setEntityPropertyValues<IHttpMediaLocation>(properties, httpMediaLocationPropertyNamesWithId)
+          )
+          break
+
+        case ContentDirectoryKnownClasses.VIDEOMEDIA:
+          await createVideoMedia(
+            arg,
+            doneList,
+            decode.setEntityPropertyValues<IVideoMedia>(properties, videoMediaPropertyNamesWithId),
+            nextEntityIdBeforeTransaction
+          )
+          break
+
+        case ContentDirectoryKnownClasses.VIDEO:
+          await createVideo(
+            arg,
+            doneList,
+            decode.setEntityPropertyValues<IVideo>(properties, videoPropertyNamesWithId),
+            nextEntityIdBeforeTransaction
+          )
+          break
+
+        case ContentDirectoryKnownClasses.LANGUAGE:
+          await createLanguage(arg, decode.setEntityPropertyValues<ILanguage>(properties, languagePropertyNamesWIthId))
+          break
+
+        case ContentDirectoryKnownClasses.VIDEOMEDIAENCODING:
+          await createVideoMediaEncoding(
+            arg,
+            decode.setEntityPropertyValues<IVideoMediaEncoding>(properties, videoMediaEncodingPropertyNamesWithId)
+          )
+          break
+
+        case ContentDirectoryKnownClasses.LICENSE:
+          await createLicense(
+            arg,
+            classEntityMap,
+            decode.setEntityPropertyValues<ILicense>(properties, licensePropertyNamesWithId),
+            nextEntityIdBeforeTransaction
+          )
+          break
+        case ContentDirectoryKnownClasses.MEDIALOCATION:
+          await createMediaLocation(
+            arg,
+            classEntityMap,
+            decode.setEntityPropertyValues<IMediaLocation>(properties, mediaLocationPropertyNamesWithId),
+            nextEntityIdBeforeTransaction
+          )
+          break
+
+        default:
+          console.log(`Unknown class name: ${className}`)
+          break
+      }
     }
   }
 }
@@ -185,12 +271,14 @@ async function batchAddSchemaSupportToEntity(
  * @param entities list of entities those properties values updated
  */
 async function batchUpdatePropertyValue(db: DB, createEntityOperations: ICreateEntityOperation[], entities: IEntity[]) {
+  const entityIdBeforeTransaction = (await getNextEntityId(db)) - createEntityOperations.length
+
   for (const entity of entities) {
     const { entityId, indexOf, properties } = entity
     // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-    const id = entityId ? entityId.toString() : indexOf!.toString()
+    const id = entityId ? entityId.toString() : entityIdBeforeTransaction - indexOf!
 
-    const where: IWhereCond = { where: { id } }
+    const where: IWhereCond = { where: { id: id.toString() } }
     const className = await getClassName(db, entity, createEntityOperations)
     if (className === undefined) {
       console.log(`Can not update entity properties values. Unknown class name`)
@@ -199,10 +287,11 @@ async function batchUpdatePropertyValue(db: DB, createEntityOperations: ICreateE
 
     switch (className) {
       case ContentDirectoryKnownClasses.CHANNEL:
-        updateChannelEntityPropertyValues(
+        await updateChannelEntityPropertyValues(
           db,
           where,
-          decode.setEntityPropertyValues<IChannel>(properties, CategoryPropertyNamesWithId)
+          decode.setEntityPropertyValues<IChannel>(properties, channelPropertyNamesWithId),
+          entityIdBeforeTransaction
         )
         break
 
@@ -250,7 +339,8 @@ async function batchUpdatePropertyValue(db: DB, createEntityOperations: ICreateE
         await updateVideoMediaEntityPropertyValues(
           db,
           where,
-          decode.setEntityPropertyValues<IVideoMedia>(properties, videoPropertyNamesWithId)
+          decode.setEntityPropertyValues<IVideoMedia>(properties, videoPropertyNamesWithId),
+          entityIdBeforeTransaction
         )
         break
 
@@ -258,7 +348,8 @@ async function batchUpdatePropertyValue(db: DB, createEntityOperations: ICreateE
         await updateVideoEntityPropertyValues(
           db,
           where,
-          decode.setEntityPropertyValues<IVideo>(properties, videoPropertyNamesWithId)
+          decode.setEntityPropertyValues<IVideo>(properties, videoPropertyNamesWithId),
+          entityIdBeforeTransaction
         )
         break
 
@@ -277,6 +368,22 @@ async function batchUpdatePropertyValue(db: DB, createEntityOperations: ICreateE
           decode.setEntityPropertyValues<IVideoMediaEncoding>(properties, videoMediaEncodingPropertyNamesWithId)
         )
         break
+      case ContentDirectoryKnownClasses.LICENSE:
+        await updateLicenseEntityPropertyValues(
+          db,
+          where,
+          decode.setEntityPropertyValues<ILicense>(properties, licensePropertyNamesWithId),
+          entityIdBeforeTransaction
+        )
+        break
+      case ContentDirectoryKnownClasses.MEDIALOCATION:
+        await updateMediaLocationEntityPropertyValues(
+          db,
+          where,
+          decode.setEntityPropertyValues<IMediaLocation>(properties, mediaLocationPropertyNamesWithId),
+          entityIdBeforeTransaction
+        )
+        break
 
       default:
         console.log(`Unknown class name: ${className}`)

query-node/mappings/types.ts

@@ -34,6 +34,11 @@ export interface MemberControllerAccount extends BaseJoystreamMember {
   controllerAccount: Buffer
 }
 
+export interface IReference {
+  entityId: number
+  existing: boolean
+}
+
 export interface IChannel {
   title: string
   description: string
@@ -41,7 +46,7 @@ export interface IChannel {
   avatarPhotoURL: string
   isPublic: boolean
   isCurated: boolean
-  language: number
+  language?: IReference
 }
 
 export interface ICategory {
@@ -79,32 +84,42 @@ export interface IVideoMediaEncoding {
 }
 
 export interface IVideoMedia {
-  encoding: number
+  encoding?: IReference
   pixelWidth: number
   pixelHeight: number
   size: number
-  location: number
+  location?: IReference
 }
 
 export interface IVideo {
   // referenced entity's id
-  channel: number
+  channel?: IReference
   // referenced entity's id
-  category: number
+  category?: IReference
   title: string
   description: string
   duration: number
   skippableIntroDuration?: number
   thumbnailURL: string
-  language: number
+  language?: IReference
   // referenced entity's id
-  media: number
+  media?: IReference
   hasMarketing?: boolean
   publishedBeforeJoystream?: number
   isPublic: boolean
   isCurated: boolean
   isExplicit: boolean
-  license: number
+  license?: IReference
+}
+
+export interface ILicense {
+  knownLicense?: IReference
+  userDefinedLicense?: IReference
+}
+
+export interface IMediaLocation {
+  httpMediaLocation?: IReference
+  joystreamMediaLocation?: IReference
 }
 
 export enum OperationType {
@@ -135,9 +150,15 @@ export interface IBatchOperation {
 }
 
 export interface IProperty {
-  [propertyId: string]: any
-  // propertyId: string;
-  // value: any;
+  // PropertId: Value
+  // [propertyId: string]: any
+
+  id: string
+  value: any
+
+  // If reference.exising is false then reference.entityId is the index that entity is at
+  // in the transaction batch operation
+  reference?: IReference
 }
 
 export interface IEntity {
@@ -166,5 +187,8 @@ export interface ICreateEntityOperation {
 export interface IDBBlockId {
   db: DB
   block: number
+  // Entity id
   id: string
 }
+
+export type ClassEntityMap = Map<string, IEntity[]>

query-node/package.json

@@ -3,7 +3,7 @@
 	"version": "1.0.0",
 	"description": "GraphQL server and Substrate indexer. Generated with ♥ by Hydra-CLI",
 	"scripts": {
-		"build": "yarn codegen:all && (ln -s ../../../../../node_modules/typeorm/cli.js generated/graphql-server/node_modules/.bin/typeorm || :) && tsc --build tsconfig.json",
+		"build": "./build.sh",
 		"test": "echo \"Error: no test specified\" && exit 1",
 		"clean": "rm -rf ./generated",
 		"processor:start": "(cd ./generated/indexer && yarn && DEBUG=${DEBUG} yarn start:processor)",
@@ -19,12 +19,13 @@
 		"codegen:all": "yarn hydra-cli codegen && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
 		"codegen:indexer": "yarn hydra-cli codegen --no-graphql && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
 		"codegen:server": "yarn hydra-cli codegen --no-indexer",
-		"docker:up": "docker-compose up -d"
+		"docker:up": "docker-compose up -d",
+    "cd-classes": "ts-node scripts/get-class-id-and-name.ts"
 	},
 	"author": "",
 	"license": "ISC",
 	"devDependencies": {
-		"@dzlzv/hydra-cli": "^0.0.17"
+		"@dzlzv/hydra-cli": "^0.0.19"
 	},
 	"dependencies": {
 		"@joystream/types": "^0.14.0",

query-node/schema.graphql

@@ -8,8 +8,8 @@ type Block @entity {
   "Block number as a string"
   id: ID!
   block: Int!
-  timestamp: Int!
-  nework: Network!
+  timestamp: BigInt!
+  network: Network!
 }
 
 "Stored information about a registered user"
@@ -55,6 +55,14 @@ type ClassEntity @entity {
   happenedIn: Block!
 }
 
+"Keep track of the next entity id"
+type NextEntityId @entity {
+  "Constant field is set to '1'"
+  id: ID!
+
+  nextId: Int!
+}
+
 #### High Level Derivative Entities ####
 
 type Language @entity {
@@ -94,9 +102,9 @@ type Channel @entity {
   isCurated: Boolean!
 
   "The primary langauge of the channel's content"
-  languageId: Int
+  language: Language!
 
-  # videos: [Video!] @derivedFrom(field: "channel")
+  videos: [Video!] @derivedFrom(field: "channel")
 
   happenedIn: Block!
 }
@@ -111,7 +119,7 @@ type Category @entity {
   "The description of the category"
   description: String
 
-  # videos: [Video!] @derivedFrom(field: "category")
+  videos: [Video!] @derivedFrom(field: "category")
 
   happenedIn: Block!
 }
@@ -122,6 +130,8 @@ type VideoMediaEncoding @entity {
   id: ID!
 
   name: String!
+
+  happenedIn: Block!
 }
 
 type KnownLicense @entity {
@@ -153,6 +163,38 @@ type UserDefinedLicense @entity {
   happenedIn: Block!
 }
 
+type License @entity {
+  "Runtime entity identifier (EntityId)"
+  id: ID!
+
+  # One of the following field will be non-null
+
+  "Reference to a known license"
+  knownLicense: KnownLicense
+
+  "Reference to user-defined license"
+  userdefinedLicense: UserDefinedLicense
+
+  happenedIn: Block!
+}
+
+type MediaLocation @entity {
+  "Runtime entity identifier (EntityId)"
+  id: ID!
+
+  # One of the following field will be non-null
+
+  "A reference to HttpMediaLocation"
+  httpMediaLocation: HttpMediaLocation
+
+  "A reference to JoystreamMediaLocation"
+  joystreamMediaLocation: JoystreamMediaLocation
+
+  videoMedia: VideoMedia @derivedFrom(field: "location")
+
+  happenedIn: Block!
+}
+
 type JoystreamMediaLocation @entity {
   "Runtime entity identifier (EntityId)"
   id: ID!
@@ -181,7 +223,7 @@ type VideoMedia @entity {
   id: ID!
 
   "Encoding of the video media object"
-  encodingId: Int!
+  encoding: VideoMediaEncoding!
 
   "Video media width in pixels"
   pixelWidth: Int!
@@ -192,13 +234,10 @@ type VideoMedia @entity {
   "Video media size in bytes"
   size: Int
 
-  # video: Video! @derivedFrom(field: "media")
-
-  # One of the location field will be non-null
+  video: Video @derivedFrom(field: "media")
 
-  # httpMediaLocation: HttpMediaLocation
-  # joystreamMediaLocation: JoystreamMediaLocation
-  locationId: Int!
+  "Location of the video media object"
+  location: MediaLocation!
 
   happenedIn: Block!
 }
@@ -208,10 +247,10 @@ type Video @entity {
   id: ID!
 
   "Reference to member's channel"
-  channelId: Int!
+  channel: Channel!
 
   "Reference to a video category"
-  categoryId: Int!
+  category: Category!
 
   "The title of the video"
   title: String! @fulltext(query: "titles")
@@ -229,10 +268,10 @@ type Video @entity {
   thumbnailURL: String!
 
   "Video's main langauge"
-  languageId: Int
+  language: Language
 
   "Reference to VideoMedia"
-  videoMediaId: Int!
+  media: VideoMedia!
 
   "Whether or not Video contains marketing"
   hasMarketing: Boolean
@@ -249,8 +288,7 @@ type Video @entity {
   "Whether the Video contains explicit material."
   isExplicit: Boolean!
 
-  # Lincense
-  licenseId: Int!
+  license: License!
 
   happenedIn: Block!
 }

query-node/scripts/get-class-id-and-name.ts

@@ -0,0 +1,21 @@
+import { ApiPromise, WsProvider } from '@polkadot/api'
+import { types as joyTypes } from '@joystream/types'
+import * as BN from 'bn.js'
+
+async function main() {
+  // Initialize the api
+  const provider = new WsProvider('ws://127.0.0.1:9944')
+  const api = await ApiPromise.create({ provider, types: joyTypes })
+
+  const n = await api.query.contentDirectory.nextClassId()
+  const nextClassId = new BN(n.toJSON() as string).toNumber()
+  for (let id = 0; id < nextClassId; id++) {
+    const cls = await api.query.contentDirectory.classById(new BN(id))
+    const { name } = cls.toJSON() as never
+    console.log(id, name)
+  }
+}
+
+main()
+  .then(() => process.exit())
+  .catch(console.error)

tests/network-tests/.env

@@ -1,5 +1,7 @@
 # Address of the Joystream node.
 NODE_URL = ws://127.0.0.1:9944
+# Address of the Joystream query node.
+QUERY_NODE_URL = http://127.0.0.1:8080/graphql
 # Account which is expected to provide sufficient funds to test accounts.
 TREASURY_ACCOUNT_URI = //Alice
 # Sudo Account

tests/network-tests/package.json

@@ -21,7 +21,8 @@
     "bn.js": "^4.11.8",
     "dotenv": "^8.2.0",
     "fs": "^0.0.1-security",
-    "uuid": "^7.0.3"
+    "uuid": "^7.0.3",
+    "@apollo/client": "^3.2.5"
   },
   "devDependencies": {
     "@polkadot/ts": "^0.3.14",

tests/network-tests/src/Api.ts

@@ -13,7 +13,7 @@ import {
   Opening as WorkingGroupOpening,
 } from '@joystream/types/working-group'
 import { ElectionStake, Seat } from '@joystream/types/council'
-import { AccountInfo, Balance, BalanceOf, BlockNumber, Event, EventRecord } from '@polkadot/types/interfaces'
+import { AccountInfo, Hash, Balance, BalanceOf, BlockNumber, Event, EventRecord } from '@polkadot/types/interfaces'
 import BN from 'bn.js'
 import { SubmittableExtrinsic } from '@polkadot/api/types'
 import { Sender } from './sender'
@@ -30,6 +30,12 @@ import {
 } from '@joystream/types/hiring'
 import { FillOpeningParameters, ProposalId } from '@joystream/types/proposals'
 import { v4 as uuid } from 'uuid'
+import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
+import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
+import { initializeContentDir, InputParser, ExtrinsicsHelper } from 'cd-schemas'
+import { OperationType } from '@joystream/types/content-directory'
+import { gql, ApolloClient, ApolloQueryResult, NormalizedCacheObject } from '@apollo/client'
+
 import Debugger from 'debug'
 const debug = Debugger('api')
 
@@ -39,11 +45,11 @@ export enum WorkingGroups {
 }
 
 export class Api {
-  private readonly api: ApiPromise
-  private readonly sender: Sender
-  private readonly keyring: Keyring
+  protected readonly api: ApiPromise
+  protected readonly sender: Sender
+  protected readonly keyring: Keyring
   // source of funds for all new accounts
-  private readonly treasuryAccount: string
+  protected readonly treasuryAccount: string
 
   public static async create(provider: WsProvider, treasuryAccountUri: string, sudoAccountUri: string): Promise<Api> {
     let connectAttempts = 0
@@ -1707,6 +1713,7 @@ export class Api {
     ).filter((addr) => addr !== '')
   }
   */
+
   public async terminateApplication(
     leader: string,
     applicationId: ApplicationId,
@@ -1918,4 +1925,131 @@ export class Api {
   public getMaxWorkersCount(module: WorkingGroups): BN {
     return this.api.createType('u32', this.api.consts[module].maxWorkerNumberLimit)
   }
+
+  async sendContentDirectoryTransaction(operations: OperationType[]): Promise<void> {
+    const transaction = this.api.tx.contentDirectory.transaction(
+      { Lead: null }, // We use member with id 0 as actor (in this case we assume this is Alice)
+      operations // We provide parsed operations as second argument
+    )
+    const lead = (await this.getGroupLead(WorkingGroups.ContentDirectoryWorkingGroup)) as Worker
+    await this.sender.signAndSend(transaction, lead.role_account_id, false)
+  }
+
+  public async createChannelEntity(channel: ChannelEntity): Promise<void> {
+    // Create the parser with known entity schemas (the ones in content-directory-schemas/inputs)
+    const parser = InputParser.createWithKnownSchemas(
+      this.api,
+      // The second argument is an array of entity batches, following standard entity batch syntax ({ className, entries }):
+      [
+        {
+          className: 'Channel',
+          entries: [channel], // We could specify multiple entries here, but in this case we only need one
+        },
+      ]
+    )
+    // We parse the input into CreateEntity and AddSchemaSupportToEntity operations
+    const operations = await parser.getEntityBatchOperations()
+    return await this.sendContentDirectoryTransaction(operations)
+  }
+
+  public async createVideoEntity(video: VideoEntity): Promise<void> {
+    // Create the parser with known entity schemas (the ones in content-directory-schemas/inputs)
+    const parser = InputParser.createWithKnownSchemas(
+      this.api,
+      // The second argument is an array of entity batches, following standard entity batch syntax ({ className, entries }):
+      [
+        {
+          className: 'Video',
+          entries: [video], // We could specify multiple entries here, but in this case we only need one
+        },
+      ]
+    )
+    // We parse the input into CreateEntity and AddSchemaSupportToEntity operations
+    const operations = await parser.getEntityBatchOperations()
+    return await this.sendContentDirectoryTransaction(operations)
+  }
+
+  public async updateChannelEntity(
+    channelUpdateInput: Record<string, any>,
+    uniquePropValue: Record<string, any>
+  ): Promise<void> {
+    // Create the parser with known entity schemas (the ones in content-directory-schemas/inputs)
+    const parser = InputParser.createWithKnownSchemas(this.api)
+
+    // We can reuse InputParser's `findEntityIdByUniqueQuery` method to find entityId of the channel we
+    // created in ./createChannel.ts example (normally we would probably use some other way to do it, ie.: query node)
+    const CHANNEL_ID = await parser.findEntityIdByUniqueQuery(uniquePropValue, 'Channel') // Use getEntityUpdateOperations to parse the update input
+    const updateOperations = await parser.getEntityUpdateOperations(
+      channelUpdateInput,
+      'Channel', // Class name
+      CHANNEL_ID // Id of the entity we want to update
+    )
+    return await this.sendContentDirectoryTransaction(updateOperations)
+  }
+
+  public async initializeContentDirectory(leadKeyPair: KeyringPair) {
+    await initializeContentDir(this.api, leadKeyPair)
+  }
+}
+
+export class QueryNodeApi extends Api {
+  private readonly queryNodeProvider: ApolloClient<NormalizedCacheObject>
+
+  public static async new(
+    provider: WsProvider,
+    queryNodeProvider: ApolloClient<NormalizedCacheObject>,
+    treasuryAccountUri: string,
+    sudoAccountUri: string
+  ): Promise<QueryNodeApi> {
+    let connectAttempts = 0
+    while (true) {
+      connectAttempts++
+      debug(`Connecting to chain, attempt ${connectAttempts}..`)
+      try {
+        const api = await ApiPromise.create({ provider, types })
+
+        // Wait for api to be connected and ready
+        await api.isReady
+
+        // If a node was just started up it might take a few seconds to start producing blocks
+        // Give it a few seconds to be ready.
+        await Utils.wait(5000)
+
+        return new QueryNodeApi(api, queryNodeProvider, treasuryAccountUri, sudoAccountUri)
+      } catch (err) {
+        if (connectAttempts === 3) {
+          throw new Error('Unable to connect to chain')
+        }
+      }
+      await Utils.wait(5000)
+    }
+  }
+
+  constructor(
+    api: ApiPromise,
+    queryNodeProvider: ApolloClient<NormalizedCacheObject>,
+    treasuryAccountUri: string,
+    sudoAccountUri: string
+  ) {
+    super(api, treasuryAccountUri, sudoAccountUri)
+    this.queryNodeProvider = queryNodeProvider
+  }
+
+  public async getChannelbyTitle(title: string): Promise<ApolloQueryResult<any>> {
+    const GET_CHANNEL_BY_TITLE = gql`
+      query($title: String!) {
+        channels(where: { title_eq: $title }) {
+          title
+          description
+          coverPhotoUrl
+          avatarPhotoUrl
+          isPublic
+          isCurated
+          languageId
+        }
+      }
+    `
+
+    return await this.queryNodeProvider.query({ query: GET_CHANNEL_BY_TITLE, variables: { title } })
+  }
 }

tests/network-tests/src/fixtures/contentDirectoryModule.ts

@@ -0,0 +1,65 @@
+import { QueryNodeApi } from '../Api'
+import BN from 'bn.js'
+import { assert } from 'chai'
+import { Seat } from '@joystream/types/council'
+import { v4 as uuid } from 'uuid'
+import { Utils } from '../utils'
+import { Fixture } from '../Fixture'
+import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
+import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
+
+export class CreateChannelFixture implements Fixture {
+  private api: QueryNodeApi
+  public channelEntity: ChannelEntity
+
+  public constructor(api: QueryNodeApi, channelEntity: ChannelEntity) {
+    this.api = api
+    this.channelEntity = channelEntity
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    await this.api.createChannelEntity(this.channelEntity)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class CreateVideoFixture implements Fixture {
+  private api: QueryNodeApi
+  private videoEntity: VideoEntity
+
+  public constructor(api: QueryNodeApi, videoEntity: VideoEntity) {
+    this.api = api
+    this.videoEntity = videoEntity
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    await this.api.createVideoEntity(this.videoEntity)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}
+
+export class UpdateChannelFixture implements Fixture {
+  private api: QueryNodeApi
+  private channelUpdateInput: Record<string, any>
+  private uniquePropValue: Record<string, any>
+
+  public constructor(api: QueryNodeApi, channelUpdateInput: Record<string, any>, uniquePropValue: Record<string, any>) {
+    this.api = api
+    this.channelUpdateInput = channelUpdateInput
+    this.uniquePropValue = uniquePropValue
+  }
+
+  public async runner(expectFailure: boolean): Promise<void> {
+    await this.api.updateChannelEntity(this.channelUpdateInput, this.uniquePropValue)
+
+    if (expectFailure) {
+      throw new Error('Successful fixture run while expecting failure')
+    }
+  }
+}

tests/network-tests/src/flows/contentDirectory/contentDirectoryInitialization.ts

@@ -0,0 +1,7 @@
+import { Api, WorkingGroups } from '../../Api'
+import { assert } from 'chai'
+import { KeyringPair } from '@polkadot/keyring/types'
+
+export default async function initializeContentDirectory(api: Api, leadKeyPair: KeyringPair) {
+  await api.initializeContentDirectory(leadKeyPair)
+}

tests/network-tests/src/flows/contentDirectory/creatingChannel.ts

@@ -0,0 +1,43 @@
+import { QueryNodeApi } from '../../Api'
+import { Utils } from '../../utils'
+import { CreateChannelFixture } from '../../fixtures/contentDirectoryModule'
+import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
+import { assert } from 'chai'
+import { KeyringPair } from '@polkadot/keyring/types'
+
+export function createSimpleChannelFixture(api: QueryNodeApi): CreateChannelFixture {
+  const channelEntity: ChannelEntity = {
+    title: 'Example channel',
+    description: 'This is an example channel',
+    // We can use "existing" syntax to reference either an on-chain entity or other entity that's part of the same batch.
+    // Here we reference language that we assume was added by initialization script (initialize:dev), as it is part of
+    // input/entityBatches/LanguageBatch.json
+    language: { existing: { code: 'EN' } },
+    coverPhotoUrl: '',
+    avatarPhotoURL: '',
+    isPublic: true,
+  }
+  return new CreateChannelFixture(api, channelEntity)
+}
+
+export default async function channelCreation(api: QueryNodeApi) {
+  const createChannelHappyCaseFixture = createSimpleChannelFixture(api)
+
+  await createChannelHappyCaseFixture.runner(false)
+
+  // Temporary solution (wait 2 minutes)
+  await Utils.wait(120000)
+
+  // Ensure newly created channel was parsed by query node
+  const result = await api.getChannelbyTitle(createChannelHappyCaseFixture.channelEntity.title)
+  const queriedChannel = result.data.channels[0]
+
+  assert(queriedChannel.title === createChannelHappyCaseFixture.channelEntity.title, 'Should be equal')
+  assert(queriedChannel.description === createChannelHappyCaseFixture.channelEntity.description, 'Should be equal')
+  assert(queriedChannel.coverPhotoUrl === createChannelHappyCaseFixture.channelEntity.coverPhotoUrl, 'Should be equal')
+  assert(
+    queriedChannel.avatarPhotoUrl === createChannelHappyCaseFixture.channelEntity.avatarPhotoURL,
+    'Should be equal'
+  )
+  assert(queriedChannel.isPublic === createChannelHappyCaseFixture.channelEntity.isPublic, 'Should be equal')
+}

tests/network-tests/src/flows/contentDirectory/creatingVideo.ts

@@ -0,0 +1,49 @@
+import { QueryNodeApi, WorkingGroups } from '../../Api'
+import { CreateVideoFixture } from '../../fixtures/contentDirectoryModule'
+import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
+import { assert } from 'chai'
+
+export function createVideoReferencingChannelFixture(api: QueryNodeApi): CreateVideoFixture {
+  const videoEntity: VideoEntity = {
+    title: 'Example video',
+    description: 'This is an example video',
+    // We reference existing language and category by their unique properties with "existing" syntax
+    // (those referenced here are part of inputs/entityBatches)
+    language: { existing: { code: 'EN' } },
+    category: { existing: { name: 'Education' } },
+    // We use the same "existing" syntax to reference a channel by unique property (title)
+    // In this case it's a channel that we created in createChannel example
+    channel: { existing: { title: 'Example channel' } },
+    media: {
+      // We use "new" syntax to sygnalize we want to create a new VideoMedia entity that will be related to this Video entity
+      new: {
+        // We use "exisiting" enconding from inputs/entityBatches/VideoMediaEncodingBatch.json
+        encoding: { existing: { name: 'H.263_MP4' } },
+        pixelHeight: 600,
+        pixelWidth: 800,
+        // We create nested VideoMedia->MediaLocation->HttpMediaLocation relations using the "new" syntax
+        location: { new: { httpMediaLocation: { new: { url: 'https://testnet.joystream.org/' } } } },
+      },
+    },
+    // Here we use combined "new" and "existing" syntaxes to create Video->License->KnownLicense relations
+    license: {
+      new: {
+        knownLicense: {
+          // This license can be found in inputs/entityBatches/KnownLicenseBatch.json
+          existing: { code: 'CC_BY' },
+        },
+      },
+    },
+    duration: 3600,
+    thumbnailURL: '',
+    isExplicit: false,
+    isPublic: true,
+  }
+  return new CreateVideoFixture(api, videoEntity)
+}
+
+export default async function createVideo(api: QueryNodeApi) {
+  const createVideoHappyCaseFixture = createVideoReferencingChannelFixture(api)
+
+  await createVideoHappyCaseFixture.runner(false)
+}

tests/network-tests/src/flows/contentDirectory/updatingChannel.ts

@@ -0,0 +1,21 @@
+import { QueryNodeApi, WorkingGroups } from '../../Api'
+import { UpdateChannelFixture } from '../../fixtures/contentDirectoryModule'
+import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
+import { assert } from 'chai'
+
+export function createUpdateChannelTitleFixture(api: QueryNodeApi): UpdateChannelFixture {
+  // Create partial channel entity, only containing the fields we wish to update
+  const channelUpdateInput: Partial<ChannelEntity> = {
+    title: 'Updated channel title',
+  }
+
+  const uniquePropVal: Record<string, any> = { title: 'Example channel' }
+
+  return new UpdateChannelFixture(api, channelUpdateInput, uniquePropVal)
+}
+
+export default async function updateChannel(api: QueryNodeApi) {
+  const createVideoHappyCaseFixture = createUpdateChannelTitleFixture(api)
+
+  await createVideoHappyCaseFixture.runner(false)
+}

tests/network-tests/src/flows/workingGroup/leaderSetup.ts

@@ -3,13 +3,17 @@ import BN from 'bn.js'
 import { PaidTermId } from '@joystream/types/members'
 import { SudoHireLeadFixture } from '../../fixtures/sudoHireLead'
 import { assert } from 'chai'
+import { KeyringPair } from '@polkadot/keyring/types'
 
 // Worker application happy case scenario
-export default async function leaderSetup(api: Api, env: NodeJS.ProcessEnv, group: WorkingGroups) {
+export default async function leaderSetup(
+  api: Api,
+  env: NodeJS.ProcessEnv,
+  group: WorkingGroups
+): Promise<KeyringPair> {
   const lead = await api.getGroupLead(group)
-  if (lead) {
-    return
-  }
+
+  assert(!lead, `Lead is already set`)
 
   const leadKeyPair = api.createKeyPairs(1)[0]
   const paidTerms: PaidTermId = api.createPaidTermId(new BN(+env.MEMBERSHIP_PAID_TERMS!))
@@ -37,4 +41,6 @@ export default async function leaderSetup(api: Api, env: NodeJS.ProcessEnv, grou
   const hiredLead = await api.getGroupLead(group)
   assert(hiredLead, `${group} group Lead was not hired!`)
   assert(hiredLead!.role_account_id.eq(leadKeyPair.address))
+
+  return leadKeyPair
 }

tests/network-tests/src/scenarios/content-directory.ts

@@ -1,7 +1,12 @@
 import { WsProvider } from '@polkadot/api'
-import { Api, WorkingGroups } from '../Api'
+import { Api, QueryNodeApi, WorkingGroups } from '../Api'
 import { config } from 'dotenv'
 import leaderSetup from '../flows/workingGroup/leaderSetup'
+import initializeContentDirectory from '../flows/contentDirectory/contentDirectoryInitialization'
+import createChannel from '../flows/contentDirectory/creatingChannel'
+import createVideo from '../flows/contentDirectory/creatingVideo'
+import updateChannel from '../flows/contentDirectory/updatingChannel'
+import { ApolloClient, InMemoryCache } from '@apollo/client'
 
 const scenario = async () => {
   // Load env variables
@@ -11,13 +16,34 @@ const scenario = async () => {
   // Connect api to the chain
   const nodeUrl: string = env.NODE_URL || 'ws://127.0.0.1:9944'
   const provider = new WsProvider(nodeUrl)
-  const api: Api = await Api.create(provider, env.TREASURY_ACCOUNT_URI || '//Alice', env.SUDO_ACCOUNT_URI || '//Alice')
 
-  await leaderSetup(api, env, WorkingGroups.ContentDirectoryWorkingGroup)
+  const queryNodeUrl: string = env.QUERY_NODE_URL || 'http://127.0.0.1:8080/graphql'
+
+  const queryNodeProvider = new ApolloClient({
+    uri: queryNodeUrl,
+    cache: new InMemoryCache(),
+  })
+
+  const api: QueryNodeApi = await QueryNodeApi.new(
+    provider,
+    queryNodeProvider,
+    env.TREASURY_ACCOUNT_URI || '//Alice',
+    env.SUDO_ACCOUNT_URI || '//Alice'
+  )
+
+  const leadKeyPair = await leaderSetup(api, env, WorkingGroups.ContentDirectoryWorkingGroup)
 
   // Some flows that use the curator lead to perform some tests...
   //
 
+  await initializeContentDirectory(api, leadKeyPair)
+
+  await createChannel(api)
+
+  await createVideo(api)
+
+  await updateChannel(api)
+
   // Note: disconnecting and then reconnecting to the chain in the same process
   // doesn't seem to work!
   api.close()

utils/api-scripts/src/status.ts

@@ -12,7 +12,7 @@ async function main() {
 
   // Create the API and wait until ready
   let api: ApiPromise
-  let retry = 3
+  let retry = 6
   while (true) {
     try {
       api = await ApiPromise.create({ provider, types })

yarn.lock

@@ -23,6 +23,25 @@
     call-me-maybe "^1.0.1"
     js-yaml "^3.13.1"
 
+"@apollo/client@^3.2.5":
+  version "3.2.5"
+  resolved "https://registry.yarnpkg.com/@apollo/client/-/client-3.2.5.tgz#24e0a6faa1d231ab44807af237c6227410c75c4d"
+  integrity sha512-zpruxnFMz6K94gs2pqc3sidzFDbQpKT5D6P/J/I9s8ekHZ5eczgnRp6pqXC86Bh7+44j/btpmOT0kwiboyqTnA==
+  dependencies:
+    "@graphql-typed-document-node/core" "^3.0.0"
+    "@types/zen-observable" "^0.8.0"
+    "@wry/context" "^0.5.2"
+    "@wry/equality" "^0.2.0"
+    fast-json-stable-stringify "^2.0.0"
+    graphql-tag "^2.11.0"
+    hoist-non-react-statics "^3.3.2"
+    optimism "^0.13.0"
+    prop-types "^15.7.2"
+    symbol-observable "^2.0.0"
+    ts-invariant "^0.4.4"
+    tslib "^1.10.0"
+    zen-observable "^0.8.14"
+
 "@apollo/protobufjs@^1.0.3":
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/@apollo/protobufjs/-/protobufjs-1.0.5.tgz#a78b726147efc0795e74c8cb8a11aafc6e02f773"
@@ -1396,7 +1415,7 @@
     typeorm-model-generator "^0.4.2"
     warthog "https://github.com/metmirr/warthog/releases/download/v2.20.0/warthog-v2.20.0.tgz"
 
-"@dzlzv/hydra-indexer-lib@0.0.19-legacy.1.26.1", "@dzlzv/hydra-indexer-lib@^0.0.19-legacy.1.26.1":
+"@dzlzv/hydra-indexer-lib@^0.0.19-legacy.1.26.1":
   version "0.0.19-legacy.1.26.1"
   resolved "https://registry.yarnpkg.com/@dzlzv/hydra-indexer-lib/-/hydra-indexer-lib-0.0.19-legacy.1.26.1.tgz#346b564845b2014f7a4d9b3976c03e30da8dd309"
   integrity sha512-4pwaSDRIo/1MqxjfSotjv91fkIj/bfZcZx5nqjB9gRT85X28b3WqkqTFrzlGsGGbvUFWAx4WIeQKnY1yrpX89Q==
@@ -1683,6 +1702,11 @@
   dependencies:
     prop-types "^15.7.2"
 
+"@graphql-typed-document-node/core@^3.0.0":
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/@graphql-typed-document-node/core/-/core-3.1.0.tgz#0eee6373e11418bfe0b5638f654df7a4ca6a3950"
+  integrity sha512-wYn6r8zVZyQJ6rQaALBEln5B1pzxb9shV5Ef97kTvn6yVGrqyXVnDqnU24MXnFubR+rZjBY9NWuxX3FB2sTsjg==
+
 "@hapi/address@2.x.x":
   version "2.1.4"
   resolved "https://registry.yarnpkg.com/@hapi/address/-/address-2.1.4.tgz#5d67ed43f3fd41a69d4b9ff7b56e7c0d1d0a81e5"
@@ -3376,7 +3400,7 @@
     is-ipfs "^0.6.0"
     recursive-fs "^1.1.2"
 
-"@polkadot/api-contract@1.26.1", "@polkadot/api-contract@^1.26.1":
+"@polkadot/api-contract@^1.26.1":
   version "1.26.1"
   resolved "https://registry.yarnpkg.com/@polkadot/api-contract/-/api-contract-1.26.1.tgz#a8b52ef469ab8bbddb83191f8d451e31ffd76142"
   integrity sha512-zLGA/MHUJf12vanUEUBBRqpHVAONHWztoHS0JTIWUUS2+3GEXk6hGw+7PPtBDfDsLj0LgU/Qna1bLalC/zyl5w==
@@ -5458,6 +5482,11 @@
   resolved "https://registry.yarnpkg.com/@types/yup/-/yup-0.26.37.tgz#7a52854ac602ba0dc969bebc960559f7464a1686"
   integrity sha512-cDqR/ez4+iAVQYOwadXjKX4Dq1frtnDGV2GNVKj3aUVKVCKRvsr8esFk66j+LgeeJGmrMcBkkfCf3zk13MjV7A==
 
+"@types/zen-observable@^0.8.0":
+  version "0.8.1"
+  resolved "https://registry.yarnpkg.com/@types/zen-observable/-/zen-observable-0.8.1.tgz#5668c0bce55a91f2b9566b1d8a4c0a8dbbc79764"
+  integrity sha512-wmk0xQI6Yy7Fs/il4EpOcflG4uonUpYGqvZARESLc2oy4u69fkatFLbJOeW4Q6awO15P4rduAe6xkwHevpXcUQ==
+
 "@typescript-eslint/eslint-plugin@3.8.0":
   version "3.8.0"
   resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-3.8.0.tgz#f82947bcdd9a4e42be7ad80dfd61f1dc411dd1df"
@@ -5937,6 +5966,13 @@
     "@webassemblyjs/wast-parser" "1.9.0"
     "@xtuc/long" "4.2.2"
 
+"@wry/context@^0.5.2":
+  version "0.5.2"
+  resolved "https://registry.yarnpkg.com/@wry/context/-/context-0.5.2.tgz#f2a5d5ab9227343aa74c81e06533c1ef84598ec7"
+  integrity sha512-B/JLuRZ/vbEKHRUiGj6xiMojST1kHhu4WcreLfNN7q9DqQFrb97cWgf/kiYsPSUCAMVN0HzfFc8XjJdzgZzfjw==
+  dependencies:
+    tslib "^1.9.3"
+
 "@wry/equality@^0.1.2":
   version "0.1.11"
   resolved "https://registry.yarnpkg.com/@wry/equality/-/equality-0.1.11.tgz#35cb156e4a96695aa81a9ecc4d03787bc17f1790"
@@ -5944,6 +5980,13 @@
   dependencies:
     tslib "^1.9.3"
 
+"@wry/equality@^0.2.0":
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/@wry/equality/-/equality-0.2.0.tgz#a312d1b6a682d0909904c2bcd355b02303104fb7"
+  integrity sha512-Y4d+WH6hs+KZJUC8YKLYGarjGekBrhslDbf/R20oV+AakHPINSitHfDRQz3EGcEWc1luXYNUvMhawWtZVWNGvQ==
+  dependencies:
+    tslib "^1.9.3"
+
 "@xtuc/ieee754@^1.2.0":
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz#eef014a3145ae477a1cbc00cd1e552336dceb790"
@@ -7836,11 +7879,21 @@ bluebird@^3.1.1, bluebird@^3.3.5, bluebird@^3.5.1, bluebird@^3.5.3, bluebird@^3.
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
   integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
 
-bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.1.1, bn.js@^4.11.8, bn.js@^4.4.0, bn.js@^5.1.1, bn.js@^5.1.2, bn.js@^5.1.3:
+bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.1.1, bn.js@^4.11.8, bn.js@^4.4.0:
+  version "4.11.9"
+  resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.11.9.tgz#26d556829458f9d1e81fc48952493d0ba3507828"
+  integrity sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==
+
+bn.js@^5.1.1, bn.js@^5.1.2:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-5.1.2.tgz#c9686902d3c9a27729f43ab10f9d79c2004da7b0"
   integrity sha512-40rZaf3bUNKTVYu9sIeeEGOg7g14Yvnj9kH7b50EiwX0Q7A6umbvfI5tvHaOERH0XigqKkfLkFQxzb4e6CIXnA==
 
+bn.js@^5.1.3:
+  version "5.1.3"
+  resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-5.1.3.tgz#beca005408f642ebebea80b042b4d18d2ac0ee6b"
+  integrity sha512-GkTiFpjFtUzU9CbMeJ5iazkCzGL3jrhzerzZIuqLABjbwRaFt33I9tUdSNryIptM+RxDet6OKm2WnLXzW51KsQ==
+
 body-parser@1.19.0, body-parser@^1.18.3, body-parser@^1.19.0:
   version "1.19.0"
   resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.19.0.tgz#96b2709e57c9c4e09a6fd66a8fd979844f69f08a"
@@ -10486,7 +10539,7 @@ date-utils@*:
   resolved "https://registry.yarnpkg.com/date-utils/-/date-utils-1.2.21.tgz#61fb16cdc1274b3c9acaaffe9fc69df8720a2b64"
   integrity sha1-YfsWzcEnSzyayq/+n8ad+HIKK2Q=
 
-dateformat@^3.0.0:
+dateformat@^3.0.0, dateformat@^3.0.2:
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-3.0.3.tgz#a6e37499a4d9a9cf85ef5872044d62901c9889ae"
   integrity sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==
@@ -10499,6 +10552,11 @@ dateformat@~1.0.4-1.2.3:
     get-stdin "^4.0.1"
     meow "^3.3.0"
 
+datejs@^1.0.0-rc3:
+  version "1.0.0-rc3"
+  resolved "https://registry.yarnpkg.com/datejs/-/datejs-1.0.0-rc3.tgz#bffa1efedefeb41fdd8a242af55afa01fb58de57"
+  integrity sha1-v/oe/t7+tB/diiQq9Vr6AftY3lc=
+
 de-indent@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/de-indent/-/de-indent-1.0.2.tgz#b2038e846dc33baa5796128d0804b455b8c1e21d"
@@ -14306,7 +14364,7 @@ graphql-subscriptions@^1.0.0, graphql-subscriptions@^1.1.0:
   dependencies:
     iterall "^1.2.1"
 
-graphql-tag@^2.9.2:
+graphql-tag@^2.11.0, graphql-tag@^2.9.2:
   version "2.11.0"
   resolved "https://registry.yarnpkg.com/graphql-tag/-/graphql-tag-2.11.0.tgz#1deb53a01c46a7eb401d6cb59dec86fa1cccbffd"
   integrity sha512-VmsD5pJqWJnQZMUeRwrDhfgoyqcfwEkvtpANqcoUG8/tOLkwNgU9mzub/Mc78OJMhHjx7gfAMTxzdG43VGg3bA==
@@ -14658,7 +14716,7 @@ hogan.js@^3.0.2:
     mkdirp "0.3.0"
     nopt "1.0.10"
 
-hoist-non-react-statics@^3.0.0:
+hoist-non-react-statics@^3.0.0, hoist-non-react-statics@^3.3.2:
   version "3.3.2"
   resolved "https://registry.yarnpkg.com/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz#ece0acaf71d62c2969c2ec59feff42a4b1a85b45"
   integrity sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==
@@ -19205,7 +19263,7 @@ lodash@^4.17.11, lodash@^4.17.12, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548"
   integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==
 
-lodash@^4.17.20:
+lodash@^4.17.20, lodash@^4.3.0:
   version "4.17.20"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52"
   integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==
@@ -21572,6 +21630,13 @@ opn@^5.5.0:
   dependencies:
     is-wsl "^1.1.0"
 
+optimism@^0.13.0:
+  version "0.13.0"
+  resolved "https://registry.yarnpkg.com/optimism/-/optimism-0.13.0.tgz#c08904e1439a0eb9e7f86183dafa06cc715ff351"
+  integrity sha512-6JAh3dH+YUE4QUdsgUw8nUQyrNeBKfAEKOHMlLkQ168KhIYFIxzPsHakWrRXDnTO+x61RJrS3/2uEt6W0xlocA==
+  dependencies:
+    "@wry/context" "^0.5.2"
+
 optimist@^0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/optimist/-/optimist-0.6.1.tgz#da3ea74686fa21a19a111c326e90eb15a0196686"
@@ -27018,6 +27083,11 @@ symbol-observable@^1.0.4, symbol-observable@^1.1.0, symbol-observable@^1.2.0:
   resolved "https://registry.yarnpkg.com/symbol-observable/-/symbol-observable-1.2.0.tgz#c22688aed4eab3cdc2dfeacbb561660560a00804"
   integrity sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==
 
+symbol-observable@^2.0.0:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/symbol-observable/-/symbol-observable-2.0.3.tgz#5b521d3d07a43c351055fa43b8355b62d33fd16a"
+  integrity sha512-sQV7phh2WCYAn81oAkakC5qjq2Ml0g8ozqz03wOGnx9dDlG1de6yrF+0RAzSJD8fPUow3PTSMf2SAbOGxb93BA==
+
 symbol-tree@^3.2.2, symbol-tree@^3.2.4:
   version "3.2.4"
   resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2"
@@ -27697,7 +27767,7 @@ ts-dedent@^1.1.0:
   resolved "https://registry.yarnpkg.com/ts-dedent/-/ts-dedent-1.1.1.tgz#68fad040d7dbd53a90f545b450702340e17d18f3"
   integrity sha512-UGTRZu1evMw4uTPyYF66/KFd22XiU+jMaIuHrkIHQ2GivAXVlLV0v/vHrpOuTRf9BmpNHi/SO7Vd0rLu0y57jg==
 
-ts-invariant@^0.4.0:
+ts-invariant@^0.4.0, ts-invariant@^0.4.4:
   version "0.4.4"
   resolved "https://registry.yarnpkg.com/ts-invariant/-/ts-invariant-0.4.4.tgz#97a523518688f93aafad01b0e80eb803eb2abd86"
   integrity sha512-uEtWkFM/sdZvRNNDL3Ehu4WVpwaulhwQszV8mrtcdeE8nN00BV9mAmQ88RkrBhFgl9gMgvjJLAQcZbnPXI9mlA==
@@ -28093,7 +28163,12 @@ typescript-formatter@^7.2.2:
     commandpost "^1.0.0"
     editorconfig "^0.15.0"
 
-typescript@3.5.2, typescript@^3.0.3, typescript@^3.7.2, typescript@^3.7.5, typescript@^3.8.3, typescript@^3.9.5, typescript@^3.9.6, typescript@^3.9.7:
+typescript@3.5.2:
+  version "3.5.2"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.5.2.tgz#a09e1dc69bc9551cadf17dba10ee42cf55e5d56c"
+  integrity sha512-7KxJovlYhTX5RaRbUdkAXN1KUZ8PwWlTzQdHV6xNqvuFOs7+WBo10TQUqT19Q/Jz2hk5v9TQDIhyLhhJY4p5AA==
+
+typescript@^3.0.3, typescript@^3.7.2, typescript@^3.7.5, typescript@^3.8.3, typescript@^3.9.5, typescript@^3.9.6, typescript@^3.9.7:
   version "3.9.7"
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.7.tgz#98d600a5ebdc38f40cb277522f12dc800e9e25fa"
   integrity sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==
@@ -30131,7 +30206,7 @@ zen-observable-ts@^0.8.21:
     tslib "^1.9.3"
     zen-observable "^0.8.0"
 
-zen-observable@^0.8.0:
+zen-observable@^0.8.0, zen-observable@^0.8.14:
   version "0.8.15"
   resolved "https://registry.yarnpkg.com/zen-observable/-/zen-observable-0.8.15.tgz#96415c512d8e3ffd920afd3889604e30b9eaac15"
   integrity sha512-PQ2PC7R9rslx84ndNBZB/Dkv8V8fZEpk83RLgXtYd0fwUgEjseMn1Dgajh2x6S8QbZAFa9p2qVCEuYZNgve0dQ==