Merge branch 'sumer' into sumer-new-substrate-update-apps

query-node/mappings/src/common.ts

@@ -28,6 +28,16 @@ export function inconsistentState(extraInfo: string, data?: unknown): void {
   logger.error(errorMessage, data)
 }
 
+/*
+  Reports that metadata inserted by the user are not entirely valid, but the problem can be overcome.
+*/
+export function invalidMetadata(extraInfo: string, data?: unknown): void {
+  const errorMessage = 'Invalid metadata: ' + extraInfo
+
+  // log error
+  logger.info(errorMessage, data)
+}
+
 /*
   Prepares data object from content parameters.
 */

query-node/mappings/src/content/utils.ts

@@ -9,10 +9,10 @@
 import { SubstrateEvent } from '@dzlzv/hydra-common'
 import { DatabaseManager } from '@dzlzv/hydra-db-utils'
 import { Bytes } from '@polkadot/types'
-import ISO6391 from 'iso-639-1';
-import { u64 } from '@polkadot/types/primitive';
+import ISO6391 from 'iso-639-1'
+import { u64 } from '@polkadot/types/primitive'
 import { FindConditions } from 'typeorm'
-import * as jspb from "google-protobuf";
+import * as jspb from "google-protobuf"
 
 // protobuf definitions
 import {
@@ -30,7 +30,7 @@ import {
 } from '../../../generated/types'
 
 import {
-  inconsistentState,
+  invalidMetadata,
   logger,
   prepareDataObject,
 } from '../common'
@@ -99,6 +99,69 @@ export interface IReadProtobufArgumentsWithAssets extends IReadProtobufArguments
   contentOwner: typeof DataObjectOwner
 }
 
+/*
+  This class represents one of 3 possible states when changing property read from metadata.
+  NoChange - don't change anything (used when invalid metadata are encountered)
+  Unset - unset the value (used when the unset is requested in runtime)
+  Change - set the new value
+*/
+export class PropertyChange<T> {
+
+  static newUnset<T>() {
+    return new PropertyChange<T>('unset')
+  }
+
+  static newNoChange<T>() {
+    return new PropertyChange<T>('nochange')
+  }
+
+  static newChange<T>(value: T) {
+    return new PropertyChange<T>('change', value)
+  }
+
+  private type: string
+  private value?: T
+
+  private constructor(type: 'change' | 'nochange' | 'unset', value?: T) {
+    this.type = type
+    this.value = value
+  }
+
+  public isUnset(): boolean {
+    return this.type == 'unset'
+  }
+
+  public isNoChange(): boolean {
+    return this.type == 'nochange'
+  }
+
+  public isValue(): boolean {
+    return this.type == 'change'
+  }
+
+  public getValue(): T | undefined {
+    return this.type == 'change'
+      ? this.value
+      : undefined
+  }
+
+  /*
+    Integrates the value into the given dictionary.
+  */
+  public integrateInto(object: Object, key: string): void {
+    if (this.isNoChange()) {
+      return
+    }
+
+    if (this.isUnset()) {
+      delete object[key]
+      return
+    }
+
+    object[key] = this.value
+  }
+}
+
 /*
   Reads information from the event and protobuf metadata and constructs changeset that's fit to be used when saving to db.
 */
@@ -107,7 +170,7 @@ export async function readProtobuf<T extends ChannelCategory | VideoCategory>(
   parameters: IReadProtobufArguments,
 ): Promise<Partial<T>> {
   // true option here is crucial, it indicates that we want just the underlying bytes (by default it will also include bytes encoding the length)
-  const metaU8a = parameters.metadata.toU8a(true);
+  const metaU8a = parameters.metadata.toU8a(true)
 
   // process channel category
   if (type instanceof ChannelCategory) {
@@ -140,7 +203,7 @@ export async function readProtobufWithAssets<T extends Channel | Video>(
   parameters: IReadProtobufArgumentsWithAssets,
 ): Promise<Partial<T>> {
   // true option here is crucial, it indicates that we want just the underlying bytes (by default it will also include bytes encoding the length)
-  const metaU8a = parameters.metadata.toU8a(true);
+  const metaU8a = parameters.metadata.toU8a(true)
 
   // process channel
   if (type instanceof Channel) {
@@ -177,7 +240,9 @@ export async function readProtobufWithAssets<T extends Channel | Video>(
 
     // prepare language if needed
     if ('language' in metaAsObject) {
-      result.language = await prepareLanguage(metaAsObject.language, parameters.db, parameters.blockNumber)
+      const language = await prepareLanguage(metaAsObject.language, parameters.db, parameters.blockNumber)
+      delete metaAsObject.language // make sure temporary value will not interfere
+      language.integrateInto(result, 'language')
     }
 
     return result as Partial<T>
@@ -191,16 +256,22 @@ export async function readProtobufWithAssets<T extends Channel | Video>(
 
     // prepare video category if needed
     if ('category' in metaAsObject) {
-      result.category = await prepareVideoCategory(metaAsObject.category, parameters.db)
+      const category = await prepareVideoCategory(metaAsObject.category, parameters.db)
+      delete metaAsObject.category // make sure temporary value will not interfere
+      category.integrateInto(result, 'category')
     }
 
     // prepare media meta information if needed
-    if ('mediaType' in metaAsObject) {
+    if ('mediaType' in metaAsObject || 'mediaPixelWidth' in metaAsObject || 'mediaPixelHeight' in metaAsObject) {
       // prepare video file size if poosible
       const videoSize = await extractVideoSize(parameters.assets, metaAsObject.video)
 
       result.mediaMetadata = await prepareVideoMetadata(metaAsObject, videoSize, parameters.blockNumber)
+
+      // remove extra values
       delete metaAsObject.mediaType
+      delete metaAsObject.mediaPixelWidth
+      delete metaAsObject.mediaPixelHeight
     }
 
     // prepare license if needed
@@ -217,7 +288,7 @@ export async function readProtobufWithAssets<T extends Channel | Video>(
         blockNumber: parameters.blockNumber,
         contentOwner: parameters.contentOwner,
       })
-      integrateAsset('thumbnail', result, asset) // changes `result` inline!
+      integrateAsset('thumbnailPhoto', result, asset) // changes `result` inline!
       delete metaAsObject.thumbnailPhoto
     }
 
@@ -236,7 +307,9 @@ export async function readProtobufWithAssets<T extends Channel | Video>(
 
     // prepare language if needed
     if ('language' in metaAsObject) {
-      result.language = await prepareLanguage(metaAsObject.language, parameters.db, parameters.blockNumber)
+      const language = await prepareLanguage(metaAsObject.language, parameters.db, parameters.blockNumber)
+      delete metaAsObject.language // make sure temporary value will not interfere
+      language.integrateInto(result, 'language')
     }
 
     // prepare information about media published somewhere else before Joystream if needed.
@@ -265,12 +338,8 @@ export async function convertContentActorToChannelOwner(db: DatabaseManager, con
 
     // ensure member exists
     if (!member) {
-      inconsistentState(`Actor is non-existing member`, memberId)
-      return {
-        // this will clear fields
-        ownerMember: undefined,
-        ownerCuratorGroup: undefined,
-      }
+      invalidMetadata(`Actor is non-existing member`, memberId)
+      return {} // this will keep fields unchanged
     }
 
     return {
@@ -285,12 +354,8 @@ export async function convertContentActorToChannelOwner(db: DatabaseManager, con
 
     // ensure curator group exists
     if (!curatorGroup) {
-      inconsistentState('Actor is non-existing curator group', curatorGroupId)
-      return {
-        // this will clear fields
-        ownerMember: undefined,
-        ownerCuratorGroup: undefined,
-      }
+      invalidMetadata('Actor is non-existing curator group', curatorGroupId)
+      return {} // this will keep fields unchanged
     }
 
     return {
@@ -381,28 +446,30 @@ interface IExtractAssetParameters {
 /*
   Selects asset from provided set of assets and prepares asset data fit to be saved to db.
 */
-async function extractAsset(parameters: IExtractAssetParameters): Promise<AssetStorageOrUrls | undefined> {
+async function extractAsset(parameters: IExtractAssetParameters): Promise<PropertyChange<AssetStorageOrUrls>> {
   // is asset being unset?
   if (parameters.assetIndex === undefined) {
-    return undefined
+    return PropertyChange.newUnset()
   }
 
   // ensure asset index is valid
   if (parameters.assetIndex >= parameters.assets.length) {
-    inconsistentState(`Non-existing asset extraction requested`, {
+    invalidMetadata(`Non-existing asset extraction requested`, {
       assetsProvided: parameters.assets.length,
       assetIndex: parameters.assetIndex,
     })
-    return undefined
+    return PropertyChange.newNoChange()
   }
 
   // convert asset to data object record
-  return convertAsset({
+  const asset = await convertAsset({
     rawAsset: parameters.assets[parameters.assetIndex],
     db: parameters.db,
     blockNumber: parameters.blockNumber,
     contentOwner: parameters.contentOwner,
   })
+
+  return PropertyChange.newChange(asset)
 }
 
 /*
@@ -411,30 +478,38 @@ async function extractAsset(parameters: IExtractAssetParameters): Promise<AssetS
 
   Changes `result` argument!
 */
-function integrateAsset<T>(propertyName: string, result: Object, asset: AssetStorageOrUrls | undefined) {
+function integrateAsset<T>(propertyName: string, result: Object, asset: PropertyChange<AssetStorageOrUrls>): void {
   // helpers - property names
   const nameUrl = propertyName + 'Urls'
   const nameDataObject = propertyName + 'DataObject'
   const nameAvailability = propertyName + 'Availability'
 
-  if (asset === undefined) {
+  if (asset.isNoChange()) {
+    return
+  }
+
+  if (asset.isUnset()) {
     result[nameUrl] = []
     result[nameAvailability] = AssetAvailability.INVALID
     result[nameDataObject] = undefined // plan deletion (will have effect when saved to db)
 
-    return result
+    return
   }
 
-  // is asset saved in storage?
-  if (!isAssetInStorage(asset)) {
+  const newValue = asset.getValue() as AssetStorageOrUrls
+
+  // is asset available on external URL(s)
+  if (!isAssetInStorage(newValue)) {
     // (un)set asset's properties
-    result[nameUrl] = asset
+    result[nameUrl] = newValue
     result[nameAvailability] = AssetAvailability.ACCEPTED
     result[nameDataObject] = undefined // plan deletion (will have effect when saved to db)
 
-    return result
+    return
   }
 
+  // asset saved in storage
+
   // prepare conversion table between liaison judgment and asset availability
   const conversionTable = {
     [LiaisonJudgement.ACCEPTED]: AssetAvailability.ACCEPTED,
@@ -443,8 +518,8 @@ function integrateAsset<T>(propertyName: string, result: Object, asset: AssetSto
 
   // (un)set asset's properties
   result[nameUrl] = [] // plan deletion (will have effect when saved to db)
-  result[nameAvailability] = conversionTable[asset.liaisonJudgement]
-  result[nameDataObject] = asset
+  result[nameAvailability] = conversionTable[newValue.liaisonJudgement]
+  result[nameDataObject] = newValue
 }
 
 async function extractVideoSize(assets: NewAsset[], assetIndex: number | undefined): Promise<number | undefined> {
@@ -455,7 +530,7 @@ async function extractVideoSize(assets: NewAsset[], assetIndex: number | undefin
 
   // ensure asset index is valid
   if (assetIndex > assets.length) {
-    inconsistentState(`Non-existing asset video size extraction requested`, {assetsProvided: assets.length, assetIndex})
+    invalidMetadata(`Non-existing asset video size extraction requested`, {assetsProvided: assets.length, assetIndex})
     return undefined
   }
 
@@ -476,19 +551,19 @@ async function extractVideoSize(assets: NewAsset[], assetIndex: number | undefin
   return videoSize
 }
 
-async function prepareLanguage(languageIso: string | undefined, db: DatabaseManager, blockNumber: number): Promise<Language | undefined> {
+async function prepareLanguage(languageIso: string | undefined, db: DatabaseManager, blockNumber: number): Promise<PropertyChange<Language>> {
   // is language being unset?
   if (languageIso === undefined) {
-    return undefined
+    return PropertyChange.newUnset()
   }
 
   // validate language string
-  const isValidIso = ISO6391.validate(languageIso);
+  const isValidIso = ISO6391.validate(languageIso)
 
   // ensure language string is valid
   if (!isValidIso) {
-    inconsistentState(`Invalid language ISO-639-1 provided`, languageIso)
-    return undefined
+    invalidMetadata(`Invalid language ISO-639-1 provided`, languageIso)
+    return PropertyChange.newNoChange()
   }
 
   // load language
@@ -496,7 +571,7 @@ async function prepareLanguage(languageIso: string | undefined, db: DatabaseMana
 
   // return existing language if any
   if (language) {
-    return language;
+    return PropertyChange.newChange(language)
   }
 
 
@@ -512,7 +587,7 @@ async function prepareLanguage(languageIso: string | undefined, db: DatabaseMana
 
   await db.save<Language>(newLanguage)
 
-  return newLanguage
+  return PropertyChange.newChange(newLanguage)
 }
 
 async function prepareLicense(licenseProtobuf: LicenseMetadata.AsObject | undefined): Promise<License | undefined> {
@@ -536,6 +611,9 @@ async function prepareLicense(licenseProtobuf: LicenseMetadata.AsObject | undefi
 }
 
 async function prepareVideoMetadata(videoProtobuf: VideoMetadata.AsObject, videoSize: number | undefined, blockNumber: number): Promise<VideoMediaMetadata> {
+  // TODO: handle situations when only some metadata are set (e.g. pixelWidth and mediaType is defined, but pixelHeight is missing)
+  // TODO: handle update of VideoMediaEncoding and VideoMediaMetadata
+  //       right now when only some of mediaType(mb partial), mediaPixelWidth, or mediaPixelHeight is set, the update discards previous values
   // create new encoding info
   const encoding = new VideoMediaEncoding({
     ...videoProtobuf.mediaType,
@@ -563,10 +641,10 @@ async function prepareVideoMetadata(videoProtobuf: VideoMetadata.AsObject, video
   return videoMeta
 }
 
-async function prepareVideoCategory(categoryId: number | undefined, db: DatabaseManager): Promise<VideoCategory | undefined> {
+async function prepareVideoCategory(categoryId: number | undefined, db: DatabaseManager): Promise<PropertyChange<VideoCategory>> {
   // is category being unset?
   if (categoryId === undefined) {
-    return undefined
+    return PropertyChange.newUnset()
   }
 
   // load video category
@@ -574,11 +652,11 @@ async function prepareVideoCategory(categoryId: number | undefined, db: Database
 
   // ensure video category exists
   if (!category) {
-    inconsistentState('Non-existing video category association with video requested', categoryId)
-    return undefined
+    invalidMetadata('Non-existing video category association with video requested', categoryId)
+    return PropertyChange.newNoChange()
   }
 
-  return category
+  return PropertyChange.newChange(category)
 }
 
 function convertMetadataToObject<T extends Object>(metadata: jspb.Message): T {

query-node/package.json

@@ -21,6 +21,7 @@
     "db:processor:migrate": "hydra-processor migrate --env ../.env",
     "db:migrate": "yarn db:schema:migrate && yarn db:processor:migrate",
     "db:bootstrap": "yarn db:create && yarn db:prepare && yarn db:migrate",
+    "db:reset": "yarn db:drop && yarn db:prepare && yarn db:migrate",
     "bootstrap": "yarn codegen && yarn db:drop && yarn db:bootstrap",
     "codegen": "hydra-cli codegen",
     "codegen:noinstall": "hydra-cli codegen --no-install",