Query node: Undeterministic ids fix

distributor-node/config.yml

@@ -34,6 +34,7 @@ operatorApi:
   hmacSecret: this-is-not-so-secret
 keys:
   - suri: //Alice
+  - suri: //testing//worker//Distribution//0
   # - mnemonic: "escape naive annual throw tragic achieve grunt verify cram note harvest problem"
   #   type: ed25519
   # - keyfile: "/path/to/keyfile.json"

query-node/mappings/common.ts

@@ -6,6 +6,7 @@ import { metaToObject } from '@joystream/metadata-protobuf/utils'
 import { AnyMetadataClass, DecodedMetadataObject } from '@joystream/metadata-protobuf/types'
 
 export const CURRENT_NETWORK = Network.GIZA
+
 /*
   Simple logger enabling error and informational reporting.
 
@@ -223,3 +224,10 @@ export async function getById<T extends BaseModel>(
 
   return result
 }
+
+export function deterministicEntityId(createdInEvent: SubstrateEvent, additionalIdentifier?: string | number): string {
+  return (
+    `${createdInEvent.blockNumber}-${createdInEvent.indexInBlock}` +
+    (additionalIdentifier ? `-${additionalIdentifier}` : '')
+  )
+}

query-node/mappings/content/utils.ts

@@ -1,5 +1,5 @@
 import { DatabaseManager, EventContext, StoreContext } from '@joystream/hydra-common'
-import { FindConditions, Raw } from 'typeorm'
+import { FindConditions } from 'typeorm'
 import {
   IVideoMetadata,
   IPublishedBeforeJoystream,
@@ -8,7 +8,7 @@ import {
   IChannelMetadata,
 } from '@joystream/metadata-protobuf'
 import { integrateMeta, isSet, isValidLanguageCode } from '@joystream/metadata-protobuf/utils'
-import { invalidMetadata, inconsistentState, logger } from '../common'
+import { invalidMetadata, inconsistentState, logger, deterministicEntityId } from '../common'
 import {
   // primary entities
   CuratorGroup,
@@ -209,9 +209,8 @@ async function processVideoMediaEncoding(
   const encoding =
     existingVideoMediaEncoding ||
     new VideoMediaEncoding({
+      id: deterministicEntityId(event),
       createdAt: new Date(event.blockTimestamp),
-      createdById: '1',
-      updatedById: '1',
     })
   // integrate media encoding-related data
   integrateMeta(encoding, metadata, ['codecName', 'container', 'mimeMediaType'])
@@ -231,10 +230,9 @@ async function processVideoMediaMetadata(
   const videoMedia =
     existingVideoMedia ||
     new VideoMediaMetadata({
+      id: deterministicEntityId(event),
       createdInBlock: event.blockNumber,
       createdAt: new Date(event.blockTimestamp),
-      createdById: '1',
-      updatedById: '1',
     })
 
   // integrate media-related data
@@ -364,13 +362,11 @@ async function processLanguage(
 
   // create new language
   const newLanguage = new Language({
+    id: deterministicEntityId(event),
     iso: languageIso,
     createdInBlock: event.blockNumber,
     createdAt: new Date(event.blockTimestamp),
     updatedAt: new Date(event.blockTimestamp),
-    // TODO: remove these lines after Hydra auto-fills the values when cascading save (remove them on all places)
-    createdById: '1',
-    updatedById: '1',
   })
 
   await store.save<Language>(newLanguage)
@@ -397,9 +393,8 @@ async function updateVideoLicense(
     license =
       previousLicense ||
       new License({
+        id: deterministicEntityId(event),
         createdAt: new Date(event.blockTimestamp),
-        createdById: '1',
-        updatedById: '1',
       })
     license.updatedAt = new Date(event.blockTimestamp)
     integrateMeta(license, licenseMetadata, ['attribution', 'code', 'customText'])

query-node/mappings/storage/index.ts

@@ -76,6 +76,7 @@ export async function storage_StorageOperatorMetadataSet({ event, store }: Event
   const [bucketId, , metadataBytes] = new Storage.StorageOperatorMetadataSetEvent(event).params
   const storageBucket = await getStorageBucketWithOperatorMetadata(store, bucketId.toString())
   storageBucket.operatorMetadata = await processStorageOperatorMetadata(
+    event,
     store,
     storageBucket.operatorMetadata,
     metadataBytes
@@ -276,7 +277,7 @@ export async function storage_DistributionBucketFamilyMetadataSet({
   const [familyId, metadataBytes] = new Storage.DistributionBucketFamilyMetadataSetEvent(event).params
 
   const family = await getDistributionBucketFamilyWithMetadata(store, familyId.toString())
-  family.metadata = await processDistributionBucketFamilyMetadata(store, family.metadata, metadataBytes)
+  family.metadata = await processDistributionBucketFamilyMetadata(event, store, family.metadata, metadataBytes)
 
   await store.save<DistributionBucketFamily>(family)
 }
@@ -426,7 +427,7 @@ export async function storage_DistributionBucketMetadataSet({
   const [workerId, bucketId, metadataBytes] = new Storage.DistributionBucketMetadataSetEvent(event).params
 
   const operator = await getDistributionBucketOperatorWithMetadata(store, distributionOperatorId(bucketId, workerId))
-  operator.metadata = await processDistributionOperatorMetadata(store, operator.metadata, metadataBytes)
+  operator.metadata = await processDistributionOperatorMetadata(event, store, operator.metadata, metadataBytes)
 
   await store.save<DistributionBucketOperator>(operator)
 }

query-node/mappings/storage/metadata.ts

@@ -1,4 +1,4 @@
-import { DatabaseManager } from '@joystream/hydra-common'
+import { DatabaseManager, SubstrateEvent } from '@joystream/hydra-common'
 import {
   DistributionBucketFamilyMetadata,
   DistributionBucketOperatorMetadata,
@@ -11,7 +11,7 @@ import {
   GeographicalAreaSubdivistion,
   DistributionBucketFamilyGeographicArea,
 } from 'query-node/dist/model'
-import { deserializeMetadata, invalidMetadata } from '../common'
+import { deserializeMetadata, deterministicEntityId, invalidMetadata } from '../common'
 import { Bytes } from '@polkadot/types'
 import {
   DistributionBucketOperatorMetadata as DistributionBucketOperatorMetadataProto,
@@ -33,11 +33,12 @@ const protobufContinentToGraphlContinent: { [key in GeographicalAreaProto.Contin
 }
 
 async function processNodeLocationMetadata(
+  event: SubstrateEvent,
   store: DatabaseManager,
   current: NodeLocationMetadata | undefined,
   meta: INodeLocationMetadata
 ): Promise<NodeLocationMetadata> {
-  const nodeLocation = current || new NodeLocationMetadata()
+  const nodeLocation = current || new NodeLocationMetadata({ id: deterministicEntityId(event) })
   if (isSet(meta.city)) {
     nodeLocation.city = meta.city
   }
@@ -45,7 +46,7 @@ async function processNodeLocationMetadata(
     if (isEmptyObject(meta.coordinates)) {
       nodeLocation.coordinates = null as any
     } else {
-      const coordinates = current?.coordinates || new GeoCoordinates()
+      const coordinates = current?.coordinates || new GeoCoordinates({ id: deterministicEntityId(event) })
       coordinates.latitude = meta.coordinates.latitude || coordinates.latitude || 0
       coordinates.longitude = meta.coordinates.longitude || coordinates.longitude || 0
       await store.save<GeoCoordinates>(coordinates)
@@ -65,6 +66,7 @@ async function processNodeLocationMetadata(
 }
 
 export async function processDistributionOperatorMetadata(
+  event: SubstrateEvent,
   store: DatabaseManager,
   current: DistributionBucketOperatorMetadata | undefined,
   metadataBytes: Bytes
@@ -73,14 +75,14 @@ export async function processDistributionOperatorMetadata(
   if (!meta) {
     return current
   }
-  const metadataEntity = current || new DistributionBucketOperatorMetadata()
+  const metadataEntity = current || new DistributionBucketOperatorMetadata({ id: deterministicEntityId(event) })
   if (isSet(meta.endpoint)) {
     metadataEntity.nodeEndpoint = meta.endpoint
   }
   if (isSet(meta.location)) {
     metadataEntity.nodeLocation = isEmptyObject(meta.location)
       ? (null as any)
-      : await processNodeLocationMetadata(store, metadataEntity.nodeLocation, meta.location)
+      : await processNodeLocationMetadata(event, store, metadataEntity.nodeLocation, meta.location)
   }
   if (isSet(meta.extra)) {
     metadataEntity.extra = meta.extra
@@ -92,6 +94,7 @@ export async function processDistributionOperatorMetadata(
 }
 
 export async function processStorageOperatorMetadata(
+  event: SubstrateEvent,
   store: DatabaseManager,
   current: StorageBucketOperatorMetadata | undefined,
   metadataBytes: Bytes
@@ -100,14 +103,14 @@ export async function processStorageOperatorMetadata(
   if (!meta) {
     return current
   }
-  const metadataEntity = current || new StorageBucketOperatorMetadata()
+  const metadataEntity = current || new StorageBucketOperatorMetadata({ id: deterministicEntityId(event) })
   if (isSet(meta.endpoint)) {
     metadataEntity.nodeEndpoint = meta.endpoint || (null as any)
   }
   if (isSet(meta.location)) {
     metadataEntity.nodeLocation = isEmptyObject(meta.location)
       ? (null as any)
-      : await processNodeLocationMetadata(store, metadataEntity.nodeLocation, meta.location)
+      : await processNodeLocationMetadata(event, store, metadataEntity.nodeLocation, meta.location)
   }
   if (isSet(meta.extra)) {
     metadataEntity.extra = meta.extra || (null as any)
@@ -119,6 +122,7 @@ export async function processStorageOperatorMetadata(
 }
 
 export async function processDistributionBucketFamilyMetadata(
+  event: SubstrateEvent,
   store: DatabaseManager,
   current: DistributionBucketFamilyMetadata | undefined,
   metadataBytes: Bytes
@@ -127,7 +131,7 @@ export async function processDistributionBucketFamilyMetadata(
   if (!meta) {
     return current
   }
-  const metadataEntity = current || new DistributionBucketFamilyMetadata()
+  const metadataEntity = current || new DistributionBucketFamilyMetadata({ id: deterministicEntityId(event) })
   if (isSet(meta.region)) {
     metadataEntity.region = meta.region || (null as any)
   }
@@ -138,7 +142,7 @@ export async function processDistributionBucketFamilyMetadata(
     metadataEntity.latencyTestTargets = meta.latencyTestTargets.filter((t) => t)
   }
 
-  await store.save<DistributionBucketOperatorMetadata>(metadataEntity)
+  await store.save<DistributionBucketFamilyMetadata>(metadataEntity)
 
   // Update areas after metadata is saved (since we need an id to reference)
   if (isSet(meta.areas)) {
@@ -161,9 +165,7 @@ export async function processDistributionBucketFamilyMetadata(
             }
             area.id = `${metadataEntity.id}-C-${continent.code}`
             area.area = continent
-          }
-
-          if (a.countryCode) {
+          } else if (a.countryCode) {
             if (!isValidCountryCode(a.countryCode)) {
               return invalidMetadata(`Invalid country code: ${a.countryCode}`)
             }
@@ -171,9 +173,7 @@ export async function processDistributionBucketFamilyMetadata(
             country.code = a.countryCode
             area.id = `${metadataEntity.id}-c-${country.code}`
             area.area = country
-          }
-
-          if (a.subdivisionCode) {
+          } else if (a.subdivisionCode) {
             if (!isValidSubdivisionCode(a.subdivisionCode)) {
               return invalidMetadata(`Invalid subdivision code: ${a.subdivisionCode}`)
             }
@@ -181,6 +181,8 @@ export async function processDistributionBucketFamilyMetadata(
             subdivision.code = a.subdivisionCode
             area.id = `${metadataEntity.id}-s-${subdivision.code}`
             area.area = subdivision
+          } else {
+            return
           }
 
           await store.save<DistributionBucketFamilyGeographicArea>(area)