Update to Hydra 3.1.0-alpha.13 + related temporary fixes

package.json

@@ -38,7 +38,8 @@
     "typescript": "^4.3.5",
     "bn.js": "^5.1.2",
     "rxjs": "^7.2.0",
-    "typeorm": "^0.2.31",
+    "typeorm": "0.2.34",
+    "@joystream/warthog": "2.39.0",
     "pg": "^8.4.0",
     "chalk": "^4.0.0"
   },

query-node/codegen/package.json

@@ -5,7 +5,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@dzlzv/hydra-cli": "3.1.0-alpha.0",
-    "@dzlzv/hydra-typegen": "3.1.0-alpha.0"
+    "@joystream/hydra-cli": "3.1.0-alpha.13",
+    "@joystream/hydra-typegen": "3.1.0-alpha.13"
   }
 }

query-node/mappings/common.ts

@@ -5,11 +5,11 @@ import {
   ExtrinsicArg,
   EventContext,
   StoreContext,
-} from '@dzlzv/hydra-common'
+} from '@joystream/hydra-common'
 import { Bytes } from '@polkadot/types'
 import { WorkingGroup, WorkerId, ThreadId, ContentParameters } from '@joystream/types/augment/all'
 import { Worker, Event, Network, DataObject, LiaisonJudgement, DataObjectOwner } from 'query-node/dist/model'
-import { BaseModel } from 'warthog'
+import { BaseModel } from '@joystream/warthog'
 import { ContentParameters as Custom_ContentParameters } from '@joystream/types/storage'
 import { registry } from '@joystream/types'
 import { metaToObject } from '@joystream/metadata-protobuf/utils'

query-node/mappings/content/channel.ts

@@ -1,7 +1,7 @@
 /*
 eslint-disable @typescript-eslint/naming-convention
 */
-import { EventContext, StoreContext } from '@dzlzv/hydra-common'
+import { EventContext, StoreContext } from '@joystream/hydra-common'
 import { In } from 'typeorm'
 import { AccountId } from '@polkadot/types/interfaces'
 import { Option } from '@polkadot/types/codec'

query-node/mappings/content/curatorGroup.ts

@@ -1,7 +1,7 @@
 /*
 eslint-disable @typescript-eslint/naming-convention
 */
-import { EventContext, StoreContext } from '@dzlzv/hydra-common'
+import { EventContext, StoreContext } from '@joystream/hydra-common'
 import { FindConditions } from 'typeorm'
 import { CuratorGroup } from 'query-node/dist/model'
 import { Content } from '../generated/types'

query-node/mappings/content/utils.ts

@@ -6,7 +6,7 @@
 //         every time query node codegen is run (that will overwrite said manual changes)
 //       - verify in integration tests that the records are trully created/updated/removed as expected
 
-import { DatabaseManager, EventContext, StoreContext } from '@dzlzv/hydra-common'
+import { DatabaseManager, EventContext, StoreContext } from '@joystream/hydra-common'
 import ISO6391 from 'iso-639-1'
 import { FindConditions } from 'typeorm'
 import {

query-node/mappings/content/video.ts

@@ -1,7 +1,7 @@
 /*
 eslint-disable @typescript-eslint/naming-convention
 */
-import { EventContext, StoreContext } from '@dzlzv/hydra-common'
+import { EventContext, StoreContext } from '@joystream/hydra-common'
 import { In } from 'typeorm'
 import { Content } from '../generated/types'
 import { deserializeMetadata, inconsistentState, logger } from '../common'

query-node/mappings/forum.ts

@@ -1,7 +1,7 @@
 /*
 eslint-disable @typescript-eslint/naming-convention
 */
-import { EventContext, StoreContext, DatabaseManager } from '@dzlzv/hydra-common'
+import { EventContext, StoreContext, DatabaseManager } from '@joystream/hydra-common'
 import {
   bytesToString,
   deserializeMetadata,

query-node/mappings/genesis.ts

@@ -1,4 +1,4 @@
-import { StoreContext } from '@dzlzv/hydra-common'
+import { StoreContext } from '@joystream/hydra-common'
 import BN from 'bn.js'
 import { MembershipSystemSnapshot, WorkingGroup } from 'query-node/dist/model'
 import { membershipSystem, workingGroups } from './genesis-data'

query-node/mappings/membership.ts

@@ -1,7 +1,7 @@
 /*
 eslint-disable @typescript-eslint/naming-convention
 */
-import { EventContext, StoreContext, DatabaseManager, SubstrateEvent } from '@dzlzv/hydra-common'
+import { EventContext, StoreContext, DatabaseManager, SubstrateEvent } from '@joystream/hydra-common'
 import { Members } from './generated/types'
 import { MemberId, BuyMembershipParameters, InviteMembershipParameters } from '@joystream/types/augment/all'
 import { MembershipMetadata } from '@joystream/metadata-protobuf'

query-node/mappings/package.json

@@ -7,13 +7,14 @@
   "scripts": {
     "build": "rm -rf lib && tsc --build tsconfig.json && cp ./generated/types/typedefs.json ./lib/generated/types/typedefs.json",
     "lint": "echo \"Skippinng\"",
-    "clean": "rm -rf lib"
+    "clean": "rm -rf lib",
+    "postinstall": "yarn ts-node ./scripts/postInstall.ts"
   },
   "dependencies": {
-    "@dzlzv/hydra-common": "3.1.0-alpha.0",
-    "@dzlzv/hydra-db-utils": "3.1.0-alpha.0",
+    "@joystream/hydra-common": "3.1.0-alpha.13",
+    "@joystream/hydra-db-utils": "3.1.0-alpha.13",
     "@joystream/types": "^0.17.0",
-    "warthog": "https://github.com/metmirr/warthog/releases/download/v2.30.0/warthog-v2.30.0.tgz",
+    "@joystream/warthog": "2.39.0",
     "@joystream/metadata-protobuf": "^1.0.0",
     "iso-639-1": "^2.1.8"
   },

query-node/mappings/proposals.ts

@@ -1,7 +1,7 @@
 /*
 eslint-disable @typescript-eslint/naming-convention
 */
-import { SubstrateEvent, DatabaseManager, EventContext, StoreContext } from '@dzlzv/hydra-common'
+import { SubstrateEvent, DatabaseManager, EventContext, StoreContext } from '@joystream/hydra-common'
 import { ProposalDetails as RuntimeProposalDetails } from '@joystream/types/augment/all'
 import BN from 'bn.js'
 import {

query-node/mappings/proposalsDiscussion.ts

@@ -1,7 +1,7 @@
 /*
 eslint-disable @typescript-eslint/naming-convention
 */
-import { EventContext, StoreContext, DatabaseManager } from '@dzlzv/hydra-common'
+import { EventContext, StoreContext, DatabaseManager } from '@joystream/hydra-common'
 import {
   Membership,
   ProposalDiscussionPostStatusActive,

query-node/mappings/scripts/postInstall.ts

@@ -0,0 +1,35 @@
+// A script to be executed post query-node install, that may include workarounds in Hydra node_modules
+import fs from 'fs'
+import path from 'path'
+
+// FIXME: Temporarly remove broken sanitizeNullCharacter call
+const subscribersJsPath = path.resolve(
+  __dirname,
+  '../../../node_modules/@joystream/hydra-processor/lib/db/subscribers.js'
+)
+const subscribersJsContent = fs.readFileSync(subscribersJsPath).toString()
+fs.writeFileSync(
+  subscribersJsPath,
+  subscribersJsContent.replace(/sanitizeNullCharacter\(entity, field\);/g, '//sanitizeNullCharacter(entity, field)')
+)
+
+// FIXME: Temporarly replace broken relations resolution in @joystream/warthog
+const dataLoaderJsPath = path.resolve(
+  __dirname,
+  '../../../node_modules/@joystream/warthog/dist/middleware/DataLoaderMiddleware.js'
+)
+const dataLoaderJsContent = fs.readFileSync(dataLoaderJsPath).toString()
+const dataLoaderJsContentLines = dataLoaderJsContent.split('\n')
+dataLoaderJsContentLines.splice(
+  dataLoaderJsContentLines.findIndex((l) => l.match(/return context\.connection\.relationIdLoader/)),
+  0,
+  `return Promise.all(
+    entities.map(entity => context.connection.relationLoader.load(relation, entity))
+  ).then(function (results) {
+    return results.map(function (related) {
+      return (relation.isManyToOne || relation.isOneToOne) ? related[0] : related
+    })
+  })
+  `
+)
+fs.writeFileSync(dataLoaderJsPath, dataLoaderJsContentLines.join('\n'))

query-node/mappings/storage.ts

@@ -1,7 +1,7 @@
 /*
 eslint-disable @typescript-eslint/naming-convention
 */
-import { EventContext, StoreContext, DatabaseManager } from '@dzlzv/hydra-common'
+import { EventContext, StoreContext, DatabaseManager } from '@joystream/hydra-common'
 import { FindConditions, In, Raw } from 'typeorm'
 import {
   createDataObject,

query-node/mappings/workingGroups.ts

@@ -1,7 +1,7 @@
 /*
 eslint-disable @typescript-eslint/naming-convention
 */
-import { EventContext, StoreContext, DatabaseManager, SubstrateEvent } from '@dzlzv/hydra-common'
+import { EventContext, StoreContext, DatabaseManager, SubstrateEvent } from '@joystream/hydra-common'
 
 import { StorageWorkingGroup as WorkingGroups } from './generated/types'
 import {

query-node/package.json

@@ -41,7 +41,7 @@
     "tslib": "^2.0.0",
     "@types/bn.js": "^4.11.6",
     "bn.js": "^5.1.2",
-    "@dzlzv/hydra-processor": "3.1.0-alpha.0",
+    "@joystream/hydra-processor": "3.1.0-alpha.13",
     "envsub": "4.0.7"
   },
   "volta": {

query-node/scripts/initializeDefaultSchemas.ts

@@ -1,155 +0,0 @@
-/* eslint-disable import/first */
-import 'reflect-metadata'
-
-import { loadConfig } from '../generated/graphql-server/src/config'
-loadConfig()
-
-import BN from 'bn.js'
-import { nanoid } from 'nanoid'
-import { SnakeNamingStrategy } from '@dzlzv/hydra-db-utils'
-import { createConnection, ConnectionOptions, getConnection, EntityManager } from 'typeorm'
-
-import { Video } from '../generated/graphql-server/src/modules/video/video.model'
-import { Channel } from '../generated/graphql-server/src/modules/channel/channel.model'
-import { Block, Network } from '../generated/graphql-server/src/modules/block/block.model'
-import { Category } from '../generated/graphql-server/src/modules/category/category.model'
-import { VideoMedia } from '../generated/graphql-server/src/modules/video-media/video-media.model'
-import { LicenseEntity } from '../generated/graphql-server/src/modules/license-entity/license-entity.model'
-import { JoystreamMediaLocation, KnownLicense } from '../generated/graphql-server/src/modules/variants/variants.model'
-import { KnownLicenseEntity } from '../generated/graphql-server/src/modules/known-license-entity/known-license-entity.model'
-import { VideoMediaEncoding } from '../generated/graphql-server/src/modules/video-media-encoding/video-media-encoding.model'
-import { MediaLocationEntity } from '../generated/graphql-server/src/modules/media-location-entity/media-location-entity.model'
-import { HttpMediaLocationEntity } from '../generated/graphql-server/src/modules/http-media-location-entity/http-media-location-entity.model'
-import { JoystreamMediaLocationEntity } from '../generated/graphql-server/src/modules/joystream-media-location-entity/joystream-media-location-entity.model'
-
-function getConnectionOptions() {
-  const connectionConfig: ConnectionOptions = {
-    type: 'postgres',
-    host: process.env.WARTHOG_DB_HOST,
-    port: parseInt(process.env.WARTHOG_DB_PORT!),
-    username: process.env.WARTHOG_DB_USERNAME,
-    password: process.env.WARTHOG_DB_PASSWORD,
-    database: process.env.WARTHOG_DB_DATABASE,
-    entities: [process.env.WARTHOG_DB_ENTITIES!],
-    namingStrategy: new SnakeNamingStrategy(),
-    logging: true,
-  }
-  return connectionConfig
-}
-
-export async function main(): Promise<void> {
-  console.log(`Initializing...`)
-  await createConnection(getConnectionOptions())
-  await getConnection().transaction(async (db: EntityManager) => {
-    const id = '0'
-    const createdAt = new Date()
-    const createdById = '0'
-    const version = 0
-
-    // ///////// Block /////////////////
-    const happenedIn = new Block({
-      createdAt,
-      createdById,
-      version,
-      block: 0,
-      timestamp: new BN(Date.now()),
-      network: Network.BABYLON,
-    })
-    await db.save<Block>(happenedIn)
-    // ///////// Block /////////////////
-
-    const commonProperties = { id, happenedIn, createdAt, createdById, version }
-
-    // ///////// HttpMediaLocationEntity /////////////////
-    const httpMediaLocation = new HttpMediaLocationEntity({
-      ...commonProperties,
-      url: '5FyzfM2YtZa75hHYCAo5evNS8bH8P4Kw8EyXqKkC5upVSDBQ',
-    })
-    await db.save<HttpMediaLocationEntity>(httpMediaLocation)
-    // ///////// HttpMediaLocationEntity /////////////////
-
-    // ///////// JoystreamMediaLocationEntity /////////////////
-    const joyMediaLocation = new JoystreamMediaLocationEntity({
-      ...commonProperties,
-      dataObjectId: '5FyzfM2YtZa75hHYCAo5evNS8bH8P4Kw8EyXqKkC5upVSDBQ',
-    })
-    await db.save<JoystreamMediaLocationEntity>(joyMediaLocation)
-    // ///////// JoystreamMediaLocationEntity /////////////////
-
-    // ///////// KnownLicenseEntity /////////////////
-    const knownLicense = new KnownLicenseEntity({ ...commonProperties, code: 'NA' })
-    await db.save<KnownLicenseEntity>(knownLicense)
-    // ///////// KnownLicenseEntity /////////////////
-
-    // ///////// License /////////////////
-    const k = new KnownLicense()
-    k.code = knownLicense.code
-    const license = new LicenseEntity({ ...commonProperties, type: k })
-    await db.save<LicenseEntity>(license)
-    // ///////// License /////////////////
-
-    // ///////// MediaLocationEntity /////////////////
-    const mediaLocEntity = new MediaLocationEntity({ ...commonProperties, joystreamMediaLocation: joyMediaLocation })
-    await db.save<MediaLocationEntity>(mediaLocEntity)
-    // ///////// MediaLocationEntity /////////////////
-
-    // ///////// Channel /////////////////
-    const channel = new Channel({
-      ...commonProperties,
-      handle: `Channel(0) - ${nanoid()}`,
-      description: `Channel 0`,
-      isPublic: false,
-      isCurated: false,
-    })
-    await db.save<Channel>(channel)
-    // ///////// Channel /////////////////
-
-    // ///////// Category /////////////////
-    const category = new Category({ ...commonProperties, name: `Other` })
-    await db.save<Category>(category)
-    // ///////// Category /////////////////
-
-    // ///////// VideoMediaEncoding /////////////////
-    const videoMediaEncod = new VideoMediaEncoding({ ...commonProperties, name: 'NA' })
-    await db.save<VideoMediaEncoding>(videoMediaEncod)
-    // ///////// VideoMediaEncoding /////////////////
-
-    // ///////// VideoMedia /////////////////
-    const location = new JoystreamMediaLocation()
-    location.dataObjectId = joyMediaLocation.dataObjectId
-    const videoMedia = new VideoMedia({
-      ...commonProperties,
-      location,
-      locationEntity: mediaLocEntity,
-      encoding: videoMediaEncod,
-      pixelHeight: 0,
-      pixelWidth: 0,
-    })
-    await db.save<VideoMedia>(videoMedia)
-    // ///////// VideoMedia /////////////////
-
-    // ///////// Video /////////////////
-    const v = new Video({ ...commonProperties })
-    v.category = category
-    v.channel = channel
-    v.media = videoMedia
-    v.license = license
-    v.title = `Video(0)`
-    v.description = `Video(0)`
-    v.duration = 0
-    v.thumbnailUrl = 'https://eu-central-1.linodeobjects.com/joystream/1.png'
-    v.isPublic = false
-    v.isCurated = false
-    v.isExplicit = false
-    v.isFeatured = false
-    await db.save<Video>(v)
-    // ///////// Video /////////////////
-  })
-}
-
-main()
-  .then(() => {
-    console.log(`Done.`)
-    process.exit()
-  })
-  .catch(console.log)