浏览代码

Query node: Further setup adjustments

Leszek Wiesner 3 年之前
父节点
当前提交
bf4306ed45

+ 38 - 12
docker-compose.yml

@@ -64,7 +64,7 @@ services:
       POSTGRES_PASSWORD: ${DB_PASS}
       POSTGRES_DB: ${INDEXER_DB_NAME}
 
-  graphql-server:
+  graphql-server: &graphql-server
     image: joystream/apps
     restart: unless-stopped
     build:
@@ -80,9 +80,20 @@ services:
       - "127.0.0.1:8081:${GRAPHQL_SERVER_PORT}"
     depends_on:
       - db
-    command: ["workspace", "query-node-root", "server:start:prod"]
+    command: ["workspace", "query-node-root", "query-node:start:prod"]
 
-  processor:
+  graphql-server-mnt:
+    <<: *graphql-server
+    image: node:14
+    build: .
+    volumes:
+      - type: bind
+        source: .
+        target: /joystream
+    working_dir: /joystream
+    command: ["yarn", "workspace", "query-node-root", "query-node:start:prod"]
+
+  processor: &processor
     image: joystream/apps
     restart: unless-stopped
     build:
@@ -92,16 +103,30 @@ services:
       # relative to working directory where docker-compose was run from
       - .env
     environment:
-      - POLL_INTERVAL_MS=300
-      - DEBUG=hydra-processor:*
-      - DB_HOST=db
-      - INDEXER_ENDPOINT_URL=http://indexer-api-gateway:4002/graphql
+      - INDEXER_ENDPOINT_URL=http://hydra-indexer-gateway:${WARTHOG_APP_PORT}/graphql
+      - TYPEORM_HOST=db
+      - TYPEORM_DATABASE=${DB_NAME}
+      - DEBUG=index-builder:*
+      - WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944
+    volumes:
+      - ./types/augment/all/defs.json:/joystream/query-node/mappings/lib/generated/types/typedefs.json
     depends_on:
-      - indexer-api-gateway
+      - hydra-indexer-gateway
     command: ["workspace", "query-node-root", "processor:start"]
 
+  processor-mnt:
+    <<: *processor
+    image: node:14
+    build: .
+    volumes:
+      - type: bind
+        source: .
+        target: /joystream
+    working_dir: /joystream
+    command: ["yarn", "workspace", "query-node-root", "processor:start"]
+
   indexer:
-    image: joystream/hydra-indexer:2.1.0-beta.5
+    image: joystream/hydra-indexer:3.0.0
     restart: unless-stopped
     env_file:
       # relative to working directory where docker-compose was run from
@@ -116,13 +141,14 @@ services:
       - DB_HOST=db
     depends_on:
       - db
+      - redis
     volumes:
       - ./types/augment/all/defs.json:/home/hydra/packages/hydra-indexer/types.json
     command: >
-      sh -c 'yarn db:bootstrap && yarn start:prod'
+      sh -c "yarn db:bootstrap && yarn start:prod"
 
-  indexer-api-gateway:
-    image: joystream/hydra-indexer-gateway:2.1.0-beta.5
+  hydra-indexer-gateway:
+    image: joystream/hydra-indexer-gateway:3.0.0
     restart: unless-stopped
     env_file:
       # relative to working directory where docker-compose was run from

+ 0 - 1
query-node/bootstrap/index.ts

@@ -1 +0,0 @@
-export { bootMembers } from './members';

+ 0 - 41
query-node/bootstrap/members.ts

@@ -1,41 +0,0 @@
-import { Member } from '../generated/graphql-server/src/modules/member/member.model';
-import { DB, getLogger } from '../generated/indexer';
-
-import { ApiPromise } from '@polkadot/api';
-import { Hash } from '@polkadot/types/interfaces';
-import { Option } from '@polkadot/types/codec';
-import type { Profile } from '@joystream/types/lib/members';
-import { Codec } from '@polkadot/types/types';
-
-const logger = getLogger();
-
-export async function bootMembers(api: ApiPromise, db: DB) {
-  let blkHeight: number = process.env.BLOCK_HEIGHT ? parseInt(process.env.BLOCK_HEIGHT) : 0;
-  let blkHash: Hash = await api.rpc.chain.getBlockHash(blkHeight);
-  let ids = await api.query.members.membersCreated.at(blkHash);
-  let num: number = parseInt(ids.toString());
-
-  for (let i = 0; i < num; i++) {
-    let profileOpt = (await api.query.members.memberProfile.at(blkHash, i)) as Option<Profile & Codec>;
-    let profile: Profile | null = profileOpt.unwrapOr(null);
-
-    if (!profile) {
-      continue;
-    }
-
-    let member = new Member();
-    member.memberId = i.toString();
-    member.handle = profile.handle.toString();
-    member.avatarUri = profile.avatar_uri.toString();
-    member.about = profile.about.toString();
-
-    member.rootAccount = Buffer.from(profile.root_account);
-    member.controllerAccount = Buffer.from(profile.controller_account);
-    member.registeredAtBlock = profile.registered_at_block.toString();
-
-    logger.trace(`Saving member: ${JSON.stringify(member, null, 2)}`);
-    await db.save<Member>(member);
-    logger.info(`Saved members: ${i}/${num}`);
-  }
-  logger.info(`Done bootstrapping members!`);
-}

+ 0 - 19
query-node/bootstrap/package.json

@@ -1,19 +0,0 @@
-{
-  "name": "mappings",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.ts",
-  "scripts": {
-    "build": "tsc --build tsconfig.json",
-    "postinstall": "tsc --build tsconfig.json"
-  },
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "@joystream/types": "^0.8.0",
-    "@polkadot/api": "^1.14.1",
-    "@polkadot/types": "^1.14.1",
-    "log4js": "^6.2.1",
-    "typeorm": "^0.2.24"
-  }
-}

+ 0 - 18
query-node/bootstrap/tsconfig.json

@@ -1,18 +0,0 @@
-{
-	"compilerOptions": {
-		"declaration": true,
-		"importHelpers": true,
-		"module": "commonjs",
-		"outDir": "lib",
-		"rootDir": "./..",
-		"strict": true,
-		"target": "es2017",
-		"experimentalDecorators": true,
-		"emitDecoratorMetadata": true,
-		"skipLibCheck": true,
-		"sourceMap": true,
-		"inlineSources": false
-	},
-	"include": ["./*.ts"],
-	"exclude": ["node_modules"]
-}

+ 6 - 3
query-node/build.sh

@@ -14,10 +14,13 @@ yarn clean
 yarn codegen:noinstall
 yarn typegen # if this fails try to run this command outside of yarn workspaces
 
+# We run yarn again to ensure graphql-server dependencies are installed
+# and are inline with root workspace resolutions
+yarn
+
 yarn workspace query-node codegen
 yarn workspace query-node build
+
 yarn workspace query-node-mappings build
 
-# We run yarn again to ensure graphql-server dependencies are installed
-# and are inline with root workspace resolutions
-yarn
+

+ 0 - 15
query-node/db-migrate.sh

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
-cd $SCRIPT_PATH
-
-set -a
-. ../.env
-set +a
-
-yarn workspace query-node-root db:schema:migrate
-
-yarn hydra-processor migrate --env=../.env
-
-yarn workspace query-node-root db:init

+ 17 - 0
query-node/kill.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+cd $SCRIPT_PATH
+
+set -a
+. ../.env
+set +a
+
+# Only remove query-node related services
+docker-compose rm -vsf processor-mnt
+docker-compose rm -vsf graphql-server-mnt
+docker-compose rm -vsf indexer
+docker-compose rm -vsf hydra-indexer-gateway
+docker-compose rm -vsf redis
+docker-compose rm -vsf db

+ 2 - 2
query-node/mappings/package.json

@@ -12,8 +12,8 @@
   },
   "dependencies": {
     "@polkadot/types": "4.2.1",
-    "@joystream/hydra-common": "3.1.0-alpha.7",
-    "@joystream/hydra-db-utils": "3.1.0-alpha.7",
+    "@joystream/hydra-common": "3.1.0-alpha.1",
+    "@joystream/hydra-db-utils": "3.1.0-alpha.1",
     "@joystream/content-metadata-protobuf": "^1.1.0",
     "@joystream/types": "^0.17.0",
     "@joystream/warthog": "2.35.0"

+ 1 - 0
query-node/mappings/tsconfig.json

@@ -12,6 +12,7 @@
     "experimentalDecorators": true,
     "emitDecoratorMetadata": true,
     "skipLibCheck": true,
+    "resolveJsonModule": true,
     "baseUrl": ".",
     "paths": {
       "@polkadot/types/augment": ["../../types/augment/augment-types.ts"]

+ 4 - 5
query-node/package.json

@@ -8,8 +8,7 @@
     "lint": "echo \"Skippinng\"",
     "clean": "rm -rf ./generated",
     "clean:query-node": "rm -rf ./generated/graphql-server",
-    "processor:bootstrap": "./bootstrap.sh",
-    "processor:start": "DEBUG=${DEBUG} yarn processor:bootstrap && hydra-processor run -e ../.env",
+    "processor:start": "DEBUG=${DEBUG} hydra-processor run -e ../.env",
     "query-node:build": "yarn workspace query-node build",
     "query-node:start:dev": "yarn workspace query-node start:dev",
     "query-node:start:prod": "yarn workspace query-node start:prod",
@@ -41,11 +40,11 @@
     "@types/bn.js": "^4.11.6",
     "bn.js": "^5.1.2",
     "@polkadot/metadata": "^4.1.1",
-    "@joystream/hydra-cli": "3.1.0-alpha.7",
-    "@joystream/hydra-processor": "3.1.0-alpha.7"
+    "@joystream/hydra-cli": "3.1.0-alpha.1",
+    "@joystream/hydra-processor": "3.1.0-alpha.1"
   },
   "devDependencies": {
-    "@joystream/hydra-typegen": "3.1.0-alpha.7"
+    "@joystream/hydra-typegen": "3.1.0-alpha.1"
   },
   "volta": {
 		"extends": "../package.json"

+ 0 - 9
query-node/processor-start.sh

@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
-cd $SCRIPT_PATH
-
-yarn
-
-yarn hydra-processor run --env=../.env

+ 0 - 21
query-node/scripts/get-class-id-and-name.ts

@@ -1,21 +0,0 @@
-import { ApiPromise, WsProvider } from '@polkadot/api'
-import { types as joyTypes } from '@joystream/types'
-import * as BN from 'bn.js'
-
-async function main() {
-  // Initialize the api
-  const provider = new WsProvider('ws://127.0.0.1:9944')
-  const api = await ApiPromise.create({ provider, types: joyTypes })
-
-  const n = await api.query.contentDirectory.nextClassId()
-  const nextClassId = new BN(n.toJSON() as string).toNumber()
-  for (let id = 0; id < nextClassId; id++) {
-    const cls = await api.query.contentDirectory.classById(new BN(id))
-    const { name } = cls.toJSON() as never
-    console.log(id, name)
-  }
-}
-
-main()
-  .then(() => process.exit())
-  .catch(console.error)

+ 0 - 155
query-node/scripts/initializeDefaultSchemas.ts

@@ -1,155 +0,0 @@
-/* eslint-disable import/first */
-import 'reflect-metadata'
-
-import { loadConfig } from '../generated/graphql-server/src/config'
-loadConfig()
-
-import BN from 'bn.js'
-import { nanoid } from 'nanoid'
-import { SnakeNamingStrategy } from '@dzlzv/hydra-db-utils'
-import { createConnection, ConnectionOptions, getConnection, EntityManager } from 'typeorm'
-
-import { Video } from '../generated/graphql-server/src/modules/video/video.model'
-import { Channel } from '../generated/graphql-server/src/modules/channel/channel.model'
-import { Block, Network } from '../generated/graphql-server/src/modules/block/block.model'
-import { Category } from '../generated/graphql-server/src/modules/category/category.model'
-import { VideoMedia } from '../generated/graphql-server/src/modules/video-media/video-media.model'
-import { LicenseEntity } from '../generated/graphql-server/src/modules/license-entity/license-entity.model'
-import { JoystreamMediaLocation, KnownLicense } from '../generated/graphql-server/src/modules/variants/variants.model'
-import { KnownLicenseEntity } from '../generated/graphql-server/src/modules/known-license-entity/known-license-entity.model'
-import { VideoMediaEncoding } from '../generated/graphql-server/src/modules/video-media-encoding/video-media-encoding.model'
-import { MediaLocationEntity } from '../generated/graphql-server/src/modules/media-location-entity/media-location-entity.model'
-import { HttpMediaLocationEntity } from '../generated/graphql-server/src/modules/http-media-location-entity/http-media-location-entity.model'
-import { JoystreamMediaLocationEntity } from '../generated/graphql-server/src/modules/joystream-media-location-entity/joystream-media-location-entity.model'
-
-function getConnectionOptions() {
-  const connectionConfig: ConnectionOptions = {
-    type: 'postgres',
-    host: process.env.WARTHOG_DB_HOST,
-    port: parseInt(process.env.WARTHOG_DB_PORT!),
-    username: process.env.WARTHOG_DB_USERNAME,
-    password: process.env.WARTHOG_DB_PASSWORD,
-    database: process.env.WARTHOG_DB_DATABASE,
-    entities: [process.env.WARTHOG_DB_ENTITIES!],
-    namingStrategy: new SnakeNamingStrategy(),
-    logging: true,
-  }
-  return connectionConfig
-}
-
-export async function main(): Promise<void> {
-  console.log(`Initializing...`)
-  await createConnection(getConnectionOptions())
-  await getConnection().transaction(async (db: EntityManager) => {
-    const id = '0'
-    const createdAt = new Date()
-    const createdById = '0'
-    const version = 0
-
-    // ///////// Block /////////////////
-    const happenedIn = new Block({
-      createdAt,
-      createdById,
-      version,
-      block: 0,
-      timestamp: new BN(Date.now()),
-      network: Network.BABYLON,
-    })
-    await db.save<Block>(happenedIn)
-    // ///////// Block /////////////////
-
-    const commonProperties = { id, happenedIn, createdAt, createdById, version }
-
-    // ///////// HttpMediaLocationEntity /////////////////
-    const httpMediaLocation = new HttpMediaLocationEntity({
-      ...commonProperties,
-      url: '5FyzfM2YtZa75hHYCAo5evNS8bH8P4Kw8EyXqKkC5upVSDBQ',
-    })
-    await db.save<HttpMediaLocationEntity>(httpMediaLocation)
-    // ///////// HttpMediaLocationEntity /////////////////
-
-    // ///////// JoystreamMediaLocationEntity /////////////////
-    const joyMediaLocation = new JoystreamMediaLocationEntity({
-      ...commonProperties,
-      dataObjectId: '5FyzfM2YtZa75hHYCAo5evNS8bH8P4Kw8EyXqKkC5upVSDBQ',
-    })
-    await db.save<JoystreamMediaLocationEntity>(joyMediaLocation)
-    // ///////// JoystreamMediaLocationEntity /////////////////
-
-    // ///////// KnownLicenseEntity /////////////////
-    const knownLicense = new KnownLicenseEntity({ ...commonProperties, code: 'NA' })
-    await db.save<KnownLicenseEntity>(knownLicense)
-    // ///////// KnownLicenseEntity /////////////////
-
-    // ///////// License /////////////////
-    const k = new KnownLicense()
-    k.code = knownLicense.code
-    const license = new LicenseEntity({ ...commonProperties, type: k })
-    await db.save<LicenseEntity>(license)
-    // ///////// License /////////////////
-
-    // ///////// MediaLocationEntity /////////////////
-    const mediaLocEntity = new MediaLocationEntity({ ...commonProperties, joystreamMediaLocation: joyMediaLocation })
-    await db.save<MediaLocationEntity>(mediaLocEntity)
-    // ///////// MediaLocationEntity /////////////////
-
-    // ///////// Channel /////////////////
-    const channel = new Channel({
-      ...commonProperties,
-      handle: `Channel(0) - ${nanoid()}`,
-      description: `Channel 0`,
-      isPublic: false,
-      isCurated: false,
-    })
-    await db.save<Channel>(channel)
-    // ///////// Channel /////////////////
-
-    // ///////// Category /////////////////
-    const category = new Category({ ...commonProperties, name: `Other` })
-    await db.save<Category>(category)
-    // ///////// Category /////////////////
-
-    // ///////// VideoMediaEncoding /////////////////
-    const videoMediaEncod = new VideoMediaEncoding({ ...commonProperties, name: 'NA' })
-    await db.save<VideoMediaEncoding>(videoMediaEncod)
-    // ///////// VideoMediaEncoding /////////////////
-
-    // ///////// VideoMedia /////////////////
-    const location = new JoystreamMediaLocation()
-    location.dataObjectId = joyMediaLocation.dataObjectId
-    const videoMedia = new VideoMedia({
-      ...commonProperties,
-      location,
-      locationEntity: mediaLocEntity,
-      encoding: videoMediaEncod,
-      pixelHeight: 0,
-      pixelWidth: 0,
-    })
-    await db.save<VideoMedia>(videoMedia)
-    // ///////// VideoMedia /////////////////
-
-    // ///////// Video /////////////////
-    const v = new Video({ ...commonProperties })
-    v.category = category
-    v.channel = channel
-    v.media = videoMedia
-    v.license = license
-    v.title = `Video(0)`
-    v.description = `Video(0)`
-    v.duration = 0
-    v.thumbnailUrl = 'https://eu-central-1.linodeobjects.com/joystream/1.png'
-    v.isPublic = false
-    v.isCurated = false
-    v.isExplicit = false
-    v.isFeatured = false
-    await db.save<Video>(v)
-    // ///////// Video /////////////////
-  })
-}
-
-main()
-  .then(() => {
-    console.log(`Done.`)
-    process.exit()
-  })
-  .catch(console.log)

+ 30 - 0
query-node/start.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -e
+
+SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+cd $SCRIPT_PATH
+
+set -a
+. ../.env
+set +a
+
+# Start the joystream-node first to allow fetching Olympia metadata during build (typegen)
+docker-compose up -d joystream-node
+
+# Only run codegen if no generated files found
+[ ! -d "generated/" ] && yarn build
+
+# Bring up db
+docker-compose up -d db
+
+# Make sure we use dev config for db migrations (prevents "Cannot create database..." and some other errors)
+yarn workspace query-node config:dev
+
+# Migrate the databases
+yarn workspace query-node-root db:prepare
+yarn workspace query-node-root db:migrate
+
+docker-compose up -d graphql-server-mnt
+
+# Starting up processor will bring up all services it depends on
+docker-compose up -d processor-mnt

文件差异内容过多而无法显示
+ 136 - 130
yarn.lock


部分文件因为文件数量过多而无法显示