query node - members and workers bootstrapping

query-node/bootstrap.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+
+SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+cd $SCRIPT_PATH
+
+# Load and export variables from root .env file into shell environment
+set -a
+. ../.env
+. ./generated/graphql-server/.env
+set +a
+
+BOOTSTRAP_DATA_FOLDER=`pwd`/mappings/bootstrap/data node ./mappings/lib/mappings/bootstrap/index.js

query-node/bootstrap/index.ts

@@ -1 +0,0 @@
-export { bootMembers } from './members';

query-node/bootstrap/members.ts

@@ -1,41 +0,0 @@
-import { Member } from '../generated/graphql-server/src/modules/member/member.model';
-import { DB, getLogger } from '../generated/indexer';
-
-import { ApiPromise } from '@polkadot/api';
-import { Hash } from '@polkadot/types/interfaces';
-import { Option } from '@polkadot/types/codec';
-import type { Profile } from '@joystream/types/lib/members';
-import { Codec } from '@polkadot/types/types';
-
-const logger = getLogger();
-
-export async function bootMembers(api: ApiPromise, db: DB) {
-  let blkHeight: number = process.env.BLOCK_HEIGHT ? parseInt(process.env.BLOCK_HEIGHT) : 0;
-  let blkHash: Hash = await api.rpc.chain.getBlockHash(blkHeight);
-  let ids = await api.query.members.membersCreated.at(blkHash);
-  let num: number = parseInt(ids.toString());
-
-  for (let i = 0; i < num; i++) {
-    let profileOpt = (await api.query.members.memberProfile.at(blkHash, i)) as Option<Profile & Codec>;
-    let profile: Profile | null = profileOpt.unwrapOr(null);
-
-    if (!profile) {
-      continue;
-    }
-
-    let member = new Member();
-    member.memberId = i.toString();
-    member.handle = profile.handle.toString();
-    member.avatarUri = profile.avatar_uri.toString();
-    member.about = profile.about.toString();
-
-    member.rootAccount = Buffer.from(profile.root_account);
-    member.controllerAccount = Buffer.from(profile.controller_account);
-    member.registeredAtBlock = profile.registered_at_block.toString();
-
-    logger.trace(`Saving member: ${JSON.stringify(member, null, 2)}`);
-    await db.save<Member>(member);
-    logger.info(`Saved members: ${i}/${num}`);
-  }
-  logger.info(`Done bootstrapping members!`);
-}

query-node/bootstrap/package.json

@@ -1,19 +0,0 @@
-{
-  "name": "mappings",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.ts",
-  "scripts": {
-    "build": "tsc --build tsconfig.json",
-    "postinstall": "tsc --build tsconfig.json"
-  },
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "@joystream/types": "^0.8.0",
-    "@polkadot/api": "^1.14.1",
-    "@polkadot/types": "^1.14.1",
-    "log4js": "^6.2.1",
-    "typeorm": "^0.2.24"
-  }
-}

query-node/bootstrap/tsconfig.json

@@ -1,18 +0,0 @@
-{
-	"compilerOptions": {
-		"declaration": true,
-		"importHelpers": true,
-		"module": "commonjs",
-		"outDir": "lib",
-		"rootDir": "./..",
-		"strict": true,
-		"target": "es2017",
-		"experimentalDecorators": true,
-		"emitDecoratorMetadata": true,
-		"skipLibCheck": true,
-		"sourceMap": true,
-		"inlineSources": false
-	},
-	"include": ["./*.ts"],
-	"exclude": ["node_modules"]
-}

query-node/mappings/bootstrap/data/members.json

@@ -0,0 +1 @@
+[]

query-node/mappings/bootstrap/data/workers.json

@@ -0,0 +1 @@
+{}

query-node/mappings/bootstrap/index.ts

@@ -0,0 +1,66 @@
+import { createDBConnection } from '@dzlzv/hydra-processor/lib/db'
+import { DatabaseManager, makeDatabaseManager } from '@dzlzv/hydra-db-utils'
+import { Connection, getManager } from 'typeorm'
+
+import { bootMembers, IBootstrapMember } from './members';
+import { bootWorkers, IBootstrapWorker, IBootstrapWorkers } from './workers';
+import { Worker, WorkerType } from 'query-node'
+import { FindConditions } from 'typeorm'
+import fs from 'fs'
+import path from 'path'
+
+
+init()
+
+async function init() {
+    const [databaseManager, connection] = await createDatabaseManager()
+    const data = loadData()
+
+    if (await isDbInitialized(databaseManager)) {
+        await connection.close()
+        return
+    }
+
+    await bootMembers(databaseManager, data.members)
+    await bootWorkers(databaseManager, data.workers)
+
+    await connection.close()
+}
+
+async function isDbInitialized(db: DatabaseManager): Promise<boolean> {
+    // simple way to check if db is bootstrapped already - check if there is at least 1 storage provider
+    const membership = await db.get(Worker, {
+        where: {
+          type: WorkerType.STORAGE,
+        } as FindConditions<Worker>
+    })
+
+    return !!membership
+}
+
+
+//async function createDatabaseManager(): Promise<DatabaseManager> {
+async function createDatabaseManager(): Promise<[DatabaseManager, Connection]> {
+    // paths in `entities` should be the same as `entities` set in `manifest.yml`
+    const entities = [
+        'generated/graphql-server/dist/**/*.model.js'
+    ]
+
+    const connection = await createDBConnection(entities)
+    const entityManager = getManager(connection.name)
+    const databaseManager = makeDatabaseManager(entityManager)
+
+    return [databaseManager, connection]
+}
+
+interface IBootstrapData {
+    members: IBootstrapMember[]
+    workers: IBootstrapWorkers
+}
+
+function loadData(): IBootstrapData {
+    return {
+        members: JSON.parse(fs.readFileSync(process.env.BOOTSTRAP_DATA_FOLDER + '/members.json').toString()),
+        workers: JSON.parse(fs.readFileSync(process.env.BOOTSTRAP_DATA_FOLDER + '/workers.json').toString()),
+    }
+}

query-node/mappings/bootstrap/members.ts

@@ -0,0 +1,44 @@
+//import { Connection } from 'typeorm'
+import { DatabaseManager } from '@dzlzv/hydra-db-utils'
+import {
+  logger,
+} from '../src/common'
+import { MembershipEntryMethod, Membership } from 'query-node'
+
+export interface IBootstrapMember {
+  member_id: number
+  root_account: string,
+  controller_account: string,
+  handle: string,
+  avatar_uri: string,
+  about: string,
+  registered_at_time: number
+}
+
+//export async function bootMembers(members: IBootstrapMember[], db: Connection): Promise<void> {
+export async function bootMembers(db: DatabaseManager, members: IBootstrapMember[]): Promise<void> {
+  for (const rawMember of members) {
+    // create new membership
+    const member = new Membership({
+      // main data
+      id: rawMember.member_id.toString(),
+      rootAccount: rawMember.root_account,
+      controllerAccount: rawMember.controller_account,
+      handle: rawMember.handle,
+      about: rawMember.about,
+      avatarUri: rawMember.avatar_uri,
+      createdInBlock: 0,
+      entry: MembershipEntryMethod.GENESIS,
+
+      // fill in auto-generated fields
+      createdAt: new Date(rawMember.registered_at_time),
+      updatedAt: new Date(rawMember.registered_at_time),
+    })
+
+    // save membership
+    await db.save<Membership>(member)
+
+    // emit log event
+    logger.info('Member has been bootstrapped', {id: rawMember.member_id})
+  }
+}

query-node/mappings/bootstrap/workers.ts

@@ -0,0 +1,39 @@
+import { DatabaseManager } from '@dzlzv/hydra-db-utils'
+import { Worker, WorkerType } from 'query-node'
+import {logger} from '../src/common'
+
+export interface IBootstrapWorkers {
+  storage: IBootstrapWorker[]
+  gateway: IBootstrapWorker[]
+}
+
+export interface IBootstrapWorker {
+  role_account_id: string
+}
+
+export async function bootWorkers(db: DatabaseManager, workers: IBootstrapWorkers): Promise<void> {
+  await bootWorkersInGroup(db, workers.storage, WorkerType.STORAGE)
+  await bootWorkersInGroup(db, workers.gateway, WorkerType.GATEWAY)
+}
+
+export async function bootWorkersInGroup(db: DatabaseManager, workers: IBootstrapWorker[], workerType: WorkerType): Promise<void> {
+  if (!workers) {
+    return
+  }
+
+  for (const rawWorker of workers) {
+    // create new membership
+    const worker = new Worker({
+      // main data
+      workerId: rawWorker.role_account_id,
+      type: workerType,
+      isActive: true,
+    })
+
+    // save worker
+    await db.save<Worker>(worker)
+
+    // emit log event
+    logger.info('Worker has been bootstrapped', {id: rawWorker.role_account_id, workerType})
+  }
+}

query-node/package.json

@@ -8,7 +8,8 @@
     "lint": "echo \"Skippinng\"",
     "clean": "rm -rf ./generated",
     "clean:query-node": "rm -rf ./generated/graphql-server",
-    "processor:start": "DEBUG=${DEBUG} hydra-processor run -e ../.env",
+    "processor:bootstrap": "./bootstrap.sh",
+    "processor:start": "DEBUG=${DEBUG} yarn processor:bootstrap && hydra-processor run -e ../.env",
     "query-node:build": "yarn workspace query-node build",
     "query-node:start:dev": "yarn workspace query-node start:dev",
     "query-node:start:prod": "yarn workspace query-node start:prod",