refactor council report generator

 - update README, types
 - lib/rewards.ts: getWorkerRewards
 - replace computeCuratorsReward()
 - use this.saveStats()

community-contributions/report-generator/src/lib/api.ts

@@ -0,0 +1,281 @@
+import { ApiPromise } from "@polkadot/api";
+import { Option, u32, Vec } from "@polkadot/types";
+import type { Codec, Observable } from "@polkadot/types/types";
+import {
+  AccountId,
+  Balance,
+  BalanceOf,
+  BlockNumber,
+  EraIndex,
+  EventRecord,
+  Hash,
+  Moment,
+} from "@polkadot/types/interfaces";
+import { SignedBlock } from "@polkadot/types/interfaces/runtime";
+
+import { PostId, ThreadId } from "@joystream/types/common";
+import { CategoryId } from "@joystream/types/forum";
+import { ElectionStake, SealedVote, Seats } from "@joystream/types/council";
+import { Mint, MintId } from "@joystream/types/mint";
+import { MemberId, Membership } from "@joystream/types/members";
+import { WorkerId } from "@joystream/types/working-group";
+import { Stake, StakeId } from "@joystream/types/stake";
+import {
+  RewardRelationship,
+  RewardRelationshipId,
+} from "@joystream/types/recurring-rewards";
+
+import { Entity, EntityId } from "@joystream/types/content-directory";
+import { ContentId, DataObject } from "@joystream/types/media";
+
+import { SpendingParams } from "@joystream/types/proposals";
+import { ProposalId, WorkerOf } from "@joystream/types/augment-codec/all";
+import { ProposalDetails, ProposalOf } from "@joystream/types/augment/types";
+
+// blocks
+export const getBlock = (api: ApiPromise, hash: Hash): Promise<SignedBlock> =>
+  api.rpc.chain.getBlock(hash);
+
+export const getBlockHash = (api: ApiPromise, block: number): Promise<Hash> =>
+  api.rpc.chain.getBlockHash(block);
+
+export const getTimestamp = (api: ApiPromise, hash: Hash): Promise<Moment> =>
+  api.query.timestamp.now.at(hash);
+
+export const getIssuance = (api: ApiPromise, hash: Hash): Promise<Balance> =>
+  api.query.balances.totalIssuance.at(hash);
+
+export const getEvents = (
+  api: ApiPromise,
+  hash: Hash
+): Promise<Vec<EventRecord>> => api.query.system.events.at(hash);
+
+export const getEra = (
+  api: ApiPromise,
+  hash: Hash
+): Promise<Option<EraIndex>> => api.query.staking.currentEra.at(hash);
+
+export const getEraStake = async (
+  api: ApiPromise,
+  hash: Hash,
+  era: EraIndex
+): Promise<number> =>
+  (await api.query.staking.erasTotalStake.at(hash, era)).toNumber();
+
+// council
+export const getCouncil = (api: ApiPromise, hash: Hash): Promise<Seats> =>
+  api.query.council.activeCouncil.at(hash);
+
+export const getCouncilRound = async (
+  api: ApiPromise,
+  hash: Hash
+): Promise<number> =>
+  ((await api.query.councilElection.round.at(hash)) as u32).toNumber();
+
+export const getCouncilSize = async (
+  api: ApiPromise,
+  hash: Hash
+): Promise<number> =>
+  ((await api.query.councilElection.councilSize.at(hash)) as u32).toNumber();
+
+export const getCouncilApplicants = (
+  api: ApiPromise,
+  hash: Hash
+): Promise<Vec<AccountId>> => api.query.councilElection.applicants.at(hash);
+
+export const getCouncilApplicantStakes = (
+  api: ApiPromise,
+  hash: Hash,
+  applicant: AccountId
+): Promise<ElectionStake> =>
+  api.query.councilElection.applicantStakes.at(hash, applicant);
+
+export const getCouncilCommitments = (
+  api: ApiPromise,
+  hash: Hash
+): Promise<Vec<Hash>> => api.query.councilElection.commitments.at(hash);
+
+export const getCouncilPayoutInterval = (
+  api: ApiPromise,
+  hash: Hash
+): Promise<Option<BlockNumber>> => api.query.council.payoutInterval.at(hash);
+
+export const getCouncilPayout = (
+  api: ApiPromise,
+  hash: Hash
+): Promise<BalanceOf> => api.query.council.amountPerPayout.at(hash);
+
+const periods = [
+  "announcingPeriod",
+  "votingPeriod",
+  "revealingPeriod",
+  "newTermDuration",
+];
+export const getCouncilPeriods = (
+  api: ApiPromise,
+  hash: Hash
+): Promise<number>[] =>
+  periods.map(async (period: string) =>
+    ((await api.query.councilElection[period].at(
+      hash
+    )) as BlockNumber).toNumber()
+  );
+
+// working groups
+export const getNextWorker = async (
+  api: ApiPromise,
+  group: string,
+  hash: Hash
+): Promise<number> =>
+  ((await api.query[group].nextWorkerId.at(hash)) as WorkerId).toNumber();
+
+export const getWorker = (
+  api: ApiPromise,
+  group: string,
+  hash: Hash,
+  id: number
+): Promise<WorkerOf> => api.query[group].workerById.at(hash, id);
+
+export const getWorkers = (
+  api: ApiPromise,
+  group: string,
+  hash: Hash
+): Promise<number> => api.query[group].activeWorkerCount.at(hash);
+
+export const getStake = async (
+  api: ApiPromise,
+  id: StakeId | number
+): Promise<Stake> => (await api.query.stake.stakes(id)) as Stake;
+
+export const getWorkerReward = (
+  api: ApiPromise,
+  hash: Hash,
+  id: RewardRelationshipId | number
+): Promise<RewardRelationship> =>
+  api.query.recurringRewards.rewardRelationships.at(hash, id);
+
+// mints
+export const getCouncilMint = (api: ApiPromise, hash: Hash): Promise<MintId> =>
+  api.query.council.councilMint.at(hash);
+
+export const getGroupMint = (
+  api: ApiPromise,
+  group: string,
+  hash: Hash
+): Promise<MintId> => api.query[group].mint.at(hash);
+
+export const getMintsCreated = async (
+  api: ApiPromise,
+  hash: Hash
+): Promise<number> => parseInt(await api.query.minting.mintsCreated.at(hash));
+
+export const getMint = (
+  api: ApiPromise,
+  hash: Hash,
+  id: MintId | number
+): Promise<Mint> => api.query.minting.mints.at(hash, id);
+
+// members
+export const getNextMember = async (
+  api: ApiPromise,
+  hash: Hash
+): Promise<number> =>
+  ((await api.query.members.nextMemberId.at(hash)) as MemberId).toNumber();
+
+export const getMember = (
+  api: ApiPromise,
+  hash: Hash,
+  id: MemberId
+): Promise<Membership> => api.query.members.membershipById.at(hash, id);
+
+// forum
+export const getNextPost = async (
+  api: ApiPromise,
+  hash: Hash
+): Promise<number> =>
+  ((await api.query.forum.nextPostId.at(hash)) as PostId).toNumber();
+
+export const getNextThread = async (
+  api: ApiPromise,
+  hash: Hash
+): Promise<number> =>
+  ((await api.query.forum.nextThreadId.at(hash)) as ThreadId).toNumber();
+
+export const getNextCategory = async (
+  api: ApiPromise,
+  hash: Hash
+): Promise<number> =>
+  ((await api.query.forum.nextCategoryId.at(hash)) as CategoryId).toNumber();
+
+// proposals
+export const getProposalCount = async (
+  api: ApiPromise,
+  hash: Hash
+): Promise<number> =>
+  ((await api.query.proposalsEngine.proposalCount.at(hash)) as u32).toNumber();
+
+export const getProposalInfo = async (
+  api: ApiPromise,
+  id: ProposalId
+): Promise<ProposalOf> =>
+  (await api.query.proposalsEngine.proposals(id)) as ProposalOf;
+
+export const getProposalDetails = async (
+  api: ApiPromise,
+  id: ProposalId
+): Promise<ProposalDetails> =>
+  (await api.query.proposalsCodex.proposalDetailsByProposalId(
+    id
+  )) as ProposalDetails;
+
+// validators
+export const getValidatorCount = async (
+  api: ApiPromise,
+  hash: Hash
+): Promise<number> =>
+  ((await api.query.staking.validatorCount.at(hash)) as u32).toNumber();
+
+export const getValidators = (
+  api: ApiPromise,
+  hash: Hash
+): Promise<Option<Vec<AccountId>>> =>
+  api.query.staking.snapshotValidators.at(hash);
+
+// media
+export const getNextEntity = async (
+  api: ApiPromise,
+  hash: Hash
+): Promise<number> =>
+  ((await api.query.contentDirectory.nextEntityId.at(
+    hash
+  )) as EntityId).toNumber();
+
+export const getNextChannel = async (
+  api: ApiPromise,
+  hash: Hash
+): Promise<number> => api.query.content.nextChannelId.at(hash);
+
+export const getNextVideo = async (
+  api: ApiPromise,
+  hash: Hash
+): Promise<number> => api.query.content.nextVideoId.at(hash);
+
+export const getEntity = (
+  api: ApiPromise,
+  hash: Hash,
+  id: number
+): Promise<Entity> => api.query.contentDirectory.entityById.at(hash, id);
+
+export const getDataObjects = async (
+  api: ApiPromise
+): Promise<Map<ContentId, DataObject>> =>
+  ((await api.query.dataDirectory.dataByContentId.entries()) as unknown) as Map<
+    ContentId,
+    DataObject
+  >;
+
+export const getDataObject = async (
+  api: ApiPromise,
+  id: ContentId
+): Promise<Option<DataObject>> =>
+  (await api.query.dataDirectory.dataByContentId(id)) as Option<DataObject>;

community-contributions/report-generator/src/lib/index.ts

@@ -0,0 +1,13 @@
+import { Mint, MintId } from "@joystream/types/mint";
+import { Moment } from "@polkadot/types/interfaces";
+
+export const getPercent = (value1: number, value2: number): number => {
+  if (value1 == 0) return value2 > 0 ? Infinity : 0;
+  return Number(((value2 * 100) / value1 - 100).toFixed(2));
+};
+
+export const momentToString = (moment: Moment) =>
+  new Date(moment.toNumber()).toLocaleDateString("en-US");
+
+export const getTotalMinted = (mint: Mint) =>
+  Number(mint.getField("total_minted").toString());

community-contributions/report-generator/src/lib/rewards.ts

@@ -0,0 +1,127 @@
+import { ApiPromise } from "@polkadot/api";
+import { Option, Vec } from "@polkadot/types";
+import { AccountId, Balance } from "@polkadot/types/interfaces";
+import { Hash } from "@polkadot/types/interfaces";
+import { Mint, MintId } from "@joystream/types/mint";
+import { Stake } from "@joystream/types/stake";
+import { WorkerOf } from "@joystream/types/augment-codec/all";
+import { Bounty, CacheEvent, MintStatistics, WorkerReward } from "../types";
+import {
+  RewardRelationship,
+  RewardRelationshipId,
+} from "@joystream/types/recurring-rewards";
+
+import { getPercent, getTotalMinted, momentToString } from "./";
+import {
+  getBlock,
+  getBlockHash,
+  getMint,
+  getNextWorker,
+  getWorker,
+  getWorkerReward,
+  getStake,
+  getValidators,
+  getValidatorCount,
+} from "./api";
+
+export const filterMethods = {
+  getBurnedTokens: ({ section, method }: CacheEvent) =>
+    section === "balances" && method === "Transfer",
+  newValidatorsRewards: ({ section, method }: CacheEvent) =>
+    section === "staking" && method === "Reward",
+};
+
+export const getWorkerRewards = async (
+  api: ApiPromise,
+  group: string,
+  hash: Hash
+): Promise<WorkerReward[]> => {
+  let workers = Array<WorkerReward>();
+  const nextWorkerId = await getNextWorker(api, group, hash);
+
+  for (let id = 0; id < nextWorkerId; ++id) {
+    const worker: WorkerOf = await getWorker(api, group, hash, id);
+
+    // TODO workers fired before the end will be missed out
+    if (!worker.is_active) continue;
+    let stake: Stake, reward: RewardRelationship;
+
+    if (worker.role_stake_profile.isSome) {
+      const roleStakeProfile = worker.role_stake_profile.unwrap();
+      stake = await getStake(api, roleStakeProfile.stake_id);
+    }
+
+    if (worker.reward_relationship.isSome) {
+      // TODO changing salaries are not reflected
+      const rewardId: RewardRelationshipId = worker.reward_relationship.unwrap();
+      reward = await getWorkerReward(api, hash, rewardId);
+    }
+    workers.push({ id, stake, reward });
+  }
+  return workers;
+};
+
+export const getBurnedTokens = (
+  burnAddress: string,
+  blocks: [number, CacheEvent[]][]
+): number => {
+  let tokensBurned = 0;
+  blocks.forEach(([key, transfers]) =>
+    transfers.forEach((transfer) => {
+      let receiver = transfer.data[1] as AccountId;
+      let amount = transfer.data[2] as Balance;
+      if (receiver.toString() === burnAddress) tokensBurned = Number(amount);
+    })
+  );
+  return tokensBurned;
+};
+
+export const getMintInfo = async (
+  api: ApiPromise,
+  mintId: MintId,
+  startHash: Hash,
+  endHash: Hash
+): Promise<MintStatistics> => {
+  const startMint: Mint = await getMint(api, startHash, mintId);
+  const endMint: Mint = await getMint(api, endHash, mintId);
+  let stats = new MintStatistics();
+  stats.startMinted = getTotalMinted(startMint);
+  stats.endMinted = getTotalMinted(endMint);
+  stats.diffMinted = stats.endMinted - stats.startMinted;
+  stats.percMinted = getPercent(stats.startMinted, stats.endMinted);
+  return stats;
+};
+
+export const getValidatorsRewards = (
+  blocks: [number, CacheEvent[]][]
+): number => {
+  let newValidatorRewards = 0;
+  blocks.forEach(([key, validatorRewards]) =>
+    validatorRewards.forEach(
+      (reward: CacheEvent) => (newValidatorRewards += Number(reward.data[1]))
+    )
+  );
+  return newValidatorRewards;
+};
+
+export const getActiveValidators = async (
+  api: ApiPromise,
+  hash: Hash,
+  searchPreviousBlocks: boolean = false
+): Promise<AccountId[]> => {
+  const block = await getBlock(api, hash);
+  let currentBlockNr = block.block.header.number.toNumber();
+  let activeValidators: AccountId[];
+  do {
+    const hash: Hash = await getBlockHash(api, currentBlockNr);
+    const validators: Option<Vec<AccountId>> = await getValidators(api, hash);
+    if (!validators.isEmpty) {
+      let max = await getValidatorCount(api, hash);
+      activeValidators = Array.from(validators.unwrap()).slice(0, max);
+    }
+
+    if (searchPreviousBlocks) --currentBlockNr;
+    else ++currentBlockNr;
+  } while (activeValidators == undefined);
+  return activeValidators;
+};

contributions/tech/report-generator/.gitignore

@@ -1,5 +1,4 @@
 .idea/*
-lib/*
 node_modules
 yarn.lock
 report.md

contributions/tech/report-generator/README.md

@@ -4,17 +4,14 @@ This scripts collects some information from Joystream chain. \
 It was created to allow the council to generate a report in the finish of the council round. \
 It takes some minutes to complete the report, multiple runs, with the same block range, will be quicker since it has a "cache" system for the block events.  
 
- ## Setup
- ```
- yarn && yarn build
- ```
+## Setup
 
- ## Usage
- ```
-node lib/generator.js <start block> <end block> 
- ```
+`yarn && yarn build`
+
+## Usage
+
+`node lib/generator.js <start block> <end block>`
 
 ## Example
- ```
-node lib/generator.js 57601 234038 
- ```
+
+`node lib/generator.js 57601 234038`

contributions/tech/report-generator/src/types.ts

@@ -1,6 +1,9 @@
 import { GenericEventData } from "@polkadot/types/generic/Event";
+import { Stake } from "@joystream/types/stake";
+import { RewardRelationship } from "@joystream/types/recurring-rewards";
 
 export class Statistics {
+  [key: string]: number | string;
   councilRound: number = 0;
   councilMembers: number = 0;
 
@@ -34,7 +37,6 @@ export class Statistics {
   percNewThreads: number = 0;
 
   startPosts: number = 0;
-  // endPosts: number = 0;
   newPosts: number = 0;
   endPosts: number = 0;
   percNewPosts: number = 0;
@@ -245,3 +247,9 @@ export class CacheEvent {
     public data: GenericEventData
   ) {}
 }
+
+export interface WorkerReward {
+  id: number;
+  stake: Stake;
+  reward: RewardRelationship;
+}