Improve create / update of eras & add stake field to eras

server/joystream/index.ts

@@ -73,7 +73,7 @@ const getTimestamp = async (api: Api, hash?: string) =>
     .utc(
       hash
         ? await api.query.timestamp.now.at(hash)
-        : await api.query.timestamp.now()
+        : await api.query.timestamp.now(),
     )
     .valueOf()
 
@@ -91,7 +91,7 @@ const addBlock = async (
     posts: 0,
     proposals: 0,
     proposalPosts: 0,
-  }
+  },
 ): Promise<Status> => {
   const id = +header.number
   const last = await Block.findByPk(id - 1)
@@ -108,7 +108,7 @@ const addBlock = async (
   block.setValidator(account.id)
 
   const currentEra = Number(await api.query.staking.currentEra())
-  const era = await Era.findOrCreate({ where: { id: currentEra } })
+  await Era.findOrCreate({ where: { id: currentEra } })
   await block.setEra(currentEra)
   io.emit('block', await Block.findByIdWithIncludes(block.id))
 
@@ -119,8 +119,11 @@ const addBlock = async (
   const q = queue.length ? ` (${queue.length} queued${f})` : ''
   console.log(`[Joystream] block ${block.id} ${handle}${q}`)
 
-  processEvents(api, id)
-  //updateEra(api, io, status, currentEra)
+  await processEvents(api, id)
+  if (await isEraInfoAvailable(api, id)) {
+    await updateEra(api, currentEra, id)
+  }
+
   //updateBalances(api, id)
   return updateStatus(api, status, currentEra)
 }
@@ -128,52 +131,34 @@ const addBlock = async (
 const addBlockRange = async (
   api: Api,
   startBlock: number,
-  endBlock: number
+  endBlock: number,
 ) => {
   const previousHash = await getBlockHash(api, startBlock - 1)
   let previousEra = (await api.query.staking.activeEra.at(
-    previousHash
+    previousHash,
   )) as Option<ActiveEraInfo>
 
   for (let i = startBlock; i < endBlock; i++) {
+    console.log('[Joystream] Processing block ' + i)
+
     const hash = await getBlockHash(api, i)
     const blockEra = (await api.query.staking.activeEra.at(
-      hash
+      hash,
     )) as Option<ActiveEraInfo>
+    const currentEra = blockEra.unwrap().index.toNumber();
+
     if (
-      blockEra.unwrap().index.toNumber() ===
+      currentEra ===
       previousEra.unwrap().index.toNumber()
     ) {
       continue
     }
 
-    let totalValidators = (await api.query.staking.snapshotValidators.at(
-      hash
-    )) as Option<Vec<AccountId>>
-    if (totalValidators.isEmpty) {
-      continue
+    if (await isEraInfoAvailable(api, i)) {
+      await Era.findOrCreate({ where: { id: currentEra } })
+      await updateEra(api, currentEra, i)
+      previousEra = blockEra
     }
-    console.log(`found validators: ${totalValidators.unwrap().length}`)
-
-    const totalNrValidators = totalValidators.unwrap().length
-    const maxSlots = Number(
-      (await api.query.staking.validatorCount.at(hash)).toString()
-    )
-    const actives = Math.min(maxSlots, totalNrValidators)
-    const waiting =
-      totalNrValidators > maxSlots ? totalNrValidators - maxSlots : 0
-
-    const timestamp = (await api.query.timestamp.now.at(hash)) as Moment
-    const date = new Date(timestamp.toNumber())
-    await Era.create({
-      id: blockEra.unwrap().index.toNumber(),
-      waiting: waiting,
-      actives: actives,
-      maxSlots: maxSlots,
-      timestamp: date,
-    })
-
-    previousEra = blockEra
   }
 }
 
@@ -200,7 +185,7 @@ const updateStatus = async (api: Api, old: Status, era: number) => {
     status.channels > old.channels && fetchChannel(api, status.channels)
     status.categories > old.categories && fetchCategory(api, status.categories)
     status.proposalPosts > old.proposalPosts &&
-      fetchProposalPosts(api, status.proposalPosts)
+    fetchProposalPosts(api, status.proposalPosts)
   }
   return status
 }
@@ -218,7 +203,7 @@ const fetchAll = async (api: Api, status: Status) => {
   for (let id = status.proposals; id > 0; id--) {
     queue.push(() => fetchProposal(api, id))
   }
-  queue.push(() => fetchProposalPosts(api, status.proposalPosts))
+  // queue.push(() => fetchProposalPosts(api, status.proposalPosts))
 
   for (let id = status.channels; id > 0; id--) {
     queue.push(() => fetchChannel(api, id))
@@ -257,44 +242,43 @@ const processEvents = async (api: Api, blockId: number) => {
   // TODO catch votes, posts, proposals?
 }
 
-const updateEra = async (api: Api, io: any, status: any, era: number) => {
-  const now: number = moment().valueOf()
-  if (lastUpdate + 60000 > now) return status
-  //console.log(`updating status`, lastUpdate)
-  lastUpdate = now
-
-  // session.disabledValidators: Vec<u32>
-  // check online: imOnline.keys
-  // imOnline.authoredBlocks: 2
-  // session.currentIndex: 17,081
-
-  //const lastReward = Number(await api.query.staking.erasValidatorReward(era))
-  //console.debug(`last reward`, era, lastReward)
-  //if (lastReward > 0) {} // TODO save lastReward
-
-  const nominatorEntries = await api.query.staking.nominators.entries()
-  const nominators = nominatorEntries.map((n: any) => String(n[0].toHuman()))
-
-  const rewardPoints = await api.query.staking.erasRewardPoints(era)
-  const validatorEntries = await api.query.session.validators()
-  const validators = validatorEntries.map((v: any) => String(v))
-
-  // TODO staking.bondedEras: Vec<(EraIndex,SessionIndex)>
-  const stashes = await api.derive.staking.stashes()
-  fetching = `stakes`
-  //if (!stashes) return
-
-  for (let validator of stashes) {
-    try {
-      const prefs = await api.query.staking.erasValidatorPrefs(era, validator)
-      const commission = Number(prefs.commission) / 10000000
-
-      const data = await api.query.staking.erasStakers(era, validator)
-      let { total, own, others } = data.toJSON()
-    } catch (e) {
-      console.warn(`Failed to fetch stakes for ${validator} in era ${era}`, e)
-    }
+const isEraInfoAvailable = async (api: Api, blockId: number) => {
+  const hash = await getBlockHash(api, blockId)
+  let totalValidators = (await api.query.staking.snapshotValidators.at(
+    hash,
+  )) as Option<Vec<AccountId>>
+  return !totalValidators.isEmpty
+}
+
+const updateEra = async (api: Api, eraId: number, blockId: number) => {
+
+  let dbEra = await Era.findByPk(eraId)
+  if (dbEra.actives !== null) {
+    console.log(`[Joystream] Era ${eraId} contains info, skipping update...`)
+    return
   }
+
+  const hash = await getBlockHash(api, blockId)
+  let totalValidators = (await api.query.staking.snapshotValidators.at(
+    hash,
+  )) as Option<Vec<AccountId>>
+
+  console.log(`[Joystream] Processing era ${eraId}`)
+
+  const totalNrValidators = totalValidators.unwrap().length
+  dbEra.maxSlots = Number(
+    (await api.query.staking.validatorCount.at(hash)).toString(),
+  )
+  dbEra.actives = Math.min(dbEra.maxSlots, totalNrValidators)
+  dbEra.waiting =
+    totalNrValidators > dbEra.maxSlots ? totalNrValidators - dbEra.maxSlots : 0
+
+  const chainTimestamp = (await api.query.timestamp.now.at(hash)) as Moment
+  dbEra.timestamp = new Date(chainTimestamp.toNumber())
+
+  dbEra.stake = await api.query.staking.erasTotalStake.at(hash, eraId)
+
+  await dbEra.save();
 }
 
 const validatorStatus = async (api: Api, blockId: number) => {
@@ -315,7 +299,7 @@ const validatorStatus = async (api: Api, blockId: number) => {
 const getAccountAtBlock = (
   api: Api,
   hash: string,
-  account: string
+  account: string,
 ): Promise<AccountInfo> => api.query.system.account.at(hash, account)
 
 // TODO when to cal
@@ -523,12 +507,12 @@ const fetchCouncil = async (api: Api, round: number) => {
                   stake: Number(stake),
                   consulId: consul.id,
                   memberId: id,
-                })
-              )
-            )
-          )
-        )
-      )
+                }),
+              ),
+            ),
+          ),
+        ),
+      ),
     )
   } catch (e) {
     console.error(`Failed to save council ${round}`, e)
@@ -556,7 +540,7 @@ const fetchProposalPosts = async (api: Api, max: number) => {
     fetching = `proposal posts ${threadId} ${postId}`
     const post = await api.query.proposalsDiscussion.postThreadIdByPostId(
       threadId,
-      postId
+      postId,
     )
     if (post.text.length) {
       console.log(postId, threadId, post.text.toHuman())
@@ -565,13 +549,18 @@ const fetchProposalPosts = async (api: Api, max: number) => {
   }
 }
 
-const findCouncilAtBlock = (api: Api, block: number) =>
-  Council.findOne({
+const findCouncilAtBlock = (api: Api, block: number) => {
+  if (!block) {
+    console.error(`[findCouncilAtBlock] empty block`)
+    return
+  }
+  return Council.findOne({
     where: {
       start: { [Op.lte]: block },
       end: { [Op.gte]: block - VOTINGDURATION },
     },
   })
+}
 
 const fetchProposalVotes = async (api: Api, proposal: ProposalDetail) => {
   if (!proposal) return console.error(`[fetchProposalVotes] empty proposal`)
@@ -589,7 +578,7 @@ const fetchProposalVotes = async (api: Api, proposal: ProposalDetail) => {
       where: { councilRound: { [Op.or]: councils } },
     })
     consuls.map(({ id, memberId }: any) =>
-      fetchProposalVoteByConsul(api, proposal.id, id, memberId)
+      fetchProposalVoteByConsul(api, proposal.id, id, memberId),
     )
   } catch (e) {
     console.log(`failed to fetch votes of proposal ${proposal.id}`, e)
@@ -600,7 +589,7 @@ const fetchProposalVoteByConsul = async (
   api: Api,
   proposalId: number,
   consulId: number,
-  memberId: number
+  memberId: number,
 ): Promise<any> => {
   fetching = `vote by ${consulId} for proposal ${proposalId}`
   const exists = await ProposalVote.findOne({
@@ -621,7 +610,7 @@ const fetchProposalVoteByConsul = async (
 // accounts
 const fetchMemberByAccount = async (
   api: Api,
-  account: string
+  account: string,
 ): Promise<MemberType | undefined> => {
   if (!account) {
     console.error(`fetchMemberByAccount called without account`)
@@ -635,7 +624,7 @@ const fetchMemberByAccount = async (
 
 const fetchMember = async (
   api: Api,
-  id: number
+  id: number,
 ): Promise<MemberType | undefined> => {
   if (id <= 0) return
   const exists = await Member.findByPk(+id)