backend: queueing

package.json

@@ -1,9 +1,8 @@
 {
   "name": "jsstats",
   "version": "0.0.1",
-  "private": true,
+    "private": true,
   "dependencies": {
-    "@joystream/types": "^0.14.0",
     "axios": "^0.21.1",
     "bootstrap": "^4.3.1",
     "chalk": "^4.1.0",
@@ -54,8 +53,13 @@
     ]
   },
   "devDependencies": {
+    "@joystream/types": "^0.14.0",
     "@types/express": "^4.17.11",
+    "@types/morgan": "^1.9.2",
     "@types/node": "^14.14.31",
+    "@types/node-fetch": "^2.5.8",
+    "@types/socket.io": "^2.1.13",
+    "discord.js": "^12.5.1",
     "ts-node": "^9.1.1",
     "typescript": "^4.1.5"
   }

server/db/index.ts

@@ -2,4 +2,4 @@ import db from './db'
 
 require('./models')
 
-module.exports = db
+export default db

server/db/models/block.ts

@@ -8,7 +8,6 @@ const Block = db.define('block', {
   },
   timestamp: DataTypes.DATE,
   blocktime: DataTypes.INTEGER,
-  author: DataTypes.STRING,
 })
 
 export default Block

server/db/models/category.ts

@@ -6,8 +6,9 @@ const Category = db.define('category', {
     type: DataTypes.INTEGER,
     primaryKey: true,
   },
-  title: DataTypes.STRING,
-  description: DataTypes.STRING,
+  createdAt: DataTypes.INTEGER,
+  title: DataTypes.TEXT,
+  description: DataTypes.TEXT,
   position: DataTypes.INTEGER,
   deleted: DataTypes.BOOLEAN,
   archived: DataTypes.BOOLEAN,

server/db/models/channel.ts

@@ -6,9 +6,16 @@ const Channel = db.define('channel', {
     type: DataTypes.INTEGER,
     primaryKey: true,
   },
-  timestamp: DataTypes.DATE,
-  blocktime: DataTypes.INTEGER,
-  author: DataTypes.STRING,
+  createdAt: DataTypes.INTEGER,
+  handle: DataTypes.STRING,
+  title: DataTypes.STRING,
+  description: DataTypes.TEXT,
+  avatar: DataTypes.TEXT,
+  banner: DataTypes.TEXT,
+  content: DataTypes.STRING,
+  curation: DataTypes.STRING,
+  principal: DataTypes.INTEGER,
+  publicationStatus: DataTypes.BOOLEAN,
 })
 
 export default Channel

server/db/models/council.ts

@@ -6,9 +6,8 @@ const Council = db.define('council', {
     type: DataTypes.INTEGER,
     primaryKey: true,
   },
-  timestamp: DataTypes.DATE,
-  blocktime: DataTypes.INTEGER,
-  author: DataTypes.STRING,
+  block: DataTypes.INTEGER,
+  round: DataTypes.INTEGER,
 })
 
 export default Council

server/db/models/index.ts

@@ -3,16 +3,24 @@ import Channel from './channel'
 import Council from './council'
 import Proposal from './proposal'
 import Member from './member'
-
 import Category from './category'
 import Thread from './thread'
 import Post from './post'
 
+Block.belongsTo(Member, { as: 'author' })
+
+Council.hasMany(Member, { as: 'seat' })
+
+Channel.belongsTo(Member, { as: 'owner' })
+
 Category.hasMany(Thread)
 Category.belongsTo(Member, { as: 'moderator' })
+
 Thread.belongsTo(Category)
 Thread.belongsTo(Member, { as: 'author' })
+Thread.belongsTo(Member, { as: 'moderator' })
 Thread.hasMany(Post)
+
 Post.belongsTo(Thread)
 Post.belongsTo(Member, { as: 'author' })
 

server/db/models/member.ts

@@ -6,9 +6,10 @@ const Member = db.define('member', {
     type: DataTypes.INTEGER,
     primaryKey: true,
   },
-  timestamp: DataTypes.DATE,
-  blocktime: DataTypes.INTEGER,
-  author: DataTypes.STRING,
+  createdAt: DataTypes.INTEGER,
+  account: DataTypes.STRING,
+  handle: DataTypes.STRING,
+  about: DataTypes.TEXT,
 })
 
 export default Member

server/db/models/post.ts

@@ -6,8 +6,8 @@ const Post = db.define('post', {
     type: DataTypes.INTEGER,
     primaryKey: true,
   },
-  text: DataTypes.STRING,
-  createdAt: DataTypes.DATE,
+  text: DataTypes.TEXT,
+  createdAt: DataTypes.INTEGER,
 })
 
 export default Post

server/db/models/proposal.ts

@@ -6,8 +6,8 @@ const Proposal = db.define('proposal', {
     type: DataTypes.INTEGER,
     primaryKey: true,
   },
-  createdAt: DataTypes.DATE,
-  finalizedAt: DataTypes.DATE,
+  createdAt: DataTypes.INTEGER,
+  finalizedAt: DataTypes.INTEGER,
   title: DataTypes.STRING,
   type: DataTypes.STRING,
   stage: DataTypes.STRING,

server/db/models/thread.ts

@@ -6,9 +6,9 @@ const Thread = db.define('thread', {
     type: DataTypes.INTEGER,
     primaryKey: true,
   },
-  title: DataTypes.STRING,
+  createdAt: DataTypes.INTEGER,
+  title: DataTypes.TEXT,
   nrInCategory: DataTypes.INTEGER,
-  moderation: DataTypes.STRING  
 })
 
 export default Thread

server/db/seed.ts

@@ -10,7 +10,7 @@ import {
   Thread,
 } from './models'
 
-const blocks = [] //require('../../blocks.json')
+const blocks :any[]= [] //require('../../blocks.json')
 
 async function runSeed() {
   await db.sync({ force: true })

server/index.ts

@@ -1,12 +1,21 @@
-const db = require('./db')
+import express from 'express'
+import path from 'path'
+import morgan from 'morgan'
+import socketio from 'socket.io'
+
+import db from './db'
+//const db = require('./db')
 const pg = require('pg')
 delete pg.native
 
-const express = require('express')
+const PORT: number = process.env.PORT ? +process.env.PORT : 3500
+
 const app = express()
-const path = require('path')
-const morgan = require('morgan')
-const socketio = require('socket.io')
+const server = app.listen((PORT: number) => {
+  console.log(`[Express] Listening on port ${PORT}`)
+})
+const io: any = socketio(server)
+require('./socket')(io)
 
 //const cors = require("cors");
 //const passport = require('passport')
@@ -15,16 +24,6 @@ const socketio = require('socket.io')
 //const SequelizeStore = require('connect-session-sequelize')(session.Store)
 //const sessionStore = new SequelizeStore({ db })
 
-const PORT = process.env.PORT || 3500
-//const URL = ["http://localhost:3050"];
-
-const server = app.listen(PORT, () => {
-  console.log(`[Express] Listening on port ${PORT}`)
-})
-
-const io = socketio(server)
-require('./socket')(io)
-
 app.use(morgan('dev'))
 //app.use(cors({ credentials: true, origin: URL }))
 // passport.use(
@@ -100,13 +99,5 @@ app.use((err: any, req: any, res: any, next: any) => {
   next()
 })
 
-const startListening = () => {}
-
-const startApp = async () => {
-  //await sessionStore.sync();
-  await startListening()
-}
-
-startApp()
-
-module.exports = {}
+//module.exports = {}
+//export {}

server/joystream/index.ts

@@ -1,24 +1,77 @@
-const { Block } = require('../db/models')
+import {
+  Block,
+  Channel,
+  Council,
+  Proposal,
+  Category,
+  Post,
+  Thread,
+  Member,
+} from '../db/models'
+const models: { [key: string]: any } = {
+  channel: Channel,
+  proposal: Proposal,
+  category: Category,
+  thread: Thread,
+  post: Post,
+  block: Block,
+  council: Council,
+  member: Member,
+}
 
-const get = require('./lib/getters')
-const axios = require('axios')
-const moment = require('moment')
+import * as get from './lib/getters'
+import axios from 'axios'
+import moment from 'moment'
 
 import { VoteKind } from '@joystream/types/proposals'
 import {
   Api,
   Handles,
   IState,
-  Member,
-  Category,
-  Channel,
-  Post,
+  MemberType,
+  CategoryType,
+  ChannelType,
+  PostType,
   Seat,
-  Thread,
+  ThreadType,
+  CouncilType,
   ProposalDetail,
   Status,
 } from '../types'
 
+// queuing
+let lastUpdate = 0
+const queue: any[] = []
+let inProgress = false
+const enqueue = (fn: any) => {
+  queue.push(fn)
+  processNext()
+}
+const processNext = async () => {
+  if (inProgress) return
+  inProgress = true
+  const task = queue.pop()
+  if (task) await task()
+
+  inProgress = false
+  //processNext()
+  //return queue.length
+}
+
+const save = async (model: any, data: any) => {
+  const Model = models[model]
+  try {
+    const exists = await Model.findByPk(data.id)
+    if (exists) return exists.update(data)
+  } catch (e) {}
+  //console.debug(`saving ${data.id}`, `queued tasks: ${queue.length}`)
+  try {
+    return Model.create(data)
+  } catch (e) {
+    console.warn(`Failed to save ${Model}`, e.message)
+  }
+}
+
 const addBlock = async (
   api: Api,
   io: any,
@@ -26,69 +79,58 @@ const addBlock = async (
   status: Status
 ): Promise<Status> => {
   const id = +header.number
-  const exists = await Block.findByPk(id)
-  if (exists) return status
-
-  const timestamp = (await api.query.timestamp.now()).toNumber()
-
   const last = await Block.findByPk(id - 1)
-
-  const blocktime = last ? timestamp - last.timestamp : 6000
   const author = header.author?.toString()
-
-  const block = await Block.create({ id, timestamp, blocktime, author })
-  console.log('[Joystream] block', block.id, block.blocktime, block.author)
+  const member = await fetchMemberByAccount(api, author)
+  const timestamp = moment.utc(await api.query.timestamp.now()).valueOf()
+  const blocktime = last ? timestamp - last.timestamp : 6000
+  const block = await save('block', { id, timestamp, blocktime })
+  if (member && member.id) block.setAuthor(member)
   io.emit('block', block)
-  updateAll(api, io, status)
-  return status
+
+  const handle = member ? member.handle : author
+  const queued = `(queued: ${queue.length})`
+  console.log(`[Joystream] block ${block.id} ${handle} ${queued}`)
+  return updateStatus(api, io, status)
 }
 
-const processEvents = (api: Api, blockHash) => {
+const processEvents = (api: Api, blockHash: string) => {
   const blockEvents = api.query.system.events.at(blockHash)
   // TODO as Vec<EventRecord>
-  let transfers = blockEvents.filter((event) => {
+  let transfers = blockEvents.filter((event: any) => {
     return event.section == 'balances' && event.method == 'Transfer'
   })
-  let validatorRewards = blockEvents.filter((event) => {
+  let validatorRewards = blockEvents.filter((event: any) => {
     return event.section == 'staking' && event.method == 'Reward'
   })
 }
 
-// from frontend
-const updateAll = async (api: Api, io: any, status: any) => {
-  const proposalCount = await get.proposalCount(api)
-  if (proposalCount > status.proposalCount) {
-    fetchProposal(api, proposalCount)
-    status.proposalCount = proposalCount
-  }
-
-  const currentChannel = await get.currentChannelId(api)
-  if (currentChannel > status.lastChannel)
-    status.lastChannel = await fetchChannels(api, currentChannel)
-
-  const currentCategory = await get.currentCategoryId(api)
-  if (currentCategory > status.lastCategory)
-    status.lastCategory = await fetchCategories(api, currentCategory)
-
-  const currentPost = await get.currentPostId(api)
-  if (currentPost > status.lastPost)
-    status.lastPost = await fetchPosts(api, currentPost)
-
-  const currentThread = await get.currentThreadId(api)
-  if (currentThread > status.lastThread)
-    status.lastThread = await fetchThreads(api, currentThread)
-
-  const postCount = await api.query.proposalsDiscussion.postCount()
-  // TODO save proposalComments: Number(postCount)
-
-  const currentEra = Number(await api.query.staking.currentEra())
-  if (currentEra > status.era) {
-    status.era = currentEra
+const updateStatus = async (api: Api, io: any, status: any) => {
+  const era = Number(await api.query.staking.currentEra())
+  if (era > status.era) {
     fetchStakes(api, status.era, status.validators)
     fetchLastReward(api, status.era - 1)
-  } else if (status.lastReward === 0) fetchLastReward(api, currentEra)
+  } else if (status.lastReward === 0) fetchLastReward(api, era)
+  fetchEraRewardPoints(api, Number(era))
 
-  fetchEraRewardPoints(api, Number(status.era))
+  //const postCount = await api.query.proposalsDiscussion.postCount()
+  // TODO save proposalComments: Number(postCount)
+
+  const now: number = moment().valueOf()
+  if (lastUpdate + 60000 > now) return status
+  //console.log(`updating status`, lastUpdate)
+  lastUpdate = now
+  processNext()
+  return {
+    members: await fetchMembers(api),
+    categories: await fetchCategories(api),
+    threads: await fetchThreads(api),
+    proposals: await fetchProposals(api),
+    channels: await fetchChannels(api),
+    posts: await fetchPosts(api),
+    queued: queue.length,
+    era,
+  }
 }
 
 const fetchLastReward = async (api: Api, era: number) => {
@@ -106,192 +148,226 @@ const fetchTokenomics = async () => {
   // TODO save 'tokenomics', data
 }
 
-const fetchChannels = async (api: Api, lastId: number) => {
-  const channels = [] // TOOD await Channel.findAll()
-  for (let id = lastId; id > 0; id--) {
-    if (channels.find((c) => c.id === id)) continue
-    console.debug(`Fetching channel ${id}`)
-    const data = await api.query.contentWorkingGroup.channelById(id)
-
-    const handle = String(data.handle)
-    const title = String(data.title)
-    const description = String(data.description)
-    const avatar = String(data.avatar)
-    const banner = String(data.banner)
-    const content = String(data.content)
-    const ownerId = Number(data.owner)
-    const accountId = String(data.role_account)
-    const publicationStatus =
-      data.publication_status === 'Public' ? true : false
-    const curation = String(data.curation_status)
-    const createdAt = data.created
-    const principal = Number(data.principal_id)
-
-    const channel = {
-      id,
-      handle,
-      title,
-      description,
-      avatar,
-      banner,
-      content,
-      ownerId,
-      accountId,
-      publicationStatus,
-      curation,
-      createdAt,
-      principal,
-    }
-    // TODO Channel.create(channel)
-  }
+const fetchChannels = async (api: Api) => {
+  const lastId = await get.currentChannelId(api)
+  for (let id = lastId; id > 0; id--) enqueue(() => fetchChannel(api, id))
   return lastId
 }
+const fetchChannel = async (api: Api, id: number) => {
+  const exists = await Channel.findByPk(id)
+  if (exists) return exists
 
-const fetchCategories = async (api: Api, lastId: number) => {
-  const categories = [] // TODO await Category.findAll()
-  for (let id = lastId; id > 0; id--) {
-    if (categories.find((c) => c.id === id)) continue
-    console.debug(`fetching category ${id}`)
-    const data = await api.query.forum.categoryById(id)
-
-    const threadId = Number(data.thread_id)
-    const title = String(data.title)
-    const description = String(data.description)
-    const createdAt = Number(data.created_at.block)
-    const deleted = data.deleted
-    const archived = data.archived
-    const subcategories = Number(data.num_direct_subcategories)
-    const moderatedThreads = Number(data.num_direct_moderated_threads)
-    const unmoderatedThreads = Number(data.num_direct_unmoderated_threads)
-    const position = Number(data.position_in_parent_category)
-    const moderatorId = String(data.moderator_id)
-
-    const category = {
-      id,
-      threadId,
-      title,
-      description,
-      createdAt,
-      deleted,
-      archived,
-      subcategories,
-      moderatedThreads,
-      unmoderatedThreads,
-      position,
-      moderatorId,
-    }
-    //TODO Category.create(
+  console.debug(`Fetching channel ${id}`)
+  const data = await api.query.contentWorkingGroup.channelById(id)
+
+  const handle = String(data.handle)
+  const title = String(data.title)
+  const description = String(data.description)
+  const avatar = String(data.avatar)
+  const banner = String(data.banner)
+  const content = String(data.content)
+  const ownerId = Number(data.owner)
+  const accountId = String(data.role_account)
+  const publicationStatus = data.publication_status === 'Public' ? true : false
+  const curation = String(data.curation_status)
+  const createdAt = +data.created
+  const principal = Number(data.principal_id)
+
+  const channel = {
+    id,
+    handle,
+    title,
+    description,
+    avatar,
+    banner,
+    content,
+    publicationStatus,
+    curation,
+    createdAt,
+    principal,
   }
+  const chan = await save('channel', channel)
+  const owner = await fetchMember(api, ownerId)
+  chan.setOwner(owner)
+}
+
+const fetchCategories = async (api: Api) => {
+  const lastId = await get.currentCategoryId(api)
+  for (let id = lastId; id > 0; id--) enqueue(() => fetchCategory(api, id))
   return lastId
 }
+const fetchCategory = async (api: Api, id: number) => {
+  const exists = await Category.findByPk(id)
+  if (exists) return exists
 
-const fetchPosts = async (api: Api, lastId: number) => {
-  const posts = [] // TODO Post.findAll()
-  for (let id = lastId; id > 0; id--) {
-    if (posts.find((p) => p.id === id)) continue
-    console.debug(`fetching post ${id}`)
-    const data = await api.query.forum.postById(id)
+  console.debug(`fetching category ${id}`)
+  const data = await api.query.forum.categoryById(id)
+
+  const threadId = +data.thread_id
+  const title = String(data.title)
+  const description = String(data.description)
+  const createdAt = +data.created_at.block
+  const deleted = data.deleted
+  const archived = data.archived
+  const subcategories = Number(data.num_direct_subcategories)
+  const moderatedThreads = Number(data.num_direct_moderated_threads)
+  const unmoderatedThreads = Number(data.num_direct_unmoderated_threads)
+  const position = +data.position_in_parent_category // TODO sometimes NaN
+  const moderator: string = String(data.moderator_id) // account
+
+  const cat = {
+    id,
+    title,
+    description,
+    createdAt,
+    deleted,
+    archived,
+    subcategories,
+    moderatedThreads,
+    unmoderatedThreads,
+    //position,
+  }
+  const category = await save('category', cat)
+  const mod = await fetchMemberByAccount(api, moderator)
+  if (mod) category.setModerator(mod)
+  return category
+}
 
-    const threadId = Number(data.thread_id)
-    const text = data.current_text
-    //const moderation = data.moderation;
-    //const history = data.text_change_history;
-    //const createdAt = moment(data.created_at);
-    const createdAt = data.created_at
-    const authorId = String(data.author_id)
+const fetchPosts = async (api: Api) => {
+  const lastId = await get.currentPostId(api)
+  for (let id = lastId; id > 0; id--) enqueue(() => fetchPost(api, id))
+  return lastId
+}
+const fetchPost = async (api: Api, id: number) => {
+  const exists = await Post.findByPk(id)
+  if (exists) return exists
 
-    // TODO Post.create({ id, threadId, text, authorId, createdAt })
-  }
+  console.debug(`fetching post ${id}`)
+  const data = await api.query.forum.postById(id)
+
+  const threadId = Number(data.thread_id)
+  const text = data.current_text
+  const moderation = data.moderation
+  //const history = data.text_change_history;
+  const createdAt = data.created_at.block
+  const author: string = String(data.author_id)
+
+  const post = await save('post', { id, text, createdAt })
+  const thread = await fetchThread(api, threadId)
+  if (thread) post.setThread(thread)
+  const member = await fetchMemberByAccount(api, author)
+  if (member) post.setAuthor(member)
+  const mod = await fetchMemberByAccount(api, moderation)
+  return post
+}
+
+const fetchThreads = async (api: Api) => {
+  const lastId = await get.currentThreadId(api)
+  for (let id = lastId; id > 0; id--) enqueue(() => fetchThread(api, id))
   return lastId
 }
+const fetchThread = async (api: Api, id: number) => {
+  const exists = await Thread.findByPk(id)
+  if (exists) return exists
 
-const fetchThreads = async (api: Api, lastId: number) => {
-  const threads = [] //TODO Thread.findAll()
-  for (let id = lastId; id > 0; id--) {
-    if (threads.find((t) => t.id === id)) continue
-    console.debug(`fetching thread ${id}`)
-    const data = await api.query.forum.threadById(id)
-
-    const title = String(data.title)
-    const categoryId = Number(data.category_id)
-    const nrInCategory = Number(data.nr_in_category)
-    const moderation = data.moderation
-    const createdAt = String(data.created_at.block)
-    const authorId = String(data.author_id)
-
-    const thread = {
-      id,
-      title,
-      categoryId,
-      nrInCategory,
-      moderation,
-      createdAt,
-      authorId,
-    }
-    // TODO Thread.create(
+  console.debug(`fetching thread ${id}`)
+  const data = await api.query.forum.threadById(id)
+
+  const title = String(data.title)
+  const categoryId = Number(data.category_id)
+  const nrInCategory = Number(data.nr_in_category)
+  const moderation = data.moderation
+  const createdAt = +data.created_at.block
+  const account = String(data.author_id)
+
+  const thread = await save('thread', { id, title, nrInCategory, createdAt })
+  const category = await fetchCategory(api, categoryId)
+  if (category) thread.setCategory(category)
+  const author = await fetchMemberByAccount(api, account)
+  if (author) thread.setAuthor(author)
+  if (moderation) {
+    /* TODO
+  Error: Invalid value ModerationAction(3) [Map] {
+[1]   'moderated_at' => BlockAndTime(2) [Map] {
+[1]     'block' => <BN: 4f4ff>,
+[1]     'time' => <BN: 17526e65a40>,
+[1]     registry: TypeRegistry {},
+[1]     block: [Getter],
+[1]     time: [Getter],
+[1]     typeDefs: { block: [Function: U32], time: [Function: U64] }
+[1]   },
+[1]   'moderator_id'
+[1]   'rationale' => [String (Text): 'Irrelevant as posted in another thread.'] {
+*/
+    //const mod = await fetchMemberByAccount(api, moderation)
+    //if (mod) thread.setModeration(mod)
   }
-  return lastId
+  return thread
 }
 
-const fetchCouncils = async (api: Api, status: any) => {
-  let councils = [] // await Council.findAll()
+const fetchCouncils = async (api: Api, lastBlock: number) => {
+  const round = await api.query.councilElection.round()
+  let councils: CouncilType[] = await Council.findAll()
   const cycle = 201600
 
-  for (let round = 0; round < status.round; round++) {
+  for (let round = 0; round < round; round++) {
     const block = 57601 + round * cycle
-    if (councils[round] || block > status.block) continue
-
-    console.debug(`Fetching council at block ${block}`)
-    const blockHash = await api.rpc.chain.getBlockHash(block)
-    if (!blockHash) continue
-
-    // TODO Council.create(await api.query.council.activeCouncil.at(blockHash))
+    if (councils.find((c) => c.round === round) || block > lastBlock) continue
+    //enqueue(() => fetchCouncil(api, block))
   }
 }
 
-const fetchProposals = async (api: Api) => {
-  const proposalCount = await get.proposalCount(api)
-  for (let i = proposalCount; i > 0; i--) fetchProposal(api, i)
+const fetchCouncil = async (api: Api, block: number) => {
+  console.debug(`Fetching council at block ${block}`)
+  const blockHash = await api.rpc.chain.getBlockHash(block)
+  if (!blockHash)
+    return console.error(`Error: empty blockHash fetchCouncil ${block}`)
+  const council = await api.query.council.activeCouncil.at(blockHash)
+  return save('council', council)
 }
 
+const fetchProposals = async (api: Api) => {
+  const lastId = await get.proposalCount(api)
+  for (let i = lastId; i > 0; i--) enqueue(() => fetchProposal(api, i))
+}
 const fetchProposal = async (api: Api, id: number) => {
-  const exists = null // TODO await Proposa.findByPk(id)
+  const exists = await Proposal.findByPk(id)
+  if (exists) return exists
 
-  if (exists && exists.stage === 'Finalized')
-    if (exists.votesByAccount && exists.votesByAccount.length) return
-    else return fetchVotesPerProposal(api, exists)
+  //if (exists && exists.stage === 'Finalized')
+  //if (exists.votesByAccount && exists.votesByAccount.length) return
+  //else return //TODO fetchVotesPerProposal(api, exists)
 
   console.debug(`Fetching proposal ${id}`)
   const proposal = await get.proposalDetail(api, id)
-  //TODO Proposal.create(proposal)
-  fetchVotesPerProposal(api, proposal)
+  save('proposal', proposal)
+  //TODO fetchVotesPerProposal(api, proposal)
 }
 
 const fetchVotesPerProposal = async (api: Api, proposal: ProposalDetail) => {
-  const { votesByAccount } = proposal
-  const proposals = [] // TODO await Proposal.findAll()
-  const councils = [] // TODO await Council.findAll()
+  if (proposal.votesByAccount && proposal.votesByAccount.length) return
 
-  if (votesByAccount && votesByAccount.length) return
+  const proposals = await Proposal.findAll()
+  const councils = await Council.findAll()
 
   console.debug(`Fetching proposal votes (${proposal.id})`)
-  let members = []
-  councils.map((seats) =>
+  let members: MemberType[] = []
+  councils.map((seats: Seat[]) =>
     seats.forEach(async (seat: Seat) => {
       if (members.find((member) => member.account === seat.member)) return
-      const member = null // TODO await Member.findOne({ account: seat.member })
+      const member = await Member.findOne({ where: { account: seat.member } })
       member && members.push(member)
     })
   )
 
   const { id } = proposal
-  proposal.votesByAccount = await Promise.all(
+  const votesByAccount = await Promise.all(
     members.map(async (member) => {
       const vote = await fetchVoteByProposalByVoter(api, id, member.id)
       return { vote, handle: member.handle }
     })
   )
-  // TODO save proposal.votesByAccount
+  Proposal.findByPk(id).then((p: any) => p.update({ votesByAccount }))
 }
 
 const fetchVoteByProposalByVoter = async (
@@ -338,7 +414,7 @@ const fetchValidators = async (api: Api) => {
 const fetchStakes = async (api: Api, era: number, validators: string[]) => {
   // TODO staking.bondedEras: Vec<(EraIndex,SessionIndex)>
   console.debug(`fetching stakes`)
-  const stashes = [] // TODO Stash.findAll()
+  const stashes: any[] = [] // TODO await Stash.findAll()
   if (!stashes) return
   stashes.forEach(async (validator: string) => {
     try {
@@ -362,38 +438,36 @@ const fetchEraRewardPoints = async (api: Api, era: number) => {
 }
 
 // accounts
-const fetchMembers = async (api: Api, lastId: number) => {
-  for (let id = lastId; id > 0; id--) {
-    fetchMember(api, id)
-  }
+const fetchMembers = async (api: Api) => {
+  const lastId = await api.query.members.nextMemberId()
+  for (let id = lastId - 1; id > 0; id--) enqueue(() => fetchMember(api, id))
+  return lastId
 }
 
 const fetchMemberByAccount = async (
   api: Api,
   account: string
-): Promise<Member> => {
-  const exists = null // TODO await Member.findOne({account}
+): Promise<MemberType | undefined> => {
+  const exists = await Member.findOne({ where: { account } })
   if (exists) return exists
 
-  const id = await get.memberIdByAccount(api, account)
-  if (!id)
-    return { id: -1, handle: `unknown`, account, about: ``, registeredAt: 0 }
-  // TODO return member
+  const id: number = Number(await get.memberIdByAccount(api, account))
+  return id ? fetchMember(api, id) : undefined
 }
 
-const fetchMember = async (api: Api, id: number): Promise<Member> => {
-  const exists = null // TODO await Member.findOne({id}
-  if (exists) return exists
-
-  console.debug(`Fetching member ${id}`)
+const fetchMember = async (api: Api, id: number): Promise<MemberType> => {
+  try {
+    const exists = await Member.findByPk(id)
+    if (exists) return exists
+  } catch (e) {
+    console.debug(`Fetching member ${id}`)
+  }
   const membership = await get.membership(api, id)
-
   const handle = String(membership.handle)
   const account = String(membership.root_account)
   const about = String(membership.about)
-  const registeredAt = Number(membership.registered_at_block)
-  const member = null // TODO await Member.create({ id, handle, account, registeredAt, about })
-  return member
+  const createdAt = +membership.registered_at_block
+  return save('member', { id, handle, createdAt, about })
 }
 
 const fetchReports = () => {
@@ -428,7 +502,8 @@ const fetchGithubDir = async (url: string) => {
       const match = o.name.match(/^(.+)\.md$/)
       const name = match ? match[1] : o.name
       if (o.type === 'file') {
-        // TODO save await fetchGithubFile(o.download_url)
+        const file = await fetchGithubFile(o.download_url)
+        // TODO save file
       } else fetchGithubDir(o.url)
     }
   )

server/joystream/lib/announcements.ts

@@ -1,4 +1,4 @@
-import { Api, Council, ProposalDetail, Proposals, Summary } from "../../types";
+import { Api, ProposalDetail, Proposals, Summary } from "../../types";
 import { BlockNumber } from "@polkadot/types/interfaces";
 import { Channel } from "@joystream/types/augment";
 import { Category, Thread, Post } from "@joystream/types/forum";
@@ -126,53 +126,6 @@ export const threads = async (
   return current;
 };
 
-// announce latest proposals
-export const proposals = async (
-  api: Api,
-  prop: Proposals,
-  sendMessage: (msg: string) => void
-): Promise<Proposals> => {
-  let { current, last, active, executing } = prop;
-
-  for (let id: number = +last + 1; id <= current; id++) {
-    const proposal: ProposalDetail = await proposalDetail(api, id);
-    const { createdAt, message, parameters } = proposal;
-    const votingEndsAt = createdAt + +parameters.votingPeriod
-    const msg = `Proposal ${id} <b>created</b> at block ${createdAt}.\r\n${message}\r\nYou can vote until block ${votingEndsAt}.`;
-    sendMessage(msg);
-    active.push(id);
-  }
-
-  for (const id of active) {
-    const proposal: ProposalDetail = await proposalDetail(api, id);
-    const { finalizedAt, message, parameters, result, stage } = proposal;
-    if (stage === "Finalized") {
-      let label: string = result;
-      if (result === "Approved") {
-        const executed = +parameters.gracePeriod > 0 ? false : true;
-        label = executed ? "Executed" : "Finalized";
-        if (!executed) executing.push(id);
-      }
-      const msg = `Proposal ${id} <b>${label}</b> at block ${finalizedAt}.\r\n${message}`;
-      sendMessage(msg);
-      active = active.filter((a) => a !== id);
-    }
-  }
-
-  for (const id of executing) {
-    const proposal = await proposalDetail(api, id);
-    const { exec, finalizedAt, message, parameters } = proposal;
-    const execStatus = exec ? Object.keys(exec)[0] : "";
-    const label = execStatus === "Executed" ? "has been" : "failed to be";
-    const block = +finalizedAt + +parameters.gracePeriod;
-    const msg = `Proposal ${id} <b>${label} executed</b> at block ${block}.\r\n${message}`;
-    sendMessage(msg);
-    executing = executing.filter((e) => e !== id);
-  }
-
-  return { current, last: current, active, executing };
-};
-
 // heartbeat
 
 const getAverage = (array: number[]) =>

server/joystream/lib/getters.ts

@@ -1,26 +1,26 @@
-import { formatProposalMessage } from "./announcements";
-import fetch from "node-fetch";
+import { formatProposalMessage } from './announcements'
+import fetch from 'node-fetch'
 
 //types
 
-import { Api, ProposalArray, ProposalDetail } from "../../types";
+import { Api, ProposalArray, ProposalDetail } from '../../types'
 import {
   ChannelId,
   PostId,
   ProposalDetailsOf,
   ThreadId,
-} from "@joystream/types/augment";
-import { Category, CategoryId } from "@joystream/types/forum";
-import { MemberId, Membership } from "@joystream/types/members";
-import { Proposal } from "@joystream/types/proposals";
-import { AccountId } from "@polkadot/types/interfaces";
+} from '@joystream/types/augment'
+import { Category, CategoryId } from '@joystream/types/forum'
+import { MemberId, Membership } from '@joystream/types/members'
+import { Proposal } from '@joystream/types/proposals'
+import { AccountId } from '@polkadot/types/interfaces'
 
 // channel
 
 export const currentChannelId = async (api: Api): Promise<number> => {
-  const id: ChannelId = await api.query.contentWorkingGroup.nextChannelId();
-  return Number(id) - 1;
-};
+  const id: ChannelId = await api.query.contentWorkingGroup.nextChannelId()
+  return Number(id) - 1
+}
 
 // members
 
@@ -28,117 +28,115 @@ export const membership = async (
   api: Api,
   id: MemberId | number
 ): Promise<Membership> => {
-  return await api.query.members.membershipById(id);
-};
+  return await api.query.members.membershipById(id)
+}
 
 export const memberHandle = async (api: Api, id: MemberId): Promise<string> => {
-  const member: Membership = await membership(api, id);
-  return member.handle.toJSON();
-};
+  const member: Membership = await membership(api, id)
+  return member.handle.toJSON()
+}
 
 export const memberIdByAccount = async (
   api: Api,
   account: AccountId | string
 ): Promise<MemberId | number> => {
-  const ids = await api.query.members.memberIdsByRootAccountId(account);
-  return ids.length ? ids[0] : 0;
-};
+  const ids = await api.query.members.memberIdsByRootAccountId(account)
+  return ids.length ? ids[0] : 0
+}
 
 export const memberHandleByAccount = async (
   api: Api,
   account: AccountId | string
 ): Promise<string> => {
-  const id: MemberId = await api.query.members.memberIdsByRootAccountId(
-    account
-  );
-  const handle: string = await memberHandle(api, id);
-  return handle === "joystream_storage_member" ? "joystream" : handle;
-};
+  const id: MemberId = await api.query.members.memberIdsByRootAccountId(account)
+  const handle: string = await memberHandle(api, id)
+  return handle === 'joystream_storage_member' ? 'joystream' : handle
+}
 
 // forum
 
 export const categoryById = async (api: Api, id: number): Promise<Category> => {
-  const category: Category = await api.query.forum.categoryById(id);
-  return category;
-};
+  const category: Category = await api.query.forum.categoryById(id)
+  return category
+}
 
 export const currentPostId = async (api: Api): Promise<number> => {
-  const postId: PostId = await api.query.forum.nextPostId();
-  return Number(postId) - 1;
-};
+  const postId: PostId = await api.query.forum.nextPostId()
+  return Number(postId) - 1
+}
 
 export const currentThreadId = async (api: Api): Promise<number> => {
-  const threadId: ThreadId = await api.query.forum.nextThreadId();
-  return Number(threadId) - 1;
-};
+  const threadId: ThreadId = await api.query.forum.nextThreadId()
+  return Number(threadId) - 1
+}
 
 export const currentCategoryId = async (api: Api): Promise<number> => {
-  const categoryId: CategoryId = await api.query.forum.nextCategoryId();
-  return Number(categoryId) - 1;
-};
+  const categoryId: CategoryId = await api.query.forum.nextCategoryId()
+  return Number(categoryId) - 1
+}
 
 // proposals
 
 export const proposalCount = async (api: Api): Promise<number> => {
-  const proposalCount: any = await api.query.proposalsEngine.proposalCount();
-  return proposalCount.toJSON() || 0;
-};
+  const proposalCount: any = await api.query.proposalsEngine.proposalCount()
+  return proposalCount.toJSON() || 0
+}
 
 export const activeProposalCount = async (api: Api): Promise<number> => {
-  const proposalCount: number = await api.query.proposalsEngine.activeProposalCount();
-  return proposalCount || 0;
-};
+  const proposalCount: number = await api.query.proposalsEngine.activeProposalCount()
+  return proposalCount || 0
+}
 
 export const pendingProposals = async (api: Api): Promise<ProposalArray> => {
   const pending: ProposalArray = await api.query.proposalsEngine.pendingExecutionProposalIds(
     await activeProposalCount(api)
-  );
+  )
   //const pending: ProposalArray = pendingProposals.toJSON();
-  if (pending.length) console.debug("pending proposals", pending);
-  return pending;
-};
+  if (pending.length) console.debug('pending proposals', pending)
+  return pending
+}
 
 export const activeProposals = async (api: Api): Promise<ProposalArray> => {
   const active: ProposalArray = await api.query.proposalsEngine.activeProposalIds(
     await activeProposalCount(api)
-  );
+  )
   //const active: ProposalArray = result.toJSON();
-  if (active.length) console.debug("active proposals", active);
-  return active;
-};
+  if (active.length) console.debug('active proposals', active)
+  return active
+}
 
 const getProposalType = async (api: Api, id: number): Promise<string> => {
   const details: ProposalDetailsOf = await api.query.proposalsCodex.proposalDetailsByProposalId(
     id
-  );
-  const [type]: string[] = Object.getOwnPropertyNames(details.toJSON());
-  return type;
-};
+  )
+  const [type]: string[] = Object.getOwnPropertyNames(details.toJSON())
+  return type
+}
 
 export const proposalDetail = async (
   api: Api,
   id: number
 ): Promise<ProposalDetail> => {
-  const proposal: Proposal = await api.query.proposalsEngine.proposals(id);
-  const status: { [key: string]: any } = proposal.status;
-  const stage: string = status.isActive ? "Active" : "Finalized";
-  const { finalizedAt, proposalStatus } = status[`as${stage}`];
+  const proposal: Proposal = await api.query.proposalsEngine.proposals(id)
+  const status: { [key: string]: any } = proposal.status
+  const stage: string = status.isActive ? 'Active' : 'Finalized'
+  const { finalizedAt, proposalStatus } = status[`as${stage}`]
   const result: string = proposalStatus
-    ? (proposalStatus.isApproved && "Approved") ||
-      (proposalStatus.isCanceled && "Canceled") ||
-      (proposalStatus.isExpired && "Expired") ||
-      (proposalStatus.isRejected && "Rejected") ||
-      (proposalStatus.isSlashed && "Slashed") ||
-      (proposalStatus.isVetoed && "Vetoed")
-    : "Pending";
-  const exec = proposalStatus ? proposalStatus["Approved"] : null;
-
-  const { description, parameters, proposerId, votingResults } = proposal;
-  const author: string = await memberHandle(api, proposerId);
-  const title: string = proposal.title.toString();
-  const type: string = await getProposalType(api, id);
-  const args: string[] = [String(id), title, type, stage, result, author];
-  const message: string = formatProposalMessage(args);
+    ? (proposalStatus.isApproved && 'Approved') ||
+      (proposalStatus.isCanceled && 'Canceled') ||
+      (proposalStatus.isExpired && 'Expired') ||
+      (proposalStatus.isRejected && 'Rejected') ||
+      (proposalStatus.isSlashed && 'Slashed') ||
+      (proposalStatus.isVetoed && 'Vetoed')
+    : 'Pending'
+  const exec = proposalStatus ? proposalStatus['Approved'] : null
+
+  const { description, parameters, proposerId, votingResults } = proposal
+  const author: string = await memberHandle(api, proposerId)
+  const title: string = proposal.title.toString()
+  const type: string = await getProposalType(api, id)
+  const args: string[] = [String(id), title, type, stage, result, author]
+  const message: string = formatProposalMessage(args)
   const createdAt: number = Number(proposal.createdAt)
 
   return {
@@ -146,7 +144,7 @@ export const proposalDetail = async (
     title,
     createdAt,
     finalizedAt,
-    parameters,
+    parameters: JSON.stringify(parameters),
     message,
     stage,
     result,
@@ -155,26 +153,26 @@ export const proposalDetail = async (
     votes: votingResults,
     type,
     author,
-    authorId: Number(proposerId)
-  };
-};
+    authorId: Number(proposerId),
+  }
+}
 
 // storage providers
 export const providerStatus = async (domain: string): Promise<boolean> => {
   try {
-    const res = await fetch(`https://${domain}:5001/api/v0/version`);
-    return res.status >= 400 ? false : true;
+    const res = await fetch(`https://${domain}:5001/api/v0/version`)
+    return res.status >= 400 ? false : true
   } catch (e) {
-    return false;
+    return false
   }
-};
+}
 
 export const nextOpeningId = async (api: Api): Promise<number> => {
-  const id = await api.query.storageWorkingGroup.nextOpeningId();
-  return id.toJSON();
-};
+  const id = await api.query.storageWorkingGroup.nextOpeningId()
+  return id.toJSON()
+}
 
 export const nextWorkerId = async (api: Api): Promise<number> => {
-  const id = await api.query.storageWorkingGroup.nextWorkerId();
-  return id.toJSON();
-};
+  const id = await api.query.storageWorkingGroup.nextWorkerId()
+  return id.toJSON()
+}

server/socket.ts

@@ -1,27 +1,31 @@
-import { Api, Status } from './types'
+import { Api, Header, Status } from './types'
 const { types } = require('@joystream/types')
 
 const { Block } = require('./db/models')
-
 const { ApiPromise, WsProvider } = require('@polkadot/api')
+const { addBlock } = require('./joystream')
+const chalk = require('chalk')
+
 // TODO allow alternative backends
 const wsLocation = 'ws://localhost:9944'
 // 'wss://rome-rpc-endpoint.joystream.org:9944'
 
-const { addBlock } = require('./joystream')
-
-module.exports = (io) => {
+module.exports = (io: any) => {
   const handleUpstream = async (api: Api) => {
     let status: Status = {}
-    api.derive.chain.subscribeNewHeads(
-      async (header) => (status = await addBlock(api, io, header, status))
-    )
+    let lastHeader: Header
+    api.derive.chain.subscribeNewHeads(async (header: Header) => {
+      if (lastHeader && lastHeader.number === header.number)
+        return console.debug(`skipping seen block`)
+      lastHeader = header
+      status = await addBlock(api, io, header, status)
+    })
   }
 
-  io.on('connection', async (socket) => {
+  io.on('connection', async (socket: any) => {
     console.log(chalk.green(`[socket.io] Connection: ${socket.id}`))
 
-    socket.on('get blocks', async (limit) => {
+    socket.on('get blocks', async (limit: number) => {
       const blocks = await Block.findAll({ limit, order: [['id', 'DESC']] })
       socket.emit('blocks', blocks)
     })

server/tsconfig.json

@@ -1,10 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "es6",
-    "module": "commonjs",
-    "rootDir": "./",
-    "outDir": "./build",
-    "esModuleInterop": true,
-    "strict": false
-  }
-}

server/types.ts

@@ -76,11 +76,15 @@ export interface Backer {
   stake: number;
 }
 
-export interface Council {
+export interface CouncilType {
   round: number;
   last: string;
 }
 
+export interface CouncilModel {
+
+}
+
 export interface Options {
   verbose: number;
   channel: boolean;
@@ -93,7 +97,7 @@ export interface ProposalDetail {
   createdAt: number;
   finalizedAt: number;
   message: string;
-  parameters: ProposalParameters;
+  parameters: string;
   stage: any;
   result: string;
   exec: any;
@@ -127,7 +131,7 @@ export interface Proposals {
   executing: ProposalArray;
 }
 
-export interface Channel {
+export interface ChannelType {
   id: number;
   handle: string;
   title: string;
@@ -143,7 +147,7 @@ export interface Channel {
   principal: number;
 }
 
-export interface Category {
+export interface CategoryType {
   id: number;
   threadId: number;
   title: string;
@@ -158,7 +162,7 @@ export interface Category {
   moderatorId: string;
 }
 
-export interface Post {
+export interface PostType {
   id: number;
   text: string;
   threadId: number;
@@ -166,7 +170,7 @@ export interface Post {
   createdAt: { block: number; time: number };
 }
 
-export interface Thread {
+export interface ThreadType {
   id: number;
   title: string;
   categoryId: number;
@@ -176,7 +180,7 @@ export interface Thread {
   authorId: string;
 }
 
-export interface Member {
+export interface MemberType {
   account: string;
   handle: string;
   id: number;
@@ -184,10 +188,10 @@ export interface Member {
   about: string;
 }
 
-export interface Block {
-  id: number;
+export interface Header {
+  number: number;
   timestamp: number;
-  duration: number;
+  author: string
 }
 
 export interface Summary {