Merge branch 'giza_staging' into distributor-node-staging

tests/network-tests/.env

@@ -1,61 +1,62 @@
 # Address of the Joystream node.
-NODE_URL = ws://127.0.0.1:9944
+NODE_URL=ws://127.0.0.1:9944
 # Address of the Joystream query node.
-QUERY_NODE_URL = http://127.0.0.1:8081/graphql
+QUERY_NODE_URL=http://127.0.0.1:8081/graphql
 # Account which is expected to provide sufficient funds to test accounts.
-TREASURY_ACCOUNT_URI = //Alice
+TREASURY_ACCOUNT_URI=//Alice
 # Sudo Account
-SUDO_ACCOUNT_URI = //Alice
+SUDO_ACCOUNT_URI=//Alice
 # Amount of members able to buy membership in membership creation test.
-MEMBERSHIP_CREATION_N = 2
+MEMBERSHIP_CREATION_N=2
 # ID of the membership paid terms used in membership creation test.
-MEMBERSHIP_PAID_TERMS = 0
+MEMBERSHIP_PAID_TERMS=0
 # Council stake amount for first K accounts in council election test.
-COUNCIL_STAKE_GREATER_AMOUNT = 3000
+COUNCIL_STAKE_GREATER_AMOUNT=3000
 # Council stake amount for first the rest participants in council election test.
-COUNCIL_STAKE_LESSER_AMOUNT = 2000
+COUNCIL_STAKE_LESSER_AMOUNT=2000
 # Number of members with greater stake in council election test.
-COUNCIL_ELECTION_K = 2
+COUNCIL_ELECTION_K=2
 # Balance to spend using spending proposal
-SPENDING_BALANCE = 1000
+SPENDING_BALANCE=1000
 # Minting capacity increment for content working group minting capacity test.
-MINTING_CAPACITY_INCREMENT = 20
+MINTING_CAPACITY_INCREMENT=20
 # Minting capacity for council mint for spending proposal.
-COUNCIL_MINTING_CAPACITY = 100000
+COUNCIL_MINTING_CAPACITY=100000
 # Stake amount for Rome runtime upgrade proposal
-RUNTIME_UPGRADE_PROPOSAL_STAKE = 100000
+RUNTIME_UPGRADE_PROPOSAL_STAKE=100000
 # Validator count increment for Validator count test.
-VALIDATOR_COUNT_INCREMENT = 2
+VALIDATOR_COUNT_INCREMENT=2
 # Constantinople runtime path
-RUNTIME_WASM_PATH = ../../target/release/wbuild/joystream-node-runtime/joystream_node_runtime.compact.wasm
+RUNTIME_WASM_PATH=../../target/release/wbuild/joystream-node-runtime/joystream_node_runtime.compact.wasm
 # Working group size N
-WORKING_GROUP_N = 3
+WORKING_GROUP_N=3
 # Working group application stake
-WORKING_GROUP_APPLICATION_STAKE = 10
+WORKING_GROUP_APPLICATION_STAKE=10
 # Working group role stake
-WORKING_GROUP_ROLE_STAKE = 10
+WORKING_GROUP_ROLE_STAKE=10
 # Reward interval for working group tests
-LONG_REWARD_INTERVAL = 99999
+LONG_REWARD_INTERVAL=99999
 # First reward interval for working group reward test
-SHORT_FIRST_REWARD_INTERVAL = 3
+SHORT_FIRST_REWARD_INTERVAL=3
 # Reward interval for working group reward test
-SHORT_REWARD_INTERVAL = 3
+SHORT_REWARD_INTERVAL=3
 # Payout amount for working group tests
-PAYOUT_AMOUNT = 3
+PAYOUT_AMOUNT=3
 # Payout amount for leader-related proposals tests
-ALTERED_PAYOUT_AMOUNT = 7
+ALTERED_PAYOUT_AMOUNT=7
 # Mint capacity for storage working group
-STORAGE_WORKING_GROUP_MINTING_CAPACITY = 100000
+STORAGE_WORKING_GROUP_MINTING_CAPACITY=100000
 # Default unstaking period for storage working group
-STORAGE_WORKING_GROUP_UNSTAKING_PERIOD = 1
+STORAGE_WORKING_GROUP_UNSTAKING_PERIOD=1
 # Slash value for manage working group lead testing scenario
-SLASH_AMOUNT = 2
+SLASH_AMOUNT=2
 # Stake decrement amount for manage working group lead testing scenario
-STAKE_DECREMENT = 3
+STAKE_DECREMENT=3
 # Mint capacity increment value for working gorup mint capacity test
-MINT_CAPACITY_INCREMENT = 1000
-# Mini-secret or mnemonic used in SURI for deterministic key derivation
-SURI_MINI_SECRET = ""
-# The starting key id to use when running a scenario. This will allow scenario
+MINT_CAPACITY_INCREMENT=1000
 # to be able to use all accounts generated in a prior scenario run against the same chain
-START_KEY_ID = 0
+START_KEY_ID=0
+# Mini-secret or mnemonic used in SURI for deterministic key derivation
+SURI_MINI_SECRET=""
+# Storage node address to download content from
+STORAGE_NODE_URL=http://localhost:3001/asset/v0

tests/network-tests/.eslintrc.js

@@ -1,12 +1,12 @@
 module.exports = {
-    env: {
-        node: true,
-    },
+  env: {
+    node: true,
+  },
   rules: {
-      'no-async-promise-executor': 'off',
-      'no-useless-return': 'off',
-      'new-cap': 'off',
-      // Disabled because of the false positive bug: https://github.com/eslint/eslint/issues/11899
-      'require-atomic-updates': 'off',
-  }
+    'no-async-promise-executor': 'off',
+    'no-useless-return': 'off',
+    'new-cap': 'off',
+    // Disabled because of the false positive bug: https://github.com/eslint/eslint/issues/11899
+    'require-atomic-updates': 'off',
+  },
 }

tests/network-tests/node-utils.sh

@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+
+# style reference https://google.github.io/styleguide/shellguide.html
+
+set -e
+
+SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
+cd $SCRIPT_PATH
+
+# Log only to stderr
+# Only output from this script should be the container id of the node at the very end
+
+# Location that will be mounted as the /data volume in containers
+# This is where the initial members and balances files and generated chainspec files will be located.
+export DATA_PATH=${DATA_PATH:=$(pwd)/data}
+mkdir -p ${DATA_PATH}
+
+# Initial account balance for sudo account
+SUDO_INITIAL_BALANCE=${SUDO_INITIAL_BALANCE:=100000000}
+SUDO_ACCOUNT_URI=${SUDO_ACCOUNT_URI:="//Alice"}
+SUDO_ACCOUNT=$(docker run --rm --pull=always docker.io/parity/subkey:2.0.1 inspect ${SUDO_ACCOUNT_URI} --output-type json | jq .ss58Address -r)
+
+# Source of funds for all new accounts that are created in the tests.
+TREASURY_INITIAL_BALANCE=${TREASURY_INITIAL_BALANCE:=100000000}
+TREASURY_ACCOUNT_URI=${TREASURY_ACCOUNT_URI:=$SUDO_ACCOUNT_URI}
+TREASURY_ACCOUNT=$(docker run --rm --pull=always docker.io/parity/subkey:2.0.1 inspect ${TREASURY_ACCOUNT_URI} --output-type json | jq .ss58Address -r)
+
+>&2 echo "sudo account from suri: ${SUDO_ACCOUNT}"
+>&2 echo "treasury account from suri: ${TREASURY_ACCOUNT}"
+
+# Prevent joystream cli from prompting
+export AUTO_CONFIRM=true
+
+export JOYSTREAM_NODE_TAG=${RUNTIME_TAG}
+
+#######################################
+# create initial-balances.json & initial-members.json files
+# Globals:
+#   SUDO_INITIAL_BALANCES
+#   SUDO_ACCOUNT
+#   TREASURY_ACCOUNT
+#   TREASURY_INITIAL_BALANCE
+#   DATA_PATH
+# Arguments:
+#   None
+#######################################
+function create_initial_config {
+    echo "{
+  \"balances\":[
+    [\"$SUDO_ACCOUNT\", $SUDO_INITIAL_BALANCE],
+    [\"$TREASURY_ACCOUNT\", $TREASURY_INITIAL_BALANCE]
+  ]
+}" > ${DATA_PATH}/initial-balances.json
+
+    # Remember if there are initial members at genesis query-node needs to be bootstrapped
+    # or any events processed for this member will cause processor to fail.
+    if [ "${MAKE_SUDO_MEMBER}" == true ]
+    then
+	echo "
+    [{
+      \"member_id\":0,
+      \"root_account\":\"$SUDO_ACCOUNT\",
+      \"controller_account\":\"$SUDO_ACCOUNT\",
+      \"handle\":\"sudosudo\",
+      \"avatar_uri\":\"https://sudo.com/avatar.png\",
+      \"about\":\"Sudo\",
+      \"registered_at_time\":0
+    }]
+  " > ${DATA_PATH}/initial-members.json
+    else
+	echo "[]" > ${DATA_PATH}/initial-members.json
+    fi
+}
+
+#######################################
+# create human-readable chainspec file
+# Globals:
+#   SUDO_ACCOUNT
+#   DATA_PATH
+# Arguments:
+#   None
+#######################################
+function create_chainspec_file {
+    # Create a chain spec file
+    docker run --rm -v ${DATA_PATH}:/data --entrypoint ./chain-spec-builder \
+	   joystream/node:${RUNTIME_TAG} \
+	   new \
+	   --authority-seeds Alice \
+	   --sudo-account ${SUDO_ACCOUNT} \
+	   --deployment dev \
+	   --chain-spec-path /data/chain-spec.json \
+	   --initial-balances-path /data/initial-balances.json \
+	   --initial-members-path /data/initial-members.json
+}
+
+#######################################
+# convert human-readable chainspec into
+# raw chainspec
+# Globals:
+#   DATA_PATH
+# Arguments:
+#   None
+#######################################
+function convert_chainspec {
+    docker run --rm -v ${DATA_PATH}:/data joystream/node:${RUNTIME_TAG} build-spec \
+	   --raw --disable-default-bootnode \
+	   --chain /data/chain-spec.json > ${DATA_PATH}/chain-spec-raw.json
+}
+
+#######################################
+# cleanup docker logs and shuts down container
+# Globals:
+#   CONTAINER_ID
+# Arguments:
+#   None
+#######################################
+function cleanup() {
+    # if [[ -z $CONTAINER_ID ]]; then
+    # 	docker logs ${CONTAINER_ID} --tail 15
+    # fi
+    docker-compose -f ../../docker-compose.yml down -v
+    find ./assets/ -name '[A-Z0-9]*__rejectedContent.json' -delete
+    rm -rf $DATA_PATH
+}
+
+#######################################
+# Start a chain with generated chain spec
+# Globals:
+#   DATA_PATH
+# Arguments:
+#   None
+#######################################
+function start_node {
+    docker-compose -f ../../docker-compose.yml run \
+		   -d -v ${DATA_PATH}:/spec --name "${JOYSTREAM_NODE_TAG}" \
+		   -p 9944:9944 -p 9933:9933 joystream-node \
+		   --alice --validator --unsafe-ws-external --unsafe-rpc-external \
+		   --rpc-methods Unsafe --rpc-cors=all -l runtime \
+		   --chain /spec/chain-spec-raw.json \
+		   --base-path /data
+}

tests/network-tests/run-migration-tests.sh

@@ -4,97 +4,82 @@ set -e
 SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
 cd $SCRIPT_PATH
 
-# Location to store runtime WASM for runtime upgrade
-DATA_PATH=$PWD/data
-
-# The joystream/node docker image tag to start chain
-export RUNTIME=${RUNTIME:=latest}
-
 # The joystream/node docker image tag which contains WASM runtime to upgrade chain with
-TARGET_RUNTIME=${TARGET_RUNTIME:=latest}
-
-# Prevent joystream cli from prompting
-export AUTO_CONFIRM=true
-
-# Create chainspec with Alice (sudo) as member so we can use her in joystream-cli
-CONTAINER_ID=$(MAKE_SUDO_MEMBER=true ./run-test-node-docker.sh)
-
-function cleanup() {
-    docker logs ${CONTAINER_ID} --tail 15
-    docker-compose -f ../../docker-compose.yml down -v
-    rm ./assets/TestChannel__rejectedContent.json || true
-    rm ./assets/TestVideo__rejectedContent.json || true
+TARGET_RUNTIME_TAG=${TARGET_RUNTIME_TAG:=latest}
+# The joystream/node docker image tag to start the chain with
+RUNTIME_TAG=${RUNTIME_TAG:=sumer}
+# Post migration assertions by means of typescript scenarios required
+POST_MIGRATION_ASYNC_ASSERTIONS=${POST_MIGRATION_ASYNC_ASSERTIONS:=$true}
+# source common function used for node setup
+source ./node-utils.sh
+
+#######################################
+# use fork-off to generate a chainspec file with the current s
+# Globals:
+#   DATA_PATH
+# Arguments:
+#   None
+#######################################
+function fork_off_init() {
+    # chain-spec-raw already existing
+
+    if ! [[ -f ${DATA_PATH}/storage.json ]]; then
+        curl http://testnet-rpc-3-uk.joystream.org:9933 -H \
+            "Content-type: application/json" -d \
+            '{"jsonrpc":"2.0","id":1,"method":"state_getPairs","params":["0x"]}' \
+            > ${DATA_PATH}/storage.json
+    fi
+
+    if ! [[ -f ${DATA_PATH}/schema.json ]]; then
+        cp $SCRIPT_PATH/../../types/augment/all/defs.json ${DATA_PATH}/schema.json
+    fi
+
+    id=$(docker create joystream/node:${TARGET_RUNTIME_TAG})
+    docker cp $id:/joystream/runtime.compact.wasm ${DATA_PATH}/runtime.wasm
+
+    # RPC endpoint for live RUNTIME testnet 
+    WS_RPC_ENDPOINT="wss://testnet-rpc-3-uk.joystream.org" \
+        yarn workspace api-scripts tsnode-strict src/fork-off.ts
 }
 
-function pre_migration_hook() {
-  sleep 10 # needed otherwise docker image won't be ready yet
-  # Display runtime version
-  yarn workspace api-scripts tsnode-strict src/status.ts | grep Runtime
-
-  # assume older version of joystream-cli is installed globally. So we run these commands to
-  # work against older runtime. Assert it is version  `@joystream/cli/0.5.1` ?
-  joystream-cli --version
-
-  joystream-cli account:choose --address 5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY # Alice
-  echo "creating 1 channel"
-  joystream-cli content:createChannel --input=./assets/TestChannel.json --context=Member || true
-  echo "adding 1 video to the above channel"
-  joystream-cli content:createVideo -c 1 --input=./assets/TestVideo.json || true
-
-  # Confirm channel and video created successfully
-  joystream-cli content:videos 1
-  joystream-cli content:channel 1
+function export_chainspec_file_to_disk() {
+    echo "**** Initializing node database by exporting state ****"
+    # write the initial genesis state to db, in order to avoid waiting for an arbitrary amount of time 
+    docker-compose -f ../../docker-compose.yml run \
+		   -v ${DATA_PATH}:/spec joystream-node export-state \
+		   --chain /spec/chain-spec-raw.json \
+		   --base-path /data --pruning archive > ${DATA_PATH}/exported-state.json
 }
 
-function post_migration_hook() {
-  echo "*** verify existence of the 5 new groups ***"
-  yarn joystream-cli working-groups:overview --group=operationsAlpha
-  yarn joystream-cli working-groups:overview --group=operationsBeta
-  yarn joystream-cli working-groups:overview --group=operationsGamma
-  yarn joystream-cli working-groups:overview --group=curators
-  yarn joystream-cli working-groups:overview --group=distributors
-
-  echo "*** verify previously created channel and video are cleared ***"
-  # Allow a few blocks for migration to complete
-  sleep 12
-  
-  # FIXME: Howto assert these fail as expected. They should report video and channel do no exist
-  # Can we get json output to more easily parse result of query?
-  set +e
-  yarn joystream-cli content:channel 1
-  if [ $? -eq 0 ]; then
-    echo "Unexpected channel was found"
-    exit -1
-  fi
-  # This cammand doesn't give error exit code if videos not found in a channel
-  yarn joystream-cli content:videos 1
-}    
-
-trap cleanup EXIT
-
-if [ "$TARGET_RUNTIME" == "$RUNTIME" ]; then
-  echo "Not Performing a runtime upgrade."
-else
-  pre_migration_hook
-
-  # Copy new runtime wasm file from target joystream/node image
-  echo "Extracting wasm blob from target joystream/node image."
-  id=$(docker create joystream/node:${TARGET_RUNTIME})
-  docker cp $id:/joystream/runtime.compact.wasm ${DATA_PATH}
-  docker rm $id
-
-  echo "Performing runtime upgrade."
-  yarn workspace api-scripts tsnode-strict \
-    src/dev-set-runtime-code.ts -- ${DATA_PATH}/runtime.compact.wasm
-
-  echo "Runtime upgraded."
-
-  # Display runtime version
-  yarn workspace api-scripts tsnode-strict src/status.ts | grep Runtime
-
-  echo "Performing migration tests"
-
-  post_migration_hook
+# entrypoint
+function main {
+    CONTAINER_ID=""
+
+    echo "**** CREATING EMPTY CHAINSPEC ****"
+    create_initial_config
+    create_chainspec_file
+    convert_chainspec
+    echo "**** EMPTY CHAINSPEC CREATED SUCCESSFULLY ****"
+
+    # use forkoff to update chainspec with the live state + update runtime code
+    fork_off_init
+
+    export JOYSTREAM_NODE_TAG=$RUNTIME_TAG
+
+    # export chain-spec BEFORE starting the node
+    export_chainspec_file_to_disk
+    
+    echo "***** STARTING NODE WITH FORKED STATE *****"
+    CONTAINER_ID=$(start_node)
+
+    if ( $POST_MIGRATION_ASYNC_ASSERTIONS ); then
+        sleep 120
+        # verify assertion using typsecript
+        echo "***** POST MIGRATION TYPESCRIPT *****"
+        yarn workspace network-tests node-ts-strict src/scenarios/post-migration.ts
+    fi
+}
 
-  echo "Done with migrations tests"
-fi
+# main entrypoint
+main || :
+cleanup

tests/network-tests/src/Api.ts

@@ -32,7 +32,7 @@ import { FillOpeningParameters, ProposalId } from '@joystream/types/proposals'
 // import { v4 as uuid } from 'uuid'
 import { extendDebug } from './Debugger'
 import { InvertedPromise } from './InvertedPromise'
-import { VideoId } from '@joystream/types/content'
+import { VideoId, VideoCategoryId } from '@joystream/types/content'
 import { ChannelId } from '@joystream/types/common'
 import { ChannelCategoryMetadata, VideoCategoryMetadata } from '@joystream/metadata-protobuf'
 import { metadataToBytes } from '../../../cli/lib/helpers/serialization'
@@ -1790,6 +1790,18 @@ export class Api {
     return (await this.api.query.members.membershipById(memberId))?.controller_account.toString()
   }
 
+  public async getNumberOfOutstandingVideos(): Promise<number> {
+    return (await this.api.query.content.videoById.entries<VideoId>()).length
+  }
+
+  public async getNumberOfOutstandingChannels(): Promise<number> {
+    return (await this.api.query.content.channelById.entries<ChannelId>()).length
+  }
+
+  public async getNumberOfOutstandingVideoCategories(): Promise<number> {
+    return (await this.api.query.content.videoCategoryById.entries<VideoCategoryId>()).length
+  }
+
   // Create a mock channel, throws on failure
   async createMockChannel(memberId: number, memberControllerAccount?: string): Promise<ChannelId> {
     memberControllerAccount = memberControllerAccount || (await this.getMemberControllerAccount(memberId))

tests/network-tests/src/misc/postMigrationAssertionsFlow.ts

@@ -0,0 +1,68 @@
+import { assert } from 'chai'
+import { FlowProps } from '../Flow'
+import { extendDebug } from '../Debugger'
+import { Utils } from '../utils'
+
+export default async function postMigrationAssertions({ api }: FlowProps): Promise<void> {
+  const debug = extendDebug('flow:postMigrationAssertions')
+  debug('Started')
+
+  debug('Ensure migration is done')
+
+  let channelMigration = await api.query.content.channelMigration()
+  let videoMigration = await api.query.content.videoMigration()
+
+  // wait for migration to be done and checking that index do actually change
+  while (
+    channelMigration.current_id.toNumber() < channelMigration.final_id.toNumber() ||
+    videoMigration.current_id.toNumber() < videoMigration.final_id.toNumber()
+  ) {
+    // wait 6 seconds until next block is produced
+    await Utils.wait(6000)
+
+    const channelMigrationNew = await api.query.content.channelMigration()
+    const videoMigrationNew = await api.query.content.videoMigration()
+
+    // check invariant in order to prevent infinite loop
+    if (
+      channelMigrationNew.current_id.toNumber() > channelMigration.current_id.toNumber() ||
+      videoMigrationNew.current_id.toNumber() > videoMigration.current_id.toNumber()
+    ) {
+      // update migration variables
+      channelMigration = channelMigrationNew
+      videoMigration = videoMigrationNew
+    } else {
+      throw new Error('Migration status not changing')
+    }
+  }
+
+  debug('Check all new  working groups have been correctly initialized')
+
+  const wgBeta = await api.query.operationsWorkingGroupBeta.activeWorkerCount()
+  const wgGamma = await api.query.operationsWorkingGroupGamma.activeWorkerCount()
+  const wgGateway = await api.query.gatewayWorkingGroup.activeWorkerCount()
+
+  assert.equal(wgBeta.toNumber(), 0)
+  assert.equal(wgGamma.toNumber(), 0)
+  assert.equal(wgGateway.toNumber(), 0)
+
+  debug('Checking that Video, Channel, Categories  counters have not been re-set')
+
+  const nextVideoCategoryId = await api.query.content.nextVideoCategoryId()
+  const nextVideoId = await api.query.content.nextVideoId()
+  const nextChannelId = await api.query.content.nextChannelId()
+
+  assert(nextVideoCategoryId.toNumber() > 1)
+  assert(nextVideoId.toNumber() > 1)
+  assert(nextChannelId.toNumber() > 1)
+
+  debug('Checking that number of outstanding channels & videos == 0')
+
+  const numChannels = await api.getNumberOfOutstandingChannels()
+  const numVideos = await api.getNumberOfOutstandingVideos()
+
+  assert.equal(numChannels, 0)
+  assert.equal(numVideos, 0)
+
+  debug('Done')
+}

tests/network-tests/src/scenarios/post-migration.ts

@@ -0,0 +1,6 @@
+import postMigrationAssertions from '../misc/postMigrationAssertionsFlow'
+import { scenario } from '../Scenario'
+
+scenario(async ({ job }) => {
+  job('Verify post-migration chain state', postMigrationAssertions)
+})

tests/network-tests/tsconfig.json

@@ -12,7 +12,7 @@
     "resolveJsonModule": true,
     "paths": {
       "@polkadot/types/augment": ["../../types/augment-codec/augment-types.ts"],
-      "@polkadot/api/augment": [ "../../types/augment-codec/augment-api.ts"]
+      "@polkadot/api/augment": ["../../types/augment-codec/augment-api.ts"]
     }
   }
 }

utils/api-scripts/src/fork-off.ts

@@ -0,0 +1,125 @@
+import fs = require('fs')
+import path = require('path')
+import { xxhashAsHex } from '@polkadot/util-crypto'
+import { ApiPromise, WsProvider } from '@polkadot/api'
+const execSync = require('child_process').execSync
+
+// paths & env variables
+let alice = process.env.SUDO_ACCOUNT
+// bad error handling TODO: fix process.env
+let schemaPath = path.join(process.env.DATA_PATH || '', 'schema.json')
+let wasmPath = path.join(process.env.DATA_PATH || '', 'runtime.wasm') || ''
+let hexPath = path.join(process.env.DATA_PATH || '', 'runtime.hex') || ''
+let specPath = path.join(process.env.DATA_PATH || '', 'chain-spec-raw.json')
+let storagePath = path.join(process.env.DATA_PATH || '', 'storage.json')
+
+// this might not be of much use
+const provider = new WsProvider(process.env.WS_RPC_ENDPOINT || 'ws://localhost:9944')
+/**
+ * All module prefixes except those mentioned in the skippedModulesPrefix will be added to this by the script.
+ * If you want to add any past module or part of a skipped module, add the prefix here manually.
+ *
+ * Any storage value’s hex can be logged via console.log(api.query.<module>.<call>.key([...opt params])),
+ * e.g. console.log(api.query.timestamp.now.key()).
+ *
+ * If you want a map/doublemap key prefix, you can do it via .keyPrefix(),
+ * e.g. console.log(api.query.system.account.keyPrefix()).
+ *
+ * For module hashing, do it via xxhashAsHex,
+ * e.g. console.log(xxhashAsHex('System', 128)).
+ */
+let prefixes = ['0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9' /* System.Account */]
+const skippedModulesPrefix = [
+  'System',
+  'Session',
+  'Babe',
+  'Grandpa',
+  'GrandpaFinality',
+  'FinalityTracker',
+  'Authorship',
+]
+
+// Apparently not needed: To review
+// async function fixParachinStates(api: ApiPromise, chainSpec: any) {
+//     const skippedKeys = [
+//         api.query["parasScheduler"].sessionStartBlock.key()
+//     ];
+//     for (const k of skippedKeys) {
+//         delete chainSpec.genesis.raw.top[k];
+//     }
+// }
+
+async function main() {
+  // hexdump of runtime wasm binary, running it from the shell gives bad format error
+  execSync('cat ' + wasmPath + ' | hexdump -ve \'/1 "%02x"\' > ' + hexPath)
+
+  let api
+  if (!fs.existsSync(schemaPath)) {
+    console.log('Custom Schema missing, using default schema.')
+    api = await ApiPromise.create({ provider })
+  } else {
+    const types = JSON.parse(fs.readFileSync(schemaPath, 'utf8'))
+    api = await ApiPromise.create({
+      provider,
+      types,
+    })
+  }
+
+  // storage.json is guaranteed to exists
+
+  let metadata = await api.rpc.state.getMetadata()
+  // Populate the prefixes array
+  let modules = metadata.asLatest.modules
+  modules.forEach((module) => {
+    if (module.storage) {
+      if (!skippedModulesPrefix.includes(module.name.toString())) {
+        prefixes.push(xxhashAsHex(module.name.toString(), 128))
+      }
+    }
+  })
+
+  // blank starting chainspec guaranteed to exist
+
+  let storage: Storage = JSON.parse(fs.readFileSync(storagePath, 'utf8'))
+  let chainSpec = JSON.parse(fs.readFileSync(specPath, 'utf8'))
+
+  // Modify chain name and id
+  chainSpec.name = chainSpec.name + '-fork'
+  chainSpec.id = chainSpec.id + '-fork'
+  chainSpec.protocolId = chainSpec.protocolId
+
+  // Grab the items to be moved, then iterate through and insert into storage
+  storage.result
+    .filter((i) => prefixes.some((prefix) => i[0].startsWith(prefix)))
+    .forEach(([key, value]) => (chainSpec.genesis.raw.top[key] = value))
+
+  // Delete System.LastRuntimeUpgrade to ensure that the on_runtime_upgrade event is triggered
+  delete chainSpec.genesis.raw.top['0x26aa394eea5630e07c48ae0c9558cef7f9cce9c888469bb1a0dceaa129672ef8']
+
+  //    fixParachinStates(api, chainSpec);
+
+  // Set the code to the current runtime code: this replaces the set code transaction
+  chainSpec.genesis.raw.top['0x3a636f6465'] = '0x' + fs.readFileSync(hexPath, 'utf8').trim()
+
+  // To prevent the validator set from changing mid-test, set Staking.ForceEra to ForceNone ('0x02')
+  chainSpec.genesis.raw.top['0x5f3e4907f716ac89b6347d15ececedcaf7dad0317324aecae8744b87fc95f2f3'] = '0x02'
+
+  if (alice !== '') {
+    // Set sudo key to //Alice
+    chainSpec.genesis.raw.top['0x5c0d1176a568c1f92944340dbfed9e9c530ebca703c85910e7164cb7d1c9e47b'] =
+      '0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d'
+  }
+
+  fs.writeFileSync(specPath, JSON.stringify(chainSpec, null, 4))
+
+  console.log('****** INITIAL CHAINSPEC UPDATED TO REFLECT LIVE STATE ******')
+  process.exit()
+}
+
+main()
+
+interface Storage {
+  'jsonrpc': string
+  'result': Array<[string, string]>
+  'id': string
+}