Merge branch 'babylon' into joystream-cli

.dockerignore

@@ -1,4 +1,4 @@
-**target*
+target/
 **node_modules*
 .tmp/
 .vscode/

.env

@@ -0,0 +1,57 @@
+COMPOSE_PROJECT_NAME=joystream
+
+###########################
+#     Common settings     #
+###########################
+
+# The env variables below are by default used by all services and should be
+# overriden in local env files (e.g. ./generated/indexer) if needed
+# DB config
+DB_NAME=query_node
+DB_USER=postgres
+DB_PASS=postgres
+DB_HOST=localhost
+DB_PORT=5432
+DEBUG=index-builder:*
+TYPEORM_LOGGING=error
+
+###########################
+#    Indexer options      #
+###########################
+
+# Substrate endpoint to source events from
+WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944/
+# Block height to start indexing from.
+# Note, that if there are already some indexed events, this setting is ignored
+BLOCK_HEIGHT=0
+
+# Custom types to register for Substrate API
+# TYPE_REGISTER_PACKAGE_NAME=
+# TYPE_REGISTER_PACKAGE_VERSION=
+# TYPE_REGISTER_FUNCTION=
+
+# Redis cache server
+REDIS_URI=redis://localhost:6379/0
+
+###########################
+#    Processor options    #
+###########################
+
+# Where the mapping scripts are located, relative to ./generated/indexer
+TYPES_JSON=../../typedefs.json
+
+# Indexer GraphQL API endpoint to fetch indexed events
+INDEXER_ENDPOINT_URL=http://localhost:4100/graphql
+
+# Block height from which the processor starts. Note that if
+# there are already processed events in the database, this setting is ignored
+BLOCK_HEIGHT=0
+
+###############################
+#    Processor GraphQL API    #
+###############################
+
+GRAPHQL_SERVER_PORT=4002
+GRAPHQL_SERVER_HOST=localhost
+WARTHOG_APP_PORT=4002
+WARTHOG_APP_HOST=localhost

.github/workflows/content-directory-schemas.yml

@@ -17,4 +17,4 @@ jobs:
     - name: validate
       run: |
         yarn install --frozen-lockfile
-        yarn workspace cd-schemas checks --quiet
+        yarn workspace @joystream/cd-schemas checks --quiet

.github/workflows/run-network-tests.yml

@@ -77,7 +77,7 @@ jobs:
         with:
           name: ${{ steps.compute_shasum.outputs.shasum }}-joystream-node-docker-image.tar.gz
           path: joystream-node-docker-image.tar.gz
-  
+
   basic_runtime_with_upgrade:
     name: Integration Tests (Runtime Upgrade)
     needs: build_images
@@ -146,11 +146,11 @@ jobs:
       - name: Install packages and dependencies
         run: yarn install --frozen-lockfile
       - name: Ensure tests are runnable
-        run: yarn workspace cd-schemas checks --quiet
+        run: yarn workspace @joystream/cd-schemas checks --quiet
       - name: Start chain
-        run: docker-compose up -d
+        run: docker-compose up -d joystream-node
       - name: Initialize the content directory
-        run: yarn workspace cd-schemas initialize:dev
+        run: yarn workspace @joystream/cd-schemas initialize:dev
 
   query_node:
     name: Query Node Integration Tests
@@ -179,7 +179,7 @@ jobs:
       # integration tests
       - name: Execute Tests
         run: query-node/run-tests.sh
-  
+
   storage_node:
     name: Storage Node Tests
     needs: build_images
@@ -202,10 +202,14 @@ jobs:
       - name: Build storage node
         run: yarn workspace storage-node build
       - name: Start Services
-        run: docker-compose --file docker-compose-with-storage.yml up -d
-      - name: Add development storage node and initialize content directory
-        run: DEBUG=* yarn storage-cli dev-init
-      - name: Try uploading
+        run: |
+          docker-compose up -d ipfs
+          docker-compose up -d joystream-node
+      - name: Configure and start development storage node
+        run: |
+          DEBUG=* yarn storage-cli dev-init
+          docker-compose up -d colossus
+      - name: Test uploading
         run: |
           WAIT_TIME=90
           export DEBUG=joystream:*

README.md

@@ -108,9 +108,7 @@ A step by step guide to setup a full node and validator on the Joystream testnet
 ### Integration tests
 
 ```bash
-docker-compose up -d
-DEBUG=* yarn workspace network-tests test-run src/scenarios/full.ts
-docker-compose down
+tests/network-tests/run-tests.sh
 ```
 
 ### Contributing

apps.Dockerfile

@@ -7,9 +7,7 @@ COPY . /joystream
 # to ensure dev dependencies are installed.
 RUN yarn install --frozen-lockfile
 
-# Pioneer is failing to build only on github actions workflow runner
-# Error: packages/page-staking/src/index.tsx(24,21): error TS2307: Cannot find module './Targets' or its corresponding type declarations.
-# RUN yarn workspace pioneer build
+RUN yarn workspace pioneer build
 RUN yarn workspace storage-node build
 RUN yarn workspace query-node-root build
 

build.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+set -e
+
+yarn
+yarn workspace @joystream/types build
+yarn workspace @joystream/cd-schemas generate:all
+yarn workspace @joystream/cd-schemas build
+yarn workspace @joystream/cli build
+yarn workspace query-node-root build
+yarn workspace storage-node build
+# Not strictly needed during development, we run "yarn workspace pioneer start" to start
+# a dev instance, but will show highlight build issues
+yarn workspace pioneer build
+
+if ! command -v docker-compose &> /dev/null
+then
+  echo "docker-compose not found, skipping docker build!"
+else
+  # Build joystream/apps docker image
+  docker-compose build pioneer
+
+  # Optionally build joystream/node docker image
+  # TODO: Try to fetch a cached joystream/node image
+  # if one is found matching code shasum instead of building
+  while true
+  do
+    read -p "Rebuild joystream/node docker image? (y/N): " answer2
+
+    case $answer2 in
+    [yY]* ) docker-compose build joystream-node
+            break;;
+
+    [nN]* ) break;;
+
+    * )     break;;
+    esac
+  done
+fi
+
+# Build cargo crates: native binaries joystream/node, wasm runtime, and chainspec builder.
+while true
+do
+  read -p "Compile joystream node native binary? (y/N): " answer1
+
+  case $answer1 in
+   [yY]* ) yarn cargo-checks
+           yarn cargo-build
+           break;;
+
+   [nN]* ) break;;
+
+   * )     break;;
+  esac
+done

cli/README.md

@@ -87,17 +87,26 @@ When using the CLI for the first time there are a few common steps you might wan
 * [`joystream-cli content-directory:curatorGroups`](#joystream-cli-content-directorycuratorgroups)
 * [`joystream-cli content-directory:entities CLASSNAME [PROPERTIES]`](#joystream-cli-content-directoryentities-classname-properties)
 * [`joystream-cli content-directory:entity ID`](#joystream-cli-content-directoryentity-id)
+* [`joystream-cli content-directory:initialize`](#joystream-cli-content-directoryinitialize)
+* [`joystream-cli content-directory:removeCuratorFromGroup [GROUPID] [CURATORID]`](#joystream-cli-content-directoryremovecuratorfromgroup-groupid-curatorid)
 * [`joystream-cli content-directory:removeCuratorGroup [ID]`](#joystream-cli-content-directoryremovecuratorgroup-id)
+* [`joystream-cli content-directory:removeEntity ID`](#joystream-cli-content-directoryremoveentity-id)
 * [`joystream-cli content-directory:removeMaintainerFromClass [CLASSNAME] [GROUPID]`](#joystream-cli-content-directoryremovemaintainerfromclass-classname-groupid)
 * [`joystream-cli content-directory:setCuratorGroupStatus [ID] [STATUS]`](#joystream-cli-content-directorysetcuratorgroupstatus-id-status)
 * [`joystream-cli content-directory:updateClassPermissions [CLASSNAME]`](#joystream-cli-content-directoryupdateclasspermissions-classname)
 * [`joystream-cli council:info`](#joystream-cli-councilinfo)
 * [`joystream-cli help [COMMAND]`](#joystream-cli-help-command)
 * [`joystream-cli media:createChannel`](#joystream-cli-mediacreatechannel)
+* [`joystream-cli media:curateContent`](#joystream-cli-mediacuratecontent)
+* [`joystream-cli media:featuredVideos`](#joystream-cli-mediafeaturedvideos)
 * [`joystream-cli media:myChannels`](#joystream-cli-mediamychannels)
 * [`joystream-cli media:myVideos`](#joystream-cli-mediamyvideos)
+* [`joystream-cli media:removeChannel [ID]`](#joystream-cli-mediaremovechannel-id)
+* [`joystream-cli media:removeVideo [ID]`](#joystream-cli-mediaremovevideo-id)
+* [`joystream-cli media:setFeaturedVideos VIDEOIDS`](#joystream-cli-mediasetfeaturedvideos-videoids)
 * [`joystream-cli media:updateChannel [ID]`](#joystream-cli-mediaupdatechannel-id)
 * [`joystream-cli media:updateVideo [ID]`](#joystream-cli-mediaupdatevideo-id)
+* [`joystream-cli media:updateVideoLicense [ID]`](#joystream-cli-mediaupdatevideolicense-id)
 * [`joystream-cli media:uploadVideo FILEPATH`](#joystream-cli-mediauploadvideo-filepath)
 * [`joystream-cli working-groups:application WGAPPLICATIONID`](#joystream-cli-working-groupsapplication-wgapplicationid)
 * [`joystream-cli working-groups:createOpening`](#joystream-cli-working-groupscreateopening)
@@ -109,6 +118,7 @@ When using the CLI for the first time there are a few common steps you might wan
 * [`joystream-cli working-groups:opening WGOPENINGID`](#joystream-cli-working-groupsopening-wgopeningid)
 * [`joystream-cli working-groups:openings`](#joystream-cli-working-groupsopenings)
 * [`joystream-cli working-groups:overview`](#joystream-cli-working-groupsoverview)
+* [`joystream-cli working-groups:setDefaultGroup`](#joystream-cli-working-groupssetdefaultgroup)
 * [`joystream-cli working-groups:slashWorker WORKERID`](#joystream-cli-working-groupsslashworker-workerid)
 * [`joystream-cli working-groups:startAcceptingApplications WGOPENINGID`](#joystream-cli-working-groupsstartacceptingapplications-wgopeningid)
 * [`joystream-cli working-groups:startReviewPeriod WGOPENINGID`](#joystream-cli-working-groupsstartreviewperiod-wgopeningid)
@@ -319,7 +329,9 @@ USAGE
 
 OPTIONS
   -i, --input=input    Path to JSON file to use as input (if not specified - the input can be provided interactively)
-  -o, --output=output  Path where the output JSON file should be placed (can be then reused as input)
+
+  -o, --output=output  Path to the directory where the output JSON file should be placed (the output file can be then
+                       reused as input)
 ```
 
 _See code: [src/commands/content-directory/addClassSchema.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/addClassSchema.ts)_
@@ -389,7 +401,9 @@ USAGE
 
 OPTIONS
   -i, --input=input    Path to JSON file to use as input (if not specified - the input can be provided interactively)
-  -o, --output=output  Path where the output JSON file should be placed (can be then reused as input)
+
+  -o, --output=output  Path to the directory where the output JSON file should be placed (the output file can be then
+                       reused as input)
 ```
 
 _See code: [src/commands/content-directory/createClass.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/createClass.ts)_
@@ -469,6 +483,35 @@ ARGUMENTS
 
 _See code: [src/commands/content-directory/entity.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/entity.ts)_
 
+## `joystream-cli content-directory:initialize`
+
+Initialize content directory with input data from @joystream/content library or custom, provided one. Requires lead access.
+
+```
+USAGE
+  $ joystream-cli content-directory:initialize
+
+OPTIONS
+  --rootInputsDir=rootInputsDir  Custom inputs directory (must follow @joystream/content directory structure)
+```
+
+_See code: [src/commands/content-directory/initialize.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/initialize.ts)_
+
+## `joystream-cli content-directory:removeCuratorFromGroup [GROUPID] [CURATORID]`
+
+Remove Curator from Curator Group.
+
+```
+USAGE
+  $ joystream-cli content-directory:removeCuratorFromGroup [GROUPID] [CURATORID]
+
+ARGUMENTS
+  GROUPID    ID of the Curator Group
+  CURATORID  ID of the curator
+```
+
+_See code: [src/commands/content-directory/removeCuratorFromGroup.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/removeCuratorFromGroup.ts)_
+
 ## `joystream-cli content-directory:removeCuratorGroup [ID]`
 
 Remove existing Curator Group.
@@ -483,6 +526,23 @@ ARGUMENTS
 
 _See code: [src/commands/content-directory/removeCuratorGroup.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/removeCuratorGroup.ts)_
 
+## `joystream-cli content-directory:removeEntity ID`
+
+Removes a single entity by id (can be executed in Member, Curator or Lead context)
+
+```
+USAGE
+  $ joystream-cli content-directory:removeEntity ID
+
+ARGUMENTS
+  ID  ID of the entity to remove
+
+OPTIONS
+  --context=(Member|Curator|Lead)  Actor context to execute the command in (Member/Curator/Lead)
+```
+
+_See code: [src/commands/content-directory/removeEntity.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/content-directory/removeEntity.ts)_
+
 ## `joystream-cli content-directory:removeMaintainerFromClass [CLASSNAME] [GROUPID]`
 
 Remove maintainer (Curator Group) from class.
@@ -565,11 +625,40 @@ USAGE
 
 OPTIONS
   -i, --input=input    Path to JSON file to use as input (if not specified - the input can be provided interactively)
-  -o, --output=output  Path where the output JSON file should be placed (can be then reused as input)
+
+  -o, --output=output  Path to the directory where the output JSON file should be placed (the output file can be then
+                       reused as input)
 ```
 
 _See code: [src/commands/media/createChannel.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/media/createChannel.ts)_
 
+## `joystream-cli media:curateContent`
+
+Set the curation status of given entity (Channel/Video). Requires Curator access.
+
+```
+USAGE
+  $ joystream-cli media:curateContent
+
+OPTIONS
+  -c, --className=(Channel|Video)   (required) Name of the class of the entity to curate (Channel/Video)
+  -s, --status=(Accepted|Censored)  (required) Specifies the curation status (Accepted/Censored)
+  --id=id                           (required) ID of the entity to curate
+```
+
+_See code: [src/commands/media/curateContent.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/media/curateContent.ts)_
+
+## `joystream-cli media:featuredVideos`
+
+Show a list of currently featured videos.
+
+```
+USAGE
+  $ joystream-cli media:featuredVideos
+```
+
+_See code: [src/commands/media/featuredVideos.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/media/featuredVideos.ts)_
+
 ## `joystream-cli media:myChannels`
 
 Show the list of channels associated with current account's membership.
@@ -595,6 +684,51 @@ OPTIONS
 
 _See code: [src/commands/media/myVideos.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/media/myVideos.ts)_
 
+## `joystream-cli media:removeChannel [ID]`
+
+Removes a channel (required controller access).
+
+```
+USAGE
+  $ joystream-cli media:removeChannel [ID]
+
+ARGUMENTS
+  ID  ID of the Channel entity
+```
+
+_See code: [src/commands/media/removeChannel.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/media/removeChannel.ts)_
+
+## `joystream-cli media:removeVideo [ID]`
+
+Remove given Video entity and associated entities (VideoMedia, License) from content directory.
+
+```
+USAGE
+  $ joystream-cli media:removeVideo [ID]
+
+ARGUMENTS
+  ID  ID of the Video entity
+```
+
+_See code: [src/commands/media/removeVideo.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/media/removeVideo.ts)_
+
+## `joystream-cli media:setFeaturedVideos VIDEOIDS`
+
+Set currently featured videos (requires lead/maintainer access).
+
+```
+USAGE
+  $ joystream-cli media:setFeaturedVideos VIDEOIDS
+
+ARGUMENTS
+  VIDEOIDS  Comma-separated video ids
+
+OPTIONS
+  --add  If provided - currently featured videos will not be removed.
+```
+
+_See code: [src/commands/media/setFeaturedVideos.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/media/setFeaturedVideos.ts)_
+
 ## `joystream-cli media:updateChannel [ID]`
 
 Update one of the owned channels on Joystream (requires a membership).
@@ -608,14 +742,18 @@ ARGUMENTS
 
 OPTIONS
   -i, --input=input    Path to JSON file to use as input (if not specified - the input can be provided interactively)
-  -o, --output=output  Path where the output JSON file should be placed (can be then reused as input)
+
+  -o, --output=output  Path to the directory where the output JSON file should be placed (the output file can be then
+                       reused as input)
+
+  --asCurator          Provide this flag in order to use Curator context for the update
 ```
 
 _See code: [src/commands/media/updateChannel.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/media/updateChannel.ts)_
 
 ## `joystream-cli media:updateVideo [ID]`
 
-Update existing video information (requires a membership).
+Update existing video information (requires controller/maintainer access).
 
 ```
 USAGE
@@ -623,10 +761,27 @@ USAGE
 
 ARGUMENTS
   ID  ID of the Video to update
+
+OPTIONS
+  --asCurator  Specify in order to update the video as curator
 ```
 
 _See code: [src/commands/media/updateVideo.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/media/updateVideo.ts)_
 
+## `joystream-cli media:updateVideoLicense [ID]`
+
+Update existing video license (requires controller/maintainer access).
+
+```
+USAGE
+  $ joystream-cli media:updateVideoLicense [ID]
+
+ARGUMENTS
+  ID  ID of the Video
+```
+
+_See code: [src/commands/media/updateVideoLicense.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/media/updateVideoLicense.ts)_
+
 ## `joystream-cli media:uploadVideo FILEPATH`
 
 Upload a new Video to a channel (requires a membership).
@@ -657,9 +812,8 @@ ARGUMENTS
   WGAPPLICATIONID  Working Group Application ID
 
 OPTIONS
-  -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
-                     executed
-                     Available values are: storageProviders, curators.
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/application.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/application.ts)_
@@ -673,19 +827,20 @@ USAGE
   $ joystream-cli working-groups:createOpening
 
 OPTIONS
-  -c, --createDraftOnly      If provided - the extrinsic will not be executed. Use this flag if you only want to create
-                             a draft.
+  -e, --edit                               If provided along with --input - launches in edit mode allowing to modify the
+                                           input before sending the exstinsic
 
-  -d, --useDraft             Whether to create the opening from existing draft.
-                             If provided without --draftName - the list of choices will be displayed.
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
 
-  -g, --group=group          (required) [default: storageProviders] The working group context in which the command
-                             should be executed
-                             Available values are: storageProviders, curators.
+  -i, --input=input                        Path to JSON file to use as input (if not specified - the input can be
+                                           provided interactively)
 
-  -n, --draftName=draftName  Name of the draft to create the opening from.
+  -o, --output=output                      Path to the file where the output JSON should be saved (this output can be
+                                           then reused as input)
 
-  -s, --skipPrompts          Whether to skip all prompts when adding from draft (will use all default values)
+  --dryRun                                 If provided along with --output - skips sending the actual extrinsic(can be
+                                           used to generate a "draft" which can be provided as input later)
 ```
 
 _See code: [src/commands/working-groups/createOpening.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/createOpening.ts)_
@@ -702,9 +857,8 @@ ARGUMENTS
   WORKERID  Worker ID
 
 OPTIONS
-  -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
-                     executed
-                     Available values are: storageProviders, curators.
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/decreaseWorkerStake.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/decreaseWorkerStake.ts)_
@@ -721,9 +875,8 @@ ARGUMENTS
   WORKERID  Worker ID
 
 OPTIONS
-  -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
-                     executed
-                     Available values are: storageProviders, curators.
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/evictWorker.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/evictWorker.ts)_
@@ -740,9 +893,8 @@ ARGUMENTS
   WGOPENINGID  Working Group Opening ID
 
 OPTIONS
-  -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
-                     executed
-                     Available values are: storageProviders, curators.
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/fillOpening.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/fillOpening.ts)_
@@ -756,9 +908,8 @@ USAGE
   $ joystream-cli working-groups:increaseStake
 
 OPTIONS
-  -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
-                     executed
-                     Available values are: storageProviders, curators.
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/increaseStake.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/increaseStake.ts)_
@@ -772,9 +923,8 @@ USAGE
   $ joystream-cli working-groups:leaveRole
 
 OPTIONS
-  -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
-                     executed
-                     Available values are: storageProviders, curators.
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/leaveRole.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/leaveRole.ts)_
@@ -791,9 +941,8 @@ ARGUMENTS
   WGOPENINGID  Working Group Opening ID
 
 OPTIONS
-  -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
-                     executed
-                     Available values are: storageProviders, curators.
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/opening.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/opening.ts)_
@@ -807,9 +956,8 @@ USAGE
   $ joystream-cli working-groups:openings
 
 OPTIONS
-  -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
-                     executed
-                     Available values are: storageProviders, curators.
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/openings.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/openings.ts)_
@@ -823,13 +971,27 @@ USAGE
   $ joystream-cli working-groups:overview
 
 OPTIONS
-  -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
-                     executed
-                     Available values are: storageProviders, curators.
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/overview.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/overview.ts)_
 
+## `joystream-cli working-groups:setDefaultGroup`
+
+Change the default group context for working-groups commands.
+
+```
+USAGE
+  $ joystream-cli working-groups:setDefaultGroup
+
+OPTIONS
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
+```
+
+_See code: [src/commands/working-groups/setDefaultGroup.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/setDefaultGroup.ts)_
+
 ## `joystream-cli working-groups:slashWorker WORKERID`
 
 Slashes given worker stake. Requires lead access.
@@ -842,9 +1004,8 @@ ARGUMENTS
   WORKERID  Worker ID
 
 OPTIONS
-  -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
-                     executed
-                     Available values are: storageProviders, curators.
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/slashWorker.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/slashWorker.ts)_
@@ -861,9 +1022,8 @@ ARGUMENTS
   WGOPENINGID  Working Group Opening ID
 
 OPTIONS
-  -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
-                     executed
-                     Available values are: storageProviders, curators.
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/startAcceptingApplications.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/startAcceptingApplications.ts)_
@@ -880,9 +1040,8 @@ ARGUMENTS
   WGOPENINGID  Working Group Opening ID
 
 OPTIONS
-  -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
-                     executed
-                     Available values are: storageProviders, curators.
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/startReviewPeriod.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/startReviewPeriod.ts)_
@@ -899,9 +1058,8 @@ ARGUMENTS
   WGAPPLICATIONID  Working Group Application ID
 
 OPTIONS
-  -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
-                     executed
-                     Available values are: storageProviders, curators.
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/terminateApplication.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/terminateApplication.ts)_
@@ -918,9 +1076,8 @@ ARGUMENTS
   ACCOUNTADDRESS  New reward account address (if omitted, one of the existing CLI accounts can be selected)
 
 OPTIONS
-  -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
-                     executed
-                     Available values are: storageProviders, curators.
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/updateRewardAccount.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/updateRewardAccount.ts)_
@@ -937,9 +1094,8 @@ ARGUMENTS
   ACCOUNTADDRESS  New role account address (if omitted, one of the existing CLI accounts can be selected)
 
 OPTIONS
-  -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
-                     executed
-                     Available values are: storageProviders, curators.
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/updateRoleAccount.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/updateRoleAccount.ts)_
@@ -956,9 +1112,8 @@ ARGUMENTS
   WORKERID  Worker ID
 
 OPTIONS
-  -g, --group=group  (required) [default: storageProviders] The working group context in which the command should be
-                     executed
-                     Available values are: storageProviders, curators.
+  -g, --group=(storageProviders|curators)  The working group context in which the command should be executed
+                                           Available values are: storageProviders, curators.
 ```
 
 _See code: [src/commands/working-groups/updateWorkerReward.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/updateWorkerReward.ts)_

cli/package.json

@@ -125,5 +125,9 @@
     "format": "prettier ./ --write",
     "generate:schema-typings": "rm -rf ./src/json-schemas/typings && json2ts -i ./src/json-schemas/ -o ./src/json-schemas/typings/"
   },
-  "types": "lib/index.d.ts"
+  "types": "lib/index.d.ts",
+  "volta": {
+    "node": "12.18.2",
+    "yarn": "1.22.4"
+  }
 }

cli/src/base/ContentDirectoryCommandBase.ts

@@ -1,7 +1,7 @@
 import ExitCodes from '../ExitCodes'
 import { WorkingGroups } from '../Types'
-import { ReferenceProperty } from 'cd-schemas/types/extrinsics/AddClassSchema'
-import { FlattenRelations } from 'cd-schemas/types/utility'
+import { ReferenceProperty } from '@joystream/cd-schemas/types/extrinsics/AddClassSchema'
+import { FlattenRelations } from '@joystream/cd-schemas/types/utility'
 import { BOOL_PROMPT_OPTIONS } from '../helpers/prompting'
 import {
   Class,
@@ -234,14 +234,14 @@ export default abstract class ContentDirectoryCommandBase extends RolesCommandBa
     }
 
     if (requireSchema && !entity.supported_schemas.toArray().length) {
-      this.error(`${requiredClass || ''}Entity of id ${id} has no schema support added!`)
+      this.error(`${requiredClass || ''} entity of id ${id} has no schema support added!`)
     }
 
     return entity
   }
 
-  async getAndParseKnownEntity<T>(id: string | number): Promise<FlattenRelations<T>> {
-    const entity = await this.getEntity(id)
+  async getAndParseKnownEntity<T>(id: string | number, className?: string): Promise<FlattenRelations<T>> {
+    const entity = await this.getEntity(id, className)
     return this.parseToKnownEntityJson<T>(entity)
   }
 
@@ -337,7 +337,7 @@ export default abstract class ContentDirectoryCommandBase extends RolesCommandBa
     const defaultValues = entityClass.properties
       .map((p) => p.name.toString())
       .reduce((d, propName) => {
-        if (includedProps?.includes(propName)) {
+        if (!includedProps || includedProps.includes(propName)) {
           d[propName] = chalk.grey('[not set]')
         }
         return d

cli/src/base/MediaCommandBase.ts

@@ -1,5 +1,5 @@
 import ContentDirectoryCommandBase from './ContentDirectoryCommandBase'
-import { VideoEntity } from 'cd-schemas/types/entities'
+import { VideoEntity } from '@joystream/cd-schemas/types/entities'
 import fs from 'fs'
 import { DistinctQuestion } from 'inquirer'
 import path from 'path'

cli/src/commands/content-directory/addClassSchema.ts

@@ -1,7 +1,7 @@
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
-import AddClassSchemaSchema from 'cd-schemas/schemas/extrinsics/AddClassSchema.schema.json'
-import { AddClassSchema } from 'cd-schemas/types/extrinsics/AddClassSchema'
-import { InputParser } from 'cd-schemas'
+import AddClassSchemaSchema from '@joystream/cd-schemas/schemas/extrinsics/AddClassSchema.schema.json'
+import { AddClassSchema } from '@joystream/cd-schemas/types/extrinsics/AddClassSchema'
+import { InputParser } from '@joystream/cd-schemas'
 import { JsonSchemaPrompter, JsonSchemaCustomPrompts } from '../../helpers/JsonSchemaPrompt'
 import { JSONSchema } from '@apidevtools/json-schema-ref-parser'
 import { IOFlags, getInputJson, saveOutputJson } from '../../helpers/InputOutput'

cli/src/commands/content-directory/createClass.ts

@@ -1,7 +1,7 @@
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
-import CreateClassSchema from 'cd-schemas/schemas/extrinsics/CreateClass.schema.json'
-import { CreateClass } from 'cd-schemas/types/extrinsics/CreateClass'
-import { InputParser } from 'cd-schemas'
+import CreateClassSchema from '@joystream/cd-schemas/schemas/extrinsics/CreateClass.schema.json'
+import { CreateClass } from '@joystream/cd-schemas/types/extrinsics/CreateClass'
+import { InputParser } from '@joystream/cd-schemas'
 import { JsonSchemaPrompter, JsonSchemaCustomPrompts } from '../../helpers/JsonSchemaPrompt'
 import { JSONSchema } from '@apidevtools/json-schema-ref-parser'
 import { IOFlags, getInputJson, saveOutputJson } from '../../helpers/InputOutput'

cli/src/commands/content-directory/initialize.ts

@@ -1,21 +1,28 @@
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
-import { CreateClass } from 'cd-schemas/types/extrinsics/CreateClass'
-import { getInputs, InputParser, ExtrinsicsHelper } from 'cd-schemas'
-import { AddClassSchema } from 'cd-schemas/types/extrinsics/AddClassSchema'
-import { EntityBatch } from 'cd-schemas/types/EntityBatch'
+import { InputParser, ExtrinsicsHelper, getInitializationInputs } from '@joystream/cd-schemas'
+import { flags } from '@oclif/command'
 
 export default class InitializeCommand extends ContentDirectoryCommandBase {
   static description =
-    'Initialize content directory with input data from @joystream/content library. Requires lead access.'
+    'Initialize content directory with input data from @joystream/content library or custom, provided one. Requires lead access.'
+
+  static flags = {
+    rootInputsDir: flags.string({
+      required: false,
+      description: 'Custom inputs directory (must follow @joystream/content directory structure)',
+    }),
+  }
 
   async run() {
     const account = await this.getRequiredSelectedAccount()
     await this.requireLead()
     await this.requestAccountDecoding(account)
 
-    const classInputs = getInputs<CreateClass>('classes').map(({ data }) => data)
-    const schemaInputs = getInputs<AddClassSchema>('schemas').map(({ data }) => data)
-    const entityBatchInputs = getInputs<EntityBatch>('entityBatches').map(({ data }) => data)
+    const {
+      flags: { rootInputsDir },
+    } = this.parse(InitializeCommand)
+
+    const { classInputs, schemaInputs, entityBatchInputs } = getInitializationInputs(rootInputsDir)
 
     const currentClasses = await this.getApi().availableClasses()
 

cli/src/commands/content-directory/updateClassPermissions.ts

@@ -1,8 +1,8 @@
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
-import CreateClassSchema from 'cd-schemas/schemas/extrinsics/CreateClass.schema.json'
+import CreateClassSchema from '@joystream/cd-schemas/schemas/extrinsics/CreateClass.schema.json'
 import chalk from 'chalk'
 import { JsonSchemaCustomPrompts, JsonSchemaPrompter } from '../../helpers/JsonSchemaPrompt'
-import { CreateClass } from 'cd-schemas/types/extrinsics/CreateClass'
+import { CreateClass } from '@joystream/cd-schemas/types/extrinsics/CreateClass'
 import { JSONSchema } from '@apidevtools/json-schema-ref-parser'
 
 export default class UpdateClassPermissionsCommand extends ContentDirectoryCommandBase {

cli/src/commands/media/createChannel.ts

@@ -1,11 +1,13 @@
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
-import ChannelEntitySchema from 'cd-schemas/schemas/entities/ChannelEntity.schema.json'
-import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
-import { InputParser } from 'cd-schemas'
+import ChannelEntitySchema from '@joystream/cd-schemas/schemas/entities/ChannelEntity.schema.json'
+import { ChannelEntity } from '@joystream/cd-schemas/types/entities/ChannelEntity'
+import { InputParser } from '@joystream/cd-schemas'
 import { IOFlags, getInputJson, saveOutputJson } from '../../helpers/InputOutput'
 import { JSONSchema } from '@apidevtools/json-schema-ref-parser'
 import { JsonSchemaCustomPrompts, JsonSchemaPrompter } from '../../helpers/JsonSchemaPrompt'
+
 import { flags } from '@oclif/command'
+import _ from 'lodash'
 
 export default class CreateChannelCommand extends ContentDirectoryCommandBase {
   static description = 'Create a new channel on Joystream (requires a membership).'
@@ -34,7 +36,7 @@ export default class CreateChannelCommand extends ContentDirectoryCommandBase {
 
       const prompter = new JsonSchemaPrompter<ChannelEntity>(channelJsonSchema, undefined, customPrompts)
 
-      inputJson = await prompter.promptAll(true)
+      inputJson = await prompter.promptAll()
     }
 
     this.jsonPrettyPrint(JSON.stringify(inputJson))
@@ -42,7 +44,7 @@ export default class CreateChannelCommand extends ContentDirectoryCommandBase {
       confirm || (await this.simplePrompt({ type: 'confirm', message: 'Do you confirm the provided input?' }))
 
     if (confirmed) {
-      saveOutputJson(output, `${inputJson.title}Channel.json`, inputJson)
+      saveOutputJson(output, `${_.startCase(inputJson.handle)}Channel.json`, inputJson)
       const inputParser = InputParser.createWithKnownSchemas(this.getOriginalApi(), [
         {
           className: 'Channel',

cli/src/commands/media/curateContent.ts

@@ -1,8 +1,8 @@
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
-import { InputParser } from 'cd-schemas'
+import { InputParser } from '@joystream/cd-schemas'
 import { flags } from '@oclif/command'
-import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
-import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
+import { ChannelEntity } from '@joystream/cd-schemas/types/entities/ChannelEntity'
+import { VideoEntity } from '@joystream/cd-schemas/types/entities/VideoEntity'
 
 const CLASSES = ['Channel', 'Video'] as const
 const STATUSES = ['Accepted', 'Censored'] as const

cli/src/commands/media/featuredVideos.ts

@@ -0,0 +1,36 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import { displayTable } from '../../helpers/display'
+import { FeaturedVideoEntity, VideoEntity } from '@joystream/cd-schemas/types/entities'
+import chalk from 'chalk'
+
+export default class FeaturedVideosCommand extends ContentDirectoryCommandBase {
+  static description = 'Show a list of currently featured videos.'
+
+  async run() {
+    const featuredEntries = await this.entitiesByClassAndOwner('FeaturedVideo')
+    const featured = await Promise.all(
+      featuredEntries
+        .filter(([, entity]) => entity.supported_schemas.toArray().length) // Ignore FeaturedVideo entities without schema
+        .map(([, entity]) => this.parseToKnownEntityJson<FeaturedVideoEntity>(entity))
+    )
+
+    const videoIds: number[] = featured.map(({ video: videoId }) => videoId)
+
+    const videos = await Promise.all(videoIds.map((videoId) => this.getAndParseKnownEntity<VideoEntity>(videoId)))
+
+    if (videos.length) {
+      displayTable(
+        videos.map(({ title, channel }, index) => ({
+          featuredVideoEntityId: featuredEntries[index][0].toNumber(),
+          videoId: videoIds[index],
+          channelId: channel,
+          title,
+        })),
+        3
+      )
+      this.log(`\nTIP: Use ${chalk.bold('content-directory:entity ID')} command to see more details about given video`)
+    } else {
+      this.log(`No videos have been featured yet! Set some with ${chalk.bold('media:setFeaturedVideos')}`)
+    }
+  }
+}

cli/src/commands/media/myChannels.ts

@@ -1,5 +1,5 @@
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
-import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
+import { ChannelEntity } from '@joystream/cd-schemas/types/entities/ChannelEntity'
 import { displayTable } from '../../helpers/display'
 import chalk from 'chalk'
 
@@ -9,7 +9,7 @@ export default class MyChannelsCommand extends ContentDirectoryCommandBase {
   async run() {
     const memberId = await this.getRequiredMemberId()
 
-    const props: (keyof ChannelEntity)[] = ['title', 'isPublic']
+    const props: (keyof ChannelEntity)[] = ['handle', 'isPublic']
 
     const list = await this.createEntityList('Channel', props, [], memberId)
 

cli/src/commands/media/myVideos.ts

@@ -1,5 +1,5 @@
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
-import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
+import { VideoEntity } from '@joystream/cd-schemas/types/entities/VideoEntity'
 import { displayTable } from '../../helpers/display'
 import chalk from 'chalk'
 import { flags } from '@oclif/command'

cli/src/commands/media/removeChannel.ts

@@ -1,7 +1,7 @@
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
 import { Entity } from '@joystream/types/content-directory'
 import { createType } from '@joystream/types'
-import { ChannelEntity } from 'cd-schemas/types/entities'
+import { ChannelEntity } from '@joystream/cd-schemas/types/entities'
 
 export default class RemoveChannelCommand extends ContentDirectoryCommandBase {
   static description = 'Removes a channel (required controller access).'
@@ -29,13 +29,13 @@ export default class RemoveChannelCommand extends ContentDirectoryCommandBase {
       channelId = parseInt(id)
       channelEntity = await this.getEntity(channelId, 'Channel', memberId)
     } else {
-      const [id, channel] = await this.promptForEntityEntry('Select a channel to remove', 'Channel', 'title', memberId)
+      const [id, channel] = await this.promptForEntityEntry('Select a channel to remove', 'Channel', 'handle', memberId)
       channelId = id.toNumber()
       channelEntity = channel
     }
     const channel = await this.parseToKnownEntityJson<ChannelEntity>(channelEntity)
 
-    await this.requireConfirmation(`Are you sure you want to remove "${channel.title}" channel?`)
+    await this.requireConfirmation(`Are you sure you want to remove "${channel.handle}" channel?`)
 
     const api = this.getOriginalApi()
     this.log(`Removing Channel entity (ID: ${channelId})...`)

cli/src/commands/media/removeVideo.ts

@@ -1,6 +1,6 @@
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
 import { Entity } from '@joystream/types/content-directory'
-import { VideoEntity } from 'cd-schemas/types/entities'
+import { VideoEntity } from '@joystream/cd-schemas/types/entities'
 import { createType } from '@joystream/types'
 
 export default class RemoveVideoCommand extends ContentDirectoryCommandBase {

cli/src/commands/media/setFeaturedVideos.ts

@@ -0,0 +1,79 @@
+import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
+import { VideoEntity } from '@joystream/cd-schemas/types/entities'
+import { InputParser, ExtrinsicsHelper } from '@joystream/cd-schemas'
+import { FlattenRelations } from '@joystream/cd-schemas/types/utility'
+import { flags } from '@oclif/command'
+import { createType } from '@joystream/types'
+
+export default class SetFeaturedVideosCommand extends ContentDirectoryCommandBase {
+  static description = 'Set currently featured videos (requires lead/maintainer access).'
+  static args = [
+    {
+      name: 'videoIds',
+      required: true,
+      description: 'Comma-separated video ids',
+    },
+  ]
+
+  static flags = {
+    add: flags.boolean({
+      description: 'If provided - currently featured videos will not be removed.',
+      required: false,
+    }),
+  }
+
+  async run() {
+    const account = await this.getRequiredSelectedAccount()
+    let actor = createType('Actor', { Lead: null })
+    try {
+      await this.getRequiredLead()
+    } catch (e) {
+      actor = await this.getCuratorContext(['FeaturedVideo'])
+    }
+
+    await this.requestAccountDecoding(account)
+
+    const {
+      args: { videoIds },
+      flags: { add },
+    } = this.parse(SetFeaturedVideosCommand)
+
+    const ids: number[] = videoIds.split(',').map((id: string) => parseInt(id))
+
+    const videos: [number, FlattenRelations<VideoEntity>][] = (
+      await Promise.all(ids.map((id) => this.getAndParseKnownEntity<VideoEntity>(id, 'Video')))
+    ).map((video, index) => [ids[index], video])
+
+    this.log(
+      `Featured videos that will ${add ? 'be added to' : 'replace'} existing ones:`,
+      videos.map(([id, { title }]) => ({ id, title }))
+    )
+
+    await this.requireConfirmation('Do you confirm the provided input?')
+
+    if (!add) {
+      const currentlyFeaturedIds = (await this.entitiesByClassAndOwner('FeaturedVideo')).map(([id]) => id.toNumber())
+      const removeTxs = currentlyFeaturedIds.map((id) =>
+        this.getOriginalApi().tx.contentDirectory.removeEntity(actor, id)
+      )
+
+      if (currentlyFeaturedIds.length) {
+        this.log(`Removing existing FeaturedVideo entities (${currentlyFeaturedIds.join(', ')})...`)
+
+        const txHelper = new ExtrinsicsHelper(this.getOriginalApi())
+        await txHelper.sendAndCheck(account, removeTxs, 'The removal of existing FeaturedVideo entities failed')
+      }
+    }
+
+    this.log('Adding new FeaturedVideo entities...')
+    const featuredVideoEntries = videos.map(([id]) => ({ video: id }))
+    const inputParser = InputParser.createWithKnownSchemas(this.getOriginalApi(), [
+      {
+        className: 'FeaturedVideo',
+        entries: featuredVideoEntries,
+      },
+    ])
+    const operations = await inputParser.getEntityBatchOperations()
+    await this.sendAndFollowNamedTx(account, 'contentDirectory', 'transaction', [actor, operations])
+  }
+}

cli/src/commands/media/updateChannel.ts

@@ -1,13 +1,14 @@
 import ContentDirectoryCommandBase from '../../base/ContentDirectoryCommandBase'
-import ChannelEntitySchema from 'cd-schemas/schemas/entities/ChannelEntity.schema.json'
-import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
-import { InputParser } from 'cd-schemas'
+import ChannelEntitySchema from '@joystream/cd-schemas/schemas/entities/ChannelEntity.schema.json'
+import { ChannelEntity } from '@joystream/cd-schemas/types/entities/ChannelEntity'
+import { InputParser } from '@joystream/cd-schemas'
 import { IOFlags, getInputJson, saveOutputJson } from '../../helpers/InputOutput'
 import { JSONSchema } from '@apidevtools/json-schema-ref-parser'
 import { JsonSchemaCustomPrompts, JsonSchemaPrompter } from '../../helpers/JsonSchemaPrompt'
 import { Actor, Entity } from '@joystream/types/content-directory'
 import { flags } from '@oclif/command'
 import { createType } from '@joystream/types'
+import _ from 'lodash'
 
 export default class UpdateChannelCommand extends ContentDirectoryCommandBase {
   static description = 'Update one of the owned channels on Joystream (requires a membership).'
@@ -51,7 +52,7 @@ export default class UpdateChannelCommand extends ContentDirectoryCommandBase {
       channelId = parseInt(id)
       channelEntity = await this.getEntity(channelId, 'Channel', memberId)
     } else {
-      const [id, channel] = await this.promptForEntityEntry('Select a channel to update', 'Channel', 'title', memberId)
+      const [id, channel] = await this.promptForEntityEntry('Select a channel to update', 'Channel', 'handle', memberId)
       channelId = id.toNumber()
       channelEntity = channel
     }
@@ -80,14 +81,14 @@ export default class UpdateChannelCommand extends ContentDirectoryCommandBase {
 
       const prompter = new JsonSchemaPrompter<ChannelEntity>(channelJsonSchema, currentValues, customPrompts)
 
-      inputJson = await prompter.promptAll(true)
+      inputJson = await prompter.promptAll()
     }
 
     this.jsonPrettyPrint(JSON.stringify(inputJson))
     const confirmed = await this.simplePrompt({ type: 'confirm', message: 'Do you confirm the provided input?' })
 
     if (confirmed) {
-      saveOutputJson(output, `${inputJson.title}Channel.json`, inputJson)
+      saveOutputJson(output, `${_.startCase(inputJson.handle)}Channel.json`, inputJson)
       const inputParser = InputParser.createWithKnownSchemas(this.getOriginalApi())
       const updateOperations = await inputParser.getEntityUpdateOperations(inputJson, 'Channel', channelId)
       this.log('Sending the extrinsic...')

cli/src/commands/media/updateVideo.ts

@@ -1,6 +1,6 @@
-import VideoEntitySchema from 'cd-schemas/schemas/entities/VideoEntity.schema.json'
-import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
-import { InputParser } from 'cd-schemas'
+import VideoEntitySchema from '@joystream/cd-schemas/schemas/entities/VideoEntity.schema.json'
+import { VideoEntity } from '@joystream/cd-schemas/types/entities/VideoEntity'
+import { InputParser } from '@joystream/cd-schemas'
 import { JSONSchema } from '@apidevtools/json-schema-ref-parser'
 import { JsonSchemaCustomPrompts, JsonSchemaPrompter } from '../../helpers/JsonSchemaPrompt'
 import { Actor, Entity } from '@joystream/types/content-directory'
@@ -83,7 +83,7 @@ export default class UpdateVideoCommand extends MediaCommandBase {
       'category',
       'title',
       'description',
-      'thumbnailURL',
+      'thumbnailUrl',
       'duration',
       'isPublic',
       'isExplicit',

cli/src/commands/media/updateVideoLicense.ts

@@ -1,6 +1,6 @@
 import MediaCommandBase from '../../base/MediaCommandBase'
-import { LicenseEntity, VideoEntity } from 'cd-schemas/types/entities'
-import { InputParser } from 'cd-schemas'
+import { LicenseEntity, VideoEntity } from '@joystream/cd-schemas/types/entities'
+import { InputParser } from '@joystream/cd-schemas'
 import { Entity } from '@joystream/types/content-directory'
 import { createType } from '@joystream/types'
 

cli/src/commands/media/uploadVideo.ts

@@ -1,8 +1,8 @@
-import VideoEntitySchema from 'cd-schemas/schemas/entities/VideoEntity.schema.json'
-import VideoMediaEntitySchema from 'cd-schemas/schemas/entities/VideoMediaEntity.schema.json'
-import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
-import { VideoMediaEntity } from 'cd-schemas/types/entities/VideoMediaEntity'
-import { InputParser } from 'cd-schemas'
+import VideoEntitySchema from '@joystream/cd-schemas/schemas/entities/VideoEntity.schema.json'
+import VideoMediaEntitySchema from '@joystream/cd-schemas/schemas/entities/VideoMediaEntity.schema.json'
+import { VideoEntity } from '@joystream/cd-schemas/types/entities/VideoEntity'
+import { VideoMediaEntity } from '@joystream/cd-schemas/types/entities/VideoMediaEntity'
+import { InputParser } from '@joystream/cd-schemas'
 import { JSONSchema } from '@apidevtools/json-schema-ref-parser'
 import { JsonSchemaPrompter } from '../../helpers/JsonSchemaPrompt'
 import { flags } from '@oclif/command'
@@ -368,7 +368,7 @@ export default class UploadVideoCommand extends MediaCommandBase {
       channelId = await this.promptForEntityId(
         'Select a channel to publish the video under',
         'Channel',
-        'title',
+        'handle',
         memberId
       )
     } else {

cli/src/helpers/InputOutput.ts

@@ -5,7 +5,7 @@ import fs from 'fs'
 import path from 'path'
 import Ajv from 'ajv'
 import $RefParser, { JSONSchema } from '@apidevtools/json-schema-ref-parser'
-import { getSchemasLocation } from 'cd-schemas'
+import { getSchemasLocation } from '@joystream/cd-schemas'
 import chalk from 'chalk'
 
 // Default schema path for resolving refs

cli/src/helpers/JsonSchemaPrompt.ts

@@ -4,7 +4,7 @@ import _ from 'lodash'
 import RefParser, { JSONSchema } from '@apidevtools/json-schema-ref-parser'
 import chalk from 'chalk'
 import { BOOL_PROMPT_OPTIONS } from './prompting'
-import { getSchemasLocation } from 'cd-schemas'
+import { getSchemasLocation } from '@joystream/cd-schemas'
 import path from 'path'
 
 type CustomPromptMethod = () => Promise<any>
@@ -106,7 +106,11 @@ export class JsonSchemaPrompter<JsonResult> {
     if (schema.oneOf) {
       const oneOf = schema.oneOf as JSONSchema[]
       const options = this.oneOfToOptions(oneOf, currentValue)
-      const { choosen } = await inquirer.prompt({ name: 'choosen', message: propDisplayName, type: 'list', ...options })
+      const choosen = await this.inquirerSinglePrompt({
+        message: propDisplayName,
+        type: 'list',
+        ...options,
+      })
       if (choosen !== options.default) {
         _.set(this.filledObject, propertyPath, undefined) // Clear any previous value if different variant selected
       }
@@ -128,18 +132,13 @@ export class JsonSchemaPrompter<JsonResult> {
         const required = allPropsRequired || (Array.isArray(schema.required) && schema.required.includes(pName))
 
         if (!required) {
-          confirmed = (
-            await inquirer.prompt([
-              {
-                message: `Do you want to provide optional ${chalk.greenBright(objectPropertyPath)}?`,
-                type: 'confirm',
-                name: 'confirmed',
-                default:
-                  _.get(this.filledObject, objectPropertyPath) !== undefined &&
-                  _.get(this.filledObject, objectPropertyPath) !== null,
-              },
-            ])
-          ).confirmed
+          confirmed = await this.inquirerSinglePrompt({
+            message: `Do you want to provide optional ${chalk.greenBright(objectPropertyPath)}?`,
+            type: 'confirm',
+            default:
+              _.get(this.filledObject, objectPropertyPath) !== undefined &&
+              _.get(this.filledObject, objectPropertyPath) !== null,
+          })
         }
         if (confirmed) {
           value[pName] = await this.prompt(pSchema, objectPropertyPath)
@@ -207,14 +206,11 @@ export class JsonSchemaPrompter<JsonResult> {
     let currItem = 0
     const result = []
     while (currItem < maxItems) {
-      const { next } = await inquirer.prompt([
-        {
-          ...BOOL_PROMPT_OPTIONS,
-          name: 'next',
-          message: `Do you want to add another item to ${this.propertyDisplayName(propertyPath)} array?`,
-          default: _.get(this.filledObject, `${propertyPath}[${currItem}]`) !== undefined,
-        },
-      ])
+      const next = await this.inquirerSinglePrompt({
+        ...BOOL_PROMPT_OPTIONS,
+        message: `Do you want to add another item to ${this.propertyDisplayName(propertyPath)} array?`,
+        default: _.get(this.filledObject, `${propertyPath}[${currItem}]`) !== undefined,
+      })
       if (!next) {
         break
       }
@@ -228,20 +224,17 @@ export class JsonSchemaPrompter<JsonResult> {
   }
 
   private async promptSimple(promptOptions: DistinctQuestion, propertyPath: string, normalize?: (v: any) => any) {
-    const { result } = await inquirer.prompt([
-      {
-        ...promptOptions,
-        name: 'result',
-        validate: (v) => {
-          v = normalize ? normalize(v) : v
-          return (
-            this.setValueAndGetError(propertyPath, v) ||
-            (promptOptions.validate ? promptOptions.validate(v) : true) ||
-            true
-          )
-        },
+    const result = await this.inquirerSinglePrompt({
+      ...promptOptions,
+      validate: (v) => {
+        v = normalize ? normalize(v) : v
+        return (
+          this.setValueAndGetError(propertyPath, v) ||
+          (promptOptions.validate ? promptOptions.validate(v) : true) ||
+          true
+        )
       },
-    ])
+    })
 
     return result
   }
@@ -291,4 +284,16 @@ export class JsonSchemaPrompter<JsonResult> {
     await this.prompt(mainSchema.properties![p] as JSONSchema, p, customPrompt)
     return this.filledObject[p] as Exclude<JsonResult[P], undefined>
   }
+
+  async inquirerSinglePrompt(question: DistinctQuestion) {
+    const { result } = await inquirer.prompt([
+      {
+        ...question,
+        name: 'result',
+        prefix: '',
+      },
+    ])
+
+    return result
+  }
 }

cli/src/helpers/display.ts

@@ -44,7 +44,7 @@ export function displayTable(rows: { [k: string]: string | number }[], cellHoriz
   const maxLength = (columnName: string) =>
     rows.reduce((maxLength, row) => {
       const val = row[columnName]
-      const valLength = typeof val === 'string' ? val.length : val.toString().length
+      const valLength = typeof val === 'string' ? val.length : val !== undefined ? val.toString().length : 0
       return Math.max(maxLength, valLength)
     }, columnName.length)
   const columnDef = (columnName: string) => ({

content-directory-schemas/README.md

@@ -14,7 +14,7 @@ In order to make this documentation as clear as possible it is important to make
 In order to intialize the content directory on a development chain based on data that is provided in form of json files inside `/inputs` directory (`classes`, `schemas` and example entities - `entityBatches`), we can run:
 
 ```
-yarn workspace cd-schemas initialize:dev
+yarn workspace @joystream/cd-schemas initialize:dev
 ```
 
 This will handle:
@@ -51,7 +51,7 @@ For more context, see: https://code.visualstudio.com/docs/languages/json
 
 ### Validate inputs and `json-schemas` via a command
 
-All inputs inside `inputs` directory and `json-schemas` used to validate those inputs can also be validated using `yarn workspace cd-schemas validate` command. This is mainly to facilitate checking the validity of `.json` and `.schema.json` files inside `content-directory-schemas` through CI.
+All inputs inside `inputs` directory and `json-schemas` used to validate those inputs can also be validated using `yarn workspace @joystream/cd-schemas validate` command. This is mainly to facilitate checking the validity of `.json` and `.schema.json` files inside `content-directory-schemas` through CI.
 
 ### Entity batches
 
@@ -109,7 +109,7 @@ We can do it by either using `"new"` or `"existing"` keyword.
 There is a script that provides an easy way of converting `runtime-schemas` (based on inputs from `inputs/schemas`) to `json-schemas` (`.schema.json` files) which allow validating the input (ie. json files) describing some specific entities. It can be run with:
 
 ```
-yarn workspace cd-schemas generate:entity-schemas
+yarn workspace @joystream/cd-schemas generate:entity-schemas
 ```
 
 Those `json-schemas` are currently mainly used for validating the inputs inside `inputs/entityBatches`.
@@ -125,7 +125,7 @@ The generated `json-schemas` include:
 Thanks to the `json-schema-to-typescript` library, we can very simply generate Typescript interfaces based on existing `json-schemas`. This can be done via:
 
 ```
-yarn workspace cd-schemas generate:types
+yarn workspace @joystream/cd-schemas generate:types
 ```
 
 This command will generate:
@@ -153,19 +153,19 @@ The best way to ilustrate this would be by providing some examples:
 
 #### Creating a channel
 ```
-  import { InputParser } from 'cd-schemas'
-  import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
+  import { InputParser } from '@joystream/cd-schemas'
+  import { ChannelEntity } from '@joystream/cd-schemas/types/entities/ChannelEntity'
   // Other imports...
 
   async main() {
     // Initialize the api, SENDER_KEYPAIR and SENDER_MEMBER_ID...
 
     const channel: ChannelEntity = {
-      title: 'Example channel',
+      handle: 'Example channel',
       description: 'This is an example channel',
       language: { existing: { code: 'EN' } },
       coverPhotoUrl: '',
-      avatarPhotoURL: '',
+      avatarPhotoUrl: '',
       isPublic: true,
     }
 
@@ -182,12 +182,12 @@ The best way to ilustrate this would be by providing some examples:
       .signAndSend(SENDER_KEYPAIR)
   }
 ```
-_Full example with comments can be found in `content-directory-schemas/examples/createChannel.ts` and ran with `yarn workspace cd-schemas example:createChannel`_
+_Full example with comments can be found in `content-directory-schemas/examples/createChannel.ts` and ran with `yarn workspace @joystream/cd-schemas example:createChannel`_
 
 #### Creating a video
 ```
-import { InputParser } from 'cd-schemas'
-import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
+import { InputParser } from '@joystream/cd-schemas'
+import { VideoEntity } from '@joystream/cd-schemas/types/entities/VideoEntity'
 // ...
 
 async main() {
@@ -198,7 +198,7 @@ async main() {
     description: 'This is an example video',
     language: { existing: { code: 'EN' } },
     category: { existing: { name: 'Education' } },
-    channel: { existing: { title: 'Example channel' } },
+    channel: { existing: { handle: 'Example channel' } },
     media: {
       new: {
         encoding: { existing: { name: 'H.263_MP4' } },
@@ -221,7 +221,7 @@ async main() {
       },
     },
     duration: 3600,
-    thumbnailURL: '',
+    thumbnailUrl: '',
     isExplicit: false,
     isPublic: true,
   }
@@ -239,25 +239,25 @@ async main() {
     .signAndSend(SENDER_KEYPAIR)
 }
 ```
-_Full example with comments can be found in `content-directory-schemas/examples/createVideo.ts` and ran with `yarn workspace cd-schemas example:createChannel`_
+_Full example with comments can be found in `content-directory-schemas/examples/createVideo.ts` and ran with `yarn workspace @joystream/cd-schemas example:createChannel`_
 
-#### Update channel title
+#### Update channel handle
 
 ```
-import { InputParser } from 'cd-schemas'
-import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
+import { InputParser } from '@joystream/cd-schemas'
+import { ChannelEntity } from '@joystream/cd-schemas/types/entities/ChannelEntity'
 // ...
 
 async function main() {
   // ...
 
   const channelUpdateInput: Partial<ChannelEntity> = {
-    title: 'Updated channel title',
+    handle: 'Updated channel handle',
   }
 
   const parser = InputParser.createWithKnownSchemas(api)
 
-  const CHANNEL_ID = await parser.findEntityIdByUniqueQuery({ title: 'Example channel' }, 'Channel')
+  const CHANNEL_ID = await parser.findEntityIdByUniqueQuery({ handle: 'Example channel' }, 'Channel')
 
   const updateOperations = await parser.getEntityUpdateOperations(channelUpdateInput, 'Channel', CHANNEL_ID)
 
@@ -266,7 +266,7 @@ async function main() {
     .signAndSend(SENDER_KEYPAIR)
 }
 ```
-_Full example with comments can be found in `content-directory-schemas/examples/updateChannelTitle.ts` and ran with `yarn workspace cd-schemas example:updateChannelTitle`_
+_Full example with comments can be found in `content-directory-schemas/examples/updateChannelHandle.ts` and ran with `yarn workspace @joystream/cd-schemas example:updateChannelHandle`_
 
 Note: Updates can also inlucde `new` and `existing` keywords. In case `new` is specified inside the update - `CreateEntity` and `AddSchemaSupportToEntity` operations will be included as part of the operations returned by `InputParser.getEntityUpdateOperations`.
 

content-directory-schemas/examples/createChannel.ts

@@ -1,9 +1,9 @@
 import { ApiPromise, WsProvider } from '@polkadot/api'
 import { types as joyTypes } from '@joystream/types'
 import { Keyring } from '@polkadot/keyring'
-// Import input parser and channel entity from cd-schemas (we use it as library here)
-import { InputParser } from 'cd-schemas'
-import { ChannelEntity } from 'cd-schemas/types/entities'
+// Import input parser and channel entity from @joystream/cd-schemas (we use it as library here)
+import { InputParser } from '@joystream/cd-schemas'
+import { ChannelEntity } from '@joystream/cd-schemas/types/entities'
 
 async function main() {
   // Initialize the api
@@ -16,14 +16,14 @@ async function main() {
   const [ALICE] = keyring.getPairs()
 
   const channel: ChannelEntity = {
-    title: 'Example channel',
+    handle: 'Example channel',
     description: 'This is an example channel',
     // We can use "existing" syntax to reference either an on-chain entity or other entity that's part of the same batch.
     // Here we reference language that we assume was added by initialization script (initialize:dev), as it is part of
     // input/entityBatches/LanguageBatch.json
     language: { existing: { code: 'EN' } },
     coverPhotoUrl: '',
-    avatarPhotoURL: '',
+    avatarPhotoUrl: '',
     isPublic: true,
   }
   // Create the parser with known entity schemas (the ones in content-directory-schemas/inputs)

content-directory-schemas/examples/createChannelWithoutTransaction.ts

@@ -1,10 +1,10 @@
 import { ApiPromise, WsProvider } from '@polkadot/api'
 import { types as joyTypes } from '@joystream/types'
 import { Keyring } from '@polkadot/keyring'
-// Import input parser and channel entity from cd-schemas (we use it as library here)
-import { InputParser } from 'cd-schemas'
-import { ChannelEntity } from 'cd-schemas/types/entities'
-import { FlattenRelations } from 'cd-schemas/types/utility'
+// Import input parser and channel entity from @joystream/cd-schemas (we use it as library here)
+import { InputParser } from '@joystream/cd-schemas'
+import { ChannelEntity } from '@joystream/cd-schemas/types/entities'
+import { FlattenRelations } from '@joystream/cd-schemas/types/utility'
 import { EntityId } from '@joystream/types/content-directory'
 
 // Alternative way of creating a channel using separate extrinsics (instead of contentDirectory.transaction)
@@ -26,11 +26,11 @@ async function main() {
 
   // We use FlattenRelations to exlude { new } and { existing } (which are not allowed if we want to parse only a single entity)
   const channel: FlattenRelations<ChannelEntity> = {
-    title: 'Example channel 2',
+    handle: 'Example channel 2',
     description: 'This is an example channel',
     language: languageEntityId,
     coverPhotoUrl: '',
-    avatarPhotoURL: '',
+    avatarPhotoUrl: '',
     isPublic: true,
   }
 

content-directory-schemas/examples/createVideo.ts

@@ -1,9 +1,9 @@
 import { ApiPromise, WsProvider } from '@polkadot/api'
 import { types as joyTypes } from '@joystream/types'
 import { Keyring } from '@polkadot/keyring'
-// Import input parser and video entity from cd-schemas (we use it as library here)
-import { InputParser } from 'cd-schemas'
-import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
+// Import input parser and video entity from @joystream/cd-schemas (we use it as library here)
+import { InputParser } from '@joystream/cd-schemas'
+import { VideoEntity } from '@joystream/cd-schemas/types/entities/VideoEntity'
 
 async function main() {
   // Initialize the api
@@ -22,9 +22,9 @@ async function main() {
     // (those referenced here are part of inputs/entityBatches)
     language: { existing: { code: 'EN' } },
     category: { existing: { name: 'Education' } },
-    // We use the same "existing" syntax to reference a channel by unique property (title)
+    // We use the same "existing" syntax to reference a channel by unique property (handle)
     // In this case it's a channel that we created in createChannel example
-    channel: { existing: { title: 'Example channel' } },
+    channel: { existing: { handle: 'Example channel' } },
     media: {
       // We use "new" syntax to sygnalize we want to create a new VideoMedia entity that will be related to this Video entity
       new: {
@@ -46,7 +46,7 @@ async function main() {
       },
     },
     duration: 3600,
-    thumbnailURL: '',
+    thumbnailUrl: '',
     isExplicit: false,
     isPublic: true,
   }

content-directory-schemas/examples/updateChannelTitle.ts

@@ -1,9 +1,9 @@
 import { ApiPromise, WsProvider } from '@polkadot/api'
 import { types as joyTypes } from '@joystream/types'
 import { Keyring } from '@polkadot/keyring'
-// Import input parser and channel entity from cd-schemas (we use it as library here)
-import { InputParser } from 'cd-schemas'
-import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
+// Import input parser and channel entity from @joystream/cd-schemas (we use it as library here)
+import { InputParser } from '@joystream/cd-schemas'
+import { ChannelEntity } from '@joystream/cd-schemas/types/entities/ChannelEntity'
 
 async function main() {
   // Initialize the api
@@ -17,7 +17,7 @@ async function main() {
 
   // Create partial channel entity, only containing the fields we wish to update
   const channelUpdateInput: Partial<ChannelEntity> = {
-    title: 'Updated channel title',
+    handle: 'Updated channel handle',
   }
 
   // Create the parser with known entity schemas (the ones in content-directory-schemas/inputs)
@@ -25,7 +25,7 @@ async function main() {
 
   // We can reuse InputParser's `findEntityIdByUniqueQuery` method to find entityId of the channel we
   // created in ./createChannel.ts example (normally we would probably use some other way to do it, ie.: query node)
-  const CHANNEL_ID = await parser.findEntityIdByUniqueQuery({ title: 'Example channel' }, 'Channel')
+  const CHANNEL_ID = await parser.findEntityIdByUniqueQuery({ handle: 'Example channel' }, 'Channel')
 
   // Use getEntityUpdateOperations to parse the update input
   const updateOperations = await parser.getEntityUpdateOperations(

content-directory-schemas/examples/updateChannelTitleWithoutTransaction.ts

@@ -1,10 +1,10 @@
 import { ApiPromise, WsProvider } from '@polkadot/api'
 import { types as joyTypes } from '@joystream/types'
 import { Keyring } from '@polkadot/keyring'
-// Import input parser and channel entity from cd-schemas (we use it as library here)
-import { InputParser } from 'cd-schemas'
-import { ChannelEntity } from 'cd-schemas/types/entities'
-import { FlattenRelations } from 'cd-schemas/types/utility'
+// Import input parser and channel entity from @joystream/cd-schemas (we use it as library here)
+import { InputParser } from '@joystream/cd-schemas'
+import { ChannelEntity } from '@joystream/cd-schemas/types/entities'
+import { FlattenRelations } from '@joystream/cd-schemas/types/utility'
 
 // Alternative way of update a channel using updateEntityPropertyValues extrinsic
 async function main() {
@@ -19,7 +19,7 @@ async function main() {
 
   // Create partial channel entity, only containing the fields we wish to update
   const channelUpdateInput: Partial<FlattenRelations<ChannelEntity>> = {
-    title: 'Updated channel title 2',
+    handle: 'Updated channel handle 2',
   }
 
   // Create the parser with known entity schemas (the ones in content-directory-schemas/inputs)
@@ -28,7 +28,7 @@ async function main() {
   // We can reuse InputParser's `findEntityIdByUniqueQuery` method to find entityId of the channel we
   // created in ./createChannelWithoutTransaction.ts example
   // (normally we would probably use some other way to do it, ie.: query node)
-  const CHANNEL_ID = await parser.findEntityIdByUniqueQuery({ title: 'Example channel 2' }, 'Channel')
+  const CHANNEL_ID = await parser.findEntityIdByUniqueQuery({ handle: 'Example channel 2' }, 'Channel')
 
   // We use parser to create input property values map
   const newPropertyValues = await parser.parseToInputEntityValuesMap(channelUpdateInput, 'Channel')

content-directory-schemas/inputs/classes/FeaturedVideoClass.json

@@ -0,0 +1,6 @@
+{
+  "name": "FeaturedVideo",
+  "description": "Featured video references",
+  "maximum_entities_count": 400,
+  "default_entity_creation_voucher_upper_bound": 50
+}

content-directory-schemas/inputs/classes/index.js

@@ -0,0 +1,18 @@
+const EXPECTED_CLASS_ORDER = [
+  'Channel',
+  'ContentCategory',
+  'HttpMediaLocation',
+  'JoystreamMediaLocation',
+  'KnownLicense',
+  'Language',
+  'License',
+  'MediaLocation',
+  'UserDefinedLicense',
+  'Video',
+  'VideoMedia',
+  'VideoMediaEncoding',
+  'FeaturedVideo',
+]
+
+// Exports class input jsons in a predictable order
+module.exports = EXPECTED_CLASS_ORDER.map((className) => require(`./${className}Class.json`))

content-directory-schemas/inputs/entityBatches/ChannelBatch.json

@@ -1,13 +0,0 @@
-{
-  "className": "Channel",
-  "entries": [
-    {
-      "title": "Joystream Cartoons",
-      "description": "Joystream Cartoons channel",
-      "language": { "existing": { "code": "EN" } },
-      "coverPhotoUrl": "https://user-images.githubusercontent.com/4144334/91547902-7e90db00-e91c-11ea-9f5c-45d4921928d5.png",
-      "avatarPhotoURL": "https://user-images.githubusercontent.com/4144334/91546674-ba2aa580-e91a-11ea-96e2-abc7654c0461.png",
-      "isPublic": true
-    }
-  ]
-}

content-directory-schemas/inputs/entityBatches/VideoBatch.json

@@ -1,63 +0,0 @@
-{
-  "className": "Video",
-  "entries": [
-    {
-      "title": "Caminades 2",
-      "description": "Caminandes 2: Gran Dillama",
-      "language": { "existing": { "code": "EN" } },
-      "category": { "existing": { "name": "Film & Animation" } },
-      "channel": { "existing": { "title": "Joystream Cartoons" } },
-      "duration": 146,
-      "hasMarketing": false,
-      "isPublic": true,
-      "media": {
-        "new": {
-          "encoding": { "existing": { "name": "H.264_MP4" } },
-          "location": {
-            "new": {
-              "httpMediaLocation": {
-                "new": {
-                  "url": "http://www.caminandes.com/download/02_gran_dillama_1080p.zip"
-                }
-              }
-            }
-          },
-          "pixelWidth": 1920,
-          "pixelHeight": 1080
-        }
-      },
-      "thumbnailURL": "http://www.caminandes.com/wp-content/uploads/2016/02/web_header4.png",
-      "isExplicit": false,
-      "license": { "new": { "knownLicense": { "existing": { "code": "CC_BY" } } } }
-    },
-    {
-      "title": "Caminades 3",
-      "description": "Caminandes 3: Llamigos",
-      "language": { "existing": { "code": "EN" } },
-      "category": { "existing": { "name": "Film & Animation" } },
-      "channel": { "existing": { "title": "Joystream Cartoons" } },
-      "duration": 150,
-      "hasMarketing": false,
-      "isPublic": true,
-      "media": {
-        "new": {
-          "encoding": { "existing": { "name": "H.264_MP4" } },
-          "location": {
-            "new": {
-              "httpMediaLocation": {
-                "new": {
-                  "url": "http://www.caminandes.com/download/03_caminandes_llamigos_1080p.mp4"
-                }
-              }
-            }
-          },
-          "pixelWidth": 1920,
-          "pixelHeight": 1080
-        }
-      },
-      "thumbnailURL": "http://www.caminandes.com/wp-content/uploads/2016/02/web_header4.png",
-      "isExplicit": false,
-      "license": { "new": { "knownLicense": { "existing": { "code": "CC_BY" } } } }
-    }
-  ]
-}

content-directory-schemas/inputs/schemas/ChannelSchema.json

@@ -2,8 +2,8 @@
   "className": "Channel",
   "newProperties": [
     {
-      "name": "title",
-      "description": "The title of the Channel",
+      "name": "handle",
+      "description": "The handle of the Channel",
       "required": true,
       "unique": true,
       "property_type": { "Single": { "Text": 64 } }
@@ -17,13 +17,13 @@
     {
       "name": "coverPhotoUrl",
       "description": "Url for Channel's cover (background) photo. Recommended ratio: 16:9.",
-      "required": true,
+      "required": false,
       "property_type": { "Single": { "Text": 256 } }
     },
     {
-      "name": "avatarPhotoURL",
+      "name": "avatarPhotoUrl",
       "description": "Channel's avatar photo.",
-      "required": true,
+      "required": false,
       "property_type": { "Single": { "Text": 256 } }
     },
     {

content-directory-schemas/inputs/schemas/FeaturedVideoSchema.json

@@ -0,0 +1,12 @@
+{
+  "className": "FeaturedVideo",
+  "newProperties": [
+    {
+      "name": "video",
+      "description": "Reference to a video",
+      "required": true,
+      "unique": true,
+      "property_type": { "Single": { "Reference": { "className": "Video" } } }
+    }
+  ]
+}

content-directory-schemas/inputs/schemas/VideoSchema.json

@@ -38,7 +38,7 @@
       "property_type": { "Single": "Uint16" }
     },
     {
-      "name": "thumbnailURL",
+      "name": "thumbnailUrl",
       "description": "Video thumbnail url (recommended ratio: 16:9)",
       "required": true,
       "property_type": { "Single": { "Text": 256 } }

content-directory-schemas/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "cd-schemas",
+  "name": "@joystream/cd-schemas",
   "version": "0.1.0",
   "description": "JSON schemas, inputs and related tooling for Joystream content directory 2.0",
   "author": "Joystream contributors",
@@ -19,9 +19,9 @@
     "initialize:dev": "yarn initialize:alice-as-lead && yarn initialize:content-dir",
     "example:createChannel": "ts-node ./examples/createChannel.ts",
     "example:createVideo": "ts-node ./examples/createVideo.ts",
-    "example:updateChannelTitle": "ts-node ./examples/updateChannelTitle.ts",
+    "example:updateChannelHandle": "ts-node ./examples/updateChannelHandle.ts",
     "example:createChannelWithoutTransaction": "ts-node ./examples/createChannelWithoutTransaction.ts",
-    "example:updateChannelTitlelWithoutTransaction": "ts-node ./examples/updateChannelTitleWithoutTransaction.ts"
+    "example:updateChannelHandlelWithoutTransaction": "ts-node ./examples/updateChannelHandleWithoutTransaction.ts"
   },
   "dependencies": {
     "ajv": "6.12.5",
@@ -44,5 +44,9 @@
   "bugs": {
     "url": "https://github.com/Joystream/joystream/issues"
   },
-  "homepage": "https://github.com/Joystream/joystream"
+  "homepage": "https://github.com/Joystream/joystream",
+  "volta": {
+    "node": "12.18.2",
+    "yarn": "1.22.4"
+  }
 }

content-directory-schemas/scripts/initializeContentDir.ts

@@ -1,20 +1,15 @@
-import { CreateClass } from '../types/extrinsics/CreateClass'
-import { AddClassSchema } from '../types/extrinsics/AddClassSchema'
 import { types } from '@joystream/types'
 import { ApiPromise, WsProvider } from '@polkadot/api'
-import { getInputs } from '../src/helpers/inputs'
+import { getInitializationInputs } from '../src/helpers/inputs'
 import fs from 'fs'
 import path from 'path'
-import { EntityBatch } from '../types/EntityBatch'
 import { InputParser } from '../src/helpers/InputParser'
 import { ExtrinsicsHelper, getAlicePair } from '../src/helpers/extrinsics'
 
 // Save entity operations output here for easier debugging
 const ENTITY_OPERATIONS_OUTPUT_PATH = path.join(__dirname, '../operations.json')
 
-const classInputs = getInputs<CreateClass>('classes').map(({ data }) => data)
-const schemaInputs = getInputs<AddClassSchema>('schemas').map(({ data }) => data)
-const entityBatchInputs = getInputs<EntityBatch>('entityBatches').map(({ data }) => data)
+const { classInputs, schemaInputs, entityBatchInputs } = getInitializationInputs()
 
 async function main() {
   // Init api

content-directory-schemas/scripts/inputSchemasToEntitySchemas.ts

@@ -42,7 +42,7 @@ const HashPropertyDef = ({ Hash: maxLength }: HashProperty): JSONSchema7 => ({
 
 const ReferencePropertyDef = ({ Reference: ref }: ReferenceProperty): JSONSchema7 => ({
   'oneOf': [
-    onePropertyObjectDef('new', { '$ref': `./${ref.className}Entity.schema.json` }),
+    onePropertyObjectDef('new', { '$ref': `../entities/${ref.className}Entity.schema.json` }),
     onePropertyObjectDef('existing', { '$ref': `../entityReferences/${ref.className}Ref.schema.json` }),
     PRIMITIVE_PROPERTY_DEFS.definitions.Uint64 as JSONSchema7,
   ],

content-directory-schemas/src/helpers/InputParser.ts

@@ -18,7 +18,7 @@ import { ApiPromise } from '@polkadot/api'
 import { JoyBTreeSet } from '@joystream/types/common'
 import { CreateClass } from '../../types/extrinsics/CreateClass'
 import { EntityBatch } from '../../types/EntityBatch'
-import { getInputs } from './inputs'
+import { getInitializationInputs, getInputs } from './inputs'
 
 type SimpleEntityValue = string | boolean | number | string[] | boolean[] | number[] | undefined | null
 // Input without "new" or "extising" keywords
@@ -38,12 +38,8 @@ export class InputParser {
   private classIdByNameMap = new Map<string, number>()
 
   static createWithInitialInputs(api: ApiPromise): InputParser {
-    return new InputParser(
-      api,
-      getInputs<CreateClass>('classes').map(({ data }) => data),
-      getInputs<AddClassSchema>('schemas').map(({ data }) => data),
-      getInputs<EntityBatch>('entityBatches').map(({ data }) => data)
-    )
+    const { classInputs, schemaInputs, entityBatchInputs } = getInitializationInputs()
+    return new InputParser(api, classInputs, schemaInputs, entityBatchInputs)
   }
 
   static createWithKnownSchemas(api: ApiPromise, entityBatches?: EntityBatch[]): InputParser {
@@ -198,7 +194,7 @@ export class InputParser {
         const schemaPropertyType = schema.newProperties.find((p) => p.name === propertyName)!.property_type
         // Handle entities "nested" via "new"
         if (isSingle(schemaPropertyType) && isReference(schemaPropertyType.Single)) {
-          if (Object.keys(propertyValue).includes('new')) {
+          if (propertyValue !== null && Object.keys(propertyValue).includes('new')) {
             const refEntitySchema = this.schemaByClassName(schemaPropertyType.Single.Reference.className)
             this.includeEntityInputInUniqueQueryMap(propertyValue.new, refEntitySchema)
           }

content-directory-schemas/src/helpers/inputs.ts

@@ -1,5 +1,7 @@
 import path from 'path'
 import fs from 'fs'
+import { CreateClass, AddClassSchema } from '../../types/extrinsics'
+import { EntityBatch } from '../../types/EntityBatch'
 
 export const INPUTS_LOCATION = path.join(__dirname, '../../inputs')
 export const INPUT_TYPES = ['classes', 'schemas', 'entityBatches'] as const
@@ -9,12 +11,30 @@ export type FetchedInput<Schema = any> = { fileName: string; data: Schema }
 
 export const getInputsLocation = (inputType: InputType) => path.join(INPUTS_LOCATION, inputType)
 
-export function getInputs<Schema = any>(inputType: InputType): FetchedInput<Schema>[] {
-  return fs.readdirSync(getInputsLocation(inputType)).map((fileName) => {
-    const inputJson = fs.readFileSync(path.join(INPUTS_LOCATION, inputType, fileName)).toString()
-    return {
+export function getInputs<Schema = any>(
+  inputType: InputType,
+  rootInputsLocation = INPUTS_LOCATION
+): FetchedInput<Schema>[] {
+  const inputs: FetchedInput<Schema>[] = []
+  fs.readdirSync(path.join(rootInputsLocation, inputType)).forEach((fileName) => {
+    const inputFilePath = path.join(rootInputsLocation, inputType, fileName)
+    if (path.extname(inputFilePath) !== '.json') {
+      return
+    }
+    const inputJson = fs.readFileSync(inputFilePath).toString()
+    inputs.push({
       fileName,
       data: JSON.parse(inputJson) as Schema,
-    }
+    })
   })
+  return inputs
+}
+
+export function getInitializationInputs(rootInputsLocation = INPUTS_LOCATION) {
+  return {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    classInputs: require('../../inputs/classes/index.js') as CreateClass[],
+    schemaInputs: getInputs<AddClassSchema>('schemas').map(({ data }) => data),
+    entityBatchInputs: getInputs<EntityBatch>('entityBatches').map(({ data }) => data),
+  }
 }

content-directory-schemas/src/index.ts

@@ -1,6 +1,6 @@
 export { ExtrinsicsHelper, getAlicePair } from './helpers/extrinsics'
 export { InputParser } from './helpers/InputParser'
-export { getInputs, getInputsLocation } from './helpers/inputs'
+export { getInputs, getInitializationInputs, getInputsLocation } from './helpers/inputs'
 export { isReference, isSingle } from './helpers/propertyType'
 export { getSchemasLocation } from './helpers/schemas'
 export { default as initializeContentDir } from './helpers/initialize'

docker-compose-with-storage.yml

@@ -1,39 +0,0 @@
-version: '3'
-services:
-  ipfs:
-    image: ipfs/go-ipfs:latest
-    ports:
-      - '127.0.0.1:5001:5001'
-      - '127.0.0.1:8080:8080'
-    entrypoint: ''
-    command: |
-      /bin/sh -c "
-        set -e
-        /usr/local/bin/start_ipfs config profile apply lowpower
-        /usr/local/bin/start_ipfs config --json Gateway.PublicGateways '{\"localhost\": null }'
-        /sbin/tini -- /usr/local/bin/start_ipfs daemon --migrate=true
-      "
-  chain:
-    image: joystream/node
-    build:
-      context: .
-      dockerfile: joystream-node.Dockerfile
-    ports:
-      - '127.0.0.1:9944:9944'
-    command: --dev --ws-external --base-path /data --log runtime
-
-  colossus:
-    image: joystream/apps
-    restart: on-failure
-    depends_on:
-      - "chain"
-      - "ipfs"
-    build:
-      context: .
-      dockerfile: apps.Dockerfile
-    ports:
-      - '127.0.0.1:3001:3001'
-    command: colossus --dev --ws-provider ws://chain:9944 --ipfs-host ipfs
-    environment:
-      - DEBUG=*
-

docker-compose.yml

@@ -1,17 +1,150 @@
-# Compiles new joystream node image if local image not found,
-# and runs local development chain.
-# To prevent build run docker-compose with "--no-build" arg
-version: "3"
+# Compiles new joystream/node and joystream/apps images if local images not found
+# and runs a complete joystream development network
+# To prevent build of docker images run docker-compose with "--no-build" arg
+version: "3.4"
 services:
   joystream-node:
-    image: joystream/node
+    image: joystream/node:latest
     build:
       # context is relative to the compose file
       context: .
       # dockerfile is relative to the context
       dockerfile: joystream-node.Dockerfile
     container_name: joystream-node
-    command: --dev --alice --validator --unsafe-ws-external --rpc-cors=all --log runtime
+    volumes:
+      - /data
+    command: --dev --alice --validator --unsafe-ws-external --rpc-cors=all --log runtime --base-path /data
     ports:
-      - "9944:9944"
-  
+      - "127.0.0.1:9944:9944"
+
+  ipfs:
+    image: ipfs/go-ipfs:latest
+    ports:
+      - '127.0.0.1:5001:5001'
+      - '127.0.0.1:8080:8080'
+    volumes:
+      - /data/ipfs
+    entrypoint: ''
+    command: |
+      /bin/sh -c "
+        set -e
+        /usr/local/bin/start_ipfs config profile apply lowpower
+        /usr/local/bin/start_ipfs config --json Gateway.PublicGateways '{\"localhost\": null }'
+        /sbin/tini -- /usr/local/bin/start_ipfs daemon --migrate=true
+      "
+
+  colossus:
+    image: joystream/apps
+    restart: on-failure
+    depends_on:
+      - "joystream-node"
+      - "ipfs"
+    build:
+      context: .
+      dockerfile: apps.Dockerfile
+    ports:
+      - '127.0.0.1:3001:3001'
+    command: colossus --dev --ws-provider ${WS_PROVIDER_ENDPOINT_URI} --ipfs-host ipfs
+    environment:
+      - DEBUG=*
+
+  db:
+    image: postgres:12
+    restart: always
+    ports:
+      - "127.0.0.1:${DB_PORT}:5432"
+    volumes:
+      - /var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: ${DB_USER}
+      POSTGRES_PASSWORD: ${DB_PASS}
+      POSTGRES_DB: ${DB_NAME}
+
+  graphql-server:
+    image: joystream/apps
+    restart: unless-stopped
+    build: 
+      context: .
+      dockerfile: apps.Dockerfile
+    env_file:
+      # relative to working directory where docker-compose was run from 
+      - .env
+    environment:
+      - DB_HOST=db
+    ports:
+      - "127.0.0.1:8081:${GRAPHQL_SERVER_PORT}"
+    depends_on: 
+      - db
+    command: ["workspace", "query-node-root", "server:start:prod"]
+
+  processor:
+    image: joystream/apps
+    restart: unless-stopped
+    build: 
+      context: .
+      dockerfile: apps.Dockerfile
+    env_file:
+      # relative to working directory where docker-compose was run from 
+      - .env
+    environment:
+      - INDEXER_ENDPOINT_URL=http://indexer-api-gateway:4000/graphql
+      - DB_HOST=db
+      - TYPEORM_HOST=db
+      - DEBUG=index-builder:*
+      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
+    depends_on:
+      - indexer-api-gateway
+    command: ["workspace", "query-node-root", "processor:start"]
+
+  indexer:
+    image: joystream/apps
+    restart: unless-stopped
+    build: 
+      context: .
+      dockerfile: apps.Dockerfile
+    env_file:
+      # relative to working directory where docker-compose was run from 
+      - .env 
+    environment:
+      - TYPEORM_HOST=db
+      - INDEXER_WORKERS=5
+      - PROCESSOR_POLL_INTERVAL=1000 # refresh every second 
+      - REDIS_URI=redis://redis:6379/0
+      - DEBUG=index-builder:*
+      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
+    depends_on: 
+      - db
+    command: ["workspace", "query-node-root", "indexer:start"] 
+
+  indexer-api-gateway:
+    image: joystream/hydra-indexer-gateway:latest
+    restart: unless-stopped
+    environment:
+      - WARTHOG_STARTER_DB_DATABASE=${DB_NAME}
+      - WARTHOG_STARTER_DB_HOST=db 
+      - WARTHOG_STARTER_DB_PASSWORD=${DB_PASS}
+      - WARTHOG_STARTER_DB_PORT=${DB_PORT}
+      - WARTHOG_STARTER_DB_USERNAME=${DB_USER}
+      - WARTHOG_STARTER_REDIS_URI=redis://redis:6379/0 
+      - PORT=4000
+    ports:
+      - "127.0.0.1:4000:4000"
+    depends_on:
+      - redis
+      - db
+      - indexer
+
+  redis:
+    image: redis:6.0-alpine
+    restart: always
+    ports:
+      - "127.0.0.1:6379:6379"
+
+  pioneer:
+    image: joystream/apps
+    build:
+      context: .
+      dockerfile: apps.Dockerfile
+    ports:
+      - "127.0.0.1:3000:3000"
+    command: workspace pioneer start

joystream-node.Dockerfile

@@ -1,4 +1,11 @@
-FROM joystream/rust-builder AS builder
+FROM liuchong/rustup:1.46.0 AS rustup
+RUN rustup component add rustfmt clippy
+RUN rustup install nightly-2020-05-23 --force
+RUN rustup target add wasm32-unknown-unknown --toolchain nightly-2020-05-23
+RUN apt-get update && \
+  apt-get install -y curl git gcc xz-utils sudo pkg-config unzip clang libc6-dev-i386
+
+FROM rustup AS builder
 LABEL description="Compiles all workspace artifacts"
 WORKDIR /joystream
 COPY . /joystream

package.json

@@ -4,7 +4,9 @@
   "version": "1.0.0",
   "license": "GPL-3.0-only",
   "scripts": {
-    "postinstall": "yarn workspace @joystream/types build && yarn workspace cd-schemas generate:all && yarn workspace cd-schemas build && yarn workspace @joystream/cli build",
+    "postinstall": "yarn workspace @joystream/types build && yarn workspace @joystream/cd-schemas generate:all && yarn workspace @joystream/cd-schemas build && yarn workspace @joystream/cli build",
+    "build": "./build.sh",
+    "start": "./start.sh",
     "cargo-checks": "devops/git-hooks/pre-commit && devops/git-hooks/pre-push",
     "cargo-build": "scripts/cargo-build.sh"
   },
@@ -50,5 +52,9 @@
   "engines": {
     "node": ">=12.18.0",
     "yarn": "^1.22.0"
+  },
+  "volta": {
+    "node": "12.18.2",
+    "yarn": "1.22.4"
   }
 }

pioneer/package.json

@@ -93,5 +93,8 @@
     "sass-loader": "^8.0.0",
     "style-loader": "^1.0.0",
     "@joystream/types": "link:../types"
+  },
+  "volta": {
+    "extends": "../package.json"
   }
 }

query-node/.env

@@ -1,3 +1,5 @@
+COMPOSE_PROJECT_NAME=joystream
+
 # Project name
 PROJECT_NAME=query_node
 

query-node/docker-compose.yml

@@ -1,91 +0,0 @@
-version: "3.4"
-
-services:
-  db:
-    image: postgres:12
-    restart: always
-    ports:
-      - "${DB_PORT}:5432"
-    volumes:
-      - /var/lib/postgresql/data
-    environment:
-      POSTGRES_USER: ${DB_USER}
-      POSTGRES_PASSWORD: ${DB_PASS}
-      POSTGRES_DB: ${DB_NAME}
-
-  graphql-server:
-    image: joystream/apps
-    restart: unless-stopped
-    build: 
-      context: ../
-      dockerfile: apps.Dockerfile
-    env_file:
-      - .env
-    environment:
-      - DB_HOST=db
-    ports:
-      - "8080:${GRAPHQL_SERVER_PORT}"
-    depends_on: 
-      - db
-    command: ["workspace", "query-node-root", "server:start:prod"]
-
-  processor:
-    image: joystream/apps
-    restart: unless-stopped
-    build: 
-      context: ../
-      dockerfile: apps.Dockerfile
-    env_file:
-      - .env
-    environment:
-      - INDEXER_ENDPOINT_URL=http://indexer-api-gateway:4000/graphql
-      - DB_HOST=db
-      - TYPEORM_HOST=db
-      - DEBUG=index-builder:*
-      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
-    depends_on:
-      - indexer-api-gateway
-    command: ["workspace", "query-node-root", "processor:start"]
-  
-  indexer:
-    image: joystream/apps
-    restart: unless-stopped
-    build: 
-      context: ../
-      dockerfile: apps.Dockerfile
-    env_file:
-      - .env 
-    environment:
-      - TYPEORM_HOST=db
-      - INDEXER_WORKERS=5
-      - PROCESSOR_POLL_INTERVAL=1000 # refresh every second 
-      - REDIS_URI=redis://redis:6379/0
-      - DEBUG=index-builder:*
-      - WS_PROVIDER_ENDPOINT_URI=${WS_PROVIDER_ENDPOINT_URI}
-    depends_on: 
-      - db
-    command: ["workspace", "query-node-root", "indexer:start"] 
-  
-  indexer-api-gateway:
-    image: joystream/hydra-indexer-gateway:latest
-    restart: unless-stopped
-    environment:
-      - WARTHOG_STARTER_DB_DATABASE=${DB_NAME}
-      - WARTHOG_STARTER_DB_HOST=db 
-      - WARTHOG_STARTER_DB_PASSWORD=${DB_PASS}
-      - WARTHOG_STARTER_DB_PORT=${DB_PORT}
-      - WARTHOG_STARTER_DB_USERNAME=${DB_USER}
-      - WARTHOG_STARTER_REDIS_URI=redis://redis:6379/0 
-      - PORT=4000
-    ports:
-      - "4000:4000"
-    depends_on:
-      - redis
-      - db
-      - indexer
-    
-  redis:
-    image: redis:6.0-alpine
-    restart: always
-    ports:
-      - "6379:6379"

query-node/indexer-tsconfig.json

@@ -14,7 +14,8 @@
     "baseUrl": ".",
     "paths": {
       "@polkadot/types/augment": ["../../../types/augment-codec/augment-types.ts"]
-    }
+    },
+    "esModuleInterop": true
   },
   "exclude": ["node_modules"]
 }

query-node/mappings/content-directory/content-dir-consts.ts

@@ -1,16 +1,18 @@
-import { IPropertyIdWithName } from '../types'
+import { IPropertyWithId } from '../types'
 
 // Content directory predefined class names
 export enum ContentDirectoryKnownClasses {
   CHANNEL = 'Channel',
   CATEGORY = 'Category',
+  HTTPMEDIALOCATION = 'HttpMediaLocation',
+  JOYSTREAMMEDIALOCATION = 'JoystreamMediaLocation',
   KNOWNLICENSE = 'KnownLicense',
+  LANGUAGE = 'Language',
+  LICENSE = 'License',
+  MEDIALOCATION = 'MediaLocation',
   USERDEFINEDLICENSE = 'UserDefinedLicense',
-  JOYSTREAMMEDIALOCATION = 'JoystreamMediaLocation',
-  HTTPMEDIALOCATION = 'HttpMediaLocation',
-  VIDEOMEDIA = 'VideoMedia',
   VIDEO = 'Video',
-  LANGUAGE = 'Language',
+  VIDEOMEDIA = 'VideoMedia',
   VIDEOMEDIAENCODING = 'VideoMediaEncoding',
 }
 
@@ -18,85 +20,97 @@ export enum ContentDirectoryKnownClasses {
 export const contentDirectoryClassNamesWithId: { classId: number; name: string }[] = [
   { name: ContentDirectoryKnownClasses.CHANNEL, classId: 1 },
   { name: ContentDirectoryKnownClasses.CATEGORY, classId: 2 },
+  { name: ContentDirectoryKnownClasses.HTTPMEDIALOCATION, classId: 3 },
+  { name: ContentDirectoryKnownClasses.JOYSTREAMMEDIALOCATION, classId: 4 },
   { name: ContentDirectoryKnownClasses.KNOWNLICENSE, classId: 5 },
-  { name: ContentDirectoryKnownClasses.USERDEFINEDLICENSE, classId: 9 },
   { name: ContentDirectoryKnownClasses.LANGUAGE, classId: 6 },
-  { name: ContentDirectoryKnownClasses.JOYSTREAMMEDIALOCATION, classId: 4 },
-  { name: ContentDirectoryKnownClasses.HTTPMEDIALOCATION, classId: 3 },
-  { name: ContentDirectoryKnownClasses.VIDEOMEDIA, classId: 11 },
+  { name: ContentDirectoryKnownClasses.LICENSE, classId: 7 },
+  { name: ContentDirectoryKnownClasses.MEDIALOCATION, classId: 8 },
+  { name: ContentDirectoryKnownClasses.USERDEFINEDLICENSE, classId: 9 },
   { name: ContentDirectoryKnownClasses.VIDEO, classId: 10 },
+  { name: ContentDirectoryKnownClasses.VIDEOMEDIA, classId: 11 },
   { name: ContentDirectoryKnownClasses.VIDEOMEDIAENCODING, classId: 12 },
 ]
 
-export const CategoryPropertyNamesWithId: IPropertyIdWithName = {
-  0: 'name',
-  1: 'description',
+export const categoryPropertyNamesWithId: IPropertyWithId = {
+  0: { name: 'name', type: 'string', required: true },
+  1: { name: 'description', type: 'string', required: false },
+}
+
+export const channelPropertyNamesWithId: IPropertyWithId = {
+  0: { name: 'handle', type: 'string', required: true },
+  1: { name: 'description', type: 'string', required: false },
+  2: { name: 'coverPhotoUrl', type: 'string', required: false },
+  3: { name: 'avatarPhotoUrl', type: 'string', required: false },
+  4: { name: 'isPublic', type: 'boolean', required: true },
+  5: { name: 'isCurated', type: 'boolean', required: false },
+  6: { name: 'language', type: 'number', required: false },
+}
+
+export const licensePropertyNamesWithId: IPropertyWithId = {
+  0: { name: 'knownLicense', type: 'number', required: false },
+  1: { name: 'userDefinedLicense', type: 'number', required: false },
 }
 
-export const channelPropertyNamesWithId: IPropertyIdWithName = {
-  0: 'title',
-  1: 'description',
-  2: 'coverPhotoURL',
-  3: 'avatarPhotoURL',
-  4: 'isPublic',
-  5: 'isCurated',
-  6: 'language',
+export const knownLicensePropertyNamesWIthId: IPropertyWithId = {
+  0: { name: 'code', type: 'string', required: true },
+  1: { name: 'name', type: 'string', required: false },
+  2: { name: 'description', type: 'string', required: false },
+  3: { name: 'url', type: 'string', required: false },
 }
 
-export const knownLicensePropertyNamesWIthId: IPropertyIdWithName = {
-  0: 'code',
-  1: 'name',
-  2: 'description',
-  3: 'url',
+export const languagePropertyNamesWIthId: IPropertyWithId = {
+  0: { name: 'name', type: 'string', required: true },
+  1: { name: 'code', type: 'string', required: true },
 }
 
-export const languagePropertyNamesWIthId: IPropertyIdWithName = {
-  0: 'name',
-  1: 'code',
+export const userDefinedLicensePropertyNamesWithId: IPropertyWithId = {
+  0: { name: 'content', type: 'string', required: false },
 }
 
-export const userDefinedLicensePropertyNamesWithId: IPropertyIdWithName = {
-  0: 'content',
+export const mediaLocationPropertyNamesWithId: IPropertyWithId = {
+  0: { name: 'httpMediaLocation', type: 'number', required: false },
+  1: { name: 'joystreamMediaLocation', type: 'number', required: false },
 }
 
-export const joystreamMediaLocationPropertyNamesWithId: IPropertyIdWithName = {
-  0: 'dataObjectId',
+export const joystreamMediaLocationPropertyNamesWithId: IPropertyWithId = {
+  0: { name: 'dataObjectId', type: 'string', required: true },
 }
 
-export const httpMediaLocationPropertyNamesWithId: IPropertyIdWithName = {
-  0: 'url',
-  1: 'port',
+export const httpMediaLocationPropertyNamesWithId: IPropertyWithId = {
+  0: { name: 'url', type: 'string', required: false },
+  1: { name: 'port', type: 'number', required: false },
 }
 
-export const videoMediaEncodingPropertyNamesWithId: IPropertyIdWithName = {
-  0: 'name',
+export const videoMediaEncodingPropertyNamesWithId: IPropertyWithId = {
+  0: { name: 'name', type: 'string', required: true },
 }
 
-export const videoMediaPropertyNamesWithId: IPropertyIdWithName = {
-  0: 'encoding',
-  1: 'pixelWidth',
-  2: 'pixelHeight',
-  3: 'size',
-  4: 'location',
+export const videoMediaPropertyNamesWithId: IPropertyWithId = {
+  0: { name: 'encoding', type: 'number', required: true },
+  1: { name: 'pixelWidth', type: 'number', required: true },
+  2: { name: 'pixelHeight', type: 'number', required: true },
+  3: { name: 'size', type: 'number', required: false },
+  4: { name: 'location', type: 'number', required: true },
 }
 
-export const videoPropertyNamesWithId: IPropertyIdWithName = {
+export const videoPropertyNamesWithId: IPropertyWithId = {
   // referenced entity's id
-  0: 'channel',
+  0: { name: 'channel', type: 'number', required: true },
   // referenced entity's id
-  1: 'category',
-  2: 'title',
-  3: 'description',
-  4: 'duration',
-  5: 'skippableIntroDuration',
-  6: 'thumbnailURL',
-  7: 'language',
+  1: { name: 'category', type: 'number', required: true },
+  2: { name: 'title', type: 'string', required: false },
+  3: { name: 'description', type: 'string', required: false },
+  4: { name: 'duration', type: 'number', required: true },
+  5: { name: 'skippableIntroDuration', type: 'number', required: false },
+  6: { name: 'thumbnailUrl', type: 'string', required: true },
+  7: { name: 'language', type: 'number', required: false },
   // referenced entity's id
-  8: 'media',
-  9: 'hasMarketing',
-  10: 'publishedBeforeJoystream',
-  11: 'isPublic',
-  12: 'isExplicit',
-  13: 'license',
-  14: 'isCurated',
+  8: { name: 'media', type: 'number', required: true },
+  9: { name: 'hasMarketing', type: 'boolean', required: false },
+  10: { name: 'publishedBeforeJoystream', type: 'number', required: false },
+  11: { name: 'isPublic', type: 'boolean', required: true },
+  12: { name: 'isExplicit', type: 'boolean', required: true },
+  13: { name: 'license', type: 'number', required: true },
+  14: { name: 'isCurated', type: 'boolean', required: true },
 }

query-node/mappings/content-directory/decode.ts

@@ -1,35 +1,46 @@
 import { SubstrateEvent } from '../../generated/indexer'
 import {
-  IPropertyIdWithName,
   IClassEntity,
   IProperty,
   IBatchOperation,
   ICreateEntityOperation,
   IEntity,
+  IReference,
+  IPropertyWithId,
 } from '../types'
+import Debug from 'debug'
 
 import { ParametrizedClassPropertyValue, UpdatePropertyValuesOperation } from '@joystream/types/content-directory'
 import { createType } from '@joystream/types'
 
+const debug = Debug('mappings:cd:decode')
+
 function stringIfyEntityId(event: SubstrateEvent): string {
   const { 1: entityId } = event.params
   return entityId.value as string
 }
 
-function setProperties<T>({ extrinsic, blockNumber }: SubstrateEvent, propNamesWithId: IPropertyIdWithName): T {
+function setProperties<T>({ extrinsic, blockNumber }: SubstrateEvent, propNamesWithId: IPropertyWithId): T {
   if (extrinsic === undefined) throw Error('Undefined extrinsic')
 
   const { 3: newPropertyValues } = extrinsic!.args
-  const properties: { [key: string]: any } = {}
+  const properties: { [key: string]: any; reference?: IReference } = {}
 
   for (const [k, v] of Object.entries(newPropertyValues.value)) {
-    const propertyName = propNamesWithId[k]
-    const propertyValue = createType('InputPropertyValue', v as any)
-      .asType('Single')
-      .value.toJSON()
-    properties[propertyName] = propertyValue
+    const prop = propNamesWithId[k]
+    const singlePropVal = createType('InputPropertyValue', v as any).asType('Single')
+
+    if (singlePropVal.isOfType('Reference')) {
+      properties[prop.name] = { entityId: singlePropVal.asType('Reference').toJSON(), existing: true }
+    } else {
+      const val = singlePropVal.value.toJSON()
+      if (typeof val !== prop.type && !prop.required) properties[prop.name] = undefined
+      else properties[prop.name] = val
+    }
   }
   properties.version = blockNumber
+
+  debug(`Entity properties: ${JSON.stringify(properties)}`)
   return properties as T
 }
 
@@ -48,16 +59,18 @@ function getClassEntity(event: SubstrateEvent): IClassEntity {
  * @param properties
  * @param propertyNamesWithId
  */
-function setEntityPropertyValues<T>(properties: IProperty[], propertyNamesWithId: IPropertyIdWithName): T {
-  const entityProperties: { [key: string]: any } = {}
+function setEntityPropertyValues<T>(properties: IProperty[], propertyNamesWithId: IPropertyWithId): T {
+  const entityProperties: { [key: string]: any; reference?: IReference } = {}
 
   for (const [propId, propName] of Object.entries(propertyNamesWithId)) {
     // get the property value by id
-    const p = properties.find((p) => p.propertyId === propId)
-    const propertyValue = p ? p.value : undefined
-    entityProperties[propName] = propertyValue
+    const p = properties.find((p) => p.id === propId)
+    if (!p) continue
+
+    if (typeof p.value !== propName.type && !propName.required) entityProperties[propName.name] = undefined
+    else entityProperties[propName.name] = p.reference ? p.reference : p.value
   }
-  // console.log(entityProperties);
+  debug(`Entity properties: ${JSON.stringify(entityProperties)}`)
   return entityProperties as T
 }
 
@@ -70,20 +83,27 @@ function getEntityProperties(propertyValues: ParametrizedClassPropertyValue[]):
     const v = createType('ParametrizedPropertyValue', pv.value)
     const propertyId = pv.in_class_index.toJSON()
 
+    let reference
     let value
     if (v.isOfType('InputPropertyValue')) {
       const inputPropVal = v.asType('InputPropertyValue')
       value = inputPropVal.isOfType('Single')
         ? inputPropVal.asType('Single').value.toJSON()
         : inputPropVal.asType('Vector').value.toJSON()
+
+      if (inputPropVal.isOfType('Single')) {
+        if (inputPropVal.asType('Single').isOfType('Reference')) {
+          reference = { entityId: value as number, existing: true }
+        }
+      }
     } else if (v.isOfType('InternalEntityJustAdded')) {
-      // const inputPropVal = v.asType('InternalEntityJustAdded');
       value = v.asType('InternalEntityJustAdded').toJSON()
+      reference = { entityId: value as number, existing: false }
     } else {
       // TODO: Add support for v.asType('InternalEntityVec')
       throw Error('InternalEntityVec property type is not supported yet!')
     }
-    properties.push({ propertyId: `${propertyId}`, value })
+    properties.push({ id: `${propertyId}`, value, reference })
   })
   return properties
 }

query-node/mappings/content-directory/entity-helper.ts

@@ -1,476 +0,0 @@
-import { DB, SubstrateEvent } from '../../generated/indexer'
-import { Channel } from '../../generated/graphql-server/src/modules/channel/channel.model'
-import { Category } from '../../generated/graphql-server/src/modules/category/category.model'
-import { KnownLicense } from '../../generated/graphql-server/src/modules/known-license/known-license.model'
-import { UserDefinedLicense } from '../../generated/graphql-server/src/modules/user-defined-license/user-defined-license.model'
-import { JoystreamMediaLocation } from '../../generated/graphql-server/src/modules/joystream-media-location/joystream-media-location.model'
-import { HttpMediaLocation } from '../../generated/graphql-server/src/modules/http-media-location/http-media-location.model'
-import { VideoMedia } from '../../generated/graphql-server/src/modules/video-media/video-media.model'
-import { Video } from '../../generated/graphql-server/src/modules/video/video.model'
-import { Block, Network } from '../../generated/graphql-server/src/modules/block/block.model'
-import { Language } from '../../generated/graphql-server/src/modules/language/language.model'
-import { VideoMediaEncoding } from '../../generated/graphql-server/src/modules/video-media-encoding/video-media-encoding.model'
-import { ClassEntity } from '../../generated/graphql-server/src/modules/class-entity/class-entity.model'
-import { decode } from './decode'
-import {
-  CategoryPropertyNamesWithId,
-  channelPropertyNamesWithId,
-  httpMediaLocationPropertyNamesWithId,
-  joystreamMediaLocationPropertyNamesWithId,
-  knownLicensePropertyNamesWIthId,
-  languagePropertyNamesWIthId,
-  userDefinedLicensePropertyNamesWithId,
-  videoMediaEncodingPropertyNamesWithId,
-  videoPropertyNamesWithId,
-  contentDirectoryClassNamesWithId,
-  ContentDirectoryKnownClasses,
-} from './content-dir-consts'
-import {
-  ICategory,
-  IChannel,
-  ICreateEntityOperation,
-  IDBBlockId,
-  IEntity,
-  IHttpMediaLocation,
-  IJoystreamMediaLocation,
-  IKnownLicense,
-  ILanguage,
-  IUserDefinedLicense,
-  IVideo,
-  IVideoMedia,
-  IVideoMediaEncoding,
-  IWhereCond,
-} from '../types'
-
-async function createBlockOrGetFromDatabase(db: DB, blockNumber: number): Promise<Block> {
-  let b = await db.get(Block, { where: { block: blockNumber } })
-  if (b === undefined) {
-    // TODO: get timestamp from the event or extrinsic
-    b = new Block({ block: blockNumber, nework: Network.BABYLON, timestamp: 123 })
-    await db.save<Block>(b)
-  }
-  return b
-}
-
-async function createChannel({ db, block, id }: IDBBlockId, p: IChannel): Promise<void> {
-  // const { properties: p } = decode.channelEntity(event);
-  const channel = new Channel()
-
-  channel.version = block
-  channel.id = id
-  channel.title = p.title
-  channel.description = p.description
-  channel.isCurated = p.isCurated || false
-  channel.isPublic = p.isPublic
-  channel.coverPhotoUrl = p.coverPhotoURL
-  channel.avatarPhotoUrl = p.avatarPhotoURL
-  channel.languageId = p.language
-  channel.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save(channel)
-}
-
-async function createCategory({ db, block, id }: IDBBlockId, p: ICategory): Promise<void> {
-  // const p = decode.categoryEntity(event);
-  const category = new Category()
-
-  category.id = id
-  category.name = p.name
-  category.description = p.description
-  category.version = block
-  category.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save(category)
-}
-
-async function createKnownLicense({ db, block, id }: IDBBlockId, p: IKnownLicense): Promise<void> {
-  const knownLicence = new KnownLicense()
-
-  knownLicence.id = id
-  knownLicence.code = p.code
-  knownLicence.name = p.name
-  knownLicence.description = p.description
-  knownLicence.url = p.url
-  knownLicence.version = block
-  knownLicence.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save(knownLicence)
-}
-
-async function createUserDefinedLicense({ db, block, id }: IDBBlockId, p: IUserDefinedLicense): Promise<void> {
-  const userDefinedLicense = new UserDefinedLicense()
-
-  userDefinedLicense.id = id
-  userDefinedLicense.content = p.content
-  userDefinedLicense.version = block
-  userDefinedLicense.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save(userDefinedLicense)
-}
-
-async function createJoystreamMediaLocation({ db, block, id }: IDBBlockId, p: IJoystreamMediaLocation): Promise<void> {
-  const joyMediaLoc = new JoystreamMediaLocation()
-
-  joyMediaLoc.id = id
-  joyMediaLoc.dataObjectId = p.dataObjectId
-  joyMediaLoc.version = block
-  joyMediaLoc.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save(joyMediaLoc)
-}
-
-async function createHttpMediaLocation({ db, block, id }: IDBBlockId, p: IHttpMediaLocation): Promise<void> {
-  const httpMediaLoc = new HttpMediaLocation()
-
-  httpMediaLoc.id = id
-  httpMediaLoc.url = p.url
-  httpMediaLoc.port = p.port
-  httpMediaLoc.version = block
-  httpMediaLoc.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save(httpMediaLoc)
-}
-
-async function createVideoMedia({ db, block, id }: IDBBlockId, p: IVideoMedia): Promise<void> {
-  const videoMedia = new VideoMedia()
-
-  videoMedia.id = id
-  videoMedia.encodingId = p.encoding
-  videoMedia.locationId = p.location
-  videoMedia.pixelHeight = p.pixelHeight
-  videoMedia.pixelWidth = p.pixelWidth
-  videoMedia.size = p.size
-  videoMedia.version = block
-  videoMedia.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save(videoMedia)
-}
-
-async function createVideo({ db, block, id }: IDBBlockId, p: IVideo): Promise<void> {
-  const video = new Video()
-
-  video.id = id
-  video.title = p.title
-  video.description = p.description
-  video.categoryId = p.category
-  video.channelId = p.channel
-  video.duration = p.duration
-  video.hasMarketing = p.hasMarketing
-  // TODO: needs to be handled correctly, from runtime CurationStatus is coming
-  video.isCurated = p.isCurated || true
-  video.isExplicit = p.isExplicit
-  video.isPublic = p.isPublic
-  video.languageId = p.language
-  video.licenseId = p.license
-  video.videoMediaId = p.media
-  video.publishedBeforeJoystream = p.publishedBeforeJoystream
-  video.skippableIntroDuration = p.skippableIntroDuration
-  video.thumbnailUrl = p.thumbnailURL
-  video.version = block
-  video.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save<Video>(video)
-}
-
-async function createLanguage({ db, block, id }: IDBBlockId, p: ILanguage): Promise<void> {
-  const language = new Language()
-  language.id = id
-  language.name = p.name
-  language.code = p.code
-  language.version = block
-  language.happenedIn = await createBlockOrGetFromDatabase(db, block)
-
-  await db.save<Language>(language)
-}
-
-async function createVideoMediaEncoding({ db, block, id }: IDBBlockId, p: IVideoMediaEncoding): Promise<void> {
-  const encoding = new VideoMediaEncoding()
-
-  encoding.id = id
-  encoding.name = p.name
-  encoding.version = block
-  // happenedIn is not defined in the graphql schema!
-  // encoding.happenedIn = await createBlockOrGetFromDatabase(db, block)
-  await db.save<VideoMediaEncoding>(encoding)
-}
-
-async function batchCreateClassEntities(db: DB, block: number, operations: ICreateEntityOperation[]): Promise<void> {
-  // Create entities before adding schema support
-  operations.map(async ({ classId }, index) => {
-    const c = new ClassEntity()
-    c.id = index.toString()
-    c.classId = classId
-    c.version = block
-    c.happenedIn = await createBlockOrGetFromDatabase(db, block)
-    await db.save<ClassEntity>(c)
-  })
-}
-
-async function getClassName(
-  db: DB,
-  entity: IEntity,
-  createEntityOperations: ICreateEntityOperation[]
-): Promise<string | undefined> {
-  const { entityId, indexOf } = entity
-  if (entityId === undefined && indexOf === undefined) {
-    throw Error(`Can not determine class of the entity`)
-  }
-
-  let classId: number | undefined
-  // Is newly created entity in the same transaction
-  if (indexOf !== undefined) {
-    classId = createEntityOperations[indexOf].classId
-  } else {
-    const ce = await db.get(ClassEntity, { where: { id: entityId } })
-    if (ce === undefined) console.log(`Class not found for the entity: ${entityId}`)
-    classId = ce ? ce.classId : undefined
-  }
-
-  const c = contentDirectoryClassNamesWithId.find((c) => c.classId === classId)
-  // TODO: stop execution, class should be created before entity creation
-  if (c === undefined) console.log(`Not recognized class id: ${classId}`)
-  return c ? c.name : undefined
-}
-
-async function removeChannel(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(Channel, where)
-  if (record === undefined) throw Error(`Channel not found`)
-  await db.remove<Channel>(record)
-}
-async function removeCategory(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(Category, where)
-  if (record === undefined) throw Error(`Category not found`)
-  await db.remove<Category>(record)
-}
-async function removeVideoMedia(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(VideoMedia, where)
-  if (record === undefined) throw Error(`VideoMedia not found`)
-  await db.remove<VideoMedia>(record)
-}
-async function removeVideo(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(Video, where)
-  if (record === undefined) throw Error(`Video not found`)
-  await db.remove<Video>(record)
-}
-async function removeUserDefinedLicense(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(UserDefinedLicense, where)
-  if (record === undefined) throw Error(`UserDefinedLicense not found`)
-  await db.remove<UserDefinedLicense>(record)
-}
-async function removeKnownLicense(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(KnownLicense, where)
-  if (record === undefined) throw Error(`KnownLicense not found`)
-  await db.remove<KnownLicense>(record)
-}
-async function removeHttpMediaLocation(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(HttpMediaLocation, where)
-  if (record === undefined) throw Error(`HttpMediaLocation not found`)
-  await db.remove<HttpMediaLocation>(record)
-}
-async function removeJoystreamMediaLocation(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(JoystreamMediaLocation, where)
-  if (record === undefined) throw Error(`JoystreamMediaLocation not found`)
-  await db.remove<JoystreamMediaLocation>(record)
-}
-async function removeLanguage(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(Language, where)
-  if (record === undefined) throw Error(`Language not found`)
-  await db.remove<Language>(record)
-}
-async function removeVideoMediaEncoding(db: DB, where: IWhereCond): Promise<void> {
-  const record = await db.get(VideoMediaEncoding, where)
-  if (record === undefined) throw Error(`Language not found`)
-  await db.remove<VideoMediaEncoding>(record)
-}
-
-// ========Entity property value updates========
-
-async function updateCategoryEntityPropertyValues(db: DB, where: IWhereCond, props: ICategory): Promise<void> {
-  const record = await db.get(Category, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<Category>(record)
-}
-async function updateChannelEntityPropertyValues(db: DB, where: IWhereCond, props: IChannel): Promise<void> {
-  const record = await db.get(Channel, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<Channel>(record)
-}
-async function updateVideoMediaEntityPropertyValues(db: DB, where: IWhereCond, props: IVideoMedia): Promise<void> {
-  const record = await db.get(VideoMedia, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<VideoMedia>(record)
-}
-async function updateVideoEntityPropertyValues(db: DB, where: IWhereCond, props: IVideo): Promise<void> {
-  const record = await db.get(Video, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<Video>(record)
-}
-async function updateUserDefinedLicenseEntityPropertyValues(
-  db: DB,
-  where: IWhereCond,
-  props: IUserDefinedLicense
-): Promise<void> {
-  const record = await db.get(UserDefinedLicense, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<UserDefinedLicense>(record)
-}
-async function updateKnownLicenseEntityPropertyValues(db: DB, where: IWhereCond, props: IKnownLicense): Promise<void> {
-  const record = await db.get(KnownLicense, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<KnownLicense>(record)
-}
-async function updateHttpMediaLocationEntityPropertyValues(
-  db: DB,
-  where: IWhereCond,
-  props: IHttpMediaLocation
-): Promise<void> {
-  const record = await db.get(HttpMediaLocation, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<HttpMediaLocation>(record)
-}
-async function updateJoystreamMediaLocationEntityPropertyValues(
-  db: DB,
-  where: IWhereCond,
-  props: IJoystreamMediaLocation
-): Promise<void> {
-  const record = await db.get(JoystreamMediaLocation, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<JoystreamMediaLocation>(record)
-}
-async function updateLanguageEntityPropertyValues(db: DB, where: IWhereCond, props: ILanguage): Promise<void> {
-  const record = await db.get(Language, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<Language>(record)
-}
-async function updateVideoMediaEncodingEntityPropertyValues(
-  db: DB,
-  where: IWhereCond,
-  props: IVideoMediaEncoding
-): Promise<void> {
-  const record = await db.get(VideoMediaEncoding, where)
-  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
-  Object.assign(record, props)
-  await db.save<VideoMediaEncoding>(record)
-}
-
-async function updateEntityPropertyValues(
-  db: DB,
-  event: SubstrateEvent,
-  where: IWhereCond,
-  className: string
-): Promise<void> {
-  switch (className) {
-    case ContentDirectoryKnownClasses.CHANNEL:
-      updateChannelEntityPropertyValues(db, where, decode.setProperties<IChannel>(event, channelPropertyNamesWithId))
-      break
-
-    case ContentDirectoryKnownClasses.CATEGORY:
-      await updateCategoryEntityPropertyValues(
-        db,
-        where,
-        decode.setProperties<ICategory>(event, CategoryPropertyNamesWithId)
-      )
-      break
-
-    case ContentDirectoryKnownClasses.KNOWNLICENSE:
-      await updateKnownLicenseEntityPropertyValues(
-        db,
-        where,
-        decode.setProperties<IKnownLicense>(event, knownLicensePropertyNamesWIthId)
-      )
-      break
-
-    case ContentDirectoryKnownClasses.USERDEFINEDLICENSE:
-      await updateUserDefinedLicenseEntityPropertyValues(
-        db,
-        where,
-        decode.setProperties<IUserDefinedLicense>(event, userDefinedLicensePropertyNamesWithId)
-      )
-      break
-
-    case ContentDirectoryKnownClasses.JOYSTREAMMEDIALOCATION:
-      await updateJoystreamMediaLocationEntityPropertyValues(
-        db,
-        where,
-        decode.setProperties<IJoystreamMediaLocation>(event, joystreamMediaLocationPropertyNamesWithId)
-      )
-      break
-
-    case ContentDirectoryKnownClasses.HTTPMEDIALOCATION:
-      await updateHttpMediaLocationEntityPropertyValues(
-        db,
-        where,
-        decode.setProperties<IHttpMediaLocation>(event, httpMediaLocationPropertyNamesWithId)
-      )
-      break
-
-    case ContentDirectoryKnownClasses.VIDEOMEDIA:
-      await updateVideoMediaEntityPropertyValues(
-        db,
-        where,
-        decode.setProperties<IVideoMedia>(event, videoPropertyNamesWithId)
-      )
-      break
-
-    case ContentDirectoryKnownClasses.VIDEO:
-      await updateVideoEntityPropertyValues(db, where, decode.setProperties<IVideo>(event, videoPropertyNamesWithId))
-      break
-
-    case ContentDirectoryKnownClasses.LANGUAGE:
-      await updateLanguageEntityPropertyValues(
-        db,
-        where,
-        decode.setProperties<ILanguage>(event, languagePropertyNamesWIthId)
-      )
-      break
-
-    case ContentDirectoryKnownClasses.VIDEOMEDIAENCODING:
-      await updateVideoMediaEncodingEntityPropertyValues(
-        db,
-        where,
-        decode.setProperties<IVideoMediaEncoding>(event, videoMediaEncodingPropertyNamesWithId)
-      )
-      break
-
-    default:
-      throw new Error(`Unknown class name: ${className}`)
-  }
-}
-
-export {
-  createCategory,
-  createChannel,
-  createVideoMedia,
-  createVideo,
-  createUserDefinedLicense,
-  createKnownLicense,
-  createHttpMediaLocation,
-  createJoystreamMediaLocation,
-  createLanguage,
-  createVideoMediaEncoding,
-  removeCategory,
-  removeChannel,
-  removeVideoMedia,
-  removeVideo,
-  removeUserDefinedLicense,
-  removeKnownLicense,
-  removeHttpMediaLocation,
-  removeJoystreamMediaLocation,
-  removeLanguage,
-  removeVideoMediaEncoding,
-  createBlockOrGetFromDatabase,
-  batchCreateClassEntities,
-  getClassName,
-  updateCategoryEntityPropertyValues,
-  updateChannelEntityPropertyValues,
-  updateVideoMediaEntityPropertyValues,
-  updateVideoEntityPropertyValues,
-  updateUserDefinedLicenseEntityPropertyValues,
-  updateHttpMediaLocationEntityPropertyValues,
-  updateJoystreamMediaLocationEntityPropertyValues,
-  updateKnownLicenseEntityPropertyValues,
-  updateLanguageEntityPropertyValues,
-  updateVideoMediaEncodingEntityPropertyValues,
-  updateEntityPropertyValues,
-}

query-node/mappings/content-directory/entity/create.ts

@@ -0,0 +1,438 @@
+import { DB } from '../../../generated/indexer'
+import { Channel } from '../../../generated/graphql-server/src/modules/channel/channel.model'
+import { Category } from '../../../generated/graphql-server/src/modules/category/category.model'
+import { KnownLicenseEntity } from '../../../generated/graphql-server/src/modules/known-license-entity/known-license-entity.model'
+import { UserDefinedLicenseEntity } from '../../../generated/graphql-server/src/modules/user-defined-license-entity/user-defined-license-entity.model'
+import { JoystreamMediaLocationEntity } from '../../../generated/graphql-server/src/modules/joystream-media-location-entity/joystream-media-location-entity.model'
+import { HttpMediaLocationEntity } from '../../../generated/graphql-server/src/modules/http-media-location-entity/http-media-location-entity.model'
+import { VideoMedia } from '../../../generated/graphql-server/src/modules/video-media/video-media.model'
+import { Video } from '../../../generated/graphql-server/src/modules/video/video.model'
+import { Block, Network } from '../../../generated/graphql-server/src/modules/block/block.model'
+import { Language } from '../../../generated/graphql-server/src/modules/language/language.model'
+import { VideoMediaEncoding } from '../../../generated/graphql-server/src/modules/video-media-encoding/video-media-encoding.model'
+import { ClassEntity } from '../../../generated/graphql-server/src/modules/class-entity/class-entity.model'
+import { LicenseEntity } from '../../../generated/graphql-server/src/modules/license-entity/license-entity.model'
+import { MediaLocationEntity } from '../../../generated/graphql-server/src/modules/media-location-entity/media-location-entity.model'
+
+import { contentDirectoryClassNamesWithId } from '../content-dir-consts'
+import {
+  ClassEntityMap,
+  ICategory,
+  IChannel,
+  ICreateEntityOperation,
+  IDBBlockId,
+  IEntity,
+  IHttpMediaLocation,
+  IJoystreamMediaLocation,
+  IKnownLicense,
+  ILanguage,
+  ILicense,
+  IMediaLocation,
+  IUserDefinedLicense,
+  IVideo,
+  IVideoMedia,
+  IVideoMediaEncoding,
+} from '../../types'
+import { getOrCreate } from '../get-or-create'
+import BN from 'bn.js'
+import {
+  HttpMediaLocation,
+  JoystreamMediaLocation,
+  KnownLicense,
+  UserDefinedLicense,
+} from '../../../generated/graphql-server/src/modules/variants/variants.model'
+
+async function createBlockOrGetFromDatabase(db: DB, blockNumber: number): Promise<Block> {
+  let b = await db.get(Block, { where: { block: blockNumber } })
+  if (b === undefined) {
+    // TODO: get timestamp from the event or extrinsic
+    b = new Block({ block: blockNumber, network: Network.BABYLON, timestamp: new BN(Date.now()) })
+    await db.save<Block>(b)
+  }
+  return b
+}
+
+async function createChannel(
+  { db, block, id }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  p: IChannel,
+  nextEntityIdBeforeTransaction: number
+): Promise<Channel> {
+  const record = await db.get(Channel, { where: { id } })
+  if (record) return record
+
+  const channel = new Channel()
+
+  channel.version = block
+  channel.id = id
+  channel.handle = p.handle
+  channel.description = p.description
+  channel.isCurated = p.isCurated || false
+  channel.isPublic = p.isPublic
+  channel.coverPhotoUrl = p.coverPhotoUrl
+  channel.avatarPhotoUrl = p.avatarPhotoUrl
+
+  channel.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  const { language } = p
+  if (language) {
+    channel.language = await getOrCreate.language(
+      { db, block, id },
+      classEntityMap,
+      language,
+      nextEntityIdBeforeTransaction
+    )
+  }
+  await db.save(channel)
+  return channel
+}
+
+async function createCategory({ db, block, id }: IDBBlockId, p: ICategory): Promise<Category> {
+  const record = await db.get(Category, { where: { id } })
+  if (record) return record
+
+  const category = new Category()
+
+  category.id = id
+  category.name = p.name
+  category.description = p.description
+  category.version = block
+  category.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save(category)
+  return category
+}
+
+async function createKnownLicense({ db, block, id }: IDBBlockId, p: IKnownLicense): Promise<KnownLicenseEntity> {
+  const record = await db.get(KnownLicenseEntity, { where: { id } })
+  if (record) return record
+
+  const knownLicence = new KnownLicenseEntity()
+
+  knownLicence.id = id
+  knownLicence.code = p.code
+  knownLicence.name = p.name
+  knownLicence.description = p.description
+  knownLicence.url = p.url
+  knownLicence.version = block
+  knownLicence.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save(knownLicence)
+  return knownLicence
+}
+
+async function createUserDefinedLicense(
+  { db, block, id }: IDBBlockId,
+  p: IUserDefinedLicense
+): Promise<UserDefinedLicenseEntity> {
+  const record = await db.get(UserDefinedLicenseEntity, { where: { id } })
+  if (record) return record
+
+  const userDefinedLicense = new UserDefinedLicenseEntity()
+
+  userDefinedLicense.id = id
+  userDefinedLicense.content = p.content
+  userDefinedLicense.version = block
+  userDefinedLicense.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save<UserDefinedLicenseEntity>(userDefinedLicense)
+  return userDefinedLicense
+}
+
+async function createJoystreamMediaLocation(
+  { db, block, id }: IDBBlockId,
+  p: IJoystreamMediaLocation
+): Promise<JoystreamMediaLocationEntity> {
+  const record = await db.get(JoystreamMediaLocationEntity, { where: { id } })
+  if (record) return record
+
+  const joyMediaLoc = new JoystreamMediaLocationEntity()
+
+  joyMediaLoc.id = id
+  joyMediaLoc.dataObjectId = p.dataObjectId
+  joyMediaLoc.version = block
+  joyMediaLoc.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save(joyMediaLoc)
+  return joyMediaLoc
+}
+
+async function createHttpMediaLocation(
+  { db, block, id }: IDBBlockId,
+  p: IHttpMediaLocation
+): Promise<HttpMediaLocationEntity> {
+  const record = await db.get(HttpMediaLocationEntity, { where: { id } })
+  if (record) return record
+
+  const httpMediaLoc = new HttpMediaLocationEntity()
+
+  httpMediaLoc.id = id
+  httpMediaLoc.url = p.url
+  httpMediaLoc.port = p.port
+  httpMediaLoc.version = block
+  httpMediaLoc.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save(httpMediaLoc)
+  return httpMediaLoc
+}
+
+async function createVideoMedia(
+  { db, block, id }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  p: IVideoMedia,
+  nextEntityIdBeforeTransaction: number
+): Promise<VideoMedia> {
+  const videoMedia = new VideoMedia()
+
+  videoMedia.id = id
+  videoMedia.pixelHeight = p.pixelHeight
+  videoMedia.pixelWidth = p.pixelWidth
+  videoMedia.size = p.size
+  videoMedia.version = block
+  const { encoding, location } = p
+  if (encoding !== undefined) {
+    videoMedia.encoding = await getOrCreate.videoMediaEncoding(
+      { db, block, id },
+      classEntityMap,
+      encoding,
+      nextEntityIdBeforeTransaction
+    )
+  }
+  if (location !== undefined) {
+    const m = await getOrCreate.mediaLocation(
+      { db, block, id },
+      classEntityMap,
+      location,
+      nextEntityIdBeforeTransaction
+    )
+    videoMedia.locationEntity = m
+    const { httpMediaLocation, joystreamMediaLocation } = m
+    if (httpMediaLocation) {
+      const mediaLoc = new HttpMediaLocation()
+      mediaLoc.isTypeOf = 'HttpMediaLocation'
+      mediaLoc.port = httpMediaLocation.port
+      mediaLoc.url = httpMediaLocation.url
+      videoMedia.location = mediaLoc
+    }
+    if (joystreamMediaLocation) {
+      const mediaLoc = new JoystreamMediaLocation()
+      mediaLoc.isTypeOf = 'JoystreamMediaLocation'
+      mediaLoc.dataObjectId = joystreamMediaLocation.dataObjectId
+      videoMedia.location = mediaLoc
+    }
+  }
+
+  videoMedia.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save<VideoMedia>(videoMedia)
+  return videoMedia
+}
+
+async function createVideo(
+  { db, block, id }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  p: IVideo,
+  nextEntityIdBeforeTransaction: number
+): Promise<Video> {
+  const record = await db.get(Video, { where: { id } })
+  if (record) return record
+
+  const video = new Video()
+
+  video.id = id
+  video.title = p.title
+  video.description = p.description
+  video.duration = p.duration
+  video.hasMarketing = p.hasMarketing
+  // TODO: needs to be handled correctly, from runtime CurationStatus is coming
+  video.isCurated = p.isCurated || true
+  video.isExplicit = p.isExplicit
+  video.isPublic = p.isPublic
+  video.publishedBeforeJoystream = p.publishedBeforeJoystream
+  video.skippableIntroDuration = p.skippableIntroDuration
+  video.thumbnailUrl = p.thumbnailUrl
+  video.version = block
+
+  const { language, license, category, channel, media } = p
+  if (language !== undefined) {
+    video.language = await getOrCreate.language(
+      { db, block, id },
+      classEntityMap,
+      language,
+      nextEntityIdBeforeTransaction
+    )
+  }
+  if (license !== undefined) {
+    const { knownLicense, userdefinedLicense } = await getOrCreate.license(
+      { db, block, id },
+      classEntityMap,
+      license,
+      nextEntityIdBeforeTransaction
+    )
+    if (knownLicense) {
+      const lic = new KnownLicense()
+      lic.code = knownLicense.code
+      lic.description = knownLicense.description
+      lic.isTypeOf = 'KnownLicense'
+      lic.name = knownLicense.name
+      lic.url = knownLicense.url
+      video.license = lic
+    }
+    if (userdefinedLicense) {
+      const lic = new UserDefinedLicense()
+      lic.content = userdefinedLicense.content
+      lic.isTypeOf = 'UserDefinedLicense'
+      video.license = lic
+    }
+  }
+  if (category !== undefined) {
+    video.category = await getOrCreate.category(
+      { db, block, id },
+      classEntityMap,
+      category,
+      nextEntityIdBeforeTransaction
+    )
+  }
+  if (channel !== undefined) {
+    video.channel = await getOrCreate.channel({ db, block, id }, classEntityMap, channel, nextEntityIdBeforeTransaction)
+  }
+  if (media !== undefined) {
+    video.media = await getOrCreate.videoMedia({ db, block, id }, classEntityMap, media, nextEntityIdBeforeTransaction)
+  }
+
+  video.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save<Video>(video)
+  return video
+}
+
+async function createLanguage({ db, block, id }: IDBBlockId, p: ILanguage): Promise<Language> {
+  const record = await db.get(Language, { where: { id } })
+  if (record) return record
+
+  const language = new Language()
+  language.id = id
+  language.name = p.name
+  language.code = p.code
+  language.version = block
+  language.happenedIn = await createBlockOrGetFromDatabase(db, block)
+
+  await db.save<Language>(language)
+  return language
+}
+
+async function createVideoMediaEncoding(
+  { db, block, id }: IDBBlockId,
+  p: IVideoMediaEncoding
+): Promise<VideoMediaEncoding> {
+  const record = await db.get(VideoMediaEncoding, { where: { id } })
+  if (record) return record
+
+  const encoding = new VideoMediaEncoding()
+  encoding.id = id
+  encoding.name = p.name
+  encoding.version = block
+  encoding.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save<VideoMediaEncoding>(encoding)
+  return encoding
+}
+
+async function createLicense(
+  { db, block, id }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  p: ILicense,
+  nextEntityIdBeforeTransaction: number
+): Promise<LicenseEntity> {
+  const record = await db.get(LicenseEntity, { where: { id } })
+  if (record) return record
+
+  const { knownLicense, userDefinedLicense } = p
+
+  const license = new LicenseEntity()
+  license.id = id
+  if (knownLicense !== undefined) {
+    license.knownLicense = await getOrCreate.knownLicense(
+      { db, block, id },
+      classEntityMap,
+      knownLicense,
+      nextEntityIdBeforeTransaction
+    )
+  }
+  if (userDefinedLicense !== undefined) {
+    license.userdefinedLicense = await getOrCreate.userDefinedLicense(
+      { db, block, id },
+      classEntityMap,
+      userDefinedLicense,
+      nextEntityIdBeforeTransaction
+    )
+  }
+  license.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save<LicenseEntity>(license)
+  return license
+}
+
+async function createMediaLocation(
+  { db, block, id }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  p: IMediaLocation,
+  nextEntityIdBeforeTransaction: number
+): Promise<MediaLocationEntity> {
+  const { httpMediaLocation, joystreamMediaLocation } = p
+
+  const location = new MediaLocationEntity()
+  location.id = id
+  if (httpMediaLocation !== undefined) {
+    location.httpMediaLocation = await getOrCreate.httpMediaLocation(
+      { db, block, id },
+      classEntityMap,
+      httpMediaLocation,
+      nextEntityIdBeforeTransaction
+    )
+  }
+  if (joystreamMediaLocation !== undefined) {
+    location.joystreamMediaLocation = await getOrCreate.joystreamMediaLocation(
+      { db, block, id },
+      classEntityMap,
+      joystreamMediaLocation,
+      nextEntityIdBeforeTransaction
+    )
+  }
+  location.happenedIn = await createBlockOrGetFromDatabase(db, block)
+  await db.save<MediaLocationEntity>(location)
+  return location
+}
+
+async function getClassName(
+  db: DB,
+  entity: IEntity,
+  createEntityOperations: ICreateEntityOperation[]
+): Promise<string | undefined> {
+  const { entityId, indexOf } = entity
+  if (entityId === undefined && indexOf === undefined) {
+    throw Error(`Can not determine class of the entity`)
+  }
+
+  let classId: number | undefined
+  // Is newly created entity in the same transaction
+  if (indexOf !== undefined) {
+    classId = createEntityOperations[indexOf].classId
+  } else {
+    const ce = await db.get(ClassEntity, { where: { id: entityId } })
+    if (ce === undefined) console.log(`Class not found for the entity: ${entityId}`)
+    classId = ce ? ce.classId : undefined
+  }
+
+  const c = contentDirectoryClassNamesWithId.find((c) => c.classId === classId)
+  // TODO: stop execution, class should be created before entity creation
+  if (c === undefined) console.log(`Not recognized class id: ${classId}`)
+  return c ? c.name : undefined
+}
+
+export {
+  createCategory,
+  createChannel,
+  createVideoMedia,
+  createVideo,
+  createUserDefinedLicense,
+  createKnownLicense,
+  createHttpMediaLocation,
+  createJoystreamMediaLocation,
+  createLanguage,
+  createVideoMediaEncoding,
+  createLicense,
+  createMediaLocation,
+  createBlockOrGetFromDatabase,
+  getClassName,
+}

query-node/mappings/content-directory/entity.ts → query-node/mappings/content-directory/entity/index.ts

@@ -1,17 +1,23 @@
 import Debug from 'debug'
-import { DB, SubstrateEvent } from '../../generated/indexer'
-import { ClassEntity } from '../../generated/graphql-server/src/modules/class-entity/class-entity.model'
+import { DB, SubstrateEvent } from '../../../generated/indexer'
+import { ClassEntity } from '../../../generated/graphql-server/src/modules/class-entity/class-entity.model'
 
-import { decode } from './decode'
+import { decode } from '../decode'
+import {
+  updateCategoryEntityPropertyValues,
+  updateChannelEntityPropertyValues,
+  updateVideoMediaEntityPropertyValues,
+  updateVideoEntityPropertyValues,
+  updateUserDefinedLicenseEntityPropertyValues,
+  updateHttpMediaLocationEntityPropertyValues,
+  updateJoystreamMediaLocationEntityPropertyValues,
+  updateKnownLicenseEntityPropertyValues,
+  updateLanguageEntityPropertyValues,
+  updateVideoMediaEncodingEntityPropertyValues,
+  updateLicenseEntityPropertyValues,
+  updateMediaLocationEntityPropertyValues,
+} from './update'
 import {
-  createCategory,
-  createChannel,
-  createVideoMedia,
-  createVideo,
-  createUserDefinedLicense,
-  createKnownLicense,
-  createHttpMediaLocation,
-  createJoystreamMediaLocation,
   removeCategory,
   removeChannel,
   removeVideoMedia,
@@ -22,22 +28,24 @@ import {
   removeJoystreamMediaLocation,
   removeLanguage,
   removeVideoMediaEncoding,
+  removeLicense,
+  removeMediaLocation,
+} from './remove'
+import {
+  createCategory,
+  createChannel,
+  createVideoMedia,
+  createVideo,
+  createUserDefinedLicense,
+  createKnownLicense,
+  createHttpMediaLocation,
+  createJoystreamMediaLocation,
   createLanguage,
   createVideoMediaEncoding,
-  updateCategoryEntityPropertyValues,
-  updateChannelEntityPropertyValues,
-  updateVideoMediaEntityPropertyValues,
-  updateVideoEntityPropertyValues,
-  updateUserDefinedLicenseEntityPropertyValues,
-  updateHttpMediaLocationEntityPropertyValues,
-  updateJoystreamMediaLocationEntityPropertyValues,
-  updateKnownLicenseEntityPropertyValues,
-  updateLanguageEntityPropertyValues,
-  updateVideoMediaEncodingEntityPropertyValues,
   createBlockOrGetFromDatabase,
-} from './entity-helper'
+} from './create'
 import {
-  CategoryPropertyNamesWithId,
+  categoryPropertyNamesWithId,
   channelPropertyNamesWithId,
   httpMediaLocationPropertyNamesWithId,
   joystreamMediaLocationPropertyNamesWithId,
@@ -48,7 +56,7 @@ import {
   videoPropertyNamesWithId,
   contentDirectoryClassNamesWithId,
   ContentDirectoryKnownClasses,
-} from './content-dir-consts'
+} from '../content-dir-consts'
 
 import {
   IChannel,
@@ -63,7 +71,11 @@ import {
   IVideoMediaEncoding,
   IDBBlockId,
   IWhereCond,
-} from '../types'
+  IEntity,
+  ILicense,
+  IMediaLocation,
+} from '../../types'
+import { getOrCreate } from '../get-or-create'
 
 const debug = Debug('mappings:content-directory')
 
@@ -91,11 +103,16 @@ async function contentDirectory_EntitySchemaSupportAdded(db: DB, event: Substrat
 
   switch (cls.name) {
     case ContentDirectoryKnownClasses.CHANNEL:
-      await createChannel(arg, decode.setProperties<IChannel>(event, channelPropertyNamesWithId))
+      await createChannel(
+        arg,
+        new Map<string, IEntity[]>(),
+        decode.setProperties<IChannel>(event, channelPropertyNamesWithId),
+        0 // ignored
+      )
       break
 
     case ContentDirectoryKnownClasses.CATEGORY:
-      await createCategory(arg, decode.setProperties<ICategory>(event, CategoryPropertyNamesWithId))
+      await createCategory(arg, decode.setProperties<ICategory>(event, categoryPropertyNamesWithId))
       break
 
     case ContentDirectoryKnownClasses.KNOWNLICENSE:
@@ -124,11 +141,21 @@ async function contentDirectory_EntitySchemaSupportAdded(db: DB, event: Substrat
       break
 
     case ContentDirectoryKnownClasses.VIDEOMEDIA:
-      await createVideoMedia(arg, decode.setProperties<IVideoMedia>(event, videoPropertyNamesWithId))
+      await createVideoMedia(
+        arg,
+        new Map<string, IEntity[]>(),
+        decode.setProperties<IVideoMedia>(event, videoPropertyNamesWithId),
+        0 // ignored
+      )
       break
 
     case ContentDirectoryKnownClasses.VIDEO:
-      await createVideo(arg, decode.setProperties<IVideo>(event, videoPropertyNamesWithId))
+      await createVideo(
+        arg,
+        new Map<string, IEntity[]>(),
+        decode.setProperties<IVideo>(event, videoPropertyNamesWithId),
+        0 // ignored
+      )
       break
 
     case ContentDirectoryKnownClasses.LANGUAGE:
@@ -162,7 +189,7 @@ async function contentDirectory_EntityRemoved(db: DB, event: SubstrateEvent): Pr
 
   const cls = contentDirectoryClassNamesWithId.find((c) => c.classId === classEntity.classId)
   if (cls === undefined) {
-    console.log('Undefined class')
+    console.log('Unknown class')
     return
   }
 
@@ -206,6 +233,14 @@ async function contentDirectory_EntityRemoved(db: DB, event: SubstrateEvent): Pr
       await removeVideoMediaEncoding(db, where)
       break
 
+    case ContentDirectoryKnownClasses.LICENSE:
+      await removeLicense(db, where)
+      break
+
+    case ContentDirectoryKnownClasses.MEDIALOCATION:
+      await removeMediaLocation(db, where)
+      break
+
     default:
       throw new Error(`Unknown class name: ${cls.name}`)
   }
@@ -224,17 +259,18 @@ async function contentDirectory_EntityCreated(db: DB, event: SubstrateEvent): Pr
   classEntity.version = event.blockNumber
   classEntity.happenedIn = await createBlockOrGetFromDatabase(db, event.blockNumber)
   await db.save<ClassEntity>(classEntity)
+
+  await getOrCreate.nextEntityId(db, c.entityId + 1)
 }
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
 async function contentDirectory_EntityPropertyValuesUpdated(db: DB, event: SubstrateEvent): Promise<void> {
-  debug(`EntityPropertyValuesUpdated event: ${JSON.stringify(event)}`)
-
   const { extrinsic } = event
-
   if (extrinsic && extrinsic.method === 'transaction') return
   if (extrinsic === undefined) throw Error(`Extrinsic data not found for event: ${event.id}`)
 
+  debug(`EntityPropertyValuesUpdated event: ${JSON.stringify(event)}`)
+
   const { 2: newPropertyValues } = extrinsic.args
   const entityId = decode.stringIfyEntityId(event)
 
@@ -253,14 +289,14 @@ async function contentDirectory_EntityPropertyValuesUpdated(db: DB, event: Subst
 
   switch (cls.name) {
     case ContentDirectoryKnownClasses.CHANNEL:
-      updateChannelEntityPropertyValues(db, where, decode.setProperties<IChannel>(event, channelPropertyNamesWithId))
+      updateChannelEntityPropertyValues(db, where, decode.setProperties<IChannel>(event, channelPropertyNamesWithId), 0)
       break
 
     case ContentDirectoryKnownClasses.CATEGORY:
       await updateCategoryEntityPropertyValues(
         db,
         where,
-        decode.setProperties<ICategory>(event, CategoryPropertyNamesWithId)
+        decode.setProperties<ICategory>(event, categoryPropertyNamesWithId)
       )
       break
 
@@ -300,12 +336,13 @@ async function contentDirectory_EntityPropertyValuesUpdated(db: DB, event: Subst
       await updateVideoMediaEntityPropertyValues(
         db,
         where,
-        decode.setProperties<IVideoMedia>(event, videoPropertyNamesWithId)
+        decode.setProperties<IVideoMedia>(event, videoPropertyNamesWithId),
+        0
       )
       break
 
     case ContentDirectoryKnownClasses.VIDEO:
-      await updateVideoEntityPropertyValues(db, where, decode.setProperties<IVideo>(event, videoPropertyNamesWithId))
+      await updateVideoEntityPropertyValues(db, where, decode.setProperties<IVideo>(event, videoPropertyNamesWithId), 0)
       break
 
     case ContentDirectoryKnownClasses.LANGUAGE:
@@ -324,6 +361,24 @@ async function contentDirectory_EntityPropertyValuesUpdated(db: DB, event: Subst
       )
       break
 
+    case ContentDirectoryKnownClasses.LICENSE:
+      await updateLicenseEntityPropertyValues(
+        db,
+        where,
+        decode.setProperties<ILicense>(event, videoMediaEncodingPropertyNamesWithId),
+        0
+      )
+      break
+
+    case ContentDirectoryKnownClasses.MEDIALOCATION:
+      await updateMediaLocationEntityPropertyValues(
+        db,
+        where,
+        decode.setProperties<IMediaLocation>(event, videoMediaEncodingPropertyNamesWithId),
+        0
+      )
+      break
+
     default:
       throw new Error(`Unknown class name: ${cls.name}`)
   }

query-node/mappings/content-directory/entity/remove.ts

@@ -0,0 +1,145 @@
+import { DB } from '../../../generated/indexer'
+import { Channel } from '../../../generated/graphql-server/src/modules/channel/channel.model'
+import { Category } from '../../../generated/graphql-server/src/modules/category/category.model'
+import { KnownLicenseEntity } from '../../../generated/graphql-server/src/modules/known-license-entity/known-license-entity.model'
+import { UserDefinedLicenseEntity } from '../../../generated/graphql-server/src/modules/user-defined-license-entity/user-defined-license-entity.model'
+import { JoystreamMediaLocationEntity } from '../../../generated/graphql-server/src/modules/joystream-media-location-entity/joystream-media-location-entity.model'
+import { HttpMediaLocationEntity } from '../../../generated/graphql-server/src/modules/http-media-location-entity/http-media-location-entity.model'
+import { VideoMedia } from '../../../generated/graphql-server/src/modules/video-media/video-media.model'
+import { Video } from '../../../generated/graphql-server/src/modules/video/video.model'
+import { Language } from '../../../generated/graphql-server/src/modules/language/language.model'
+import { VideoMediaEncoding } from '../../../generated/graphql-server/src/modules/video-media-encoding/video-media-encoding.model'
+import { LicenseEntity } from '../../../generated/graphql-server/src/modules/license-entity/license-entity.model'
+import { MediaLocationEntity } from '../../../generated/graphql-server/src/modules/media-location-entity/media-location-entity.model'
+
+import { IWhereCond } from '../../types'
+
+async function removeChannel(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(Channel, where)
+  if (record === undefined) throw Error(`Channel not found`)
+  if (record.videos) record.videos.map(async (v) => await removeVideo(db, { where: { id: v.id } }))
+  await db.remove<Channel>(record)
+}
+async function removeCategory(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(Category, where)
+  if (record === undefined) throw Error(`Category not found`)
+  if (record.videos) record.videos.map(async (v) => await removeVideo(db, { where: { id: v.id } }))
+  await db.remove<Category>(record)
+}
+async function removeVideoMedia(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(VideoMedia, where)
+  if (record === undefined) throw Error(`VideoMedia not found`)
+  if (record.video) await db.remove<Video>(record.video)
+  await db.remove<VideoMedia>(record)
+}
+async function removeVideo(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(Video, where)
+  if (record === undefined) throw Error(`Video not found`)
+  await db.remove<Video>(record)
+}
+
+async function removeLicense(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(LicenseEntity, where)
+  if (record === undefined) throw Error(`License not found`)
+
+  const { knownLicense, userdefinedLicense } = record
+  let videos: Video[] = []
+
+  if (knownLicense) {
+    videos = await db.getMany(Video, {
+      where: {
+        license: {
+          isTypeOf: 'KnownLicense',
+          code: knownLicense.code,
+          description: knownLicense.description,
+          name: knownLicense.name,
+          url: knownLicense.url,
+        },
+      },
+    })
+  }
+  if (userdefinedLicense) {
+    videos = await db.getMany(Video, {
+      where: { license: { isTypeOf: 'UserDefinedLicense', content: userdefinedLicense.content } },
+    })
+  }
+  // Remove all the videos under this license
+  videos.map(async (v) => await removeVideo(db, { where: { id: v.id } }))
+  await db.remove<LicenseEntity>(record)
+}
+async function removeUserDefinedLicense(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(UserDefinedLicenseEntity, where)
+  if (record === undefined) throw Error(`UserDefinedLicense not found`)
+  if (record.licenseentityuserdefinedLicense)
+    record.licenseentityuserdefinedLicense.map(async (l) => await removeLicense(db, { where: { id: l.id } }))
+  await db.remove<UserDefinedLicenseEntity>(record)
+}
+async function removeKnownLicense(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(KnownLicenseEntity, where)
+  if (record === undefined) throw Error(`KnownLicense not found`)
+  if (record.licenseentityknownLicense)
+    record.licenseentityknownLicense.map(async (k) => await removeLicense(db, { where: { id: k.id } }))
+  await db.remove<KnownLicenseEntity>(record)
+}
+async function removeMediaLocation(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(MediaLocationEntity, where)
+  if (record === undefined) throw Error(`MediaLocation not found`)
+  if (record.videoMedia) await removeVideo(db, { where: { id: record.videoMedia.id } })
+
+  const { httpMediaLocation, joystreamMediaLocation } = record
+
+  let videoMedia: VideoMedia | undefined
+  if (httpMediaLocation) {
+    videoMedia = await db.get(VideoMedia, {
+      where: { location: { isTypeOf: 'HttpMediaLocation', url: httpMediaLocation.url, port: httpMediaLocation.port } },
+    })
+  }
+  if (joystreamMediaLocation) {
+    videoMedia = await db.get(VideoMedia, {
+      where: { location: { isTypeOf: 'JoystreamMediaLocation', dataObjectId: joystreamMediaLocation.dataObjectId } },
+    })
+  }
+  if (videoMedia) await db.remove<VideoMedia>(videoMedia)
+  await db.remove<MediaLocationEntity>(record)
+}
+async function removeHttpMediaLocation(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(HttpMediaLocationEntity, where)
+  if (record === undefined) throw Error(`HttpMediaLocation not found`)
+  if (record.medialocationentityhttpMediaLocation)
+    record.medialocationentityhttpMediaLocation.map(async (v) => await removeMediaLocation(db, { where: { id: v.id } }))
+  await db.remove<HttpMediaLocationEntity>(record)
+}
+async function removeJoystreamMediaLocation(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(JoystreamMediaLocationEntity, where)
+  if (record === undefined) throw Error(`JoystreamMediaLocation not found`)
+  if (record.medialocationentityjoystreamMediaLocation)
+    record.medialocationentityjoystreamMediaLocation.map(async (v) => await removeVideo(db, { where: { id: v.id } }))
+  await db.remove<JoystreamMediaLocationEntity>(record)
+}
+async function removeLanguage(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(Language, where)
+  if (record === undefined) throw Error(`Language not found`)
+  if (record.channellanguage) record.channellanguage.map(async (c) => await removeChannel(db, { where: { id: c.id } }))
+  if (record.videolanguage) record.videolanguage.map(async (v) => await removeVideo(db, { where: { id: v.id } }))
+  await db.remove<Language>(record)
+}
+async function removeVideoMediaEncoding(db: DB, where: IWhereCond): Promise<void> {
+  const record = await db.get(VideoMediaEncoding, where)
+  if (record === undefined) throw Error(`Language not found`)
+  await db.remove<VideoMediaEncoding>(record)
+}
+
+export {
+  removeCategory,
+  removeChannel,
+  removeVideoMedia,
+  removeVideo,
+  removeUserDefinedLicense,
+  removeKnownLicense,
+  removeHttpMediaLocation,
+  removeJoystreamMediaLocation,
+  removeLanguage,
+  removeVideoMediaEncoding,
+  removeMediaLocation,
+  removeLicense,
+}

query-node/mappings/content-directory/entity/update.ts

@@ -0,0 +1,298 @@
+import { DB } from '../../../generated/indexer'
+import { Channel } from '../../../generated/graphql-server/src/modules/channel/channel.model'
+import { Category } from '../../../generated/graphql-server/src/modules/category/category.model'
+import { KnownLicenseEntity } from '../../../generated/graphql-server/src/modules/known-license-entity/known-license-entity.model'
+import { UserDefinedLicenseEntity } from '../../../generated/graphql-server/src/modules/user-defined-license-entity/user-defined-license-entity.model'
+import { VideoMedia } from '../../../generated/graphql-server/src/modules/video-media/video-media.model'
+import { Video } from '../../../generated/graphql-server/src/modules/video/video.model'
+import { Language } from '../../../generated/graphql-server/src/modules/language/language.model'
+import { VideoMediaEncoding } from '../../../generated/graphql-server/src/modules/video-media-encoding/video-media-encoding.model'
+import { LicenseEntity } from '../../../generated/graphql-server/src/modules/license-entity/license-entity.model'
+import { MediaLocationEntity } from '../../../generated/graphql-server/src/modules/media-location-entity/media-location-entity.model'
+import { HttpMediaLocationEntity } from '../../../generated/graphql-server/src/modules/http-media-location-entity/http-media-location-entity.model'
+import { JoystreamMediaLocationEntity } from '../../../generated/graphql-server/src/modules/joystream-media-location-entity/joystream-media-location-entity.model'
+
+import {
+  ICategory,
+  IChannel,
+  IHttpMediaLocation,
+  IJoystreamMediaLocation,
+  IKnownLicense,
+  ILanguage,
+  ILicense,
+  IMediaLocation,
+  IReference,
+  IUserDefinedLicense,
+  IVideo,
+  IVideoMedia,
+  IVideoMediaEncoding,
+  IWhereCond,
+} from '../../types'
+import {
+  HttpMediaLocation,
+  JoystreamMediaLocation,
+  KnownLicense,
+  UserDefinedLicense,
+} from '../../../generated/graphql-server/src/modules/variants/variants.model'
+
+function getEntityIdFromReferencedField(ref: IReference, entityIdBeforeTransaction: number): string {
+  const { entityId, existing } = ref
+  const id = existing ? entityId : entityIdBeforeTransaction + entityId
+  return id.toString()
+}
+
+async function updateMediaLocationEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: IMediaLocation,
+  entityIdBeforeTransaction: number
+): Promise<void> {
+  const { httpMediaLocation, joystreamMediaLocation } = props
+  const record = await db.get(MediaLocationEntity, where)
+  if (record === undefined) throw Error(`MediaLocation entity not found: ${where.where.id}`)
+
+  if (httpMediaLocation) {
+    const id = getEntityIdFromReferencedField(httpMediaLocation, entityIdBeforeTransaction)
+    record.httpMediaLocation = await db.get(HttpMediaLocationEntity, { where: { id } })
+  }
+  if (joystreamMediaLocation) {
+    const id = getEntityIdFromReferencedField(joystreamMediaLocation, entityIdBeforeTransaction)
+    record.joystreamMediaLocation = await db.get(JoystreamMediaLocationEntity, { where: { id } })
+  }
+  await db.save<MediaLocationEntity>(record)
+}
+
+async function updateLicenseEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: ILicense,
+  entityIdBeforeTransaction: number
+): Promise<void> {
+  const record = await db.get(LicenseEntity, where)
+  if (record === undefined) throw Error(`License entity not found: ${where.where.id}`)
+
+  const { knownLicense, userDefinedLicense } = props
+  if (knownLicense) {
+    const id = getEntityIdFromReferencedField(knownLicense, entityIdBeforeTransaction)
+    record.knownLicense = await db.get(KnownLicenseEntity, { where: { id } })
+  }
+  if (userDefinedLicense) {
+    const id = getEntityIdFromReferencedField(userDefinedLicense, entityIdBeforeTransaction)
+    record.userdefinedLicense = await db.get(UserDefinedLicenseEntity, { where: { id } })
+  }
+  await db.save<LicenseEntity>(record)
+}
+
+async function updateCategoryEntityPropertyValues(db: DB, where: IWhereCond, props: ICategory): Promise<void> {
+  const record = await db.get(Category, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+  Object.assign(record, props)
+  await db.save<Category>(record)
+}
+async function updateChannelEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: IChannel,
+  entityIdBeforeTransaction: number
+): Promise<void> {
+  const record = await db.get(Channel, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+
+  let lang: Language | undefined = record.language
+  if (props.language) {
+    const id = getEntityIdFromReferencedField(props.language, entityIdBeforeTransaction)
+    lang = await db.get(Language, { where: { id } })
+    if (lang === undefined) throw Error(`Language entity not found: ${id}`)
+    props.language = undefined
+  }
+  Object.assign(record, props)
+
+  record.language = lang
+  await db.save<Channel>(record)
+}
+async function updateVideoMediaEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: IVideoMedia,
+  entityIdBeforeTransaction: number
+): Promise<void> {
+  const record = await db.get(VideoMedia, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+
+  let enco: VideoMediaEncoding | undefined
+  let mediaLoc: HttpMediaLocation | JoystreamMediaLocation = record.location
+  const { encoding, location } = props
+  if (encoding) {
+    const id = getEntityIdFromReferencedField(encoding, entityIdBeforeTransaction)
+    enco = await db.get(VideoMediaEncoding, { where: { id } })
+    if (enco === undefined) throw Error(`VideoMediaEncoding entity not found: ${id}`)
+    props.encoding = undefined
+  }
+
+  if (location) {
+    const id = getEntityIdFromReferencedField(location, entityIdBeforeTransaction)
+    const mLoc = await db.get(MediaLocationEntity, { where: { id } })
+    if (!mLoc) throw Error(`MediaLocation entity not found: ${id}`)
+    const { httpMediaLocation, joystreamMediaLocation } = mLoc
+
+    if (httpMediaLocation) {
+      mediaLoc = new HttpMediaLocation()
+      mediaLoc.isTypeOf = typeof HttpMediaLocation
+      mediaLoc.url = httpMediaLocation.url
+      mediaLoc.port = httpMediaLocation.port
+    }
+    if (joystreamMediaLocation) {
+      mediaLoc = new JoystreamMediaLocation()
+      mediaLoc.isTypeOf = typeof JoystreamMediaLocation
+      mediaLoc.dataObjectId = joystreamMediaLocation.dataObjectId
+    }
+    props.location = undefined
+  }
+  Object.assign(record, props)
+
+  record.encoding = enco || record.encoding
+  record.location = mediaLoc
+  await db.save<VideoMedia>(record)
+}
+async function updateVideoEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: IVideo,
+  entityIdBeforeTransaction: number
+): Promise<void> {
+  const record = await db.get<Video>(Video, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+
+  let chann: Channel | undefined
+  let cat: Category | undefined
+  let lang: Language | undefined
+  let vMedia: VideoMedia | undefined
+  let lic: KnownLicense | UserDefinedLicense = record.license
+
+  const { channel, category, language, media, license } = props
+  if (channel) {
+    const id = getEntityIdFromReferencedField(channel, entityIdBeforeTransaction)
+    chann = await db.get(Channel, { where: { id } })
+    if (!chann) throw Error(`Channel entity not found: ${id}`)
+    props.channel = undefined
+  }
+  if (category) {
+    const id = getEntityIdFromReferencedField(category, entityIdBeforeTransaction)
+    cat = await db.get(Category, { where: { id } })
+    if (!cat) throw Error(`Category entity not found: ${id}`)
+    props.category = undefined
+  }
+  if (media) {
+    const id = getEntityIdFromReferencedField(media, entityIdBeforeTransaction)
+    vMedia = await db.get(VideoMedia, { where: { id } })
+    if (!vMedia) throw Error(`VideoMedia entity not found: ${id}`)
+    props.media = undefined
+  }
+  if (license) {
+    const id = getEntityIdFromReferencedField(license, entityIdBeforeTransaction)
+    const licenseEntity = await db.get(LicenseEntity, {
+      where: { id },
+      relations: ['knownLicense', 'userdefinedLicense'],
+    })
+    if (!licenseEntity) throw Error(`License entity not found: ${id}`)
+    const { knownLicense, userdefinedLicense } = licenseEntity
+    if (knownLicense) {
+      lic = new KnownLicense()
+      lic.code = knownLicense.code
+      lic.description = knownLicense.description
+      lic.isTypeOf = 'KnownLicense'
+      lic.name = knownLicense.name
+      lic.url = knownLicense.url
+    }
+    if (userdefinedLicense) {
+      lic = new UserDefinedLicense()
+      lic.content = userdefinedLicense.content
+      lic.isTypeOf = 'UserDefinedLicense'
+    }
+    props.license = undefined
+  }
+  if (language) {
+    const id = getEntityIdFromReferencedField(language, entityIdBeforeTransaction)
+    lang = await db.get(Language, { where: { id } })
+    if (!lang) throw Error(`Language entity not found: ${id}`)
+    props.language = undefined
+  }
+
+  Object.assign(record, props)
+
+  record.channel = chann || record.channel
+  record.category = cat || record.category
+  record.media = vMedia || record.media
+  record.license = lic
+  record.language = lang
+
+  await db.save<Video>(record)
+}
+async function updateUserDefinedLicenseEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: IUserDefinedLicense
+): Promise<void> {
+  const record = await db.get(UserDefinedLicenseEntity, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+  Object.assign(record, props)
+  await db.save<UserDefinedLicenseEntity>(record)
+}
+async function updateKnownLicenseEntityPropertyValues(db: DB, where: IWhereCond, props: IKnownLicense): Promise<void> {
+  const record = await db.get(KnownLicenseEntity, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+  Object.assign(record, props)
+  await db.save<KnownLicenseEntity>(record)
+}
+async function updateHttpMediaLocationEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: IHttpMediaLocation
+): Promise<void> {
+  const record = await db.get(HttpMediaLocationEntity, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+  Object.assign(record, props)
+  await db.save<HttpMediaLocationEntity>(record)
+}
+
+async function updateJoystreamMediaLocationEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: IJoystreamMediaLocation
+): Promise<void> {
+  const record = await db.get(JoystreamMediaLocationEntity, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+  Object.assign(record, props)
+  await db.save<JoystreamMediaLocationEntity>(record)
+}
+async function updateLanguageEntityPropertyValues(db: DB, where: IWhereCond, props: ILanguage): Promise<void> {
+  const record = await db.get(Language, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+  Object.assign(record, props)
+  await db.save<Language>(record)
+}
+async function updateVideoMediaEncodingEntityPropertyValues(
+  db: DB,
+  where: IWhereCond,
+  props: IVideoMediaEncoding
+): Promise<void> {
+  const record = await db.get(VideoMediaEncoding, where)
+  if (record === undefined) throw Error(`Entity not found: ${where.where.id}`)
+  Object.assign(record, props)
+  await db.save<VideoMediaEncoding>(record)
+}
+
+export {
+  updateCategoryEntityPropertyValues,
+  updateChannelEntityPropertyValues,
+  updateVideoMediaEntityPropertyValues,
+  updateVideoEntityPropertyValues,
+  updateUserDefinedLicenseEntityPropertyValues,
+  updateHttpMediaLocationEntityPropertyValues,
+  updateJoystreamMediaLocationEntityPropertyValues,
+  updateKnownLicenseEntityPropertyValues,
+  updateLanguageEntityPropertyValues,
+  updateVideoMediaEncodingEntityPropertyValues,
+  updateLicenseEntityPropertyValues,
+  updateMediaLocationEntityPropertyValues,
+}

query-node/mappings/content-directory/get-or-create.ts

@@ -0,0 +1,410 @@
+import { Channel } from '../../generated/graphql-server/src/modules/channel/channel.model'
+import { Category } from '../../generated/graphql-server/src/modules/category/category.model'
+import { KnownLicenseEntity } from '../../generated/graphql-server/src/modules/known-license-entity/known-license-entity.model'
+import { UserDefinedLicenseEntity } from '../../generated/graphql-server/src/modules/user-defined-license-entity/user-defined-license-entity.model'
+import { JoystreamMediaLocationEntity } from '../../generated/graphql-server/src/modules/joystream-media-location-entity/joystream-media-location-entity.model'
+import { HttpMediaLocationEntity } from '../../generated/graphql-server/src/modules/http-media-location-entity/http-media-location-entity.model'
+import { VideoMedia } from '../../generated/graphql-server/src/modules/video-media/video-media.model'
+import { Language } from '../../generated/graphql-server/src/modules/language/language.model'
+import { VideoMediaEncoding } from '../../generated/graphql-server/src/modules/video-media-encoding/video-media-encoding.model'
+import { LicenseEntity } from '../../generated/graphql-server/src/modules/license-entity/license-entity.model'
+import { MediaLocationEntity } from '../../generated/graphql-server/src/modules/media-location-entity/media-location-entity.model'
+import { NextEntityId } from '../../generated/graphql-server/src/modules/next-entity-id/next-entity-id.model'
+
+import { decode } from './decode'
+import {
+  categoryPropertyNamesWithId,
+  channelPropertyNamesWithId,
+  httpMediaLocationPropertyNamesWithId,
+  joystreamMediaLocationPropertyNamesWithId,
+  knownLicensePropertyNamesWIthId,
+  languagePropertyNamesWIthId,
+  licensePropertyNamesWithId,
+  mediaLocationPropertyNamesWithId,
+  userDefinedLicensePropertyNamesWithId,
+  videoMediaEncodingPropertyNamesWithId,
+  videoPropertyNamesWithId,
+} from './content-dir-consts'
+import {
+  ClassEntityMap,
+  ICategory,
+  IChannel,
+  IDBBlockId,
+  IEntity,
+  IHttpMediaLocation,
+  IJoystreamMediaLocation,
+  IKnownLicense,
+  ILanguage,
+  ILicense,
+  IMediaLocation,
+  IReference,
+  IUserDefinedLicense,
+  IVideoMedia,
+  IVideoMediaEncoding,
+} from '../types'
+
+import {
+  createCategory,
+  createChannel,
+  createVideoMedia,
+  createUserDefinedLicense,
+  createKnownLicense,
+  createHttpMediaLocation,
+  createJoystreamMediaLocation,
+  createLanguage,
+  createVideoMediaEncoding,
+  createLicense,
+  createMediaLocation,
+} from './entity/create'
+
+import { DB } from '../../generated/indexer'
+
+// Keep track of the next entity id
+async function nextEntityId(db: DB, nextEntityId: number): Promise<void> {
+  let e = await db.get(NextEntityId, { where: { id: '1' } })
+  if (!e) e = new NextEntityId({ id: '1' })
+  e.nextId = nextEntityId
+  await db.save<NextEntityId>(e)
+}
+
+function generateEntityIdFromIndex(index: number): string {
+  return `${index}`
+}
+
+function findEntity(entityId: number, className: string, classEntityMap: ClassEntityMap): IEntity {
+  const newlyCreatedEntities = classEntityMap.get(className)
+  if (newlyCreatedEntities === undefined) throw Error(`Couldn't find '${className}' entities in the classEntityMap`)
+  const entity = newlyCreatedEntities.find((e) => e.indexOf === entityId)
+  if (!entity) throw Error(`Unknown ${className} entity id: ${entityId}`)
+
+  // Remove the inserted entity from the list
+  classEntityMap.set(
+    className,
+    newlyCreatedEntities.filter((e) => e.entityId !== entityId)
+  )
+  return entity
+}
+
+async function language(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  language: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<Language> {
+  let lang
+  const { entityId, existing } = language
+  if (existing) {
+    lang = await db.get(Language, { where: { id: entityId.toString() } })
+    if (!lang) throw Error(`Language entity not found`)
+    return lang
+  }
+
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+  // could be created in the transaction
+  lang = await db.get(Language, { where: { id } })
+  if (lang) return lang
+
+  // get the entity from list of newly created entities and insert into db
+  const { properties } = findEntity(entityId, 'Language', classEntityMap)
+  return await createLanguage(
+    { db, block, id },
+    decode.setEntityPropertyValues<ILanguage>(properties, languagePropertyNamesWIthId)
+  )
+}
+
+async function videoMediaEncoding(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  encoding: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<VideoMediaEncoding> {
+  let vmEncoding
+  const { entityId, existing } = encoding
+  if (existing) {
+    vmEncoding = await db.get(VideoMediaEncoding, { where: { id: entityId.toString() } })
+    if (!vmEncoding) throw Error(`VideoMediaEncoding entity not found`)
+    return vmEncoding
+  }
+
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+
+  // could be created in the transaction
+  vmEncoding = await db.get(VideoMediaEncoding, { where: { id } })
+  if (vmEncoding) return vmEncoding
+
+  const { properties } = findEntity(entityId, 'VideoMediaEncoding', classEntityMap)
+  return await createVideoMediaEncoding(
+    { db, block, id },
+    decode.setEntityPropertyValues<IVideoMediaEncoding>(properties, videoMediaEncodingPropertyNamesWithId)
+  )
+}
+
+async function videoMedia(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  media: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<VideoMedia> {
+  let videoM: VideoMedia | undefined
+  const { entityId, existing } = media
+  if (existing) {
+    videoM = await db.get(VideoMedia, { where: { id: entityId.toString() } })
+    if (!videoM) throw Error(`VideoMedia entity not found`)
+    return videoM
+  }
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+
+  // could be created in the transaction
+  videoM = await db.get(VideoMedia, { where: { id } })
+  if (videoM) return videoM
+
+  const { properties } = findEntity(entityId, 'VideoMedia', classEntityMap)
+  return await createVideoMedia(
+    { db, block, id },
+    classEntityMap,
+    decode.setEntityPropertyValues<IVideoMedia>(properties, videoPropertyNamesWithId),
+    nextEntityIdBeforeTransaction
+  )
+}
+
+async function knownLicense(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  knownLicense: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<KnownLicenseEntity> {
+  let kLicense: KnownLicenseEntity | undefined
+  const { entityId, existing } = knownLicense
+  if (existing) {
+    kLicense = await db.get(KnownLicenseEntity, { where: { id: entityId.toString() } })
+    if (!kLicense) throw Error(`KnownLicense entity not found`)
+    return kLicense
+  }
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+  // could be created in the transaction
+  kLicense = await db.get(KnownLicenseEntity, { where: { id } })
+  if (kLicense) return kLicense
+
+  const { properties } = findEntity(entityId, 'KnownLicense', classEntityMap)
+  return await createKnownLicense(
+    { db, block, id },
+    decode.setEntityPropertyValues<IKnownLicense>(properties, knownLicensePropertyNamesWIthId)
+  )
+}
+async function userDefinedLicense(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  userDefinedLicense: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<UserDefinedLicenseEntity> {
+  let udLicense: UserDefinedLicenseEntity | undefined
+  const { entityId, existing } = userDefinedLicense
+  if (existing) {
+    udLicense = await db.get(UserDefinedLicenseEntity, { where: { id: entityId.toString() } })
+    if (!udLicense) throw Error(`UserDefinedLicense entity not found`)
+    return udLicense
+  }
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+  // could be created in the transaction
+  udLicense = await db.get(UserDefinedLicenseEntity, {
+    where: { id },
+  })
+  if (udLicense) return udLicense
+
+  const { properties } = findEntity(entityId, 'UserDefinedLicense', classEntityMap)
+  return await createUserDefinedLicense(
+    { db, block, id },
+    decode.setEntityPropertyValues<IUserDefinedLicense>(properties, userDefinedLicensePropertyNamesWithId)
+  )
+}
+
+async function channel(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  channel: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<Channel> {
+  let chann: Channel | undefined
+  const { entityId, existing } = channel
+
+  if (existing) {
+    chann = await db.get(Channel, { where: { id: entityId.toString() } })
+    if (!chann) throw Error(`Channel entity not found`)
+    return chann
+  }
+
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+  // could be created in the transaction
+  chann = await db.get(Channel, { where: { id } })
+  if (chann) return chann
+
+  const { properties } = findEntity(entityId, 'Channel', classEntityMap)
+  return await createChannel(
+    { db, block, id },
+    classEntityMap,
+    decode.setEntityPropertyValues<IChannel>(properties, channelPropertyNamesWithId),
+    nextEntityIdBeforeTransaction
+  )
+}
+
+async function category(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  category: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<Category> {
+  let cat: Category | undefined
+  const { entityId, existing } = category
+
+  if (existing) {
+    cat = await db.get(Category, { where: { id: entityId.toString() } })
+    if (!cat) throw Error(`Category entity not found`)
+    return cat
+  }
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+  // could be created in the transaction
+  cat = await db.get(Category, { where: { id } })
+  if (cat) return cat
+
+  const { properties } = findEntity(entityId, 'Category', classEntityMap)
+  return await createCategory(
+    { db, block, id },
+    decode.setEntityPropertyValues<ICategory>(properties, categoryPropertyNamesWithId)
+  )
+}
+
+async function httpMediaLocation(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  httpMediaLoc: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<HttpMediaLocationEntity | undefined> {
+  let loc: HttpMediaLocationEntity | undefined
+  const { entityId, existing } = httpMediaLoc
+
+  if (existing) {
+    loc = await db.get(HttpMediaLocationEntity, { where: { id: entityId.toString() } })
+    if (!loc) throw Error(`HttpMediaLocation entity not found`)
+    return loc
+  }
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+
+  // could be created in the transaction
+  loc = await db.get(HttpMediaLocationEntity, {
+    where: { id },
+  })
+  if (loc) return loc
+
+  const { properties } = findEntity(entityId, 'HttpMediaLocation', classEntityMap)
+  return await createHttpMediaLocation(
+    { db, block, id },
+    decode.setEntityPropertyValues<IHttpMediaLocation>(properties, httpMediaLocationPropertyNamesWithId)
+  )
+}
+
+async function joystreamMediaLocation(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  joyMediaLoc: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<JoystreamMediaLocationEntity | undefined> {
+  let loc: JoystreamMediaLocationEntity | undefined
+  const { entityId, existing } = joyMediaLoc
+
+  if (existing) {
+    loc = await db.get(JoystreamMediaLocationEntity, { where: { id: entityId.toString() } })
+    if (!loc) throw Error(`JoystreamMediaLocation entity not found`)
+    return loc
+  }
+
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+
+  // could be created in the transaction
+  loc = await db.get(JoystreamMediaLocationEntity, {
+    where: { id },
+  })
+  if (loc) return loc
+
+  const { properties } = findEntity(entityId, 'JoystreamMediaLocation', classEntityMap)
+  return await createJoystreamMediaLocation(
+    { db, block, id },
+    decode.setEntityPropertyValues<IJoystreamMediaLocation>(properties, joystreamMediaLocationPropertyNamesWithId)
+  )
+}
+
+async function license(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  license: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<LicenseEntity> {
+  let lic: LicenseEntity | undefined
+  const { entityId, existing } = license
+
+  if (existing) {
+    lic = await db.get(LicenseEntity, { where: { id: entityId.toString() } })
+    if (!lic) throw Error(`License entity not found`)
+    return lic
+  }
+
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+  // could be created in the transaction
+  lic = await db.get(LicenseEntity, { where: { id }, relations: ['knownLicense', 'userdefinedLicense'] })
+  if (lic) return lic
+
+  const { properties } = findEntity(entityId, 'License', classEntityMap)
+  return await createLicense(
+    { db, block, id },
+    classEntityMap,
+    decode.setEntityPropertyValues<ILicense>(properties, licensePropertyNamesWithId),
+    nextEntityIdBeforeTransaction
+  )
+}
+
+async function mediaLocation(
+  { db, block }: IDBBlockId,
+  classEntityMap: ClassEntityMap,
+  location: IReference,
+  nextEntityIdBeforeTransaction: number
+): Promise<MediaLocationEntity> {
+  let loc: MediaLocationEntity | undefined
+  const { entityId, existing } = location
+  if (existing) {
+    loc = await db.get(MediaLocationEntity, { where: { id: entityId.toString() } })
+    if (!loc) throw Error(`MediaLocation entity not found`)
+    return loc
+  }
+  const id = generateEntityIdFromIndex(nextEntityIdBeforeTransaction + entityId)
+
+  // could be created in the transaction
+  loc = await db.get(MediaLocationEntity, {
+    where: { id },
+    relations: ['httpMediaLocation', 'joystreamMediaLocation'],
+  })
+  if (loc) {
+    return loc
+  }
+
+  const { properties } = findEntity(entityId, 'MediaLocation', classEntityMap)
+  return await createMediaLocation(
+    { db, block, id },
+    classEntityMap,
+    decode.setEntityPropertyValues<IMediaLocation>(properties, mediaLocationPropertyNamesWithId),
+    nextEntityIdBeforeTransaction
+  )
+}
+
+export const getOrCreate = {
+  language,
+  videoMediaEncoding,
+  videoMedia,
+  knownLicense,
+  userDefinedLicense,
+  channel,
+  category,
+  joystreamMediaLocation,
+  httpMediaLocation,
+  license,
+  mediaLocation,
+  nextEntityId,
+}

query-node/mappings/content-directory/transaction.ts

@@ -1,8 +1,12 @@
 import Debug from 'debug'
 
 import { DB, SubstrateEvent } from '../../generated/indexer'
+import { NextEntityId } from '../../generated/graphql-server/src/modules/next-entity-id/next-entity-id.model'
+import { ClassEntity } from '../../generated/graphql-server/src/modules/class-entity/class-entity.model'
+
 import { decode } from './decode'
 import {
+  ClassEntityMap,
   ICategory,
   IChannel,
   ICreateEntityOperation,
@@ -12,6 +16,8 @@ import {
   IJoystreamMediaLocation,
   IKnownLicense,
   ILanguage,
+  ILicense,
+  IMediaLocation,
   IUserDefinedLicense,
   IVideo,
   IVideoMedia,
@@ -19,7 +25,7 @@ import {
   IWhereCond,
 } from '../types'
 import {
-  CategoryPropertyNamesWithId,
+  categoryPropertyNamesWithId,
   channelPropertyNamesWithId,
   knownLicensePropertyNamesWIthId,
   userDefinedLicensePropertyNamesWithId,
@@ -30,19 +36,10 @@ import {
   videoPropertyNamesWithId,
   languagePropertyNamesWIthId,
   ContentDirectoryKnownClasses,
+  licensePropertyNamesWithId,
+  mediaLocationPropertyNamesWithId,
 } from './content-dir-consts'
 import {
-  createCategory,
-  createChannel,
-  createVideoMedia,
-  createVideo,
-  createUserDefinedLicense,
-  createKnownLicense,
-  createHttpMediaLocation,
-  createJoystreamMediaLocation,
-  createLanguage,
-  createVideoMediaEncoding,
-  getClassName,
   updateCategoryEntityPropertyValues,
   updateChannelEntityPropertyValues,
   updateVideoMediaEntityPropertyValues,
@@ -53,10 +50,36 @@ import {
   updateKnownLicenseEntityPropertyValues,
   updateLanguageEntityPropertyValues,
   updateVideoMediaEncodingEntityPropertyValues,
-  batchCreateClassEntities,
-} from './entity-helper'
+  updateLicenseEntityPropertyValues,
+  updateMediaLocationEntityPropertyValues,
+} from './entity/update'
 
-const debug = Debug('mappings:content-directory')
+import {
+  createCategory,
+  createChannel,
+  createVideoMedia,
+  createVideo,
+  createUserDefinedLicense,
+  createKnownLicense,
+  createHttpMediaLocation,
+  createJoystreamMediaLocation,
+  createLanguage,
+  createVideoMediaEncoding,
+  getClassName,
+  createLicense,
+  createMediaLocation,
+  createBlockOrGetFromDatabase,
+} from './entity/create'
+import { getOrCreate } from './get-or-create'
+
+const debug = Debug('mappings:cd:transaction')
+
+async function getNextEntityId(db: DB): Promise<number> {
+  const e = await db.get(NextEntityId, { where: { id: '1' } })
+  // Entity creation happens before addSchemaSupport so this should never happen
+  if (!e) throw Error(`NextEntityId table doesn't have any record`)
+  return e.nextId
+}
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
 export async function contentDirectory_TransactionCompleted(db: DB, event: SubstrateEvent): Promise<void> {
@@ -80,11 +103,30 @@ export async function contentDirectory_TransactionCompleted(db: DB, event: Subst
 
   // Create entities before adding schema support
   // We need this to know which entity belongs to which class(we will need to know to update/create
-  // Channel, Video etc.). For example if there is
-  // a property update operation there is no class id
+  // Channel, Video etc.). For example if there is a property update operation there is no class id
   await batchCreateClassEntities(db, block, createEntityOperations)
-  await batchUpdatePropertyValue(db, createEntityOperations, updatePropertyValuesOperations)
+
   await batchAddSchemaSupportToEntity(db, createEntityOperations, addSchemaSupportToEntityOperations, block)
+
+  await batchUpdatePropertyValue(db, createEntityOperations, updatePropertyValuesOperations)
+}
+
+async function batchCreateClassEntities(db: DB, block: number, operations: ICreateEntityOperation[]): Promise<void> {
+  const nId = await db.get(NextEntityId, { where: { id: '1' } })
+  let nextId = nId ? nId.nextId : 1 // start entity id from 1
+
+  for (const { classId } of operations) {
+    const c = new ClassEntity({
+      id: nextId.toString(), // entity id
+      classId: classId,
+      version: block,
+      happenedIn: await createBlockOrGetFromDatabase(db, block),
+    })
+    await db.save<ClassEntity>(c)
+    nextId++
+  }
+
+  await getOrCreate.nextEntityId(db, nextId)
 }
 
 /**
@@ -100,80 +142,124 @@ async function batchAddSchemaSupportToEntity(
   entities: IEntity[],
   block: number
 ) {
-  // find the related entity ie. Channel, Video etc
-  for (const entity of entities) {
-    const { entityId, indexOf, properties } = entity
-
-    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-    const id = entityId ? entityId.toString() : indexOf!.toString()
+  const classEntityMap: ClassEntityMap = new Map<string, IEntity[]>()
 
+  for (const entity of entities) {
     const className = await getClassName(db, entity, createEntityOperations)
-    if (className === undefined) continue
-
-    const arg: IDBBlockId = { db, block, id }
-
-    switch (className) {
-      case ContentDirectoryKnownClasses.CATEGORY:
-        await createCategory(arg, decode.setEntityPropertyValues<ICategory>(properties, CategoryPropertyNamesWithId))
-        break
-
-      case ContentDirectoryKnownClasses.CHANNEL:
-        await createChannel(arg, decode.setEntityPropertyValues<IChannel>(properties, channelPropertyNamesWithId))
-        break
-
-      case ContentDirectoryKnownClasses.KNOWNLICENSE:
-        await createKnownLicense(
-          arg,
-          decode.setEntityPropertyValues<IKnownLicense>(properties, knownLicensePropertyNamesWIthId)
-        )
-        break
-
-      case ContentDirectoryKnownClasses.USERDEFINEDLICENSE:
-        await createUserDefinedLicense(
-          arg,
-          decode.setEntityPropertyValues<IUserDefinedLicense>(properties, userDefinedLicensePropertyNamesWithId)
-        )
-        break
-
-      case ContentDirectoryKnownClasses.JOYSTREAMMEDIALOCATION:
-        await createJoystreamMediaLocation(
-          arg,
-          decode.setEntityPropertyValues<IJoystreamMediaLocation>(properties, joystreamMediaLocationPropertyNamesWithId)
-        )
-        break
-
-      case ContentDirectoryKnownClasses.HTTPMEDIALOCATION:
-        await createHttpMediaLocation(
-          arg,
-          decode.setEntityPropertyValues<IHttpMediaLocation>(properties, httpMediaLocationPropertyNamesWithId)
-        )
-        break
-
-      case ContentDirectoryKnownClasses.VIDEOMEDIA:
-        await createVideoMedia(
-          arg,
-          decode.setEntityPropertyValues<IVideoMedia>(properties, videoMediaPropertyNamesWithId)
-        )
-        break
-
-      case ContentDirectoryKnownClasses.VIDEO:
-        await createVideo(arg, decode.setEntityPropertyValues<IVideo>(properties, videoPropertyNamesWithId))
-        break
-
-      case ContentDirectoryKnownClasses.LANGUAGE:
-        await createLanguage(arg, decode.setEntityPropertyValues<ILanguage>(properties, languagePropertyNamesWIthId))
-        break
-
-      case ContentDirectoryKnownClasses.VIDEOMEDIAENCODING:
-        await createVideoMediaEncoding(
-          arg,
-          decode.setEntityPropertyValues<IVideoMediaEncoding>(properties, videoMediaEncodingPropertyNamesWithId)
-        )
-        break
+    if (className !== undefined) {
+      const es = classEntityMap.get(className)
+      classEntityMap.set(className, es ? [...es, entity] : [entity])
+    }
+  }
 
-      default:
-        console.log(`Unknown class name: ${className}`)
-        break
+  // This is a copy of classEntityMap, we will use it to keep track of items.
+  // We will remove items from this list whenever we insert them into db
+  const doneList: ClassEntityMap = new Map(classEntityMap.entries())
+
+  const nextEntityIdBeforeTransaction = (await getNextEntityId(db)) - createEntityOperations.length
+
+  for (const [className, entities] of classEntityMap) {
+    for (const entity of entities) {
+      const { entityId, indexOf, properties } = entity
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      const id = entityId !== undefined ? entityId : indexOf! + nextEntityIdBeforeTransaction
+      const arg: IDBBlockId = { db, block, id: id.toString() }
+
+      switch (className) {
+        case ContentDirectoryKnownClasses.CATEGORY:
+          await createCategory(arg, decode.setEntityPropertyValues<ICategory>(properties, categoryPropertyNamesWithId))
+          break
+
+        case ContentDirectoryKnownClasses.CHANNEL:
+          await createChannel(
+            arg,
+            doneList,
+            decode.setEntityPropertyValues<IChannel>(properties, channelPropertyNamesWithId),
+            nextEntityIdBeforeTransaction
+          )
+          break
+
+        case ContentDirectoryKnownClasses.KNOWNLICENSE:
+          await createKnownLicense(
+            arg,
+            decode.setEntityPropertyValues<IKnownLicense>(properties, knownLicensePropertyNamesWIthId)
+          )
+          break
+
+        case ContentDirectoryKnownClasses.USERDEFINEDLICENSE:
+          await createUserDefinedLicense(
+            arg,
+            decode.setEntityPropertyValues<IUserDefinedLicense>(properties, userDefinedLicensePropertyNamesWithId)
+          )
+          break
+
+        case ContentDirectoryKnownClasses.JOYSTREAMMEDIALOCATION:
+          await createJoystreamMediaLocation(
+            arg,
+            decode.setEntityPropertyValues<IJoystreamMediaLocation>(
+              properties,
+              joystreamMediaLocationPropertyNamesWithId
+            )
+          )
+          break
+
+        case ContentDirectoryKnownClasses.HTTPMEDIALOCATION:
+          await createHttpMediaLocation(
+            arg,
+            decode.setEntityPropertyValues<IHttpMediaLocation>(properties, httpMediaLocationPropertyNamesWithId)
+          )
+          break
+
+        case ContentDirectoryKnownClasses.VIDEOMEDIA:
+          await createVideoMedia(
+            arg,
+            doneList,
+            decode.setEntityPropertyValues<IVideoMedia>(properties, videoMediaPropertyNamesWithId),
+            nextEntityIdBeforeTransaction
+          )
+          break
+
+        case ContentDirectoryKnownClasses.VIDEO:
+          await createVideo(
+            arg,
+            doneList,
+            decode.setEntityPropertyValues<IVideo>(properties, videoPropertyNamesWithId),
+            nextEntityIdBeforeTransaction
+          )
+          break
+
+        case ContentDirectoryKnownClasses.LANGUAGE:
+          await createLanguage(arg, decode.setEntityPropertyValues<ILanguage>(properties, languagePropertyNamesWIthId))
+          break
+
+        case ContentDirectoryKnownClasses.VIDEOMEDIAENCODING:
+          await createVideoMediaEncoding(
+            arg,
+            decode.setEntityPropertyValues<IVideoMediaEncoding>(properties, videoMediaEncodingPropertyNamesWithId)
+          )
+          break
+
+        case ContentDirectoryKnownClasses.LICENSE:
+          await createLicense(
+            arg,
+            classEntityMap,
+            decode.setEntityPropertyValues<ILicense>(properties, licensePropertyNamesWithId),
+            nextEntityIdBeforeTransaction
+          )
+          break
+        case ContentDirectoryKnownClasses.MEDIALOCATION:
+          await createMediaLocation(
+            arg,
+            classEntityMap,
+            decode.setEntityPropertyValues<IMediaLocation>(properties, mediaLocationPropertyNamesWithId),
+            nextEntityIdBeforeTransaction
+          )
+          break
+
+        default:
+          console.log(`Unknown class name: ${className}`)
+          break
+      }
     }
   }
 }
@@ -185,12 +271,14 @@ async function batchAddSchemaSupportToEntity(
  * @param entities list of entities those properties values updated
  */
 async function batchUpdatePropertyValue(db: DB, createEntityOperations: ICreateEntityOperation[], entities: IEntity[]) {
+  const entityIdBeforeTransaction = (await getNextEntityId(db)) - createEntityOperations.length
+
   for (const entity of entities) {
     const { entityId, indexOf, properties } = entity
     // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-    const id = entityId ? entityId.toString() : indexOf!.toString()
+    const id = entityId ? entityId.toString() : entityIdBeforeTransaction - indexOf!
 
-    const where: IWhereCond = { where: { id } }
+    const where: IWhereCond = { where: { id: id.toString() } }
     const className = await getClassName(db, entity, createEntityOperations)
     if (className === undefined) {
       console.log(`Can not update entity properties values. Unknown class name`)
@@ -199,10 +287,11 @@ async function batchUpdatePropertyValue(db: DB, createEntityOperations: ICreateE
 
     switch (className) {
       case ContentDirectoryKnownClasses.CHANNEL:
-        updateChannelEntityPropertyValues(
+        await updateChannelEntityPropertyValues(
           db,
           where,
-          decode.setEntityPropertyValues<IChannel>(properties, CategoryPropertyNamesWithId)
+          decode.setEntityPropertyValues<IChannel>(properties, channelPropertyNamesWithId),
+          entityIdBeforeTransaction
         )
         break
 
@@ -210,7 +299,7 @@ async function batchUpdatePropertyValue(db: DB, createEntityOperations: ICreateE
         await updateCategoryEntityPropertyValues(
           db,
           where,
-          decode.setEntityPropertyValues<ICategory>(properties, CategoryPropertyNamesWithId)
+          decode.setEntityPropertyValues<ICategory>(properties, categoryPropertyNamesWithId)
         )
         break
 
@@ -250,7 +339,8 @@ async function batchUpdatePropertyValue(db: DB, createEntityOperations: ICreateE
         await updateVideoMediaEntityPropertyValues(
           db,
           where,
-          decode.setEntityPropertyValues<IVideoMedia>(properties, videoPropertyNamesWithId)
+          decode.setEntityPropertyValues<IVideoMedia>(properties, videoPropertyNamesWithId),
+          entityIdBeforeTransaction
         )
         break
 
@@ -258,7 +348,8 @@ async function batchUpdatePropertyValue(db: DB, createEntityOperations: ICreateE
         await updateVideoEntityPropertyValues(
           db,
           where,
-          decode.setEntityPropertyValues<IVideo>(properties, videoPropertyNamesWithId)
+          decode.setEntityPropertyValues<IVideo>(properties, videoPropertyNamesWithId),
+          entityIdBeforeTransaction
         )
         break
 
@@ -277,6 +368,22 @@ async function batchUpdatePropertyValue(db: DB, createEntityOperations: ICreateE
           decode.setEntityPropertyValues<IVideoMediaEncoding>(properties, videoMediaEncodingPropertyNamesWithId)
         )
         break
+      case ContentDirectoryKnownClasses.LICENSE:
+        await updateLicenseEntityPropertyValues(
+          db,
+          where,
+          decode.setEntityPropertyValues<ILicense>(properties, licensePropertyNamesWithId),
+          entityIdBeforeTransaction
+        )
+        break
+      case ContentDirectoryKnownClasses.MEDIALOCATION:
+        await updateMediaLocationEntityPropertyValues(
+          db,
+          where,
+          decode.setEntityPropertyValues<IMediaLocation>(properties, mediaLocationPropertyNamesWithId),
+          entityIdBeforeTransaction
+        )
+        break
 
       default:
         console.log(`Unknown class name: ${className}`)

query-node/mappings/types.ts

@@ -34,14 +34,19 @@ export interface MemberControllerAccount extends BaseJoystreamMember {
   controllerAccount: Buffer
 }
 
+export interface IReference {
+  entityId: number
+  existing: boolean
+}
+
 export interface IChannel {
-  title: string
+  handle: string
   description: string
-  coverPhotoURL: string
-  avatarPhotoURL: string
+  coverPhotoUrl: string
+  avatarPhotoUrl: string
   isPublic: boolean
   isCurated: boolean
-  language: number
+  language?: IReference
 }
 
 export interface ICategory {
@@ -79,32 +84,42 @@ export interface IVideoMediaEncoding {
 }
 
 export interface IVideoMedia {
-  encoding: number
+  encoding?: IReference
   pixelWidth: number
   pixelHeight: number
   size: number
-  location: number
+  location?: IReference
 }
 
 export interface IVideo {
   // referenced entity's id
-  channel: number
+  channel?: IReference
   // referenced entity's id
-  category: number
+  category?: IReference
   title: string
   description: string
   duration: number
   skippableIntroDuration?: number
-  thumbnailURL: string
-  language: number
+  thumbnailUrl: string
+  language?: IReference
   // referenced entity's id
-  media: number
+  media?: IReference
   hasMarketing?: boolean
   publishedBeforeJoystream?: number
   isPublic: boolean
   isCurated: boolean
   isExplicit: boolean
-  license: number
+  license?: IReference
+}
+
+export interface ILicense {
+  knownLicense?: IReference
+  userDefinedLicense?: IReference
+}
+
+export interface IMediaLocation {
+  httpMediaLocation?: IReference
+  joystreamMediaLocation?: IReference
 }
 
 export enum OperationType {
@@ -135,9 +150,15 @@ export interface IBatchOperation {
 }
 
 export interface IProperty {
-  [propertyId: string]: any
-  // propertyId: string;
-  // value: any;
+  // PropertId: Value
+  // [propertyId: string]: any
+
+  id: string
+  value: any
+
+  // If reference.exising is false then reference.entityId is the index that entity is at
+  // in the transaction batch operation
+  reference?: IReference
 }
 
 export interface IEntity {
@@ -149,9 +170,14 @@ export interface IEntity {
   properties: IProperty[]
 }
 
-export interface IPropertyIdWithName {
-  // propertyId - property name
-  [propertyId: string]: string
+export interface IPropertyDef {
+  name: string
+  type: string
+  required: boolean
+}
+
+export interface IPropertyWithId {
+  [inClassIndex: string]: IPropertyDef
 }
 
 export interface IWhereCond {
@@ -166,5 +192,8 @@ export interface ICreateEntityOperation {
 export interface IDBBlockId {
   db: DB
   block: number
+  // Entity id
   id: string
 }
+
+export type ClassEntityMap = Map<string, IEntity[]>

query-node/package.json

@@ -16,22 +16,27 @@
 		"db:schema:migrate": "(cd ./generated/graphql-server && yarn db:create && yarn db:sync && yarn db:migrate)",
 		"db:indexer:migrate": "(cd ./generated/indexer && yarn db:migrate)",
 		"db:migrate": "yarn db:schema:migrate && yarn db:indexer:migrate",
-		"codegen:all": "yarn hydra-cli codegen && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
-		"codegen:indexer": "yarn hydra-cli codegen --no-graphql && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
-		"codegen:server": "yarn hydra-cli codegen --no-indexer",
-		"docker:up": "docker-compose up -d"
+		"codegen:all": "yarn hydra-cli codegen --no-install && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
+		"codegen:indexer": "yarn hydra-cli codegen --no-install --no-graphql && cp indexer-tsconfig.json generated/indexer/tsconfig.json",
+		"codegen:server": "yarn hydra-cli codegen --no-install --no-indexer",
+		"cd-classes": "ts-node scripts/get-class-id-and-name.ts"
 	},
 	"author": "",
 	"license": "ISC",
 	"devDependencies": {
-		"@dzlzv/hydra-cli": "^0.0.17"
+		"@dzlzv/hydra-cli": "^0.0.21"
 	},
 	"dependencies": {
+		"@dzlzv/hydra-indexer-lib": "^0.0.19-legacy.1.26.1",
 		"@joystream/types": "^0.14.0",
 		"@types/bn.js": "^4.11.6",
 		"@types/debug": "^4.1.5",
 		"bn.js": "^5.1.2",
 		"debug": "^4.2.0",
+		"dotenvi": "^0.9.1",
 		"tslib": "^2.0.0"
+	},
+	"volta": {
+		"extends": "../package.json"
 	}
 }

query-node/run-tests.sh

@@ -7,8 +7,8 @@ cd $SCRIPT_PATH
 function cleanup() {
     # Show tail end of logs for the processor and indexer containers to
     # see any possible errors
-    (echo "## Processor Logs ##" && docker logs query-node_processor_1 --tail 50) || :
-    (echo "## Indexer Logs ##" && docker logs query-node_indexer_1 --tail 50) || :
+    (echo "## Processor Logs ##" && docker logs joystream_processor_1 --tail 50) || :
+    (echo "## Indexer Logs ##" && docker logs joystream_indexer_1 --tail 50) || :
     docker-compose down -v
 }
 
@@ -23,9 +23,14 @@ export WS_PROVIDER_ENDPOINT_URI=ws://joystream-node:9944/
 # typeorm commandline is used by db:migrate step below.
 ln -s ../../../../../node_modules/typeorm/cli.js generated/graphql-server/node_modules/.bin/typeorm || :
 
-yarn db:up
+# clean start
+docker-compose down -v
+
+docker-compose up -d db
 yarn db:migrate
-yarn docker:up
+docker-compose up -d graphql-server
+# Starting up processor will bring up all services it depends on
+docker-compose up -d processor
 
 # Run tests
-ATTACH_TO_NETWORK=query-node_default ../tests/network-tests/run-tests.sh content-directory
+ATTACH_TO_NETWORK=joystream_default ../tests/network-tests/run-tests.sh content-directory

query-node/schema.graphql

@@ -8,8 +8,8 @@ type Block @entity {
   "Block number as a string"
   id: ID!
   block: Int!
-  timestamp: Int!
-  nework: Network!
+  timestamp: BigInt!
+  network: Network!
 }
 
 "Stored information about a registered user"
@@ -18,7 +18,7 @@ type Member @entity {
   id: ID!
 
   "The unique handle chosen by member"
-  handle: String @unique @fulltext(query: "handles")
+  handle: String @unique @fulltext(query: "membersByHandle")
 
   "A Url to member's Avatar image"
   avatarUri: String
@@ -55,6 +55,14 @@ type ClassEntity @entity {
   happenedIn: Block!
 }
 
+"Keep track of the next entity id"
+type NextEntityId @entity {
+  "Constant field is set to '1'"
+  id: ID!
+
+  nextId: Int!
+}
+
 #### High Level Derivative Entities ####
 
 type Language @entity {
@@ -76,16 +84,16 @@ type Channel @entity {
   # owner: Member!
 
   "The title of the Channel"
-  title: String! @fulltext(query: "titles")
+  handle: String! @fulltext(query: "search")
 
   "The description of a Channel"
   description: String!
 
   "Url for Channel's cover (background) photo. Recommended ratio: 16:9."
-  coverPhotoURL: String!
+  coverPhotoUrl: String
 
   "Channel's avatar photo."
-  avatarPhotoURL: String!
+  avatarPhotoUrl: String
 
   "Flag signaling whether a channel is public."
   isPublic: Boolean!
@@ -94,9 +102,9 @@ type Channel @entity {
   isCurated: Boolean!
 
   "The primary langauge of the channel's content"
-  languageId: Int
+  language: Language
 
-  # videos: [Video!] @derivedFrom(field: "channel")
+  videos: [Video!] @derivedFrom(field: "channel")
 
   happenedIn: Block!
 }
@@ -106,12 +114,12 @@ type Category @entity {
   id: ID!
 
   "The name of the category"
-  name: String! @unique @fulltext(query: "names")
+  name: String! @unique @fulltext(query: "categoriesByName")
 
   "The description of the category"
   description: String
 
-  # videos: [Video!] @derivedFrom(field: "category")
+  videos: [Video!] @derivedFrom(field: "category")
 
   happenedIn: Block!
 }
@@ -122,9 +130,11 @@ type VideoMediaEncoding @entity {
   id: ID!
 
   name: String!
+
+  happenedIn: Block!
 }
 
-type KnownLicense @entity {
+type KnownLicenseEntity @entity {
   "Runtime entity identifier (EntityId)"
   id: ID!
 
@@ -143,7 +153,7 @@ type KnownLicense @entity {
   happenedIn: Block!
 }
 
-type UserDefinedLicense @entity {
+type UserDefinedLicenseEntity @entity {
   "Runtime entity identifier (EntityId)"
   id: ID!
 
@@ -153,7 +163,39 @@ type UserDefinedLicense @entity {
   happenedIn: Block!
 }
 
-type JoystreamMediaLocation @entity {
+type LicenseEntity @entity {
+  "Runtime entity identifier (EntityId)"
+  id: ID!
+
+  # One of the following field will be non-null
+
+  "Reference to a known license"
+  knownLicense: KnownLicenseEntity
+
+  "Reference to user-defined license"
+  userdefinedLicense: UserDefinedLicenseEntity
+
+  happenedIn: Block!
+}
+
+type MediaLocationEntity @entity {
+  "Runtime entity identifier (EntityId)"
+  id: ID!
+
+  # One of the following field will be non-null
+
+  "A reference to HttpMediaLocation"
+  httpMediaLocation: HttpMediaLocationEntity
+
+  "A reference to JoystreamMediaLocation"
+  joystreamMediaLocation: JoystreamMediaLocationEntity
+
+  videoMedia: VideoMedia @derivedFrom(field: "locationEntity")
+
+  happenedIn: Block!
+}
+
+type JoystreamMediaLocationEntity @entity {
   "Runtime entity identifier (EntityId)"
   id: ID!
 
@@ -163,7 +205,7 @@ type JoystreamMediaLocation @entity {
   happenedIn: Block!
 }
 
-type HttpMediaLocation @entity {
+type HttpMediaLocationEntity @entity {
   "Runtime entity identifier (EntityId)"
   id: ID!
 
@@ -181,7 +223,7 @@ type VideoMedia @entity {
   id: ID!
 
   "Encoding of the video media object"
-  encodingId: Int!
+  encoding: VideoMediaEncoding!
 
   "Video media width in pixels"
   pixelWidth: Int!
@@ -192,13 +234,12 @@ type VideoMedia @entity {
   "Video media size in bytes"
   size: Int
 
-  # video: Video! @derivedFrom(field: "media")
+  video: Video @derivedFrom(field: "media")
 
-  # One of the location field will be non-null
+  "Location of the video media object"
+  location: MediaLocation!
 
-  # httpMediaLocation: HttpMediaLocation
-  # joystreamMediaLocation: JoystreamMediaLocation
-  locationId: Int!
+  locationEntity: MediaLocationEntity
 
   happenedIn: Block!
 }
@@ -208,13 +249,13 @@ type Video @entity {
   id: ID!
 
   "Reference to member's channel"
-  channelId: Int!
+  channel: Channel!
 
   "Reference to a video category"
-  categoryId: Int!
+  category: Category!
 
   "The title of the video"
-  title: String! @fulltext(query: "titles")
+  title: String! @fulltext(query: "search")
 
   "The description of the Video"
   description: String!
@@ -226,13 +267,13 @@ type Video @entity {
   skippableIntroDuration: Int
 
   "Video thumbnail url (recommended ratio: 16:9)"
-  thumbnailURL: String!
+  thumbnailUrl: String!
 
   "Video's main langauge"
-  languageId: Int
+  language: Language
 
   "Reference to VideoMedia"
-  videoMediaId: Int!
+  media: VideoMedia!
 
   "Whether or not Video contains marketing"
   hasMarketing: Boolean
@@ -249,8 +290,43 @@ type Video @entity {
   "Whether the Video contains explicit material."
   isExplicit: Boolean!
 
-  # Lincense
-  licenseId: Int!
+  license: License!
 
   happenedIn: Block!
 }
+
+type JoystreamMediaLocation @variant {
+  "Id of the data object in the Joystream runtime dataDirectory module"
+  dataObjectId: String!
+}
+
+type HttpMediaLocation @variant {
+  "The http url pointing to the media"
+  url: String!
+
+  "The port to use when connecting to the http url (defaults to 80)"
+  port: Int
+}
+
+type KnownLicense @variant {
+  "Short, commonly recognized code of the licence (ie. CC_BY_SA)"
+  code: String!
+
+  "Full, descriptive name of the license (ie. Creative Commons - Attribution-NonCommercial-NoDerivs)"
+  name: String
+
+  "Short description of the license conditions"
+  description: String
+
+  "An url pointing to full license content"
+  url: String
+}
+
+type UserDefinedLicense @variant {
+  "Custom license content"
+  content: String!
+}
+
+union MediaLocation = HttpMediaLocation | JoystreamMediaLocation
+
+union License = KnownLicense | UserDefinedLicense

query-node/scripts/get-class-id-and-name.ts

@@ -0,0 +1,21 @@
+import { ApiPromise, WsProvider } from '@polkadot/api'
+import { types as joyTypes } from '@joystream/types'
+import * as BN from 'bn.js'
+
+async function main() {
+  // Initialize the api
+  const provider = new WsProvider('ws://127.0.0.1:9944')
+  const api = await ApiPromise.create({ provider, types: joyTypes })
+
+  const n = await api.query.contentDirectory.nextClassId()
+  const nextClassId = new BN(n.toJSON() as string).toNumber()
+  for (let id = 0; id < nextClassId; id++) {
+    const cls = await api.query.contentDirectory.classById(new BN(id))
+    const { name } = cls.toJSON() as never
+    console.log(id, name)
+  }
+}
+
+main()
+  .then(() => process.exit())
+  .catch(console.error)

rust-builder.Dockerfile

@@ -1,8 +0,0 @@
-FROM liuchong/rustup:1.46.0 AS builder
-LABEL description="Rust and WASM build environment for joystream and substrate"
-
-WORKDIR /setup
-COPY setup.sh /setup
-ENV TERM=xterm
-
-RUN ./setup.sh

scripts/runtime-code-shasum.sh

@@ -7,17 +7,16 @@ export WORKSPACE_ROOT=`cargo metadata --offline --no-deps --format-version 1 | j
 
 cd ${WORKSPACE_ROOT}
 
-# srot/owner/group/mtime arguments only work with gnu version of tar.
-# So if you run this on Mac the default version of tar is `bsdtar`
-# and you will not get an idempotent result.
-# Install gnu-tar with brew
-#   brew install gnu-tar
-#   export PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH"
-tar -c --sort=name --owner=root:0 --group=root:0 --mtime='UTC 2020-01-01' \
+TAR=tar
+if [[ "$OSTYPE" == "darwin"* ]]; then
+	TAR=gtar
+fi
+
+# sort/owner/group/mtime arguments only work with gnu version of tar!
+${TAR} -c --sort=name --owner=root:0 --group=root:0 --mtime='UTC 2020-01-01' \
     Cargo.lock \
     Cargo.toml \
     runtime \
     runtime-modules \
     utils/chain-spec-builder \
     joystream-node.Dockerfile | shasum | cut -d " " -f 1
-

setup.sh

@@ -2,11 +2,8 @@
 
 set -e
 
-# If OS is supported will install:
-#  - build tools and any other dependencies required for rust and substrate
-#  - rustup - rust insaller
-#  - rust compiler and toolchains
-#  - skips installing substrate and subkey
+# If OS is supported will install build tools for rust and substrate.
+# Skips installing substrate itself and subkey
 curl https://getsubstrate.io -sSf | bash -s -- --fast
 
 source ~/.cargo/env
@@ -19,13 +16,21 @@ rustup component add rustfmt clippy
 rustup install nightly-2020-05-23 --force
 rustup target add wasm32-unknown-unknown --toolchain nightly-2020-05-23
 
-# Ensure the stable toolchain is still the default
-rustup default stable
+# Sticking with older version of compiler to ensure working build
+rustup install 1.46.0
+rustup default 1.46.0
 
-# TODO: Install additional tools...
+if [[ "$OSTYPE" == "linux-gnu" ]]; then
+    apt-get install -y coreutils clang jq curl gcc xz-utils sudo pkg-config unzip clang libc6-dev-i386
+    apt-get install -y docker.io docker-compose
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+    brew install b2sum gnu-tar jq curl
+    echo "It is recommended to setup Docker desktop from: https://www.docker.com/products/docker-desktop"
+fi
 
-# - b2sum
-# - nodejs
-# - npm
-# - yarn
-# .... ?
+# Volta nodejs, npm, yarn tools manager
+curl https://get.volta.sh | bash
+
+volta install node@12
+volta install yarn
+volta install npx

start.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -e
+
+# Run a complete joystream development network on your machine using docker.
+# Make sure to run build.sh prior to running this script.
+
+# Clean start!
+docker-compose down -v
+
+function down()
+{
+    # Stop containers and clear volumes
+    docker-compose down -v
+}
+
+trap down EXIT
+
+# Run a local development chain
+docker-compose up -d joystream-node
+
+## Storage Infrastructure
+# Configure a dev storage node and start storage node
+DEBUG=joystream:storage-cli:dev yarn storage-cli dev-init
+docker-compose up -d colossus
+# Initialise the content directory with standard classes, schemas and initial entities
+yarn workspace @joystream/cd-schemas initialize:dev
+
+## Query Node Infrastructure
+# Initialize a new database for the query node infrastructure
+docker-compose up -d db
+yarn workspace query-node-root db:migrate
+# Startup all query-node infrastructure services
+docker-compose up -d graphql-server
+docker-compose up -d processor
+
+echo "press Ctrl+C to shutdown"
+
+# Start a dev instance of pioneer and wait for exit
+docker-compose up pioneer

storage-node/README.md

@@ -35,6 +35,7 @@ _Building_
 
 ```bash
 $ yarn install
+$ yarn build
 ```
 
 The command will install dependencies, and make a `colossus` executable available:

storage-node/docker-compose.yaml

@@ -1,29 +0,0 @@
-version: '3'
-services:
-  ipfs:
-    image: ipfs/go-ipfs:latest
-    ports:
-      - '127.0.0.1:5001:5001'
-      - '127.0.0.1:8080:8080'
-    volumes:
-      - ipfs-data:/data/ipfs
-    entrypoint: ''
-    command: |
-      /bin/sh -c "
-        set -e
-        /usr/local/bin/start_ipfs config profile apply lowpower
-        /usr/local/bin/start_ipfs config --json Gateway.PublicGateways '{\"localhost\": null }'
-        /sbin/tini -- /usr/local/bin/start_ipfs daemon --migrate=true
-      "
-  chain:
-    image: joystream/node:latest
-    ports:
-      - '127.0.0.1:9944:9944'
-    volumes:
-      - chain-data:/data
-    command: --dev --ws-external --base-path /data
-volumes:
-  ipfs-data:
-    driver: local
-  chain-data:
-    driver: local

storage-node/package.json

@@ -48,5 +48,8 @@
     "prettier": "^2.0.5",
     "typescript": "^3.9.6",
     "wsrun": "^3.6.5"
+  },
+  "volta": {
+    "extends": "../package.json"
   }
 }

storage-node/packages/cli/package.json

@@ -26,6 +26,9 @@
   "engines": {
     "node": ">=12.18.0"
   },
+  "volta": {
+    "extends": "../package.json"
+  },
   "scripts": {
     "test": "mocha 'dist/test/**/*.js'",
     "lint": "eslint --ext .js,.ts . && tsc --noEmit --pretty",

storage-node/packages/colossus/package.json

@@ -31,6 +31,9 @@
   "engines": {
     "node": ">=12.18.0"
   },
+  "volta": {
+    "extends": "../package.json"
+  },
   "scripts": {
     "test": "mocha 'test/**/*.js'",
     "lint": "eslint 'paths/**/*.js' 'lib/**/*.js'",

storage-node/packages/helios/package.json

@@ -14,5 +14,8 @@
     "@types/bn.js": "^4.11.5",
     "axios": "^0.19.0",
     "bn.js": "^4.11.8"
+  },
+  "volta": {
+    "extends": "../package.json"
   }
 }

storage-node/start-dev.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-# Avoid pulling joystream/node from docker hub. It is most likely
-# not the version that we want to work with. Either you should
-# build it locally or pull it down manually.
-if ! docker inspect joystream/node:latest > /dev/null 2>&1;
-then
-  echo "Didn't find local joystream/node:latest docker image."
-  exit 1
-fi
-
-SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
-cd $SCRIPT_PATH
-
-# stop prior run and clear volumes
-# docker-compose down -v
-
-# Run a development joystream-node chain and ipfs daemon in the background
-docker-compose up -d
-
-function down()
-{
-    # Stop containers and clear volumes
-    docker-compose down -v
-}
-
-trap down EXIT
-
-# configure the dev chain
-DEBUG=joystream:storage-cli:dev yarn storage-cli dev-init
-
-# Run the tests
-# Tests sometimes fail, so skip for now
-# yarn workspace storage-node test
-
-# Start Colossus storage-node
-DEBUG=joystream:* yarn colossus --dev

storage-node/stop-dev.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-# stop prior run and clear volumes
-docker-compose down -v

tests/network-tests/.env

@@ -1,7 +1,7 @@
 # Address of the Joystream node.
 NODE_URL = ws://127.0.0.1:9944
 # Address of the Joystream query node.
-QUERY_NODE_URL = http://127.0.0.1:8080/graphql
+QUERY_NODE_URL = http://127.0.0.1:8081/graphql
 # Account which is expected to provide sufficient funds to test accounts.
 TREASURY_ACCOUNT_URI = //Alice
 # Sudo Account

tests/network-tests/package.json

@@ -32,5 +32,8 @@
     "prettier": "2.0.2",
     "ts-node": "^8.8.1",
     "typescript": "^3.8.3"
+  },
+  "volta": {
+    "extends": "../../package.json"
   }
 }

tests/network-tests/run-tests.sh

@@ -75,7 +75,7 @@ trap cleanup EXIT
 
 # Initialize content-directory
 # sleep 15
-# yarn workspace cd-schemas initialize:dev
+# yarn workspace @joystream/cd-schemas initialize:dev
 # NOTE: Skipping this step and let the scenarios do this setup instead
 # or align the scenario expectations of the initial state to match
 # with what we do here.
@@ -108,4 +108,4 @@ SCENARIO=$1
 SCENARIO=${SCENARIO:=full}
 
 # Execute the tests
-time DEBUG=* yarn workspace network-tests test-run src/scenarios/${SCENARIO}.ts
+time DEBUG=* yarn workspace network-tests test-run src/scenarios/${SCENARIO}.ts

tests/network-tests/src/Api.ts

@@ -30,9 +30,9 @@ import {
 } from '@joystream/types/hiring'
 import { FillOpeningParameters, ProposalId } from '@joystream/types/proposals'
 import { v4 as uuid } from 'uuid'
-import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
-import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
-import { initializeContentDir, InputParser, ExtrinsicsHelper } from 'cd-schemas'
+import { ChannelEntity } from '@joystream/cd-schemas/types/entities/ChannelEntity'
+import { VideoEntity } from '@joystream/cd-schemas/types/entities/VideoEntity'
+import { initializeContentDir, InputParser, ExtrinsicsHelper } from '@joystream/cd-schemas'
 import { OperationType } from '@joystream/types/content-directory'
 import { gql, ApolloClient, ApolloQueryResult, NormalizedCacheObject } from '@apollo/client'
 

tests/network-tests/src/fixtures/contentDirectoryModule.ts

@@ -5,8 +5,8 @@ import { Seat } from '@joystream/types/council'
 import { v4 as uuid } from 'uuid'
 import { Utils } from '../utils'
 import { Fixture } from '../Fixture'
-import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
-import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
+import { ChannelEntity } from '@joystream/cd-schemas/types/entities/ChannelEntity'
+import { VideoEntity } from '@joystream/cd-schemas/types/entities/VideoEntity'
 
 export class CreateChannelFixture implements Fixture {
   private api: QueryNodeApi

tests/network-tests/src/flows/contentDirectory/creatingChannel.ts

@@ -1,20 +1,20 @@
 import { QueryNodeApi } from '../../Api'
 import { Utils } from '../../utils'
 import { CreateChannelFixture } from '../../fixtures/contentDirectoryModule'
-import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
+import { ChannelEntity } from '@joystream/cd-schemas/types/entities/ChannelEntity'
 import { assert } from 'chai'
 import { KeyringPair } from '@polkadot/keyring/types'
 
 export function createSimpleChannelFixture(api: QueryNodeApi): CreateChannelFixture {
   const channelEntity: ChannelEntity = {
-    title: 'Example channel',
+    handle: 'Example channel',
     description: 'This is an example channel',
     // We can use "existing" syntax to reference either an on-chain entity or other entity that's part of the same batch.
     // Here we reference language that we assume was added by initialization script (initialize:dev), as it is part of
     // input/entityBatches/LanguageBatch.json
     language: { existing: { code: 'EN' } },
     coverPhotoUrl: '',
-    avatarPhotoURL: '',
+    avatarPhotoUrl: '',
     isPublic: true,
   }
   return new CreateChannelFixture(api, channelEntity)
@@ -29,14 +29,14 @@ export default async function channelCreation(api: QueryNodeApi) {
   await Utils.wait(120000)
 
   // Ensure newly created channel was parsed by query node
-  const result = await api.getChannelbyTitle(createChannelHappyCaseFixture.channelEntity.title)
+  const result = await api.getChannelbyTitle(createChannelHappyCaseFixture.channelEntity.handle)
   const queriedChannel = result.data.channels[0]
 
-  assert(queriedChannel.title === createChannelHappyCaseFixture.channelEntity.title, 'Should be equal')
+  assert(queriedChannel.title === createChannelHappyCaseFixture.channelEntity.handle, 'Should be equal')
   assert(queriedChannel.description === createChannelHappyCaseFixture.channelEntity.description, 'Should be equal')
   assert(queriedChannel.coverPhotoUrl === createChannelHappyCaseFixture.channelEntity.coverPhotoUrl, 'Should be equal')
   assert(
-    queriedChannel.avatarPhotoUrl === createChannelHappyCaseFixture.channelEntity.avatarPhotoURL,
+    queriedChannel.avatarPhotoUrl === createChannelHappyCaseFixture.channelEntity.avatarPhotoUrl,
     'Should be equal'
   )
   assert(queriedChannel.isPublic === createChannelHappyCaseFixture.channelEntity.isPublic, 'Should be equal')

tests/network-tests/src/flows/contentDirectory/creatingVideo.ts

@@ -1,6 +1,6 @@
 import { QueryNodeApi, WorkingGroups } from '../../Api'
 import { CreateVideoFixture } from '../../fixtures/contentDirectoryModule'
-import { VideoEntity } from 'cd-schemas/types/entities/VideoEntity'
+import { VideoEntity } from '@joystream/cd-schemas/types/entities/VideoEntity'
 import { assert } from 'chai'
 
 export function createVideoReferencingChannelFixture(api: QueryNodeApi): CreateVideoFixture {
@@ -11,9 +11,9 @@ export function createVideoReferencingChannelFixture(api: QueryNodeApi): CreateV
     // (those referenced here are part of inputs/entityBatches)
     language: { existing: { code: 'EN' } },
     category: { existing: { name: 'Education' } },
-    // We use the same "existing" syntax to reference a channel by unique property (title)
+    // We use the same "existing" syntax to reference a channel by unique property (handle)
     // In this case it's a channel that we created in createChannel example
-    channel: { existing: { title: 'Example channel' } },
+    channel: { existing: { handle: 'Example channel' } },
     media: {
       // We use "new" syntax to sygnalize we want to create a new VideoMedia entity that will be related to this Video entity
       new: {
@@ -35,7 +35,7 @@ export function createVideoReferencingChannelFixture(api: QueryNodeApi): CreateV
       },
     },
     duration: 3600,
-    thumbnailURL: '',
+    thumbnailUrl: '',
     isExplicit: false,
     isPublic: true,
   }

tests/network-tests/src/flows/contentDirectory/updatingChannel.ts

@@ -1,21 +1,21 @@
 import { QueryNodeApi, WorkingGroups } from '../../Api'
 import { UpdateChannelFixture } from '../../fixtures/contentDirectoryModule'
-import { ChannelEntity } from 'cd-schemas/types/entities/ChannelEntity'
+import { ChannelEntity } from '@joystream/cd-schemas/types/entities/ChannelEntity'
 import { assert } from 'chai'
 
-export function createUpdateChannelTitleFixture(api: QueryNodeApi): UpdateChannelFixture {
+export function createUpdateChannelHandleFixture(api: QueryNodeApi): UpdateChannelFixture {
   // Create partial channel entity, only containing the fields we wish to update
   const channelUpdateInput: Partial<ChannelEntity> = {
-    title: 'Updated channel title',
+    handle: 'Updated channel handle',
   }
 
-  const uniquePropVal: Record<string, any> = { title: 'Example channel' }
+  const uniquePropVal: Record<string, any> = { handle: 'Example channel' }
 
   return new UpdateChannelFixture(api, channelUpdateInput, uniquePropVal)
 }
 
 export default async function updateChannel(api: QueryNodeApi) {
-  const createVideoHappyCaseFixture = createUpdateChannelTitleFixture(api)
+  const createVideoHappyCaseFixture = createUpdateChannelHandleFixture(api)
 
   await createVideoHappyCaseFixture.runner(false)
 }

types/package.json

@@ -59,5 +59,9 @@
   "bugs": {
     "url": "https://github.com/Joystream/joystream/issues"
   },
-  "homepage": "https://github.com/Joystream/joystream"
+  "homepage": "https://github.com/Joystream/joystream",
+  "volta": {
+    "node": "12.18.2",
+    "yarn": "1.22.4"
+  }
 }

utils/api-scripts/package.json

@@ -22,5 +22,8 @@
     "@polkadot/ts": "^0.1.56",
     "typescript": "^3.9.7",
     "ts-node": "^8.6.2"
+  },
+  "volta": {
+    "extends": "../../package.json"
   }
 }

utils/api-scripts/src/dev-set-runtime-code.ts

@@ -32,7 +32,7 @@ async function main() {
   const provider = new WsProvider('ws://127.0.0.1:9944')
 
   let api: ApiPromise
-  let retry = 3
+  let retry = 6
   while (true) {
     try {
       api = await ApiPromise.create({ provider, types })

yarn.lock

@@ -1388,10 +1388,10 @@
     ajv "^6.12.0"
     ajv-keywords "^3.4.1"
 
-"@dzlzv/hydra-cli@^0.0.17":
-  version "0.0.17"
-  resolved "https://registry.yarnpkg.com/@dzlzv/hydra-cli/-/hydra-cli-0.0.17.tgz#56ccae132f76e738724cdc5f0abcd47ff25df530"
-  integrity sha512-ixrjGn6a7UG7ecHYKWTHpcxbdi6X32NbtyCuewm4YGFdb+v0/Eg5zWhFbg1PbMUW9GllC4MiIjDF7Bh1fh9t7Q==
+"@dzlzv/hydra-cli@^0.0.21":
+  version "0.0.21"
+  resolved "https://registry.yarnpkg.com/@dzlzv/hydra-cli/-/hydra-cli-0.0.21.tgz#4f99f125d81fdf1f962e0c31b3acf937a75bd3c1"
+  integrity sha512-pHZ6Qms4DcOpl8eYWllwonkH8nHtDwLuj/P1O+5S0dl6aKJ2tYmvxpmH0OfBLUACf5K8g6eSMs36M37ospSQ8g==
   dependencies:
     "@oclif/command" "^1.5.20"
     "@oclif/config" "^1"
@@ -1411,11 +1411,12 @@
     listr "^0.14.3"
     lodash "^4.17.15"
     mustache "^4.0.1"
+    pluralize "^8.0.0"
     tslib "1.11.2"
     typeorm-model-generator "^0.4.2"
-    warthog "https://github.com/metmirr/warthog/releases/download/v2.20.0/warthog-v2.20.0.tgz"
+    warthog "https://github.com/metmirr/warthog/releases/download/v2.22.0/warthog-v2.22.0.tgz"
 
-"@dzlzv/hydra-indexer-lib@^0.0.19-legacy.1.26.1":
+"@dzlzv/hydra-indexer-lib@0.0.19-legacy.1.26.1", "@dzlzv/hydra-indexer-lib@^0.0.19-legacy.1.26.1":
   version "0.0.19-legacy.1.26.1"
   resolved "https://registry.yarnpkg.com/@dzlzv/hydra-indexer-lib/-/hydra-indexer-lib-0.0.19-legacy.1.26.1.tgz#346b564845b2014f7a4d9b3976c03e30da8dd309"
   integrity sha512-4pwaSDRIo/1MqxjfSotjv91fkIj/bfZcZx5nqjB9gRT85X28b3WqkqTFrzlGsGGbvUFWAx4WIeQKnY1yrpX89Q==
@@ -3400,7 +3401,7 @@
     is-ipfs "^0.6.0"
     recursive-fs "^1.1.2"
 
-"@polkadot/api-contract@^1.26.1":
+"@polkadot/api-contract@1.26.1", "@polkadot/api-contract@^1.26.1":
   version "1.26.1"
   resolved "https://registry.yarnpkg.com/@polkadot/api-contract/-/api-contract-1.26.1.tgz#a8b52ef469ab8bbddb83191f8d451e31ffd76142"
   integrity sha512-zLGA/MHUJf12vanUEUBBRqpHVAONHWztoHS0JTIWUUS2+3GEXk6hGw+7PPtBDfDsLj0LgU/Qna1bLalC/zyl5w==
@@ -4938,7 +4939,7 @@
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.159.tgz#61089719dc6fdd9c5cb46efc827f2571d1517065"
   integrity sha512-gF7A72f7WQN33DpqOWw9geApQPh4M3PxluMtaHxWHXEGSN12/WbcEk/eNSqWNQcQhF66VSZ06vCF94CrHwXJDg==
 
-"@types/lodash@^4.14.148", "@types/lodash@^4.14.161":
+"@types/lodash@^4.14.148":
   version "4.14.164"
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.164.tgz#52348bcf909ac7b4c1bcbeda5c23135176e5dfa0"
   integrity sha512-fXCEmONnrtbYUc5014avwBeMdhHHO8YJCkOBflUL9EoJBSKZ1dei+VO74fA7JkTHZ1GvZack2TyIw5U+1lT8jg==
@@ -4948,6 +4949,11 @@
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.157.tgz#fdac1c52448861dfde1a2e1515dbc46e54926dc8"
   integrity sha512-Ft5BNFmv2pHDgxV5JDsndOWTRJ+56zte0ZpYLowp03tW+K+t8u8YMOzAnpuqPgzX6WO1XpDIUm7u04M8vdDiVQ==
 
+"@types/lodash@^4.14.161":
+  version "4.14.165"
+  resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.165.tgz#74d55d947452e2de0742bad65270433b63a8c30f"
+  integrity sha512-tjSSOTHhI5mCHTy/OOXYIhi2Wt1qcbHmuXD1Ha7q70CgI/I71afO4XtLb/cVexki1oVYchpul/TOuu3Arcdxrg==
+
 "@types/long@^4.0.0":
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9"
@@ -7879,21 +7885,11 @@ bluebird@^3.1.1, bluebird@^3.3.5, bluebird@^3.5.1, bluebird@^3.5.3, bluebird@^3.
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
   integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==
 
-bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.1.1, bn.js@^4.11.8, bn.js@^4.4.0:
-  version "4.11.9"
-  resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.11.9.tgz#26d556829458f9d1e81fc48952493d0ba3507828"
-  integrity sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==
-
-bn.js@^5.1.1, bn.js@^5.1.2:
+bn.js@^4.0.0, bn.js@^4.1.0, bn.js@^4.1.1, bn.js@^4.11.8, bn.js@^4.4.0, bn.js@^5.1.1, bn.js@^5.1.2, bn.js@^5.1.3:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-5.1.2.tgz#c9686902d3c9a27729f43ab10f9d79c2004da7b0"
   integrity sha512-40rZaf3bUNKTVYu9sIeeEGOg7g14Yvnj9kH7b50EiwX0Q7A6umbvfI5tvHaOERH0XigqKkfLkFQxzb4e6CIXnA==
 
-bn.js@^5.1.3:
-  version "5.1.3"
-  resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-5.1.3.tgz#beca005408f642ebebea80b042b4d18d2ac0ee6b"
-  integrity sha512-GkTiFpjFtUzU9CbMeJ5iazkCzGL3jrhzerzZIuqLABjbwRaFt33I9tUdSNryIptM+RxDet6OKm2WnLXzW51KsQ==
-
 body-parser@1.19.0, body-parser@^1.18.3, body-parser@^1.19.0:
   version "1.19.0"
   resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.19.0.tgz#96b2709e57c9c4e09a6fd66a8fd979844f69f08a"
@@ -11341,7 +11337,7 @@ dotenv@^6.2.0:
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-6.2.0.tgz#941c0410535d942c8becf28d3f357dbd9d476064"
   integrity sha512-HygQCKUBSFl8wKQZBSemMywRWcEDNidvNbjGVyZu3nbZ8qq9ubiPoGLMdRDpfSrpkkm9BXYFkpKxxFX38o/76w==
 
-dotenvi@^0.9.0:
+dotenvi@^0.9.0, dotenvi@^0.9.1:
   version "0.9.1"
   resolved "https://registry.yarnpkg.com/dotenvi/-/dotenvi-0.9.1.tgz#e280012ee9d201a0c57cb1f6e43559603b6f0fb4"
   integrity sha512-gM9HKu6P8BS+jBQRcJRdWKkbIA35Ztszr2FEqp1oKYLMfdTWDumLNi9xlIeEAFc2C4DeOwsYcNi+mMl5OWGtcw==
@@ -16146,10 +16142,10 @@ is-color-stop@^1.0.0:
     rgb-regex "^1.0.1"
     rgba-regex "^1.0.0"
 
-is-core-module@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.0.0.tgz#58531b70aed1db7c0e8d4eb1a0a2d1ddd64bd12d"
-  integrity sha512-jq1AH6C8MuteOoBPwkxHafmByhL9j5q4OaPGdbuD+ZtQJVzH+i6E3BJDQcBA09k57i2Hh2yQbEG8yObZ0jdlWw==
+is-core-module@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.1.0.tgz#a4cc031d9b1aca63eecbd18a650e13cb4eeab946"
+  integrity sha512-YcV7BgVMRFRua2FqQzKtTDMz8iCuLEyGKjr70q8Zm1yy2qKcurbFEd79PAdHV77oL3NrAaOVQIbMmiHQCHB7ZA==
   dependencies:
     has "^1.0.3"
 
@@ -22401,6 +22397,11 @@ pg-protocol@^1.3.0:
   resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.3.0.tgz#3c8fb7ca34dbbfcc42776ce34ac5f537d6e34770"
   integrity sha512-64/bYByMrhWULUaCd+6/72c9PMWhiVFs3EVxl9Ct6a3v/U8+rKgqP2w+kKg/BIGgMJyB+Bk/eNivT32Al+Jghw==
 
+pg-protocol@^1.4.0:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.4.0.tgz#43a71a92f6fe3ac559952555aa3335c8cb4908be"
+  integrity sha512-El+aXWcwG/8wuFICMQjM5ZSAm6OWiJicFdNYo+VY3QP+8vI4SvLIWVe51PppTzMhikUJR+PsyIFKqfdXPz/yxA==
+
 pg-types@1.*:
   version "1.13.0"
   resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.13.0.tgz#75f490b8a8abf75f1386ef5ec4455ecf6b345c63"
@@ -22451,7 +22452,20 @@ pg@^7.12.1:
     pgpass "1.x"
     semver "4.3.2"
 
-pg@^8.0.3, pg@^8.3.3:
+pg@^8.0.3:
+  version "8.5.0"
+  resolved "https://registry.yarnpkg.com/pg/-/pg-8.5.0.tgz#c29332763ffd51ce52b07dc20dc2337f4d213d08"
+  integrity sha512-h+KHEwce67pAQilZhMCpCx1RC7rR1US7mdjwvKzHRaRxKQxbbFtv5UlwjzqILQ1dwhK+RVGqOVcahE/2KOcaeA==
+  dependencies:
+    buffer-writer "2.0.0"
+    packet-reader "1.0.0"
+    pg-connection-string "^2.4.0"
+    pg-pool "^3.2.2"
+    pg-protocol "^1.4.0"
+    pg-types "^2.1.0"
+    pgpass "1.x"
+
+pg@^8.3.3:
   version "8.4.2"
   resolved "https://registry.yarnpkg.com/pg/-/pg-8.4.2.tgz#2aa58166a23391e91d56a7ea57c6d99931c0642a"
   integrity sha512-E9FlUrrc7w3+sbRmL1CSw99vifACzB2TjhMM9J5w9D1LIg+6un0jKkpHS1EQf2CWhKhec2bhrBLVMmUBDbjPRQ==
@@ -25101,11 +25115,11 @@ resolve@1.1.7:
   integrity sha1-IDEU2CrSxe2ejgQRs5ModeiJ6Xs=
 
 resolve@1.x, resolve@^1.0.0:
-  version "1.18.1"
-  resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.18.1.tgz#018fcb2c5b207d2a6424aee361c5a266da8f4130"
-  integrity sha512-lDfCPaMKfOJXjy0dPayzPdF1phampNWr3qFCjAu+rw/qbQmr5jWH5xN2hwh9QKfw9E5v4hwV7A+jrCmL8yjjqA==
+  version "1.19.0"
+  resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.19.0.tgz#1af5bf630409734a067cae29318aac7fa29a267c"
+  integrity sha512-rArEXAgsBG4UgRGcynxWIWKFvh/XZCcS8UJdHhwy91zwAvCZIbcs+vAbflgBnNjYMs/i/i+/Ux6IZhML1yPvxg==
   dependencies:
-    is-core-module "^2.0.0"
+    is-core-module "^2.1.0"
     path-parse "^1.0.6"
 
 resolve@^1.1.6, resolve@^1.1.7, resolve@^1.11.0, resolve@^1.11.1, resolve@^1.17.0, resolve@^1.2.0:
@@ -28163,12 +28177,7 @@ typescript-formatter@^7.2.2:
     commandpost "^1.0.0"
     editorconfig "^0.15.0"
 
-typescript@3.5.2:
-  version "3.5.2"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.5.2.tgz#a09e1dc69bc9551cadf17dba10ee42cf55e5d56c"
-  integrity sha512-7KxJovlYhTX5RaRbUdkAXN1KUZ8PwWlTzQdHV6xNqvuFOs7+WBo10TQUqT19Q/Jz2hk5v9TQDIhyLhhJY4p5AA==
-
-typescript@^3.0.3, typescript@^3.7.2, typescript@^3.7.5, typescript@^3.8.3, typescript@^3.9.5, typescript@^3.9.6, typescript@^3.9.7:
+typescript@3.5.2, typescript@^3.0.3, typescript@^3.7.2, typescript@^3.7.5, typescript@^3.8.3, typescript@^3.9.5, typescript@^3.9.6, typescript@^3.9.7:
   version "3.9.7"
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.7.tgz#98d600a5ebdc38f40cb277522f12dc800e9e25fa"
   integrity sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==
@@ -29137,9 +29146,9 @@ warning@^4.0.2, warning@^4.0.3:
   dependencies:
     loose-envify "^1.0.0"
 
-"warthog@https://github.com/metmirr/warthog/releases/download/v2.20.0/warthog-v2.20.0.tgz":
-  version "2.20.0"
-  resolved "https://github.com/metmirr/warthog/releases/download/v2.20.0/warthog-v2.20.0.tgz#c655c7da3279b958d8fd6549be30d9eb811d48c7"
+"warthog@https://github.com/metmirr/warthog/releases/download/v2.22.0/warthog-v2.22.0.tgz":
+  version "2.22.0"
+  resolved "https://github.com/metmirr/warthog/releases/download/v2.22.0/warthog-v2.22.0.tgz#33162a65e2897c79f9b5c6c214565e85685108d3"
   dependencies:
     "@types/app-root-path" "^1.2.4"
     "@types/bn.js" "^4.11.6"