Fix/introduce average newness for ypp (#137)

* Adjust logic at creation

* Adjust logic on manager

* Transfer sub-weights to env

* Adjust resolver

* CR fixes

* Update src/utils/VideoRelevanceManager.ts

* Update .env

---------

Co-authored-by: Leszek Wiesner <leszek@jsgenesis.com>

.env

@@ -20,7 +20,7 @@ SUPPORT_NEW_CATEGORIES=true
 KILL_SWITCH_ON=false
 VIDEO_VIEW_PER_IP_TIME_LIMIT=86400 # 86400 seconds = 24 hours
 VIDEO_RELEVANCE_VIEWS_TICK=50 # every 50 views video relevance score will be recalculated
-RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5]" # [newness (30 days - days since created) weight, views weight, comments weight, rections weights]
+RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5, [7,3]]" # [newness (negative number of days since created) weight, views weight, comments weight, rections weights, [joystream creation weight, YT creation weight]]
 
 # Operator API secret
 OPERATOR_SECRET=this-is-not-so-secret-change-it

src/mappings/content/video.ts

@@ -8,12 +8,10 @@ import { DecodedMetadataObject } from '@joystream/metadata-protobuf/types'
 import { integrateMeta } from '@joystream/metadata-protobuf/utils'
 import { Channel, Video, VideoViewEvent } from '../../model'
 import { EventHandlerContext } from '../../utils/events'
-import { deserializeMetadata, u8aToBytes } from '../utils'
+import { deserializeMetadata, u8aToBytes, videoRelevanceManager } from '../utils'
 import { processVideoMetadata } from './metadata'
 import { deleteVideo, encodeAssets, processAppActionMetadata, processNft } from './utils'
 import { generateAppActionCommitment } from '@joystream/js/utils'
-import { config, ConfigVariable } from '../../utils/config'
-import { NEWNESS_SECONDS_DIVIDER } from '../../utils/VideoRelevanceManager'
 
 export async function processVideoCreatedEvent({
   overlay,
@@ -28,10 +26,6 @@ export async function processVideoCreatedEvent({
 
   const videoId = contentId.toString()
   const viewsNum = await overlay.getEm().getRepository(VideoViewEvent).countBy({ videoId })
-  const [newnessWeight, viewsWeight] = await config.get(
-    ConfigVariable.RelevanceWeights,
-    overlay.getEm()
-  )
   const video = overlay.getRepository(Video).new({
     id: videoId,
     createdAt: new Date(block.timestamp),
@@ -46,13 +40,11 @@ export async function processVideoCreatedEvent({
     reactionsCount: 0,
     viewsNum,
     // First we need to dic by 1k to match postgres epoch (in seconds) then apply the further dividers
-    videoRelevance: +(
-      (30 - (Date.now() - new Date(block.timestamp).getTime()) / (1000 * NEWNESS_SECONDS_DIVIDER)) *
-        newnessWeight +
-      viewsNum * viewsWeight
-    ).toFixed(2),
+    videoRelevance: 0,
   })
 
+  videoRelevanceManager.scheduleRecalcForVideo(videoId)
+
   // fetch related channel and owner
   const channel = await overlay.getRepository(Channel).getByIdOrFail(channelId.toString())
 
@@ -140,6 +132,10 @@ export async function processVideoUpdatedEvent({
   }
 
   if (videoMetadataUpdate) {
+    if ('publishedBeforeJoystream' in videoMetadataUpdate) {
+      delete videoMetadataUpdate.publishedBeforeJoystream
+    }
+
     await processVideoMetadata(
       overlay,
       block,

src/server-extension/resolvers/AdminResolver/index.ts

@@ -59,7 +59,13 @@ export class AdminResolver {
     const em = await this.em()
     await config.set(
       ConfigVariable.RelevanceWeights,
-      [args.newnessWeight, args.viewsWeight, args.commentsWeight, args.reactionsWeight],
+      [
+        args.newnessWeight,
+        args.viewsWeight,
+        args.commentsWeight,
+        args.reactionsWeight,
+        [args.joysteamTimestampSubWeight, args.ytTimestampSubWeight],
+      ],
       em
     )
     await videoRelevanceManager.updateVideoRelevanceValue(em, true)

src/server-extension/resolvers/AdminResolver/types.ts

@@ -14,6 +14,12 @@ export class SetVideoWeightsInput {
 
   @Field(() => Float, { nullable: false })
   reactionsWeight!: number
+
+  @Field(() => Float, { nullable: false })
+  joysteamTimestampSubWeight!: number
+
+  @Field(() => Float, { nullable: false })
+  ytTimestampSubWeight!: number
 }
 
 @ObjectType()

src/utils/VideoRelevanceManager.ts

@@ -25,29 +25,45 @@ export class VideoRelevanceManager {
 
   async updateVideoRelevanceValue(em: EntityManager, forceUpdateAll?: boolean) {
     if (this.videosToUpdate.size || forceUpdateAll) {
-      const [newnessWeight, viewsWeight, commentsWeight, reactionsWeight] = await config.get(
-        ConfigVariable.RelevanceWeights,
-        em
-      )
+      const [
+        newnessWeight,
+        viewsWeight,
+        commentsWeight,
+        reactionsWeight,
+        [joystreamTimestampWeight, ytTimestampWeight] = [7, 3],
+      ] = await config.get(ConfigVariable.RelevanceWeights, em)
       await em.query(`
-        UPDATE "video"
-        SET
-          "video_relevance" = ROUND(
-          ((30 - (extract(epoch from now() - created_at) / ${NEWNESS_SECONDS_DIVIDER})) * ${newnessWeight}) +
-          (views_num * ${viewsWeight}) +
-          (
-            comments_count * ${commentsWeight} 
-          ) +
-          (
-            reactions_count * ${reactionsWeight} 
-          ), 2)
+        WITH weighted_timestamp AS (
+    SELECT 
+        id,
+        (
+          extract(epoch from created_at)*${joystreamTimestampWeight} +
+          COALESCE(extract(epoch from published_before_joystream), extract(epoch from created_at))*${ytTimestampWeight}
+        ) / ${joystreamTimestampWeight + ytTimestampWeight} as wtEpoch
+    FROM 
+        "video" 
         ${
           forceUpdateAll
             ? ''
             : `WHERE "id" IN (${[...this.videosToUpdate.values()]
                 .map((id) => `'${id}'`)
                 .join(', ')})`
-        }`)
+        }
+        )
+    UPDATE 
+        "video"
+    SET
+        "video_relevance" = ROUND(
+        (extract(epoch from now()) - wtEpoch) / (60 * 60 * 24) * ${newnessWeight * -1} +
+        (views_num * ${viewsWeight}) +
+        (comments_count * ${commentsWeight}) +
+        (reactions_count * ${reactionsWeight}), 
+            2)
+    FROM
+        weighted_timestamp
+    WHERE
+        "video".id = weighted_timestamp.id;
+        `)
       this.videosToUpdate.clear()
     }
   }

src/utils/config.ts

@@ -37,7 +37,7 @@ export const configVariables = {
   [ConfigVariable.KillSwitch]: boolType,
   [ConfigVariable.VideoViewPerIpTimeLimit]: numberType,
   [ConfigVariable.VideoRelevanceViewsTick]: numberType,
-  [ConfigVariable.RelevanceWeights]: jsonType<[number, number, number, number]>(),
+  [ConfigVariable.RelevanceWeights]: jsonType<[number, number, number, number, [number, number]]>(),
   [ConfigVariable.AppPrivateKey]: stringType,
 } as const