Change DistributionBucketFamilyMetadata standard

metadata-protobuf/package.json

@@ -41,7 +41,9 @@
     "long": "^4.0.0",
     "@types/long": "^4.0.1",
     "i18n-iso-countries": "^6.8.0",
-    "iso-639-1": "^2.1.9"
+    "iso-639-1": "^2.1.9",
+    "iso-3166-2": "^1.0.0",
+    "@types/iso-3166-2": "^1.0.0"
   },
   "devDependencies": {
     "@types/chai": "^4.2.11",

metadata-protobuf/proto/Storage.proto

@@ -23,8 +23,26 @@ message DistributionBucketOperatorMetadata {
   optional string extra = 3; // Additional information about the node / node operator
 }
 
+message GeographicalArea {
+  enum Continent {
+    AF = 1;
+    NA = 2;
+    OC = 3;
+    AN = 4;
+    AS = 5;
+    EU = 6;
+    SA = 7;
+  }
+  oneof code {
+    Continent continent = 1;
+    string country_code = 2; // ISO 3166-1 alpha-2 country code
+    string subdivision_code = 3; // ISO 3166-2 subdivision code
+  }
+}
+
 message DistributionBucketFamilyMetadata {
   optional string region = 1; // ID / name of the region covered by the distribution family (ie. us-east-1). Should be unique.
   optional string description = 2; // Additional, more specific description of the region
-  repeated GeoCoordiantes boundary = 3; // Geographical boundary of the region, defined as polygon through array of coordinates (providing [{}] will unset the current value)
+  repeated GeographicalArea areas = 3; // Standarized geographical areas covered by the family (providing [{}] will unset the current value)
+  repeated string latency_test_targets = 4; // List of targets (hosts/ips) best suited latency measurements for this family
 }

metadata-protobuf/src/utils.ts

@@ -1,6 +1,7 @@
 import { AnyMessage, AnyMetadataClass, DecodedMetadataObject } from './types'
 import countries from 'i18n-iso-countries'
 import langs from 'iso-639-1'
+import subdivisions from 'iso-3166-2'
 
 export function isSet<T>(v: T | null | undefined): v is T {
   return v !== null && v !== undefined
@@ -42,3 +43,8 @@ export function isValidCountryCode(code: string): boolean {
 export function isValidLanguageCode(code: string): boolean {
   return langs.validate(code)
 }
+
+// According to ISO 3166-2 standard
+export function isValidSubdivisionCode(code: string): boolean {
+  return !!subdivisions.subdivision(code)
+}

query-node/mappings/storage/index.ts

@@ -171,7 +171,7 @@ async function getDistributionBucketFamilyWithMetadata(
 ): Promise<DistributionBucketFamily> {
   const family = await store.get(DistributionBucketFamily, {
     where: { id },
-    relations: ['metadata', 'metadata.boundary'],
+    relations: ['metadata', 'metadata.areas'],
   })
   if (!family) {
     throw new Error(`DistributionBucketFamily not found by id: ${id}`)

query-node/mappings/storage/metadata.ts

@@ -5,16 +5,32 @@ import {
   StorageBucketOperatorMetadata,
   GeoCoordinates,
   NodeLocationMetadata,
+  Continent,
+  GeographicalAreaContinent,
+  GeographicalAreaCountry,
+  GeographicalAreaSubdivistion,
+  DistributionBucketFamilyGeographicArea,
 } from 'query-node/dist/model'
-import { deserializeMetadata } from '../common'
+import { deserializeMetadata, invalidMetadata } from '../common'
 import { Bytes } from '@polkadot/types'
 import {
   DistributionBucketOperatorMetadata as DistributionBucketOperatorMetadataProto,
   StorageBucketOperatorMetadata as StorageBucketOperatorMetadataProto,
   DistributionBucketFamilyMetadata as DistributionBucketFamilyMetadataProto,
   INodeLocationMetadata,
+  GeographicalArea as GeographicalAreaProto,
 } from '@joystream/metadata-protobuf'
-import { isSet, isEmptyObject, isValidCountryCode } from '@joystream/metadata-protobuf/utils'
+import { isSet, isEmptyObject, isValidCountryCode, isValidSubdivisionCode } from '@joystream/metadata-protobuf/utils'
+
+const protobufContinentToGraphlContinent: { [key in GeographicalAreaProto.Continent]: Continent } = {
+  [GeographicalAreaProto.Continent.AF]: Continent.AF,
+  [GeographicalAreaProto.Continent.AN]: Continent.AN,
+  [GeographicalAreaProto.Continent.AS]: Continent.AS,
+  [GeographicalAreaProto.Continent.EU]: Continent.EU,
+  [GeographicalAreaProto.Continent.NA]: Continent.NA,
+  [GeographicalAreaProto.Continent.OC]: Continent.OC,
+  [GeographicalAreaProto.Continent.SA]: Continent.SA,
+}
 
 async function processNodeLocationMetadata(
   store: DatabaseManager,
@@ -118,24 +134,57 @@ export async function processDistributionBucketFamilyMetadata(
   if (isSet(meta.description)) {
     metadataEntity.description = meta.description || (null as any)
   }
+  if (isSet(meta.latencyTestTargets)) {
+    metadataEntity.latencyTestTargets = meta.latencyTestTargets
+  }
 
   await store.save<DistributionBucketOperatorMetadata>(metadataEntity)
 
-  // Update boundary after metadata is saved (since we need an id to reference)
-  if (isSet(meta.boundary)) {
-    await Promise.all((metadataEntity.boundary || []).map((coords) => store.remove<GeoCoordinates>(coords)))
+  // Update areas after metadata is saved (since we need an id to reference)
+  if (isSet(meta.areas)) {
+    // Drop current areas
+    await Promise.all(metadataEntity.areas?.map((a) => store.remove<DistributionBucketFamilyGeographicArea>(a)) || [])
+    // Save new areas
     await Promise.all(
-      meta.boundary
-        .filter((c) => !isEmptyObject(c))
-        .map(({ latitude, longitude }) =>
-          store.save<GeoCoordinates>(
-            new GeoCoordinates({
-              latitude: latitude || 0,
-              longitude: longitude || 0,
-              boundarySourceBucketFamilyMeta: metadataEntity,
-            })
-          )
-        )
+      meta.areas
+        .filter((a) => !isEmptyObject(a))
+        .map(async (a) => {
+          const area = new DistributionBucketFamilyGeographicArea({
+            distributionBucketFamilyMetadata: metadataEntity,
+          })
+
+          if (a.continent) {
+            const continent = new GeographicalAreaContinent()
+            continent.code = protobufContinentToGraphlContinent[a.continent]
+            if (!continent.code) {
+              return invalidMetadata(`Unrecognized continent enum variant: ${a.continent}`)
+            }
+            area.id = `${metadataEntity.id}-C-${continent.code}`
+            area.area = continent
+          }
+
+          if (a.countryCode) {
+            if (!isValidCountryCode(a.countryCode)) {
+              return invalidMetadata(`Invalid country code: ${a.countryCode}`)
+            }
+            const country = new GeographicalAreaCountry()
+            country.code = a.countryCode
+            area.id = `${metadataEntity.id}-c-${country.code}`
+            area.area = country
+          }
+
+          if (a.subdivisionCode) {
+            if (!isValidSubdivisionCode(a.subdivisionCode)) {
+              return invalidMetadata(`Invalid subdivision code: ${a.subdivisionCode}`)
+            }
+            const subdivision = new GeographicalAreaSubdivistion()
+            subdivision.code = a.subdivisionCode
+            area.id = `${metadataEntity.id}-s-${subdivision.code}`
+            area.area = subdivision
+          }
+
+          await store.save<DistributionBucketFamilyGeographicArea>(area)
+        })
     )
   }
 

query-node/schemas/storage.graphql

@@ -42,11 +42,34 @@ union StorageBucketOperatorStatus = StorageBucketOperatorStatusMissing | Storage
 type GeoCoordinates @entity {
   latitude: Float!
   longitude: Float!
+}
+
+enum Continent {
+  AF
+  NA
+  OC
+  AN
+  AS
+  EU
+  SA
+}
+
+type GeographicalAreaContinent @variant {
+  code: Continent
+}
+
+type GeographicalAreaCountry @variant {
+  "ISO 3166-1 alpha-2 country code"
+  code: String
+}
 
-  "Optional DistributionBucketFamilyMetadata reference in case the coordinates are part of a region boundary"
-  boundarySourceBucketFamilyMeta: DistributionBucketFamilyMetadata
+type GeographicalAreaSubdivistion @variant {
+  "ISO 3166-2 subdivision code"
+  code: String
 }
 
+union GeographicalArea = GeographicalAreaContinent | GeographicalAreaCountry | GeographicalAreaSubdivistion
+
 type NodeLocationMetadata @entity {
   "ISO 3166-1 alpha-2 country code (2 letters)"
   countryCode: String
@@ -185,6 +208,17 @@ type StorageDataObject @entity {
   ipfsHash: String!
 }
 
+type DistributionBucketFamilyGeographicArea @entity {
+  "{metadataId}-{(C|c|s)}-{code}"
+  id: ID!
+
+  "Geographical area (continent / country / subdivision)"
+  area: GeographicalArea!
+
+  "Related distribution bucket family metadata"
+  distributionBucketFamilyMetadata: DistributionBucketFamilyMetadata!
+}
+
 type DistributionBucketFamilyMetadata @entity {
   "Name of the geographical region covered by the family (ie.: us-east-1)"
   region: String
@@ -192,8 +226,11 @@ type DistributionBucketFamilyMetadata @entity {
   "Optional, more specific description of the region covered by the family"
   description: String
 
-  "Optional region boundary as geocoordiantes polygon"
-  boundary: [GeoCoordinates!] @derivedFrom(field: "boundarySourceBucketFamilyMeta")
+  "Geographical areas covered by the family"
+  areas: [DistributionBucketFamilyGeographicArea!] @derivedFrom(field: "distributionBucketFamilyMetadata")
+
+  "List of targets (hosts/ips) best suited latency measurements for the family"
+  latencyTestTargets: [String!]
 }
 
 type DistributionBucketOperatorMetadata @entity {

yarn.lock

@@ -4861,6 +4861,11 @@
   resolved "https://registry.yarnpkg.com/@types/is-function/-/is-function-1.0.0.tgz#1b0b819b1636c7baf0d6785d030d12edf70c3e83"
   integrity "sha1-GwuBmxY2x7rw1nhdAw0S7fcMPoM= sha512-iTs9HReBu7evG77Q4EC8hZnqRt57irBDkK9nvmHroiOIVwYMQc4IvYvdRgwKfYepunIY7Oh/dBuuld+Gj9uo6w=="
 
+"@types/iso-3166-2@^1.0.0":
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/@types/iso-3166-2/-/iso-3166-2-1.0.0.tgz#98a7b6e505c031e83c5aac13c9c49f9d9f6ca234"
+  integrity sha512-DYDyoRyPyxBeI9bYoTXLfsOZH12m1anrWEj9LU5Sl9rgsJ4soJMYf/7byozM+64Smn6/a1i079eQLGuPykwaHQ==
+
 "@types/isomorphic-fetch@^0.0.35":
   version "0.0.35"
   resolved "https://registry.yarnpkg.com/@types/isomorphic-fetch/-/isomorphic-fetch-0.0.35.tgz#c1c0d402daac324582b6186b91f8905340ea3361"
@@ -16922,6 +16927,10 @@ iso-639-1@^2.1.9:
   version "2.1.9"
   resolved "https://registry.yarnpkg.com/iso-639-1/-/iso-639-1-2.1.9.tgz#e41b11d4f1808e5316d0252c3fa16eeb9b37bb58"
   integrity sha512-owRu9up+Cpx/hwSzm83j6G8PtC7U99UCtPVItsafefNfEgMl+pi8KBwhXwJkJfp6IouyYWFxj8n24SvCWpKZEQ==
+iso-3166-2@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/iso-3166-2/-/iso-3166-2-1.0.0.tgz#20c5cda527b56bfc7409c6802d9bff0119086131"
+  integrity sha512-xLAazfKZzwlsg/Zz/GQGQk3jJez5/2ORrjD3TjSuqz/arMht/xTK49c0GOE3afO/gEd9tHtBVVlfBla01unUng==
 
 iso-constants@^0.1.2:
   version "0.1.2"