feat!: deserialization for backwards compatibility

breaking(tl-utils): signature of `generateTypescriptDefinitionsForTlSchema` and `generateTypescriptDefinitionsForTlEntry`

eslint.config.js

@@ -2,7 +2,11 @@ import antfu from '@antfu/eslint-config'
 
 export default antfu({
     type: 'lib',
-    ignores: ['e2e/runtime/*'],
+    ignores: [
+        'e2e/runtime/*',
+        'packages/tl/**/*.js',
+        'packages/tl/**/*.d.ts',
+    ],
     typescript: process.env.CI
         ? {
             tsconfigPath: 'tsconfig.json',

packages/core/src/utils/binary/compat.test.ts

@@ -0,0 +1,39 @@
+import { hex } from '@fuman/utils'
+import Long from 'long'
+import { describe, expect, it } from 'vitest'
+import { deserializeObjectWithCompat } from './compat.js'
+
+describe('binary/compat', () => {
+    it('should correctly read emojiStatus from layer 197', () => {
+        const data = hex.decode('9d619b920000000000000000')
+        expect(deserializeObjectWithCompat(data)).toEqual({
+            _: 'emojiStatus',
+            documentId: Long.ZERO,
+        })
+    })
+
+    it('should correctly read emojiStatus from layer 198', () => {
+        const data = hex.decode('8a06ffe7000000000000000000000000')
+        expect(deserializeObjectWithCompat(data)).toEqual({
+            _: 'emojiStatus',
+            documentId: Long.ZERO,
+        })
+    })
+
+    it('should correctly read emojiStatus from 197 inside channelAdminLogEventActionChangeEmojiStatus', () => {
+        // rather unlikely case where emojiStatus from different layers is inside the same object.
+        // still useful to test it tho
+        const data = hex.decode('b1fea93e9d619b9200000000000000008a06ffe7000000000000000000000000')
+        expect(deserializeObjectWithCompat(data)).toEqual({
+            _: 'channelAdminLogEventActionChangeEmojiStatus',
+            prevValue: {
+                _: 'emojiStatus',
+                documentId: Long.ZERO,
+            },
+            newValue: {
+                _: 'emojiStatus',
+                documentId: Long.ZERO,
+            },
+        })
+    })
+})

packages/core/src/utils/binary/compat.ts

@@ -0,0 +1,90 @@
+import type { tl } from '@mtcute/tl'
+import type { tlCompat } from '@mtcute/tl/compat'
+import { objectEntries } from '@fuman/utils'
+import { TlBinaryReader } from '@mtcute/tl-runtime'
+import { __tlReaderMap } from '@mtcute/tl/binary/reader.js'
+import { __tlReaderMapCompat } from '@mtcute/tl/compat/reader.js'
+
+function replaceType<
+    Input extends tlCompat.TlObject,
+    NewTypeName extends tl.TlObject['_'],
+>(obj: Input, type: NewTypeName): Omit<Input, '_'> & { _: NewTypeName } {
+    // modifying the object is safe because we have created the object ourselves inside the original reader fn
+    return Object.assign(obj, { _: type })
+}
+
+function mapCompatStarGift(obj: tlCompat.TypeStarGift): tl.TypeStarGift {
+    switch (obj._) {
+        case 'starGiftUnique_layer197':
+            return {
+                ...obj,
+                _: 'starGiftUnique',
+                ownerId: obj.ownerId ? { _: 'peerUser', userId: obj.ownerId } : undefined,
+            }
+        case 'starGiftUnique_layer198':
+            return replaceType(obj, 'starGiftUnique')
+        default:
+            return obj
+    }
+}
+
+function mapCompatObject(obj: tlCompat.TlObject): tl.TlObject {
+    switch (obj._) {
+        case 'starGiftUnique_layer197':
+        case 'starGiftUnique_layer198':
+            return mapCompatStarGift(obj)
+        case 'emojiStatus_layer197':
+            return {
+                _: 'emojiStatus',
+                documentId: obj.documentId,
+            }
+        case 'messageMediaDocument_layer197':
+            return replaceType(obj, 'messageMediaDocument')
+        case 'channelFull_layer197':
+            return replaceType(obj, 'channelFull')
+        case 'messageActionStarGiftUnique_layer197':
+            return {
+                ...obj,
+                _: 'messageActionStarGiftUnique',
+                gift: mapCompatStarGift(obj.gift),
+            }
+        case 'messageActionStarGift_layer197':
+            return {
+                ...obj,
+                _: 'messageActionStarGift',
+                gift: mapCompatStarGift(obj.gift),
+            }
+        default:
+            return obj
+    }
+}
+
+function wrapReader(reader: (r: unknown) => unknown) {
+    return (r: unknown) => mapCompatObject(reader(r) as tlCompat.TlObject)
+}
+
+function getCombinedReaderMap(): Record<number, (r: unknown) => unknown> {
+    const ret: Record<number, (r: unknown) => unknown> = {
+        ...__tlReaderMap,
+    }
+
+    for (const [id, reader] of objectEntries(__tlReaderMapCompat)) {
+        ret[id] = wrapReader(reader)
+    }
+
+    return ret
+}
+
+const _combinedReaderMap = /* @__PURE__ */ getCombinedReaderMap()
+
+/**
+ * Deserialize a TL object previously serialized with {@link serializeObject},
+ * with backwards compatibility for older versions of the schema.
+ *
+ * > **Note**: only some types from some layers are supported for backward compatibility,
+ * > for the complete list please see [TYPES_FOR_COMPAT](https://github.com/mtcute/mtcute/blob/master/packages/tl/scripts/constants.ts)
+ * > or [compat.tl](https://github.com/mtcute/mtcute/blob/master/packages/tl/data/compat.tl)
+ */
+export function deserializeObjectWithCompat(data: Uint8Array): tl.TlObject {
+    return TlBinaryReader.deserializeObject(_combinedReaderMap, data)
+}

packages/core/src/utils/index.ts

@@ -6,6 +6,7 @@ export * from '../storage/service/base.js'
 export * from '../storage/service/default-dcs.js'
 // end todo
 export * from './bigint-utils.js'
+export * from './binary/compat.js'
 export * from './binary/serialization.js'
 export * from './crypto/index.js'
 export * from './dcs.js'

packages/tl-runtime/src/reader.ts

@@ -29,6 +29,8 @@ export class TlBinaryReader {
 
     pos = 0
 
+    _objectMapper?: (obj: any) => any
+
     /**
      * @param objectsMap  Readers map
      * @param data  Buffer to read from

packages/tl-utils/src/codegen/types.test.ts

@@ -41,12 +41,14 @@ describe('generateTypescriptDefinitionsForTlEntry', () => {
         it('adds usage info comments', () => {
             const entries = parseTlToEntries('---functions---\ntest = Test;\ntestBot = Test;')
             const [result, resultBot] = entries.map(it =>
-                generateTypescriptDefinitionsForTlEntry(it, 'tl.', {
+                generateTypescriptDefinitionsForTlEntry(it, {
-                    base: {},
+                    errors: {
-                    errors: {},
+                        base: {},
-                    throws: { test: ['FOO', 'BAR'] },
+                        errors: {},
-                    userOnly: { test: 1 },
+                        throws: { test: ['FOO', 'BAR'] },
-                    botOnly: { testBot: 1 },
+                        userOnly: { test: 1 },
+                        botOnly: { testBot: 1 },
+                    },
                 }),
             )
 
@@ -82,7 +84,7 @@ describe('generateTypescriptDefinitionsForTlEntry', () => {
 
     it('generates code with raw flags for constructors with flags', () => {
         const entry = parseTlToEntries('test flags:# flags2:# = Test;')[0]
-        expect(generateTypescriptDefinitionsForTlEntry(entry, undefined, undefined, true)).toMatchSnapshot()
+        expect(generateTypescriptDefinitionsForTlEntry(entry, { withFlags: true })).toMatchSnapshot()
     })
 })
 
@@ -91,7 +93,7 @@ describe('generateTypescriptDefinitionsForTlSchema', () => {
         const entries = parseTlToEntries(tl.join('\n'))
         const schema = parseFullTlSchema(entries)
 
-        let [codeTs, codeJs] = generateTypescriptDefinitionsForTlSchema(schema, 0)
+        let [codeTs, codeJs] = generateTypescriptDefinitionsForTlSchema(schema)
 
         // skip prelude
         codeTs = codeTs.substring(codeTs.indexOf('-readonly [P in keyof T]: T[P]') + 37, codeTs.length - 1)

packages/tl-utils/src/codegen/types.ts

@@ -90,10 +90,17 @@ function entryFullTypeName(entry: TlEntry): string {
  */
 export function generateTypescriptDefinitionsForTlEntry(
     entry: TlEntry,
-    baseNamespace = 'tl.',
+    params?: {
-    errors?: TlErrors,
+        baseNamespace?: string
-    withFlags = false,
+        errors?: TlErrors
+        withFlags?: boolean
+        extends?: {
+            ownSchema: TlFullSchema
+            namespace: string
+        }
+    },
 ): string {
+    const { baseNamespace = 'tl.', errors, withFlags = false, extends: extendsSchema } = params ?? {}
     let ret = ''
 
     let comment = ''
@@ -176,8 +183,19 @@ export function generateTypescriptDefinitionsForTlEntry(
             typeFinal = true
         }
 
+        let typeNamespace = baseNamespace
+
+        if (extendsSchema) {
+            // ensure this type is defined in our schema, otherwise we should use the base schema namespace
+            const { ownSchema, namespace } = extendsSchema
+            const exists = arg.type[0].match(/[A-Z]/) ? arg.type in ownSchema.unions : arg.type in ownSchema.classes
+            if (!exists) {
+                typeNamespace = `${namespace}.`
+            }
+        }
+
         if (!typeFinal) {
-            type = fullTypeName(arg.type, baseNamespace, {
+            type = fullTypeName(arg.type, typeNamespace, {
                 typeModifiers: arg.typeModifiers,
             })
         }
@@ -190,13 +208,42 @@ export function generateTypescriptDefinitionsForTlEntry(
     return ret
 }
 
-const PRELUDE = `
+/**
-import _Long from 'long';
+ * Generate TypeScript definitions for a given TL schema
+ *
+ * @param schema  TL schema to generate definitions for
+ * @returns  Tuple containing `[ts, js]` code
+ */
+export function generateTypescriptDefinitionsForTlSchema(
+    schema: TlFullSchema,
+    params?: {
+        /** Layer of the schema */
+        layer?: number
+        /** Namespace of the schema */
+        namespace?: string
+        /** Errors information object */
+        errors?: TlErrors
+        /** Whether to skip importing _Long */
+        skipLongImport?: boolean
+        /** Whether to only generate typings and don't emit any JS helper functions typings */
+        onlyTypings?: boolean
+        /** Namespace of another schema that this one extends */
+        extends?: string
+    },
+): [string, string] {
+    const {
+        layer = 0,
+        namespace = 'tl',
+        errors,
+        skipLongImport = false,
+        onlyTypings = false,
+        extends: extendsSchema,
+    } = params ?? {}
+    let ts = `${skipLongImport ? '' : 'import _Long from \'long\';'}
+export declare namespace ${namespace} {
+    const LAYER = ${layer};
 
-export declare namespace $NS$ {
+    ${onlyTypings ? '' : 'function $extendTypes(types: Record<string, string>): void'}
-    const LAYER = $LAYER$;
-
-    function $extendTypes(types: Record<string, string>): void
 
     type Long = _Long;
     type RawLong = Uint8Array;
@@ -211,8 +258,7 @@ export declare namespace $NS$ {
     }
 `
 
-const PRELUDE_JS = `
+    let js = `exports.${namespace} = {};
-exports.$NS$ = {};
 (function(ns) {
 var _types = void 0;
 function _isAny(type) {
@@ -225,26 +271,12 @@ ns.$extendTypes = function(types) {
         types.hasOwnProperty(i) && (_types[i] = types[i])
     }
 }
-ns.LAYER = $LAYER$;
+ns.LAYER = ${layer};
 `
 
-/**
+    if (extendsSchema) {
- * Generate TypeScript definitions for a given TL schema
+        ts += '    type AnyToNever<T> = any extends T ? never : T;\n'
- *
+    }
- * @param schema  TL schema to generate definitions for
- * @param layer  Layer of the schema
- * @param namespace  namespace of the schema
- * @param errors  Errors information object
- * @returns  Tuple containing `[ts, js]` code
- */
-export function generateTypescriptDefinitionsForTlSchema(
-    schema: TlFullSchema,
-    layer: number,
-    namespace = 'tl',
-    errors?: TlErrors,
-): [string, string] {
-    let ts = PRELUDE.replace('$NS$', namespace).replace('$LAYER$', String(layer))
-    let js = PRELUDE_JS.replace('$NS$', namespace).replace('$LAYER$', String(layer))
 
     if (errors) {
         const [_ts, _js] = generateCodeForErrors(errors, 'ns.')
@@ -269,7 +301,15 @@ export function generateTypescriptDefinitionsForTlSchema(
                 unions[entry.type] = 1
             }
 
-            ts += `${indent(indentSize, generateTypescriptDefinitionsForTlEntry(entry, `${namespace}.`))}\n`
+            ts += `${indent(indentSize, generateTypescriptDefinitionsForTlEntry(entry, {
+                baseNamespace: `${namespace}.`,
+                extends: extendsSchema
+                    ? {
+                        ownSchema: schema,
+                        namespace: extendsSchema,
+                    }
+                    : undefined,
+            }))}\n`
         })
 
         ts += indent(indentSize, 'interface RpcCallReturn')
@@ -338,10 +378,16 @@ export function generateTypescriptDefinitionsForTlSchema(
                 ts += fullTypeName(entry.name, `${namespace}.`)
             })
 
+            if (extendsSchema) {
+                ts += ` | AnyToNever<${extendsSchema}.${typeName}>`
+            }
+
             ts += '\n'
 
-            ts += `${indent(indentSize, `function isAny${typeWithoutNs}(o: object): o is ${typeName}`)}\n`
+            if (!onlyTypings) {
-            js += `ns.isAny${typeWithoutNs} = _isAny('${name}');\n`
+                ts += `${indent(indentSize, `function isAny${typeWithoutNs}(o: object): o is ${typeName}`)}\n`
+                js += `ns.isAny${typeWithoutNs} = _isAny('${name}');\n`
+            }
         }
 
         if (ns) {
@@ -383,6 +429,9 @@ export function generateTypescriptDefinitionsForTlSchema(
                     })}`,
             )}\n`
     })
+    if (extendsSchema) {
+        ts += `${indent(8, `| ${extendsSchema}.TlObject`)}\n`
+    }
 
     ts += '}'
 

packages/tl/.gitignore

@@ -2,5 +2,7 @@
 /index.js
 binary/reader.js
 binary/writer.js
+/compat/index.d.ts
+/compat/reader.js
 
 /diff.json

packages/tl/compat/reader.d.ts

@@ -0,0 +1 @@
+export const __tlReaderMapCompat: Record<number, (r: unknown) => unknown>

packages/tl/scripts/constants.ts

@@ -30,6 +30,8 @@ export const ESM_PRELUDE = `// This file is auto-generated. Do not edit.
 Object.defineProperty(exports, "__esModule", { value: true });
 `
 
+// these types and their descendants are supported for backward compatibility,
+// and will get included into compat.tl on schema bump
 export const TYPES_FOR_COMPAT: string[] = [
     'message',
     'messageService',

packages/tl/scripts/gen-code.ts

@@ -10,11 +10,14 @@ import { join } from 'node:path'
 
 import {
     generateReaderCodeForTlEntries,
+    generateTlEntriesDifference,
     generateTypescriptDefinitionsForTlSchema,
     generateWriterCodeForTlEntries,
     parseFullTlSchema,
+    parseTlToEntries,
+    stringifyArgumentType,
 } from '@mtcute/tl-utils'
-import { __dirname, API_SCHEMA_JSON_FILE, ERRORS_JSON_FILE, ESM_PRELUDE, MTP_SCHEMA_JSON_FILE } from './constants.js'
+import { __dirname, API_SCHEMA_JSON_FILE, COMPAT_TL_FILE, ERRORS_JSON_FILE, ESM_PRELUDE, MTP_SCHEMA_JSON_FILE } from './constants.js'
 import { unpackTlSchema } from './schema.js'
 
 const OUT_TYPINGS_FILE = join(__dirname, '../index.d.ts')
@@ -22,12 +25,15 @@ const OUT_TYPINGS_JS_FILE = join(__dirname, '../index.js')
 const OUT_READERS_FILE = join(__dirname, '../binary/reader.js')
 const OUT_WRITERS_FILE = join(__dirname, '../binary/writer.js')
 
+const OUT_TYPINGS_COMPAT_FILE = join(__dirname, '../compat/index.d.ts')
+const OUT_READERS_COMPAT_FILE = join(__dirname, '../compat/reader.js')
+
 async function generateTypings(apiSchema: TlFullSchema, apiLayer: number, mtpSchema: TlFullSchema, errors: TlErrors) {
     console.log('Generating typings...')
-    const [apiTs, apiJs] = generateTypescriptDefinitionsForTlSchema(apiSchema, apiLayer, undefined, errors)
+    const [apiTs, apiJs] = generateTypescriptDefinitionsForTlSchema(apiSchema, { layer: apiLayer, errors })
-    const [mtpTs, mtpJs] = generateTypescriptDefinitionsForTlSchema(mtpSchema, 0, 'mtp')
+    const [mtpTs, mtpJs] = generateTypescriptDefinitionsForTlSchema(mtpSchema, { layer: 0, namespace: 'mtp', skipLongImport: true })
 
-    await writeFile(OUT_TYPINGS_FILE, `${apiTs}\n\n${mtpTs.replace("import _Long from 'long';", '')}`)
+    await writeFile(OUT_TYPINGS_FILE, `${apiTs}\n\n${mtpTs}`)
     await writeFile(OUT_TYPINGS_JS_FILE, `${ESM_PRELUDE + apiJs}\n\n${mtpJs}`)
 }
 
@@ -90,6 +96,67 @@ async function generateWriters(apiSchema: TlFullSchema, mtpSchema: TlFullSchema)
     await writeFile(OUT_WRITERS_FILE, ESM_PRELUDE + code)
 }
 
+async function generateCompatCode(compatSchema: TlFullSchema, currentSchema: TlFullSchema) {
+    console.log('Generating compat code...')
+
+    // update compat schema with documentation about the diff with the current schema
+    for (const entry of compatSchema.entries) {
+        const origName = entry.name.replace(/_layer\d+$/, '')
+        const existing = currentSchema.entries.find(it => it.name === origName)
+        if (existing) {
+            const lines: string[] = ['Compared to the current schema, changes from this entry:\n']
+            const diff = generateTlEntriesDifference({ ...entry, name: origName }, existing)
+            if (diff.arguments) {
+                if (diff.arguments.added.length) {
+                    lines.push('Added arguments:')
+                    for (const arg of diff.arguments.added) {
+                        lines.push(`    ${arg.name}: ${stringifyArgumentType(arg.type, arg.typeModifiers)}`)
+                    }
+                }
+                if (diff.arguments.removed.length) {
+                    lines.push(`Removed arguments: ${diff.arguments.removed.map(it => it.name).join(', ')}`)
+                }
+                if (diff.arguments.modified.length) {
+                    let first = true
+                    for (const mod of diff.arguments.modified) {
+                        if (!mod.type) continue
+                        if (first) {
+                            lines.push('Changed arguments:')
+                            first = false
+                        }
+                        lines.push(`    ${mod.name}: ${mod.type.old} => ${mod.type.new}`)
+                    }
+                }
+
+                entry.comment = lines.join('\n')
+            } else {
+                entry.comment = 'No changes' // (??)
+            }
+        } else {
+            entry.comment = 'Entry was removed from the schema'
+        }
+    }
+
+    // readers
+    {
+        const code = generateReaderCodeForTlEntries(removeInternalEntries(compatSchema.entries), {
+            variableName: 'm',
+            includeMethods: false,
+        })
+        await writeFile(OUT_READERS_COMPAT_FILE, `${ESM_PRELUDE + code}\nexports.__tlReaderMapCompat = m;`)
+    }
+
+    // typings
+    {
+        const [codeTs] = generateTypescriptDefinitionsForTlSchema(compatSchema, {
+            namespace: 'tlCompat',
+            onlyTypings: true,
+            extends: 'tl',
+        })
+        await writeFile(OUT_TYPINGS_COMPAT_FILE, `import { tl } from '../index.d.ts';\n${codeTs}`)
+    }
+}
+
 // put common errors to the top so they are parsed first
 const ERRORS_ORDER = ['FLOOD_WAIT_%d', 'FILE_MIGRATE_%d', 'NETWORK_MIGRATE_%d', 'PHONE_MIGRATE_%d', 'STATS_MIGRATE_%d']
 
@@ -118,10 +185,12 @@ async function main() {
         JSON.parse(await readFile(API_SCHEMA_JSON_FILE, 'utf8')) as TlPackedSchema,
     )
     const mtpSchema = parseFullTlSchema(JSON.parse(await readFile(MTP_SCHEMA_JSON_FILE, 'utf8')) as TlEntry[])
+    const compatSchema = parseFullTlSchema(parseTlToEntries(await readFile(COMPAT_TL_FILE, 'utf8')))
 
     await generateTypings(apiSchema, apiLayer, mtpSchema, errors)
     await generateReaders(apiSchema, mtpSchema)
     await generateWriters(apiSchema, mtpSchema)
+    await generateCompatCode(compatSchema, apiSchema)
 
     console.log('Done!')
 }