kylefang/mtcute
1
0
Fork 0
mirror of https://github.com/zhigang1992/mtcute.git synced 2026-01-12 17:32:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat!: deserialization for backwards compatibility

Browse Source 
breaking(tl-utils): signature of `generateTypescriptDefinitionsForTlSchema` and `generateTypescriptDefinitionsForTlEntry`
This commit is contained in:
alina sireneva
2025-02-20 23:45:30 +03:00
parent 591aff7576
commit d7a9ffe702
11 changed files with 307 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
eslint.config.js
View File

@@ -2,7 +2,11 @@ import antfu from '@antfu/eslint-config'
export default antfu({
type: 'lib',
ignores: ['e2e/runtime/*'],
ignores: [
'e2e/runtime/*',
'packages/tl/**/*.js',
'packages/tl/**/*.d.ts',
],
typescript: process.env.CI
? {
tsconfigPath: 'tsconfig.json',

39
packages/core/src/utils/binary/compat.test.ts Normal file
View File

@@ -0,0 +1,39 @@
import { hex } from '@fuman/utils'
import Long from 'long'
import { describe, expect, it } from 'vitest'
import { deserializeObjectWithCompat } from './compat.js'
describe('binary/compat', () => {
it('should correctly read emojiStatus from layer 197', () => {
const data = hex.decode('9d619b920000000000000000')
expect(deserializeObjectWithCompat(data)).toEqual({
_: 'emojiStatus',
documentId: Long.ZERO,
})
})
it('should correctly read emojiStatus from layer 198', () => {
const data = hex.decode('8a06ffe7000000000000000000000000')
expect(deserializeObjectWithCompat(data)).toEqual({
_: 'emojiStatus',
documentId: Long.ZERO,
})
})
it('should correctly read emojiStatus from 197 inside channelAdminLogEventActionChangeEmojiStatus', () => {
// rather unlikely case where emojiStatus from different layers is inside the same object.
// still useful to test it tho
const data = hex.decode('b1fea93e9d619b9200000000000000008a06ffe7000000000000000000000000')
expect(deserializeObjectWithCompat(data)).toEqual({
_: 'channelAdminLogEventActionChangeEmojiStatus',
prevValue: {
_: 'emojiStatus',
documentId: Long.ZERO,
},
newValue: {
_: 'emojiStatus',
documentId: Long.ZERO,
},
})
})
})

90
packages/core/src/utils/binary/compat.ts Normal file
View File

@@ -0,0 +1,90 @@
import type { tl } from '@mtcute/tl'
import type { tlCompat } from '@mtcute/tl/compat'
import { objectEntries } from '@fuman/utils'
import { TlBinaryReader } from '@mtcute/tl-runtime'
import { __tlReaderMap } from '@mtcute/tl/binary/reader.js'
import { __tlReaderMapCompat } from '@mtcute/tl/compat/reader.js'
function replaceType<
Input extends tlCompat.TlObject,
NewTypeName extends tl.TlObject['_'],
>(obj: Input, type: NewTypeName): Omit<Input, '_'> & { _: NewTypeName } {
// modifying the object is safe because we have created the object ourselves inside the original reader fn
return Object.assign(obj, { _: type })
}
function mapCompatStarGift(obj: tlCompat.TypeStarGift): tl.TypeStarGift {
switch (obj._) {
case 'starGiftUnique_layer197':
return {
...obj,
_: 'starGiftUnique',
ownerId: obj.ownerId ? { _: 'peerUser', userId: obj.ownerId } : undefined,
}
case 'starGiftUnique_layer198':
return replaceType(obj, 'starGiftUnique')
default:
return obj
}
}
function mapCompatObject(obj: tlCompat.TlObject): tl.TlObject {
switch (obj._) {
case 'starGiftUnique_layer197':
case 'starGiftUnique_layer198':
return mapCompatStarGift(obj)
case 'emojiStatus_layer197':
return {
_: 'emojiStatus',
documentId: obj.documentId,
}
case 'messageMediaDocument_layer197':
return replaceType(obj, 'messageMediaDocument')
case 'channelFull_layer197':
return replaceType(obj, 'channelFull')
case 'messageActionStarGiftUnique_layer197':
return {
...obj,
_: 'messageActionStarGiftUnique',
gift: mapCompatStarGift(obj.gift),
}
case 'messageActionStarGift_layer197':
return {
...obj,
_: 'messageActionStarGift',
gift: mapCompatStarGift(obj.gift),
}
default:
return obj
}
}
function wrapReader(reader: (r: unknown) => unknown) {
return (r: unknown) => mapCompatObject(reader(r) as tlCompat.TlObject)
}
function getCombinedReaderMap(): Record<number, (r: unknown) => unknown> {
const ret: Record<number, (r: unknown) => unknown> = {
...__tlReaderMap,
}
for (const [id, reader] of objectEntries(__tlReaderMapCompat)) {
ret[id] = wrapReader(reader)
}
return ret
}
const _combinedReaderMap = /* @__PURE__ */ getCombinedReaderMap()
/**
* Deserialize a TL object previously serialized with {@link serializeObject},
* with backwards compatibility for older versions of the schema.
*
* > **Note**: only some types from some layers are supported for backward compatibility,
* > for the complete list please see [TYPES_FOR_COMPAT](https://github.com/mtcute/mtcute/blob/master/packages/tl/scripts/constants.ts)
* > or [compat.tl](https://github.com/mtcute/mtcute/blob/master/packages/tl/data/compat.tl)
*/
export function deserializeObjectWithCompat(data: Uint8Array): tl.TlObject {
return TlBinaryReader.deserializeObject(_combinedReaderMap, data)
}

1
packages/core/src/utils/index.ts
View File

@@ -6,6 +6,7 @@ export * from '../storage/service/base.js'
export * from '../storage/service/default-dcs.js'
// end todo
export * from './bigint-utils.js'
export * from './binary/compat.js'
export * from './binary/serialization.js'
export * from './crypto/index.js'
export * from './dcs.js'

2
packages/tl-runtime/src/reader.ts
View File

@@ -29,6 +29,8 @@ export class TlBinaryReader {
pos = 0
_objectMapper?: (obj: any) => any
/**
* @param objectsMap Readers map
* @param data Buffer to read from

18
packages/tl-utils/src/codegen/types.test.ts
View File

@@ -41,12 +41,14 @@ describe('generateTypescriptDefinitionsForTlEntry', () => {
it('adds usage info comments', () => {
const entries = parseTlToEntries('---functions---\ntest = Test;\ntestBot = Test;')
const [result, resultBot] = entries.map(it =>
generateTypescriptDefinitionsForTlEntry(it, 'tl.', {
base: {},
errors: {},
throws: { test: ['FOO', 'BAR'] },
userOnly: { test: 1 },
botOnly: { testBot: 1 },
generateTypescriptDefinitionsForTlEntry(it, {
errors: {
base: {},
errors: {},
throws: { test: ['FOO', 'BAR'] },
userOnly: { test: 1 },
botOnly: { testBot: 1 },
},
}),
)
@@ -82,7 +84,7 @@ describe('generateTypescriptDefinitionsForTlEntry', () => {
it('generates code with raw flags for constructors with flags', () => {
const entry = parseTlToEntries('test flags:# flags2:# = Test;')[0]
expect(generateTypescriptDefinitionsForTlEntry(entry, undefined, undefined, true)).toMatchSnapshot()
expect(generateTypescriptDefinitionsForTlEntry(entry, { withFlags: true })).toMatchSnapshot()
})
})
@@ -91,7 +93,7 @@ describe('generateTypescriptDefinitionsForTlSchema', () => {
const entries = parseTlToEntries(tl.join('\n'))
const schema = parseFullTlSchema(entries)
let [codeTs, codeJs] = generateTypescriptDefinitionsForTlSchema(schema, 0)
let [codeTs, codeJs] = generateTypescriptDefinitionsForTlSchema(schema)
// skip prelude
codeTs = codeTs.substring(codeTs.indexOf('-readonly [P in keyof T]: T[P]') + 37, codeTs.length - 1)

115
packages/tl-utils/src/codegen/types.ts
View File

@@ -90,10 +90,17 @@ function entryFullTypeName(entry: TlEntry): string {
*/
export function generateTypescriptDefinitionsForTlEntry(
entry: TlEntry,
baseNamespace = 'tl.',
errors?: TlErrors,
withFlags = false,
params?: {
baseNamespace?: string
errors?: TlErrors
withFlags?: boolean
extends?: {
ownSchema: TlFullSchema
namespace: string
}
},
): string {
const { baseNamespace = 'tl.', errors, withFlags = false, extends: extendsSchema } = params ?? {}
let ret = ''
let comment = ''
@@ -176,8 +183,19 @@ export function generateTypescriptDefinitionsForTlEntry(
typeFinal = true
}
let typeNamespace = baseNamespace
if (extendsSchema) {
// ensure this type is defined in our schema, otherwise we should use the base schema namespace
const { ownSchema, namespace } = extendsSchema
const exists = arg.type[0].match(/[A-Z]/) ? arg.type in ownSchema.unions : arg.type in ownSchema.classes
if (!exists) {
typeNamespace = `${namespace}.`
}
}
if (!typeFinal) {
type = fullTypeName(arg.type, baseNamespace, {
type = fullTypeName(arg.type, typeNamespace, {
typeModifiers: arg.typeModifiers,
})
}
@@ -190,13 +208,42 @@ export function generateTypescriptDefinitionsForTlEntry(
return ret
}
const PRELUDE = `
import _Long from 'long';
/**
* Generate TypeScript definitions for a given TL schema
*
* @param schema TL schema to generate definitions for
* @returns Tuple containing `[ts, js]` code
*/
export function generateTypescriptDefinitionsForTlSchema(
schema: TlFullSchema,
params?: {
/** Layer of the schema */
layer?: number
/** Namespace of the schema */
namespace?: string
/** Errors information object */
errors?: TlErrors
/** Whether to skip importing _Long */
skipLongImport?: boolean
/** Whether to only generate typings and don't emit any JS helper functions typings */
onlyTypings?: boolean
/** Namespace of another schema that this one extends */
extends?: string
},
): [string, string] {
const {
layer = 0,
namespace = 'tl',
errors,
skipLongImport = false,
onlyTypings = false,
extends: extendsSchema,
} = params ?? {}
let ts = `${skipLongImport ? '' : 'import _Long from \'long\';'}
export declare namespace ${namespace} {
const LAYER = ${layer};
export declare namespace $NS$ {
const LAYER = $LAYER$;
function $extendTypes(types: Record<string, string>): void
${onlyTypings ? '' : 'function $extendTypes(types: Record<string, string>): void'}
type Long = _Long;
type RawLong = Uint8Array;
@@ -211,8 +258,7 @@ export declare namespace $NS$ {
}
`
const PRELUDE_JS = `
exports.$NS$ = {};
let js = `exports.${namespace} = {};
(function(ns) {
var _types = void 0;
function _isAny(type) {
@@ -225,26 +271,12 @@ ns.$extendTypes = function(types) {
types.hasOwnProperty(i) && (_types[i] = types[i])
}
}
ns.LAYER = $LAYER$;
ns.LAYER = ${layer};
`
/**
* Generate TypeScript definitions for a given TL schema
*
* @param schema TL schema to generate definitions for
* @param layer Layer of the schema
* @param namespace namespace of the schema
* @param errors Errors information object
* @returns Tuple containing `[ts, js]` code
*/
export function generateTypescriptDefinitionsForTlSchema(
schema: TlFullSchema,
layer: number,
namespace = 'tl',
errors?: TlErrors,
): [string, string] {
let ts = PRELUDE.replace('$NS$', namespace).replace('$LAYER$', String(layer))
let js = PRELUDE_JS.replace('$NS$', namespace).replace('$LAYER$', String(layer))
if (extendsSchema) {
ts += ' type AnyToNever<T> = any extends T ? never : T;\n'
}
if (errors) {
const [_ts, _js] = generateCodeForErrors(errors, 'ns.')
@@ -269,7 +301,15 @@ export function generateTypescriptDefinitionsForTlSchema(
unions[entry.type] = 1
}
ts += `${indent(indentSize, generateTypescriptDefinitionsForTlEntry(entry, `${namespace}.`))}\n`
ts += `${indent(indentSize, generateTypescriptDefinitionsForTlEntry(entry, {
baseNamespace: `${namespace}.`,
extends: extendsSchema
? {
ownSchema: schema,
namespace: extendsSchema,
}
: undefined,
}))}\n`
})
ts += indent(indentSize, 'interface RpcCallReturn')
@@ -338,10 +378,16 @@ export function generateTypescriptDefinitionsForTlSchema(
ts += fullTypeName(entry.name, `${namespace}.`)
})
if (extendsSchema) {
ts += ` | AnyToNever<${extendsSchema}.${typeName}>`
}
ts += '\n'
ts += `${indent(indentSize, `function isAny${typeWithoutNs}(o: object): o is ${typeName}`)}\n`
js += `ns.isAny${typeWithoutNs} = _isAny('${name}');\n`
if (!onlyTypings) {
ts += `${indent(indentSize, `function isAny${typeWithoutNs}(o: object): o is ${typeName}`)}\n`
js += `ns.isAny${typeWithoutNs} = _isAny('${name}');\n`
}
}
if (ns) {
@@ -383,6 +429,9 @@ export function generateTypescriptDefinitionsForTlSchema(
})}`,
)}\n`
})
if (extendsSchema) {
ts += `${indent(8, `| ${extendsSchema}.TlObject`)}\n`
}
ts += '}'

2
packages/tl/.gitignore vendored
View File

@@ -2,5 +2,7 @@
/index.js
binary/reader.js
binary/writer.js
/compat/index.d.ts
/compat/reader.js
/diff.json

1
packages/tl/compat/reader.d.ts vendored Normal file
View File

@@ -0,0 +1 @@
export const __tlReaderMapCompat: Record<number, (r: unknown) => unknown>

2
packages/tl/scripts/constants.ts
View File

@@ -30,6 +30,8 @@ export const ESM_PRELUDE = `// This file is auto-generated. Do not edit.
Object.defineProperty(exports, "__esModule", { value: true });
`
// these types and their descendants are supported for backward compatibility,
// and will get included into compat.tl on schema bump
export const TYPES_FOR_COMPAT: string[] = [
'message',
'messageService',

77
packages/tl/scripts/gen-code.ts
View File

@@ -10,11 +10,14 @@ import { join } from 'node:path'
import {
generateReaderCodeForTlEntries,
generateTlEntriesDifference,
generateTypescriptDefinitionsForTlSchema,
generateWriterCodeForTlEntries,
parseFullTlSchema,
parseTlToEntries,
stringifyArgumentType,
} from '@mtcute/tl-utils'
import { __dirname, API_SCHEMA_JSON_FILE, ERRORS_JSON_FILE, ESM_PRELUDE, MTP_SCHEMA_JSON_FILE } from './constants.js'
import { __dirname, API_SCHEMA_JSON_FILE, COMPAT_TL_FILE, ERRORS_JSON_FILE, ESM_PRELUDE, MTP_SCHEMA_JSON_FILE } from './constants.js'
import { unpackTlSchema } from './schema.js'
const OUT_TYPINGS_FILE = join(__dirname, '../index.d.ts')
@@ -22,12 +25,15 @@ const OUT_TYPINGS_JS_FILE = join(__dirname, '../index.js')
const OUT_READERS_FILE = join(__dirname, '../binary/reader.js')
const OUT_WRITERS_FILE = join(__dirname, '../binary/writer.js')
const OUT_TYPINGS_COMPAT_FILE = join(__dirname, '../compat/index.d.ts')
const OUT_READERS_COMPAT_FILE = join(__dirname, '../compat/reader.js')
async function generateTypings(apiSchema: TlFullSchema, apiLayer: number, mtpSchema: TlFullSchema, errors: TlErrors) {
console.log('Generating typings...')
const [apiTs, apiJs] = generateTypescriptDefinitionsForTlSchema(apiSchema, apiLayer, undefined, errors)
const [mtpTs, mtpJs] = generateTypescriptDefinitionsForTlSchema(mtpSchema, 0, 'mtp')
const [apiTs, apiJs] = generateTypescriptDefinitionsForTlSchema(apiSchema, { layer: apiLayer, errors })
const [mtpTs, mtpJs] = generateTypescriptDefinitionsForTlSchema(mtpSchema, { layer: 0, namespace: 'mtp', skipLongImport: true })
await writeFile(OUT_TYPINGS_FILE, `${apiTs}\n\n${mtpTs.replace("import _Long from 'long';", '')}`)
await writeFile(OUT_TYPINGS_FILE, `${apiTs}\n\n${mtpTs}`)
await writeFile(OUT_TYPINGS_JS_FILE, `${ESM_PRELUDE + apiJs}\n\n${mtpJs}`)
}
@@ -90,6 +96,67 @@ async function generateWriters(apiSchema: TlFullSchema, mtpSchema: TlFullSchema)
await writeFile(OUT_WRITERS_FILE, ESM_PRELUDE + code)
}
async function generateCompatCode(compatSchema: TlFullSchema, currentSchema: TlFullSchema) {
console.log('Generating compat code...')
// update compat schema with documentation about the diff with the current schema
for (const entry of compatSchema.entries) {
const origName = entry.name.replace(/_layer\d+$/, '')
const existing = currentSchema.entries.find(it => it.name === origName)
if (existing) {
const lines: string[] = ['Compared to the current schema, changes from this entry:\n']
const diff = generateTlEntriesDifference({ ...entry, name: origName }, existing)
if (diff.arguments) {
if (diff.arguments.added.length) {
lines.push('Added arguments:')
for (const arg of diff.arguments.added) {
lines.push(` ${arg.name}: ${stringifyArgumentType(arg.type, arg.typeModifiers)}`)
}
}
if (diff.arguments.removed.length) {
lines.push(`Removed arguments: ${diff.arguments.removed.map(it => it.name).join(', ')}`)
}
if (diff.arguments.modified.length) {
let first = true
for (const mod of diff.arguments.modified) {
if (!mod.type) continue
if (first) {
lines.push('Changed arguments:')
first = false
}
lines.push(` ${mod.name}: ${mod.type.old} => ${mod.type.new}`)
}
}
entry.comment = lines.join('\n')
} else {
entry.comment = 'No changes' // (??)
}
} else {
entry.comment = 'Entry was removed from the schema'
}
}
// readers
{
const code = generateReaderCodeForTlEntries(removeInternalEntries(compatSchema.entries), {
variableName: 'm',
includeMethods: false,
})
await writeFile(OUT_READERS_COMPAT_FILE, `${ESM_PRELUDE + code}\nexports.__tlReaderMapCompat = m;`)
}
// typings
{
const [codeTs] = generateTypescriptDefinitionsForTlSchema(compatSchema, {
namespace: 'tlCompat',
onlyTypings: true,
extends: 'tl',
})
await writeFile(OUT_TYPINGS_COMPAT_FILE, `import { tl } from '../index.d.ts';\n${codeTs}`)
}
}
// put common errors to the top so they are parsed first
const ERRORS_ORDER = ['FLOOD_WAIT_%d', 'FILE_MIGRATE_%d', 'NETWORK_MIGRATE_%d', 'PHONE_MIGRATE_%d', 'STATS_MIGRATE_%d']
@@ -118,10 +185,12 @@ async function main() {
JSON.parse(await readFile(API_SCHEMA_JSON_FILE, 'utf8')) as TlPackedSchema,
)
const mtpSchema = parseFullTlSchema(JSON.parse(await readFile(MTP_SCHEMA_JSON_FILE, 'utf8')) as TlEntry[])
const compatSchema = parseFullTlSchema(parseTlToEntries(await readFile(COMPAT_TL_FILE, 'utf8')))
await generateTypings(apiSchema, apiLayer, mtpSchema, errors)
await generateReaders(apiSchema, mtpSchema)
await generateWriters(apiSchema, mtpSchema)
await generateCompatCode(compatSchema, apiSchema)
console.log('Done!')
}