Usync: Barebones Usync Protocol support (#960)

* feature(feature/usync-mex): initial commit

* chore: fix merge commit

* chore:lint
This commit is contained in:
Rajeh Taher
2024-12-22 23:38:41 +02:00
committed by GitHub
parent 8333a25fca
commit f1f49ad2c8
14 changed files with 525 additions and 147 deletions

View File

@@ -8,7 +8,8 @@ import { chatModificationToAppPatch, ChatMutationMap, decodePatches, decodeSyncd
import { makeMutex } from '../Utils/make-mutex' import { makeMutex } from '../Utils/make-mutex'
import processMessage from '../Utils/process-message' import processMessage from '../Utils/process-message'
import { BinaryNode, getBinaryNodeChild, getBinaryNodeChildren, jidNormalizedUser, reduceBinaryNodeToDictionary, S_WHATSAPP_NET } from '../WABinary' import { BinaryNode, getBinaryNodeChild, getBinaryNodeChildren, jidNormalizedUser, reduceBinaryNodeToDictionary, S_WHATSAPP_NET } from '../WABinary'
import { makeSocket } from './socket' import { USyncQuery, USyncUser } from '../WAUSync'
import { makeUSyncSocket } from './usync'
const MAX_SYNC_ATTEMPTS = 2 const MAX_SYNC_ATTEMPTS = 2
@@ -21,7 +22,7 @@ export const makeChatsSocket = (config: SocketConfig) => {
shouldIgnoreJid, shouldIgnoreJid,
shouldSyncHistoryMessage, shouldSyncHistoryMessage,
} = config } = config
const sock = makeSocket(config) const sock = makeUSyncSocket(config)
const { const {
ev, ev,
ws, ws,
@@ -139,83 +140,47 @@ export const makeChatsSocket = (config: SocketConfig) => {
}) })
} }
/** helper function to run a generic IQ query */
const interactiveQuery = async(userNodes: BinaryNode[], queryNode: BinaryNode) => {
const result = await query({
tag: 'iq',
attrs: {
to: S_WHATSAPP_NET,
type: 'get',
xmlns: 'usync',
},
content: [
{
tag: 'usync',
attrs: {
sid: generateMessageTag(),
mode: 'query',
last: 'true',
index: '0',
context: 'interactive',
},
content: [
{
tag: 'query',
attrs: {},
content: [queryNode]
},
{
tag: 'list',
attrs: {},
content: userNodes
}
]
}
],
})
const usyncNode = getBinaryNodeChild(result, 'usync')
const listNode = getBinaryNodeChild(usyncNode, 'list')
const users = getBinaryNodeChildren(listNode, 'user')
return users
}
const onWhatsApp = async(...jids: string[]) => { const onWhatsApp = async(...jids: string[]) => {
const query = { tag: 'contact', attrs: {} } const usyncQuery = new USyncQuery()
const list = jids.map((jid) => { .withContactProtocol()
// insures only 1 + is there
const content = `+${jid.replace('+', '')}`
return { for(const jid of jids) {
tag: 'user', const phone = `+${jid.replace('+', '').split('@')[0].split(':')[0]}`
attrs: {}, usyncQuery.withUser(new USyncUser().withPhone(phone))
content: [{ }
tag: 'contact',
attrs: {},
content,
}],
}
})
const results = await interactiveQuery(list, query)
return results.map(user => { const results = await sock.executeUSyncQuery(usyncQuery)
const contact = getBinaryNodeChild(user, 'contact')
return { exists: contact?.attrs.type === 'in', jid: user.attrs.jid } if(results) {
}).filter(item => item.exists) return results.list.filter((a) => !!a.contact).map(({ contact, id }) => ({ jid: id, exists: contact }))
}
} }
const fetchStatus = async(jid: string) => { const fetchStatus = async(...jids: string[]) => {
const [result] = await interactiveQuery( const usyncQuery = new USyncQuery()
[{ tag: 'user', attrs: { jid } }], .withStatusProtocol()
{ tag: 'status', attrs: {} }
) for(const jid of jids) {
usyncQuery.withUser(new USyncUser().withId(jid))
}
const result = await sock.executeUSyncQuery(usyncQuery)
if(result) { if(result) {
const status = getBinaryNodeChild(result, 'status') return result.list
return { }
status: status?.content!.toString(), }
setAt: new Date(+(status?.attrs.t || 0) * 1000)
} const fetchDisappearingDuration = async(...jids: string[]) => {
const usyncQuery = new USyncQuery()
.withDisappearingModeProtocol()
for(const jid of jids) {
usyncQuery.withUser(new USyncUser().withId(jid))
}
const result = await sock.executeUSyncQuery(usyncQuery)
if(result) {
return result.list
} }
} }
@@ -1021,6 +986,7 @@ export const makeChatsSocket = (config: SocketConfig) => {
onWhatsApp, onWhatsApp,
fetchBlocklist, fetchBlocklist,
fetchStatus, fetchStatus,
fetchDisappearingDuration,
updateProfilePicture, updateProfilePicture,
removeProfilePicture, removeProfilePicture,
updateProfileStatus, updateProfileStatus,

View File

@@ -7,6 +7,7 @@ import { AnyMessageContent, MediaConnInfo, MessageReceiptType, MessageRelayOptio
import { aggregateMessageKeysNotFromMe, assertMediaContent, bindWaitForEvent, decryptMediaRetryData, encodeSignedDeviceIdentity, encodeWAMessage, encryptMediaRetryRequest, extractDeviceJids, generateMessageIDV2, generateWAMessage, getStatusCodeForMediaRetry, getUrlFromDirectPath, getWAUploadToServer, normalizeMessageContent, parseAndInjectE2ESessions, unixTimestampSeconds } from '../Utils' import { aggregateMessageKeysNotFromMe, assertMediaContent, bindWaitForEvent, decryptMediaRetryData, encodeSignedDeviceIdentity, encodeWAMessage, encryptMediaRetryRequest, extractDeviceJids, generateMessageIDV2, generateWAMessage, getStatusCodeForMediaRetry, getUrlFromDirectPath, getWAUploadToServer, normalizeMessageContent, parseAndInjectE2ESessions, unixTimestampSeconds } from '../Utils'
import { getUrlInfo } from '../Utils/link-preview' import { getUrlInfo } from '../Utils/link-preview'
import { areJidsSameUser, BinaryNode, BinaryNodeAttributes, getBinaryNodeChild, getBinaryNodeChildren, isJidGroup, isJidUser, jidDecode, jidEncode, jidNormalizedUser, JidWithDevice, S_WHATSAPP_NET } from '../WABinary' import { areJidsSameUser, BinaryNode, BinaryNodeAttributes, getBinaryNodeChild, getBinaryNodeChildren, isJidGroup, isJidUser, jidDecode, jidEncode, jidNormalizedUser, JidWithDevice, S_WHATSAPP_NET } from '../WABinary'
import { USyncQuery, USyncUser } from '../WAUSync'
import { makeGroupsSocket } from './groups' import { makeGroupsSocket } from './groups'
export const makeMessagesSocket = (config: SocketConfig) => { export const makeMessagesSocket = (config: SocketConfig) => {
@@ -27,7 +28,6 @@ export const makeMessagesSocket = (config: SocketConfig) => {
upsertMessage, upsertMessage,
query, query,
fetchPrivacySettings, fetchPrivacySettings,
generateMessageTag,
sendNode, sendNode,
groupMetadata, groupMetadata,
groupToggleEphemeral, groupToggleEphemeral,
@@ -144,72 +144,54 @@ export const makeMessagesSocket = (config: SocketConfig) => {
logger.debug('not using cache for devices') logger.debug('not using cache for devices')
} }
const users: BinaryNode[] = [] const toFetch: string[] = []
jids = Array.from(new Set(jids)) jids = Array.from(new Set(jids))
for(let jid of jids) { for(let jid of jids) {
const user = jidDecode(jid)?.user const user = jidDecode(jid)?.user
jid = jidNormalizedUser(jid) jid = jidNormalizedUser(jid)
if(useCache) {
const devices = userDevicesCache.get<JidWithDevice[]>(user!)
if(devices) {
deviceResults.push(...devices)
const devices = userDevicesCache.get<JidWithDevice[]>(user!) logger.trace({ user }, 'using cache for devices')
if(devices && useCache) { } else {
deviceResults.push(...devices) toFetch.push(jid)
}
logger.trace({ user }, 'using cache for devices')
} else { } else {
users.push({ tag: 'user', attrs: { jid } }) toFetch.push(jid)
} }
} }
if(!users.length) { if(!toFetch.length) {
return deviceResults return deviceResults
} }
const iq: BinaryNode = { const query = new USyncQuery()
tag: 'iq', .withContext('message')
attrs: { .withDeviceProtocol()
to: S_WHATSAPP_NET,
type: 'get',
xmlns: 'usync',
},
content: [
{
tag: 'usync',
attrs: {
sid: generateMessageTag(),
mode: 'query',
last: 'true',
index: '0',
context: 'message',
},
content: [
{
tag: 'query',
attrs: { },
content: [
{
tag: 'devices',
attrs: { version: '2' }
}
]
},
{ tag: 'list', attrs: { }, content: users }
]
},
],
}
const result = await query(iq)
const extracted = extractDeviceJids(result, authState.creds.me!.id, ignoreZeroDevices)
const deviceMap: { [_: string]: JidWithDevice[] } = {}
for(const item of extracted) { for(const jid of toFetch) {
deviceMap[item.user] = deviceMap[item.user] || [] query.withUser(new USyncUser().withId(jid))
deviceMap[item.user].push(item)
deviceResults.push(item)
} }
for(const key in deviceMap) { const result = await sock.executeUSyncQuery(query)
userDevicesCache.set(key, deviceMap[key])
if(result) {
const extracted = extractDeviceJids(result?.list, authState.creds.me!.id, ignoreZeroDevices)
const deviceMap: { [_: string]: JidWithDevice[] } = {}
for(const item of extracted) {
deviceMap[item.user] = deviceMap[item.user] || []
deviceMap[item.user].push(item)
deviceResults.push(item)
}
for(const key in deviceMap) {
userDevicesCache.set(key, deviceMap[key])
}
} }
return deviceResults return deviceResults

81
src/Socket/usync.ts Normal file
View File

@@ -0,0 +1,81 @@
import { Boom } from '@hapi/boom'
import { SocketConfig } from '../Types'
import { BinaryNode, S_WHATSAPP_NET } from '../WABinary'
import { USyncQuery } from '../WAUSync'
import { makeSocket } from './socket'
export const makeUSyncSocket = (config: SocketConfig) => {
const sock = makeSocket(config)
const {
generateMessageTag,
query,
} = sock
const executeUSyncQuery = async(usyncQuery: USyncQuery) => {
if(usyncQuery.protocols.length === 0) {
throw new Boom('USyncQuery must have at least one protocol')
}
// todo: validate users, throw WARNING on no valid users
// variable below has only validated users
const validUsers = usyncQuery.users
const userNodes = validUsers.map((user) => {
return {
tag: 'user',
attrs: {
jid: !user.phone ? user.id : undefined,
},
content: usyncQuery.protocols
.map((a) => a.getUserElement(user))
.filter(a => a !== null)
} as BinaryNode
})
const listNode: BinaryNode = {
tag: 'list',
attrs: {},
content: userNodes
}
const queryNode: BinaryNode = {
tag: 'query',
attrs: {},
content: usyncQuery.protocols.map((a) => a.getQueryElement())
}
const iq = {
tag: 'iq',
attrs: {
to: S_WHATSAPP_NET,
type: 'get',
xmlns: 'usync',
},
content: [
{
tag: 'usync',
attrs: {
context: usyncQuery.context,
mode: usyncQuery.mode,
sid: generateMessageTag(),
last: 'true',
index: '0',
},
content: [
queryNode,
listNode
]
}
],
}
const result = await query(iq)
return usyncQuery.parseUSyncQueryResult(result)
}
return {
...sock,
executeUSyncQuery,
}
}

27
src/Types/USync.ts Normal file
View File

@@ -0,0 +1,27 @@
import { BinaryNode } from '../WABinary'
import { USyncUser } from '../WAUSync'
/**
* Defines the interface for a USyncQuery protocol
*/
export interface USyncQueryProtocol {
/**
* The name of the protocol
*/
name: string
/**
* Defines what goes inside the query part of a USyncQuery
*/
getQueryElement: () => BinaryNode
/**
* Defines what goes inside the user part of a USyncQuery
*/
getUserElement: (user: USyncUser) => BinaryNode | null
/**
* Parse the result of the query
* @param data Data from the result
* @returns Whatever the protocol is supposed to return
*/
parser: (data: BinaryNode) => unknown
}

View File

@@ -3,6 +3,7 @@ import { KEY_BUNDLE_TYPE } from '../Defaults'
import { SignalRepository } from '../Types' import { SignalRepository } from '../Types'
import { AuthenticationCreds, AuthenticationState, KeyPair, SignalIdentity, SignalKeyStore, SignedKeyPair } from '../Types/Auth' import { AuthenticationCreds, AuthenticationState, KeyPair, SignalIdentity, SignalKeyStore, SignedKeyPair } from '../Types/Auth'
import { assertNodeErrorFree, BinaryNode, getBinaryNodeChild, getBinaryNodeChildBuffer, getBinaryNodeChildren, getBinaryNodeChildUInt, jidDecode, JidWithDevice, S_WHATSAPP_NET } from '../WABinary' import { assertNodeErrorFree, BinaryNode, getBinaryNodeChild, getBinaryNodeChildBuffer, getBinaryNodeChildren, getBinaryNodeChildUInt, jidDecode, JidWithDevice, S_WHATSAPP_NET } from '../WABinary'
import { DeviceListData, ParsedDeviceInfo, USyncQueryResultList } from '../WAUSync'
import { Curve, generateSignalPubKey } from './crypto' import { Curve, generateSignalPubKey } from './crypto'
import { encodeBigEndian } from './generics' import { encodeBigEndian } from './generics'
@@ -114,30 +115,24 @@ export const parseAndInjectE2ESessions = async(
} }
} }
export const extractDeviceJids = (result: BinaryNode, myJid: string, excludeZeroDevices: boolean) => { export const extractDeviceJids = (result: USyncQueryResultList[], myJid: string, excludeZeroDevices: boolean) => {
const { user: myUser, device: myDevice } = jidDecode(myJid)! const { user: myUser, device: myDevice } = jidDecode(myJid)!
const extracted: JidWithDevice[] = [] const extracted: JidWithDevice[] = []
for(const node of result.content as BinaryNode[]) {
const list = getBinaryNodeChild(node, 'list')?.content
if(list && Array.isArray(list)) { for(const userResult of result) {
for(const item of list) { const { devices, id } = userResult as { devices: ParsedDeviceInfo, id: string }
const { user } = jidDecode(item.attrs.jid)! const { user } = jidDecode(id)!
const devicesNode = getBinaryNodeChild(item, 'devices') const deviceList = devices?.deviceList as DeviceListData[]
const deviceListNode = getBinaryNodeChild(devicesNode, 'device-list') if(Array.isArray(deviceList)) {
if(Array.isArray(deviceListNode?.content)) { for(const { id: device, keyIndex } of deviceList) {
//eslint-disable-next-line max-depth if(
for(const { tag, attrs } of deviceListNode!.content) { (!excludeZeroDevices || device !== 0) && // if zero devices are not-excluded, or device is non zero
const device = +attrs.id (myUser !== user || myDevice !== device) && // either different user or if me user, not this device
//eslint-disable-next-line max-depth (device === 0 || !!keyIndex) // ensure that "key-index" is specified for "non-zero" devices, produces a bad req otherwise
if( ) {
tag === 'device' && // ensure the "device" tag extracted.push({ user, device })
(!excludeZeroDevices || device !== 0) && // if zero devices are not-excluded, or device is non zero
(myUser !== user || myDevice !== device) && // either different user or if me user, not this device
(device === 0 || !!attrs['key-index']) // ensure that "key-index" is specified for "non-zero" devices, produces a bad req otherwise
) {
extracted.push({ user, device })
}
}
} }
} }
} }

View File

@@ -0,0 +1,32 @@
import { USyncQueryProtocol } from '../../Types/USync'
import { assertNodeErrorFree, BinaryNode } from '../../WABinary'
import { USyncUser } from '../USyncUser'
export class USyncContactProtocol implements USyncQueryProtocol {
name = 'contact'
getQueryElement(): BinaryNode {
return {
tag: 'contact',
attrs: {},
}
}
getUserElement(user: USyncUser): BinaryNode {
//TODO: Implement type / username fields (not yet supported)
return {
tag: 'contact',
attrs: {},
content: user.phone,
}
}
parser(node: BinaryNode): boolean {
if(node.tag === 'contact') {
assertNodeErrorFree(node)
return node?.attrs?.type === 'in'
}
return false
}
}

View File

@@ -0,0 +1,78 @@
import { USyncQueryProtocol } from '../../Types/USync'
import { assertNodeErrorFree, BinaryNode, getBinaryNodeChild } from '../../WABinary'
//import { USyncUser } from '../USyncUser'
export type KeyIndexData = {
timestamp: number
signedKeyIndex?: Uint8Array
expectedTimestamp?: number
}
export type DeviceListData = {
id: number
keyIndex?: number
isHosted?: boolean
}
export type ParsedDeviceInfo = {
deviceList?: DeviceListData[]
keyIndex?: KeyIndexData
}
export class USyncDeviceProtocol implements USyncQueryProtocol {
name = 'devices'
getQueryElement(): BinaryNode {
return {
tag: 'devices',
attrs: {
version: '2',
},
}
}
getUserElement(/* user: USyncUser */): BinaryNode | null {
//TODO: Implement device phashing, ts and expectedTs
//TODO: if all are not present, return null <- current behavior
//TODO: otherwise return a node w tag 'devices' w those as attrs
return null
}
parser(node: BinaryNode): ParsedDeviceInfo {
const deviceList: DeviceListData[] = []
let keyIndex: KeyIndexData | undefined = undefined
if(node.tag === 'devices') {
assertNodeErrorFree(node)
const deviceListNode = getBinaryNodeChild(node, 'device-list')
const keyIndexNode = getBinaryNodeChild(node, 'key-index-list')
if(Array.isArray(deviceListNode?.content)) {
for(const { tag, attrs } of deviceListNode!.content) {
const id = +attrs.id
const keyIndex = +attrs['key-index']
if(tag === 'device') {
deviceList.push({
id,
keyIndex,
isHosted: !!(attrs['is_hosted'] && attrs['is_hosted'] === 'true')
})
}
}
}
if(keyIndexNode?.tag === 'key-index-list') {
keyIndex = {
timestamp: +keyIndexNode.attrs['ts'],
signedKeyIndex: keyIndexNode?.content as Uint8Array,
expectedTimestamp: keyIndexNode.attrs['expected_ts'] ? +keyIndexNode.attrs['expected_ts'] : undefined
}
}
}
return {
deviceList,
keyIndex
}
}
}

View File

@@ -0,0 +1,35 @@
import { USyncQueryProtocol } from '../../Types/USync'
import { assertNodeErrorFree, BinaryNode } from '../../WABinary'
export type DisappearingModeData = {
duration: number
setAt?: Date
}
export class USyncDisappearingModeProtocol implements USyncQueryProtocol {
name = 'disappearing_mode'
getQueryElement(): BinaryNode {
return {
tag: 'disappearing_mode',
attrs: {},
}
}
getUserElement(): null {
return null
}
parser(node: BinaryNode): DisappearingModeData | undefined {
if(node.tag === 'status') {
assertNodeErrorFree(node)
const duration: number = +node?.attrs.duration
const setAt = new Date(+(node?.attrs.t || 0) * 1000)
return {
duration,
setAt,
}
}
}
}

View File

@@ -0,0 +1,44 @@
import { USyncQueryProtocol } from '../../Types/USync'
import { assertNodeErrorFree, BinaryNode } from '../../WABinary'
export type StatusData = {
status?: string | null
setAt?: Date
}
export class USyncStatusProtocol implements USyncQueryProtocol {
name = 'status'
getQueryElement(): BinaryNode {
return {
tag: 'status',
attrs: {},
}
}
getUserElement(): null {
return null
}
parser(node: BinaryNode): StatusData | undefined {
if(node.tag === 'status') {
assertNodeErrorFree(node)
let status: string | null = node?.content!.toString()
const setAt = new Date(+(node?.attrs.t || 0) * 1000)
if(!status) {
if(+node.attrs?.code === 401) {
status = ''
} else {
status = null
}
} else if(typeof status === 'string' && status.length === 0) {
status = null
}
return {
status,
setAt,
}
}
}
}

View File

@@ -0,0 +1,4 @@
export * from './USyncDeviceProtocol'
export * from './USyncContactProtocol'
export * from './USyncStatusProtocol'
export * from './USyncDisappearingModeProtocol'

103
src/WAUSync/USyncQuery.ts Normal file
View File

@@ -0,0 +1,103 @@
import { USyncQueryProtocol } from '../Types/USync'
import { BinaryNode, getBinaryNodeChild } from '../WABinary'
import { USyncContactProtocol, USyncDeviceProtocol, USyncDisappearingModeProtocol, USyncStatusProtocol } from './Protocols'
import { USyncUser } from './USyncUser'
export type USyncQueryResultList = { [protocol: string]: unknown, id: string }
export type USyncQueryResult = {
list: USyncQueryResultList[]
sideList: USyncQueryResultList[]
}
export class USyncQuery {
protocols: USyncQueryProtocol[]
users: USyncUser[]
context: string
mode: string
constructor() {
this.protocols = []
this.users = []
this.context = 'interactive'
this.mode = 'query'
}
withMode(mode: string) {
this.mode = mode
return this
}
withContext(context: string) {
this.context = context
return this
}
withUser(user: USyncUser) {
this.users.push(user)
return this
}
parseUSyncQueryResult(result: BinaryNode): USyncQueryResult | undefined {
if(result.attrs.type !== 'result') {
return
}
const protocolMap = Object.fromEntries(this.protocols.map((protocol) => {
return [protocol.name, protocol.parser]
}))
const queryResult: USyncQueryResult = {
// TODO: implement errors etc.
list: [],
sideList: [],
}
const usyncNode = getBinaryNodeChild(result, 'usync')
//TODO: implement error backoff, refresh etc.
//TODO: see if there are any errors in the result node
//const resultNode = getBinaryNodeChild(usyncNode, 'result')
const listNode = getBinaryNodeChild(usyncNode, 'list')
if(Array.isArray(listNode?.content) && typeof listNode !== 'undefined') {
queryResult.list = listNode.content.map((node) => {
const id = node?.attrs.jid
const data = Array.isArray(node?.content) ? Object.fromEntries(node.content.map((content) => {
const protocol = content.tag
const parser = protocolMap[protocol]
if(parser) {
return [protocol, parser(content)]
} else {
return [protocol, null]
}
}).filter(([, b]) => b !== null) as [string, unknown][]) : {}
return { ...data, id }
})
}
//TODO: implement side list
//const sideListNode = getBinaryNodeChild(usyncNode, 'side_list')
return queryResult
}
withDeviceProtocol() {
this.protocols.push(new USyncDeviceProtocol())
return this
}
withContactProtocol() {
this.protocols.push(new USyncContactProtocol())
return this
}
withStatusProtocol() {
this.protocols.push(new USyncStatusProtocol())
return this
}
withDisappearingModeProtocol() {
this.protocols.push(new USyncDisappearingModeProtocol())
return this
}
}

27
src/WAUSync/USyncUser.ts Normal file
View File

@@ -0,0 +1,27 @@
export class USyncUser {
id: string
lid: string
phone: string
type: string
withId(id: string) {
this.id = id
return this
}
withLid(lid: string) {
this.lid = lid
return this
}
withPhone(phone: string) {
this.phone = phone
return this
}
withType(type: string) {
this.type = type
return this
}
}

3
src/WAUSync/index.ts Normal file
View File

@@ -0,0 +1,3 @@
export * from './Protocols'
export * from './USyncQuery'
export * from './USyncUser'

View File

@@ -7,6 +7,7 @@ export * from './Store'
export * from './Defaults' export * from './Defaults'
export * from './WABinary' export * from './WABinary'
export * from './WAM' export * from './WAM'
export * from './WAUSync'
export type WASocket = ReturnType<typeof makeWASocket> export type WASocket = ReturnType<typeof makeWASocket>
export { makeWASocket } export { makeWASocket }