Usync: Barebones Usync Protocol support (#960)

* feature(feature/usync-mex): initial commit

* chore: fix merge commit

* chore:lint
This commit is contained in:
Rajeh Taher
2024-12-22 23:38:41 +02:00
committed by GitHub
parent 8333a25fca
commit f1f49ad2c8
14 changed files with 525 additions and 147 deletions

View File

@@ -8,7 +8,8 @@ import { chatModificationToAppPatch, ChatMutationMap, decodePatches, decodeSyncd
import { makeMutex } from '../Utils/make-mutex'
import processMessage from '../Utils/process-message'
import { BinaryNode, getBinaryNodeChild, getBinaryNodeChildren, jidNormalizedUser, reduceBinaryNodeToDictionary, S_WHATSAPP_NET } from '../WABinary'
import { makeSocket } from './socket'
import { USyncQuery, USyncUser } from '../WAUSync'
import { makeUSyncSocket } from './usync'
const MAX_SYNC_ATTEMPTS = 2
@@ -21,7 +22,7 @@ export const makeChatsSocket = (config: SocketConfig) => {
shouldIgnoreJid,
shouldSyncHistoryMessage,
} = config
const sock = makeSocket(config)
const sock = makeUSyncSocket(config)
const {
ev,
ws,
@@ -139,83 +140,47 @@ export const makeChatsSocket = (config: SocketConfig) => {
})
}
/** helper function to run a generic IQ query */
const interactiveQuery = async(userNodes: BinaryNode[], queryNode: BinaryNode) => {
const result = await query({
tag: 'iq',
attrs: {
to: S_WHATSAPP_NET,
type: 'get',
xmlns: 'usync',
},
content: [
{
tag: 'usync',
attrs: {
sid: generateMessageTag(),
mode: 'query',
last: 'true',
index: '0',
context: 'interactive',
},
content: [
{
tag: 'query',
attrs: {},
content: [queryNode]
},
{
tag: 'list',
attrs: {},
content: userNodes
}
]
}
],
})
const usyncNode = getBinaryNodeChild(result, 'usync')
const listNode = getBinaryNodeChild(usyncNode, 'list')
const users = getBinaryNodeChildren(listNode, 'user')
return users
}
const onWhatsApp = async(...jids: string[]) => {
const query = { tag: 'contact', attrs: {} }
const list = jids.map((jid) => {
// insures only 1 + is there
const content = `+${jid.replace('+', '')}`
const usyncQuery = new USyncQuery()
.withContactProtocol()
return {
tag: 'user',
attrs: {},
content: [{
tag: 'contact',
attrs: {},
content,
}],
}
})
const results = await interactiveQuery(list, query)
for(const jid of jids) {
const phone = `+${jid.replace('+', '').split('@')[0].split(':')[0]}`
usyncQuery.withUser(new USyncUser().withPhone(phone))
}
return results.map(user => {
const contact = getBinaryNodeChild(user, 'contact')
return { exists: contact?.attrs.type === 'in', jid: user.attrs.jid }
}).filter(item => item.exists)
const results = await sock.executeUSyncQuery(usyncQuery)
if(results) {
return results.list.filter((a) => !!a.contact).map(({ contact, id }) => ({ jid: id, exists: contact }))
}
}
const fetchStatus = async(jid: string) => {
const [result] = await interactiveQuery(
[{ tag: 'user', attrs: { jid } }],
{ tag: 'status', attrs: {} }
)
const fetchStatus = async(...jids: string[]) => {
const usyncQuery = new USyncQuery()
.withStatusProtocol()
for(const jid of jids) {
usyncQuery.withUser(new USyncUser().withId(jid))
}
const result = await sock.executeUSyncQuery(usyncQuery)
if(result) {
const status = getBinaryNodeChild(result, 'status')
return {
status: status?.content!.toString(),
setAt: new Date(+(status?.attrs.t || 0) * 1000)
}
return result.list
}
}
const fetchDisappearingDuration = async(...jids: string[]) => {
const usyncQuery = new USyncQuery()
.withDisappearingModeProtocol()
for(const jid of jids) {
usyncQuery.withUser(new USyncUser().withId(jid))
}
const result = await sock.executeUSyncQuery(usyncQuery)
if(result) {
return result.list
}
}
@@ -1021,6 +986,7 @@ export const makeChatsSocket = (config: SocketConfig) => {
onWhatsApp,
fetchBlocklist,
fetchStatus,
fetchDisappearingDuration,
updateProfilePicture,
removeProfilePicture,
updateProfileStatus,

View File

@@ -7,6 +7,7 @@ import { AnyMessageContent, MediaConnInfo, MessageReceiptType, MessageRelayOptio
import { aggregateMessageKeysNotFromMe, assertMediaContent, bindWaitForEvent, decryptMediaRetryData, encodeSignedDeviceIdentity, encodeWAMessage, encryptMediaRetryRequest, extractDeviceJids, generateMessageIDV2, generateWAMessage, getStatusCodeForMediaRetry, getUrlFromDirectPath, getWAUploadToServer, normalizeMessageContent, parseAndInjectE2ESessions, unixTimestampSeconds } from '../Utils'
import { getUrlInfo } from '../Utils/link-preview'
import { areJidsSameUser, BinaryNode, BinaryNodeAttributes, getBinaryNodeChild, getBinaryNodeChildren, isJidGroup, isJidUser, jidDecode, jidEncode, jidNormalizedUser, JidWithDevice, S_WHATSAPP_NET } from '../WABinary'
import { USyncQuery, USyncUser } from '../WAUSync'
import { makeGroupsSocket } from './groups'
export const makeMessagesSocket = (config: SocketConfig) => {
@@ -27,7 +28,6 @@ export const makeMessagesSocket = (config: SocketConfig) => {
upsertMessage,
query,
fetchPrivacySettings,
generateMessageTag,
sendNode,
groupMetadata,
groupToggleEphemeral,
@@ -144,72 +144,54 @@ export const makeMessagesSocket = (config: SocketConfig) => {
logger.debug('not using cache for devices')
}
const users: BinaryNode[] = []
const toFetch: string[] = []
jids = Array.from(new Set(jids))
for(let jid of jids) {
const user = jidDecode(jid)?.user
jid = jidNormalizedUser(jid)
if(useCache) {
const devices = userDevicesCache.get<JidWithDevice[]>(user!)
if(devices) {
deviceResults.push(...devices)
const devices = userDevicesCache.get<JidWithDevice[]>(user!)
if(devices && useCache) {
deviceResults.push(...devices)
logger.trace({ user }, 'using cache for devices')
logger.trace({ user }, 'using cache for devices')
} else {
toFetch.push(jid)
}
} else {
users.push({ tag: 'user', attrs: { jid } })
toFetch.push(jid)
}
}
if(!users.length) {
if(!toFetch.length) {
return deviceResults
}
const iq: BinaryNode = {
tag: 'iq',
attrs: {
to: S_WHATSAPP_NET,
type: 'get',
xmlns: 'usync',
},
content: [
{
tag: 'usync',
attrs: {
sid: generateMessageTag(),
mode: 'query',
last: 'true',
index: '0',
context: 'message',
},
content: [
{
tag: 'query',
attrs: { },
content: [
{
tag: 'devices',
attrs: { version: '2' }
}
]
},
{ tag: 'list', attrs: { }, content: users }
]
},
],
}
const result = await query(iq)
const extracted = extractDeviceJids(result, authState.creds.me!.id, ignoreZeroDevices)
const deviceMap: { [_: string]: JidWithDevice[] } = {}
const query = new USyncQuery()
.withContext('message')
.withDeviceProtocol()
for(const item of extracted) {
deviceMap[item.user] = deviceMap[item.user] || []
deviceMap[item.user].push(item)
deviceResults.push(item)
for(const jid of toFetch) {
query.withUser(new USyncUser().withId(jid))
}
for(const key in deviceMap) {
userDevicesCache.set(key, deviceMap[key])
const result = await sock.executeUSyncQuery(query)
if(result) {
const extracted = extractDeviceJids(result?.list, authState.creds.me!.id, ignoreZeroDevices)
const deviceMap: { [_: string]: JidWithDevice[] } = {}
for(const item of extracted) {
deviceMap[item.user] = deviceMap[item.user] || []
deviceMap[item.user].push(item)
deviceResults.push(item)
}
for(const key in deviceMap) {
userDevicesCache.set(key, deviceMap[key])
}
}
return deviceResults

81
src/Socket/usync.ts Normal file
View File

@@ -0,0 +1,81 @@
import { Boom } from '@hapi/boom'
import { SocketConfig } from '../Types'
import { BinaryNode, S_WHATSAPP_NET } from '../WABinary'
import { USyncQuery } from '../WAUSync'
import { makeSocket } from './socket'
export const makeUSyncSocket = (config: SocketConfig) => {
const sock = makeSocket(config)
const {
generateMessageTag,
query,
} = sock
const executeUSyncQuery = async(usyncQuery: USyncQuery) => {
if(usyncQuery.protocols.length === 0) {
throw new Boom('USyncQuery must have at least one protocol')
}
// todo: validate users, throw WARNING on no valid users
// variable below has only validated users
const validUsers = usyncQuery.users
const userNodes = validUsers.map((user) => {
return {
tag: 'user',
attrs: {
jid: !user.phone ? user.id : undefined,
},
content: usyncQuery.protocols
.map((a) => a.getUserElement(user))
.filter(a => a !== null)
} as BinaryNode
})
const listNode: BinaryNode = {
tag: 'list',
attrs: {},
content: userNodes
}
const queryNode: BinaryNode = {
tag: 'query',
attrs: {},
content: usyncQuery.protocols.map((a) => a.getQueryElement())
}
const iq = {
tag: 'iq',
attrs: {
to: S_WHATSAPP_NET,
type: 'get',
xmlns: 'usync',
},
content: [
{
tag: 'usync',
attrs: {
context: usyncQuery.context,
mode: usyncQuery.mode,
sid: generateMessageTag(),
last: 'true',
index: '0',
},
content: [
queryNode,
listNode
]
}
],
}
const result = await query(iq)
return usyncQuery.parseUSyncQueryResult(result)
}
return {
...sock,
executeUSyncQuery,
}
}

27
src/Types/USync.ts Normal file
View File

@@ -0,0 +1,27 @@
import { BinaryNode } from '../WABinary'
import { USyncUser } from '../WAUSync'
/**
* Defines the interface for a USyncQuery protocol
*/
export interface USyncQueryProtocol {
/**
* The name of the protocol
*/
name: string
/**
* Defines what goes inside the query part of a USyncQuery
*/
getQueryElement: () => BinaryNode
/**
* Defines what goes inside the user part of a USyncQuery
*/
getUserElement: (user: USyncUser) => BinaryNode | null
/**
* Parse the result of the query
* @param data Data from the result
* @returns Whatever the protocol is supposed to return
*/
parser: (data: BinaryNode) => unknown
}

View File

@@ -3,6 +3,7 @@ import { KEY_BUNDLE_TYPE } from '../Defaults'
import { SignalRepository } from '../Types'
import { AuthenticationCreds, AuthenticationState, KeyPair, SignalIdentity, SignalKeyStore, SignedKeyPair } from '../Types/Auth'
import { assertNodeErrorFree, BinaryNode, getBinaryNodeChild, getBinaryNodeChildBuffer, getBinaryNodeChildren, getBinaryNodeChildUInt, jidDecode, JidWithDevice, S_WHATSAPP_NET } from '../WABinary'
import { DeviceListData, ParsedDeviceInfo, USyncQueryResultList } from '../WAUSync'
import { Curve, generateSignalPubKey } from './crypto'
import { encodeBigEndian } from './generics'
@@ -114,30 +115,24 @@ export const parseAndInjectE2ESessions = async(
}
}
export const extractDeviceJids = (result: BinaryNode, myJid: string, excludeZeroDevices: boolean) => {
export const extractDeviceJids = (result: USyncQueryResultList[], myJid: string, excludeZeroDevices: boolean) => {
const { user: myUser, device: myDevice } = jidDecode(myJid)!
const extracted: JidWithDevice[] = []
for(const node of result.content as BinaryNode[]) {
const list = getBinaryNodeChild(node, 'list')?.content
if(list && Array.isArray(list)) {
for(const item of list) {
const { user } = jidDecode(item.attrs.jid)!
const devicesNode = getBinaryNodeChild(item, 'devices')
const deviceListNode = getBinaryNodeChild(devicesNode, 'device-list')
if(Array.isArray(deviceListNode?.content)) {
//eslint-disable-next-line max-depth
for(const { tag, attrs } of deviceListNode!.content) {
const device = +attrs.id
//eslint-disable-next-line max-depth
if(
tag === 'device' && // ensure the "device" tag
(!excludeZeroDevices || device !== 0) && // if zero devices are not-excluded, or device is non zero
(myUser !== user || myDevice !== device) && // either different user or if me user, not this device
(device === 0 || !!attrs['key-index']) // ensure that "key-index" is specified for "non-zero" devices, produces a bad req otherwise
) {
extracted.push({ user, device })
}
}
for(const userResult of result) {
const { devices, id } = userResult as { devices: ParsedDeviceInfo, id: string }
const { user } = jidDecode(id)!
const deviceList = devices?.deviceList as DeviceListData[]
if(Array.isArray(deviceList)) {
for(const { id: device, keyIndex } of deviceList) {
if(
(!excludeZeroDevices || device !== 0) && // if zero devices are not-excluded, or device is non zero
(myUser !== user || myDevice !== device) && // either different user or if me user, not this device
(device === 0 || !!keyIndex) // ensure that "key-index" is specified for "non-zero" devices, produces a bad req otherwise
) {
extracted.push({ user, device })
}
}
}

View File

@@ -0,0 +1,32 @@
import { USyncQueryProtocol } from '../../Types/USync'
import { assertNodeErrorFree, BinaryNode } from '../../WABinary'
import { USyncUser } from '../USyncUser'
export class USyncContactProtocol implements USyncQueryProtocol {
name = 'contact'
getQueryElement(): BinaryNode {
return {
tag: 'contact',
attrs: {},
}
}
getUserElement(user: USyncUser): BinaryNode {
//TODO: Implement type / username fields (not yet supported)
return {
tag: 'contact',
attrs: {},
content: user.phone,
}
}
parser(node: BinaryNode): boolean {
if(node.tag === 'contact') {
assertNodeErrorFree(node)
return node?.attrs?.type === 'in'
}
return false
}
}

View File

@@ -0,0 +1,78 @@
import { USyncQueryProtocol } from '../../Types/USync'
import { assertNodeErrorFree, BinaryNode, getBinaryNodeChild } from '../../WABinary'
//import { USyncUser } from '../USyncUser'
export type KeyIndexData = {
timestamp: number
signedKeyIndex?: Uint8Array
expectedTimestamp?: number
}
export type DeviceListData = {
id: number
keyIndex?: number
isHosted?: boolean
}
export type ParsedDeviceInfo = {
deviceList?: DeviceListData[]
keyIndex?: KeyIndexData
}
export class USyncDeviceProtocol implements USyncQueryProtocol {
name = 'devices'
getQueryElement(): BinaryNode {
return {
tag: 'devices',
attrs: {
version: '2',
},
}
}
getUserElement(/* user: USyncUser */): BinaryNode | null {
//TODO: Implement device phashing, ts and expectedTs
//TODO: if all are not present, return null <- current behavior
//TODO: otherwise return a node w tag 'devices' w those as attrs
return null
}
parser(node: BinaryNode): ParsedDeviceInfo {
const deviceList: DeviceListData[] = []
let keyIndex: KeyIndexData | undefined = undefined
if(node.tag === 'devices') {
assertNodeErrorFree(node)
const deviceListNode = getBinaryNodeChild(node, 'device-list')
const keyIndexNode = getBinaryNodeChild(node, 'key-index-list')
if(Array.isArray(deviceListNode?.content)) {
for(const { tag, attrs } of deviceListNode!.content) {
const id = +attrs.id
const keyIndex = +attrs['key-index']
if(tag === 'device') {
deviceList.push({
id,
keyIndex,
isHosted: !!(attrs['is_hosted'] && attrs['is_hosted'] === 'true')
})
}
}
}
if(keyIndexNode?.tag === 'key-index-list') {
keyIndex = {
timestamp: +keyIndexNode.attrs['ts'],
signedKeyIndex: keyIndexNode?.content as Uint8Array,
expectedTimestamp: keyIndexNode.attrs['expected_ts'] ? +keyIndexNode.attrs['expected_ts'] : undefined
}
}
}
return {
deviceList,
keyIndex
}
}
}

View File

@@ -0,0 +1,35 @@
import { USyncQueryProtocol } from '../../Types/USync'
import { assertNodeErrorFree, BinaryNode } from '../../WABinary'
export type DisappearingModeData = {
duration: number
setAt?: Date
}
export class USyncDisappearingModeProtocol implements USyncQueryProtocol {
name = 'disappearing_mode'
getQueryElement(): BinaryNode {
return {
tag: 'disappearing_mode',
attrs: {},
}
}
getUserElement(): null {
return null
}
parser(node: BinaryNode): DisappearingModeData | undefined {
if(node.tag === 'status') {
assertNodeErrorFree(node)
const duration: number = +node?.attrs.duration
const setAt = new Date(+(node?.attrs.t || 0) * 1000)
return {
duration,
setAt,
}
}
}
}

View File

@@ -0,0 +1,44 @@
import { USyncQueryProtocol } from '../../Types/USync'
import { assertNodeErrorFree, BinaryNode } from '../../WABinary'
export type StatusData = {
status?: string | null
setAt?: Date
}
export class USyncStatusProtocol implements USyncQueryProtocol {
name = 'status'
getQueryElement(): BinaryNode {
return {
tag: 'status',
attrs: {},
}
}
getUserElement(): null {
return null
}
parser(node: BinaryNode): StatusData | undefined {
if(node.tag === 'status') {
assertNodeErrorFree(node)
let status: string | null = node?.content!.toString()
const setAt = new Date(+(node?.attrs.t || 0) * 1000)
if(!status) {
if(+node.attrs?.code === 401) {
status = ''
} else {
status = null
}
} else if(typeof status === 'string' && status.length === 0) {
status = null
}
return {
status,
setAt,
}
}
}
}

View File

@@ -0,0 +1,4 @@
export * from './USyncDeviceProtocol'
export * from './USyncContactProtocol'
export * from './USyncStatusProtocol'
export * from './USyncDisappearingModeProtocol'

103
src/WAUSync/USyncQuery.ts Normal file
View File

@@ -0,0 +1,103 @@
import { USyncQueryProtocol } from '../Types/USync'
import { BinaryNode, getBinaryNodeChild } from '../WABinary'
import { USyncContactProtocol, USyncDeviceProtocol, USyncDisappearingModeProtocol, USyncStatusProtocol } from './Protocols'
import { USyncUser } from './USyncUser'
export type USyncQueryResultList = { [protocol: string]: unknown, id: string }
export type USyncQueryResult = {
list: USyncQueryResultList[]
sideList: USyncQueryResultList[]
}
export class USyncQuery {
protocols: USyncQueryProtocol[]
users: USyncUser[]
context: string
mode: string
constructor() {
this.protocols = []
this.users = []
this.context = 'interactive'
this.mode = 'query'
}
withMode(mode: string) {
this.mode = mode
return this
}
withContext(context: string) {
this.context = context
return this
}
withUser(user: USyncUser) {
this.users.push(user)
return this
}
parseUSyncQueryResult(result: BinaryNode): USyncQueryResult | undefined {
if(result.attrs.type !== 'result') {
return
}
const protocolMap = Object.fromEntries(this.protocols.map((protocol) => {
return [protocol.name, protocol.parser]
}))
const queryResult: USyncQueryResult = {
// TODO: implement errors etc.
list: [],
sideList: [],
}
const usyncNode = getBinaryNodeChild(result, 'usync')
//TODO: implement error backoff, refresh etc.
//TODO: see if there are any errors in the result node
//const resultNode = getBinaryNodeChild(usyncNode, 'result')
const listNode = getBinaryNodeChild(usyncNode, 'list')
if(Array.isArray(listNode?.content) && typeof listNode !== 'undefined') {
queryResult.list = listNode.content.map((node) => {
const id = node?.attrs.jid
const data = Array.isArray(node?.content) ? Object.fromEntries(node.content.map((content) => {
const protocol = content.tag
const parser = protocolMap[protocol]
if(parser) {
return [protocol, parser(content)]
} else {
return [protocol, null]
}
}).filter(([, b]) => b !== null) as [string, unknown][]) : {}
return { ...data, id }
})
}
//TODO: implement side list
//const sideListNode = getBinaryNodeChild(usyncNode, 'side_list')
return queryResult
}
withDeviceProtocol() {
this.protocols.push(new USyncDeviceProtocol())
return this
}
withContactProtocol() {
this.protocols.push(new USyncContactProtocol())
return this
}
withStatusProtocol() {
this.protocols.push(new USyncStatusProtocol())
return this
}
withDisappearingModeProtocol() {
this.protocols.push(new USyncDisappearingModeProtocol())
return this
}
}

27
src/WAUSync/USyncUser.ts Normal file
View File

@@ -0,0 +1,27 @@
export class USyncUser {
id: string
lid: string
phone: string
type: string
withId(id: string) {
this.id = id
return this
}
withLid(lid: string) {
this.lid = lid
return this
}
withPhone(phone: string) {
this.phone = phone
return this
}
withType(type: string) {
this.type = type
return this
}
}

3
src/WAUSync/index.ts Normal file
View File

@@ -0,0 +1,3 @@
export * from './Protocols'
export * from './USyncQuery'
export * from './USyncUser'

View File

@@ -7,6 +7,7 @@ export * from './Store'
export * from './Defaults'
export * from './WABinary'
export * from './WAM'
export * from './WAUSync'
export type WASocket = ReturnType<typeof makeWASocket>
export { makeWASocket }