From 266e695b3dfe30479c27462e818f4e5ccd635035 Mon Sep 17 00:00:00 2001 From: Adhiraj Date: Fri, 1 May 2020 22:13:46 +0530 Subject: [PATCH] Bug Fixes + ConversationExtract --- .DS_Store | Bin 10244 -> 10244 bytes .gitignore | 1 + ConversationExtract.js | 96 +++++++++++++++++++++++++++++++++++++++++ WhatsAppWeb.Query.js | 36 ++++++++++------ WhatsAppWeb.Recv.js | 19 ++++---- WhatsAppWeb.Send.js | 9 ++-- 6 files changed, 134 insertions(+), 27 deletions(-) create mode 100644 ConversationExtract.js diff --git a/.DS_Store b/.DS_Store index 1b30681ffed874a5860b9da91349684ea359b1d6..f5e2a23c443554aed41352813922b351d48a08ca 100644 GIT binary patch delta 124 zcmZn(XbG6$FDlExz`)4BAi%&7&ydNG$B@R5&!D%la2or>2Hwr=94s7+vYT53EE(0s z7@UDpc?@L?sSHI7#SDoIB|r@z5m$x^AiD@ClFU#td4r_L<`;rj7&reAKFqv1TVf9T X#)c5a&Fl)lST-*e6=Ry5E%6or1lJ*z delta 439 zcmZn(XbG6$FRH}Az`)4BAi%(2#*hTWDGaF$<{JyAu}^H^-OSFx!ch-W!N*_#lr>^V zL{`P1$B@fVz+mE;lb@WFlb-}s2Q-FJ9EdCag8@(kq}PJM7^u$@Xtoi934 should the Id of the chat be recorded + * */ +function extractChats (authCreds, outputFile, produceAnonData) { + let client = new WhatsAppWeb() // instantiate an instance + if (authCreds) { // login if creds are present + client.login(authCreds) + } else { // create a new connection otherwise + client.connect() + } + // internal extract function + const extract = function () { + fs.writeFileSync(outputFile, "chat,input,output\n") // write header to file + + let rows = 0 + let chats = Object.keys(client.chats) + + const extractChat = function (index) { + const id = chats[index] + console.log("extracting for " + id + "...") + + var curInput = "" + var curOutput = "" + var lastMessage + return client.getAllMessages (id, m => { + var text + if (!m.message) { // if message not present, return + return + } else if (m.message.conversation) { // if its a plain text message + text = m.message.conversation + } else if (m.message.extendedTextMessage && m.message.extendedTextMessage.contextInfo) { // if its a reply to a previous message + const mText = m.message.extendedTextMessage.text + const quotedMessage = m.message.extendedTextMessage.contextInfo.quotedMessage + // if it's like a '.' and the quoted message has no text, then just forget it + if (mText.length <= 2 && !quotedMessage.conversation) { + return + } + // if somebody sent like a '.', then the text should be the quoted message + if (mText.length <= 2) { + text = quotedMessage.conversation + } else { // otherwise just use this text + text = mText + } + } else { + return + } + // if the person who sent the message has switched, flush the row + if (lastMessage && !m.key.fromMe && lastMessage.key.fromMe) { + + let row = "" + (produceAnonData ? "" : id) + ",\"" + curInput + "\",\"" + curOutput + "\"\n" + fs.appendFileSync (outputFile, row) + rows += 1 + curInput = "" + curOutput = "" + } + + if (m.key.fromMe) { + curOutput += curOutput === "" ? text : ("\n"+text) + } else { + curInput += curInput === "" ? text : ("\n"+text) + } + + lastMessage = m + }, 50, "after") // load from the start, in chunks of 50 + .then (() => console.log("finished extraction for " + id)) + .then (() => { + if (index+1 < chats.length) { + return extractChat(index+1) + } + }) + } + + extractChat(0) + .then (() => { + console.log("extracted all; total " + rows + " rows") + client.disconnect () + }) + } + + client.handlers.onConnected = () => { + // start extracting 4 seconds after the connection + setTimeout(extract, 4000) + } + client.handlers.onUnreadMessage = (message) => { + + } + client.handlers.onError = (error) => { + console.log("got error: " + error) + } +} +let creds = JSON.parse(fs.readFileSync("auth_info.json")) +extractChats(creds, "output.csv", true) \ No newline at end of file diff --git a/WhatsAppWeb.Query.js b/WhatsAppWeb.Query.js index 4cbd541..0192163 100644 --- a/WhatsAppWeb.Query.js +++ b/WhatsAppWeb.Query.js @@ -28,10 +28,10 @@ module.exports = function(WhatsAppWeb) { {epoch: this.msgCount.toString(), type: "contacts"}, null ] - return this.query(json, true) // this has to be an encrypted query + return this.query(json, [10, 64]) // this has to be an encrypted query } // load messages from a group or sender - WhatsAppWeb.prototype.getMessages = function (jid, count, beforeMessage=null) { + WhatsAppWeb.prototype.getMessages = function (jid, count, indexMessage=null, mode="before") { // construct JSON let json = [ "query", @@ -39,34 +39,44 @@ module.exports = function(WhatsAppWeb) { epoch: this.msgCount.toString(), type: "message", jid: jid, - kind: "before", + kind: mode, owner: "true", count: count.toString() }, null ] - // if we have some index before which we want to query - if (beforeMessage) { - json[1].index = beforeMessage.id - json[1].owner = beforeMessage.fromMe ? "true" : "false" + // if we have some index from which we want to query + if (indexMessage) { + json[1].index = indexMessage.id + json[1].owner = indexMessage.fromMe ? "true" : "false" } - return this.query(json, true) + + return this.query(json, [10, 128]) } // loads all the conversation you've had with given ID - WhatsAppWeb.prototype.getAllMessages = function (jid, onMessage, chunkSize=25) { + WhatsAppWeb.prototype.getAllMessages = function (jid, onMessage, chunkSize=25, mode="before") { var offsetID = null const loadMessage = () => { - return this.getMessages(jid, chunkSize, offsetID) + return this.getMessages(jid, chunkSize, offsetID, mode) .then (json => { if (json[2]) { // callback with most recent message first (descending order of date) - for (var i = json[2].length-1; i >= 0;i--) { - onMessage(json[2][i][2]) + let lastMessage + if (mode === "before") { + for (var i = json[2].length-1; i >= 0;i--) { + onMessage(json[2][i][2]) + lastMessage = json[2][i][2] + } + } else { + for (var i = 0; i < json[2].length;i++) { + onMessage(json[2][i][2]) + lastMessage = json[2][i][2] + } } // if there are still more messages if (json[2].length >= chunkSize) { - offsetID = json[2][0][2].key // get the oldest message + offsetID = lastMessage.key // get the last message return new Promise ( (resolve, reject) => { // send query after 200 ms setTimeout( () => loadMessage().then (resolve).catch(reject), 200) diff --git a/WhatsAppWeb.Recv.js b/WhatsAppWeb.Recv.js index b1d57e2..c877e3c 100644 --- a/WhatsAppWeb.Recv.js +++ b/WhatsAppWeb.Recv.js @@ -27,12 +27,7 @@ module.exports = function(WhatsAppWeb) { const messageTag = message.slice(0, commaIndex) //console.log(messageTag) if (data.length === 0) { - // got an empty message, usually get one after sending a message or something - if (this.callbacks[messageTag]) { - const q = this.callbacks[messageTag] - q.callback() - delete this.callbacks[messageTag] - } + // got an empty message, usually get one after sending a query with the 128 tag return } @@ -84,6 +79,7 @@ module.exports = function(WhatsAppWeb) { } return case "action": + /* this is when some action was taken on a chat or that we recieve a message. json[1] tells us more about the message, it can be null @@ -122,13 +118,12 @@ module.exports = function(WhatsAppWeb) { this.chats[id].messages.push(message[2]) // append this message to the array } - - const id = json[0][2].key.remoteJid // get the ID whose chats we just processed this.clearUnreadMessages(id) // forward to the handler any any unread messages } return case "response": + //console.log(json[1]) // if it is the list of all the people the WhatsApp account has chats with if (json[1].type === "chat") { @@ -149,6 +144,8 @@ module.exports = function(WhatsAppWeb) { this.handlers.gotContact(json[2]) } return + } else if (json[1].type === "message") { + } break case "Presence": @@ -214,8 +211,8 @@ module.exports = function(WhatsAppWeb) { const chat = this.chats[id] // get the chat var j = 0 let unreadMessages = chat.user.count - while (unreadMessages > 0) { - if (!chat.messages[j].key.fromMe) { // only forward if the message is from the sender + while (unreadMessages > 0 && chat.messages[j]) { + if (!chat.messages[j].key.fromMe && this.handlers.onUnreadMessage) { // only forward if the message is from the sender this.handlers.onUnreadMessage( chat.messages[j] ) // send off the unread message unreadMessages -= 1 // reduce } @@ -236,7 +233,7 @@ module.exports = function(WhatsAppWeb) { this.chats[ message.key.remoteJid ].messages.splice(0, 0, message) } - if (!message.key.fromMe) { // if this message was sent to us, notify the handler + if (!message.key.fromMe && this.handlers.onUnreadMessage) { // if this message was sent to us, notify the handler this.handlers.onUnreadMessage ( message ) } } diff --git a/WhatsAppWeb.Send.js b/WhatsAppWeb.Send.js index a9b476a..132ab5e 100644 --- a/WhatsAppWeb.Send.js +++ b/WhatsAppWeb.Send.js @@ -28,6 +28,9 @@ module.exports = function(WhatsAppWeb) { } // send a text message to someone, optionally you can provide a quoted message & the timestamp for the message WhatsAppWeb.prototype.sendTextMessage = function (id, txt, quoted=null, timestamp=null) { + if (typeof txt !== "string") { + return Promise.reject("") + } let message if (quoted) { message = { @@ -149,9 +152,9 @@ module.exports = function(WhatsAppWeb) { const json = [ "action", {epoch: this.msgCount.toString(), type: "relay" }, - [ ['message', null, messageJSON] ] + [ ["message", null, messageJSON] ] ] - return this.query(json, [16, 128]) + return this.query(json, [16, 64]) } // send query message to WhatsApp servers; returns a promise WhatsAppWeb.prototype.query = function (json, binaryTags=null) { @@ -169,7 +172,7 @@ module.exports = function(WhatsAppWeb) { // send a binary message, the tags parameter tell WhatsApp what the message is all about WhatsAppWeb.prototype.sendBinary = function (json, tags) { const binary = this.encoder.write(json) // encode the JSON to the WhatsApp binary format - + var buff = Utils.aesEncrypt(binary, this.authInfo.encKey) // encrypt it using AES and our encKey const sign = Utils.hmacSign(buff, this.authInfo.macKey) // sign the message using HMAC and our macKey const tag = Utils.generateMessageTag()