Backup and end-to-end test!

This commit is contained in:
Scott Nonnenberg 2018-04-20 14:55:33 -07:00
parent c3acf43c47
commit a7d44d3344
No known key found for this signature in database
GPG Key ID: 5F82280C35134661
6 changed files with 477 additions and 28 deletions

View File

@ -468,6 +468,7 @@ async function readAttachment(dir, attachment, name, options) {
const data = await readFileAsArrayBuffer(targetPath);
const isEncrypted = !_.isUndefined(key);
if (isEncrypted) {
attachment.data = await crypto.decryptSymmetric(key, data);
} else {
@ -475,6 +476,65 @@ async function readAttachment(dir, attachment, name, options) {
}
}
async function writeThumbnail(attachment, options) {
const {
dir,
message,
index,
key,
newKey,
} = options;
const filename = `${_getAnonymousAttachmentFileName(message, index)}-thumbnail`;
const target = path.join(dir, filename);
const { thumbnail } = attachment;
if (!thumbnail || !thumbnail.data) {
return;
}
await writeEncryptedAttachment(target, thumbnail.data, {
key,
newKey,
filename,
dir,
});
}
async function writeThumbnails(rawQuotedAttachments, options) {
const { name } = options;
const { loadAttachmentData } = Signal.Migrations;
const promises = rawQuotedAttachments.map(async (attachment) => {
if (!attachment || !attachment.thumbnail || !attachment.thumbnail.path) {
return attachment;
}
return Object.assign(
{},
attachment,
{ thumbnail: await loadAttachmentData(attachment.thumbnail) }
);
});
const attachments = await Promise.all(promises);
try {
await Promise.all(_.map(
attachments,
(attachment, index) => writeThumbnail(attachment, Object.assign({}, options, {
index,
}))
));
} catch (error) {
console.log(
'writeThumbnails: error exporting conversation',
name,
':',
error && error.stack ? error.stack : error
);
throw error;
}
}
async function writeAttachment(attachment, options) {
const {
dir,
@ -485,26 +545,16 @@ async function writeAttachment(attachment, options) {
} = options;
const filename = _getAnonymousAttachmentFileName(message, index);
const target = path.join(dir, filename);
if (fs.existsSync(target)) {
if (newKey) {
console.log(`Deleting attachment ${filename}; key has changed`);
fs.unlinkSync(target);
} else {
console.log(`Skipping attachment ${filename}; already exists`);
return;
}
}
if (!Attachment.hasData(attachment)) {
throw new TypeError("'attachment.data' is required");
}
const ciphertext = await crypto.encryptSymmetric(key, attachment.data);
const writer = await createFileAndWriter(dir, filename);
const stream = createOutputStream(writer);
stream.write(Buffer.from(ciphertext));
await stream.close();
await writeEncryptedAttachment(target, attachment.data, {
key,
newKey,
filename,
dir,
});
}
async function writeAttachments(rawAttachments, options) {
@ -531,6 +581,32 @@ async function writeAttachments(rawAttachments, options) {
}
}
async function writeEncryptedAttachment(target, data, options = {}) {
const {
key,
newKey,
filename,
dir,
} = options;
if (fs.existsSync(target)) {
if (newKey) {
console.log(`Deleting attachment ${filename}; key has changed`);
fs.unlinkSync(target);
} else {
console.log(`Skipping attachment ${filename}; already exists`);
return;
}
}
const ciphertext = await crypto.encryptSymmetric(key, data);
const writer = await createFileAndWriter(dir, filename);
const stream = createOutputStream(writer);
stream.write(Buffer.from(ciphertext));
await stream.close();
}
function _sanitizeFileName(filename) {
return filename.toString().replace(/[^a-z0-9.,+()'#\- ]/gi, '_');
}
@ -542,6 +618,7 @@ async function exportConversation(db, conversation, options) {
dir,
attachmentsDir,
key,
newKey,
} = options;
if (!name) {
throw new Error('Need a name!');
@ -610,6 +687,7 @@ async function exportConversation(db, conversation, options) {
}
// eliminate attachment data from the JSON, since it will go to disk
// Note: this is for legacy messages only, which stored attachment data in the db
message.attachments = _.map(
attachments,
attachment => _.omit(attachment, ['data'])
@ -629,18 +707,34 @@ async function exportConversation(db, conversation, options) {
const jsonString = JSON.stringify(stringify(message));
stream.write(jsonString);
console.log({ backupMessage: message });
if (attachments && attachments.length > 0) {
const exportAttachments = () => writeAttachments(attachments, {
dir: attachmentsDir,
name,
message,
key,
newKey,
});
// eslint-disable-next-line more/no-then
promiseChain = promiseChain.then(exportAttachments);
}
const quoteThumbnails = message.quote && message.quote.attachments;
if (quoteThumbnails && quoteThumbnails.length > 0) {
const exportQuoteThumbnails = () => writeThumbnails(quoteThumbnails, {
dir: attachmentsDir,
name,
message,
key,
newKey,
});
// eslint-disable-next-line more/no-then
promiseChain = promiseChain.then(exportQuoteThumbnails);
}
count += 1;
cursor.continue();
} else {
@ -701,6 +795,7 @@ function exportConversations(db, options) {
messagesDir,
attachmentsDir,
key,
newKey,
} = options;
if (!messagesDir) {
@ -747,6 +842,7 @@ function exportConversations(db, options) {
dir,
attachmentsDir,
key,
newKey,
});
};
@ -808,11 +904,23 @@ function loadAttachments(dir, getName, options) {
options = options || {};
const { message } = options;
const promises = _.map(message.attachments, (attachment, index) => {
const attachmentPromises = _.map(message.attachments, (attachment, index) => {
const name = getName(message, index, attachment);
return readAttachment(dir, attachment, name, options);
});
return Promise.all(promises);
const quoteAttachments = message.quote && message.quote.attachments;
const thumbnailPromises = _.map(quoteAttachments, (attachment, index) => {
const thumbnail = attachment && attachment.thumbnail;
if (!thumbnail) {
return null;
}
const name = `${getName(message, index, thumbnail)}-thumbnail`;
return readAttachment(dir, thumbnail, name, options);
});
return Promise.all(attachmentPromises.concat(thumbnailPromises));
}
function saveMessage(db, message) {
@ -922,7 +1030,8 @@ async function importConversation(db, dir, options) {
return false;
}
if (message.attachments && message.attachments.length) {
if ((message.attachments && message.attachments.length) ||
(message.quote && message.quote.attachments && message.quote.attachments.length)) {
const importMessage = async () => {
const getName = attachmentsDir
? _getAnonymousAttachmentFileName

View File

@ -243,9 +243,12 @@ exports.createAttachmentDataWriter = (writeExistingAttachmentData) => {
const message = exports.initializeSchemaVersion(rawMessage);
const { attachments } = message;
const hasAttachments = attachments && attachments.length > 0;
if (!hasAttachments) {
const { attachments, quote } = message;
const hasFilesToWrite =
(quote && quote.attachments && quote.attachments.length > 0) ||
(attachments && attachments.length > 0);
if (!hasFilesToWrite) {
return message;
}
@ -256,7 +259,7 @@ exports.createAttachmentDataWriter = (writeExistingAttachmentData) => {
return message;
}
attachments.forEach((attachment) => {
(attachments || []).forEach((attachment) => {
if (!Attachment.hasData(attachment)) {
throw new TypeError("'attachment.data' is required during message import");
}
@ -266,13 +269,29 @@ exports.createAttachmentDataWriter = (writeExistingAttachmentData) => {
}
});
const messageWithoutAttachmentData = Object.assign({}, message, {
attachments: await Promise.all(attachments.map(async (attachment) => {
await writeExistingAttachmentData(attachment);
return omit(attachment, ['data']);
})),
const writeThumbnails = exports._mapQuotedAttachments(async (thumbnail) => {
const { data, path } = thumbnail;
// we want to be bulletproof to thumbnails without data
if (!data || !path) {
return thumbnail;
}
await writeExistingAttachmentData(thumbnail);
return omit(thumbnail, ['data']);
});
const messageWithoutAttachmentData = Object.assign(
{},
await writeThumbnails(message),
{
attachments: await Promise.all((attachments || []).map(async (attachment) => {
await writeExistingAttachmentData(attachment);
return omit(attachment, ['data']);
})),
}
);
return messageWithoutAttachmentData;
};
};

View File

@ -114,6 +114,7 @@
"eslint-plugin-mocha": "^4.12.1",
"eslint-plugin-more": "^0.3.1",
"extract-zip": "^1.6.6",
"glob": "^7.1.2",
"grunt": "^1.0.1",
"grunt-cli": "^1.2.0",
"grunt-contrib-concat": "^1.0.1",

View File

@ -205,3 +205,17 @@ window.Signal.Workflow.MessageDataMigrator =
// We pull this in last, because the native module involved appears to be sensitive to
// /tmp mounted as noexec on Linux.
require('./js/spell_check');
if (window.config.environment === 'test') {
/* eslint-disable global-require, import/no-extraneous-dependencies */
window.test = {
fs: require('fs'),
glob: require('glob'),
fse: require('fs-extra'),
tmp: require('tmp'),
path: require('path'),
basePath: __dirname,
attachmentsPath,
};
/* eslint-enable global-require, import/no-extraneous-dependencies */
}

View File

@ -1,5 +1,8 @@
/* global Signal: false */
/* global Whisper: false */
/* global assert: false */
/* global textsecure: false */
/* global _: false */
'use strict';
@ -223,4 +226,270 @@ describe('Backup', () => {
);
});
});
describe('end-to-end', () => {
it('exports then imports to produce the same data we started with', async () => {
const {
attachmentsPath,
fs,
fse,
glob,
path,
tmp,
} = window.test;
const {
upgradeMessageSchema,
loadAttachmentData,
} = window.Signal.Migrations;
const key = new Uint8Array([
1, 3, 4, 5, 6, 7, 8, 11,
23, 34, 1, 34, 3, 5, 45, 45,
1, 3, 4, 5, 6, 7, 8, 11,
23, 34, 1, 34, 3, 5, 45, 45,
]);
const attachmentsPattern = path.join(attachmentsPath, '**');
const OUR_NUMBER = '+12025550000';
const CONTACT_ONE_NUMBER = '+12025550001';
async function wrappedLoadAttachment(attachment) {
return _.omit(await loadAttachmentData(attachment), ['path']);
}
async function clearAllData() {
await textsecure.storage.protocol.removeAllData();
await fse.emptyDir(attachmentsPath);
}
function removeId(model) {
return _.omit(model, ['id']);
}
const twoSlashes = /.*\/.*\/.*/;
function removeDirs(dirs) {
return _.filter(dirs, (fullDir) => {
const dir = fullDir.replace(attachmentsPath, '');
return twoSlashes.test(dir);
});
}
function _mapQuotedAttachments(mapper) {
return async (message, context) => {
if (!message.quote) {
return message;
}
const wrappedMapper = async (attachment) => {
if (!attachment || !attachment.thumbnail) {
return attachment;
}
return Object.assign({}, attachment, {
thumbnail: await mapper(attachment.thumbnail, context),
});
};
const quotedAttachments = (message.quote && message.quote.attachments) || [];
return Object.assign({}, message, {
quote: Object.assign({}, message.quote, {
attachments: await Promise.all(quotedAttachments.map(wrappedMapper)),
}),
});
};
}
async function loadAllFilesFromDisk(message) {
const loadThumbnails = _mapQuotedAttachments((thumbnail) => {
// we want to be bulletproof to thumbnails without data
if (!thumbnail.path) {
return thumbnail;
}
return wrappedLoadAttachment(thumbnail);
});
const promises = (message.attachments || []).map(attachment =>
wrappedLoadAttachment(attachment));
return Object.assign(
{},
await loadThumbnails(message),
{
attachments: await Promise.all(promises),
}
);
}
let backupDir;
try {
const ATTACHMENT_COUNT = 2;
const MESSAGE_COUNT = 1;
const CONVERSATION_COUNT = 1;
const messageWithAttachments = {
conversationId: CONTACT_ONE_NUMBER,
body: 'Totally!',
source: OUR_NUMBER,
received_at: 1524185933350,
timestamp: 1524185933350,
errors: [],
attachments: [{
contentType: 'image/gif',
fileName: 'sad_cat.gif',
data: new Uint8Array([
1, 2, 3, 4, 5, 6, 7, 8,
1, 2, 3, 4, 5, 6, 7, 8,
1, 2, 3, 4, 5, 6, 7, 8,
1, 2, 3, 4, 5, 6, 7, 8,
]).buffer,
}],
quote: {
text: "Isn't it cute?",
author: CONTACT_ONE_NUMBER,
id: 12345678,
attachments: [{
contentType: 'audio/mp3',
fileName: 'song.mp3',
}, {
contentType: 'image/gif',
fileName: 'happy_cat.gif',
thumbnail: {
contentType: 'image/png',
data: new Uint8Array([
2, 2, 3, 4, 5, 6, 7, 8,
1, 2, 3, 4, 5, 6, 7, 8,
1, 2, 3, 4, 5, 6, 7, 8,
1, 2, 3, 4, 5, 6, 7, 8,
]).buffer,
},
}],
},
};
console.log('Backup test: Clear all data');
await clearAllData();
console.log('Backup test: Create models, save to db/disk');
const message = await upgradeMessageSchema(messageWithAttachments);
console.log({ message });
const messageModel = new Whisper.Message(message);
await window.wrapDeferred(messageModel.save());
const conversation = {
active_at: 1524185933350,
color: 'orange',
expireTimer: 0,
id: CONTACT_ONE_NUMBER,
lastMessage: 'Heyo!',
name: 'Someone Somewhere',
profileAvatar: {
contentType: 'image/jpeg',
data: new Uint8Array([
3, 2, 3, 4, 5, 6, 7, 8,
1, 2, 3, 4, 5, 6, 7, 8,
1, 2, 3, 4, 5, 6, 7, 8,
1, 2, 3, 4, 5, 6, 7, 8,
]).buffer,
size: 64,
},
profileKey: new Uint8Array([
4, 2, 3, 4, 5, 6, 7, 8,
1, 2, 3, 4, 5, 6, 7, 8,
1, 2, 3, 4, 5, 6, 7, 8,
1, 2, 3, 4, 5, 6, 7, 8,
]).buffer,
profileName: 'Someone! 🤔',
profileSharing: true,
timestamp: 1524185933350,
tokens: [
'someone somewhere',
'someone',
'somewhere',
'2025550001',
'12025550001',
],
type: 'private',
unreadCount: 0,
verified: 0,
};
console.log({ conversation });
const conversationModel = new Whisper.Conversation(conversation);
await window.wrapDeferred(conversationModel.save());
console.log('Backup test: Ensure that all attachments were saved to disk');
const attachmentFiles = removeDirs(glob.sync(attachmentsPattern));
console.log({ attachmentFiles });
assert.strictEqual(ATTACHMENT_COUNT, attachmentFiles.length);
console.log('Backup test: Export!');
backupDir = tmp.dirSync().name;
console.log({ backupDir });
await Signal.Backup.exportToDirectory(backupDir, { key });
console.log('Backup test: Ensure that messages.zip exists');
const zipPath = path.join(backupDir, 'messages.zip');
const messageZipExists = fs.existsSync(zipPath);
assert.strictEqual(true, messageZipExists);
console.log('Backup test: Ensure that all attachments made it to backup dir');
const backupAttachmentPattern = path.join(backupDir, 'attachments/*');
const backupAttachments = glob.sync(backupAttachmentPattern);
console.log({ backupAttachments });
assert.strictEqual(ATTACHMENT_COUNT, backupAttachments.length);
console.log('Backup test: Clear all data');
await clearAllData();
console.log('Backup test: Import!');
await Signal.Backup.importFromDirectory(backupDir, { key });
console.log('Backup test: ensure that all attachments were imported');
const recreatedAttachmentFiles = removeDirs(glob.sync(attachmentsPattern));
console.log({ recreatedAttachmentFiles });
assert.strictEqual(ATTACHMENT_COUNT, recreatedAttachmentFiles.length);
assert.deepEqual(attachmentFiles, recreatedAttachmentFiles);
console.log('Backup test: Check messages');
const messageCollection = new Whisper.MessageCollection();
await window.wrapDeferred(messageCollection.fetch());
assert.strictEqual(messageCollection.length, MESSAGE_COUNT);
const messageFromDB = removeId(messageCollection.at(0).attributes);
console.log({ messageFromDB, message });
assert.deepEqual(messageFromDB, message);
console.log('Backup test: check that all attachments were successfully imported');
const messageWithAttachmentsFromDB = await loadAllFilesFromDisk(messageFromDB);
console.log({ messageWithAttachmentsFromDB, messageWithAttachments });
assert.deepEqual(
_.omit(messageWithAttachmentsFromDB, ['schemaVersion']),
messageWithAttachments
);
console.log('Backup test: check conversations');
const conversationCollection = new Whisper.ConversationCollection();
await window.wrapDeferred(conversationCollection.fetch());
assert.strictEqual(conversationCollection.length, CONVERSATION_COUNT);
const conversationFromDB = conversationCollection.at(0).attributes;
console.log({ conversationFromDB, conversation });
assert.deepEqual(
conversationFromDB,
_.omit(conversation, ['profileAvatar'])
);
console.log('Backup test: Clear all data');
await clearAllData();
console.log('Backup test: Complete!');
} finally {
if (backupDir) {
console.log({ backupDir });
console.log('Deleting', backupDir);
await fse.remove(backupDir);
}
}
});
});
});

View File

@ -67,6 +67,43 @@ describe('Message', () => {
await Message.createAttachmentDataWriter(writeExistingAttachmentData)(input);
assert.deepEqual(actual, expected);
});
it('should process quote attachment thumbnails', async () => {
const input = {
body: 'Imagine there is no heaven…',
schemaVersion: 4,
attachments: [],
quote: {
attachments: [{
thumbnail: {
path: 'ab/abcdefghi',
data: stringToArrayBuffer('Its easy if you try'),
},
}],
},
};
const expected = {
body: 'Imagine there is no heaven…',
schemaVersion: 4,
attachments: [],
quote: {
attachments: [{
thumbnail: {
path: 'ab/abcdefghi',
},
}],
},
};
const writeExistingAttachmentData = (attachment) => {
assert.equal(attachment.path, 'ab/abcdefghi');
assert.deepEqual(attachment.data, stringToArrayBuffer('Its easy if you try'));
};
const actual =
await Message.createAttachmentDataWriter(writeExistingAttachmentData)(input);
assert.deepEqual(actual, expected);
});
});
describe('initializeSchemaVersion', () => {