Split database migrations into pre- and post-attachment migration
- Run light-weight migrations before attachment migration. - Run regular migrations after attachments have been moved to disk.
This commit is contained in:
Daniel Gasienica
parent
3841154295
commit
d16178638e
|
|
@ -11,12 +11,13 @@
|
|||
/* global Whisper: false */
|
||||
/* global wrapDeferred: false */
|
||||
|
||||
;(function() {
|
||||
;(async function() {
|
||||
'use strict';
|
||||
|
||||
const { IdleDetector, MessageDataMigrator } = Signal.Workflow;
|
||||
const { Errors, Message } = window.Signal.Types;
|
||||
const { upgradeMessageSchema } = window.Signal.Migrations;
|
||||
const { Migrations0DatabaseWithAttachmentData } = window.Signal.Database;
|
||||
const { Views } = window.Signal;
|
||||
|
||||
// Implicitly used in `indexeddb-backbonejs-adapter`:
|
||||
|
|
@ -75,13 +76,17 @@
|
|||
return accountManager;
|
||||
};
|
||||
|
||||
const cancelInitializationMessage = Views.Initialization.setMessage();
|
||||
console.log('Start IndexedDB migrations');
|
||||
storage.fetch();
|
||||
|
||||
|
||||
/* eslint-enable */
|
||||
/* jshint ignore:start */
|
||||
const cancelInitializationMessage = Views.Initialization.setMessage();
|
||||
console.log('Start IndexedDB migrations');
|
||||
|
||||
console.log('Migrate database with attachments');
|
||||
await Migrations0DatabaseWithAttachmentData.run({ Backbone });
|
||||
|
||||
console.log('Migrate database without attachments');
|
||||
storage.fetch();
|
||||
|
||||
const NUM_MESSAGE_UPGRADES_PER_IDLE = 2;
|
||||
const idleDetector = new IdleDetector();
|
||||
idleDetector.on('idle', async () => {
|
||||
|
|
|
|||
133
js/database.js
133
js/database.js
|
|
@ -2,13 +2,11 @@
|
|||
/* global Backbone: false */
|
||||
/* global _: false */
|
||||
|
||||
/* eslint-disable more/no-then */
|
||||
|
||||
// eslint-disable-next-line func-names
|
||||
(function () {
|
||||
'use strict';
|
||||
|
||||
const { Migrations } = window.Signal;
|
||||
const { Migrations1DatabaseWithoutAttachmentData } = window.Signal.Database;
|
||||
|
||||
window.Whisper = window.Whisper || {};
|
||||
window.Whisper.Database = window.Whisper.Database || {};
|
||||
|
|
@ -125,132 +123,5 @@
|
|||
request.onsuccess = resolve;
|
||||
}));
|
||||
|
||||
Whisper.Database.migrations = [
|
||||
{
|
||||
version: '12.0',
|
||||
migrate(transaction, next) {
|
||||
console.log('migration 12.0');
|
||||
console.log('creating object stores');
|
||||
const messages = transaction.db.createObjectStore('messages');
|
||||
messages.createIndex('conversation', ['conversationId', 'received_at'], {
|
||||
unique: false,
|
||||
});
|
||||
messages.createIndex('receipt', 'sent_at', { unique: false });
|
||||
messages.createIndex('unread', ['conversationId', 'unread'], { unique: false });
|
||||
messages.createIndex('expires_at', 'expires_at', { unique: false });
|
||||
|
||||
const conversations = transaction.db.createObjectStore('conversations');
|
||||
conversations.createIndex('inbox', 'active_at', { unique: false });
|
||||
conversations.createIndex('group', 'members', {
|
||||
unique: false,
|
||||
multiEntry: true,
|
||||
});
|
||||
conversations.createIndex('type', 'type', {
|
||||
unique: false,
|
||||
});
|
||||
conversations.createIndex('search', 'tokens', {
|
||||
unique: false,
|
||||
multiEntry: true,
|
||||
});
|
||||
|
||||
transaction.db.createObjectStore('groups');
|
||||
|
||||
transaction.db.createObjectStore('sessions');
|
||||
transaction.db.createObjectStore('identityKeys');
|
||||
transaction.db.createObjectStore('preKeys');
|
||||
transaction.db.createObjectStore('signedPreKeys');
|
||||
transaction.db.createObjectStore('items');
|
||||
|
||||
console.log('creating debug log');
|
||||
transaction.db.createObjectStore('debug');
|
||||
|
||||
next();
|
||||
},
|
||||
},
|
||||
{
|
||||
version: '13.0',
|
||||
migrate(transaction, next) {
|
||||
console.log('migration 13.0');
|
||||
console.log('Adding fields to identity keys');
|
||||
const identityKeys = transaction.objectStore('identityKeys');
|
||||
const request = identityKeys.openCursor();
|
||||
const promises = [];
|
||||
request.onsuccess = (event) => {
|
||||
const cursor = event.target.result;
|
||||
if (cursor) {
|
||||
const attributes = cursor.value;
|
||||
attributes.timestamp = 0;
|
||||
attributes.firstUse = false;
|
||||
attributes.nonblockingApproval = false;
|
||||
attributes.verified = 0;
|
||||
promises.push(new Promise(((resolve, reject) => {
|
||||
const putRequest = identityKeys.put(attributes, attributes.id);
|
||||
putRequest.onsuccess = resolve;
|
||||
putRequest.onerror = (e) => {
|
||||
console.log(e);
|
||||
reject(e);
|
||||
};
|
||||
})));
|
||||
cursor.continue();
|
||||
} else {
|
||||
// no more results
|
||||
Promise.all(promises).then(() => {
|
||||
next();
|
||||
});
|
||||
}
|
||||
};
|
||||
request.onerror = (event) => {
|
||||
console.log(event);
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
version: '14.0',
|
||||
migrate(transaction, next) {
|
||||
console.log('migration 14.0');
|
||||
console.log('Adding unprocessed message store');
|
||||
const unprocessed = transaction.db.createObjectStore('unprocessed');
|
||||
unprocessed.createIndex('received', 'timestamp', { unique: false });
|
||||
next();
|
||||
},
|
||||
},
|
||||
{
|
||||
version: '15.0',
|
||||
migrate(transaction, next) {
|
||||
console.log('migration 15.0');
|
||||
console.log('Adding messages index for de-duplication');
|
||||
const messages = transaction.objectStore('messages');
|
||||
messages.createIndex('unique', ['source', 'sourceDevice', 'sent_at'], {
|
||||
unique: true,
|
||||
});
|
||||
next();
|
||||
},
|
||||
},
|
||||
{
|
||||
version: '16.0',
|
||||
migrate(transaction, next) {
|
||||
console.log('migration 16.0');
|
||||
console.log('Dropping log table, since we now log to disk');
|
||||
transaction.db.deleteObjectStore('debug');
|
||||
next();
|
||||
},
|
||||
},
|
||||
{
|
||||
version: 17,
|
||||
async migrate(transaction, next) {
|
||||
console.log('migration 17');
|
||||
console.log('Start migration to database version 17');
|
||||
|
||||
const start = Date.now();
|
||||
await Migrations.V17.run(transaction);
|
||||
const duration = Date.now() - start;
|
||||
|
||||
console.log(
|
||||
'Complete migration to database version 17.',
|
||||
`Duration: ${duration}ms`
|
||||
);
|
||||
next();
|
||||
},
|
||||
},
|
||||
];
|
||||
Whisper.Database.migrations = Migrations1DatabaseWithoutAttachmentData.migrations;
|
||||
}());
|
||||
|
|
|
|||
|
|
@ -1,55 +0,0 @@
|
|||
const Message = require('../../types/message');
|
||||
|
||||
|
||||
exports.run = async (transaction) => {
|
||||
const messagesStore = transaction.objectStore('messages');
|
||||
|
||||
console.log('Initialize messages schema version');
|
||||
const numUpgradedMessages = await _initializeMessageSchemaVersion(messagesStore);
|
||||
console.log('Complete messages schema version initialization', { numUpgradedMessages });
|
||||
|
||||
console.log('Create index from attachment schema version to attachment');
|
||||
messagesStore.createIndex('schemaVersion', 'schemaVersion', { unique: false });
|
||||
};
|
||||
|
||||
const _initializeMessageSchemaVersion = messagesStore =>
|
||||
new Promise((resolve, reject) => {
|
||||
const messagePutOperations = [];
|
||||
|
||||
const cursorRequest = messagesStore.openCursor();
|
||||
cursorRequest.onsuccess = async (event) => {
|
||||
const cursor = event.target.result;
|
||||
const hasMoreData = Boolean(cursor);
|
||||
if (!hasMoreData) {
|
||||
await Promise.all(messagePutOperations);
|
||||
return resolve(messagePutOperations.length);
|
||||
}
|
||||
|
||||
const message = cursor.value;
|
||||
const messageWithSchemaVersion = Message.initializeSchemaVersion(message);
|
||||
messagePutOperations.push(putItem(
|
||||
messagesStore,
|
||||
messageWithSchemaVersion,
|
||||
messageWithSchemaVersion.id
|
||||
));
|
||||
|
||||
return cursor.continue();
|
||||
};
|
||||
|
||||
cursorRequest.onerror = event =>
|
||||
reject(event.target.error);
|
||||
});
|
||||
|
||||
// putItem :: IDBObjectStore -> Item -> Key -> Promise Item
|
||||
const putItem = (store, item, key) =>
|
||||
new Promise((resolve, reject) => {
|
||||
try {
|
||||
const request = store.put(item, key);
|
||||
request.onsuccess = event =>
|
||||
resolve(event.target.result);
|
||||
request.onerror = event =>
|
||||
reject(event.target.error);
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
|
|
@ -0,0 +1,166 @@
|
|||
const isFunction = require('lodash/isFunction');
|
||||
const isObject = require('lodash/isObject');
|
||||
|
||||
|
||||
// IMPORTANT: The migrations below are run on a database that may be very large
|
||||
// due to attachments being directly stored inside the database. Please avoid
|
||||
// any expensive operations, e.g. modifying all messages / attachments, etc., as
|
||||
// it may cause out-of-memory errors for users with long histories:
|
||||
// https://github.com/signalapp/Signal-Desktop/issues/2163
|
||||
const migrations = [
|
||||
{
|
||||
version: '12.0',
|
||||
migrate(transaction, next) {
|
||||
console.log('Migration 12');
|
||||
console.log('creating object stores');
|
||||
const messages = transaction.db.createObjectStore('messages');
|
||||
messages.createIndex('conversation', ['conversationId', 'received_at'], {
|
||||
unique: false,
|
||||
});
|
||||
messages.createIndex('receipt', 'sent_at', { unique: false });
|
||||
messages.createIndex('unread', ['conversationId', 'unread'], { unique: false });
|
||||
messages.createIndex('expires_at', 'expires_at', { unique: false });
|
||||
|
||||
const conversations = transaction.db.createObjectStore('conversations');
|
||||
conversations.createIndex('inbox', 'active_at', { unique: false });
|
||||
conversations.createIndex('group', 'members', {
|
||||
unique: false,
|
||||
multiEntry: true,
|
||||
});
|
||||
conversations.createIndex('type', 'type', {
|
||||
unique: false,
|
||||
});
|
||||
conversations.createIndex('search', 'tokens', {
|
||||
unique: false,
|
||||
multiEntry: true,
|
||||
});
|
||||
|
||||
transaction.db.createObjectStore('groups');
|
||||
|
||||
transaction.db.createObjectStore('sessions');
|
||||
transaction.db.createObjectStore('identityKeys');
|
||||
transaction.db.createObjectStore('preKeys');
|
||||
transaction.db.createObjectStore('signedPreKeys');
|
||||
transaction.db.createObjectStore('items');
|
||||
|
||||
console.log('creating debug log');
|
||||
transaction.db.createObjectStore('debug');
|
||||
|
||||
next();
|
||||
},
|
||||
},
|
||||
{
|
||||
version: '13.0',
|
||||
migrate(transaction, next) {
|
||||
console.log('Migration 13');
|
||||
console.log('Adding fields to identity keys');
|
||||
const identityKeys = transaction.objectStore('identityKeys');
|
||||
const request = identityKeys.openCursor();
|
||||
const promises = [];
|
||||
request.onsuccess = (event) => {
|
||||
const cursor = event.target.result;
|
||||
if (cursor) {
|
||||
const attributes = cursor.value;
|
||||
attributes.timestamp = 0;
|
||||
attributes.firstUse = false;
|
||||
attributes.nonblockingApproval = false;
|
||||
attributes.verified = 0;
|
||||
promises.push(new Promise(((resolve, reject) => {
|
||||
const putRequest = identityKeys.put(attributes, attributes.id);
|
||||
putRequest.onsuccess = resolve;
|
||||
putRequest.onerror = (e) => {
|
||||
console.log(e);
|
||||
reject(e);
|
||||
};
|
||||
})));
|
||||
cursor.continue();
|
||||
} else {
|
||||
// no more results
|
||||
// eslint-disable-next-line more/no-then
|
||||
Promise.all(promises).then(() => {
|
||||
next();
|
||||
});
|
||||
}
|
||||
};
|
||||
request.onerror = (event) => {
|
||||
console.log(event);
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
version: '14.0',
|
||||
migrate(transaction, next) {
|
||||
console.log('Migration 14');
|
||||
console.log('Adding unprocessed message store');
|
||||
const unprocessed = transaction.db.createObjectStore('unprocessed');
|
||||
unprocessed.createIndex('received', 'timestamp', { unique: false });
|
||||
next();
|
||||
},
|
||||
},
|
||||
{
|
||||
version: '15.0',
|
||||
migrate(transaction, next) {
|
||||
console.log('Migration 15');
|
||||
console.log('Adding messages index for de-duplication');
|
||||
const messages = transaction.objectStore('messages');
|
||||
messages.createIndex('unique', ['source', 'sourceDevice', 'sent_at'], {
|
||||
unique: true,
|
||||
});
|
||||
next();
|
||||
},
|
||||
},
|
||||
{
|
||||
version: '16.0',
|
||||
migrate(transaction, next) {
|
||||
console.log('Migration 16');
|
||||
console.log('Dropping log table, since we now log to disk');
|
||||
transaction.db.deleteObjectStore('debug');
|
||||
next();
|
||||
},
|
||||
},
|
||||
{
|
||||
version: 17,
|
||||
async migrate(transaction, next) {
|
||||
console.log('Migration 17');
|
||||
console.log('Start migration to database version 17');
|
||||
|
||||
const start = Date.now();
|
||||
|
||||
const messagesStore = transaction.objectStore('messages');
|
||||
console.log('Create index from attachment schema version to attachment');
|
||||
messagesStore.createIndex('schemaVersion', 'schemaVersion', { unique: false });
|
||||
|
||||
const duration = Date.now() - start;
|
||||
|
||||
console.log(
|
||||
'Complete migration to database version 17.',
|
||||
`Duration: ${duration}ms`
|
||||
);
|
||||
next();
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const database = {
|
||||
id: 'signal',
|
||||
nolog: true,
|
||||
migrations,
|
||||
};
|
||||
|
||||
exports.run = ({ Backbone } = {}) => {
|
||||
if (!isObject(Backbone) || !isObject(Backbone.Collection) ||
|
||||
!isFunction(Backbone.Collection.extend)) {
|
||||
throw new TypeError('"Backbone" is required');
|
||||
}
|
||||
|
||||
const migrationCollection = new (Backbone.Collection.extend({
|
||||
database,
|
||||
storeName: 'conversations',
|
||||
}))();
|
||||
|
||||
return new Promise((resolve) => {
|
||||
// NOTE: This `then` refers to a jQuery `Deferred`:
|
||||
// eslint-disable-next-line more/no-then
|
||||
migrationCollection.fetch().then(() => resolve());
|
||||
});
|
||||
};
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
exports.migrations = [
|
||||
{
|
||||
version: 18,
|
||||
async migrate(transaction, next) {
|
||||
console.log('Migration 18');
|
||||
next();
|
||||
},
|
||||
},
|
||||
];
|
||||
|
|
@ -127,12 +127,16 @@
|
|||
window.Signal = {};
|
||||
window.Signal.Backup = require('./js/modules/backup');
|
||||
window.Signal.Crypto = require('./js/modules/crypto');
|
||||
window.Signal.Database = {};
|
||||
window.Signal.Database.Migrations0DatabaseWithAttachmentData =
|
||||
require('./js/modules/migrations/migrations_0_database_with_attachment_data');
|
||||
window.Signal.Database.Migrations1DatabaseWithoutAttachmentData =
|
||||
require('./js/modules/migrations/migrations_1_database_without_attachment_data');
|
||||
window.Signal.Logs = require('./js/modules/logs');
|
||||
window.Signal.Migrations = {};
|
||||
window.Signal.Migrations.loadAttachmentData = Attachment.loadData(readAttachmentData);
|
||||
window.Signal.Migrations.deleteAttachmentData = Attachment.deleteData(deleteAttachmentData);
|
||||
window.Signal.Migrations.upgradeMessageSchema = upgradeMessageSchema;
|
||||
window.Signal.Migrations.V17 = require('./js/modules/migrations/17');
|
||||
window.Signal.OS = require('./js/modules/os');
|
||||
window.Signal.Types = {};
|
||||
window.Signal.Types.Attachment = Attachment;
|
||||
|
|
|
|||
Loading…
Reference in New Issue