jonny
/
Signal-Desktop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Generate thumbnails for new video attachments, video quotes

This commit is contained in:
Scott Nonnenberg 2018-04-25 18:32:46 -07:00
parent 0e99ca61a2
commit ac0b50d20f
No known key found for this signature in database
GPG Key ID: 5F82280C35134661
5 changed files with 222 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
background.html
View File

@ -237,7 +237,12 @@
</div>
</script>
<script type='text/x-tmpl-mustache' id='attachment-preview'>
<img src='{{ source }}' class='preview' />
<div class='image-container'>
<img src='{{ source }}' class='preview' />
<div class='outer'>
<div class='play icon'></div>
</div>
</div>
<a class='x close' alt='remove attachment' href='#'></a>
</script>
<script type='text/x-tmpl-mustache' id='file-view'>

83
js/models/conversations.js
View File

@ -629,7 +629,11 @@
const attachmentWithData = await loadAttachmentData(attachment);
const { data, contentType } = attachmentWithData;
const objectUrl = this.makeObjectUrl(data, contentType);
const thumbnail = await Whisper.FileInputView.makeThumbnail(128, objectUrl);
const thumbnail = Signal.Util.GoogleChrome.isImageTypeSupported(contentType)
? await Whisper.FileInputView.makeThumbnail(128, objectUrl)
: await Whisper.FileInputView.makeVideoThumbnail(128, objectUrl);
URL.revokeObjectURL(objectUrl);
const arrayBuffer = await this.blobToArrayBuffer(thumbnail);
@ -654,7 +658,8 @@
attachments: await Promise.all((attachments || []).map(async (attachment) => {
const { contentType } = attachment;
const willMakeThumbnail =
Signal.Util.GoogleChrome.isImageTypeSupported(contentType);
Signal.Util.GoogleChrome.isImageTypeSupported(contentType) ||
Signal.Util.GoogleChrome.isVideoTypeSupported(contentType);
return {
contentType,
@ -1154,9 +1159,14 @@
const { attachments, id, author } = quote;
const first = attachments[0];
if (!first || first.thumbnail) {
return true;
}
// Maybe in the future we could try to pull the thumbnail from a video ourselves,
// but for now we will rely on incoming thumbnails only.
if (!Signal.Util.GoogleChrome.isImageTypeSupported(first.contentType)) {
if (!Signal.Util.GoogleChrome.isImageTypeSupported(first.contentType) &&
!Signal.Util.GoogleChrome.isVideoTypeSupported(first.contentType)) {
return false;
}
@ -1194,9 +1204,12 @@
const { attachments } = quote;
const first = attachments[0];
// Maybe in the future we could try to pull thumbnails video ourselves,
// but for now we will rely on incoming thumbnails only.
if (!first || !Signal.Util.GoogleChrome.isImageTypeSupported(first.contentType)) {
if (!first || first.thumbnail) {
return;
}
if (!Signal.Util.GoogleChrome.isImageTypeSupported(first.contentType) &&
!Signal.Util.GoogleChrome.isVideoTypeSupported(first.contentType)) {
return;
}
@ -1230,18 +1243,26 @@
if (!thumbnail) {
return false;
}
const thumbnailWithData = await loadAttachmentData(thumbnail);
thumbnailWithData.objectUrl = this.makeObjectUrl(
thumbnailWithData.data,
thumbnailWithData.contentType
);
try {
const thumbnailWithData = await loadAttachmentData(thumbnail);
thumbnailWithData.objectUrl = this.makeObjectUrl(
thumbnailWithData.data,
thumbnailWithData.contentType
);
// If we update this data in place, there's the risk that this data could be
// saved back to the database
// eslint-disable-next-line no-param-reassign
message.quoteThumbnail = thumbnailWithData;
// If we update this data in place, there's the risk that this data could be
// saved back to the database
// eslint-disable-next-line no-param-reassign
message.quoteThumbnail = thumbnailWithData;
return true;
return true;
} catch (error) {
console.log(
'loadQuoteThumbnail: had trouble loading thumbnail data from disk',
error && error.stack ? error.stack : error
);
return false;
}
},
async processQuotes(messages) {
const lookup = this.makeMessagesLookup(messages);
@ -1259,21 +1280,16 @@
return;
}
// 1. Check to see if we've already loaded the target message into memory
// 1. Load provided thumbnail
const gotThumbnail = await this.loadQuoteThumbnail(message, quote);
// 2. Check to see if we've already loaded the target message into memory
const { author, id } = quote;
const key = this.makeKey(author, id);
const quotedMessage = lookup[key];
if (quotedMessage) {
// eslint-disable-next-line no-param-reassign
await this.loadQuotedMessage(message, quotedMessage);
// Note: in the future when we generate our own thumbnail we won't need to rely
// on incoming thumbnail if we have our local message in hand.
if (!message.quotedMessage.imageUrl) {
await this.loadQuoteThumbnail(message, quote);
}
this.forceRender(message);
return;
}
@ -1286,28 +1302,15 @@
}
// We've don't want to go to the database or load thumbnails a second time.
if (message.quoteIsProcessed) {
if (message.quoteIsProcessed || gotThumbnail) {
return;
}
// eslint-disable-next-line no-param-reassign
message.quoteIsProcessed = true;
// 2. Go to the database for the real referenced attachment
// 3. As a last resort, go to the database to generate a thumbnail on-demand
const loaded = await this.loadQuotedMessageFromDatabase(message, id);
if (loaded) {
// Note: in the future when we generate our own thumbnail we won't need to rely
// on incoming thumbnail if we have our local message in hand.
if (!message.quotedMessageFromDatabase.imageUrl) {
await this.loadQuoteThumbnail(message, quote);
}
this.forceRender(message);
return;
}
// 3. Finally, use the provided thumbnail
const gotThumbnail = await this.loadQuoteThumbnail(message, quote);
if (gotThumbnail) {
this.forceRender(message);
}
});

4
js/models/messages.js
View File

@ -181,10 +181,6 @@
URL.revokeObjectURL(this.quoteThumbnail.objectUrl);
this.quoteThumbnail = null;
}
if (this.quotedMessageFromDatabase) {
this.quotedMessageFromDatabase.unload();
this.quotedMessageFromDatabase = null;
}
if (this.quotedMessage) {
this.quotedMessage = null;
}

208
js/views/file_input_view.js
View File

@ -3,6 +3,8 @@
/* global i18n: false */
/* global loadImage: false */
/* global Backbone: false */
/* global _: false */
/* global Signal: false */
// eslint-disable-next-line func-names
(function () {
@ -61,6 +63,63 @@
}));
}
function makeVideoScreenshot(objectUrl) {
return new Promise(((resolve, reject) => {
const video = document.createElement('video');
function capture() {
const canvas = document.createElement('canvas');
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
canvas.getContext('2d').drawImage(video, 0, 0, canvas.width, canvas.height);
const image = window.dataURLToBlobSync(canvas.toDataURL('image/png'));
video.removeEventListener('canplay', capture);
resolve(image);
}
video.addEventListener('canplay', capture);
video.addEventListener('error', (error) => {
console.log(
'makeVideoThumbnail error',
error && error.stack ? error.stack : error
);
reject(error);
});
video.src = objectUrl;
}));
}
function makeObjectUrl(data, contentType) {
const blob = new Blob([data], {
type: contentType,
});
return URL.createObjectURL(blob);
}
function blobToArrayBuffer(blob) {
return new Promise((resolve, reject) => {
const fileReader = new FileReader();
fileReader.onload = e => resolve(e.target.result);
fileReader.onerror = reject;
fileReader.onabort = reject;
fileReader.readAsArrayBuffer(blob);
});
}
async function makeVideoThumbnail(size, videoObjectUrl) {
const blob = await makeVideoScreenshot(videoObjectUrl);
const arrayBuffer = await blobToArrayBuffer(blob);
const screenshotObjectUrl = makeObjectUrl(arrayBuffer, 'image/png');
return makeThumbnail(size, screenshotObjectUrl);
}
Whisper.FileInputView = Backbone.View.extend({
tagName: 'span',
className: 'file-input',
@ -103,10 +162,18 @@
}
},
addThumb(src) {
addThumb(src, options = {}) {
_.defaults(options, { addPlayIcon: false });
this.$('.avatar').hide();
this.thumb.src = src;
this.$('.attachment-previews').append(this.thumb.render().el);
if (options.addPlayIcon) {
this.$el.addClass('video-attachment');
} else {
this.$el.removeClass('video-attachment');
}
this.thumb.$('img')[0].onload = () => {
this.$el.trigger('force-resize');
};
@ -160,69 +227,81 @@
}));
},
previewImages() {
async previewImages() {
this.clearForm();
const file = this.file || this.$input.prop('files')[0];
if (!file) { return; }
let type = file.type.split('/')[0];
if (file.type === 'image/tiff') {
type = 'file';
if (!file) {
return;
}
switch (type) {
case 'audio': this.addThumb('images/audio.svg'); break;
case 'video': this.addThumb('images/video.svg'); break;
const contentType = file.type;
const renderVideoPreview = async () => {
// we use the variable on this here to ensure cleanup if we're interrupted
this.previewObjectUrl = URL.createObjectURL(file);
const thumbnail = await makeVideoScreenshot(this.previewObjectUrl);
URL.revokeObjectURL(this.previewObjectUrl);
const arrayBuffer = await blobToArrayBuffer(thumbnail);
this.previewObjectUrl = makeObjectUrl(arrayBuffer, 'image/png');
this.addThumb(this.previewObjectUrl, { addPlayIcon: true });
};
const renderImagePreview = async () => {
if (!MIME.isJPEG(file.type)) {
this.previewObjectUrl = URL.createObjectURL(file);
this.addThumb(this.previewObjectUrl);
return;
}
const dataUrl = await window.autoOrientImage(file);
this.addThumb(dataUrl);
};
if (Signal.Util.GoogleChrome.isImageTypeSupported(contentType)) {
renderImagePreview();
} else if (Signal.Util.GoogleChrome.isVideoTypeSupported(contentType)) {
renderVideoPreview();
} else if (MIME.isAudio(contentType)) {
this.addThumb('images/audio.svg');
} else {
this.addThumb('images/file.svg');
}
const blob = await this.autoScale(file);
let limitKb = 1000000;
const blobType = file.type === 'image/gif'
? 'gif'
: contentType.split('/')[0];
switch (blobType) {
case 'image':
if (!MIME.isJPEG(file.type)) {
this.previewObjectUrl = URL.createObjectURL(file);
this.addThumb(this.previewObjectUrl);
break;
}
// NOTE: Temporarily allow `then` until we convert the entire file
// to `async` / `await`:
// eslint-disable-next-line more/no-then
window.autoOrientImage(file)
.then(dataURL => this.addThumb(dataURL));
break;
limitKb = 6000; break;
case 'gif':
limitKb = 25000; break;
case 'audio':
limitKb = 100000; break;
case 'video':
limitKb = 100000; break;
default:
this.addThumb('images/file.svg'); break;
limitKb = 100000; break;
}
if ((blob.size / 1024).toFixed(4) >= limitKb) {
const units = ['kB', 'MB', 'GB'];
let u = -1;
let limit = limitKb * 1000;
do {
limit /= 1000;
u += 1;
} while (limit >= 1000 && u < units.length - 1);
const toast = new Whisper.FileSizeToast({
model: { limit, units: units[u] },
});
toast.$el.insertAfter(this.$el);
toast.render();
this.deleteFiles();
}
// NOTE: Temporarily allow `then` until we convert the entire file
// to `async` / `await`:
// eslint-disable-next-line more/no-then
this.autoScale(file).then((blob) => {
let limitKb = 1000000;
const blobType = file.type === 'image/gif' ? 'gif' : type;
switch (blobType) {
case 'image':
limitKb = 6000; break;
case 'gif':
limitKb = 25000; break;
case 'audio':
limitKb = 100000; break;
case 'video':
limitKb = 100000; break;
default:
limitKb = 100000; break;
}
if ((blob.size / 1024).toFixed(4) >= limitKb) {
const units = ['kB', 'MB', 'GB'];
let u = -1;
let limit = limitKb * 1000;
do {
limit /= 1000;
u += 1;
} while (limit >= 1000 && u < units.length - 1);
const toast = new Whisper.FileSizeToast({
model: { limit, units: units[u] },
});
toast.$el.insertAfter(this.$el);
toast.render();
this.deleteFiles();
}
});
},
hasFiles() {
@ -262,7 +341,7 @@
.then(setFlags(attachmentFlags));
},
getThumbnail() {
async getThumbnail() {
// Scale and crop an image to 256px square
const size = 256;
const file = this.file || this.$input.prop('files')[0];
@ -275,11 +354,10 @@
const objectUrl = URL.createObjectURL(file);
// eslint-disable-next-line more/no-then
return makeThumbnail(size, objectUrl).then((arrayBuffer) => {
URL.revokeObjectURL(objectUrl);
return this.readFile(arrayBuffer);
});
const arrayBuffer = await makeThumbnail(size, objectUrl);
URL.revokeObjectURL(objectUrl);
return this.readFile(arrayBuffer);
},
// File -> Promise Attachment
@ -370,4 +448,6 @@
});
Whisper.FileInputView.makeThumbnail = makeThumbnail;
Whisper.FileInputView.makeVideoThumbnail = makeVideoThumbnail;
Whisper.FileInputView.makeVideoScreenshot = makeVideoScreenshot;
}());

29
stylesheets/_conversation.scss
View File

@ -927,6 +927,33 @@ span.status {
form.send {
background: #ffffff;
&.video-attachment {
.image-container {
position: relative;
}
.outer {
position: absolute;
top: 0;
right: 0;
bottom: 0;
left: 0;
display: flex;
text-align: center;
display: flex;
align-items: center;
justify-content: center;
.play.icon {
height: 30px;
width: 30px;
@include color-svg('../images/play.svg', white);
}
}
}
}
input, textarea {
@ -935,8 +962,10 @@ span.status {
.attachment-previews {
padding: 0 36px;
.attachment-preview {
padding: 13px 10px 0;
}
img {
border: 2px solid #ddd;