Use consistent User-Agent when uploading debug logs

This commit is contained in:
Evan Hahn 2021-01-06 11:23:15 -06:00 committed by Scott Nonnenberg
parent d0146a1613
commit 8ccf402497
9 changed files with 331 additions and 138 deletions

View File

@ -1,4 +1,4 @@
// Copyright 2017-2020 Signal Messenger, LLC
// Copyright 2017-2021 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
/* eslint-env node */
@ -8,7 +8,7 @@
const electron = require('electron');
const _ = require('lodash');
const debuglogs = require('./modules/debuglogs');
const { uploadDebugLogs } = require('../ts/logging/debuglogs');
const Privacy = require('./modules/privacy');
const { createBatcher } = require('../ts/util/batcher');
@ -97,7 +97,7 @@ function fetch() {
});
}
const publish = debuglogs.upload;
const publish = uploadDebugLogs;
// A modern logging interface for the browser

View File

@ -1,75 +0,0 @@
// Copyright 2018-2020 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
/* eslint-env node */
/* global window */
const FormData = require('form-data');
const got = require('got');
const pify = require('pify');
const { gzip } = require('zlib');
const BASE_URL = 'https://debuglogs.org';
const VERSION = window.getVersion();
const USER_AGENT = `Signal Desktop ${VERSION}`;
// Workaround: Submitting `FormData` using native `FormData::submit` procedure
// as integration with `got` results in S3 error saying we havent set the
// `Content-Length` header:
// https://github.com/sindresorhus/got/pull/466
const submitFormData = (form, url) =>
new Promise((resolve, reject) => {
form.submit(url, (error, response) => {
if (error) {
return reject(error);
}
const { statusCode } = response;
if (statusCode !== 204) {
return reject(
new Error(`Failed to upload to S3, got status ${statusCode}`)
);
}
return resolve();
});
});
// upload :: String -> Promise URL
exports.upload = async content => {
const signedForm = await got.get(BASE_URL, {
json: true,
headers: {
'user-agent': USER_AGENT,
},
});
if (!signedForm.body) {
throw new Error('Failed to retrieve token');
}
const { fields, url } = signedForm.body;
const form = new FormData();
// The API expects `key` to be the first field:
form.append('key', fields.key);
Object.entries(fields)
.filter(([key]) => key !== 'key')
.forEach(([key, value]) => {
form.append(key, value);
});
const contentBuffer = await pify(gzip)(Buffer.from(content, 'utf8'));
const contentType = 'application/gzip';
form.append('Content-Type', contentType);
form.append('file', contentBuffer, {
contentType,
filename: `signal-desktop-debug-log-${VERSION}.txt.gz`,
});
window.log.info('Debug log upload starting...');
// WORKAROUND: See comment on `submitFormData`:
// await got.post(url, { body: form });
await submitFormData(form, url);
window.log.info('Debug log upload complete.');
return `${BASE_URL}/${fields.key}`;
};

View File

@ -1,4 +1,4 @@
// Copyright 2015-2020 Signal Messenger, LLC
// Copyright 2015-2021 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
/* global i18n: false */
@ -73,7 +73,10 @@
this.$('.result').addClass('loading');
try {
const publishedLogURL = await window.log.publish(text);
const publishedLogURL = await window.log.publish(
text,
window.getVersion()
);
const view = new Whisper.DebugLogLinkView({
url: publishedLogURL,
el: this.$('.result'),

View File

@ -84,12 +84,12 @@
"fast-glob": "3.2.1",
"filesize": "3.6.1",
"firstline": "1.2.1",
"form-data": "2.3.2",
"form-data": "3.0.0",
"fs-extra": "5.0.0",
"fuse.js": "3.4.4",
"glob": "7.1.6",
"google-libphonenumber": "3.2.6",
"got": "8.2.0",
"got": "8.3.2",
"history": "4.9.0",
"intl-tel-input": "12.1.15",
"jquery": "3.5.0",

View File

@ -1,3 +1,6 @@
// Copyright 2021 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
/* eslint-disable no-console */
// To replicate logic we have on the client side

78
ts/logging/debuglogs.ts Normal file
View File

@ -0,0 +1,78 @@
// Copyright 2018-2021 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import FormData from 'form-data';
import { gzip } from 'zlib';
import pify from 'pify';
import got from 'got';
import { getUserAgent } from '../util/getUserAgent';
const BASE_URL = 'https://debuglogs.org';
const isObject = (value: unknown): value is Record<string, unknown> =>
typeof value === 'object' && !Array.isArray(value) && Boolean(value);
const parseTokenBody = (
body: unknown
): { fields: Record<string, unknown>; url: string } => {
if (!isObject(body)) {
throw new Error('Token body is not an object');
}
const { fields, url } = body as Record<string, unknown>;
if (!isObject(fields)) {
throw new Error('Token body\'s "fields" key is not an object');
}
if (typeof url !== 'string') {
throw new Error('Token body\'s "url" key is not a string');
}
let parsedUrl: URL;
try {
parsedUrl = new URL(url);
} catch (err) {
throw new Error("Token body's URL was not a valid URL");
}
if (parsedUrl.protocol !== 'https:') {
throw new Error("Token body's URL was not HTTPS");
}
return { fields, url };
};
export const uploadDebugLogs = async (
content: string,
appVersion: string
): Promise<string> => {
const headers = { 'User-Agent': getUserAgent(appVersion) };
const signedForm = await got.get(BASE_URL, { json: true, headers });
const { fields, url } = parseTokenBody(signedForm.body);
const form = new FormData();
// The API expects `key` to be the first field:
form.append('key', fields.key);
Object.entries(fields)
.filter(([key]) => key !== 'key')
.forEach(([key, value]) => {
form.append(key, value);
});
const contentBuffer = await pify(gzip)(Buffer.from(content, 'utf8'));
const contentType = 'application/gzip';
form.append('Content-Type', contentType);
form.append('file', contentBuffer, {
contentType,
filename: `signal-desktop-debug-log-${appVersion}.txt.gz`,
});
window.log.info('Debug log upload starting...');
const { statusCode } = await got.post(url, { headers, body: form });
if (statusCode !== 204) {
throw new Error(`Failed to upload to S3, got status ${statusCode}`);
}
window.log.info('Debug log upload complete.');
return `${BASE_URL}/${fields.key}`;
};

View File

@ -0,0 +1,127 @@
// Copyright 2021 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { assert } from 'chai';
import * as sinon from 'sinon';
import got from 'got';
import FormData from 'form-data';
import * as util from 'util';
import * as zlib from 'zlib';
import { uploadDebugLogs } from '../../logging/debuglogs';
const gzip: (_: zlib.InputType) => Promise<Buffer> = util.promisify(zlib.gzip);
describe('uploadDebugLogs', () => {
beforeEach(function beforeEach() {
this.sandbox = sinon.createSandbox();
this.sandbox.stub(process, 'platform').get(() => 'linux');
this.fakeGet = this.sandbox.stub(got, 'get');
this.fakePost = this.sandbox.stub(got, 'post');
this.fakeGet.resolves({
body: {
fields: {
foo: 'bar',
key: 'abc123',
},
url: 'https://example.com/fake-upload',
},
});
this.fakePost.resolves({ statusCode: 204 });
});
afterEach(function afterEach() {
this.sandbox.restore();
});
it('makes a request to get the S3 bucket, then uploads it there', async function test() {
assert.strictEqual(
await uploadDebugLogs('hello world', '1.2.3'),
'https://debuglogs.org/abc123'
);
sinon.assert.calledOnce(this.fakeGet);
sinon.assert.calledWith(this.fakeGet, 'https://debuglogs.org', {
json: true,
headers: { 'User-Agent': 'Signal-Desktop/1.2.3 Linux' },
});
const compressedContent = await gzip('hello world');
sinon.assert.calledOnce(this.fakePost);
sinon.assert.calledWith(this.fakePost, 'https://example.com/fake-upload', {
headers: { 'User-Agent': 'Signal-Desktop/1.2.3 Linux' },
body: sinon.match((value: unknown) => {
if (!(value instanceof FormData)) {
return false;
}
// `FormData` doesn't offer high-level APIs for fetching data, so we do this.
const buffer = value.getBuffer();
assert(
buffer.includes(compressedContent),
'gzipped content was not in body'
);
return true;
}, 'FormData'),
});
});
it("rejects if we can't get a token", async function test() {
this.fakeGet.rejects(new Error('HTTP request failure'));
let err: unknown;
try {
await uploadDebugLogs('hello world', '1.2.3');
} catch (e) {
err = e;
}
assert.instanceOf(err, Error);
});
it('rejects with an invalid token body', async function test() {
const bodies = [
null,
{},
{ fields: {} },
{ fields: { nokey: 'ok' } },
{ fields: { key: '123' } },
{ fields: { key: '123' }, url: { not: 'a string' } },
{ fields: { key: '123' }, url: 'http://notsecure.example.com' },
{ fields: { key: '123' }, url: 'not a valid URL' },
];
// We want to make sure these run serially, so we can't use `Promise.all`. They're
// async, so we can't use `forEach`. `for ... of` is a reasonable option here.
// eslint-disable-next-line no-restricted-syntax
for (const body of bodies) {
this.fakeGet.resolves({ body });
let err: unknown;
try {
// Again, these should be run serially.
// eslint-disable-next-line no-await-in-loop
await uploadDebugLogs('hello world', '1.2.3');
} catch (e) {
err = e;
}
assert.instanceOf(err, Error);
}
});
it("rejects if the upload doesn't return a 204", async function test() {
this.fakePost.resolves({ statusCode: 400 });
let err: unknown;
try {
await uploadDebugLogs('hello world', '1.2.3');
} catch (e) {
err = e;
}
assert.instanceOf(err, Error);
});
});

View File

@ -212,38 +212,6 @@
"reasonCategory": "usageTrusted",
"updated": "2020-03-25T15:45:04.024Z"
},
{
"rule": "jQuery-append(",
"path": "js/modules/debuglogs.js",
"line": " form.append('key', fields.key);",
"lineNumber": 53,
"reasonCategory": "falseMatch",
"updated": "2020-03-20T20:40:34.498Z"
},
{
"rule": "jQuery-append(",
"path": "js/modules/debuglogs.js",
"line": " form.append(key, value);",
"lineNumber": 57,
"reasonCategory": "falseMatch",
"updated": "2020-03-20T20:40:34.498Z"
},
{
"rule": "jQuery-append(",
"path": "js/modules/debuglogs.js",
"line": " form.append('Content-Type', contentType);",
"lineNumber": 62,
"reasonCategory": "falseMatch",
"updated": "2020-03-20T20:40:34.498Z"
},
{
"rule": "jQuery-append(",
"path": "js/modules/debuglogs.js",
"line": " form.append('file', contentBuffer, {",
"lineNumber": 63,
"reasonCategory": "falseMatch",
"updated": "2020-09-11T17:24:56.124Z"
},
{
"rule": "jQuery-load(",
"path": "js/modules/emojis.js",
@ -406,7 +374,7 @@
"rule": "jQuery-$(",
"path": "js/views/debug_log_view.js",
"line": " el: this.$('.result'),",
"lineNumber": 79,
"lineNumber": 82,
"reasonCategory": "usageTrusted",
"updated": "2020-05-01T17:11:39.527Z",
"reasonDetail": "Protected from arbitrary input"
@ -415,7 +383,7 @@
"rule": "jQuery-$(",
"path": "js/views/debug_log_view.js",
"line": " this.$('.loading').removeClass('loading');",
"lineNumber": 81,
"lineNumber": 84,
"reasonCategory": "usageTrusted",
"updated": "2020-05-01T17:11:39.527Z",
"reasonDetail": "Protected from arbitrary input"
@ -424,7 +392,7 @@
"rule": "jQuery-$(",
"path": "js/views/debug_log_view.js",
"line": " this.$('.link').focus().select();",
"lineNumber": 83,
"lineNumber": 86,
"reasonCategory": "usageTrusted",
"updated": "2020-11-18T03:39:29.033Z",
"reasonDetail": "Protected from arbitrary input"
@ -433,7 +401,7 @@
"rule": "jQuery-$(",
"path": "js/views/debug_log_view.js",
"line": " this.$('.loading').removeClass('loading');",
"lineNumber": 89,
"lineNumber": 92,
"reasonCategory": "usageTrusted",
"updated": "2020-11-18T03:39:29.033Z",
"reasonDetail": "Protected from arbitrary input"
@ -442,7 +410,7 @@
"rule": "jQuery-$(",
"path": "js/views/debug_log_view.js",
"line": " this.$('.result').text(i18n('debugLogError'));",
"lineNumber": 90,
"lineNumber": 93,
"reasonCategory": "usageTrusted",
"updated": "2020-11-18T03:39:29.033Z",
"reasonDetail": "Protected from arbitrary input"
@ -13570,6 +13538,30 @@
"reasonCategory": "falseMatch",
"updated": "2019-07-19T17:16:02.404Z"
},
{
"rule": "jQuery-append(",
"path": "node_modules/request/node_modules/form-data/lib/form_data.js",
"line": " append(header);",
"lineNumber": 73,
"reasonCategory": "falseMatch",
"updated": "2020-12-17T18:08:07.752Z"
},
{
"rule": "jQuery-append(",
"path": "node_modules/request/node_modules/form-data/lib/form_data.js",
"line": " append(value);",
"lineNumber": 74,
"reasonCategory": "falseMatch",
"updated": "2020-12-17T18:08:07.752Z"
},
{
"rule": "jQuery-append(",
"path": "node_modules/request/node_modules/form-data/lib/form_data.js",
"line": " append(footer);",
"lineNumber": 75,
"reasonCategory": "falseMatch",
"updated": "2020-12-17T18:08:07.752Z"
},
{
"rule": "jQuery-append(",
"path": "node_modules/request/request.js",
@ -14891,6 +14883,70 @@
"updated": "2020-10-26T19:12:24.410Z",
"reasonDetail": "Only used to focus the element."
},
{
"rule": "jQuery-append(",
"path": "ts/logging/debuglogs.js",
"line": " form.append('key', fields.key);",
"lineNumber": 45,
"reasonCategory": "falseMatch",
"updated": "2020-12-17T18:08:07.752Z"
},
{
"rule": "jQuery-append(",
"path": "ts/logging/debuglogs.js",
"line": " form.append(key, value);",
"lineNumber": 49,
"reasonCategory": "falseMatch",
"updated": "2020-12-17T18:08:07.752Z"
},
{
"rule": "jQuery-append(",
"path": "ts/logging/debuglogs.js",
"line": " form.append('Content-Type', contentType);",
"lineNumber": 53,
"reasonCategory": "falseMatch",
"updated": "2020-12-17T18:08:07.752Z"
},
{
"rule": "jQuery-append(",
"path": "ts/logging/debuglogs.js",
"line": " form.append('file', contentBuffer, {",
"lineNumber": 54,
"reasonCategory": "falseMatch",
"updated": "2020-12-17T18:08:07.752Z"
},
{
"rule": "jQuery-append(",
"path": "ts/logging/debuglogs.ts",
"line": " form.append('key', fields.key);",
"lineNumber": 55,
"reasonCategory": "falseMatch",
"updated": "2020-12-17T18:08:07.752Z"
},
{
"rule": "jQuery-append(",
"path": "ts/logging/debuglogs.ts",
"line": " form.append(key, value);",
"lineNumber": 59,
"reasonCategory": "falseMatch",
"updated": "2020-12-17T18:08:07.752Z"
},
{
"rule": "jQuery-append(",
"path": "ts/logging/debuglogs.ts",
"line": " form.append('Content-Type', contentType);",
"lineNumber": 64,
"reasonCategory": "falseMatch",
"updated": "2020-12-17T18:08:07.752Z"
},
{
"rule": "jQuery-append(",
"path": "ts/logging/debuglogs.ts",
"line": " form.append('file', contentBuffer, {",
"lineNumber": 65,
"reasonCategory": "falseMatch",
"updated": "2020-12-17T18:08:07.752Z"
},
{
"rule": "React-createRef",
"path": "ts/quill/mentions/completion.js",
@ -15149,4 +15205,4 @@
"reasonCategory": "falseMatch",
"updated": "2020-09-08T23:07:22.682Z"
}
]
]

View File

@ -7702,16 +7702,7 @@ fork-ts-checker-webpack-plugin@1.5.0:
tapable "^1.0.0"
worker-rpc "^0.1.0"
form-data@2.3.2, form-data@~2.3.2:
version "2.3.2"
resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.2.tgz#4970498be604c20c005d4f5c23aecd21d6b49099"
integrity sha1-SXBJi+YEwgwAXU9cI67NIda0kJk=
dependencies:
asynckit "^0.4.0"
combined-stream "1.0.6"
mime-types "^2.1.12"
form-data@^3.0.0:
form-data@3.0.0, form-data@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.0.tgz#31b7e39c85f1355b7139ee0c647cf0de7f83c682"
integrity sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==
@ -7728,6 +7719,15 @@ form-data@~2.1.1:
combined-stream "^1.0.5"
mime-types "^2.1.12"
form-data@~2.3.2:
version "2.3.2"
resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.2.tgz#4970498be604c20c005d4f5c23aecd21d6b49099"
integrity sha1-SXBJi+YEwgwAXU9cI67NIda0kJk=
dependencies:
asynckit "^0.4.0"
combined-stream "1.0.6"
mime-types "^2.1.12"
format@^0.2.2:
version "0.2.2"
resolved "https://registry.yarnpkg.com/format/-/format-0.2.2.tgz#d6170107e9efdc4ed30c9dc39016df942b5cb58b"
@ -8332,10 +8332,10 @@ google-libphonenumber@3.2.6:
resolved "https://registry.yarnpkg.com/google-libphonenumber/-/google-libphonenumber-3.2.6.tgz#3d725b48ff44706b80246e77f95f2c2fdc6fd729"
integrity sha512-6QCQAaKJlSd/1dUqvdQf7zzfb3uiZHsG8yhCfOdCVRfMuPZ/VDIEB47y5SYwjPQJPs7ebfW5jj6PeobB9JJ4JA==
got@8.2.0:
version "8.2.0"
resolved "https://registry.yarnpkg.com/got/-/got-8.2.0.tgz#0d11a071d05046348a2f5c0a5fa047fb687fdfc6"
integrity sha512-giadqJpXIwjY+ZsuWys8p2yjZGhOHiU4hiJHjS/oeCxw1u8vANQz3zPlrxW2Zw/siCXsSMI3hvzWGcnFyujyAg==
got@8.3.2:
version "8.3.2"
resolved "https://registry.yarnpkg.com/got/-/got-8.3.2.tgz#1d23f64390e97f776cac52e5b936e5f514d2e937"
integrity sha512-qjUJ5U/hawxosMryILofZCkm3C84PLJS/0grRIpjAwu+Lkxxj5cxeCU25BG0/3mDSpXKTyZr8oh8wIgLaH0QCw==
dependencies:
"@sindresorhus/is" "^0.7.0"
cacheable-request "^2.1.1"
@ -8347,7 +8347,7 @@ got@8.2.0:
isurl "^1.0.0-alpha5"
lowercase-keys "^1.0.0"
mimic-response "^1.0.0"
p-cancelable "^0.3.0"
p-cancelable "^0.4.0"
p-timeout "^2.0.1"
pify "^3.0.0"
safe-buffer "^5.1.1"
@ -11995,9 +11995,10 @@ osenv@^0.1.4:
os-homedir "^1.0.0"
os-tmpdir "^1.0.0"
p-cancelable@^0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-0.3.0.tgz#b9e123800bcebb7ac13a479be195b507b98d30fa"
p-cancelable@^0.4.0:
version "0.4.1"
resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-0.4.1.tgz#35f363d67d52081c8d9585e37bcceb7e0bbcb2a0"
integrity sha512-HNa1A8LvB1kie7cERyy21VNeHb2CWJJYqyyC2o3klWFfMGlFmWv2Z7sFgZH8ZiaYL95ydToKTFVXgMV/Os0bBQ==
p-cancelable@^1.0.0:
version "1.1.0"