// Copyright (C) 2020 The Android Open Source Project // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. import {defer} from './deferred'; import {Time} from './time'; export const BUCKET_NAME = 'perfetto-ui-data'; export const MIME_JSON = 'application/json; charset=utf-8'; export const MIME_BINARY = 'application/octet-stream'; export interface GcsUploaderArgs { /** * The mime-type to use for the upload. If undefined uses * application/octet-stream. */ mimeType?: string; /** * The name to use for the uploaded file. By default it uses a hash of * the passed data/blob and uses content-addressing. */ fileName?: string; /** An optional callback that is invoked upon upload progress (or failure) */ onProgress?: (uploader: GcsUploader) => void; } /** * A utility class to handle uploads of possibly large files to * Google Cloud Storage. * It returns immediately if the file exists already */ export class GcsUploader { state: 'UPLOADING' | 'UPLOADED' | 'ERROR' = 'UPLOADING'; error = ''; totalSize = 0; uploadedSize = 0; uploadedUrl = ''; uploadedFileName = ''; private args: GcsUploaderArgs; private onProgress: (_: GcsUploader) => void; private req: XMLHttpRequest; private donePromise = defer(); private startTime = performance.now(); constructor(data: Blob | ArrayBuffer | string, args: GcsUploaderArgs) { this.args = args; this.onProgress = args.onProgress ?? ((_: GcsUploader) => {}); this.req = new XMLHttpRequest(); this.start(data); } async start(data: Blob | ArrayBuffer | string) { let fname = this.args.fileName; if (fname === undefined) { // If the file name is unspecified, hash the contents. if (data instanceof Blob) { fname = await hashFileStreaming(data); } else { fname = await sha1(data); } } this.uploadedFileName = fname; this.uploadedUrl = `https://storage.googleapis.com/${BUCKET_NAME}/${fname}`; // Check if the file has been uploaded already. If so, skip. const res = await fetch( `https://www.googleapis.com/storage/v1/b/${BUCKET_NAME}/o/${fname}`, ); if (res.status === 200) { console.log( `Skipping upload of ${this.uploadedUrl} because it exists already`, ); this.state = 'UPLOADED'; this.donePromise.resolve(); return; } const reqUrl = 'https://www.googleapis.com/upload/storage/v1/b/' + `${BUCKET_NAME}/o?uploadType=media` + `&name=${fname}&predefinedAcl=publicRead`; this.req.onabort = (e: ProgressEvent) => this.onRpcEvent(e); this.req.onerror = (e: ProgressEvent) => this.onRpcEvent(e); this.req.upload.onprogress = (e: ProgressEvent) => this.onRpcEvent(e); this.req.onloadend = (e: ProgressEvent) => this.onRpcEvent(e); this.req.open('POST', reqUrl, /* async= */ true); const mimeType = this.args.mimeType ?? MIME_BINARY; this.req.setRequestHeader('Content-Type', mimeType); this.req.send(data); } waitForCompletion(): Promise { return this.donePromise; } abort() { if (this.state === 'UPLOADING') { this.req.abort(); } } getEtaString() { let str = `${Math.ceil((100 * this.uploadedSize) / this.totalSize)}%`; str += ` (${(this.uploadedSize / 1e6).toFixed(2)} MB)`; const elapsed = (performance.now() - this.startTime) / 1000; const rate = this.uploadedSize / elapsed; const etaSecs = Math.round((this.totalSize - this.uploadedSize) / rate); str += ' - ETA: ' + Time.toTimecode(Time.fromSeconds(etaSecs)).dhhmmss; return str; } private onRpcEvent(e: ProgressEvent) { let done = false; switch (e.type) { case 'progress': this.uploadedSize = e.loaded; this.totalSize = e.total; break; case 'abort': this.state = 'ERROR'; this.error = 'Upload aborted'; break; case 'error': this.state = 'ERROR'; this.error = `${this.req.status} - ${this.req.statusText}`; break; case 'loadend': done = true; if (this.req.status === 200) { this.state = 'UPLOADED'; } else if (this.state === 'UPLOADING') { this.state = 'ERROR'; this.error = `${this.req.status} - ${this.req.statusText}`; } break; default: return; } this.onProgress(this); if (done) { this.donePromise.resolve(); } } } /** * Computes the SHA-1 of a string or ArrayBuffer(View) * @param data a string or ArrayBuffer to hash. */ async function sha1(data: string | ArrayBuffer): Promise { let buffer: ArrayBuffer; if (typeof data === 'string') { buffer = new TextEncoder().encode(data); } else { buffer = data; } const digest = await crypto.subtle.digest('SHA-1', buffer); return digestToHex(digest); } /** * Converts a hash for the given file in streaming mode, without loading the * whole file into memory. The result is "a" SHA-1 but is not the same of * `shasum -a 1 file`. The reason for this is that the crypto APIs support * only one-shot digest computation and lack the usual update() + digest() * chunked API. So we end up computing a SHA-1 of the concatenation of the * SHA-1 of each chunk. * Speed: ~800 MB/s on a M2 Macbook Air 2023. * @param file The file to hash. * @returns A hex-encoded string containing the hash of the file. */ async function hashFileStreaming(file: Blob): Promise { const CHUNK_SIZE = 32 * 1024 * 1024; // 32MB const totalChunks = Math.ceil(file.size / CHUNK_SIZE); let chunkDigests = ''; for (let i = 0; i < totalChunks; i++) { const start = i * CHUNK_SIZE; const end = Math.min(start + CHUNK_SIZE, file.size); const chunk = await file.slice(start, end).arrayBuffer(); const digest = await crypto.subtle.digest('SHA-1', chunk); chunkDigests += digestToHex(digest); } return sha1(chunkDigests); } /** * Converts the return value of crypto.digest() to a hex string. * @param digest an array of bytes containing the digest * @returns hex-encoded string of the digest. */ function digestToHex(digest: ArrayBuffer): string { return Array.from(new Uint8Array(digest)) .map((x) => x.toString(16).padStart(2, '0')) .join(''); }