feat: deprecate composite checksum

Author: AllanZhengYP

packages/storage/__tests__/providers/s3/apis/internal/uploadData/multipartHandlers.test.ts

@@ -75,11 +75,7 @@ const getZeroDelayTimeout = () =>
 
 const mockCalculateContentCRC32Mock = () => {
 	mockCalculateContentCRC32.mockReset();
-	mockCalculateContentCRC32.mockResolvedValue({
-		checksumArrayBuffer: new ArrayBuffer(0),
-		checksum: 'mockChecksum',
-		seed: 0,
-	});
+	mockCalculateContentCRC32.mockResolvedValue('mockChecksum');
 };
 const mockCalculateContentCRC32Reset = () => {
 	mockCalculateContentCRC32.mockReset();
@@ -298,17 +294,12 @@ describe('getMultipartUploadHandlers with key', () => {
 				await multipartUploadJob();
 
 				/**
-				 * final crc32 calculation calls calculateContentCRC32 3 times
-				 * 1 time for each of the 2 parts
-				 * 1 time to combine the resulting hash for each of the two parts
-				 *
-				 * uploading each part calls calculateContentCRC32 1 time each
-				 *
-				 * 1 time for optionsHash
-				 *
-				 * these steps results in 6 calls in total
+				 * `calculateContentCRC32` is called 4 times with Full-body checksum:
+				 * * 1 time when calculating final crc32 for the whole object to be uploaded.
+				 * * 1 time when calculating optionsHash.
+				 * * 1 time each when uploading part calls, 2 calls in total.
 				 */
-				expect(calculateContentCRC32).toHaveBeenCalledTimes(6);
+				expect(calculateContentCRC32).toHaveBeenCalledTimes(4);
 				expect(calculateContentMd5).not.toHaveBeenCalled();
 				expect(mockUploadPart).toHaveBeenCalledTimes(2);
 				expect(mockUploadPart).toHaveBeenCalledWith(
@@ -389,7 +380,7 @@ describe('getMultipartUploadHandlers with key', () => {
 				file.size,
 			);
 			await multipartUploadJob();
-			expect(file.slice).toHaveBeenCalledTimes(10_000 * 2); // S3 limit of parts count double for crc32 calculations
+			expect(file.slice).toHaveBeenCalledTimes(10_000);
 			expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1);
 			expect(mockUploadPart).toHaveBeenCalledTimes(10_000);
 			expect(mockCompleteMultipartUpload).toHaveBeenCalledTimes(1);
@@ -1112,17 +1103,12 @@ describe('getMultipartUploadHandlers with path', () => {
 				await multipartUploadJob();
 
 				/**
-				 * final crc32 calculation calls calculateContentCRC32 3 times
-				 * 1 time for each of the 2 parts
-				 * 1 time to combine the resulting hash for each of the two parts
-				 *
-				 * uploading each part calls calculateContentCRC32 1 time each
-				 *
-				 * 1 time for optionsHash
-				 *
-				 * these steps results in 6 calls in total
+				 * `calculateContentCRC32` is called 4 times with Full-body checksum:
+				 * * 1 time when calculating final crc32 for the whole object to be uploaded.
+				 * * 1 time when calculating optionsHash.
+				 * * 1 time each when uploading part calls, 2 calls in total.
 				 */
-				expect(calculateContentCRC32).toHaveBeenCalledTimes(6);
+				expect(calculateContentCRC32).toHaveBeenCalledTimes(4);
 				expect(calculateContentMd5).not.toHaveBeenCalled();
 				expect(mockUploadPart).toHaveBeenCalledTimes(2);
 				expect(mockUploadPart).toHaveBeenCalledWith(
@@ -1203,7 +1189,7 @@ describe('getMultipartUploadHandlers with path', () => {
 				file.size,
 			);
 			await multipartUploadJob();
-			expect(file.slice).toHaveBeenCalledTimes(10_000 * 2); // S3 limit of parts count double for crc32 calculations
+			expect(file.slice).toHaveBeenCalledTimes(10_000);
 			expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1);
 			expect(mockUploadPart).toHaveBeenCalledTimes(10_000);
 			expect(mockCompleteMultipartUpload).toHaveBeenCalledTimes(1);

packages/storage/__tests__/providers/s3/utils/crc32.test.ts

@@ -1,131 +1,82 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import {
-	TextDecoder as TextDecoderPolyfill,
-	TextEncoder as TextEncoderPolyfill,
-} from 'node:util';
+import { TextEncoder as TextEncoderPolyfill } from 'node:util';
 
 import { calculateContentCRC32 } from '../../../../src/providers/s3/utils/crc32';
 
 global.TextEncoder = TextEncoderPolyfill as any;
-global.TextDecoder = TextDecoderPolyfill as any;
 
 const MB = 1024 * 1024;
 const getBlob = (size: number) => new Blob(['1'.repeat(size)]);
 const encoder = new TextEncoder();
-const decoder = new TextDecoder();
 
 describe('calculate crc32', () => {
 	describe.each([
 		{
 			type: 'file',
 			size: '4B',
 			data: new File(['data'], 'someName'),
-			expected: {
-				checksum: 'rfPzYw==',
-				checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer,
-				seed: 2918445923,
-			},
+			expected: 'rfPzYw==',
 		},
 		{
 			type: 'blob',
 			size: '4B',
 			data: new Blob(['data']),
-			expected: {
-				checksum: 'rfPzYw==',
-				checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer,
-				seed: 2918445923,
-			},
+			expected: 'rfPzYw==',
 		},
 		{
 			type: 'string',
 			size: '4B',
 			data: 'data',
-			expected: {
-				checksum: 'rfPzYw==',
-				checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer,
-				seed: 2918445923,
-			},
+			expected: 'rfPzYw==',
 		},
 		{
 			type: 'arrayBuffer',
 			size: '4B',
 			data: encoder.encode('data').buffer,
-			expected: {
-				checksum: 'rfPzYw==',
-				checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer,
-				seed: 2918445923,
-			},
+			expected: 'rfPzYw==',
 		},
 		{
 			type: 'arrayBufferView',
 			size: '4B',
 			data: new DataView(encoder.encode('1234 data 5678').buffer, 5, 4),
-			expected: {
-				checksum: 'rfPzYw==',
-				checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer,
-				seed: 2918445923,
-			},
+			expected: 'rfPzYw==',
 		},
 		{
 			type: 'file',
 			size: '8MB',
 			data: new File([getBlob(8 * MB)], 'someName'),
-			expected: {
-				checksum: '/YBlgg==',
-				checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer,
-				seed: 4253050242,
-			},
+			expected: '/YBlgg==',
 		},
 		{
 			type: 'blob',
 			size: '8MB',
 			data: getBlob(8 * MB),
-			expected: {
-				checksum: '/YBlgg==',
-				checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer,
-				seed: 4253050242,
-			},
+			expected: '/YBlgg==',
 		},
 		{
 			type: 'string',
 			size: '8MB',
 			data: '1'.repeat(8 * MB),
-			expected: {
-				checksum: '/YBlgg==',
-				checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer,
-				seed: 4253050242,
-			},
+			expected: '/YBlgg==',
 		},
 		{
 			type: 'arrayBuffer',
 			size: '8MB',
 			data: encoder.encode('1'.repeat(8 * MB)).buffer,
-			expected: {
-				checksum: '/YBlgg==',
-				checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer,
-				seed: 4253050242,
-			},
+			expected: '/YBlgg==',
 		},
 		{
 			type: 'arrayBufferView',
 			size: '8MB',
 			data: encoder.encode('1'.repeat(8 * MB)),
-			expected: {
-				checksum: '/YBlgg==',
-				checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer,
-				seed: 4253050242,
-			},
+			expected: '/YBlgg==',
 		},
 	])('output for data type of $type with size $size', ({ data, expected }) => {
 		it('should match expected checksum results', async () => {
-			const result = (await calculateContentCRC32(data))!;
-			expect(result.checksum).toEqual(expected.checksum);
-			expect(result.seed).toEqual(expected.seed);
-			expect(decoder.decode(result.checksumArrayBuffer)).toEqual(
-				decoder.decode(expected.checksumArrayBuffer),
-			);
+			const result = await calculateContentCRC32(data);
+			expect(result).toEqual(expected);
 		});
 	});
 });