1// Copyright 2022 The Pigweed Authors 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); you may not 4// use this file except in compliance with the License. You may obtain a copy of 5// the License at 6// 7// https://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12// License for the specific language governing permissions and limitations under 13// the License. 14 15/* eslint-env browser */ 16 17import { Frame, Encoder, Decoder } from 'pigweedjs/pw_hdlc'; 18import { Detokenizer } from './detokenizer'; 19 20const CSV = ` 2164636261, ,"regular token" 2286fc33f3, ,"base64 token" 230d6bd33c, ,"Regular Token: %s and Nested Token: %s" 2497185e6f, ,"(token: %s, string: %s, int: %d, float: %f)" 25451d86ed, ,"Cat" 26`; 27 28function generateFrame(text: string): Frame { 29 const uintArray = new TextEncoder().encode(text); 30 const encodedFrame = new Encoder().uiFrame(1, uintArray); 31 const decodedFrames = Array.from(new Decoder().process(encodedFrame)); 32 return decodedFrames[0]; 33} 34 35describe('Detokenizer', () => { 36 let detokenizer: Detokenizer; 37 38 beforeEach(() => { 39 detokenizer = new Detokenizer(CSV); 40 }); 41 42 it('parses a base64 correct frame properly', () => { 43 const frame = generateFrame('$8zP8hg=='); 44 expect(detokenizer.detokenizeBase64(frame)).toEqual('base64 token'); 45 }); 46 it('parses a correct frame properly', () => { 47 const frame = generateFrame('abcde'); 48 expect(detokenizer.detokenize(frame)).toEqual('regular token'); 49 }); 50 it('failure to detokenize returns original string', () => { 51 expect(detokenizer.detokenize(generateFrame('aabbcc'))).toEqual('aabbcc'); 52 expect(detokenizer.detokenizeBase64(generateFrame('$8zP7hg=='))).toEqual( 53 '$8zP7hg==', 54 ); 55 }); 56 it('recursive detokenize all nested base64 tokens', () => { 57 expect( 58 detokenizer.detokenizeBase64( 59 generateFrame( 60 '$PNNrDQkkN1lZZFJRPT0lJGIxNFlsd2trTjFsWlpGSlJQVDBGUTJGdFpXeFlwSENkUHc9PQ==', 61 ), 62 ), 63 ).toEqual( 64 'Regular Token: Cat and Nested Token: (token: Cat, string: Camel, int: 44, float: 1.2300000190734863)', 65 ); 66 }); 67 68 it('recursion detokenize with limits on max recursion', () => { 69 expect( 70 detokenizer.detokenizeBase64( 71 generateFrame( 72 '$PNNrDQkkN1lZZFJRPT0lJGIxNFlsd2trTjFsWlpGSlJQVDBGUTJGdFpXeFlwSENkUHc9PQ==', 73 ), 74 1, 75 ), 76 ).toEqual( 77 'Regular Token: Cat and Nested Token: (token: $7YYdRQ==, string: Camel, int: 44, float: 1.2300000190734863)', 78 ); 79 }); 80}); 81