xref: /aosp_15_r20/external/lz4/examples/blockStreaming_lineByLine.c (revision 27162e4e17433d5aa7cb38e7b6a433a09405fc7f)
1*27162e4eSAndroid Build Coastguard Worker // LZ4 streaming API example : line-by-line logfile compression
2*27162e4eSAndroid Build Coastguard Worker // by Takayuki Matsuoka
3*27162e4eSAndroid Build Coastguard Worker 
4*27162e4eSAndroid Build Coastguard Worker 
5*27162e4eSAndroid Build Coastguard Worker #if defined(_MSC_VER) && (_MSC_VER <= 1800)  /* Visual Studio <= 2013 */
6*27162e4eSAndroid Build Coastguard Worker #  define _CRT_SECURE_NO_WARNINGS
7*27162e4eSAndroid Build Coastguard Worker #  define snprintf sprintf_s
8*27162e4eSAndroid Build Coastguard Worker #endif
9*27162e4eSAndroid Build Coastguard Worker #include "lz4.h"
10*27162e4eSAndroid Build Coastguard Worker 
11*27162e4eSAndroid Build Coastguard Worker #include <stdio.h>
12*27162e4eSAndroid Build Coastguard Worker #include <stdint.h>
13*27162e4eSAndroid Build Coastguard Worker #include <stdlib.h>
14*27162e4eSAndroid Build Coastguard Worker #include <string.h>
15*27162e4eSAndroid Build Coastguard Worker 
write_uint16(FILE * fp,uint16_t i)16*27162e4eSAndroid Build Coastguard Worker static size_t write_uint16(FILE* fp, uint16_t i)
17*27162e4eSAndroid Build Coastguard Worker {
18*27162e4eSAndroid Build Coastguard Worker     return fwrite(&i, sizeof(i), 1, fp);
19*27162e4eSAndroid Build Coastguard Worker }
20*27162e4eSAndroid Build Coastguard Worker 
write_bin(FILE * fp,const void * array,int arrayBytes)21*27162e4eSAndroid Build Coastguard Worker static size_t write_bin(FILE* fp, const void* array, int arrayBytes)
22*27162e4eSAndroid Build Coastguard Worker {
23*27162e4eSAndroid Build Coastguard Worker     return fwrite(array, 1, arrayBytes, fp);
24*27162e4eSAndroid Build Coastguard Worker }
25*27162e4eSAndroid Build Coastguard Worker 
read_uint16(FILE * fp,uint16_t * i)26*27162e4eSAndroid Build Coastguard Worker static size_t read_uint16(FILE* fp, uint16_t* i)
27*27162e4eSAndroid Build Coastguard Worker {
28*27162e4eSAndroid Build Coastguard Worker     return fread(i, sizeof(*i), 1, fp);
29*27162e4eSAndroid Build Coastguard Worker }
30*27162e4eSAndroid Build Coastguard Worker 
read_bin(FILE * fp,void * array,int arrayBytes)31*27162e4eSAndroid Build Coastguard Worker static size_t read_bin(FILE* fp, void* array, int arrayBytes)
32*27162e4eSAndroid Build Coastguard Worker {
33*27162e4eSAndroid Build Coastguard Worker     return fread(array, 1, arrayBytes, fp);
34*27162e4eSAndroid Build Coastguard Worker }
35*27162e4eSAndroid Build Coastguard Worker 
36*27162e4eSAndroid Build Coastguard Worker 
test_compress(FILE * outFp,FILE * inpFp,size_t messageMaxBytes,size_t ringBufferBytes)37*27162e4eSAndroid Build Coastguard Worker static void test_compress(
38*27162e4eSAndroid Build Coastguard Worker     FILE* outFp,
39*27162e4eSAndroid Build Coastguard Worker     FILE* inpFp,
40*27162e4eSAndroid Build Coastguard Worker     size_t messageMaxBytes,
41*27162e4eSAndroid Build Coastguard Worker     size_t ringBufferBytes)
42*27162e4eSAndroid Build Coastguard Worker {
43*27162e4eSAndroid Build Coastguard Worker     LZ4_stream_t* const lz4Stream = LZ4_createStream();
44*27162e4eSAndroid Build Coastguard Worker     const size_t cmpBufBytes = LZ4_COMPRESSBOUND(messageMaxBytes);
45*27162e4eSAndroid Build Coastguard Worker     char* const cmpBuf = (char*) malloc(cmpBufBytes);
46*27162e4eSAndroid Build Coastguard Worker     char* const inpBuf = (char*) malloc(ringBufferBytes);
47*27162e4eSAndroid Build Coastguard Worker     int inpOffset = 0;
48*27162e4eSAndroid Build Coastguard Worker 
49*27162e4eSAndroid Build Coastguard Worker     for ( ; ; )
50*27162e4eSAndroid Build Coastguard Worker     {
51*27162e4eSAndroid Build Coastguard Worker         char* const inpPtr = &inpBuf[inpOffset];
52*27162e4eSAndroid Build Coastguard Worker 
53*27162e4eSAndroid Build Coastguard Worker #if 0
54*27162e4eSAndroid Build Coastguard Worker         // Read random length data to the ring buffer.
55*27162e4eSAndroid Build Coastguard Worker         const int randomLength = (rand() % messageMaxBytes) + 1;
56*27162e4eSAndroid Build Coastguard Worker         const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength);
57*27162e4eSAndroid Build Coastguard Worker         if (0 == inpBytes) break;
58*27162e4eSAndroid Build Coastguard Worker #else
59*27162e4eSAndroid Build Coastguard Worker         // Read line to the ring buffer.
60*27162e4eSAndroid Build Coastguard Worker         int inpBytes = 0;
61*27162e4eSAndroid Build Coastguard Worker         if (!fgets(inpPtr, (int) messageMaxBytes, inpFp))
62*27162e4eSAndroid Build Coastguard Worker             break;
63*27162e4eSAndroid Build Coastguard Worker         inpBytes = (int) strlen(inpPtr);
64*27162e4eSAndroid Build Coastguard Worker #endif
65*27162e4eSAndroid Build Coastguard Worker 
66*27162e4eSAndroid Build Coastguard Worker         {
67*27162e4eSAndroid Build Coastguard Worker             const int cmpBytes = LZ4_compress_fast_continue(
68*27162e4eSAndroid Build Coastguard Worker                 lz4Stream, inpPtr, cmpBuf, inpBytes, (int) cmpBufBytes, 1);
69*27162e4eSAndroid Build Coastguard Worker             if (cmpBytes <= 0) break;
70*27162e4eSAndroid Build Coastguard Worker             write_uint16(outFp, (uint16_t) cmpBytes);
71*27162e4eSAndroid Build Coastguard Worker             write_bin(outFp, cmpBuf, cmpBytes);
72*27162e4eSAndroid Build Coastguard Worker 
73*27162e4eSAndroid Build Coastguard Worker             // Add and wraparound the ringbuffer offset
74*27162e4eSAndroid Build Coastguard Worker             inpOffset += inpBytes;
75*27162e4eSAndroid Build Coastguard Worker             if ((size_t)inpOffset >= ringBufferBytes - messageMaxBytes) inpOffset = 0;
76*27162e4eSAndroid Build Coastguard Worker         }
77*27162e4eSAndroid Build Coastguard Worker     }
78*27162e4eSAndroid Build Coastguard Worker     write_uint16(outFp, 0);
79*27162e4eSAndroid Build Coastguard Worker 
80*27162e4eSAndroid Build Coastguard Worker     free(inpBuf);
81*27162e4eSAndroid Build Coastguard Worker     free(cmpBuf);
82*27162e4eSAndroid Build Coastguard Worker     LZ4_freeStream(lz4Stream);
83*27162e4eSAndroid Build Coastguard Worker }
84*27162e4eSAndroid Build Coastguard Worker 
85*27162e4eSAndroid Build Coastguard Worker 
test_decompress(FILE * outFp,FILE * inpFp,size_t messageMaxBytes,size_t ringBufferBytes)86*27162e4eSAndroid Build Coastguard Worker static void test_decompress(
87*27162e4eSAndroid Build Coastguard Worker     FILE* outFp,
88*27162e4eSAndroid Build Coastguard Worker     FILE* inpFp,
89*27162e4eSAndroid Build Coastguard Worker     size_t messageMaxBytes,
90*27162e4eSAndroid Build Coastguard Worker     size_t ringBufferBytes)
91*27162e4eSAndroid Build Coastguard Worker {
92*27162e4eSAndroid Build Coastguard Worker     LZ4_streamDecode_t* const lz4StreamDecode = LZ4_createStreamDecode();
93*27162e4eSAndroid Build Coastguard Worker     char* const cmpBuf = (char*) malloc(LZ4_COMPRESSBOUND(messageMaxBytes));
94*27162e4eSAndroid Build Coastguard Worker     char* const decBuf = (char*) malloc(ringBufferBytes);
95*27162e4eSAndroid Build Coastguard Worker     int decOffset = 0;
96*27162e4eSAndroid Build Coastguard Worker 
97*27162e4eSAndroid Build Coastguard Worker     for ( ; ; )
98*27162e4eSAndroid Build Coastguard Worker     {
99*27162e4eSAndroid Build Coastguard Worker         uint16_t cmpBytes = 0;
100*27162e4eSAndroid Build Coastguard Worker 
101*27162e4eSAndroid Build Coastguard Worker         if (read_uint16(inpFp, &cmpBytes) != 1) break;
102*27162e4eSAndroid Build Coastguard Worker         if (cmpBytes == 0) break;
103*27162e4eSAndroid Build Coastguard Worker         if (read_bin(inpFp, cmpBuf, cmpBytes) != cmpBytes) break;
104*27162e4eSAndroid Build Coastguard Worker 
105*27162e4eSAndroid Build Coastguard Worker         {
106*27162e4eSAndroid Build Coastguard Worker             char* const decPtr = &decBuf[decOffset];
107*27162e4eSAndroid Build Coastguard Worker             const int decBytes = LZ4_decompress_safe_continue(
108*27162e4eSAndroid Build Coastguard Worker                 lz4StreamDecode, cmpBuf, decPtr, cmpBytes, (int) messageMaxBytes);
109*27162e4eSAndroid Build Coastguard Worker             if (decBytes <= 0) break;
110*27162e4eSAndroid Build Coastguard Worker             write_bin(outFp, decPtr, decBytes);
111*27162e4eSAndroid Build Coastguard Worker 
112*27162e4eSAndroid Build Coastguard Worker             // Add and wraparound the ringbuffer offset
113*27162e4eSAndroid Build Coastguard Worker             decOffset += decBytes;
114*27162e4eSAndroid Build Coastguard Worker             if ((size_t)decOffset >= ringBufferBytes - messageMaxBytes) decOffset = 0;
115*27162e4eSAndroid Build Coastguard Worker         }
116*27162e4eSAndroid Build Coastguard Worker     }
117*27162e4eSAndroid Build Coastguard Worker 
118*27162e4eSAndroid Build Coastguard Worker     free(decBuf);
119*27162e4eSAndroid Build Coastguard Worker     free(cmpBuf);
120*27162e4eSAndroid Build Coastguard Worker     LZ4_freeStreamDecode(lz4StreamDecode);
121*27162e4eSAndroid Build Coastguard Worker }
122*27162e4eSAndroid Build Coastguard Worker 
123*27162e4eSAndroid Build Coastguard Worker 
compare(FILE * f0,FILE * f1)124*27162e4eSAndroid Build Coastguard Worker static int compare(FILE* f0, FILE* f1)
125*27162e4eSAndroid Build Coastguard Worker {
126*27162e4eSAndroid Build Coastguard Worker     int result = 0;
127*27162e4eSAndroid Build Coastguard Worker     const size_t tempBufferBytes = 65536;
128*27162e4eSAndroid Build Coastguard Worker     char* const b0 = (char*) malloc(tempBufferBytes);
129*27162e4eSAndroid Build Coastguard Worker     char* const b1 = (char*) malloc(tempBufferBytes);
130*27162e4eSAndroid Build Coastguard Worker 
131*27162e4eSAndroid Build Coastguard Worker     while(0 == result)
132*27162e4eSAndroid Build Coastguard Worker     {
133*27162e4eSAndroid Build Coastguard Worker         const size_t r0 = fread(b0, 1, tempBufferBytes, f0);
134*27162e4eSAndroid Build Coastguard Worker         const size_t r1 = fread(b1, 1, tempBufferBytes, f1);
135*27162e4eSAndroid Build Coastguard Worker 
136*27162e4eSAndroid Build Coastguard Worker         result = (int) r0 - (int) r1;
137*27162e4eSAndroid Build Coastguard Worker 
138*27162e4eSAndroid Build Coastguard Worker         if (0 == r0 || 0 == r1) break;
139*27162e4eSAndroid Build Coastguard Worker         if (0 == result) result = memcmp(b0, b1, r0);
140*27162e4eSAndroid Build Coastguard Worker     }
141*27162e4eSAndroid Build Coastguard Worker 
142*27162e4eSAndroid Build Coastguard Worker     free(b1);
143*27162e4eSAndroid Build Coastguard Worker     free(b0);
144*27162e4eSAndroid Build Coastguard Worker     return result;
145*27162e4eSAndroid Build Coastguard Worker }
146*27162e4eSAndroid Build Coastguard Worker 
147*27162e4eSAndroid Build Coastguard Worker 
main(int argc,char * argv[])148*27162e4eSAndroid Build Coastguard Worker int main(int argc, char* argv[])
149*27162e4eSAndroid Build Coastguard Worker {
150*27162e4eSAndroid Build Coastguard Worker     enum {
151*27162e4eSAndroid Build Coastguard Worker         MESSAGE_MAX_BYTES   = 1024,
152*27162e4eSAndroid Build Coastguard Worker         RING_BUFFER_BYTES   = 1024 * 256 + MESSAGE_MAX_BYTES,
153*27162e4eSAndroid Build Coastguard Worker     };
154*27162e4eSAndroid Build Coastguard Worker 
155*27162e4eSAndroid Build Coastguard Worker     char inpFilename[256] = { 0 };
156*27162e4eSAndroid Build Coastguard Worker     char lz4Filename[256] = { 0 };
157*27162e4eSAndroid Build Coastguard Worker     char decFilename[256] = { 0 };
158*27162e4eSAndroid Build Coastguard Worker 
159*27162e4eSAndroid Build Coastguard Worker     if (argc < 2)
160*27162e4eSAndroid Build Coastguard Worker     {
161*27162e4eSAndroid Build Coastguard Worker         printf("Please specify input filename\n");
162*27162e4eSAndroid Build Coastguard Worker         return 0;
163*27162e4eSAndroid Build Coastguard Worker     }
164*27162e4eSAndroid Build Coastguard Worker 
165*27162e4eSAndroid Build Coastguard Worker     snprintf(inpFilename, 256, "%s", argv[1]);
166*27162e4eSAndroid Build Coastguard Worker     snprintf(lz4Filename, 256, "%s.lz4s", argv[1]);
167*27162e4eSAndroid Build Coastguard Worker     snprintf(decFilename, 256, "%s.lz4s.dec", argv[1]);
168*27162e4eSAndroid Build Coastguard Worker 
169*27162e4eSAndroid Build Coastguard Worker     printf("inp = [%s]\n", inpFilename);
170*27162e4eSAndroid Build Coastguard Worker     printf("lz4 = [%s]\n", lz4Filename);
171*27162e4eSAndroid Build Coastguard Worker     printf("dec = [%s]\n", decFilename);
172*27162e4eSAndroid Build Coastguard Worker 
173*27162e4eSAndroid Build Coastguard Worker     // compress
174*27162e4eSAndroid Build Coastguard Worker     {
175*27162e4eSAndroid Build Coastguard Worker         FILE* inpFp = fopen(inpFilename, "rb");
176*27162e4eSAndroid Build Coastguard Worker         FILE* outFp = fopen(lz4Filename, "wb");
177*27162e4eSAndroid Build Coastguard Worker 
178*27162e4eSAndroid Build Coastguard Worker         test_compress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES);
179*27162e4eSAndroid Build Coastguard Worker 
180*27162e4eSAndroid Build Coastguard Worker         fclose(outFp);
181*27162e4eSAndroid Build Coastguard Worker         fclose(inpFp);
182*27162e4eSAndroid Build Coastguard Worker     }
183*27162e4eSAndroid Build Coastguard Worker 
184*27162e4eSAndroid Build Coastguard Worker     // decompress
185*27162e4eSAndroid Build Coastguard Worker     {
186*27162e4eSAndroid Build Coastguard Worker         FILE* inpFp = fopen(lz4Filename, "rb");
187*27162e4eSAndroid Build Coastguard Worker         FILE* outFp = fopen(decFilename, "wb");
188*27162e4eSAndroid Build Coastguard Worker 
189*27162e4eSAndroid Build Coastguard Worker         test_decompress(outFp, inpFp, MESSAGE_MAX_BYTES, RING_BUFFER_BYTES);
190*27162e4eSAndroid Build Coastguard Worker 
191*27162e4eSAndroid Build Coastguard Worker         fclose(outFp);
192*27162e4eSAndroid Build Coastguard Worker         fclose(inpFp);
193*27162e4eSAndroid Build Coastguard Worker     }
194*27162e4eSAndroid Build Coastguard Worker 
195*27162e4eSAndroid Build Coastguard Worker     // verify
196*27162e4eSAndroid Build Coastguard Worker     {
197*27162e4eSAndroid Build Coastguard Worker         FILE* inpFp = fopen(inpFilename, "rb");
198*27162e4eSAndroid Build Coastguard Worker         FILE* decFp = fopen(decFilename, "rb");
199*27162e4eSAndroid Build Coastguard Worker 
200*27162e4eSAndroid Build Coastguard Worker         const int cmp = compare(inpFp, decFp);
201*27162e4eSAndroid Build Coastguard Worker         if (0 == cmp)
202*27162e4eSAndroid Build Coastguard Worker             printf("Verify : OK\n");
203*27162e4eSAndroid Build Coastguard Worker         else
204*27162e4eSAndroid Build Coastguard Worker             printf("Verify : NG\n");
205*27162e4eSAndroid Build Coastguard Worker 
206*27162e4eSAndroid Build Coastguard Worker         fclose(decFp);
207*27162e4eSAndroid Build Coastguard Worker         fclose(inpFp);
208*27162e4eSAndroid Build Coastguard Worker     }
209*27162e4eSAndroid Build Coastguard Worker 
210*27162e4eSAndroid Build Coastguard Worker     return 0;
211*27162e4eSAndroid Build Coastguard Worker }
212