fix: double type is not handled correctly in tsz compress on windows (#34847)

* fix: failed compress cases on windows

test_compress_basic.py
test_compress_alter_table.py

* fix ai review comments

Author: localvar

contrib/TSZ/sz/inc/ByteToolkit.h

@@ -15,12 +15,13 @@ extern "C" {
 #endif
 
 #include <stdio.h>
+#include <stdint.h>
 
-int bytesToInt_bigEndian(unsigned char* bytes);
-void intToBytes_bigEndian(unsigned char *b, unsigned int num);
+int bytesToInt32_bigEndian(unsigned char* bytes);
+void int32ToBytes_bigEndian(unsigned char *b, unsigned int num);
 
-long bytesToLong_bigEndian(unsigned char* b);
-void longToBytes_bigEndian(unsigned char *b, long num);
+uint64_t bytesToInt64_bigEndian(unsigned char* b);
+void int64ToBytes_bigEndian(unsigned char *b, uint64_t num);
 
 short getExponent_float(float value);
 short getPrecisionReqLength_float(float precision);

contrib/TSZ/sz/inc/CompressElement.h

@@ -19,7 +19,7 @@ extern "C" {
 typedef struct DoubleValueCompressElement
 {
 	double data;
-	long curValue;
+	uint64_t curValue;
 	unsigned char curBytes[8]; //big_endian
 	int reqBytesLength;
 	int resiBitsLength;
@@ -28,7 +28,7 @@ typedef struct DoubleValueCompressElement
 typedef struct FloatValueCompressElement
 {
 	float data;  // diffValue + medianValue
-	int curValue; // diff int value
+	uint32_t curValue; // diff int value
 	unsigned char curBytes[4]; // dif bytes value diffValue->iValue big_endian
 	int reqBytesLength;
 	int resiBitsLength;

contrib/TSZ/sz/inc/Huffman.h

@@ -10,6 +10,8 @@
 #ifndef _Huffman_H
 #define _Huffman_H
 
+#include <stdint.h>
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -32,7 +34,7 @@ typedef struct HuffmanTree {
 	node *qqq, *qq; //the root node of the HuffmanTree is qq[1]
 	int n_nodes; //n_nodes is for compression
 	int qend; 
-	unsigned long **code;
+	uint64_t **code;
 	unsigned char *cout;
 	int n_inode; //n_inode is for decompression
 	int maxBitCount;
@@ -45,7 +47,7 @@ node new_node(HuffmanTree *huffmanTree, size_t freq, unsigned int c, node a, nod
 node new_node2(HuffmanTree *huffmanTree, unsigned int c, unsigned char t);
 void qinsert(HuffmanTree *huffmanTree, node n);
 node qremove(HuffmanTree *huffmanTree);
-void build_code(HuffmanTree *huffmanTree, node n, int len, unsigned long out1, unsigned long out2);
+void build_code(HuffmanTree *huffmanTree, node n, int len, uint64_t out1, uint64_t out2);
 void init(HuffmanTree *huffmanTree, int *s, size_t length);
 void init_static(HuffmanTree *huffmanTree, int *s, size_t length);
 void encode(HuffmanTree *huffmanTree, int *s, size_t length, unsigned char *out, size_t *outSize);

contrib/TSZ/sz/inc/dataCompression.h

@@ -39,7 +39,7 @@ float min_f(float a, float b);
 float max_f(float a, float b);
 double getRealPrecision_double(double valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status);
 double getRealPrecision_float(float valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status);
-double getRealPrecision_int(long valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status);
+double getRealPrecision_int(int64_t valueRangeSize, int errBoundMode, double absErrBound, double relBoundRatio, int *status);
 void symTransform_8bytes(unsigned char data[8]);
 void symTransform_2bytes(unsigned char data[2]);
 void symTransform_4bytes(unsigned char data[4]);

contrib/TSZ/sz/inc/sz.h

@@ -115,15 +115,15 @@ typedef union lint32
 
 typedef union lint64
 {
-	long lvalue;
-	unsigned long ulvalue;
+	int64_t lvalue;
+	uint64_t ulvalue;
 	unsigned char byte[8];
 } lint64;
 
 typedef union ldouble
 {
     double value;
-    unsigned long lvalue;
+    uint64_t lvalue;
     unsigned char byte[8];
 } ldouble;
 

contrib/TSZ/sz/inc/utility.h

@@ -18,8 +18,8 @@ extern "C" {
 
 
 int is_lossless_compressed_data(unsigned char* compressedBytes, size_t cmpSize);
-unsigned long sz_lossless_compress(int losslessCompressor, unsigned char* data, unsigned long dataLength, unsigned char* compressBytes);
-unsigned long sz_lossless_decompress(int losslessCompressor, unsigned char* compressBytes, unsigned long cmpSize, unsigned char** oriData, unsigned long targetOriSize);
+size_t sz_lossless_compress(int losslessCompressor, unsigned char* data, size_t dataLength, unsigned char* compressBytes);
+size_t sz_lossless_decompress(int losslessCompressor, unsigned char* compressBytes, size_t cmpSize, unsigned char** oriData, size_t targetOriSize);
 
 
 #ifdef __cplusplus

contrib/TSZ/sz/src/ByteToolkit.c

@@ -11,7 +11,7 @@
 #include <string.h>
 #include "sz.h" 	
 
-INLINE int bytesToInt_bigEndian(unsigned char* bytes)
+INLINE int bytesToInt32_bigEndian(unsigned char* bytes)
 {
 	int res;
 	unsigned char* des = (unsigned char*)&res;
@@ -26,7 +26,7 @@ INLINE int bytesToInt_bigEndian(unsigned char* bytes)
  * @unsigned char *b the variable to store the converted bytes (length=4)
  * @unsigned int num
  * */
-INLINE void intToBytes_bigEndian(unsigned char *b, unsigned int num)
+INLINE void int32ToBytes_bigEndian(unsigned char *b, unsigned int num)
 {
 	unsigned char* sou =(unsigned char*)&num;
 	b[0] = sou[3];
@@ -38,10 +38,10 @@ INLINE void intToBytes_bigEndian(unsigned char *b, unsigned int num)
 /**
  * @endianType: refers to the endian_type of unsigned char* b.
  * */
-INLINE long bytesToLong_bigEndian(unsigned char* b) {
+INLINE uint64_t bytesToInt64_bigEndian(unsigned char* b) {
 
-	long temp = 0;
-	long res = 0;
+	uint64_t temp = 0;
+	uint64_t res = 0;
 
 	res <<= 8;
 	temp = b[0] & 0xff;
@@ -79,11 +79,9 @@ INLINE long bytesToLong_bigEndian(unsigned char* b) {
 
 }
 
-INLINE void longToBytes_bigEndian(unsigned char *b, long num) 
+INLINE void int64ToBytes_bigEndian(unsigned char *b, uint64_t num) 
 {
 	unsigned char* sou = (unsigned char*)&num;
-  if(sizeof(num) == 8) {
-    // 8 bytes
 	b[7] = sou[0];
 	b[6] = sou[1];
 	b[5] = sou[2];
@@ -92,13 +90,6 @@ INLINE void longToBytes_bigEndian(unsigned char *b, long num)
 	b[2] = sou[5];
 	b[1] = sou[6];
 	b[0] = sou[7];
-  } else {
-	memset(b, 0, 4);
-	b[7] = sou[0];
-	b[6] = sou[1];
-	b[5] = sou[2];
-	b[4] = sou[3];
-  }
 }
 
 //TODO: debug: lfBuf.lvalue could be actually little_endian....
@@ -132,9 +123,9 @@ INLINE short getExponent_double(double value)
 
 	ldouble lbuf;
 	lbuf.value = value;
-	long lvalue = lbuf.lvalue;
+	uint64_t lvalue = lbuf.lvalue;
 
-	int expValue = (int)((lvalue & 0x7FF0000000000000) >> 52);
+	int expValue = (int)((lvalue & 0x7FF0000000000000ULL) >> 52);
 	expValue -= 1023;
 	return (short)expValue;
 }
@@ -143,11 +134,11 @@ INLINE short getPrecisionReqLength_double(double precision)
 {
 	ldouble lbuf;
 	lbuf.value = precision;
-	long lvalue = lbuf.lvalue;
+	uint64_t lvalue = lbuf.lvalue;
 
-	int expValue = (int)((lvalue & 0x7FF0000000000000) >> 52);
+	int expValue = (int)((lvalue & 0x7FF0000000000000ULL) >> 52);
 	expValue -= 1023;
-//	unsigned char the1stManBit = (unsigned char)((lvalue & 0x0008000000000000) >> 51);
+//	unsigned char the1stManBit = (unsigned char)((lvalue & 0x0008000000000000ULL) >> 51);
 //	if(the1stManBit==1)
 //		expValue--;
 	return (short)expValue;
@@ -263,18 +254,18 @@ INLINE size_t bytesToSize(unsigned char* bytes, int size_type)
 {
 	size_t result = 0;
 	if(size_type == 4)	
-		result = bytesToInt_bigEndian(bytes);//4		
+		result = bytesToInt32_bigEndian(bytes);//4		
 	else
-		result = bytesToLong_bigEndian(bytes);//8	
+		result = bytesToInt64_bigEndian(bytes);//8	
 	return result;
 }
 
 INLINE void sizeToBytes(unsigned char* outBytes, size_t size, int size_type)
 {
 	if(size_type == 4)
-		intToBytes_bigEndian(outBytes, (unsigned int)size);//4
+		int32ToBytes_bigEndian(outBytes, (unsigned int)size);//4
 	else
-		longToBytes_bigEndian(outBytes, (unsigned long)size);//8
+		int64ToBytes_bigEndian(outBytes, (int64_t)size);//8
 }
 
 void convertSZParamsToBytes(sz_params* params, unsigned char* result, char optQuantMode)

contrib/TSZ/sz/src/Huffman.c

@@ -23,12 +23,12 @@ HuffmanTree* createHuffmanTree(int stateNum)
 
 	huffmanTree->pool = (struct node_t*)malloc(huffmanTree->allNodes*2*sizeof(struct node_t));
 	huffmanTree->qqq = (node*)malloc(huffmanTree->allNodes*2*sizeof(node));
-	huffmanTree->code = (unsigned long**)malloc(huffmanTree->stateNum*sizeof(unsigned long*));
+	huffmanTree->code = (uint64_t**)malloc(huffmanTree->stateNum*sizeof(uint64_t*));
 	huffmanTree->cout = (unsigned char *)malloc(huffmanTree->stateNum*sizeof(unsigned char));
 
 	memset(huffmanTree->pool, 0, huffmanTree->allNodes*2*sizeof(struct node_t));
 	memset(huffmanTree->qqq, 0, huffmanTree->allNodes*2*sizeof(node));
-    memset(huffmanTree->code, 0, huffmanTree->stateNum*sizeof(unsigned long*));
+    memset(huffmanTree->code, 0, huffmanTree->stateNum*sizeof(uint64_t*));
     memset(huffmanTree->cout, 0, huffmanTree->stateNum*sizeof(unsigned char));
 	huffmanTree->qq = huffmanTree->qqq - 1;
 	huffmanTree->n_nodes = 0;
@@ -119,10 +119,10 @@ node qremove(HuffmanTree* huffmanTree)
  * @out2 should be 0 as well.
  * @index: the index of the byte
  * */
-void build_code(HuffmanTree *huffmanTree, node n, int len, unsigned long out1, unsigned long out2)
+void build_code(HuffmanTree *huffmanTree, node n, int len, uint64_t out1, uint64_t out2)
 {
 	if (n->t) {
-		huffmanTree->code[n->c] = (unsigned long*)malloc(2*sizeof(unsigned long));
+		huffmanTree->code[n->c] = (uint64_t*)malloc(2*sizeof(uint64_t));
 		if(len<=64)
 		{
 			if(len == 0)
@@ -234,14 +234,14 @@ void encode(HuffmanTree *huffmanTree, int *s, size_t length, unsigned char *out,
 			byteSizep = bitSize/8; //it's used to move the pointer p for next data
 			if(byteSize<=8)
 			{
-				longToBytes_bigEndian(p, (huffmanTree->code[state])[0]);
+				int64ToBytes_bigEndian(p, (huffmanTree->code[state])[0]);
 				p += byteSizep;
 			}
 			else //byteSize>8
 			{
-				longToBytes_bigEndian(p, (huffmanTree->code[state])[0]);
+				int64ToBytes_bigEndian(p, (huffmanTree->code[state])[0]);
 				p += 8;
-				longToBytes_bigEndian(p, (huffmanTree->code[state])[1]);
+				int64ToBytes_bigEndian(p, (huffmanTree->code[state])[1]);
 				p += (byteSizep - 8);
 			}
 			*outSize += byteSize;
@@ -254,8 +254,8 @@ void encode(HuffmanTree *huffmanTree, int *s, size_t length, unsigned char *out,
 			{
 				p++;
 				//(*outSize)++;
-				long newCode = (huffmanTree->code[state])[0] << lackBits;
-				longToBytes_bigEndian(p, newCode);
+				uint64_t newCode = (huffmanTree->code[state])[0] << lackBits;
+				int64ToBytes_bigEndian(p, newCode);
 
 				if(bitSize<=64)
 				{
@@ -279,7 +279,7 @@ void encode(HuffmanTree *huffmanTree, int *s, size_t length, unsigned char *out,
 						p++;
 						//(*outSize)++;
 						newCode = (huffmanTree->code[state])[1] << lackBits;
-						longToBytes_bigEndian(p, newCode);
+						int64ToBytes_bigEndian(p, newCode);
 						bitSize -= lackBits;
 						byteSize = bitSize%8==0 ? bitSize/8 : bitSize/8+1;
 						byteSizep = bitSize/8;
@@ -706,9 +706,9 @@ void encode_withTree(HuffmanTree* huffmanTree, int *s, size_t length, unsigned c
 	//printf("treeByteSize = %d\n", treeByteSize);
 
 	*out = (unsigned char*)malloc(length*sizeof(int)+treeByteSize);
-	intToBytes_bigEndian(buffer, nodeCount);
+	int32ToBytes_bigEndian(buffer, nodeCount);
 	memcpy(*out, buffer, 4);
-	intToBytes_bigEndian(buffer, huffmanTree->stateNum/2); //real number of intervals
+	int32ToBytes_bigEndian(buffer, huffmanTree->stateNum/2); //real number of intervals
 	memcpy(*out+4, buffer, 4);
 	memcpy(*out+8, treeBytes, treeByteSize);
 	free(treeBytes);
@@ -724,13 +724,13 @@ void encode_withTree(HuffmanTree* huffmanTree, int *s, size_t length, unsigned c
 void decode_withTree(HuffmanTree* huffmanTree, unsigned char *s, size_t targetLength, int *out)
 {
 	size_t encodeStartIndex;
-	size_t nodeCount = bytesToInt_bigEndian(s);
+	size_t nodeCount = bytesToInt32_bigEndian(s);
 	node root = reconstruct_HuffTree_from_bytes_anyStates(huffmanTree,s+8, nodeCount);
 
 	//sdi: Debug
 /*	build_code(root, 0, 0, 0);
 	int i;
-	unsigned long code_1, code_2;
+	uint64_t code_1, code_2;
 	for (i = 0; i < stateNum; i++)
 		if (code[i])
 		{

contrib/TSZ/sz/src/TightDataPointStorageD.c

@@ -84,13 +84,13 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
 
 	for (i = 0; i < 4; i++)
 		byteBuf[i] = flatBytes[index++];
-	int max_quant_intervals = bytesToInt_bigEndian(byteBuf);// 4	
+	int max_quant_intervals = bytesToInt32_bigEndian(byteBuf);// 4	
 
 	pde_params->maxRangeRadius = max_quant_intervals/2;
 
 	for (i = 0; i < 4; i++)
 		byteBuf[i] = flatBytes[index++];
-	(*this)->intervals = bytesToInt_bigEndian(byteBuf);// 4	
+	(*this)->intervals = bytesToInt32_bigEndian(byteBuf);// 4	
 
 	for (i = 0; i < 8; i++)
 		byteBuf[i] = flatBytes[index++];
@@ -136,7 +136,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
 	if ((*this)->ifAdtFse == 0) {
 		(*this)->typeArray = &flatBytes[index];
 		//retrieve the number of states (i.e., stateNum)
-		(*this)->allNodes = bytesToInt_bigEndian((*this)->typeArray); //the first 4 bytes store the stateNum
+		(*this)->allNodes = bytesToInt32_bigEndian((*this)->typeArray); //the first 4 bytes store the stateNum
 		(*this)->stateNum = ((*this)->allNodes+1)/2;	
 		index+=(*this)->typeArray_size;
 	} else {
@@ -147,10 +147,21 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi
 		index+=(*this)->transCodeBits_size;
 	}
 
-    // todo need check length
-	(*this)->residualMidBits_size = flatBytesLength - 1 - 1 - MetaDataByteLength - pde_exe->SZ_SIZE_TYPE - 4 - 4 - 4 - 1 - 8 
-			- pde_exe->SZ_SIZE_TYPE - pde_exe->SZ_SIZE_TYPE
-			- (*this)->leadNumArray_size - (*this)->exactMidBytes_size;
+	// todo need check length
+	(*this)->residualMidBits_size = flatBytesLength                 // total length of flatBytes
+									- 1                             // version
+									- 1                             // sameByte
+									- MetaDataByteLength_double     // meta data
+									- pde_exe->SZ_SIZE_TYPE         // data series length
+									- 4                             // max quant intervals
+									- 4                             // intervals
+									- sizeof(double)                // median value
+									- 1                             // req length
+									- sizeof(double)                // real precision
+									- pde_exe->SZ_SIZE_TYPE         // leadNumArray_size
+									- pde_exe->SZ_SIZE_TYPE         // exactMidBytes_size
+									- (*this)->leadNumArray_size    // data length of leadNumArray
+									- (*this)->exactMidBytes_size;  // data length of exactMidBytes
 	if ((*this)->ifAdtFse == 0) {
 		(*this)->residualMidBits_size = (*this)->residualMidBits_size - (*this)->typeArray_size - pde_exe->SZ_SIZE_TYPE ;
 	} else {
@@ -250,11 +261,11 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte
 
 	for(i = 0;i<exe_params->SZ_SIZE_TYPE;i++)//ST: 4 or 8 bytes
 		bytes[k++] = dsLengthBytes[i];	
-	intToBytes_bigEndian(max_quant_intervals_Bytes, confparams_cpr->max_quant_intervals);
+	int32ToBytes_bigEndian(max_quant_intervals_Bytes, confparams_cpr->max_quant_intervals);
 	for(i = 0;i<4;i++)//4
 		bytes[k++] = max_quant_intervals_Bytes[i];		
 
-	intToBytes_bigEndian(intervalsBytes, tdps->intervals);
+	int32ToBytes_bigEndian(intervalsBytes, tdps->intervals);
 	for(i = 0;i<4;i++)//4
 		bytes[k++] = intervalsBytes[i];		
 
@@ -326,9 +337,9 @@ bool convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char*
 	unsigned char dsLengthBytes[8];
 
 	if(exe_params->SZ_SIZE_TYPE==4)
-		intToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//4
+		int32ToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//4
 	else
-		longToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//8
+		int64ToBytes_bigEndian(dsLengthBytes, tdps->dataSeriesLength);//8
 
 	unsigned char sameByte = tdps->allSameData==1?(unsigned char)1:(unsigned char)0;
 	//sameByte = sameByte | (confparams_cpr->szMode << 1);