FORM  4.2.1
compress.c
Go to the documentation of this file.
1 
7 /* #[ License : */
8 /*
9  * Copyright (C) 1984-2017 J.A.M. Vermaseren
10  * When using this file you are requested to refer to the publication
11  * J.A.M.Vermaseren "New features of FORM" math-ph/0010025
12  * This is considered a matter of courtesy as the development was paid
13  * for by FOM the Dutch physics granting agency and we would like to
14  * be able to track its scientific use to convince FOM of its value
15  * for the community.
16  *
17  * This file is part of FORM.
18  *
19  * FORM is free software: you can redistribute it and/or modify it under the
20  * terms of the GNU General Public License as published by the Free Software
21  * Foundation, either version 3 of the License, or (at your option) any later
22  * version.
23  *
24  * FORM is distributed in the hope that it will be useful, but WITHOUT ANY
25  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
26  * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
27  * details.
28  *
29  * You should have received a copy of the GNU General Public License along
30  * with FORM. If not, see <http://www.gnu.org/licenses/>.
31  */
32 /* #] License : */
33 
34 #include "form3.h"
35 
36 #ifdef WITHZLIB
37 /*
38 #define GZIPDEBUG
39 
40  Low level routines for dealing with zlib during sorting and handling
41  the scratch files. Work started 5-sep-2005.
42  The .sor file handling was more or less completed on 8-sep-2005
43  The handling of the scratch files still needs some thinking.
44  Complications are:
45  gzip compression should be per expression, not per buffer.
46  No gzip compression for expressions with a bracket index.
47  Separate decompression buffers for expressions in the rhs.
48  This last one will involve more buffer work and organization.
49  Information about compression should be stored for each expr.
50  (including what method/program was used)
51  Note: Be careful with compression. By far the most compact method
52  is the original problem....
53 
54  #[ Variables :
55 
56  The following variables are to contain the intermediate buffers
57  for the inflation of the various patches in the sort file.
58  There can be up to MaxFpatches (FilePatches in the setup) and hence
59  we can have that many streams simultaneously. We set this up once
60  and only when needed.
61  (in struct A.N or AB[threadnum].N)
62  Bytef **AN.ziobufnum;
63  Bytef *AN.ziobuffers;
64 */
65 
66 /*
67  #] Variables :
68  #[ SetupOutputGZIP :
69 
70  Routine prepares a gzip output stream for the given file.
71 */
72 
73 int SetupOutputGZIP(FILEHANDLE *f)
74 {
75  GETIDENTITY
76 
77  if ( AT.SS != AT.S0 ) return(0);
78  if ( AR.NoCompress == 1 ) return(0);
79  if ( AR.gzipCompress <= 0 ) return(0);
80 
81  if ( f->ziobuffer == 0 ) {
82 /*
83  1: Allocate a struct for the gzip stream:
84 */
85  f->zsp = Malloc1(sizeof(z_stream),"output zstream");
86 /*
87  2: Allocate the output buffer.
88 */
89  f->ziobuffer =
90  (Bytef *)Malloc1(f->ziosize*sizeof(char),"output zbuffer");
91  if ( f->zsp == 0 || f->ziobuffer == 0 ) {
92  MLOCK(ErrorMessageLock);
93  MesCall("SetupOutputGZIP");
94  MUNLOCK(ErrorMessageLock);
95  Terminate(-1);
96  }
97  }
98 /*
99  3: Set the default fields:
100 */
101  f->zsp->zalloc = Z_NULL;
102  f->zsp->zfree = Z_NULL;
103  f->zsp->opaque = Z_NULL;
104 /*
105  4: Set the output space:
106 */
107  f->zsp->next_out = f->ziobuffer;
108  f->zsp->avail_out = f->ziosize;
109  f->zsp->total_out = 0;
110 /*
111  5: Set the input space:
112 */
113  f->zsp->next_in = (Bytef *)(f->PObuffer);
114  f->zsp->avail_in = (Bytef *)(f->POfill) - (Bytef *)(f->PObuffer);
115  f->zsp->total_in = 0;
116 /*
117  6: Initiate the deflation
118 */
119  if ( deflateInit(f->zsp,AR.gzipCompress) != Z_OK ) {
120  MLOCK(ErrorMessageLock);
121  MesPrint("Error from zlib: %s",f->zsp->msg);
122  MesCall("SetupOutputGZIP");
123  MUNLOCK(ErrorMessageLock);
124  Terminate(-1);
125  }
126 
127  return(0);
128 }
129 
130 /*
131  #] SetupOutputGZIP :
132  #[ PutOutputGZIP :
133 
134  Routine is called when the PObuffer of f is full.
135  The contents of it will be compressed and whenever the output buffer
136  f->ziobuffer is full it will be written and the output buffer
137  will be reset.
138  Upon exit the input buffer will be cleared.
139 */
140 
141 int PutOutputGZIP(FILEHANDLE *f)
142 {
143  GETIDENTITY
144  int zerror;
145 /*
146  First set the number of bytes in the input
147 */
148  f->zsp->next_in = (Bytef *)(f->PObuffer);
149  f->zsp->avail_in = (Bytef *)(f->POfill) - (Bytef *)(f->PObuffer);
150  f->zsp->total_in = 0;
151 
152  while ( ( zerror = deflate(f->zsp,Z_NO_FLUSH) ) == Z_OK ) {
153  if ( f->zsp->avail_out == 0 ) {
154 /*
155  ziobuffer is full. Write the output.
156 */
157 #ifdef GZIPDEBUG
158  {
159  char *s = (char *)((UBYTE *)(f->ziobuffer)+f->ziosize);
160  MLOCK(ErrorMessageLock);
161  MesPrint("%wWriting %l bytes at %10p: %d %d %d %d %d"
162  ,f->ziosize,&(f->POposition),s[-5],s[-4],s[-3],s[-2],s[-1]);
163  MUNLOCK(ErrorMessageLock);
164  }
165 #endif
166 #ifdef ALLLOCK
167  LOCK(f->pthreadslock);
168 #endif
169  if ( f == AR.hidefile ) {
170  LOCK(AS.inputslock);
171  }
172  SeekFile(f->handle,&(f->POposition),SEEK_SET);
173  if ( WriteFile(f->handle,(UBYTE *)(f->ziobuffer),f->ziosize)
174  != f->ziosize ) {
175  if ( f == AR.hidefile ) {
176  UNLOCK(AS.inputslock);
177  }
178 #ifdef ALLLOCK
179  UNLOCK(f->pthreadslock);
180 #endif
181  MLOCK(ErrorMessageLock);
182  MesPrint("%wWrite error during compressed sort. Disk full?");
183  MUNLOCK(ErrorMessageLock);
184  return(-1);
185  }
186  if ( f == AR.hidefile ) {
187  UNLOCK(AS.inputslock);
188  }
189 #ifdef ALLLOCK
190  UNLOCK(f->pthreadslock);
191 #endif
192  ADDPOS(f->filesize,f->ziosize);
193  ADDPOS(f->POposition,f->ziosize);
194 #ifdef WITHPTHREADS
195  if ( AS.MasterSort && AC.ThreadSortFileSynch ) {
196  if ( f->handle >= 0 ) SynchFile(f->handle);
197  }
198 #endif
199 /*
200  Reset the output
201 */
202  f->zsp->next_out = f->ziobuffer;
203  f->zsp->avail_out = f->ziosize;
204  f->zsp->total_out = 0;
205  }
206  else if ( f->zsp->avail_in == 0 ) {
207 /*
208  We compressed everything and it sits in ziobuffer. Finish
209 */
210  return(0);
211  }
212  else {
213  MLOCK(ErrorMessageLock);
214  MesPrint("%w avail_in = %d, avail_out = %d.",f->zsp->avail_in,f->zsp->avail_out);
215  MUNLOCK(ErrorMessageLock);
216  break;
217  }
218  }
219  MLOCK(ErrorMessageLock);
220  MesPrint("%wError in gzip handling of output. zerror = %d",zerror);
221  MUNLOCK(ErrorMessageLock);
222  return(-1);
223 }
224 
225 /*
226  #] PutOutputGZIP :
227  #[ FlushOutputGZIP :
228 
229  Routine is called to flush a stream. The compression of the input buffer
230  will be completed and the contents of f->ziobuffer will be written.
231  Both buffers will be cleared.
232 */
233 
234 int FlushOutputGZIP(FILEHANDLE *f)
235 {
236  GETIDENTITY
237  int zerror;
238 /*
239  Set the proper parameters
240 */
241  f->zsp->next_in = (Bytef *)(f->PObuffer);
242  f->zsp->avail_in = (Bytef *)(f->POfill) - (Bytef *)(f->PObuffer);
243  f->zsp->total_in = 0;
244 
245  while ( ( zerror = deflate(f->zsp,Z_FINISH) ) == Z_OK ) {
246  if ( f->zsp->avail_out == 0 ) {
247 /*
248  Write the output
249 */
250 #ifdef GZIPDEBUG
251  MLOCK(ErrorMessageLock);
252  MesPrint("%wWriting %l bytes at %10p",f->ziosize,&(f->POposition));
253  MUNLOCK(ErrorMessageLock);
254 #endif
255 #ifdef ALLLOCK
256  LOCK(f->pthreadslock);
257 #endif
258  if ( f == AR.hidefile ) {
259  UNLOCK(AS.inputslock);
260  }
261  SeekFile(f->handle,&(f->POposition),SEEK_SET);
262  if ( WriteFile(f->handle,(UBYTE *)(f->ziobuffer),f->ziosize)
263  != f->ziosize ) {
264  if ( f == AR.hidefile ) {
265  UNLOCK(AS.inputslock);
266  }
267 #ifdef ALLLOCK
268  UNLOCK(f->pthreadslock);
269 #endif
270  MLOCK(ErrorMessageLock);
271  MesPrint("%wWrite error during compressed sort. Disk full?");
272  MUNLOCK(ErrorMessageLock);
273  return(-1);
274  }
275  if ( f == AR.hidefile ) {
276  UNLOCK(AS.inputslock);
277  }
278 #ifdef ALLLOCK
279  UNLOCK(f->pthreadslock);
280 #endif
281  ADDPOS(f->filesize,f->ziosize);
282  ADDPOS(f->POposition,f->ziosize);
283 #ifdef WITHPTHREADS
284  if ( AS.MasterSort && AC.ThreadSortFileSynch ) {
285  if ( f->handle >= 0 ) SynchFile(f->handle);
286  }
287 #endif
288 /*
289  Reset the output
290 */
291  f->zsp->next_out = f->ziobuffer;
292  f->zsp->avail_out = f->ziosize;
293  f->zsp->total_out = 0;
294  }
295  }
296  if ( zerror == Z_STREAM_END ) {
297 /*
298  Write the output
299 */
300 #ifdef GZIPDEBUG
301  MLOCK(ErrorMessageLock);
302  MesPrint("%wWriting %l bytes at %10p",(LONG)(f->zsp->avail_out),&(f->POposition));
303  MUNLOCK(ErrorMessageLock);
304 #endif
305 #ifdef ALLLOCK
306  LOCK(f->pthreadslock);
307 #endif
308  if ( f == AR.hidefile ) {
309  LOCK(AS.inputslock);
310  }
311  SeekFile(f->handle,&(f->POposition),SEEK_SET);
312  if ( WriteFile(f->handle,(UBYTE *)(f->ziobuffer),f->zsp->total_out)
313  != (LONG)(f->zsp->total_out) ) {
314  if ( f == AR.hidefile ) {
315  UNLOCK(AS.inputslock);
316  }
317 #ifdef ALLLOCK
318  UNLOCK(f->pthreadslock);
319 #endif
320  MLOCK(ErrorMessageLock);
321  MesPrint("%wWrite error during compressed sort. Disk full?");
322  MUNLOCK(ErrorMessageLock);
323  return(-1);
324  }
325  if ( f == AR.hidefile ) {
326  LOCK(AS.inputslock);
327  }
328 #ifdef ALLLOCK
329  UNLOCK(f->pthreadslock);
330 #endif
331  ADDPOS(f->filesize,f->zsp->total_out);
332  ADDPOS(f->POposition,f->zsp->total_out);
333 #ifdef WITHPTHREADS
334  if ( AS.MasterSort && AC.ThreadSortFileSynch ) {
335  if ( f->handle >= 0 ) SynchFile(f->handle);
336  }
337 #endif
338 #ifdef GZIPDEBUG
339  MLOCK(ErrorMessageLock);
340  { char *s = f->ziobuffer+f->zsp->total_out;
341  MesPrint("%w Last bytes written: %d %d %d %d %d",s[-5],s[-4],s[-3],s[-2],s[-1]);
342  }
343  MesPrint("%w Perceived position in FlushOutputGZIP is %10p",&(f->POposition));
344  MUNLOCK(ErrorMessageLock);
345 #endif
346 /*
347  Reset the output
348 */
349  f->zsp->next_out = f->ziobuffer;
350  f->zsp->avail_out = f->ziosize;
351  f->zsp->total_out = 0;
352  if ( ( zerror = deflateEnd(f->zsp) ) == Z_OK ) return(0);
353  MLOCK(ErrorMessageLock);
354  if ( f->zsp->msg ) {
355  MesPrint("%wError in finishing gzip handling of output: %s",f->zsp->msg);
356  }
357  else {
358  MesPrint("%wError in finishing gzip handling of output.");
359  }
360  MUNLOCK(ErrorMessageLock);
361  }
362  else {
363  MLOCK(ErrorMessageLock);
364  MesPrint("%wError in gzip handling of output.");
365  MUNLOCK(ErrorMessageLock);
366  }
367  return(-1);
368 }
369 
370 /*
371  #] FlushOutputGZIP :
372  #[ SetupAllInputGZIP :
373 
374  Routine prepares all gzip input streams for a merge.
375 
376  Problem (29-may-2008): If we never use GZIP compression, this routine
377  will still allocate the array space. This is an enormous amount!
378  It places an effective restriction on the value of SortIOsize
379 */
380 
381 int SetupAllInputGZIP(SORTING *S)
382 {
383  GETIDENTITY
384  int i, NumberOpened = 0;
385  z_streamp zsp;
386 /*
387  This code was added 29-may-2008 by JV to prevent further processing if
388  there is no compression at all (usually).
389 */
390  for ( i = 0; i < S->inNum; i++ ) {
391  if ( S->fpincompressed[i] ) break;
392  }
393  if ( i >= S->inNum ) return(0);
394 
395  if ( S->zsparray == 0 ) {
396  S->zsparray = (z_streamp)Malloc1(sizeof(z_stream)*S->MaxFpatches,"input zstreams");
397  if ( S->zsparray == 0 ) {
398  MLOCK(ErrorMessageLock);
399  MesCall("SetupAllInputGZIP");
400  MUNLOCK(ErrorMessageLock);
401  Terminate(-1);
402  }
403 /*
404  We add 128 bytes in the hope that if it can happen that it goes
405  outside the buffer during decompression, it does not do damage.
406 */
407  AN.ziobuffers = (Bytef *)Malloc1(S->MaxFpatches*(S->file.ziosize+128)*sizeof(Bytef),"input raw buffers");
408 /*
409  This seems to be one of the really stupid errors:
410  We allocate way too much space. Way way way too much.
411  AN.ziobufnum = (Bytef **)Malloc1(S->MaxFpatches*S->file.ziosize*sizeof(Bytef *),"input raw pointers");
412 */
413  AN.ziobufnum = (Bytef **)Malloc1(S->MaxFpatches*sizeof(Bytef *),"input raw pointers");
414  if ( AN.ziobuffers == 0 || AN.ziobufnum == 0 ) {
415  MLOCK(ErrorMessageLock);
416  MesCall("SetupAllInputGZIP");
417  MUNLOCK(ErrorMessageLock);
418  Terminate(-1);
419  }
420  for ( i = 0 ; i < S->MaxFpatches; i++ ) {
421  AN.ziobufnum[i] = AN.ziobuffers + i * (S->file.ziosize+128);
422  }
423  }
424  for ( i = 0; i < S->inNum; i++ ) {
425 #ifdef GZIPDEBUG
426  MLOCK(ErrorMessageLock);
427  MesPrint("%wPreparing z-stream %d with compression %d",i,S->fpincompressed[i]);
428  MUNLOCK(ErrorMessageLock);
429 #endif
430  if ( S->fpincompressed[i] ) {
431  zsp = &(S->zsparray[i]);
432 /*
433  1: Set the default fields:
434 */
435  zsp->zalloc = Z_NULL;
436  zsp->zfree = Z_NULL;
437  zsp->opaque = Z_NULL;
438 /*
439  2: Set the output space:
440 */
441  zsp->next_out = Z_NULL;
442  zsp->avail_out = 0;
443  zsp->total_out = 0;
444 /*
445  3: Set the input space temporarily:
446 */
447  zsp->next_in = Z_NULL;
448  zsp->avail_in = 0;
449  zsp->total_in = 0;
450 /*
451  4: Initiate the inflation
452 */
453  if ( inflateInit(zsp) != Z_OK ) {
454  MLOCK(ErrorMessageLock);
455  if ( zsp->msg ) MesPrint("%wError from inflateInit: %s",zsp->msg);
456  else MesPrint("%wError from inflateInit");
457  MesCall("SetupAllInputGZIP");
458  MUNLOCK(ErrorMessageLock);
459  Terminate(-1);
460  }
461  NumberOpened++;
462  }
463  }
464  return(NumberOpened);
465 }
466 
467 /*
468  #] SetupAllInputGZIP :
469  #[ FillInputGZIP :
470 
471  Routine is called when we need new input in the specified buffer.
472  This buffer is used for the output and we keep reading and uncompressing
473  input till either this buffer is full or the input stream is finished.
474  The return value is the number of bytes in the buffer.
475 */
476 
477 LONG FillInputGZIP(FILEHANDLE *f, POSITION *position, UBYTE *buffer, LONG buffersize, int numstream)
478 {
479  GETIDENTITY
480  int zerror;
481  LONG readsize, toread = 0;
482  SORTING *S = AT.SS;
483  z_streamp zsp;
484  POSITION pos;
485  if ( S->fpincompressed[numstream] ) {
486  zsp = &(S->zsparray[numstream]);
487  zsp->next_out = (Bytef *)buffer;
488  zsp->avail_out = buffersize;
489  zsp->total_out = 0;
490  if ( zsp->avail_in == 0 ) {
491 /*
492  First loading of the input
493 */
494  if ( ISGEPOSINC(S->fPatchesStop[numstream],*position,f->ziosize) ) {
495  toread = f->ziosize;
496  }
497  else {
498  DIFPOS(pos,S->fPatchesStop[numstream],*position);
499  toread = (LONG)(BASEPOSITION(pos));
500  }
501  if ( toread > 0 ) {
502 #ifdef GZIPDEBUG
503  MLOCK(ErrorMessageLock);
504  MesPrint("%w-+Reading %l bytes in stream %d at position %10p; stop at %10p",toread,numstream,position,&(S->fPatchesStop[numstream]));
505  MUNLOCK(ErrorMessageLock);
506 #endif
507 #ifdef ALLLOCK
508  LOCK(f->pthreadslock);
509 #endif
510  SeekFile(f->handle,position,SEEK_SET);
511  readsize = ReadFile(f->handle,(UBYTE *)(AN.ziobufnum[numstream]),toread);
512  SeekFile(f->handle,position,SEEK_CUR);
513 #ifdef ALLLOCK
514  UNLOCK(f->pthreadslock);
515 #endif
516 #ifdef GZIPDEBUG
517  MLOCK(ErrorMessageLock);
518  { char *s = AN.ziobufnum[numstream]+readsize;
519  MesPrint("%w read: %l +Last bytes read: %d %d %d %d %d in %s, newpos = %10p",readsize,s[-5],s[-4],s[-3],s[-2],s[-1],f->name,position);
520  }
521  MUNLOCK(ErrorMessageLock);
522 #endif
523  if ( readsize == 0 ) {
524  zsp->next_in = AN.ziobufnum[numstream];
525  zsp->avail_in = f->ziosize;
526  zsp->total_in = 0;
527  return(zsp->total_out);
528  }
529  if ( readsize < 0 ) {
530  MLOCK(ErrorMessageLock);
531  MesPrint("%wFillInputGZIP: Read error during compressed sort.");
532  MUNLOCK(ErrorMessageLock);
533  return(-1);
534  }
535  ADDPOS(f->filesize,readsize);
536  ADDPOS(f->POposition,readsize);
537 /*
538  Set the input
539 */
540  zsp->next_in = AN.ziobufnum[numstream];
541  zsp->avail_in = readsize;
542  zsp->total_in = 0;
543  }
544  }
545  if ( toread > 0 || zsp->avail_in ) {
546  while ( ( zerror = inflate(zsp,Z_NO_FLUSH) ) == Z_OK ) {
547  if ( zsp->avail_out == 0 ) {
548 /*
549  Finish
550 */
551  return((LONG)(zsp->total_out));
552  }
553  if ( zsp->avail_in == 0 ) {
554 
555  if ( ISEQUALPOS(S->fPatchesStop[numstream],*position) ) {
556 /*
557  We finished this stream. Try to terminate.
558 */
559  zerror = Z_STREAM_END;
560  break;
561 /*
562  if ( ( zerror = inflate(zsp,Z_SYNC_FLUSH) ) == Z_OK ) {
563  return((LONG)(zsp->total_out));
564  }
565  else
566  break;
567 */
568 /*
569 #ifdef GZIPDEBUG
570  MLOCK(ErrorMessageLock);
571  MesPrint("%wClosing stream %d",numstream);
572 #endif
573  readsize = zsp->total_out;
574 #ifdef GZIPDEBUG
575  if ( readsize > 0 ) {
576  WORD *s = (WORD *)(buffer+zsp->total_out);
577  MesPrint("%w Last words: %d %d %d %d %d",s[-5],s[-4],s[-3],s[-2],s[-1]);
578  }
579  else {
580  MesPrint("%w No words");
581  }
582  MUNLOCK(ErrorMessageLock);
583 #endif
584  if ( ( zerror = inflateEnd(zsp) ) == Z_OK ) return(readsize);
585  break;
586 */
587  }
588 /*
589  Read more input
590 */
591 #ifdef GZIPDEBUG
592  if ( numstream == 0 ) {
593  MLOCK(ErrorMessageLock);
594  MesPrint("%wWant to read in stream 0 at position %10p",position);
595  MUNLOCK(ErrorMessageLock);
596  }
597 #endif
598  if ( ISGEPOSINC(S->fPatchesStop[numstream],*position,f->ziosize) ) {
599  toread = f->ziosize;
600  }
601  else {
602  DIFPOS(pos,S->fPatchesStop[numstream],*position);
603  toread = (LONG)(BASEPOSITION(pos));
604  }
605 #ifdef GZIPDEBUG
606  MLOCK(ErrorMessageLock);
607  MesPrint("%w--Reading %l bytes in stream %d at position %10p",toread,numstream,position);
608  MUNLOCK(ErrorMessageLock);
609 #endif
610 #ifdef ALLLOCK
611  LOCK(f->pthreadslock);
612 #endif
613  SeekFile(f->handle,position,SEEK_SET);
614  readsize = ReadFile(f->handle,(UBYTE *)(AN.ziobufnum[numstream]),toread);
615  SeekFile(f->handle,position,SEEK_CUR);
616 #ifdef ALLLOCK
617  UNLOCK(f->pthreadslock);
618 #endif
619 #ifdef GZIPDEBUG
620  MLOCK(ErrorMessageLock);
621  { char *s = AN.ziobufnum[numstream]+readsize;
622  MesPrint("%w Last bytes read: %d %d %d %d %d",s[-5],s[-4],s[-3],s[-2],s[-1]);
623  }
624  MUNLOCK(ErrorMessageLock);
625 #endif
626  if ( readsize == 0 ) {
627  zsp->next_in = AN.ziobufnum[numstream];
628  zsp->avail_in = f->ziosize;
629  zsp->total_in = 0;
630  return(zsp->total_out);
631  }
632  if ( readsize < 0 ) {
633  MLOCK(ErrorMessageLock);
634  MesPrint("%wFillInputGZIP: Read error during compressed sort.");
635  MUNLOCK(ErrorMessageLock);
636  return(-1);
637  }
638  ADDPOS(f->filesize,readsize);
639  ADDPOS(f->POposition,readsize);
640 /*
641  Reset the input
642 */
643  zsp->next_in = AN.ziobufnum[numstream];
644  zsp->avail_in = readsize;
645  zsp->total_in = 0;
646  }
647  else {
648  break;
649  }
650  }
651  }
652  else {
653  zerror = Z_STREAM_END;
654  zsp->total_out = 0;
655  }
656 #ifdef GZIPDEBUG
657  MLOCK(ErrorMessageLock);
658  MesPrint("%w zerror = %d in stream %d. At position %10p",zerror,numstream,position);
659  MUNLOCK(ErrorMessageLock);
660 #endif
661  if ( zerror == Z_STREAM_END ) {
662 /*
663  Reset the input
664 */
665  zsp->next_in = Z_NULL;
666  zsp->avail_in = 0;
667  zsp->total_in = 0;
668 /*
669  Make the final call and finish
670 */
671 #ifdef GZIPDEBUG
672  MLOCK(ErrorMessageLock);
673  MesPrint("%wClosing stream %d",numstream);
674 #endif
675  readsize = zsp->total_out;
676 #ifdef GZIPDEBUG
677  if ( readsize > 0 ) {
678  WORD *s = (WORD *)(buffer+zsp->total_out);
679  MesPrint("%w -Last words: %d %d %d %d %d",s[-5],s[-4],s[-3],s[-2],s[-1]);
680  }
681  else {
682  MesPrint("%w No words");
683  }
684  MUNLOCK(ErrorMessageLock);
685 #endif
686  if ( zsp->zalloc != Z_NULL ) {
687  zerror = inflateEnd(zsp);
688  zsp->zalloc = Z_NULL;
689  }
690  if ( zerror == Z_OK || zerror == Z_STREAM_END ) return(readsize);
691  }
692 
693  MLOCK(ErrorMessageLock);
694  MesPrint("%wFillInputGZIP: Error in gzip handling of input. zerror = %d",zerror);
695  MUNLOCK(ErrorMessageLock);
696  return(-1);
697  }
698  else {
699 #ifdef GZIPDEBUG
700  MLOCK(ErrorMessageLock);
701  MesPrint("%w++Reading %l bytes at position %10p",buffersize,position);
702  MUNLOCK(ErrorMessageLock);
703 #endif
704 #ifdef ALLLOCK
705  LOCK(f->pthreadslock);
706 #endif
707  SeekFile(f->handle,position,SEEK_SET);
708  readsize = ReadFile(f->handle,buffer,buffersize);
709  SeekFile(f->handle,position,SEEK_CUR);
710 #ifdef ALLLOCK
711  UNLOCK(f->pthreadslock);
712 #endif
713  if ( readsize < 0 ) {
714  MLOCK(ErrorMessageLock);
715  MesPrint("%wFillInputGZIP: Read error during uncompressed sort.");
716  MesPrint("%w++Reading %l bytes at position %10p",buffersize,position);
717  MUNLOCK(ErrorMessageLock);
718  }
719  return(readsize);
720  }
721 }
722 
723 /*
724  #] FillInputGZIP :
725  #[ ClearSortGZIP :
726 */
727 
728 void ClearSortGZIP(FILEHANDLE *f)
729 {
730  if ( f->ziobuffer ) {
731  M_free(f->ziobuffer,"output zbuffer");
732  M_free(f->zsp,"output zstream");
733  f->ziobuffer = 0;
734  }
735 }
736 
737 /*
738  #] ClearSortGZIP :
739 */
740 #endif
Definition: structs.h:633
Definition: structs.h:1086
int handle
Definition: structs.h:661