From: "Larry D Bohan, Jr" Subject: Re: faster gunzip Date: 04 May 2000 00:00:00 GMT Message-ID: Content-Transfer-Encoding: 7bit References: <8esfrh$rc9@gap.cco.caltech.edu> Content-Type: text/plain; charset=us-ascii X-Complaints-To: abuse@verio.net X-Trace: ord-read.news.verio.net 957477939 199.217.9.201 (Thu, 04 May 2000 22:05:39 GMT) Organization: Verio MIME-Version: 1.0 NNTP-Posting-Date: Thu, 04 May 2000 22:05:39 GMT Newsgroups: comp.os.vms On 4 May 2000 18:34:25 GMT, mathog@seqaxp.bio.caltech.edu (David Mathog) wrote: >gunzip 1.2-4 on our OpenVMS 7.2-1 DS10 is very much slower than gunzip on >a DS10 running Linux/Alpha. A big part of that is because on Linux you >don't really get the data onto disk unless you force it there - instead it >is cached into the free memory on the system. We've hashed over the pluses >and minuses of that method before and I don't want to start another thread >on it here. this was posted on C.O.V. a few weeks/months ago. if the speed of gnuzip is really irritating, you might perhaps try to apply the techniques seen here. I was saving this example for the day I ran into something similarly irritating. the perceived difference in speed btw this example and a regular $COPY is impressive. Even while already using other techniques to speed up $COPY (large extents, disls w/ lotsa of contiguous free space, etc) It would be interesting if this BUFOBJ implementation could be made optionally available for $COPY, $CONVERT, etc. (i imagine (hyper)SORT and BACKUP already have similar optimizations) or for that matter, the C run-time library. /* * FAST_IO_COPY.C - disk file copy Fast I/O example. * This program requires OpenVMS V7.0 (VAX or Alpha). * The DEC C V5.2 (or newer) compiler is preferred. */ #include atrdef #include descrip #include fibdef #include iodef #include iosadef #include lib$routines #include rms #include ssdef #include starlet #include stdio #include stdlib #include string #include "fatdef.h" #define NBYTES_PER_BLOCK 512 #define MAX_BUFFERS 16 #define MAX_BLOCKS_PER_XFER 127 #define MAX_BYTES_PER_XFER (NBYTES_PER_BLOCK*MAX_BLOCKS_PER_XFER) #define MAX_BYTES_PER_BUFFER (NBYTES_PER_BLOCK+MAX_BYTES_PER_XFER) #pragma extern_model save #pragma extern_model common_block shr #pragma nostandard extern volatile _align (page) char buffer[MAX_BUFFERS][MAX_BYTES_PER_BUFFER]; #pragma standard extern volatile fandle ifan[MAX_BUFFERS]; extern volatile fandle ofan[MAX_BUFFERS]; extern volatile int iohan[MAX_BUFFERS][2]; extern struct FAB ifab; extern struct NAM inam; extern char iexpand[NAM$C_MAXRSS+1]; extern char iresult[NAM$C_MAXRSS+1]; extern struct FAB ofab; extern struct NAM onam; extern char oexpand[NAM$C_MAXRSS+1]; extern char oresult[NAM$C_MAXRSS+1]; extern struct XABFHC xabfhc; extern struct XABSUM xabsum; extern struct XABALL xaball[255]; extern struct XABKEY xabkey[255]; extern char knm[255][32]; extern volatile int ivbn; extern volatile int iefn; extern volatile int nreads_outstanding; extern volatile int nwrites_outstanding; extern volatile int max_reads_outstanding; extern volatile int max_writes_outstanding; extern volatile int nbuffers; extern volatile int nblocks_per_xfer; extern volatile int nbytes_per_xfer; extern volatile int nbytes_per_buffer; #pragma extern_model restore void read_completion_ast(iosa *piosa) { register int status = piosa->iosa$l_status; register int nbytes = piosa->iosa$ih_count; register int ibuf = piosa->iosa$q_context_q[0]; if (nreads_outstanding > max_reads_outstanding) max_reads_outstanding = nreads_outstanding; nreads_outstanding--; if ((status != SS$_ENDOFFILE) && !(status & 1)) { int msgvec[2] = {1,0}; msgvec[1] = status; printf("Could not perform input from VBN %0d to buffer #%0d.\n", piosa->iosa$q_context_q[1], ibuf); sys$putmsg(msgvec,0,0,0); goto read_return; } if (nbytes) { ifab.fab$l_ctx++; status = sys$io_perform(ofan[ibuf], ofab.fab$l_stv, &buffer[ibuf][0], &buffer[ibuf][NBYTES_PER_BLOCK], nbytes, piosa->iosa$q_context_q[1]); if (!(status & 1)) { int msgvec[2] = {1,0}; msgvec[1] = status; printf( "Could not start output (%0d bytes) to VBN %0d from buffer #%0d.\n", nbytes, piosa->iosa$q_context_q[1], ibuf); sys$putmsg(msgvec,0,0,0); goto read_return; } nwrites_outstanding++; } read_return: if ((nreads_outstanding == 0) && (nwrites_outstanding == 0)) sys$setef(iefn); return; } /* read_completion_ast */ void write_completion_ast(iosa *piosa) { register int status = piosa->iosa$l_status; register int nbytes = piosa->iosa$ih_count; register int ibuf = piosa->iosa$q_context_q[0]; if (nwrites_outstanding > max_writes_outstanding) max_writes_outstanding = nwrites_outstanding; nwrites_outstanding--; if (!(status & 1)) { int msgvec[2] = {1,0}; msgvec[1] = status; printf( "Could not perform output (%0d bytes) to VBN %0d from buffer #%0d.\n", nbytes, piosa->iosa$q_context_q[1], ibuf); sys$putmsg(msgvec,0,0,0); goto write_return; } ofab.fab$l_ctx++; if (ivbn <= ifab.fab$l_alq) { piosa->iosa$ih_count = nbytes_per_xfer; piosa->iosa$q_context_q[1] = ivbn; status = sys$io_perform(ifan[ibuf], ifab.fab$l_stv, &buffer[ibuf][0], &buffer[ibuf][NBYTES_PER_BLOCK], nbytes_per_xfer, ivbn); if (!(status & 1)) { int msgvec[2] = {1,0}; msgvec[1] = status; printf( "Could not start input from VBN %0d to buffer #%0d.\n", ivbn, ibuf); sys$putmsg(msgvec,0,0,0); goto write_return; } ivbn += nblocks_per_xfer; nreads_outstanding++; } write_return: if ((nreads_outstanding == 0) && (nwrites_outstanding == 0)) sys$setef(iefn); return; } /* write_completion_ast */ int fast_io_copy(int argc, char *argv[]) #pragma nostandard main_program #pragma standard { register iosa *piosa = 0; register int status = 0; register int ibuf = 0; volatile char *inadr[2] = {0,0}; char *retadr[2] = {0,0}; unsigned short int iosb[4]; struct atrdef atrlst[2]; struct fibdef fib; struct dsc$descriptor_s fibdsc; FAT fat; /* * Require input and output file specifications. */ if (argc < 3) { printf("Usage: %s \n", argv[0]); return 1; } /* * Allow additional arguments to change either * the buffer count or the transfer size in blocks. */ nbuffers = MAX_BUFFERS; nblocks_per_xfer = MAX_BLOCKS_PER_XFER; for (ibuf=3; ibuf MAX_BUFFERS)) nbuffers = MAX_BUFFERS; else if (nbuffers < 0) nbuffers = 1; } } else if (!strncmp(argv[ibuf], "-nblocks", 8)) { ibuf++; if (ibuf < argc) { nblocks_per_xfer = atoi(argv[ibuf]); if ((nblocks_per_xfer == 0) || (nblocks_per_xfer > MAX_BLOCKS_PER_XFER)) nblocks_per_xfer = MAX_BLOCKS_PER_XFER; else if (nblocks_per_xfer < 0) nblocks_per_xfer = 1; } } else { printf("Unrecognized argument #%0d: \"%s\".\n", ibuf, argv[ibuf]); } } nbytes_per_xfer = NBYTES_PER_BLOCK * nblocks_per_xfer; nbytes_per_buffer = NBYTES_PER_BLOCK + nbytes_per_xfer; printf("Using %0d buffers, with %0d blocks (%0d bytes) per transfer.\n", nbuffers, nblocks_per_xfer, nbytes_per_xfer); for (ibuf=0; ibuf read requests. */ for ( ibuf=0, ivbn=1; ((ibufiosa$l_status = 1; piosa->iosa$ih_count = nbytes_per_xfer; piosa->iosa$q_context_q[0] = ibuf; piosa->iosa$q_context_q[1] = ivbn; status = sys$io_perform(ifan[ibuf], ifab.fab$l_stv, &buffer[ibuf][0], &buffer[ibuf][NBYTES_PER_BLOCK], nbytes_per_xfer, ivbn); if (!(status & 1)) { sys$setast(1); printf("Could not start input from VBN %0d to buffer #%0d.\n", ivbn, ibuf); return status; } ivbn += nblocks_per_xfer; nreads_outstanding++; sys$setast(1); } /* * Wait for the disk reads and writes to complete. */ sys$waitfr(iefn); lib$show_timer(0); printf(" Total Fast I/O Reads: %0d; Writes: %0d.\n", ifab.fab$l_ctx, ofab.fab$l_ctx); printf("Maximum Outstanding Reads: %0d; Writes: %0d.\n", max_reads_outstanding, max_writes_outstanding); /* * Deassign the input channel. */ status = sys$dassgn(ifab.fab$l_stv); if (!(status & 1)) { printf("Could not deassign input channel.\n"); return status; } /* * Completely re-initialize the file information block. The file ID and * directory ID fields must be zeroed, or the XQP will attempt to truncate * and modify the attributes of the *original* or *previous* file, *not* * the file that is currently accessed on the channel. */ bzero((char*)&fib, sizeof(fib)); fib.fib$l_acctl = FIB$M_WRITETHRU; fib.fib$w_exctl = FIB$M_TRUNC; fib.fib$l_exsz = 0; fib.fib$l_exvbn = xabfhc.xab$l_ebk; if (xabfhc.xab$w_ffb) fib.fib$l_exvbn++; fib.fib$w_verlimit = 0; fib.fib$l_acl_status = 0; /* * Copy the file header characteristics to the file attributes. */ fat.fat$b_rtype = xabfhc.xab$b_rfo; fat.fat$b_rattrib = xabfhc.xab$b_atr; fat.fat$w_rsize = xabfhc.xab$w_lrl; fat.fat$w_hiblkl = xabfhc.xab$w_hbk0; /* ignored */ fat.fat$w_hiblkh = xabfhc.xab$w_hbk2; /* ignored */ fat.fat$w_efblkl = xabfhc.xab$w_ebk0; fat.fat$w_efblkh = xabfhc.xab$w_ebk2; fat.fat$w_ffbyte = xabfhc.xab$w_ffb; fat.fat$b_bktsize = xabfhc.xab$b_bkz; fat.fat$b_vfcsize = xabfhc.xab$b_hsz; fat.fat$w_maxrec = xabfhc.xab$w_mrz; fat.fat$w_defext = xabfhc.xab$w_dxq; fat.fat$w_gbc = xabfhc.xab$w_gbc; fat.fat$w_versions = xabfhc.xab$w_verlimit; /* * Modify and truncate the file. */ status = sys$qiow(iefn, ofab.fab$l_stv, IO$_MODIFY, iosb, 0, 0, &fibdsc, 0, 0, 0, &atrlst, 0); if (status & 1) status = iosb[0]; if (status & 1) { printf("Output file truncated at VBN %0u by %0u blocks.\n", fib.fib$l_exvbn, fib.fib$l_exsz); printf("New end-of-file block is %0u, first free byte is %0u.\n", xabfhc.xab$l_ebk, xabfhc.xab$w_ffb); } else { printf("Could not truncate output file to EBK %0u, FFB %0u.\n", xabfhc.xab$l_ebk, xabfhc.xab$w_ffb); } /* * Deassign the output channel. */ status = sys$dassgn(ofab.fab$l_stv); if (!(status & 1)) { printf("Could not deassign output channel.\n"); return status; } } /* input file search loop */ if ((status != RMS$_NMF) && !(status & 1)) { unsigned int msgvec[3]; msgvec[0] = 2; msgvec[1] = ifab.fab$l_sts; msgvec[2] = ifab.fab$l_stv; printf("Could not search for input file.\n"); sys$putmsg(msgvec,0,0,0); } /* * Clean up after buffer objects. * This apparently gets done at image exit anyway. */ for (ibuf=0; ibuf