From:	SMTP%"munroe@DMC.COM"  6-JUN-1994 09:25:34.27
To:	EVERHART
CC:	
Subj:	Patch: Limit the size of NNTP batch collections.

Message-Id: <9406042254.AA06213@uu7.psi.com>
Date:         Sat, 4 Jun 1994 11:27:32 EDT
Reply-To: munroe@DMC.COM
Sender: ANU-NEWS Discussion <ANU-NEWS@VM1.NoDak.EDU>
From: "FABRICATI DIEM, PVNC!" <munroe@DMC.COM>
Organization: Doyle, Munroe Consultants, Inc., Hudson, MA
Subject:      Patch: Limit the size of NNTP batch collections.
To: Multiple recipients of list ANU-NEWS <ANU-NEWS@VM1.NoDak.EDU>

Well, I decided there WASN'T any way to limit NNTP id collection
sizes, so I implemented one.  If the news.sys entry is of the
form:

...:N100:...

NEWS will limit the number of items it collects in a single file
for NNTP to 100 (sometimes somewhat larger, but never smaller).
The attached VMSSHARE file contains the patches to the sources of
6.1B8 necessary to implement this:
$! ------------------ CUT HERE -----------------------
$ v='f$verify(f$trnlnm("SHARE_VERIFY"))'
$!
$! This archive created by VMS_SHARE Version 7.2-007  22-FEB-1990
$!   On  4-JUN-1994 11:24:56.19   By user MUNROE
$!
$! This VMS_SHARE Written by:
$!    Andy Harper, Kings College London UK
$!
$! Acknowledgements to:
$!    James Gray       - Original VMS_SHARE
$!    Michael Bednarek - Original Concept and implementation
$!
$! TO UNPACK THIS SHARE FILE, CONCATENATE ALL PARTS IN ORDER
$! AND EXECUTE AS A COMMAND PROCEDURE  (  @name  )
$!
$! THE FOLLOWING FILE(S) WILL BE CREATED AFTER UNPACKING:
$!       1. NEWSDEFINE.DIFF;1
$!       2. NEWSDIST.DIFF;1
$!       3. NEWSSITE.DIFF;1
$!
$set="set"
$set symbol/scope=(nolocal,noglobal)
$f=f$parse("SHARE_TEMP","SYS$SCRATCH:.TMP_"+f$getjpi("","PID"))
$e="write sys$error  ""%UNPACK"", "
$w="write sys$output ""%UNPACK"", "
$ if f$trnlnm("SHARE_LOG") then $ w = "!"
$ ve=f$getsyi("version")
$ if ve-f$extract(0,1,ve) .ges. "4.4" then $ goto START
$ e "-E-OLDVER, Must run at least VMS 4.4"
$ v=f$verify(v)
$ exit 44
$UNPACK: SUBROUTINE ! P1=filename, P2=checksum
$ if f$search(P1) .eqs. "" then $ goto file_absent
$ e "-W-EXISTS, File ''P1' exists. Skipped."
$ delete 'f'*
$ exit
$file_absent:
$ if f$parse(P1) .nes. "" then $ goto dirok
$ dn=f$parse(P1,,,"DIRECTORY")
$ w "-I-CREDIR, Creating directory ''dn'."
$ create/dir 'dn'
$ if $status then $ goto dirok
$ e "-E-CREDIRFAIL, Unable to create ''dn'. File skipped."
$ delete 'f'*
$ exit
$dirok:
$ w "-I-PROCESS, Processing file ''P1'."
$ if .not. f$verify() then $ define/user sys$output nl:
$ EDIT/TPU/NOSEC/NODIS/COM=SYS$INPUT 'f'/OUT='P1'
PROCEDURE Unpacker ON_ERROR ENDON_ERROR;SET(FACILITY_NAME,"UNPACK");SET(
SUCCESS,OFF);SET(INFORMATIONAL,OFF);f:=GET_INFO(COMMAND_LINE,"file_name");b:=
CREATE_BUFFER(f,f);p:=SPAN(" ")@r&LINE_END;POSITION(BEGINNING_OF(b));
LOOP EXITIF SEARCH(p,FORWARD)=0;POSITION(r);ERASE(r);ENDLOOP;POSITION(
BEGINNING_OF(b));g:=0;LOOP EXITIF MARK(NONE)=END_OF(b);x:=ERASE_CHARACTER(1);
IF g=0 THEN IF x="X" THEN MOVE_VERTICAL(1);ENDIF;IF x="V" THEN APPEND_LINE;
MOVE_HORIZONTAL(-CURRENT_OFFSET);MOVE_VERTICAL(1);ENDIF;IF x="+" THEN g:=1;
ERASE_LINE;ENDIF;ELSE IF x="-" THEN IF INDEX(CURRENT_LINE,"+-+-+-+-+-+-+-+")=
1 THEN g:=0;ENDIF;ENDIF;ERASE_LINE;ENDIF;ENDLOOP;t:="0123456789ABCDEF";
POSITION(BEGINNING_OF(b));LOOP r:=SEARCH("`",FORWARD);EXITIF r=0;POSITION(r);
ERASE(r);x1:=INDEX(t,ERASE_CHARACTER(1))-1;x2:=INDEX(t,ERASE_CHARACTER(1))-1;
COPY_TEXT(ASCII(16*x1+x2));ENDLOOP;WRITE_FILE(b,GET_INFO(COMMAND_LINE,
"output_file"));ENDPROCEDURE;Unpacker;QUIT;
$ delete/nolog 'f'*
$ CHECKSUM 'P1'
$ IF CHECKSUM$CHECKSUM .eqs. P2 THEN $ EXIT
$ e "-E-CHKSMFAIL, Checksum of ''P1' failed."
$ ENDSUBROUTINE
$START:
$ create 'f'
X*** user4:`5Banu-news.beta8.news_src`5Dnewsdefine.h.3
X--- newsdefine.h
X**************
X*** 19,24
X  **      - added IDMAP struct and NEWS_`5BM`7CV`5D_ITMIDSSET for message-ID
V handling
X  **    6.1b7  Charles Bailey  bailey@genetics.upenn.edu
X  **      - added NEWS_GRPFIL_RSZ preprocessor symbol
X  **--
X  **/
X  `0C
X--- 19,26 -----
X  **      - added IDMAP struct and NEWS_`5BM`7CV`5D_ITMIDSSET for message-ID
V handling
X  **    6.1b7  Charles Bailey  bailey@genetics.upenn.edu
X  **      - added NEWS_GRPFIL_RSZ preprocessor symbol
X+ **    6.1b8  Dick Munroe munroe@dmc.com
X+ **`09- added estimated article count.
X  **--
X  **/
X  `0C
X**************
X*** 442,447
X      char *sys_type;
X      char *sys_file;
X      int sys_batch_size;
X      int sys_article_count;
X      int sys_article_bytes;
X      int sys_id_count;`09`09/* IHAVE ID's */
X--- 444,450 -----
X      char *sys_type;
X      char *sys_file;
X      int sys_batch_size;
X+     int sys_estimated_article_count ;
X      int sys_article_count;
X      int sys_article_bytes;
X      int sys_id_count;`09`09/* IHAVE ID's */
$ CALL UNPACK NEWSDEFINE.DIFF;1 1510433691
$ create 'f'
X*** user4:`5Banu-news.beta8.news_src`5Dnewsdist.c;5
X--- newsdist.c
X**************
X*** 69,74
X  **`09V6.1b8`0928-Jan-1994`09terry@spcvxb.spc.edu  (Terry Kennedy)
X  **`09    - Fixed string overrun in sys_remote_send which clobbered stat
X  **`09      buffer for item file.
X  **--
X  **/
X  `0C
X--- 69,79 -----
X  **`09V6.1b8`0928-Jan-1994`09terry@spcvxb.spc.edu  (Terry Kennedy)
X  **`09    - Fixed string overrun in sys_remote_send which clobbered stat
X  **`09      buffer for item file.
X+ **`09     `20
X+ **`09V6.1b8`0904-Jun-1994`09munroe@dmc.com
X+ **`09    - Rig a mechanism for limiting the sizes of individual NNTP batch
Ves.
X+ **`09      The Type of Batch field can be of the form Nnumber where number
V is
X+ **`09      the number of articles to be included in each batch.
X  **--
X  **/
X  `0C
X**************
X*** 255,261
X `20
X    if (!type `7C`7C !*type `7C`7C *type == 'm') return(size);    `20
X    if (*type == 'b') ++type;
X!   if (*type == 'n') ++type;
X    if (isdigit(*type)) size = atoi(type);`09
X    return(size);
X  `7D
X--- 260,269 -----
X `20
X    if (!type `7C`7C !*type `7C`7C *type == 'm') return(size);    `20
X    if (*type == 'b') ++type;
X!   if (*type == 'n') `7B
X!     ++type;
X!     size = NEWS_NNTP_BATCH_SIZE;
X!     `7D
X    if (isdigit(*type)) size = atoi(type);`09
X    return(size);
X  `7D
X**************
X*** 410,416
X      `7D
X    tmp->sys_enext = 0;
X  /* wcf 7/22/92 */
X!   tmp->sys_article_count = tmp->sys_article_bytes = tmp->sys_id_count = 0;
X    if (!sysfile) sysfile = tmp;
X    else `7B
X      while (t1->sys_enext) t1 = t1->sys_enext;
X--- 418,424 -----
X      `7D
X    tmp->sys_enext = 0;
X  /* wcf 7/22/92 */
X!   tmp->sys_article_count = tmp->sys_article_bytes = tmp->sys_id_count = tm
Vp->sys_estimated_article_count = 0;
X    if (!sysfile) sysfile = tmp;
X    else `7B
X      while (t1->sys_enext) t1 = t1->sys_enext;
X**************
X*** 933,938
X  `09      if (!(fpo = fopen(s->sys_file,"a","shr=nil","mbc=64")))
X  `09`09fpo = fopen(s->sys_file,"w","shr=nil","mbc=64");
X  `09      `7D
X  #if !CLOSE_BATCHFILES
X  `09    s->sys_fp = fpo;
X  #endif
X--- 941,973 -----
X  `09      if (!(fpo = fopen(s->sys_file,"a","shr=nil","mbc=64")))
X  `09`09fpo = fopen(s->sys_file,"w","shr=nil","mbc=64");
X  `09      `7D
X+ `09    else `7B
X+ `09      /*
X+ `09      ** Assume that each entry in the collection file is about 40 byte
Vs
X+ `09      ** long.  From inspection this seems to be about the right number
V.
X+ `09      ** Only do this calculation once, when the file is opened.  After
X+ `09      ** that it's redundant.
X+ `09      */
X+ `09     `20
X+ `09      fstat(fileno(fpo), &s->sys_statb);
X+ `09      s->sys_estimated_article_count = (s->sys_statb.st_size + 39) / 40
V ;
X+ `09      `7D
X+ #if !CLOSE_BATCHFILES
X+ `09    s->sys_fp = fpo;
X+ #endif
X+ `09    `7D
X+ `09  if (((s->sys_id_count + s->sys_estimated_article_count) % s->sys_batc
Vh_size) == 0) `7B
X+ `09    /*
X+ `09    ** By our calculations, the number of articles in the current batch
X+ `09    ** file has overflowed the number specified in the news.sys file
X+ `09    ** entry.  Close the current batch, open a new one and adjust the
X+ `09    ** estimated article count so that we will write the correct number
X+ `09    ** of news items into the next batch.
X+ `09    */
X+ `09   `20
X+ `09    fclose(fpo) ;
X+ `09    fpo = fopen(s->sys_file,"w","shr=nil","mbc=64");
X+ `09    s->sys_estimated_article_count = - s->sys_id_count ;
X  #if !CLOSE_BATCHFILES
X  `09    s->sys_fp = fpo;
X  #endif
$ CALL UNPACK NEWSDIST.DIFF;1 1715022676
$ create 'f'
X*** user4:`5Banu-news.beta8.news_src`5Dnewssite.h;2
X--- newssite.h
X**************
X*** 37,42
X  #define NEWS_BATCH_SIZE 250000
X `20
X  /*
X   *  RECHECK_TIMER
X   *  The configuration files are periodically re-checked to ensure that any
V edit
X   *  changes made to the files are read into all active NEWS sessions. The
V timer
X--- 37,50 -----
X  #define NEWS_BATCH_SIZE 250000
X `20
X  /*
X+  *  NEWS_NNTP_BATCH_SIZE
X+  *  This is the maximum size of news batchs files for NNTP sites (in
X+  *  articles, not bytes).
X+  */
X+`20
X+ #define NEWS_NNTP_BATCH_SIZE 250
X+`20
X+ /*
X   *  RECHECK_TIMER
X   *  The configuration files are periodically re-checked to ensure that any
V edit
X   *  changes made to the files are read into all active NEWS sessions. The
V timer
$ CALL UNPACK NEWSSITE.DIFF;1 520435535
$ v=f$verify(v)
$ EXIT

--
Dick Munroe                             Internet: munroe@dmc.com
Acorn Software, Inc.                    UUCP: ...uunet!thehulk!munroe
267 Cox St.                             Office: (508) 568-1618
Hudson, Ma. 01749 USA                   FAX: (508) 562-1133

GET CONNECTED!!! Send mail to info@dmc.com to find out about DMConnection.
