Everhart,Glenn From: mckinneyj@cpgtsa.dcs.saic.com Sent: Friday, April 03, 1998 4:21 PM To: Info-VAX@Mvb.Saic.Com Subject: Re: Deleting >46,000 files In article <35224CB3.577B@ix.netcom.com>, Dominic Olivastro writes: > A user just created a command procedure to do the following: > > 1) Delete all files in a working directory > 2) Create new files in that directory > 3) Process those files. > > This procedure is repeated weekly. > > It looks simple enough except for one problem. In step 2 he created > over 46000 files! So the second week he ran it, it was stuck in step 1 > for over a day, and still he had deleted only the "A*.*;*" files > > Two questions: > > 1. How do I get rid of these files? > > 2. What is a better way to proceded? Should I, eg, have him make not 1 > directory with 46000 files, but find a logical way to break them up > into, say, 46 directories with 1000 files each? Would that help? > > Dom > > -- > Dominic Olivastro email: ChiRsrch@ix.netcom.com > CHI Research, Inc visit our homepage: > http://www.ChiResearch.com > Many have commented on how to rid yourself of the files; some suggesting that you write a program to reverse delete the contents of the directory. Here's some code that I once wrote to do just that... has "minor" flaws in that it is unable to handle logical search lists as part of the input spec and also doesn't handle elipsis. Never found the time to add these but it works for me. #ifdef __DECC #pragma module DELETE_STAR_DOT_STAR "V2.0" #else #module DELETE_STAR_DOT_STAR "V2.0" #endif /* DELETE_STAR_DOT_STAR.C * * hack to delete all files from a directory * * open the directory, read in the file names and sort it in decending * order so that we can trim files from the back end of the directory * first and avoid the overhead of RMS repeatedly compressing out * the spaces vacated by filenames should we delete in ascending order * * This 2nd version of the program directly call RMS's sys$erase for * each version of the filespec rather than lib$delete_file once for * a wildcarded version of filespec. sys$erase proves to be significantly * faster and less I/O intensive. * * Note: no support yet for search list logicals as part of filespec; * be specific in what you ask for; relative filespecs are ok. * * $ CC/PREFIX=ALL DELETE_STAR_DOT_STAR * $ LINK/NOTRACE DELETE_STAR_DOT_STAR * $ DSDS :== $SYS$DISK:[]DELETE_STAR_DOT_STAR * $ DSDS ddcu:[a.b.c] * */ /* #define DEBUG 1 /* define DEBUG for verbose mode */ #include #include #include #include #include #include #include #include #include #include /* some VAX DECC SSDEF.H header files don't include SS$_INVARG */ #ifndef SS$_INVARG #define SS$_INVARG 4042 #endif #include #include /* * A valid directory entry consists of 7FFF in the first word, the length of * the file name string in the second byte of next word followed immediately * by the file name string... if the file name string ends on a word boundary * then the next word is the version else skip a byte and next word is version. * This version number will be the highest version of the file. The next three * words will comprise the FID. If there are multiple versions of the file * then a quadword bucket (word_version,3word_FID) will follow for each * other version of the file. The buckets are in order of decending version. * * DIRECTORY RECORD is variable length depending upon multiple versions of file * WORD = %x7FFF * BYTE * BYTE = (byte count of file name) * ASCII = (file name - must be even number of bytes with %x00 pad) * WORD = (version number) * WORD*3 = (File ID) * WORD = (version number) ...if necessary * WORD*3 = (File ID) ...if necessary * WORD = (version number) ...if necessary * WORD*3 = (File ID) ...if necessary * ... ...if necessary */ #define PROLOG 4 /* WORD+BYTE+BYTE before filename */ #define VER 2 /* size of VER in directory record */ #define FID 6 /* size of FID in directory record */ #define BLOCK 512 /* block size */ #define NAME 39 /* max length of file name */ #define EXT 39 /* max length of file extension */ #define DIR 3 /* length of .DIR file extension */ #define VSN 6 /* max length of version (;nnnnn) */ void delete_file(); int main(int argc, char *argv[]) /* argv[1]=DirectoryName */ { int r0; /* return status */ int fd; /* file descriptor for OPEN */ int bytes; /* bytes read */ short *p0; /* a handy pointer to short */ char *p1, *p2; /* a couple of char pointers */ char fna[BLOCK], temp[BLOCK]; /* input parse areas */ char filespec[NAME+1+EXT+VSN+1]; /* filename+ext+version+NUL */ union /* the Directory record */ { char rec0[BLOCK]; /* in it's entirety */ struct /* and, the important stuff */ { short flag; /* always %x7FFF */ char filler; char count; /* length of filename */ char filename[NAME+1+EXT]; /* filename+.+ext */ } rec1; } buffer; #define record buffer.rec0 #define count buffer.rec1.count #define filename buffer.rec1.filename /* SORT control buffer */ short key_buffer[5] = { 1, /* number of keys */ DSC$K_DTYPE_T, /* descriptor type */ 1, /* order (0=ascend,1=descend) */ 0, /* offset of key */ sizeof(filespec)-1 }; /* length of key */ short lrl = sizeof(filespec) - 1; /* longest record length */ struct dsc$descriptor sor_desc; /* sort I/O record descriptor */ sor_desc.dsc$w_length = sizeof(filespec) - 1; /* setup descriptor */ sor_desc.dsc$b_dtype = DSC$K_DTYPE_T; sor_desc.dsc$b_class = DSC$K_CLASS_S; sor_desc.dsc$a_pointer = filespec; if (argc != 2) /* get dir from command line */ { printf("Usage: $ D(elete)S(tar)D(ot)S(tar) ddcu:[directory]\n"); return(SS$_INSFARG); } if (strstr(argv[1], "...") != NULL) /* ellipsis illegal */ { printf("%%DSDS-E-ILLEGAL_ELLIPSIS, use of ellipsis illegal\n"); return(SS$_INVARG); } /* parse the input and reformat the directory spec into a file spec */ strcpy(temp, argv[1]); /* make a temporary copy */ if (strstr(temp, "]") == NULL) /* it's not a directory */ return(SS$_BADIRECTORY); else { strtok(temp, "]"); /* get rid of the end "]" */ if (strstr(temp, ".") == NULL) /* relative top lvl directory */ { p1 = temp; /* from the start of string */ p2 = strstr(temp, "["); /* locate the "[" */ if (p1 == p2) /* if it's first character */ { /* then */ p1++; /* just shift it over one */ strcpy(fna, p1); /* character and we're done */ } else { /* else */ *p2 = '\0'; /* replace it with a NUL */ strcpy(fna, p1); /* use device (it's first) */ p2++; /* and directory (after "[") */ strcat(fna, p2); /* to create file spec */ } strcat(fna, ".DIR"); /* finally add extension */ } else /* in relative subdirectory */ { p2 = p1 = temp; /* start at beginning */ p1 = temp + strlen(temp) - 1; /* to find end */ for (p1; p1 > p2; p1--) /* and moving backwards */ { if (*p1 == '.') *p1 = ']'; /* find the "." and */ if (*p1 == ']') break; /* replace it with "]" */ } strcpy(fna, temp); /* move into final buffer */ strcat(fna, ".DIR"); /* and add extension */ } } #ifdef DEBUG printf("%%DEBUG-I-DIROPEN, Opening directory %s\n", fna); #endif fd = open(fna, O_RDONLY, 0); /* now try and open it */ if (fd == -1) return(RMS$_DNF); #ifdef DEBUG printf("%%DEBUG-I-SORINIT, Initializing SORT\n"); #endif r0 = sor$begin_sort(&key_buffer, &lrl, 0,0,0,0,0,0,0 ); /* initialize sort */ if (!(r0 & 1)) return(r0); #ifdef DEBUG printf("%%DEBUG-I-READDIR, Reading directory records\n"); #endif memset(record, '\0', sizeof(record)); /* NUL fill the buffer */ while ((bytes = read(fd, record, sizeof(record))) > 0) /* read directory */ { /* and, */ memset(filespec, '\0', sizeof(filespec)); /* NUL fill buffer */ p1 = record + PROLOG + count; /* locate version */ if (count & 1) p1++; /* if count is odd */ for(p1; p1-record < bytes; p1=p1+FID+VER) /* get each version */ { p0 = (short *) p1; strncpy(filespec, filename, count); /* start building filespec */ sprintf(filespec, "%s;%d", filespec, *p0); /* add version */ sor_desc.dsc$w_length = strlen(filespec); /* fixup descriptor */ #ifdef DEBUG printf("%%DEBUG-I-FOUNDFILE, Found filespec %s\n", filespec); #endif r0 = sor$release_rec(&sor_desc, 0); /* give SORT the filespec */ if (!(r0 & 1)) return(r0); memset(filespec, '\0', sizeof(filespec)); /* NUL fill buffer */ } memset(record, '\0', sizeof(record)); /* NUL fill the buffer */ } #ifdef DEBUG printf("%%DEBUG-I-DIRCLOSE, Closing directory %s\n", fna); #endif close(fd); /* done with input directory */ #ifdef DEBUG printf("%%DEBUG-I-SORT, Performing SORT\n"); #endif r0 = sor$sort_merge(0); /* sort dir descending order */ if (!(r0 & 1)) return(r0); #ifdef DEBUG printf("%%DEBUG-I-RETRIEVE, Retrieving SORTed records\n"); #endif for (;;) { sor_desc.dsc$w_length = sizeof(filespec); /* fixup descriptor */ r0 = sor$return_rec(&sor_desc, 0, 0); /* receive a record back */ if (r0 == SS$_ENDOFFILE) break; if (!(r0 & 1)) return(r0); #ifdef DEBUG printf("%%DEBUG-I-DELETE, Deleting file %s\n",filespec); #endif delete_file(filespec, argv[1]); } #ifdef DEBUG printf("%%DEBUG-I-SORTCOMP, SORT complete\n"); #endif r0 = sor$end_sort(0); /* clean up after the sort */ if (!(r0 & 1)) return(r0); #ifdef DEBUG printf("%%DEBUG-I-PROCCOMP, Processing complete\n"); #endif return(SS$_NORMAL); /* done */ } /* delete the file */ void delete_file(char *fna, char *dna) { int r0; struct FAB fab; struct { short argc; short options; long desc[2]; } msgvec; fab = cc$rms_fab; /* initialize fab */ fab.fab$l_fna = fna; /* set fna */ fab.fab$b_fns = strlen(fna); /* and fns */ fab.fab$l_dna = dna; /* set dna */ fab.fab$b_dns = strlen(dna); /* and fns */ r0 = sys$erase(&fab); /* for the erase service */ if (r0 != RMS$_NORMAL) /* if delete fails */ { msgvec.argc = 2; /* load a RMS message vector */ msgvec.options = 15; /* with all 4 bits set */ msgvec.desc[0] = fab.fab$l_sts; /* status */ msgvec.desc[1] = fab.fab$l_stv; /* and status value */ printf("%%DSDS-W-FILNOTDEL, error deleting %s\n", fna); sys$putmsg(&msgvec, 0, 0, 0); /* and deliver the message */ } return; }