[SLL] limit on number of files in a directory and hashed dir vs. flat dir file access time
Chuck Wolber
chuckw at quantumlinux.com
Mon Nov 5 08:12:04 PST 2007
Nice work Ana. I would post this to the LKML and see if anyone takes it
up.
..Chuck..
On Sat, 3 Nov 2007, Ana wrote:
> I'm attempting a new test, using a simple C program.
>
> It looks like there is a small increase with every 10x increase of the
> file count, until the 1,000,000 file count attempt. There's a 3x
> increase in system time spent between 100,000 and 1,000,000 file-count
> tests. At least that's how I'm reading the result. I'm open to other
> interpretations and to all suggestions about how to improve the test.
> If you want more information (about my system or whatnot) just ask.
>
> This test measures access time for many files in one directory. The
> random-open-close test chooses a random file name from a known set of
> existing file names and does an open() and close() on it. For every run
> of the test, the number of files in the directory increase by 10x, but
> the number of rand-open-close operations is constant at 100,000,000. I
> use the times() system call to measure time.
>
> (I don't know much about times()... I think the values returned
> correspond to jiffies, which increment (I think) during the timer
> interrupt. So, jiffies, I think, can freeze if interrupts are disabled
> for much time. Please correct me if I'm wrong.)
>
> A future version of this will divide the files into a hierarchy of
> directories.
>
> I hope this interests somebody.
>
> - Ana
>
>
>
> output from last night's test:
>
> -------------------------------------------------------------------------
> ana at betty:/mnt/var/tmp/test$ sh runlongertest.sh
> doing: 100
> Sat Nov 3 01:26:00 PDT 2007
> User Ticks spent: 8220
> Syst Ticks spent: 45222
> Sat Nov 3 01:34:55 PDT 2007
> doing: 1000
> Sat Nov 3 01:34:55 PDT 2007
> User Ticks spent: 8847
> Syst Ticks spent: 49277
> Sat Nov 3 01:44:36 PDT 2007
> doing: 10000
> Sat Nov 3 01:44:36 PDT 2007
> User Ticks spent: 7729
> Syst Ticks spent: 50677
> Sat Nov 3 01:54:20 PDT 2007
> doing: 100000
> Sat Nov 3 01:56:08 PDT 2007
> User Ticks spent: 8380
> Syst Ticks spent: 51541
> Sat Nov 3 02:06:07 PDT 2007
> doing: 1000000
> Sat Nov 3 06:07:10 PDT 2007
> User Ticks spent: 10207
> Syst Ticks spent: 1633793
> Sat Nov 3 13:05:33 PDT 2007
> ana at betty:/mnt/var/tmp/test$
> -------------------------------------------------------------------------
>
>
> the shell wrapper
> -------------------------------------------------------------------------
> function die
> {
> echo error
> exit 1
> }
>
>
> i=100
> while [ $i -lt 10000000 ];
> do
> echo doing: $i
> ./testjig create $i $i > /dev/null || die
> date
> ./testjig randopenclose 100000000 $i || die
> #./testjig randopenclose 1000 $i || die
> date
> let i*=10;
> done
> -------------------------------------------------------------------------
>
>
>
> the quickly conceived and written C program for the file operations
> -------------------------------------------------------------------------
> #include <stdlib.h>
> #include <stdio.h>
> #include <sys/types.h>
> #include <sys/stat.h>
> #include <fcntl.h>
> #include <unistd.h>
> #include <sys/times.h>
> #include <errno.h>
>
> #define MAXFILECNT 16777215
>
> #define OPTCREATE 1
> #define OPTDELETE 2
> #define OPTOPNCLS 3
> #define OPTRNDOPNCLS 4
> #define OPTSTAT 5
> #define OPTRNDSTAT 6
> #define OPTNOOP -1
>
> static int filename( char * fname, int n, int max );
> static int createfile( char * fname );
> static int do_option( int opt, int count, int max );
> static int parse_option( const char * opt );
>
> static void usage( void );
>
> static void start_timer( void );
> static void stop_timer( void );
> static void report_timer( void );
>
>
> int main(int argc, char ** argv)
> {
> int max = 0;
> int count = 0;
> int opt = OPTNOOP;
> int reslt = 0;
>
> if( argc < 4 )
> {
> usage();
> return 1;
> }
>
> opt = parse_option( argv[1] );
> if( opt == OPTNOOP )
> {
> usage();
> return 2;
> }
>
> count = atoi(argv[2]);
> if( count < 1 )
> {
> usage();
> return 3;
> }
>
> max = atoi(argv[3]);
> if( max < 1 )
> {
> usage();
> return 4;
> }
> else if( max > MAXFILECNT )
> {
> printf("specified max (%d) is too great; resetting to: %d\n", max, MAXFILECNT);
> max = MAXFILECNT;
> }
>
> srand(getpid()*time());
>
> start_timer();
>
> reslt = do_option( opt, count, max );
>
> stop_timer();
> report_timer();
>
> return reslt;
> }
>
> #define FNAMELENGH 128
> #if 1
> static int filename( char * fname, int n, int max )
> {
> int fnum = n % max;
> return (0 < snprintf(fname, FNAMELENGH, "files/%06x.t", fnum));
> }
> #else
> static int filename( char * fname, int n, int max )
> {
> char number[16];
> int fnum = n % max;
> int err = 0;
>
> err = snprintf(number, sizeof(number), "%06x", fnum);
> if( err < 6 )
> return 0;
>
> err = snprintf(fname, FNAMELENGH, "files/%c/%c/%c/%c/%c/%c/%s.t",
> number[0], number[1], number[2], number[3], number[4], number[5], number );
> if( err < 26 )
> return 0;
>
> return 1;
> }
> #endif
>
> static int createfile( char * fname )
> {
> int fd;
> if( -1 < (fd = open(fname, O_WRONLY|O_CREAT, 00777)) )
> close(fd);
> return (fd < 0);
> }
>
> static int openclose( char * fname )
> {
> int fd = -1;
> if( -1 < (fd = open(fname, O_RDONLY)) )
> close(fd);
> return (fd < 0);
> }
>
>
>
> static int do_option( int opt, int count, int max )
> {
> int i, fd, fnumber;
> int error = 0;
> char fname[FNAMELENGH];
> struct stat statbuf;
>
> if( opt == OPTCREATE && count != max )
> count = max;
>
> for( i = 0; i < count; i++ )
> {
> switch( opt )
> {
> case OPTCREATE:
> if( filename(fname, i, max) )
> createfile( fname );
> break;
> case OPTDELETE:
> if( filename(fname, i, max) )
> unlink( fname );
> break;
> case OPTOPNCLS:
> if( filename(fname, i, max) )
> error = openclose( fname );
> break;
> case OPTRNDOPNCLS:
> fnumber = 1 + (int) (max * (rand() / (RAND_MAX + 1.0)));
> if( filename(fname, fnumber, max) )
> error = openclose( fname );
> break;
> case OPTSTAT:
> if( filename(fname, i, max) )
> error = stat(fname, &statbuf);
> break;
> case OPTRNDSTAT:
> fnumber = 1 + (int) (max * (rand() / (RAND_MAX + 1.0)));
> if( filename(fname, fnumber, max) )
> error = stat(fname, &statbuf);
> break;
> default:
> errno = EINVAL;
> error = -1;
> }
>
> if( error )
> break;
> }
>
> if( error )
> printf("quitting because of error: %m\n");
>
> return 0;
> }
>
> static int parse_option( const char * opt )
> {
> if( 0 == strcmp( opt, "create" ) ) return OPTCREATE;
> if( 0 == strcmp( opt, "delete" ) ) return OPTDELETE;
> if( 0 == strcmp( opt, "openclose" ) ) return OPTOPNCLS;
> if( 0 == strcmp( opt, "randopenclose" ) ) return OPTRNDOPNCLS;
> if( 0 == strcmp( opt, "stat" ) ) return OPTSTAT;
> if( 0 == strcmp( opt, "randstat" ) ) return OPTRNDSTAT;
> return OPTNOOP;
> }
>
> static void usage( void )
> {
> printf("usage: tester <operation> ops_count max_fileno\n");
> printf(" operations: create delete openclose randopenclose stat randstat\n");
> }
>
>
>
>
> static struct tms timer_start; /* Stores the starting time*/
> static struct tms timer_stop; /* Stores the ending time*/
>
> static void start_timer( void )
> {
> times(&timer_start);
> }
>
> static void stop_timer( void )
> {
> times(&timer_stop);
> }
>
> static void report_timer( void )
> {
> printf("User Ticks spent: %ld\n", ( timer_stop.tms_utime - timer_start.tms_utime ));
> printf("Syst Ticks spent: %ld\n", ( timer_stop.tms_stime - timer_start.tms_stime ));
> }
> -------------------------------------------------------------------------
>
>
> !DSPAM:472cffa2221001787922047!
>
--
http://www.quantumlinux.com
Quantum Linux Laboratories, LLC.
ACCELERATING Business with Open Technology
"Stay Hungry. Stay Foolish."
-The Whole Earth Catalog
More information about the linux-list
mailing list