[SLL] limit on number of files in a directory and hashed dir vs. flat dir file access time
Ana
christiana at hipointcoffee.com
Sat Nov 3 16:09:11 PDT 2007
I'm attempting a new test, using a simple C program.
It looks like there is a small increase with every 10x increase of the
file count, until the 1,000,000 file count attempt. There's a 3x
increase in system time spent between 100,000 and 1,000,000 file-count
tests. At least that's how I'm reading the result. I'm open to other
interpretations and to all suggestions about how to improve the test.
If you want more information (about my system or whatnot) just ask.
This test measures access time for many files in one directory. The
random-open-close test chooses a random file name from a known set of
existing file names and does an open() and close() on it. For every run
of the test, the number of files in the directory increase by 10x, but
the number of rand-open-close operations is constant at 100,000,000. I
use the times() system call to measure time.
(I don't know much about times()... I think the values returned
correspond to jiffies, which increment (I think) during the timer
interrupt. So, jiffies, I think, can freeze if interrupts are disabled
for much time. Please correct me if I'm wrong.)
A future version of this will divide the files into a hierarchy of
directories.
I hope this interests somebody.
- Ana
output from last night's test:
-------------------------------------------------------------------------
ana at betty:/mnt/var/tmp/test$ sh runlongertest.sh
doing: 100
Sat Nov 3 01:26:00 PDT 2007
User Ticks spent: 8220
Syst Ticks spent: 45222
Sat Nov 3 01:34:55 PDT 2007
doing: 1000
Sat Nov 3 01:34:55 PDT 2007
User Ticks spent: 8847
Syst Ticks spent: 49277
Sat Nov 3 01:44:36 PDT 2007
doing: 10000
Sat Nov 3 01:44:36 PDT 2007
User Ticks spent: 7729
Syst Ticks spent: 50677
Sat Nov 3 01:54:20 PDT 2007
doing: 100000
Sat Nov 3 01:56:08 PDT 2007
User Ticks spent: 8380
Syst Ticks spent: 51541
Sat Nov 3 02:06:07 PDT 2007
doing: 1000000
Sat Nov 3 06:07:10 PDT 2007
User Ticks spent: 10207
Syst Ticks spent: 1633793
Sat Nov 3 13:05:33 PDT 2007
ana at betty:/mnt/var/tmp/test$
-------------------------------------------------------------------------
the shell wrapper
-------------------------------------------------------------------------
function die
{
echo error
exit 1
}
i=100
while [ $i -lt 10000000 ];
do
echo doing: $i
./testjig create $i $i > /dev/null || die
date
./testjig randopenclose 100000000 $i || die
#./testjig randopenclose 1000 $i || die
date
let i*=10;
done
-------------------------------------------------------------------------
the quickly conceived and written C program for the file operations
-------------------------------------------------------------------------
#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/times.h>
#include <errno.h>
#define MAXFILECNT 16777215
#define OPTCREATE 1
#define OPTDELETE 2
#define OPTOPNCLS 3
#define OPTRNDOPNCLS 4
#define OPTSTAT 5
#define OPTRNDSTAT 6
#define OPTNOOP -1
static int filename( char * fname, int n, int max );
static int createfile( char * fname );
static int do_option( int opt, int count, int max );
static int parse_option( const char * opt );
static void usage( void );
static void start_timer( void );
static void stop_timer( void );
static void report_timer( void );
int main(int argc, char ** argv)
{
int max = 0;
int count = 0;
int opt = OPTNOOP;
int reslt = 0;
if( argc < 4 )
{
usage();
return 1;
}
opt = parse_option( argv[1] );
if( opt == OPTNOOP )
{
usage();
return 2;
}
count = atoi(argv[2]);
if( count < 1 )
{
usage();
return 3;
}
max = atoi(argv[3]);
if( max < 1 )
{
usage();
return 4;
}
else if( max > MAXFILECNT )
{
printf("specified max (%d) is too great; resetting to: %d\n", max, MAXFILECNT);
max = MAXFILECNT;
}
srand(getpid()*time());
start_timer();
reslt = do_option( opt, count, max );
stop_timer();
report_timer();
return reslt;
}
#define FNAMELENGH 128
#if 1
static int filename( char * fname, int n, int max )
{
int fnum = n % max;
return (0 < snprintf(fname, FNAMELENGH, "files/%06x.t", fnum));
}
#else
static int filename( char * fname, int n, int max )
{
char number[16];
int fnum = n % max;
int err = 0;
err = snprintf(number, sizeof(number), "%06x", fnum);
if( err < 6 )
return 0;
err = snprintf(fname, FNAMELENGH, "files/%c/%c/%c/%c/%c/%c/%s.t",
number[0], number[1], number[2], number[3], number[4], number[5], number );
if( err < 26 )
return 0;
return 1;
}
#endif
static int createfile( char * fname )
{
int fd;
if( -1 < (fd = open(fname, O_WRONLY|O_CREAT, 00777)) )
close(fd);
return (fd < 0);
}
static int openclose( char * fname )
{
int fd = -1;
if( -1 < (fd = open(fname, O_RDONLY)) )
close(fd);
return (fd < 0);
}
static int do_option( int opt, int count, int max )
{
int i, fd, fnumber;
int error = 0;
char fname[FNAMELENGH];
struct stat statbuf;
if( opt == OPTCREATE && count != max )
count = max;
for( i = 0; i < count; i++ )
{
switch( opt )
{
case OPTCREATE:
if( filename(fname, i, max) )
createfile( fname );
break;
case OPTDELETE:
if( filename(fname, i, max) )
unlink( fname );
break;
case OPTOPNCLS:
if( filename(fname, i, max) )
error = openclose( fname );
break;
case OPTRNDOPNCLS:
fnumber = 1 + (int) (max * (rand() / (RAND_MAX + 1.0)));
if( filename(fname, fnumber, max) )
error = openclose( fname );
break;
case OPTSTAT:
if( filename(fname, i, max) )
error = stat(fname, &statbuf);
break;
case OPTRNDSTAT:
fnumber = 1 + (int) (max * (rand() / (RAND_MAX + 1.0)));
if( filename(fname, fnumber, max) )
error = stat(fname, &statbuf);
break;
default:
errno = EINVAL;
error = -1;
}
if( error )
break;
}
if( error )
printf("quitting because of error: %m\n");
return 0;
}
static int parse_option( const char * opt )
{
if( 0 == strcmp( opt, "create" ) ) return OPTCREATE;
if( 0 == strcmp( opt, "delete" ) ) return OPTDELETE;
if( 0 == strcmp( opt, "openclose" ) ) return OPTOPNCLS;
if( 0 == strcmp( opt, "randopenclose" ) ) return OPTRNDOPNCLS;
if( 0 == strcmp( opt, "stat" ) ) return OPTSTAT;
if( 0 == strcmp( opt, "randstat" ) ) return OPTRNDSTAT;
return OPTNOOP;
}
static void usage( void )
{
printf("usage: tester <operation> ops_count max_fileno\n");
printf(" operations: create delete openclose randopenclose stat randstat\n");
}
static struct tms timer_start; /* Stores the starting time*/
static struct tms timer_stop; /* Stores the ending time*/
static void start_timer( void )
{
times(&timer_start);
}
static void stop_timer( void )
{
times(&timer_stop);
}
static void report_timer( void )
{
printf("User Ticks spent: %ld\n", ( timer_stop.tms_utime - timer_start.tms_utime ));
printf("Syst Ticks spent: %ld\n", ( timer_stop.tms_stime - timer_start.tms_stime ));
}
-------------------------------------------------------------------------
More information about the linux-list
mailing list