Skip Menu |

Preferred bug tracker

Please visit the preferred bug tracker to report your issue.

This queue is for tickets about the Email-Delete CPAN distribution.

Report information
The Basics
Id: 33912
Status: resolved
Priority: 0/
Queue: Email-Delete

People
Owner: Nobody in particular
Requestors: whynot [...] pozharski.name
Cc:
AdminCc:

Bug Information
Severity: Important
Broken in: (no value)
Fixed in: (no value)



Subject: E::D::Maildir is B<very> slow
Subject: bugfix
diff -ur orig.Email-Delete-1.022/META.yml bugfix.Email-Delete-1.022/META.yml --- orig.Email-Delete-1.022/META.yml 2007-02-15 15:24:47.000000000 +0200 +++ bugfix.Email-Delete-1.022/META.yml 2008-03-03 00:07:16.000000000 +0200 @@ -10,7 +10,6 @@ Email::FolderType: 0.7 Email::LocalDelivery: 0.213 Email::Simple: 1.92 - File::Find::Rule: 0.28 Test::More: 0.47 meta-spec: url: http://module-build.sourceforge.net/META-spec-v1.2.html diff -ur orig.Email-Delete-1.022/Makefile.PL bugfix.Email-Delete-1.022/Makefile.PL --- orig.Email-Delete-1.022/Makefile.PL 2007-02-15 15:22:48.000000000 +0200 +++ bugfix.Email-Delete-1.022/Makefile.PL 2008-03-03 00:07:16.000000000 +0200 @@ -10,7 +10,6 @@ 'Email::FolderType' => '0.7', 'Email::LocalDelivery' => '0.213', # avoid 0.21[12], mangle tests 'Email::Simple' => '1.92', - 'File::Find::Rule' => '0.28', 'Test::More' => '0.47', }, VERSION_FROM => 'lib/Email/Delete.pm', diff -ur orig.Email-Delete-1.022/lib/Email/Delete/Maildir.pm bugfix.Email-Delete-1.022/lib/Email/Delete/Maildir.pm --- orig.Email-Delete-1.022/lib/Email/Delete/Maildir.pm 2006-08-14 19:34:48.000000000 +0300 +++ bugfix.Email-Delete-1.022/lib/Email/Delete/Maildir.pm 2008-03-07 17:39:57.000000000 +0200 @@ -5,24 +5,24 @@ use vars qw[$VERSION]; $VERSION = sprintf "%d.%02d", split m/\./, (qw$Revision: 1.1 $)[1]; -use File::Find::Rule; use Email::Simple; sub delete_message { my %args = @_; - my @files = File::Find::Rule->file - ->grep(sub { - local $_ = shift; - local *MSG; - open MSG, $_ or return; - my $msg = Email::Simple->new(do{ - local $/; <MSG>; - }); - close MSG; - $args{matching}->($msg); - }) - ->in($args{from}); + my @files; +# Whatever in F<tmp/> is undelivered yet, right? + foreach my $sect ( qw( new cur ) ) { +# What if C<$args{from}> is something but directory? Never mind, just skip it. + opendir my($dh), "$args{from}/$sect" or next; + while(my $mail = readdir $dh) { +# Faild to open subfolder? Here? Immaterial, go away. + -f "$args{from}/$sect/$mail" or next; + open my $fh, '<', "$args{from}/$sect/$mail" or next; + my $msg = Email::Simple->new(do { local $/; <$fh>; }) or next; + $args{matching}->($msg) and push @files, "$args{from}/$sect/$mail"; + }; + }; return unlink @files; }
Subject: bugsample
#!/usr/bin/perl # $Id$ package main; use strict; use warnings; use vars qw($VERSION); $VERSION = q(0.0.0); use File::Temp qw(tempfile tempdir); use Email::Simple; use Email::Simple::Creator; use Email::LocalDelivery; use Email::Delete; use Benchmark qw(timethese cmpthese timediff); use Getopt::Std; our($opt_a, $opt_b, $opt_d, $opt_f, $opt_m, $opt_r, $opt_x); my $body = 'x' x 72 . "\n"; sub dump_stuff ($$) { my($lib, $mbox) = ( @_ ); my $stuff = Email::Simple->create( header => [ From => q(nobody@localhost.localdomain), To => q(adm@localhost.localdomain), ], body => $body x (256 * $opt_x),); for my $flag (0 .. $opt_m - 1) { $stuff->header_set(Subject => qq(This is $flag mail for $lib)); $stuff->header_set(q(X-Bugsample) => $flag % $opt_f ? 0 : 1); Email::LocalDelivery->deliver($stuff->as_string, $mbox); }; }; sub bench_stuff ($$) { defined(my $pid = fork) or die qq|can't fork: $!\n|; $pid and return; my $lib = shift @_; my $arch = shift @_; my $mbox; if($arch eq q(mbox)) { (undef, $mbox) = tempfile qq(ed-bugsample.$lib.XXXXXX), DIR => q(/tmp); } elsif($arch eq q(maildir)) { $mbox = tempdir qq(ed-bugsample.$lib.XXXXXX), DIR => q(/tmp); mkdir qq($mbox/$_) foreach(qw(tmp new cur)); }; dump_stuff $lib, $mbox; eval qq|use lib q(./$lib.Email-Delete-1.022/blib/lib);|; if($lib ne q(none) && !$opt_a) { for(1 .. ($opt_m + 1)/$opt_f) { my $flag = 0; Email::Delete::delete_message( from => $mbox, matching => sub { warn q(looking at: ), $_[0]->header(q(Subject)), "\n" if($ENV{VERBOSE}); $flag and return; return ++$flag if(int shift(@_)->header(q(X-Bugsample))); return; }); }; } elsif($lib ne q(none)) { Email::Delete::delete_message( from => $mbox, matching => sub { warn q(looking at: ), $_[0]->header(q(Subject)), "\n" if($ENV{VERBOSE}); return int shift(@_)->header(q(X-Bugsample)); }); }; defined($pid = fork) or die qq|can't fork to cleanup: $!\n|; $pid and exit 0; if(-f $mbox) { unlink $mbox; } elsif(-d $mbox) { for my $kind (qw(tmp new cur)) { opendir my $dh, qq($mbox/$kind) or die qq|can't opendir ($mbox/$kind): $!\n|; while(my $drop = readdir $dh) { -f qq($mbox/$kind/$drop) or next; unlink qq($mbox/$kind/$drop); }; rmdir qq($mbox/$kind) or die qq|can't rmdir ($mbox/$kind): $!\n|; }; rmdir qq($mbox) or die qq|can't rmdir ($mbox): $!\n|; }; exit 0; }; if(grep { m/.*help$/; } @ARGV) { print STDERR <<"END_OF_HELP"; orig.Email-Delete-1.022 - original upstream distribution bugfix.Email-Delete-1.022 - patched both have C<perl Makefile.PL> and C<make> done orig_ - upstream code bugfix_ - bugfix code none_ - void loop _d - maildir _b - mbox -a - do one run -b - skip mbox -d - skip maildir -f ratio - kill each <ratio> mail -m number - fill folder with <number> mails -r times - cycle tests number <times> -x factor - enlarge each body <factor> times \$ENV{VERBOSE} - enable messaging inside loop END_OF_HELP exit 0; }; getopts q(abdm:r:x:f:); $opt_m = 10 unless($opt_m); $opt_r = 5 unless($opt_r); $opt_x = 1 unless($opt_x); $opt_f = 1 unless($opt_f); my $rc = timethese $opt_r, { !$opt_b ? ( none_b => sub { bench_stuff q(none), q(mbox); undef until(wait == -1); }, orig_b => sub { bench_stuff q(orig), q(mbox); undef until(wait == -1); },) : ( ), !$opt_d ? ( none_d => sub { bench_stuff q(none), q(maildir); undef until(wait == -1); }, orig_d => sub { bench_stuff q(orig), q(maildir); undef until(wait == -1); }, bugfix_d => sub { bench_stuff q(bugfix), q(maildir); undef until(wait == -1); },) : ( ), }, q(nop); cmpthese $rc, q(nop); # vim: set filetype=perl
From: whynot [...] pozharski.name
On Sat Mar 08 17:04:34 2008, whynot wrote: Show quoted text
> --- orig.Email-Delete-1.022/lib/Email/Delete/Maildir.pm 2006-08-14 > 19:34:48.000000000 +0300 > +++ bugfix.Email-Delete-1.022/lib/Email/Delete/Maildir.pm 2008-03-07 > 17:39:57.000000000 +0200 > @@ -5,24 +5,24 @@
(oops, failed to navigate form properly.) Provided F<bugsample> is far from minimal but, hopefully, complete. To operate properly, F<bugsample> needs pristine F<orig.Email-Delete-1.022> and fixed F<bugfix.Email-Delete-1.022>. Sorry for inconvenience, I should have provided CLI options to assign those at run-time. Provided F<bugsample> emulates 2 modes: [1] application knows in advance what to delete (with B<-a> option); [2] application runs through folder each time again (without B<-a> option). Orthogonal options: [3] number of mails in mailbox (B<-m number>); [4] percent of mails to kill (B<-f number>); [5] size of mail (times of base mail) (B<-x number>). Here are gross results: -m1 27% 27% -m150 3% 21247% -f1 8% 9508% -f20 1746% 1708% -x1 20% 1417% -x5 9% 9547% Those stats was collected with C<-f<number> -m25> in 2nd place and default 10 mails in the 3rd. Since it seems to be PFM F<bugsample> allows to expose some internal workings; running C<VERBOSE=yes ./bugsample -r1 -b -m2> clearly shows what's the problem. B<-b> option disables benchmarking E::D::Mbox (sorry, it should be default). I was just curious how E::D::Maildir compares to E::D::Mbox. Mbox is 18 times faster in [2] and, honestly, 3 times slower in [1] (at moderate 10 mails). Please, be careful with F<stats.dump.log>, I've lost ~30 hours to count those.

Message body is not shown because it is too large.

From: whynot [...] pozharski.name
On Sat Mar 08 17:48:17 2008, whynot wrote: Show quoted text
> On Sat Mar 08 17:04:34 2008, whynot wrote:
> > --- orig.Email-Delete-1.022/lib/Email/Delete/Maildir.pm 2006-08-14 > > 19:34:48.000000000 +0300 > > +++ bugfix.Email-Delete-1.022/lib/Email/Delete/Maildir.pm 2008-03-07 > > 17:39:57.000000000 +0200 > > @@ -5,24 +5,24 @@
> > (oops, failed to navigate form properly.)
(surely, should sleep more.) === contents (perl -V) goes below === Summary of my perl5 (revision 5 version 8 subversion 8) configuration: Platform: osname=linux, osvers=2.6.22-3-amd64, archname=i486-linux-gnu-thread-multi uname='linux deneb 2.6.22-3-amd64 #1 smp thu oct 11 15:23:23 utc 2007 i686 gnulinux ' config_args='-Dusethreads -Duselargefiles -Dccflags=-DDEBIAN -Dcccdlflags=-fPIC -Darchname=i486-linux-gnu -Dprefix=/usr -Dprivlib=/usr/share/perl/5.8 -Darchlib=/usr/lib/perl/5.8 -Dvendorprefix=/usr -Dvendorlib=/usr/share/perl5 -Dvendorarch=/usr/lib/perl5 -Dsiteprefix=/usr/local -Dsitelib=/usr/local/share/perl/5.8.8 -Dsitearch=/usr/local/lib/perl/5.8.8 -Dman1dir=/usr/share/man/man1 -Dman3dir=/usr/share/man/man3 -Dsiteman1dir=/usr/local/man/man1 -Dsiteman3dir=/usr/local/man/man3 -Dman1ext=1 -Dman3ext=3perl -Dpager=/usr/bin/sensible-pager -Uafs -Ud_csh -Uusesfio -Uusenm -Duseshrplib -Dlibperl=libperl.so.5.8.8 -Dd_dosuid -des' hint=recommended, useposix=true, d_sigaction=define usethreads=define use5005threads=undef useithreads=define usemultiplicity=define useperlio=define d_sfio=undef uselargefiles=define usesocks=undef use64bitint=undef use64bitall=undef uselongdouble=undef usemymalloc=n, bincompat5005=undef Compiler: cc='cc', ccflags ='-D_REENTRANT -D_GNU_SOURCE -DTHREADS_HAVE_PIDS -DDEBIAN -fno-strict-aliasing -pipe -I/usr/local/include -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64', optimize='-O2', cppflags='-D_REENTRANT -D_GNU_SOURCE -DTHREADS_HAVE_PIDS -DDEBIAN -fno-strict-aliasing -pipe -I/usr/local/include' ccversion='', gccversion='4.1.2 20061115 (prerelease) (Debian 4.1.1-21)', gccosandvers='' intsize=4, longsize=4, ptrsize=4, doublesize=8, byteorder=1234 d_longlong=define, longlongsize=8, d_longdbl=define, longdblsize=12 ivtype='long', ivsize=4, nvtype='double', nvsize=8, Off_t='off_t', lseeksize=8 alignbytes=4, prototype=define Linker and Libraries: ld='cc', ldflags =' -L/usr/local/lib' libpth=/usr/local/lib /lib /usr/lib libs=-lgdbm -lgdbm_compat -ldb -ldl -lm -lpthread -lc -lcrypt perllibs=-ldl -lm -lpthread -lc -lcrypt libc=/lib/libc-2.3.6.so, so=so, useshrplib=true, libperl=libperl.so.5.8.8 gnulibc_version='2.3.6' Dynamic Linking: dlsrc=dl_dlopen.xs, dlext=so, d_dlsymun=undef, ccdlflags='-Wl,-E' cccdlflags='-fPIC', lddlflags='-shared -L/usr/local/lib' Characteristics of this binary (from libperl): Compile-time options: MULTIPLICITY PERL_IMPLICIT_CONTEXT PERL_MALLOC_WRAP THREADS_HAVE_PIDS USE_ITHREADS USE_LARGE_FILES USE_PERLIO USE_REENTRANT_API Built under linux Compiled at Nov 5 2007 06:11:48 @INC: /etc/perl /usr/local/lib/perl/5.8.8 /usr/local/share/perl/5.8.8 /usr/lib/perl5 /usr/share/perl5 /usr/lib/perl/5.8 /usr/share/perl/5.8 /usr/local/lib/site_perl . === contents (perl -V) gone above === === contents (perl -mFile::Find::Rule -e 'print $File::Find::Rule::VERSION') goes below === 0.30 === contents (perl -mFile::Find::Rule -e 'print $File::Find::Rule::VERSION') gone above ===
2.001 released -- rjbs