Bug #52306 for Parse-BACKPAN-Packages: List of distributions

I would like to get a list of all distributions on BackPAN so I can iterate through them. for my $distname ($backpan->dists) { for my $release ($backpan->distributions($distname)) { print $release->filename; } } The data can be gotten from a simple addition to _init_dist_by. Attached is a patch which adds dists() to do this. The terminology is a bit messed up (see my other ticket). I decided to have it return an array ref rather than an array like everything else does due to the size of the list (20k+). YMMV. It also changes the tests to use a local cache directory. This allows the tests to be broken up into multiple files without slowing everythign down.

From 6fc7da64fe2bf97ba9445c35c654e02164fd12e4 Mon Sep 17 00:00:00 2001 From: Michael G. Schwern <schwern@pobox.com> Date: Mon, 30 Nov 2009 21:54:40 -0800 Subject: [PATCH] Add "dists" to get all the distribution names. Make the tests use a local cache so they run faster as we add more test files. --- lib/Parse/BACKPAN/Packages.pm | 32 +++++++++++++++++++++++++------- t/00cache.t | 22 ++++++++++++++++++++++ t/dists.t | 24 ++++++++++++++++++++++++ t/lib/TestUtils.pm | 26 ++++++++++++++++++++++++++ t/simple.t | 11 ++++------- 5 files changed, 101 insertions(+), 14 deletions(-) create mode 100644 t/00cache.t create mode 100644 t/dists.t create mode 100644 t/lib/TestUtils.pm diff --git a/lib/Parse/BACKPAN/Packages.pm b/lib/Parse/BACKPAN/Packages.pm index 8b3c28a..2e3f53e 100644 --- a/lib/Parse/BACKPAN/Packages.pm +++ b/lib/Parse/BACKPAN/Packages.pm @@ -10,24 +10,31 @@ use LWP::UserAgent; use Parse::BACKPAN::Packages::File; use Parse::BACKPAN::Packages::Distribution; use base qw( Class::Accessor::Fast ); -__PACKAGE__->mk_accessors(qw( files dists_by no_cache )); +__PACKAGE__->mk_accessors(qw( files dists_by dists no_cache cache_dir )); our $VERSION = '0.36'; sub new { my $class = shift; my $self = $class->SUPER::new(@_); + my $dist_data; if ( !$self->no_cache ) { - my $cache = App::Cache->new( { ttl => 60 * 60 } ); + my %cache_opts; + $cache_opts{ttl} = 60 * 60; + $cache_opts{directory} = $self->cache_dir if $self->cache_dir; + + my $cache = App::Cache->new( \%cache_opts ); $self->files( $cache->get_code( 'files', sub { $self->_init_files() } ) ); - $self->dists_by( - $cache->get_code( 'dists_by', sub { $self->_init_dists_by() } ) ); + $dist_data = $cache->get_code( 'dists', sub { $self->_init_dists() } ); } else { $self->files( $self->_init_files() ); - $self->dists_by( $self->_init_dists_by() ); + $dist_data = $self->_init_dists(); } + $self->dists_by($dist_data->{dist_by}); + $self->dists($dist_data->{dists}); + return $self; } @@ -120,7 +127,7 @@ sub authors { return sort keys %$dists_by; } -sub _init_dists_by { +sub _init_dists { my ($self) = shift; my @files; @@ -133,17 +140,22 @@ sub _init_dists_by { @files = sort { $a->date <=> $b->date } @files; my $dist_by; + my $dists; foreach my $file (@files) { my $i = CPAN::DistnameInfo->new( $file->prefix ); my ( $dist, $cpanid ) = ( $i->dist, $i->cpanid ); next unless $dist && $cpanid; + $dists->{$dist}++; $dist_by->{$cpanid}{$dist}++; } my %dists_by = map { $_ => [ keys %{ $dist_by->{$_} } ] } keys %$dist_by; - return \%dists_by; + return { + dist_by => \%dists_by, + dists => [keys %$dists] + }; } sub size { @@ -215,6 +227,12 @@ that you can pass them into the distributions_by method: my @authors = $p->authors; +=head2 dists + + my $distributions = $p->dists; + +Returns an array ref of the names of all the distributions in BackPAN. + =head2 distributions The distributions method returns a list of objects representing all diff --git a/t/00cache.t b/t/00cache.t new file mode 100644 index 0000000..e9f4d0e --- /dev/null +++ b/t/00cache.t @@ -0,0 +1,22 @@ +#!perl + +# A test to clear and populate the cache so subsequent tests run faster + +use strict; +use warnings; +use Test::More tests => 4; +use lib 'lib', "t/lib"; +use_ok("Parse::BACKPAN::Packages"); + +use TestUtils; + +# Clear out any leftover cache +TestUtils::clear_cache(); + +# Populate the cache +my $p = new_backpan(); +isa_ok( $p, "Parse::BACKPAN::Packages" ); +cmp_ok( $p->size, '>=', 5_597_434_696, "backpan is at least 5.6G" ); + +my $files = $p->files; +cmp_ok( scalar( keys %$files ), '>=', 105_996 ); diff --git a/t/dists.t b/t/dists.t new file mode 100644 index 0000000..d2eac1f --- /dev/null +++ b/t/dists.t @@ -0,0 +1,24 @@ +#!perl + +use strict; +use warnings; + +use lib 'lib', 't/lib'; + +use Test::More tests => 3; +use TestUtils; + +my $p = new_backpan(); + +my $dists = $p->dists; +cmp_ok scalar @$dists, '>=', 21911; + +my %dists = map { $_ => 1 } @$dists; +ok $dists{"Acme-Pony"}; + +# Pick a distribution at random, it should have releases. +my $dist = $dists->[rand @$dists]; +my @releases = $p->distributions($dist); +is $releases[0]->dist, $dist, "found releases for $dist"; + +1; diff --git a/t/lib/TestUtils.pm b/t/lib/TestUtils.pm new file mode 100644 index 0000000..9a00c6e --- /dev/null +++ b/t/lib/TestUtils.pm @@ -0,0 +1,26 @@ +package TestUtils; + +use strict; +use warnings; + +use File::Spec; +use File::Path; +use Parse::BACKPAN::Packages; + +use base "Exporter"; +our @EXPORT = qw(new_backpan); + +sub cache_dir { + return File::Spec->rel2abs("t/cache"); +} + +sub clear_cache { + rmtree cache_dir(); +} + +sub new_backpan { + my $cache_dir = File::Spec->rel2abs("t/cache"); + return Parse::BACKPAN::Packages->new( { cache_dir => $cache_dir } ); +} + +1; diff --git a/t/simple.t b/t/simple.t index 36e2c79..0e19e3f 100644 --- a/t/simple.t +++ b/t/simple.t @@ -1,15 +1,12 @@ #!perl use strict; use warnings; -use Test::More tests => 86; -use lib 'lib'; +use Test::More tests => 84; +use lib 'lib', 't/lib'; +use TestUtils; use_ok("Parse::BACKPAN::Packages"); -my $p = Parse::BACKPAN::Packages->new( { no_cache => 1 } ); -ok( $p->size >= 5_597_434_696, "backpan is at least 5.6G" ); - -my $files = $p->files; -ok( scalar( keys %$files ) >= 105_996 ); +my $p = new_backpan(); my $file = $p->file("authors/id/L/LB/LBROCARD/Acme-Colour-0.16.tar.gz"); is( $file->prefix, "authors/id/L/LB/LBROCARD/Acme-Colour-0.16.tar.gz" ); -- 1.6.5.2