| Subject: | kr_exception from 401 page |
I recently used the POE::Component::Client::UserAgent module, and it has
worked against numerous pages. There is nothing special about my script
(attached), it makes HTTP Requests and spits back the response- but when
my script requested: http://dominio-free-gratis.info
the response->status_line was: '401 Unauthorized'
followed by the unfriendly 'crash' with message:
Usage: $h->header($field, ...) at (eval
43)[/Library/Perl/5.8.8/HTTP/Message.pm:390] line 1
I found that commenting out line 280 of /POE/Resource/Events.pm out and
replacing with 'warning $exception' temporarily works around the issue
as long as you have $poe_kernel->run() in a loop to restart ... though
you end up loosing whatever queries where in the session that 'crashed.'
Being an intermediate Perl debugger and a novice POE user, I'm not
entirely sure what is causing kr_exception to be set for this case (line
267 of POE::Resource::Events); but here is the traceback:
( for reference all line 167 is: "printf "%s ERROR: %s\n", $suspect,
$response->status_line;" where $suspect is $request->uri() ... and both
$response and $request are passed in from the postback which is just a
simple 'GET.' )
main::response(./scan_domain.pl:167):
167: printf "%s ERROR: %s\n", $suspect, $response->status_line;
DB<2> n
http://dominio-free-gratis.info ERROR: 401 Unauthorized
POE::Kernel::_dispatch_event(/Library/Perl/5.8.8/POE/Kernel.pm:1053):
1053: if (ref($@) or $@ ne '') {
DB<2> n
POE::Kernel::_dispatch_event(/Library/Perl/5.8.8/POE/Kernel.pm:1121):
1121: @$etc = ( );
DB<2> n
POE::Kernel::_dispatch_event(/Library/Perl/5.8.8/POE/Kernel.pm:1123):
1123: if (TRACE_STATISTICS) {
DB<2> n
POE::Kernel::_dispatch_event(/Library/Perl/5.8.8/POE/Kernel.pm:1137):
1137: if (defined $return and substr(ref($return), 0, 5) eq 'POE::') {
DB<2> n
POE::Kernel::_dispatch_event(/Library/Perl/5.8.8/POE/Kernel.pm:1142):
1142: $kr_active_session = $hold_active_session;
DB<2> n
POE::Kernel::_dispatch_event(/Library/Perl/5.8.8/POE/Kernel.pm:1145):
1145: $kr_active_event = $hold_active_event;
DB<2> n
POE::Kernel::_dispatch_event(/Library/Perl/5.8.8/POE/Kernel.pm:1147):
1147: if (TRACE_EVENTS) {
DB<2> n
POE::Kernel::_dispatch_event(/Library/Perl/5.8.8/POE/Kernel.pm:1155):
1155: return unless $self->_data_ses_exists($session);
DB<2> n
POE::Kernel::_dispatch_event(/Library/Perl/5.8.8/POE/Kernel.pm:1175):
1175: if ($type & ET_POST) {
DB<2> n
POE::Kernel::_dispatch_event(/Library/Perl/5.8.8/POE/Kernel.pm:1176):
1176: $self->_data_ses_collect_garbage($session)
1177: if $self->_data_ses_exists($session);
DB<2> n
POE::Kernel::_dispatch_event(/Library/Perl/5.8.8/POE/Kernel.pm:1178):
1178: $self->_data_ses_collect_garbage($source_session)
1179: if ( $session != $source_session and
1180: $self->_data_ses_exists($source_session)
1181: );
DB<2> n
POE::Kernel::_dispatch_event(/Library/Perl/5.8.8/POE/Kernel.pm:1196):
1196: return @$return if wantarray;
DB<2> n
POE::Kernel::_dispatch_event(/Library/Perl/5.8.8/POE/Kernel.pm:1197):
1197: return $return;
DB<2> n
POE::Kernel::_data_ev_dispatch_due(/Library/Perl/5.8.8/POE/Resource/Events.pm:267):
267: if ($POE::Kernel::kr_exception) {
DB<2> n
POE::Kernel::_data_ev_dispatch_due(/Library/Perl/5.8.8/POE/Resource/Events.pm:245):
245: last if $next_time > $now;
DB<2> n
POE::Kernel::_data_ev_dispatch_due(/Library/Perl/5.8.8/POE/Resource/Events.pm:247):
247: my ($due_time, $id, $event) = $kr_queue->dequeue_next();
DB<2> n
POE::Kernel::_data_ev_dispatch_due(/Library/Perl/5.8.8/POE/Resource/Events.pm:249):
249: if (TRACE_EVENTS) {
DB<2> n
POE::Kernel::_data_ev_dispatch_due(/Library/Perl/5.8.8/POE/Resource/Events.pm:258):
258: if ($due_time < $now) {
DB<2> n
POE::Kernel::_data_ev_dispatch_due(/Library/Perl/5.8.8/POE/Resource/Events.pm:259):
259: $self->_data_stat_add('blocked', 1);
DB<2> n
POE::Kernel::_data_ev_dispatch_due(/Library/Perl/5.8.8/POE/Resource/Events.pm:260):
260: $self->_data_stat_add('blocked_seconds', $now - $due_time);
DB<2> n
POE::Kernel::_data_ev_dispatch_due(/Library/Perl/5.8.8/POE/Resource/Events.pm:263):
263: $self->_data_ev_refcount_dec($event->[EV_SOURCE],
$event->[EV_SESSION]);
DB<2> n
POE::Kernel::_data_ev_dispatch_due(/Library/Perl/5.8.8/POE/Resource/Events.pm:264):
264: $self->_dispatch_event(@$event, $due_time, $id);
DB<2> n
POE::Kernel::_data_ev_dispatch_due(/Library/Perl/5.8.8/POE/Resource/Events.pm:267):
267: if ($POE::Kernel::kr_exception) {
DB<2> n
POE::Kernel::_data_ev_dispatch_due(/Library/Perl/5.8.8/POE/Resource/Events.pm:271):
271: my $exception = $POE::Kernel::kr_exception;
DB<2> n
POE::Kernel::_data_ev_dispatch_due(/Library/Perl/5.8.8/POE/Resource/Events.pm:272):
272: $POE::Kernel::kr_exception = undef;
DB<2> n
POE::Kernel::_data_ev_dispatch_due(/Library/Perl/5.8.8/POE/Resource/Events.pm:275):
275: POE::Kernel->stop();
DB<2> n
POE::Kernel::_data_ev_dispatch_due(/Library/Perl/5.8.8/POE/Resource/Events.pm:278):
278: die $exception;
DB<2> n
Usage: $h->header($field, ...) at (eval
43)[/Library/Perl/5.8.8/HTTP/Message.pm:390] line 1
at /Library/Perl/5.8.8/POE/Resource/Events.pm line 278
POE::Kernel::_data_ev_dispatch_due('POE::Kernel=ARRAY(0x9252e0)')
called at /Library/Perl/5.8.8/POE/Loop/Select.pm line 321
POE::Kernel::loop_do_timeslice('POE::Kernel=ARRAY(0x9252e0)') called at
/Library/Perl/5.8.8/POE/Loop/Select.pm line 329
POE::Kernel::loop_run('POE::Kernel=ARRAY(0x9252e0)') called at
/Library/Perl/5.8.8/POE/Kernel.pm line 1272
POE::Kernel::run('POE::Kernel=ARRAY(0x9252e0)') called at
./scan_domain.pl line 67
Thanks,
Mike
| Subject: | scan.pl |
#!/usr/bin/perl -w
use strict;
use warnings;
use Carp;
$| = 1;
use POE; # import lots of constants
use POE::Component::Client::UserAgent;
my $domain_file = $ARGV[0];
if ((!-e $domain_file) || (!-r $domain_file)) { croak "Error file to process does not exist or cannot be read from: $domain_file\n"; }
open(FH, "$domain_file") or croak "Error opening file to process: $domain_file\n";
my $scanned_domains = 0;
my $eof = 0;
sub INITIAL_COUNT () { 100 }
my $USERAGENT_ALIAS = "useragent";
my $USERAGENT_NAME = 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)';
my $USERAGENT_TIMEOUT = 15;
my @KEYWORDS = (
# This is populated with strings of keywords that i am looking for
);
sub get_next_domain {
local *MYFH = shift;
my $next_domain = <MYFH>;
if (!defined $next_domain) {
$eof = 1;
return undef;
}
chomp $next_domain;
return $next_domain;
}
# now run POE!
while (!$eof) {
# create client session
POE::Session->create (
package_states => [ main => [ "_start", "start_next_scan", "response" ] ]
#inline_states => {
# _start => \&_start,
# start_next_scan => \&start_next_scan,
# response => \&response
#},
);
$poe_kernel->run;
print "Looping\n";
}
close(FH);
print "NUMBER OF DOMAINS SCANNED: $scanned_domains\n";
exit 0;
# this is the first event to arrive
sub _start
{
# create the PoCoCl::UserAgent session
POE::Component::Client::UserAgent -> new(
{
alias => $USERAGENT_ALIAS,
agent => $USERAGENT_NAME,
timeout => $USERAGENT_TIMEOUT
}
) or croak "Error creating POE UserAgent: $!\n";
for ( 1 .. INITIAL_COUNT ) {
$_[KERNEL]->yield("start_next_scan");
}
}
sub start_next_scan {
my ( $kernel, $heap ) = @_[ KERNEL, HEAP ];
my $next_domain = get_next_domain(*FH);
return unless defined $next_domain;
printf("SCANNING: %s\n", $next_domain);
$scanned_domains++;
if (defined($next_domain)) {
# hand it our request
$_[KERNEL] -> post (
# default alias is 'useragent'
useragent => 'request',
{
request => HTTP::Request -> new (GET => 'http://'.$next_domain),
# let UserAgent know where to deliver the response
response => $_[SESSION] -> postback ('response')
}
);
# Once we are done posting requests, we can post a shutdown event
# to the PoCoCl::UserAgent session. Responses will still be returned.
$_[KERNEL] -> post (useragent => 'shutdown');
}
}
# Here is where the response arrives. Actually in this example we
# would get more than one response, as hotmail home page is a mere
# redirect to some address at passport.com. The component processes
# the redirect automatically by default.
sub response
{
# @{$_[ARG0]} is the list we passed to postback()
# after the event name, empty in this example
# @{$_[ARG1]} is the list PoCoCl::UserAgent is passing back to us
my ($request, $response, $entry) = @{$_[ARG1]};
# These are:
# - HTTP::Request
# - HTTP::Response
# - LWP::Parallel::USerAgent::Entry
$_[KERNEL]->yield("start_next_scan");
my $suspect = $request->uri();
print "Processing response from: $suspect\n";
if ($response->is_success) {
printf("CONNECTED: %s\n", $request->uri);
my $html = $response->content;
#print "Received Content:\n$html\n";
$html =~ s/\n//g;
foreach my $keyword (@KEYWORDS) {
if ($html =~ /($keyword)/i) {
print "\n$suspect KEYWORD MATCH $keyword\n";
}
}
}
elsif ($response->is_redirect) {
printf("REDIRECTED FROM %s TO %s\n", $suspect, $response->base());
#printf("This was the last request for %s: %s\n", $request->uri(), $response->request->uri());
my $html = $response->content;
#print "Received Content:\n$html\n";
$html =~ s/\n//g;
foreach my $keyword (@KEYWORDS) {
if ($html =~ /($keyword)/i) {
print "\n$suspect KEYWORD MATCH $keyword\n";
}
}
}
elsif ($response->is_error) {
printf "%s ERROR: %s\n", $suspect, $response->status_line;
print "Made it this far";
}
}