| Subject: | Memory leak when fetching binary columns |
| Date: | Fri, 08 Sep 2006 12:26:47 -0400 |
| To: | bug-DBD-Pg [...] rt.cpan.org |
| From: | Stephen Marshall <smarshall [...] wsi.com> |
There is a memory leak that occurs when fetching data with bytea
columns. The leak occurs in the dequote_bytea() function.
The patch for this bug is attached. I have also posted the patch on the
DBD-Pg development site on gborg (patch named
memory_leak_fetch_binary_columns).
I found a memory leak in DBD-Pg-1.49. I was able to reproduce it in all
versions back to 1.45, but it was not in version 1.43 or 1.42.
I found the leak to occur on both these platforms:
RedHat Enterprise Linux 3.0
perl 5.8.0
uname -a: Linux rainbow 2.4.21-32.0.1.EL-wsi-smp #1 SMP Mon Jun 6
15:27:49 EDT 2005 i686 i686 i386 GNU/Linux
RedHat Enterprise Linux 4.0 -
perl 5.8.5
uname -a: Linux sysh 2.6.9-42.0.2.ELsmp #1 SMP Thu Aug 17 18:00:32
EDT 2006 i686 i686 i386 GNU/Linux
The dequote_bytea function does in place replacement of the escaped
ASCII text with its binary equivalent. Thus it require no additional
memory to be allocated. The use of memory allocation in this function
using New() and Renew() are unnecessary and cause memory to be leaked
with each fetch.
The bug can be readily seen by executing this query repeatedly and
getting the results with fetchrow_hashref or fetchrow_arrayref.
SELECT decode(repeat('\\\\377\\\\000', 20000), 'escape') as bindata
This query generates a 40,000 byte bytea column. If a 40,000 text
column is used, the leak does not occur.
The process size can be monitored with top or an equivalent utility. A
full test script that exercises the problem is attached.
*** quote.c.1.49 2006-09-08 11:26:30.000000000 -0400
--- quote.c 2006-09-08 11:17:43.000000000 -0400
***************
*** 304,312 ****
if (NULL == string)
return;
-
- New(0, result, strlen((char *)string)+1, unsigned char);
-
result = string;
while (*string != '\0') {
--- 304,309 ----
***************
*** 334,340 ****
}
}
result = '\0';
- Renew(result, (*retlen), unsigned char);
string = result - (*retlen);
return;
}
--- 331,336 ----
#!/usr/bin/perl
#
# Test program to exercise listen for notification and test for
# memory leaks.
#
#------------------------------------------------------------------------------
#
# Require and import any standard Perl modules.
#
use Getopt::Std;
use DBI;
no encoding;
use strict;
#------------------------------------------------------------------------------
# Interpret the configurable parameters for this script.
#------------------------------------------------------------------------------
#
# Set defaults for global variables. The use of globals is needed to allow
# appropriate cleanup from signal handlers.
#
$main::is_verbose = 0; # Output is NOT verbose
#
# Set defaults for configurable parameters.
#
my $appname = "test_dbdpg_prepare";
my $event = "Test_DBDPG_prepare";
my $dbname = "skybase";
my $dbhost = undef;
my $dbuser = "nownet";
my $timeout = 0.1; # in seconds
my $use_binary = 1;
my $use_server_prepare = 0;
my $use_hashref = 1;
#
# Get options from line command.
#
my %opts;
getopts('TAPd:e:h:U:s:v', \%opts);
if ($opts{A}) # Use fetchrow_arrayref, rather than fetchrow_hashref
{
$use_hashref = 0;
}
if ($opts{T}) # Enable text query, rather than binary
{
$use_binary = 0;
}
if ($opts{P}) # Enable server side prepared statements.
{
$use_server_prepare = 1;
}
if ($opts{v}) # Enable verbose mode.
{
$main::is_verbose = 1;
}
if ($opts{d}) # Specify database to connect to
{
$dbname = $opts{d};
}
if ($opts{h}) # Specify database host to connect to
{
$dbhost = $opts{h};
}
if ($opts{U}) # Specify database user
{
$dbuser = $opts{U};
}
if ($opts{e}) # Specify event to listen for
{
$event = $opts{e};
}
if ($opts{s}) # Specify sleep interval (fractional seconds)
{
$timeout = $opts{s};
}
loginfo("Starting ", $appname, ", pid = ", $$, "\n");
#
# Connect to the database and get the database connection's socket.
#
my $dsn = "dbi:Pg:dbname=$dbname";
$dsn .= join("", ";host=", $dbhost) if defined($dbhost);
my %attr = (AutoCommit => 1, RaiseError => 1, PrintError => 0);
my $dbh = DBI->connect($dsn, $dbuser, undef, \%attr);
if (!defined($dbh))
{
die "Error connecting to ", $dsn, " as user ", $dbuser,
": ", DBI->errstr, "\n";
}
loginfo("Connected to ", $dsn, " as user ", $dbuser, "\n");
#
# Define a query to retrieve a 40,000 byte value.
# It varies whether the data is binary or textual.
#
my $qry = undef;
if ($use_binary) # Binary string query
{
$qry = "SELECT decode(repeat('\\\\377\\\\000', 20000), 'escape') as bindata";
}
else # text string query
{
$qry = "SELECT repeat('10', 20000) as textdata";
}
my $sth = $dbh->prepare($qry, {pg_server_prepare => $use_server_prepare});
my $first_time = 1;
for (my $event_cnt = 0; ; $event_cnt++)
{
#
# Use select as to sleep with sub-second precision.
#
my $nfound = select( undef, undef, undef, $timeout);
# Execute query and process all the results.
$sth->execute();
if ($use_hashref) # Use fetchrow_hashref
{
while (defined(my $hash_ref = $sth->fetchrow_hashref))
{
}
}
else # use fetchrow_arrayref
{
while (defined(my $array_ref = $sth->fetchrow_arrayref))
{
}
}
loginfo("Event ", $event_cnt, ": ", $qry, ": pg_server_prepare = ",
$use_server_prepare, ", use_hashref = ", $use_hashref, "\n");
} # end of event loop
return 0; # Exit the application
#------------------------------------------------------------------------------
# Routine to produce diagnostic messages. logverbose only writes if verbose
# mode is enabled; logerror always writes.
#------------------------------------------------------------------------------
sub logerror
{
if ($main::is_verbose)
{
print STDERR @_;
}
}
sub loginfo
{
print STDERR "PID ", $$, ": ", @_;
}
#------------------------------------------------------------------------------
# Routine to provide online documentation for this script.
#------------------------------------------------------------------------------
sub usage()
{
die <<"TextOfUsage";
Name:
test_dbdpg_prepare - Issues prepared test queries.
Synopsis:
text_dbdpg_prepare [-v][-d <dbname>][-e <event>][-h <dbhost>][-s <ms>]
The following options are supported:
-v enables verbose output to stderr.
-d specifies an alternate database.
-e specifies an alternate event for triggering processing.
-h specifies database hostname
-s specifies seconds to sleep between notifications (can be fractional)
TextOfUsage
}