Re: Assitance needed for the resolution of memory leak

[Sasmit Utkarsh <utkarshsasmit@xxxxxxxxx>]

HI Adrian,

I have a main program which actually parses the input data from the files and inserts them to a table in postgres db. The input can have many duplicates and we cannot avoid it for now but we have to achieve the insertions of the records as quickly as possible. Below are the functions which are called from main to insert the record in a loop untill it reaches the end of the input file. If the input file has all unique records then there are no leaks reported by addresssanitizer tool but when there are duplicates it reports leaks. 

Like for an ex: I have provided the test run for a file which has 4 duplicate entries in the input file and below is the leak reported. I have followed 
consistently using PQclear to free all PGresult objects. Not sure why it references to PQmakeEmptyPGresult and pqResultAlloc even though I have not used it explicitly. Is anything missing here? How can I avoid memory leaks? Please assist here

addresssanitizer report :

 TOTAL rows inserted into fs_data table = 0
 TOTAL rows FAILED to insert into table = 4
 END TRANSACtION
 CLOSE shc_data
 FINISH

=================================================================
==336368==ERROR: LeakSanitizer: detected memory leaks

Direct leak of 640 byte(s) in 4 object(s) allocated from:
    #0 0xf7a1d0bb in __interceptor_malloc (/nix/store/3qz5qcx23d522i1fivjgh2nw0bs0pywf-gcc-10.3.0-lib/lib/libasan.so.6+0xaf0bb)
    #1 0xf792b7f3 in PQmakeEmptyPGresult (/nix/store/0rh8jnm4xgav5916nasd1d2ciy4nccsp-postgresql-13.7-lib/lib/libpq.so.5+0x117f3)

Indirect leak of 8192 byte(s) in 4 object(s) allocated from:
    #0 0xf7a1d0bb in __interceptor_malloc (/nix/store/3qz5qcx23d522i1fivjgh2nw0bs0pywf-gcc-10.3.0-lib/lib/libasan.so.6+0xaf0bb)
    #1 0xf792b3bc in pqResultAlloc (/nix/store/0rh8jnm4xgav5916nasd1d2ciy4nccsp-postgresql-13.7-lib/lib/libpq.so.5+0x113bc)

SUMMARY: AddressSanitizer: 8832 byte(s) leaked in 8 allocation(s).

Insert function:

int postgreSQL_insert(PGconn *conn, int fa, int ft, int ord, unsigned char *data, int blk_size, char *recid, char *xaddr)
{
PGresult  *res;
int rc = 0;
int rc2;
char *ErrorMsg;
char cur_recid[5] = "\0\0\0\0\0";
int ft3;
int ord3;
char Command[250];;

uint32_t fa2 = htonl(fa);
uint32_t ft2 = htonl(ft);
uint32_t ord2 = htonl(ord);
uint32_t blk_size2 = htonl(blk_size);
uint32_t xaddr_size = strlen(xaddr);
uint32_t recid_size = strlen(recid);

char *fa_val  = (char *)&fa2;
char *ft_val  = (char *)&ft2;
char *ord_val = (char *)&ord2;
char *blk_size_val = (char *)&blk_size2;

int   nParams               = 7;
//char *command               = "INSERT INTO fs_data (file_address, face_type, ordinal, xaddr, recid, blk_size, data) VALUES ($1, $2, $3, $4, $5, $6, $7)";
//char *command               = "INSERT INTO fs_data (file_address, face_type, ordinal, xaddr, recid, blk_size, data) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (file_address) DO NOTHING";
char *command               = "INSERT INTO fs_data (file_address, face_type, ordinal, xaddr, recid, blk_size, data) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (file_address) DO NOTHING RETURNING file_address";
Oid   paramTypes[7]         = {23, 23, 23, 25, 25, 23, 17};   // {int, int, int, text, text, int, bytea}
const char *paramValues[7]  = {fa_val, ft_val, ord_val, xaddr, recid, blk_size_val, data};
int   paramLengths[7]       = {sizeof(fa2), sizeof(ft2), sizeof(ord2), xaddr_size, recid_size, sizeof(blk_size2), blk_size};
int   paramFormats[7]       = {1, 1, 1, 1, 1, 1, 1}; // identify all parameters as binary
int   resultFormat          = 1;               // Result will be in binary format.

if(init_only == false)
{
rc = postgreSQL_delete(conn, fa);
}

if ( sql_debug ) { fprintf(stderr,"PostgreSQL_insert INPUT fa %i ft %i ord %i xaddr %s id %s blk_size %i\n",fa,ft,ord,xaddr,recid,blk_size); }
/* INSERT row in table fs_data */

if ( sql_debug ) { fprintf(stderr,"Command : (%s)\n", command); }
if ( sql_debug ) { fprintf(stderr,"Param. 0: faddr       (%i)\n", fa); }
if ( sql_debug ) { fprintf(stderr,"Param. 1: face_type   (%i)\n", ft); }
if ( sql_debug ) { fprintf(stderr,"Param. 2: ordinal     (%i)\n", ord); }
if ( sql_debug ) { fprintf(stderr,"Param. 3: xaddr       (%s)\n", xaddr); }
if ( sql_debug ) { fprintf(stderr,"Param. 4: recid       (%s)\n", recid); }
if ( sql_debug ) { fprintf(stderr,"Param. 5: blk_size    (%i)\n", blk_size); }
res = PQexecParams(conn, // shc_data database connection
  command, // "INSERT INTO fs_data ...)";
  nParams, // parameter count
  paramTypes, // integer,      integer,   integer, bytea
  paramValues, // file_address, face_type, ordinal, data
  paramLengths, // Lenght of each parameter
  paramFormats, // all parameters are binary format
  resultFormat); // result is binary


//fprintf(stderr," res = %s\n",PQresStatus(PQresultStatus(res))); // show the value of res

if ( ( PQresultStatus(res) == PGRES_TUPLES_OK)
 && (PQntuples(res) == 0) ) // if tuples == 0, we hit the ON CONFLICT, so report the ERROR
{
rc = 1;
fprintf(stderr, "\nERROR: DUPLICATE file address %s\n", xaddr);
fprintf(stderr, "FAIL : INSERT into fs_data file_address = %-11i (%s) recid = %s face_type = %i ordinal = %i\n", fa,xaddr,recid,ft,ord);
rc2 = SQL_SELECT_recid(fa, conn, cur_recid);
if(rc2 == 0)
{
sprintf(Command,"SELECT face_type, ordinal from fs_data where file_address=%i",fa);
//fprintf(stderr,"Command = %s\n",Command);
res = PQexec(conn, Command);
               if (PQresultStatus(res) == PGRES_TUPLES_OK)
{
int num_rows = PQntuples(res);
if (num_rows > 0)
{
ft3 = atoi(PQgetvalue(res, 0, 0));
ord3= atoi(PQgetvalue(res, 0, 1));
fprintf(stderr, "CURRENT data:   in fs_data file_address = %-11i (%s) recid = %s face_type = %i ordinal = %i\n", fa,xaddr,cur_recid,ft3,ord3);
}
else
{
fprintf(stderr,"FAIL postgreSQL_insert: No rows returned for file_address=%i\n",fa);
}
}
else
fprintf(stderr,"FAIL postgreSQL_insert: Query execution FAILED: %s\n", PQresultErrorMessage(res));

PQclear(res);
}
}
else if ( PQresultStatus(res) != PGRES_TUPLES_OK) // something else went wrong
{
rc = 1;
fprintf(stderr,"ERROR: %s\n",PQresultErrorMessage(res));
PQclear(res);
}
else
{
rc = 0;
if ( sql_debug ) { fprintf(stderr,"INSERT SUCCESSFUL\n"); }
PQclear(res);
}

if ( sql_debug1 ) { printPostgreSQL_SELECT_results(fa,conn); }

return rc;
}

Regards,

Sasmit Utkarsh

+91-7674022625

On Thu, Oct 12, 2023 at 1:06 AM Adrian Klaver <adrian.klaver@xxxxxxxxxxx> wrote:

On 10/11/23 12:15, Sasmit Utkarsh wrote:
> Hi Team,
>
> I'm trying to insert multiple duplicate rows into the table using libpq
> in C. I'm getting a memory leak reported from addresssanitizer tool.
> Please let me know if i can share the code snippet

That is the only way you are going to get an answer, so share away.

Also include the report information from addresssanitizer.

>
>
> Regards,
> Sasmit Utkarsh
> +91-7674022625

--
Adrian Klaver
adrian.klaver@xxxxxxxxxxx