dbi Blog

Welcome to part II of a three-part article on extending gawk with a SQLite binding. Part I is here. Part II is followed by Part III, which give some explanations for the code presented here and shows how to use the extension with a stress test.
Here, I’ll list the source code of the extension and give instructions to compile and use it in gawk. Beware though that the code should be taken with several grains of salt, actually a whole wheelbarrow of it, because it has been only superficially tested. Some more serious testing is required in order to trust it entirely. So, caveat emptor !
I assume the source code of gawk is already installed (see for example the instructions here). We still need the SQLite source code. Go here and download the amalgamation zip file. Unzip it somewhere and then copy the file sqlite3.c to the gawk extension directory, ~/dmgawk/gawk-4.2.1/extension. Compile it with the command below:

gcc -c sqlite3.c -DHAVE_READLINE -fPIC -lpthread -ldl

Now, edit the file Makefile.am and add the references to the new extension, as shown below:
vi Makefile.am pkgextension_LTLIBRARIES = filefuncs.la ... sqlite_gawk.la <----- noinst_LTLIBRARIES = ... time_la_SOURCES = time.c time_la_LDFLAGS = $(MY_MODULE_FLAGS) time_la_LIBADD = $(MY_LIBS) sqlite_gawk_la_SOURCES = sqlite_gawk.c <----- sqlite_gawk_la_LDFLAGS = $(MY_MODULE_FLAGS) <----- sqlite_gawk_la_LIBADD = $(MY_LIBS) -lpthread -ldl -lreadline <----- ...
Save and quit; that’s all for the make file;
We are still in the extension directory. Let’s edit the interface sqlite_gawk.c now and insert the code below;
vi sqlite_gawk.c

/*
 * sqlite-gawk.c - an interface to sqlite() library;
 * Cesare Cervini
 * dbi-services.com
 * 8/2018
*/
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
 
#include <stdio.h>
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
 
#include <sys/types.h>
#include <sys/stat.h>
 
#include "gawkapi.h"
 
// extension;
#include <time.h>
#include <errno.h>
#include <limits.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <sys/types.h>
#include <regex.h>
#include <sqlite3.h>
 
#include "gettext.h"
#define _(msgid)  gettext(msgid)
#define N_(msgid) msgid
 
static const gawk_api_t *api;   /* for convenience macros to work */
static awk_ext_id_t ext_id;
static const char *ext_version = "an interface to sqlite3: version 1.0";
 
int plugin_is_GPL_compatible;
 
/* internal structure and variables */
/*
internally stores the db handles so an integer is returned to gawk as the index of a array;
*/
#define MAX_DB 100
static unsigned nb_sqlite_free_handles = MAX_DB;
static sqlite3 *sqlite_handles[MAX_DB];
 
/* init_sqlite_handles */
/*
since gawk does not know pointers, we use integers as db handles, which are actually indexes into a table of sqllite db handles;
initializes the sqlite_handles array to null pointers and resets the number of free handles;
called at program start time;
*/
static awk_bool_t init_sqlite_handles(void) {
   for (unsigned i = 0; i < MAX_DB; i++)
      sqlite_handles[i] = NULL;
   nb_sqlite_free_handles = MAX_DB;
 
   register_ext_version(ext_version);
 
   return awk_true;
}
 
/*
readfile() and writefile() functions for blob I/Os from/to file into/from memory;
e.g.:
   INSERT INTO chickenhouse my_blob = readfile('chicken_run.mp4');
   SELECT writefile("'Mary Poppins Returns.mp4'", my_blob) FROM children_movies;
they are taken directly from the source file sqlite3.c, i.e. sqlite-s shell program;
those functions are later registered in do_sqlite_open() via a call to sqlite_create_function() for use as extending SQL functions;
to be done while opening a db in do_sqlite_open() and for each opened db where the extended functions must be available, i.e. all for short;
*/
 
// ------------------------------------------- begin of imported functions from sqllite3.c ---------------------------------------------
/*
** This function is used in place of stat().  On Windows, special handling
** is required in order for the included time to be returned as UTC.  On all
** other systems, this function simply calls stat().
*/
static int fileStat(
  const char *zPath,
  struct stat *pStatBuf
){
  return stat(zPath, pStatBuf);
}
 
/*
** Set the result stored by context ctx to a blob containing the 
** contents of file zName.
*/
static void readFileContents(sqlite3_context *ctx, const char *zName){
  FILE *in;
  long nIn;
  void *pBuf;
 
  in = fopen(zName, "rb");
  if( in==0 ) return;
  fseek(in, 0, SEEK_END);
  nIn = ftell(in);
  rewind(in);
  pBuf = sqlite3_malloc( nIn );
  if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
    sqlite3_result_blob(ctx, pBuf, nIn, sqlite3_free);
  }else{
    sqlite3_free(pBuf);
  }
  fclose(in);
}
 
/*
** Implementation of the "readfile(X)" SQL function.  The entire content
** of the file named X is read and returned as a BLOB.  NULL is returned
** if the file does not exist or is unreadable.
*/
static void readfileFunc(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  const char *zName;
  (void)(argc);  /* Unused parameter */
  zName = (const char*)sqlite3_value_text(argv[0]);
  if( zName==0 ) return;
  readFileContents(context, zName);
}
 
/*
** This function does the work for the writefile() UDF. Refer to 
** header comments at the top of this file for details.
*/
static int writeFile(
  sqlite3_context *pCtx,          /* Context to return bytes written in */
  const char *zFile,              /* File to write */
  sqlite3_value *pData,           /* Data to write */
  mode_t mode,                    /* MODE parameter passed to writefile() */
  sqlite3_int64 mtime             /* MTIME parameter (or -1 to not set time) */
){
  if( S_ISLNK(mode) ){
    const char *zTo = (const char*)sqlite3_value_text(pData);
    if( symlink(zTo, zFile)<0 ) return 1;
  }else
  {
    if( S_ISDIR(mode) ){
      if( mkdir(zFile, mode) ){
        /* The mkdir() call to create the directory failed. This might not
        ** be an error though - if there is already a directory at the same
        ** path and either the permissions already match or can be changed
        ** to do so using chmod(), it is not an error.  */
        struct stat sStat;
        if( errno!=EEXIST
         || 0!=fileStat(zFile, &sStat)
         || !S_ISDIR(sStat.st_mode)
         || ((sStat.st_mode&0777)!=(mode&0777) && 0!=chmod(zFile, mode&0777))
        ){
          return 1;
        }
      }
    }else{
      sqlite3_int64 nWrite = 0;
      const char *z;
      int rc = 0;
      FILE *out = fopen(zFile, "wb");
      if( out==0 ) return 1;
      z = (const char*)sqlite3_value_blob(pData);
      if( z ){
        sqlite3_int64 n = fwrite(z, 1, sqlite3_value_bytes(pData), out);
        nWrite = sqlite3_value_bytes(pData);
        if( nWrite!=n ){
          rc = 1;
        }
      }
      fclose(out);
      if( rc==0 && mode && chmod(zFile, mode & 0777) ){
        rc = 1;
      }
      if( rc ) return 2;
      sqlite3_result_int64(pCtx, nWrite);
    }
  }
 
  if( mtime>=0 ){
#if defined(AT_FDCWD) && 0 /* utimensat() is not universally available */
    /* Recent unix */
    struct timespec times[2];
    times[0].tv_nsec = times[1].tv_nsec = 0;
    times[0].tv_sec = time(0);
    times[1].tv_sec = mtime;
    if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){
      return 1;
    }
#else
    /* Legacy unix */
    struct timeval times[2];
    times[0].tv_usec = times[1].tv_usec = 0;
    times[0].tv_sec = time(0);
    times[1].tv_sec = mtime;
    if( utimes(zFile, times) ){
      return 1;
    }
#endif
  }
 
  return 0;
}
 
/*
** Argument zFile is the name of a file that will be created and/or written
** by SQL function writefile(). This function ensures that the directory
** zFile will be written to exists, creating it if required. The permissions
** for any path components created by this function are set to (mode&0777).
**
** If an OOM condition is encountered, SQLITE_NOMEM is returned. Otherwise,
** SQLITE_OK is returned if the directory is successfully created, or
** SQLITE_ERROR otherwise.
*/
static int makeDirectory(
  const char *zFile,
  mode_t mode
){
  char *zCopy = sqlite3_mprintf("%s", zFile);
  int rc = SQLITE_OK;
 
  if( zCopy==0 ){
    rc = SQLITE_NOMEM;
  }else{
    int nCopy = (int)strlen(zCopy);
    int i = 1;
 
    while( rc==SQLITE_OK ){
      struct stat sStat;
      int rc2;
 
      for(; zCopy[i]!='/' && i<nCopy; i++);
      if( i==nCopy ) break;
      zCopy[i] = '';
 
      rc2 = fileStat(zCopy, &sStat);
      if( rc2!=0 ){
        if( mkdir(zCopy, mode & 0777) ) rc = SQLITE_ERROR;
      }else{
        if( !S_ISDIR(sStat.st_mode) ) rc = SQLITE_ERROR;
      }
      zCopy[i] = '/';
      i++;
    }
 
    sqlite3_free(zCopy);
  }
 
  return rc;
}
 
/*
** Set the error message contained in context ctx to the results of
** vprintf(zFmt, ...).
*/
static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){
  char *zMsg = 0;
  va_list ap;
  va_start(ap, zFmt);
  zMsg = sqlite3_vmprintf(zFmt, ap);
  sqlite3_result_error(ctx, zMsg, -1);
  sqlite3_free(zMsg);
  va_end(ap);
}
 
/*
** Implementation of the "writefile(W,X[,Y[,Z]]])" SQL function.  
** Refer to header comments at the top of this file for details.
*/
static void writefileFunc(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  const char *zFile;
  mode_t mode = 0;
  int res;
  sqlite3_int64 mtime = -1;
 
  if( argc4 ){
    sqlite3_result_error(context, 
        "wrong number of arguments to function writefile()", -1
    );
    return;
  }
 
  zFile = (const char*)sqlite3_value_text(argv[0]);
  if( zFile==0 ) return;
  if( argc>=3 ){
    mode = (mode_t)sqlite3_value_int(argv[2]);
  }
  if( argc==4 ){
    mtime = sqlite3_value_int64(argv[3]);
  }
 
  res = writeFile(context, zFile, argv[1], mode, mtime);
  if( res==1 && errno==ENOENT ){
    if( makeDirectory(zFile, mode)==SQLITE_OK ){
      res = writeFile(context, zFile, argv[1], mode, mtime);
    }
  }
 
  if( argc>2 && res!=0 ){
    if( S_ISLNK(mode) ){
      ctxErrorMsg(context, "failed to create symlink: %s", zFile);
    }else if( S_ISDIR(mode) ){
      ctxErrorMsg(context, "failed to create directory: %s", zFile);
    }else{
      ctxErrorMsg(context, "failed to write file: %s", zFile);
    }
  }
}
// ------------------------------------------- end of imported functions from sqllite3.c ---------------------------------------------
 
/* get_free_sqlite_handle */
/*
looks for a free slot in sqlite_handles;
return its index if found, -1 otherwise;
*/
static unsigned get_free_sqlite_handle(void) {
   if (0 == nb_sqlite_free_handles) {
       fprintf(stderr, "maximum of open db [%d] reached, no free handles !n", MAX_DB);
       return -1;
   }
   for (unsigned i = 0; i < MAX_DB; i++)
      if (NULL == sqlite_handles[i])
         return i;
   // should never come so far;
   return -1;
}
 
/* do_sqllite_open */
/* returns -1 if error, a db handle in the range 0 .. MAX_DB - 1 otherwise; */
static awk_value_t *
do_sqlite_open(int nargs, awk_value_t *result, struct awk_ext_func *unused) {
   awk_value_t db_name;
   short int ret;
 
   assert(result != NULL);
 
   unsigned int db_handle = get_free_sqlite_handle();
   if (-1 == db_handle)
      return make_number(-1, result);
 
   if (get_argument(0, AWK_STRING, &db_name)) {
      sqlite3 *db;
 
      ret = sqlite3_open(db_name.str_value.str, &db);
 
      if (ret) {
         char error_string[1000];
         sprintf(error_string, "sqlite3_open(): cannot open database [%s], error %sn", db_name.str_value.str, sqlite3_errmsg(db));
         fprintf(stderr, "%sn", error_string);
         update_ERRNO_string(_(error_string));
         ret = -1;
      }
      else {
         sqlite_handles[db_handle] = db;
         nb_sqlite_free_handles--;
         ret = db_handle;
 
         // register the extension functions readfile() and writefile() for blobs;
         ret = sqlite3_create_function(db, "readfile", 1, SQLITE_UTF8, 0, readfileFunc, 0, 0);
         if (ret == SQLITE_OK) {
            ret = sqlite3_create_function(db, "writefile", -1, SQLITE_UTF8, 0, writefileFunc, 0, 0);
            if (SQLITE_OK != ret)
               fprintf(stderr, "%sn", "could not register function writefile()");
         }
         else if (SQLITE_OK != ret)
            fprintf(stderr, "%sn", "could not register function readfile()");
      }
   }
   else {
      update_ERRNO_string(_("sqlite3_open(): missing parameter database name"));
      ret = -1;
   }
 
   return make_number(ret, result);
}
 
/* do_sqllite_close */
/* returns -1 if error, 0 otherwise; */
static awk_value_t *
do_sqlite_close(int nargs, awk_value_t *result, struct awk_ext_func *unused) {
   awk_value_t db_handle;
   int ret;
 
   assert(result != NULL);
 
   if (get_argument(0, AWK_NUMBER, &db_handle)) {
      sqlite3_close(sqlite_handles[(int) db_handle.num_value]);
      sqlite_handles[(int) db_handle.num_value] = NULL;
      nb_sqlite_free_handles++;
      ret = 0;
   }
   else {
      update_ERRNO_string(_("sqlite3_close(): missing parameter database handle"));
      ret = -1;
   }
   return make_number(ret, result);
}
 
/* do_sqllite_exec */
/*
returns -1 if error, 0 otherwise;
sqlite_exec is overloaded;
if 2 parameters, usual DML/DDL statements;
if 6 parameters, then incremental blob I/O;
sqlite_exec(db, db_name, table, column, rowid, readfile(file_name))
or
sqlite_exec(db, db_name, table, column, rowid, writefile(file_name))
implements sqlite3'c shell readfile()/writefile() syntax with incremental blob I/Os;
Example of usage:
first, get the rowid of the row that contains the blob to access;
   sqlite_select(db, "select rowid from <table> where <condition>", array)
then, call the sqlite_exec function with the tuple (<db_name>, <table>, <blob_column>, <rowid>) and the action to do, either readfile() or writefile();
   sqlite_exec(db, <db_name>, '<table>', '<blob_column>', array[0]["rowid"], readfile(file_name))
   sqlite_exec(db, <db_name>, '<table>', '<blob_column>', array[0]["rowid"], writefile(file_name))
e.g.:
   rc = sqlite_exec(my_db, "main", "test_with_blob", "my_blob", a_test[0]["rowid"], "readfile(/home/dmadmin/setup_files/documentum.tar)")
note how the file name is not quoted;
in case of readfile(), if the blob's size changes, an update of the blob filled with zero-byte bytes and with the new size is first performed, then the blob is reopened;
see doc here for incremental blob I/Os: https://sqlite.org/c3ref/blob_open.html;
int sqlite3_blob_open(sqlite3*, const char *zDb, const char *zTable, const char *zColumn, sqlite3_int64 iRow, int flags, sqlite3_blob **ppBlob);
int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64);
int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset);
int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset);
int sqlite3_blob_close(sqlite3_blob *);
*/
static awk_value_t *
do_sqlite_exec(int nargs, awk_value_t *result, struct awk_ext_func *unused) {
   awk_value_t db_handle;
   int ret = 1;
 
   assert(result != NULL);
 
   if (!get_argument(0, AWK_NUMBER, &db_handle)) {
      fprintf(stderr, "in do_sqlite_exec, cannot get the db handle argumentn");
      ret = -1;
      goto end;
   }
   if (2 == nargs) {
      awk_value_t sql_stmt;
      if (!get_argument(1, AWK_STRING, &sql_stmt))  {
         fprintf(stderr, "in do_sqlite_exec, cannot get the sql_stmt argumentn");
         ret = -1;
         goto end;
      }
      char *errorMessg = NULL;
      ret = sqlite3_exec(sqlite_handles[(int) db_handle.num_value], sql_stmt.str_value.str, NULL, NULL, &errorMessg);
      if (SQLITE_OK != ret) {
         fprintf(stderr, "in do_sqlite_exec, SQL error %s while executing [%s]n", sqlite3_errmsg(sqlite_handles[(int) db_handle.num_value]), sql_stmt.str_value.str);
         sqlite3_free(errorMessg);
         ret = -1;
         goto end;
      }
   }
   else if (6 == nargs) {
      awk_value_t arg, number_value;
      char *db_name = NULL, *table_name = NULL, *column_name = NULL, *file_stmt = NULL, *file_name = NULL;
 
      if (!get_argument(1, AWK_STRING, &arg))  {
         fprintf(stderr, "in do_sqlite_exec, cannot get the db_name argumentn");
         ret = -1;
         goto abort;
      }
      db_name = strdup(arg.str_value.str);
 
      if (!get_argument(2, AWK_STRING, &arg))  {
         fprintf(stderr, "in do_sqlite_exec, cannot get the table_name argumentn");
         ret = -1;
         goto abort;
      }
      table_name = strdup(arg.str_value.str);
 
      if (!get_argument(3, AWK_STRING, &arg))  {
         fprintf(stderr, "in do_sqlite_exec, cannot get the column_name argumentn");
         ret = -1;
         goto abort;
      }
      column_name = strdup(arg.str_value.str);
       
      if (!get_argument(4, AWK_NUMBER, &number_value))  {
         fprintf(stderr, "in do_sqlite_exec, cannot get the rowid argumentn");
         ret = -1;
         goto abort;
      }
      long int rowid = number_value.num_value;
       
      if (!get_argument(5, AWK_STRING, &arg))  {
         fprintf(stderr, "in do_sqlite_exec, cannot get the readfile()/writefile() argumentn");
         ret = -1;
         goto abort;
      }
      file_stmt = strdup(arg.str_value.str);
       
      unsigned short bRead2Blob;
      char *RE_readfile = "^readfile\(([^)]+)\)$";
      char *RE_writefile = "^writefile\(([^)]+)\)$";
      regex_t RE;
      regmatch_t pmatches[2];
 
      if (regcomp(&RE, RE_readfile, REG_EXTENDED)) {
         fprintf(stderr, "in do_sqlite_exec, error compiling REs %sn", RE_readfile);
         ret = -1;
         goto abort;
      }
      if (regexec(&RE, file_stmt, 2, pmatches, 0)) {
         // no call to readfile() requested, try writefile();
         regfree(&RE);
         if (regcomp(&RE, RE_writefile, REG_EXTENDED)) {
            fprintf(stderr, "in do_sqlite_exec, error compiling REs %sn", RE_writefile);
            ret = -1;
            goto abort;
         }
         if (regexec(&RE, file_stmt, 2, pmatches, 0)) {
            fprintf(stderr, "in do_sqlite_exec, error executing RE %s and RE %s against %s;nneither readfile(file_name) nor writefile(file_name) was foundn", RE_readfile, RE_writefile, file_stmt);
            ret = -1;
            goto abort;
         }
         else bRead2Blob = 0;
      }
      else bRead2Blob = 1;
      file_name = strndup(file_stmt + pmatches[1].rm_so, pmatches[1].rm_eo - pmatches[1].rm_so);
      regfree(&RE);
      sqlite3_blob *pBlob;
      if (bRead2Blob) {
         ret = sqlite3_blob_open(sqlite_handles[(int) db_handle.num_value], db_name, table_name, column_name, rowid, 1, &pBlob);
         if (SQLITE_OK != ret) {
            fprintf(stderr, "in do_sqlite_exec, at reading blob, with parameters: db_name=%s, table_name=%s, column_name=%s, rowid=%ld, file statement=%s, error in sqlite3_blob_open %sn%sn",
                            db_name, table_name, column_name, rowid, file_stmt,
                            sqlite3_errmsg(sqlite_handles[(int) db_handle.num_value]), sqlite3_errstr(ret));
            ret = -1;
            goto abort;
         }
 
         FILE *fs_in = fopen(file_name, "r");
         if (NULL == fs_in) {
            fprintf(stderr, "in do_sqlite_exec, error opening file %s for readingn", file_name);
            ret = -1;
            goto local_abort_w;
         }
 
         // will the blob size change ?
         fseek(fs_in, 0, SEEK_END);
         unsigned long file_size = ftell(fs_in);
         rewind(fs_in);
         unsigned long blobSize = sqlite3_blob_bytes(pBlob);
         if (file_size != blobSize) {
            // yes, must first update the blob with the new size and reopen it;
            char stmt[500];
            char *errorMessg = NULL;
            sprintf(stmt, "update %s set %s = zeroblob(%ld) where rowid = %ld", table_name, column_name, file_size, rowid);
            ret = sqlite3_exec(sqlite_handles[(int) db_handle.num_value], stmt, NULL, NULL, &errorMessg);
            if (SQLITE_OK != ret) {
               fprintf(stderr, "in do_sqlite_exec, SQL error %s while changing the blob's size through [%s]:n%sn", sqlite3_errmsg(sqlite_handles[(int) db_handle.num_value]), stmt, errorMessg);
               sqlite3_free(errorMessg);
               ret = -1;
               goto local_abort_w;
            }
            ret = sqlite3_blob_reopen(pBlob, rowid);
            if (SQLITE_OK != ret) {
               fprintf(stderr, "in do_sqlite_exec, error while reopening the blob: %sn%sn", sqlite3_errmsg(sqlite_handles[(int) db_handle.num_value]), sqlite3_errstr(ret));
               ret = -1;
               goto local_abort_w;
            }
         }
 
         // let's work with a 10 MiB large buffer;
         unsigned long BUFFER_SIZE = 10 * 1024 * 1024;
         char *pBuffer = (char *) malloc(sizeof(char) * BUFFER_SIZE);
         unsigned long nbBytes;
         unsigned long offset = 0;
         while ((nbBytes = fread(pBuffer, sizeof(char), BUFFER_SIZE, fs_in)) > 0) {
            ret = sqlite3_blob_write(pBlob, pBuffer, nbBytes, offset);
            if (SQLITE_OK != ret) {
               fprintf(stderr, "in do_sqlite_exec, sqlite3_blob_write, error %sn%sn", sqlite3_errmsg(sqlite_handles[(int) db_handle.num_value]), sqlite3_errstr(ret));
               ret = -1;
               free(pBuffer);
               goto local_abort_w;
            }
            offset += nbBytes;
         }
         free(pBuffer);
local_abort_w:
         fclose(fs_in);
         ret = sqlite3_blob_close(pBlob);
         if (SQLITE_OK != ret) {
            fprintf(stderr, "in do_sqlite_exec, sqlite3_blob_close, error %sn%sn", sqlite3_errmsg(sqlite_handles[(int) db_handle.num_value]), sqlite3_errstr(ret));
            ret = -1;
         }
      }
      else {
         ret = sqlite3_blob_open(sqlite_handles[(int) db_handle.num_value], db_name, table_name, column_name, rowid, 0, &pBlob);
         if (SQLITE_OK != ret) {
            fprintf(stderr, "in do_sqlite_exec at writing blob, error %d in sqlite3_blob_open with parameters: db_name=%s, table_name=%s, column_name=%s, rowid=%ld, file statement=%sn",
                            ret,
                            db_name, table_name, column_name, rowid, file_stmt);
            ret = -1;
            goto abort;
         }
         unsigned long BUFFER_SIZE = 10 * 1024 * 1024;
         char *pBuffer = (char *) malloc(sizeof(char) * BUFFER_SIZE);
         unsigned long offset = 0;
         FILE *fs_out = fopen(file_name, "w");
         if (NULL == fs_out) {
            fprintf(stderr, "in do_sqlite_exec, error %d opening file %s for writingn", errno, file_name);
            ret = -1;
            goto local_abort_r;
         }
         unsigned long blobSize = sqlite3_blob_bytes(pBlob);
         if (BUFFER_SIZE >= blobSize) {
            ret = sqlite3_blob_read(pBlob, pBuffer, blobSize, offset);
            if (SQLITE_OK != ret) {
               fprintf(stderr, "in do_sqlite_exec, sqlite3_blob_read, error %sn%sn", sqlite3_errmsg(sqlite_handles[(int) db_handle.num_value]), sqlite3_errstr(ret));
               ret = -1;
               goto local_abort_r;
            }
            unsigned long nbBytes = fwrite(pBuffer, sizeof(char), BUFFER_SIZE, fs_out);
            if (nbBytes < blobSize) {
               fprintf(stderr, "in do_sqlite_exec, error in fwrite()n");
               ret = -1;
               goto local_abort_r;
            }
         }
         else {
            unsigned long nbBytes;
            while ((nbBytes = (blobSize <= BUFFER_SIZE ? blobSize : BUFFER_SIZE)) > 0) {
               ret = sqlite3_blob_read(pBlob, pBuffer, nbBytes, offset);
               if (SQLITE_OK != ret) {
                  fprintf(stderr, "in do_sqlite_exec, sqlite3_blob_read, error %sn%sn", sqlite3_errmsg(sqlite_handles[(int) db_handle.num_value]), sqlite3_errstr(ret));
                  ret = -1;
                  goto local_abort_r;
               }
               ret = fwrite(pBuffer, sizeof(char), nbBytes, fs_out);
               if (ret < nbBytes) {
                  fprintf(stderr, "in do_sqlite_exec, error in fwrite()n");
                  ret = -1;
                  goto local_abort_r;
               }
               offset += nbBytes;
               blobSize -= nbBytes;
            }
         }
local_abort_r:
         fclose(fs_out);
         free(pBuffer);
         ret = sqlite3_blob_close(pBlob);
         if (SQLITE_OK != ret) {
            fprintf(stderr, "in do_sqlite_exec, processing of writefile(), sqlite3_blob_close, error %sn%sn", sqlite3_errmsg(sqlite_handles[(int) db_handle.num_value]), sqlite3_errstr(ret));
            ret = -1;
         }
      }
abort:
      free(db_name);
      free(table_name);
      free(column_name);
      free(file_stmt);
      free(file_name);
   }
   else {
      fprintf(stderr, "in do_sqlite_exec, unsupported number of parameters in statement while processing [%d]n", nargs);
      ret = -1;
   }
end:
   return make_number(ret, result);
}
 
static unsigned max(unsigned n1, unsigned n2) {
   if (n1 > n2)
      return n1;
   else return n2;
}
 
// this struct is used to pass parameters to the callbacks;
typedef struct DISPLAYED_TABLE {
   char *sqlStmt;
 
   unsigned short bHeaderPrinted;
   char *COL_SEPARATOR;
   char *ELLIPSIS;
   unsigned short len_ellipsis;
   unsigned short MAX_WIDTH;
   unsigned short MIN_WIDTH;
 
   unsigned nb_columns;
   unsigned *max_col_widths;
   unsigned *actual_col_widths;
   char *col_overflow_action;
   char **headers;
   unsigned widest_column;
 
   char *size_list;  // list of blank- or comma-separated column widths;
 
   unsigned long NR;
   unsigned short bStoreOrDisplay;
   awk_array_t gawk_array;
 
   unsigned short bEpilog;
} DISPLAYED_TABLE;
 
void cleanup(DISPLAYED_TABLE *dt) {
   if (dt -> sqlStmt)
      free(dt -> sqlStmt);
   if (dt -> max_col_widths)
      free(dt -> max_col_widths);
   if (dt -> actual_col_widths)
      free(dt -> actual_col_widths);
   for (unsigned i = 0; i < dt -> nb_columns; i++) {
      if (dt -> headers)
         free(dt -> headers[i]);
   }
   if (dt -> headers)
      free(dt -> headers);
   if (dt -> size_list)
      free(dt -> size_list);
}
 
// strip the trailing blanks so they are not counted toward the column width;
// and returns the number of characters from the beginning;
// former version attempted to insert a  terminator but it caused error when the string resides in code area (e.g. SELECT sqlite_version()) because modifications are not allowed there for obvious reasons;
unsigned getUsefulLen(char * str) {
   unsigned len = strlen(str);
   char *p = str + len - 1;
   while (' ' == *p && p > str) p--;
   if (' ' != *p)
      len = p - str + 1;
   else
      len = 0;
   return(len);
}
 
char *fillStr(char *S, char ch, unsigned max_len) {
   S[max_len] = '';
   for (char *p = S; max_len; p++, max_len--)
      *p = ch;
   return S;
}
 
/* select_callback_raw */
// displays the data without truncation nor cleaning up tainling blanks;
// columns are separated by dt -> COL_SEPARATOR, which is useful to import data as CSV;
static int select_callback_raw(void *vdt, int nb_columns, char **column_values, char **column_names) {
   DISPLAYED_TABLE *dt = (DISPLAYED_TABLE *) vdt;
 
   if (dt -> bEpilog) {
      printf("%ld rows selectedn", dt -> NR);
      cleanup(dt);
      return 0;
   }
 
   if (!dt -> bHeaderPrinted) {
      // header has not been printed yet, print it and afterwards print the first row;
      for (unsigned i = 0; i < nb_columns; i++)
         printf("%s%s", column_names[i], i  COL_SEPARATOR : "");
      printf("n");
      dt -> bHeaderPrinted = 1;
   }
 
   for (unsigned i = 0; i < nb_columns; i++)
      printf("%s%s", column_values[i], i  COL_SEPARATOR : "");
   printf("n");
   dt -> NR++;
   return 0;
}
 
/* select_callback_draft */
/*
display the data in maximum 15-character wide columns, with possible truncation, in which case an ellipsis (...) is appended;
at the end, the optimum widths for each column are listed so they can be passed as a string list in the call to sqlite_select(,, "....") to avoid truncation;
this output is convenient as a quick draft;
*/
static int select_callback_draft(void *vdt, int nb_columns, char **column_values, char **column_names) {
   DISPLAYED_TABLE *dt = (DISPLAYED_TABLE *) vdt;
 
   char col_str[dt -> MAX_WIDTH + 1];
 
   if (dt -> bEpilog) {
      printf("%ld rows selectedn", dt -> NR);
 
      printf("nOptimum column widthsn");
      printf("=====================n");
      printf("for query: %sn", dt -> sqlStmt);
      for (unsigned i = 0; i < dt > nb_columns; i++)
         printf("%-*s  %dn", dt -> widest_column + 5, dt -> headers[i], dt -> max_col_widths[i]);
 
      cleanup(dt);
      return 0;
   }
 
   if (!dt -> bHeaderPrinted) {
      // header has not been printed yet, print it and afterwards print the first row;
      dt -> nb_columns = nb_columns; 
      dt -> max_col_widths = (unsigned *) malloc(sizeof(unsigned) * nb_columns);
      dt -> actual_col_widths = (unsigned *) malloc(sizeof(unsigned) * nb_columns);
      dt -> headers = (char **) malloc(sizeof(char *) * nb_columns);
 
      char *header_line = NULL;
 
      for (unsigned i = 0; i &t; nb_columns; i++) {
         char *tmp_s;
         unsigned len = strlen(column_names[i]);
         dt -> max_col_widths[i] = len;
         dt -> widest_column = max(dt -> widest_column, len);
         if (len > dt -> MAX_WIDTH) {
            // column overflow, apply a truncation with ellipsis;
            dt -> actual_col_widths[i] = dt -> MAX_WIDTH;
            strncpy(col_str, column_names[i], dt -> MAX_WIDTH - dt -> len_ellipsis);
            col_str[dt -> MAX_WIDTH - dt -> len_ellipsis] = '';
            strcat(col_str, dt -> ELLIPSIS);
            tmp_s = col_str;
         }
         else if (len %lt; dt -> MIN_WIDTH) {
            dt -> actual_col_widths[i] = dt -> MIN_WIDTH;
            tmp_s = column_names[i];
         }
         else {
            dt -> actual_col_widths[i] = len;
            tmp_s = column_names[i];
         }
         printf("%-*s%s", dt -> actual_col_widths[i], tmp_s, i  COL_SEPARATOR : "");
         dt -> headers[i] = strdup(column_names[i]);
      }
      printf("n");
 
      for (unsigned i = 0; i < nb_columns; i++) {
         header_line = (char *) realloc(header_line, sizeof(char) * dt -> actual_col_widths[i]);
         fillStr(header_line, '-', dt -> actual_col_widths[i]);
         printf("%s%s", header_line, i  COL_SEPARATOR : "");
      }
      printf("n");
      free(header_line);
 
      dt -> bHeaderPrinted = 1;
   }
   // header has been printed, print the rows now;
   for (unsigned i = 0; i < nb_columns; i++) {
      char *tmp_s;
      unsigned len = getUsefulLen(column_values[i]);
      dt -> max_col_widths[i] = max(dt -> max_col_widths[i], len);
      if (len > dt -> actual_col_widths[i]) {
         strncpy(col_str, column_values[i], dt -> actual_col_widths[i] - dt -> len_ellipsis);
         col_str[dt -> actual_col_widths[i] - dt -> len_ellipsis] = '';
         strcat(col_str, dt -> ELLIPSIS);
         tmp_s = col_str;
      }
      else {
         tmp_s = column_values[i];
      }
      printf("%-*.*s%s", dt -> actual_col_widths[i], dt -> actual_col_widths[i], tmp_s, i  COL_SEPARATOR : "");
   }
   printf("n");
   dt -> NR++;
   return 0;
}
 
/* printConstrained */
// prints the row's column in constrained column widths;
static void printConstrained(DISPLAYED_TABLE *dt, char **data, unsigned nb_columns) {
   // let's replicate the data because they will be modified locally;
   char **ldata = (char **) malloc(sizeof(char *) * nb_columns);
   for (unsigned i = 0; i < nb_columns; i++)
      ldata[i] = strndup(data[i], getUsefulLen(data[i]));
   unsigned bWrapOccured;
   do {
      bWrapOccured = 0;
      for (unsigned i = 0; i < nb_columns; i++) {
         char *col_str = NULL;
         unsigned len = strlen(ldata[i]);
         dt -> actual_col_widths[i] = dt -> max_col_widths[i];
         if (len > dt -> max_col_widths[i]) {
            // column width overflow, apply the requested action: either wrap-around, truncate with ellipsis or truncate without ellipsis;
            if ('e' == dt -> col_overflow_action[i]) {
               if (dt -> max_col_widths[i] < dt -gt&; len_ellipsis)
                  dt -> actual_col_widths[i] = dt -> len_ellipsis;
               col_str = strndup(ldata[i], dt -> actual_col_widths[i] - dt -> len_ellipsis);
               col_str[dt -> actual_col_widths[i] - dt -> len_ellipsis] = '';
               strcat(col_str, dt -> ELLIPSIS);
               sprintf(ldata[i], "%*s", len, " ");
            }
            else if ('t' == dt -> col_overflow_action[i]) {
               col_str = strndup(ldata[i], dt -> actual_col_widths[i]);
               sprintf(ldata[i], "%*s", len, " ");
            }
            else if ('w' == dt -> col_overflow_action[i]) {
               col_str = strndup(ldata[i], dt -> actual_col_widths[i]);
               // shift the column names by as many printed characters;
               // the new column names will be printed at the next cycle of the inner loop 
               unsigned j;
               for (j = dt -> actual_col_widths[i]; j < len; j++)
                  ldata[i][j - dt -> actual_col_widths[i]] = ldata[i][j];
               ldata[i][len - dt -> actual_col_widths[i]] = '';
               bWrapOccured = 1;
            }
         }
         else {
            col_str = strdup(ldata[i]);
            // no wrap-around necessary here but prepare the str for the next cycle just in case;
            sprintf(ldata[i], "%*s", len, " ");
         }
         printf("%-*s%s", dt -> actual_col_widths[i], col_str, i  COL_SEPARATOR : "");
         free(col_str);
      }
      printf("n");
   } while (bWrapOccured);
   for (unsigned i = 0; i < nb_columns; i++)
      free(ldata[i]);
   free(ldata);
}
 
/* select_callback_sized */
// displays the columns within predetermined sizes, wrap-around if overflow;
static int select_callback_sized(void *vdt, int nb_columns, char **column_values, char **column_names) {
   DISPLAYED_TABLE *dt = (DISPLAYED_TABLE *) vdt;
 
   if (dt -> bEpilog) {
      printf("%ld rows selectedn", dt -> NR);
      cleanup(dt);
      return 0;
   }
 
   if (!dt -> bHeaderPrinted) {
      // header has not been printed yet, print it and afterwards print the first row;
      dt -> actual_col_widths = (unsigned *) malloc(sizeof(unsigned) * nb_columns);
      if (dt -> nb_columns < nb_columns) {
         unsigned last_width = dt -> max_col_widths[dt -> nb_columns - 1];
         fprintf(stderr, "warning: missing column sizes, extending the last provided one %dn", last_width);
         dt -> max_col_widths = (unsigned *) realloc(dt -> max_col_widths, sizeof(unsigned) * nb_columns);
         dt -> col_overflow_action = (char *) realloc(dt -> col_overflow_action, sizeof(char) * nb_columns);
         char last_overflow_action = dt -> col_overflow_action[dt -> nb_columns - 1];
         for (unsigned i = dt -> nb_columns; i &t; nb_columns; i++) {
            dt -> max_col_widths[i] = last_width;
            dt -> col_overflow_action[i] = last_overflow_action;
         }
         dt -> nb_columns = nb_columns;
      }
      else if (dt -> nb_columns > nb_columns) {
         fprintf(stderr, "warning: too many columns widths given, %d vs actual %d, ignoring the %d in excessn", dt -> nb_columns, nb_columns, dt -> nb_columns - nb_columns);
         dt -> nb_columns = nb_columns;
      }
      printConstrained(dt, column_names, nb_columns);
 
      char *header_line = NULL;
      for (unsigned i = 0; i < nb_columns; i++) {
         header_line = (char *) realloc(header_line, sizeof(char) * dt -> actual_col_widths[i]);
         fillStr(header_line, '-', dt -> actual_col_widths[i]);
         printf("%s%s", header_line, i < nb_columns - 1 ? dt -> COL_SEPARATOR : "");
      }
      printf("n");
      free(header_line);
      dt -> bHeaderPrinted = 1;
   }
   printConstrained(dt, column_values, nb_columns);
   dt -> NR++;
   return 0;
}
 
/* select_callback_array */
/*
returs the database rows into the gawk associative array passed as parameter;
its structure is as follows:
array[0] = sub-array_0
array[1] = sub-array_1
...
array[count-1] = sub-array_count-1
where the sub-arrays are associative arrays too with structure:
sub-array0[col1] = value1
sub-array0[col2] = value2
...
sub-array0[coln] = valuen
sub-array1[col1] = value1
...
sub-array1[coln] = valuen
...
sub-arraym[col1] = value1
...
sub-arraym[coln] = valuen
Said otherwise, the returned array is an array of associative arrays whose first dimension contains the rows and second dimension contains the columns, 
i.e. it' a table of database rows or an array of hashes, or a list of dictionaries;
in perl linguo, it's an array of hashes;
in python, it would be an array of dictionaries;
*/
static int select_callback_array(void *vdt, int nb_columns, char **column_values, char **column_names) {
   DISPLAYED_TABLE *dt = (DISPLAYED_TABLE *) vdt;
 
   if (dt -> bEpilog) {
      printf("%ld rows selectedn", dt -> NR);
      cleanup(dt);
      return 0;
   }
 
   awk_array_t row;
   awk_value_t value;
   awk_value_t row_index;
   awk_value_t col_index, col_value;
 
   if (!dt -> bHeaderPrinted) {
      // create the main array once;
      // doesn't work; keep the code in case a fix is found;
      //db_table = create_array();
      //value.val_type = AWK_ARRAY;
      //value.array_cookie = db_table; 
    
      // add it to gawk's symbol table so it appear magically in gawk's script namespace;
      //if (!sym_update(dt -> array_name, &value)) 
      //   fatal(ext_id, "in select_callback_array, creation of table array %s failedn", dt -> array_name);
      //db_table = value.array_cookie;
 
      // nothing special to do here;
      dt -> bHeaderPrinted = 1;
   }
   char index_str[50];
   unsigned len = sprintf(index_str, "%ld", dt -> NR);
   make_const_string(index_str, len, &row_index);
 
   // create the sub-array for each row;
   // indexes are the column names and values are the column values;
   row = create_array();
   value.val_type = AWK_ARRAY;
   value.array_cookie = row;
   if (! set_array_element(dt -> gawk_array, &row_index, &value))
      fatal(ext_id, "in select_callback_array, creation of row array %ld failedn", dt -> NR);
   row  = value.array_cookie;
 
   for (unsigned i = 0; i  < nb_columns; i++) {
      make_const_string(column_names[i], strlen(column_names[i]), &col_index);
      make_const_string(column_values[i], strlen(column_values[i]), &col_value);
      if (! set_array_element(row, &col_index, &col_value))
         fatal(ext_id, "in select_callback_array, assigned value %s to index %s at row %ld failedn", column_values[i], column_names[i], dt -> NR);
   }
 
   dt -> NR++;
   return 0;
}
 
/* do_sqllite_select */
/*
generic select entry point;
possible invocations:
Case: call profile:                                          --> action;
   0: sqlite_select(db, sql_stmt)                            --> draft output, default fixed width columns, with needed column widths list at the end;
   1: sqlite_select(db, sql_stmt, "")                        --> raw output, no truncation, | as default separator;
   2: sqlite_select(db, sql_stmt, "separator-string")        --> raw output, no truncation, use given string as separator;
   2: sqlite_select(db, sql_stmt, "list-of-columns-widths")  --> fixed sized column output, a w|t|e suffix is allowed for wrapping-around or truncating too large columns without or with ellipsis;
   3: sqlite_select(db, sql_stmt, dummy, gawk_array)         --> raw output into the gawk associative array gawk_array;
the appropriate callback will be called based on the invocation's profile;
returns -1 if error, 0 otherwise;
*/
static awk_value_t *
do_sqlite_select(int nargs, awk_value_t *result, struct awk_ext_func *unused) {
   awk_value_t db_handle, sql_stmt, col_sizes;
   int ret = 0;
 
   assert(result != NULL);
 
   if (!get_argument(0, AWK_NUMBER, &db_handle)) {
      fprintf(stderr, "in do_sqlite_select, cannot get the db handle argumentn");
      ret = -1;
      goto quit;
   }
   if (!get_argument(1, AWK_STRING, &sql_stmt)) {
      fprintf(stderr, "do_sqlite_select, cannot get the sql_stmt argumentn");
      ret = -1;
      goto quit;
   }
   DISPLAYED_TABLE dt;
   dt.sqlStmt = strdup(sql_stmt.str_value.str);
   dt.bHeaderPrinted = 0;
   dt.COL_SEPARATOR = "  ";
   dt.ELLIPSIS = "..."; dt.len_ellipsis = strlen(dt.ELLIPSIS); 
   dt.MAX_WIDTH = 15;
   dt.MIN_WIDTH = dt.len_ellipsis + 5;
   dt.nb_columns = 0;
   dt.max_col_widths = NULL;
   dt.actual_col_widths = NULL;
   dt.col_overflow_action = NULL;
   dt.headers = NULL;
   dt.widest_column = 0;
   dt.size_list = NULL;
   dt.NR = 0;
   dt.bStoreOrDisplay = 1;
   dt.gawk_array = NULL;
   dt.bEpilog = 0;
 
   unsigned short bCase;
   unsigned short bFoundSeparator = 0;
   char *errorMessg = NULL;
 
   if (4 == nargs) {
      bCase = 3;
      awk_value_t value;
      if (!get_argument(3, AWK_ARRAY, &value))
         fatal(ext_id, "in do_sqlite_select, accessing the gawk array parameter failedn");
      dt.gawk_array = value.array_cookie;
      clear_array(dt.gawk_array);
   }
   else if (get_argument(2, AWK_STRING, &col_sizes)) {
      if (0 == strlen(col_sizes.str_value.str))
         // raw, unformatted output;
         bCase = 1;
      else {
         // columns are output with constrained widths and possible wrapping-around or truncation with/without ellipsis;
         bCase = 2;
         char *width_str, *tmp_str, *next_tok_iter;
         long width_value;
         tmp_str = strdup(col_sizes.str_value.str);
         next_tok_iter = tmp_str;
         while ((width_str = strtok(next_tok_iter, " ,/"))) {
            errno = 0;
            char *overflow_action_suffix;
            width_value = strtol(width_str, &overflow_action_suffix, 10);
            if ((errno == ERANGE && (width_value == LONG_MAX || width_value == LONG_MIN)) ||
                (errno != 0 && width_value == 0) ||
                (width_value < 0)) {
               if (0 == dt.nb_columns) {
                  // let's take this as a separator for select_callback_raw();
                  dt.COL_SEPARATOR = width_str;
                  bFoundSeparator = 1;
                  bCase = 0;
               }
               else {
                  fprintf(stderr, "invalid number in size string [%s], exiting ...n", width_str);
                  if (dt.nb_columns > 0) {
                     free(dt.max_col_widths);
                     free(dt.col_overflow_action);
                  }
                  free(tmp_str);
                  ret = -1;
                  goto quit;
               }
            }
            else if (bFoundSeparator) {
               // nothing else is accepted after a separator;
               fprintf(stderr, "separator [%s] must be the only parameter in raw output, exiting ...n", dt.COL_SEPARATOR);
               free(tmp_str);
               ret = -1;
               goto quit;
            }
            dt.max_col_widths = (unsigned *) realloc(dt.max_col_widths, sizeof(unsigned) * (dt.nb_columns + 1));
            dt.col_overflow_action = (char *) realloc(dt.col_overflow_action, sizeof(char) * (dt.nb_columns + 1));
            dt.max_col_widths[dt.nb_columns] = width_value;
            if (NULL == overflow_action_suffix || ! *overflow_action_suffix)
               dt.col_overflow_action[dt.nb_columns] = 'e';
            else if ('t' == *overflow_action_suffix || 'w' == *overflow_action_suffix || 'e' == *overflow_action_suffix)
               dt.col_overflow_action[dt.nb_columns] = *overflow_action_suffix;
            else if (0 == dt.nb_columns) {
               bCase = 0;
               dt.COL_SEPARATOR = strdup(width_str);
               bFoundSeparator = 1;
               dt.nb_columns++;
               break;      
            }
            else {
               // allowed overflow suffix is one of t, w or e;
               fprintf(stderr, "invalid overflow action suffix [%c]; it must be one of w (wrap-around), t (truncation without ellipsis) or e (truncation with ellipsis), exiting ...n", *overflow_action_suffix);
               free(tmp_str);
               ret = -1;
               goto quit;
            }
            if ('e' == dt.col_overflow_action[dt.nb_columns] && width_value < dt.len_ellipsis) {
               fprintf(stderr, "column [%d] has maximum width [%ld] and requests a truncation with ellipsis [%s] but a minimum width of [%d] characters is necessary for this, assuming that minimum widthn", dt.nb_columns, width_value, dt.ELLIPSIS, dt.len_ellipsis);
               dt.max_col_widths[dt.nb_columns] = dt.len_ellipsis;
            }
            dt.nb_columns++;
            next_tok_iter = NULL;
         }
         free(tmp_str);
      }
   }
   else
      // draft output, i.e. default column width, possible truncation, optimal column widths listed at the end;
      bCase = 0;
 
   switch (bCase) {
      case 0: ret = sqlite3_exec(sqlite_handles[(int) db_handle.num_value], sql_stmt.str_value.str, select_callback_draft, &dt, &errorMessg);
              break;
      case 1: if (!bFoundSeparator)
                 // use default separator
                 dt.COL_SEPARATOR = "|";
              ret = sqlite3_exec(sqlite_handles[(int) db_handle.num_value], sql_stmt.str_value.str, select_callback_raw, &dt, &errorMessg);
              break;
      case 2: ret = sqlite3_exec(sqlite_handles[(int) db_handle.num_value], sql_stmt.str_value.str, select_callback_sized, &dt, &errorMessg);
              break;
      case 3: ret = sqlite3_exec(sqlite_handles[(int) db_handle.num_value], sql_stmt.str_value.str, select_callback_array, &dt, &errorMessg);
              break;
      default: fprintf(stderr, "programming error: did you not forget a case ?n");
   }
   if (SQLITE_OK == ret) {
      dt.bEpilog = 1;
      0 == bCase ? select_callback_draft(&dt, 0, NULL, NULL) :
      1 == bCase ? select_callback_raw(&dt, 0, NULL, NULL) :
      2 == bCase ? select_callback_sized(&dt, 0, NULL, NULL) :
      3 == bCase ? select_callback_array(&dt, 0, NULL, NULL) :
      0;
   }
   else {
      fprintf(stderr, "do_sqlite_select, SQL error %s while executing [%s]n", errorMessg, sql_stmt.str_value.str);
      sqlite3_free(errorMessg);
   }
quit:
   return make_number(ret, result);
}
 
/* these are the exported functions along with their min and max arities; */
   static awk_ext_func_t func_table[] = {
        {"sqlite_open", do_sqlite_open, 1, 1, awk_false, NULL},
        {"sqlite_close", do_sqlite_close, 1, 1, awk_false, NULL},
        {"sqlite_exec", do_sqlite_exec, 6, 2, awk_false, NULL},
        {"sqlite_select", do_sqlite_select, 4, 2, awk_false, NULL},
};
 
static awk_bool_t (*init_func)(void) = init_sqlite_handles;
 
/* define the dl_load function using the boilerplate macro */
 
dl_load_func(func_table, sqlite_gawk, "")

Quite the extension ! Sorry for this lengthy listing but there is a lot of stuff going on here.
Next, let’s make the awk and the new extension. Here are the incantations:
pwd /home/dmadmin/dmgawk/gawk-4.2.1/extension ./configure make cd .libs; gcc -o sqlite_gawk.so -shared sqlite_gawk.o ../sqlite3.o -pthread
That’s it. As said elsewhere, an additional sudo make install will install the new gawk and its extension to their canonical locations, i.e. /usr/local/bin/gawk for gawk and /usr/local/lib/gawk for the extensions. But for the moment, let’s test it; for this, we still need a test gawk script.
vi tsqlite.awk

# test program for the sqlite_gawk, interface to sqlite3;
# Cesare Cervini
# dbi-services.com
# 8/2018
 
@load "sqlite_gawk"
 
BEGIN {
   my_db = sqlite_open("/home/dmadmin/sqlite-amalgamation-3240000/test.db")
   print "db opened:", my_db
 
   my_db2 = sqlite_open("/home/dmadmin/sqlite-amalgamation-3240000/test.db")
   print "db opened:", my_db2
 
   sqlite_close(my_db)
   sqlite_close(my_db2)
 
   my_db = sqlite_open("/home/dmadmin/sqlite-amalgamation-3240000/test.db")
   print "db opened:", my_db
 
   printf "n"
 
   rc = sqlite_exec(my_db, "CREATE TABLE IF NOT EXISTS test1(n1 NUMBER, s1 TEXT, s2 CHAR(100))")
   print "return code = ", rc
 
   rc = sqlite_exec(my_db, "INSERT INTO test1(n1, s1, s2) VALUES(100, "hello1", "hello0101")")
   print "return code = ", rc
   rc = sqlite_exec(my_db, "INSERT INTO test1(n1, s1, s2) VALUES(200, "hello2", "hello0102")")
   print "return code = ", rc
   rc = sqlite_exec(my_db, "INSERT INTO test1(n1, s1, s2) VALUES(300, "hello3", "hello0103")")
   print "return code = ", rc
   rc = sqlite_exec(my_db, "INSERT INTO test1(n1, s1, s2) VALUES(400, "hello4", "hello0104")")
   print "return code = ", rc
   rc = sqlite_exec(my_db, "INSERT INTO test1(n1, s1, s2) VALUES(400, "hello5 with spaces       ", "hello0105 with spaces              ")")
   print "return code = ", rc
   rc = sqlite_exec(my_db, "INSERT INTO test1(n1, s1, s2) VALUES(400, "hello6 with spaces        ", "hello0106   ")")
   print "return code = ", rc
   printf "n"
 
   stmt = "SELECT * FROM test1";
   split("", a_test)
   print "sqlite_select(my_db, " stmt ", 0, a_test)"
   rc = sqlite_select(my_db, stmt, 0, a_test)
   dumparray("a_test", a_test);
   for (row in a_test) {
      printf("row %d: ", row)
      for (col in a_test[row])
         printf("  %s = %s", col, a_test[row][col])
      printf "n"
   }
   printf "n"
 
   # print in draft format;
   stmt = "SELECT name FROM sqlite_master WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%' ORDER BY 1"
   print "sqlite_select(my_db, "" stmt "")"
   rc = sqlite_select(my_db, stmt)
   print "return code = ", rc
   printf "n"
 
   # print in draft format;
   stmt = "SELECT sql FROM sqlite_master ORDER BY tbl_name, type DESC, name"
   print "sqlite_select(my_db, "" stmt "", "100")"
   rc = sqlite_select(my_db, stmt , "100")
   print "return code = ", rc
   printf "n"
 
   # print in draft format;
   stmt = "SELECT * FROM test1"
   print "sqlite_select(my_db, " stmt ")"
   rc = sqlite_select(my_db, stmt)
   print "return code = ", rc
   printf "n"
 
   # print in raw format with non default separator;
   stmt = "SELECT * FROM test1"
   print "sqlite_select(my_db, " stmt ", "||")"
   rc = sqlite_select(my_db, stmt, "||")
   print "return code = ", rc
   printf "n"
 
   # now that we know the needed column widths, let's used them;
   # trailing spaces are removed to compact the column somewhat;
   stmt = "SELECT * FROM test1"
   print "sqlite_select(my_db, " stmt ", "3 18 21")"
   rc = sqlite_select(my_db, stmt, "3 18 21")
   print "return code = ", rc
   printf "n"
 
   # print in raw format, with default | separator;
   stmt = "SELECT * FROM test1"
   print "sqlite_select(my_db, " stmt ", "")"
   rc = sqlite_select(my_db, stmt, "")
   print "return code = ", rc
   printf "n"
 
   stmt = "INSERT INTO test1(n1, s1, s2) VALUES(400, "hello6-with-spaces        ", "hello0106-12345")"
   print "sqlite_exec(my_db, " stmt ")"
   rc = sqlite_exec(my_db, stmt)
   print "return code = ", rc
   printf "n"
 
   stmt = "SELECT * FROM test1"
   print "sqlite_select(my_db, " stmt ", "2e 15e 10w")"
   rc = sqlite_select(my_db, stmt, "2e 15e 10w")
   print "return code = ", rc
   printf "n"
 
   stmt = "SELECT count(*) FROM test1"
   print "sqlite_select(my_db," stmt ")"
   rc = sqlite_select(my_db, stmt)
   print "return code = ", rc
   printf "n"
 
   stmt = "DELETE FROM test1"
   print "sqlite_exec(my_db, " stmt ")"
   rc = sqlite_exec(my_db, stmt)
   print "return code = ", rc
   printf "n"
 
   stmt = "SELECT count(*) FROM test1"
   print "sqlite_select(my_db," stmt ")"
   rc = sqlite_select(my_db, stmt)
   print "return code = ", rc
   printf "n"
 
   rc = sqlite_exec(my_db, "CREATE TABLE IF NOT EXISTS test_with_blob(n1 NUMBER, my_blob BLOB)")
   print "return code = ", rc
 
   rc = sqlite_exec(my_db, "DELETE FROM test_with_blob")
   print "return code = ", rc
 
   stmt = "INSERT INTO test_with_blob(n1, my_blob) VALUES(1, readfile("gawk-4.2.1.tar.gz"))"
   print "sqlite_exec(my_db," stmt ")"
   #rc = sqlite_exec(my_db, stmt)
   print "return code = ", rc
   printf "n"
 
    
   stmt = "SELECT n1, writefile('yy' || rowid, my_blob) FROM test_with_blob"
   print "sqlite_select(my_db, " stmt ")"
   rc = sqlite_exec(my_db, stmt)
   print "return code = ", rc
   printf "n"
 
   # file too large, > 3 Gb, fails silently;
   # do don't do it;
   # stmt = "INSERT INTO test_with_blob(n1, my_blob) VALUES(1000, readfile("/home/dmadmin/setup_files/documentum.tar"))" 
 
   # this one is OK at 68 Mb;
   stmt = "INSERT INTO test_with_blob(n1, my_blob) VALUES(1000, readfile("/home/dmadmin/setup_files/instantclient-basic-linux.x64-12.2.0.1.0.zip"))"
   print "sqlite_exec(my_db," stmt ")"
   rc = sqlite_exec(my_db, stmt)
   print "return code = ", rc
   printf "n"
 
   stmt = "SELECT n1, writefile('"yy' || rowid || '"', my_blob) FROM test_with_blob where n1 = 1000" 
   print "sqlite_select(my_db, " stmt ")"
   #rc = sqlite_exec(my_db, stmt)
   print "return code = ", rc
   printf "n"
 
   stmt = "INSERT INTO test_with_blob(n1, my_blob) VALUES(5000, readfile('/home/dmadmin/dmgawk/gawk-4.2.1/extension/sqlite_gawk.c'))"
   print "sqlite_exec(my_db," stmt ")"
   rc = sqlite_exec(my_db, stmt)
   print "return code = ", rc
   printf "n"
 
   stmt = "UPDATE test_with_blob set my_blob = readfile('/home/dmadmin/dmgawk/gawk-4.2.1/extension/sqlite_gawk.c') where n1 = 1000"
   print "sqlite_exec(my_db," stmt ")"
   rc = sqlite_exec(my_db, stmt)
   print "return code = ", rc
   printf "n"
 
   # xx is a 999'000'000 bytes file; the import using a memory buffer with that size takes some time to complete;
   # the incremental blob I/Os below seem faster;
   # to make one, use: dd if=/dev/zero of=xx count=990 bs=1000000
   stmt = "UPDATE test_with_blob set my_blob = readfile('/home/dmadmin/dmgawk/xx') where n1 = 1000"
   print "sqlite_exec(my_db," stmt ")"
   rc = sqlite_exec(my_db, stmt)
   print "return code = ", rc
   printf "n"
 
   # this is needed to enforce typing of a_array to array;
   # split("", a_test)
   delete(a_test)
   print "sqlite_select(db, "select rowid from test_with_blob where n1 = 1000 limit 1", 0, a_test)"
   sqlite_select(db, "select rowid from test_with_blob where n1 = 1000 limit 1", 0, a_test)
   print "after getting blob"
   dumparray("a_test", a_test)
   print "sqlite_exec(my_db, 'main', 'test_with_blob', 'my_blob', " a_test[0]["rowid"] ", writefile(~/dmgawk/my_blob_" a_test[0]["rowid"] "))"
   rc = sqlite_exec(my_db, "main", "test_with_blob", "my_blob", a_test[0]["rowid"], "writefile(/home/dmadmin/dmgawk/my_blob_" a_test[0]["rowid"] ")")
   print "return code = ", rc
   printf "n"
 
   #print "sqlite_exec(my_db, 'main', 'test_with_blob', 'my_blob', " a_test[0]["rowid"] ", readfile(/home/dmadmin/setup_files/documentum.tar))"
   #rc = sqlite_exec(my_db, "main", "test_with_blob", "my_blob", a_test[0]["rowid"], "readfile(/home/dmadmin/setup_files/documentum.tar)")
   #rc = sqlite_exec(my_db, "main", "test_with_blob", "my_blob", a_test[0]["rowid"], "readfile(/home/dmadmin/setup_files/patch.bin)")
   rc = sqlite_exec(my_db, "main", "test_with_blob", "my_blob", a_test[0]["rowid"], "readfile(/home/dmadmin/dmgawk/xx)")
   print "return code = ", rc
   printf "n"
 
   stmt = "SELECT n1, hex(my_blob) FROM test_with_blob where n1 = 2000 limit 1"
   stmt = "SELECT n1, my_blob FROM test_with_blob where n1 = 2000 limit 1"
   stmt = "SELECT n1, substr(my_blob, 1) FROM test_with_blob where n1 = 2000 limit 1"
   rc = sqlite_select(my_db, stmt)
   rc = sqlite_select(my_db, stmt, "10 100w")
   print "return code = ", rc
   printf "n"
 
   stmt = "SELECT n1, replace(my_blob, 'n', '\n') as 'noLF' FROM test_with_blob where n1 = 5000 limit 2"
   print "sqlite_select(my_db," stmt ", 10, 100w)"
   rc = sqlite_select(my_db, stmt, "10, 100w")
   print "return code = ", rc
   printf "n"
 
   sqlite_close(my_db)
 
   exit(0)
}
 
function dumparray(name, array, i) {
   for (i in array)
      if (isarray(array[i]))
         dumparray(name "["" i ""]", array[i])
      else
         printf("%s["%s"] = %sn", name, i, array[i])
      }

To execute the test:
AWKLIBPATH=gawk-4.2.1/extension/.libs gawk-4.2.1/gawk -f tsqlite.awk db opened: 0 db opened: 0 db opened: 0 return code = 0 return code = 0 return code = 0 return code = 0 return code = 0 return code = 0 sqlite_select(my_db, SELECT * FROM test1, 0, a_test) 6 rows selected a_test["0"]["n1"] = 100 a_test["0"]["s1"] = hello1 a_test["0"]["s2"] = hello0101 a_test["1"]["n1"] = 200 a_test["1"]["s1"] = hello2 a_test["1"]["s2"] = hello0102 a_test["2"]["n1"] = 300 a_test["2"]["s1"] = hello3 a_test["2"]["s2"] = hello0103 a_test["3"]["n1"] = 400 a_test["3"]["s1"] = hello4 a_test["3"]["s2"] = hello0104 a_test["4"]["n1"] = 400 a_test["4"]["s1"] = hello5 with spaces a_test["4"]["s2"] = hello0105 with spaces a_test["5"]["n1"] = 400 a_test["5"]["s1"] = hello6 with spaces a_test["5"]["s2"] = hello0106 row 0: n1 = 100 s1 = hello1 s2 = hello0101 row 1: n1 = 200 s1 = hello2 s2 = hello0102 row 2: n1 = 300 s1 = hello3 s2 = hello0103 row 3: n1 = 400 s1 = hello4 s2 = hello0104 row 4: n1 = 400 s1 = hello5 with spaces s2 = hello0105 with spaces row 5: n1 = 400 s1 = hello6 with spaces s2 = hello0106 sqlite_select(my_db, "SELECT name FROM sqlite_master WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%' ORDER BY 1") name -------- test test1 test_... 3 rows selected 1 columns displayed Optimum column widths ===================== for query: SELECT name FROM sqlite_master WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%' ORDER BY 1 name 14 return code = 0 sqlite_select(my_db, "SELECT sql FROM sqlite_master ORDER BY tbl_name, type DESC, name", "100") sql ---------------------------------------------------------------------------------------------------- CREATE TABLE test(a1 number) CREATE TABLE test1(n1 NUMBER, s1 TEXT, s2 CHAR(100)) CREATE TABLE test_with_blob(n1 NUMBER, my_blob BLOB) 3 rows selected return code = 0 sqlite_select(my_db, SELECT * FROM test1) n1 s1 s2 -------- -------- -------- 100 hello1 hello... 200 hello2 hello... 300 hello3 hello... 400 hello4 hello... 400 hello... hello... 400 hello... hello... 6 rows selected 3 columns displayed Optimum column widths ===================== for query: SELECT * FROM test1 n1 3 s1 18 s2 21 return code = 0 sqlite_select(my_db, SELECT * FROM test1, "||") n1 ||s1 ||s2 --------||--------||-------- 100 ||hello1 ||hello... 200 ||hello2 ||hello... 300 ||hello3 ||hello... 400 ||hello4 ||hello... 400 ||hello...||hello... 400 ||hello...||hello... 6 rows selected 3 columns displayed Optimum column widths ===================== for query: SELECT * FROM test1 n1 3 s1 18 s2 21 return code = 0 sqlite_select(my_db, SELECT * FROM test1, "3 18 21") n1 s1 s2 --- ------------------ --------------------- 100 hello1 hello0101 200 hello2 hello0102 300 hello3 hello0103 400 hello4 hello0104 400 hello5 with spaces hello0105 with spaces 400 hello6 with spaces hello0106 6 rows selected return code = 0 sqlite_select(my_db, SELECT * FROM test1, "") n1|s1|s2 100|hello1|hello0101 200|hello2|hello0102 300|hello3|hello0103 400|hello4|hello0104 400|hello5 with spaces |hello0105 with spaces 400|hello6 with spaces |hello0106 6 rows selected return code = 0 sqlite_exec(my_db, INSERT INTO test1(n1, s1, s2) VALUES(400, "hello6-with-spaces ", "hello0106-12345")) return code = 0 sqlite_select(my_db, SELECT * FROM test1, "2e 15e 10w") column [0] has maximum width [2] and requests a truncation with ellipsis [...] but a minimum width of [3] characters is necessary for this, assuming that minimum width n1 s1 s2 --- --------------- ---------- 100 hello1 hello0101 200 hello2 hello0102 300 hello3 hello0103 400 hello4 hello0104 400 hello5 with ... hello0105 with space s 400 hello6 with ... hello0106 400 hello6-with-... hello0106- 12345 7 rows selected return code = 0 sqlite_select(my_db,SELECT count(*) FROM test1) count(*) -------- 7 1 rows selected 1 columns displayed Optimum column widths ===================== for query: SELECT count(*) FROM test1 count(*) 8 return code = 0 sqlite_exec(my_db, DELETE FROM test1) return code = 0 sqlite_select(my_db,SELECT count(*) FROM test1) count(*) -------- 0 1 rows selected 1 columns displayed Optimum column widths ===================== for query: SELECT count(*) FROM test1 count(*) 8 return code = 0 return code = 0 return code = 0 sqlite_exec(my_db,INSERT INTO test_with_blob(n1, my_blob) VALUES(1, readfile("gawk-4.2.1.tar.gz"))) return code = 0 sqlite_select(my_db, SELECT n1, writefile('yy' || rowid, my_blob) FROM test_with_blob) return code = 0 sqlite_exec(my_db,INSERT INTO test_with_blob(n1, my_blob) VALUES(1000, readfile("/home/dmadmin/setup_files/instantclient-basic-linux.x64-12.2.0.1.0.zip"))) return code = 0 sqlite_select(my_db, SELECT n1, writefile('"yy' || rowid || '"', my_blob) FROM test_with_blob where n1 = 1000) return code = 0 sqlite_exec(my_db,INSERT INTO test_with_blob(n1, my_blob) VALUES(5000, readfile('/home/dmadmin/dmgawk/gawk-4.2.1/extension/sqlite_gawk.c'))) return code = 0 sqlite_exec(my_db,UPDATE test_with_blob set my_blob = readfile('/home/dmadmin/dmgawk/gawk-4.2.1/extension/sqlite_gawk.c') where n1 = 1000) return code = 0 sqlite_exec(my_db,UPDATE test_with_blob set my_blob = readfile('/home/dmadmin/dmgawk/xx') where n1 = 1000) return code = 0 sqlite_select(db, "select rowid from test_with_blob where n1 = 1000 limit 1", 0, a_test) 1 rows selected after getting blob a_test["0"]["rowid"] = 1 sqlite_exec(my_db, 'main', 'test_with_blob', 'my_blob', 1, writefile(~/dmgawk/my_blob_1)) return code = 0 sqlite_exec(my_db, 'main', 'test_with_blob', 'my_blob', 1, readfile(/home/dmadmin/setup_files/documentum.tar)) return code = 0 sqlite_select(my_db) return code = 0 sqlite_select(my_db,SELECT n1, replace(my_blob, 'n', 'n') as 'noLF' FROM test_with_blob where n1 = 5000 limit 2, 10, 100w) n1 noLF ---------- ---------------------------------------------------------------------------------------------------- 5000 /*n * sqlite-gawk.c - an interface to sqlite() library;n * Cesare Cervinin * dbi-services.comn * 8/2018n*/n#ifdef HAVE_CONFIG_Hn#include n#endifnn#include n#include n#include n#include n#include nn#include n#inc lude nn#include "gawkapi.h"nn// extension;n#include n#include n# include n#include n#include n#include n#inclu de n#include nn#include "gettext.h"n#define _(msgid) gettext(msgid)n#defi ne N_(msgid) msgidnnstatic const gawk_api_t *api; /* for convenience macros to work */nstatic a wk_ext_id_t ext_id;nstatic const char *ext_version = "an interface to sqlite3: version 1.0";nnint plugin_is_GPL_compatible;nn/* internal structure and variables */n/*ninternally stores the db h ... c)(void) = init_sqlite_handles;nn/* define the dl_load function using the boilerplate macro */nn dl_load_func(func_table, sqlite_gawk, "")nn 1 rows selected return code = 0
That was a very long second part. If you are still there, please turn now to part Part III for some explanation of all this.

Post Views: 1,444

A SQLite extension for gawk (part II)

Oracle Team

Leave a Reply:

Related blog articles

Oracle Team

Leave a Reply:

Related blog articles

M-Files IMPACT Global Conference 2025 – Day 3

M-Files IMPACT Global Conference 2025 – Day 2

M-Files IMPACT Global Conference 2025 – Day 1

Set SQL Server Trace Flags Automatically and Smoothly