contrib/malete/mlt/core.h

/*
        The Malete project - the Z39.2/Z39.50 database framework of OpenIsis.
        Version 0.9.x (patchlevel see file Version)
        Copyright (C) 2001-2003 by Erik Grziwotz, erik@openisis.org

        This library is free software; you can redistribute it and/or
        modify it under the terms of the GNU Lesser General Public
        License as published by the Free Software Foundation; either
        version 2.1 of the License, or (at your option) any later version.

        This library is distributed in the hope that it will be useful,
        but WITHOUT ANY WARRANTY; without even the implied warranty of
        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
        See the GNU Lesser General Public License for more details.

        You should have received a copy of the GNU Lesser General Public
        License along with this library; if not, write to the Free Software
        Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

        see README for more information
EOH */
#ifndef CORE_H

#include <string.h> /* various str* and mem* */

/*
        $Id: core.h,v 1.4 2005/05/29 10:30:36 kripke Exp $
        full interface of the Malete core
*/

#define CORE_VERSION "1.0.3"



/* ****************************************************************** */
/*                                                                    */
/* COMPILER FEATURES                                                  */
/*                                                                    */
/* ****************************************************************** */

#ifndef __STDC__ /* defined by ISO C */
#error "need ANSI/ISO C compiler"
#endif
/* "cpp -dM </dev/null" lists nonstandard machine and OS macros */

/* optimizing */
#ifdef __GNUC__
#       define OPT_INLINE __inline__ /* OPT_INLINE type func () */
#else
#       define OPT_INLINE
#endif
#if defined( __i386__ ) && defined( __GNUC__ )
/* called function pops args; makes most functions slightly faster */
/* type OPT_STDCALL func () on both declaration and definition */
#       define OPT_STDCALL __attribute__((stdcall))
#else
#       define OPT_STDCALL
#endif
#if defined( __i386__ ) && defined( __GNUC__ ) && defined(NDEBUG)
/* more aggressive: up to 3 args in registers, fails with -pg */
/* type OPT_REGPARM func () in declaration is sufficient */
/* also using in definition allows fallback to stdcall */
#       define OPT_REGPARM __attribute__((regparm(3)))
#else
#       define OPT_REGPARM OPT_STDCALL
#endif

/* CPU features */
#if defined( __sparc__ ) || defined( __ppc__ )
#       define CPU_BIG_ENDIAN
#endif
#if defined( __sparc__ )
#       define CPU_NEED_ALIGN
#endif
/* max bits of PAGE_SIZE; min is always 12 */
/* grep -r 'fine PAGE_SHIFT' /usr/src/linux/include/asm-* : 12..16 */
#if defined( __ia64__ )
#       define CPU_PAGE_SHIFT 16
#elif defined( __arm__ )
#       define CPU_PAGE_SHIFT 15
#elif defined( __i386__ ) || defined( __ppc__ )
#       define CPU_PAGE_SHIFT 12 /* there are more not exceeding 12 bits */
#else /* however 13 bits is not too much waste */
#       define CPU_PAGE_SHIFT 13 /* sparc, sparc64, alpha, m68k, cris */
#endif

/* 64 bit integer */
#ifdef __GNUC__
#       define LOLO_BUILTIN long long
#       define LOLO( v ) v##LL
#       define LULU( v ) v##ULL
#elif defined( _MSC_VER ) || defined( __BORLANDC__ )
#       define LOLO_BUILTIN __int64
#       define LOLO( v ) v /* is there some special suffix ??? */
#       define LULU( v ) v
#elif 0xFFFFFFFFL != ~0L /* 64 bit compiler ? */
#       define LOLO_BUILTIN long
#       define LOLO( v ) v##L
#       define LULU( v ) v##UL
#else
#       error "please use a compiler providing a 64 bit integer type. thanks."
#endif
typedef LOLO_BUILTIN lolo;
typedef unsigned LOLO_BUILTIN lulu;



/* ****************************************************************** */
/*                                                                    */
/* BUILD FEATURES                                                     */
/*                                                                    */
/* ****************************************************************** */

/* do not support the ENV_SHARED mode */
#if defined(BUILD_SHMODE) /* is broken */
#       undef BUILD_SHMODE
#endif



/* ****************************************************************** */
/*                                                                    */
/* DATA STRUCTURES AND UTILITIES                                      */
/*                                                                    */
/* ****************************************************************** */

/* some characters */
#define TAB 9 /* horizontal, that is */
#define LF 10 /* LineFeed a.k.a. newline - '\n' isn't really well defined */
#define VT 11 /* vertical, used as newline replacement */
#define CR 13 /* for windoze, telnet and the like */

extern int a2i ( const char *p, int l );
extern int a2il ( const char *p, int l, int *res );
extern int a2id ( const char *p, int l, int dflt );
extern int i2a ( char *p, int i );
extern int u2a ( char *p, unsigned u );
/* print exactly n digits, do not add 0 byte */
extern void u2a0 ( char *p, unsigned u, unsigned n );

extern const char b36dig[36]; /* '0'..'9', 'a'..'z' */
/* 0..35 '0'..'9', 'a'..'z' and 'A'..'Z', 36 else */
extern const char b36val[256];

extern const unsigned char lat1up[256]; /* latin 1 uppercase */
extern const unsigned char lat1ct[256]; /* latin 1 ctype */
enum { /* character type bits */
        CT_WHITE = 0x01, /* all <= ' ' */
        CT_DIGIT = 0x02, /* 0..9 */
        CT_ALPHA = 0x04, /* 'A'..'Z','a'..'z' */
        CT_IDENT = 0x08, /* digits,alphas,underscore */
        CT_SPECL = 0x10, /* all other ASCIIs */
        CT_LATIN = 0x20  /* ident + non ASCII Latin1 alphas */
};
enum { /* character type values */
        CT_W = CT_WHITE,
        CT_D = CT_DIGIT|CT_IDENT|CT_LATIN,
        CT_A = CT_ALPHA|CT_IDENT|CT_LATIN,
        CT_I = CT_IDENT|CT_LATIN,
        CT_S = CT_SPECL,
        CT_L = CT_LATIN,
        CT_N = 0 /* other C1 control or symbol */
};
#define CT_IS(type, c) (CT_##type == lat1ct[(unsigned char)(c)])
#define CT_HAS(flg, c) (CT_##flg & lat1ct[(unsigned char)(c)])


extern void mTr ( char *bytes, char from, char to, int len );


typedef struct Fld {
        int      tag;
        unsigned len; /* must use at most 31 bits, highest is temporarily abused */
        char    *val; /* not necessarily 0 terminated */
} Fld;

/*
        v (value) functions on single field
*/
#define V2I( f ) a2i( (f)->val, (f)->len )

/* field has primary value key length l */
#define VKEY( f, k, l ) ( \
        (f)->len >= (l) && !memcmp( (f)->val, k, l ) \
        && ((f)->len == (l) || TAB == (f)->val[(l)]) \
)

/* dup a field as 0-terminated string */
#define VDUPZ( f ) ((char*)mDupz((f)->val, (f)->len))

/*
        snip tab-separated subfields from value,
        setting tag to the subfield identifier and len/val to the contents.
        if opt is not 0, it lists the options wanted (* for the primary).
        dst->val should be 0-initialized;
        if it's greater than src->val, search starts at dst->val+dst->len,
        @return 0 if not found, else 1+dst->len
*/
extern int vGet ( Fld *dst, const Fld *src, const char *opt );
/*
        access to the primary value never fails, if initialized, so return len
*/
#define VPRI( dst, src ) ((dst)->val = 0, vGet(dst, src, 0), (dst)->len)
#define VEQZ( f, str ) (!strncmp((f)->val,str,(f)->len) && !str[(f)->len])

/* sign(a - b) */
extern int vCmp ( const Fld *a, const Fld *b );
/* a > b */
extern int vGt ( const Fld *a, const Fld *b );

/* comparision function type for rSort */
typedef int VGt ( const Fld *a, const Fld *b );


/*
        r (record) functions on an array of fields
        they expect the tag of the first field to be the negative number of fields.
        The value of the first field ("header") may contain various meta info.
        For lists representing database "records",
        the canonical format is [no[@pos]][<TAB>leader].
        Leader can be used e.g. to hold a Z39.2 leader as used by MARC.
        Other (protocol) lists should start with some type identifier.

        it's a matter of taste whether you like and use this typedef.
        as it's not going to be changed but only here to clarify the intend,
        the core functions do not use it.
*/
typedef const Fld *Rec;

#define RLEN( r )  (-(r)->tag)
#define REND( r )  ((r)-(r)->tag)
#define RLAST( r ) (REND(r)-1)

extern unsigned rSiz ( const Fld *r );

extern const Fld *rGet ( const Fld *r, int tag, int *pos );

/*
        get field with given tag and primary value or empty pv
*/
extern const Fld *rKey ( const Fld *r, int tag, const char *key );

/*
        create a new const rec in a contigous mAlloced piece of mem
        if siz is 0, rSiz() is used
*/
extern const Fld *rDup ( const Fld *src, unsigned siz );


extern int rSer ( char *buf, const Fld *rec );

extern void rSort ( Fld *rec, VGt *gt );
extern void rSortTag ( Fld *rec );
extern void rSortVal ( Fld *rec ); /* == rSort(vGt) */


#define DEFBLKLEN 8000 /* default buffer block length */
#define DEFFIELDS 40 /* default number of fields */

typedef struct LBlk { /* chained buffer block */
        struct LBlk *nxt;
        unsigned     siz;
        char         byt[DEFBLKLEN]; /* actual len may vary */
} LBlk;


/*
        A full-fledged, modifiable list.
*/
typedef struct List {
        Fld      *fld; /* fields list, initially fl0 */
        unsigned  fav; /* fields available at end of *fld buffer */
        int       siz; /* used secondary buffer size minus holes (add buf-blk.byt) */
        char     *buf; /* pointing into blk->byt */
        char     *end; /* of blk->byt */
        LBlk     *blk; /* buffer chain, initially bl0 */
        char     *bok; /* if == buf, buffers contain the serialization */
        Fld       fl0[DEFFIELDS];
        LBlk      bl0;
} List;


#define LLEN( l )  RLEN((l)->fld)
#define LEND( l )  REND((l)->fld)
#define LLAST( l ) RLAST((l)->fld)


/*
        initialize list and set header.
        if fmt is 0, no header is printed, and the list left empty.
        (i.e. the first field added will become the header).
        @return l
*/
extern List *lInit ( List *l, const char *fmt, ... );
/*
        completely clear all fields and buffers.
*/
extern List *lClr ( List *l );
/*
        clear all fields and buffers, but keep the header.
*/
extern List *lReset ( List *l );
/*
        clear all fields and buffers, keep nothing
*/
extern void OPT_STDCALL lFini ( List *l );

/*
        increase available space.
        If buffer has a flusher, call it.
        fields -1 means we want to append (keep last field contigous, do NOT flush).
        fields >0 means reserve at least mode fields.
*/
extern int OPT_REGPARM lExtend ( List *l, unsigned need, int fields );



/* bytes available at buf */
#define LAVL( l ) ((unsigned)((l)->end - (l)->buf))
#define LSIZ( l ) ((unsigned)((l)->buf - (l)->blk->byt + (l)->siz))

/*
        fragment used to start a new field with tag t.
        If the list was empty, the field will become the header,
        ignoring the tag.
*/
#define LDEFNEWF( l, t ) \
        int __i = (l)->fld->tag; \
        Fld *__f = (l)->fld - __i--; \
        __f->tag = t; \
        (l)->fld->tag = __i;

/* add field tag t reserving n bytes space */
#define LNEWF( l, t, n ) do { \
        if ( ((l)->fav && LAVL(l) >= n) || lExtend( l, n, 1 ) ) { \
                LDEFNEWF( l, t ) \
                __f->val = (l)->buf; \
                __f->len = 0; \
                (l)->fav --; \
        } } while(0)

/* add field tag t using n bytes prefilled space
        must be preextended for one field
*/
#define LPREF( l, t, n ) do { \
        if ( (l)->fav ) { \
                LDEFNEWF( l, t ) \
                __f->val = (l)->buf; \
                __f->len = n; \
                (l)->fav --; \
                (l)->buf += n; \
        } } while(0)


/* add field tag t with value v n bytes long */
#define LADD( l, t, v, n ) do { \
        if ( ((l)->fav && LAVL(l) >= n) || lExtend( l, n, 1 ) ) { \
                LDEFNEWF( l, t ) \
                memcpy( __f->val = (l)->buf, v, __f->len = n ); \
                (l)->fav --; \
                (l)->buf += n; \
        } } while(0)

/* append value v n bytes long */
#define LAPP( l, v, n ) do { \
        if ( LAVL(l) >= n || lExtend( l, n, -1 ) ) { \
                Fld *__f = LLAST(l); \
                memcpy( (l)->buf, v, n ); \
                __f->len += n; \
                (l)->buf += n; \
        } } while(0)

/* add field tag t with int value i */
#define LADDI( l, t, i ) do { \
        if ( ((l)->fav && LAVL(l) >= 12) || lExtend( l, 12, 1 ) ) { \
                LDEFNEWF( l, t ) \
                (l)->buf += __f->len = i2a( __f->val = (l)->buf, i ); \
                (l)->fav --; \
        } } while(0)

#define LADDS( l, t, s ) LADD( l, t, s, strlen(s) )
#define LAPPS( l, s )    LAPP( l, s, strlen(s) )
#define LADDF( l, f )    LADD( l, f->tag, f->val, f->len )


/*
        append canonical options fields.
        args starting with '-' are treated as options and appended as
        tab separated subfield w/o the '-'.
        If the first arg does not start with '-', it is used as primary value.
        Other non-options are appended as subfields indicated by '@'.
        In order to skip the program's name, from main() use with argc-1,argv+1.
*/
extern int lArgv ( List *l, int tag, int argc, const char **argv );
/*
        same using 0-terminated list.
extern int lArgs ( List *l, int tag, const char *arg, ... );
*/
extern List *lVar ( List *l, int argc, const char **argv );

/* if siz is 0, rSiz() is used */
extern int lCpy ( List *l, const Fld *src, unsigned siz );
#define LCPY( l, src ) lCpy( l, (src)->fld, LSIZ(src) )

/* create canned version of list as new record */
#define LCAN( l ) rDup( (l)->fld, LSIZ(l) )


/*
        Print a field to list.
        a fmt of 0 indicates varargs.
*/
extern int lOut ( List *to, int tag, const char *fmt, ... );


/*
        set the first field with tag to val (of len, -1 for strlen)
        kill additional fields with tag
        if no field found, add one at end
extern int lSet ( List *l, int tag, const char *val, unsigned len );
*/


enum { /* list parse state */
        LPS_SOR, /* 0, at start of record */
        LPS_SOL = 0x1000000, /* at start of line */
        LPS_TAG = 0x2000000, /* in tag (only digits seen in line) */
        LPS_VAL = 0x3000000, /* in val (some non-digit seen) */
        LPS_NEG = 0x4000000, /* found '-' at start of line */
        LPS_CR  = 0x8000000, /* kill carriage return character */
        LPS_LEN = 0x0ffffff
};
extern int lParse (List *l, const char *txt, int lps);


typedef struct Sink Sink;
typedef void sink (Sink *that, int eor); /* clean all or part of buffer(s) */

struct Sink {
        List  lst;
        sink *snk; /* sinking function */
        void *dst; /* destination */
        int   off; /* start of current record in list, negative after partial flush */
};

#define SINK(s) (s)->snk(s, 0)
#define SEOR(s) (s)->snk(s, 1)
#define SCPY(s, l) do { LCPY(&(s)->lst, l); SEOR(s); } while (0)
#define SCPYR(s, r) do { lCpy(&(s)->lst, r, 0); SEOR(s); } while (0)

/* ****************************************************************** */
/*                                                                    */
/* ENVIRONMENT AND SYSTEM                                             */
/*                                                                    */
/* ****************************************************************** */

#ifndef WIN32
typedef int file;
#define FIL_NONE -1
#else
typedef void *file;
#define FIL_NONE 0
#endif

/* environment.
*/
typedef struct Env {
        List     *opt; /* options (header is packed command line) */
        file      in;
        Sink     *out; /* a proper record sink expecting complete records */
        Sink     *err; /* a usually field buffered sink for eOut messages */
        int       log; /* level */
        int       flg;
        int       wri; /* write mode none/excl/shared */
        int       sig; /* interrupted by signal */
        unsigned  psz; /* system page size = 1<<psh enforced min 4K */
        int       psh; /* page shift (bits) 12..CPU_PAGE_SHIFT */
        unsigned  rml; /* r memory map limit (max pages per pointer map) */
        unsigned  qml; /* q memory map limit (max pages per tree map) */
        /* LBlk     *blk[5]; for memory management */
} Env;

enum { /* env flags */
        ENV_BUFFER = 1, /* env.err is buffered */
        ENV_MSYNC = 2 /* use msync */
};
enum { /* env writing mode */
        ENV_RO,    /* readonly - not writing */
        ENV_EXCL  /* exclusive access (the default) */
#ifdef BUILD_SHMODE
        , ENV_SHARED /* shared access */
#endif
};
enum { /* signal code */
        ENV_CANCEL = 1, /* abort current request */
        ENV_TERM /* abort current request and exit */
};

extern Env env; /* main environment */

enum { /* our very own errno */
        ERR_OK    = 0, /* 0 is no error, also read(2)'s EINTR, EAGAIN */
        ERR_NO    = -1, /* get details from errno */
        /* error level FATAL: we are wrong */
        ERR_IDIOT = -2, /* caught programming error */
        ERR_TRASH = -3, /* database internal consistency */
        LOG_FATAL   = ERR_TRASH, /* fatal internal errors: we can't go on */
        /* error levels SYERR,ERROR,IOERR: system or user was wrong */
        ERR_NOMEM = -4, /* out of memory, also open(2)'s EMFILE, ENFILE */
        ERR_IO    = -5, /* real IO error, also write(2)'s ENOSPC, EPIPE */
        ERR_BUSY  = -6, /* object is busy */
        LOG_SYSERR  = ERR_BUSY, /* problem with system ressources: bad file, no mem */
        ERR_BADF  = -7, /* bad file, also read(2)'s EINVAL, some of open(2) */
        ERR_FAULT = -8, /* 0 pointer or bad sized buffer given */
        ERR_INVAL = -9, /* general invalid parameters, any EINVAL errno */
        LOG_ERROR   = ERR_INVAL, /* unusable input, database or query */
        ERR_AGAIN = -10, /* no data at intr or nonblock */
        LOG_IOERR   = ERR_AGAIN, /* problem on IO */
        LOG_WARN    = -11, /* bad input */
        /* logging levels: nothing was wrong */
        LOG_INFO    = -12, /* some major event like opening a db */
        LOG_VERBOSE = -13, /* any event like reading a record */
        LOG_DEBUG   = -14, /* lots of processing details (debug built only) */
        LOG_TRACE   = -15, /* database content (log_str) */
        LOG_ALL     = LOG_TRACE
};


#define EADD(t, v, n) LADD(&env.out->lst, t, v, n)
#define EAPP(v, n)    LAPP(&env.out->lst, v, n)
#define EADDS(t, s)   LADDS(&env.out->lst, t, s)
#define EAPPS(s)      LAPPS(&env.out->lst, s)
#define EADDI(t, i)   LADDI(&env.out->lst, t, i)
#define EADDF(f)      LADDF(&env.out->lst, f)

/*
        Append a field with abs(tag) to env.out for non-negative tags,
        else to env.err if tag >= env's loglevel.
        For system errors, additional info is appended.
        env.err's eof is called, if any.
        returns tag

        supports only a small subset of printf formats -- see the src
        has %b (bytes), which is like %.*s, but prints the string in hex
*/
extern int eOut ( int tag, const char *fmt, ... );
extern int eRr ( int tag, const char *fmt, ... );
#ifndef NDEBUG
# define LOG_DBG eRr
#elif defined( __GNUC__ )
# define LOG_DBG( args... )
#else
# define LOG_DBG (void) /* compiler should dispose statement off */
#endif

extern void cInit ( List *args, Sink *out, Sink *err );
/* typical usage from main() */
#define CINIT(argl) cInit(lVar(lInit(argl,0), argc-1, argv+1), 0, 0)


extern void *mAlloc ( int size );
extern void *mDup ( const void *str, int size );
/* cp siz bytes, append a 0 byte */
extern char *mDupz ( const char *str, int size );
extern void *mTryAlloc ( int size );
extern void  mFree ( void *mem );
extern Fld  *mFldAlloc ( int nfields );
extern void  mFldFree ( Fld *fld );
extern LBlk *mBlkAlloc ( int size );
extern void  mBlkFree ( LBlk *blk );
extern List *mListAlloc ( const char *name ); /* lInitialized with name */
extern void  mListFree ( List *l );
#define mFldAlloc(n) ((Fld*)mAlloc((n)*sizeof(Fld)))
#define mFldFree mFree
#define mBlkFree mFree
#define MFREE(p) do { mFree(p); (p)=0; } while(0)


extern int tUpd ( lolo *tm );
extern int tGtf ( char *buf, lolo *tm );
extern char *tGtfm ( char *buf, lolo *tm );
extern void tSleep ( lolo tm );


/* ************************************************************
        disk files (block devices)
*/


enum {
        /* basic open flags */
        FIL_RD    = 0x001, /* shall be opened for input */
        FIL_WR    = 0x002, /* shall be opened for output */
        FIL_RDWR  = 0x003, /* shall be opened for both */
        FIL_TRY   = 0x004, /* do not complain if open fails */
        /* write flags */
        FIL_CREAT = 0x010, /* shall be created */
        FIL_TRUNC = 0x020, /* shall be truncated */
        FIL_SYNC  = 0x040, /* syncing output */
        /* lock flags */
        FIL_TLOCK = 0x100, /* try locking (EX with WR) */
        FIL_BLOCK = 0x200, /* blocking lock (EX with WR) */
        FIL_FLOCK = 0x300 /* any locking is set */
};

extern int fOpen ( file *f, const char *name, int flags );
extern int fClose ( file *f );
extern int fSize ( file f );
extern unsigned fTime ( file f ); /* mtime sec */


/*
        Like the syscalls, this returns the number of bytes on success, 0 on eof.
        fPwrite repeats and does not return an error when interrupted.
        On error, a negative value is returned.
*/
extern int fRead ( file *f, void *buf, unsigned len );
extern int fWrite ( file *f, const void *buf, unsigned len );
extern int fPread ( file *f, void *buf, unsigned len, unsigned off );
extern int fPwrite ( file *f, const void *buf, unsigned len, unsigned off );
extern int fSeek ( file *f, unsigned off );
extern int fTrunc ( file *f, unsigned length );
#ifdef BUILD_SHMODE
/* remove a full file lock as set by fOpen */
extern void fUnlock ( file f );
/*
        lock byte n
        use TLOCK or BLOCK, possibly with WR, to lock, 0 to unlock.
*/
extern int fLock ( file f, unsigned n, int flg );
#define FLOCK(f,n,flg) (ENV_SHARED==env.wri && fLock(f,n,flg))
#else
#define FLOCK(f,n,flg) 0
#endif
#define FLOCKSH(f,n) FLOCK(f,n,FIL_BLOCK)
#define FLOCKEX(f,n) FLOCK(f,n,FIL_BLOCK|FIL_WR)
#define FLOCKUN(f,n) FLOCK(f,n,0)

extern int fSlurp ( char **buf, int sz, const char *name, int opt );

/* record-oriented sink to (file)dst.
        Expects the list to contain a proper record.
        If the header starts with a digit, a W<TAB> is prepended.
        An empty header is ommited.
*/
extern void fSinkr (Sink *that, int eor);
/* line-oriented sink (field values only) to (file)dst.
        leaves the header alone, ignores tags and blank values.
        yet prints a blank line on eor.
*/
extern void fSinkl (Sink *that, int eor);

typedef struct {
        file     fil;
        int      flg;
        char    *map;
        unsigned npg; /* in pages of env.psz */
        unsigned lim; /* max pages to map */
#ifdef WIN32
        char    *nam; /* for shared mapping */
        void    *hdl; /* "mapping object" */
#endif
} FMap;

extern int fMOpen ( FMap *fm, const char *name, int flags );
extern int fMClose ( FMap *fm );
extern int fMap ( FMap *fm, unsigned npg );
extern int fMSync ( FMap *fm, unsigned page );


/*
        file input buffer structure for fGets.
        suitable for both temp fix buffers and, with some care, for List buffers.
*/
typedef struct {
        file      f; /* file to read from */
        unsigned  n; /* current line number (1 based) */
        unsigned  o; /* offset of p from file start */
        char     *b; /* buffer base (const) */
        unsigned  s; /* buffer size (const) */
        char     *p; /* start of current line */
        unsigned  l; /* line len */
        unsigned  m; /* more bytes after p+l (including the LF) */
} FBuf;
/* sloppy but convenient initializer macro */
#define FIL_BUF( fb, fil, buf )  do { \
        fb.f = fil; fb.b = fb.p = buf; fb.s = sizeof(buf); \
        fb.n = 1; fb.o = fb.l = fb.m = 0; \
        } while (0)
/* even more convenient macro, must be end of data def */
#define FIL_DEFBUF( f ) \
        char buf[0x2000]; \
        FBuf fb; \
        FIL_BUF(fb, f, buf);
/*
        set p and l to next line, lines are terminated by LF.
        if l, advance p after current line.
        if m, skip next byte.
        if m, search for newline.
        if no newline found, mv p downto b and read more chars upto s.
        if l==s, buffer is exhausted (and m is 0).
        else if m, p[l] is a newline.
        else if FIL_NONE == f,
        no bytes could be read (an eof was seen or somebody set NBLOCK).
        return whether we got a line
*/
extern int fGets ( FBuf *fb );
extern int fGetr ( List *l, FBuf *fb );



/* ****************************************************************** */
/*                                                                    */
/* DATABASE                                                           */
/*                                                                    */
/* ****************************************************************** */

enum { /* flags for record and query data/index file pairs */
        DX_OPEN  = 0x1, /* open */
        DX_WRITE = 0x2, /* open for writing */
        DX_ASYNC = 0x4, /* no synced write */
        DX_MODIF = 0x8  /* modified */
};
typedef struct Rdx {
        file        mrd;
        FMap        mrx;
        int         flg;
        int         mid; /* in records == maxid */
        int         rdl; /* length of data file in bytes */
        int         ptl; /* pointer bytes, by now always 8 */
        int         typ; /* type of pointer file */
} Rdx;

extern int rInit ( Rdx *rdx );
extern void rFini ( Rdx *rdx );

/*
        read record rid.
        if mpos and the rec is found at mpos or higher,
        rRead will backtrack to earlier versions.
*/
extern int rRead ( List *l, Rdx *rdx, int rid, unsigned mpos );
/*
        write a record
        use rid 0 to get new rid
        use opos -1 if you don't care about old pos
        to be transparent here, the rid@pos as found in standard recs
        is ignored and should be pre-skipped up to the leader.
        specify the record size if known, with 0, rSiz will be used
        @return the record id written (> 0) on success, <= 0 on error
*/
extern int rWrite ( Rdx *rdx, const Fld *r, int rid, int opos, unsigned siz );


typedef struct Cdx Cdx;

typedef struct Qdx { /* actually it's a B-Link-Tree */
        file           mqd; /* the leaves file */
        FMap           mqx; /* the tree file */
        const Cdx     *cdx; /* the collation */
        int            flg; /* flags: writeable */
        unsigned char  typ; /* cfg: leaf block type */
        unsigned char  ksz; /* cfg: max key length, default 255 */
        unsigned char  ptr; /* cfg: inverted file pointer type or plain value size */
        unsigned char  let; /* cfg: pct free on load */
        /* members set automatically:  */
        unsigned char  vsz; /* value size, min 4, default 8 */
        unsigned char  uni; /* value unique length (see qSet) */
        unsigned char  ftp; /* fork block type */
        unsigned char  dpt; /* depth (level of root over bottom > 0) */
        unsigned       lsz; /* leaf block size computed from type */
        unsigned       lln; /* # leaf blocks in index */
        unsigned       fln; /* # fork blocks in index */
        /* members considered internal:  */
        struct QLoad  *qld;
} Qdx;

enum { /* btree block type, size, flg */
        QDX_TYPMSK = 0xC0, /* highest 2 bits: basic type */
        QDX_LEAF   = 0x00, /* leaf block, portable */
        QDX_FORKLE = 0x40, /* fork block little endian */
        QDX_FORKBE = 0x80, /* fork block big endian */
        QDX_LEAFPV = 0xC0, /* leaf plain values (forks don't care) */
        /* next 2 bits 0x30 for future extensions */
        QDX_COMPRS = 0x08, /* flag compressed keys (not yet supported) */
        QDX_SIZMSK = 0x07, /* lowest 3 bits: blocksize */
        QDX_LEAF0K = 0x00, /* 1/2K blocks 0x0200 9+0 bits */
        QDX_LEAF1K = 0x01, /* 1K blocks 0x0400 */
        QDX_LEAF2K = 0x02, /* 2K blocks 0x0800 */
        QDX_LEAF4K = 0x03, /* 4K blocks 0x1000 */
        QDX_LEAF8K = 0x04  /* 8K blocks 0x2000, max for leaves */
};
enum {
        QDX_MAXVALPERLEAF = 0x800, /* max 8K / min 4 bytes vsz */
        QDX_LEAFSH =   9, /* leaf size shift+(0..4) ~ 512 bytes - 8K */
        QDX_FORKSH =  12  /* fork size shift+(0..4) ~ 4K - 64K */
};

typedef struct {
        unsigned char len;
        unsigned char byt[255];
}       Val;

typedef struct {
        Val           val;
        unsigned char len;
        unsigned char byt[255];
}       Key;


typedef struct QLoop QLoop;
typedef int QCb ( QLoop *self );

struct QLoop {
        QCb   *qcb;
        Qdx   *qdx;
        int    flg;
        Key    key;
        Key    to;
        /* set on callback: */
        Key    cur;
        unsigned            nvals;
        const unsigned char *vals;
};

enum { /* flags */
        QLOOP = 0, /* loop endless */
        /* stop based on QLoop.key */
        QEQ   = 1, /* loop while == key */
        QPF   = 2, /* loop on prefix key */
        /* stop based on QLoop.to */
        QUPTO = 4, /* loop while < to */
        QINCL = 5, /* loop while <= to */
        QSTOP = 7, /* mask for stop mode */
        QSKIP = 8, /* skip the from key */
        QSAME = 0x10 /* callback on same key */
};


extern int qInit ( Qdx *bt );
extern void qFini ( Qdx *bt );

/*
        load a sorted series of keys and hits into index.
        call repeatedly, using a key with val.len 0 in last call
*/
extern int qLoad ( Qdx *bt, Key *key );
extern int qLoadf ( Qdx *bt, file *f );

/*
        write the key-value pair to the index and return 0, unless:
        - the value is zero on the 1st uni bytes
          and there is already such a value (unique key, return 3)
        - there is an all-zero value for the key (stopword, return 2)
        - it is already there (full duplicate found, return 1)
        With plain values, only full duplicates are checked.
        Uni is usually the length of the initial segment of value,
        which is the rid in fulltext mode, else the tag.
        Where a unique key is found, the value is copied to key.
*/
extern int qSet ( Qdx *bt, Key *key );
enum {
        QST_OK,
        QST_FOUND,
        QST_STPWRD,
        QST_UNIKEY
};
/*
        delete a key-value pair. return 1 if found, else 0
*/
extern int qDel ( Qdx *bt, Key *key );

extern int qLoop ( QLoop *self );


/*
        standard values structured as pointers
        denoting a record and position where key occurred.
        The value structure is up to 3 big endian unsigned numbers:
        0..2 bytes for tag,
        3+(0..3) bytes for rid (in fulltext mode: before the tag) and
        0..4 bytes for pos,
        totalling from 4 (vsz min.) to 12 bytes.
*/
typedef struct { /* where a key has a hit */
        unsigned short tag; /* while neg. tags are allowed, sorting is unsigned */
        unsigned short ext; /* extend row id to six bytes (used as db number) */
        unsigned       rid; /* row id */
        unsigned       pos; /* word pos, usually with field occ<<16 */
}       Ptr;

enum {
        QDX_TAGMSK = 0xC0, /* mask length of tag 0..2 */
        QDX_TAG1   = 0x40,
        QDX_TAG2   = 0x80,
        QDX_TAGSH  =    6, /* ... shifted by 6 */
        QDX_RIDMSK = 0x30, /* mask length of rid 0..3 */
        QDX_RIDMIN =    3, /* based on 3 */
        QDX_RIDSH  =    4, /* ... shifted by 4 */
        QDX_RID3   = 0x00,
        QDX_RID4   = 0x10,
        QDX_RID5   = 0x20,
        QDX_RID6   = 0x30,
        QDX_FULTXT = 0x08, /* traditional fulltext ordering */
        QDX_POSMSK = 0x07, /* mask length pos info 0..4 */
        /* default settings based on 3 byte rid */
        QDX_ISIS   = QDX_RID3|QDX_TAG2|QDX_FULTXT|3,
        /* 0x8B, 8 byte fulltext 3+2+3 IFP format */
        QDX_STDDB  = QDX_TAG1|QDX_RID3
        /* 0x40, 4 byte 3 rid + 1 for 256 field tags */
};

/* decode a value to a pointer */
extern void qRdVal ( Ptr *ptr, const unsigned char *val, unsigned char typ );
/* create a value from hit. */
extern void qMkVal ( Val *val, Ptr *ptr, unsigned char typ );
/* decode a key to plaintext */
extern int qRdKey ( Qdx *qdx, char *plain, int l, Key *key );
/* create a key from plaintext, truncating to ksz. */
extern void qMkKey ( Qdx *qdx, Key *key, char *b, int l );
/* create a key and value from line. */
extern int qMkKeyVal ( Qdx *qdx, Key *key, char *b, int l );


typedef struct {
        Qdx *qdx;
        int  del;
        Key  pfx;
        Ptr  ptr;
} QSet;
/*
        set or del one keys
        basically this behaves like qMkKey, qMkVal, qSet
        ptr.pos is incremented
        with pfx, pfx is prepended to the key (before qMkKey)
*/
extern int qSetKeyVal (QSet *qst, char *val, int len);
/*
        split the value into words and qSetKeyVal each
        return the number of entries made
*/
extern int qSetKeyVals (QSet *qst, char *val, int len);


#define CDX_MAXSEQ 15   /* max byte sequence length */


extern const Cdx *cOpen ( const Fld *src );
extern int cEnc ( const Cdx *cdx, Key *key, unsigned char *b, int l, int w );
extern int cDec ( const Cdx *cdx, unsigned char *b, int l, Key *key );


enum { /* see Metadata.txt */
        MET_OPT = 001,
        MET_UNU = 002,
        MET_CTP = 003,
        MET_COL = 004,
        MET_VER = 005,
        MET_FLD = 006
};

typedef struct Db {
        char       *nam;
        char       *pat;
        int         flg;
        int         mnt; /* mount count */
        struct Db  *nxt; /* linked list */
        const Fld  *opt; /* inner meta data */
        Rdx         rdx;
        Qdx         qdx;
} Db;

extern Db *dOpen (const char *dbname);
extern void dClose (Db *db);
extern void dCloseAll ();

#define dRead(l,db,rid) ((rid)?rRead(l,&(db)->rdx,rid,0):lCpy(l,(db)->opt,0))
#define dWrite(db,r,rid) rWrite(&(db)->rdx,r,rid,-1,0)

#define dSet(db,key) qSet(&(db)->qdx,key)
#define dDel(db,key) qDel(&(db)->qdx,key)
#define dLoop(db,ql) ((ql)->qdx=&(db)->qdx, qLoop(ql))

#define CORE_H
#endif /* CORE_H */

---