- developers-list - monetdb.org

Re: [Monetdb-developers] [Monetdb-pf-checkins] pathfinder/modules/pftijah nexi.c, 1.49, 1.50 pftijah_tokenize.l, 1.12, 1.13 pftijah_util.mx, 1.2, 1.3 serialize_pftijah.mx, 1.41, 1.42
by Sjoerd Mullender 27 Feb '07

27 Feb '07

On 2007-02-27 16:43, Jan Flokstra wrote: > Update of /cvsroot/monetdb/pathfinder/modules/pftijah > In directory sc8-pr-cvs7.sourceforge.net:/tmp/cvs-serv27686 > > Modified Files: > nexi.c pftijah_tokenize.l pftijah_util.mx serialize_pftijah.mx > Log Message: > * repair BBP refcount bug for BAT Is this a fix which also applies to the stable branch? > * reimplement the direct bat acces methods in pftijah serialization for more > speed (and clarity). > > * Start optimizing the the pftijah tokenizer. The flex functions are called once > per handle_character() call. This leads to 2 malloc's per call. I tried to > do without the malloc's but this caused to a lot of strange results:-) > I am now planning to craft the flexer by hand. The first small experiment > shows there is a lot to gain there. (25% speedup in indexing time). > > > > Index: serialize_pftijah.mx > =================================================================== > RCS file: /cvsroot/monetdb/pathfinder/modules/pftijah/serialize_pftijah.mx,v > retrieving revision 1.41 > retrieving revision 1.42 > diff -u -d -r1.41 -r1.42 > --- serialize_pftijah.mx 23 Feb 2007 15:11:07 -0000 1.41 > +++ serialize_pftijah.mx 27 Feb 2007 15:43:37 -0000 1.42 > @@ -31,8 +31,8 @@ > > extern int handleTijahTerm(struct tijahContextStruct *ctx, char* term); > > -extern int useFlexScanner(char* buf, int len, struct tijahContextStruct* tjCtx); /* FLEX */ > -extern char* flexScanOneTerm(char* buf, int len); > +extern int useFlexScanner(char* buf, struct tijahContextStruct* tjCtx); /* FLEX */ > +extern char* flexScanOneTerm(char* buf); > > extern char* normalizeTerm(struct tijahContextStruct *ctx, char* term ); > > @@ -70,15 +70,10 @@ > typedef struct dbat_struct { > const char* name; > BAT* bat; > - int oid_mark; > - int max_i; > - int max_sz; > - bit dflt; /* fill with default value during extend */ > - int dflt_int; /* the default int value */ > - chr dflt_chr; /* the default chr value */ > - oid dflt_oid; /* the default oid value */ > - /* */ > - union { /* cast to perform direct indexex insert in [void,any] BATs */ > + oid raw_max; > + oid seqbase; > + oid seq_max; > + union { /* cast to perform direct indexe insert in [void,any] BATs */ > void* voidCAST; /* the basecast */ > chr* chrCAST; /* cast for [void,chr] BAT */ > int* intCAST; /* cast for [void,int] BAT */ > @@ -89,7 +84,6 @@ > int dbat_init(const char* name, dbat* dbat, BAT* b) { > dbat->name = name; > dbat->bat = b; > - dbat->dflt = FALSE; > if ( dbat->bat->htype != TYPE_void ) { > stream_printf(GDKerr,"ERROR: dbat_init(%s) non void BAT\n",dbat->name); > return 0; > @@ -98,31 +92,25 @@ > stream_printf(GDKerr,"ERROR: dbat_init(%s) unknown ttype(%d)\n",dbat->name,dbat->bat->ttype); > return 0; > } > - dbat->oid_mark = b->hseqbase; > - dbat->max_i = dbat->max_sz = BATcount(dbat->bat); > + dbat->seqbase = (oid)b->hseqbase; > + dbat->raw_max = (oid)BATcount(dbat->bat); > + dbat->seq_max = dbat->raw_max + dbat->seqbase; > dbat->cast.voidCAST = (void*)BUNfirst(dbat->bat); > /* */ > return 1; > } > > -int dbat_finalize(dbat* dbat) { > - BAT* b = dbat->bat; > > +int dbat_finalize(dbat* dbat, int topidx) { > void* top; > + BAT* b = dbat->bat; > > - int bottomTop = dbat->max_i; > - if ( 0 ) stream_printf(GDKout,"dbat_finalize(size=%d)\n",dbat->max_i); > + topidx -= (int)dbat->seqbase; > + int bottomTop = topidx; > switch( b->ttype ) { > case TYPE_int : > top = &dbat->cast.intCAST[bottomTop]; > break; > - case TYPE_chr: { > - b->batBuns->free = dbat->max_i; > - BATsetcount(b, dbat->max_i); > - b->tsorted = 0; > - b->batDirty = TRUE; /* VERY important this one */ > - return 1; > - } > case TYPE_oid: > top = &dbat->cast.oidCAST[bottomTop]; > break; > @@ -137,7 +125,7 @@ > /* */ > dbat->name = NULL; > dbat->bat = NULL; > - dbat->max_i = dbat->max_sz = 0; > + dbat->raw_max = dbat->seqbase = 0; > /* */ > return 1; > } > @@ -145,9 +133,14 @@ > #define MINCHUNK 8192 > #define MAXCHUNK 67108864 > > -int dbat_extend(dbat* dbat, int i_mark) { > - /* CHECK THIS if ( i_mark ) i = i_mark - dbat->oid_mark; */ > - size_t newsize = MAX(MIN(MAX(MINCHUNK,dbat->max_sz*2),(size_t)(dbat->max_sz+MAXCHUNK)),(size_t)i_mark); > +int dbat_extend(dbat* dbat, oid min_i /*raw-index*/, size_t forced_size) { > + size_t newsize; > + > + if ( forced_size ) { > + newsize = forced_size; > + } else { > + newsize = MAX(MIN(MAX(MINCHUNK,dbat->raw_max*2),(size_t)(dbat->raw_max+MAXCHUNK)),(size_t)min_i); > + } > > /* first check if the number of BUN's < INT_MAX. If this was the case > * and the previous time INT_MAX was returned this means the BAT cannot > @@ -156,94 +149,34 @@ > if ( newsize > INT_MAX ) { > newsize = INT_MAX; > > - if ( dbat->max_sz == INT_MAX ) { > + if ( dbat->raw_max == INT_MAX ) { > GDKerror("dbat_extend: BATextend[\"%s\"](size>INT_MAX) fails\n","incomplete"); > return -1; > } > } > - if ( 0 ) { stream_printf(GDKout,"dbat_extend[%s](%d -> %d)\n",dbat->name,dbat->max_sz,newsize); } > - dbat->max_sz= newsize; > +#if 0 > + stream_printf(GDKout,"dbat_extend[%s](%d -> %d)\n",dbat->name,dbat->raw_max,newsize); > +#endif > + dbat->raw_max= newsize; > + dbat->seq_max = dbat->raw_max + dbat->seqbase; > if ( !(dbat->bat = BATextend(dbat->bat,newsize)) ) { > GDKerror("dbat_extend: BATextend[\"%s\"](to %d) fails\n","incomplete",newsize); > return -1; > } > dbat->cast.voidCAST = (void*)BUNfirst(dbat->bat); > - /* > - * now check if there's a default value handler used > - * > - */ > - if ( dbat->dflt ) { > - switch( dbat->bat->ttype ) { > - case TYPE_int : { > - int v = dbat->dflt_int; > - int *to = &dbat->cast.intCAST[dbat->max_sz]; > - for(register int *p = &dbat->cast.intCAST[dbat->max_i]; p<to;) > - *p++ = v; > - break; > - } > - case TYPE_chr: { > - chr v = dbat->dflt_chr; > - chr *to = &dbat->cast.chrCAST[dbat->max_sz]; > - for(register chr *p = &dbat->cast.chrCAST[dbat->max_i]; p<to;) > - *p++ = v; > - break; > - } > - case TYPE_oid: { > - oid v = dbat->dflt_oid; > - oid *to = &dbat->cast.oidCAST[dbat->max_sz]; > - for(register oid *p = &dbat->cast.oidCAST[dbat->max_i]; p<to;) > - *p++ = v; > - break; > - } > - default: > - GDKerror("dbat_extend: bad ttype\n"); > - return -1; > - } > - } > - /* */ > return 1; > } > > int dbat_sizeHint(dbat* dbat, int sizeHint_mark) { > - int sizeHint = sizeHint_mark - dbat->oid_mark; > - int estimate = dbat->max_i + sizeHint; > - > - return dbat_extend(dbat, estimate); > -} > - > -INLINE static int dbat_set_oid(dbat* dbat, int pos_mark, oid v) { > - register int pos; > + int sizeHint = sizeHint_mark - dbat->seqbase; > + int estimate = dbat->raw_max + sizeHint; > > - if ( (pos=pos_mark - dbat->oid_mark) < dbat->max_i ) { > - dbat->cast.oidCAST[pos] = v; > - return 1; > - } else { > - if ( pos >= dbat->max_sz ) { > - if ( dbat_extend(dbat,pos) < 0 ) > - return -1; > - } > - dbat->max_i = pos + 1; > - dbat->cast.oidCAST[pos] = v; > - return 1; > - } > + return dbat_extend(dbat, estimate, 0); > } > > -INLINE static int dbat_set_int(dbat* dbat, int pos_mark, int v) { > - register int pos; > +#define dbat_set_oid(DBAT,I,V) (DBAT)->cast.oidCAST[I-(DBAT)->seqbase] = V > > - if ( (pos=pos_mark - dbat->oid_mark) < dbat->max_i ) { > - dbat->cast.intCAST[pos] = v; > - return 1; > - } else { > - if ( pos >= dbat->max_sz ) { > - if ( dbat_extend(dbat,pos) < 0 ) > - return -1; > - } > - dbat->max_i = pos + 1; > - dbat->cast.intCAST[pos] = v; > - return 1; > - } > -} > +#define dbat_set_int(DBAT,I,V) (DBAT)->cast.intCAST[I-(DBAT)->seqbase] = V > > /************************************************ > * > @@ -315,9 +248,6 @@ > > /************************************************ > * > - * > - * First the temporary shredder for Tijah by JF > - * > */ > > INLINE static oid > @@ -328,15 +258,15 @@ > BUN bun; > > HASHfnd_str(bun, tjctx->hm_globalTag, (str)t); > - if ( bun ) > - /* if ( (bun = BUNfnd(tjctx->hm_globalTag,t)) ) OLD */ > + if ( bun ) { > return *(oid*)BUNtail(tjctx->hm_globalTag,bun); > - else { > - if ( !BUNins(tjctx->b_globalTag, &tjctx->n_globalTag, (str)t, FALSE) ) { > + } else { > + if ( BUNins(tjctx->b_globalTag, &tjctx->n_globalTag, (str)t, FALSE) ) { > + return tjctx->n_globalTag++; > + } else { > GDKerror("INSERT OF \"%s\" in globalTag fails.\n"); > return oid_nil; > - } else > - return tjctx->n_globalTag++; > + } > } > #endif > } > @@ -349,10 +279,7 @@ > BUN bun; > > HASHfnd_str(bun, tjctx->hm_globalTerm, (str)t); > - if ( bun ) { > - return *(oid*)BUNtail(tjctx->hm_globalTerm,bun); > - } else > - return oid_nil; > + return ( bun ? *(oid*)BUNtail(tjctx->hm_globalTerm,bun) : oid_nil ); > } > > INLINE static oid > @@ -366,22 +293,35 @@ > if ( bun ) > return *(oid*)BUNtail(tjctx->hm_globalTerm,bun); > else { > - if ( !BUNins(tjctx->b_globalTerm, &tjctx->n_globalTerm, (str)t, FALSE)){ > + if ( BUNins(tjctx->b_globalTerm, &tjctx->n_globalTerm, (str)t, FALSE)){ > + return tjctx->n_globalTerm++; > + } else { > GDKerror("INSERT OF \"%s\" in globalTerm fails.\n"); > return oid_nil; > - } else > - return tjctx->n_globalTerm++; > + } > } > #endif > } > > -#define tj_add2plane(TJCTX,O) \ > - ((dbat_set_oid(&(TJCTX)->dbat_collPre, (TJCTX)->tijahPre, O) < 0) \ > - ? oid_nil : ((oid)(TJCTX)->tijahPre++)) > +INLINE oid tj_extend_plane(struct tijahContextStruct *tjctx) { > + oid base = tjctx->tijahPre - tjctx->dbat_collPre.seqbase; > > -#define insertPreSize(TJCTX,POS,SIZE) \ > - dbat_set_int(&TJCTX->dbat_collSize,(int)POS,SIZE) > + if ( base >= tjctx->dbat_collPre.raw_max ) { > + if ( dbat_extend(&tjctx->dbat_collPre,base, 0) < 0 ) > + return oid_nil; > + /* IMPORTANT: the size of the two bats is synchronized by the use > + * of the forced size (last) parameter of dbat_extend > + */ > + if ( dbat_extend(&tjctx->dbat_collSize,base,tjctx->dbat_collPre.raw_max) < 0 ) > + return oid_nil; > + } > + return tjctx->tijahPre++; > +} > > +#define tj_newPre(TJCTX) \ > + (((TJCTX)->tijahPre < (TJCTX)->dbat_collPre.seq_max) \ > + ? \ > + ((oid)(TJCTX)->tijahPre++) : tj_extend_plane(TJCTX)) > > int > handleTijahTerm(struct tijahContextStruct *tjctx, char* term) { > @@ -397,13 +337,13 @@ > } > } > if ( (termOid = tj_termOid(tjctx, term)) == oid_nil ) > - return 0; > + return -1; > } > if ( termOid ) { /* term is not a stopword */ > - if ( (tjPre = tj_add2plane(tjctx, termOid)) == oid_nil ) > + if ( (tjPre = tj_newPre(tjctx) ) == oid_nil ) > return 0; > - if ( insertPreSize(tjctx,tjPre,0) < 0 ) > - return -1; > + dbat_set_oid(&tjctx->dbat_collPre, tjPre, termOid); > + dbat_set_int(&tjctx->dbat_collSize,(int)tjPre,0); > #ifdef TJ_TRACE > if ( TJ_TRACE ) stream_printf(GDKout,"C[%s]:TERM: \"%s\", termoid=%d, tjPre=%d\n",tjctx->name,term,termOid,tjPre); > #endif > @@ -416,40 +356,13 @@ > return 1; > } > > -/************ > - * > - * The part where the Strings from Pathfinder are shredded into words > - * by Tijah. The USE_FLEX macro determines if the strings is shredded > - * by Hennings fancy flex scanner or Jan's simple strtok() scanner. > - */ > - > -const char* obsoleteNexiChars = " \t\n\r,:;&*%$#!@="; > - > -int > -useStrtokScanner(tjCtx* tjctx, char* s) > -{ > - char *t; > - int sz = 0; > - > -#ifdef TJ_TRACE > - if (TJ_TRACE) stream_printf(GDKout,"C[%s]:CHARACTERS:\n",tjctx->name); > -#endif > - if ( (t = strtok(s,obsoleteNexiChars)) ) do { > - /* not the empty string here */ > - if ( handleTijahTerm(tjctx,t) < 0 ) > - return -1; > - sz++; > - } while ( (t=strtok(NULL,obsoleteNexiChars)) ); > - return 1; > -} > - > /************************************************ > * > * Now the real output handlers > */ > > > -#ifdef notused > +#if 0 > static int > handle_sizeHint(XqueryCtx* ctx, int hinted_size) { > tjCtx* tjctx = (tjCtx*)ctx->driverWs; > @@ -502,14 +415,12 @@ > return (str)str_nil; > } > > -#define GUESSFORCE FALSE > - > /* > * Replace the value of a collection parameter int the collection parameter > * bat > */ > static int replaceCollParam(tjCtx* tjctx, str param, str val) { > - return ( BUNreplace(tjctx->b_collParam,param,val,GUESSFORCE) != NULL ); > + return ( BUNreplace(tjctx->b_collParam,param,val,FALSE) != NULL ); > } > > static BAT* > @@ -894,10 +805,10 @@ > /* if ( DOEMIT(tjctx) ) { */ > if ( (termOid = tj_tagOid(tjctx, name)) == oid_nil ) > return 0; > - if ( (tjPre = tj_add2plane(tjctx, termOid)) == oid_nil ) > + if ( (tjPre = tj_newPre(tjctx) ) == oid_nil ) > return 0; > + dbat_set_oid(&tjctx->dbat_collPre, tjPre, termOid); > if ( tj_pushTag(tjctx,tjPre) < 0 ) return 0; > - if ( 0 ) stream_printf(GDKout,"C[%s]:startElement: \"%s\", termoid=%d, Tijah pre#=%d, Pathfinder pre#=%d\n",tjctx->name,name,termOid,tjPre,pre); > #ifdef TJ_TRACE > if ( TJ_TRACE ) stream_printf(GDKout,"C[%s]:startElement: \"%s\", termoid=%d, Tijah pre#=%d, Pathfinder pre#=%d\n",tjctx->name,name,termOid,tjPre,pre); > #endif > @@ -913,8 +824,7 @@ > --tjctx->doc_height; > oid start = tj_popTag(tjctx); /* oid of the first node of the element */ > int size = tjctx->tijahPre - start - 1; /* the Tijah element size */ > - if ( insertPreSize(tjctx,start,size) < 0 ) > - return 0; > + dbat_set_int(&tjctx->dbat_collSize,(int)start,size); > #ifdef TJ_TRACE > if ( TJ_TRACE ) stream_printf(GDKout,"C[%s]:empty_endElement: \"%s\"\n", tjctx->name,""); > #endif > @@ -934,8 +844,7 @@ > /* if ( DOEMIT(tjctx) ) { */ > oid start = tj_popTag(tjctx); /* oid of the first node of the element */ > int size = tjctx->tijahPre - start - 1; /* the Tijah element size */ > - if ( insertPreSize(tjctx,start,size) < 0 ) > - return 0; > + dbat_set_int(&tjctx->dbat_collSize,(int)start,size); > #ifdef TJ_TRACE > if ( TJ_TRACE ) stream_printf(GDKout,"C[%s]:endElement: \"%s\"\n", tjctx->name,name); > #endif > @@ -944,8 +853,6 @@ > return 1; > } > > -#define USE_FLEX 1 > - > /** > * Output generation handler. Handles equivalent of * SAX characters() event. > */ > @@ -954,28 +861,23 @@ > EMPTY_CHECK; > tjCtx* tjctx = (tjCtx*)ctx->driverWs; > > + register char* p = (char*)ch; > + while( *p && isspace(*p) ) p++; > + if ( !*p ) > + return 1; > #ifdef TJ_TRACE > - if ( TJ_TRACE ) stream_printf(GDKout,"C[%s]:handle_characters(%s) start\n",tjctx->name, (char*)ch); > + if ( TJ_TRACE ) stream_printf(GDKout,"C[%s]:handle_characters(%s) start\n",tjctx->name, p); > #endif > > if ( DOEMIT(tjctx) ) { > -#ifdef USE_FLEX > - return useFlexScanner((char*)ch,strlen((char*)ch),tjctx); > -#else > - return useStrtokScanner(tjctx,(char*)ch); > -#endif > + return useFlexScanner(p,tjctx); > } > return 1; > } > > char* normalizeTerm(struct tijahContextStruct *tjctx, char* term ) { > char *res; > -#ifdef USE_FLEX > - res = flexScanOneTerm((char*)term,strlen((char*)term)); > -#else > - res = strtok(term,obsoleteNexiChars); > -#endif > - /* INCOMPLETE, should make shure tijahContext is always avail. here */ > + res = flexScanOneTerm((char*)term); > if ( res && tjctx && tjctx->stemCtx->stem) { > if ( !(res = (char*)tjctx->stemCtx->stem(tjctx->stemCtx,(char*)res)) ) { > /* must be a stopword */ > @@ -986,14 +888,6 @@ > } > > int CMDtj_normalizeTerm(char** res, str term, str stemmer) { > -//Leave tokenization disabled for now > -// char* tokenized; > -//#ifdef USE_FLEX > -// tokenized = flexScanOneTerm(term,strlen(term)); > -//#else > -// tokenized = strtok(term,obsoleteNexiChars); > -//#endif > - > tjStemCtx* stemCtx = getStemmingContext( stemmer ); > > if ( stemCtx->stem ) { > @@ -1123,13 +1017,9 @@ > #ifdef TJ_TRACE > if ( TJ_TRACE ) stream_printf(GDKout,"C[%s]:FINISH INDEXING\n",tjctx->name); > #endif > - > - /* feature not used anymore ????? */ > - if ( 0 /* ROEL CHANGE VIRTUAL ROOT SIZE HERE */ ) > - insertPreSize(tjctx,0,tjctx->tijahPre - 1); > - if ( dbat_finalize(&tjctx->dbat_collPre) < 0 ) > + if ( dbat_finalize(&tjctx->dbat_collPre, tjctx->tijahPre) < 0 ) > return GDK_FAIL; > - if ( dbat_finalize(&tjctx->dbat_collSize) < 0 ) > + if ( dbat_finalize(&tjctx->dbat_collSize, tjctx->tijahPre) < 0 ) > return GDK_FAIL; > #ifdef TJ_TRACE > if ( TJ_TRACE ) stream_printf(GDKout,"C[%s]:FINALIZED DIRECT BATS\n",tjctx->name); > > Index: pftijah_tokenize.l > =================================================================== > RCS file: /cvsroot/monetdb/pathfinder/modules/pftijah/pftijah_tokenize.l,v > retrieving revision 1.12 > retrieving revision 1.13 > diff -u -d -r1.12 -r1.13 > --- pftijah_tokenize.l 9 Jan 2007 15:44:39 -0000 1.12 > +++ pftijah_tokenize.l 27 Feb 2007 15:43:37 -0000 1.13 > @@ -115,7 +115,40 @@ > > %% > > -int useFlexScanner(char* buf, int len, struct tijahContextStruct* tjCtx) { > +int OPT0useFlexScanner(char* buf, struct tijahContextStruct* tjCtx) { > + /* UPDATE: this delivers very strange testset results and should not be > + * used I think. > + */ > + /* This is an optimized version of the flex scanner which does not copy the > + * input buffer. The only strange thing about this interface is that it > + * requires 2 YY_END_OF_BUFFER_CHAR (eg. 0) at the end of the buffer. The > + * size of the buffer is inclusive the 2 0's. > + * The last zero is toggled with its original value to prevent corruption > + * of memory management tables. This was for me the only way to prevent > + * copying here. > + */ > + int len = strlen(buf); > + char remember = buf[len+1]; > + buf[len+1] = YY_END_OF_BUFFER_CHAR; > + YY_BUFFER_STATE myBuf = yy_scan_buffer(buf, len+2); > + > + if ( !myBuf ) { > + stream_printf(GDKout,"# useFlexScanner: unable to get setup non-copy buffer."); > + return 0; > + } > + while ( pftijah_tokenizelex() ) { > + /* stream_printf(GDKout,"# scan(%s).\n",pftijah_tokenizetext); */ > + if ( !handleTijahTerm(tjCtx, pftijah_tokenizetext) ) > + return 0; > + } > + yy_delete_buffer(myBuf); > + buf[len+1] = remember; > + return 1; > +} > + > +int useFlexScanner(char* buf, struct tijahContextStruct* tjCtx) { > + // the original > + int len = strlen(buf); > YY_BUFFER_STATE myBuf = yy_scan_bytes(buf, len); > while (pftijah_tokenizelex()) { > if ( !handleTijahTerm(tjCtx, pftijah_tokenizetext) ) > @@ -125,6 +158,40 @@ > return 1; > } > > +int OPT2useFlexScanner(char* input, struct tijahContextStruct* tjctx) > +{ > + /* the fast function. This function is in the pftijah context with lots > + * of small strings to tokenize many times faster as the flex and the > + * strtok() methods which seem to have a rather larger overhead > + */ > + register char* s = input; > + register char x; > +// #define EMIT x=*s; *s=0; stream_printf(GDKout,"#[%s]\n",base);if (!handleTijahTerm(tjctx,base)) return 0; *s=x > +#define EMIT x=*s; *s=0; if (!handleTijahTerm(tjctx,base)) return 0; *s=x > + > + while ( 1 ) { > + while ( isspace( *s ) ) s++; > + if ( *s ) { > + char* base = s; > + if ( isalnum(*s) ) { > + if ( isdigit(*s) ) { > + while ( isdigit(*++s) ) ; > + EMIT; > + } else { > + if (isupper(*s)) *s=tolower(*s); > + while ( isalnum(*++s) ) if (isupper(*s)) *s=tolower(*s); > + EMIT; > + } > + } else { > + // INCOMPLETE, ENTITIES HERE > + // stream_printf(GDKout,"#[SKIPPING:%c]\n",*s); > + s++; > + } > + } else > + return 1; > + } > +} > + > char* tijah_tokenize_string(char* buf, int len, char* outbuf) { > int cnt = 0; > YY_BUFFER_STATE myBuf = yy_scan_bytes(buf, len); > @@ -137,9 +204,10 @@ > return outbuf; > } > > -char* flexScanOneTerm(char* buf, int len) { > +char* flexScanOneTerm(char* buf) { > char *res; > char resBUFF[256]; > + int len = strlen(buf); > > YY_BUFFER_STATE myBuf = yy_scan_bytes(buf, len); > if ( pftijah_tokenizelex() ) { > > Index: nexi.c > =================================================================== > RCS file: /cvsroot/monetdb/pathfinder/modules/pftijah/nexi.c,v > retrieving revision 1.49 > retrieving revision 1.50 > diff -u -d -r1.49 -r1.50 > --- nexi.c 23 Feb 2007 15:11:05 -0000 1.49 > +++ nexi.c 27 Feb 2007 15:43:37 -0000 1.50 > @@ -455,6 +455,7 @@ > /* > * Now find out if the collection is fragmented or not. > */ > + /* INCOMPLETE, ERROR HERE WITH REFCOUNTS IN HEAD */ > BAT* fb = pftu_lookup_bat(pftu_batname1("tj_%s_fragments",(char*)parserCtx->collection,0)); > if ( ! fb ) { > stream_printf(GDKerr,"Error: cannot find fragments bat for collection \"%s\".\n",parserCtx->collection); > @@ -471,6 +472,8 @@ > parserCtx->ffPfx = ""; > parserCtx->flastPfx = ", str(1)"; > } > + BBPunfix(BBPcacheid(fb)); > + fb = NULL; > // Some special cases for NLLR, since NLLR only works with COARSE2 at the moment > if ( txt_retr_model->model == MODEL_NLLR ) { > // Switch to COARSE2 algebra for NLLR > > Index: pftijah_util.mx > =================================================================== > RCS file: /cvsroot/monetdb/pathfinder/modules/pftijah/pftijah_util.mx,v > retrieving revision 1.2 > retrieving revision 1.3 > diff -u -d -r1.2 -r1.3 > --- pftijah_util.mx 9 Jan 2007 17:15:23 -0000 1.2 > +++ pftijah_util.mx 27 Feb 2007 15:43:37 -0000 1.3 > @@ -73,6 +73,7 @@ > if ( b == bat_nil ) { > return NULL; > } else { > + BBPfix(b); > return BBPdescriptor(b); > } > } > > > ------------------------------------------------------------------------- > Take Surveys. Earn Cash. Influence the Future of IT > Join SourceForge.net's Techsay panel and you'll get the chance to share your > opinions on IT & business topics through brief surveys-and earn cash > http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV > _______________________________________________ > Monetdb-pf-checkins mailing list > Monetdb-pf-checkins(a)lists.sourceforge.net > https://lists.sourceforge.net/lists/listinfo/monetdb-pf-checkins -- Sjoerd Mullender

2 1

[Monetdb-developers] MonetDB5 compilation in windows...
by Romulo Goncalves 26 Feb '07

26 Feb '07

During the compilation of MonetDB5 in windows I got the error: cl -GF -W3 -wd4273 -wd4102 -MD -nologo -Zi -Ox -I. -I.\..\.. "-IC:\Pthreads\include" -I.\..\..\..\src\tools "-I..\mal" -I.\..\..\..\..\clients\NT\include -I.\..\..\..\..\clients\NT\include\MonetDB\mapilib "-IC:\cygwin\home\goncalve\MonetDB\Mone tDB_SQL\MonetDB\NT\include" "-IC:\cygwin\home\goncalve\MonetDB\MonetDB_SQL\MonetDB\NT\include\common" "-IC:\cygwin\home\goncal ve\MonetDB\MonetDB_SQL\MonetDB\NT\include\gdk" "-IC:\cygwin\home\goncalve\MonetDB\MonetDB_SQL\MonetDB\NT\include\monet" "-IC:\ cygwin\home\goncalve\MonetDB\MonetDB_SQL\MonetDB\NT\include\plain" "-IC:\cygwin\home\goncalve\MonetDB\MonetDB_SQL\MonetDB\NT\i nclude\mapi" -I.\..\..\..\src\tools "-I..\mal" -I.\..\..\..\..\clients\NT\include -I.\..\..\..\..\clients\NT\include\MonetDB\ mapilib "-IC:\cygwin\home\goncalve\MonetDB\MonetDB_SQL\MonetDB\NT\include" "-IC:\cygwin\home\goncalve\MonetDB\MonetDB_SQL\Mone tDB\NT\include\common" "-IC:\cygwin\home\goncalve\MonetDB\MonetDB_SQL\MonetDB\NT\include\gdk" "-IC:\cygwin\home\goncalve\Monet DB\MonetDB_SQL\MonetDB\NT\include\monet" "-IC:\cygwin\home\goncalve\MonetDB\MonetDB_SQL\MonetDB\NT\include\plain" "-IC:\cygwin \home\goncalve\MonetDB\MonetDB_SQL\MonetDB\NT\include\mapi" -c mserver5.c mserver5.c mserver5.mx(161) : error C2146: syntax error : missing ')' before identifier 'VERSION' mserver5.mx(161) : error C2144: syntax error : '<Unknown>' should be preceded by '<Unknown>' mserver5.mx(161) : error C2144: syntax error : '<Unknown>' should be preceded by '<Unknown>' mserver5.mx(161) : error C2143: syntax error : missing ')' before 'identifier' mserver5.mx(161) : error C2059: syntax error : ')' mserver5.mx(179) : error C2065: 'VERSION' : undeclared identifier NMAKE : fatal error U1077: 'cl' : return code '0x2' Stop. I tried to correct it, but I cannot find it. I checked the last commits to the files mserver.mx and I do not find any syntax error. Any idea? Regards, Romulo

2 1

[Monetdb-developers] stable branch
by Sjoerd Mullender 26 Feb '07

26 Feb '07

You are free to check in bug fixes on the stable branch again. -- Sjoerd Mullender

1 0

[Monetdb-developers] Bug fix release of MonetDB suite
by Sjoerd Mullender 26 Feb '07

26 Feb '07

The MonetDB team at CWI, The Netherlands, is happy to announce the availability of a new bug-fix release for the major "Venus" release. New versions of all components of the MonetDB suite are released. MonetDB/XQuery The handling of updates was incomplete. Several memory mapping and concurrency problems have been fixed. Moreover, empty pages that resulted from deletion of nodes are now properly handed back to the system for reuse. MonetDB/XRPC Security and concurrency handling has been improved. MonetDB/SQL Foreign key constraint checking fixed; View in view dependencies bug fixed; Resolved crashes caused by large views; Prepared statements with more than 30 arguments now work. MonetDB/API Haskell and TCL libraries were broken and have been removed; Perl and Python libraries were broken and have been fixed. The current version of the protocol is not compatible with the previous version. Bugs fixed: [ 1579510 ] XQ: update error for "(1,2)/." [ 1590523 ] XQ: substring-after [ 1636828 ] fork() behaves strangely [ 1637867 ] XQ: seqcast int? to numeric unsupported [ 1639108 ] mserver crashes on delete query [ 1640248 ] PF: ~500 tests randomly fail on Solaris x86 [ 1644245 ] M5 & SQL5: various tests hang/timeout on Darwin [ 1648303 ] database dump does not save/restore auto-increment sequences [ 1650469 ] SQL: Armada sinks MonetDB ships [ 1651567 ] yet another delete bug, foreign key related [ 1651599 ] caching of inserts causes extreme memory consumption [ 1655818 ] SQL:Error inserting float in numeric column using JDBC [ 1655857 ] XQ: call to builtin function crashes XRPC client [ 1659889 ] SQL: sequence not restored properly [ 1644610 ] Conflict between normal query and XRPC query (and many more that hadn't been reported) -- Sjoerd Mullender

1 0

[Monetdb-developers] Clients compilaiton in windows...
by Romulo Goncalves 26 Feb '07

26 Feb '07

Compiling the clients in windows gave the error: copy ".\..\..\..\..\..\src\php\Cimpl\examples\monetdb_query_demo.php" "monetdb_query_demo.php" 1 file(s) copied. cl -GF -W3 -wd4273 -wd4102 -MD -nologo -Zi -Ox -I. -I.\..\..\.. "-IC:\Pthreads\include" -I.\..\..\..\..\src\php\Cimpl "-I..\..\mapilib" -I.\..\..\..\..\src\php\Cimpl\..\..\mapilib -I.\..\..\..\..\..\MonetDB\NT\include -I.\..\..\..\..\..\MonetD \NT\include\common -I.\..\..\..\..\..\MonetDB\NT\include\gdk -I.\..\..\..\..\..\php-5.1.4 -I.\..\..\..\..\..\php-5.1.4\main - .\..\..\..\..\..\php-5.1.4\TSRM -I.\..\..\..\..\..\php-5.1.4\Zend -DPHP_WIN32 -DZEND_WIN32 -DZEND_DEBUG=0 -DZTS=1 -DLIBMONE DB -DCOMPILE_DL_MONETDB -Fo"php_monetdb.obj" -c ".\..\..\..\..\src\php\Cimpl\php_monetdb.c" php_monetdb.c E:\Visual Studio .NET 2003\Vc7\PlatformSDK\Include\WS2tcpip.h(647) : warning C4142: benign redefinition of type ..\..\..\..\src\php\Cimpl\php_monetdb.c(1721) : error C2198: 'zend_exception_get_default' : too few arguments for call throug pointer-to-function ..\..\..\..\src\php\Cimpl\php_monetdb.c(1736) : error C2198: 'zend_exception_get_default' : too few arguments for call throug pointer-to-function ..\..\..\..\src\php\Cimpl\php_monetdb.c(1746) : error C2198: 'zend_exception_get_default' : too few arguments for call throug pointer-to-function NMAKE : fatal error U1077: 'cl' : return code '0x2' Stop. NMAKE : fatal error U1077: 'cd' : return code '0x2' Stop. NMAKE : fatal error U1077: 'cd' : return code '0x2' Stop. NMAKE : fatal error U1077: 'cd' : return code '0x2' Stop. NMAKE : fatal error U1077: '"E:\Visual Studio .NET 2003\VC7\BIN\nmake.exe"' : return code '0x2' Stop. I check my zend_exceptions and the function is defined as: ZEND_API zend_class_entry *zend_exception_get_default(TSRMLS_D); My php is php-5.1.4 What values should I use? I used 0 and it compiles, but for sure it is not the best option. Regards, Romulo

3 4

Re: [Monetdb-developers] [Monetdb-pf-checkins] pathfinder/modules/pftijah pftijah.mx, 1.92.2.17, 1.92.2.18
by Henning Rode 26 Feb '07

26 Feb '07

roel, wat was hier eigenlijk aan de hand? eigenlijk hoef je "left" toch niet nog een keer te sorteren? "left" is toch altijd gesorteerd, of niet? -henning Roel van Os wrote: > Update of /cvsroot/monetdb/pathfinder/modules/pftijah > In directory sc8-pr-cvs7.sourceforge.net:/tmp/cvs-serv12912 > > Modified Files: > Tag: XQuery_0-16 > pftijah.mx > Log Message: > PF/Tijah: fix some sortedness issues. These only seem to occur on large-scale > querying, not on the current Tests. > > > Index: pftijah.mx > =================================================================== > RCS file: /cvsroot/monetdb/pathfinder/modules/pftijah/pftijah.mx,v > retrieving revision 1.92.2.17 > retrieving revision 1.92.2.18 > diff -u -d -r1.92.2.17 -r1.92.2.18 > --- pftijah.mx 23 Feb 2007 09:10:04 -0000 1.92.2.17 > +++ pftijah.mx 23 Feb 2007 12:12:57 -0000 1.92.2.18 > @@ -1371,7 +1371,7 @@ > } > > > - var ctx := region; > + var ctx := region.sort(); > var pre_size := load( "tj_" + collName + "_size1"); > var desc := scj_desc( pre_size, ctx, count(pre_size) ); > var result := nodes( desc ).project( dbl(scoreBase) ); > @@ -1589,7 +1589,7 @@ > var pre_size := bat("tj_" + collName + "_size" + ind); > > # evaluate doc/term (anc/desc) relationship > - var elem_tid := _containing_desc3(left.mark(0@0), pre_tid, pre_size); > + var elem_tid := _containing_desc3(left.sort().mark(0@0), pre_tid, pre_size); > pre_tid := nil; > if (elem_tid.count() = 0) {return new(oid,dbl);} > > @@ -1632,7 +1632,7 @@ > > # evaluate doc/term (anc/desc) relationship > var t2 := time(); > - var elem_tid := _containing_desc3(left.mark(0@0), pre_tid, pre_size); > + var elem_tid := _containing_desc3(left.sort().mark(0@0), pre_tid, pre_size); > var t3 := time(); > pre_tid := nil; > if (elem_tid.count() = 0) {return new(oid,dbl);} > @@ -1679,7 +1679,7 @@ > > # evaluate doc/term (anc/desc) relationship > var t2 := time(); > - var elem_tid := _containing_desc(left.mark(0@0), pre_tid, pre_size); > + var elem_tid := _containing_desc(left.sort().mark(0@0), pre_tid, pre_size); > var t3 := time(); > pre_tid := nil; > if (elem_tid.count() = 0) {return new(oid,dbl);} > @@ -1870,7 +1870,7 @@ > var pre_size := bat("tj_" + collName + "_size1"); > > # See which document contain the query terms we create a bat of [doc, term-id]: > - var doc_tid := _containing_desc(ctx.mark(0(a)0).sort(), tid_pre.reverse(), pre_size); > + var doc_tid := _containing_desc(ctx.sort().mark(0@0), tid_pre.reverse(), pre_size); > > # len(doc): [doc, size] > var doc_len := [dbl](bat("tj_" + collName + "_size1").semijoin(doc_tid)); > > > ------------------------------------------------------------------------- > Take Surveys. Earn Cash. Influence the Future of IT > Join SourceForge.net's Techsay panel and you'll get the chance to share your > opinions on IT & business topics through brief surveys-and earn cash > http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV > _______________________________________________ > Monetdb-pf-checkins mailing list > Monetdb-pf-checkins(a)lists.sourceforge.net > https://lists.sourceforge.net/lists/listinfo/monetdb-pf-checkins

1 0

[Monetdb-developers] windows compilaton...
by Romulo Goncalves 26 Feb '07

26 Feb '07

During the compilation of clients (cvs head) I got the following errors: cd "MonetSQLdb" && nmake /nologo "prefix=C:\cygwin\home\goncalve\MonetDB\MonetDB_SQL\clients\NT" all cl -GF -W3 -wd4273 -wd4102 -MD -nologo -Zi -Ox -I. -I.\..\..\.. "-IC:\Pthreads\include" -I.\..\..\..\..\src\python\Cim pl "-I..\..\mapilib" -I.\..\..\..\..\..\MonetDB\NT\include -I.\..\..\..\..\..\MonetDB\NT\include\common -I.\..\..\..\..\..\Mon etDB\NT\include\gdk -IC:\Python25\include -LD -Fe_MapiLib.pyd MapiLib.py.obj /link ..\..\mapilib\libMapi.lib C:\Python25\libs \python25.lib Creating library _MapiLib.lib and object _MapiLib.exp MapiLib.py.obj : error LNK2019: unresolved external symbol __imp__mapi_setBlocking referenced in function __wrap_mapi_setBlock ing MapiLib.py.obj : error LNK2019: unresolved external symbol __imp__mapi_is_blocked referenced in function __wrap_mapi_is_blocke d _MapiLib.pyd : fatal error LNK1120: 2 unresolved externals NMAKE : fatal error U1077: 'cl' : return code '0x2' Stop. NMAKE : fatal error U1077: 'cd' : return code '0x2' Stop. NMAKE : fatal error U1077: 'cd' : return code '0x2' Stop. NMAKE : fatal error U1077: 'cd' : return code '0x2' Stop. NMAKE : fatal error U1077: '"E:\Visual Studio .NET 2003\VC7\BIN\nmake.exe"' : return code '0x2' Stop. I already updated and compiled buildtools and MonetDB. Maybe Niels or Fabian know the reason of this error. Regards, Romulo

3 8

[Monetdb-developers] check-ins
by Henning Rode 26 Feb '07

26 Feb '07

hej amsterdam, what's the status of the release? or more important, when are we allowed to do check-ins again? and second question: which machine on the test web is actually running icc? i remember we checked the test-web on friday and besides the rare sun-os all systems looked pretty green on our test-cases. so, where exactly should we have spotted the icc compilation problems? best -henning

2 1

Re: [Monetdb-developers] [Monetdb-pf-checkins] pathfinder/runtime/xrpc/export export.xq, 1.1.2.1, 1.1.2.2
by Sjoerd Mullender 25 Feb '07

25 Feb '07

On 02/25/2007 11:02 PM, Ying Zhang wrote: > Update of /cvsroot/monetdb/pathfinder/runtime/xrpc/export > In directory sc8-pr-cvs7.sourceforge.net:/tmp/cvs-serv16912 > > Modified Files: > Tag: XQuery_0-16 > export.xq > Log Message: > A copy-paste error in the NS identifier. > > Since this change won't affect compilation, I check it in without > explicit permission from Sjoerd. :P It does, however, affect the binary and source distributions that we make, and so you should have asked permission. But you're in luck: I'm going to use tonight's build. > > > Index: export.xq > =================================================================== > RCS file: /cvsroot/monetdb/pathfinder/runtime/xrpc/export/export.xq,v > retrieving revision 1.1.2.1 > retrieving revision 1.1.2.2 > diff -u -d -r1.1.2.1 -r1.1.2.2 > --- export.xq 21 Feb 2007 16:25:48 -0000 1.1.2.1 > +++ export.xq 25 Feb 2007 22:02:54 -0000 1.1.2.2 > @@ -11,5 +11,5 @@ > > (: =================== HTTP =================== :) > > -declare function admin:GET($uri as xs:string) > +declare function export:GET($uri as xs:string) > { doc($uri) }; > > > ------------------------------------------------------------------------- > Take Surveys. Earn Cash. Influence the Future of IT > Join SourceForge.net's Techsay panel and you'll get the chance to share your > opinions on IT & business topics through brief surveys-and earn cash > http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV > _______________________________________________ > Monetdb-pf-checkins mailing list > Monetdb-pf-checkins(a)lists.sourceforge.net > https://lists.sourceforge.net/lists/listinfo/monetdb-pf-checkins -- Sjoerd Mullender

1 0

Re: [Monetdb-developers] [Monetdb-checkins] buildtools/autogen/autogen am.py, 1.30.2.1, 1.30.2.2
by Sjoerd Mullender 25 Feb '07

25 Feb '07

On 02/25/2007 01:51 PM, Niels Nes wrote: > Update of /cvsroot/monetdb/buildtools/autogen/autogen > In directory sc8-pr-cvs7.sourceforge.net:/tmp/cvs-serv2017/autogen/autogen > > Modified Files: > Tag: MonetDB_1-16 > am.py > Log Message: > we need to distribute *.def files as the rules to build them are part of > the rules.mk file (which isn't distributed) Did you really mean to check in those print statements? > > Index: am.py > =================================================================== > RCS file: /cvsroot/monetdb/buildtools/autogen/autogen/am.py,v > retrieving revision 1.30.2.1 > retrieving revision 1.30.2.2 > diff -u -d -r1.30.2.1 -r1.30.2.2 > --- am.py 13 Feb 2007 10:13:23 -0000 1.30.2.1 > +++ am.py 25 Feb 2007 12:51:11 -0000 1.30.2.2 > @@ -19,7 +19,7 @@ > from codegen import find_org > > #automake_ext = ['c', 'h', 'y', 'l', 'glue.c'] > -automake_ext = ['o', 'lo', 'c', 'h', 'tab.c', 'tab.h', 'yy.c', 'glue.c', 'proto.h', 'py.c', 'pm.c', ''] > +automake_ext = ['o', 'lo', 'c', 'h', 'tab.c', 'tab.h', 'yy.c', 'glue.c', 'proto.h', 'py.c', 'pm.c', 'def', ''] > buildtools_ext = ['mx', 'm', 'y', 'l', 'brg'] > > am_assign = "+=" > @@ -153,9 +153,11 @@ > pf = f > > # built source if has dep and ext != cur ext > + print(pf) > if not(cond) and deps.has_key(pf) and pf not in am['BUILT_SOURCES']: > pfb, pfext = split_filename(pf) > sfb, sfext = split_filename(deps[pf][0]) > + print('test %s' % pf) > if sfext != pfext: > if pfext in automake_ext: > dist = None > @@ -622,6 +624,7 @@ > def am_library(fd, var, libmap, am): > name = var[4:] > > + print("am_library\n"); > sep = "" > pref = 'lib' > if libmap.has_key("NAME"): > @@ -702,6 +705,7 @@ > deps = [] > for target in libmap['TARGETS']: > t, ext = split_filename(target) > + print(t,ext) > if ext in scripts_ext: > if target not in SCRIPTS: > SCRIPTS.append(target) > > > ------------------------------------------------------------------------- > Take Surveys. Earn Cash. Influence the Future of IT > Join SourceForge.net's Techsay panel and you'll get the chance to share your > opinions on IT & business topics through brief surveys-and earn cash > http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV > _______________________________________________ > Monetdb-checkins mailing list > Monetdb-checkins(a)lists.sourceforge.net > https://lists.sourceforge.net/lists/listinfo/monetdb-checkins -- Sjoerd Mullender

1 0